openFOAM-RANS-to-GPU/scripts/verify_airfrans_stepper.py

1936 lines
78 KiB
Python
Executable file

#!/usr/bin/env python3
"""Verifier harness for the migrated AirfRANS OpenFOAM stepper case."""
from __future__ import annotations
import argparse
import dataclasses
import hashlib
import json
import math
import os
import shutil
import subprocess
import sys
import time
import traceback
from collections.abc import Iterable, Mapping
from pathlib import Path
from typing import Any
def _drop_ambient_pythonpath() -> None:
pythonpath = os.environ.pop("PYTHONPATH", "")
for entry in pythonpath.split(os.pathsep):
if not entry:
continue
while entry in sys.path:
sys.path.remove(entry)
_drop_ambient_pythonpath()
import numpy as np
from openfoam_env import apply_openfoam_env, openfoam_env
from prepare_airfrans_stepper_case import (
DEFAULT_SOURCE,
REQUIRED_COMPARISON_FIELDS,
load_case_manifest,
prepare_case,
write_case_manifest,
)
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_WORK = ROOT / "tmp/airfrans_stepper_verify"
PRIMARY_FIELDS = ("U", "p", "phi")
TURBULENCE_FIELDS = ("nut", "k", "omega")
REQUIRED_FIELDS = REQUIRED_COMPARISON_FIELDS
BACKEND_CHOICES = ("auto", "cpu", "gpu")
FULL_GPU_RANS_GUARD = "scripts/verify_gpu_rans_solver.sh"
GPU_PRIMITIVE_PROOF = "scripts/verify_gpu_algorithm.sh"
GPU_PRIMITIVE_NAME = "cell_flux_imbalance"
AIRFRANS_VERIFIER_SCOPE = (
"AirfRANS/OpenFOAM stepper parity verifier; --backend gpu is a GPU-owned "
"RANS solver contract with no CPU fallback or primitive-only acceptance"
)
LOADING_FAILURE = "loading_or_parsing_failure"
ORACLE_FAILURE = "openfoam_oracle_failure"
STEPPER_FAILURE = "stepper_execution_failure"
COMPARISON_FAILURE = "numerical_comparison_failure"
BACKEND_FAILURE = "backend_execution_failure"
INTERNAL_FAILURE = "internal_harness_failure"
EXIT_CODES = {
LOADING_FAILURE: 2,
ORACLE_FAILURE: 3,
STEPPER_FAILURE: 4,
COMPARISON_FAILURE: 5,
BACKEND_FAILURE: 6,
INTERNAL_FAILURE: 7,
}
STAGE_OBSERVABILITY_GROUPS = (
{
"name": "momentum_assembly",
"split_stages": ("assemble_momentum_terms", "assemble_UEqn"),
"run_one_stages": ("assemble_UEqn",),
"split_outputs": {
"assemble_momentum_terms": ("terms",),
"assemble_UEqn": ("UEqn",),
},
},
{
"name": "pressure_assembly",
"split_stages": ("compute_pressure_inputs", "assemble_pEqn"),
"run_one_stages": ("compute_pressure_inputs", "assemble_pEqn"),
"split_outputs": {
"compute_pressure_inputs": ("HbyA", "phiHbyA", "rAU"),
"assemble_pEqn": ("pEqn",),
},
},
{
"name": "linear_solve_results",
"split_stages": ("solve_UEqn", "solve_pEqn"),
"run_one_stages": ("solve_UEqn", "solve_pEqn"),
"split_outputs": {
"solve_UEqn": ("performance", "field_after"),
"solve_pEqn": ("performance", "p", "phi"),
},
},
{
"name": "final_correction",
"split_stages": ("correct_velocity_pressure_flux",),
"run_one_stages": ("update_phi_from_pEqn_flux", "correct_velocity_pressure_flux"),
"split_outputs": {
"correct_velocity_pressure_flux": ("U", "p", "phi"),
},
},
{
"name": "turbulence_updates",
"split_stages": ("momentum_transport_predict", "momentum_transport_correct"),
"run_one_stages": ("momentum_transport_predict", "momentum_transport_correct"),
"split_outputs": {
"momentum_transport_predict": ("case_path", "solver_name"),
"momentum_transport_correct": ("U", "p", "phi", "nut", "k", "omega"),
},
},
)
FIELD_COMPARISON_ATTRIBUTION = {
"U": ("final_correction", "linear_solve_results", "momentum_assembly", "turbulence_updates"),
"p": ("linear_solve_results", "pressure_assembly", "final_correction"),
"phi": ("final_correction", "linear_solve_results", "pressure_assembly"),
"nut": ("turbulence_updates",),
"k": ("turbulence_updates",),
"omega": ("turbulence_updates",),
}
GPU_RUNTIME_UNAVAILABLE = "gpu_runtime_unavailable"
GPU_NUMERICAL_MISMATCH = "gpu_numerical_mismatch"
GPU_INPUT_SCHEMA_VERSION = 1
DIFFERENTIABILITY_VARIABLES = {
"fields": REQUIRED_FIELDS,
"mesh_state": ("points", "faces", "owner", "neighbour", "cells", "V", "C", "Cf", "Sf", "magSf", "patches"),
"case_parameters": ("Uinf", "nu", "rhoInf", "angle_of_attack", "liftDir", "dragDir", "solver_tolerances"),
"losses": ("per_field_linf_error", "per_field_l2_error", "oracle_parity_allclose"),
}
NON_DIFFERENTIABLE_BOUNDARIES = (
{
"name": "openfoam_case_io",
"category": "io",
"reason": "OpenFOAM dictionaries, mesh files, and field files are parsed and written as discrete external data.",
},
{
"name": "mesh_topology_and_patch_addressing",
"category": "topology",
"reason": "Face/cell connectivity, owner/neighbour addressing, and boundary patch membership are discrete indices.",
},
{
"name": "boundary_condition_selection",
"category": "discrete_boundary_choice",
"reason": "Patch types, coupled/constraint flags, and boundary condition dictionaries select code paths.",
},
{
"name": "simple_pimple_and_linear_solver_control",
"category": "convergence_control",
"reason": "Iteration counts, stopping criteria, and solver convergence branches are discrete control flow.",
},
{
"name": "turbulence_bounding_and_limiters",
"category": "limiter",
"reason": "k/omega/nut bounding and model limiters introduce clipping or branch-dependent updates.",
},
{
"name": "openfoam_cpp_backend_calls",
"category": "unsupported_solver_operation",
"reason": "The current registered backend executes OpenFOAM C++ operations and exposes no AD, JVP, or VJP API.",
},
)
CUSTOM_GRADIENT_REQUIREMENTS = (
{
"operation": "sparse_linear_solves",
"stage_groups": ("linear_solve_results",),
"reason": "Implicit sparse solves require an adjoint or custom VJP rather than differentiating solver iterations blindly.",
},
{
"operation": "pressure_velocity_correction",
"stage_groups": ("pressure_assembly", "final_correction"),
"reason": "The coupled pressure/flux/velocity correction needs a consistent custom gradient for the assembled operators.",
},
{
"operation": "turbulence_model_correctors_and_limiters",
"stage_groups": ("turbulence_updates",),
"reason": "Model correctors, wall functions, and bounding need explicit subgradient or smoothing choices.",
},
{
"operation": "mesh_and_boundary_discrete_choices",
"stage_groups": (),
"reason": "Topology and patch-type changes are not differentiable; only fixed-topology numeric values can be checked.",
},
)
class HarnessError(Exception):
"""Categorized verifier failure that can be serialized into the report."""
def __init__(
self,
category: str,
step: str,
message: str,
*,
details: Mapping[str, Any] | None = None,
cause: BaseException | None = None,
) -> None:
super().__init__(message)
self.category = category
self.step = step
self.details = dict(details or {})
self.cause = cause
def to_dict(self) -> dict[str, Any]:
out: dict[str, Any] = {
"category": self.category,
"step": self.step,
"message": str(self),
"details": self.details,
}
if self.cause is not None:
out["cause"] = {
"type": type(self.cause).__name__,
"message": str(self.cause),
}
return json_ready(out)
class BackendExecutionError(Exception):
"""Backend selection or execution failure before numerical comparison."""
def __init__(self, step: str, message: str, *, details: Mapping[str, Any] | None = None) -> None:
super().__init__(message)
self.step = step
self.details = dict(details or {})
def to_harness_error(self) -> HarnessError:
return HarnessError(BACKEND_FAILURE, self.step, str(self), details=self.details, cause=self)
def json_ready(value: Any) -> Any:
"""Convert report values into strict JSON-compatible data."""
if dataclasses.is_dataclass(value) and not isinstance(value, type):
return json_ready(dataclasses.asdict(value))
if isinstance(value, Path):
return str(value)
if isinstance(value, np.ndarray):
return array_stats(value)
if isinstance(value, np.generic):
return json_ready(value.item())
if isinstance(value, float):
return value if math.isfinite(value) else None
if isinstance(value, (str, int, bool)) or value is None:
return value
if isinstance(value, Mapping):
return {str(key): json_ready(item) for key, item in value.items()}
if isinstance(value, (list, tuple, set)):
return [json_ready(item) for item in value]
return repr(value)
def array_shape(array: Any | None) -> list[int] | None:
if array is None:
return None
return [int(dim) for dim in np.asarray(array).shape]
def finite_float(value: Any) -> float | None:
try:
number = float(value)
except (TypeError, ValueError):
return None
return number if math.isfinite(number) else None
def array_stats(array: Any) -> dict[str, Any]:
arr = np.asarray(array)
out: dict[str, Any] = {
"shape": array_shape(arr),
"dtype": str(arr.dtype),
"size": int(arr.size),
}
if arr.size == 0 or not np.issubdtype(arr.dtype, np.number):
return out
finite = np.isfinite(arr)
out["finite_count"] = int(np.count_nonzero(finite))
out["nonfinite_count"] = int(arr.size - out["finite_count"])
if out["finite_count"]:
finite_values = arr[finite]
out.update(
{
"min": finite_float(np.min(finite_values)),
"max": finite_float(np.max(finite_values)),
"mean": finite_float(np.mean(finite_values)),
}
)
return out
def value_at(array: np.ndarray, index: tuple[int, ...]) -> Any:
value = array[index]
if isinstance(value, np.generic):
return json_ready(value.item())
if isinstance(value, np.ndarray):
return json_ready(value.tolist())
return json_ready(value)
def entity_value_at(array: np.ndarray, entity_index: int | None) -> Any:
if entity_index is None or array.ndim == 0:
return None
value = array[entity_index]
if isinstance(value, np.generic):
return json_ready(value.item())
if isinstance(value, np.ndarray):
return json_ready(value.tolist())
return json_ready(value)
def update_hash_text(digest: "hashlib._Hash", value: str) -> None:
encoded = value.encode("utf-8")
digest.update(len(encoded).to_bytes(8, "little"))
digest.update(encoded)
def update_hash_array(digest: "hashlib._Hash", label: str, array: Any) -> None:
arr = np.ascontiguousarray(np.asarray(array))
update_hash_text(digest, label)
update_hash_text(digest, str(arr.dtype))
update_hash_text(digest, repr(tuple(int(dim) for dim in arr.shape)))
digest.update(arr.tobytes())
def source_summary(source: Any) -> dict[str, Any]:
return {
"file": getattr(source, "file", ""),
"function": getattr(source, "function", ""),
"lines": json_ready(getattr(source, "lines", None)),
}
def boundary_field_summary(patch: Any) -> dict[str, Any]:
return {
"type": getattr(patch, "type", ""),
"values_shape": array_shape(getattr(patch, "values", None)),
"fixes_value": bool(getattr(patch, "fixes_value", False)),
"assignable": bool(getattr(patch, "assignable", False)),
"coupled": bool(getattr(patch, "coupled", False)),
"updated": bool(getattr(patch, "updated", False)),
"patch_internal_shape": array_shape(getattr(patch, "patch_internal", None)),
"value_internal_coeffs_shape": array_shape(getattr(patch, "value_internal_coeffs", None)),
"value_boundary_coeffs_shape": array_shape(getattr(patch, "value_boundary_coeffs", None)),
"gradient_internal_coeffs_shape": array_shape(getattr(patch, "gradient_internal_coeffs", None)),
"gradient_boundary_coeffs_shape": array_shape(getattr(patch, "gradient_boundary_coeffs", None)),
}
def field_summary(field: Any) -> dict[str, Any]:
boundary = getattr(field, "boundary", {})
return {
"name": getattr(field, "name", ""),
"kind": getattr(field, "kind", ""),
"dimensions": getattr(field, "dimensions", ""),
"entity_kind": getattr(field, "entity_kind", ""),
"entity_count": int(getattr(field, "entity_count", 0)),
"internal": array_stats(getattr(field, "internal")),
"boundary": {name: boundary_field_summary(patch) for name, patch in boundary.items()},
}
def solve_summary(performance: Any) -> dict[str, Any]:
return {
"solver_name": getattr(performance, "solver_name", ""),
"field_name": getattr(performance, "field_name", ""),
"initial_residual": json_ready(getattr(performance, "initial_residual", None)),
"final_residual": json_ready(getattr(performance, "final_residual", None)),
"n_iterations": json_ready(getattr(performance, "n_iterations", None)),
"converged": bool(getattr(performance, "converged", False)),
"singular": bool(getattr(performance, "singular", False)),
}
def matrix_summary(matrix: Any) -> dict[str, Any]:
derived: dict[str, Any] = {}
for name in ("A", "H", "H1", "flux", "face_flux_correction"):
value = getattr(matrix, name, None)
if callable(value):
value = value()
if value is not None:
derived[name] = field_summary(value)
for name in ("residual", "D", "DD"):
value = getattr(matrix, name, None)
if callable(value):
value = value()
if value is not None:
derived[name] = array_stats(value)
return {
"name": getattr(matrix, "name", ""),
"field_name": getattr(matrix, "field_name", ""),
"value_rank": getattr(matrix, "value_rank", ""),
"dimensions": getattr(matrix, "dimensions", ""),
"has_diag": bool(getattr(matrix, "has_diag", False)),
"has_upper": bool(getattr(matrix, "has_upper", False)),
"has_lower": bool(getattr(matrix, "has_lower", False)),
"diagonal": bool(getattr(matrix, "diagonal", False)),
"symmetric": bool(getattr(matrix, "symmetric", False)),
"asymmetric": bool(getattr(matrix, "asymmetric", False)),
"diag": array_stats(getattr(matrix, "diag")),
"upper": None if getattr(matrix, "upper", None) is None else array_stats(getattr(matrix, "upper")),
"lower": None if getattr(matrix, "lower", None) is None else array_stats(getattr(matrix, "lower")),
"source": array_stats(getattr(matrix, "source")),
"psi": field_summary(getattr(matrix, "psi")),
"internal_coeff_shapes": [array_shape(item) for item in getattr(matrix, "internal_coeffs", [])],
"boundary_coeff_shapes": [array_shape(item) for item in getattr(matrix, "boundary_coeffs", [])],
"derived": derived,
}
def summarize_value(value: Any) -> Any:
if hasattr(value, "diag") and hasattr(value, "field_name") and hasattr(value, "source"):
return matrix_summary(value)
if hasattr(value, "internal") and hasattr(value, "entity_kind") and hasattr(value, "boundary"):
return field_summary(value)
if hasattr(value, "solver_name") and hasattr(value, "initial_residual"):
return solve_summary(value)
if hasattr(value, "name") and hasattr(value, "phase") and hasattr(value, "outputs"):
return transform_summary(value)
if isinstance(value, Mapping):
return {str(key): summarize_value(item) for key, item in value.items()}
if isinstance(value, list):
return [summarize_value(item) for item in value]
if isinstance(value, tuple):
return [summarize_value(item) for item in value]
return json_ready(value)
def transform_summary(result: Any) -> dict[str, Any]:
return {
"name": getattr(result, "name", ""),
"phase": getattr(result, "phase", ""),
"changed_fields": json_ready(getattr(result, "changed_fields", [])),
"source": source_summary(getattr(result, "source", None)),
"metadata": json_ready(getattr(result, "metadata", {})),
"outputs": summarize_value(getattr(result, "outputs", {})),
}
def graph_stage_summaries(result: Any) -> list[dict[str, Any]]:
return [transform_summary(entry) for entry in getattr(result, "outputs", {}).get("graph", [])]
def stage_observability_report(mode: str, stages: list[dict[str, Any]], *, evidence: Mapping[str, Any]) -> dict[str, Any]:
stage_by_name = {stage["name"]: stage for stage in stages}
failures: list[dict[str, Any]] = []
groups = []
for group in STAGE_OBSERVABILITY_GROUPS:
group_name = group["name"]
required_stages = tuple(group[f"{mode}_stages"])
missing_stages = [name for name in required_stages if name not in stage_by_name]
output_requirements = group.get(f"{mode}_outputs", {})
output_checks = {}
for stage_name, required_outputs in output_requirements.items():
stage = stage_by_name.get(stage_name)
if stage is None:
continue
outputs = stage.get("outputs", {})
output_keys = set(outputs) if isinstance(outputs, Mapping) else set()
missing_outputs = [name for name in required_outputs if name not in output_keys]
output_checks[stage_name] = {
"required": list(required_outputs),
"available": sorted(output_keys),
"missing": missing_outputs,
}
if missing_outputs:
failures.append(
{
"path": f"stage_observability.{mode}.{group_name}.{stage_name}.outputs",
"message": "missing inspectable stage output",
"expected": list(required_outputs),
"actual": sorted(output_keys),
}
)
if missing_stages:
failures.append(
{
"path": f"stage_observability.{mode}.{group_name}.stages",
"message": "missing required solver stage",
"expected": list(required_stages),
"actual": list(stage_by_name),
}
)
groups.append(
{
"name": group_name,
"required_stages": list(required_stages),
"observed": not missing_stages and not any(check["missing"] for check in output_checks.values()),
"missing_stages": missing_stages,
"output_checks": output_checks,
}
)
if failures:
raise HarnessError(
STEPPER_FAILURE,
f"{mode}_stage_observability",
f"{mode} solver-stage observability is incomplete",
details={"failures": failures},
)
return {
"mode": mode,
"graph": [stage["name"] for stage in stages],
"groups": groups,
"evidence": json_ready(evidence),
}
def selected_field_summaries(fields: Mapping[str, Any]) -> dict[str, Any]:
return {name: field_summary(fields[name]) for name in REQUIRED_FIELDS if name in fields}
def field_dict_to_mapping(fields: Any) -> dict[str, Any]:
if isinstance(fields, Mapping):
return dict(fields)
if hasattr(fields, "fields") and isinstance(fields.fields, Mapping):
return dict(fields.fields)
return {name: getattr(fields, name) for name in REQUIRED_FIELDS if hasattr(fields, name)}
def mesh_patch_table(mesh: Any) -> list[dict[str, Any]]:
patches = []
for patch in getattr(mesh, "boundary", []):
patches.append(
{
"index": int(getattr(patch, "index", 0)),
"name": getattr(patch, "name", ""),
"type": getattr(patch, "type", ""),
"start": int(getattr(patch, "start", 0)),
"size": int(getattr(patch, "size", 0)),
"coupled": bool(getattr(patch, "coupled", False)),
"constraint": bool(getattr(patch, "constraint", False)),
}
)
return patches
def mesh_topology_digest(mesh: Any) -> str:
digest = hashlib.sha256()
for name in ("n_points", "n_faces", "n_internal_faces", "n_cells"):
update_hash_text(digest, f"{name}={int(getattr(mesh, name))}")
update_hash_array(digest, "faces.offsets", mesh.faces.offsets)
update_hash_array(digest, "faces.values", mesh.faces.values)
update_hash_array(digest, "cells.offsets", mesh.cells.offsets)
update_hash_array(digest, "cells.values", mesh.cells.values)
update_hash_array(digest, "owner", mesh.owner)
update_hash_array(digest, "neighbour", mesh.neighbour)
ldu = getattr(mesh, "ldu", {})
if isinstance(ldu, Mapping):
if "lower_addr" in ldu:
update_hash_array(digest, "ldu.lower_addr", ldu["lower_addr"])
if "upper_addr" in ldu:
update_hash_array(digest, "ldu.upper_addr", ldu["upper_addr"])
for entry in ldu.get("patch_addr", []):
update_hash_text(digest, f"ldu.patch_index={entry.get('patch_index')}")
update_hash_array(digest, "ldu.patch_addr", entry.get("addr", []))
for patch in mesh_patch_table(mesh):
update_hash_text(digest, json.dumps(patch, sort_keys=True))
return digest.hexdigest()
def mesh_geometry_digest(mesh: Any) -> str:
digest = hashlib.sha256()
for name in ("points", "V", "C", "Cf", "Sf", "magSf"):
update_hash_array(digest, name, getattr(mesh, name))
return digest.hexdigest()
def mesh_identity(mesh: Any) -> dict[str, Any]:
return {
"n_points": int(mesh.n_points),
"n_faces": int(mesh.n_faces),
"n_internal_faces": int(mesh.n_internal_faces),
"n_cells": int(mesh.n_cells),
"patches": mesh_patch_table(mesh),
"topology_sha256": mesh_topology_digest(mesh),
"geometry_sha256": mesh_geometry_digest(mesh),
}
def compare_mesh_identity(mode: str, actual: Mapping[str, Any], expected: Mapping[str, Any]) -> tuple[dict[str, Any], list[dict[str, Any]]]:
keys = ("n_points", "n_faces", "n_internal_faces", "n_cells", "topology_sha256", "geometry_sha256", "patches")
differences = {
key: {"actual": actual.get(key), "expected": expected.get(key)}
for key in keys
if actual.get(key) != expected.get(key)
}
report = {"matches_oracle": not differences, "differences": differences}
if not differences:
return report, []
return report, [{"mode": mode, "kind": "mesh_identity", "differences": differences}]
def field_comparison_attribution(
mode: str,
field: str,
reason: str,
stage_graph: Iterable[str] | None,
) -> dict[str, Any]:
observed_stages = set(stage_graph or ())
candidates = []
for group_name in FIELD_COMPARISON_ATTRIBUTION.get(field, ()):
group = next(item for item in STAGE_OBSERVABILITY_GROUPS if item["name"] == group_name)
stages = list(group.get(f"{mode}_stages", ()))
present_stages = [stage for stage in stages if not observed_stages or stage in observed_stages]
candidates.append(
{
"group": group_name,
"stages": present_stages,
"evidence_path": f"stage_observability.{mode}.{group_name}",
}
)
likely = candidates[0] if candidates else {"group": "unknown", "stages": [], "evidence_path": None}
out = {
"field": field,
"reason": reason,
"likely_stage_group": likely["group"],
"likely_stages": likely["stages"],
"candidate_stage_groups": candidates,
"evidence_path": likely["evidence_path"],
"basis": "field-to-stage dependency map for the selected solver graph",
}
if reason in {"missing_field", "shape_mismatch"}:
out["precondition_failure"] = "field_export_or_state_mapping"
out["field_evidence_path"] = f"modes.{mode}.fields.{field}"
out["basis"] = "field is absent or has the wrong topology; inspect export/state mapping first, then the mapped solver stages"
return out
def field_compare_report(name: str, actual_field: Any, expected_field: Any, *, rtol: float, atol: float) -> tuple[dict[str, Any], dict[str, Any] | None]:
actual = np.asarray(actual_field.internal)
expected = np.asarray(expected_field.internal)
actual_entity_kind = getattr(actual_field, "entity_kind", "")
expected_entity_kind = getattr(expected_field, "entity_kind", "")
report: dict[str, Any] = {
"field": name,
"actual_shape": array_shape(actual),
"expected_shape": array_shape(expected),
"shape_matches": actual.shape == expected.shape,
"actual_entity_kind": actual_entity_kind,
"expected_entity_kind": expected_entity_kind,
"entity_kind_matches": actual_entity_kind == expected_entity_kind,
"rtol": rtol,
"atol": atol,
"comparison": "numpy.allclose(equal_nan=False)",
}
if actual.shape != expected.shape:
report.update({"allclose": False, "reason": "shape_mismatch"})
return report, {"field": name, "reason": "shape_mismatch", **report}
if actual.size == 0:
report.update(
{
"allclose": True,
"max_abs": 0.0,
"mean_abs": 0.0,
"location": None,
"actual_at_max": None,
"expected_at_max": None,
}
)
return report, None
diff = np.abs(actual - expected)
finite = np.isfinite(diff)
if not np.all(finite):
flat_index = int(np.flatnonzero(~finite)[0])
max_abs: float | None = None
else:
flat_index = int(np.argmax(diff))
max_abs = finite_float(diff.reshape(-1)[flat_index])
max_index = tuple(int(item) for item in np.unravel_index(flat_index, diff.shape))
entity_index = max_index[0] if max_index else None
component_index = list(max_index[1:]) if len(max_index) > 1 else None
actual_at_max = value_at(actual, max_index)
expected_at_max = value_at(expected, max_index)
tolerance_at_max = None
if isinstance(expected_at_max, (int, float)):
tolerance_at_max = atol + rtol * abs(float(expected_at_max))
finite_diff = diff[finite]
allclose = bool(np.allclose(actual, expected, rtol=rtol, atol=atol, equal_nan=False))
report.update(
{
"allclose": allclose,
"max_abs": max_abs,
"mean_abs": finite_float(np.mean(finite_diff)) if finite_diff.size else None,
"location": {
"array_index": list(max_index),
"entity_kind": getattr(actual_field, "entity_kind", ""),
"entity_index": entity_index,
"component_index": component_index,
},
"actual_at_max": actual_at_max,
"expected_at_max": expected_at_max,
"actual_entity_at_max": entity_value_at(actual, entity_index),
"expected_entity_at_max": entity_value_at(expected, entity_index),
"tolerance_at_max": finite_float(tolerance_at_max),
"nonfinite_error_count": int(diff.size - np.count_nonzero(finite)),
}
)
if allclose:
return report, None
report["reason"] = "value_mismatch"
return report, {"field": name, "reason": "value_mismatch", **report}
def compare_fields(
mode: str,
actual: Mapping[str, Any],
expected: Mapping[str, Any],
*,
rtol: float,
atol: float,
stage_graph: Iterable[str] | None = None,
) -> tuple[dict[str, Any], list[dict[str, Any]]]:
report: dict[str, Any] = {}
mismatches: list[dict[str, Any]] = []
for name in REQUIRED_FIELDS:
if name not in actual or name not in expected:
missing = {
"field": name,
"reason": "missing_field",
"missing_actual": name not in actual,
"missing_expected": name not in expected,
"actual_fields": sorted(actual),
"expected_fields": sorted(expected),
}
attribution = field_comparison_attribution(mode, name, "missing_field", stage_graph)
report[name] = {"field": name, "allclose": False, "attribution": attribution, **missing}
mismatches.append({"mode": mode, "attribution": attribution, **missing})
continue
field_report, mismatch = field_compare_report(name, actual[name], expected[name], rtol=rtol, atol=atol)
report[name] = field_report
if mismatch is not None:
attribution = field_comparison_attribution(mode, name, mismatch["reason"], stage_graph)
field_report["attribution"] = attribution
mismatch["attribution"] = attribution
mismatches.append({"mode": mode, **mismatch})
return report, mismatches
def run_openfoam_command(cmd: list[str], *, log_path: Path, timeout: int) -> dict[str, Any]:
started = time.monotonic()
env = openfoam_env()
log_path.parent.mkdir(parents=True, exist_ok=True)
try:
with log_path.open("w") as log:
subprocess.run(cmd, cwd=ROOT, env=env, check=True, stdout=log, stderr=subprocess.STDOUT, timeout=timeout)
except subprocess.TimeoutExpired as exc:
raise HarnessError(
ORACLE_FAILURE,
Path(cmd[0]).name,
f"OpenFOAM command timed out after {timeout}s: {' '.join(cmd)}",
details={"cmd": cmd, "timeout_seconds": timeout, "log_path": log_path},
cause=exc,
) from exc
except subprocess.CalledProcessError as exc:
raise HarnessError(
ORACLE_FAILURE,
Path(cmd[0]).name,
f"OpenFOAM command failed with exit code {exc.returncode}: {' '.join(cmd)}",
details={"cmd": cmd, "returncode": exc.returncode, "log_path": log_path},
cause=exc,
) from exc
return {
"cmd": cmd,
"returncode": 0,
"log_path": log_path,
"duration_seconds": round(time.monotonic() - started, 6),
"timeout_seconds": timeout,
}
def patch_start_from_latest(case: Path) -> None:
path = case / "system/controlDict"
text = path.read_text()
old = "startFrom startTime;"
new = "startFrom latestTime;"
if old not in text:
raise AssertionError(f"{path}: missing {old!r}")
path.write_text(text.replace(old, new, 1))
def import_foam() -> Any:
apply_openfoam_env()
import foam_stepper as foam
return foam
def clone_prepared_case(prepared: Path, dest: Path, source: Path, metadata: Any) -> dict[str, Any]:
if dest.exists():
shutil.rmtree(dest)
shutil.copytree(prepared, dest, copy_function=shutil.copy2)
return write_case_manifest(source, dest, metadata)
def prepared_case_digest(manifest: Mapping[str, Any]) -> str:
prepared = manifest.get("prepared_case", {})
if not isinstance(prepared, Mapping):
return ""
return str(prepared.get("files_sha256", ""))
def prepared_file_count(manifest: Mapping[str, Any]) -> int:
prepared = manifest.get("prepared_case", {})
files = prepared.get("files", []) if isinstance(prepared, Mapping) else []
return len(files) if isinstance(files, list) else 0
def compare_prepared_case_manifests(manifests: Mapping[str, Mapping[str, Any]]) -> dict[str, dict[str, Any]]:
baseline = prepared_case_digest(manifests["prepared"])
return {
role: {
"matches_prepared": prepared_case_digest(manifest) == baseline,
"files_sha256": prepared_case_digest(manifest),
"file_count": prepared_file_count(manifest),
}
for role, manifest in manifests.items()
}
def prepare_work_cases(source: Path, work: Path, *, include_split: bool) -> dict[str, Any]:
if work.exists():
shutil.rmtree(work)
work.mkdir(parents=True)
prepared_case = work / "prepared_case"
oracle = work / "oracle_case"
run_one = work / "run_one_case"
split = work / "split_case" if include_split else None
prepared_meta = prepare_case(source, prepared_case, end_time=1)
manifests: dict[str, Any] = {"prepared": load_case_manifest(prepared_case)}
manifests["oracle"] = clone_prepared_case(prepared_case, oracle, source, prepared_meta)
manifests["run_one"] = clone_prepared_case(prepared_case, run_one, source, prepared_meta)
if split is not None:
manifests["split"] = clone_prepared_case(prepared_case, split, source, prepared_meta)
return {
"prepared_case": prepared_case,
"oracle_case": oracle,
"run_one_case": run_one,
"split_case": split,
"metadata": {
"prepared": prepared_meta,
"oracle": prepared_meta,
"run_one": prepared_meta,
"split": prepared_meta if split is not None else None,
},
"manifests": manifests,
"prepared_case_identity": compare_prepared_case_manifests(manifests),
}
def _gpu_backend_module() -> Any:
try:
from foam_stepper.gpu import backend as gpu_backend
except Exception as exc:
raise BackendExecutionError(
"select_backend",
"GPU runtime unavailable: failed to import reusable foam_stepper.gpu backend",
details={
"requested": "gpu",
"failure_kind": GPU_RUNTIME_UNAVAILABLE,
"provider": "quadrants_cuda_rans_solver",
"used_cpu_fallback": False,
"cause": {"type": type(exc).__name__, "message": str(exc)},
},
) from exc
return gpu_backend
def _wrap_gpu_backend_error(exc: BaseException) -> BackendExecutionError:
return BackendExecutionError(
str(getattr(exc, "step", "execute_gpu_solver_contract")),
str(exc),
details=dict(getattr(exc, "details", {}) or {}),
)
def _call_gpu_backend(function_name: str, *args: Any, **kwargs: Any) -> Any:
gpu_backend = _gpu_backend_module()
try:
return getattr(gpu_backend, function_name)(*args, **kwargs)
except gpu_backend.BackendExecutionError as exc:
raise _wrap_gpu_backend_error(exc) from exc
def prepare_gpu_solver_inputs(foam: Any, stepper: Any, case: Path, prepared: Mapping[str, Any], backend: Mapping[str, Any]) -> dict[str, Any]:
return _call_gpu_backend("prepare_gpu_solver_inputs", foam, stepper, case, prepared, backend)
def run_gpu_solver_stage_smoke(stepper: Any, backend: Mapping[str, Any], case: Path) -> dict[str, Any]:
return _call_gpu_backend("run_gpu_solver_stage_smoke", stepper, backend, case)
def gpu_solver_stage_report(gpu_run: Mapping[str, Any]) -> dict[str, Any]:
return _call_gpu_backend("gpu_solver_stage_report", gpu_run)
def gpu_solver_input_blocker_summary(gpu_inputs: Mapping[str, Any]) -> dict[str, Any]:
return _call_gpu_backend("gpu_solver_input_blocker_summary", gpu_inputs)
def gpu_solver_contract_blocker(backend: Mapping[str, Any], case: Path) -> BackendExecutionError | None:
gpu_backend = _gpu_backend_module()
try:
blocker = gpu_backend.gpu_solver_contract_blocker(backend, case)
except gpu_backend.BackendExecutionError as exc:
raise _wrap_gpu_backend_error(exc) from exc
if blocker is None:
return None
return _wrap_gpu_backend_error(blocker)
def record_gpu_contract_failure(report: dict[str, Any], blocker: BackendExecutionError, gpu_inputs: Mapping[str, Any] | None = None) -> None:
if gpu_inputs is not None:
blocker.details["gpu_solver_inputs"] = gpu_solver_input_blocker_summary(gpu_inputs)
report["backend"]["contract_preflight"] = json_ready(
{
"status": "failed",
"step": blocker.step,
"failure_kind": blocker.details.get("failure_kind"),
"checked_before_oracle": True,
"gpu_inputs_prepared": gpu_inputs is not None,
"reason": "GPU requests must fail as incomplete before any CPU/OpenFOAM solver path can be substituted as the backend.",
}
)
raise blocker.to_harness_error() from blocker
def select_execution_backend(requested: str) -> dict[str, Any]:
if requested not in BACKEND_CHOICES:
raise BackendExecutionError(
"select_backend",
f"unknown backend {requested!r}",
details={"requested": requested, "choices": list(BACKEND_CHOICES)},
)
if requested == "gpu":
return _call_gpu_backend("select_execution_backend", requested)
return {
"requested": requested,
"selected": "cpu",
"device": "host",
"provider": "foam_stepper_cpu",
"available_backends": ["cpu"],
"gpu_device_present": any(Path(path).exists() for path in ("/dev/nvidia0", "/dev/dri/renderD128")),
"capabilities": [
"foam_stepper_python_bridge",
"openfoam_case_loader",
"oracle_comparison",
],
"used_cpu_fallback": False,
}
def run_backend_iteration(stepper: Any, backend: Mapping[str, Any], case: Path) -> dict[str, Any]:
selected = backend.get("selected")
if selected == "gpu":
return _call_gpu_backend("run_backend_iteration", stepper, backend, case)
if selected != "cpu":
raise BackendExecutionError(
"execute_backend",
f"backend {selected!r} is not executable",
details={"backend": backend, "case": case},
)
result = stepper.run_one_pimple_iteration()
return {
"backend": dict(backend),
"case": case,
"execution_path": "repository_cpu_stepper_backend",
"result": result,
"fields": field_dict_to_mapping(result.outputs["fields"]),
}
def run_gpu_split_iteration(foam: Any, stepper: Any, backend: Mapping[str, Any], case: Path) -> dict[str, Any]:
return _call_gpu_backend("run_gpu_split_iteration", foam, stepper, backend, case)
def make_stepper(foam: Any, case: Path, label: str) -> Any:
try:
return foam.Case(case).make_stepper()
except Exception as exc:
raise HarnessError(
LOADING_FAILURE,
f"load_{label}_case",
f"failed to load {label} case with foam_stepper: {case}",
details={"case": case},
cause=exc,
) from exc
def read_mesh_identity(stepper: Any, label: str) -> dict[str, Any]:
try:
return mesh_identity(stepper.mesh())
except Exception as exc:
raise HarnessError(
LOADING_FAILURE,
f"read_{label}_mesh",
f"failed to read {label} mesh identity",
details={"case": getattr(stepper, "case_path", "")},
cause=exc,
) from exc
def read_fields(stepper: Any, label: str) -> dict[str, Any]:
try:
return field_dict_to_mapping(stepper.fields())
except Exception as exc:
raise HarnessError(
LOADING_FAILURE,
f"read_{label}_fields",
f"failed to read {label} fields",
details={"case": getattr(stepper, "case_path", "")},
cause=exc,
) from exc
def validation_details(exc: BaseException) -> dict[str, Any]:
if hasattr(exc, "to_dict"):
return json_ready(exc.to_dict())
return {"type": type(exc).__name__, "message": str(exc)}
def export_solver_state_checked(foam: Any, stepper: Any, label: str, fields: Mapping[str, Any] | None = None) -> dict[str, Any]:
try:
field_source = fields if fields is not None else stepper.fields()
return foam.export_solver_state(stepper.mesh(), field_source, required_fields=REQUIRED_FIELDS)
except Exception as exc:
raise HarnessError(
LOADING_FAILURE,
f"export_{label}_state",
f"failed to export explicit {label} solver state",
details=validation_details(exc),
cause=exc,
) from exc
def export_matrix_state_checked(foam: Any, matrix: Any, solver_state: Mapping[str, Any], label: str) -> dict[str, Any]:
try:
return foam.export_matrix_state(matrix, mesh_state=solver_state)
except Exception as exc:
raise HarnessError(
STEPPER_FAILURE,
f"export_{label}_matrix_state",
f"failed to export explicit {label} matrix state",
details=validation_details(exc),
cause=exc,
) from exc
def checked_step(stages: list[dict[str, Any]], name: str, fn: Any) -> Any:
try:
result = fn()
except Exception as exc:
raise HarnessError(
STEPPER_FAILURE,
name,
f"split-step stage failed: {name}",
cause=exc,
) from exc
stages.append(transform_summary(result))
return result
def run_split_iteration(foam: Any, stepper: Any) -> dict[str, Any]:
stages: list[dict[str, Any]] = []
checked_step(stages, "pre_solve", stepper.pre_solve)
checked_step(stages, "advance_time", stepper.advance_time)
begin = checked_step(stages, "begin_pimple_iteration", stepper.begin_pimple_iteration)
if begin.outputs.get("active") is not True:
raise HarnessError(
STEPPER_FAILURE,
"begin_pimple_iteration",
"split-step PIMPLE iteration was inactive",
details={"outputs": summarize_value(begin.outputs)},
)
checked_step(stages, "fv_models_correct", stepper.fv_models_correct)
checked_step(stages, "pre_predictor", stepper.pre_predictor)
checked_step(stages, "momentum_transport_predict", stepper.momentum_transport_predictor)
terms = checked_step(stages, "assemble_momentum_terms", stepper.assemble_momentum_terms)
UEqn = checked_step(stages, "assemble_UEqn", stepper.assemble_momentum_matrix)
checked_step(stages, "relax_UEqn", stepper.relax_matrix)
checked_step(stages, "constrain_UEqn", stepper.constrain_matrix)
checked_step(stages, "solve_UEqn", stepper.solve_momentum)
checked_step(stages, "compute_pressure_inputs", stepper.compute_pressure_inputs)
pEqn = checked_step(stages, "assemble_pEqn", stepper.assemble_pressure_matrix)
checked_step(stages, "solve_pEqn", stepper.solve_pressure)
checked_step(stages, "correct_velocity_pressure_flux", stepper.correct_velocity_pressure_flux)
checked_step(stages, "momentum_transport_correct", stepper.momentum_transport_corrector)
checked_step(stages, "end_pimple_iteration", stepper.end_pimple_iteration)
checked_step(stages, "post_solve", lambda: stepper.post_solve(write=False))
try:
fields = field_dict_to_mapping(stepper.fields())
except Exception as exc:
raise HarnessError(
STEPPER_FAILURE,
"split_fields",
"failed to read split-step fields after execution",
cause=exc,
) from exc
solver_state = export_solver_state_checked(foam, stepper, "split", fields)
momentum_terms = [term.get("name", "") for term in terms.outputs.get("terms", [])]
UEqn_matrix = UEqn.outputs["UEqn"]
pEqn_matrix = pEqn.outputs["pEqn"]
matrix_states = {
"UEqn": foam.describe_matrix_state(export_matrix_state_checked(foam, UEqn_matrix, solver_state, "UEqn")),
"pEqn": foam.describe_matrix_state(export_matrix_state_checked(foam, pEqn_matrix, solver_state, "pEqn")),
}
observability = stage_observability_report(
"split",
stages,
evidence={
"split_step_execution": True,
"momentum_terms": momentum_terms,
"matrix_states": sorted(matrix_states),
"turbulence_fields": visible_turbulence_fields(fields),
},
)
return {
"fields": fields,
"state": solver_state,
"state_summary": foam.describe_solver_state(solver_state),
"stages": stages,
"graph": [stage["name"] for stage in stages],
"momentum_terms": momentum_terms,
"UEqn": matrix_summary(UEqn_matrix),
"pEqn": matrix_summary(pEqn_matrix),
"matrix_states": matrix_states,
"observability": observability,
}
def visible_turbulence_fields(fields: Mapping[str, Any]) -> list[str]:
return [name for name in TURBULENCE_FIELDS if name in fields]
def differentiability_report(backend: Mapping[str, Any], *, case_name: str) -> dict[str, Any]:
capabilities = tuple(backend.get("capabilities", ()))
autodiff_available = any(capability in capabilities for capability in ("autodiff", "jvp", "vjp", "gradient"))
status = "differentiable" if autodiff_available else "not_differentiable"
return {
"status": status,
"backend": {
"requested": backend.get("requested"),
"selected": backend.get("selected"),
"provider": backend.get("provider"),
"capabilities": list(capabilities),
},
"variables": DIFFERENTIABILITY_VARIABLES,
"claims": [
{
"name": "current_migrated_solver_path",
"differentiable": autodiff_available,
"variables": DIFFERENTIABILITY_VARIABLES,
"applies_to": {
"modes": ["run_one", "split"],
"fields": list(REQUIRED_FIELDS),
"parameters": list(DIFFERENTIABILITY_VARIABLES["case_parameters"]),
"losses": list(DIFFERENTIABILITY_VARIABLES["losses"]),
},
"evidence": {
"backend_capabilities": list(capabilities),
"reason": "no registered backend capability exposes AD/JVP/VJP for solver updates"
if not autodiff_available
else "backend advertises differentiability capabilities",
},
}
],
"non_differentiable": list(NON_DIFFERENTIABLE_BOUNDARIES),
"custom_gradient_required": list(CUSTOM_GRADIENT_REQUIREMENTS),
"checks": [
{
"name": "solver_sensitivity_against_finite_difference",
"status": "not_run",
"fixed_problem_setup": {
"case": case_name,
"fields": list(REQUIRED_FIELDS),
"mesh_topology": "fixed prepared AirfRANS mesh",
"backend": backend.get("selected"),
},
"computed_sensitivity": None,
"independent_numerical_check": None,
"reason": "no differentiable backend or custom gradient is currently available, so no computed sensitivity is claimed",
}
],
}
def command_timing_summary(command: Mapping[str, Any]) -> dict[str, Any]:
cmd = list(command.get("cmd") or ())
executable = Path(str(cmd[0])).name if cmd else None
log_path = str(command.get("log_path", ""))
role = "openfoam_command"
if executable == "checkMesh":
role = "oracle_mesh_check"
elif executable == "foamRun" and "oracle" in log_path:
role = "oracle_solver"
return {
"role": role,
"executable": executable,
"cmd": cmd,
"duration_seconds": command.get("duration_seconds"),
"timeout_seconds": command.get("timeout_seconds"),
"log_path": command.get("log_path"),
"returncode": command.get("returncode"),
}
def gpu_timing_summary(source: Mapping[str, Any], *, label: str) -> dict[str, Any]:
profiler = source.get("profiler", {}) if isinstance(source.get("profiler"), Mapping) else {}
timing = source.get("timing", {}) if isinstance(source.get("timing"), Mapping) else {}
return {
"label": label,
"execution_path": source.get("execution_path"),
"timing": timing,
"profiler": {
"available": profiler.get("available"),
"profile_record_count": profiler.get("profile_record_count"),
"device_time_ms_total": profiler.get("device_time_ms_total"),
"expected_kernel_entrypoints": profiler.get("expected_kernel_entrypoints"),
"generated_kernel_names": profiler.get("generated_kernel_names"),
},
}
def build_timing_evidence(report: Mapping[str, Any]) -> dict[str, Any]:
commands = [command_timing_summary(command) for command in report.get("commands", [])]
oracle_solver = next((command for command in commands if command.get("role") == "oracle_solver"), None)
gpu_sources: dict[str, Any] = {}
gpu_stage_smoke = report.get("gpu_solver_stages")
if isinstance(gpu_stage_smoke, Mapping) and gpu_stage_smoke.get("status") == "executed":
gpu_sources["stage_preflight"] = gpu_timing_summary(gpu_stage_smoke, label="stage_preflight")
for mode in ("run_one", "split"):
mode_gpu = report.get("modes", {}).get(mode, {}).get("gpu_solver", {})
if isinstance(mode_gpu, Mapping) and mode_gpu.get("status") == "executed":
gpu_sources[mode] = gpu_timing_summary(mode_gpu, label=mode)
gpu_solver_timing_ok = any(
isinstance(source.get("timing"), Mapping) and source["timing"].get("gpu_solver_wall_seconds") is not None
for source in gpu_sources.values()
)
gpu_profiler_ok = any(
isinstance(source.get("profiler"), Mapping) and source["profiler"].get("available") is True and source["profiler"].get("profile_record_count", 0) > 0
for source in gpu_sources.values()
)
openfoam_timing_ok = oracle_solver is not None and oracle_solver.get("duration_seconds") is not None
return {
"schema_version": 1,
"status": "reported" if commands or gpu_sources else "not_reported",
"openfoam_reference": {
"oracle_solver_duration_seconds": None if oracle_solver is None else oracle_solver.get("duration_seconds"),
"commands": commands,
},
"gpu_solver": gpu_sources,
"criteria": {
"openfoam_reference_timing": openfoam_timing_ok,
"gpu_solver_wall_timing": gpu_solver_timing_ok,
"gpu_profiler_timing": gpu_profiler_ok,
},
}
def enabled_mode_names(report: Mapping[str, Any]) -> list[str]:
return [mode for mode in ("run_one", "split") if report.get("modes", {}).get(mode, {}).get("enabled", False)]
def comparison_regression_summary(comparisons: Mapping[str, Any]) -> dict[str, Any]:
out: dict[str, Any] = {}
for field in REQUIRED_FIELDS:
data = comparisons.get(field, {})
out[field] = {
"actual_shape": data.get("actual_shape"),
"expected_shape": data.get("expected_shape"),
"shape_matches": data.get("shape_matches"),
"allclose": data.get("allclose"),
"max_abs": data.get("max_abs"),
"mean_abs": data.get("mean_abs"),
"tolerance_at_max": data.get("tolerance_at_max"),
"location": data.get("location"),
}
return out
def build_verifier_evidence(report: Mapping[str, Any]) -> dict[str, Any]:
modes = enabled_mode_names(report)
prepared_identity = report.get("case_preparation", {}).get("prepared_case_identity", {})
prepared_ok = bool(prepared_identity) and all(identity.get("matches_prepared") for identity in prepared_identity.values())
mesh_ok_by_mode = {
mode: bool(report.get("modes", {}).get(mode, {}).get("mesh_comparison", {}).get("matches_oracle"))
for mode in modes
}
comparison_ok_by_mode = {}
observability_ok_by_mode = {}
regression_modes: dict[str, Any] = {}
for mode in modes:
mode_report = report.get("modes", {}).get(mode, {})
comparisons = mode_report.get("comparisons", {})
comparison_ok_by_mode[mode] = all(
comparisons.get(field, {}).get("shape_matches") is True and comparisons.get(field, {}).get("allclose") is True
for field in REQUIRED_FIELDS
)
observability = report.get("stage_observability", {}).get(mode, {})
groups = observability.get("groups", [])
observability_ok_by_mode[mode] = bool(groups) and all(group.get("observed") is True for group in groups)
regression_modes[mode] = {
"enabled": True,
"case_role": mode,
"mesh_matches_oracle": mesh_ok_by_mode[mode],
"observed_stage_groups": [group.get("name") for group in groups if group.get("observed")],
"fields": comparison_regression_summary(comparisons),
}
timing = report.get("timing", {}) if isinstance(report.get("timing"), Mapping) else {}
timing_criteria = timing.get("criteria", {}) if isinstance(timing.get("criteria"), Mapping) else {}
if report.get("backend", {}).get("selected") == "gpu":
timing_ok = (
timing_criteria.get("openfoam_reference_timing") is True
and timing_criteria.get("gpu_solver_wall_timing") is True
and timing_criteria.get("gpu_profiler_timing") is True
)
else:
timing_ok = timing.get("status") in {"reported", "not_reported"}
criteria = {
"prepared_case_identity": prepared_ok,
"mesh_identity": bool(mesh_ok_by_mode) and all(mesh_ok_by_mode.values()),
"field_comparisons": bool(comparison_ok_by_mode) and all(comparison_ok_by_mode.values()),
"stage_observability": bool(observability_ok_by_mode) and all(observability_ok_by_mode.values()),
"backend_selected": report.get("backend", {}).get("selected") is not None,
"timing_reported": timing_ok,
"differentiability_reported": report.get("differentiability", {}).get("status") in {"differentiable", "not_differentiable"},
}
passed = all(criteria.values())
oracle_mesh = report.get("mesh_identity", {}).get("oracle", {})
prepared_digest = prepared_identity.get("prepared", {}).get("files_sha256")
return {
"passed": passed,
"status_basis": "passed all verifier evidence criteria" if passed else "one or more verifier evidence criteria failed",
"criteria": criteria,
"mode_criteria": {
"mesh_identity": mesh_ok_by_mode,
"field_comparisons": comparison_ok_by_mode,
"stage_observability": observability_ok_by_mode,
},
"entrypoint": report.get("harness_entrypoint", {}),
"regression_summary": {
"schema_version": report.get("harness", {}).get("schema_version"),
"source_case": report.get("problem_boundary", {}).get("airfrans_simulation"),
"prepared_case_files_sha256": prepared_digest,
"backend_selected": report.get("backend", {}).get("selected"),
"differentiability_status": report.get("differentiability", {}).get("status"),
"mesh_topology_sha256": oracle_mesh.get("topology_sha256"),
"mesh_geometry_sha256": oracle_mesh.get("geometry_sha256"),
"modes": regression_modes,
},
}
def base_report(args: argparse.Namespace) -> dict[str, Any]:
report_path = args.report if args.report is not None else args.work / "verifier_report.json"
return {
"harness": {
"name": "airfrans_stepper_verifier",
"spec": "VERIFIER_HARNESS_SPEC.md",
"schema_version": 1,
"scope": AIRFRANS_VERIFIER_SCOPE,
},
"status": "running",
"failure": None,
"paths": {
"root": ROOT,
"source": args.source,
"work": args.work,
"report": report_path,
},
"harness_entrypoint": {
"command": "uv run python scripts/verify_airfrans_stepper.py --work <work-dir> --report <report.json>",
"arguments": {
"source": args.source,
"work": args.work,
"report": report_path,
"backend": args.backend,
"skip_split": args.skip_split,
"rtol": args.rtol,
"atol": args.atol,
},
"prepares_case": True,
"runs_oracle": True,
"runs_migrated_path": True,
"writes_report": True,
"scope": AIRFRANS_VERIFIER_SCOPE,
},
"source_policy": "raw AirfRANS source is read-only; all solver runs use prepared work-directory copies",
"tolerances": {
"rtol": args.rtol,
"atol": args.atol,
"policy": "CPU stepper parity with the repository OpenFOAM oracle must pass np.allclose for every required field.",
},
"comparison_policy": {
"function": "numpy.allclose(actual, expected, rtol, atol, equal_nan=False)",
"scope": "internal arrays for U, p, phi, nut, k, and omega",
"backend_tolerances": {
"cpu": {
"rtol": args.rtol,
"atol": args.atol,
"reason": "strict CPU parity against the repository OpenFOAM oracle",
},
"gpu": {
"rtol": args.rtol,
"atol": args.atol,
"reason": "full GPU RANS backend must own solver execution and satisfy OpenFOAM oracle parity; primitive GPU diagnostics are not acceptance",
"availability_checked_during_backend_selection": True,
"full_solver_guard": FULL_GPU_RANS_GUARD,
"primitive_evidence": {
"name": GPU_PRIMITIVE_NAME,
"path": GPU_PRIMITIVE_PROOF,
"counts_as_full_gpu_rans_solver": False,
},
},
},
},
"backend": {
"requested": args.backend,
"selected": None,
"failure_category": BACKEND_FAILURE,
"execution_scope": AIRFRANS_VERIFIER_SCOPE,
},
"required_fields": {
"primary": list(PRIMARY_FIELDS),
"turbulence": list(TURBULENCE_FIELDS),
"all": list(REQUIRED_FIELDS),
},
"problem_boundary": {
"airfrans_simulation": args.source.name,
"selected_source_case": args.source,
"prepared_case_policy": "prepare one reproducible OpenFOAM v14 case and clone it into isolated oracle/run_one/split execution cases",
"oracle_output": {
"case_role": "oracle",
"producer": "foamRun -solver incompressibleFluid -noFunctionObjects",
"time": "1",
"fields": list(REQUIRED_FIELDS),
},
"repository_outputs": {
"run_one": "migrated run_one path selected by the requested backend; gpu requests must use the GPU solver contract and reject CPU fallback",
"split": "split solver stages selected by the requested backend",
"time": "1",
"fields": list(REQUIRED_FIELDS),
},
"mesh_identity_evidence": [
"n_points",
"n_faces",
"n_internal_faces",
"n_cells",
"patches",
"topology_sha256",
"geometry_sha256",
],
},
"commands": [],
"case_preparation": {},
"mesh_identity": {},
"modes": {
"run_one": {"enabled": True},
"split": {"enabled": not args.skip_split},
},
"tracked_turbulence_fields": {},
"comparison_mismatches": [],
"comparison_attribution": [],
"verifier_evidence": {
"passed": False,
"status_basis": "not_evaluated",
},
"timing": {
"status": "not_evaluated",
},
"differentiability": {
"status": "not_evaluated",
"reason": "backend has not been selected yet",
},
"state_exports": {},
"gpu_solver_inputs": {
"schema_version": GPU_INPUT_SCHEMA_VERSION,
"status": "not_requested",
},
"stage_observability": {
"contract": json_ready(STAGE_OBSERVABILITY_GROUPS),
},
}
def run_harness(args: argparse.Namespace, report: dict[str, Any]) -> None:
try:
prepared = prepare_work_cases(args.source, args.work, include_split=not args.skip_split)
except Exception as exc:
raise HarnessError(
LOADING_FAILURE,
"prepare_cases",
"failed to prepare reproducible AirfRANS work cases",
details={"source": args.source, "work": args.work},
cause=exc,
) from exc
oracle_case = prepared["oracle_case"]
run_one_case = prepared["run_one_case"]
split_case = prepared["split_case"]
report["case_preparation"] = json_ready(prepared)
identity_mismatches = {
role: identity
for role, identity in prepared["prepared_case_identity"].items()
if not identity["matches_prepared"]
}
if identity_mismatches:
raise HarnessError(
LOADING_FAILURE,
"prepare_case_identity",
"prepared execution case copies differ from the reproducible baseline case",
details={"prepared_case_identity": identity_mismatches},
)
try:
backend = select_execution_backend(args.backend)
except BackendExecutionError as exc:
raise exc.to_harness_error() from exc
report["backend"] = json_ready(backend)
report["differentiability"] = json_ready(differentiability_report(backend, case_name=args.source.name))
if backend.get("selected") == "gpu":
try:
foam = import_foam()
except Exception as exc:
raise HarnessError(
LOADING_FAILURE,
"import_foam_stepper",
"failed to import foam_stepper before GPU input preparation",
cause=exc,
) from exc
gpu_input_stepper = make_stepper(foam, run_one_case, "gpu_inputs")
report["mesh_identity"]["gpu_inputs"] = read_mesh_identity(gpu_input_stepper, "gpu_inputs")
try:
gpu_inputs = prepare_gpu_solver_inputs(foam, gpu_input_stepper, run_one_case, prepared, backend)
except BackendExecutionError as exc:
raise exc.to_harness_error() from exc
report["gpu_solver_inputs"] = json_ready(gpu_inputs)
try:
gpu_stage_smoke = run_gpu_solver_stage_smoke(gpu_input_stepper, backend, run_one_case)
except BackendExecutionError as exc:
raise exc.to_harness_error() from exc
report["gpu_solver_stages"] = json_ready(gpu_solver_stage_report(gpu_stage_smoke))
gpu_contract_blocker = gpu_solver_contract_blocker(backend, run_one_case)
if gpu_contract_blocker is not None:
record_gpu_contract_failure(report, gpu_contract_blocker, gpu_inputs)
report["commands"].append(
run_openfoam_command(["checkMesh", "-case", str(oracle_case), "-constant"], log_path=args.work / "checkMesh.log", timeout=120)
)
report["commands"].append(
run_openfoam_command(
["foamRun", "-case", str(oracle_case), "-solver", "incompressibleFluid", "-noFunctionObjects"],
log_path=args.work / "foamRun_oracle.log",
timeout=600,
)
)
try:
patch_start_from_latest(oracle_case)
except Exception as exc:
raise HarnessError(
LOADING_FAILURE,
"select_oracle_latest_time",
"failed to configure oracle case to load latestTime output",
details={"case": oracle_case},
cause=exc,
) from exc
try:
foam = import_foam()
except Exception as exc:
raise HarnessError(
LOADING_FAILURE,
"import_foam_stepper",
"failed to import foam_stepper in the repository OpenFOAM environment",
cause=exc,
) from exc
oracle_stepper = make_stepper(foam, oracle_case, "oracle")
oracle_mesh = read_mesh_identity(oracle_stepper, "oracle")
oracle_fields = read_fields(oracle_stepper, "oracle")
report["mesh_identity"]["oracle"] = oracle_mesh
report["modes"]["oracle"] = {
"case": oracle_case,
"fields": selected_field_summaries(oracle_fields),
}
report["tracked_turbulence_fields"]["oracle"] = visible_turbulence_fields(oracle_fields)
oracle_state = export_solver_state_checked(foam, oracle_stepper, "oracle", oracle_fields)
report["state_exports"]["oracle"] = foam.describe_solver_state(oracle_state)
run_one_stepper = make_stepper(foam, run_one_case, "run_one")
run_one_mesh = read_mesh_identity(run_one_stepper, "run_one")
report["mesh_identity"]["run_one"] = run_one_mesh
mesh_report, mesh_mismatches = compare_mesh_identity("run_one", run_one_mesh, oracle_mesh)
report["modes"]["run_one"]["mesh_comparison"] = mesh_report
try:
backend_run = run_backend_iteration(run_one_stepper, backend, run_one_case)
run_one_result = backend_run["result"]
run_one_fields = backend_run["fields"]
except BackendExecutionError as exc:
raise exc.to_harness_error() from exc
except Exception as exc:
raise HarnessError(
STEPPER_FAILURE,
"run_one_pimple_iteration",
"full one-iteration backend execution failed",
details={"case": run_one_case, "backend": backend},
cause=exc,
) from exc
run_one_stages = graph_stage_summaries(run_one_result)
run_one_observability = stage_observability_report(
"run_one",
run_one_stages,
evidence={
"one_iteration_execution": True,
"compared_fields": list(REQUIRED_FIELDS),
"turbulence_fields": visible_turbulence_fields(run_one_fields),
"backend": backend,
},
)
report["stage_observability"]["run_one"] = run_one_observability
run_one_comparisons, run_one_mismatches = compare_fields(
"run_one",
run_one_fields,
oracle_fields,
rtol=args.rtol,
atol=args.atol,
stage_graph=[stage["name"] for stage in run_one_stages],
)
report["modes"]["run_one"].update(
{
"case": run_one_case,
"execution_path": backend_run["execution_path"],
"backend": backend_run["backend"],
"stage": transform_summary(run_one_result),
"graph": [stage["name"] for stage in run_one_stages],
"fields": selected_field_summaries(run_one_fields),
"comparisons": run_one_comparisons,
"observability": run_one_observability,
}
)
if backend.get("selected") == "gpu":
report["modes"]["run_one"]["gpu_solver"] = json_ready(backend_run.get("gpu_solver", {}))
report["tracked_turbulence_fields"]["run_one"] = visible_turbulence_fields(run_one_fields)
run_one_state = export_solver_state_checked(foam, run_one_stepper, "run_one", run_one_fields)
report["state_exports"]["run_one"] = foam.describe_solver_state(run_one_state)
mismatches = mesh_mismatches + run_one_mismatches
if split_case is not None:
split_stepper = make_stepper(foam, split_case, "split")
split_mesh = read_mesh_identity(split_stepper, "split")
report["mesh_identity"]["split"] = split_mesh
split_mesh_report, split_mesh_mismatches = compare_mesh_identity("split", split_mesh, oracle_mesh)
report["modes"]["split"]["mesh_comparison"] = split_mesh_report
if backend.get("selected") == "gpu":
split_result = run_gpu_split_iteration(foam, split_stepper, backend, split_case)
else:
split_result = run_split_iteration(foam, split_stepper)
report["stage_observability"]["split"] = split_result["observability"]
split_fields = split_result["fields"]
split_comparisons, split_mismatches = compare_fields(
"split",
split_fields,
oracle_fields,
rtol=args.rtol,
atol=args.atol,
stage_graph=split_result["graph"],
)
report["modes"]["split"].update(
{
"case": split_case,
"graph": split_result["graph"],
"stages": split_result["stages"],
"momentum_terms": split_result["momentum_terms"],
"UEqn": split_result["UEqn"],
"pEqn": split_result["pEqn"],
"fields": selected_field_summaries(split_fields),
"comparisons": split_comparisons,
"observability": split_result["observability"],
"execution_path": split_result.get("execution_path"),
"backend": split_result.get("backend"),
"gpu_solver": split_result.get("gpu_solver"),
}
)
report["tracked_turbulence_fields"]["split"] = visible_turbulence_fields(split_fields)
report["state_exports"]["split"] = split_result["state_summary"]
report["state_exports"]["split_matrices"] = split_result["matrix_states"]
mismatches.extend(split_mesh_mismatches)
mismatches.extend(split_mismatches)
comparison_attribution = [mismatch["attribution"] for mismatch in mismatches if "attribution" in mismatch]
report["comparison_attribution"] = json_ready(comparison_attribution)
report["comparison_mismatches"] = json_ready(mismatches)
report["timing"] = json_ready(build_timing_evidence(report))
report["verifier_evidence"] = json_ready(build_verifier_evidence(report))
if mismatches:
raise HarnessError(
COMPARISON_FAILURE,
"compare_oracle_stepper_outputs",
f"{len(mismatches)} verifier comparison mismatch(es) observed",
details={
"failure_kind": GPU_NUMERICAL_MISMATCH if backend.get("selected") == "gpu" else COMPARISON_FAILURE,
"mismatches": mismatches,
"likely_causes": comparison_attribution,
},
)
def write_report(report: Mapping[str, Any], path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(json_ready(report), indent=2, sort_keys=True, allow_nan=False) + "\n")
def fmt_sci(value: Any) -> str:
if value is None:
return "none"
try:
return f"{float(value):.3e}"
except (TypeError, ValueError):
return str(value)
def format_location(location: Mapping[str, Any] | None) -> str:
if not location:
return "none"
component = location.get("component_index")
component_text = "" if component is None else f" component={component}"
return f"{location.get('entity_kind')}[{location.get('entity_index')}]{component_text}"
def print_human_summary(report: Mapping[str, Any]) -> None:
status = report.get("status")
print(f"airfrans_stepper_verification {status}")
print(f"source={report['paths']['source']}")
print(f"work={report['paths']['work']}")
print(f"report={report['paths']['report']}")
entrypoint = report.get("harness_entrypoint") or {}
print(f"harness_entrypoint={entrypoint.get('command')}")
print(f"harness_scope={entrypoint.get('scope') or report.get('harness', {}).get('scope')}")
print(f"rtol={report['tolerances']['rtol']} atol={report['tolerances']['atol']}")
backend = report.get("backend", {})
print(f"backend_requested={backend.get('requested')} backend_selected={backend.get('selected')}")
differentiability = report.get("differentiability") or {}
if differentiability:
claims = differentiability.get("claims") or []
checks = differentiability.get("checks") or []
print(f"differentiability_status={differentiability.get('status')}")
print(f"differentiability_claims={len(claims)} non_differentiable_boundaries={len(differentiability.get('non_differentiable') or [])}")
print(f"differentiability_checks={[check.get('status') for check in checks]}")
evidence = report.get("verifier_evidence") or {}
if evidence:
print(f"evidence_passed={evidence.get('passed')} evidence_status_basis={evidence.get('status_basis')}")
print(f"evidence_criteria={evidence.get('criteria')}")
if status != "passed":
failure = report.get("failure") or {}
print(f"failure_category={failure.get('category')}")
print(f"failure_step={failure.get('step')}")
print(f"failure_message={failure.get('message')}")
timing = report.get("timing") or {}
if timing.get("status") == "reported":
openfoam = timing.get("openfoam_reference") or {}
gpu_solver = timing.get("gpu_solver") or {}
run_one_timing = ((gpu_solver.get("run_one") or {}).get("timing") or {})
run_one_profiler = ((gpu_solver.get("run_one") or {}).get("profiler") or {})
print(f"openfoam_oracle_duration_seconds={openfoam.get('oracle_solver_duration_seconds')}")
print(f"gpu_solver_run_one_wall_seconds={run_one_timing.get('gpu_solver_wall_seconds')}")
print(f"gpu_solver_run_one_device_time_ms={run_one_profiler.get('device_time_ms_total')}")
mismatches = (failure.get("details") or {}).get("mismatches") or report.get("comparison_mismatches") or []
if mismatches:
first = mismatches[0]
attribution = first.get("attribution") or {}
print(f"comparison_failure_mode={first.get('mode')}")
print(f"comparison_failure_field={first.get('field')}")
print(f"comparison_failure_reason={first.get('reason')}")
print(f"comparison_failure_likely_stage_group={attribution.get('likely_stage_group')}")
print(f"comparison_failure_likely_stages={attribution.get('likely_stages')}")
print(f"comparison_failure_max_abs={fmt_sci(first.get('max_abs'))}")
print(f"comparison_failure_location={format_location(first.get('location'))}")
return
oracle_mesh = report["mesh_identity"]["oracle"]
patch_names = [patch["name"] for patch in oracle_mesh["patches"]]
print(f"oracle_case={report['case_preparation']['oracle_case']}")
prepared_identity = report["case_preparation"].get("prepared_case_identity", {})
prepared_digest = prepared_identity.get("prepared", {}).get("files_sha256")
print(f"prepared_case={report['case_preparation']['prepared_case']}")
print(f"prepared_case_files_sha256={prepared_digest}")
print(f"run_one_case={report['modes']['run_one']['case']}")
if report["modes"].get("split", {}).get("enabled"):
print(f"split_case={report['modes']['split']['case']}")
print(f"mesh_cells={oracle_mesh['n_cells']}")
print(f"mesh_internal_faces={oracle_mesh['n_internal_faces']}")
print(f"mesh_patches={patch_names}")
print(f"mesh_topology_sha256={oracle_mesh['topology_sha256']}")
for mode in ("run_one", "split"):
mode_report = report["modes"].get(mode, {})
if not mode_report.get("enabled", False):
continue
for field_name in REQUIRED_FIELDS:
data = mode_report.get("comparisons", {}).get(field_name)
if not data:
continue
print(
f"mode={mode} field={field_name} actual_shape={tuple(data['actual_shape'])} "
f"expected_shape={tuple(data['expected_shape'])} max_abs={fmt_sci(data.get('max_abs'))} "
f"location={format_location(data.get('location'))} allclose={data['allclose']}"
)
graph = mode_report.get("graph")
if graph:
print(f"{mode}_graph={graph}")
observability = mode_report.get("observability")
if observability:
observed_groups = [group["name"] for group in observability.get("groups", []) if group.get("observed")]
print(f"{mode}_observed_stage_groups={observed_groups}")
split_report = report["modes"].get("split", {})
if split_report.get("enabled") and "UEqn" in split_report and "pEqn" in split_report:
print(f"split_momentum_terms={split_report.get('momentum_terms')}")
print(f"split_UEqn_diag_shape={tuple(split_report['UEqn']['diag']['shape'])}")
print(f"split_pEqn_diag_shape={tuple(split_report['pEqn']['diag']['shape'])}")
print(f"visible_turbulence_fields={report['tracked_turbulence_fields']}")
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--source", type=Path, default=DEFAULT_SOURCE)
parser.add_argument("--work", type=Path, default=DEFAULT_WORK)
parser.add_argument("--report", type=Path, default=None, help="JSON report path; defaults to WORK/verifier_report.json")
parser.add_argument("--rtol", type=float, default=1e-8)
parser.add_argument("--atol", type=float, default=1e-8)
parser.add_argument("--backend", choices=BACKEND_CHOICES, default="auto", help="Backend request; gpu means the full RANS solver backend and currently fails until that backend exists")
parser.add_argument("--skip-split", action="store_true", help="Skip the explicit split-step parity check for local debugging")
return parser.parse_args(argv)
def main(argv: list[str] | None = None) -> int:
args = parse_args(argv)
args.source = args.source.resolve()
args.work = args.work.resolve()
if args.report is None:
args.report = args.work / "verifier_report.json"
else:
args.report = args.report.resolve()
report = base_report(args)
exit_code = 0
try:
run_harness(args, report)
evidence = report.get("verifier_evidence")
if not isinstance(evidence, Mapping) or evidence.get("passed") is not True:
raise HarnessError(
INTERNAL_FAILURE,
"verifier_evidence",
"verifier completed without passing evidence criteria",
details={"verifier_evidence": evidence},
)
report["status"] = "passed"
except HarnessError as exc:
report["status"] = "failed"
report["failure"] = exc.to_dict()
exit_code = EXIT_CODES.get(exc.category, EXIT_CODES[INTERNAL_FAILURE])
except Exception as exc: # pragma: no cover - keeps CLI failures categorized in the report.
report["status"] = "failed"
report["failure"] = json_ready(
{
"category": INTERNAL_FAILURE,
"step": "run_harness",
"message": str(exc),
"cause": {"type": type(exc).__name__, "message": str(exc)},
"traceback": traceback.format_exc(),
}
)
exit_code = EXIT_CODES[INTERNAL_FAILURE]
try:
write_report(report, args.report)
except Exception as exc:
report["status"] = "failed"
report["failure"] = json_ready(
{
"category": INTERNAL_FAILURE,
"step": "write_report",
"message": f"failed to write verifier report: {args.report}",
"cause": {"type": type(exc).__name__, "message": str(exc)},
}
)
exit_code = EXIT_CODES[INTERNAL_FAILURE]
print_human_summary(json_ready(report))
return exit_code
if __name__ == "__main__":
raise SystemExit(main())