1936 lines
78 KiB
Python
Executable file
1936 lines
78 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Verifier harness for the migrated AirfRANS OpenFOAM stepper case."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import dataclasses
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import traceback
|
|
from collections.abc import Iterable, Mapping
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
def _drop_ambient_pythonpath() -> None:
|
|
pythonpath = os.environ.pop("PYTHONPATH", "")
|
|
for entry in pythonpath.split(os.pathsep):
|
|
if not entry:
|
|
continue
|
|
while entry in sys.path:
|
|
sys.path.remove(entry)
|
|
|
|
|
|
_drop_ambient_pythonpath()
|
|
|
|
import numpy as np
|
|
|
|
from openfoam_env import apply_openfoam_env, openfoam_env
|
|
from prepare_airfrans_stepper_case import (
|
|
DEFAULT_SOURCE,
|
|
REQUIRED_COMPARISON_FIELDS,
|
|
load_case_manifest,
|
|
prepare_case,
|
|
write_case_manifest,
|
|
)
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
DEFAULT_WORK = ROOT / "tmp/airfrans_stepper_verify"
|
|
PRIMARY_FIELDS = ("U", "p", "phi")
|
|
TURBULENCE_FIELDS = ("nut", "k", "omega")
|
|
REQUIRED_FIELDS = REQUIRED_COMPARISON_FIELDS
|
|
BACKEND_CHOICES = ("auto", "cpu", "gpu")
|
|
FULL_GPU_RANS_GUARD = "scripts/verify_gpu_rans_solver.sh"
|
|
GPU_PRIMITIVE_PROOF = "scripts/verify_gpu_algorithm.sh"
|
|
GPU_PRIMITIVE_NAME = "cell_flux_imbalance"
|
|
AIRFRANS_VERIFIER_SCOPE = (
|
|
"AirfRANS/OpenFOAM stepper parity verifier; --backend gpu is a GPU-owned "
|
|
"RANS solver contract with no CPU fallback or primitive-only acceptance"
|
|
)
|
|
|
|
LOADING_FAILURE = "loading_or_parsing_failure"
|
|
ORACLE_FAILURE = "openfoam_oracle_failure"
|
|
STEPPER_FAILURE = "stepper_execution_failure"
|
|
COMPARISON_FAILURE = "numerical_comparison_failure"
|
|
BACKEND_FAILURE = "backend_execution_failure"
|
|
INTERNAL_FAILURE = "internal_harness_failure"
|
|
|
|
EXIT_CODES = {
|
|
LOADING_FAILURE: 2,
|
|
ORACLE_FAILURE: 3,
|
|
STEPPER_FAILURE: 4,
|
|
COMPARISON_FAILURE: 5,
|
|
BACKEND_FAILURE: 6,
|
|
INTERNAL_FAILURE: 7,
|
|
}
|
|
|
|
STAGE_OBSERVABILITY_GROUPS = (
|
|
{
|
|
"name": "momentum_assembly",
|
|
"split_stages": ("assemble_momentum_terms", "assemble_UEqn"),
|
|
"run_one_stages": ("assemble_UEqn",),
|
|
"split_outputs": {
|
|
"assemble_momentum_terms": ("terms",),
|
|
"assemble_UEqn": ("UEqn",),
|
|
},
|
|
},
|
|
{
|
|
"name": "pressure_assembly",
|
|
"split_stages": ("compute_pressure_inputs", "assemble_pEqn"),
|
|
"run_one_stages": ("compute_pressure_inputs", "assemble_pEqn"),
|
|
"split_outputs": {
|
|
"compute_pressure_inputs": ("HbyA", "phiHbyA", "rAU"),
|
|
"assemble_pEqn": ("pEqn",),
|
|
},
|
|
},
|
|
{
|
|
"name": "linear_solve_results",
|
|
"split_stages": ("solve_UEqn", "solve_pEqn"),
|
|
"run_one_stages": ("solve_UEqn", "solve_pEqn"),
|
|
"split_outputs": {
|
|
"solve_UEqn": ("performance", "field_after"),
|
|
"solve_pEqn": ("performance", "p", "phi"),
|
|
},
|
|
},
|
|
{
|
|
"name": "final_correction",
|
|
"split_stages": ("correct_velocity_pressure_flux",),
|
|
"run_one_stages": ("update_phi_from_pEqn_flux", "correct_velocity_pressure_flux"),
|
|
"split_outputs": {
|
|
"correct_velocity_pressure_flux": ("U", "p", "phi"),
|
|
},
|
|
},
|
|
{
|
|
"name": "turbulence_updates",
|
|
"split_stages": ("momentum_transport_predict", "momentum_transport_correct"),
|
|
"run_one_stages": ("momentum_transport_predict", "momentum_transport_correct"),
|
|
"split_outputs": {
|
|
"momentum_transport_predict": ("case_path", "solver_name"),
|
|
"momentum_transport_correct": ("U", "p", "phi", "nut", "k", "omega"),
|
|
},
|
|
},
|
|
)
|
|
|
|
FIELD_COMPARISON_ATTRIBUTION = {
|
|
"U": ("final_correction", "linear_solve_results", "momentum_assembly", "turbulence_updates"),
|
|
"p": ("linear_solve_results", "pressure_assembly", "final_correction"),
|
|
"phi": ("final_correction", "linear_solve_results", "pressure_assembly"),
|
|
"nut": ("turbulence_updates",),
|
|
"k": ("turbulence_updates",),
|
|
"omega": ("turbulence_updates",),
|
|
}
|
|
|
|
GPU_RUNTIME_UNAVAILABLE = "gpu_runtime_unavailable"
|
|
GPU_NUMERICAL_MISMATCH = "gpu_numerical_mismatch"
|
|
GPU_INPUT_SCHEMA_VERSION = 1
|
|
|
|
DIFFERENTIABILITY_VARIABLES = {
|
|
"fields": REQUIRED_FIELDS,
|
|
"mesh_state": ("points", "faces", "owner", "neighbour", "cells", "V", "C", "Cf", "Sf", "magSf", "patches"),
|
|
"case_parameters": ("Uinf", "nu", "rhoInf", "angle_of_attack", "liftDir", "dragDir", "solver_tolerances"),
|
|
"losses": ("per_field_linf_error", "per_field_l2_error", "oracle_parity_allclose"),
|
|
}
|
|
|
|
NON_DIFFERENTIABLE_BOUNDARIES = (
|
|
{
|
|
"name": "openfoam_case_io",
|
|
"category": "io",
|
|
"reason": "OpenFOAM dictionaries, mesh files, and field files are parsed and written as discrete external data.",
|
|
},
|
|
{
|
|
"name": "mesh_topology_and_patch_addressing",
|
|
"category": "topology",
|
|
"reason": "Face/cell connectivity, owner/neighbour addressing, and boundary patch membership are discrete indices.",
|
|
},
|
|
{
|
|
"name": "boundary_condition_selection",
|
|
"category": "discrete_boundary_choice",
|
|
"reason": "Patch types, coupled/constraint flags, and boundary condition dictionaries select code paths.",
|
|
},
|
|
{
|
|
"name": "simple_pimple_and_linear_solver_control",
|
|
"category": "convergence_control",
|
|
"reason": "Iteration counts, stopping criteria, and solver convergence branches are discrete control flow.",
|
|
},
|
|
{
|
|
"name": "turbulence_bounding_and_limiters",
|
|
"category": "limiter",
|
|
"reason": "k/omega/nut bounding and model limiters introduce clipping or branch-dependent updates.",
|
|
},
|
|
{
|
|
"name": "openfoam_cpp_backend_calls",
|
|
"category": "unsupported_solver_operation",
|
|
"reason": "The current registered backend executes OpenFOAM C++ operations and exposes no AD, JVP, or VJP API.",
|
|
},
|
|
)
|
|
|
|
CUSTOM_GRADIENT_REQUIREMENTS = (
|
|
{
|
|
"operation": "sparse_linear_solves",
|
|
"stage_groups": ("linear_solve_results",),
|
|
"reason": "Implicit sparse solves require an adjoint or custom VJP rather than differentiating solver iterations blindly.",
|
|
},
|
|
{
|
|
"operation": "pressure_velocity_correction",
|
|
"stage_groups": ("pressure_assembly", "final_correction"),
|
|
"reason": "The coupled pressure/flux/velocity correction needs a consistent custom gradient for the assembled operators.",
|
|
},
|
|
{
|
|
"operation": "turbulence_model_correctors_and_limiters",
|
|
"stage_groups": ("turbulence_updates",),
|
|
"reason": "Model correctors, wall functions, and bounding need explicit subgradient or smoothing choices.",
|
|
},
|
|
{
|
|
"operation": "mesh_and_boundary_discrete_choices",
|
|
"stage_groups": (),
|
|
"reason": "Topology and patch-type changes are not differentiable; only fixed-topology numeric values can be checked.",
|
|
},
|
|
)
|
|
|
|
|
|
class HarnessError(Exception):
|
|
"""Categorized verifier failure that can be serialized into the report."""
|
|
|
|
def __init__(
|
|
self,
|
|
category: str,
|
|
step: str,
|
|
message: str,
|
|
*,
|
|
details: Mapping[str, Any] | None = None,
|
|
cause: BaseException | None = None,
|
|
) -> None:
|
|
super().__init__(message)
|
|
self.category = category
|
|
self.step = step
|
|
self.details = dict(details or {})
|
|
self.cause = cause
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
out: dict[str, Any] = {
|
|
"category": self.category,
|
|
"step": self.step,
|
|
"message": str(self),
|
|
"details": self.details,
|
|
}
|
|
if self.cause is not None:
|
|
out["cause"] = {
|
|
"type": type(self.cause).__name__,
|
|
"message": str(self.cause),
|
|
}
|
|
return json_ready(out)
|
|
|
|
class BackendExecutionError(Exception):
|
|
"""Backend selection or execution failure before numerical comparison."""
|
|
|
|
def __init__(self, step: str, message: str, *, details: Mapping[str, Any] | None = None) -> None:
|
|
super().__init__(message)
|
|
self.step = step
|
|
self.details = dict(details or {})
|
|
|
|
def to_harness_error(self) -> HarnessError:
|
|
return HarnessError(BACKEND_FAILURE, self.step, str(self), details=self.details, cause=self)
|
|
|
|
|
|
|
|
|
|
def json_ready(value: Any) -> Any:
|
|
"""Convert report values into strict JSON-compatible data."""
|
|
|
|
if dataclasses.is_dataclass(value) and not isinstance(value, type):
|
|
return json_ready(dataclasses.asdict(value))
|
|
if isinstance(value, Path):
|
|
return str(value)
|
|
if isinstance(value, np.ndarray):
|
|
return array_stats(value)
|
|
if isinstance(value, np.generic):
|
|
return json_ready(value.item())
|
|
if isinstance(value, float):
|
|
return value if math.isfinite(value) else None
|
|
if isinstance(value, (str, int, bool)) or value is None:
|
|
return value
|
|
if isinstance(value, Mapping):
|
|
return {str(key): json_ready(item) for key, item in value.items()}
|
|
if isinstance(value, (list, tuple, set)):
|
|
return [json_ready(item) for item in value]
|
|
return repr(value)
|
|
|
|
|
|
def array_shape(array: Any | None) -> list[int] | None:
|
|
if array is None:
|
|
return None
|
|
return [int(dim) for dim in np.asarray(array).shape]
|
|
|
|
|
|
def finite_float(value: Any) -> float | None:
|
|
try:
|
|
number = float(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
return number if math.isfinite(number) else None
|
|
|
|
|
|
def array_stats(array: Any) -> dict[str, Any]:
|
|
arr = np.asarray(array)
|
|
out: dict[str, Any] = {
|
|
"shape": array_shape(arr),
|
|
"dtype": str(arr.dtype),
|
|
"size": int(arr.size),
|
|
}
|
|
if arr.size == 0 or not np.issubdtype(arr.dtype, np.number):
|
|
return out
|
|
|
|
finite = np.isfinite(arr)
|
|
out["finite_count"] = int(np.count_nonzero(finite))
|
|
out["nonfinite_count"] = int(arr.size - out["finite_count"])
|
|
if out["finite_count"]:
|
|
finite_values = arr[finite]
|
|
out.update(
|
|
{
|
|
"min": finite_float(np.min(finite_values)),
|
|
"max": finite_float(np.max(finite_values)),
|
|
"mean": finite_float(np.mean(finite_values)),
|
|
}
|
|
)
|
|
return out
|
|
|
|
|
|
def value_at(array: np.ndarray, index: tuple[int, ...]) -> Any:
|
|
value = array[index]
|
|
if isinstance(value, np.generic):
|
|
return json_ready(value.item())
|
|
if isinstance(value, np.ndarray):
|
|
return json_ready(value.tolist())
|
|
return json_ready(value)
|
|
|
|
|
|
def entity_value_at(array: np.ndarray, entity_index: int | None) -> Any:
|
|
if entity_index is None or array.ndim == 0:
|
|
return None
|
|
value = array[entity_index]
|
|
if isinstance(value, np.generic):
|
|
return json_ready(value.item())
|
|
if isinstance(value, np.ndarray):
|
|
return json_ready(value.tolist())
|
|
return json_ready(value)
|
|
|
|
|
|
def update_hash_text(digest: "hashlib._Hash", value: str) -> None:
|
|
encoded = value.encode("utf-8")
|
|
digest.update(len(encoded).to_bytes(8, "little"))
|
|
digest.update(encoded)
|
|
|
|
|
|
def update_hash_array(digest: "hashlib._Hash", label: str, array: Any) -> None:
|
|
arr = np.ascontiguousarray(np.asarray(array))
|
|
update_hash_text(digest, label)
|
|
update_hash_text(digest, str(arr.dtype))
|
|
update_hash_text(digest, repr(tuple(int(dim) for dim in arr.shape)))
|
|
digest.update(arr.tobytes())
|
|
|
|
|
|
def source_summary(source: Any) -> dict[str, Any]:
|
|
return {
|
|
"file": getattr(source, "file", ""),
|
|
"function": getattr(source, "function", ""),
|
|
"lines": json_ready(getattr(source, "lines", None)),
|
|
}
|
|
|
|
|
|
def boundary_field_summary(patch: Any) -> dict[str, Any]:
|
|
return {
|
|
"type": getattr(patch, "type", ""),
|
|
"values_shape": array_shape(getattr(patch, "values", None)),
|
|
"fixes_value": bool(getattr(patch, "fixes_value", False)),
|
|
"assignable": bool(getattr(patch, "assignable", False)),
|
|
"coupled": bool(getattr(patch, "coupled", False)),
|
|
"updated": bool(getattr(patch, "updated", False)),
|
|
"patch_internal_shape": array_shape(getattr(patch, "patch_internal", None)),
|
|
"value_internal_coeffs_shape": array_shape(getattr(patch, "value_internal_coeffs", None)),
|
|
"value_boundary_coeffs_shape": array_shape(getattr(patch, "value_boundary_coeffs", None)),
|
|
"gradient_internal_coeffs_shape": array_shape(getattr(patch, "gradient_internal_coeffs", None)),
|
|
"gradient_boundary_coeffs_shape": array_shape(getattr(patch, "gradient_boundary_coeffs", None)),
|
|
}
|
|
|
|
|
|
def field_summary(field: Any) -> dict[str, Any]:
|
|
boundary = getattr(field, "boundary", {})
|
|
return {
|
|
"name": getattr(field, "name", ""),
|
|
"kind": getattr(field, "kind", ""),
|
|
"dimensions": getattr(field, "dimensions", ""),
|
|
"entity_kind": getattr(field, "entity_kind", ""),
|
|
"entity_count": int(getattr(field, "entity_count", 0)),
|
|
"internal": array_stats(getattr(field, "internal")),
|
|
"boundary": {name: boundary_field_summary(patch) for name, patch in boundary.items()},
|
|
}
|
|
|
|
|
|
def solve_summary(performance: Any) -> dict[str, Any]:
|
|
return {
|
|
"solver_name": getattr(performance, "solver_name", ""),
|
|
"field_name": getattr(performance, "field_name", ""),
|
|
"initial_residual": json_ready(getattr(performance, "initial_residual", None)),
|
|
"final_residual": json_ready(getattr(performance, "final_residual", None)),
|
|
"n_iterations": json_ready(getattr(performance, "n_iterations", None)),
|
|
"converged": bool(getattr(performance, "converged", False)),
|
|
"singular": bool(getattr(performance, "singular", False)),
|
|
}
|
|
|
|
|
|
def matrix_summary(matrix: Any) -> dict[str, Any]:
|
|
derived: dict[str, Any] = {}
|
|
for name in ("A", "H", "H1", "flux", "face_flux_correction"):
|
|
value = getattr(matrix, name, None)
|
|
if callable(value):
|
|
value = value()
|
|
if value is not None:
|
|
derived[name] = field_summary(value)
|
|
|
|
for name in ("residual", "D", "DD"):
|
|
value = getattr(matrix, name, None)
|
|
if callable(value):
|
|
value = value()
|
|
if value is not None:
|
|
derived[name] = array_stats(value)
|
|
|
|
return {
|
|
"name": getattr(matrix, "name", ""),
|
|
"field_name": getattr(matrix, "field_name", ""),
|
|
"value_rank": getattr(matrix, "value_rank", ""),
|
|
"dimensions": getattr(matrix, "dimensions", ""),
|
|
"has_diag": bool(getattr(matrix, "has_diag", False)),
|
|
"has_upper": bool(getattr(matrix, "has_upper", False)),
|
|
"has_lower": bool(getattr(matrix, "has_lower", False)),
|
|
"diagonal": bool(getattr(matrix, "diagonal", False)),
|
|
"symmetric": bool(getattr(matrix, "symmetric", False)),
|
|
"asymmetric": bool(getattr(matrix, "asymmetric", False)),
|
|
"diag": array_stats(getattr(matrix, "diag")),
|
|
"upper": None if getattr(matrix, "upper", None) is None else array_stats(getattr(matrix, "upper")),
|
|
"lower": None if getattr(matrix, "lower", None) is None else array_stats(getattr(matrix, "lower")),
|
|
"source": array_stats(getattr(matrix, "source")),
|
|
"psi": field_summary(getattr(matrix, "psi")),
|
|
"internal_coeff_shapes": [array_shape(item) for item in getattr(matrix, "internal_coeffs", [])],
|
|
"boundary_coeff_shapes": [array_shape(item) for item in getattr(matrix, "boundary_coeffs", [])],
|
|
"derived": derived,
|
|
}
|
|
|
|
|
|
def summarize_value(value: Any) -> Any:
|
|
if hasattr(value, "diag") and hasattr(value, "field_name") and hasattr(value, "source"):
|
|
return matrix_summary(value)
|
|
if hasattr(value, "internal") and hasattr(value, "entity_kind") and hasattr(value, "boundary"):
|
|
return field_summary(value)
|
|
if hasattr(value, "solver_name") and hasattr(value, "initial_residual"):
|
|
return solve_summary(value)
|
|
if hasattr(value, "name") and hasattr(value, "phase") and hasattr(value, "outputs"):
|
|
return transform_summary(value)
|
|
if isinstance(value, Mapping):
|
|
return {str(key): summarize_value(item) for key, item in value.items()}
|
|
if isinstance(value, list):
|
|
return [summarize_value(item) for item in value]
|
|
if isinstance(value, tuple):
|
|
return [summarize_value(item) for item in value]
|
|
return json_ready(value)
|
|
|
|
|
|
def transform_summary(result: Any) -> dict[str, Any]:
|
|
return {
|
|
"name": getattr(result, "name", ""),
|
|
"phase": getattr(result, "phase", ""),
|
|
"changed_fields": json_ready(getattr(result, "changed_fields", [])),
|
|
"source": source_summary(getattr(result, "source", None)),
|
|
"metadata": json_ready(getattr(result, "metadata", {})),
|
|
"outputs": summarize_value(getattr(result, "outputs", {})),
|
|
}
|
|
|
|
def graph_stage_summaries(result: Any) -> list[dict[str, Any]]:
|
|
return [transform_summary(entry) for entry in getattr(result, "outputs", {}).get("graph", [])]
|
|
|
|
|
|
def stage_observability_report(mode: str, stages: list[dict[str, Any]], *, evidence: Mapping[str, Any]) -> dict[str, Any]:
|
|
stage_by_name = {stage["name"]: stage for stage in stages}
|
|
failures: list[dict[str, Any]] = []
|
|
groups = []
|
|
for group in STAGE_OBSERVABILITY_GROUPS:
|
|
group_name = group["name"]
|
|
required_stages = tuple(group[f"{mode}_stages"])
|
|
missing_stages = [name for name in required_stages if name not in stage_by_name]
|
|
output_requirements = group.get(f"{mode}_outputs", {})
|
|
output_checks = {}
|
|
for stage_name, required_outputs in output_requirements.items():
|
|
stage = stage_by_name.get(stage_name)
|
|
if stage is None:
|
|
continue
|
|
outputs = stage.get("outputs", {})
|
|
output_keys = set(outputs) if isinstance(outputs, Mapping) else set()
|
|
missing_outputs = [name for name in required_outputs if name not in output_keys]
|
|
output_checks[stage_name] = {
|
|
"required": list(required_outputs),
|
|
"available": sorted(output_keys),
|
|
"missing": missing_outputs,
|
|
}
|
|
if missing_outputs:
|
|
failures.append(
|
|
{
|
|
"path": f"stage_observability.{mode}.{group_name}.{stage_name}.outputs",
|
|
"message": "missing inspectable stage output",
|
|
"expected": list(required_outputs),
|
|
"actual": sorted(output_keys),
|
|
}
|
|
)
|
|
if missing_stages:
|
|
failures.append(
|
|
{
|
|
"path": f"stage_observability.{mode}.{group_name}.stages",
|
|
"message": "missing required solver stage",
|
|
"expected": list(required_stages),
|
|
"actual": list(stage_by_name),
|
|
}
|
|
)
|
|
groups.append(
|
|
{
|
|
"name": group_name,
|
|
"required_stages": list(required_stages),
|
|
"observed": not missing_stages and not any(check["missing"] for check in output_checks.values()),
|
|
"missing_stages": missing_stages,
|
|
"output_checks": output_checks,
|
|
}
|
|
)
|
|
if failures:
|
|
raise HarnessError(
|
|
STEPPER_FAILURE,
|
|
f"{mode}_stage_observability",
|
|
f"{mode} solver-stage observability is incomplete",
|
|
details={"failures": failures},
|
|
)
|
|
return {
|
|
"mode": mode,
|
|
"graph": [stage["name"] for stage in stages],
|
|
"groups": groups,
|
|
"evidence": json_ready(evidence),
|
|
}
|
|
|
|
|
|
def selected_field_summaries(fields: Mapping[str, Any]) -> dict[str, Any]:
|
|
return {name: field_summary(fields[name]) for name in REQUIRED_FIELDS if name in fields}
|
|
|
|
|
|
def field_dict_to_mapping(fields: Any) -> dict[str, Any]:
|
|
if isinstance(fields, Mapping):
|
|
return dict(fields)
|
|
if hasattr(fields, "fields") and isinstance(fields.fields, Mapping):
|
|
return dict(fields.fields)
|
|
return {name: getattr(fields, name) for name in REQUIRED_FIELDS if hasattr(fields, name)}
|
|
|
|
|
|
def mesh_patch_table(mesh: Any) -> list[dict[str, Any]]:
|
|
patches = []
|
|
for patch in getattr(mesh, "boundary", []):
|
|
patches.append(
|
|
{
|
|
"index": int(getattr(patch, "index", 0)),
|
|
"name": getattr(patch, "name", ""),
|
|
"type": getattr(patch, "type", ""),
|
|
"start": int(getattr(patch, "start", 0)),
|
|
"size": int(getattr(patch, "size", 0)),
|
|
"coupled": bool(getattr(patch, "coupled", False)),
|
|
"constraint": bool(getattr(patch, "constraint", False)),
|
|
}
|
|
)
|
|
return patches
|
|
|
|
|
|
def mesh_topology_digest(mesh: Any) -> str:
|
|
digest = hashlib.sha256()
|
|
for name in ("n_points", "n_faces", "n_internal_faces", "n_cells"):
|
|
update_hash_text(digest, f"{name}={int(getattr(mesh, name))}")
|
|
update_hash_array(digest, "faces.offsets", mesh.faces.offsets)
|
|
update_hash_array(digest, "faces.values", mesh.faces.values)
|
|
update_hash_array(digest, "cells.offsets", mesh.cells.offsets)
|
|
update_hash_array(digest, "cells.values", mesh.cells.values)
|
|
update_hash_array(digest, "owner", mesh.owner)
|
|
update_hash_array(digest, "neighbour", mesh.neighbour)
|
|
ldu = getattr(mesh, "ldu", {})
|
|
if isinstance(ldu, Mapping):
|
|
if "lower_addr" in ldu:
|
|
update_hash_array(digest, "ldu.lower_addr", ldu["lower_addr"])
|
|
if "upper_addr" in ldu:
|
|
update_hash_array(digest, "ldu.upper_addr", ldu["upper_addr"])
|
|
for entry in ldu.get("patch_addr", []):
|
|
update_hash_text(digest, f"ldu.patch_index={entry.get('patch_index')}")
|
|
update_hash_array(digest, "ldu.patch_addr", entry.get("addr", []))
|
|
for patch in mesh_patch_table(mesh):
|
|
update_hash_text(digest, json.dumps(patch, sort_keys=True))
|
|
return digest.hexdigest()
|
|
|
|
|
|
def mesh_geometry_digest(mesh: Any) -> str:
|
|
digest = hashlib.sha256()
|
|
for name in ("points", "V", "C", "Cf", "Sf", "magSf"):
|
|
update_hash_array(digest, name, getattr(mesh, name))
|
|
return digest.hexdigest()
|
|
|
|
|
|
def mesh_identity(mesh: Any) -> dict[str, Any]:
|
|
return {
|
|
"n_points": int(mesh.n_points),
|
|
"n_faces": int(mesh.n_faces),
|
|
"n_internal_faces": int(mesh.n_internal_faces),
|
|
"n_cells": int(mesh.n_cells),
|
|
"patches": mesh_patch_table(mesh),
|
|
"topology_sha256": mesh_topology_digest(mesh),
|
|
"geometry_sha256": mesh_geometry_digest(mesh),
|
|
}
|
|
|
|
|
|
def compare_mesh_identity(mode: str, actual: Mapping[str, Any], expected: Mapping[str, Any]) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
|
keys = ("n_points", "n_faces", "n_internal_faces", "n_cells", "topology_sha256", "geometry_sha256", "patches")
|
|
differences = {
|
|
key: {"actual": actual.get(key), "expected": expected.get(key)}
|
|
for key in keys
|
|
if actual.get(key) != expected.get(key)
|
|
}
|
|
report = {"matches_oracle": not differences, "differences": differences}
|
|
if not differences:
|
|
return report, []
|
|
return report, [{"mode": mode, "kind": "mesh_identity", "differences": differences}]
|
|
|
|
|
|
def field_comparison_attribution(
|
|
mode: str,
|
|
field: str,
|
|
reason: str,
|
|
stage_graph: Iterable[str] | None,
|
|
) -> dict[str, Any]:
|
|
observed_stages = set(stage_graph or ())
|
|
candidates = []
|
|
for group_name in FIELD_COMPARISON_ATTRIBUTION.get(field, ()):
|
|
group = next(item for item in STAGE_OBSERVABILITY_GROUPS if item["name"] == group_name)
|
|
stages = list(group.get(f"{mode}_stages", ()))
|
|
present_stages = [stage for stage in stages if not observed_stages or stage in observed_stages]
|
|
candidates.append(
|
|
{
|
|
"group": group_name,
|
|
"stages": present_stages,
|
|
"evidence_path": f"stage_observability.{mode}.{group_name}",
|
|
}
|
|
)
|
|
|
|
likely = candidates[0] if candidates else {"group": "unknown", "stages": [], "evidence_path": None}
|
|
out = {
|
|
"field": field,
|
|
"reason": reason,
|
|
"likely_stage_group": likely["group"],
|
|
"likely_stages": likely["stages"],
|
|
"candidate_stage_groups": candidates,
|
|
"evidence_path": likely["evidence_path"],
|
|
"basis": "field-to-stage dependency map for the selected solver graph",
|
|
}
|
|
if reason in {"missing_field", "shape_mismatch"}:
|
|
out["precondition_failure"] = "field_export_or_state_mapping"
|
|
out["field_evidence_path"] = f"modes.{mode}.fields.{field}"
|
|
out["basis"] = "field is absent or has the wrong topology; inspect export/state mapping first, then the mapped solver stages"
|
|
return out
|
|
|
|
|
|
def field_compare_report(name: str, actual_field: Any, expected_field: Any, *, rtol: float, atol: float) -> tuple[dict[str, Any], dict[str, Any] | None]:
|
|
actual = np.asarray(actual_field.internal)
|
|
expected = np.asarray(expected_field.internal)
|
|
actual_entity_kind = getattr(actual_field, "entity_kind", "")
|
|
expected_entity_kind = getattr(expected_field, "entity_kind", "")
|
|
report: dict[str, Any] = {
|
|
"field": name,
|
|
"actual_shape": array_shape(actual),
|
|
"expected_shape": array_shape(expected),
|
|
"shape_matches": actual.shape == expected.shape,
|
|
"actual_entity_kind": actual_entity_kind,
|
|
"expected_entity_kind": expected_entity_kind,
|
|
"entity_kind_matches": actual_entity_kind == expected_entity_kind,
|
|
"rtol": rtol,
|
|
"atol": atol,
|
|
"comparison": "numpy.allclose(equal_nan=False)",
|
|
}
|
|
|
|
if actual.shape != expected.shape:
|
|
report.update({"allclose": False, "reason": "shape_mismatch"})
|
|
return report, {"field": name, "reason": "shape_mismatch", **report}
|
|
|
|
if actual.size == 0:
|
|
report.update(
|
|
{
|
|
"allclose": True,
|
|
"max_abs": 0.0,
|
|
"mean_abs": 0.0,
|
|
"location": None,
|
|
"actual_at_max": None,
|
|
"expected_at_max": None,
|
|
}
|
|
)
|
|
return report, None
|
|
|
|
diff = np.abs(actual - expected)
|
|
finite = np.isfinite(diff)
|
|
if not np.all(finite):
|
|
flat_index = int(np.flatnonzero(~finite)[0])
|
|
max_abs: float | None = None
|
|
else:
|
|
flat_index = int(np.argmax(diff))
|
|
max_abs = finite_float(diff.reshape(-1)[flat_index])
|
|
|
|
max_index = tuple(int(item) for item in np.unravel_index(flat_index, diff.shape))
|
|
entity_index = max_index[0] if max_index else None
|
|
component_index = list(max_index[1:]) if len(max_index) > 1 else None
|
|
actual_at_max = value_at(actual, max_index)
|
|
expected_at_max = value_at(expected, max_index)
|
|
tolerance_at_max = None
|
|
if isinstance(expected_at_max, (int, float)):
|
|
tolerance_at_max = atol + rtol * abs(float(expected_at_max))
|
|
|
|
finite_diff = diff[finite]
|
|
allclose = bool(np.allclose(actual, expected, rtol=rtol, atol=atol, equal_nan=False))
|
|
report.update(
|
|
{
|
|
"allclose": allclose,
|
|
"max_abs": max_abs,
|
|
"mean_abs": finite_float(np.mean(finite_diff)) if finite_diff.size else None,
|
|
"location": {
|
|
"array_index": list(max_index),
|
|
"entity_kind": getattr(actual_field, "entity_kind", ""),
|
|
"entity_index": entity_index,
|
|
"component_index": component_index,
|
|
},
|
|
"actual_at_max": actual_at_max,
|
|
"expected_at_max": expected_at_max,
|
|
"actual_entity_at_max": entity_value_at(actual, entity_index),
|
|
"expected_entity_at_max": entity_value_at(expected, entity_index),
|
|
"tolerance_at_max": finite_float(tolerance_at_max),
|
|
"nonfinite_error_count": int(diff.size - np.count_nonzero(finite)),
|
|
}
|
|
)
|
|
if allclose:
|
|
return report, None
|
|
report["reason"] = "value_mismatch"
|
|
return report, {"field": name, "reason": "value_mismatch", **report}
|
|
|
|
|
|
def compare_fields(
|
|
mode: str,
|
|
actual: Mapping[str, Any],
|
|
expected: Mapping[str, Any],
|
|
*,
|
|
rtol: float,
|
|
atol: float,
|
|
stage_graph: Iterable[str] | None = None,
|
|
) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
|
report: dict[str, Any] = {}
|
|
mismatches: list[dict[str, Any]] = []
|
|
for name in REQUIRED_FIELDS:
|
|
if name not in actual or name not in expected:
|
|
missing = {
|
|
"field": name,
|
|
"reason": "missing_field",
|
|
"missing_actual": name not in actual,
|
|
"missing_expected": name not in expected,
|
|
"actual_fields": sorted(actual),
|
|
"expected_fields": sorted(expected),
|
|
}
|
|
attribution = field_comparison_attribution(mode, name, "missing_field", stage_graph)
|
|
report[name] = {"field": name, "allclose": False, "attribution": attribution, **missing}
|
|
mismatches.append({"mode": mode, "attribution": attribution, **missing})
|
|
continue
|
|
field_report, mismatch = field_compare_report(name, actual[name], expected[name], rtol=rtol, atol=atol)
|
|
report[name] = field_report
|
|
if mismatch is not None:
|
|
attribution = field_comparison_attribution(mode, name, mismatch["reason"], stage_graph)
|
|
field_report["attribution"] = attribution
|
|
mismatch["attribution"] = attribution
|
|
mismatches.append({"mode": mode, **mismatch})
|
|
return report, mismatches
|
|
|
|
|
|
def run_openfoam_command(cmd: list[str], *, log_path: Path, timeout: int) -> dict[str, Any]:
|
|
started = time.monotonic()
|
|
env = openfoam_env()
|
|
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
with log_path.open("w") as log:
|
|
subprocess.run(cmd, cwd=ROOT, env=env, check=True, stdout=log, stderr=subprocess.STDOUT, timeout=timeout)
|
|
except subprocess.TimeoutExpired as exc:
|
|
raise HarnessError(
|
|
ORACLE_FAILURE,
|
|
Path(cmd[0]).name,
|
|
f"OpenFOAM command timed out after {timeout}s: {' '.join(cmd)}",
|
|
details={"cmd": cmd, "timeout_seconds": timeout, "log_path": log_path},
|
|
cause=exc,
|
|
) from exc
|
|
except subprocess.CalledProcessError as exc:
|
|
raise HarnessError(
|
|
ORACLE_FAILURE,
|
|
Path(cmd[0]).name,
|
|
f"OpenFOAM command failed with exit code {exc.returncode}: {' '.join(cmd)}",
|
|
details={"cmd": cmd, "returncode": exc.returncode, "log_path": log_path},
|
|
cause=exc,
|
|
) from exc
|
|
return {
|
|
"cmd": cmd,
|
|
"returncode": 0,
|
|
"log_path": log_path,
|
|
"duration_seconds": round(time.monotonic() - started, 6),
|
|
"timeout_seconds": timeout,
|
|
}
|
|
|
|
|
|
def patch_start_from_latest(case: Path) -> None:
|
|
path = case / "system/controlDict"
|
|
text = path.read_text()
|
|
old = "startFrom startTime;"
|
|
new = "startFrom latestTime;"
|
|
if old not in text:
|
|
raise AssertionError(f"{path}: missing {old!r}")
|
|
path.write_text(text.replace(old, new, 1))
|
|
|
|
|
|
def import_foam() -> Any:
|
|
apply_openfoam_env()
|
|
import foam_stepper as foam
|
|
|
|
return foam
|
|
|
|
|
|
def clone_prepared_case(prepared: Path, dest: Path, source: Path, metadata: Any) -> dict[str, Any]:
|
|
if dest.exists():
|
|
shutil.rmtree(dest)
|
|
shutil.copytree(prepared, dest, copy_function=shutil.copy2)
|
|
return write_case_manifest(source, dest, metadata)
|
|
|
|
|
|
def prepared_case_digest(manifest: Mapping[str, Any]) -> str:
|
|
prepared = manifest.get("prepared_case", {})
|
|
if not isinstance(prepared, Mapping):
|
|
return ""
|
|
return str(prepared.get("files_sha256", ""))
|
|
|
|
|
|
def prepared_file_count(manifest: Mapping[str, Any]) -> int:
|
|
prepared = manifest.get("prepared_case", {})
|
|
files = prepared.get("files", []) if isinstance(prepared, Mapping) else []
|
|
return len(files) if isinstance(files, list) else 0
|
|
|
|
|
|
def compare_prepared_case_manifests(manifests: Mapping[str, Mapping[str, Any]]) -> dict[str, dict[str, Any]]:
|
|
baseline = prepared_case_digest(manifests["prepared"])
|
|
return {
|
|
role: {
|
|
"matches_prepared": prepared_case_digest(manifest) == baseline,
|
|
"files_sha256": prepared_case_digest(manifest),
|
|
"file_count": prepared_file_count(manifest),
|
|
}
|
|
for role, manifest in manifests.items()
|
|
}
|
|
|
|
|
|
def prepare_work_cases(source: Path, work: Path, *, include_split: bool) -> dict[str, Any]:
|
|
if work.exists():
|
|
shutil.rmtree(work)
|
|
work.mkdir(parents=True)
|
|
prepared_case = work / "prepared_case"
|
|
oracle = work / "oracle_case"
|
|
run_one = work / "run_one_case"
|
|
split = work / "split_case" if include_split else None
|
|
|
|
prepared_meta = prepare_case(source, prepared_case, end_time=1)
|
|
manifests: dict[str, Any] = {"prepared": load_case_manifest(prepared_case)}
|
|
manifests["oracle"] = clone_prepared_case(prepared_case, oracle, source, prepared_meta)
|
|
manifests["run_one"] = clone_prepared_case(prepared_case, run_one, source, prepared_meta)
|
|
if split is not None:
|
|
manifests["split"] = clone_prepared_case(prepared_case, split, source, prepared_meta)
|
|
|
|
return {
|
|
"prepared_case": prepared_case,
|
|
"oracle_case": oracle,
|
|
"run_one_case": run_one,
|
|
"split_case": split,
|
|
"metadata": {
|
|
"prepared": prepared_meta,
|
|
"oracle": prepared_meta,
|
|
"run_one": prepared_meta,
|
|
"split": prepared_meta if split is not None else None,
|
|
},
|
|
"manifests": manifests,
|
|
"prepared_case_identity": compare_prepared_case_manifests(manifests),
|
|
}
|
|
|
|
|
|
def _gpu_backend_module() -> Any:
|
|
try:
|
|
from foam_stepper.gpu import backend as gpu_backend
|
|
except Exception as exc:
|
|
raise BackendExecutionError(
|
|
"select_backend",
|
|
"GPU runtime unavailable: failed to import reusable foam_stepper.gpu backend",
|
|
details={
|
|
"requested": "gpu",
|
|
"failure_kind": GPU_RUNTIME_UNAVAILABLE,
|
|
"provider": "quadrants_cuda_rans_solver",
|
|
"used_cpu_fallback": False,
|
|
"cause": {"type": type(exc).__name__, "message": str(exc)},
|
|
},
|
|
) from exc
|
|
return gpu_backend
|
|
|
|
|
|
def _wrap_gpu_backend_error(exc: BaseException) -> BackendExecutionError:
|
|
return BackendExecutionError(
|
|
str(getattr(exc, "step", "execute_gpu_solver_contract")),
|
|
str(exc),
|
|
details=dict(getattr(exc, "details", {}) or {}),
|
|
)
|
|
|
|
|
|
def _call_gpu_backend(function_name: str, *args: Any, **kwargs: Any) -> Any:
|
|
gpu_backend = _gpu_backend_module()
|
|
try:
|
|
return getattr(gpu_backend, function_name)(*args, **kwargs)
|
|
except gpu_backend.BackendExecutionError as exc:
|
|
raise _wrap_gpu_backend_error(exc) from exc
|
|
|
|
|
|
def prepare_gpu_solver_inputs(foam: Any, stepper: Any, case: Path, prepared: Mapping[str, Any], backend: Mapping[str, Any]) -> dict[str, Any]:
|
|
return _call_gpu_backend("prepare_gpu_solver_inputs", foam, stepper, case, prepared, backend)
|
|
|
|
|
|
def run_gpu_solver_stage_smoke(stepper: Any, backend: Mapping[str, Any], case: Path) -> dict[str, Any]:
|
|
return _call_gpu_backend("run_gpu_solver_stage_smoke", stepper, backend, case)
|
|
|
|
|
|
def gpu_solver_stage_report(gpu_run: Mapping[str, Any]) -> dict[str, Any]:
|
|
return _call_gpu_backend("gpu_solver_stage_report", gpu_run)
|
|
|
|
|
|
def gpu_solver_input_blocker_summary(gpu_inputs: Mapping[str, Any]) -> dict[str, Any]:
|
|
return _call_gpu_backend("gpu_solver_input_blocker_summary", gpu_inputs)
|
|
|
|
|
|
def gpu_solver_contract_blocker(backend: Mapping[str, Any], case: Path) -> BackendExecutionError | None:
|
|
gpu_backend = _gpu_backend_module()
|
|
try:
|
|
blocker = gpu_backend.gpu_solver_contract_blocker(backend, case)
|
|
except gpu_backend.BackendExecutionError as exc:
|
|
raise _wrap_gpu_backend_error(exc) from exc
|
|
if blocker is None:
|
|
return None
|
|
return _wrap_gpu_backend_error(blocker)
|
|
|
|
|
|
def record_gpu_contract_failure(report: dict[str, Any], blocker: BackendExecutionError, gpu_inputs: Mapping[str, Any] | None = None) -> None:
|
|
if gpu_inputs is not None:
|
|
blocker.details["gpu_solver_inputs"] = gpu_solver_input_blocker_summary(gpu_inputs)
|
|
report["backend"]["contract_preflight"] = json_ready(
|
|
{
|
|
"status": "failed",
|
|
"step": blocker.step,
|
|
"failure_kind": blocker.details.get("failure_kind"),
|
|
"checked_before_oracle": True,
|
|
"gpu_inputs_prepared": gpu_inputs is not None,
|
|
"reason": "GPU requests must fail as incomplete before any CPU/OpenFOAM solver path can be substituted as the backend.",
|
|
}
|
|
)
|
|
|
|
raise blocker.to_harness_error() from blocker
|
|
|
|
|
|
def select_execution_backend(requested: str) -> dict[str, Any]:
|
|
if requested not in BACKEND_CHOICES:
|
|
raise BackendExecutionError(
|
|
"select_backend",
|
|
f"unknown backend {requested!r}",
|
|
details={"requested": requested, "choices": list(BACKEND_CHOICES)},
|
|
)
|
|
|
|
if requested == "gpu":
|
|
return _call_gpu_backend("select_execution_backend", requested)
|
|
|
|
return {
|
|
"requested": requested,
|
|
"selected": "cpu",
|
|
"device": "host",
|
|
"provider": "foam_stepper_cpu",
|
|
"available_backends": ["cpu"],
|
|
"gpu_device_present": any(Path(path).exists() for path in ("/dev/nvidia0", "/dev/dri/renderD128")),
|
|
"capabilities": [
|
|
"foam_stepper_python_bridge",
|
|
"openfoam_case_loader",
|
|
"oracle_comparison",
|
|
],
|
|
"used_cpu_fallback": False,
|
|
}
|
|
|
|
|
|
def run_backend_iteration(stepper: Any, backend: Mapping[str, Any], case: Path) -> dict[str, Any]:
|
|
selected = backend.get("selected")
|
|
if selected == "gpu":
|
|
return _call_gpu_backend("run_backend_iteration", stepper, backend, case)
|
|
if selected != "cpu":
|
|
raise BackendExecutionError(
|
|
"execute_backend",
|
|
f"backend {selected!r} is not executable",
|
|
details={"backend": backend, "case": case},
|
|
)
|
|
result = stepper.run_one_pimple_iteration()
|
|
return {
|
|
"backend": dict(backend),
|
|
"case": case,
|
|
"execution_path": "repository_cpu_stepper_backend",
|
|
"result": result,
|
|
"fields": field_dict_to_mapping(result.outputs["fields"]),
|
|
}
|
|
|
|
|
|
def run_gpu_split_iteration(foam: Any, stepper: Any, backend: Mapping[str, Any], case: Path) -> dict[str, Any]:
|
|
return _call_gpu_backend("run_gpu_split_iteration", foam, stepper, backend, case)
|
|
|
|
|
|
def make_stepper(foam: Any, case: Path, label: str) -> Any:
|
|
try:
|
|
return foam.Case(case).make_stepper()
|
|
except Exception as exc:
|
|
raise HarnessError(
|
|
LOADING_FAILURE,
|
|
f"load_{label}_case",
|
|
f"failed to load {label} case with foam_stepper: {case}",
|
|
details={"case": case},
|
|
cause=exc,
|
|
) from exc
|
|
|
|
|
|
def read_mesh_identity(stepper: Any, label: str) -> dict[str, Any]:
|
|
try:
|
|
return mesh_identity(stepper.mesh())
|
|
except Exception as exc:
|
|
raise HarnessError(
|
|
LOADING_FAILURE,
|
|
f"read_{label}_mesh",
|
|
f"failed to read {label} mesh identity",
|
|
details={"case": getattr(stepper, "case_path", "")},
|
|
cause=exc,
|
|
) from exc
|
|
|
|
|
|
def read_fields(stepper: Any, label: str) -> dict[str, Any]:
|
|
try:
|
|
return field_dict_to_mapping(stepper.fields())
|
|
except Exception as exc:
|
|
raise HarnessError(
|
|
LOADING_FAILURE,
|
|
f"read_{label}_fields",
|
|
f"failed to read {label} fields",
|
|
details={"case": getattr(stepper, "case_path", "")},
|
|
cause=exc,
|
|
) from exc
|
|
|
|
def validation_details(exc: BaseException) -> dict[str, Any]:
|
|
if hasattr(exc, "to_dict"):
|
|
return json_ready(exc.to_dict())
|
|
return {"type": type(exc).__name__, "message": str(exc)}
|
|
|
|
|
|
def export_solver_state_checked(foam: Any, stepper: Any, label: str, fields: Mapping[str, Any] | None = None) -> dict[str, Any]:
|
|
try:
|
|
field_source = fields if fields is not None else stepper.fields()
|
|
return foam.export_solver_state(stepper.mesh(), field_source, required_fields=REQUIRED_FIELDS)
|
|
except Exception as exc:
|
|
raise HarnessError(
|
|
LOADING_FAILURE,
|
|
f"export_{label}_state",
|
|
f"failed to export explicit {label} solver state",
|
|
details=validation_details(exc),
|
|
cause=exc,
|
|
) from exc
|
|
|
|
|
|
def export_matrix_state_checked(foam: Any, matrix: Any, solver_state: Mapping[str, Any], label: str) -> dict[str, Any]:
|
|
try:
|
|
return foam.export_matrix_state(matrix, mesh_state=solver_state)
|
|
except Exception as exc:
|
|
raise HarnessError(
|
|
STEPPER_FAILURE,
|
|
f"export_{label}_matrix_state",
|
|
f"failed to export explicit {label} matrix state",
|
|
details=validation_details(exc),
|
|
cause=exc,
|
|
) from exc
|
|
|
|
|
|
def checked_step(stages: list[dict[str, Any]], name: str, fn: Any) -> Any:
|
|
try:
|
|
result = fn()
|
|
except Exception as exc:
|
|
raise HarnessError(
|
|
STEPPER_FAILURE,
|
|
name,
|
|
f"split-step stage failed: {name}",
|
|
cause=exc,
|
|
) from exc
|
|
stages.append(transform_summary(result))
|
|
return result
|
|
|
|
|
|
def run_split_iteration(foam: Any, stepper: Any) -> dict[str, Any]:
|
|
stages: list[dict[str, Any]] = []
|
|
checked_step(stages, "pre_solve", stepper.pre_solve)
|
|
checked_step(stages, "advance_time", stepper.advance_time)
|
|
begin = checked_step(stages, "begin_pimple_iteration", stepper.begin_pimple_iteration)
|
|
if begin.outputs.get("active") is not True:
|
|
raise HarnessError(
|
|
STEPPER_FAILURE,
|
|
"begin_pimple_iteration",
|
|
"split-step PIMPLE iteration was inactive",
|
|
details={"outputs": summarize_value(begin.outputs)},
|
|
)
|
|
|
|
checked_step(stages, "fv_models_correct", stepper.fv_models_correct)
|
|
checked_step(stages, "pre_predictor", stepper.pre_predictor)
|
|
checked_step(stages, "momentum_transport_predict", stepper.momentum_transport_predictor)
|
|
terms = checked_step(stages, "assemble_momentum_terms", stepper.assemble_momentum_terms)
|
|
UEqn = checked_step(stages, "assemble_UEqn", stepper.assemble_momentum_matrix)
|
|
checked_step(stages, "relax_UEqn", stepper.relax_matrix)
|
|
checked_step(stages, "constrain_UEqn", stepper.constrain_matrix)
|
|
checked_step(stages, "solve_UEqn", stepper.solve_momentum)
|
|
checked_step(stages, "compute_pressure_inputs", stepper.compute_pressure_inputs)
|
|
pEqn = checked_step(stages, "assemble_pEqn", stepper.assemble_pressure_matrix)
|
|
checked_step(stages, "solve_pEqn", stepper.solve_pressure)
|
|
checked_step(stages, "correct_velocity_pressure_flux", stepper.correct_velocity_pressure_flux)
|
|
checked_step(stages, "momentum_transport_correct", stepper.momentum_transport_corrector)
|
|
checked_step(stages, "end_pimple_iteration", stepper.end_pimple_iteration)
|
|
checked_step(stages, "post_solve", lambda: stepper.post_solve(write=False))
|
|
|
|
try:
|
|
fields = field_dict_to_mapping(stepper.fields())
|
|
except Exception as exc:
|
|
raise HarnessError(
|
|
STEPPER_FAILURE,
|
|
"split_fields",
|
|
"failed to read split-step fields after execution",
|
|
cause=exc,
|
|
) from exc
|
|
|
|
solver_state = export_solver_state_checked(foam, stepper, "split", fields)
|
|
|
|
momentum_terms = [term.get("name", "") for term in terms.outputs.get("terms", [])]
|
|
UEqn_matrix = UEqn.outputs["UEqn"]
|
|
pEqn_matrix = pEqn.outputs["pEqn"]
|
|
matrix_states = {
|
|
"UEqn": foam.describe_matrix_state(export_matrix_state_checked(foam, UEqn_matrix, solver_state, "UEqn")),
|
|
"pEqn": foam.describe_matrix_state(export_matrix_state_checked(foam, pEqn_matrix, solver_state, "pEqn")),
|
|
}
|
|
observability = stage_observability_report(
|
|
"split",
|
|
stages,
|
|
evidence={
|
|
"split_step_execution": True,
|
|
"momentum_terms": momentum_terms,
|
|
"matrix_states": sorted(matrix_states),
|
|
"turbulence_fields": visible_turbulence_fields(fields),
|
|
},
|
|
)
|
|
return {
|
|
"fields": fields,
|
|
"state": solver_state,
|
|
"state_summary": foam.describe_solver_state(solver_state),
|
|
"stages": stages,
|
|
"graph": [stage["name"] for stage in stages],
|
|
"momentum_terms": momentum_terms,
|
|
"UEqn": matrix_summary(UEqn_matrix),
|
|
"pEqn": matrix_summary(pEqn_matrix),
|
|
"matrix_states": matrix_states,
|
|
"observability": observability,
|
|
}
|
|
|
|
|
|
def visible_turbulence_fields(fields: Mapping[str, Any]) -> list[str]:
|
|
return [name for name in TURBULENCE_FIELDS if name in fields]
|
|
|
|
|
|
def differentiability_report(backend: Mapping[str, Any], *, case_name: str) -> dict[str, Any]:
|
|
capabilities = tuple(backend.get("capabilities", ()))
|
|
autodiff_available = any(capability in capabilities for capability in ("autodiff", "jvp", "vjp", "gradient"))
|
|
status = "differentiable" if autodiff_available else "not_differentiable"
|
|
return {
|
|
"status": status,
|
|
"backend": {
|
|
"requested": backend.get("requested"),
|
|
"selected": backend.get("selected"),
|
|
"provider": backend.get("provider"),
|
|
"capabilities": list(capabilities),
|
|
},
|
|
"variables": DIFFERENTIABILITY_VARIABLES,
|
|
"claims": [
|
|
{
|
|
"name": "current_migrated_solver_path",
|
|
"differentiable": autodiff_available,
|
|
"variables": DIFFERENTIABILITY_VARIABLES,
|
|
"applies_to": {
|
|
"modes": ["run_one", "split"],
|
|
"fields": list(REQUIRED_FIELDS),
|
|
"parameters": list(DIFFERENTIABILITY_VARIABLES["case_parameters"]),
|
|
"losses": list(DIFFERENTIABILITY_VARIABLES["losses"]),
|
|
},
|
|
"evidence": {
|
|
"backend_capabilities": list(capabilities),
|
|
"reason": "no registered backend capability exposes AD/JVP/VJP for solver updates"
|
|
if not autodiff_available
|
|
else "backend advertises differentiability capabilities",
|
|
},
|
|
}
|
|
],
|
|
"non_differentiable": list(NON_DIFFERENTIABLE_BOUNDARIES),
|
|
"custom_gradient_required": list(CUSTOM_GRADIENT_REQUIREMENTS),
|
|
"checks": [
|
|
{
|
|
"name": "solver_sensitivity_against_finite_difference",
|
|
"status": "not_run",
|
|
"fixed_problem_setup": {
|
|
"case": case_name,
|
|
"fields": list(REQUIRED_FIELDS),
|
|
"mesh_topology": "fixed prepared AirfRANS mesh",
|
|
"backend": backend.get("selected"),
|
|
},
|
|
"computed_sensitivity": None,
|
|
"independent_numerical_check": None,
|
|
"reason": "no differentiable backend or custom gradient is currently available, so no computed sensitivity is claimed",
|
|
}
|
|
],
|
|
}
|
|
|
|
|
|
def command_timing_summary(command: Mapping[str, Any]) -> dict[str, Any]:
|
|
cmd = list(command.get("cmd") or ())
|
|
executable = Path(str(cmd[0])).name if cmd else None
|
|
log_path = str(command.get("log_path", ""))
|
|
role = "openfoam_command"
|
|
if executable == "checkMesh":
|
|
role = "oracle_mesh_check"
|
|
elif executable == "foamRun" and "oracle" in log_path:
|
|
role = "oracle_solver"
|
|
return {
|
|
"role": role,
|
|
"executable": executable,
|
|
"cmd": cmd,
|
|
"duration_seconds": command.get("duration_seconds"),
|
|
"timeout_seconds": command.get("timeout_seconds"),
|
|
"log_path": command.get("log_path"),
|
|
"returncode": command.get("returncode"),
|
|
}
|
|
|
|
|
|
def gpu_timing_summary(source: Mapping[str, Any], *, label: str) -> dict[str, Any]:
|
|
profiler = source.get("profiler", {}) if isinstance(source.get("profiler"), Mapping) else {}
|
|
timing = source.get("timing", {}) if isinstance(source.get("timing"), Mapping) else {}
|
|
return {
|
|
"label": label,
|
|
"execution_path": source.get("execution_path"),
|
|
"timing": timing,
|
|
"profiler": {
|
|
"available": profiler.get("available"),
|
|
"profile_record_count": profiler.get("profile_record_count"),
|
|
"device_time_ms_total": profiler.get("device_time_ms_total"),
|
|
"expected_kernel_entrypoints": profiler.get("expected_kernel_entrypoints"),
|
|
"generated_kernel_names": profiler.get("generated_kernel_names"),
|
|
},
|
|
}
|
|
|
|
|
|
def build_timing_evidence(report: Mapping[str, Any]) -> dict[str, Any]:
|
|
commands = [command_timing_summary(command) for command in report.get("commands", [])]
|
|
oracle_solver = next((command for command in commands if command.get("role") == "oracle_solver"), None)
|
|
gpu_sources: dict[str, Any] = {}
|
|
gpu_stage_smoke = report.get("gpu_solver_stages")
|
|
if isinstance(gpu_stage_smoke, Mapping) and gpu_stage_smoke.get("status") == "executed":
|
|
gpu_sources["stage_preflight"] = gpu_timing_summary(gpu_stage_smoke, label="stage_preflight")
|
|
for mode in ("run_one", "split"):
|
|
mode_gpu = report.get("modes", {}).get(mode, {}).get("gpu_solver", {})
|
|
if isinstance(mode_gpu, Mapping) and mode_gpu.get("status") == "executed":
|
|
gpu_sources[mode] = gpu_timing_summary(mode_gpu, label=mode)
|
|
|
|
gpu_solver_timing_ok = any(
|
|
isinstance(source.get("timing"), Mapping) and source["timing"].get("gpu_solver_wall_seconds") is not None
|
|
for source in gpu_sources.values()
|
|
)
|
|
gpu_profiler_ok = any(
|
|
isinstance(source.get("profiler"), Mapping) and source["profiler"].get("available") is True and source["profiler"].get("profile_record_count", 0) > 0
|
|
for source in gpu_sources.values()
|
|
)
|
|
openfoam_timing_ok = oracle_solver is not None and oracle_solver.get("duration_seconds") is not None
|
|
return {
|
|
"schema_version": 1,
|
|
"status": "reported" if commands or gpu_sources else "not_reported",
|
|
"openfoam_reference": {
|
|
"oracle_solver_duration_seconds": None if oracle_solver is None else oracle_solver.get("duration_seconds"),
|
|
"commands": commands,
|
|
},
|
|
"gpu_solver": gpu_sources,
|
|
"criteria": {
|
|
"openfoam_reference_timing": openfoam_timing_ok,
|
|
"gpu_solver_wall_timing": gpu_solver_timing_ok,
|
|
"gpu_profiler_timing": gpu_profiler_ok,
|
|
},
|
|
}
|
|
|
|
def enabled_mode_names(report: Mapping[str, Any]) -> list[str]:
|
|
return [mode for mode in ("run_one", "split") if report.get("modes", {}).get(mode, {}).get("enabled", False)]
|
|
|
|
|
|
def comparison_regression_summary(comparisons: Mapping[str, Any]) -> dict[str, Any]:
|
|
out: dict[str, Any] = {}
|
|
for field in REQUIRED_FIELDS:
|
|
data = comparisons.get(field, {})
|
|
out[field] = {
|
|
"actual_shape": data.get("actual_shape"),
|
|
"expected_shape": data.get("expected_shape"),
|
|
"shape_matches": data.get("shape_matches"),
|
|
"allclose": data.get("allclose"),
|
|
"max_abs": data.get("max_abs"),
|
|
"mean_abs": data.get("mean_abs"),
|
|
"tolerance_at_max": data.get("tolerance_at_max"),
|
|
"location": data.get("location"),
|
|
}
|
|
return out
|
|
|
|
|
|
def build_verifier_evidence(report: Mapping[str, Any]) -> dict[str, Any]:
|
|
modes = enabled_mode_names(report)
|
|
prepared_identity = report.get("case_preparation", {}).get("prepared_case_identity", {})
|
|
prepared_ok = bool(prepared_identity) and all(identity.get("matches_prepared") for identity in prepared_identity.values())
|
|
|
|
mesh_ok_by_mode = {
|
|
mode: bool(report.get("modes", {}).get(mode, {}).get("mesh_comparison", {}).get("matches_oracle"))
|
|
for mode in modes
|
|
}
|
|
comparison_ok_by_mode = {}
|
|
observability_ok_by_mode = {}
|
|
regression_modes: dict[str, Any] = {}
|
|
for mode in modes:
|
|
mode_report = report.get("modes", {}).get(mode, {})
|
|
comparisons = mode_report.get("comparisons", {})
|
|
comparison_ok_by_mode[mode] = all(
|
|
comparisons.get(field, {}).get("shape_matches") is True and comparisons.get(field, {}).get("allclose") is True
|
|
for field in REQUIRED_FIELDS
|
|
)
|
|
observability = report.get("stage_observability", {}).get(mode, {})
|
|
groups = observability.get("groups", [])
|
|
observability_ok_by_mode[mode] = bool(groups) and all(group.get("observed") is True for group in groups)
|
|
regression_modes[mode] = {
|
|
"enabled": True,
|
|
"case_role": mode,
|
|
"mesh_matches_oracle": mesh_ok_by_mode[mode],
|
|
"observed_stage_groups": [group.get("name") for group in groups if group.get("observed")],
|
|
"fields": comparison_regression_summary(comparisons),
|
|
}
|
|
|
|
timing = report.get("timing", {}) if isinstance(report.get("timing"), Mapping) else {}
|
|
timing_criteria = timing.get("criteria", {}) if isinstance(timing.get("criteria"), Mapping) else {}
|
|
if report.get("backend", {}).get("selected") == "gpu":
|
|
timing_ok = (
|
|
timing_criteria.get("openfoam_reference_timing") is True
|
|
and timing_criteria.get("gpu_solver_wall_timing") is True
|
|
and timing_criteria.get("gpu_profiler_timing") is True
|
|
)
|
|
else:
|
|
timing_ok = timing.get("status") in {"reported", "not_reported"}
|
|
|
|
criteria = {
|
|
"prepared_case_identity": prepared_ok,
|
|
"mesh_identity": bool(mesh_ok_by_mode) and all(mesh_ok_by_mode.values()),
|
|
"field_comparisons": bool(comparison_ok_by_mode) and all(comparison_ok_by_mode.values()),
|
|
"stage_observability": bool(observability_ok_by_mode) and all(observability_ok_by_mode.values()),
|
|
"backend_selected": report.get("backend", {}).get("selected") is not None,
|
|
"timing_reported": timing_ok,
|
|
"differentiability_reported": report.get("differentiability", {}).get("status") in {"differentiable", "not_differentiable"},
|
|
}
|
|
passed = all(criteria.values())
|
|
oracle_mesh = report.get("mesh_identity", {}).get("oracle", {})
|
|
prepared_digest = prepared_identity.get("prepared", {}).get("files_sha256")
|
|
return {
|
|
"passed": passed,
|
|
"status_basis": "passed all verifier evidence criteria" if passed else "one or more verifier evidence criteria failed",
|
|
"criteria": criteria,
|
|
"mode_criteria": {
|
|
"mesh_identity": mesh_ok_by_mode,
|
|
"field_comparisons": comparison_ok_by_mode,
|
|
"stage_observability": observability_ok_by_mode,
|
|
},
|
|
"entrypoint": report.get("harness_entrypoint", {}),
|
|
"regression_summary": {
|
|
"schema_version": report.get("harness", {}).get("schema_version"),
|
|
"source_case": report.get("problem_boundary", {}).get("airfrans_simulation"),
|
|
"prepared_case_files_sha256": prepared_digest,
|
|
"backend_selected": report.get("backend", {}).get("selected"),
|
|
"differentiability_status": report.get("differentiability", {}).get("status"),
|
|
"mesh_topology_sha256": oracle_mesh.get("topology_sha256"),
|
|
"mesh_geometry_sha256": oracle_mesh.get("geometry_sha256"),
|
|
"modes": regression_modes,
|
|
},
|
|
}
|
|
|
|
|
|
|
|
def base_report(args: argparse.Namespace) -> dict[str, Any]:
|
|
report_path = args.report if args.report is not None else args.work / "verifier_report.json"
|
|
return {
|
|
"harness": {
|
|
"name": "airfrans_stepper_verifier",
|
|
"spec": "VERIFIER_HARNESS_SPEC.md",
|
|
"schema_version": 1,
|
|
"scope": AIRFRANS_VERIFIER_SCOPE,
|
|
},
|
|
"status": "running",
|
|
"failure": None,
|
|
"paths": {
|
|
"root": ROOT,
|
|
"source": args.source,
|
|
"work": args.work,
|
|
"report": report_path,
|
|
},
|
|
"harness_entrypoint": {
|
|
"command": "uv run python scripts/verify_airfrans_stepper.py --work <work-dir> --report <report.json>",
|
|
"arguments": {
|
|
"source": args.source,
|
|
"work": args.work,
|
|
"report": report_path,
|
|
"backend": args.backend,
|
|
"skip_split": args.skip_split,
|
|
"rtol": args.rtol,
|
|
"atol": args.atol,
|
|
},
|
|
"prepares_case": True,
|
|
"runs_oracle": True,
|
|
"runs_migrated_path": True,
|
|
"writes_report": True,
|
|
"scope": AIRFRANS_VERIFIER_SCOPE,
|
|
},
|
|
"source_policy": "raw AirfRANS source is read-only; all solver runs use prepared work-directory copies",
|
|
"tolerances": {
|
|
"rtol": args.rtol,
|
|
"atol": args.atol,
|
|
"policy": "CPU stepper parity with the repository OpenFOAM oracle must pass np.allclose for every required field.",
|
|
},
|
|
"comparison_policy": {
|
|
"function": "numpy.allclose(actual, expected, rtol, atol, equal_nan=False)",
|
|
"scope": "internal arrays for U, p, phi, nut, k, and omega",
|
|
"backend_tolerances": {
|
|
"cpu": {
|
|
"rtol": args.rtol,
|
|
"atol": args.atol,
|
|
"reason": "strict CPU parity against the repository OpenFOAM oracle",
|
|
},
|
|
"gpu": {
|
|
"rtol": args.rtol,
|
|
"atol": args.atol,
|
|
"reason": "full GPU RANS backend must own solver execution and satisfy OpenFOAM oracle parity; primitive GPU diagnostics are not acceptance",
|
|
"availability_checked_during_backend_selection": True,
|
|
"full_solver_guard": FULL_GPU_RANS_GUARD,
|
|
"primitive_evidence": {
|
|
"name": GPU_PRIMITIVE_NAME,
|
|
"path": GPU_PRIMITIVE_PROOF,
|
|
"counts_as_full_gpu_rans_solver": False,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
"backend": {
|
|
"requested": args.backend,
|
|
"selected": None,
|
|
"failure_category": BACKEND_FAILURE,
|
|
"execution_scope": AIRFRANS_VERIFIER_SCOPE,
|
|
},
|
|
"required_fields": {
|
|
"primary": list(PRIMARY_FIELDS),
|
|
"turbulence": list(TURBULENCE_FIELDS),
|
|
"all": list(REQUIRED_FIELDS),
|
|
},
|
|
"problem_boundary": {
|
|
"airfrans_simulation": args.source.name,
|
|
"selected_source_case": args.source,
|
|
"prepared_case_policy": "prepare one reproducible OpenFOAM v14 case and clone it into isolated oracle/run_one/split execution cases",
|
|
"oracle_output": {
|
|
"case_role": "oracle",
|
|
"producer": "foamRun -solver incompressibleFluid -noFunctionObjects",
|
|
"time": "1",
|
|
"fields": list(REQUIRED_FIELDS),
|
|
},
|
|
"repository_outputs": {
|
|
"run_one": "migrated run_one path selected by the requested backend; gpu requests must use the GPU solver contract and reject CPU fallback",
|
|
"split": "split solver stages selected by the requested backend",
|
|
"time": "1",
|
|
"fields": list(REQUIRED_FIELDS),
|
|
},
|
|
"mesh_identity_evidence": [
|
|
"n_points",
|
|
"n_faces",
|
|
"n_internal_faces",
|
|
"n_cells",
|
|
"patches",
|
|
"topology_sha256",
|
|
"geometry_sha256",
|
|
],
|
|
},
|
|
"commands": [],
|
|
"case_preparation": {},
|
|
"mesh_identity": {},
|
|
"modes": {
|
|
"run_one": {"enabled": True},
|
|
"split": {"enabled": not args.skip_split},
|
|
},
|
|
"tracked_turbulence_fields": {},
|
|
"comparison_mismatches": [],
|
|
"comparison_attribution": [],
|
|
"verifier_evidence": {
|
|
"passed": False,
|
|
"status_basis": "not_evaluated",
|
|
},
|
|
"timing": {
|
|
"status": "not_evaluated",
|
|
},
|
|
"differentiability": {
|
|
"status": "not_evaluated",
|
|
"reason": "backend has not been selected yet",
|
|
},
|
|
"state_exports": {},
|
|
"gpu_solver_inputs": {
|
|
"schema_version": GPU_INPUT_SCHEMA_VERSION,
|
|
"status": "not_requested",
|
|
},
|
|
"stage_observability": {
|
|
"contract": json_ready(STAGE_OBSERVABILITY_GROUPS),
|
|
},
|
|
}
|
|
|
|
|
|
def run_harness(args: argparse.Namespace, report: dict[str, Any]) -> None:
|
|
try:
|
|
prepared = prepare_work_cases(args.source, args.work, include_split=not args.skip_split)
|
|
except Exception as exc:
|
|
raise HarnessError(
|
|
LOADING_FAILURE,
|
|
"prepare_cases",
|
|
"failed to prepare reproducible AirfRANS work cases",
|
|
details={"source": args.source, "work": args.work},
|
|
cause=exc,
|
|
) from exc
|
|
|
|
oracle_case = prepared["oracle_case"]
|
|
run_one_case = prepared["run_one_case"]
|
|
split_case = prepared["split_case"]
|
|
report["case_preparation"] = json_ready(prepared)
|
|
|
|
identity_mismatches = {
|
|
role: identity
|
|
for role, identity in prepared["prepared_case_identity"].items()
|
|
if not identity["matches_prepared"]
|
|
}
|
|
if identity_mismatches:
|
|
raise HarnessError(
|
|
LOADING_FAILURE,
|
|
"prepare_case_identity",
|
|
"prepared execution case copies differ from the reproducible baseline case",
|
|
details={"prepared_case_identity": identity_mismatches},
|
|
)
|
|
|
|
|
|
try:
|
|
backend = select_execution_backend(args.backend)
|
|
except BackendExecutionError as exc:
|
|
raise exc.to_harness_error() from exc
|
|
report["backend"] = json_ready(backend)
|
|
report["differentiability"] = json_ready(differentiability_report(backend, case_name=args.source.name))
|
|
if backend.get("selected") == "gpu":
|
|
try:
|
|
foam = import_foam()
|
|
except Exception as exc:
|
|
raise HarnessError(
|
|
LOADING_FAILURE,
|
|
"import_foam_stepper",
|
|
"failed to import foam_stepper before GPU input preparation",
|
|
cause=exc,
|
|
) from exc
|
|
gpu_input_stepper = make_stepper(foam, run_one_case, "gpu_inputs")
|
|
report["mesh_identity"]["gpu_inputs"] = read_mesh_identity(gpu_input_stepper, "gpu_inputs")
|
|
try:
|
|
gpu_inputs = prepare_gpu_solver_inputs(foam, gpu_input_stepper, run_one_case, prepared, backend)
|
|
except BackendExecutionError as exc:
|
|
raise exc.to_harness_error() from exc
|
|
report["gpu_solver_inputs"] = json_ready(gpu_inputs)
|
|
try:
|
|
gpu_stage_smoke = run_gpu_solver_stage_smoke(gpu_input_stepper, backend, run_one_case)
|
|
except BackendExecutionError as exc:
|
|
raise exc.to_harness_error() from exc
|
|
report["gpu_solver_stages"] = json_ready(gpu_solver_stage_report(gpu_stage_smoke))
|
|
gpu_contract_blocker = gpu_solver_contract_blocker(backend, run_one_case)
|
|
if gpu_contract_blocker is not None:
|
|
record_gpu_contract_failure(report, gpu_contract_blocker, gpu_inputs)
|
|
report["commands"].append(
|
|
run_openfoam_command(["checkMesh", "-case", str(oracle_case), "-constant"], log_path=args.work / "checkMesh.log", timeout=120)
|
|
)
|
|
report["commands"].append(
|
|
run_openfoam_command(
|
|
["foamRun", "-case", str(oracle_case), "-solver", "incompressibleFluid", "-noFunctionObjects"],
|
|
log_path=args.work / "foamRun_oracle.log",
|
|
timeout=600,
|
|
)
|
|
)
|
|
|
|
try:
|
|
patch_start_from_latest(oracle_case)
|
|
except Exception as exc:
|
|
raise HarnessError(
|
|
LOADING_FAILURE,
|
|
"select_oracle_latest_time",
|
|
"failed to configure oracle case to load latestTime output",
|
|
details={"case": oracle_case},
|
|
cause=exc,
|
|
) from exc
|
|
|
|
try:
|
|
foam = import_foam()
|
|
except Exception as exc:
|
|
raise HarnessError(
|
|
LOADING_FAILURE,
|
|
"import_foam_stepper",
|
|
"failed to import foam_stepper in the repository OpenFOAM environment",
|
|
cause=exc,
|
|
) from exc
|
|
|
|
oracle_stepper = make_stepper(foam, oracle_case, "oracle")
|
|
oracle_mesh = read_mesh_identity(oracle_stepper, "oracle")
|
|
oracle_fields = read_fields(oracle_stepper, "oracle")
|
|
report["mesh_identity"]["oracle"] = oracle_mesh
|
|
report["modes"]["oracle"] = {
|
|
"case": oracle_case,
|
|
"fields": selected_field_summaries(oracle_fields),
|
|
}
|
|
report["tracked_turbulence_fields"]["oracle"] = visible_turbulence_fields(oracle_fields)
|
|
oracle_state = export_solver_state_checked(foam, oracle_stepper, "oracle", oracle_fields)
|
|
report["state_exports"]["oracle"] = foam.describe_solver_state(oracle_state)
|
|
|
|
|
|
run_one_stepper = make_stepper(foam, run_one_case, "run_one")
|
|
run_one_mesh = read_mesh_identity(run_one_stepper, "run_one")
|
|
report["mesh_identity"]["run_one"] = run_one_mesh
|
|
mesh_report, mesh_mismatches = compare_mesh_identity("run_one", run_one_mesh, oracle_mesh)
|
|
report["modes"]["run_one"]["mesh_comparison"] = mesh_report
|
|
|
|
try:
|
|
backend_run = run_backend_iteration(run_one_stepper, backend, run_one_case)
|
|
run_one_result = backend_run["result"]
|
|
run_one_fields = backend_run["fields"]
|
|
except BackendExecutionError as exc:
|
|
raise exc.to_harness_error() from exc
|
|
except Exception as exc:
|
|
raise HarnessError(
|
|
STEPPER_FAILURE,
|
|
"run_one_pimple_iteration",
|
|
"full one-iteration backend execution failed",
|
|
details={"case": run_one_case, "backend": backend},
|
|
cause=exc,
|
|
) from exc
|
|
|
|
run_one_stages = graph_stage_summaries(run_one_result)
|
|
run_one_observability = stage_observability_report(
|
|
"run_one",
|
|
run_one_stages,
|
|
evidence={
|
|
"one_iteration_execution": True,
|
|
"compared_fields": list(REQUIRED_FIELDS),
|
|
"turbulence_fields": visible_turbulence_fields(run_one_fields),
|
|
"backend": backend,
|
|
},
|
|
)
|
|
report["stage_observability"]["run_one"] = run_one_observability
|
|
|
|
run_one_comparisons, run_one_mismatches = compare_fields(
|
|
"run_one",
|
|
run_one_fields,
|
|
oracle_fields,
|
|
rtol=args.rtol,
|
|
atol=args.atol,
|
|
stage_graph=[stage["name"] for stage in run_one_stages],
|
|
)
|
|
report["modes"]["run_one"].update(
|
|
{
|
|
"case": run_one_case,
|
|
"execution_path": backend_run["execution_path"],
|
|
"backend": backend_run["backend"],
|
|
"stage": transform_summary(run_one_result),
|
|
"graph": [stage["name"] for stage in run_one_stages],
|
|
"fields": selected_field_summaries(run_one_fields),
|
|
"comparisons": run_one_comparisons,
|
|
"observability": run_one_observability,
|
|
}
|
|
)
|
|
if backend.get("selected") == "gpu":
|
|
report["modes"]["run_one"]["gpu_solver"] = json_ready(backend_run.get("gpu_solver", {}))
|
|
report["tracked_turbulence_fields"]["run_one"] = visible_turbulence_fields(run_one_fields)
|
|
run_one_state = export_solver_state_checked(foam, run_one_stepper, "run_one", run_one_fields)
|
|
report["state_exports"]["run_one"] = foam.describe_solver_state(run_one_state)
|
|
|
|
|
|
mismatches = mesh_mismatches + run_one_mismatches
|
|
|
|
if split_case is not None:
|
|
split_stepper = make_stepper(foam, split_case, "split")
|
|
split_mesh = read_mesh_identity(split_stepper, "split")
|
|
report["mesh_identity"]["split"] = split_mesh
|
|
split_mesh_report, split_mesh_mismatches = compare_mesh_identity("split", split_mesh, oracle_mesh)
|
|
report["modes"]["split"]["mesh_comparison"] = split_mesh_report
|
|
|
|
if backend.get("selected") == "gpu":
|
|
split_result = run_gpu_split_iteration(foam, split_stepper, backend, split_case)
|
|
else:
|
|
split_result = run_split_iteration(foam, split_stepper)
|
|
report["stage_observability"]["split"] = split_result["observability"]
|
|
split_fields = split_result["fields"]
|
|
split_comparisons, split_mismatches = compare_fields(
|
|
"split",
|
|
split_fields,
|
|
oracle_fields,
|
|
rtol=args.rtol,
|
|
atol=args.atol,
|
|
stage_graph=split_result["graph"],
|
|
)
|
|
report["modes"]["split"].update(
|
|
{
|
|
"case": split_case,
|
|
"graph": split_result["graph"],
|
|
"stages": split_result["stages"],
|
|
"momentum_terms": split_result["momentum_terms"],
|
|
"UEqn": split_result["UEqn"],
|
|
"pEqn": split_result["pEqn"],
|
|
"fields": selected_field_summaries(split_fields),
|
|
"comparisons": split_comparisons,
|
|
"observability": split_result["observability"],
|
|
"execution_path": split_result.get("execution_path"),
|
|
"backend": split_result.get("backend"),
|
|
"gpu_solver": split_result.get("gpu_solver"),
|
|
}
|
|
)
|
|
report["tracked_turbulence_fields"]["split"] = visible_turbulence_fields(split_fields)
|
|
report["state_exports"]["split"] = split_result["state_summary"]
|
|
report["state_exports"]["split_matrices"] = split_result["matrix_states"]
|
|
mismatches.extend(split_mesh_mismatches)
|
|
mismatches.extend(split_mismatches)
|
|
|
|
comparison_attribution = [mismatch["attribution"] for mismatch in mismatches if "attribution" in mismatch]
|
|
report["comparison_attribution"] = json_ready(comparison_attribution)
|
|
report["comparison_mismatches"] = json_ready(mismatches)
|
|
report["timing"] = json_ready(build_timing_evidence(report))
|
|
report["verifier_evidence"] = json_ready(build_verifier_evidence(report))
|
|
if mismatches:
|
|
raise HarnessError(
|
|
COMPARISON_FAILURE,
|
|
"compare_oracle_stepper_outputs",
|
|
f"{len(mismatches)} verifier comparison mismatch(es) observed",
|
|
details={
|
|
"failure_kind": GPU_NUMERICAL_MISMATCH if backend.get("selected") == "gpu" else COMPARISON_FAILURE,
|
|
"mismatches": mismatches,
|
|
"likely_causes": comparison_attribution,
|
|
},
|
|
)
|
|
|
|
|
|
def write_report(report: Mapping[str, Any], path: Path) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(json_ready(report), indent=2, sort_keys=True, allow_nan=False) + "\n")
|
|
|
|
|
|
def fmt_sci(value: Any) -> str:
|
|
if value is None:
|
|
return "none"
|
|
try:
|
|
return f"{float(value):.3e}"
|
|
except (TypeError, ValueError):
|
|
return str(value)
|
|
|
|
|
|
def format_location(location: Mapping[str, Any] | None) -> str:
|
|
if not location:
|
|
return "none"
|
|
component = location.get("component_index")
|
|
component_text = "" if component is None else f" component={component}"
|
|
return f"{location.get('entity_kind')}[{location.get('entity_index')}]{component_text}"
|
|
|
|
|
|
def print_human_summary(report: Mapping[str, Any]) -> None:
|
|
status = report.get("status")
|
|
print(f"airfrans_stepper_verification {status}")
|
|
print(f"source={report['paths']['source']}")
|
|
print(f"work={report['paths']['work']}")
|
|
print(f"report={report['paths']['report']}")
|
|
entrypoint = report.get("harness_entrypoint") or {}
|
|
print(f"harness_entrypoint={entrypoint.get('command')}")
|
|
print(f"harness_scope={entrypoint.get('scope') or report.get('harness', {}).get('scope')}")
|
|
print(f"rtol={report['tolerances']['rtol']} atol={report['tolerances']['atol']}")
|
|
backend = report.get("backend", {})
|
|
print(f"backend_requested={backend.get('requested')} backend_selected={backend.get('selected')}")
|
|
differentiability = report.get("differentiability") or {}
|
|
if differentiability:
|
|
claims = differentiability.get("claims") or []
|
|
checks = differentiability.get("checks") or []
|
|
print(f"differentiability_status={differentiability.get('status')}")
|
|
print(f"differentiability_claims={len(claims)} non_differentiable_boundaries={len(differentiability.get('non_differentiable') or [])}")
|
|
print(f"differentiability_checks={[check.get('status') for check in checks]}")
|
|
evidence = report.get("verifier_evidence") or {}
|
|
if evidence:
|
|
print(f"evidence_passed={evidence.get('passed')} evidence_status_basis={evidence.get('status_basis')}")
|
|
print(f"evidence_criteria={evidence.get('criteria')}")
|
|
|
|
if status != "passed":
|
|
failure = report.get("failure") or {}
|
|
print(f"failure_category={failure.get('category')}")
|
|
print(f"failure_step={failure.get('step')}")
|
|
print(f"failure_message={failure.get('message')}")
|
|
timing = report.get("timing") or {}
|
|
if timing.get("status") == "reported":
|
|
openfoam = timing.get("openfoam_reference") or {}
|
|
gpu_solver = timing.get("gpu_solver") or {}
|
|
run_one_timing = ((gpu_solver.get("run_one") or {}).get("timing") or {})
|
|
run_one_profiler = ((gpu_solver.get("run_one") or {}).get("profiler") or {})
|
|
print(f"openfoam_oracle_duration_seconds={openfoam.get('oracle_solver_duration_seconds')}")
|
|
print(f"gpu_solver_run_one_wall_seconds={run_one_timing.get('gpu_solver_wall_seconds')}")
|
|
print(f"gpu_solver_run_one_device_time_ms={run_one_profiler.get('device_time_ms_total')}")
|
|
mismatches = (failure.get("details") or {}).get("mismatches") or report.get("comparison_mismatches") or []
|
|
if mismatches:
|
|
first = mismatches[0]
|
|
attribution = first.get("attribution") or {}
|
|
print(f"comparison_failure_mode={first.get('mode')}")
|
|
print(f"comparison_failure_field={first.get('field')}")
|
|
print(f"comparison_failure_reason={first.get('reason')}")
|
|
print(f"comparison_failure_likely_stage_group={attribution.get('likely_stage_group')}")
|
|
print(f"comparison_failure_likely_stages={attribution.get('likely_stages')}")
|
|
print(f"comparison_failure_max_abs={fmt_sci(first.get('max_abs'))}")
|
|
print(f"comparison_failure_location={format_location(first.get('location'))}")
|
|
return
|
|
|
|
oracle_mesh = report["mesh_identity"]["oracle"]
|
|
patch_names = [patch["name"] for patch in oracle_mesh["patches"]]
|
|
print(f"oracle_case={report['case_preparation']['oracle_case']}")
|
|
prepared_identity = report["case_preparation"].get("prepared_case_identity", {})
|
|
prepared_digest = prepared_identity.get("prepared", {}).get("files_sha256")
|
|
print(f"prepared_case={report['case_preparation']['prepared_case']}")
|
|
print(f"prepared_case_files_sha256={prepared_digest}")
|
|
print(f"run_one_case={report['modes']['run_one']['case']}")
|
|
if report["modes"].get("split", {}).get("enabled"):
|
|
print(f"split_case={report['modes']['split']['case']}")
|
|
print(f"mesh_cells={oracle_mesh['n_cells']}")
|
|
print(f"mesh_internal_faces={oracle_mesh['n_internal_faces']}")
|
|
print(f"mesh_patches={patch_names}")
|
|
print(f"mesh_topology_sha256={oracle_mesh['topology_sha256']}")
|
|
|
|
for mode in ("run_one", "split"):
|
|
mode_report = report["modes"].get(mode, {})
|
|
if not mode_report.get("enabled", False):
|
|
continue
|
|
for field_name in REQUIRED_FIELDS:
|
|
data = mode_report.get("comparisons", {}).get(field_name)
|
|
if not data:
|
|
continue
|
|
print(
|
|
f"mode={mode} field={field_name} actual_shape={tuple(data['actual_shape'])} "
|
|
f"expected_shape={tuple(data['expected_shape'])} max_abs={fmt_sci(data.get('max_abs'))} "
|
|
f"location={format_location(data.get('location'))} allclose={data['allclose']}"
|
|
)
|
|
graph = mode_report.get("graph")
|
|
if graph:
|
|
print(f"{mode}_graph={graph}")
|
|
observability = mode_report.get("observability")
|
|
if observability:
|
|
observed_groups = [group["name"] for group in observability.get("groups", []) if group.get("observed")]
|
|
print(f"{mode}_observed_stage_groups={observed_groups}")
|
|
|
|
split_report = report["modes"].get("split", {})
|
|
if split_report.get("enabled") and "UEqn" in split_report and "pEqn" in split_report:
|
|
print(f"split_momentum_terms={split_report.get('momentum_terms')}")
|
|
print(f"split_UEqn_diag_shape={tuple(split_report['UEqn']['diag']['shape'])}")
|
|
print(f"split_pEqn_diag_shape={tuple(split_report['pEqn']['diag']['shape'])}")
|
|
|
|
print(f"visible_turbulence_fields={report['tracked_turbulence_fields']}")
|
|
|
|
|
|
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--source", type=Path, default=DEFAULT_SOURCE)
|
|
parser.add_argument("--work", type=Path, default=DEFAULT_WORK)
|
|
parser.add_argument("--report", type=Path, default=None, help="JSON report path; defaults to WORK/verifier_report.json")
|
|
parser.add_argument("--rtol", type=float, default=1e-8)
|
|
parser.add_argument("--atol", type=float, default=1e-8)
|
|
parser.add_argument("--backend", choices=BACKEND_CHOICES, default="auto", help="Backend request; gpu means the full RANS solver backend and currently fails until that backend exists")
|
|
parser.add_argument("--skip-split", action="store_true", help="Skip the explicit split-step parity check for local debugging")
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = parse_args(argv)
|
|
args.source = args.source.resolve()
|
|
args.work = args.work.resolve()
|
|
if args.report is None:
|
|
args.report = args.work / "verifier_report.json"
|
|
else:
|
|
args.report = args.report.resolve()
|
|
|
|
report = base_report(args)
|
|
exit_code = 0
|
|
try:
|
|
run_harness(args, report)
|
|
evidence = report.get("verifier_evidence")
|
|
if not isinstance(evidence, Mapping) or evidence.get("passed") is not True:
|
|
raise HarnessError(
|
|
INTERNAL_FAILURE,
|
|
"verifier_evidence",
|
|
"verifier completed without passing evidence criteria",
|
|
details={"verifier_evidence": evidence},
|
|
)
|
|
report["status"] = "passed"
|
|
except HarnessError as exc:
|
|
report["status"] = "failed"
|
|
report["failure"] = exc.to_dict()
|
|
exit_code = EXIT_CODES.get(exc.category, EXIT_CODES[INTERNAL_FAILURE])
|
|
except Exception as exc: # pragma: no cover - keeps CLI failures categorized in the report.
|
|
report["status"] = "failed"
|
|
report["failure"] = json_ready(
|
|
{
|
|
"category": INTERNAL_FAILURE,
|
|
"step": "run_harness",
|
|
"message": str(exc),
|
|
"cause": {"type": type(exc).__name__, "message": str(exc)},
|
|
"traceback": traceback.format_exc(),
|
|
}
|
|
)
|
|
exit_code = EXIT_CODES[INTERNAL_FAILURE]
|
|
|
|
try:
|
|
write_report(report, args.report)
|
|
except Exception as exc:
|
|
report["status"] = "failed"
|
|
report["failure"] = json_ready(
|
|
{
|
|
"category": INTERNAL_FAILURE,
|
|
"step": "write_report",
|
|
"message": f"failed to write verifier report: {args.report}",
|
|
"cause": {"type": type(exc).__name__, "message": str(exc)},
|
|
}
|
|
)
|
|
exit_code = EXIT_CODES[INTERNAL_FAILURE]
|
|
|
|
print_human_summary(json_ready(report))
|
|
return exit_code
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|