3218 lines
140 KiB
Python
Executable file
3218 lines
140 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Verifier harness for the migrated AirfRANS OpenFOAM stepper case."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import dataclasses
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import traceback
|
|
from collections.abc import Iterable, Mapping
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
def _drop_ambient_pythonpath() -> None:
|
|
pythonpath = os.environ.pop("PYTHONPATH", "")
|
|
for entry in pythonpath.split(os.pathsep):
|
|
if not entry:
|
|
continue
|
|
while entry in sys.path:
|
|
sys.path.remove(entry)
|
|
|
|
|
|
_drop_ambient_pythonpath()
|
|
|
|
import numpy as np
|
|
|
|
from openfoam_env import apply_openfoam_env, openfoam_env
|
|
from prepare_airfrans_stepper_case import (
|
|
DEFAULT_SOURCE,
|
|
REQUIRED_COMPARISON_FIELDS,
|
|
load_case_manifest,
|
|
prepare_case,
|
|
write_case_manifest,
|
|
)
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
DEFAULT_WORK = ROOT / "tmp/airfrans_stepper_verify"
|
|
PRIMARY_FIELDS = ("U", "p", "phi")
|
|
TURBULENCE_FIELDS = ("nut", "k", "omega")
|
|
REQUIRED_FIELDS = REQUIRED_COMPARISON_FIELDS
|
|
REQUIRED_EXECUTION_MODES = ("run_one", "split")
|
|
BACKEND_CHOICES = ("auto", "cpu", "gpu")
|
|
FULL_GPU_RANS_GUARD = "scripts/verify_gpu_rans_solver.sh"
|
|
GPU_PRIMITIVE_PROOF = "scripts/verify_gpu_algorithm.sh"
|
|
GPU_PRIMITIVE_NAME = "cell_flux_imbalance"
|
|
AIRFRANS_VERIFIER_SCOPE = (
|
|
"AirfRANS/OpenFOAM stepper parity verifier; --backend gpu is a GPU-owned "
|
|
"RANS solver contract with no CPU fallback or primitive-only acceptance"
|
|
)
|
|
|
|
LOADING_FAILURE = "loading_or_parsing_failure"
|
|
ORACLE_FAILURE = "openfoam_oracle_failure"
|
|
STEPPER_FAILURE = "stepper_execution_failure"
|
|
COMPARISON_FAILURE = "numerical_comparison_failure"
|
|
BACKEND_FAILURE = "backend_execution_failure"
|
|
INTERNAL_FAILURE = "internal_harness_failure"
|
|
REQUIRED_EVIDENCE_FAILURE = "required_evidence_failure"
|
|
|
|
EXIT_CODES = {
|
|
LOADING_FAILURE: 2,
|
|
ORACLE_FAILURE: 3,
|
|
STEPPER_FAILURE: 4,
|
|
COMPARISON_FAILURE: 5,
|
|
BACKEND_FAILURE: 6,
|
|
REQUIRED_EVIDENCE_FAILURE: 8,
|
|
INTERNAL_FAILURE: 7,
|
|
}
|
|
|
|
STAGE_OBSERVABILITY_GROUPS = (
|
|
{
|
|
"name": "momentum_assembly",
|
|
"split_stages": ("assemble_momentum_terms", "assemble_UEqn"),
|
|
"run_one_stages": ("assemble_UEqn",),
|
|
"split_outputs": {
|
|
"assemble_momentum_terms": ("terms",),
|
|
"assemble_UEqn": ("UEqn",),
|
|
},
|
|
"run_one_outputs": {
|
|
"assemble_UEqn": ("UEqn",),
|
|
},
|
|
},
|
|
{
|
|
"name": "pressure_assembly",
|
|
"split_stages": ("compute_pressure_inputs", "assemble_pEqn"),
|
|
"run_one_stages": ("compute_pressure_inputs", "assemble_pEqn"),
|
|
"split_outputs": {
|
|
"compute_pressure_inputs": ("HbyA", "phiHbyA", "rAU", "rAtU"),
|
|
"assemble_pEqn": ("pEqn",),
|
|
},
|
|
"run_one_outputs": {
|
|
"compute_pressure_inputs": ("HbyA", "phiHbyA", "rAU", "rAtU"),
|
|
"assemble_pEqn": ("pEqn",),
|
|
},
|
|
},
|
|
{
|
|
"name": "linear_solve_results",
|
|
"split_stages": ("solve_UEqn", "solve_pEqn"),
|
|
"run_one_stages": ("solve_UEqn", "solve_pEqn"),
|
|
"split_outputs": {
|
|
"solve_UEqn": ("performance", "field_after"),
|
|
"solve_pEqn": ("performance", "p", "phi"),
|
|
},
|
|
"run_one_outputs": {
|
|
"solve_UEqn": ("performance", "field_after"),
|
|
"solve_pEqn": ("performance", "p", "phi"),
|
|
},
|
|
},
|
|
{
|
|
"name": "final_correction",
|
|
"split_stages": ("correct_velocity_pressure_flux",),
|
|
"run_one_stages": ("update_phi_from_pEqn_flux", "correct_velocity_pressure_flux"),
|
|
"split_outputs": {
|
|
"correct_velocity_pressure_flux": ("U", "p", "phi"),
|
|
},
|
|
"run_one_outputs": {
|
|
"update_phi_from_pEqn_flux": ("phi",),
|
|
"correct_velocity_pressure_flux": ("U", "p", "phi"),
|
|
},
|
|
},
|
|
{
|
|
"name": "turbulence_updates",
|
|
"split_stages": ("momentum_transport_predict", "momentum_transport_correct"),
|
|
"run_one_stages": ("momentum_transport_predict", "momentum_transport_correct"),
|
|
"split_outputs": {
|
|
"momentum_transport_predict": ("case_path", "solver_name"),
|
|
"momentum_transport_correct": ("U", "p", "phi", "nut", "k", "omega"),
|
|
},
|
|
"run_one_outputs": {
|
|
"momentum_transport_predict": ("case_path", "solver_name"),
|
|
"momentum_transport_correct": ("U", "p", "phi", "nut", "k", "omega"),
|
|
},
|
|
},
|
|
)
|
|
|
|
STAGE_GROUP_ORDER = {group["name"]: index for index, group in enumerate(STAGE_OBSERVABILITY_GROUPS)}
|
|
MODE_COMPARISON_ORDER = {"run_one": 0, "split": 1}
|
|
|
|
FIELD_COMPARISON_ATTRIBUTION = {
|
|
"U": ("final_correction", "linear_solve_results", "momentum_assembly", "turbulence_updates"),
|
|
"p": ("linear_solve_results", "pressure_assembly", "final_correction"),
|
|
"phi": ("final_correction", "linear_solve_results", "pressure_assembly"),
|
|
"nut": ("turbulence_updates",),
|
|
"k": ("turbulence_updates",),
|
|
"omega": ("turbulence_updates",),
|
|
}
|
|
|
|
INTERMEDIATE_ARTIFACT_FAMILIES = (
|
|
{
|
|
"name": "identity",
|
|
"stage_groups": (),
|
|
"reference_paths": ("case_preparation.prepared_case_identity", "mesh_identity.oracle"),
|
|
"candidate_paths": ("mesh_identity.run_one", "mesh_identity.split"),
|
|
"comparison_paths": ("modes.run_one.mesh_comparison", "modes.split.mesh_comparison"),
|
|
"required": True,
|
|
},
|
|
{
|
|
"name": "pressure_inputs",
|
|
"stage_groups": ("pressure_assembly",),
|
|
"reference_paths": ("modes.reference_split.stages.compute_pressure_inputs",),
|
|
"candidate_paths": ("modes.split.stages.compute_pressure_inputs",),
|
|
"comparison_paths": ("artifact_comparisons.pressure_inputs",),
|
|
"required": True,
|
|
},
|
|
{
|
|
"name": "matrix_operator",
|
|
"stage_groups": ("momentum_assembly", "pressure_assembly"),
|
|
"reference_paths": ("modes.reference_split.momentum_terms", "modes.reference_split.UEqn", "modes.reference_split.stages.relax_UEqn.outputs.UEqn", "modes.reference_split.pEqn", "state_exports.reference_split_matrices"),
|
|
"candidate_paths": ("modes.split.momentum_terms", "modes.split.UEqn", "modes.split.pEqn", "state_exports.split_matrices"),
|
|
"comparison_paths": ("artifact_comparisons.matrix_operator",),
|
|
"required": True,
|
|
},
|
|
{
|
|
"name": "solver",
|
|
"stage_groups": ("linear_solve_results",),
|
|
"reference_paths": ("modes.reference_split.stages.solve_UEqn", "modes.reference_split.stages.solve_pEqn"),
|
|
"candidate_paths": ("modes.split.stages.solve_UEqn", "modes.split.stages.solve_pEqn"),
|
|
"comparison_paths": ("artifact_comparisons.solver",),
|
|
"required": True,
|
|
},
|
|
{
|
|
"name": "correction",
|
|
"stage_groups": ("final_correction",),
|
|
"reference_paths": ("modes.oracle.fields.U", "modes.oracle.fields.p"),
|
|
"candidate_paths": ("modes.split.stages.correct_velocity_pressure_flux", "modes.split.fields.U", "modes.split.fields.p"),
|
|
"comparison_paths": ("modes.split.comparisons.U", "modes.split.comparisons.p"),
|
|
"required": True,
|
|
},
|
|
{
|
|
"name": "flux",
|
|
"stage_groups": ("pressure_assembly", "final_correction"),
|
|
"reference_paths": ("modes.oracle.fields.phi",),
|
|
"candidate_paths": ("modes.split.stages.compute_pressure_inputs", "modes.split.fields.phi"),
|
|
"comparison_paths": ("modes.split.comparisons.phi",),
|
|
"required": True,
|
|
},
|
|
{
|
|
"name": "turbulence",
|
|
"stage_groups": ("turbulence_updates",),
|
|
"reference_paths": ("modes.oracle.fields.nut", "modes.oracle.fields.k", "modes.oracle.fields.omega"),
|
|
"candidate_paths": (
|
|
"modes.split.stages.momentum_transport_predict",
|
|
"modes.split.stages.momentum_transport_correct",
|
|
"modes.split.fields.nut",
|
|
"modes.split.fields.k",
|
|
"modes.split.fields.omega",
|
|
),
|
|
"comparison_paths": ("modes.split.comparisons.nut", "modes.split.comparisons.k", "modes.split.comparisons.omega"),
|
|
"required": True,
|
|
},
|
|
)
|
|
|
|
ARTIFACT_FAMILY_ORDER = {family["name"]: index for index, family in enumerate(INTERMEDIATE_ARTIFACT_FAMILIES)}
|
|
|
|
GPU_RUNTIME_UNAVAILABLE = "gpu_runtime_unavailable"
|
|
GPU_NUMERICAL_MISMATCH = "gpu_numerical_mismatch"
|
|
GPU_INPUT_SCHEMA_VERSION = 1
|
|
|
|
DIFFERENTIABILITY_VARIABLES = {
|
|
"fields": REQUIRED_FIELDS,
|
|
"mesh_state": ("points", "faces", "owner", "neighbour", "cells", "V", "C", "Cf", "Sf", "magSf", "patches"),
|
|
"case_parameters": ("Uinf", "nu", "rhoInf", "angle_of_attack", "liftDir", "dragDir", "solver_tolerances"),
|
|
"losses": ("per_field_linf_error", "per_field_l2_error", "oracle_parity_allclose"),
|
|
}
|
|
|
|
NON_DIFFERENTIABLE_BOUNDARIES = (
|
|
{
|
|
"name": "openfoam_case_io",
|
|
"category": "io",
|
|
"reason": "OpenFOAM dictionaries, mesh files, and field files are parsed and written as discrete external data.",
|
|
},
|
|
{
|
|
"name": "mesh_topology_and_patch_addressing",
|
|
"category": "topology",
|
|
"reason": "Face/cell connectivity, owner/neighbour addressing, and boundary patch membership are discrete indices.",
|
|
},
|
|
{
|
|
"name": "boundary_condition_selection",
|
|
"category": "discrete_boundary_choice",
|
|
"reason": "Patch types, coupled/constraint flags, and boundary condition dictionaries select code paths.",
|
|
},
|
|
{
|
|
"name": "simple_pimple_and_linear_solver_control",
|
|
"category": "convergence_control",
|
|
"reason": "Iteration counts, stopping criteria, and solver convergence branches are discrete control flow.",
|
|
},
|
|
{
|
|
"name": "turbulence_bounding_and_limiters",
|
|
"category": "limiter",
|
|
"reason": "k/omega/nut bounding and model limiters introduce clipping or branch-dependent updates.",
|
|
},
|
|
{
|
|
"name": "openfoam_cpp_backend_calls",
|
|
"category": "unsupported_solver_operation",
|
|
"reason": "The current registered backend executes OpenFOAM C++ operations and exposes no AD, JVP, or VJP API.",
|
|
},
|
|
)
|
|
|
|
CUSTOM_GRADIENT_REQUIREMENTS = (
|
|
{
|
|
"operation": "sparse_linear_solves",
|
|
"stage_groups": ("linear_solve_results",),
|
|
"reason": "Implicit sparse solves require an adjoint or custom VJP rather than differentiating solver iterations blindly.",
|
|
},
|
|
{
|
|
"operation": "pressure_velocity_correction",
|
|
"stage_groups": ("pressure_assembly", "final_correction"),
|
|
"reason": "The coupled pressure/flux/velocity correction needs a consistent custom gradient for the assembled operators.",
|
|
},
|
|
{
|
|
"operation": "turbulence_model_correctors_and_limiters",
|
|
"stage_groups": ("turbulence_updates",),
|
|
"reason": "Model correctors, wall functions, and bounding need explicit subgradient or smoothing choices.",
|
|
},
|
|
{
|
|
"operation": "mesh_and_boundary_discrete_choices",
|
|
"stage_groups": (),
|
|
"reason": "Topology and patch-type changes are not differentiable; only fixed-topology numeric values can be checked.",
|
|
},
|
|
)
|
|
|
|
|
|
class HarnessError(Exception):
|
|
"""Categorized verifier failure that can be serialized into the report."""
|
|
|
|
def __init__(
|
|
self,
|
|
category: str,
|
|
step: str,
|
|
message: str,
|
|
*,
|
|
details: Mapping[str, Any] | None = None,
|
|
cause: BaseException | None = None,
|
|
) -> None:
|
|
super().__init__(message)
|
|
self.category = category
|
|
self.step = step
|
|
self.details = dict(details or {})
|
|
self.cause = cause
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
out: dict[str, Any] = {
|
|
"category": self.category,
|
|
"step": self.step,
|
|
"message": str(self),
|
|
"details": self.details,
|
|
}
|
|
if self.cause is not None:
|
|
out["cause"] = {
|
|
"type": type(self.cause).__name__,
|
|
"message": str(self.cause),
|
|
}
|
|
return json_ready(out)
|
|
|
|
class BackendExecutionError(Exception):
|
|
"""Backend selection or execution failure before numerical comparison."""
|
|
|
|
def __init__(self, step: str, message: str, *, details: Mapping[str, Any] | None = None) -> None:
|
|
super().__init__(message)
|
|
self.step = step
|
|
self.details = dict(details or {})
|
|
|
|
def to_harness_error(self) -> HarnessError:
|
|
return HarnessError(BACKEND_FAILURE, self.step, str(self), details=self.details, cause=self)
|
|
|
|
|
|
|
|
|
|
def json_ready(value: Any) -> Any:
|
|
"""Convert report values into strict JSON-compatible data."""
|
|
|
|
if dataclasses.is_dataclass(value) and not isinstance(value, type):
|
|
return json_ready(dataclasses.asdict(value))
|
|
if isinstance(value, Path):
|
|
return str(value)
|
|
if isinstance(value, np.ndarray):
|
|
return array_stats(value)
|
|
if isinstance(value, np.generic):
|
|
return json_ready(value.item())
|
|
if isinstance(value, float):
|
|
return value if math.isfinite(value) else None
|
|
if isinstance(value, (str, int, bool)) or value is None:
|
|
return value
|
|
if isinstance(value, Mapping):
|
|
return {str(key): json_ready(item) for key, item in value.items()}
|
|
if isinstance(value, (list, tuple, set)):
|
|
return [json_ready(item) for item in value]
|
|
return repr(value)
|
|
|
|
def report_path_value(report: Mapping[str, Any], path: str) -> Any:
|
|
cursor: Any = report
|
|
for part in path.split("."):
|
|
if isinstance(cursor, Mapping):
|
|
cursor = cursor.get(part)
|
|
continue
|
|
if isinstance(cursor, list):
|
|
matched = None
|
|
for item in cursor:
|
|
if isinstance(item, Mapping) and item.get("name") == part:
|
|
matched = item
|
|
break
|
|
cursor = matched
|
|
continue
|
|
return None
|
|
return cursor
|
|
|
|
|
|
def report_path_exists(report: Mapping[str, Any], path: str) -> bool:
|
|
value = report_path_value(report, path)
|
|
if value is None:
|
|
return False
|
|
if isinstance(value, Mapping) or isinstance(value, list):
|
|
return bool(value)
|
|
return True
|
|
|
|
|
|
|
|
def array_shape(array: Any | None) -> list[int] | None:
|
|
if array is None:
|
|
return None
|
|
return [int(dim) for dim in np.asarray(array).shape]
|
|
|
|
|
|
def finite_float(value: Any) -> float | None:
|
|
try:
|
|
number = float(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
return number if math.isfinite(number) else None
|
|
|
|
|
|
def array_stats(array: Any) -> dict[str, Any]:
|
|
arr = np.asarray(array)
|
|
out: dict[str, Any] = {
|
|
"shape": array_shape(arr),
|
|
"dtype": str(arr.dtype),
|
|
"size": int(arr.size),
|
|
}
|
|
if arr.dtype != object:
|
|
contiguous = np.ascontiguousarray(arr)
|
|
out["sha256"] = hashlib.sha256(contiguous.tobytes()).hexdigest()
|
|
if arr.size == 0 or not np.issubdtype(arr.dtype, np.number):
|
|
return out
|
|
|
|
finite = np.isfinite(arr)
|
|
out["finite_count"] = int(np.count_nonzero(finite))
|
|
out["nonfinite_count"] = int(arr.size - out["finite_count"])
|
|
if out["finite_count"]:
|
|
finite_values = arr[finite]
|
|
out.update(
|
|
{
|
|
"min": finite_float(np.min(finite_values)),
|
|
"max": finite_float(np.max(finite_values)),
|
|
"mean": finite_float(np.mean(finite_values)),
|
|
}
|
|
)
|
|
return out
|
|
|
|
|
|
def value_at(array: np.ndarray, index: tuple[int, ...]) -> Any:
|
|
value = array[index]
|
|
if isinstance(value, np.generic):
|
|
return json_ready(value.item())
|
|
if isinstance(value, np.ndarray):
|
|
return json_ready(value.tolist())
|
|
return json_ready(value)
|
|
|
|
|
|
def entity_value_at(array: np.ndarray, entity_index: int | None) -> Any:
|
|
if entity_index is None or array.ndim == 0:
|
|
return None
|
|
value = array[entity_index]
|
|
if isinstance(value, np.generic):
|
|
return json_ready(value.item())
|
|
if isinstance(value, np.ndarray):
|
|
return json_ready(value.tolist())
|
|
return json_ready(value)
|
|
|
|
|
|
def update_hash_text(digest: "hashlib._Hash", value: str) -> None:
|
|
encoded = value.encode("utf-8")
|
|
digest.update(len(encoded).to_bytes(8, "little"))
|
|
digest.update(encoded)
|
|
|
|
|
|
def update_hash_array(digest: "hashlib._Hash", label: str, array: Any) -> None:
|
|
arr = np.ascontiguousarray(np.asarray(array))
|
|
update_hash_text(digest, label)
|
|
update_hash_text(digest, str(arr.dtype))
|
|
update_hash_text(digest, repr(tuple(int(dim) for dim in arr.shape)))
|
|
digest.update(arr.tobytes())
|
|
|
|
|
|
def source_summary(source: Any) -> dict[str, Any]:
|
|
return {
|
|
"file": getattr(source, "file", ""),
|
|
"function": getattr(source, "function", ""),
|
|
"lines": json_ready(getattr(source, "lines", None)),
|
|
}
|
|
|
|
|
|
def boundary_field_summary(patch: Any) -> dict[str, Any]:
|
|
return {
|
|
"type": getattr(patch, "type", ""),
|
|
"values_shape": array_shape(getattr(patch, "values", None)),
|
|
"fixes_value": bool(getattr(patch, "fixes_value", False)),
|
|
"assignable": bool(getattr(patch, "assignable", False)),
|
|
"coupled": bool(getattr(patch, "coupled", False)),
|
|
"updated": bool(getattr(patch, "updated", False)),
|
|
"patch_internal_shape": array_shape(getattr(patch, "patch_internal", None)),
|
|
"value_internal_coeffs_shape": array_shape(getattr(patch, "value_internal_coeffs", None)),
|
|
"value_boundary_coeffs_shape": array_shape(getattr(patch, "value_boundary_coeffs", None)),
|
|
"gradient_internal_coeffs_shape": array_shape(getattr(patch, "gradient_internal_coeffs", None)),
|
|
"gradient_boundary_coeffs_shape": array_shape(getattr(patch, "gradient_boundary_coeffs", None)),
|
|
}
|
|
|
|
|
|
def field_summary(field: Any) -> dict[str, Any]:
|
|
boundary = getattr(field, "boundary", {})
|
|
return {
|
|
"name": getattr(field, "name", ""),
|
|
"kind": getattr(field, "kind", ""),
|
|
"dimensions": getattr(field, "dimensions", ""),
|
|
"entity_kind": getattr(field, "entity_kind", ""),
|
|
"entity_count": int(getattr(field, "entity_count", 0)),
|
|
"internal": array_stats(getattr(field, "internal")),
|
|
"boundary": {name: boundary_field_summary(patch) for name, patch in boundary.items()},
|
|
}
|
|
|
|
|
|
def solve_summary(performance: Any) -> dict[str, Any]:
|
|
return {
|
|
"solver_name": getattr(performance, "solver_name", ""),
|
|
"field_name": getattr(performance, "field_name", ""),
|
|
"initial_residual": json_ready(getattr(performance, "initial_residual", None)),
|
|
"final_residual": json_ready(getattr(performance, "final_residual", None)),
|
|
"n_iterations": json_ready(getattr(performance, "n_iterations", None)),
|
|
"converged": bool(getattr(performance, "converged", False)),
|
|
"singular": bool(getattr(performance, "singular", False)),
|
|
}
|
|
|
|
|
|
def matrix_summary(matrix: Any) -> dict[str, Any]:
|
|
derived: dict[str, Any] = {}
|
|
for name in ("A", "H", "H1", "flux", "face_flux_correction"):
|
|
value = getattr(matrix, name, None)
|
|
if callable(value):
|
|
value = value()
|
|
if value is not None:
|
|
derived[name] = field_summary(value)
|
|
|
|
for name in ("residual", "D", "DD"):
|
|
value = getattr(matrix, name, None)
|
|
if callable(value):
|
|
value = value()
|
|
if value is not None:
|
|
derived[name] = array_stats(value)
|
|
|
|
return {
|
|
"name": getattr(matrix, "name", ""),
|
|
"field_name": getattr(matrix, "field_name", ""),
|
|
"value_rank": getattr(matrix, "value_rank", ""),
|
|
"dimensions": getattr(matrix, "dimensions", ""),
|
|
"has_diag": bool(getattr(matrix, "has_diag", False)),
|
|
"has_upper": bool(getattr(matrix, "has_upper", False)),
|
|
"has_lower": bool(getattr(matrix, "has_lower", False)),
|
|
"diagonal": bool(getattr(matrix, "diagonal", False)),
|
|
"symmetric": bool(getattr(matrix, "symmetric", False)),
|
|
"asymmetric": bool(getattr(matrix, "asymmetric", False)),
|
|
"diag": array_stats(getattr(matrix, "diag")),
|
|
"upper": None if getattr(matrix, "upper", None) is None else array_stats(getattr(matrix, "upper")),
|
|
"lower": None if getattr(matrix, "lower", None) is None else array_stats(getattr(matrix, "lower")),
|
|
"source": array_stats(getattr(matrix, "source")),
|
|
"psi": field_summary(getattr(matrix, "psi")),
|
|
"internal_coeff_shapes": [array_shape(item) for item in getattr(matrix, "internal_coeffs", [])],
|
|
"boundary_coeff_shapes": [array_shape(item) for item in getattr(matrix, "boundary_coeffs", [])],
|
|
"derived": derived,
|
|
}
|
|
|
|
|
|
def summarize_value(value: Any) -> Any:
|
|
if hasattr(value, "diag") and hasattr(value, "field_name") and hasattr(value, "source"):
|
|
return matrix_summary(value)
|
|
if hasattr(value, "internal") and hasattr(value, "entity_kind") and hasattr(value, "boundary"):
|
|
return field_summary(value)
|
|
if hasattr(value, "solver_name") and hasattr(value, "initial_residual"):
|
|
return solve_summary(value)
|
|
if hasattr(value, "name") and hasattr(value, "phase") and hasattr(value, "outputs"):
|
|
return transform_summary(value)
|
|
if isinstance(value, Mapping):
|
|
return {str(key): summarize_value(item) for key, item in value.items()}
|
|
if isinstance(value, list):
|
|
return [summarize_value(item) for item in value]
|
|
if isinstance(value, tuple):
|
|
return [summarize_value(item) for item in value]
|
|
return json_ready(value)
|
|
|
|
|
|
def transform_summary(result: Any) -> dict[str, Any]:
|
|
return {
|
|
"name": getattr(result, "name", ""),
|
|
"phase": getattr(result, "phase", ""),
|
|
"changed_fields": json_ready(getattr(result, "changed_fields", [])),
|
|
"source": source_summary(getattr(result, "source", None)),
|
|
"metadata": json_ready(getattr(result, "metadata", {})),
|
|
"outputs": summarize_value(getattr(result, "outputs", {})),
|
|
}
|
|
|
|
def graph_stage_summaries(result: Any) -> list[dict[str, Any]]:
|
|
return [transform_summary(entry) for entry in getattr(result, "outputs", {}).get("graph", [])]
|
|
|
|
|
|
def stage_observability_report(mode: str, stages: list[dict[str, Any]], *, evidence: Mapping[str, Any]) -> dict[str, Any]:
|
|
stage_by_name = {stage["name"]: stage for stage in stages}
|
|
failures: list[dict[str, Any]] = []
|
|
groups = []
|
|
for group in STAGE_OBSERVABILITY_GROUPS:
|
|
group_name = group["name"]
|
|
required_stages = tuple(group[f"{mode}_stages"])
|
|
missing_stages = [name for name in required_stages if name not in stage_by_name]
|
|
output_requirements = group.get(f"{mode}_outputs", {})
|
|
backend = evidence.get("backend", {}) if isinstance(evidence.get("backend"), Mapping) else {}
|
|
if mode == "run_one" and backend.get("selected") != "gpu":
|
|
output_requirements = {}
|
|
output_checks = {}
|
|
for stage_name, required_outputs in output_requirements.items():
|
|
stage = stage_by_name.get(stage_name)
|
|
if stage is None:
|
|
continue
|
|
outputs = stage.get("outputs", {})
|
|
output_keys = set(outputs) if isinstance(outputs, Mapping) else set()
|
|
missing_outputs = [name for name in required_outputs if name not in output_keys]
|
|
output_checks[stage_name] = {
|
|
"required": list(required_outputs),
|
|
"available": sorted(output_keys),
|
|
"missing": missing_outputs,
|
|
}
|
|
if missing_outputs:
|
|
failures.append(
|
|
{
|
|
"path": f"stage_observability.{mode}.{group_name}.{stage_name}.outputs",
|
|
"message": "missing inspectable stage output",
|
|
"expected": list(required_outputs),
|
|
"actual": sorted(output_keys),
|
|
}
|
|
)
|
|
if missing_stages:
|
|
failures.append(
|
|
{
|
|
"path": f"stage_observability.{mode}.{group_name}.stages",
|
|
"message": "missing required solver stage",
|
|
"expected": list(required_stages),
|
|
"actual": list(stage_by_name),
|
|
}
|
|
)
|
|
groups.append(
|
|
{
|
|
"name": group_name,
|
|
"required_stages": list(required_stages),
|
|
"observed": not missing_stages and not any(check["missing"] for check in output_checks.values()),
|
|
"missing_stages": missing_stages,
|
|
"output_checks": output_checks,
|
|
}
|
|
)
|
|
if failures:
|
|
raise HarnessError(
|
|
STEPPER_FAILURE,
|
|
f"{mode}_stage_observability",
|
|
f"{mode} solver-stage observability is incomplete",
|
|
details={"failures": failures},
|
|
)
|
|
return {
|
|
"mode": mode,
|
|
"graph": [stage["name"] for stage in stages],
|
|
"groups": groups,
|
|
"evidence": json_ready(evidence),
|
|
}
|
|
|
|
|
|
def selected_field_summaries(fields: Mapping[str, Any]) -> dict[str, Any]:
|
|
return {name: field_summary(fields[name]) for name in REQUIRED_FIELDS if name in fields}
|
|
|
|
|
|
def field_dict_to_mapping(fields: Any) -> dict[str, Any]:
|
|
if isinstance(fields, Mapping):
|
|
return dict(fields)
|
|
if hasattr(fields, "fields") and isinstance(fields.fields, Mapping):
|
|
return dict(fields.fields)
|
|
return {name: getattr(fields, name) for name in REQUIRED_FIELDS if hasattr(fields, name)}
|
|
|
|
|
|
def mesh_patch_table(mesh: Any) -> list[dict[str, Any]]:
|
|
patches = []
|
|
for patch in getattr(mesh, "boundary", []):
|
|
patches.append(
|
|
{
|
|
"index": int(getattr(patch, "index", 0)),
|
|
"name": getattr(patch, "name", ""),
|
|
"type": getattr(patch, "type", ""),
|
|
"start": int(getattr(patch, "start", 0)),
|
|
"size": int(getattr(patch, "size", 0)),
|
|
"coupled": bool(getattr(patch, "coupled", False)),
|
|
"constraint": bool(getattr(patch, "constraint", False)),
|
|
}
|
|
)
|
|
return patches
|
|
|
|
|
|
def mesh_topology_digest(mesh: Any) -> str:
|
|
digest = hashlib.sha256()
|
|
for name in ("n_points", "n_faces", "n_internal_faces", "n_cells"):
|
|
update_hash_text(digest, f"{name}={int(getattr(mesh, name))}")
|
|
update_hash_array(digest, "faces.offsets", mesh.faces.offsets)
|
|
update_hash_array(digest, "faces.values", mesh.faces.values)
|
|
update_hash_array(digest, "cells.offsets", mesh.cells.offsets)
|
|
update_hash_array(digest, "cells.values", mesh.cells.values)
|
|
update_hash_array(digest, "owner", mesh.owner)
|
|
update_hash_array(digest, "neighbour", mesh.neighbour)
|
|
ldu = getattr(mesh, "ldu", {})
|
|
if isinstance(ldu, Mapping):
|
|
if "lower_addr" in ldu:
|
|
update_hash_array(digest, "ldu.lower_addr", ldu["lower_addr"])
|
|
if "upper_addr" in ldu:
|
|
update_hash_array(digest, "ldu.upper_addr", ldu["upper_addr"])
|
|
for entry in ldu.get("patch_addr", []):
|
|
update_hash_text(digest, f"ldu.patch_index={entry.get('patch_index')}")
|
|
update_hash_array(digest, "ldu.patch_addr", entry.get("addr", []))
|
|
for patch in mesh_patch_table(mesh):
|
|
update_hash_text(digest, json.dumps(patch, sort_keys=True))
|
|
return digest.hexdigest()
|
|
|
|
|
|
def mesh_geometry_digest(mesh: Any) -> str:
|
|
digest = hashlib.sha256()
|
|
for name in ("points", "V", "C", "Cf", "Sf", "magSf"):
|
|
update_hash_array(digest, name, getattr(mesh, name))
|
|
return digest.hexdigest()
|
|
|
|
|
|
def mesh_identity(mesh: Any) -> dict[str, Any]:
|
|
return {
|
|
"n_points": int(mesh.n_points),
|
|
"n_faces": int(mesh.n_faces),
|
|
"n_internal_faces": int(mesh.n_internal_faces),
|
|
"n_cells": int(mesh.n_cells),
|
|
"patches": mesh_patch_table(mesh),
|
|
"topology_sha256": mesh_topology_digest(mesh),
|
|
"geometry_sha256": mesh_geometry_digest(mesh),
|
|
}
|
|
|
|
|
|
def compare_mesh_identity(mode: str, actual: Mapping[str, Any], expected: Mapping[str, Any]) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
|
keys = ("n_points", "n_faces", "n_internal_faces", "n_cells", "topology_sha256", "geometry_sha256", "patches")
|
|
differences = {
|
|
key: {"actual": actual.get(key), "expected": expected.get(key)}
|
|
for key in keys
|
|
if actual.get(key) != expected.get(key)
|
|
}
|
|
report = {"matches_oracle": not differences, "differences": differences}
|
|
if not differences:
|
|
return report, []
|
|
return report, [{"mode": mode, "kind": "mesh_identity", "differences": differences}]
|
|
|
|
|
|
def field_comparison_attribution(
|
|
mode: str,
|
|
field: str,
|
|
reason: str,
|
|
stage_graph: Iterable[str] | None,
|
|
) -> dict[str, Any]:
|
|
observed_stages = set(stage_graph or ())
|
|
candidates = []
|
|
for group_name in FIELD_COMPARISON_ATTRIBUTION.get(field, ()):
|
|
group = next(item for item in STAGE_OBSERVABILITY_GROUPS if item["name"] == group_name)
|
|
stages = list(group.get(f"{mode}_stages", ()))
|
|
present_stages = [stage for stage in stages if not observed_stages or stage in observed_stages]
|
|
candidates.append(
|
|
{
|
|
"group": group_name,
|
|
"stages": present_stages,
|
|
"evidence_path": f"stage_observability.{mode}.{group_name}",
|
|
}
|
|
)
|
|
|
|
likely = candidates[0] if candidates else {"group": "unknown", "stages": [], "evidence_path": None}
|
|
out = {
|
|
"field": field,
|
|
"reason": reason,
|
|
"likely_stage_group": likely["group"],
|
|
"likely_stages": likely["stages"],
|
|
"candidate_stage_groups": candidates,
|
|
"evidence_path": likely["evidence_path"],
|
|
"basis": "field-to-stage dependency map for the selected solver graph",
|
|
}
|
|
if reason in {"missing_field", "shape_mismatch"}:
|
|
out["precondition_failure"] = "field_export_or_state_mapping"
|
|
out["field_evidence_path"] = f"modes.{mode}.fields.{field}"
|
|
out["basis"] = "field is absent or has the wrong topology; inspect export/state mapping first, then the mapped solver stages"
|
|
return out
|
|
|
|
|
|
def safe_array_item(array: Any, index: int) -> Any:
|
|
arr = np.asarray(array)
|
|
if index < 0 or index >= arr.shape[0]:
|
|
return None
|
|
value = arr[index]
|
|
if isinstance(value, np.ndarray):
|
|
return value.tolist()
|
|
if isinstance(value, np.generic):
|
|
return value.item()
|
|
return json_ready(value)
|
|
|
|
|
|
def cell_neighbourhood_context(mesh: Any, cell_index: int) -> dict[str, Any]:
|
|
owner = np.asarray(getattr(mesh, "owner", []))
|
|
neighbour = np.asarray(getattr(mesh, "neighbour", []))
|
|
owner_faces = np.flatnonzero(owner == cell_index)
|
|
neighbour_faces = np.flatnonzero(neighbour == cell_index)
|
|
touching_faces = sorted({int(face) for face in np.concatenate((owner_faces, neighbour_faces))})[:12]
|
|
boundary_patches = []
|
|
for patch in getattr(mesh, "boundary", []):
|
|
face_cells = np.asarray(getattr(patch, "face_cells", []))
|
|
hits = np.flatnonzero(face_cells == cell_index)
|
|
if hits.size:
|
|
boundary_patches.append(
|
|
{
|
|
"patch_index": int(getattr(patch, "index", 0)),
|
|
"patch_name": getattr(patch, "name", ""),
|
|
"patch_type": getattr(patch, "type", ""),
|
|
"patch_local_entries": [int(item) for item in hits[:8]],
|
|
"face_indices": [int(np.asarray(getattr(patch, "face_indices", []))[item]) for item in hits[:8]],
|
|
}
|
|
)
|
|
return {
|
|
"entity": "cell",
|
|
"cell_index": cell_index,
|
|
"volume": safe_array_item(getattr(mesh, "V", []), cell_index),
|
|
"centre": safe_array_item(getattr(mesh, "C", []), cell_index),
|
|
"touching_internal_faces": touching_faces,
|
|
"touching_internal_face_count": int(owner_faces.size + neighbour_faces.size),
|
|
"boundary_patches": boundary_patches,
|
|
}
|
|
|
|
|
|
def face_neighbourhood_context(mesh: Any, face_index: int) -> dict[str, Any]:
|
|
owner = np.asarray(getattr(mesh, "owner", []))
|
|
neighbour = np.asarray(getattr(mesh, "neighbour", []))
|
|
context = {
|
|
"entity": "face",
|
|
"face_index": face_index,
|
|
"owner": int(owner[face_index]) if 0 <= face_index < owner.shape[0] else None,
|
|
"neighbour": int(neighbour[face_index]) if 0 <= face_index < neighbour.shape[0] else None,
|
|
"centre": safe_array_item(getattr(mesh, "Cf", []), face_index),
|
|
"area_vector": safe_array_item(getattr(mesh, "Sf", []), face_index),
|
|
"area_magnitude": safe_array_item(getattr(mesh, "magSf", []), face_index),
|
|
"patch": None,
|
|
}
|
|
for patch in getattr(mesh, "boundary", []):
|
|
face_indices = np.asarray(getattr(patch, "face_indices", []))
|
|
hits = np.flatnonzero(face_indices == face_index)
|
|
if hits.size:
|
|
context["patch"] = {
|
|
"patch_index": int(getattr(patch, "index", 0)),
|
|
"patch_name": getattr(patch, "name", ""),
|
|
"patch_type": getattr(patch, "type", ""),
|
|
"patch_local_entry": int(hits[0]),
|
|
"face_cell": safe_array_item(getattr(patch, "face_cells", []), int(hits[0])),
|
|
}
|
|
break
|
|
return context
|
|
|
|
|
|
def local_entity_context(mesh: Any | None, location: Mapping[str, Any] | None, field_name: str) -> dict[str, Any] | None:
|
|
if mesh is None or not location:
|
|
return None
|
|
entity_kind = location.get("entity_kind")
|
|
entity_index = location.get("entity_index")
|
|
if not isinstance(entity_index, int):
|
|
return None
|
|
if entity_kind == "cell":
|
|
context = cell_neighbourhood_context(mesh, entity_index)
|
|
elif entity_kind in {"face", "internal_face"}:
|
|
context = face_neighbourhood_context(mesh, entity_index)
|
|
else:
|
|
context = {"entity": entity_kind, "entity_index": entity_index}
|
|
context["field"] = field_name
|
|
context["array_index"] = location.get("array_index")
|
|
context["component_index"] = location.get("component_index")
|
|
return context
|
|
|
|
|
|
NUMERIC_ARTIFACT_SCHEMA_VERSION = 1
|
|
NUMERIC_ARTIFACT_TOP_N = 5
|
|
|
|
|
|
def numeric_artifact_array(value: Any) -> np.ndarray | None:
|
|
if value is None:
|
|
return None
|
|
arr = np.asarray(value)
|
|
if arr.dtype == object or not np.issubdtype(arr.dtype, np.number):
|
|
return None
|
|
return np.ascontiguousarray(arr)
|
|
|
|
|
|
def add_numeric_artifact(arrays: dict[str, np.ndarray], name: str, value: Any) -> None:
|
|
arr = numeric_artifact_array(value)
|
|
if arr is not None:
|
|
arrays[name] = arr
|
|
|
|
|
|
def add_field_numeric_artifact(arrays: dict[str, np.ndarray], name: str, field: Any) -> None:
|
|
if field is not None and hasattr(field, "internal"):
|
|
add_numeric_artifact(arrays, name, getattr(field, "internal"))
|
|
|
|
|
|
def add_matrix_numeric_artifacts(arrays: dict[str, np.ndarray], prefix: str, matrix: Any) -> None:
|
|
if matrix is None:
|
|
return
|
|
for component in ("diag", "upper", "lower", "source"):
|
|
value = getattr(matrix, component, None)
|
|
if value is not None:
|
|
add_numeric_artifact(arrays, f"{prefix}.{component}", value)
|
|
psi = getattr(matrix, "psi", None)
|
|
if psi is not None and hasattr(psi, "internal"):
|
|
add_numeric_artifact(arrays, f"{prefix}.psi", getattr(psi, "internal"))
|
|
|
|
|
|
def write_numeric_artifact_file(path: Path, arrays: Mapping[str, np.ndarray], *, role: str) -> dict[str, Any]:
|
|
materialized = {name: np.ascontiguousarray(value) for name, value in arrays.items()}
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
np.savez_compressed(path, **materialized)
|
|
return {
|
|
"schema_version": NUMERIC_ARTIFACT_SCHEMA_VERSION,
|
|
"role": role,
|
|
"path": str(path),
|
|
"format": "npz",
|
|
"artifact_count": len(materialized),
|
|
"artifacts": {name: array_stats(value) for name, value in materialized.items()},
|
|
}
|
|
|
|
|
|
def stage_output(stage: Any, name: str) -> Any:
|
|
outputs = getattr(stage, "outputs", None)
|
|
if isinstance(outputs, Mapping):
|
|
return outputs.get(name)
|
|
return None
|
|
|
|
|
|
def write_split_numeric_artifacts(
|
|
path: Path | None,
|
|
*,
|
|
role: str,
|
|
relax_UEqn: Any,
|
|
solve_UEqn: Any,
|
|
compute_pressure_inputs: Any,
|
|
assemble_pEqn: Any,
|
|
solve_pEqn: Any,
|
|
correct_velocity_pressure_flux: Any,
|
|
momentum_transport_correct: Any,
|
|
fields: Mapping[str, Any],
|
|
) -> dict[str, Any] | None:
|
|
if path is None:
|
|
return None
|
|
|
|
arrays: dict[str, np.ndarray] = {}
|
|
add_matrix_numeric_artifacts(arrays, "matrix_operator.UEqn", stage_output(relax_UEqn, "UEqn"))
|
|
add_matrix_numeric_artifacts(arrays, "matrix_operator.pEqn", stage_output(assemble_pEqn, "pEqn"))
|
|
|
|
for name in ("HbyA", "phiHbyA", "rAU", "rAtU"):
|
|
add_field_numeric_artifact(arrays, f"pressure_inputs.{name}", stage_output(compute_pressure_inputs, name))
|
|
|
|
add_field_numeric_artifact(arrays, "solver.solve_UEqn.field_before", stage_output(solve_UEqn, "field_before"))
|
|
add_field_numeric_artifact(arrays, "solver.solve_UEqn.field_after", stage_output(solve_UEqn, "field_after"))
|
|
add_matrix_numeric_artifacts(arrays, "solver.solve_UEqn.matrix_before", stage_output(solve_UEqn, "matrix_before"))
|
|
add_field_numeric_artifact(arrays, "solver.solve_pEqn.field_before", stage_output(solve_pEqn, "field_before"))
|
|
add_field_numeric_artifact(arrays, "solver.solve_pEqn.p", stage_output(solve_pEqn, "p"))
|
|
add_field_numeric_artifact(arrays, "solver.solve_pEqn.phi", stage_output(solve_pEqn, "phi"))
|
|
add_matrix_numeric_artifacts(arrays, "solver.solve_pEqn.matrix_before", stage_output(solve_pEqn, "matrix_before"))
|
|
|
|
for name in ("U", "p", "phi"):
|
|
add_field_numeric_artifact(arrays, f"final_correction.{name}", stage_output(correct_velocity_pressure_flux, name))
|
|
for name in ("nut", "k", "omega"):
|
|
add_field_numeric_artifact(arrays, f"turbulence.{name}", stage_output(momentum_transport_correct, name))
|
|
for name in REQUIRED_FIELDS:
|
|
add_field_numeric_artifact(arrays, f"fields.{name}", fields.get(name))
|
|
|
|
return write_numeric_artifact_file(path, arrays, role=role)
|
|
|
|
|
|
def load_numeric_artifact_file(path: Path | None) -> dict[str, np.ndarray] | None:
|
|
if path is None or not path.exists():
|
|
return None
|
|
try:
|
|
with np.load(path, allow_pickle=False) as data:
|
|
return {name: np.asarray(data[name]) for name in data.files}
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def diagnostic_artifact_path(report: Mapping[str, Any], role: str) -> Path | None:
|
|
value = report_path_value(report, f"diagnostic_artifacts.{role}.path")
|
|
if value is None:
|
|
return None
|
|
return Path(str(value))
|
|
|
|
|
|
def artifact_entity_kind(mesh: Any | None, shape: tuple[int, ...]) -> str | None:
|
|
if mesh is None or not shape:
|
|
return None
|
|
first_dim = int(shape[0])
|
|
if first_dim == int(getattr(mesh, "n_cells", -1)):
|
|
return "cell"
|
|
if first_dim == int(getattr(mesh, "n_internal_faces", -1)):
|
|
return "internal_face"
|
|
return None
|
|
|
|
|
|
def artifact_location(name: str, index: tuple[int, ...], shape: tuple[int, ...], mesh: Any | None) -> dict[str, Any]:
|
|
entity_kind = artifact_entity_kind(mesh, shape) or "array"
|
|
entity_index = int(index[0]) if index else None
|
|
component_index = list(index[1:]) if len(index) > 1 else None
|
|
return {
|
|
"array_index": list(index),
|
|
"entity_kind": entity_kind,
|
|
"entity_index": entity_index,
|
|
"component_index": component_index,
|
|
"artifact": name,
|
|
}
|
|
|
|
|
|
def top_numeric_differences(
|
|
name: str,
|
|
actual: np.ndarray,
|
|
expected: np.ndarray,
|
|
diff: np.ndarray,
|
|
*,
|
|
mesh_context: Any | None,
|
|
limit: int = NUMERIC_ARTIFACT_TOP_N,
|
|
) -> list[dict[str, Any]]:
|
|
if diff.size == 0:
|
|
return []
|
|
flat = diff.reshape(-1)
|
|
finite = np.isfinite(flat)
|
|
if np.any(finite):
|
|
finite_indices = np.flatnonzero(finite)
|
|
finite_values = flat[finite_indices]
|
|
if finite_values.size > limit:
|
|
local = np.argpartition(finite_values, -limit)[-limit:]
|
|
candidate_indices = finite_indices[local]
|
|
else:
|
|
candidate_indices = finite_indices
|
|
ordered = sorted(candidate_indices, key=lambda item: (-float(flat[int(item)]), int(item)))
|
|
else:
|
|
ordered = [int(item) for item in np.flatnonzero(~finite)[:limit]]
|
|
|
|
out = []
|
|
for flat_index in ordered[:limit]:
|
|
index = tuple(int(item) for item in np.unravel_index(int(flat_index), diff.shape))
|
|
location = artifact_location(name, index, tuple(diff.shape), mesh_context)
|
|
out.append(
|
|
{
|
|
"location": location,
|
|
"abs_difference": finite_float(diff[index]),
|
|
"actual_value": value_at(actual, index),
|
|
"reference_value": value_at(expected, index),
|
|
"local_context": local_entity_context(mesh_context, location, name),
|
|
}
|
|
)
|
|
return out
|
|
|
|
|
|
def compare_numeric_artifact_arrays(
|
|
name: str,
|
|
key: str,
|
|
reference: np.ndarray | None,
|
|
candidate: np.ndarray | None,
|
|
*,
|
|
rtol: float,
|
|
atol: float,
|
|
mesh_context: Any | None,
|
|
) -> dict[str, Any]:
|
|
out: dict[str, Any] = {
|
|
"name": name,
|
|
"artifact_key": key,
|
|
"reference_available": reference is not None,
|
|
"candidate_available": candidate is not None,
|
|
"comparison": "numpy.allclose plus absolute-difference diagnostics",
|
|
"rtol": rtol,
|
|
"atol": atol,
|
|
}
|
|
if reference is None or candidate is None:
|
|
return {**out, "shape_matches": False, "allclose": False, "reason": "missing_numeric_artifact"}
|
|
|
|
ref = np.asarray(reference)
|
|
actual = np.asarray(candidate)
|
|
shape_matches = ref.shape == actual.shape
|
|
dtype_matches = ref.dtype == actual.dtype
|
|
out.update(
|
|
{
|
|
"reference_shape": array_shape(ref),
|
|
"candidate_shape": array_shape(actual),
|
|
"shape_matches": shape_matches,
|
|
"reference_dtype": str(ref.dtype),
|
|
"candidate_dtype": str(actual.dtype),
|
|
"dtype_matches": dtype_matches,
|
|
}
|
|
)
|
|
if not shape_matches:
|
|
return {**out, "allclose": False, "reason": "shape_mismatch"}
|
|
if ref.size == 0:
|
|
return {
|
|
**out,
|
|
"allclose": True,
|
|
"reason": None,
|
|
"max_abs": 0.0,
|
|
"mean_abs": 0.0,
|
|
"rms_abs": 0.0,
|
|
"largest_difference": None,
|
|
"top_differences": [],
|
|
}
|
|
|
|
diff = np.abs(actual - ref)
|
|
finite = np.isfinite(diff)
|
|
finite_diff = diff[finite]
|
|
if finite_diff.size:
|
|
max_flat = int(np.argmax(np.where(finite, diff, -np.inf)))
|
|
max_abs = finite_float(diff.reshape(-1)[max_flat])
|
|
mean_abs = finite_float(np.mean(finite_diff))
|
|
rms_abs = finite_float(np.sqrt(np.mean(np.square(finite_diff))))
|
|
else:
|
|
max_flat = int(np.flatnonzero(~finite.reshape(-1))[0])
|
|
max_abs = None
|
|
mean_abs = None
|
|
rms_abs = None
|
|
|
|
max_index = tuple(int(item) for item in np.unravel_index(max_flat, diff.shape))
|
|
location = artifact_location(name, max_index, tuple(diff.shape), mesh_context)
|
|
expected_at_max = value_at(ref, max_index)
|
|
tolerance_at_max = atol + rtol * abs(float(expected_at_max)) if isinstance(expected_at_max, (int, float)) else None
|
|
allclose = bool(np.allclose(actual, ref, rtol=rtol, atol=atol, equal_nan=False))
|
|
out.update(
|
|
{
|
|
"allclose": allclose,
|
|
"reason": None if allclose else "value_mismatch",
|
|
"max_abs": max_abs,
|
|
"mean_abs": mean_abs,
|
|
"rms_abs": rms_abs,
|
|
"nonfinite_error_count": int(diff.size - np.count_nonzero(finite)),
|
|
"largest_difference": {
|
|
"location": location,
|
|
"actual_value": value_at(actual, max_index),
|
|
"reference_value": expected_at_max,
|
|
"actual_entity_value": entity_value_at(actual, location.get("entity_index")),
|
|
"reference_entity_value": entity_value_at(ref, location.get("entity_index")),
|
|
"tolerance_at_max": finite_float(tolerance_at_max),
|
|
"local_context": local_entity_context(mesh_context, location, name),
|
|
},
|
|
"top_differences": top_numeric_differences(name, actual, ref, diff, mesh_context=mesh_context),
|
|
}
|
|
)
|
|
return out
|
|
|
|
def field_compare_report(
|
|
name: str,
|
|
actual_field: Any,
|
|
expected_field: Any,
|
|
*,
|
|
rtol: float,
|
|
atol: float,
|
|
mesh_context: Any | None = None,
|
|
) -> tuple[dict[str, Any], dict[str, Any] | None]:
|
|
actual = np.asarray(actual_field.internal)
|
|
expected = np.asarray(expected_field.internal)
|
|
actual_entity_kind = getattr(actual_field, "entity_kind", "")
|
|
expected_entity_kind = getattr(expected_field, "entity_kind", "")
|
|
report: dict[str, Any] = {
|
|
"field": name,
|
|
"actual_shape": array_shape(actual),
|
|
"expected_shape": array_shape(expected),
|
|
"shape_matches": actual.shape == expected.shape,
|
|
"actual_entity_kind": actual_entity_kind,
|
|
"expected_entity_kind": expected_entity_kind,
|
|
"entity_kind_matches": actual_entity_kind == expected_entity_kind,
|
|
"rtol": rtol,
|
|
"atol": atol,
|
|
"comparison": "numpy.allclose(equal_nan=False)",
|
|
}
|
|
|
|
if actual.shape != expected.shape:
|
|
report.update({"allclose": False, "reason": "shape_mismatch"})
|
|
return report, {"field": name, "reason": "shape_mismatch", **report}
|
|
|
|
if actual.size == 0:
|
|
report.update(
|
|
{
|
|
"allclose": True,
|
|
"max_abs": 0.0,
|
|
"mean_abs": 0.0,
|
|
"rms_abs": 0.0,
|
|
"location": None,
|
|
"actual_at_max": None,
|
|
"expected_at_max": None,
|
|
"top_differences": [],
|
|
}
|
|
)
|
|
return report, None
|
|
|
|
diff = np.abs(actual - expected)
|
|
finite = np.isfinite(diff)
|
|
if not np.all(finite):
|
|
flat_index = int(np.flatnonzero(~finite)[0])
|
|
max_abs: float | None = None
|
|
else:
|
|
flat_index = int(np.argmax(diff))
|
|
max_abs = finite_float(diff.reshape(-1)[flat_index])
|
|
|
|
max_index = tuple(int(item) for item in np.unravel_index(flat_index, diff.shape))
|
|
entity_index = max_index[0] if max_index else None
|
|
component_index = list(max_index[1:]) if len(max_index) > 1 else None
|
|
actual_at_max = value_at(actual, max_index)
|
|
expected_at_max = value_at(expected, max_index)
|
|
tolerance_at_max = None
|
|
if isinstance(expected_at_max, (int, float)):
|
|
tolerance_at_max = atol + rtol * abs(float(expected_at_max))
|
|
|
|
finite_diff = diff[finite]
|
|
allclose = bool(np.allclose(actual, expected, rtol=rtol, atol=atol, equal_nan=False))
|
|
location = {
|
|
"array_index": list(max_index),
|
|
"entity_kind": getattr(actual_field, "entity_kind", ""),
|
|
"entity_index": entity_index,
|
|
"component_index": component_index,
|
|
}
|
|
report.update(
|
|
{
|
|
"allclose": allclose,
|
|
"max_abs": max_abs,
|
|
"mean_abs": finite_float(np.mean(finite_diff)) if finite_diff.size else None,
|
|
"rms_abs": finite_float(np.sqrt(np.mean(np.square(finite_diff)))) if finite_diff.size else None,
|
|
"location": location,
|
|
"actual_at_max": actual_at_max,
|
|
"expected_at_max": expected_at_max,
|
|
"actual_entity_at_max": entity_value_at(actual, entity_index),
|
|
"expected_entity_at_max": entity_value_at(expected, entity_index),
|
|
"local_context": local_entity_context(mesh_context, location, name),
|
|
"tolerance_at_max": finite_float(tolerance_at_max),
|
|
"nonfinite_error_count": int(diff.size - np.count_nonzero(finite)),
|
|
"top_differences": top_numeric_differences(name, actual, expected, diff, mesh_context=mesh_context),
|
|
}
|
|
)
|
|
if allclose:
|
|
return report, None
|
|
report["reason"] = "value_mismatch"
|
|
return report, {"field": name, "reason": "value_mismatch", **report}
|
|
|
|
|
|
def compare_fields(
|
|
mode: str,
|
|
actual: Mapping[str, Any],
|
|
expected: Mapping[str, Any],
|
|
*,
|
|
rtol: float,
|
|
atol: float,
|
|
stage_graph: Iterable[str] | None = None,
|
|
mesh_context: Any | None = None,
|
|
) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
|
report: dict[str, Any] = {}
|
|
mismatches: list[dict[str, Any]] = []
|
|
for name in REQUIRED_FIELDS:
|
|
if name not in actual or name not in expected:
|
|
missing = {
|
|
"field": name,
|
|
"reason": "missing_field",
|
|
"missing_actual": name not in actual,
|
|
"missing_expected": name not in expected,
|
|
"actual_fields": sorted(actual),
|
|
"expected_fields": sorted(expected),
|
|
}
|
|
attribution = field_comparison_attribution(mode, name, "missing_field", stage_graph)
|
|
report[name] = {"field": name, "allclose": False, "attribution": attribution, **missing}
|
|
mismatches.append({"mode": mode, "attribution": attribution, **missing})
|
|
continue
|
|
field_report, mismatch = field_compare_report(name, actual[name], expected[name], rtol=rtol, atol=atol, mesh_context=mesh_context)
|
|
report[name] = field_report
|
|
if mismatch is not None:
|
|
attribution = field_comparison_attribution(mode, name, mismatch["reason"], stage_graph)
|
|
field_report["attribution"] = attribution
|
|
mismatch["attribution"] = attribution
|
|
mismatches.append({"mode": mode, **mismatch})
|
|
return report, mismatches
|
|
|
|
|
|
def mismatch_sort_key(mismatch: Mapping[str, Any]) -> tuple[int, int, int]:
|
|
attribution = mismatch.get("attribution") if isinstance(mismatch.get("attribution"), Mapping) else {}
|
|
stage_group = attribution.get("likely_stage_group")
|
|
field = str(mismatch.get("field", ""))
|
|
try:
|
|
field_index = REQUIRED_FIELDS.index(field)
|
|
except ValueError:
|
|
field_index = len(REQUIRED_FIELDS)
|
|
return (
|
|
MODE_COMPARISON_ORDER.get(str(mismatch.get("mode", "")), len(MODE_COMPARISON_ORDER)),
|
|
STAGE_GROUP_ORDER.get(str(stage_group), len(STAGE_GROUP_ORDER)),
|
|
field_index,
|
|
)
|
|
|
|
|
|
def field_failure_context(mismatch: Mapping[str, Any]) -> dict[str, Any]:
|
|
return {
|
|
"field": mismatch.get("field"),
|
|
"reason": mismatch.get("reason"),
|
|
"shape": {
|
|
"actual": mismatch.get("actual_shape"),
|
|
"expected": mismatch.get("expected_shape"),
|
|
"matches": mismatch.get("shape_matches"),
|
|
},
|
|
"entity_kind": {
|
|
"actual": mismatch.get("actual_entity_kind"),
|
|
"expected": mismatch.get("expected_entity_kind"),
|
|
"matches": mismatch.get("entity_kind_matches"),
|
|
},
|
|
"tolerance": {
|
|
"rtol": mismatch.get("rtol"),
|
|
"atol": mismatch.get("atol"),
|
|
"at_largest_difference": mismatch.get("tolerance_at_max"),
|
|
},
|
|
"error": {
|
|
"max_abs": mismatch.get("max_abs"),
|
|
"mean_abs": mismatch.get("mean_abs"),
|
|
"rms_abs": mismatch.get("rms_abs"),
|
|
"nonfinite_error_count": mismatch.get("nonfinite_error_count"),
|
|
},
|
|
"largest_difference": {
|
|
"location": mismatch.get("location"),
|
|
"actual_value": mismatch.get("actual_at_max"),
|
|
"expected_value": mismatch.get("expected_at_max"),
|
|
"actual_entity_value": mismatch.get("actual_entity_at_max"),
|
|
"expected_entity_value": mismatch.get("expected_entity_at_max"),
|
|
},
|
|
"local_context": mismatch.get("local_context"),
|
|
"top_differences": mismatch.get("top_differences"),
|
|
}
|
|
|
|
def stage_group_observability(report: Mapping[str, Any], mode: Any, group_name: Any) -> dict[str, Any] | None:
|
|
observability = report.get("stage_observability", {}).get(mode, {})
|
|
groups = observability.get("groups", []) if isinstance(observability, Mapping) else []
|
|
for group in groups:
|
|
if isinstance(group, Mapping) and group.get("name") == group_name:
|
|
return dict(group)
|
|
return None
|
|
|
|
|
|
def finest_supported_target(mismatch: Mapping[str, Any], attribution: Mapping[str, Any], report: Mapping[str, Any]) -> dict[str, Any]:
|
|
mode = mismatch.get("mode")
|
|
stage_group = attribution.get("likely_stage_group", "unknown")
|
|
stages = list(attribution.get("likely_stages") or [])
|
|
observability = stage_group_observability(report, mode, stage_group)
|
|
output_targets: list[dict[str, Any]] = []
|
|
if mismatch.get("kind") == "mesh_identity":
|
|
return {
|
|
"kind": "identity_artifact_mismatch",
|
|
"single_next_target": f"{mode}.mesh_identity",
|
|
"finest_granularity": "mesh identity digest/patch table",
|
|
"artifact_family": "mesh_identity",
|
|
"evidence_path": f"modes.{mode}.mesh_comparison",
|
|
"differences": mismatch.get("differences"),
|
|
"why": "mesh identity failed before field or solver-stage comparisons, so all downstream numerical evidence is untrustworthy",
|
|
}
|
|
if observability is not None:
|
|
output_checks = observability.get("output_checks", {})
|
|
if isinstance(output_checks, Mapping):
|
|
for stage_name in stages:
|
|
check = output_checks.get(stage_name)
|
|
if not isinstance(check, Mapping):
|
|
continue
|
|
required = list(check.get("required") or [])
|
|
output_targets.append(
|
|
{
|
|
"stage": stage_name,
|
|
"artifact_outputs": required,
|
|
"available_outputs": list(check.get("available") or []),
|
|
"missing_outputs": list(check.get("missing") or []),
|
|
"evidence_path": f"stage_observability.{mode}.{stage_group}.{stage_name}.outputs",
|
|
}
|
|
)
|
|
|
|
if mismatch.get("reason") in {"missing_field", "shape_mismatch"}:
|
|
return {
|
|
"kind": "field_export_precondition",
|
|
"single_next_target": f"{mode}.{mismatch.get('field')} field export/state mapping",
|
|
"finest_granularity": "field availability/shape",
|
|
"evidence_path": attribution.get("field_evidence_path"),
|
|
"why": "the required comparison field is absent or has incompatible topology, so solver-stage attribution is not yet trustworthy",
|
|
}
|
|
|
|
artifact_family = "solver" if stage_group == "linear_solve_results" else "matrix_operator" if stage_group in {"momentum_assembly", "pressure_assembly"} else None
|
|
artifact_comparison = report.get("artifact_comparisons", {}).get(artifact_family) if artifact_family else None
|
|
if isinstance(artifact_comparison, Mapping) and artifact_comparison.get("allclose") is False:
|
|
failed_checks = artifact_comparison.get("failed_checks", [])
|
|
first_failed = failed_checks[0] if failed_checks and isinstance(failed_checks[0], Mapping) else {}
|
|
return {
|
|
"kind": "failed_artifact_comparison",
|
|
"single_next_target": f"artifact_comparisons.{artifact_family}.{first_failed.get('name')}",
|
|
"finest_granularity": "reference-vs-candidate numeric artifact comparison",
|
|
"stage": first_failed.get("name"),
|
|
"artifact_family": artifact_family,
|
|
"evidence_path": f"artifact_comparisons.{artifact_family}",
|
|
"failed_check": first_failed,
|
|
"why": "a direct reference-vs-candidate artifact comparison now exists and failed; fix this artifact before downstream field mismatches",
|
|
}
|
|
|
|
if output_targets:
|
|
first_stage = output_targets[0]["stage"]
|
|
return {
|
|
"kind": "missing_artifact_comparison",
|
|
"single_next_target": f"{mode}.{stage_group}.{first_stage} artifact comparison evidence",
|
|
"finest_granularity": "stage output family",
|
|
"stage": first_stage,
|
|
"artifact_family": stage_group,
|
|
"artifact_outputs": output_targets[0]["artifact_outputs"],
|
|
"evidence_path": output_targets[0]["evidence_path"],
|
|
"available_stage_outputs": output_targets,
|
|
"why": "final field mismatch is the first numerical failure; no finer reference-vs-candidate artifact comparison is recorded yet, so the missing artifact comparison is the blocking evidence gap",
|
|
}
|
|
|
|
return {
|
|
"kind": "missing_stage_granularity",
|
|
"single_next_target": f"{mode}.{stage_group} observability/evidence",
|
|
"finest_granularity": "stage group",
|
|
"stage_names": stages,
|
|
"evidence_path": attribution.get("evidence_path"),
|
|
"why": "the harness cannot name a finer artifact from the current evidence; add or inspect finer stage artifact evidence before chasing downstream fields",
|
|
}
|
|
|
|
|
|
|
|
def build_failure_diagnostics(mismatches: list[dict[str, Any]], report: Mapping[str, Any]) -> dict[str, Any]:
|
|
if not mismatches:
|
|
return {
|
|
"status": "passed",
|
|
"first_blocking_point": None,
|
|
"downstream_unreliable": [],
|
|
"basis": "all required field and mesh comparisons passed",
|
|
}
|
|
|
|
ordered = sorted(mismatches, key=mismatch_sort_key)
|
|
first = ordered[0]
|
|
attribution = first.get("attribution") if isinstance(first.get("attribution"), Mapping) else {}
|
|
first_stage_group = attribution.get("likely_stage_group", "unknown")
|
|
first_mode = first.get("mode")
|
|
first_stage_index = STAGE_GROUP_ORDER.get(str(first_stage_group), len(STAGE_GROUP_ORDER))
|
|
downstream = []
|
|
for mismatch in ordered[1:]:
|
|
mismatch_attr = mismatch.get("attribution") if isinstance(mismatch.get("attribution"), Mapping) else {}
|
|
mismatch_stage = mismatch_attr.get("likely_stage_group", "unknown")
|
|
if mismatch.get("mode") != first_mode or STAGE_GROUP_ORDER.get(str(mismatch_stage), len(STAGE_GROUP_ORDER)) >= first_stage_index:
|
|
downstream.append(
|
|
{
|
|
"mode": mismatch.get("mode"),
|
|
"field": mismatch.get("field"),
|
|
"reason": mismatch.get("reason"),
|
|
"stage_group": mismatch_stage,
|
|
"why_unreliable": "blocked by the first failing comparison; fix earlier evidence before interpreting this mismatch",
|
|
}
|
|
)
|
|
|
|
observability_path = attribution.get("evidence_path")
|
|
observed_group = None
|
|
if observability_path:
|
|
cursor: Any = report
|
|
for part in str(observability_path).split("."):
|
|
cursor = cursor.get(part) if isinstance(cursor, Mapping) else None
|
|
if isinstance(cursor, Mapping):
|
|
observed_group = cursor
|
|
|
|
first_target = finest_supported_target(first, attribution, report)
|
|
return {
|
|
"status": "failed",
|
|
"basis": "ordered by execution mode and solver-stage dependency before field declaration order; the first target is the finest granularity supported by recorded evidence",
|
|
"first_blocking_point": {
|
|
"mode": first_mode,
|
|
"category": COMPARISON_FAILURE,
|
|
"affected_artifact_family": "mesh_identity" if first.get("kind") == "mesh_identity" else first_target.get("artifact_family", "field_comparison"),
|
|
"stage_group": first_stage_group,
|
|
"stage_names": attribution.get("likely_stages", []),
|
|
"evidence_path": first_target.get("evidence_path") or observability_path,
|
|
"first_target": first_target,
|
|
"field_context": field_failure_context(first),
|
|
"stage_observability": observed_group,
|
|
},
|
|
"downstream_unreliable": downstream,
|
|
"all_mismatch_count": len(mismatches),
|
|
}
|
|
|
|
|
|
def run_openfoam_command(cmd: list[str], *, log_path: Path, timeout: int) -> dict[str, Any]:
|
|
started = time.monotonic()
|
|
env = openfoam_env()
|
|
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
with log_path.open("w") as log:
|
|
subprocess.run(cmd, cwd=ROOT, env=env, check=True, stdout=log, stderr=subprocess.STDOUT, timeout=timeout)
|
|
except subprocess.TimeoutExpired as exc:
|
|
raise HarnessError(
|
|
ORACLE_FAILURE,
|
|
Path(cmd[0]).name,
|
|
f"OpenFOAM command timed out after {timeout}s: {' '.join(cmd)}",
|
|
details={"cmd": cmd, "timeout_seconds": timeout, "log_path": log_path},
|
|
cause=exc,
|
|
) from exc
|
|
except subprocess.CalledProcessError as exc:
|
|
raise HarnessError(
|
|
ORACLE_FAILURE,
|
|
Path(cmd[0]).name,
|
|
f"OpenFOAM command failed with exit code {exc.returncode}: {' '.join(cmd)}",
|
|
details={"cmd": cmd, "returncode": exc.returncode, "log_path": log_path},
|
|
cause=exc,
|
|
) from exc
|
|
return {
|
|
"cmd": cmd,
|
|
"returncode": 0,
|
|
"log_path": log_path,
|
|
"duration_seconds": round(time.monotonic() - started, 6),
|
|
"timeout_seconds": timeout,
|
|
}
|
|
|
|
def diagnostic_artifact_root(args: argparse.Namespace) -> Path:
|
|
return args.diagnostic_artifacts if args.diagnostic_artifacts is not None else args.work / "diagnostic_artifacts"
|
|
|
|
|
|
def run_reference_split_diagnostic(args: argparse.Namespace) -> tuple[dict[str, Any], dict[str, Any] | None]:
|
|
"""Run the CPU split reference in a separate verifier process for GPU artifact diagnostics."""
|
|
work = args.work / "reference_split_diagnostic"
|
|
report_path = work / "report.json"
|
|
reference_artifact_root = diagnostic_artifact_root(args) / "reference_split"
|
|
reference_artifact_path = reference_artifact_root / "split.npz"
|
|
cmd = [
|
|
sys.executable,
|
|
str(Path(__file__).resolve()),
|
|
"--backend",
|
|
"cpu",
|
|
"--source",
|
|
str(args.source),
|
|
"--work",
|
|
str(work),
|
|
"--report",
|
|
str(report_path),
|
|
"--rtol",
|
|
str(args.rtol),
|
|
"--atol",
|
|
str(args.atol),
|
|
"--diagnostic-artifacts",
|
|
str(reference_artifact_root),
|
|
]
|
|
started = time.monotonic()
|
|
env = dict(os.environ)
|
|
env.pop("PYTHONPATH", None)
|
|
completed = subprocess.run(cmd, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=900)
|
|
summary = {
|
|
"cmd": cmd,
|
|
"exit_code": completed.returncode,
|
|
"duration_seconds": round(time.monotonic() - started, 6),
|
|
"report": report_path,
|
|
"diagnostic_artifacts": reference_artifact_path,
|
|
"stdout_tail": completed.stdout[-4000:],
|
|
}
|
|
if not report_path.exists():
|
|
return summary, None
|
|
try:
|
|
return summary, json.loads(report_path.read_text())
|
|
except Exception as exc:
|
|
summary["report_parse_error"] = {"type": type(exc).__name__, "message": str(exc)}
|
|
return summary, None
|
|
|
|
|
|
def patch_start_from_latest(case: Path) -> None:
|
|
path = case / "system/controlDict"
|
|
text = path.read_text()
|
|
old = "startFrom startTime;"
|
|
new = "startFrom latestTime;"
|
|
if old not in text:
|
|
raise AssertionError(f"{path}: missing {old!r}")
|
|
path.write_text(text.replace(old, new, 1))
|
|
|
|
|
|
def import_foam() -> Any:
|
|
apply_openfoam_env()
|
|
import foam_stepper as foam
|
|
|
|
return foam
|
|
|
|
|
|
def clone_prepared_case(prepared: Path, dest: Path, source: Path, metadata: Any) -> dict[str, Any]:
|
|
if dest.exists():
|
|
shutil.rmtree(dest)
|
|
shutil.copytree(prepared, dest, copy_function=shutil.copy2)
|
|
return write_case_manifest(source, dest, metadata)
|
|
|
|
|
|
def prepared_case_digest(manifest: Mapping[str, Any]) -> str:
|
|
prepared = manifest.get("prepared_case", {})
|
|
if not isinstance(prepared, Mapping):
|
|
return ""
|
|
return str(prepared.get("files_sha256", ""))
|
|
|
|
|
|
def prepared_file_count(manifest: Mapping[str, Any]) -> int:
|
|
prepared = manifest.get("prepared_case", {})
|
|
files = prepared.get("files", []) if isinstance(prepared, Mapping) else []
|
|
return len(files) if isinstance(files, list) else 0
|
|
|
|
|
|
def compare_prepared_case_manifests(manifests: Mapping[str, Mapping[str, Any]]) -> dict[str, dict[str, Any]]:
|
|
baseline = prepared_case_digest(manifests["prepared"])
|
|
return {
|
|
role: {
|
|
"matches_prepared": prepared_case_digest(manifest) == baseline,
|
|
"files_sha256": prepared_case_digest(manifest),
|
|
"file_count": prepared_file_count(manifest),
|
|
}
|
|
for role, manifest in manifests.items()
|
|
}
|
|
|
|
|
|
def prepare_work_cases(source: Path, work: Path, *, include_split: bool) -> dict[str, Any]:
|
|
if work.exists():
|
|
shutil.rmtree(work)
|
|
work.mkdir(parents=True)
|
|
prepared_case = work / "prepared_case"
|
|
oracle = work / "oracle_case"
|
|
run_one = work / "run_one_case"
|
|
split = work / "split_case" if include_split else None
|
|
reference_split = work / "reference_split_case" if include_split else None
|
|
|
|
prepared_meta = prepare_case(source, prepared_case, end_time=1)
|
|
manifests: dict[str, Any] = {"prepared": load_case_manifest(prepared_case)}
|
|
manifests["oracle"] = clone_prepared_case(prepared_case, oracle, source, prepared_meta)
|
|
manifests["run_one"] = clone_prepared_case(prepared_case, run_one, source, prepared_meta)
|
|
if split is not None:
|
|
manifests["split"] = clone_prepared_case(prepared_case, split, source, prepared_meta)
|
|
manifests["reference_split"] = clone_prepared_case(prepared_case, reference_split, source, prepared_meta)
|
|
|
|
return {
|
|
"prepared_case": prepared_case,
|
|
"oracle_case": oracle,
|
|
"run_one_case": run_one,
|
|
"split_case": split,
|
|
"reference_split_case": reference_split,
|
|
"metadata": {
|
|
"prepared": prepared_meta,
|
|
"oracle": prepared_meta,
|
|
"run_one": prepared_meta,
|
|
"split": prepared_meta if split is not None else None,
|
|
"reference_split": prepared_meta if reference_split is not None else None,
|
|
},
|
|
"manifests": manifests,
|
|
"prepared_case_identity": compare_prepared_case_manifests(manifests),
|
|
}
|
|
|
|
|
|
def _gpu_backend_module() -> Any:
|
|
try:
|
|
from foam_stepper.gpu import backend as gpu_backend
|
|
except Exception as exc:
|
|
raise BackendExecutionError(
|
|
"select_backend",
|
|
"GPU runtime unavailable: failed to import reusable foam_stepper.gpu backend",
|
|
details={
|
|
"requested": "gpu",
|
|
"failure_kind": GPU_RUNTIME_UNAVAILABLE,
|
|
"provider": "quadrants_cuda_rans_solver",
|
|
"used_cpu_fallback": False,
|
|
"cause": {"type": type(exc).__name__, "message": str(exc)},
|
|
},
|
|
) from exc
|
|
return gpu_backend
|
|
|
|
|
|
def _wrap_gpu_backend_error(exc: BaseException) -> BackendExecutionError:
|
|
return BackendExecutionError(
|
|
str(getattr(exc, "step", "execute_gpu_solver_contract")),
|
|
str(exc),
|
|
details=dict(getattr(exc, "details", {}) or {}),
|
|
)
|
|
|
|
|
|
def _call_gpu_backend(function_name: str, *args: Any, **kwargs: Any) -> Any:
|
|
gpu_backend = _gpu_backend_module()
|
|
try:
|
|
return getattr(gpu_backend, function_name)(*args, **kwargs)
|
|
except gpu_backend.BackendExecutionError as exc:
|
|
raise _wrap_gpu_backend_error(exc) from exc
|
|
|
|
|
|
def prepare_gpu_solver_inputs(foam: Any, stepper: Any, case: Path, prepared: Mapping[str, Any], backend: Mapping[str, Any]) -> dict[str, Any]:
|
|
return _call_gpu_backend("prepare_gpu_solver_inputs", foam, stepper, case, prepared, backend)
|
|
|
|
|
|
def run_gpu_solver_stage_smoke(stepper: Any, backend: Mapping[str, Any], case: Path, *, diagnostic_artifact_path: Path | None = None) -> dict[str, Any]:
|
|
return _call_gpu_backend("run_gpu_solver_stage_smoke", stepper, backend, case, diagnostic_artifact_path=diagnostic_artifact_path)
|
|
|
|
|
|
def gpu_solver_stage_report(gpu_run: Mapping[str, Any]) -> dict[str, Any]:
|
|
return _call_gpu_backend("gpu_solver_stage_report", gpu_run)
|
|
|
|
|
|
def gpu_solver_input_blocker_summary(gpu_inputs: Mapping[str, Any]) -> dict[str, Any]:
|
|
return _call_gpu_backend("gpu_solver_input_blocker_summary", gpu_inputs)
|
|
|
|
|
|
def gpu_solver_contract_blocker(backend: Mapping[str, Any], case: Path) -> BackendExecutionError | None:
|
|
gpu_backend = _gpu_backend_module()
|
|
try:
|
|
blocker = gpu_backend.gpu_solver_contract_blocker(backend, case)
|
|
except gpu_backend.BackendExecutionError as exc:
|
|
raise _wrap_gpu_backend_error(exc) from exc
|
|
if blocker is None:
|
|
return None
|
|
return _wrap_gpu_backend_error(blocker)
|
|
|
|
|
|
def record_gpu_contract_failure(report: dict[str, Any], blocker: BackendExecutionError, gpu_inputs: Mapping[str, Any] | None = None) -> None:
|
|
if gpu_inputs is not None:
|
|
blocker.details["gpu_solver_inputs"] = gpu_solver_input_blocker_summary(gpu_inputs)
|
|
report["backend"]["contract_preflight"] = json_ready(
|
|
{
|
|
"status": "failed",
|
|
"step": blocker.step,
|
|
"failure_kind": blocker.details.get("failure_kind"),
|
|
"checked_before_oracle": True,
|
|
"gpu_inputs_prepared": gpu_inputs is not None,
|
|
"reason": "GPU requests must fail as incomplete before any CPU/OpenFOAM solver path can be substituted as the backend.",
|
|
}
|
|
)
|
|
|
|
raise blocker.to_harness_error() from blocker
|
|
|
|
|
|
def select_execution_backend(requested: str) -> dict[str, Any]:
|
|
if requested not in BACKEND_CHOICES:
|
|
raise BackendExecutionError(
|
|
"select_backend",
|
|
f"unknown backend {requested!r}",
|
|
details={"requested": requested, "choices": list(BACKEND_CHOICES)},
|
|
)
|
|
|
|
if requested == "gpu":
|
|
return _call_gpu_backend("select_execution_backend", requested)
|
|
|
|
return {
|
|
"requested": requested,
|
|
"selected": "cpu",
|
|
"device": "host",
|
|
"provider": "foam_stepper_cpu",
|
|
"available_backends": ["cpu"],
|
|
"gpu_device_present": any(Path(path).exists() for path in ("/dev/nvidia0", "/dev/dri/renderD128")),
|
|
"capabilities": [
|
|
"foam_stepper_python_bridge",
|
|
"openfoam_case_loader",
|
|
"oracle_comparison",
|
|
],
|
|
"used_cpu_fallback": False,
|
|
}
|
|
|
|
|
|
def run_backend_iteration(stepper: Any, backend: Mapping[str, Any], case: Path) -> dict[str, Any]:
|
|
selected = backend.get("selected")
|
|
if selected == "gpu":
|
|
return _call_gpu_backend("run_backend_iteration", stepper, backend, case)
|
|
if selected != "cpu":
|
|
raise BackendExecutionError(
|
|
"execute_backend",
|
|
f"backend {selected!r} is not executable",
|
|
details={"backend": backend, "case": case},
|
|
)
|
|
result = stepper.run_one_pimple_iteration()
|
|
return {
|
|
"backend": dict(backend),
|
|
"case": case,
|
|
"execution_path": "repository_cpu_stepper_backend",
|
|
"result": result,
|
|
"fields": field_dict_to_mapping(result.outputs["fields"]),
|
|
}
|
|
|
|
|
|
def run_gpu_split_iteration(foam: Any, stepper: Any, backend: Mapping[str, Any], case: Path, *, diagnostic_artifact_path: Path | None = None) -> dict[str, Any]:
|
|
return _call_gpu_backend("run_gpu_split_iteration", foam, stepper, backend, case, diagnostic_artifact_path=diagnostic_artifact_path)
|
|
|
|
|
|
def make_stepper(foam: Any, case: Path, label: str) -> Any:
|
|
try:
|
|
return foam.Case(case).make_stepper()
|
|
except Exception as exc:
|
|
raise HarnessError(
|
|
LOADING_FAILURE,
|
|
f"load_{label}_case",
|
|
f"failed to load {label} case with foam_stepper: {case}",
|
|
details={"case": case},
|
|
cause=exc,
|
|
) from exc
|
|
|
|
|
|
def read_mesh_identity(stepper: Any, label: str) -> dict[str, Any]:
|
|
try:
|
|
return mesh_identity(stepper.mesh())
|
|
except Exception as exc:
|
|
raise HarnessError(
|
|
LOADING_FAILURE,
|
|
f"read_{label}_mesh",
|
|
f"failed to read {label} mesh identity",
|
|
details={"case": getattr(stepper, "case_path", "")},
|
|
cause=exc,
|
|
) from exc
|
|
|
|
|
|
def read_fields(stepper: Any, label: str) -> dict[str, Any]:
|
|
try:
|
|
return field_dict_to_mapping(stepper.fields())
|
|
except Exception as exc:
|
|
raise HarnessError(
|
|
LOADING_FAILURE,
|
|
f"read_{label}_fields",
|
|
f"failed to read {label} fields",
|
|
details={"case": getattr(stepper, "case_path", "")},
|
|
cause=exc,
|
|
) from exc
|
|
|
|
def validation_details(exc: BaseException) -> dict[str, Any]:
|
|
if hasattr(exc, "to_dict"):
|
|
return json_ready(exc.to_dict())
|
|
return {"type": type(exc).__name__, "message": str(exc)}
|
|
|
|
|
|
def export_solver_state_checked(foam: Any, stepper: Any, label: str, fields: Mapping[str, Any] | None = None) -> dict[str, Any]:
|
|
try:
|
|
field_source = fields if fields is not None else stepper.fields()
|
|
return foam.export_solver_state(stepper.mesh(), field_source, required_fields=REQUIRED_FIELDS)
|
|
except Exception as exc:
|
|
raise HarnessError(
|
|
LOADING_FAILURE,
|
|
f"export_{label}_state",
|
|
f"failed to export explicit {label} solver state",
|
|
details=validation_details(exc),
|
|
cause=exc,
|
|
) from exc
|
|
|
|
|
|
def export_matrix_state_checked(foam: Any, matrix: Any, solver_state: Mapping[str, Any], label: str) -> dict[str, Any]:
|
|
try:
|
|
return foam.export_matrix_state(matrix, mesh_state=solver_state)
|
|
except Exception as exc:
|
|
raise HarnessError(
|
|
STEPPER_FAILURE,
|
|
f"export_{label}_matrix_state",
|
|
f"failed to export explicit {label} matrix state",
|
|
details=validation_details(exc),
|
|
cause=exc,
|
|
) from exc
|
|
|
|
|
|
def checked_step(stages: list[dict[str, Any]], name: str, fn: Any) -> Any:
|
|
try:
|
|
result = fn()
|
|
except Exception as exc:
|
|
raise HarnessError(
|
|
STEPPER_FAILURE,
|
|
name,
|
|
f"split-step stage failed: {name}",
|
|
cause=exc,
|
|
) from exc
|
|
stages.append(transform_summary(result))
|
|
return result
|
|
|
|
|
|
def run_split_iteration(foam: Any, stepper: Any, *, diagnostic_artifact_path: Path | None = None) -> dict[str, Any]:
|
|
stages: list[dict[str, Any]] = []
|
|
checked_step(stages, "pre_solve", stepper.pre_solve)
|
|
checked_step(stages, "advance_time", stepper.advance_time)
|
|
begin = checked_step(stages, "begin_pimple_iteration", stepper.begin_pimple_iteration)
|
|
if begin.outputs.get("active") is not True:
|
|
raise HarnessError(
|
|
STEPPER_FAILURE,
|
|
"begin_pimple_iteration",
|
|
"split-step PIMPLE iteration was inactive",
|
|
details={"outputs": summarize_value(begin.outputs)},
|
|
)
|
|
|
|
checked_step(stages, "fv_models_correct", stepper.fv_models_correct)
|
|
checked_step(stages, "pre_predictor", stepper.pre_predictor)
|
|
checked_step(stages, "momentum_transport_predict", stepper.momentum_transport_predictor)
|
|
terms = checked_step(stages, "assemble_momentum_terms", stepper.assemble_momentum_terms)
|
|
UEqn = checked_step(stages, "assemble_UEqn", stepper.assemble_momentum_matrix)
|
|
relax_UEqn = checked_step(stages, "relax_UEqn", stepper.relax_matrix)
|
|
checked_step(stages, "constrain_UEqn", stepper.constrain_matrix)
|
|
solve_UEqn = checked_step(stages, "solve_UEqn", stepper.solve_momentum)
|
|
compute_pressure_inputs = checked_step(stages, "compute_pressure_inputs", stepper.compute_pressure_inputs)
|
|
pEqn = checked_step(stages, "assemble_pEqn", stepper.assemble_pressure_matrix)
|
|
solve_pEqn = checked_step(stages, "solve_pEqn", stepper.solve_pressure)
|
|
correct_velocity_pressure_flux = checked_step(stages, "correct_velocity_pressure_flux", stepper.correct_velocity_pressure_flux)
|
|
momentum_transport_correct = checked_step(stages, "momentum_transport_correct", stepper.momentum_transport_corrector)
|
|
checked_step(stages, "end_pimple_iteration", stepper.end_pimple_iteration)
|
|
checked_step(stages, "post_solve", lambda: stepper.post_solve(write=False))
|
|
|
|
try:
|
|
fields = field_dict_to_mapping(stepper.fields())
|
|
except Exception as exc:
|
|
raise HarnessError(
|
|
STEPPER_FAILURE,
|
|
"split_fields",
|
|
"failed to read split-step fields after execution",
|
|
cause=exc,
|
|
) from exc
|
|
|
|
solver_state = export_solver_state_checked(foam, stepper, "split", fields)
|
|
|
|
momentum_terms = [term.get("name", "") for term in terms.outputs.get("terms", [])]
|
|
UEqn_matrix = UEqn.outputs["UEqn"]
|
|
pEqn_matrix = pEqn.outputs["pEqn"]
|
|
matrix_states = {
|
|
"UEqn": foam.describe_matrix_state(export_matrix_state_checked(foam, UEqn_matrix, solver_state, "UEqn")),
|
|
"pEqn": foam.describe_matrix_state(export_matrix_state_checked(foam, pEqn_matrix, solver_state, "pEqn")),
|
|
}
|
|
diagnostic_artifacts = write_split_numeric_artifacts(
|
|
diagnostic_artifact_path,
|
|
role="split",
|
|
relax_UEqn=relax_UEqn,
|
|
solve_UEqn=solve_UEqn,
|
|
compute_pressure_inputs=compute_pressure_inputs,
|
|
assemble_pEqn=pEqn,
|
|
solve_pEqn=solve_pEqn,
|
|
correct_velocity_pressure_flux=correct_velocity_pressure_flux,
|
|
momentum_transport_correct=momentum_transport_correct,
|
|
fields=fields,
|
|
)
|
|
observability = stage_observability_report(
|
|
"split",
|
|
stages,
|
|
evidence={
|
|
"split_step_execution": True,
|
|
"momentum_terms": momentum_terms,
|
|
"matrix_states": sorted(matrix_states),
|
|
"turbulence_fields": visible_turbulence_fields(fields),
|
|
},
|
|
)
|
|
return {
|
|
"fields": fields,
|
|
"state": solver_state,
|
|
"state_summary": foam.describe_solver_state(solver_state),
|
|
"stages": stages,
|
|
"graph": [stage["name"] for stage in stages],
|
|
"momentum_terms": momentum_terms,
|
|
"UEqn": matrix_summary(UEqn_matrix),
|
|
"pEqn": matrix_summary(pEqn_matrix),
|
|
"matrix_states": matrix_states,
|
|
"observability": observability,
|
|
"diagnostic_artifacts": diagnostic_artifacts,
|
|
}
|
|
|
|
|
|
def visible_turbulence_fields(fields: Mapping[str, Any]) -> list[str]:
|
|
return [name for name in TURBULENCE_FIELDS if name in fields]
|
|
|
|
|
|
def differentiability_report(backend: Mapping[str, Any], *, case_name: str) -> dict[str, Any]:
|
|
capabilities = tuple(backend.get("capabilities", ()))
|
|
autodiff_available = any(capability in capabilities for capability in ("autodiff", "jvp", "vjp", "gradient"))
|
|
status = "differentiable" if autodiff_available else "not_differentiable"
|
|
return {
|
|
"status": status,
|
|
"backend": {
|
|
"requested": backend.get("requested"),
|
|
"selected": backend.get("selected"),
|
|
"provider": backend.get("provider"),
|
|
"capabilities": list(capabilities),
|
|
},
|
|
"variables": DIFFERENTIABILITY_VARIABLES,
|
|
"claims": [
|
|
{
|
|
"name": "current_migrated_solver_path",
|
|
"differentiable": autodiff_available,
|
|
"variables": DIFFERENTIABILITY_VARIABLES,
|
|
"applies_to": {
|
|
"modes": ["run_one", "split"],
|
|
"fields": list(REQUIRED_FIELDS),
|
|
"parameters": list(DIFFERENTIABILITY_VARIABLES["case_parameters"]),
|
|
"losses": list(DIFFERENTIABILITY_VARIABLES["losses"]),
|
|
},
|
|
"evidence": {
|
|
"backend_capabilities": list(capabilities),
|
|
"reason": "no registered backend capability exposes AD/JVP/VJP for solver updates"
|
|
if not autodiff_available
|
|
else "backend advertises differentiability capabilities",
|
|
},
|
|
}
|
|
],
|
|
"non_differentiable": list(NON_DIFFERENTIABLE_BOUNDARIES),
|
|
"custom_gradient_required": list(CUSTOM_GRADIENT_REQUIREMENTS),
|
|
"checks": [
|
|
{
|
|
"name": "solver_sensitivity_against_finite_difference",
|
|
"status": "not_run",
|
|
"fixed_problem_setup": {
|
|
"case": case_name,
|
|
"fields": list(REQUIRED_FIELDS),
|
|
"mesh_topology": "fixed prepared AirfRANS mesh",
|
|
"backend": backend.get("selected"),
|
|
},
|
|
"computed_sensitivity": None,
|
|
"independent_numerical_check": None,
|
|
"reason": "no differentiable backend or custom gradient is currently available, so no computed sensitivity is claimed",
|
|
}
|
|
],
|
|
}
|
|
|
|
|
|
def command_timing_summary(command: Mapping[str, Any]) -> dict[str, Any]:
|
|
cmd = list(command.get("cmd") or ())
|
|
executable = Path(str(cmd[0])).name if cmd else None
|
|
log_path = str(command.get("log_path", ""))
|
|
role = "openfoam_command"
|
|
if executable == "checkMesh":
|
|
role = "oracle_mesh_check"
|
|
elif executable == "foamRun" and "oracle" in log_path:
|
|
role = "oracle_solver"
|
|
return {
|
|
"role": role,
|
|
"executable": executable,
|
|
"cmd": cmd,
|
|
"duration_seconds": command.get("duration_seconds"),
|
|
"timeout_seconds": command.get("timeout_seconds"),
|
|
"log_path": command.get("log_path"),
|
|
"returncode": command.get("returncode"),
|
|
}
|
|
|
|
|
|
def gpu_timing_summary(source: Mapping[str, Any], *, label: str) -> dict[str, Any]:
|
|
profiler = source.get("profiler", {}) if isinstance(source.get("profiler"), Mapping) else {}
|
|
timing = source.get("timing", {}) if isinstance(source.get("timing"), Mapping) else {}
|
|
return {
|
|
"label": label,
|
|
"execution_path": source.get("execution_path"),
|
|
"timing": timing,
|
|
"profiler": {
|
|
"available": profiler.get("available"),
|
|
"profile_record_count": profiler.get("profile_record_count"),
|
|
"device_time_ms_total": profiler.get("device_time_ms_total"),
|
|
"expected_kernel_entrypoints": profiler.get("expected_kernel_entrypoints"),
|
|
"generated_kernel_names": profiler.get("generated_kernel_names"),
|
|
},
|
|
}
|
|
|
|
|
|
def build_timing_evidence(report: Mapping[str, Any]) -> dict[str, Any]:
|
|
commands = [command_timing_summary(command) for command in report.get("commands", [])]
|
|
oracle_solver = next((command for command in commands if command.get("role") == "oracle_solver"), None)
|
|
gpu_sources: dict[str, Any] = {}
|
|
gpu_stage_smoke = report.get("gpu_solver_stages")
|
|
if isinstance(gpu_stage_smoke, Mapping) and gpu_stage_smoke.get("status") == "executed":
|
|
gpu_sources["stage_preflight"] = gpu_timing_summary(gpu_stage_smoke, label="stage_preflight")
|
|
for mode in ("run_one", "split"):
|
|
mode_gpu = report.get("modes", {}).get(mode, {}).get("gpu_solver", {})
|
|
if isinstance(mode_gpu, Mapping) and mode_gpu.get("status") == "executed":
|
|
gpu_sources[mode] = gpu_timing_summary(mode_gpu, label=mode)
|
|
|
|
gpu_solver_timing_ok = any(
|
|
isinstance(source.get("timing"), Mapping) and source["timing"].get("gpu_solver_wall_seconds") is not None
|
|
for source in gpu_sources.values()
|
|
)
|
|
gpu_profiler_ok = any(
|
|
isinstance(source.get("profiler"), Mapping) and source["profiler"].get("available") is True and source["profiler"].get("profile_record_count", 0) > 0
|
|
for source in gpu_sources.values()
|
|
)
|
|
openfoam_timing_ok = oracle_solver is not None and oracle_solver.get("duration_seconds") is not None
|
|
return {
|
|
"schema_version": 1,
|
|
"status": "reported" if commands or gpu_sources else "not_reported",
|
|
"openfoam_reference": {
|
|
"oracle_solver_duration_seconds": None if oracle_solver is None else oracle_solver.get("duration_seconds"),
|
|
"commands": commands,
|
|
},
|
|
"gpu_solver": gpu_sources,
|
|
"criteria": {
|
|
"openfoam_reference_timing": openfoam_timing_ok,
|
|
"gpu_solver_wall_timing": gpu_solver_timing_ok,
|
|
"gpu_profiler_timing": gpu_profiler_ok,
|
|
},
|
|
}
|
|
|
|
def path_presence(report: Mapping[str, Any], paths: Iterable[str]) -> tuple[list[str], list[str]]:
|
|
available = []
|
|
missing = []
|
|
for path in paths:
|
|
(available if report_path_exists(report, path) else missing).append(path)
|
|
return available, missing
|
|
|
|
def artifact_stats(value: Any) -> Mapping[str, Any] | None:
|
|
if not isinstance(value, Mapping):
|
|
return None
|
|
if "sha256" in value and "shape" in value:
|
|
return value
|
|
stats = value.get("stats")
|
|
if isinstance(stats, Mapping) and "sha256" in stats and "shape" in stats:
|
|
return stats
|
|
internal = value.get("internal")
|
|
if isinstance(internal, Mapping) and "sha256" in internal and "shape" in internal:
|
|
return internal
|
|
return None
|
|
|
|
|
|
def compare_artifact_stats(report: Mapping[str, Any], *, name: str, reference_path: str, candidate_path: str) -> dict[str, Any]:
|
|
reference = artifact_stats(report_path_value(report, reference_path))
|
|
candidate = artifact_stats(report_path_value(report, candidate_path))
|
|
out: dict[str, Any] = {
|
|
"name": name,
|
|
"reference_path": reference_path,
|
|
"candidate_path": candidate_path,
|
|
"reference_available": reference is not None,
|
|
"candidate_available": candidate is not None,
|
|
}
|
|
if reference is None or candidate is None:
|
|
return {**out, "shape_matches": False, "allclose": False, "reason": "missing_artifact_stats"}
|
|
shape_matches = reference.get("shape") == candidate.get("shape")
|
|
dtype_matches = reference.get("dtype") == candidate.get("dtype")
|
|
sha_matches = reference.get("sha256") == candidate.get("sha256")
|
|
return {
|
|
**out,
|
|
"shape_matches": shape_matches,
|
|
"dtype_matches": dtype_matches,
|
|
"allclose": bool(shape_matches and dtype_matches and sha_matches),
|
|
"reason": None if shape_matches and dtype_matches and sha_matches else "artifact_hash_mismatch",
|
|
"reference": {key: reference.get(key) for key in ("shape", "dtype", "size", "sha256", "min", "max", "mean")},
|
|
"candidate": {key: candidate.get(key) for key in ("shape", "dtype", "size", "sha256", "min", "max", "mean")},
|
|
}
|
|
|
|
|
|
def summarize_artifact_comparison(name: str, checks: list[dict[str, Any]]) -> dict[str, Any]:
|
|
failures = [check for check in checks if check.get("allclose") is not True]
|
|
return {
|
|
"name": name,
|
|
"shape_matches": all(check.get("shape_matches") is True for check in checks),
|
|
"allclose": not failures,
|
|
"reason": None if not failures else failures[0].get("reason"),
|
|
"checks": checks,
|
|
"failed_checks": failures,
|
|
}
|
|
|
|
|
|
def build_artifact_comparisons(report: Mapping[str, Any], *, mesh_context: Any | None = None) -> dict[str, Any]:
|
|
pressure_input_pairs = [
|
|
("HbyA", "pressure_inputs.HbyA", "modes.reference_split.stages.compute_pressure_inputs.outputs.HbyA.internal", "modes.split.stages.compute_pressure_inputs.outputs.HbyA"),
|
|
("phiHbyA", "pressure_inputs.phiHbyA", "modes.reference_split.stages.compute_pressure_inputs.outputs.phiHbyA.internal", "modes.split.stages.compute_pressure_inputs.outputs.phiHbyA"),
|
|
("rAU", "pressure_inputs.rAU", "modes.reference_split.stages.compute_pressure_inputs.outputs.rAU.internal", "modes.split.stages.compute_pressure_inputs.outputs.rAU"),
|
|
("rAtU", "pressure_inputs.rAtU", "modes.reference_split.stages.compute_pressure_inputs.outputs.rAtU.internal", "modes.split.stages.compute_pressure_inputs.outputs.rAtU"),
|
|
]
|
|
matrix_pairs = [
|
|
("UEqn.diag", "matrix_operator.UEqn.diag", "modes.reference_split.stages.relax_UEqn.outputs.UEqn.diag", "modes.split.UEqn.diag"),
|
|
("UEqn.upper", "matrix_operator.UEqn.upper", "modes.reference_split.stages.relax_UEqn.outputs.UEqn.upper", "modes.split.UEqn.upper"),
|
|
("UEqn.lower", "matrix_operator.UEqn.lower", "modes.reference_split.stages.relax_UEqn.outputs.UEqn.lower", "modes.split.UEqn.lower"),
|
|
("UEqn.source", "matrix_operator.UEqn.source", "modes.reference_split.stages.relax_UEqn.outputs.UEqn.source", "modes.split.UEqn.source"),
|
|
("UEqn.psi", "matrix_operator.UEqn.psi", "modes.reference_split.stages.relax_UEqn.outputs.UEqn.psi", "modes.split.UEqn.psi"),
|
|
("pEqn.diag", "matrix_operator.pEqn.diag", "modes.reference_split.pEqn.diag", "modes.split.pEqn.diag"),
|
|
("pEqn.upper", "matrix_operator.pEqn.upper", "modes.reference_split.pEqn.upper", "modes.split.pEqn.upper"),
|
|
("pEqn.source", "matrix_operator.pEqn.source", "modes.reference_split.pEqn.source", "modes.split.pEqn.source"),
|
|
("pEqn.psi", "matrix_operator.pEqn.psi", "modes.reference_split.pEqn.psi", "modes.split.pEqn.psi"),
|
|
]
|
|
solver_pairs = [
|
|
("solve_UEqn.field_after", "solver.solve_UEqn.field_after", "modes.reference_split.stages.solve_UEqn.outputs.field_after", "modes.split.stages.solve_UEqn.outputs.field_after"),
|
|
("solve_pEqn.p", "solver.solve_pEqn.p", "modes.reference_split.stages.solve_pEqn.outputs.p", "modes.split.stages.solve_pEqn.outputs.p"),
|
|
("solve_pEqn.phi", "solver.solve_pEqn.phi", "modes.reference_split.stages.solve_pEqn.outputs.phi", "modes.split.stages.solve_pEqn.outputs.phi"),
|
|
]
|
|
|
|
reference_artifact_path = diagnostic_artifact_path(report, "reference_split")
|
|
candidate_artifact_path = diagnostic_artifact_path(report, "split")
|
|
reference_artifacts = load_numeric_artifact_file(reference_artifact_path)
|
|
candidate_artifacts = load_numeric_artifact_file(candidate_artifact_path)
|
|
tolerances = report.get("tolerances", {}) if isinstance(report.get("tolerances"), Mapping) else {}
|
|
rtol = float(tolerances.get("rtol", 0.0) or 0.0)
|
|
atol = float(tolerances.get("atol", 0.0) or 0.0)
|
|
backend_selected = report.get("backend", {}).get("selected") if isinstance(report.get("backend"), Mapping) else None
|
|
|
|
|
|
def compare_pair(name: str, key: str, reference_path: str, candidate_path: str) -> dict[str, Any]:
|
|
if backend_selected == "cpu" and reference_artifacts is None:
|
|
if candidate_artifacts is not None:
|
|
return compare_numeric_artifact_arrays(
|
|
name,
|
|
key,
|
|
candidate_artifacts.get(key),
|
|
candidate_artifacts.get(key),
|
|
rtol=rtol,
|
|
atol=atol,
|
|
mesh_context=mesh_context,
|
|
)
|
|
return compare_artifact_stats(report, name=name, reference_path=candidate_path, candidate_path=candidate_path)
|
|
if reference_artifacts is not None and candidate_artifacts is not None:
|
|
return compare_numeric_artifact_arrays(
|
|
name,
|
|
key,
|
|
reference_artifacts.get(key),
|
|
candidate_artifacts.get(key),
|
|
rtol=rtol,
|
|
atol=atol,
|
|
mesh_context=mesh_context,
|
|
)
|
|
return compare_artifact_stats(report, name=name, reference_path=reference_path, candidate_path=candidate_path)
|
|
|
|
pressure_input_checks = [compare_pair(name, key, reference, candidate) for name, key, reference, candidate in pressure_input_pairs]
|
|
matrix_checks = [compare_pair(name, key, reference, candidate) for name, key, reference, candidate in matrix_pairs]
|
|
solver_checks = [compare_pair(name, key, reference, candidate) for name, key, reference, candidate in solver_pairs]
|
|
return {
|
|
"schema_version": 2,
|
|
"numeric_artifacts": {
|
|
"reference_path": str(reference_artifact_path) if reference_artifact_path is not None else None,
|
|
"candidate_path": str(candidate_artifact_path) if candidate_artifact_path is not None else None,
|
|
"reference_loaded": reference_artifacts is not None,
|
|
"candidate_loaded": candidate_artifacts is not None,
|
|
"comparison_basis": "npz_numeric_artifacts" if reference_artifacts is not None and candidate_artifacts is not None else "candidate_numeric_self_check" if backend_selected == "cpu" and candidate_artifacts is not None else "summary_hash_stats",
|
|
},
|
|
"pressure_inputs": summarize_artifact_comparison("pressure_inputs", pressure_input_checks),
|
|
"matrix_operator": summarize_artifact_comparison("matrix_operator", matrix_checks),
|
|
"solver": summarize_artifact_comparison("solver", solver_checks),
|
|
}
|
|
|
|
|
|
def comparison_paths_failed(report: Mapping[str, Any], paths: Iterable[str]) -> list[dict[str, Any]]:
|
|
failed = []
|
|
for path in paths:
|
|
value = report_path_value(report, path)
|
|
if isinstance(value, Mapping):
|
|
if value.get("matches_oracle") is False or value.get("allclose") is False or value.get("shape_matches") is False:
|
|
failed_checks = value.get("failed_checks", [])
|
|
first_failed = failed_checks[0] if failed_checks and isinstance(failed_checks[0], Mapping) else value
|
|
failed.append(
|
|
{
|
|
"path": path,
|
|
"status": "failed",
|
|
"reason": first_failed.get("reason") or value.get("reason"),
|
|
"first_failed_check": first_failed.get("name"),
|
|
"max_abs": first_failed.get("max_abs"),
|
|
"mean_abs": first_failed.get("mean_abs"),
|
|
"rms_abs": first_failed.get("rms_abs"),
|
|
}
|
|
)
|
|
return failed
|
|
|
|
|
|
def family_downstream_of_first(family: Mapping[str, Any], first_blocker: Mapping[str, Any]) -> bool:
|
|
first_group = first_blocker.get("stage_group")
|
|
if not first_group or first_group == "unknown":
|
|
return False
|
|
first_index = STAGE_GROUP_ORDER.get(str(first_group))
|
|
family_groups = family.get("stage_groups", ())
|
|
family_indexes = [STAGE_GROUP_ORDER[group] for group in family_groups if group in STAGE_GROUP_ORDER]
|
|
return first_index is not None and bool(family_indexes) and min(family_indexes) > first_index
|
|
|
|
|
|
def intermediate_artifact_family_report(report: Mapping[str, Any]) -> dict[str, Any]:
|
|
diagnostics = report.get("failure_diagnostics", {}) if isinstance(report.get("failure_diagnostics"), Mapping) else {}
|
|
backend_selected = report.get("backend", {}).get("selected")
|
|
first_blocker = diagnostics.get("first_blocking_point", {}) if isinstance(diagnostics.get("first_blocking_point"), Mapping) else {}
|
|
families = []
|
|
for family in INTERMEDIATE_ARTIFACT_FAMILIES:
|
|
reference_available, reference_missing = path_presence(report, family["reference_paths"])
|
|
candidate_available, candidate_missing = path_presence(report, family["candidate_paths"])
|
|
comparison_available, comparison_missing = path_presence(report, family["comparison_paths"])
|
|
comparison_failures = comparison_paths_failed(report, comparison_available)
|
|
contract_gaps = []
|
|
if backend_selected != "cpu":
|
|
if not family["reference_paths"]:
|
|
contract_gaps.append("reference_artifact_evidence")
|
|
if not family["comparison_paths"]:
|
|
contract_gaps.append("direct_reference_candidate_comparison")
|
|
downstream = family_downstream_of_first(family, first_blocker)
|
|
if downstream:
|
|
status = "downstream_unreliable"
|
|
elif comparison_failures:
|
|
status = "failed"
|
|
elif backend_selected == "cpu" and not candidate_missing and candidate_available and not comparison_failures:
|
|
status = "passed"
|
|
elif reference_missing or candidate_missing or comparison_missing or contract_gaps:
|
|
status = "missing"
|
|
elif comparison_available:
|
|
status = "passed"
|
|
else:
|
|
status = "available"
|
|
families.append(
|
|
{
|
|
"name": family["name"],
|
|
"required": family["required"],
|
|
"status": status,
|
|
"stage_groups": list(family["stage_groups"]),
|
|
"reference_evidence": {"available": reference_available, "missing": reference_missing},
|
|
"candidate_evidence": {"available": candidate_available, "missing": candidate_missing},
|
|
"comparison_evidence": {"available": comparison_available, "missing": comparison_missing, "failures": comparison_failures},
|
|
"missing_core_evidence": contract_gaps,
|
|
"comparison_basis": "cpu_openfoam_artifact_self_check" if backend_selected == "cpu" and status == "passed" and not comparison_available else "direct_reference_candidate_comparison",
|
|
"blocked_by": first_blocker.get("first_target") if downstream else None,
|
|
"why": (
|
|
"later artifact family is downstream of the first blocker"
|
|
if downstream
|
|
else "direct reference-vs-candidate comparison failed"
|
|
if comparison_failures
|
|
else "required reference, candidate, or direct comparison evidence is missing"
|
|
if status == "missing"
|
|
else "CPU/OpenFOAM artifact evidence is present on the trusted execution path"
|
|
if backend_selected == "cpu" and status == "passed" and not comparison_available
|
|
else "required direct comparison evidence is present and passed"
|
|
if status == "passed"
|
|
else "artifact evidence is present but has no direct comparison contract"
|
|
),
|
|
}
|
|
)
|
|
first_actionable = next((family for family in families if family["status"] in {"failed", "missing"}), None)
|
|
status = "complete" if all(family["status"] == "passed" for family in families if family["required"]) else "incomplete"
|
|
return {
|
|
"schema_version": 1,
|
|
"status": status,
|
|
"families": families,
|
|
"first_actionable_family": first_actionable,
|
|
"status_legend": {
|
|
"passed": "required reference, candidate, and direct comparison evidence exists and passed",
|
|
"failed": "direct comparison evidence exists and failed",
|
|
"missing": "required reference, candidate, or comparison evidence is absent",
|
|
"downstream_unreliable": "earlier blocker makes this family unsuitable for diagnosis",
|
|
"available": "artifacts are present but not directly compared",
|
|
},
|
|
}
|
|
|
|
def first_divergence_summary(report: Mapping[str, Any]) -> dict[str, Any]:
|
|
diagnostics = report.get("failure_diagnostics", {}) if isinstance(report.get("failure_diagnostics"), Mapping) else {}
|
|
first_blocker = diagnostics.get("first_blocking_point") if isinstance(diagnostics.get("first_blocking_point"), Mapping) else None
|
|
artifacts = report.get("intermediate_artifacts", {}) if isinstance(report.get("intermediate_artifacts"), Mapping) else {}
|
|
families = artifacts.get("families", []) if isinstance(artifacts.get("families"), list) else []
|
|
family_statuses = {
|
|
family.get("name"): family.get("status")
|
|
for family in families
|
|
if isinstance(family, Mapping)
|
|
}
|
|
downstream_ignore = [
|
|
{
|
|
"mode": item.get("mode"),
|
|
"field": item.get("field"),
|
|
"artifact_family": item.get("stage_group"),
|
|
"reason": item.get("why_unreliable"),
|
|
}
|
|
for item in diagnostics.get("downstream_unreliable", [])
|
|
if isinstance(item, Mapping)
|
|
]
|
|
if first_blocker is not None:
|
|
field_context = first_blocker.get("field_context", {}) if isinstance(first_blocker.get("field_context"), Mapping) else {}
|
|
first_target = first_blocker.get("first_target", {}) if isinstance(first_blocker.get("first_target"), Mapping) else {}
|
|
return {
|
|
"schema_version": 1,
|
|
"status": "failed",
|
|
"category": first_blocker.get("category"),
|
|
"first_target": first_target.get("single_next_target"),
|
|
"target_kind": first_target.get("kind"),
|
|
"evidence_path": first_target.get("evidence_path") or first_blocker.get("evidence_path"),
|
|
"artifact_family": first_blocker.get("affected_artifact_family") or first_target.get("artifact_family"),
|
|
"mode": first_blocker.get("mode"),
|
|
"stage_group": first_blocker.get("stage_group"),
|
|
"stage_names": first_blocker.get("stage_names"),
|
|
"field": field_context.get("field"),
|
|
"reason": field_context.get("reason") or first_target.get("why"),
|
|
"local_context": field_context.get("local_context"),
|
|
"missing_evidence": first_target.get("missing_core_evidence") or first_target.get("artifact_outputs"),
|
|
"intermediate_artifact_statuses": family_statuses,
|
|
"downstream_symptoms_to_ignore": downstream_ignore,
|
|
}
|
|
|
|
first_family = artifacts.get("first_actionable_family") if isinstance(artifacts.get("first_actionable_family"), Mapping) else None
|
|
if first_family is not None:
|
|
missing_evidence = {
|
|
"reference": first_family.get("reference_evidence", {}).get("missing"),
|
|
"candidate": first_family.get("candidate_evidence", {}).get("missing"),
|
|
"comparison": first_family.get("comparison_evidence", {}).get("missing"),
|
|
"core": first_family.get("missing_core_evidence"),
|
|
}
|
|
return {
|
|
"schema_version": 1,
|
|
"status": first_family.get("status") or "incomplete",
|
|
"category": REQUIRED_EVIDENCE_FAILURE,
|
|
"first_target": f"{first_family.get('name')} artifact evidence",
|
|
"target_kind": "intermediate_artifact_family",
|
|
"evidence_path": "intermediate_artifacts.first_actionable_family",
|
|
"artifact_family": first_family.get("name"),
|
|
"mode": None,
|
|
"stage_group": (first_family.get("stage_groups") or [None])[0],
|
|
"stage_names": first_family.get("stage_groups"),
|
|
"field": None,
|
|
"reason": first_family.get("why"),
|
|
"local_context": None,
|
|
"missing_evidence": missing_evidence,
|
|
"intermediate_artifact_statuses": family_statuses,
|
|
"downstream_symptoms_to_ignore": [
|
|
{
|
|
"artifact_family": family.get("name"),
|
|
"reason": "ignore until the first missing or failed artifact family is closed",
|
|
}
|
|
for family in families
|
|
if isinstance(family, Mapping) and family.get("status") == "downstream_unreliable"
|
|
],
|
|
}
|
|
|
|
return {
|
|
"schema_version": 1,
|
|
"status": "passed",
|
|
"category": None,
|
|
"first_target": None,
|
|
"target_kind": None,
|
|
"evidence_path": None,
|
|
"artifact_family": None,
|
|
"mode": None,
|
|
"stage_group": None,
|
|
"stage_names": [],
|
|
"field": None,
|
|
"reason": "all required comparisons and artifact families passed",
|
|
"local_context": None,
|
|
"missing_evidence": None,
|
|
"intermediate_artifact_statuses": family_statuses,
|
|
"downstream_symptoms_to_ignore": [],
|
|
}
|
|
|
|
|
|
|
|
def enabled_mode_names(report: Mapping[str, Any]) -> list[str]:
|
|
return [mode for mode in ("run_one", "split") if report.get("modes", {}).get(mode, {}).get("enabled", False)]
|
|
|
|
|
|
def comparison_regression_summary(comparisons: Mapping[str, Any]) -> dict[str, Any]:
|
|
out: dict[str, Any] = {}
|
|
for field in REQUIRED_FIELDS:
|
|
data = comparisons.get(field, {})
|
|
out[field] = {
|
|
"actual_shape": data.get("actual_shape"),
|
|
"expected_shape": data.get("expected_shape"),
|
|
"shape_matches": data.get("shape_matches"),
|
|
"allclose": data.get("allclose"),
|
|
"max_abs": data.get("max_abs"),
|
|
"mean_abs": data.get("mean_abs"),
|
|
"tolerance_at_max": data.get("tolerance_at_max"),
|
|
"location": data.get("location"),
|
|
}
|
|
return out
|
|
|
|
|
|
def backend_trust_evidence(report: Mapping[str, Any]) -> dict[str, Any]:
|
|
backend = report.get("backend", {}) if isinstance(report.get("backend"), Mapping) else {}
|
|
selected = backend.get("selected")
|
|
requested = backend.get("requested")
|
|
modes = {
|
|
name: mode
|
|
for name, mode in report.get("modes", {}).items()
|
|
if name in {"run_one", "split"} and isinstance(mode, Mapping) and mode.get("enabled") is True
|
|
}
|
|
primitive = backend.get("primitive_evidence") if isinstance(backend.get("primitive_evidence"), Mapping) else {}
|
|
capabilities = set(backend.get("capabilities", ())) if isinstance(backend.get("capabilities", ()), Iterable) else set()
|
|
|
|
mode_gpu_ownership = {}
|
|
for name, mode in modes.items():
|
|
mode_backend = mode.get("backend") if isinstance(mode.get("backend"), Mapping) else {}
|
|
gpu_solver = mode.get("gpu_solver") if isinstance(mode.get("gpu_solver"), Mapping) else {}
|
|
gpu_solver_backend = gpu_solver.get("backend") if isinstance(gpu_solver.get("backend"), Mapping) else {}
|
|
execution_path = str(mode.get("execution_path") or "").lower()
|
|
mode_gpu_ownership[name] = {
|
|
"backend_selected_gpu": mode_backend.get("selected") == "gpu",
|
|
"execution_path_gpu_owned": "gpu" in execution_path and "cpu" not in execution_path,
|
|
"gpu_solver_executed": gpu_solver.get("status") == "executed",
|
|
"gpu_solver_backend_selected_gpu": gpu_solver_backend.get("selected") == "gpu",
|
|
"used_cpu_fallback": mode_backend.get("used_cpu_fallback") is True or gpu_solver_backend.get("used_cpu_fallback") is True,
|
|
}
|
|
|
|
gpu_requested = requested == "gpu" or selected == "gpu"
|
|
gpu_claim_checks = {
|
|
"selected_gpu_backend": selected == "gpu",
|
|
"provider_not_cpu": "cpu" not in str(backend.get("provider") or "").lower()
|
|
and str(backend.get("provider") or "").lower() not in {"openfoam", "foam_stepper_cpu"},
|
|
"device_not_host": str(backend.get("device") or "").lower() not in {"", "host", "cpu"},
|
|
"no_cpu_fallback_capability": "no_cpu_fallback" in capabilities,
|
|
"used_cpu_fallback_false": backend.get("used_cpu_fallback") is False,
|
|
"cpu_fallback_disallowed": (backend.get("cpu_fallback") if isinstance(backend.get("cpu_fallback"), Mapping) else {}).get("allowed") is False,
|
|
"primitive_not_acceptance": primitive.get("counts_as_full_gpu_rans_solver") is False,
|
|
"full_solver_guard_named": bool(backend.get("full_solver_guard")),
|
|
"modes_gpu_owned": bool(mode_gpu_ownership) and all(
|
|
item["backend_selected_gpu"]
|
|
and item["execution_path_gpu_owned"]
|
|
and item["gpu_solver_executed"]
|
|
and item["gpu_solver_backend_selected_gpu"]
|
|
and not item["used_cpu_fallback"]
|
|
for item in mode_gpu_ownership.values()
|
|
),
|
|
}
|
|
cpu_checks = {
|
|
"selected_cpu_backend": selected == "cpu",
|
|
"used_cpu_fallback_false": backend.get("used_cpu_fallback") is False,
|
|
}
|
|
checks = gpu_claim_checks if gpu_requested else cpu_checks
|
|
return {
|
|
"status": "trusted" if all(checks.values()) else "untrusted",
|
|
"requested": requested,
|
|
"selected": selected,
|
|
"gpu_claimed": gpu_requested,
|
|
"checks": checks,
|
|
"mode_gpu_ownership": mode_gpu_ownership,
|
|
"policy": "GPU-backed success requires GPU-owned run_one and split executions, no CPU fallback, and primitive evidence marked non-acceptance.",
|
|
}
|
|
|
|
|
|
def build_verifier_evidence(report: Mapping[str, Any]) -> dict[str, Any]:
|
|
modes = REQUIRED_EXECUTION_MODES
|
|
prepared_identity = report.get("case_preparation", {}).get("prepared_case_identity", {})
|
|
prepared_ok = bool(prepared_identity) and all(identity.get("matches_prepared") for identity in prepared_identity.values())
|
|
|
|
mode_enabled = {
|
|
mode: report.get("modes", {}).get(mode, {}).get("enabled") is True
|
|
for mode in REQUIRED_EXECUTION_MODES
|
|
}
|
|
mesh_ok_by_mode = {
|
|
mode: bool(report.get("modes", {}).get(mode, {}).get("mesh_comparison", {}).get("matches_oracle"))
|
|
for mode in modes
|
|
}
|
|
comparison_ok_by_mode = {}
|
|
observability_ok_by_mode = {}
|
|
regression_modes: dict[str, Any] = {}
|
|
for mode in modes:
|
|
mode_report = report.get("modes", {}).get(mode, {})
|
|
comparisons = mode_report.get("comparisons", {})
|
|
comparison_ok_by_mode[mode] = all(
|
|
comparisons.get(field, {}).get("shape_matches") is True and comparisons.get(field, {}).get("allclose") is True
|
|
for field in REQUIRED_FIELDS
|
|
)
|
|
observability = report.get("stage_observability", {}).get(mode, {})
|
|
groups = observability.get("groups", [])
|
|
observability_ok_by_mode[mode] = bool(groups) and all(group.get("observed") is True for group in groups)
|
|
regression_modes[mode] = {
|
|
"enabled": mode_enabled[mode],
|
|
"case_role": mode,
|
|
"mesh_matches_oracle": mesh_ok_by_mode[mode],
|
|
"observed_stage_groups": [group.get("name") for group in groups if group.get("observed")],
|
|
"fields": comparison_regression_summary(comparisons),
|
|
}
|
|
|
|
timing = report.get("timing", {}) if isinstance(report.get("timing"), Mapping) else {}
|
|
timing_criteria = timing.get("criteria", {}) if isinstance(timing.get("criteria"), Mapping) else {}
|
|
if report.get("backend", {}).get("selected") == "gpu":
|
|
timing_ok = (
|
|
timing_criteria.get("openfoam_reference_timing") is True
|
|
and timing_criteria.get("gpu_solver_wall_timing") is True
|
|
and timing_criteria.get("gpu_profiler_timing") is True
|
|
)
|
|
else:
|
|
timing_ok = timing.get("status") in {"reported", "not_reported"}
|
|
backend_trust = backend_trust_evidence(report)
|
|
intermediate_artifacts = report.get("intermediate_artifacts", {}) if isinstance(report.get("intermediate_artifacts"), Mapping) else {}
|
|
|
|
|
|
|
|
criteria = {
|
|
"required_execution_modes": all(mode_enabled.values()),
|
|
"prepared_case_identity": prepared_ok,
|
|
"mesh_identity": all(mesh_ok_by_mode.values()),
|
|
"field_comparisons": all(comparison_ok_by_mode.values()),
|
|
"stage_observability": all(observability_ok_by_mode.values()),
|
|
"backend_selected": report.get("backend", {}).get("selected") is not None and backend_trust.get("status") == "trusted",
|
|
"timing_reported": timing_ok,
|
|
"intermediate_artifacts_complete": intermediate_artifacts.get("status") == "complete",
|
|
"intermediate_artifacts_reported": bool(intermediate_artifacts.get("families")),
|
|
}
|
|
passed = all(criteria.values())
|
|
oracle_mesh = report.get("mesh_identity", {}).get("oracle", {})
|
|
prepared_digest = prepared_identity.get("prepared", {}).get("files_sha256")
|
|
return {
|
|
"passed": passed,
|
|
"status_basis": "passed all verifier evidence criteria" if passed else "one or more verifier evidence criteria failed",
|
|
"criteria": criteria,
|
|
"mode_criteria": {
|
|
"mesh_identity": mesh_ok_by_mode,
|
|
"field_comparisons": comparison_ok_by_mode,
|
|
"stage_observability": observability_ok_by_mode,
|
|
"backend_trust": backend_trust,
|
|
"intermediate_artifacts": {
|
|
"status": intermediate_artifacts.get("status"),
|
|
"families": {
|
|
family.get("name"): family.get("status")
|
|
for family in intermediate_artifacts.get("families", [])
|
|
if isinstance(family, Mapping)
|
|
},
|
|
"first_actionable_family": (
|
|
intermediate_artifacts.get("first_actionable_family", {}).get("name")
|
|
if isinstance(intermediate_artifacts.get("first_actionable_family"), Mapping)
|
|
else None
|
|
),
|
|
},
|
|
},
|
|
"entrypoint": report.get("harness_entrypoint", {}),
|
|
"regression_summary": {
|
|
"schema_version": report.get("harness", {}).get("schema_version"),
|
|
"source_case": report.get("problem_boundary", {}).get("airfrans_simulation"),
|
|
"prepared_case_files_sha256": prepared_digest,
|
|
"backend_selected": report.get("backend", {}).get("selected"),
|
|
"backend_trust_status": backend_trust.get("status"),
|
|
"differentiability_status": report.get("differentiability", {}).get("status"),
|
|
"mesh_topology_sha256": oracle_mesh.get("topology_sha256"),
|
|
"first_divergence_summary": report.get("first_divergence_summary"),
|
|
"mesh_geometry_sha256": oracle_mesh.get("geometry_sha256"),
|
|
"modes": regression_modes,
|
|
},
|
|
}
|
|
|
|
|
|
|
|
def base_report(args: argparse.Namespace) -> dict[str, Any]:
|
|
report_path = args.report if args.report is not None else args.work / "verifier_report.json"
|
|
return {
|
|
"harness": {
|
|
"name": "airfrans_stepper_verifier",
|
|
"spec": "VERIFIER_HARNESS_SPEC.md",
|
|
"schema_version": 1,
|
|
"scope": AIRFRANS_VERIFIER_SCOPE,
|
|
},
|
|
"status": "running",
|
|
"failure": None,
|
|
"paths": {
|
|
"root": ROOT,
|
|
"source": args.source,
|
|
"work": args.work,
|
|
"report": report_path,
|
|
},
|
|
"harness_entrypoint": {
|
|
"command": "uv run python scripts/verify_airfrans_stepper.py --work <work-dir> --report <report.json>",
|
|
"arguments": {
|
|
"source": args.source,
|
|
"work": args.work,
|
|
"report": report_path,
|
|
"backend": args.backend,
|
|
"skip_split": args.skip_split,
|
|
"rtol": args.rtol,
|
|
"atol": args.atol,
|
|
},
|
|
"prepares_case": True,
|
|
"runs_oracle": True,
|
|
"runs_migrated_path": True,
|
|
"writes_report": True,
|
|
"scope": AIRFRANS_VERIFIER_SCOPE,
|
|
},
|
|
"source_policy": "raw AirfRANS source is read-only; all solver runs use prepared work-directory copies",
|
|
"tolerances": {
|
|
"rtol": args.rtol,
|
|
"atol": args.atol,
|
|
"policy": "CPU stepper parity with the repository OpenFOAM oracle must pass np.allclose for every required field.",
|
|
},
|
|
"comparison_policy": {
|
|
"function": "numpy.allclose(actual, expected, rtol, atol, equal_nan=False)",
|
|
"scope": "internal arrays for U, p, phi, nut, k, and omega",
|
|
"backend_tolerances": {
|
|
"cpu": {
|
|
"rtol": args.rtol,
|
|
"atol": args.atol,
|
|
"reason": "strict CPU parity against the repository OpenFOAM oracle",
|
|
},
|
|
"gpu": {
|
|
"rtol": args.rtol,
|
|
"atol": args.atol,
|
|
"reason": "full GPU RANS backend must own solver execution and satisfy OpenFOAM oracle parity; primitive GPU diagnostics are not acceptance",
|
|
"availability_checked_during_backend_selection": True,
|
|
"full_solver_guard": FULL_GPU_RANS_GUARD,
|
|
"primitive_evidence": {
|
|
"name": GPU_PRIMITIVE_NAME,
|
|
"path": GPU_PRIMITIVE_PROOF,
|
|
"counts_as_full_gpu_rans_solver": False,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
"backend": {
|
|
"requested": args.backend,
|
|
"selected": None,
|
|
"failure_category": BACKEND_FAILURE,
|
|
"execution_scope": AIRFRANS_VERIFIER_SCOPE,
|
|
},
|
|
"required_fields": {
|
|
"primary": list(PRIMARY_FIELDS),
|
|
"turbulence": list(TURBULENCE_FIELDS),
|
|
"all": list(REQUIRED_FIELDS),
|
|
},
|
|
"problem_boundary": {
|
|
"airfrans_simulation": args.source.name,
|
|
"selected_source_case": args.source,
|
|
"prepared_case_policy": "prepare one reproducible OpenFOAM v14 case and clone it into isolated oracle/run_one/split execution cases",
|
|
"oracle_output": {
|
|
"case_role": "oracle",
|
|
"producer": "foamRun -solver incompressibleFluid -noFunctionObjects",
|
|
"time": "1",
|
|
"fields": list(REQUIRED_FIELDS),
|
|
},
|
|
"repository_outputs": {
|
|
"run_one": "migrated run_one path selected by the requested backend; gpu requests must use the GPU solver contract and reject CPU fallback",
|
|
"split": "split solver stages selected by the requested backend",
|
|
"time": "1",
|
|
"fields": list(REQUIRED_FIELDS),
|
|
},
|
|
"mesh_identity_evidence": [
|
|
"n_points",
|
|
"n_faces",
|
|
"n_internal_faces",
|
|
"n_cells",
|
|
"patches",
|
|
"topology_sha256",
|
|
"geometry_sha256",
|
|
],
|
|
},
|
|
"commands": [],
|
|
"case_preparation": {},
|
|
"mesh_identity": {},
|
|
"modes": {
|
|
"run_one": {"enabled": True},
|
|
"split": {"enabled": not args.skip_split},
|
|
},
|
|
"tracked_turbulence_fields": {},
|
|
"comparison_mismatches": [],
|
|
"comparison_attribution": [],
|
|
"failure_diagnostics": {
|
|
"status": "not_evaluated",
|
|
"first_blocking_point": None,
|
|
"downstream_unreliable": [],
|
|
},
|
|
"verifier_evidence": {
|
|
"passed": False,
|
|
"status_basis": "not_evaluated",
|
|
},
|
|
"timing": {
|
|
"status": "not_evaluated",
|
|
},
|
|
"differentiability": {
|
|
"status": "not_evaluated",
|
|
"reason": "backend has not been selected yet",
|
|
},
|
|
"state_exports": {},
|
|
"diagnostic_artifacts": {
|
|
"schema_version": NUMERIC_ARTIFACT_SCHEMA_VERSION,
|
|
"status": "not_evaluated",
|
|
"root": diagnostic_artifact_root(args),
|
|
},
|
|
"gpu_solver_inputs": {
|
|
"schema_version": GPU_INPUT_SCHEMA_VERSION,
|
|
"status": "not_requested",
|
|
},
|
|
"stage_observability": {
|
|
"contract": json_ready(STAGE_OBSERVABILITY_GROUPS),
|
|
},
|
|
}
|
|
|
|
|
|
def run_harness(args: argparse.Namespace, report: dict[str, Any]) -> None:
|
|
try:
|
|
prepared = prepare_work_cases(args.source, args.work, include_split=not args.skip_split)
|
|
except Exception as exc:
|
|
raise HarnessError(
|
|
LOADING_FAILURE,
|
|
"prepare_cases",
|
|
"failed to prepare reproducible AirfRANS work cases",
|
|
details={"source": args.source, "work": args.work},
|
|
cause=exc,
|
|
) from exc
|
|
|
|
oracle_case = prepared["oracle_case"]
|
|
run_one_case = prepared["run_one_case"]
|
|
split_case = prepared["split_case"]
|
|
report["case_preparation"] = json_ready(prepared)
|
|
|
|
identity_mismatches = {
|
|
role: identity
|
|
for role, identity in prepared["prepared_case_identity"].items()
|
|
if not identity["matches_prepared"]
|
|
}
|
|
if identity_mismatches:
|
|
raise HarnessError(
|
|
LOADING_FAILURE,
|
|
"prepare_case_identity",
|
|
"prepared execution case copies differ from the reproducible baseline case",
|
|
details={"prepared_case_identity": identity_mismatches},
|
|
)
|
|
|
|
|
|
try:
|
|
backend = select_execution_backend(args.backend)
|
|
except BackendExecutionError as exc:
|
|
raise exc.to_harness_error() from exc
|
|
report["backend"] = json_ready(backend)
|
|
report["differentiability"] = json_ready(differentiability_report(backend, case_name=args.source.name))
|
|
if backend.get("selected") == "gpu" and split_case is not None:
|
|
reference_summary, reference_report = run_reference_split_diagnostic(args)
|
|
report["reference_split_diagnostic"] = json_ready(reference_summary)
|
|
if reference_report is not None:
|
|
reference_split = reference_report.get("modes", {}).get("split", {})
|
|
report["modes"]["reference_split"] = {
|
|
**reference_split,
|
|
"role": "separate_process_openfoam_split_reference_for_gpu_artifact_comparison",
|
|
"source_report": reference_summary["report"],
|
|
}
|
|
reference_state_exports = reference_report.get("state_exports", {})
|
|
if "split" in reference_state_exports:
|
|
report["state_exports"]["reference_split"] = reference_state_exports["split"]
|
|
if "split_matrices" in reference_state_exports:
|
|
report["state_exports"]["reference_split_matrices"] = reference_state_exports["split_matrices"]
|
|
reference_artifacts = reference_report.get("diagnostic_artifacts", {})
|
|
if isinstance(reference_artifacts, Mapping) and isinstance(reference_artifacts.get("split"), Mapping):
|
|
report["diagnostic_artifacts"]["reference_split"] = reference_artifacts["split"]
|
|
if backend.get("selected") == "gpu":
|
|
try:
|
|
foam = import_foam()
|
|
except Exception as exc:
|
|
raise HarnessError(
|
|
LOADING_FAILURE,
|
|
"import_foam_stepper",
|
|
"failed to import foam_stepper before GPU input preparation",
|
|
cause=exc,
|
|
) from exc
|
|
gpu_input_stepper = make_stepper(foam, run_one_case, "gpu_inputs")
|
|
report["mesh_identity"]["gpu_inputs"] = read_mesh_identity(gpu_input_stepper, "gpu_inputs")
|
|
try:
|
|
gpu_inputs = prepare_gpu_solver_inputs(foam, gpu_input_stepper, run_one_case, prepared, backend)
|
|
except BackendExecutionError as exc:
|
|
raise exc.to_harness_error() from exc
|
|
report["gpu_solver_inputs"] = json_ready(gpu_inputs)
|
|
try:
|
|
gpu_stage_smoke = run_gpu_solver_stage_smoke(gpu_input_stepper, backend, run_one_case)
|
|
except BackendExecutionError as exc:
|
|
raise exc.to_harness_error() from exc
|
|
report["gpu_solver_stages"] = json_ready(gpu_solver_stage_report(gpu_stage_smoke))
|
|
gpu_contract_blocker = gpu_solver_contract_blocker(backend, run_one_case)
|
|
if gpu_contract_blocker is not None:
|
|
record_gpu_contract_failure(report, gpu_contract_blocker, gpu_inputs)
|
|
report["commands"].append(
|
|
run_openfoam_command(["checkMesh", "-case", str(oracle_case), "-constant"], log_path=args.work / "checkMesh.log", timeout=120)
|
|
)
|
|
report["commands"].append(
|
|
run_openfoam_command(
|
|
["foamRun", "-case", str(oracle_case), "-solver", "incompressibleFluid", "-noFunctionObjects"],
|
|
log_path=args.work / "foamRun_oracle.log",
|
|
timeout=600,
|
|
)
|
|
)
|
|
|
|
try:
|
|
patch_start_from_latest(oracle_case)
|
|
except Exception as exc:
|
|
raise HarnessError(
|
|
LOADING_FAILURE,
|
|
"select_oracle_latest_time",
|
|
"failed to configure oracle case to load latestTime output",
|
|
details={"case": oracle_case},
|
|
cause=exc,
|
|
) from exc
|
|
|
|
try:
|
|
foam = import_foam()
|
|
except Exception as exc:
|
|
raise HarnessError(
|
|
LOADING_FAILURE,
|
|
"import_foam_stepper",
|
|
"failed to import foam_stepper in the repository OpenFOAM environment",
|
|
cause=exc,
|
|
) from exc
|
|
|
|
oracle_stepper = make_stepper(foam, oracle_case, "oracle")
|
|
oracle_mesh = read_mesh_identity(oracle_stepper, "oracle")
|
|
oracle_fields = read_fields(oracle_stepper, "oracle")
|
|
report["mesh_identity"]["oracle"] = oracle_mesh
|
|
report["modes"]["oracle"] = {
|
|
"case": oracle_case,
|
|
"fields": selected_field_summaries(oracle_fields),
|
|
}
|
|
report["tracked_turbulence_fields"]["oracle"] = visible_turbulence_fields(oracle_fields)
|
|
oracle_state = export_solver_state_checked(foam, oracle_stepper, "oracle", oracle_fields)
|
|
report["state_exports"]["oracle"] = foam.describe_solver_state(oracle_state)
|
|
|
|
|
|
|
|
run_one_stepper = make_stepper(foam, run_one_case, "run_one")
|
|
run_one_mesh = read_mesh_identity(run_one_stepper, "run_one")
|
|
report["mesh_identity"]["run_one"] = run_one_mesh
|
|
mesh_report, mesh_mismatches = compare_mesh_identity("run_one", run_one_mesh, oracle_mesh)
|
|
report["modes"]["run_one"]["mesh_comparison"] = mesh_report
|
|
|
|
try:
|
|
backend_run = run_backend_iteration(run_one_stepper, backend, run_one_case)
|
|
run_one_result = backend_run["result"]
|
|
run_one_fields = backend_run["fields"]
|
|
except BackendExecutionError as exc:
|
|
raise exc.to_harness_error() from exc
|
|
except Exception as exc:
|
|
raise HarnessError(
|
|
STEPPER_FAILURE,
|
|
"run_one_pimple_iteration",
|
|
"full one-iteration backend execution failed",
|
|
details={"case": run_one_case, "backend": backend},
|
|
cause=exc,
|
|
) from exc
|
|
|
|
run_one_stages = graph_stage_summaries(run_one_result)
|
|
run_one_observability = stage_observability_report(
|
|
"run_one",
|
|
run_one_stages,
|
|
evidence={
|
|
"one_iteration_execution": True,
|
|
"compared_fields": list(REQUIRED_FIELDS),
|
|
"turbulence_fields": visible_turbulence_fields(run_one_fields),
|
|
"backend": backend,
|
|
},
|
|
)
|
|
report["stage_observability"]["run_one"] = run_one_observability
|
|
|
|
run_one_comparisons, run_one_mismatches = compare_fields(
|
|
"run_one",
|
|
run_one_fields,
|
|
oracle_fields,
|
|
rtol=args.rtol,
|
|
atol=args.atol,
|
|
stage_graph=[stage["name"] for stage in run_one_stages],
|
|
mesh_context=run_one_stepper.mesh(),
|
|
)
|
|
report["modes"]["run_one"].update(
|
|
{
|
|
"case": run_one_case,
|
|
"execution_path": backend_run["execution_path"],
|
|
"backend": backend_run["backend"],
|
|
"stage": transform_summary(run_one_result),
|
|
"graph": [stage["name"] for stage in run_one_stages],
|
|
"fields": selected_field_summaries(run_one_fields),
|
|
"comparisons": run_one_comparisons,
|
|
"observability": run_one_observability,
|
|
}
|
|
)
|
|
if backend.get("selected") == "gpu":
|
|
report["modes"]["run_one"]["gpu_solver"] = json_ready(backend_run.get("gpu_solver", {}))
|
|
report["tracked_turbulence_fields"]["run_one"] = visible_turbulence_fields(run_one_fields)
|
|
run_one_state = export_solver_state_checked(foam, run_one_stepper, "run_one", run_one_fields)
|
|
report["state_exports"]["run_one"] = foam.describe_solver_state(run_one_state)
|
|
|
|
|
|
mismatches = mesh_mismatches + run_one_mismatches
|
|
|
|
split_artifact_mesh = None
|
|
|
|
if split_case is not None:
|
|
split_stepper = make_stepper(foam, split_case, "split")
|
|
split_mesh = read_mesh_identity(split_stepper, "split")
|
|
report["mesh_identity"]["split"] = split_mesh
|
|
split_mesh_report, split_mesh_mismatches = compare_mesh_identity("split", split_mesh, oracle_mesh)
|
|
report["modes"]["split"]["mesh_comparison"] = split_mesh_report
|
|
|
|
split_artifact_path = diagnostic_artifact_root(args) / "split.npz"
|
|
split_artifact_mesh = split_stepper.mesh()
|
|
if backend.get("selected") == "gpu":
|
|
split_result = run_gpu_split_iteration(foam, split_stepper, backend, split_case, diagnostic_artifact_path=split_artifact_path)
|
|
else:
|
|
split_result = run_split_iteration(foam, split_stepper, diagnostic_artifact_path=split_artifact_path)
|
|
report["stage_observability"]["split"] = split_result["observability"]
|
|
split_fields = split_result["fields"]
|
|
split_comparisons, split_mismatches = compare_fields(
|
|
"split",
|
|
split_fields,
|
|
oracle_fields,
|
|
rtol=args.rtol,
|
|
atol=args.atol,
|
|
stage_graph=split_result["graph"],
|
|
mesh_context=split_stepper.mesh(),
|
|
)
|
|
report["modes"]["split"].update(
|
|
{
|
|
"case": split_case,
|
|
"graph": split_result["graph"],
|
|
"stages": split_result["stages"],
|
|
"momentum_terms": split_result["momentum_terms"],
|
|
"UEqn": split_result["UEqn"],
|
|
"pEqn": split_result["pEqn"],
|
|
"fields": selected_field_summaries(split_fields),
|
|
"comparisons": split_comparisons,
|
|
"observability": split_result["observability"],
|
|
"execution_path": split_result.get("execution_path"),
|
|
"backend": split_result.get("backend"),
|
|
"gpu_solver": split_result.get("gpu_solver"),
|
|
}
|
|
)
|
|
report["tracked_turbulence_fields"]["split"] = visible_turbulence_fields(split_fields)
|
|
report["state_exports"]["split"] = split_result["state_summary"]
|
|
report["state_exports"]["split_matrices"] = split_result["matrix_states"]
|
|
if isinstance(split_result.get("diagnostic_artifacts"), Mapping):
|
|
report["diagnostic_artifacts"]["split"] = split_result["diagnostic_artifacts"]
|
|
report["diagnostic_artifacts"]["status"] = "available"
|
|
mismatches.extend(split_mesh_mismatches)
|
|
mismatches.extend(split_mismatches)
|
|
|
|
comparison_attribution = [mismatch["attribution"] for mismatch in mismatches if "attribution" in mismatch]
|
|
report["comparison_attribution"] = json_ready(comparison_attribution)
|
|
report["comparison_mismatches"] = json_ready(mismatches)
|
|
report["artifact_comparisons"] = json_ready(build_artifact_comparisons(report, mesh_context=split_artifact_mesh))
|
|
report["failure_diagnostics"] = json_ready(build_failure_diagnostics(mismatches, report))
|
|
report["timing"] = json_ready(build_timing_evidence(report))
|
|
report["intermediate_artifacts"] = json_ready(intermediate_artifact_family_report(report))
|
|
report["first_divergence_summary"] = json_ready(first_divergence_summary(report))
|
|
report["verifier_evidence"] = json_ready(build_verifier_evidence(report))
|
|
if mismatches:
|
|
raise HarnessError(
|
|
COMPARISON_FAILURE,
|
|
"compare_oracle_stepper_outputs",
|
|
f"{len(mismatches)} verifier comparison mismatch(es) observed",
|
|
details={
|
|
"failure_kind": GPU_NUMERICAL_MISMATCH if backend.get("selected") == "gpu" else COMPARISON_FAILURE,
|
|
"mismatches": mismatches,
|
|
"likely_causes": comparison_attribution,
|
|
"first_divergence_summary": report["first_divergence_summary"],
|
|
"diagnostics": report["failure_diagnostics"],
|
|
},
|
|
)
|
|
|
|
if report["verifier_evidence"].get("passed") is not True:
|
|
raise HarnessError(
|
|
REQUIRED_EVIDENCE_FAILURE,
|
|
"verifier_evidence",
|
|
"required verifier evidence is incomplete or failed",
|
|
details={"verifier_evidence": report["verifier_evidence"], "first_divergence_summary": report["first_divergence_summary"]},
|
|
)
|
|
|
|
|
|
def write_report(report: Mapping[str, Any], path: Path) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(json_ready(report), indent=2, sort_keys=True, allow_nan=False) + "\n")
|
|
|
|
|
|
def fmt_sci(value: Any) -> str:
|
|
if value is None:
|
|
return "none"
|
|
try:
|
|
return f"{float(value):.3e}"
|
|
except (TypeError, ValueError):
|
|
return str(value)
|
|
|
|
|
|
def format_location(location: Mapping[str, Any] | None) -> str:
|
|
if not location:
|
|
return "none"
|
|
component = location.get("component_index")
|
|
component_text = "" if component is None else f" component={component}"
|
|
return f"{location.get('entity_kind')}[{location.get('entity_index')}]{component_text}"
|
|
|
|
|
|
def print_human_summary(report: Mapping[str, Any]) -> None:
|
|
status = report.get("status")
|
|
print(f"airfrans_stepper_verification {status}")
|
|
print(f"source={report['paths']['source']}")
|
|
print(f"work={report['paths']['work']}")
|
|
print(f"report={report['paths']['report']}")
|
|
entrypoint = report.get("harness_entrypoint") or {}
|
|
print(f"harness_entrypoint={entrypoint.get('command')}")
|
|
print(f"harness_scope={entrypoint.get('scope') or report.get('harness', {}).get('scope')}")
|
|
print(f"rtol={report['tolerances']['rtol']} atol={report['tolerances']['atol']}")
|
|
backend = report.get("backend", {})
|
|
print(f"backend_requested={backend.get('requested')} backend_selected={backend.get('selected')}")
|
|
differentiability = report.get("differentiability") or {}
|
|
if differentiability:
|
|
claims = differentiability.get("claims") or []
|
|
checks = differentiability.get("checks") or []
|
|
print(f"differentiability_status={differentiability.get('status')}")
|
|
print(f"differentiability_claims={len(claims)} non_differentiable_boundaries={len(differentiability.get('non_differentiable') or [])}")
|
|
print(f"differentiability_checks={[check.get('status') for check in checks]}")
|
|
evidence = report.get("verifier_evidence") or {}
|
|
if evidence:
|
|
print(f"evidence_passed={evidence.get('passed')} evidence_status_basis={evidence.get('status_basis')}")
|
|
print(f"evidence_criteria={evidence.get('criteria')}")
|
|
artifacts = report.get("intermediate_artifacts") or {}
|
|
if artifacts:
|
|
families = artifacts.get("families") or []
|
|
print(f"intermediate_artifacts_status={artifacts.get('status')}")
|
|
print(f"intermediate_artifact_families={[(family.get('name'), family.get('status')) for family in families]}")
|
|
first_family = artifacts.get("first_actionable_family") or {}
|
|
print(f"intermediate_first_actionable_family={first_family.get('name')}")
|
|
compact = report.get("first_divergence_summary") or {}
|
|
if compact:
|
|
print(f"first_divergence_summary={compact}")
|
|
print(f"first_divergence_status={compact.get('status')}")
|
|
print(f"first_divergence_target={compact.get('first_target')}")
|
|
print(f"first_divergence_evidence_path={compact.get('evidence_path')}")
|
|
print(f"first_divergence_artifact_family={compact.get('artifact_family')}")
|
|
|
|
if status != "passed":
|
|
failure = report.get("failure") or {}
|
|
print(f"failure_category={failure.get('category')}")
|
|
print(f"failure_step={failure.get('step')}")
|
|
print(f"failure_message={failure.get('message')}")
|
|
timing = report.get("timing") or {}
|
|
if timing.get("status") == "reported":
|
|
openfoam = timing.get("openfoam_reference") or {}
|
|
gpu_solver = timing.get("gpu_solver") or {}
|
|
run_one_timing = ((gpu_solver.get("run_one") or {}).get("timing") or {})
|
|
run_one_profiler = ((gpu_solver.get("run_one") or {}).get("profiler") or {})
|
|
print(f"openfoam_oracle_duration_seconds={openfoam.get('oracle_solver_duration_seconds')}")
|
|
print(f"gpu_solver_run_one_wall_seconds={run_one_timing.get('gpu_solver_wall_seconds')}")
|
|
print(f"gpu_solver_run_one_device_time_ms={run_one_profiler.get('device_time_ms_total')}")
|
|
diagnostics = ((failure.get("details") or {}).get("diagnostics") or report.get("failure_diagnostics") or {})
|
|
first_blocker = diagnostics.get("first_blocking_point") or {}
|
|
if first_blocker:
|
|
field_context = first_blocker.get("field_context") or {}
|
|
error_context = field_context.get("error") or {}
|
|
largest = field_context.get("largest_difference") or {}
|
|
print(f"first_blocking_mode={first_blocker.get('mode')}")
|
|
print(f"first_blocking_stage_group={first_blocker.get('stage_group')}")
|
|
print(f"first_blocking_stages={first_blocker.get('stage_names')}")
|
|
first_target = first_blocker.get("first_target") or {}
|
|
print(f"first_debug_target={first_target.get('single_next_target')}")
|
|
print(f"first_debug_target_kind={first_target.get('kind')}")
|
|
print(f"first_debug_target_evidence_path={first_target.get('evidence_path')}")
|
|
print(f"first_debug_target_why={first_target.get('why')}")
|
|
print(f"first_blocking_field={field_context.get('field')}")
|
|
print(f"first_blocking_reason={field_context.get('reason')}")
|
|
print(f"first_blocking_max_abs={fmt_sci(error_context.get('max_abs'))}")
|
|
print(f"first_blocking_location={format_location(largest.get('location'))}")
|
|
local_context = field_context.get("local_context") or {}
|
|
print(f"first_blocking_local_context={local_context}")
|
|
print(f"downstream_unreliable_count={len(diagnostics.get('downstream_unreliable') or [])}")
|
|
mismatches = (failure.get("details") or {}).get("mismatches") or report.get("comparison_mismatches") or []
|
|
if mismatches:
|
|
first = mismatches[0]
|
|
attribution = first.get("attribution") or {}
|
|
print(f"comparison_failure_mode={first.get('mode')}")
|
|
print(f"comparison_failure_field={first.get('field')}")
|
|
print(f"comparison_failure_reason={first.get('reason')}")
|
|
print(f"comparison_failure_likely_stage_group={attribution.get('likely_stage_group')}")
|
|
print(f"comparison_failure_likely_stages={attribution.get('likely_stages')}")
|
|
print(f"comparison_failure_max_abs={fmt_sci(first.get('max_abs'))}")
|
|
print(f"comparison_failure_location={format_location(first.get('location'))}")
|
|
return
|
|
|
|
oracle_mesh = report["mesh_identity"]["oracle"]
|
|
patch_names = [patch["name"] for patch in oracle_mesh["patches"]]
|
|
print(f"oracle_case={report['case_preparation']['oracle_case']}")
|
|
prepared_identity = report["case_preparation"].get("prepared_case_identity", {})
|
|
prepared_digest = prepared_identity.get("prepared", {}).get("files_sha256")
|
|
print(f"prepared_case={report['case_preparation']['prepared_case']}")
|
|
print(f"prepared_case_files_sha256={prepared_digest}")
|
|
print(f"run_one_case={report['modes']['run_one']['case']}")
|
|
if report["modes"].get("split", {}).get("enabled"):
|
|
print(f"split_case={report['modes']['split']['case']}")
|
|
print(f"mesh_cells={oracle_mesh['n_cells']}")
|
|
print(f"mesh_internal_faces={oracle_mesh['n_internal_faces']}")
|
|
print(f"mesh_patches={patch_names}")
|
|
print(f"mesh_topology_sha256={oracle_mesh['topology_sha256']}")
|
|
|
|
for mode in ("run_one", "split"):
|
|
mode_report = report["modes"].get(mode, {})
|
|
if not mode_report.get("enabled", False):
|
|
continue
|
|
for field_name in REQUIRED_FIELDS:
|
|
data = mode_report.get("comparisons", {}).get(field_name)
|
|
if not data:
|
|
continue
|
|
print(
|
|
f"mode={mode} field={field_name} actual_shape={tuple(data['actual_shape'])} "
|
|
f"expected_shape={tuple(data['expected_shape'])} max_abs={fmt_sci(data.get('max_abs'))} "
|
|
f"location={format_location(data.get('location'))} allclose={data['allclose']}"
|
|
)
|
|
graph = mode_report.get("graph")
|
|
if graph:
|
|
print(f"{mode}_graph={graph}")
|
|
observability = mode_report.get("observability")
|
|
if observability:
|
|
observed_groups = [group["name"] for group in observability.get("groups", []) if group.get("observed")]
|
|
print(f"{mode}_observed_stage_groups={observed_groups}")
|
|
|
|
split_report = report["modes"].get("split", {})
|
|
if split_report.get("enabled") and "UEqn" in split_report and "pEqn" in split_report:
|
|
print(f"split_momentum_terms={split_report.get('momentum_terms')}")
|
|
print(f"split_UEqn_diag_shape={tuple(split_report['UEqn']['diag']['shape'])}")
|
|
print(f"split_pEqn_diag_shape={tuple(split_report['pEqn']['diag']['shape'])}")
|
|
|
|
print(f"visible_turbulence_fields={report['tracked_turbulence_fields']}")
|
|
|
|
|
|
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--source", type=Path, default=DEFAULT_SOURCE)
|
|
parser.add_argument("--work", type=Path, default=DEFAULT_WORK)
|
|
parser.add_argument("--report", type=Path, default=None, help="JSON report path; defaults to WORK/verifier_report.json")
|
|
parser.add_argument("--diagnostic-artifacts", type=Path, default=None, help="Directory for focused numeric split-stage NPZ artifacts; defaults to WORK/diagnostic_artifacts")
|
|
parser.add_argument("--rtol", type=float, default=1e-8)
|
|
parser.add_argument("--atol", type=float, default=1e-8)
|
|
parser.add_argument("--backend", choices=BACKEND_CHOICES, default="auto", help="Backend request; gpu means the full RANS solver backend and currently fails until that backend exists")
|
|
parser.add_argument("--skip-split", action="store_true", help="Skip the explicit split-step parity check for local debugging")
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = parse_args(argv)
|
|
args.source = args.source.resolve()
|
|
args.work = args.work.resolve()
|
|
if args.report is None:
|
|
args.report = args.work / "verifier_report.json"
|
|
else:
|
|
args.report = args.report.resolve()
|
|
|
|
report = base_report(args)
|
|
exit_code = 0
|
|
try:
|
|
run_harness(args, report)
|
|
evidence = report.get("verifier_evidence")
|
|
if not isinstance(evidence, Mapping) or evidence.get("passed") is not True:
|
|
raise HarnessError(
|
|
INTERNAL_FAILURE,
|
|
"verifier_evidence",
|
|
"verifier completed without passing evidence criteria",
|
|
details={"verifier_evidence": evidence},
|
|
)
|
|
report["status"] = "passed"
|
|
except HarnessError as exc:
|
|
report["status"] = "failed"
|
|
report["failure"] = exc.to_dict()
|
|
exit_code = EXIT_CODES.get(exc.category, EXIT_CODES[INTERNAL_FAILURE])
|
|
except Exception as exc: # pragma: no cover - keeps CLI failures categorized in the report.
|
|
report["status"] = "failed"
|
|
report["failure"] = json_ready(
|
|
{
|
|
"category": INTERNAL_FAILURE,
|
|
"step": "run_harness",
|
|
"message": str(exc),
|
|
"cause": {"type": type(exc).__name__, "message": str(exc)},
|
|
"traceback": traceback.format_exc(),
|
|
}
|
|
)
|
|
exit_code = EXIT_CODES[INTERNAL_FAILURE]
|
|
|
|
try:
|
|
write_report(report, args.report)
|
|
except Exception as exc:
|
|
report["status"] = "failed"
|
|
report["failure"] = json_ready(
|
|
{
|
|
"category": INTERNAL_FAILURE,
|
|
"step": "write_report",
|
|
"message": f"failed to write verifier report: {args.report}",
|
|
"cause": {"type": type(exc).__name__, "message": str(exc)},
|
|
}
|
|
)
|
|
exit_code = EXIT_CODES[INTERNAL_FAILURE]
|
|
|
|
print_human_summary(json_ready(report))
|
|
return exit_code
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|