diff --git a/tools/myelin-e2e-fuzz/Cargo.toml b/tools/myelin-e2e-fuzz/Cargo.toml index 55276c8..3e7e0ee 100644 --- a/tools/myelin-e2e-fuzz/Cargo.toml +++ b/tools/myelin-e2e-fuzz/Cargo.toml @@ -4,6 +4,7 @@ version = "0.1.0" edition = "2024" license = "AGPL-3.0-only" publish = false +default-run = "myelin-e2e-fuzz" [dependencies] base64 = "0.22" @@ -13,6 +14,10 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.10" ureq = "2" +myelin-control-contract = { path = "../../crates/myelin-control-contract" } +provisioning = { path = "../../crates/provisioning" } +swactor-vastai = { path = "../vastai" } +tempfile = "3" [lints] workspace = true diff --git a/tools/myelin-e2e-fuzz/ordered_acceptance.py b/tools/myelin-e2e-fuzz/ordered_acceptance.py new file mode 100755 index 0000000..757284f --- /dev/null +++ b/tools/myelin-e2e-fuzz/ordered_acceptance.py @@ -0,0 +1,344 @@ +#!/usr/bin/env python3 +"""Run the complete local gates in order; retain every timing, including failures. + +Build artifacts first. Warm runs still time fixture creation, deployment, +readiness, workload, recovery, fault injection, evidence and cleanup. This +runner never acquires paid resources or removes caches to manufacture a cold run. +Use --build-images to record source/build provenance during the real Docker +builds. Without it, every cached image must already prove the same qualified +inputs and parent image identities. Container workers/wheels are ABI-specific: +qualification binds their build inputs, not equality with host artifact bytes. +""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import platform +import subprocess +import sys +import time + + +ROOT = Path(__file__).resolve().parents[2] +BINARY = ROOT / "target/release/myelin-e2e-fuzz" + + +TEST_PACKAGES = [ + "swactor", "myelin-e2e-fuzz", "swactor-vastai", "myelin-control-contract", "provisioning", + "myelin", "data-plane", "iroh-driver", "distribution", "swactor-process", + "swactor-process-context", +] +TEST_COMMAND = ["cargo", "test"] + [ + argument for package in TEST_PACKAGES for argument in ["-p", package] +] + ["--tests"] +def digest(path): + with path.open("rb") as source: + return hashlib.file_digest(source, "sha256").hexdigest() + + +def source_digest(): + paths = [ROOT / name for name in ( + "Cargo.toml", "Cargo.lock", "rust-toolchain.toml", ".dockerignore", "clippy.toml")] + excluded = {"target", ".git", ".venv", "__pycache__", "node_modules"} + for name in (".cargo", "src", "tests", "crates", "xtask", "apps/myelin", "tools/myelin-e2e-fuzz", + "tools/vastai", "tools/actor-control-flow-lint"): + for directory, directories, files in os.walk(ROOT / name, followlinks=False): + for entry in directories + files: + if (Path(directory) / entry).is_symlink(): + raise RuntimeError(f"source identity rejects symlink {directory}/{entry}") + directories[:] = [entry for entry in directories if entry not in excluded] + paths.extend(Path(directory) / entry for entry in files) + result = hashlib.sha256() + for path in sorted(paths): + result.update(os.fsencode(path.relative_to(ROOT))) + result.update(b"\0") + result.update(digest(path).encode()) + result.update(b"\0") + return result.hexdigest() + + +def persist(path, evidence): + temporary = path.with_suffix(".tmp") + with temporary.open("w") as output: + json.dump(evidence, output, separators=(",", ":")) + output.flush() + os.fsync(output.fileno()) + temporary.replace(path) + parent = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(parent) + finally: + os.close(parent) + + +def probe(command): + result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True, timeout=30) + if result.returncode: + raise RuntimeError(f"metadata command failed: {command[0]}: {result.stderr}") + return result.stdout.strip() + + +def runtime_image_identities(image): + roles = {"myelin-node-base:cuda12.6": "base", "myelin-node:latest": "node", image: "e2e"} + identities = {} + for name in sorted({"myelin-node-base:cuda12.6", "myelin-node:latest", image}): + value = json.loads(probe(["docker", "image", "inspect", name]))[0] + if "@sha256:" in name: + manifest = json.loads(probe(["docker", "manifest", "inspect", name])) + if (manifest.get("schemaVersion") != 2 or "manifests" in manifest + or manifest.get("config", {}).get("digest") != value["Id"]): + raise RuntimeError("runtime image requires a platform-specific registry manifest " + "whose config digest matches the tested local image") + labels = value.get("Config", {}).get("Labels") or {} + prefix = "org.swactor.myelin.e2e." + if labels.get(prefix + "provenance-version") != "1": + raise RuntimeError(f"runtime image {name} lacks build-time source provenance; rebuild it") + identities[name] = { + "id": value["Id"], "os": value["Os"], "architecture": value["Architecture"], + "variant": value.get("Variant", ""), + "repo_digests": sorted(value.get("RepoDigests") or []), + "provenance_version": 1, + "source_build_input_digest": labels.get(prefix + "source-build-input-digest", ""), + "image_role": labels.get(prefix + "image-role", ""), + "parent_image_id": labels.get(prefix + "parent-image-id"), + } + if identities[name]["image_role"] != roles[name]: + raise RuntimeError(f"runtime image {name} has the wrong build provenance role") + return identities + + +def verify_image_provenance(identities, image, source_build_input_digest): + if len(identities) != 3 or any( + value["source_build_input_digest"] != source_build_input_digest + for value in identities.values()): + raise RuntimeError("runtime images were not built from the qualified source/build inputs") + base = identities["myelin-node-base:cuda12.6"] + node = identities["myelin-node:latest"] + runtime = identities[image] + if (base["parent_image_id"] is not None + or node["parent_image_id"] != base["id"] + or runtime["parent_image_id"] != node["id"]): + raise RuntimeError("runtime image provenance does not match the qualified parent images") + + +def image_build_command(role, image, source_build_input_digest, parent_image_id=None): + suffix = {"base": ".base", "node": "", "e2e": ".e2e"}[role] + command = ["docker", "build", "-f", "apps/myelin/node-image/Dockerfile" + suffix, + "-t", image] + labels = { + "provenance-version": "1", + "source-build-input-digest": source_build_input_digest, + "image-role": role, + } + if parent_image_id is not None: + labels["parent-image-id"] = parent_image_id + command += ["--build-arg", "BASE_IMAGE=myelin-e2e-parent:" + parent_image_id.removeprefix("sha256:")] + for key, value in labels.items(): + command += ["--label", "org.swactor.myelin.e2e." + key + "=" + value] + return command + ["."] + + +def verify_qualified_inputs(evidence): + identity_path = Path(evidence["artifact_identity_path"]) + if json.loads(identity_path.read_text()) != evidence["deployment_artifacts"]: + raise RuntimeError("qualified artifact manifest changed during ordered gates") + if evidence["source_digest"] != source_digest() or any( + digest(ROOT / "target/release" / name) != expected + for name, expected in evidence["build_artifacts"].items()): + raise RuntimeError("tested source or binaries changed during ordered gates") + if evidence["image_identities"] != runtime_image_identities(evidence["configuration"]["image"]): + raise RuntimeError("deployment runtime images changed during ordered gates") + + +def execute(name, command, directory, evidence, evidence_path, env=None, warm_started=None): + started = time.monotonic() + env = dict(os.environ if env is None else env, + CARGO_TARGET_DIR=str(ROOT / "target")) + log = directory / f"{name}.log" + stage = {"name": name, "command": [str(part) for part in command], + "log": str(log), "state": "running", + "started_unix_ms": time.time_ns() // 1_000_000} + evidence["stages"].append(stage) + persist(evidence_path, evidence) + print(f"starting {name}: {log}", flush=True) + try: + if warm_started is not None and time.monotonic() - warm_started >= 600: + raise RuntimeError("complete warm workflow exhausted its timing ceiling; later gates not run") + if "image_identities" in evidence: + verify_qualified_inputs(evidence) + if "deployment_artifacts" in evidence: + identity_path = Path(evidence["artifact_identity_path"]) + if json.loads(identity_path.read_text()) != evidence["deployment_artifacts"]: + raise RuntimeError("qualified artifact manifest changed during ordered gates") + env["MYELIN_E2E_ARTIFACT_IDENTITY"] = str(identity_path) + with log.open("wb") as output: + result = subprocess.run(command, cwd=ROOT, env=env, stdout=output, + stderr=subprocess.STDOUT) + stage["exit_code"] = result.returncode + stage["state"] = "passed" if result.returncode == 0 else "failed" + except BaseException as error: + stage["state"] = "interrupted" + stage["error"] = str(error) + raise + finally: + stage["elapsed_secs"] = time.monotonic() - started + stage["completed_unix_ms"] = time.time_ns() // 1_000_000 + persist(evidence_path, evidence) + if result.returncode: + raise RuntimeError(f"{name} failed ({result.returncode}); see {log}; later gates not run") + if name.endswith("-campaign") and stage["elapsed_secs"] > 300: + raise RuntimeError(f"{name} exceeded the complete campaign timing ceiling; later gates not run") + if warm_started is not None and time.monotonic() - warm_started > 600: + raise RuntimeError("complete warm workflow exceeded its timing ceiling; later gates not run") + print(f"passed {name}: {stage['elapsed_secs']:.3f}s", flush=True) + return stage["elapsed_secs"] + + +def run(args): + root = args.artifacts.resolve() + # Do not reuse coverage or overwrite evidence from an earlier invocation. + root.mkdir(parents=True, exist_ok=False) + path = root / "ordered-acceptance.json" + evidence = {"schema_version": 5, "state": "running", "stages": [], "runs": [], + "machine": {"platform": platform.platform(), + "cpu_count": os.cpu_count(), + "cpuinfo": Path("/proc/cpuinfo").read_text(), + "memory": Path("/proc/meminfo").read_text()}, + "configuration": {"seed": args.seed, "warm_runs": args.warm_runs, + "case_deadline_secs": args.deadline_secs, + "image": args.image, "build_images": args.build_images, + "workspace": str(ROOT), "artifacts": str(root)}, + "paid_execution": "not_attempted"} + started = time.monotonic() + try: + evidence["source_digest"] = source_digest() + evidence["storage"] = probe(["df", "-T", str(root)]) + evidence["images_before_build"] = probe( + ["docker", "image", "list", "--format", "{{.Repository}}:{{.Tag}} {{.ID}}"]) + execute("build", ["cargo", "build", "--release", "-p", "myelin", + "--bins", "-p", "myelin-e2e-fuzz"], root, evidence, path) + execute("build-test-binaries", TEST_COMMAND + ["--no-run"], root, evidence, path) + execute("build-deployment-artifacts", + [str(BINARY), "--prepare-artifacts", "--deadline-secs", "3600", + "--artifacts", str(root / "build-deployment-artifacts")], + root, evidence, path) + identity_path = root / "build-deployment-artifacts/build-identity.json" + evidence["deployment_artifacts"] = json.loads(identity_path.read_text()) + evidence["artifact_identity_path"] = str(identity_path) + if args.build_images: + parent = None + for role, image in [("base", "myelin-node-base:cuda12.6"), + ("node", "myelin-node:latest"), ("e2e", args.image)]: + if parent is not None: + parent_tag = "myelin-e2e-parent:" + parent.removeprefix("sha256:") + execute(f"build-image-{role}-parent", ["docker", "tag", parent, parent_tag], + root, evidence, path) + execute(f"build-image-{role}", image_build_command( + role, image, evidence["deployment_artifacts"]["source_build_input_digest"], + parent), root, evidence, path) + if parent is not None and json.loads( + probe(["docker", "image", "inspect", parent_tag]))[0]["Id"] != parent: + raise RuntimeError("runtime image parent changed during build") + parent = json.loads(probe(["docker", "image", "inspect", image]))[0]["Id"] + evidence["build_artifacts"] = { + name: digest(ROOT / "target/release" / name) + for name in ["myelin-e2e-fuzz", "myelin-orchestrator", "myelin-worker"]} + if evidence["source_digest"] != source_digest(): + raise RuntimeError("source changed while building acceptance artifacts") + evidence["image_identities"] = runtime_image_identities(args.image) + verify_image_provenance(evidence["image_identities"], args.image, + evidence["deployment_artifacts"]["source_build_input_digest"]) + if "@sha256:" in args.image and args.image not in evidence["image_identities"][args.image]["repo_digests"]: + raise RuntimeError("runtime image is not bound to the requested registry manifest") + evidence["build_elapsed_secs"] = time.monotonic() - started + # Building with a pre-existing cache is not a cold benchmark. + evidence["cold_measurement"] = "not_claimed; existing cache retained" + for index in range(args.warm_runs): + directory = root / f"warm-{index + 1}" + directory.mkdir() + record = {"index": index + 1, "state": "running"} + evidence["runs"].append(record) + run_start = time.monotonic() + common = [str(BINARY), "--seed", str(args.seed), "--deadline-secs", + str(args.deadline_secs), "--no-build-image", "--image", args.image] + try: + execute(f"warm-{index + 1}-gate-a", + common + ["--deployment-e2e", "--deployment-nodes", "5", + "--artifacts", str(directory / "gate-a")], + directory, evidence, path, warm_started=run_start) + record["gate_a"] = "passed" + campaign_secs = execute(f"warm-{index + 1}-campaign", + common + ["--campaign", "--fixture-lifetime-secs", "43200", + "--artifacts", str(directory / "campaign")], + directory, evidence, path, warm_started=run_start) + record["campaign_elapsed_secs"] = campaign_secs + execute(f"warm-{index + 1}-failure-cases", + common + ["--nodes", "5", "--failure-cases", "--artifacts", + str(directory / "failure-cases")], directory, evidence, path, + warm_started=run_start) + execute(f"warm-{index + 1}-contract-model-safety", + TEST_COMMAND, directory, evidence, path, warm_started=run_start) + execute(f"warm-{index + 1}-scripted-provider", + ["bash", "tools/myelin-e2e-fuzz/scripted_safety_gate.sh", + str(directory / "scripted-provider")], directory, evidence, path, + env=dict(os.environ, SAFETY_GATE_ROOT=str(ROOT), + SCRIPTED_HARNESS_BINARY=str(BINARY)), warm_started=run_start) + record["state"] = "passed" + except BaseException: + record["state"] = "failed" + raise + finally: + record["elapsed_secs"] = time.monotonic() - run_start + record["inside_target"] = (record["state"] == "passed" + and record.get("campaign_elapsed_secs", float("inf")) <= 300 + and record["elapsed_secs"] <= 600) + persist(path, evidence) + if not record["inside_target"]: + raise RuntimeError(f"warm run {index + 1} passed behavior but exceeded timing target") + execute("verify-qualified-deployment-artifacts", + [str(BINARY), "--prepare-artifacts", "--deadline-secs", str(args.deadline_secs), + "--artifacts", str(root / "verify-qualified-deployment-artifacts")], + root, evidence, path) + if json.loads((root / "verify-qualified-deployment-artifacts/build-identity.json").read_text()) != evidence["deployment_artifacts"]: + raise RuntimeError("resolver-selected deployment artifacts changed during ordered gates") + verify_qualified_inputs(evidence) + evidence["state"] = "passed" + except BaseException as error: + evidence["state"] = "failed" + evidence["error"] = str(error) + raise + finally: + evidence["elapsed_secs"] = time.monotonic() - started + evidence["completed_unix_ms"] = time.time_ns() // 1_000_000 + evidence["expires_unix_ms"] = evidence["completed_unix_ms"] + 86_400_000 + persist(path, evidence) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--artifacts", type=Path, required=True) + parser.add_argument("--image", default="myelin-e2e-speed:local") + parser.add_argument("--build-images", action="store_true") + parser.add_argument("--seed", type=int, default=20260910) + parser.add_argument("--deadline-secs", type=int, default=120) + parser.add_argument("--warm-runs", type=int, default=3) + args = parser.parse_args() + if args.warm_runs < 3 or args.deadline_secs <= 0: + parser.error("at least three warm runs and a positive operation deadline are required") + if args.image in {"myelin-node-base:cuda12.6", "myelin-node:latest"}: + parser.error("the workload image must be distinct from its base and node images") + if args.build_images and "@sha256:" in args.image: + parser.error("build with a mutable image tag, publish it, then qualify its immutable " + "registry reference without --build-images") + try: + run(args) + except (OSError, RuntimeError, subprocess.SubprocessError) as error: + print(error, file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/myelin-e2e-fuzz/scripted_provider.py b/tools/myelin-e2e-fuzz/scripted_provider.py new file mode 100755 index 0000000..8fa7d79 --- /dev/null +++ b/tools/myelin-e2e-fuzz/scripted_provider.py @@ -0,0 +1,1068 @@ +#!/usr/bin/env python3 +"""Local VastAI wire fixture and independent real-CLI safety scenarios. + +Server: scripted_provider.py PORT CONTROL_JSON REQUEST_LOG [READY_FD] +Gate: scripted_provider.py --gate ROOT OUTPUT HARNESS_BINARY + +Each server owns its contracts, request ledger, and fault controls. The ready +pipe is written only after HTTP bind/listen succeeds. No credential is logged. +""" + +import json +import os +from pathlib import Path +import select +import secrets +import signal +import subprocess +import sys +import threading +import time +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlparse + + +UNRELATED_ID = 800000 +UNRELATED_LABEL = "unrelated-account-resource-marker" +UNRELATED_CONTRACTS = {UNRELATED_ID: UNRELATED_LABEL} +UNRELATED_WIRE_CONTRACTS = {str(UNRELATED_ID): UNRELATED_LABEL} +KEY_ENV = "MYELIN_SCRIPTED_VASTAI_KEY" +CREDENTIAL_ENVS = (KEY_ENV, "VAST_API_KEY", "MYELIN_VASTAI_API_KEY", "VASTAI_API_KEY") + + +def credential_values(env): + return tuple(env[name].encode() for name in CREDENTIAL_ENVS if env.get(name)) + + +def require_credential_absence(content, env, surface): + if any(value in content for value in credential_values(env)): + raise AssertionError(f"credential leaked into {surface}") + + +def scan_credentials(directory, env): + overlap = max(map(len, credential_values(env)), default=1) - 1 + files = 0 + total_bytes = 0 + for parent, directories, names in os.walk(directory, followlinks=False): + for name in directories + names: + artifact = Path(parent) / name + if artifact.is_symlink(): + raise AssertionError("credential scan cannot attest a symlinked artifact") + for name in names: + artifact = Path(parent) / name + require_credential_absence(os.fsencode(artifact), env, "artifact path") + with artifact.open("rb") as source: + carry = b"" + while chunk := source.read(1024 * 1024): + total_bytes += len(chunk) + require_credential_absence(carry + chunk, env, "durable artifact") + carry = (carry + chunk)[-overlap:] if overlap else b"" + files += 1 + return {"files": files, "bytes": total_bytes, "injected_credentials": len(credential_values(env))} + + +def default_offers(): + return [ + {"id": 1000 + index, "host_id": 2000 + index, "gpu_name": "RTX A2000", + "dph_total": 0.05, "gpu_ram": 24000, "compute_cap": 860, + "verification": "verified", "reliability2": 0.99, + "inet_down": 1000.0, "inet_up": 1000.0, + "internet_down_cost_per_tb": 0.0, "internet_up_cost_per_tb": 0.0} + for index in range(1, 6) + ] + + +def instance_status(status_mode, contract, label, ssh_endpoint=None): + host, port = ssh_endpoint or ("", 0) + return {"id": contract, "label": label, "actual_status": status_mode, + "intended_status": "running", "public_ipaddr": host, "ssh_port": port, + "status_msg": "scripted provider failed" if status_mode == "error" else "", + "disk_usage": 0.0} + + +class State: + def __init__(self, control_path, log_path): + self.lock = threading.RLock() + self.control_path = Path(control_path) + self.log_path = Path(log_path) + self.contracts = dict(UNRELATED_CONTRACTS) + self.created = {} + existing = self.control().get("initial_contracts", {}) + self.contracts.update({int(contract): label for contract, label in existing.items()}) + self.created.update({int(contract): label for contract, label in existing.items()}) + self.ssh_keys = [] + self.destroyed = set() + self.pending = {} + self.next_contract = 9000 + self.create_count = 0 + self.list_count = 0 + self.offer_count = 0 + self.next_listing = 0.0 + self.late_inserted = False + self.delete_release = threading.Event() + self.create_release = threading.Event() + self.offer_release = threading.Event() + self.sequence = 0 + + def control(self): + try: + return json.loads(self.control_path.read_text()) + except FileNotFoundError: + return {} + + def log(self, method, path, authorized, **fields): + with self.lock: + self.sequence += 1 + entry = {"sequence": self.sequence, "method": method, "path": path, + "authorized": authorized, "elapsed_ns": time.monotonic_ns(), **fields} + with self.log_path.open("a") as handle: + handle.write(json.dumps(entry) + "\n") + return self.sequence + + def reveal(self, control): + for contract, (label, after_list, after_time) in list(self.pending.items()): + if self.list_count >= after_list and time.monotonic() >= after_time: + self.contracts[contract] = label + del self.pending[contract] + self.log("EVENT", "contract-visible", True, contract_id=contract, label=label) + + +class Handler(BaseHTTPRequestHandler): + state = None + + def log_message(self, *_args): + pass + + def parse_request(self): + if self.raw_requestline.startswith(b"SSH-"): + self.state.log("SSH", "bootstrap-connection", True) + self.state.create_release.wait() + return False + if not super().parse_request(): + return False + if any(value in self.path.encode() for value in credential_values(os.environ)): + self.state.log("LEAK", "request-target", False) + self._reply(400, {"error": "credential in request target"}) + return False + return True + + @property + def authorized(self): + expected = os.environ.get(KEY_ENV) + return bool(expected) and self.headers.get("Authorization") == f"Bearer {expected}" + + def _reply(self, code, payload, **headers): + if code >= 400 and self.state.control().get("credential_echo"): + payload = dict(payload, diagnostic=os.environ[KEY_ENV]) + self.state.log("EVENT", "credential-echo", True, status=code) + body = json.dumps(payload).encode() + try: + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + for name, value in headers.items(): + self.send_header(name.replace("_", "-"), str(value)) + self.end_headers() + self.wfile.write(body) + except (BrokenPipeError, ConnectionResetError): + # Deliberately cancelled requests are part of the cleanup scenario. + pass + + def _body(self): + body = self.rfile.read(int(self.headers.get("Content-Length", "0"))) or b"{}" + require_credential_absence(body, os.environ, "provider request body") + return json.loads(body) + + def do_GET(self): + state = self.state + control = state.control() + path = urlparse(self.path).path + if not self.authorized: + state.log("GET", path, False) + return self._reply(401, {"error": "unauthorized"}) + if path == "/api/v0/bundles/": + with state.lock: + state.offer_count += 1 + offer_count = state.offer_count + state.log("GET", path, True, offer_count=offer_count) + offers = control.get("offers", default_offers()) + if offer_count > control.get("withhold_offer_responses_after", sys.maxsize): + state.offer_release.wait() + return self._reply(200, {"offers": offers}) + with state.lock: + if path == "/api/v0/ssh/": + state.log("GET", path, True) + return self._reply(200, {"ssh_keys": [ + {"id": index + 1, "public_key": key} + for index, key in enumerate(state.ssh_keys)]}) + if path == "/__scripted__/state": + state.log("GET", path, True) + return self._reply(200, {"contracts": state.contracts, "created": state.created, + "pending": list(state.pending), "destroyed": sorted(state.destroyed)}) + if path == "/api/v0/instances/": + state.list_count += 1 + now = time.monotonic() + rate_limited = ( + state.list_count <= control.get("rate_limit_listings", 0) + or now < state.next_listing + ) + if rate_limited: + retry_after = control.get("retry_after", 1) + state.next_listing = max(state.next_listing, now + retry_after) + state.log("GET", path, True, status=429, retry_after=retry_after) + return self._reply(429, {"retry_after": retry_after}, Retry_After=retry_after) + endpoint = ("127.0.0.1", self.server.server_port) if control.get("ssh_endpoint") else None + instances = [instance_status(control.get("status_mode", "error"), contract, label, endpoint) + for contract, label in sorted(state.contracts.items())] + state.log("GET", path, True, status=200, + returned_ids=sorted(state.contracts)) + return self._reply(200, {"instances": instances}) + parts = path.strip("/").split("/") + try: + contract = int(parts[3]) + except (IndexError, ValueError): + state.log("GET", path, True, status=404) + return self._reply(404, {"error": "unknown path"}) + state.reveal(control) + label = state.contracts.get(contract) + state.log("GET", path, True, status=200 if label else 404) + if label is None: + return self._reply(404, {"error": "not found"}) + endpoint = ("127.0.0.1", self.server.server_port) if control.get("ssh_endpoint") else None + return self._reply(200, {"instances": instance_status( + control.get("status_mode", "error"), contract, label, endpoint)}) + + def do_PUT(self): + state = self.state + control = state.control() + path = urlparse(self.path).path + body = self._body() + parts = path.strip("/").split("/") + label = body.get("label") + state.log("PUT", path, self.authorized, label=label, + image=body.get("image"), disk=body.get("disk")) + if not self.authorized: + return self._reply(401, {"error": "unauthorized"}) + if len(parts) != 4 or parts[2] != "asks": + return self._reply(404, {"error": "unknown path"}) + mode = control.get("create_mode", "ok") + if mode == "withheld-acceptance": + state.create_release.wait() + with state.lock: + state.create_count += 1 + mode = control.get("create_mode", "ok") + if mode == "reject": + return self._reply(400, {"error": "no_such_ask"}) + contract = state.next_contract + state.next_contract += 1 + state.created[contract] = label + if mode == "ambiguous": + state.pending[contract] = (label, state.list_count + control.get("reveal_after_listings", 2), + time.monotonic() + control.get("create_visibility_delay", 0.05)) + return self._reply(503, {"error": "create reply lost after acceptance"}) + if mode == "withheld-response": + state.pending[contract] = (label, state.list_count + 2, time.monotonic() + 0.1) + else: + state.contracts[contract] = label + if state.create_count == 1: + for offset in range(control.get("extra_owned_contracts", 0)): + extra = 9500 + offset + state.contracts[extra] = label + state.created[extra] = label + state.log("EVENT", "create-accepted-before-response", True, contract_id=contract, label=label) + if mode == "withheld-response": + state.create_release.wait() + return self._reply(200, {"new_contract": contract}) + + def do_DELETE(self): + state = self.state + control = state.control() + path = urlparse(self.path).path + request_sequence = state.log("DELETE", path, self.authorized) + if not self.authorized: + return self._reply(401, {"error": "unauthorized"}) + try: + contract = int(path.strip("/").split("/")[3]) + except (IndexError, ValueError): + return self._reply(404, {"error": "unknown contract"}) + if contract == control.get("withhold_delete"): + state.delete_release.wait() + if contract == control.get("slow_delete"): + time.sleep(control.get("slow_delete_seconds", 1.5)) + with state.lock: + label = state.contracts.pop(contract, None) + state.pending.pop(contract, None) + state.destroyed.add(contract) + if label and control.get("late_contract_after_delete") and not state.late_inserted: + state.late_inserted = True + late = control["late_contract_after_delete"] + state.contracts[late] = label + state.created[late] = label + state.log("EVENT", "delete-completed", True, contract_id=contract, + request_sequence=request_sequence, existed=label is not None) + return self._reply(200 if label is not None else 404, {"deleted": contract}) + + def do_POST(self): + state = self.state + path = urlparse(self.path).path + state.log("POST", path, self.authorized) + if not self.authorized: + return self._reply(401, {"error": "unauthorized"}) + body = self._body() + with state.lock: + if path == "/api/v0/ssh/": + key = body["ssh_key"] + if key not in state.ssh_keys: + state.ssh_keys.append(key) + return self._reply(200, {"success": True, "key": { + "id": state.ssh_keys.index(key) + 1, "public_key": key}}) + if path == "/__scripted__/restore": + for contract, label in body["contracts"].items(): + contract = int(contract) + state.contracts[contract] = label + state.destroyed.discard(contract) + state.delete_release.clear() + elif path == "/__scripted__/release-deletes": + state.delete_release.set() + else: + return self._reply(404, {"error": "unknown fault control"}) + return self._reply(200, {"applied": True}) + + +def serve(): + Handler.state = State(sys.argv[2], sys.argv[3]) + server = ThreadingHTTPServer(("127.0.0.1", int(sys.argv[1])), Handler) + server.daemon_threads = True + if len(sys.argv) > 4: + with os.fdopen(int(sys.argv[4]), "w") as ready: + ready.write(json.dumps({"port": server.server_port, "pid": os.getpid()}) + "\n") + ready.flush() + server.serve_forever() + + +def run_process(command, directory, env, stdout_path, stderr_path, timeout=150): + require_credential_absence(b"\0".join(os.fsencode(part) for part in command), env, "process arguments") + started = time.monotonic() + with stdout_path.open("w") as stdout, stderr_path.open("w") as stderr: + process = subprocess.Popen(command, cwd=directory, env=env, stdout=stdout, + stderr=stderr, start_new_session=True) + try: + status = process.wait(timeout=timeout) + except BaseException: + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + raise + return {"exit_status": status, "elapsed_seconds": time.monotonic() - started} + + +def crash_runner(command, root, env, directory, request_path, failure, boundary): + require_credential_absence(b"\0".join(os.fsencode(part) for part in command), env, "process arguments") + started = time.monotonic() + with (directory / "stdout.log").open("w") as stdout, (directory / "stderr.log").open("w") as stderr: + process = subprocess.Popen(command, cwd=root, env=env, stdout=stdout, + stderr=stderr, start_new_session=True) + try: + until = time.monotonic() + 90 + while time.monotonic() < until: + rows = [json.loads(line) for line in request_path.read_text().splitlines()] if request_path.exists() else [] + state_paths = list((directory / "campaign").glob("*/paid-state.json")) + admission = None + if len(state_paths) == 1: + paid = json.loads(state_paths[0].read_text()) + admission_path = Path(paid["state_dir"]) / "paid-admission.json" + if admission_path.exists(): + admission = json.loads(admission_path.read_text()) + creates = [row for row in rows + if row["method"] == "PUT" and row["path"].startswith("/api/v0/asks/")] + accepted = sum(row["path"] == "create-accepted-before-response" for row in rows) + offers = sum(row["path"] == "/api/v0/bundles/" for row in rows) + reached = admission is not None and { + "admission-before-create-reservation": + offers >= 3 and not admission["create_reservations"] and not admission["contracts"], + "create-reserved-before-provider-acceptance": + len(creates) == 5 and len(admission["create_reservations"]) == 5 + and accepted == 0 and not admission["contracts"], + "provider-accepted-before-response-accounting": + accepted == 5 and len(admission["create_reservations"]) == 5 + and not admission["contracts"], + "contract-accounted-before-bootstrap": + len(admission["contracts"]) == 5 and not admission["initial_bootstraps"], + "bootstrap-reserved-before-prepared-commit": + len(admission["initial_bootstraps"]) == 5 + and admission["mode"] == "preparing", + }[boundary] + if reached: + break + assert process.poll() is None, f"runner exited before {boundary}" + time.sleep(0.02) + else: + raise AssertionError(f"{boundary} was not observed before runner kill") + assert len(state_paths) == 1 and admission is not None + for role in ("runner", "orchestrator", "owner", "supervisor"): + identity = admission["cleanup"][role] + assert identity, f"missing admitted {role} identity" + require_credential_absence( + Path(f"/proc/{identity['pid']}/cmdline").read_bytes(), env, f"{role} arguments") + if failure == "workstation-loss": + # The provider is outside the failed workstation. Stop its local + # supervision first so no owner can be restarted during the crash. + signal_cleanup_process(admission_path, "supervisor", signal.SIGKILL) + if failure in ("owner-loss-automatic", "workstation-loss"): + signal_cleanup_process(admission_path, "owner", signal.SIGKILL) + if failure in ("orchestrator-loss", "workstation-loss"): + signal_cleanup_process(admission_path, "orchestrator", signal.SIGKILL) + if failure == "orchestrator-loss": + process.wait(timeout=60) + else: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=5) + before = state_paths[0].read_bytes() + repeated = run_process(command, root, env, directory / "repeat.stdout.log", directory / "repeat.stderr.log") + assert repeated["exit_status"] != 0, "repeated paid campaign overwrote ownership" + assert state_paths[0].read_bytes() == before, "refusal modified original paid ownership" + finally: + if process.poll() is None: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + return {"exit_status": process.returncode, "elapsed_seconds": time.monotonic() - started, + "crash_boundary": boundary, + "runner_killed_before_accounting": failure != "orchestrator-loss", + "cleanup_owner_killed": failure in ("owner-loss-automatic", "workstation-loss"), + "original_cleanup": admission["cleanup"], + "original_contracts": admission["contracts"], + "original_initial_bootstraps": admission["initial_bootstraps"], + "original_create_reservations": admission["create_reservations"], + "post_crash_sequence": rows[-1]["sequence"]} + + +def signal_cleanup_process(admission_path, role, sig): + identity = json.loads(admission_path.read_text())["cleanup"][role] + assert identity, f"missing cleanup {role} identity" + pidfd = os.pidfd_open(identity["pid"]) + try: + proc = Path(f"/proc/{identity['pid']}") + fields = (proc / "stat").read_text().rsplit(") ", 1)[1].split() + command = (proc / "cmdline").read_bytes().split(b"\0") + assert int(fields[19]) == identity["start_ticks"], "cleanup process incarnation changed" + assert Path("/proc/sys/kernel/random/boot_id").read_text().strip() == identity["boot_id"] + if role == "orchestrator": + assert Path(os.fsdecode(command[0])).name == "myelin-orchestrator" + assert str(admission_path.parent).encode() in command + else: + assert f"--paid-cleanup-{role}".encode() in command and str(admission_path).encode() in command + signal.pidfd_send_signal(pidfd, sig) + assert select.select([pidfd], [], [], 5)[0], f"{role} crash did not stop the admitted incarnation" + finally: + os.close(pidfd) + return identity + + +def stop_scripted_cleanup_owners(directory): + paths = [directory / "private" / "paid-admission.json"] + for paid_path in (directory / "campaign").glob("*/paid-state.json"): + paid = json.loads(paid_path.read_text()) + paths.append(Path(paid["state_dir"]) / "paid-admission.json") + for admission_path in paths: + if not admission_path.exists(): + continue + cleanup = json.loads(admission_path.read_text()).get("cleanup") or {} + spending_stopped = cleanup.get("spending_stopped", False) + # A completed worker still belongs to its supervisor's child.wait(). + # Let that supervisor reap and exit naturally before forced teardown. + roles = ("owner", "supervisor") if spending_stopped else ("supervisor", "owner") + for role in roles: + identity = (json.loads(admission_path.read_text()).get("cleanup") or {}).get(role) + if not identity: + continue + pidfd = None + try: + pidfd = os.pidfd_open(identity["pid"]) + if select.select([pidfd], [], [], 0)[0]: + continue + proc = Path(f"/proc/{identity['pid']}") + fields = (proc / "stat").read_text().rsplit(") ", 1)[1].split() + if fields[0] in ("Z", "X"): + continue + assert int(fields[19]) == identity["start_ticks"], "cleanup process incarnation changed" + assert Path("/proc/sys/kernel/random/boot_id").read_text().strip() == identity["boot_id"] + if spending_stopped and select.select([pidfd], [], [], 5)[0]: + continue + command = (proc / "cmdline").read_bytes().split(b"\0") + # Exit can race both the stat and cmdline reads. Empty argv + # alone is not proof of exit and must never authorize a signal. + if select.select([pidfd], [], [], 0)[0]: + continue + fields = (proc / "stat").read_text().rsplit(") ", 1)[1].split() + if fields[0] in ("Z", "X"): + continue + if not any(command) and select.select([pidfd], [], [], 5)[0]: + continue + assert int(fields[19]) == identity["start_ticks"], "cleanup process incarnation changed" + assert f"--paid-cleanup-{role}".encode() in command and str(admission_path).encode() in command + signal.pidfd_send_signal(pidfd, signal.SIGTERM) + assert select.select([pidfd], [], [], 5)[0], f"cleanup {role} did not stop" + except (FileNotFoundError, ProcessLookupError): + pass + finally: + if pidfd is not None: + os.close(pidfd) + + +def retained_owner_loss(binary, root, env, directory, request, ledger): + private = directory / "private" + private.mkdir(mode=0o700) + admission_path = private / "paid-admission.json" + command = [str(binary), "--scripted-retained-lifecycle", str(admission_path), + "--api-key-env", KEY_ENV] + initial = run_process(command, root, env, directory / "prepare.stdout.log", directory / "prepare.stderr.log") + assert initial["exit_status"] == 0, "scripted retained admission did not prepare" + before = json.loads(admission_path.read_text()) + assert before["mode"] == "prepared" and before["cleanup"]["retained_development_fixture"] + resumed = run_process(command, root, env, directory / "redeploy.stdout.log", directory / "redeploy.stderr.log") + assert resumed["exit_status"] == 0, "explicit retained generation was not admitted" + retained = json.loads(admission_path.read_text()) + grant = retained["retained_deployments"]["scripted-retained-generation"] + assert set(grant["bootstraps"]) == set(retained["contracts"]), "generation did not cover exact contracts" + for field in ("contracts", "create_reservations", "initial_bootstraps"): + assert retained[field] == before[field], f"redeployment changed {field}" + for field in ("limits", "started_unix_ms", "stop_unix_ms", "supervisor"): + assert retained["cleanup"][field] == before["cleanup"][field], f"redeployment changed original {field}" + # Both foreground processes have exited; no orchestrator/runner can issue + # manual cleanup. Only the independently supervised worker can delete. + for role in ("runner", "orchestrator"): + process = retained["cleanup"][role] + proc = Path(f"/proc/{process['pid']}/stat") + if proc.exists(): + fields = proc.read_text().rsplit(") ", 1)[1].split() + assert int(fields[19]) != process["start_ticks"] or fields[0] in ("Z", "X") + prior_sequence = ledger()[-1]["sequence"] if ledger() else 0 + killed = signal_cleanup_process(admission_path, "owner", signal.SIGKILL) + until = time.monotonic() + 30 + while time.monotonic() < until: + after = json.loads(admission_path.read_text()) + state = request("/__scripted__/state") + if (after["cleanup"]["complete"] and after["cleanup"]["spending_stopped"] + and state["contracts"] == UNRELATED_WIRE_CONTRACTS and not state["pending"]): + break + time.sleep(0.05) + else: + raise AssertionError("owner loss did not automatically stop retained spending") + assert after["cleanup"]["owner"] != killed, "owner was not restarted" + assert after["mode"] == "cleanup_only", "restart reopened paid permission" + for field in ("limits", "started_unix_ms", "stop_unix_ms", "supervisor"): + assert after["cleanup"][field] == before["cleanup"][field], f"owner restart changed {field}" + for field in ("contracts", "create_reservations", "initial_bootstraps", "retained_deployments"): + assert after[field] == retained[field], f"owner restart changed {field}" + ceiling_ms = min(before["deadline_unix_ms"], before["cleanup"]["stop_unix_ms"] + 300_000) + assert time.time_ns() // 1_000_000 < ceiling_ms, "original ceiling was exceeded" + rows = ledger() + assert all(row["authorized"] for row in rows), "unauthenticated lifecycle request" + assert not any(row["method"] == "PUT" or row["path"] == "/api/v0/bundles/" for row in rows), "retained lifecycle searched or acquired" + assert not any(row["method"] == "DELETE" + and int(row["path"].rstrip("/").split("/")[-1]) in UNRELATED_CONTRACTS + for row in rows if row["path"].startswith("/api/v0/instances/")) + known = set(map(int, retained["contracts"].values())) + listings = [row for row in rows if row["sequence"] > prior_sequence + and row["path"] == "/api/v0/instances/" and row.get("status") == 200] + assert listings and not known.intersection(listings[-1]["returned_ids"]), "missing typed exact absence after owner loss" + assert known <= set(state["destroyed"]), "cleanup omitted an exact retained identity" + (directory / "provider-final-state.json").write_text(json.dumps(state, indent=2)) + return {"exit_status": 0, "creates": 0, "retained_redeployment_admitted": True, + "crash_boundary": "prepared-commit-cleanup-owner-loss", + "initial_reservations_unchanged": True, "automatic_owner_restart": True, + "acquisition_slots": len(after["create_reservations"]), + "initial_bootstrap_slots": len(after["initial_bootstraps"]), + "manual_cleanup_commands": 0, "original_ceilings_preserved": True, + "created_contract_ids": [], "existing_contract_ids": sorted(known), + "destroyed_contract_ids": sorted(state["destroyed"]), "exact_absence": True, + "unrelated_preserved": True, "scope": "admission-and-real-cleanup-processes"} + + +def gate(root, output, binary): + if not __debug__: + raise RuntimeError("scripted safety proofs require Python assertions; optimization is forbidden") + gate_started = time.monotonic() + root, output, binary = Path(root).resolve(), Path(output).resolve(), Path(binary).resolve() + from ordered_acceptance import ROOT, TEST_COMMAND, digest, source_digest + + assert root == ROOT, "gate fixtures must hash the harness workspace" + rejection_build_artifacts = { + name: digest(root / "target/release" / name) + for name in ("myelin-e2e-fuzz", "myelin-orchestrator", "myelin-worker") + } + assert digest(binary) == rejection_build_artifacts["myelin-e2e-fuzz"], "scripted gate binary is not the qualified executable" + rejection_source_digest = source_digest() + rejection_image = "registry.invalid/myelin-node@sha256:" + "0" * 64 + rejected_gates = { + "missing-gates": "paid execution requires --gate-attestation from ordered_acceptance.py; operator authorization is not gate evidence", + "stale-gates": "paid gate attestation is missing, incomplete, stale, or incompatible", + "failed-gates": "paid gate stages failed or were not executed in order", + "out-of-order-gates": "missing, duplicate or out-of-order required gate warm-1-campaign", + "changed-binaries": "paid gate evidence does not attest the current binaries, sources and image", + } + output.mkdir(parents=True, exist_ok=False) + env = dict(os.environ, **{name: "scripted-" + secrets.token_hex(24) for name in CREDENTIAL_ENVS}) + marker = env[KEY_ENV] + key = output / "dummy_ed25519" + subprocess.run(["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(key), + "-C", "scripted-safety"], check=True) + duplicate_offers = default_offers() + for offer in duplicate_offers: + offer["host_id"] = 2001 + malformed_offers = default_offers() + malformed_offers[0]["dph_total"] = -1.0 + priced_offers = list(reversed(default_offers())) + priced_offers += [ + dict(default_offers()[0], id=1101, dph_total=0.01), + dict(default_offers()[0], id=1102, host_id=2102, dph_total=0.001, verification="unverified"), + dict(default_offers()[0], id=1103, host_id=59017, dph_total=0.001), + dict(default_offers()[0], id=1104, host_id=2104, dph_total=0.08), + ] + preflight = { + "zero-cost": ("--max-total-cost-usd", "0", "explicit positive paid-run ceiling"), + "invalid-cost": ("--max-hourly-cost-usd", "NaN", "must be finite and non-negative"), + "zero-lifetime": ("--fixture-lifetime-secs", "0", "resource bounds must be nonzero"), + "short-lifetime": ("--fixture-lifetime-secs", "1199", "planned campaign requires"), + "empty-image": ("--image", " ", "runtime image must not be empty"), + "missing-credential": (None, None, "credential environment"), + } + provider_admissions = { + "selected-cost-over-budget": ("--max-total-cost-usd", "1", "selected worst-case cost"), + } + crash_scenarios = { + "runner-loss": ("runner-loss", "provider-accepted-before-response-accounting"), + "owner-loss-automatic": ("owner-loss-automatic", "provider-accepted-before-response-accounting"), + "orchestrator-loss": ("orchestrator-loss", "provider-accepted-before-response-accounting"), + "workstation-loss": ("workstation-loss", "provider-accepted-before-response-accounting"), + "crash-before-create-reservation": ("runner-loss", "admission-before-create-reservation"), + "crash-after-create-reservation": ("runner-loss", "create-reserved-before-provider-acceptance"), + "crash-after-contract-accounting": ("runner-loss", "contract-accounted-before-bootstrap"), + "crash-after-bootstrap-reservation": ("runner-loss", "bootstrap-reserved-before-prepared-commit"), + } + scenarios = { + "missing-gates": {}, + "stale-gates": {}, + "failed-gates": {}, + "out-of-order-gates": {}, + "changed-binaries": {}, + **{name: {} for name in preflight}, + "campaign-resource-overflow": {}, + **{name: {} for name in provider_admissions}, + "runner-loss": {"create_mode": "withheld-response"}, + "owner-loss-automatic": {"create_mode": "withheld-response"}, + "orchestrator-loss": {"create_mode": "withheld-response"}, + "workstation-loss": {"create_mode": "withheld-response"}, + "crash-before-create-reservation": {"withhold_offer_responses_after": 2}, + "crash-after-create-reservation": {"create_mode": "withheld-acceptance"}, + "crash-after-contract-accounting": {"status_mode": "loading"}, + "crash-after-bootstrap-reservation": {"status_mode": "running", "ssh_endpoint": True}, + "retained-owner-loss": {"initial_contracts": { + str(8999 + node): f"scripted-retained-42-{node}-attempt-0" for node in range(1, 6)}}, + "duplicate-hosts": {"offers": duplicate_offers}, + "insufficient-offers": {"offers": []}, + "malformed-offers": {"offers": malformed_offers}, + "cheapest-distinct-verified": {"offers": priced_offers}, + "create-rejected": {"create_mode": "reject", "credential_echo": True}, + "rate-limited-listing": {"create_mode": "reject", "rate_limit_listings": 1, "retry_after": 1}, + "delayed-ambiguous-create": {"create_mode": "ambiguous", "reveal_after_listings": 2, + "create_visibility_delay": 0}, + "late-contract": {"status_mode": "running", "late_contract_after_delete": 9700, + "extra_owned_contracts": 2}, + "slow-delete": {"slow_delete": 9000, "slow_delete_seconds": 1.5, "extra_owned_contracts": 2, + "late_contract_after_delete": 9700}, + "cleanup-only-recovery": {"extra_owned_contracts": 2}, + } + summary = {"schema_version": 3, "state": "running", "scenarios": {}, + "provider": "loopback-scripted", "paid_resources": 0} + for name, control in scenarios.items(): + directory = output / name + directory.mkdir(parents=True, exist_ok=False) + control_path = directory / "control.json" + control_path.write_text(json.dumps(control)) + request_path = directory / "requests.jsonl" + ready_read, ready_write = os.pipe() + provider_log = (directory / "provider.log").open("w") + provider = subprocess.Popen([sys.executable, "-E", "-B", __file__, "0", str(control_path), str(request_path), + str(ready_write)], env=env, pass_fds=(ready_write,), + stdout=provider_log, stderr=provider_log) + os.close(ready_write) + try: + assert select.select([ready_read], [], [], 10)[0], "provider did not signal readiness" + ready = json.loads(os.read(ready_read, 4096)) + assert ready["pid"] == provider.pid and provider.poll() is None + url = f"http://127.0.0.1:{ready['port']}" + scenario_env = dict(env, VASTAI_BASE_URL=url, + XDG_STATE_HOME=str(directory / "private-state")) + + def request(path, body=None): + data = None if body is None else json.dumps(body).encode() + req = urllib.request.Request(url + path, data=data, + headers={"Authorization": f"Bearer {marker}", "Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=5) as response: + return json.load(response) + + def ledger(): + return [json.loads(line) for line in request_path.read_text().splitlines()] if request_path.exists() else [] + + if name == "retained-owner-loss": + summary["scenarios"][name] = retained_owner_loss( + binary, root, scenario_env, directory, request, ledger) + (output / "scripted-safety-summary.json").write_text(json.dumps(summary, indent=2)) + continue + + command = [str(binary), "--paid-vastai", "--authorize-paid-vastai", "--scripted-provider", + "--seed", "20260910", "--fixture-lifetime-secs", "43200", "--max-total-cost-usd", "5", + "--max-hourly-cost-usd", "0.5", "--ssh-identity", str(key), + "--image", "myelin-node:latest", "--api-key-env", KEY_ENV, + "--deadline-secs", "5", "--no-build-image", "--artifacts", str(directory / "campaign")] + # These fixtures reject before runtime-image or deployment-artifact inspection. + if name in rejected_gates: + command.remove("--scripted-provider") + command[command.index("--image") + 1] = rejection_image + if name != "missing-gates": + now = time.time_ns() // 1_000_000 + stages = [] + qualified_binary = str(root / "target/release/myelin-e2e-fuzz") + qualified_artifacts = directory / "qualified-local-gates" + + def stage(stage_name, argv): + stages.append({"name": stage_name, "command": argv, + "state": "passed", "exit_code": 0, "elapsed_secs": 0.001, + "started_unix_ms": now - 1000 + len(stages) * 2, + "completed_unix_ms": now - 999 + len(stages) * 2}) + + stage("build", ["cargo", "build", "--release", "-p", "myelin", + "--bins", "-p", "myelin-e2e-fuzz"]) + stage("build-test-binaries", TEST_COMMAND + ["--no-run"]) + stage("build-deployment-artifacts", + [qualified_binary, "--prepare-artifacts", "--deadline-secs", "3600", + "--artifacts", str(qualified_artifacts / "build-deployment-artifacts")]) + for index in range(1, 4): + common = [qualified_binary, "--seed", "20260910", "--deadline-secs", + "120", "--no-build-image", "--image", rejection_image] + for suffix, flags in [ + ("gate-a", ["--deployment-e2e", "--deployment-nodes", "5"]), + ("campaign", ["--campaign", "--fixture-lifetime-secs", "43200"]), + ("failure-cases", ["--nodes", "5", "--failure-cases"])]: + stage(f"warm-{index}-{suffix}", common + flags + ["--artifacts", + str(qualified_artifacts / f"warm-{index}" / suffix)]) + stage(f"warm-{index}-contract-model-safety", TEST_COMMAND) + stage(f"warm-{index}-scripted-provider", + ["bash", "tools/myelin-e2e-fuzz/scripted_safety_gate.sh", + str(qualified_artifacts / f"warm-{index}" / "scripted-provider")]) + stage("verify-qualified-deployment-artifacts", + [qualified_binary, "--prepare-artifacts", "--deadline-secs", "120", + "--artifacts", str(qualified_artifacts / "verify-qualified-deployment-artifacts")]) + # Synthetic image/deployment identities satisfy the schema, not paid + # qualification. Only the intended guard below may reject each fixture. + evidence = {"schema_version": 5, "state": "passed", "completed_unix_ms": now - 2, + "expires_unix_ms": now + 60_000, + "build_elapsed_secs": 0.02, "elapsed_secs": 0.1, + "build_artifacts": dict(rejection_build_artifacts), + "deployment_artifacts": { + "schema_version": 2, "source_build_input_digest": "0" * 64, + "executables": dict(rejection_build_artifacts), + "wheel_input_digest": "0" * 64, "wheels": {}, "payload": {}}, + "source_digest": rejection_source_digest, + "image_identities": { + image: {"id": "sha256:" + str(index) * 64, "os": "linux", + "architecture": "amd64", "variant": "", + "repo_digests": [rejection_image], + "provenance_version": 1, + "source_build_input_digest": "0" * 64, + "image_role": role, + "parent_image_id": None if index == 0 else "sha256:" + str(index - 1) * 64} + for index, (image, role) in enumerate([ + ("myelin-node-base:cuda12.6", "base"), + ("myelin-node:latest", "node"), (rejection_image, "e2e")])}, + "configuration": {"warm_runs": 3, "image": rejection_image, + "seed": 20260910, "case_deadline_secs": 120, + "build_images": False, "workspace": str(root), + "artifacts": str(qualified_artifacts)}, + "stages": stages, "runs": [{"index": index, "state": "passed", "gate_a": "passed", + "inside_target": True, "campaign_elapsed_secs": 0.001, + "elapsed_secs": 0.01} for index in range(1, 4)]} + if name == "stale-gates": + evidence["expires_unix_ms"] = now - 1 + elif name == "failed-gates": + stages[1]["state"], stages[1]["exit_code"] = "failed", 1 + elif name == "out-of-order-gates": + stages[4]["name"], stages[5]["name"] = stages[5]["name"], stages[4]["name"] + elif name == "changed-binaries": + tested = evidence["build_artifacts"]["myelin-e2e-fuzz"] + evidence["build_artifacts"]["myelin-e2e-fuzz"] = ( + ("1" if tested[0] == "0" else "0") + tested[1:]) + attestation_path = directory / "rejected-attestation.json" + attestation_path.write_text(json.dumps(evidence)) + command += ["--gate-attestation", str(attestation_path)] + if name in preflight: + flag, value, _ = preflight[name] + if flag is not None: + command[command.index(flag) + 1] = value + else: + scenario_env.pop(KEY_ENV) + if name in provider_admissions: + flag, value, _ = provider_admissions[name] + command[command.index(flag) + 1] = value + if name == "campaign-resource-overflow": + command.append("--scripted-campaign-resource-overflow") + if name in crash_scenarios: + failure, boundary = crash_scenarios[name] + result = crash_runner( + command, root, scenario_env, directory, request_path, failure, boundary) + else: + result = run_process(command, root, scenario_env, directory / "stdout.log", directory / "stderr.log") + assert result["exit_status"] != 0, f"{name}: fault scenario incorrectly passed" + rows = ledger() + creates = [row for row in rows + if row["method"] == "PUT" and row["path"].startswith("/api/v0/asks/")] + assert all(row["authorized"] for row in rows), f"{name}: unauthenticated provider work" + assert len(creates) <= 5, f"{name}: exceeded five admitted acquisition attempts" + assert len({row["path"] for row in creates}) == len(creates), f"{name}: create retried for an admitted offer" + if (name in rejected_gates or name in preflight or name in provider_admissions + or name == "campaign-resource-overflow" + or name in ("duplicate-hosts", "insufficient-offers", "malformed-offers", + "crash-before-create-reservation")): + assert not creates, f"{name}: rejected or pre-create scenario acquired contracts" + else: + assert creates, f"{name}: scenario never exercised admitted acquisition" + if name in rejected_gates: + assert not rows, "rejected gate evidence reached the provider" + expected_guard = rejected_gates[name] + stderr = (directory / "stderr.log").read_text() + assert f"myelin-e2e-behavioral-fuzz: {expected_guard}" in stderr.splitlines(), ( + f"{name}: did not reach intended rejection guard {expected_guard!r}") + result.update({"rejection_guard": expected_guard, "provider_requests_before_rejection": 0}) + if name in preflight: + assert not rows, f"{name}: admission failure reached provider" + assert preflight[name][2] in (directory / "stderr.log").read_text(), f"{name}: wrong preflight failure" + result["provider_requests_before_rejection"] = 0 + if name in provider_admissions: + assert any(row["path"] == "/api/v0/bundles/" for row in rows), f"{name}: offer-cost admission was not exercised" + assert provider_admissions[name][2] in (directory / "stderr.log").read_text(), f"{name}: wrong provider admission failure" + result["provider_requests_before_rejection"] = len(rows) + if name == "campaign-resource-overflow": + assert not rows, f"{name}: campaign resource admission reached provider" + assert "campaign resources exceed the hard admission ceilings" in ( + directory / "stderr.log").read_text(), f"{name}: wrong campaign resource admission failure" + result["provider_requests_before_rejection"] = 0 + if name == "malformed-offers": + assert any(row["path"] == "/api/v0/bundles/" for row in rows), "malformed offer response was not exercised" + if control.get("credential_echo"): + assert any(row["path"] == "credential-echo" for row in rows), "credential echo was not exercised" + state_paths = list((directory / "campaign").rglob("paid-state.json")) + paid_path = state_paths[0] if state_paths else None + if creates or name in crash_scenarios: + assert len(state_paths) == 1, f"{name}: missing unique durable cleanup state" + paid = json.loads(paid_path.read_text()) + if name not in crash_scenarios: + assert paid["phase"] not in ("planned", "acquiring", "prepared"), f"{name}: acquisition remained authorized" + assert all(row["label"] in paid["owned_labels"] for row in creates), f"{name}: unauthorized owned label" + assert all(int(row["path"].rstrip("/").split("/")[-1]) in paid["selected_offer_ids"] + for row in creates), f"{name}: unselected offer acquired" + if name == "cheapest-distinct-verified": + assert paid["selected_offer_ids"] == [1101, 1002, 1003, 1004, 1005], "price/verification/distinct-host policy selected the wrong offers" + result["selected_offer_ids"] = paid["selected_offer_ids"] + + def cleanup(suffix, seconds=15): + before = ledger()[-1]["sequence"] if ledger() else 0 + outcome = run_process([str(binary), "--cleanup-only", str(paid_path), + "--deadline-secs", str(seconds), "--api-key-env", KEY_ENV, + "--artifacts", str(directory / "cleanup-artifacts")], + root, scenario_env, directory / f"{suffix}.stdout.log", + directory / f"{suffix}.stderr.log", timeout=seconds + 15) + new_rows = [row for row in ledger() if row["sequence"] > before] + assert not any(row["method"] == "PUT" or row["path"] == "/api/v0/bundles/" + for row in new_rows), f"{name}: cleanup-only searched or acquired" + return outcome, new_rows + + if name == "workstation-loss": + recovered, _ = cleanup("workstation-recovery") + assert recovered["exit_status"] == 0, "workstation cleanup-only recovery failed" + recovered_admission = json.loads((Path(paid["state_dir"]) / "paid-admission.json").read_text()) + assert recovered_admission["mode"] == "cleanup_only" + for field in ("limits", "started_unix_ms", "stop_unix_ms"): + assert recovered_admission["cleanup"][field] == result["original_cleanup"][field], "workstation recovery renewed cleanup ceilings" + result["cleanup_only"] = recovered + result["manual_cleanup_commands"] = 1 + + if name in set(crash_scenarios) - {"workstation-loss"}: + admission_path = Path(paid["state_dir"]) / "paid-admission.json" + until = time.monotonic() + 30 + while time.monotonic() < until: + state = request("/__scripted__/state") + admission = json.loads(admission_path.read_text()) + if (state["contracts"] == UNRELATED_WIRE_CONTRACTS + and not state["pending"] and admission["cleanup"]["complete"]): + break + time.sleep(0.05) + else: + raise AssertionError("independent supervision did not automatically stop spending") + for field in ("limits", "started_unix_ms", "stop_unix_ms", "supervisor"): + assert admission["cleanup"][field] == result["original_cleanup"][field] + assert admission["mode"] == "cleanup_only" + if name == "owner-loss-automatic": + assert admission["cleanup"]["owner"] != result["original_cleanup"]["owner"] + result["automatic_cleanup"] = True + result["manual_cleanup_commands"] = 0 + + if name in crash_scenarios: + restored = json.loads((Path(paid["state_dir"]) / "paid-admission.json").read_text()) + assert restored["initial_bootstraps"] == result["original_initial_bootstraps"], "crash recovery resumed initial bootstrap" + assert restored["create_reservations"] == result["original_create_reservations"], "crash recovery changed acquisition reservations" + assert restored["contracts"] == result["original_contracts"], "crash recovery changed contract accounting" + result["acquisition_slots"] = len(restored["create_reservations"]) + result["initial_bootstrap_slots"] = len(restored["initial_bootstraps"]) + assert not any(row["sequence"] > result["post_crash_sequence"] + and (row["method"] == "PUT" or row["path"] == "/api/v0/bundles/") + for row in ledger()), "crash recovery searched or acquired" + + if name == "delayed-ambiguous-create": + initial = request("/__scripted__/state") + if initial["pending"] or len(initial["contracts"]) > 1: + assert json.loads(paid_path.read_text())["phase"] == "cleanup_only", "late create obligation was abandoned" + recovery, _ = cleanup("ambiguous-recovery") + result["cleanup_only"] = recovery + assert recovery["exit_status"] == 0, "observed ambiguous identities failed cleanup-only recovery" + assert any(row["path"] == "contract-visible" for row in ledger()), "delayed discovery was not exercised" + + if name == "cleanup-only-recovery": + prior = request("/__scripted__/state") + assert len(prior["created"]) >= 3, "independent cleanup siblings were not constructed" + control["withhold_delete"] = 9000 + control_path.write_text(json.dumps(control)) + request("/__scripted__/restore", {"contracts": prior["created"]}) + failed, cleanup_rows = cleanup("withheld-delete", seconds=1) + assert failed["exit_status"] != 0 and failed["elapsed_seconds"] < 5, "withheld deletion did not yield bounded failure" + assert json.loads(paid_path.read_text())["phase"] == "cleanup_only", "foreground expiry abandoned durable cleanup" + pending = request("/__scripted__/state") + assert set(map(int, pending["contracts"])) == set(UNRELATED_CONTRACTS) | {9000}, "withheld delete blocked healthy cleanup" + assert any(row["path"] == "delete-completed" and row["contract_id"] != 9000 for row in cleanup_rows) + request("/__scripted__/release-deletes", {}) + recovered, recovery_rows = cleanup("cleanup-recovery") + assert recovered["exit_status"] != 0, "duplicate-label accounting must remain a failure after spending stops" + assert not any(row["method"] == "PUT" or row["path"] == "/api/v0/bundles/" + for row in recovery_rows), "cleanup-only recovery reopened acquisition" + result["withheld_delete"] = failed + result["cleanup_only"] = recovered + + final = request("/__scripted__/state") + (directory / "provider-final-state.json").write_text(json.dumps(final, indent=2)) + assert final["contracts"] == UNRELATED_WIRE_CONTRACTS and not final["pending"], f"{name}: exact attributable absence not proved" + rows = ledger() + assert not any(row["method"] == "DELETE" + and int(row["path"].rstrip("/").split("/")[-1]) in UNRELATED_CONTRACTS + for row in rows if row["path"].startswith("/api/v0/instances/")), f"{name}: unrelated resource touched" + created_ids = set(map(int, final["created"])) + if created_ids: + successful_listings = [row for row in rows if row["path"] == "/api/v0/instances/" and row.get("status") == 200] + assert successful_listings and not created_ids.intersection(successful_listings[-1]["returned_ids"]), "missing final typed provider absence census" + assert created_ids <= set(final["destroyed"]), "destroy accounting omitted an attributable identity" + if name == "rate-limited-listing": + rate_limited = [row for row in rows if row.get("status") == 429] + assert rate_limited, "listing rate limit not exercised" + for limited in rate_limited: + subsequent = next(row for row in rows if row["sequence"] > limited["sequence"] + and row["path"] == "/api/v0/instances/") + assert subsequent["elapsed_ns"] - limited["elapsed_ns"] >= limited["retry_after"] * 1e9, "Retry-After was violated" + if name == "slow-delete": + completions = [row for row in rows if row["path"] == "delete-completed"] + slow = next(row for row in completions if row["contract_id"] == 9000) + healthy = [row for row in completions if row["contract_id"] in (9500, 9501)] + assert len(healthy) == 2 and all(row["elapsed_ns"] < slow["elapsed_ns"] for row in healthy), "slow delete serialized healthy deletion" + late = next(row for row in completions if row["contract_id"] == 9700) + assert late["elapsed_ns"] < slow["elapsed_ns"], "pending delete blocked late-contract discovery" + for application_root in (directory / "campaign", directory / "private-state", directory / "private"): + for artifact in application_root.rglob("*"): + if artifact.is_file(): + content = artifact.read_bytes() + assert not any(label.encode() in content for label in UNRELATED_CONTRACTS.values()), "application persisted unrelated account data" + result.update({"creates": len(creates), "created_contract_ids": sorted(created_ids), + "destroyed_contract_ids": sorted(final["destroyed"]), "exact_absence": True, + "unrelated_preserved": True, + "listing_requests": sum(row["path"] == "/api/v0/instances/" for row in rows)}) + summary["scenarios"][name] = result + (output / "scripted-safety-summary.json").write_text(json.dumps(summary, indent=2)) + except BaseException as error: + summary["state"] = "failed" + summary["failed_scenario"] = name + detail = str(error) + for value in credential_values(env): + detail = detail.replace(value.decode(), "[REDACTED]") + summary["error"] = detail + raise + finally: + try: + stop_scripted_cleanup_owners(directory) + finally: + os.close(ready_read) + provider.terminate() + try: + provider.wait(timeout=5) + except subprocess.TimeoutExpired: + provider.kill() + provider.wait() + provider_log.close() + try: + scanned = scan_credentials(directory, env) + summary["scenarios"].setdefault(name, {})["credential_scan"] = scanned + except BaseException: + summary["state"] = "failed" + summary["failed_scenario"] = name + summary["credential_absence"] = False + raise + finally: + summary["elapsed_seconds"] = time.monotonic() - gate_started + (output / "scripted-safety-summary.json").write_text(json.dumps(summary, indent=2)) + summary["credential_scan"] = scan_credentials(output, env) + summary["gate_b_coverage"] = { + "4_offer_selection": [ + "duplicate-hosts", "insufficient-offers", "malformed-offers", + "cheapest-distinct-verified", + ], + "5_admission": [ + *preflight, *provider_admissions, "campaign-resource-overflow", + ], + "6_shared_acquisition_bound": [ + *crash_scenarios, "create-rejected", "delayed-ambiguous-create", + "retained-owner-loss", + ], + "7_bootstrap_and_redeployment_bound": [ + "crash-after-bootstrap-reservation", "retained-owner-loss", + ], + "8_preparation_crash_accounting": [ + "crash-before-create-reservation", "crash-after-create-reservation", + "runner-loss", "crash-after-contract-accounting", + "crash-after-bootstrap-reservation", "retained-owner-loss", + ], + "9_cleanup_only_failure_recovery": [ + "runner-loss", "owner-loss-automatic", "orchestrator-loss", + "workstation-loss", "retained-owner-loss", "cleanup-only-recovery", + ], + "15_concurrent_exact_cleanup": [ + "late-contract", "slow-delete", "cleanup-only-recovery", + ], + "16_credential_redaction": ["create-rejected", "all-scenario-artifact-scans"], + } + summary["state"] = "passed" + summary["credential_absence"] = True + summary["elapsed_seconds"] = time.monotonic() - gate_started + (output / "scripted-safety-summary.json").write_text(json.dumps(summary, indent=2)) + (output.parent / "vastai-gate-b-scripted.json").write_text(json.dumps(summary, indent=2)) + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + if sys.argv[1] == "--gate": + try: + gate(*sys.argv[2:]) + except Exception as error: + print(f"scripted safety gate failed ({type(error).__name__}); inspect its summary and logs", file=sys.stderr) + sys.exit(1) + else: + serve() diff --git a/tools/myelin-e2e-fuzz/scripted_safety_gate.sh b/tools/myelin-e2e-fuzz/scripted_safety_gate.sh new file mode 100755 index 0000000..d238055 --- /dev/null +++ b/tools/myelin-e2e-fuzz/scripted_safety_gate.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Gate B only: run after Gate A. Every scenario invokes the real paid CLI +# against an independently owned loopback HTTP provider and request ledger. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +OUT="${1:-$ROOT/target/vastai-gate-b-scripted-1}" +cd "$ROOT" +TARGET="${CARGO_TARGET_DIR:-$ROOT/target}" +# An ordered owner can supply its already-built immutable binary. Standalone +# invocation builds both required binaries once, never once per scenario. +if [ -z "${SCRIPTED_HARNESS_BINARY:-}" ]; then + cargo build --release -q -p myelin --bins -p myelin-e2e-fuzz + SCRIPTED_HARNESS_BINARY="$TARGET/release/myelin-e2e-fuzz" +fi +exec python3 -E -B "$ROOT/tools/myelin-e2e-fuzz/scripted_provider.py" \ + --gate "$ROOT" "$OUT" "$SCRIPTED_HARNESS_BINARY" diff --git a/tools/myelin-e2e-fuzz/src/bin/oracle_repro.rs b/tools/myelin-e2e-fuzz/src/bin/oracle_repro.rs new file mode 100644 index 0000000..22db6d2 --- /dev/null +++ b/tools/myelin-e2e-fuzz/src/bin/oracle_repro.rs @@ -0,0 +1,35 @@ +use myelin_e2e_fuzz::{BehaviorCase, BehaviorOracle, CaseObservation}; +use std::env; +use std::fs; +use std::time::Instant; + +fn main() { + let args: Vec = env::args().collect(); + if args.len() != 3 { + eprintln!("usage: oracle_repro "); + std::process::exit(2); + } + let mut case: BehaviorCase = + serde_json::from_slice(&fs::read(&args[1]).expect("read case")).expect("parse case"); + let observation: CaseObservation = + serde_json::from_slice(&fs::read(&args[2]).expect("read observation")) + .expect("parse observation"); + if let Ok(budget) = std::env::var("REPRO_RACE_BUDGET") { + case.resource_bounds.max_race_states = budget.parse().expect("budget integer"); + } + let started = Instant::now(); + match BehaviorOracle::verify(&case, &observation) { + Ok(()) => { + println!("ORACLE OK in {:?}", started.elapsed()); + } + Err(violation) => { + println!( + "ORACLE VIOLATION {} {} in {:?}", + violation.invariant, + violation.detail, + started.elapsed() + ); + std::process::exit(1); + } + } +} diff --git a/tools/myelin-e2e-fuzz/src/budget.rs b/tools/myelin-e2e-fuzz/src/budget.rs new file mode 100644 index 0000000..d635b77 --- /dev/null +++ b/tools/myelin-e2e-fuzz/src/budget.rs @@ -0,0 +1,247 @@ +//! Monotonic, hierarchical execution ownership and out-of-band timing evidence. + +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Condvar, LazyLock, Mutex}; +use std::time::{Duration, Instant}; + +use serde::Serialize; +use serde_json::{Value, json}; + +#[derive(Debug)] +struct Cancellation { + cancelled: AtomicBool, + parent: Option>, +} + +#[derive(Debug, Default)] +struct Wake { + revision: Mutex, + changed: Condvar, +} + +/// Clones share cancellation and an absolute deadline. Child cancellation is +/// isolated; parent cancellation wakes and stops every descendant. +#[derive(Clone, Debug)] +pub struct Budget { + deadline: Instant, + cancellation: Arc, + wake: Arc, +} + +impl Budget { + pub fn new(duration: Duration) -> Self { + let now = Instant::now(); + Self { + // An unrepresentable allocation must fail closed, not be unbounded. + deadline: now.checked_add(duration).unwrap_or(now), + cancellation: Arc::new(Cancellation { + cancelled: AtomicBool::new(false), + parent: None, + }), + wake: Arc::new(Wake::default()), + } + } + + pub fn child(&self, duration: Duration) -> Self { + let now = Instant::now(); + Self { + deadline: self.deadline.min(now.checked_add(duration).unwrap_or(now)), + cancellation: Arc::new(Cancellation { + cancelled: AtomicBool::new(false), + parent: Some(Arc::clone(&self.cancellation)), + }), + wake: Arc::clone(&self.wake), + } + } + + pub fn remaining(&self, predicate: &str) -> Result { + let mut cancellation = Some(self.cancellation.as_ref()); + while let Some(state) = cancellation { + if state.cancelled.load(Ordering::Acquire) { + record_pending(predicate, "cancelled"); + return Err(format!("budget cancelled; pending predicate: {predicate}")); + } + cancellation = state.parent.as_deref(); + } + let remaining = self.deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + record_pending(predicate, "deadline_exhausted"); + return Err(format!("budget exhausted; pending predicate: {predicate}")); + } + Ok(remaining) + } + + pub fn check(&self, predicate: &str) -> Result<(), String> { + self.remaining(predicate).map(|_| ()) + } + + /// Interruptible pacing/reconciliation wait; never extends its owner. + pub fn wait(&self, duration: Duration, predicate: &str) -> Result<(), String> { + let started = Instant::now(); + let target = started.checked_add(duration).unwrap_or(self.deadline); + let mut revision = self + .wake + .revision + .lock() + .unwrap_or_else(|error| error.into_inner()); + let mut wake_count = 0_u64; + let result = loop { + let remaining = match self.remaining(predicate) { + Ok(remaining) => remaining, + Err(error) => break Err(error), + }; + let until_target = target.saturating_duration_since(Instant::now()); + if until_target.is_zero() { + break Ok(()); + } + let (guard, _) = self + .wake + .changed + .wait_timeout(revision, remaining.min(until_target)) + .unwrap_or_else(|error| error.into_inner()); + revision = guard; + wake_count = wake_count.saturating_add(1); + }; + drop(revision); + record_stage( + &format!("wait:{predicate}"), + started.elapsed(), + 0, + 0, + wake_count, + ); + result + } + + pub fn cancel(&self) { + let mut revision = self + .wake + .revision + .lock() + .unwrap_or_else(|error| error.into_inner()); + self.cancellation.cancelled.store(true, Ordering::Release); + *revision = revision.wrapping_add(1); + self.wake.changed.notify_all(); + } +} + +#[derive(Default, Serialize)] +struct StageEvidence { + count: u64, + elapsed_ns: u128, + max_elapsed_ns: u128, + bytes: u64, + records: u64, + wake_count: u64, +} + +#[derive(Default)] +struct ExecutionEvidence { + stages: BTreeMap, + pending: BTreeMap>, + immutable_artifacts: BTreeMap, +} + +static EVIDENCE: LazyLock> = LazyLock::new(Mutex::default); + +fn evidence() -> &'static Mutex { + &EVIDENCE +} + +/// Record runner work without changing the causal workload stdout protocol. +pub fn record_execution_stage(stage: &str, elapsed: Duration, bytes: u64, records: u64) { + record_stage(stage, elapsed, bytes, records, 0); +} + +fn record_stage(stage: &str, elapsed: Duration, bytes: u64, records: u64, wakes: u64) { + let mut evidence = evidence().lock().unwrap_or_else(|error| error.into_inner()); + let entry = evidence.stages.entry(stage.to_owned()).or_default(); + entry.count = entry.count.saturating_add(1); + entry.elapsed_ns = entry.elapsed_ns.saturating_add(elapsed.as_nanos()); + entry.max_elapsed_ns = entry.max_elapsed_ns.max(elapsed.as_nanos()); + entry.bytes = entry.bytes.saturating_add(bytes); + entry.records = entry.records.saturating_add(records); + entry.wake_count = entry.wake_count.saturating_add(wakes); +} + +fn record_pending(predicate: &str, reason: &str) { + let mut evidence = evidence().lock().unwrap_or_else(|error| error.into_inner()); + let count = evidence + .pending + .entry(predicate.to_owned()) + .or_default() + .entry(reason.to_owned()) + .or_default(); + *count = count.saturating_add(1); +} + +/// Retain verified source/build/image identities alongside timing evidence. +pub(crate) fn record_execution_identity(name: &str, identity: Value) { + evidence() + .lock() + .unwrap_or_else(|error| error.into_inner()) + .immutable_artifacts + .insert(name.to_owned(), identity); +} + +/// A versioned cumulative snapshot persisted at phase and failure boundaries, +/// separately from observations used to establish causal ordering. +pub fn execution_evidence() -> Value { + let evidence = evidence().lock().unwrap_or_else(|error| error.into_inner()); + json!({ + "schema_version": 1, + "clock": "monotonic", + "stages": evidence.stages, + "pending_predicates": evidence.pending, + "immutable_artifacts": evidence.immutable_artifacts, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn child_cancellation_does_not_cancel_parent_or_sibling() { + let parent = Budget::new(Duration::from_secs(5)); + let child = parent.child(Duration::from_secs(2)); + let sibling = parent.child(Duration::from_secs(2)); + child.clone().cancel(); + assert!(child.check("child").is_err()); + parent.check("parent").unwrap(); + sibling.check("sibling").unwrap(); + parent.cancel(); + assert!(sibling.check("parent cancelled").is_err()); + } + + #[test] + fn ancestor_cancellation_interrupts_descendant_wait() { + let parent = Budget::new(Duration::from_secs(30)); + let child = parent.child(Duration::from_secs(30)); + let started = Instant::now(); + let waiter = + std::thread::spawn(move || child.wait(Duration::from_secs(20), "withheld event")); + parent.cancel(); + assert!( + waiter + .join() + .unwrap() + .unwrap_err() + .contains("withheld event") + ); + assert!(started.elapsed() < Duration::from_secs(2)); + } + + #[test] + fn child_never_renews_expired_parent() { + let parent = Budget::new(Duration::ZERO); + let child = parent.child(Duration::from_secs(30)); + assert!( + child + .remaining("nested operation") + .unwrap_err() + .contains("nested operation") + ); + } +} diff --git a/tools/myelin-e2e-fuzz/src/campaign.rs b/tools/myelin-e2e-fuzz/src/campaign.rs new file mode 100644 index 0000000..1da7f45 --- /dev/null +++ b/tools/myelin-e2e-fuzz/src/campaign.rs @@ -0,0 +1,1318 @@ +//! Versioned campaign planning and durable artifact contracts. + +use std::collections::BTreeSet; +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde::{Deserialize, Serialize}; + +use crate::budget::Budget; +use crate::corpus::{ + SEEDED_MODEL_PATH, generated_campaign_cases, generated_campaign_cases_budgeted, +}; +use crate::coverage::CoverageLedger; +use crate::harness::ClusterHarness; +use crate::ir::{ + Action, ActionOp, AttemptPaths, BehaviorCase, CASE_SCHEMA_VERSION, CaseResources, + GENERATOR_VERSION, validate_artifact_id, +}; + +pub const CAMPAIGN_SCHEMA_VERSION: u32 = 2; +pub const ARTIFACT_SCHEMA_VERSION: u32 = 1; +pub const INITIAL_NODE_COUNT: usize = 5; +pub const NORMAL_CASE_COUNT: usize = 128; +pub const RECOVERY_CASE_COUNT: usize = 16; +pub const SURVIVOR_CASE_COUNT: usize = 32; +pub const PREPARATION_DEADLINE_SECS: u64 = 10 * 60; +pub const CLEANUP_DEADLINE_SECS: u64 = provisioning::PAID_CLEANUP_RESERVE_MS / 1_000; +pub const CAMPAIGN_DEADLINE_SECS: u64 = 5 * 60; +pub const WORKLOAD_DEADLINE_SECS: u64 = 285; +pub const DIAGNOSTIC_DEADLINE_SECS: u64 = 10; +pub const LOCAL_CLEANUP_RESERVE_SECS: u64 = 5; + +const MAX_CAMPAIGN_PAYLOAD_BYTES: u64 = 512 * 1024 * 1024; +const MAX_CAMPAIGN_ALLOCATION_BYTES: u64 = 8 * 1024 * 1024 * 1024; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProviderMode { + Mock, + Real, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct CampaignLimits { + pub fixture_lifetime_secs: u64, + pub total_cost_usd: f64, + pub total_hourly_price_usd: f64, + pub case_deadline_secs: u64, + pub max_campaign_payload_bytes: u64, + pub max_campaign_allocation_bytes: u64, + pub max_case_race_states: u32, +} + +impl CampaignLimits { + pub fn validate(&self, paid: bool) -> Result<(), String> { + if self.fixture_lifetime_secs == 0 + || self.case_deadline_secs == 0 + || self.max_campaign_payload_bytes == 0 + || self.max_campaign_allocation_bytes == 0 + || self.max_case_race_states == 0 + { + return Err( + "campaign lifetime, deadlines, and resource bounds must be nonzero".to_owned(), + ); + } + if self.max_campaign_payload_bytes > MAX_CAMPAIGN_PAYLOAD_BYTES + || self.max_campaign_allocation_bytes > MAX_CAMPAIGN_ALLOCATION_BYTES + { + return Err( + "campaign bounds exceed the 512 MiB payload or 8 GiB allocation ceiling".to_owned(), + ); + } + for (name, value) in [ + ("total_cost_usd", self.total_cost_usd), + ("total_hourly_price_usd", self.total_hourly_price_usd), + ] { + if !value.is_finite() || value < 0.0 { + return Err(format!("{name} must be finite and non-negative")); + } + if paid && value == 0.0 { + return Err(format!( + "{name} must be an explicit positive paid-run ceiling" + )); + } + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct OfferPolicy { + pub gpu_model: Option, + pub min_gpu_ram_mb: Option, + pub min_compute_cap: Option, + pub min_reliability: Option, + pub min_download_mbps: Option, + pub min_upload_mbps: Option, + pub max_hourly_price_per_node: Option, + pub blacklist_hosts: Vec, +} + +impl OfferPolicy { + pub fn validate(&self) -> Result<(), String> { + for (name, value) in [ + ("min_reliability", self.min_reliability), + ("min_download_mbps", self.min_download_mbps), + ("min_upload_mbps", self.min_upload_mbps), + ("max_hourly_price_per_node", self.max_hourly_price_per_node), + ] { + if value.is_some_and(|value| !value.is_finite() || value < 0.0) { + return Err(format!("{name} must be finite and non-negative")); + } + } + if self.min_reliability.is_some_and(|value| value > 1.0) { + return Err("min_reliability must not exceed one".to_owned()); + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct CampaignConfig { + pub provider: ProviderMode, + pub seed: u64, + pub runtime_image: String, + pub limits: CampaignLimits, + pub offers: OfferPolicy, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum PersistedFixtureEntry { + Blob { path: String, expected: Vec }, + QuiescentStream { path: String }, +} + +impl PersistedFixtureEntry { + pub fn path(&self) -> &str { + match self { + Self::Blob { path, .. } | Self::QuiescentStream { path } => path, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RecoveryCase { + pub id: String, + pub pre_restart: BehaviorCase, + pub post_restart: BehaviorCase, + pub persisted_entries: Vec, +} + +impl RecoveryCase { + pub fn validate(&self) -> Result<(), String> { + validate_artifact_id(&self.id, "recovery")?; + self.pre_restart.validate()?; + self.post_restart.validate()?; + if self.pre_restart.id == self.post_restart.id + || self.pre_restart.live_nodes != self.post_restart.live_nodes + { + return Err( + "recovery segments require distinct identities and the same exact live set" + .to_owned(), + ); + } + let owned = self.pre_restart.owned_paths(); + let mut persisted = BTreeSet::new(); + let mut blob = false; + let mut stream = false; + for entry in &self.persisted_entries { + if !persisted.insert(entry.path()) + || !owned.contains(entry.path()) + || !self + .post_restart + .read_only_fixture_paths + .contains(entry.path()) + { + return Err("persisted entries must be unique pre-restart-owned, post-restart read-only paths".to_owned()); + } + match entry { + PersistedFixtureEntry::Blob { .. } => blob = true, + PersistedFixtureEntry::QuiescentStream { .. } => stream = true, + } + } + if !blob || !stream { + return Err( + "recovery must identify both persisted blob and quiescent stream entries" + .to_owned(), + ); + } + Ok(()) + } + + pub(crate) fn for_attempt(&self, attempt: u64) -> Self { + let paths = AttemptPaths::new(&[&self.pre_restart, &self.post_restart], attempt); + let mut scoped = Self { + id: self.id.clone(), + pre_restart: self.pre_restart.with_attempt_paths(attempt, Some(&paths)), + post_restart: self.post_restart.with_attempt_paths(attempt, Some(&paths)), + persisted_entries: self.persisted_entries.clone(), + }; + for entry in &mut scoped.persisted_entries { + match entry { + PersistedFixtureEntry::Blob { path, .. } + | PersistedFixtureEntry::QuiescentStream { path } => { + *path = paths.map(path); + } + } + } + scoped + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct CampaignResourcePlan { + pub case_count: usize, + pub workload_segment_count: usize, + pub regression_count: usize, + pub readiness_gate_count: usize, + pub recovery_transition_count: usize, + pub destructive_transition_count: usize, + pub recovery_namespace_snapshot_count: usize, + pub auxiliary_probes_share_workload_budget: bool, + pub workload_budget_secs: u64, + pub diagnostic_budget_secs: u64, + pub fixture_cleanup_budget_secs: u64, + pub action_count: u64, + pub process_count: u64, + pub payload_bytes: u64, + pub allocation_bytes: u64, + pub observation_count: u64, + pub planned_runtime_secs: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct CampaignPlan { + pub schema_version: u32, + pub case_schema_version: u32, + pub generator_version: u32, + pub campaign_id: String, + pub seed: u64, + pub expected_initial_nodes: BTreeSet, + pub first_survivors: BTreeSet, + pub second_survivors: BTreeSet, + pub regressions: Vec, + pub normal: Vec, + pub recovery: Vec, + pub four_node: Vec, + pub three_node: Vec, + pub normal_coverage: CoverageLedger, + pub four_node_coverage: CoverageLedger, + pub three_node_coverage: CoverageLedger, + pub resources: CampaignResourcePlan, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ArtifactEnvelope { + pub artifact_schema_version: u32, + pub kind: String, + pub campaign_id: String, + pub written_unix_ms: u64, + pub payload: T, +} + +impl ArtifactEnvelope { + pub fn new( + kind: impl Into, + campaign_id: impl Into, + payload: T, + ) -> Result { + let campaign_id = campaign_id.into(); + validate_artifact_id(&campaign_id, "campaign")?; + let written_unix_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|error| format!("system clock precedes epoch: {error}"))? + .as_millis() + .try_into() + .map_err(|_| "artifact timestamp exceeded u64".to_owned())?; + Ok(Self { + artifact_schema_version: ARTIFACT_SCHEMA_VERSION, + kind: kind.into(), + campaign_id, + written_unix_ms, + payload, + }) + } + + pub fn validate(&self, expected_kind: &str) -> Result<(), String> { + if self.artifact_schema_version != ARTIFACT_SCHEMA_VERSION { + return Err(format!( + "artifact schema {} is incompatible with supported schema {ARTIFACT_SCHEMA_VERSION}", + self.artifact_schema_version + )); + } + if self.kind != expected_kind { + return Err(format!( + "artifact kind {:?} does not match expected {expected_kind:?}", + self.kind + )); + } + validate_artifact_id(&self.campaign_id, "campaign")?; + Ok(()) + } +} + +impl CampaignPlan { + pub fn build(config: &CampaignConfig, regressions: Vec) -> Result { + Self::build_inner(config, regressions, None) + } + + pub fn build_with_budget( + config: &CampaignConfig, + regressions: Vec, + budget: &Budget, + ) -> Result { + Self::build_inner(config, regressions, Some(budget)) + } + + fn build_inner( + config: &CampaignConfig, + regressions: Vec, + budget: Option<&Budget>, + ) -> Result { + if let Some(budget) = budget { + budget.check("campaign plan generation and admission")?; + } + config + .limits + .validate(config.provider == ProviderMode::Real)?; + config.offers.validate()?; + if config.runtime_image.trim().is_empty() { + return Err("runtime image must not be empty".to_owned()); + } + + let expected_initial_nodes = BTreeSet::from([1, 2, 3, 4, 5]); + let first_survivors = BTreeSet::from([1, 2, 3, 4]); + let second_survivors = BTreeSet::from([1, 2, 3]); + let campaign_id = format!("vastai-e2e-{:016x}", config.seed); + + let normal = generate_exact_cases( + config.seed.rotate_left(7), + NORMAL_CASE_COUNT, + &expected_initial_nodes, + "normal", + config.limits.max_case_race_states, + budget, + )?; + let recovery = generate_recovery_cases( + config.seed.rotate_left(13), + &expected_initial_nodes, + config.limits.max_case_race_states, + budget, + )?; + let four_node = generate_exact_cases( + config.seed.rotate_left(19), + SURVIVOR_CASE_COUNT, + &first_survivors, + "four-node", + config.limits.max_case_race_states, + budget, + )?; + let three_node = generate_exact_cases( + config.seed.rotate_left(29), + SURVIVOR_CASE_COUNT, + &second_survivors, + "three-node", + config.limits.max_case_race_states, + budget, + )?; + + for regression in ®ressions { + if let Some(budget) = budget { + budget.check("validate compatible persisted regression")?; + } + regression.validate()?; + if regression.live_nodes != expected_initial_nodes { + return Err(format!( + "regression {} requires incompatible live set {:?}", + regression.id, regression.live_nodes + )); + } + } + let normal_coverage = planned_coverage(&normal, &expected_initial_nodes, true, budget)?; + let four_node_coverage = planned_coverage(&four_node, &first_survivors, false, budget)?; + let three_node_coverage = planned_coverage(&three_node, &second_survivors, false, budget)?; + + let every_case = regressions + .iter() + .chain(&normal) + .chain( + recovery + .iter() + .flat_map(|case| [&case.pre_restart, &case.post_restart]), + ) + .chain(&four_node) + .chain(&three_node); + let totals = campaign_resources(every_case)?.checked_add(auxiliary_campaign_resources( + &expected_initial_nodes, + &recovery, + )?)?; + let CaseResources { + action_count, + process_count, + payload_bytes, + allocation_bytes, + observation_count, + } = totals; + if payload_bytes > config.limits.max_campaign_payload_bytes { + return Err(format!( + "planned campaign payload {payload_bytes} exceeds bound {}", + config.limits.max_campaign_payload_bytes + )); + } + if allocation_bytes > config.limits.max_campaign_allocation_bytes { + return Err(format!( + "planned campaign allocation {allocation_bytes} exceeds bound {}", + config.limits.max_campaign_allocation_bytes + )); + } + let workload_cases = NORMAL_CASE_COUNT + + RECOVERY_CASE_COUNT + + SURVIVOR_CASE_COUNT + + SURVIVOR_CASE_COUNT + + regressions.len(); + // Segments, probes, transitions and diagnosis share enforced parent + // deadlines. A per-operation ceiling is not a charge per successful case. + let workload_segment_count = workload_cases + RECOVERY_CASE_COUNT; + let fixture_cleanup_budget_secs = if config.provider == ProviderMode::Real { + CLEANUP_DEADLINE_SECS + } else { + LOCAL_CLEANUP_RESERVE_SECS + }; + let planned_runtime_secs = if config.provider == ProviderMode::Real { + PREPARATION_DEADLINE_SECS + CAMPAIGN_DEADLINE_SECS + CLEANUP_DEADLINE_SECS + } else { + CAMPAIGN_DEADLINE_SECS + }; + if planned_runtime_secs > config.limits.fixture_lifetime_secs { + return Err(format!( + "planned campaign requires {planned_runtime_secs}s but fixture lifetime ceiling is {}s", + config.limits.fixture_lifetime_secs + )); + } + + Ok(Self { + schema_version: CAMPAIGN_SCHEMA_VERSION, + case_schema_version: CASE_SCHEMA_VERSION, + generator_version: GENERATOR_VERSION, + campaign_id, + seed: config.seed, + expected_initial_nodes, + first_survivors, + second_survivors, + regressions, + normal, + recovery, + four_node, + three_node, + normal_coverage, + four_node_coverage, + three_node_coverage, + resources: CampaignResourcePlan { + case_count: workload_cases, + workload_segment_count, + regression_count: workload_cases + - NORMAL_CASE_COUNT + - RECOVERY_CASE_COUNT + - 2 * SURVIVOR_CASE_COUNT, + readiness_gate_count: 1, + recovery_transition_count: RECOVERY_CASE_COUNT, + destructive_transition_count: 2, + recovery_namespace_snapshot_count: 2 * RECOVERY_CASE_COUNT, + auxiliary_probes_share_workload_budget: true, + workload_budget_secs: WORKLOAD_DEADLINE_SECS, + diagnostic_budget_secs: DIAGNOSTIC_DEADLINE_SECS, + fixture_cleanup_budget_secs, + action_count, + process_count, + payload_bytes, + allocation_bytes, + observation_count, + planned_runtime_secs, + }, + }) + } + + pub fn validate(&self) -> Result<(), String> { + validate_artifact_id(&self.campaign_id, "campaign")?; + if self.schema_version != CAMPAIGN_SCHEMA_VERSION + || self.case_schema_version != CASE_SCHEMA_VERSION + || self.generator_version != GENERATOR_VERSION + { + return Err("campaign schema or generator version is incompatible".to_owned()); + } + if self.expected_initial_nodes.len() != INITIAL_NODE_COUNT + || self.normal.len() != NORMAL_CASE_COUNT + || self.recovery.len() != RECOVERY_CASE_COUNT + || self.four_node.len() != SURVIVOR_CASE_COUNT + || self.three_node.len() != SURVIVOR_CASE_COUNT + { + return Err("campaign shape differs from the fixed VastAI E2E contract".to_owned()); + } + if self.first_survivors.len() != INITIAL_NODE_COUNT - 1 + || self.second_survivors.len() != INITIAL_NODE_COUNT - 2 + || !self.first_survivors.is_subset(&self.expected_initial_nodes) + || !self.second_survivors.is_subset(&self.first_survivors) + { + return Err( + "each destructive transition must remove exactly one existing node".to_owned(), + ); + } + let logical_count = NORMAL_CASE_COUNT + + RECOVERY_CASE_COUNT + + 2 * SURVIVOR_CASE_COUNT + + self.regressions.len(); + if self.resources.case_count != logical_count + || self.resources.workload_segment_count != logical_count + RECOVERY_CASE_COUNT + || self.resources.regression_count != self.regressions.len() + || self.resources.readiness_gate_count != 1 + || self.resources.recovery_transition_count != RECOVERY_CASE_COUNT + || self.resources.destructive_transition_count != 2 + || self.resources.recovery_namespace_snapshot_count != 2 * RECOVERY_CASE_COUNT + || !self.resources.auxiliary_probes_share_workload_budget + || self.resources.workload_budget_secs != WORKLOAD_DEADLINE_SECS + || self.resources.diagnostic_budget_secs != DIAGNOSTIC_DEADLINE_SECS + { + return Err( + "campaign resource accounting differs from bounded phase contract".to_owned(), + ); + } + validate_phase(&self.normal, &self.expected_initial_nodes)?; + validate_phase(&self.regressions, &self.expected_initial_nodes)?; + validate_phase(&self.four_node, &self.first_survivors)?; + validate_phase(&self.three_node, &self.second_survivors)?; + for (cases, nodes, full, ledger) in [ + ( + &self.normal, + &self.expected_initial_nodes, + true, + &self.normal_coverage, + ), + ( + &self.four_node, + &self.first_survivors, + false, + &self.four_node_coverage, + ), + ( + &self.three_node, + &self.second_survivors, + false, + &self.three_node_coverage, + ), + ] { + ledger.validate_plan(&planned_coverage(cases, nodes, full, None)?)?; + } + for recovery in &self.recovery { + recovery.validate()?; + if recovery.pre_restart.live_nodes != self.expected_initial_nodes + || recovery.post_restart.live_nodes != self.expected_initial_nodes + { + return Err(format!( + "recovery case {} has an incompatible live set", + recovery.id + )); + } + } + let totals = campaign_resources( + self.regressions + .iter() + .chain(&self.normal) + .chain( + self.recovery + .iter() + .flat_map(|case| [&case.pre_restart, &case.post_restart]), + ) + .chain(&self.four_node) + .chain(&self.three_node), + )? + .checked_add(auxiliary_campaign_resources( + &self.expected_initial_nodes, + &self.recovery, + )?)?; + if self.resources.action_count != totals.action_count + || self.resources.process_count != totals.process_count + || self.resources.payload_bytes != totals.payload_bytes + || self.resources.allocation_bytes != totals.allocation_bytes + || self.resources.observation_count != totals.observation_count + { + return Err("campaign resource totals differ from checked case resources".to_owned()); + } + if totals.payload_bytes > MAX_CAMPAIGN_PAYLOAD_BYTES + || totals.allocation_bytes > MAX_CAMPAIGN_ALLOCATION_BYTES + { + return Err("campaign resources exceed the hard admission ceilings".to_owned()); + } + Ok(()) + } + + pub fn admit_selected_hourly_cost( + &self, + limits: &CampaignLimits, + selected_hourly_cost: f64, + ) -> Result<(), String> { + if !selected_hourly_cost.is_finite() || selected_hourly_cost < 0.0 { + return Err("selected hourly cost is invalid".to_owned()); + } + if selected_hourly_cost > limits.total_hourly_price_usd { + return Err(format!( + "selected hourly cost ${selected_hourly_cost:.6} exceeds ceiling ${:.6}", + limits.total_hourly_price_usd + )); + } + let worst_case = selected_hourly_cost * limits.fixture_lifetime_secs as f64 / 3_600.0; + if worst_case > limits.total_cost_usd { + return Err(format!( + "selected worst-case cost ${worst_case:.6} exceeds ceiling ${:.6}", + limits.total_cost_usd + )); + } + Ok(()) + } + + /// Builds a structurally consistent over-budget plan for the scripted + /// pre-provider admission check. The resulting plan is deliberately + /// invalid and must be rejected by [`Self::validate`]. + #[doc(hidden)] + pub fn inject_scripted_resource_overflow(&mut self) -> Result<(), String> { + if [ + &self.normal_coverage, + &self.four_node_coverage, + &self.three_node_coverage, + ] + .into_iter() + .any(|ledger| ledger.identity().is_some()) + { + return Err( + "scripted resource overflow must be injected before coverage is bound".to_owned(), + ); + } + for case in self + .normal + .iter_mut() + .chain(&mut self.four_node) + .chain(&mut self.three_node) + { + saturate_case_allocation(case)?; + } + for recovery in &mut self.recovery { + saturate_case_allocation(&mut recovery.pre_restart)?; + saturate_case_allocation(&mut recovery.post_restart)?; + } + + self.normal_coverage = + planned_coverage(&self.normal, &self.expected_initial_nodes, true, None)?; + self.four_node_coverage = + planned_coverage(&self.four_node, &self.first_survivors, false, None)?; + self.three_node_coverage = + planned_coverage(&self.three_node, &self.second_survivors, false, None)?; + + let totals = campaign_resources( + self.regressions + .iter() + .chain(&self.normal) + .chain( + self.recovery + .iter() + .flat_map(|case| [&case.pre_restart, &case.post_restart]), + ) + .chain(&self.four_node) + .chain(&self.three_node), + )? + .checked_add(auxiliary_campaign_resources( + &self.expected_initial_nodes, + &self.recovery, + )?)?; + self.resources.action_count = totals.action_count; + self.resources.process_count = totals.process_count; + self.resources.payload_bytes = totals.payload_bytes; + self.resources.allocation_bytes = totals.allocation_bytes; + self.resources.observation_count = totals.observation_count; + if totals.allocation_bytes <= MAX_CAMPAIGN_ALLOCATION_BYTES { + return Err( + "scripted resource overflow could not exceed the campaign allocation ceiling" + .to_owned(), + ); + } + Ok(()) + } +} +const MAX_CASE_ALLOCATION_BYTES: u64 = 64 * 1024 * 1024; + +fn saturate_case_allocation(case: &mut BehaviorCase) -> Result<(), String> { + let resources = case.resource_summary()?; + let ceiling = case + .resource_bounds + .max_allocation_bytes + .min(MAX_CASE_ALLOCATION_BYTES); + let additional = ceiling.saturating_sub(resources.allocation_bytes); + if additional == 0 { + return Ok(()); + } + let action = Action::exception( + ActionOp::MappingExportClose { + path: SEEDED_MODEL_PATH.to_owned(), + length: additional, + }, + crate::ir::PythonException::BufferError, + ); + let process_index = if resources.action_count < u64::from(case.resource_bounds.max_actions) + && resources.action_count < 64 + { + case.processes + .first_mut() + .ok_or_else(|| format!("case {} has no process for scripted admission", case.id))? + .actions + .push(action); + 0 + } else { + // Full random-DAG cases always contain a zero-allocation route wait. + // Replacing one keeps the action bound fixed while preserving the + // route's independently witnessed read and write endpoints. + let replacement = case + .processes + .iter() + .enumerate() + .find_map(|(process, program)| { + program + .actions + .iter() + .position(|action| { + matches!( + &action.operation, + ActionOp::Lookup { path, .. } if path == SEEDED_MODEL_PATH + ) + }) + .map(|step| (process, step)) + }) + .or_else(|| { + case.processes + .iter() + .enumerate() + .find_map(|(process, program)| { + program + .actions + .iter() + .position(|action| { + matches!(action.operation, ActionOp::AwaitEntry { .. }) + }) + .map(|step| (process, step)) + }) + }) + .ok_or_else(|| { + format!( + "case {} has no zero-allocation action for scripted admission", + case.id + ) + })?; + case.processes[replacement.0].actions[replacement.1] = action; + replacement.0 + }; + let process = &mut case.processes[process_index]; + process.access.read_prefixes.push("/models".to_owned()); + process.access.read_prefixes.sort(); + process.access.read_prefixes.dedup(); + case.read_only_fixture_paths + .insert(SEEDED_MODEL_PATH.to_owned()); + case.validate() +} + +fn generate_exact_cases( + seed: u64, + count: usize, + live_nodes: &BTreeSet, + phase: &str, + max_race_states: u32, + budget: Option<&Budget>, +) -> Result, String> { + let node_count = u8::try_from(live_nodes.len()) + .map_err(|_| "live-node count exceeds generator range".to_owned())?; + let contiguous = (1..=u64::from(node_count)).collect::>(); + let exact = live_nodes.iter().copied().collect::>(); + let mapping = contiguous.into_iter().zip(exact).collect::>(); + let mut cases = match budget { + Some(budget) => generated_campaign_cases_budgeted(seed, count, node_count, budget)?, + None => generated_campaign_cases(seed, count, node_count)?, + }; + for (index, case) in cases.iter_mut().enumerate() { + if let Some(budget) = budget { + budget.check("map campaign case to exact logical node identities")?; + } + case.id = format!("{phase}-{index:03}-{}", case.id); + case.live_nodes = live_nodes.clone(); + case.resource_bounds.max_race_states = max_race_states; + // Recovery phases share one attempt. Distinct seeds are not an + // identity boundary (e.g. rotating seed zero still gives zero). + let scope = format!("/cases/{phase}-{index:03}"); + let fixtures = &case.read_only_fixture_paths; + let scope_path = |path: &str| { + if fixtures.contains(path) { + path.to_owned() + } else if path == "/cases" { + scope.clone() + } else if let Some(suffix) = path.strip_prefix("/cases/") { + format!("{scope}/{suffix}") + } else { + path.to_owned() + } + }; + for process in &mut case.processes { + let fixture_grants = fixtures + .iter() + .filter(|path| { + process.access.read_prefixes.iter().any(|prefix| { + path.as_str() == prefix + || path + .strip_prefix(prefix.as_str()) + .is_some_and(|rest| rest.starts_with('/')) + }) + }) + .cloned() + .collect::>(); + let old_execution = format!("/runs/{}", process.access.execution_id); + process.access.execution_id = + format!("{phase}-{index:03}-{}", process.access.execution_id); + let new_execution = format!("/runs/{}", process.access.execution_id); + for prefix in process + .access + .read_prefixes + .iter_mut() + .chain(&mut process.access.write_prefixes) + { + *prefix = scope_path(prefix); + if prefix == &old_execution { + *prefix = new_execution.clone(); + } else if let Some(rest) = prefix.strip_prefix(old_execution.as_str()) + && rest.starts_with('/') + { + *prefix = format!("{new_execution}{rest}"); + } + } + process.access.read_prefixes.extend(fixture_grants); + process.access.read_prefixes.sort(); + process.access.read_prefixes.dedup(); + for action in &mut process.actions { + action.operation.map_paths(|path| *path = scope_path(path)); + } + process.logical_node_id = mapping + .iter() + .find_map(|(from, to)| (*from == process.logical_node_id).then_some(*to)) + .ok_or_else(|| format!("case {} references unmapped node", case.id))?; + } + for route in &mut case.routes { + for edge in &mut route.edges { + edge.path = scope_path(&edge.path); + edge.source = mapping + .iter() + .find_map(|(from, to)| (*from == edge.source).then_some(*to)) + .ok_or_else(|| format!("case {} route source is unmapped", case.id))?; + edge.destination = mapping + .iter() + .find_map(|(from, to)| (*from == edge.destination).then_some(*to)) + .ok_or_else(|| format!("case {} route destination is unmapped", case.id))?; + } + for path in &mut route.join_inputs { + *path = scope_path(path); + } + } + if let crate::ir::FailureInjection::SlowProcess { + parked_path, + release_path, + .. + } = &mut case.failure + { + *parked_path = scope_path(parked_path); + *release_path = scope_path(release_path); + } + case.validate()?; + } + Ok(cases) +} + +fn generate_recovery_cases( + seed: u64, + live_nodes: &BTreeSet, + max_race_states: u32, + budget: Option<&Budget>, +) -> Result, String> { + let before = generate_exact_cases( + seed, + RECOVERY_CASE_COUNT, + live_nodes, + "recovery-pre", + max_race_states, + budget, + )?; + let after = generate_exact_cases( + seed.rotate_left(31), + RECOVERY_CASE_COUNT, + live_nodes, + "recovery-post", + max_race_states, + budget, + )?; + before + .into_iter() + .zip(after) + .enumerate() + .map(|(index, (mut pre_restart, mut post_restart))| { + if let Some(budget) = budget { + budget.check("construct exact persisted recovery namespace facts")?; + } + let blob_path = format!("/cases/recovery-pre-{index:03}/persisted-blob"); + let stream_path = format!("/cases/recovery-pre-{index:03}/persisted-stream"); + let blob_payload = vec![u8::try_from(index).unwrap_or(u8::MAX); 1025]; + let stream_payload = vec![u8::try_from(index).unwrap_or(u8::MAX).wrapping_add(1); 4097]; + pre_restart.processes[0] + .actions + .push(Action::ok(ActionOp::PublishBlob { + path: blob_path.clone(), + bytes: blob_payload.clone(), + })); + pre_restart.processes[0] + .actions + .push(Action::ok(ActionOp::Lookup { + path: blob_path.clone(), + expected_kind: "blob".to_owned(), + })); + pre_restart.processes[0] + .actions + .push(Action::ok(ActionOp::ReadBlob { + path: blob_path.clone(), + expected: blob_payload.clone(), + })); + pre_restart.processes[0] + .actions + .push(Action::ok(ActionOp::StreamRoundTrip { + path: stream_path.clone(), + chunks: vec![stream_payload], + })); + pre_restart.processes[0] + .actions + .push(Action::ok(ActionOp::WaitForQuiescent { + path: stream_path.clone(), + })); + post_restart + .read_only_fixture_paths + .extend([blob_path.clone(), stream_path.clone()]); + post_restart.processes[0] + .access + .read_prefixes + .extend([blob_path.clone(), stream_path.clone()]); + // Keep the healthy branch behind the slow process's parked marker. + let probe_start = usize::from(matches!( + post_restart.failure, + crate::ir::FailureInjection::SlowProcess { .. } + )); + post_restart.processes[0].actions.splice( + probe_start..probe_start, + [ + Action::ok(ActionOp::Lookup { + path: blob_path.clone(), + expected_kind: "blob".to_owned(), + }), + Action::ok(ActionOp::ReadBlob { + path: blob_path.clone(), + expected: blob_payload.clone(), + }), + Action::ok(ActionOp::Lookup { + path: stream_path.clone(), + expected_kind: "stream".to_owned(), + }), + Action::ok(ActionOp::WaitForQuiescent { + path: stream_path.clone(), + }), + ], + ); + // Generation reserves four action slots for persistence probes. + // Never widen the healthy-case contract to admit a composition. + for case in [&pre_restart, &post_restart] { + let resources = case.resource_summary()?; + if !(16..=64).contains(&resources.action_count) + || !(2..=20).contains(&resources.process_count) + { + return Err(format!( + "recovery case {} exceeds healthy action/process bounds", + case.id + )); + } + } + pre_restart.validate()?; + post_restart.validate()?; + Ok(RecoveryCase { + id: format!("recovery-{index:02}"), + persisted_entries: vec![ + PersistedFixtureEntry::Blob { + path: blob_path, + expected: blob_payload, + }, + PersistedFixtureEntry::QuiescentStream { path: stream_path }, + ], + pre_restart, + post_restart, + }) + }) + .collect() +} +fn planned_coverage( + cases: &[BehaviorCase], + live_nodes: &BTreeSet, + full: bool, + budget: Option<&Budget>, +) -> Result { + let mut ledger = if full { + CoverageLedger::five_node(live_nodes.clone())? + } else { + CoverageLedger::survivor(live_nodes.clone())? + }; + for case in cases { + if let Some(budget) = budget { + budget.check("close planned campaign coverage")?; + } + ledger.plan_case(case)?; + } + ledger.assert_planned_closed()?; + Ok(ledger) +} + +fn validate_phase(cases: &[BehaviorCase], live_nodes: &BTreeSet) -> Result<(), String> { + for case in cases { + case.validate()?; + if &case.live_nodes != live_nodes { + return Err(format!( + "case {} requires {:?}, expected {live_nodes:?}", + case.id, case.live_nodes + )); + } + } + Ok(()) +} + +fn auxiliary_campaign_resources( + initial_nodes: &BTreeSet, + recovery: &[RecoveryCase], +) -> Result { + let nodes = initial_nodes.iter().copied().collect::>(); + // Use the executable constructors: nonce/attempt identities change paths, + // never their requested payload, buffer, endpoint, or observation counts. + let mut total = ClusterHarness::convergence_case(&nodes, 0)?.resource_summary()?; + for case in recovery { + for absent_paths in [BTreeSet::new(), case.post_restart.owned_paths()] { + let probe = ClusterHarness::persistence_probe_case( + &case.pre_restart, + &case.persisted_entries, + &nodes, + &absent_paths, + 0, + )?; + total = total.checked_add(probe.resource_summary()?)?; + } + } + Ok(total) +} + +fn campaign_resources<'a>( + cases: impl IntoIterator, +) -> Result { + cases + .into_iter() + .try_fold(CaseResources::default(), |total, case| { + total.checked_add(case.resource_summary()?) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn config(provider: ProviderMode) -> CampaignConfig { + CampaignConfig { + provider, + seed: 7, + runtime_image: "example.invalid/myelin-e2e:fixture".to_owned(), + limits: CampaignLimits { + fixture_lifetime_secs: 24 * 60 * 60, + total_cost_usd: 10.0, + total_hourly_price_usd: 2.0, + case_deadline_secs: 30, + max_campaign_payload_bytes: 512 * 1024 * 1024, + max_campaign_allocation_bytes: 8 * 1024 * 1024 * 1024, + max_case_race_states: 256, + }, + offers: OfferPolicy { + gpu_model: None, + min_gpu_ram_mb: None, + min_compute_cap: None, + min_reliability: Some(0.95), + min_download_mbps: None, + min_upload_mbps: None, + max_hourly_price_per_node: Some(0.4), + blacklist_hosts: Vec::new(), + }, + } + } + + #[test] + fn full_campaign_is_generated_before_provider_access() { + let plan = CampaignPlan::build(&config(ProviderMode::Real), Vec::new()).unwrap(); + plan.validate().unwrap(); + assert_eq!(plan.resources.case_count, 208); + assert_eq!(plan.resources.workload_segment_count, 224); + assert_eq!(plan.normal.len(), 128); + assert_eq!(plan.recovery.len(), 16); + assert_eq!(plan.four_node.len(), 32); + assert_eq!(plan.three_node.len(), 32); + let mut workload_allocation = 0_u64; + for case in plan + .normal + .iter() + .chain(&plan.four_node) + .chain(&plan.three_node) + .chain( + plan.recovery + .iter() + .flat_map(|case| [&case.pre_restart, &case.post_restart]), + ) + { + let resources = case.resource_summary().unwrap(); + assert!((16..=64).contains(&resources.action_count), "{}", case.id); + assert!((2..=20).contains(&resources.process_count), "{}", case.id); + workload_allocation += resources.allocation_bytes; + } + let readiness = ClusterHarness::convergence_case(&[1, 2, 3, 4, 5], 0) + .unwrap() + .resource_summary() + .unwrap(); + let mut without_snapshots = config(ProviderMode::Real); + without_snapshots.limits.max_campaign_allocation_bytes = + workload_allocation + readiness.allocation_bytes; + assert!(CampaignPlan::build(&without_snapshots, Vec::new()).is_err()); + } + + #[test] + fn campaign_resource_ceilings_cannot_be_relaxed() { + let mut limits = config(ProviderMode::Real).limits; + limits.validate(true).unwrap(); + limits.max_campaign_payload_bytes += 1; + assert!(limits.validate(true).is_err()); + limits.max_campaign_payload_bytes = MAX_CAMPAIGN_PAYLOAD_BYTES; + limits.max_campaign_allocation_bytes += 1; + assert!(limits.validate(true).is_err()); + } + + #[test] + fn role_dags_preserve_noncontiguous_survivor_membership() { + let live_nodes = BTreeSet::from([2, 7, 11]); + let cases = generate_exact_cases(0, 32, &live_nodes, "survivor", 256, None).unwrap(); + planned_coverage(&cases, &live_nodes, false, None).unwrap(); + for case in &cases { + assert_eq!(case.live_nodes, live_nodes); + for route in &case.routes { + for edge in &route.edges { + for (role, node) in [ + (&edge.source_role, edge.source), + (&edge.destination_role, edge.destination), + ] { + let owner = case + .processes + .iter() + .find(|process| process.id == *role) + .unwrap(); + assert!(live_nodes.contains(&node)); + assert_eq!(owner.logical_node_id, node); + } + } + } + } + } + + #[test] + fn paid_plan_rejects_missing_cost_ceiling() { + let mut invalid = config(ProviderMode::Real); + invalid.limits.total_cost_usd = 0.0; + assert!( + CampaignPlan::build(&invalid, Vec::new()) + .unwrap_err() + .contains("ceiling") + ); + } + + #[test] + fn selected_cost_is_admitted_conservatively() { + let config = config(ProviderMode::Real); + let plan = CampaignPlan::build(&config, Vec::new()).unwrap(); + assert!(plan.admit_selected_hourly_cost(&config.limits, 0.4).is_ok()); + assert!( + plan.admit_selected_hourly_cost(&config.limits, 2.1) + .is_err() + ); + } + + #[test] + fn artifact_envelope_rejects_wrong_kind_and_schema() { + let mut artifact = ArtifactEnvelope::new("campaign", "run", 7_u64).unwrap(); + artifact.validate("campaign").unwrap(); + artifact.kind = "other".to_owned(); + assert!(artifact.validate("campaign").is_err()); + artifact.kind = "campaign".to_owned(); + artifact.artifact_schema_version += 1; + assert!(artifact.validate("campaign").is_err()); + } + + #[test] + fn campaign_totals_include_roundtrip_retry_and_requested_allocations() { + let mut case = crate::corpus::stable_corpus(2, 1).remove(0); + case.processes.truncate(1); + case.processes[0].actions = vec![ + Action::ok(ActionOp::StreamRoundTrip { + path: "/cases/resources/roundtrip".to_owned(), + chunks: vec![vec![1, 2, 3]], + }), + Action::ok(ActionOp::StreamReadWithRetry { + path: "/cases/resources/retry".to_owned(), + expected: vec![4, 5], + }), + ]; + let totals = campaign_resources([&case, &case]).unwrap(); + assert_eq!(totals.payload_bytes, 10); + assert_eq!(totals.allocation_bytes, 2 * (61 + 7 * 256 * 1024)); + assert_eq!(totals.action_count, 4); + assert_eq!(totals.process_count, 2); + } + + #[test] + fn recovery_attempts_share_owned_persistence_but_not_external_fixtures() { + let blob = "/cases/recovery/persisted"; + let stream = "/cases/recovery/quiescent"; + let external = "/cases/external/fixture"; + let mut pre = crate::corpus::stable_corpus(2, 1).remove(0); + pre.processes.truncate(1); + pre.processes[0].actions = vec![ + Action::ok(ActionOp::PublishBlob { + path: blob.to_owned(), + bytes: vec![1], + }), + Action::ok(ActionOp::StreamRoundTrip { + path: stream.to_owned(), + chunks: vec![vec![2]], + }), + ]; + let mut post = pre.clone(); + post.id = "post-restart".to_owned(); + post.read_only_fixture_paths = + BTreeSet::from([blob.to_owned(), stream.to_owned(), external.to_owned()]); + post.processes[0].actions = vec![ + Action::ok(ActionOp::ReadBlob { + path: blob.to_owned(), + expected: vec![1], + }), + Action::ok(ActionOp::WaitForQuiescent { + path: stream.to_owned(), + }), + Action::ok(ActionOp::ReadBlob { + path: external.to_owned(), + expected: vec![3], + }), + ]; + let recovery = RecoveryCase { + id: "recovery".to_owned(), + pre_restart: pre, + post_restart: post, + persisted_entries: vec![ + PersistedFixtureEntry::Blob { + path: blob.to_owned(), + expected: vec![1], + }, + PersistedFixtureEntry::QuiescentStream { + path: stream.to_owned(), + }, + ], + }; + let first = recovery.for_attempt(7); + let second = recovery.for_attempt(8); + first.pre_restart.validate().unwrap(); + first.post_restart.validate().unwrap(); + for (step, entry) in first.persisted_entries.iter().enumerate() { + assert_eq!( + entry.path(), + first.pre_restart.processes[0].actions[step] + .operation + .path() + ); + assert_eq!( + entry.path(), + first.post_restart.processes[0].actions[step] + .operation + .path() + ); + assert!( + first + .post_restart + .read_only_fixture_paths + .contains(entry.path()) + ); + assert!(!first.post_restart.owned_paths().contains(entry.path())); + } + assert!( + first + .pre_restart + .owned_paths() + .is_disjoint(&second.pre_restart.owned_paths()) + ); + assert_eq!( + first.post_restart.processes[0].actions[2].operation.path(), + external + ); + assert!(first.post_restart.owned_paths().is_empty()); + } +} diff --git a/tools/myelin-e2e-fuzz/src/codegen.rs b/tools/myelin-e2e-fuzz/src/codegen.rs index f4e94ec..11a6432 100644 --- a/tools/myelin-e2e-fuzz/src/codegen.rs +++ b/tools/myelin-e2e-fuzz/src/codegen.rs @@ -1,61 +1,627 @@ //! Rendering of typed action IR into executable Python programs. -use std::collections::BTreeSet; use std::fmt::Write as _; +use std::time::Duration; use sha2::{Digest as _, Sha256}; use crate::ir::{ - Action, ActionOp, BehaviorCase, DescriptorFinish, DescriptorReadMethod, DescriptorWriteMethod, - ExpectedOutcome, FailureInjection, LaunchFailureKind, ProcessProgram, + Action, ActionClass, ActionOp, BARRIER_LAP_COMPLETED, BARRIER_TOKEN_FORWARDED, + BARRIER_TOKEN_RECEIVED, BehaviorCase, DescriptorFinish, DescriptorReadMethod, + DescriptorWriteMethod, EVIDENCE_HINT_BARRIER, EVIDENCE_HINT_EDGE_INDEX, EVIDENCE_HINT_LAP, + EVIDENCE_HINT_TOKEN, ExpectedOutcome, FailureInjection, LaunchFailureKind, ProcessProgram, + TopologyFamily, }; -/// Wall-clock budget for generated in-process namespace barriers. -/// -/// Generated Python waits for externally observable state (entry publication, -/// stream quiescence, unlink quiescence) by polling the namespace predicate -/// until this deadline; expiry fails the action instead of spinning a fixed -/// attempt count. +/// Per-action ceiling, subordinate to the process owner's remaining budget. const NAMESPACE_BARRIER_DEADLINE_SECONDS: f64 = 30.0; +// The public binding has no predicate subscription yet. Poll immediately, then +// pace adaptively; cancelling the actual binding future propagates into its Rust +// request owner. The harness also stops/reaps the contextual process on expiry, +// including a misbehaving coroutine that refuses cancellation. +const BUDGET_PYTHON: &str = r#" +def remaining(deadline, predicate): + value = deadline - asyncio.get_running_loop().time() + if value <= 0: + raise TimeoutError(f'budget expired: {predicate}') + return value + +def consume_cancelled(task): + if not task.cancelled(): + task.exception() + +async def bounded(operation, deadline, predicate): + task = asyncio.ensure_future(operation) + try: + done, _ = await asyncio.wait((task,), timeout=remaining(deadline, predicate)) + if not done: + raise TimeoutError(f'budget expired: {predicate}') + remaining(deadline, predicate) + return task.result() + finally: + if not task.done(): + task.cancel() + task.add_done_callback(consume_cancelled) + else: + consume_cancelled(task) + +async def pace(deadline, predicate, delay, cap=0.01): + await asyncio.sleep(min(delay, remaining(deadline, predicate))) + remaining(deadline, predicate) + return min(delay * 2, cap) + +def timing(predicate, started, requests, wakes, pending, kind='generated_wait'): + # Successful action output already carries the typed behavioral record. + # Emit a second stderr record only for waits or a failed/timed-out action. + if kind == 'generated_action' and not pending: + return + record = {'type': kind, 'predicate': predicate, + 'elapsed_seconds': asyncio.get_running_loop().time() - started, + 'pending': pending} + if requests is not None: + record.update(request_count=requests, wake_count=wakes) + print(json.dumps(record, separators=(',', ':')), file=sys.stderr) +async def wait_entry(data, path, deadline, quiescent=False): + predicate = f'{"quiescence" if quiescent else "publication"}: {path}' + started = asyncio.get_running_loop().time() + delay, requests, wakes, pending = 0.001, 0, 0, True + try: + while True: + requests += 1 + try: + entry = await bounded(data.lookup(path), deadline, predicate) + except OSError as error: + if quiescent or error.errno != errno.ENOENT: + raise + else: + if not quiescent: + pending = False + return entry + if entry.kind != 'stream': + raise RuntimeError(f'expected stream namespace node, observed {entry.kind}') + if not entry.active: + pending = False + return entry + delay = await pace(deadline, predicate, delay, 0.1 if quiescent else 0.01) + wakes += 1 + finally: + timing(predicate, started, requests, wakes, pending) +"#; + +const TRANSFER_PYTHON: &str = r#" +class Transfer: + def __init__(self): + self.length = 0 + self.hasher = hashlib.sha256() + self.complete = False + + def facts(self): + return {'length': self.length, 'digest': self.hasher.hexdigest(), 'complete': self.complete} + +def transferred(previous, payload, complete=False): + evidence = previous if previous is not None else Transfer() + evidence.length += len(payload) + evidence.hasher.update(payload) + evidence.complete = complete + return evidence + +def transfer_facts(evidence): + return None if evidence is None else evidence.facts() + +async def read_blob_retry(data, path, deadline): + predicate = f'blob transfer: {path}' + while True: + attempt_deadline = min(deadline, asyncio.get_running_loop().time() + 3) + try: + return await bounded(data.read_blob(path), attempt_deadline, predicate) + except TimeoutError: + remaining(deadline, predicate) + +async def observe_drop_release(data, path, deadline): + delay = 0.001 + while True: + try: + await bounded(data.lookup(path), deadline, 'dropped writer nonpublication') + except OSError as error: + if error.errno != errno.ENOENT: + raise + else: + raise RuntimeError('dropped writer published a blob') + try: + probe = await bounded(data.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_TRUNC, length=0), deadline, 'dropped writer reservation release') + except OSError as error: + if error.errno != errno.EEXIST: + raise + delay = await pace(deadline, 'dropped writer reservation release', delay) + else: + await bounded(probe.abort(), deadline, 'release reservation probe') + return +"#; + +// The decoder owns only a fixed header and one validated logical payload. +// Transport reads may split or coalesce either; their boundaries are immaterial. +const FRAMING_PYTHON: &str = r#" +class FramedWriter: + def __init__(self, writer, limit, step, path, **hints): + self.writer, self.limit = writer, limit + self.step, self.path, self.hints = step, path, hints + self.index = 0 + + async def write(self, payload): + length = len(payload) + if length > self.limit or self.index >= STREAM_FRAME_LIMIT: + raise RuntimeError('logical stream frame exceeds generated bounds') + await self.writer.write(struct.pack(' capacity: + raise RuntimeError('invalid stream readinto count') + if count == 0: + return position + else: + while not self.pending: + self.pending.release() + chunk = await self.reader.read() + if chunk is None: + self.pending = memoryview(b'') + return position + self.pending = memoryview(chunk) + del chunk + count = min(len(view) - position, len(self.pending)) + view[position:position + count] = self.pending[:count] + previous = self.pending + self.pending = previous[count:] + previous.release() + position += count + return position + + async def read(self, record=True): + count = await self.fill(self.header) + if count == 0: + return None + if count != len(self.header): + raise RuntimeError('truncated logical stream frame header') + index, length = struct.unpack('= STREAM_FRAME_LIMIT or length > self.remaining: + raise RuntimeError('logical stream frame exceeds generated bounds') + payload = bytearray(length) + if await self.fill(payload) != length: + raise RuntimeError('truncated logical stream frame payload') + if record: + emit(self.step, 'stream_read', self.path, 'barrier', + barrier={'type': 'stream_frame', 'incarnation': self.reader.incarnation, + 'index': index, 'length': length, 'digest': digest(payload)}, + **self.hints) + self.index += 1 + self.remaining -= length + return payload +"#; + +// Endpoint-owner tasks keep the public write context alive across preparation +// and transfer. Reader ownership is released by dropping the public reader; +// the binding deliberately does not expose a reader close method. +const ROUTE_PYTHON: &str = r#" +async def route_bounded(operation, deadline, predicate): + task = asyncio.ensure_future(operation) + try: + done, _ = await asyncio.wait((task,), timeout=remaining(deadline, predicate)) + if not done: + raise TimeoutError(f'budget expired: {predicate}') + remaining(deadline, predicate) + return task.result() + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + +class RouteEndpoint: + def __init__(self, spec): + self.spec = spec + loop = asyncio.get_running_loop() + self.opened = loop.create_future() + self.finished = loop.create_future() + self.handle = None + self.task = None + self.origin = None + self.incarnation = None + + async def opened_handle(self, handle): + try: + self.handle = handle + self.incarnation = handle.incarnation + spec = self.spec + emit(spec['step'], spec['action'], spec['path'], 'barrier', + barrier={'type': 'stream_opened', 'incarnation': handle.incarnation}, + **spec['hints']) + self.opened.set_result(None) + await self.finished + finally: + handle = None + + async def own(self, data, deadline): + spec = self.spec + try: + if spec['write']: + if spec['replace']: + self.origin = await mutation_origin(data, spec['path'], deadline) + async with data.write_stream(spec['path'], replace=spec['replace']) as writer: + await self.opened_handle(writer) + else: + delay = 0.001 + while True: + try: + reader = await data.read_stream(spec['path']) + break + except (OSError, swactor.SessionError, swactor.StreamError): + if not spec['retry']: + raise + delay = await pace(deadline, 'stream reattachment', delay) + try: + await self.opened_handle(reader) + finally: + del reader + except BaseException as error: + if not self.opened.done(): + self.opened.set_exception(error) + raise + finally: + self.handle = None + +class PreparedRoutes: + def __init__(self, specs): + self.endpoints = {spec['step']: RouteEndpoint(spec) for spec in specs} + self.received = {} + + async def prepare(self, data, deadline): + for endpoint in self.endpoints.values(): + endpoint.task = asyncio.create_task(endpoint.own(data, deadline)) + try: + await route_bounded( + asyncio.gather(*(endpoint.opened for endpoint in self.endpoints.values())), + deadline, 'route endpoint preparation') + except BaseException: + await self.close() + raise + + async def finish(self, step): + endpoint = self.endpoints[step] + endpoint.finished.set_result(None) + await endpoint.task + endpoint.task = None + + async def close(self): + tasks = [] + for endpoint in self.endpoints.values(): + if endpoint.task is not None: + if not endpoint.task.done() and not endpoint.task.cancelling(): + endpoint.task.cancel() + tasks.append(endpoint.task) + await asyncio.gather(*tasks, return_exceptions=True) + for endpoint in self.endpoints.values(): + endpoint.handle = None + endpoint.task = None + self.received.clear() + self.endpoints.clear() + + def retain(self, step, payload, transfer, expected_length, expected_digest): + if (transfer.length != expected_length + or transfer.hasher.hexdigest() != expected_digest + or len(payload) != expected_length): + raise RuntimeError('route read length or digest mismatch') + self.received[step] = payload + + def output(self, steps, rotation, expected_length): + length = sum(len(self.received[step]) for step in steps) + if length != expected_length: + raise RuntimeError('route relay length mismatch') + if len(steps) == 1 and rotation == 0: + return self.received[steps[0]] + payload = bytearray(length) + position = (-rotation) % length if length else 0 + for step in steps: + value = memoryview(self.received[step]) + split = min(len(value), length - position) + payload[position:position + split] = value[:split] + payload[:len(value) - split] = value[split:] + position = (position + len(value)) % length if length else 0 + value.release() + return payload +"#; + +struct RouteAction { + inputs: Vec, + rotation: usize, +} + +fn route_read_expected(operation: &ActionOp) -> Option<&[u8]> { + match operation { + ActionOp::ReadBlob { expected, .. } + | ActionOp::StreamRead { expected, .. } + | ActionOp::StreamReadWithRetry { expected, .. } + | ActionOp::GatedStreamRead { expected, .. } + | ActionOp::StreamReadInto { expected, .. } => Some(expected), + _ => None, + } +} + +fn route_actions(case: &BehaviorCase, program: &ProcessProgram) -> Vec> { + program + .actions + .iter() + .enumerate() + .map(|(step, action)| { + let is_read = route_read_expected(&action.operation).is_some(); + if !is_read + && !matches!( + &action.operation, + ActionOp::PublishBlob { .. } + | ActionOp::StreamWrite { .. } + | ActionOp::GatedStreamWrite { .. } + ) + { + return None; + } + let (route, edge_index) = case.routes.iter().find_map(|route| { + route + .edges + .iter() + .position(|edge| { + edge.path == action.operation.path() + && if is_read { + edge.destination_role == program.id + } else { + edge.source_role == program.id + } + }) + .map(|edge_index| (route, edge_index)) + })?; + let mut inputs = Vec::new(); + let mut rotation = 0; + if !is_read { + if case.topology == TopologyFamily::RingWalk && route.edges.len() > 1 { + if edge_index != 0 { + inputs.push( + program.actions[..step] + .iter() + .rposition(|prior| { + route_read_expected(&prior.operation).is_some() + && route + .edges + .iter() + .any(|edge| edge.path == prior.operation.path()) + }) + .expect("ring relay requires a preceding same-process route read"), + ); + } + } else { + let source = &route.edges[edge_index].source_role; + for incoming in route + .edges + .iter() + .filter(|edge| &edge.destination_role == source) + { + inputs.push( + program.actions[..step] + .iter() + .rposition(|prior| { + route_read_expected(&prior.operation).is_some() + && prior.operation.path() == incoming.path + }) + .expect( + "DAG relay requires every preceding same-process input read", + ), + ); + } + if !inputs.is_empty() { + rotation = edge_index + 1; + } + } + } + Some(RouteAction { inputs, rotation }) + }) + .collect() +} + +fn render_route_specs( + source: &mut String, + program: &ProcessProgram, + routes: &[Option], +) { + let specs = program + .actions + .iter() + .zip(routes) + .enumerate() + .filter_map(|(step, (action, route))| { + route.as_ref()?; + let (write, replace, retry) = match &action.operation { + ActionOp::StreamWrite { replace, .. } => (true, *replace, false), + ActionOp::GatedStreamWrite { .. } => (true, false, false), + ActionOp::StreamReadWithRetry { .. } => (false, false, true), + ActionOp::StreamRead { .. } + | ActionOp::GatedStreamRead { .. } + | ActionOp::StreamReadInto { .. } => (false, false, false), + _ => return None, + }; + let plan = EvidencePlan::resolve(action); + let mut hints = serde_json::Map::new(); + if let Some(token) = plan.token { + hints.insert("token".to_owned(), token.into()); + } + if let Some(lap) = plan.lap { + hints.insert("lap".to_owned(), lap.into()); + } + Some(serde_json::json!({ + "step": step, "action": action.operation.class().as_str(), + "path": action.operation.path(), "write": write, "replace": replace, + "retry": retry, "hints": hints, + })) + }) + .collect::>(); + let encoded = serde_json::to_string(&specs).expect("serialize route endpoint specifications"); + let _ = writeln!( + source, + "\nROUTE_ENDPOINTS = json.loads({})", + python_string(&encoded) + ); +} + pub fn render_python(program: &ProcessProgram) -> String { + render_program(program, Duration::from_secs(120), None) +} + +fn render_program( + program: &ProcessProgram, + budget: Duration, + case: Option<&BehaviorCase>, +) -> String { + let routes = case.map(|case| route_actions(case, program)); + // Route-local blob waits must not prevent this role from opening its + // independent stream endpoints. Keep unrelated primitive prefixes ahead + // of preparation, and leave every transfer and rendezvous barrier intact. + let first_route = routes.as_ref().and_then(|routes| { + program + .actions + .iter() + .zip(routes) + .position(|(action, route)| { + route.is_some() + || matches!(&action.operation, ActionOp::AwaitEntry { path, .. } + if case.is_some_and(|case| case.routes.iter().any(|route| + route.edges.iter().any(|edge| + edge.destination_role == program.id && edge.path == *path)))) + }) + }); + let route_arguments = if first_route.is_some() { + ", routes" + } else { + "" + }; let mut source = String::new(); - source - .push_str("import asyncio\nimport errno\nimport hashlib\nimport json\nimport swactor\n\n"); - source.push_str(&format!("PROCESS = {}\n", python_string(&program.id))); source.push_str( - "def emit(step, action, path, outcome, **facts):\n record = {'process': PROCESS, 'step': step, 'action': action, 'path': path, 'outcome': outcome}\n record.update(facts)\n print(json.dumps(record, sort_keys=True), flush=True)\n\ndef digest(data):\n return hashlib.sha256(data).hexdigest()\n\nasync def main(ctx):\n data = ctx.data\n", + "import asyncio\nimport errno\nimport faulthandler\nimport gc\nimport hashlib\nimport json\nimport os\nimport struct\nimport sys\n\nfaulthandler.enable(all_threads=True)\n\nimport swactor\n\n", + ); + let _ = writeln!(source, "PROCESS = {}", python_string(&program.id)); + source.push_str(BUDGET_PYTHON); + source.push_str(TRANSFER_PYTHON); + source.push_str(FRAMING_PYTHON); + let _ = writeln!( + source, + "\nSTREAM_FRAME_LIMIT = {}", + case.map_or( + crate::ir::MAX_STREAM_FRAMES, + BehaviorCase::stream_frame_limit + ) + ); + if first_route.is_some() { + source.push_str(ROUTE_PYTHON); + render_route_specs( + &mut source, + program, + routes.as_ref().expect("route context"), + ); + } + source.push_str( + "\ndef emit(step, action, path, outcome, **facts):\n record = {'process': PROCESS, 'step': step, 'action': action, 'path': path, 'outcome': outcome}\n record.update(facts)\n # The controller releases stream rendezvous at open and injects stops after\n # first writer/reader frames. Flush those milestones; terminal pipe draining\n # preserves every other record.\n barrier = facts.get('barrier') or {}\n barrier_type = barrier.get('type')\n control_milestone = (outcome == 'barrier' and\n (barrier_type in ('stream_opened', 'stream_first_frame') or\n (barrier_type == 'stream_frame' and barrier.get('index') == 0)))\n print(json.dumps(record, separators=(',', ':')), flush=control_milestone)\n\ndef digest(data):\n return hashlib.sha256(data).hexdigest()\n\nasync def mutation_origin(data, path, deadline):\n try:\n entry = await bounded(data.lookup(path), deadline, f'mutation origin: {path}')\n except OSError:\n return None\n return entry.revision\n", ); for (step, action) in program.actions.iter().enumerate() { - render_action(&mut source, step, action); + let _ = writeln!( + source, + "\nasync def action_{step}(data, deadline{route_arguments}):" + ); + render_action( + &mut source, + step, + action, + routes.as_ref().and_then(|routes| routes[step].as_ref()), + first_route == Some(step), + ); + } + source.push_str( + "\nasync def main(ctx):\n data = ctx.data\n loop = asyncio.get_running_loop()\n", + ); + let _ = writeln!( + source, + " process_deadline = loop.time() + {:?}", + budget.as_secs_f64() + ); + if first_route.is_some() { + source.push_str(" routes = PreparedRoutes(ROUTE_ENDPOINTS)\n try:\n"); + } + let indent = if first_route.is_some() { " " } else { "" }; + let bound = if first_route.is_some() { + "route_bounded" + } else { + "bounded" + }; + for (step, action) in program.actions.iter().enumerate() { + let _ = writeln!( + source, + "{indent} deadline = min(process_deadline, loop.time() + {NAMESPACE_BARRIER_DEADLINE_SECONDS})" + ); + let predicate = python_string(&format!( + "{} step {step} {}: {}", + program.id, + action.operation.class().as_str(), + action.operation.path() + )); + let _ = writeln!( + source, + "{indent} started = loop.time()\n{indent} completed = False\n{indent} try:\n{indent} await {bound}(action_{step}(data, deadline{route_arguments}), deadline, {predicate})\n{indent} completed = True\n{indent} finally:\n{indent} timing({predicate}, started, None, None, not completed, kind='generated_action')" + ); + } + if first_route.is_some() { + source.push_str(" finally:\n await routes.close()\n gc.collect()\n"); } source.push_str("\nswactor.run(main)\n"); source } -pub(crate) fn case_cleanup_paths(case: &BehaviorCase) -> Vec { - let mut paths = BTreeSet::new(); - for process in &case.processes { - for action in &process.actions { - let mut retain = |path: &str| { - let path = if let Some(suffix) = path.strip_prefix("/runs/self") { - format!("/runs/{}{}", process.access.execution_id, suffix) - } else { - path.to_owned() - }; - if path.starts_with("/cases/") || path.starts_with("/runs/") { - paths.insert(path); - } - }; - retain(action.operation.path()); - if let ActionOp::Rename { destination, .. } = &action.operation { - retain(destination); - } - } +fn render_frames(source: &mut String, frames: &[Vec]) { + source.push_str(" frames = [\n"); + for frame in frames { + let _ = writeln!( + source, + " bytes.fromhex({}),", + python_string(&hex(frame)) + ); } - paths.into_iter().collect() + source.push_str(" ]\n"); } -pub fn render_case_python(case: &BehaviorCase, program: &ProcessProgram) -> String { + +pub fn render_case_python( + case: &BehaviorCase, + program: &ProcessProgram, + budget: Duration, +) -> String { match &case.failure { FailureInjection::LaunchFailure { process, @@ -65,270 +631,855 @@ pub fn render_case_python(case: &BehaviorCase, program: &ProcessProgram) -> Stri process, kind: LaunchFailureKind::PythonRuntime, } if process == &program.id => "import swactor\n\nasync def main(ctx):\n raise RuntimeError('injected Python runtime failure')\n\nswactor.run(main)\n".to_owned(), - _ => render_python(program), + _ => render_program(program, budget, Some(case)), } } -pub(crate) fn render_cleanup_python(paths: &[String]) -> String { - let paths = serde_json::to_string(paths).expect("serialize cleanup paths"); - format!( - "import asyncio\nimport errno\nimport swactor\n\nPATHS = {paths}\n\nasync def main(ctx):\n loop = asyncio.get_running_loop()\n for path in PATHS:\n deadline = loop.time() + {NAMESPACE_BARRIER_DEADLINE_SECONDS}\n while True:\n try:\n await ctx.data.unlink(path)\n except OSError as error:\n if error.errno == errno.ENOENT:\n break\n if error.errno != errno.ENXIO:\n raise\n if loop.time() >= deadline:\n raise RuntimeError(f'namespace path did not become quiescent: {{path}}')\n await asyncio.sleep(0)\n else:\n break\n try:\n await ctx.data.lookup(path)\n except OSError as error:\n if error.errno != errno.ENOENT:\n raise\n else:\n raise AssertionError(f'cleanup left namespace path visible: {{path}}')\n\nswactor.run(main)\n" - ) +#[cfg(test)] +pub(crate) fn render_absence_probe_python( + process: &str, + paths: &[String], + budget: Duration, +) -> String { + let mut source = String::from("import asyncio\nimport errno\nimport json\nimport swactor\n"); + let _ = writeln!( + source, + "\nPROCESS = {}\nPATHS = {}\nPROBE_SECONDS = {:?}", + python_string(process), + serde_json::to_string(paths).expect("serialize absence-probe paths"), + budget.as_secs_f64() + ); + source.push_str(BUDGET_PYTHON); + source.push_str( + r#" +async def main(ctx): + loop = asyncio.get_running_loop() + deadline = loop.time() + PROBE_SECONDS + + async def probe(step, path): + try: + await bounded(ctx.data.lookup(path), deadline, f'absence lookup: {path}') + except BaseException as error: + if isinstance(error, OSError) and error.errno == errno.ENOENT: + return {'process': PROCESS, 'step': step, 'action': 'lookup', + 'path': path, 'outcome': 'expected_error', + 'errno': error.errno, 'error_type': type(error).__name__, + 'error': f'{type(error).__name__}: {error}'} + if isinstance(error, asyncio.CancelledError): + error = TimeoutError('absence probe budget expired or owner cancelled') + return {'process': PROCESS, 'step': step, 'action': 'lookup', + 'path': path, 'outcome': 'error', + 'errno': getattr(error, 'errno', None), + 'error_type': type(error).__name__, + 'error': f'{type(error).__name__}: {error}'} + return {'process': PROCESS, 'step': step, 'action': 'lookup', + 'path': path, 'outcome': 'error', 'errno': None, + 'error_type': 'PathExists', + 'error': f'PathExists: expected absence for {path}'} + + records = await asyncio.gather(*(probe(step, path) for step, path in enumerate(PATHS))) + for record in records: + print(json.dumps(record, separators=(',', ':'))) + failed = [record['path'] for record in records if record['outcome'] != 'expected_error'] + if failed: + raise RuntimeError(f'absence unproved: {failed}') + +swactor.run(main) +"#, + ); + source } -fn render_action(source: &mut String, step: usize, action: &Action) { +/// Evidence hint resolution for one rendered action. +/// +/// Hints are opaque corpus-authored labels; codegen threads the known keys +/// into emit calls without interpreting their scenario semantics. +struct EvidencePlan { + token: Option, + lap: Option, + edge_index: Option, + barrier: Option<&'static str>, +} + +impl EvidencePlan { + fn resolve(action: &Action) -> Self { + let hints = &action.evidence_hints; + let barrier = hints + .get(EVIDENCE_HINT_BARRIER) + .and_then(|value| match value.as_str() { + BARRIER_TOKEN_RECEIVED => Some(BARRIER_TOKEN_RECEIVED), + BARRIER_TOKEN_FORWARDED => Some(BARRIER_TOKEN_FORWARDED), + BARRIER_LAP_COMPLETED => Some(BARRIER_LAP_COMPLETED), + _ => None, + }); + Self { + token: hints.get(EVIDENCE_HINT_TOKEN).cloned(), + lap: hints + .get(EVIDENCE_HINT_LAP) + .and_then(|value| value.parse().ok()), + edge_index: hints + .get(EVIDENCE_HINT_EDGE_INDEX) + .and_then(|value| value.parse().ok()), + barrier, + } + } + + /// Top-level emit facts (`token=`/`lap=`) applied to ok records and + /// barrier records of this action. + fn facts(&self) -> String { + let mut facts = String::new(); + if let Some(token) = &self.token { + facts.push_str(", token="); + facts.push_str(&python_string(token)); + } + if let Some(lap) = self.lap { + let _ = write!(facts, ", lap={lap}"); + } + facts + } +} + +fn render_route_payload(source: &mut String, route: &RouteAction, bytes: &[u8]) { + if route.inputs.is_empty() { + let _ = writeln!( + source, + " payload = bytes.fromhex({})", + python_string(&hex(bytes)) + ); + } else { + let _ = writeln!( + source, + " payload = routes.output({:?}, {}, {})", + route.inputs, + route.rotation, + bytes.len() + ); + } +} + +fn render_route_frames(source: &mut String, route: &RouteAction, frames: &[Vec]) { + if route.inputs.is_empty() { + render_frames(source, frames); + } else { + let length = frames.iter().map(Vec::len).sum::(); + let _ = writeln!( + source, + " payload = routes.output({:?}, {}, {length})", + route.inputs, route.rotation + ); + let mut offset = 0; + source.push_str(" frame_ranges = [\n"); + for frame in frames { + let end = offset + frame.len(); + let _ = writeln!(source, " ({offset}, {end}),"); + offset = end; + } + source.push_str( + " ]\n frames = (memoryview(payload)[start:end] for start, end in frame_ranges)\n", + ); + } +} + +fn render_route_retain(source: &mut String, step: usize, expected: &[u8]) { + let _ = writeln!( + source, + " routes.retain({step}, payload, transfer, {}, {})", + expected.len(), + python_string(&digest(expected)) + ); +} + +fn render_stream_read_loop( + source: &mut String, + step: usize, + action: &Action, + length: usize, + hint_facts: &str, + indent: &str, + retain: bool, + first_already_read: bool, +) { let class = action.operation.class().as_str(); let path = python_string(action.operation.path()); - let _ = writeln!(source, " try:"); - match &action.operation { - ActionOp::PublishBlob { bytes, .. } => { + let buffers = match &action.operation { + ActionOp::StreamReadInto { buffer_sizes, .. } => { + serde_json::to_string(buffer_sizes).expect("serialize readinto sizes") + } + _ => "None".to_owned(), + }; + if first_already_read { + let _ = writeln!( + source, + "{indent}emit({step}, {class:?}, {path}, 'barrier', barrier={{'type': 'stream_frame', 'incarnation': incarnation_{step}, 'index': 0, 'length': len(first), 'digest': digest(first)}}{hint_facts})\n{indent}transfer = transferred(transfer, first)" + ); + if let ActionOp::GatedStreamRead { observed_path, .. } = &action.operation { + let observed_path = python_string(observed_path); let _ = writeln!( source, - " payload = bytes.fromhex({})", - python_string(&hex(bytes)) + "{indent}async with data.write_blob({observed_path}, length=0) as observed_writer:\n{indent} with observed_writer.map() as mapped:\n{indent} view = memoryview(mapped)\n{indent} view.release()" ); + } + let _ = writeln!( + source, + "{indent}emit({step}, {class:?}, {path}, 'barrier', barrier={{'type': 'stream_first_frame', 'incarnation': incarnation_{step}}}{hint_facts})" + ); + if matches!( + action.operation, + ActionOp::GatedStreamRead { + park_after_first_frame: true, + .. + } + ) { + let _ = writeln!(source, "{indent}await asyncio.Future()"); + } + let _ = writeln!(source, "{indent}del first"); + } else { + let _ = writeln!( + source, + "{indent}framed = FramedReader(reader, {length}, {step}, {path}, buffer_sizes={buffers}{hint_facts})" + ); + } + let _ = writeln!( + source, + "{indent}while True:\n{indent} chunk = await framed.read()\n{indent} if chunk is None:" + ); + if matches!(&action.operation, ActionOp::GatedStreamRead { .. }) { + let _ = writeln!( + source, + "{indent} if framed.index == 0:\n{indent} raise RuntimeError('hot stream ended before its first frame')" + ); + } + let _ = writeln!( + source, + "{indent} transfer = transferred(transfer, b'', complete=True)\n{indent} emit({step}, {class:?}, {path}, 'barrier', barrier={{'type': 'stream_eof', 'incarnation': incarnation_{step}}}{hint_facts})\n{indent} break\n{indent} transfer = transferred(transfer, chunk)\n{indent} if framed.index == 1:" + ); + if let ActionOp::GatedStreamRead { observed_path, .. } = &action.operation { + let observed_path = python_string(observed_path); + let _ = writeln!( + source, + "{indent} async with data.write_blob({observed_path}, length=0) as observed_writer:\n{indent} with observed_writer.map() as mapped:\n{indent} view = memoryview(mapped)\n{indent} view.release()" + ); + } + let _ = writeln!( + source, + "{indent} emit({step}, {class:?}, {path}, 'barrier', barrier={{'type': 'stream_first_frame', 'incarnation': incarnation_{step}}}{hint_facts})" + ); + if matches!( + action.operation, + ActionOp::GatedStreamRead { + park_after_first_frame: true, + .. + } + ) { + // Keep the public endpoint attached until the causally triggered + // stop cancels this task; never race EOF against controller relay. + let _ = writeln!(source, "{indent} await asyncio.Future()"); + } + if retain { + let _ = writeln!( + source, + "{indent} payload[transfer.length - len(chunk):transfer.length] = chunk" + ); + } + let _ = writeln!(source, "{indent} del chunk\n{indent}del framed"); +} + +fn render_route_operation( + source: &mut String, + step: usize, + action: &Action, + route: &RouteAction, + hint_facts: &str, + ok_facts: &str, +) { + let class = action.operation.class().as_str(); + let path = python_string(action.operation.path()); + match &action.operation { + ActionOp::PublishBlob { bytes, .. } => { + render_route_payload(source, route, bytes); let _ = writeln!( source, " async with data.write_blob({path}, length=len(payload)) as writer:" ); source.push_str( - " with writer.map() as mapped:\n view = memoryview(mapped)\n view[:] = payload\n view.release()\n", - ); - let _ = writeln!( - source, - " emit({step}, {class:?}, {path}, 'ok', length=len(payload), digest=digest(payload))" + " with writer.map() as mapped:\n view = memoryview(mapped)\n view[:] = payload\n view.release()\n transfer = transferred(transfer, payload, complete=True)\n", ); } - ActionOp::ReadBlob { .. } => { - let _ = writeln!(source, " blob = await data.read_blob({path})"); - source.push_str( - " with blob.map() as mapped:\n payload = bytes(mapped)\n", - ); + ActionOp::ReadBlob { expected, .. } => { let _ = writeln!( source, - " emit({step}, {class:?}, {path}, 'ok', length=len(payload), digest=digest(payload))" + " blob = await read_blob_retry(data, {path}, deadline)\n with blob.map() as mapped:\n with memoryview(mapped) as view:\n if len(view) != {}:\n raise RuntimeError('route blob length mismatch')\n payload = bytes(view)\n transfer = transferred(transfer, payload, complete=True)", + expected.len() ); + render_route_retain(source, step, expected); } + // Stream adapters are shared with non-route actions; prepared handles + // change endpoint ownership, never the wire or payload semantics. ActionOp::StreamWrite { chunks, replace, .. } => { + render_route_frames(source, route, chunks); let _ = writeln!( source, - " async with data.write_stream({path}, replace={}) as writer:", - if *replace { "True" } else { "False" } + " writer = routes.endpoints[{step}].handle\n incarnation_{step} = writer.incarnation\n framed = FramedWriter(writer, {}, {step}, {path}{hint_facts})\n try:\n for frame in frames:\n await framed.write(frame)\n transfer = transferred(transfer, frame)\n transfer = transferred(transfer, b'', complete=True)\n del framed\n await routes.finish({step})\n finally:\n writer = None", + chunks.iter().map(Vec::len).max().unwrap_or(0) ); - for chunk in chunks { + if *replace { let _ = writeln!( source, - " await writer.write(bytes.fromhex({}))", - python_string(&hex(chunk)) + " emit({step}, {class:?}, {path}, 'barrier', barrier={{'type': 'mutation_applied', 'from_revision': routes.endpoints[{step}].origin, 'to_revision': incarnation_{step}}}{hint_facts})" ); } - let payload = chunks.concat(); - let _ = writeln!( - source, - " payload = bytes.fromhex({})", - python_string(&hex(&payload)) - ); - let _ = writeln!( - source, - " emit({step}, {class:?}, {path}, 'ok', length=len(payload), digest=digest(payload))" - ); } - ActionOp::StreamRead { .. } => { - let _ = writeln!(source, " reader = await data.read_stream({path})"); - source.push_str( - " chunks = []\n while True:\n chunk = await reader.read()\n if chunk is None:\n break\n chunks.append(bytes(chunk))\n payload = b''.join(chunks)\n del reader\n", - ); - let _ = writeln!( - source, - " emit({step}, {class:?}, {path}, 'ok', length=len(payload), digest=digest(payload))" - ); - } - ActionOp::StreamReadInto { buffer_sizes, .. } => { - let sizes = serde_json::to_string(buffer_sizes).expect("serialize readinto sizes"); - let _ = writeln!(source, " reader = await data.read_stream({path})"); - let _ = writeln!(source, " buffer_sizes = {sizes}"); - source.push_str( - " chunks = []\n read_index = 0\n while True:\n target = bytearray(buffer_sizes[read_index % len(buffer_sizes)])\n count = await reader.readinto(target)\n if count == 0:\n break\n chunks.append(bytes(target[:count]))\n read_index += 1\n payload = b''.join(chunks)\n del reader\n", - ); - let _ = writeln!( - source, - " emit({step}, {class:?}, {path}, 'ok', length=len(payload), digest=digest(payload))" - ); - } - ActionOp::Lookup { .. } => { - let _ = writeln!(source, " entry = await data.lookup({path})"); - let _ = writeln!( - source, - " emit({step}, {class:?}, {path}, 'ok', kind=entry.kind, revision=entry.revision, active=entry.active)" - ); - } - ActionOp::AwaitEntry { .. } => { - source.push_str( - " loop = asyncio.get_running_loop()\n deadline = loop.time() + ", - ); - let _ = writeln!(source, "{NAMESPACE_BARRIER_DEADLINE_SECONDS}"); - source.push_str( - " while True:\n try:\n entry = await data.lookup(", - ); - let _ = writeln!(source, "{path})"); - source.push_str( - " except OSError as error:\n if error.errno != errno.ENOENT:\n raise\n if loop.time() >= deadline:\n raise TimeoutError('namespace barrier did not resolve') from error\n await asyncio.sleep(0)\n", - ); - let _ = writeln!( - source, - " emit({step}, {class:?}, {path}, 'ok', kind=entry.kind, revision=entry.revision, active=entry.active)" - ); - } - ActionOp::WaitForQuiescent { .. } => { - source.push_str( - " loop = asyncio.get_running_loop()\n deadline = loop.time() + ", - ); - let _ = writeln!(source, "{NAMESPACE_BARRIER_DEADLINE_SECONDS}"); - source.push_str(" while True:\n entry = await data.lookup("); - let _ = writeln!(source, "{path})"); - source.push_str( - " if entry.kind != 'stream':\n raise RuntimeError(f'expected stream namespace node, observed {entry.kind}')\n if not entry.active:\n break\n if loop.time() >= deadline:\n raise TimeoutError('stream namespace node did not become quiescent')\n await asyncio.sleep(0)\n", - ); - let _ = writeln!( - source, - " emit({step}, {class:?}, {path}, 'ok', kind=entry.kind, revision=entry.revision, active=entry.active)" - ); - } - ActionOp::Rename { - destination, - replace, + ActionOp::GatedStreamWrite { + frames, + release_path, .. } => { - let destination = python_string(destination); + render_route_frames(source, route, frames); + let release_path = python_string(release_path); let _ = writeln!( source, - " revision = await data.rename({path}, {destination}, replace={})", - if *replace { "True" } else { "False" } + " writer = routes.endpoints[{step}].handle\n incarnation_{step} = writer.incarnation\n framed = FramedWriter(writer, {}, {step}, {path}{hint_facts})\n try:", + frames.iter().map(Vec::len).max().unwrap_or(0) ); let _ = writeln!( source, - " emit({step}, {class:?}, {path}, 'ok', revision=revision)" + " frames = iter(frames)\n first = next(frames)\n await framed.write(first)\n transfer = transferred(transfer, first)\n await wait_entry(data, {release_path}, deadline)\n emit({step}, {class:?}, {path}, 'barrier', barrier={{'type': 'release_observed', 'path': {release_path}}}{hint_facts})\n for frame in frames:\n await framed.write(frame)\n transfer = transferred(transfer, frame)\n transfer = transferred(transfer, b'', complete=True)\n del framed\n await routes.finish({step})\n finally:\n writer = None" ); } - ActionOp::Unlink { .. } => { - let _ = writeln!(source, " revision = await data.unlink({path})"); + ActionOp::StreamRead { expected, .. } + | ActionOp::StreamReadWithRetry { expected, .. } + | ActionOp::GatedStreamRead { expected, .. } + | ActionOp::StreamReadInto { expected, .. } => { let _ = writeln!( source, - " emit({step}, {class:?}, {path}, 'ok', revision=revision)" + " reader = routes.endpoints[{step}].handle\n incarnation_{step} = reader.incarnation\n payload = bytearray({})\n try:", + expected.len() ); + render_stream_read_loop( + source, + step, + action, + expected.len(), + hint_facts, + " ", + true, + false, + ); + let _ = writeln!( + source, + " await routes.finish({step})\n finally:\n reader = None" + ); + render_route_retain(source, step, expected); } - ActionOp::DescriptorWrite { - flags, - length, - bytes, - method, - finish, - .. - } => { - match length { - Some(length) => { - let _ = writeln!( - source, - " descriptor = await data.open({path}, {flags}, length={length})" - ); - } - None => { - let _ = writeln!( - source, - " descriptor = await data.open({path}, {flags})" - ); - } + _ => unreachable!("route plan contains only route transfer actions"), + } + let _ = writeln!( + source, + " emit({step}, {class:?}, {path}, 'ok', length=transfer.length, digest=transfer.hasher.hexdigest(){ok_facts})" + ); +} + +fn render_action( + source: &mut String, + step: usize, + action: &Action, + route: Option<&RouteAction>, + prepare_routes: bool, +) { + let class = action.operation.class().as_str(); + let path = python_string(action.operation.path()); + let plan = EvidencePlan::resolve(action); + let hint_facts = plan.facts(); + let mut ok_facts = hint_facts.clone(); + let incarnation_facts = if matches!( + action.operation.class(), + ActionClass::StreamWrite | ActionClass::StreamRead + ) { + let _ = writeln!(source, " incarnation_{step} = None"); + format!(", incarnation=incarnation_{step}") + } else { + String::new() + }; + ok_facts.push_str(&incarnation_facts); + source.push_str(" transfer = None\n"); + ok_facts.push_str(", transfer=transfer_facts(transfer)"); + let descriptor_facts = if matches!( + &action.operation, + ActionOp::DescriptorWrite { .. } | ActionOp::DescriptorRead { .. } + ) { + source.push_str(" descriptor_evidence = None\n"); + ", descriptor=descriptor_evidence" + } else { + "" + }; + let _ = writeln!(source, " try:"); + if prepare_routes { + source.push_str(" await routes.prepare(data, deadline)\n"); + } + if let Some(route) = route { + render_route_operation(source, step, action, route, &hint_facts, &ok_facts); + } else { + match &action.operation { + ActionOp::PublishBlob { bytes, .. } => { + let _ = writeln!( + source, + " payload = bytes.fromhex({})", + python_string(&hex(bytes)) + ); + let _ = writeln!( + source, + " async with data.write_blob({path}, length=len(payload)) as writer:" + ); + source.push_str( + " with writer.map() as mapped:\n view = memoryview(mapped)\n view[:] = payload\n view.release()\n transfer = transferred(transfer, payload, complete=True)\n", + ); + let _ = writeln!( + source, + " emit({step}, {class:?}, {path}, 'ok', length=transfer.length, digest=transfer.hasher.hexdigest(){ok_facts})" + ); } - let _ = writeln!( - source, - " payload = bytes.fromhex({})", - python_string(&hex(bytes)) + ActionOp::ReadBlob { .. } => { + let _ = writeln!( + source, + " blob = await read_blob_retry(data, {path}, deadline)" + ); + source.push_str( + " with blob.map() as mapped:\n payload = bytes(mapped)\n transfer = transferred(transfer, payload, complete=True)\n", ); - match method { - DescriptorWriteMethod::Write => { - source.push_str(" await descriptor.write(payload)\n"); - } - DescriptorWriteMethod::WriteFrom => { - source.push_str(" await descriptor.writefrom(payload)\n"); - } - DescriptorWriteMethod::Mapping => { - source.push_str( - " mapping = await descriptor.map(offset=0, length=len(payload), writable=True)\n with mapping as mapped:\n view = memoryview(mapped)\n view[:] = payload\n view.release()\n", - ); - } + let _ = writeln!( + source, + " emit({step}, {class:?}, {path}, 'ok', length=transfer.length, digest=transfer.hasher.hexdigest(){ok_facts})" + ); } - render_descriptor_finish(source, *finish); - let _ = writeln!( - source, - " emit({step}, {class:?}, {path}, 'ok', length=len(payload), digest=digest(payload))" - ); - } - ActionOp::DescriptorRead { - flags, - expected, - method, - offset, - finish, - .. - } => { - let _ = writeln!( - source, - " descriptor = await data.open({path}, {flags})" - ); - match method { - DescriptorReadMethod::Read => { + ActionOp::StreamWrite { + chunks, replace, .. + } => { + if *replace { let _ = writeln!( source, - " payload = await descriptor.read({})", + " from_revision_{step} = await mutation_origin(data, {path}, deadline)" + ); + } + render_frames(source, chunks); + let _ = writeln!( + source, + " async with data.write_stream({path}, replace={}) as writer:", + if *replace { "True" } else { "False" } + ); + let _ = writeln!( + source, + " incarnation_{step} = writer.incarnation" + ); + let _ = writeln!( + source, + " emit({step}, {class:?}, {path}, 'barrier', barrier={{'type': 'stream_opened', 'incarnation': incarnation_{step}}}{hint_facts})" + ); + let _ = writeln!( + source, + " framed = FramedWriter(writer, {}, {step}, {path}{hint_facts})\n for frame in frames:\n await framed.write(frame)\n transfer = transferred(transfer, frame)\n transfer = transferred(transfer, b'', complete=True)\n del framed", + chunks.iter().map(Vec::len).max().unwrap_or(0) + ); + if *replace { + let _ = writeln!( + source, + " emit({step}, {class:?}, {path}, 'barrier', barrier={{'type': 'mutation_applied', 'from_revision': from_revision_{step}, 'to_revision': incarnation_{step}}}{hint_facts})" + ); + } + let _ = writeln!( + source, + " emit({step}, {class:?}, {path}, 'ok', length=transfer.length, digest=transfer.hasher.hexdigest(){ok_facts})" + ); + } + ActionOp::StreamRoundTrip { chunks, .. } => { + let _ = writeln!( + source, + " def roundtrip_opened_{step}(stream):\n nonlocal incarnation_{step}\n attached_incarnation = stream.incarnation\n if incarnation_{step} is None:\n emit({step}, {class:?}, {path}, 'barrier', barrier={{'type': 'stream_opened', 'incarnation': attached_incarnation}}{hint_facts})\n elif incarnation_{step} != attached_incarnation:\n raise RuntimeError('stream round trip incarnation mismatch')\n incarnation_{step} = attached_incarnation\n return attached_incarnation" + ); + render_frames(source, chunks); + let length = chunks.iter().map(Vec::len).sum::(); + let mut expected_digest = Sha256::new(); + for chunk in chunks { + expected_digest.update(chunk); + } + let expected_digest = format!("{:x}", expected_digest.finalize()); + let _ = writeln!(source, " async def roundtrip_writer_{step}():"); + let _ = writeln!( + source, + " async with data.write_stream({path}, replace=False) as writer:" + ); + let _ = writeln!(source, " roundtrip_opened_{step}(writer)"); + let _ = writeln!( + source, + " framed = FramedWriter(writer, {}, {step}, {path}{hint_facts})\n for frame in frames:\n await framed.write(frame)\n del framed", + chunks.iter().map(Vec::len).max().unwrap_or(0) + ); + let _ = writeln!(source, " async def roundtrip_reader_{step}():"); + source.push_str(" nonlocal transfer\n"); + let _ = writeln!( + source, + " reader = await data.read_stream({path})" + ); + let _ = writeln!( + source, + " incarnation_{step} = roundtrip_opened_{step}(reader)" + ); + render_stream_read_loop( + source, + step, + action, + length, + &hint_facts, + " ", + false, + false, + ); + source.push_str(" del reader\n"); + let _ = writeln!( + source, + " await asyncio.gather(roundtrip_writer_{step}(), roundtrip_reader_{step}())" + ); + let _ = writeln!( + source, + " if transfer.length != {length} or transfer.hasher.hexdigest() != '{expected_digest}':\n raise RuntimeError('stream round trip payload mismatch')" + ); + let _ = writeln!( + source, + " emit({step}, {class:?}, {path}, 'ok', length=transfer.length, digest=transfer.hasher.hexdigest(){ok_facts})" + ); + } + ActionOp::GatedStreamWrite { + frames, + release_path, + replace, + .. + } => { + let release_path = python_string(release_path); + if *replace { + let _ = writeln!( + source, + " from_revision_{step} = await mutation_origin(data, {path}, deadline)" + ); + } + render_frames(source, frames); + let _ = writeln!( + source, + " async with data.write_stream({path}, replace={}) as writer:", + if *replace { "True" } else { "False" } + ); + let _ = writeln!( + source, + " incarnation_{step} = writer.incarnation" + ); + let _ = writeln!( + source, + " emit({step}, {class:?}, {path}, 'barrier', barrier={{'type': 'stream_opened', 'incarnation': incarnation_{step}}}{hint_facts})" + ); + let _ = writeln!( + source, + " framed = FramedWriter(writer, {}, {step}, {path}{hint_facts})\n first = frames[0]\n await framed.write(first)\n transfer = transferred(transfer, first)", + frames.iter().map(Vec::len).max().unwrap_or(0) + ); + let _ = writeln!( + source, + " await wait_entry(data, {release_path}, deadline)\n emit({step}, {class:?}, {path}, 'barrier', barrier={{'type': 'release_observed', 'path': {release_path}}}{hint_facts})" + ); + source.push_str( + " for index in range(1, len(frames)):\n await framed.write(frames[index])\n transfer = transferred(transfer, frames[index])\n transfer = transferred(transfer, b'', complete=True)\n del framed\n", + ); + if *replace { + let _ = writeln!( + source, + " emit({step}, {class:?}, {path}, 'barrier', barrier={{'type': 'mutation_applied', 'from_revision': from_revision_{step}, 'to_revision': incarnation_{step}}}{hint_facts})" + ); + } + let _ = writeln!( + source, + " emit({step}, {class:?}, {path}, 'ok', length=transfer.length, digest=transfer.hasher.hexdigest(){ok_facts})" + ); + } + ActionOp::StreamRead { expected, .. } + | ActionOp::StreamReadWithRetry { expected, .. } + | ActionOp::GatedStreamRead { expected, .. } + | ActionOp::StreamReadInto { expected, .. } => { + let retry_gated = matches!( + &action.operation, + ActionOp::GatedStreamRead { + retry_attach: true, + .. + } + ); + if retry_gated { + let expected_payload = python_string(&hex(expected)); + let _ = writeln!( + source, + " expected_payload = bytes.fromhex({expected_payload})\n delay = 0.001\n while True:\n reader = None\n framed = None\n first = None\n try:\n reader = await bounded(data.read_stream({path}), deadline, 'stream reattachment')\n framed = FramedReader(reader, {}, {step}, {path}{hint_facts})\n first = await bounded(framed.read(record=False), deadline, 'replacement first frame')\n if first is not None and expected_payload.startswith(bytes(first)):\n break\n except (OSError, RuntimeError, swactor.SessionError, swactor.StreamError):\n pass\n framed = None\n reader = None\n delay = await pace(deadline, 'stream reattachment', delay)", expected.len() ); - } - DescriptorReadMethod::ReadInto => { + } else if matches!(&action.operation, ActionOp::StreamReadWithRetry { .. }) { let _ = writeln!( source, - " target = bytearray({})\n count = await descriptor.readinto(target)\n payload = bytes(target[:count])", - expected.len() - ); - } - DescriptorReadMethod::Mapping => { - let _ = writeln!( - source, - " mapping = await descriptor.map(offset={offset}, length={}, writable=False)", - expected.len() - ); - source.push_str( - " with mapping as mapped:\n payload = bytes(mapped)\n", + " delay = 0.001\n while True:\n try:\n reader = await bounded(data.read_stream({path}), deadline, 'stream reattachment')\n break\n except (OSError, swactor.SessionError, swactor.StreamError):\n delay = await pace(deadline, 'stream reattachment', delay)" ); + } else { + let _ = writeln!(source, " reader = await data.read_stream({path})"); } + let _ = writeln!(source, " incarnation_{step} = reader.incarnation"); + let _ = writeln!( + source, + " emit({step}, {class:?}, {path}, 'barrier', barrier={{'type': 'stream_opened', 'incarnation': incarnation_{step}}}{hint_facts})" + ); + render_stream_read_loop( + source, + step, + action, + expected.len(), + &hint_facts, + " ", + false, + retry_gated, + ); + source.push_str(" del reader\n"); + let _ = writeln!( + source, + " emit({step}, {class:?}, {path}, 'ok', length=transfer.length, digest=transfer.hasher.hexdigest(){ok_facts})" + ); } - render_descriptor_finish(source, *finish); - let _ = writeln!( - source, - " emit({step}, {class:?}, {path}, 'ok', length=len(payload), digest=digest(payload))" - ); - } - ActionOp::MappingExportClose { length, .. } => { - let _ = writeln!( - source, - " descriptor = await data.open({path}, 0)\n mapping = await descriptor.map(offset=0, length={length}, writable=False)" - ); - source.push_str( + ActionOp::Lookup { .. } => { + let _ = writeln!( + source, + " entry = await bounded(data.lookup({path}), deadline, 'namespace lookup')" + ); + let _ = writeln!( + source, + " emit({step}, {class:?}, {path}, 'ok', kind=entry.kind, revision=entry.revision, active=entry.active{ok_facts})" + ); + } + ActionOp::AwaitEntry { .. } => { + let _ = writeln!( + source, + " entry = await wait_entry(data, {path}, deadline)" + ); + let _ = writeln!( + source, + " emit({step}, {class:?}, {path}, 'ok', kind=entry.kind, revision=entry.revision, active=entry.active{ok_facts})" + ); + } + ActionOp::WaitForQuiescent { .. } => { + let _ = writeln!( + source, + " entry = await wait_entry(data, {path}, deadline, quiescent=True)" + ); + let _ = writeln!( + source, + " emit({step}, {class:?}, {path}, 'ok', kind=entry.kind, revision=entry.revision, active=entry.active{ok_facts})" + ); + } + ActionOp::Rename { + destination, + replace, + .. + } => { + let destination = python_string(destination); + let _ = writeln!( + source, + " from_revision_{step} = await mutation_origin(data, {path}, deadline)" + ); + let _ = writeln!( + source, + " revision = await data.rename({path}, {destination}, replace={})", + if *replace { "True" } else { "False" } + ); + let _ = writeln!( + source, + " emit({step}, {class:?}, {path}, 'barrier', barrier={{'type': 'mutation_applied', 'from_revision': from_revision_{step}, 'to_revision': revision}}{hint_facts})" + ); + let _ = writeln!( + source, + " emit({step}, {class:?}, {path}, 'ok', revision=revision{ok_facts})" + ); + } + ActionOp::Unlink { .. } => { + let _ = writeln!( + source, + " from_revision_{step} = await mutation_origin(data, {path}, deadline)" + ); + let _ = writeln!(source, " revision = await data.unlink({path})"); + let _ = writeln!( + source, + " emit({step}, {class:?}, {path}, 'barrier', barrier={{'type': 'mutation_applied', 'from_revision': from_revision_{step}, 'to_revision': revision}}{hint_facts})" + ); + let _ = writeln!( + source, + " emit({step}, {class:?}, {path}, 'ok', revision=revision{ok_facts})" + ); + } + ActionOp::DescriptorWrite { + flags, + length, + bytes, + method, + finish, + .. + } => { + match length { + Some(length) => { + let _ = writeln!( + source, + " descriptor = await data.open({path}, {flags}, length={length})" + ); + } + None => { + let _ = writeln!( + source, + " descriptor = await data.open({path}, {flags})" + ); + } + } + let _ = writeln!( + source, + " payload = bytes.fromhex({})", + python_string(&hex(bytes)) + ); + match method { + DescriptorWriteMethod::Write => { + source.push_str(" count = await descriptor.write(payload)\n transfer = transferred(transfer, memoryview(payload)[:count], complete=True)\n"); + } + DescriptorWriteMethod::WriteFrom => { + source.push_str(" count = await descriptor.writefrom(payload)\n transfer = transferred(transfer, memoryview(payload)[:count], complete=True)\n"); + } + DescriptorWriteMethod::Mapping => { + source.push_str( + " mapping = await descriptor.map(offset=0, length=len(payload), writable=True)\n with mapping as mapped:\n view = memoryview(mapped)\n view[:] = payload\n transfer = transferred(transfer, view, complete=True)\n view.release()\n", + ); + } + } + let descriptor = serde_json::to_string(&crate::ir::DescriptorObservation::Write { + method: *method, + finish: *finish, + terminal_results: Vec::new(), + dropped: false, + reservation_released: false, + }) + .expect("descriptor evidence is serializable"); + let _ = writeln!( + source, + " descriptor_evidence = json.loads({})", + python_string(&descriptor) + ); + render_descriptor_finish(source, *finish); + if *finish == DescriptorFinish::Drop { + let _ = writeln!( + source, + " descriptor_evidence['dropped'] = True\n await observe_drop_release(data, {path}, deadline)\n descriptor_evidence['reservation_released'] = True" + ); + } + let _ = writeln!( + source, + " emit({step}, {class:?}, {path}, 'ok', length=transfer.length, digest=transfer.hasher.hexdigest(), descriptor=descriptor_evidence{ok_facts})" + ); + } + ActionOp::DescriptorRead { + flags, + expected, + method, + offset, + finish, + .. + } => { + let _ = writeln!( + source, + " descriptor = await data.open({path}, {flags})" + ); + match method { + DescriptorReadMethod::Read => { + let _ = writeln!( + source, + " payload = await descriptor.read({})\n transfer = transferred(transfer, payload, complete=True)", + expected.len() + ); + } + DescriptorReadMethod::ReadInto => { + let _ = writeln!( + source, + " target = bytearray({})\n count = await descriptor.readinto(target)\n with memoryview(target)[:count] as view:\n transfer = transferred(transfer, view, complete=True)", + expected.len() + ); + } + DescriptorReadMethod::Mapping => { + let _ = writeln!( + source, + " mapping = await descriptor.map(offset={offset}, length={}, writable=False)", + expected.len() + ); + source.push_str( + " with mapping as mapped:\n with memoryview(mapped) as view:\n transfer = transferred(transfer, view, complete=True)\n", + ); + } + } + let descriptor = serde_json::to_string(&crate::ir::DescriptorObservation::Read { + method: *method, + finish: *finish, + terminal_results: Vec::new(), + }) + .expect("descriptor evidence is serializable"); + let _ = writeln!( + source, + " descriptor_evidence = json.loads({})", + python_string(&descriptor) + ); + render_descriptor_finish(source, *finish); + let _ = writeln!( + source, + " emit({step}, {class:?}, {path}, 'ok', length=transfer.length, digest=transfer.hasher.hexdigest(), descriptor=descriptor_evidence{ok_facts})" + ); + } + ActionOp::MappingExportClose { length, .. } => { + let _ = writeln!( + source, + " descriptor = await data.open({path}, 0)\n mapping = await descriptor.map(offset=0, length={length}, writable=False)" + ); + source.push_str( " view = memoryview(mapping)\n try:\n mapping.close()\n except BaseException as close_error:\n view.release()\n mapping.close()\n await descriptor.close()\n raise close_error\n view.release()\n await descriptor.close()\n payload = b''\n", ); - let _ = writeln!( - source, - " emit({step}, {class:?}, {path}, 'ok', length=0, digest=digest(payload))" - ); + let _ = writeln!( + source, + " emit({step}, {class:?}, {path}, 'ok', length=0, digest=digest(payload){ok_facts})" + ); + } } } - source.push_str(" except BaseException as error:\n error_number = getattr(error, 'errno', None)\n error_type = type(error).__name__\n"); + if let Some(kind) = plan.barrier { + let token = python_string(plan.token.as_deref().unwrap_or("")); + let lap = plan.lap.unwrap_or(0); + let edge_index = if kind == BARRIER_LAP_COMPLETED { + String::new() + } else { + format!(", 'edge_index': {}", plan.edge_index.unwrap_or(0)) + }; + let _ = writeln!( + source, + " emit({step}, {class:?}, {path}, 'barrier', barrier={{'type': '{kind}', 'token': {token}, 'lap': {lap}{edge_index}}}{hint_facts})" + ); + } + source.push_str(" except BaseException as error:\n if isinstance(error, asyncio.CancelledError):\n error = TimeoutError('action budget expired or owner cancelled')\n error_number = getattr(error, 'errno', None)\n error_type = type(error).__name__\n"); + if route.is_some() + && matches!( + action.operation.class(), + ActionClass::StreamWrite | ActionClass::StreamRead + ) + { + let _ = writeln!( + source, + " incarnation_{step} = routes.endpoints[{step}].incarnation" + ); + } match action.expected { ExpectedOutcome::Ok => { let _ = writeln!( source, - " emit({step}, {class:?}, {path}, 'error', errno=error_number, error_type=error_type, error=f'{{error_type}}: {{error}}')" + " emit({step}, {class:?}, {path}, 'error', errno=error_number, error_type=error_type, error=f'{{error_type}}: {{error}}', transfer=transfer_facts(transfer){descriptor_facts}{incarnation_facts})" ); source.push_str(" raise\n"); } @@ -339,12 +1490,12 @@ fn render_action(source: &mut String, step: usize, action: &Action) { let _ = writeln!(source, " if error_number != {expected}:"); let _ = writeln!( source, - " emit({step}, {class:?}, {path}, 'error', errno=error_number, error_type=error_type, error=f'{{error_type}}: {{error}}')" + " emit({step}, {class:?}, {path}, 'error', errno=error_number, error_type=error_type, error=f'{{error_type}}: {{error}}', transfer=transfer_facts(transfer){descriptor_facts}{incarnation_facts})" ); source.push_str(" raise\n"); let _ = writeln!( source, - " emit({step}, {class:?}, {path}, 'expected_error', errno=error_number, error_type=error_type, error=f'{{error_type}}: {{error}}')" + " emit({step}, {class:?}, {path}, 'expected_error', errno=error_number, error_type=error_type, error=f'{{error_type}}: {{error}}', transfer=transfer_facts(transfer){descriptor_facts}{incarnation_facts})" ); } ExpectedOutcome::Exception(expected) => { @@ -352,30 +1503,33 @@ fn render_action(source: &mut String, step: usize, action: &Action) { let _ = writeln!(source, " if error_type != {expected:?}:"); let _ = writeln!( source, - " emit({step}, {class:?}, {path}, 'error', errno=error_number, error_type=error_type, error=f'{{error_type}}: {{error}}')" + " emit({step}, {class:?}, {path}, 'error', errno=error_number, error_type=error_type, error=f'{{error_type}}: {{error}}', transfer=transfer_facts(transfer){descriptor_facts}{incarnation_facts})" ); source.push_str(" raise\n"); let _ = writeln!( source, - " emit({step}, {class:?}, {path}, 'expected_error', errno=error_number, error_type=error_type, error=f'{{error_type}}: {{error}}')" + " emit({step}, {class:?}, {path}, 'expected_error', errno=error_number, error_type=error_type, error=f'{{error_type}}: {{error}}', transfer=transfer_facts(transfer){descriptor_facts}{incarnation_facts})" ); } } } fn render_descriptor_finish(source: &mut String, finish: DescriptorFinish) { - match finish { - DescriptorFinish::Close => source.push_str(" await descriptor.close()\n"), - DescriptorFinish::Abort => source.push_str(" await descriptor.abort()\n"), + let (operation, count) = match finish { + DescriptorFinish::Close => ("await descriptor.close()", 1), + DescriptorFinish::Abort => ("await descriptor.abort()", 1), DescriptorFinish::Drop => { - source.push_str(" del descriptor\n"); - } - DescriptorFinish::CloseTwice => { - source.push_str(" await descriptor.close()\n await descriptor.close()\n"); - } - DescriptorFinish::AbortTwice => { - source.push_str(" await descriptor.abort()\n await descriptor.abort()\n"); + source.push_str(" del descriptor\n gc.collect()\n"); + return; } + DescriptorFinish::CloseTwice => ("await descriptor.close()", 2), + DescriptorFinish::AbortTwice => ("await descriptor.abort()", 2), + }; + for _ in 0..count { + let _ = writeln!(source, " try:\n {operation}"); + source.push_str( + " except BaseException as terminal_error:\n descriptor_evidence['terminal_results'].append({'outcome': 'error', 'errno': getattr(terminal_error, 'errno', None), 'error_type': type(terminal_error).__name__})\n raise\n descriptor_evidence['terminal_results'].append({'outcome': 'ok'})\n", + ); } } @@ -394,3 +1548,856 @@ fn hex(bytes: &[u8]) -> String { pub(crate) fn digest(bytes: &[u8]) -> String { format!("{:x}", Sha256::digest(bytes)) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::ir::AccessSpec; + use std::process::{Command, Stdio}; + + fn execute_generated(sources: &[String], scenario: &str) -> String { + let runner = format!( + "import asyncio,contextlib,errno,io,json,signal,sys,types\nsignal.alarm(5)\nsources = json.load(sys.stdin)\nswactor = types.ModuleType('swactor')\nswactor.run = lambda main: None\nswactor.SessionError = type('SessionError', (Exception,), {{}})\nswactor.StreamError = type('StreamError', (Exception,), {{}})\nsys.modules['swactor'] = swactor\n{scenario}" + ); + let mut child = Command::new("python3") + .args(["-c", &runner]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn generated execution regression"); + serde_json::to_writer(child.stdin.as_mut().unwrap(), sources).unwrap(); + drop(child.stdin.take()); + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "generated execution failed:\n{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).expect("generated output is UTF-8") + } + + fn generated_observation( + case: &BehaviorCase, + records: Vec>, + ) -> crate::ir::CaseObservation { + assert_eq!(records.len(), case.processes.len()); + crate::ir::CaseObservation { + case_id: case.id.clone(), + executions: case + .processes + .iter() + .zip(records) + .map(|(program, results)| crate::ir::ExecutionObservation { + process: program.id.clone(), + request_id: case.execution_request_id(program), + logical_node_id: program.logical_node_id, + lifecycle: [ + "spawned", + "process_started", + "context_ready", + "user_result", + "exited", + ] + .map(str::to_owned) + .to_vec(), + results, + terminal: true, + exit_success: true, + exit_status: Some(r#"{"kind":"code","value":0}"#.to_owned()), + stdout: String::new(), + stderr: String::new(), + }) + .collect(), + } + } + + #[test] + fn generated_descriptor_read_retains_corruption_before_ebadf() { + let mut case = crate::corpus::stable_corpus(2, 17).remove(0); + case.processes = vec![ProcessProgram { + id: "descriptor-reader".to_owned(), + logical_node_id: 1, + access: AccessSpec::unrestricted("descriptor-reader"), + depends_on: Vec::new(), + actions: vec![Action::error( + ActionOp::DescriptorRead { + path: "/models/fixture".to_owned(), + flags: libc::O_RDONLY, + expected: b"a".to_vec(), + method: DescriptorReadMethod::Read, + offset: 0, + finish: DescriptorFinish::CloseTwice, + }, + libc::EBADF, + )], + }]; + case.read_only_fixture_paths = ["/models/fixture".to_owned()].into(); + let output = execute_generated( + &[render_python(&case.processes[0])], + r#" +class Descriptor: + def __init__(self, payload, fail_read): + self.payload, self.fail_read, self.closed = payload, fail_read, False + async def read(self, count): + if self.fail_read: + raise OSError(errno.EBADF, 'read failed before transfer') + return self.payload + async def close(self): + if self.closed: + raise OSError(errno.EBADF, 'already closed') + self.closed = True + +class Data: + def __init__(self, payload, stage): + self.payload, self.stage = payload, stage + async def open(self, *args, **kwargs): + if self.stage == 'open': + raise OSError(errno.EBADF, 'open failed before transfer') + return Descriptor(self.payload, self.stage == 'read') + +async def scenario(): + all_records = [] + for payload, stage in [(b'a', None), (b'X', None), (b'a', 'open'), (b'a', 'read')]: + namespace, output = {}, io.StringIO() + exec(sources[0], namespace) + with contextlib.redirect_stdout(output): + await namespace['main'](types.SimpleNamespace(data=Data(payload, stage))) + records = [json.loads(line) for line in output.getvalue().splitlines()] + assert records[0]['outcome'] == 'expected_error' + assert records[0]['errno'] == errno.EBADF + if stage: + assert records[0]['transfer'] is None, records + all_records.append(records) + print(json.dumps(all_records)) + +asyncio.run(scenario()) +"#, + ); + let records: Vec> = + serde_json::from_str(&output).unwrap(); + let correct = generated_observation(&case, vec![records[0].clone()]); + crate::oracle::BehaviorOracle::verify(&case, &correct).unwrap(); + let corrupt = generated_observation(&case, vec![records[1].clone()]); + assert!(crate::oracle::BehaviorOracle::verify(&case, &corrupt).is_err()); + assert_eq!( + records[1][0].transfer.as_ref().unwrap().digest, + digest(b"X") + ); + assert!(records[2][0].transfer.is_none()); + assert!(records[3][0].transfer.is_none()); + } + + #[test] + fn generated_writer_drop_proves_nonpublication_release_and_reuse() { + let case = crate::corpus::stable_corpus(2, 17) + .into_iter() + .find(|case| case.id == "descriptor-drop-17") + .unwrap(); + let sources = case.processes.iter().map(render_python).collect::>(); + let output = execute_generated( + &sources, + r#" +class Descriptor: + def __init__(self, data, path): + self.data, self.path, self.payload, self.finished = data, path, b'', False + async def write(self, payload): + self.payload = payload + return len(payload) + async def abort(self): + self.finished = True + self.data.reserved.remove(self.path) + async def close(self): + self.data.blobs[self.path] = self.payload + await self.abort() + def __del__(self): + if self.finished: + return + if self.data.mode == 'publication': + self.data.blobs[self.path] = self.payload + if self.data.mode != 'leak': + self.data.reserved.remove(self.path) + +class Blob: + def __init__(self, payload): + self.payload = payload + def map(self): + return contextlib.nullcontext(self.payload) + +class Data: + def __init__(self, mode): + self.mode, self.reserved, self.blobs = mode, set(), {} + async def open(self, path, flags, length=None): + assert flags & 128, 'reuse must be exclusive' + if path in self.reserved or path in self.blobs: + raise OSError(errno.EEXIST, 'reserved or committed') + self.reserved.add(path) + return Descriptor(self, path) + async def lookup(self, path): + if path not in self.blobs: + raise OSError(errno.ENOENT, 'not committed') + return types.SimpleNamespace(kind='blob', revision=1, active=False) + async def read_blob(self, path): + return Blob(self.blobs[path]) + +async def run(mode): + data, records = Data(mode), [] + for source in sources: + namespace, output = {}, io.StringIO() + exec(source, namespace) + with contextlib.redirect_stdout(output): + try: + await namespace['main'](types.SimpleNamespace(data=data)) + finally: + for _ in range(8): + await asyncio.sleep(0) + records.append([json.loads(line) for line in output.getvalue().splitlines()]) + assert not data.reserved + assert list(data.blobs.values()) == [b'reused'] + return records + +async def scenario(): + valid = await run('release') + for mode in ['leak', 'publication']: + try: + await asyncio.wait_for(run(mode), timeout=0.1) + except (RuntimeError, TimeoutError): + pass + else: + raise AssertionError(f'{mode} received drop credit') + print(json.dumps(valid)) + +asyncio.run(scenario()) +"#, + ); + let records = serde_json::from_str(&output).unwrap(); + let observation = generated_observation(&case, records); + crate::oracle::BehaviorOracle::verify(&case, &observation).unwrap(); + } + + #[test] + fn generated_authorization_scopes_allowed_work_and_independent_rename_denials() { + let case = crate::corpus::stable_corpus(3, 17) + .into_iter() + .find(|case| case.id == "authorization-17") + .unwrap() + .for_attempt(7, true); + case.validate().unwrap(); + let sources = case.processes.iter().map(render_python).collect::>(); + let grants = serde_json::to_string( + &case + .processes + .iter() + .map(|program| &program.access) + .collect::>(), + ) + .unwrap(); + let scenario = format!( + "grants = json.loads({})\nfixture_path = {}\nfixture = bytes.fromhex({})\n{}", + python_string(&grants), + python_string(crate::corpus::SEEDED_MODEL_PATH), + python_string(&hex(crate::corpus::SEEDED_MODEL_BYTES)), + r#" +def permits(prefixes, path): + return any(prefix == '/' or path == prefix or path.startswith(prefix.rstrip('/') + '/') for prefix in prefixes) + +class Blob: + def __init__(self, payload): + self.payload = payload + def map(self): + return contextlib.nullcontext(self.payload) + +class Writer: + def __init__(self, data, path, length): + self.data, self.path, self.payload = data, path, bytearray(length) + async def __aenter__(self): + self.data.require('write_prefixes', self.path) + return self + def map(self): + return contextlib.nullcontext(self.payload) + async def __aexit__(self, error_type, error, traceback): + if error is None: + self.data.store[self.path] = bytes(self.payload) + +class Data: + def __init__(self, grant, store, denials): + self.grant, self.store, self.denials = grant, store, denials + def require(self, kind, path): + if not permits(self.grant[kind], path): + raise OSError(errno.EACCES, 'outside scoped grant') + def write_blob(self, path, length): + return Writer(self, path, length) + async def read_blob(self, path): + self.require('read_prefixes', path) + return Blob(self.store[path]) + async def lookup(self, path): + self.require('read_prefixes', path) + if path not in self.store: + raise OSError(errno.ENOENT, 'absent') + return types.SimpleNamespace(kind='blob', revision=1, active=False) + async def rename(self, source, destination, replace): + assert source in self.store, 'rename must not fail because its source is absent' + source_allowed = permits(self.grant['write_prefixes'], source) + destination_allowed = permits(self.grant['write_prefixes'], destination) + assert source_allowed != destination_allowed, 'exactly one rename direction must be denied' + self.denials.append('destination' if source_allowed else 'source') + self.require('write_prefixes', source) + self.require('write_prefixes', destination) + raise AssertionError('unauthorized rename succeeded') + +async def scenario(): + store, denials, all_records = {fixture_path: fixture}, [], [] + for source, grant in zip(sources, grants): + namespace, output = {}, io.StringIO() + exec(source, namespace) + with contextlib.redirect_stdout(output): + await namespace['main'](types.SimpleNamespace(data=Data(grant, store, denials))) + records = [json.loads(line) for line in output.getvalue().splitlines()] + assert any(record['outcome'] == 'ok' for record in records), records + assert any(record['outcome'] == 'expected_error' and record['errno'] == errno.EACCES for record in records), records + all_records.append(records) + assert denials == ['source', 'destination'], denials + print(json.dumps(all_records)) + +asyncio.run(scenario()) +"# + ); + let records = serde_json::from_str(&execute_generated(&sources, &scenario)).unwrap(); + let observation = generated_observation(&case, records); + crate::oracle::BehaviorOracle::verify(&case, &observation).unwrap(); + let mut ledger = + crate::coverage::CoverageLedger::survivor(case.live_nodes.clone()).unwrap(); + ledger.plan_case(&case).unwrap(); + ledger + .bind_identity(crate::coverage::EvidenceIdentity { + plan_digest: digest(b"authorization-plan"), + artifacts_digest: digest(b"generated-public-binding-test"), + deployment_generation: "authorization-test".to_owned(), + fixture_mapping_digest: digest(b"three-logical-nodes"), + }) + .unwrap(); + ledger.observe_case(&case, &observation).unwrap(); + } + + #[test] + fn generated_fresh_reader_first_waits_for_original_attachment() { + let case = crate::corpus::stable_corpus(2, 17) + .into_iter() + .find(|case| case.id == "active-stream-replacement-17") + .unwrap(); + let sources = case.processes.iter().map(render_python).collect::>(); + let output = execute_generated( + &sources, + r#" +class World: + def __init__(self): + self.marker = asyncio.Event() + self.fresh_release, self.final_release = asyncio.Event(), asyncio.Event() + self.old_attached, self.stale_attached = asyncio.Event(), asyncio.Event() + self.first_written, self.replaced = asyncio.Event(), asyncio.Event() + self.fresh_attached, self.fresh_waiting = asyncio.Event(), asyncio.Event() + self.fresh_attempts = 0 + self.open_order = [] + +class Writer: + def __init__(self, world, replace): + self.world, self.replace = world, replace + self.incarnation = 3 if replace else 1 + self.wire = bytearray() + async def __aenter__(self): + if self.replace: + assert self.world.marker.is_set() + await self.world.stale_attached.wait() + self.world.replaced.set() + await self.world.fresh_attached.wait() + else: + await self.world.old_attached.wait() + return self + async def __aexit__(self, *error): + return False + async def write(self, payload): + if not self.replace: + if self.world.replaced.is_set(): + raise OSError(errno.ESTALE, 'original writer displaced') + expected = b'active-first' + else: + expected = b'replacement' + self.wire.extend(payload) + wire = bytes(8) + len(expected).to_bytes(8, 'little') + expected + assert wire.startswith(self.wire) + if not self.replace and self.wire == wire: + self.world.first_written.set() + +class Reader: + def __init__(self, world, fresh): + self.world, self.fresh, self.reads = world, fresh, 0 + self.incarnation = 3 if fresh else 1 + async def read(self): + self.reads += 1 + if self.fresh: + return bytes(8) + (11).to_bytes(8, 'little') + b'replacement' if self.reads == 1 else None + if self.reads == 1: + await self.world.first_written.wait() + return bytes(8) + (12).to_bytes(8, 'little') + b'active-first' + await self.world.replaced.wait() + raise OSError(errno.ESTALE, 'original reader displaced') + +class MarkerWriter: + def __init__(self, marker): + self.marker = marker + async def __aenter__(self): + return self + def map(self): + return contextlib.nullcontext(bytearray()) + async def __aexit__(self, error_type, error, traceback): + if error is None: + self.marker.set() + +class Data: + def __init__(self, world, role): + self.world, self.role = world, role + async def lookup(self, path): + if path.endswith('/reader-attached'): + if self.role == 3: + self.world.fresh_waiting.set() + marker, revision = self.world.marker, 2 + elif path.endswith('/fresh-reader-attached'): + marker, revision = self.world.fresh_release, 4 + elif path.endswith('/release'): + marker, revision = self.world.final_release, 5 + else: + return types.SimpleNamespace(kind='stream', revision=3 if self.world.replaced.is_set() else 1, active=True) + if not marker.is_set(): + raise OSError(errno.ENOENT, 'marker not published') + return types.SimpleNamespace(kind='blob', revision=revision, active=False) + def write_blob(self, path, length): + assert length == 0 + if path.endswith('/reader-attached'): + marker = self.world.marker + elif path.endswith('/fresh-reader-attached'): + marker = self.world.fresh_release + else: + marker = self.world.final_release + return MarkerWriter(marker) + def write_stream(self, path, replace): + return Writer(self.world, replace) + async def read_stream(self, path): + if self.role == 3: + self.world.fresh_attempts += 1 + if self.world.fresh_attempts == 1: + self.world.open_order.append('stale') + self.world.stale_attached.set() + return Reader(self.world, False) + await self.world.replaced.wait() + self.world.open_order.append('replacement') + self.world.fresh_attached.set() + return Reader(self.world, True) + self.world.open_order.append('original') + self.world.old_attached.set() + return Reader(self.world, False) + +async def scenario(): + world, output, namespaces = World(), io.StringIO(), [] + for source in sources: + namespace = {} + exec(source, namespace) + namespaces.append(namespace) + with contextlib.redirect_stdout(output): + fresh = asyncio.create_task(namespaces[3]['main'](types.SimpleNamespace(data=Data(world, 3)))) + await world.fresh_waiting.wait() + assert world.open_order == [], 'fresh reader attempted attachment before original marker' + others = [asyncio.create_task(namespaces[index]['main'](types.SimpleNamespace(data=Data(world, index)))) + for index in range(3)] + await asyncio.wait_for(asyncio.gather(fresh, *others), timeout=1) + assert world.open_order == ['original', 'stale', 'replacement'], world.open_order + records = [json.loads(line) for line in output.getvalue().splitlines()] + print(json.dumps([[record for record in records if record['process'] == namespace['PROCESS']] + for namespace in namespaces])) + +asyncio.run(scenario()) +"#, + ); + let records: Vec> = + serde_json::from_str(&output).unwrap(); + for index in [0, 1] { + let final_record = records[index] + .iter() + .find(|record| record.outcome == "expected_error") + .unwrap(); + assert_eq!(final_record.errno, Some(libc::ESTALE)); + let prefix = final_record.transfer.as_ref().unwrap(); + assert!(!prefix.complete); + assert_eq!(prefix.digest, digest(b"active-first")); + } + let observation = generated_observation(&case, records); + crate::oracle::BehaviorOracle::verify(&case, &observation).unwrap(); + } + + #[test] + fn generated_topology_routes_finish_under_adverse_open_orders() { + use crate::coverage::{CoverageKey, CoverageLedger, EvidenceIdentity}; + use crate::ir::{CoverageScenario, DataKind, TopologyFamily}; + for nodes in 3..=5 { + let cases = crate::corpus::generated_campaign_cases(17, 32, nodes).unwrap(); + for family in [ + TopologyFamily::Chain, + TopologyFamily::RingWalk, + TopologyFamily::FanIn, + TopologyFamily::FanOut, + TopologyFamily::Diamond, + TopologyFamily::RandomDag, + ] { + if nodes == 3 && family == TopologyFamily::Diamond { + continue; + } + let mut case = cases + .iter() + .find(|case| { + case.topology == family + && case.routes.iter().any(|route| { + route.id.contains("-topology-") && route.kind == DataKind::Stream + }) + }) + .unwrap() + .clone(); + case.routes.retain(|route| route.id.contains("-topology-")); + let paths = case + .routes + .iter() + .flat_map(|route| route.edges.iter().map(|edge| edge.path.clone())) + .collect::>(); + for process in &mut case.processes { + process + .actions + .retain(|action| paths.contains(action.operation.path())); + } + case.processes.retain(|process| !process.actions.is_empty()); + case.scenarios = + std::collections::BTreeSet::from([CoverageScenario::ConcurrentStartup]); + if family == TopologyFamily::RingWalk { + case.scenarios.insert(CoverageScenario::RingCompletion); + } + case.failure = FailureInjection::None; + case.validate().unwrap(); + let sources = case + .processes + .iter() + .map(|process| render_case_python(&case, process, Duration::from_secs(120))) + .collect::>(); + let readers_first = case + .processes + .iter() + .enumerate() + .filter(|(_, process)| { + process.actions[0].operation.class() == ActionClass::StreamRead + }) + .map(|(index, _)| index) + .chain( + case.processes + .iter() + .enumerate() + .filter(|(_, process)| { + process.actions[0].operation.class() != ActionClass::StreamRead + }) + .map(|(index, _)| index), + ) + .collect::>(); + let scenario = format!( + "readers_first = {}\n{}", + serde_json::to_string(&readers_first).unwrap(), + r#" +class Stream: + def __init__(self, incarnation): + self.incarnation = incarnation + self.writer, self.reader = asyncio.Event(), asyncio.Event() + self.frames = asyncio.Queue(maxsize=4) + self.pending = b'' + self.eof = False + self.read_index = 0 + +class Writer: + def __init__(self, stream, started): + self.stream, self.started = stream, started + self.incarnation = stream.incarnation + async def __aenter__(self): + assert not self.stream.writer.is_set(), 'duplicate writer' + self.stream.writer.set() + self.started.set() + await self.stream.reader.wait() + return self + async def __aexit__(self, error_type, error, traceback): + if error is None: + await self.stream.frames.put(None) + async def write(self, frame): + await self.stream.frames.put(bytes(frame)) + +class Reader: + def __init__(self, stream): + self.stream, self.incarnation = stream, stream.incarnation + async def take(self, capacity): + while not self.stream.pending and not self.stream.eof: + chunk = await self.stream.frames.get() + if chunk is None: + self.stream.eof = True + else: + self.stream.pending = chunk + while not self.stream.frames.empty() and not self.stream.eof: + chunk = self.stream.frames.get_nowait() + if chunk is None: + self.stream.eof = True + else: + self.stream.pending += chunk + count = min(capacity, len(self.stream.pending)) + chunk, self.stream.pending = self.stream.pending[:count], self.stream.pending[count:] + return chunk + async def read(self): + capacity = [1, 7, 19, 4093][self.stream.read_index % 4] + self.stream.read_index += 1 + return await self.take(capacity) or None + async def readinto(self, target): + chunk = await self.take(len(target)) + target[:len(chunk)] = chunk + return len(chunk) + +class Blob: + def __init__(self, body, revision): + self.body, self.revision = body, revision + @contextlib.contextmanager + def map(self): + yield self.body + +class BlobWriter(Blob): + def __init__(self, world, path, length): + super().__init__(bytearray(length), None) + self.world, self.path = world, path + async def __aenter__(self): + return self + async def __aexit__(self, error_type, error, traceback): + if error is None: + self.revision = self.world.next_revision() + self.world.blobs[self.path] = self + +class World: + def __init__(self): + self.streams = {} + self.blobs = {} + self.revision = 0 + def next_revision(self): + self.revision += 1 + return self.revision + def stream(self, path): + if path not in self.streams: + self.streams[path] = Stream(self.next_revision()) + return self.streams[path] + +class Data: + def __init__(self, world): + self.world, self.started = world, asyncio.Event() + def write_stream(self, path, replace): + assert not replace + return Writer(self.world.stream(path), self.started) + async def read_stream(self, path): + stream = self.world.stream(path) + assert not stream.reader.is_set(), 'duplicate reader' + stream.reader.set() + self.started.set() + await stream.writer.wait() + return Reader(stream) + def write_blob(self, path, length): + self.started.set() + return BlobWriter(self.world, path, length) + async def read_blob(self, path): + self.started.set() + return self.world.blobs[path] + async def lookup(self, path): + self.started.set() + if path not in self.world.blobs: + raise OSError(errno.ENOENT, 'unpublished blob') + blob = self.world.blobs[path] + return types.SimpleNamespace(kind='blob', revision=blob.revision, active=False) + +async def run(order): + world, output, namespaces = World(), io.StringIO(), [] + for source in sources: + namespace = {} + exec(source, namespace) + namespaces.append(namespace) + tasks = [] + with contextlib.redirect_stdout(output): + for index in order: + data = Data(world) + tasks.append(asyncio.create_task(namespaces[index]['main'](types.SimpleNamespace(data=data)))) + # Admit participants in an adverse order, including blob-only DAG + # vertices whose initial namespace wait precedes any transfer. + await data.started.wait() + await asyncio.wait_for(asyncio.gather(*tasks), timeout=1) + assert all(stream.writer.is_set() and stream.reader.is_set() and stream.frames.empty() + for stream in world.streams.values()), 'unfinished rendezvous' + records = [json.loads(line) for line in output.getvalue().splitlines()] + return [[record for record in records if record['process'] == namespace['PROCESS']] + for namespace in namespaces] + +async def scenario(): + print(json.dumps([await run(list(reversed(range(len(sources))))), + await run(readers_first)])) + +asyncio.run(scenario()) +"# + ); + let observations: Vec>> = + serde_json::from_str(&execute_generated(&sources, &scenario)).unwrap(); + for records in observations { + let observation = generated_observation(&case, records); + crate::oracle::BehaviorOracle::verify(&case, &observation).unwrap_or_else( + |error| panic!("nodes={nodes} family={family:?}: {error:?}"), + ); + let mut ledger = CoverageLedger::survivor(case.live_nodes.clone()).unwrap(); + ledger.plan_case(&case).unwrap(); + ledger + .bind_identity(EvidenceIdentity { + plan_digest: digest(b"generated topology plan"), + artifacts_digest: digest(b"generated topology binding adapter"), + deployment_generation: "generated-topology".to_owned(), + fixture_mapping_digest: digest(b"generated topology nodes"), + }) + .unwrap(); + ledger.observe_case(&case, &observation).unwrap(); + assert_eq!( + ledger.observed.get(&CoverageKey::Topology { family }), + Some(&1) + ); + assert_eq!( + ledger.observed.get(&CoverageKey::Scenario { + scenario: CoverageScenario::ConcurrentStartup, + }), + Some(&1) + ); + } + } + } + } + + #[test] + fn generated_predicates_cancel_withheld_public_binding_replies() { + let operations = [ + ActionOp::AwaitEntry { + path: "/cases/withheld/entry".to_owned(), + expected_kind: "blob".to_owned(), + }, + ActionOp::WaitForQuiescent { + path: "/cases/withheld/stream".to_owned(), + }, + ActionOp::GatedStreamWrite { + path: "/cases/withheld/stream".to_owned(), + replace: false, + frames: vec![b"first".to_vec(), b"after-release".to_vec()], + release_path: "/cases/withheld/release".to_owned(), + }, + ]; + let sources = operations + .into_iter() + .map(|operation| { + render_program( + &ProcessProgram { + id: "withheld".to_owned(), + logical_node_id: 1, + access: AccessSpec::unrestricted("withheld"), + depends_on: Vec::new(), + actions: vec![Action::ok(operation)], + }, + Duration::from_millis(50), + None, + ) + }) + .collect::>(); + execute_generated( + &sources, + r#" +class Writer: + incarnation = 1 + def __init__(self): + self.frames = [] + async def __aenter__(self): + return self + async def __aexit__(self, *error): + return False + async def write(self, frame): + self.frames.append(frame) + +class Data: + def __init__(self): + self.cancelled = [] + self.writer = Writer() + async def lookup(self, path): + try: + await asyncio.Future() + finally: + self.cancelled.append(path) + def write_stream(self, *args, **kwargs): + return self.writer + +async def scenario(): + for source in sources: + namespace, output, data = {}, io.StringIO(), Data() + exec(source, namespace) + with contextlib.redirect_stdout(output): + try: + await namespace['main'](types.SimpleNamespace(data=data)) + except TimeoutError: + pass + else: + raise AssertionError('withheld reply passed') + for _ in range(8): + await asyncio.sleep(0) + records = [json.loads(line) for line in output.getvalue().splitlines()] + assert len(data.cancelled) == 1, data.cancelled + assert b'after-release' not in b''.join(data.writer.frames), 'writer crossed withheld release' + results = [record for record in records if record['outcome'] != 'barrier'] + assert len(results) == 1, records + assert results[0]['outcome'] == 'error', records + assert results[0]['error_type'] == 'TimeoutError', records + assert not [task for task in asyncio.all_tasks() if task is not asyncio.current_task()], 'leaked awaited request' + +asyncio.run(scenario()) +"#, + ); + } + + #[test] + fn absence_probe_looks_up_all_paths_concurrently_and_reports_in_order() { + let paths = (0..12) + .map(|index| format!("/cases/absence/{index}")) + .collect::>(); + let source = + render_absence_probe_python("typed-absence-probe", &paths, Duration::from_millis(150)); + execute_generated( + &[source], + r#" +class Data: + def __init__(self): + self.started = 0 + self.all_started = asyncio.Event() + async def lookup(self, path): + self.started += 1 + if self.started == 12: + self.all_started.set() + await asyncio.wait_for(self.all_started.wait(), 0.05) + raise OSError(errno.ENOENT, 'absent') + +async def scenario(): + namespace, output, data = {}, io.StringIO(), Data() + exec(sources[0], namespace) + with contextlib.redirect_stdout(output): + await namespace['main'](types.SimpleNamespace(data=data)) + records = [json.loads(line) for line in output.getvalue().splitlines()] + assert data.started == 12, data.started + assert [record['step'] for record in records] == list(range(12)), records + assert all(record['outcome'] == 'expected_error' for record in records), records + assert all(record['errno'] == errno.ENOENT for record in records), records + +asyncio.run(scenario()) +"#, + ); + } +} diff --git a/tools/myelin-e2e-fuzz/src/corpus.rs b/tools/myelin-e2e-fuzz/src/corpus.rs index d74eeab..e216f45 100644 --- a/tools/myelin-e2e-fuzz/src/corpus.rs +++ b/tools/myelin-e2e-fuzz/src/corpus.rs @@ -1,9 +1,16 @@ //! Deterministic stable/failure corpus and random short-DAG generation. +use std::collections::{BTreeMap, BTreeSet}; + +use crate::budget::Budget; use crate::ir::{ - AccessSpec, Action, ActionClass, ActionOp, BehaviorCase, DescriptorFinish, - DescriptorReadMethod, DescriptorWriteMethod, FailureInjection, LaunchFailureKind, - ProcessProgram, ProcessStopPhase, PythonException, + AccessSpec, Action, ActionClass, ActionOp, BehaviorCase, CoverageScenario, DataEdge, DataKind, + DataRoute, DescriptorFinish, DescriptorReadMethod, DescriptorWriteMethod, FailureInjection, + LaunchFailureKind, ProcessProgram, ProcessStopPhase, PythonException, TopologyFamily, +}; +use crate::ir::{ + BARRIER_LAP_COMPLETED, BARRIER_TOKEN_FORWARDED, BARRIER_TOKEN_RECEIVED, EVIDENCE_HINT_BARRIER, + EVIDENCE_HINT_EDGE_INDEX, EVIDENCE_HINT_LAP, EVIDENCE_HINT_TOKEN, }; pub(crate) const SEEDED_MODEL_PATH: &str = "/models/tiny-linear/weights"; @@ -23,7 +30,14 @@ pub fn stable_corpus(node_count: u8, seed: u64) -> Vec { cases.push(BehaviorCase { id: blob_id.clone(), seed, - node_count, + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes: vec![ ProcessProgram { id: format!("publisher-{source_node}"), @@ -59,7 +73,14 @@ pub fn stable_corpus(node_count: u8, seed: u64) -> Vec { cases.push(BehaviorCase { id: stream_id.clone(), seed, - node_count, + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes: vec![ ProcessProgram { id: format!("stream-source-{source_node}"), @@ -90,7 +111,14 @@ pub fn stable_corpus(node_count: u8, seed: u64) -> Vec { cases.push(BehaviorCase { id: format!("seeded-model-{seed}"), seed, - node_count, + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes: vec![ProcessProgram { id: "model-reader".to_owned(), logical_node_id: 1, @@ -108,7 +136,14 @@ pub fn stable_corpus(node_count: u8, seed: u64) -> Vec { cases.push(BehaviorCase { id: format!("namespace-mutations-{seed}"), seed, - node_count, + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes: vec![ProcessProgram { id: "namespace-mutator".to_owned(), logical_node_id: 1, @@ -154,7 +189,14 @@ pub fn stable_corpus(node_count: u8, seed: u64) -> Vec { cases.push(BehaviorCase { id: format!("stream-replacement-{seed}"), seed, - node_count, + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes: vec![ ProcessProgram { id: "replacement-first-writer".to_owned(), @@ -194,6 +236,9 @@ pub fn stable_corpus(node_count: u8, seed: u64) -> Vec { path: format!("{replacement_path}-quiescent"), bytes: Vec::new(), }), + // Either endpoint may create the next incarnation. This + // reader's quiescence-before-open evidence proves the + // transition even when the replacement writer joins it. Action::ok(ActionOp::StreamRead { path: replacement_path.clone(), expected: b"second".to_vec(), @@ -220,10 +265,119 @@ pub fn stable_corpus(node_count: u8, seed: u64) -> Vec { ], failure: FailureInjection::None, }); + // An active stream is displaced mid-flight: the stale writer and a + // consumer attached to the displaced incarnation observe ESTALE, and a + // fresh consumer of the replacement incarnation completes cleanly. + let active_root = format!("/cases/{seed}/active-stream-replacement"); + let active_stream = format!("{active_root}/stream"); + let active_marker = format!("{active_root}/reader-attached"); + let active_fresh_marker = format!("{active_root}/fresh-reader-attached"); + let active_release = format!("{active_root}/release"); + cases.push(BehaviorCase { + id: format!("active-stream-replacement-{seed}"), + seed, + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: [CoverageScenario::ActiveMutation].into(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), + processes: vec![ + ProcessProgram { + id: "active-replacement-writer".to_owned(), + logical_node_id: 1, + access: AccessSpec::unrestricted("active-replacement-writer"), + depends_on: Vec::new(), + actions: vec![Action::error( + ActionOp::GatedStreamWrite { + path: active_stream.clone(), + replace: false, + frames: vec![b"active-first".to_vec(), b"active-second".to_vec()], + release_path: active_release.clone(), + }, + libc::ESTALE, + )], + }, + ProcessProgram { + id: "active-replacement-reader".to_owned(), + logical_node_id: u64::from(node_count), + access: AccessSpec::unrestricted("active-replacement-reader"), + depends_on: Vec::new(), + actions: vec![Action::error( + ActionOp::GatedStreamRead { + expected: b"active-firstactive-second".to_vec(), + path: active_stream.clone(), + observed_path: active_marker.clone(), + retry_attach: false, + park_after_first_frame: false, + }, + libc::ESTALE, + )], + }, + ProcessProgram { + id: "active-replacement-replacer".to_owned(), + logical_node_id: 1, + access: AccessSpec::unrestricted("active-replacement-replacer"), + // The replacer must not depend on the writer's completion: + // the writer's gate waits for the release published + // immediately after the fenced replacement completes. + depends_on: Vec::new(), + actions: vec![ + Action::ok(ActionOp::AwaitEntry { + path: active_marker.clone(), + expected_kind: "blob".to_owned(), + }), + Action::ok(ActionOp::GatedStreamWrite { + path: active_stream.clone(), + replace: true, + frames: vec![b"replacement".to_vec()], + release_path: active_fresh_marker.clone(), + }), + Action::ok(ActionOp::PublishBlob { + path: active_release, + bytes: Vec::new(), + }), + ], + }, + ProcessProgram { + // The displaced reader's terminal event proves that the + // replacement incarnation exists. Starting the fresh reader + // earlier can bind it to the old incarnation, whose fencing + // correctly terminates the entire contextual process. + id: "active-replacement-fresh-reader".to_owned(), + logical_node_id: u64::from(node_count), + access: AccessSpec::unrestricted("active-replacement-fresh-reader"), + depends_on: vec!["active-replacement-reader".to_owned()], + actions: vec![ + Action::ok(ActionOp::AwaitEntry { + path: active_marker, + expected_kind: "blob".to_owned(), + }), + Action::ok(ActionOp::GatedStreamRead { + path: active_stream, + expected: b"replacement".to_vec(), + observed_path: active_fresh_marker, + retry_attach: true, + park_after_first_frame: false, + }), + ], + }, + ], + failure: FailureInjection::None, + }); cases.push(BehaviorCase { id: format!("authorization-{seed}"), seed, - node_count, + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: [CoverageScenario::Authorization].into(), + routes: Vec::new(), + read_only_fixture_paths: [SEEDED_MODEL_PATH.to_owned()].into(), + resource_bounds: Default::default(), processes: vec![ ProcessProgram { id: "read-only".to_owned(), @@ -234,13 +388,19 @@ pub fn stable_corpus(node_count: u8, seed: u64) -> Vec { write_prefixes: Vec::new(), }, depends_on: Vec::new(), - actions: vec![Action::error( - ActionOp::PublishBlob { - path: format!("/cases/{seed}/unauthorized-publication"), - bytes: vec![1], - }, - libc::EACCES, - )], + actions: vec![ + Action::ok(ActionOp::ReadBlob { + path: SEEDED_MODEL_PATH.to_owned(), + expected: SEEDED_MODEL_BYTES.to_vec(), + }), + Action::error( + ActionOp::PublishBlob { + path: format!("/cases/{seed}/unauthorized-publication"), + bytes: vec![1], + }, + libc::EACCES, + ), + ], }, ProcessProgram { id: "write-only".to_owned(), @@ -251,13 +411,19 @@ pub fn stable_corpus(node_count: u8, seed: u64) -> Vec { write_prefixes: vec!["/cases".to_owned()], }, depends_on: Vec::new(), - actions: vec![Action::error( - ActionOp::ReadBlob { - path: SEEDED_MODEL_PATH.to_owned(), - expected: SEEDED_MODEL_BYTES.to_vec(), - }, - libc::EACCES, - )], + actions: vec![ + Action::ok(ActionOp::PublishBlob { + path: format!("/cases/{seed}/denied/source"), + bytes: b"denied-source".to_vec(), + }), + Action::error( + ActionOp::ReadBlob { + path: SEEDED_MODEL_PATH.to_owned(), + expected: SEEDED_MODEL_BYTES.to_vec(), + }, + libc::EACCES, + ), + ], }, ProcessProgram { id: "rename-guard".to_owned(), @@ -267,8 +433,12 @@ pub fn stable_corpus(node_count: u8, seed: u64) -> Vec { read_prefixes: Vec::new(), write_prefixes: vec![format!("/cases/{seed}/allowed")], }, - depends_on: Vec::new(), + depends_on: vec!["write-only".to_owned()], actions: vec![ + Action::ok(ActionOp::PublishBlob { + path: format!("/cases/{seed}/allowed/source"), + bytes: b"allowed-source".to_vec(), + }), Action::error( ActionOp::Rename { source: format!("/cases/{seed}/denied/source"), @@ -294,7 +464,14 @@ pub fn stable_corpus(node_count: u8, seed: u64) -> Vec { cases.push(BehaviorCase { id: format!("self-isolation-{seed}"), seed, - node_count, + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes: vec![ ProcessProgram { id: "self-a".to_owned(), @@ -329,7 +506,14 @@ pub fn stable_corpus(node_count: u8, seed: u64) -> Vec { cases.push(BehaviorCase { id: id.clone(), seed, - node_count, + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes: vec![ ProcessProgram { id: "boundary-publisher".to_owned(), @@ -362,7 +546,14 @@ pub fn stable_corpus(node_count: u8, seed: u64) -> Vec { cases.push(BehaviorCase { id: format!("stream-backpressure-{seed}"), seed, - node_count, + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes: vec![ ProcessProgram { id: "backpressure-writer".to_owned(), @@ -415,9 +606,16 @@ fn descriptor_corpus(node_count: u8, seed: u64) -> Vec { let path = format!("/cases/{seed}/descriptor/method-{index}"); let bytes = boundary_payload(seed.rotate_left(index as u32 + 1), 4097 + index); cases.push(BehaviorCase { + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, id: id.clone(), seed: seed ^ index as u64, - node_count, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes: vec![ ProcessProgram { id: "descriptor-writer".to_owned(), @@ -463,7 +661,14 @@ fn descriptor_corpus(node_count: u8, seed: u64) -> Vec { cases.push(BehaviorCase { id: id.clone(), seed, - node_count, + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes: vec![ ProcessProgram { id: "unfinished-writer".to_owned(), @@ -484,13 +689,27 @@ fn descriptor_corpus(node_count: u8, seed: u64) -> Vec { logical_node_id: u64::from(node_count), access: AccessSpec::unrestricted(format!("{id}-observer")), depends_on: vec!["unfinished-writer".to_owned()], - actions: vec![Action::error( - ActionOp::Lookup { + actions: vec![ + Action::error( + ActionOp::Lookup { + path: path.clone(), + expected_kind: "blob".to_owned(), + }, + libc::ENOENT, + ), + Action::ok(ActionOp::DescriptorWrite { + path: path.clone(), + flags: libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL | libc::O_TRUNC, + length: Some(6), + bytes: b"reused".to_vec(), + method: DescriptorWriteMethod::Write, + finish: DescriptorFinish::Close, + }), + Action::ok(ActionOp::ReadBlob { path, - expected_kind: "blob".to_owned(), - }, - libc::ENOENT, - )], + expected: b"reused".to_vec(), + }), + ], }, ], failure: FailureInjection::None, @@ -501,7 +720,14 @@ fn descriptor_corpus(node_count: u8, seed: u64) -> Vec { cases.push(BehaviorCase { id: format!("descriptor-repeated-terminal-{seed}"), seed, - node_count, + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes: vec![ProcessProgram { id: "descriptor-repeater".to_owned(), logical_node_id: 1, @@ -538,7 +764,14 @@ fn descriptor_corpus(node_count: u8, seed: u64) -> Vec { cases.push(BehaviorCase { id: format!("descriptor-invalid-{seed}"), seed, - node_count, + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes: vec![ProcessProgram { id: "descriptor-invalid".to_owned(), logical_node_id: 1, @@ -600,7 +833,14 @@ fn namespace_edge_corpus(node_count: u8, seed: u64) -> Vec { cases.push(BehaviorCase { id: format!("namespace-no-replace-{seed}"), seed, - node_count, + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes: vec![ProcessProgram { id: "no-replace-mutator".to_owned(), logical_node_id: 1, @@ -641,7 +881,14 @@ fn namespace_edge_corpus(node_count: u8, seed: u64) -> Vec { cases.push(BehaviorCase { id: format!("namespace-quiescent-stream-rename-{seed}"), seed, - node_count, + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes: vec![ ProcessProgram { id: "quiescent-writer".to_owned(), @@ -692,7 +939,14 @@ fn namespace_edge_corpus(node_count: u8, seed: u64) -> Vec { cases.push(BehaviorCase { id: format!("namespace-blob-stream-collision-{seed}"), seed, - node_count, + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes: vec![ ProcessProgram { id: "blob-owner".to_owned(), @@ -726,7 +980,14 @@ fn namespace_edge_corpus(node_count: u8, seed: u64) -> Vec { cases.push(BehaviorCase { id: format!("namespace-stream-blob-collision-{seed}"), seed, - node_count, + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes: vec![ ProcessProgram { id: "stream-owner-writer".to_owned(), @@ -774,7 +1035,14 @@ fn namespace_edge_corpus(node_count: u8, seed: u64) -> Vec { cases.push(BehaviorCase { id: format!("namespace-blob-over-stream-{seed}"), seed, - node_count, + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes: vec![ ProcessProgram { id: "replace-stream-writer".to_owned(), @@ -835,7 +1103,14 @@ fn namespace_edge_corpus(node_count: u8, seed: u64) -> Vec { cases.push(BehaviorCase { id: format!("namespace-stream-over-blob-{seed}"), seed, - node_count, + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes: vec![ ProcessProgram { id: "replacement-stream-writer".to_owned(), @@ -897,7 +1172,14 @@ fn concurrent_namespace_corpus(node_count: u8, seed: u64) -> Vec { let publish_case = BehaviorCase { id: format!("concurrent-publish-{seed}"), seed, - node_count, + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes: vec![ ProcessProgram { id: "publisher-a".to_owned(), @@ -955,7 +1237,14 @@ fn concurrent_namespace_corpus(node_count: u8, seed: u64) -> Vec { let rename_case = BehaviorCase { id: format!("concurrent-rename-{seed}"), seed, - node_count, + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes: vec![ ProcessProgram { id: "rename-setup".to_owned(), @@ -1007,7 +1296,14 @@ fn concurrent_namespace_corpus(node_count: u8, seed: u64) -> Vec { let unlink_case = BehaviorCase { id: format!("concurrent-unlink-{seed}"), seed, - node_count, + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes: vec![ ProcessProgram { id: "unlink-setup".to_owned(), @@ -1076,7 +1372,14 @@ pub fn ordered_pair_corpus(node_count: u8, seed: u64) -> Vec { cases.push(BehaviorCase { id, seed: seed ^ ((left_index as u64) << 32) ^ right_index as u64, - node_count, + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes, failure: FailureInjection::None, }); @@ -1170,7 +1473,14 @@ pub fn failure_corpus(node_count: u8, seed: u64) -> Vec { let stopped_case = |phase: ProcessStopPhase, label: &str| BehaviorCase { id: format!("failure-process-stop-{label}-{seed}"), seed, - node_count, + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes: vec![ProcessProgram { id: "stopped-process".to_owned(), logical_node_id: 1, @@ -1193,9 +1503,16 @@ pub fn failure_corpus(node_count: u8, seed: u64) -> Vec { stopped_case(ProcessStopPhase::AfterContextReady, "after-context-ready"), { let launch_case = |kind: LaunchFailureKind, label: &str| BehaviorCase { + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, id: format!("failure-launch-{label}-{seed}"), seed, - node_count, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes: vec![ProcessProgram { id: "failed-launch".to_owned(), logical_node_id: 1, @@ -1241,51 +1558,6 @@ pub fn failure_corpus(node_count: u8, seed: u64) -> Vec { LaunchFailureKind::PythonRuntime, "python-runtime", ), - BehaviorCase { - id: format!("failure-node-loss-{seed}"), - seed: seed.rotate_left(9), - node_count, - processes: vec![ - ProcessProgram { - id: "lost-node-process".to_owned(), - logical_node_id: 1, - access: AccessSpec::unrestricted("failure-node-loss"), - depends_on: Vec::new(), - actions: vec![Action::ok(ActionOp::StreamWrite { - path: format!("/cases/failure/{seed}/node-loss"), - chunks: vec![vec![1, 2, 3]], - replace: false, - })], - }, - ProcessProgram { - id: "surviving-node-process".to_owned(), - logical_node_id: u64::from(node_count), - access: AccessSpec::unrestricted("failure-node-survivor"), - depends_on: vec!["lost-node-process".to_owned()], - actions: vec![Action::ok(ActionOp::Lookup { - path: SEEDED_MODEL_PATH.to_owned(), - expected_kind: "blob".to_owned(), - })], - }, - ], - failure: FailureInjection::KillNode { logical_node_id: 1 }, - }, - BehaviorCase { - id: format!("failure-orchestrator-stop-{seed}"), - seed: seed.rotate_right(7), - node_count, - processes: vec![ProcessProgram { - id: "orchestrator-stop-process".to_owned(), - logical_node_id: u64::from(node_count), - access: AccessSpec::unrestricted("failure-orchestrator-stop"), - depends_on: Vec::new(), - actions: vec![Action::ok(ActionOp::Lookup { - path: SEEDED_MODEL_PATH.to_owned(), - expected_kind: "blob".to_owned(), - })], - }], - failure: FailureInjection::StopOrchestrator, - }, ] } @@ -1298,7 +1570,14 @@ fn failure_corpus_launch_case( BehaviorCase { id: format!("failure-launch-{label}-{seed}"), seed, - node_count, + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes: vec![ProcessProgram { id: "failed-launch".to_owned(), logical_node_id: 1, @@ -1316,34 +1595,1456 @@ fn failure_corpus_launch_case( } } -pub fn random_short_dags(seed: u64, count: usize, node_count: u8) -> Vec { - let mut random = XorShift64(seed); - (0..count) - .map(|index| { - let source_node = 1 + random.next_u64() % u64::from(node_count); - let mut destination_node = 1 + random.next_u64() % u64::from(node_count); - if destination_node == source_node { - destination_node = destination_node % u64::from(node_count) + 1; +pub fn generated_campaign_cases( + seed: u64, + count: usize, + node_count: u8, +) -> Result, String> { + generated_campaign_cases_inner(seed, count, node_count, None) +} + +pub(crate) fn generated_campaign_cases_budgeted( + seed: u64, + count: usize, + node_count: u8, + budget: &Budget, +) -> Result, String> { + generated_campaign_cases_inner(seed, count, node_count, Some(budget)) +} + +fn generated_campaign_cases_inner( + seed: u64, + count: usize, + node_count: u8, + budget: Option<&Budget>, +) -> Result, String> { + if let Some(budget) = budget { + budget.check("generate deterministic campaign base cases")?; + } + if !(3..=5).contains(&node_count) { + return Err("campaign generation requires three to five nodes".to_owned()); + } + let mut cases = if node_count == 5 && count == 128 { + let mut stable = stable_corpus(node_count, seed); + if stable.len() < count { + return Err(format!( + "stable coverage corpus has {} cases, expected at least {count}", + stable.len() + )); + } + stable.drain(0..stable.len() - count); + stable + } else { + match budget { + Some(budget) => random_short_dags_budgeted(seed, count, node_count, budget)?, + None => random_short_dags(seed, count, node_count), + } + }; + for (index, case) in cases.iter_mut().enumerate() { + if let Some(budget) = budget { + budget.check("decorate complete deterministic campaign case")?; + } + decorate_campaign_case(case, index, count, node_count, seed)?; + } + if let Some(budget) = budget { + budget.check("complete deterministic campaign generation")?; + } + Ok(cases) +} + +fn decorate_campaign_case( + case: &mut BehaviorCase, + index: usize, + count: usize, + node_count: u8, + seed: u64, +) -> Result<(), String> { + let nodes = (1..=u64::from(node_count)).collect::>(); + let family = campaign_family(index, count, node_count); + case.topology = family; + if family == TopologyFamily::RingWalk { + case.scenarios.insert(CoverageScenario::RingCompletion); + } + let churn = index == 0; + if case.id.contains("authorization") { + case.scenarios.insert(CoverageScenario::Authorization); + } + // Each replacement variant proves exactly its own mutation scenario: + // the quiescent variant replaces a stream after an observed quiescence, + // the active variant displaces one mid-flight. Requiring the other + // scenario would make the case unprovable by construction. + if case.id.starts_with("active-stream-replacement-") { + case.scenarios.insert(CoverageScenario::ActiveMutation); + } else if case.id.starts_with("stream-replacement-") { + case.scenarios.insert(CoverageScenario::QuiescentMutation); + } + let topology_occurrence = if node_count < 5 || count < 128 { + index / 6 + } else { + let first = match family { + TopologyFamily::Chain => 0, + TopologyFamily::RingWalk => 26, + TopologyFamily::FanOut => 45, + TopologyFamily::FanIn => 64, + TopologyFamily::Diamond => 83, + TopologyFamily::RandomDag => 96, + TopologyFamily::Fixed => unreachable!("fixed is not a campaign topology"), + }; + index - first + }; + let supplemental_survivor_kind = count == 32 && node_count == 4 && matches!(index, 3 | 9); + let kinds: &[DataKind] = if supplemental_survivor_kind { + &[DataKind::Blob, DataKind::Stream] + } else if topology_occurrence % 2 == 0 { + &[DataKind::Blob] + } else { + // Blob and stream topology are sampled independently across the fixed + // campaign. Their planned ledgers still require every survivor edge + // and node role for both kinds before provider access. + &[DataKind::Stream] + }; + if kinds.contains(&DataKind::Stream) { + case.scenarios.insert(CoverageScenario::ConcurrentStartup); + } + if family == TopologyFamily::RandomDag { + let action_limit = if count == 16 { 60 } else { 56 }; + append_random_dag_routes(case, &nodes, seed, index, action_limit, kinds)?; + } else { + let edges = topology_edges(family, &nodes, index); + for &kind in kinds { + let label = if kind == DataKind::Blob { + "topology-blob" + } else { + "topology-stream" + }; + append_executable_route(case, kind, label, &edges, family)?; + } + } + + // Topology rotation already covers every ordered node pair for both data + // kinds across each phase. Do not duplicate those edges with two extra + // fixed routes in every case; retain the required 16-action floor below. + let hot_cold = family != TopologyFamily::RandomDag && index % 16 == 5; + if churn { + append_executable_route( + case, + DataKind::Stream, + "pair-stream", + &[(nodes[0], nodes[1])], + TopologyFamily::Fixed, + )?; + } + if churn { + append_churn_target(case, nodes[index % nodes.len()])?; + } + if matches!(index, 8 | 9) { + append_active_stream_fault(case, &nodes, index == 8); + } + if index < 4 { + append_descriptor_terminal(case, index); + } + if index == 4 && family != TopologyFamily::RandomDag { + case.scenarios.insert(CoverageScenario::FailureIsolation); + case.processes.push(ProcessProgram { + id: "expected-failed-branch".to_owned(), + logical_node_id: nodes[0], + access: AccessSpec::unrestricted(format!("{}-failed-branch", case.id)), + depends_on: Vec::new(), + actions: vec![Action::ok(ActionOp::Lookup { + path: SEEDED_MODEL_PATH.to_owned(), + expected_kind: "blob".to_owned(), + })], + }); + case.failure = FailureInjection::LaunchFailure { + process: "expected-failed-branch".to_owned(), + kind: LaunchFailureKind::MissingExecutable, + }; + } + if hot_cold { + append_hot_cold_scenario(case, &nodes); + } + if index == 6 { + append_slow_branch(case, &nodes); + } + + if case + .processes + .iter() + .flat_map(|process| &process.actions) + .any(|action| action.operation.path().starts_with("/models/")) + { + case.read_only_fixture_paths + .insert(SEEDED_MODEL_PATH.to_owned()); + } + let mut padding_cursor = 0; + while case + .processes + .iter() + .map(|process| process.actions.len()) + .sum::() + < 16 + { + let process_count = case.processes.len(); + if process_count == 0 { + return Err(format!("case {} has no process to pad", case.id)); + } + let process = &mut case.processes[padding_cursor % process_count]; + padding_cursor += 1; + process.actions.push(Action::ok(ActionOp::Lookup { + path: SEEDED_MODEL_PATH.to_owned(), + expected_kind: "blob".to_owned(), + })); + case.read_only_fixture_paths + .insert(SEEDED_MODEL_PATH.to_owned()); + } + let action_count = case + .processes + .iter() + .map(|process| process.actions.len()) + .sum::(); + if !(2..=20).contains(&case.processes.len()) || !(16..=64).contains(&action_count) { + return Err(format!( + "decorated case {} has {} processes and {action_count} actions", + case.id, + case.processes.len() + )); + } + case.validate() +} +fn append_descriptor_terminal(case: &mut BehaviorCase, index: usize) { + let (finish, expected_error) = match index { + 0 => (DescriptorFinish::Abort, None), + 1 => (DescriptorFinish::Drop, None), + 2 => (DescriptorFinish::CloseTwice, Some(libc::EBADF)), + _ => (DescriptorFinish::AbortTwice, Some(libc::EBADF)), + }; + let operation = ActionOp::DescriptorRead { + path: SEEDED_MODEL_PATH.to_owned(), + flags: libc::O_RDONLY, + expected: SEEDED_MODEL_BYTES.to_vec(), + method: DescriptorReadMethod::Read, + offset: 0, + finish, + }; + let action = match expected_error { + Some(error) => Action::error(operation, error), + None => Action::ok(operation), + }; + case.processes[0].actions.push(action); + case.read_only_fixture_paths + .insert(SEEDED_MODEL_PATH.to_owned()); +} + +fn append_slow_branch(case: &mut BehaviorCase, nodes: &[u64]) { + let parked_path = format!("/cases/{}/slow-branch/parked", case.id); + let release_path = format!("/cases/{}/slow-branch/release", case.id); + let completed_path = format!("/cases/{}/slow-branch/completed", case.id); + let healthy = case + .processes + .iter() + .map(|program| program.id.clone()) + .collect(); + for program in &mut case.processes { + if program.depends_on.is_empty() { + program.actions.insert( + 0, + Action::ok(ActionOp::AwaitEntry { + path: parked_path.clone(), + expected_kind: "blob".to_owned(), + }), + ); + } + } + case.processes.push(ProcessProgram { + id: "slow-branch".to_owned(), + logical_node_id: nodes[0], + access: AccessSpec::unrestricted(format!("{}-slow-branch", case.id)), + depends_on: Vec::new(), + actions: vec![ + Action::ok(ActionOp::PublishBlob { + path: parked_path.clone(), + bytes: Vec::new(), + }), + Action::ok(ActionOp::AwaitEntry { + path: release_path.clone(), + expected_kind: "blob".to_owned(), + }), + Action::ok(ActionOp::PublishBlob { + path: completed_path, + bytes: b"released".to_vec(), + }), + ], + }); + case.processes.push(ProcessProgram { + id: "slow-branch-release".to_owned(), + logical_node_id: nodes[1], + access: AccessSpec::unrestricted(format!("{}-slow-branch-release", case.id)), + depends_on: healthy, + actions: vec![ + Action::ok(ActionOp::AwaitEntry { + path: parked_path.clone(), + expected_kind: "blob".to_owned(), + }), + Action::ok(ActionOp::PublishBlob { + path: release_path.clone(), + bytes: Vec::new(), + }), + ], + }); + case.failure = FailureInjection::SlowProcess { + process: "slow-branch".to_owned(), + parked_path, + release_path, + }; + case.scenarios.insert(CoverageScenario::FailureIsolation); +} + +fn append_hot_cold_scenario(case: &mut BehaviorCase, nodes: &[u64]) { + // Causal order required by the plan (10.3): the hot stream is written and + // parked, its first frame is observed, both cold flows do observable work + // and terminate, only then is the release published, and the hot flow + // terminates on clean EOF. Ordering uses the first-frame observation + // barrier, the release-path lookup barrier, and process-completion + // dependencies for the release publisher, never a schedule. + case.scenarios.insert(CoverageScenario::HotColdFairness); + let stream_path = format!("/cases/{}/hot-cold/hot", case.id); + let observed_path = format!("/cases/{}/hot-cold/observed", case.id); + let release_path = format!("/cases/{}/hot-cold/release", case.id); + let cold_blob_path = format!("/cases/{}/hot-cold/cold-blob", case.id); + let cold_stream_path = format!("/cases/{}/hot-cold/cold-stream", case.id); + // The first logical frame exceeds the 256 KiB endpoint ring, so reaching + // the cold-work gate proves a multi-chunk hot transfer, not a tiny marker. + // The writer retains its active endpoint and remaining payload until release. + let frames = vec![ + boundary_payload(case.seed.rotate_left(7), 262_145), + boundary_payload(case.seed.rotate_left(11), 65_537), + ]; + let cold_blob_payload = boundary_payload(case.seed.rotate_left(3), 4_097); + let cold_stream_payload = boundary_payload(case.seed.rotate_left(5), 8_193); + let cold_stream_split = cold_stream_payload.len() / 2; + let cold_stream_chunks = vec![ + cold_stream_payload[..cold_stream_split].to_vec(), + cold_stream_payload[cold_stream_split..].to_vec(), + ]; + case.routes.push(DataRoute { + id: format!("{}-hot-cold", case.id), + kind: DataKind::Stream, + edges: vec![DataEdge { + source: nodes[0], + destination: nodes[1], + source_role: "hot-writer".to_owned(), + destination_role: "hot-reader".to_owned(), + path: stream_path.clone(), + }], + join_inputs: Vec::new(), + }); + let cold_blob_node = nodes[2]; + let cold_stream_node = nodes[nodes.len() - 1]; + case.processes.extend([ + ProcessProgram { + id: "hot-writer".to_owned(), + logical_node_id: nodes[0], + access: AccessSpec::unrestricted(format!("{}-hot-writer", case.id)), + depends_on: Vec::new(), + actions: vec![Action::ok(ActionOp::GatedStreamWrite { + path: stream_path.clone(), + replace: false, + frames: frames.clone(), + release_path: release_path.clone(), + })], + }, + ProcessProgram { + id: "hot-reader".to_owned(), + logical_node_id: nodes[1], + access: AccessSpec::unrestricted(format!("{}-hot-reader", case.id)), + depends_on: Vec::new(), + actions: vec![Action::ok(ActionOp::GatedStreamRead { + path: stream_path, + expected: frames.concat(), + observed_path: observed_path.clone(), + retry_attach: false, + park_after_first_frame: false, + })], + }, + ProcessProgram { + id: "cold-blob-flow".to_owned(), + logical_node_id: cold_blob_node, + access: AccessSpec::unrestricted(format!("{}-cold-blob-flow", case.id)), + depends_on: Vec::new(), + actions: vec![ + Action::ok(ActionOp::AwaitEntry { + path: observed_path.clone(), + expected_kind: "blob".to_owned(), + }), + Action::ok(ActionOp::PublishBlob { + path: cold_blob_path.clone(), + bytes: cold_blob_payload.clone(), + }), + Action::ok(ActionOp::ReadBlob { + path: cold_blob_path, + expected: cold_blob_payload, + }), + ], + }, + ProcessProgram { + id: "cold-stream-flow".to_owned(), + logical_node_id: cold_stream_node, + access: AccessSpec::unrestricted(format!("{}-cold-stream-flow", case.id)), + depends_on: Vec::new(), + actions: vec![ + Action::ok(ActionOp::AwaitEntry { + path: observed_path, + expected_kind: "blob".to_owned(), + }), + Action::ok(ActionOp::StreamRoundTrip { + path: cold_stream_path, + chunks: cold_stream_chunks, + }), + ], + }, + ProcessProgram { + id: "hot-release-publisher".to_owned(), + logical_node_id: cold_blob_node, + access: AccessSpec::unrestricted(format!("{}-hot-release-publisher", case.id)), + depends_on: vec!["cold-blob-flow".to_owned(), "cold-stream-flow".to_owned()], + actions: vec![Action::ok(ActionOp::PublishBlob { + path: release_path, + bytes: Vec::new(), + })], + }, + ]); +} + +fn append_churn_target(case: &mut BehaviorCase, node: u64) -> Result<(), String> { + // Keep an independent healthy stream live across the stop. Its first frame + // causes the stop, while a completion-dependent publisher releases it only + // after the target's terminal event. No elapsed-time overlap is assumed. + let edge = case + .routes + .iter() + .find(|route| route.kind == DataKind::Stream && route.id.ends_with("-pair-stream")) + .and_then(|route| route.edges.first()) + .cloned() + .ok_or("churn requires its independent pair stream")?; + let observed_path = format!("/cases/{}/churn/observed", case.id); + let release_path = format!("/cases/{}/churn/release", case.id); + for process in &mut case.processes { + for action in &mut process.actions { + if action.operation.path() != edge.path { + continue; } - let length = [0, 1, 4095, 4096, 4097, 65_537][(random.next_u64() as usize) % 6]; - let shape = random.next_u64() % 5; - let id = format!("random-{seed}-{index}"); - let path = format!("/cases/random/{seed}/{index}"); - let bytes = boundary_payload(random.next_u64(), length); - let processes = match shape { - 0 => vec![ + action.operation = match &action.operation { + ActionOp::StreamWrite { path, chunks, .. } => ActionOp::GatedStreamWrite { + path: path.clone(), + replace: false, + frames: chunks.clone(), + release_path: release_path.clone(), + }, + ActionOp::StreamRead { path, expected } + | ActionOp::StreamReadInto { path, expected, .. } => ActionOp::GatedStreamRead { + path: path.clone(), + expected: expected.clone(), + observed_path: observed_path.clone(), + retry_attach: false, + park_after_first_frame: false, + }, + _ => return Err("churn pair has an unsupported stream endpoint".to_owned()), + }; + } + } + case.scenarios.insert(CoverageScenario::ProcessChurn); + case.processes.extend([ + ProcessProgram { + id: "churn-target".to_owned(), + logical_node_id: node, + access: AccessSpec::unrestricted(format!("{}-churn-target", case.id)), + depends_on: Vec::new(), + actions: vec![Action::ok(ActionOp::StreamRead { + path: format!("/cases/{}/churn/parked", case.id), + expected: Vec::new(), + })], + }, + ProcessProgram { + id: "churn-release-publisher".to_owned(), + logical_node_id: node, + access: AccessSpec::unrestricted(format!("{}-churn-release", case.id)), + depends_on: vec!["churn-target".to_owned()], + actions: vec![Action::ok(ActionOp::PublishBlob { + path: release_path, + bytes: Vec::new(), + })], + }, + ]); + case.failure = FailureInjection::StopProcess { + process: "churn-target".to_owned(), + phase: ProcessStopPhase::AfterSiblingStreamFirstFrame, + kill_after_ms: Some(100), + }; + Ok(()) +} + +fn append_active_stream_fault(case: &mut BehaviorCase, nodes: &[u64], abort_writer: bool) { + let label = if abort_writer { + "active-stream-writer-abort" + } else { + "active-stream-reader-stop" + }; + let path = format!("/cases/{}/{label}/stream", case.id); + let observed_path = format!("/cases/{}/{label}/observed", case.id); + let release_path = format!("/cases/{}/{label}/reader-stopped", case.id); + let writer_id = format!("{label}-writer"); + let reader_id = format!("{label}-reader"); + let frames = if abort_writer { + vec![b"writer-abort-first".to_vec(), vec![0xa5; 4_097]] + } else { + vec![b"reader-stop-first".to_vec(), vec![0x5a; 524_288]] + }; + let expected = frames.concat(); + let write = if abort_writer { + ActionOp::GatedStreamWrite { + path: path.clone(), + replace: false, + frames, + release_path: format!("/cases/{}/{label}/never-release", case.id), + } + } else { + ActionOp::GatedStreamWrite { + path: path.clone(), + replace: false, + frames, + release_path: release_path.clone(), + } + }; + let read = ActionOp::GatedStreamRead { + path: path.clone(), + expected, + observed_path, + retry_attach: false, + park_after_first_frame: !abort_writer, + }; + let (writer_actions, reader_actions) = if abort_writer { + ( + vec![Action::ok(write)], + vec![ + Action::exception(read, PythonException::StreamError), + Action::ok(ActionOp::WaitForQuiescent { path }), + ], + ) + } else { + ( + vec![ + Action::exception(write, PythonException::StreamError), + Action::ok(ActionOp::WaitForQuiescent { path }), + ], + vec![Action::ok(read)], + ) + }; + case.processes.extend([ + ProcessProgram { + id: writer_id.clone(), + logical_node_id: nodes[0], + access: AccessSpec::unrestricted(format!("{}-{writer_id}", case.id)), + depends_on: Vec::new(), + actions: writer_actions, + }, + ProcessProgram { + id: reader_id.clone(), + logical_node_id: nodes[1], + access: AccessSpec::unrestricted(format!("{}-{reader_id}", case.id)), + depends_on: Vec::new(), + actions: reader_actions, + }, + ]); + if !abort_writer { + case.processes.push(ProcessProgram { + id: format!("{label}-release"), + logical_node_id: nodes[1], + access: AccessSpec::unrestricted(format!("{}-{label}-release", case.id)), + depends_on: vec![reader_id.clone()], + actions: vec![Action::ok(ActionOp::PublishBlob { + path: release_path, + bytes: Vec::new(), + })], + }); + } + case.scenarios.insert(if abort_writer { + CoverageScenario::ActiveStreamWriterAbort + } else { + CoverageScenario::ActiveStreamReaderStop + }); + case.failure = FailureInjection::StopProcess { + process: if abort_writer { writer_id } else { reader_id }, + phase: ProcessStopPhase::AfterStreamFirstFrame, + kill_after_ms: Some(100), + }; +} + +fn campaign_family(index: usize, count: usize, node_count: u8) -> TopologyFamily { + if node_count < 5 || count < 128 { + // Survivor and recovery campaigns rotate every realizable family. + // A diamond needs four distinct corners, not a relabeled chain. + return match index % 6 { + 0 => TopologyFamily::Chain, + 1 => TopologyFamily::RingWalk, + 2 => TopologyFamily::FanOut, + 3 => TopologyFamily::FanIn, + 4 if node_count >= 4 => TopologyFamily::Diamond, + _ => TopologyFamily::RandomDag, + }; + } + match index { + 0..=25 => TopologyFamily::Chain, + 26..=44 => TopologyFamily::RingWalk, + 45..=63 => TopologyFamily::FanOut, + 64..=82 => TopologyFamily::FanIn, + 83..=95 => TopologyFamily::Diamond, + _ => TopologyFamily::RandomDag, + } +} + +/// The graph vertices are process roles, not physical nodes. Preserve the +/// primitive-pair fixtures in independent vertex programs: their prefix work +/// uses only fixture/base-case paths and finishes before any route preparation. +/// Dependent base programs stay outside the graph and consume reserved slots. +fn append_random_dag_routes( + case: &mut BehaviorCase, + nodes: &[u64], + seed: u64, + index: usize, + action_limit: usize, + kinds: &[DataKind], +) -> Result<(), String> { + let mut roles = case + .processes + .iter() + .enumerate() + .filter_map(|(index, process)| process.depends_on.is_empty().then_some(index)) + .collect::>(); + let reserved = case.processes.len() - roles.len(); + // The 128-case campaign covers every 3..=20 vertex boundary once. Its + // remaining random-DAG cases stay at the cheapest nontrivial size; repeating + // intermediate vertex counts adds load without adding a boundary. + // Short recovery and survivor campaigns still vary their DAGs. + let requested = if (96..114).contains(&index) { + 3 + (index - 96) + } else if index >= 114 { + 3 + } else { + 3 + ((seed % 10) as usize + index % 10) % 10 + }; + let vertices = requested.min(20 - reserved).max(roles.len()); + let selected_kind = *kinds.last().expect("campaign topology kind"); + let large_kind = if vertices > 17 { + DataKind::Stream + } else { + selected_kind + }; + let small_kind = if large_kind == DataKind::Blob { + DataKind::Stream + } else { + DataKind::Blob + }; + let action_cost = |kind| if kind == DataKind::Blob { 3 } else { 2 }; + let base_actions = case + .processes + .iter() + .map(|process| process.actions.len()) + .sum::(); + let max_edges = + (action_limit - base_actions - 2 * action_cost(small_kind)) / action_cost(large_kind); + if !(3..=20).contains(&vertices) || vertices - 1 > max_edges { + return Err(format!("case {} cannot fit its random DAG", case.id)); + } + let mut random = XorShift64( + seed.rotate_left(23) ^ case.seed ^ (index as u64).wrapping_mul(0x9e37_79b9_7f4a_7c15), + ); + let mut edges = Vec::with_capacity(max_edges.min(vertices + 3)); + let mut incoming = vec![0_usize; vertices]; + let mut outgoing = vec![0_usize; vertices]; + for vertex in 0..vertices { + if vertex >= roles.len() { + let available = (0..vertex) + .filter(|&source| outgoing[source] < 4) + .collect::>(); + let parent = available[(random.next_u64() as usize) % available.len()]; + let parent_node = case.processes[roles[parent]].logical_node_id; + let choices = nodes + .iter() + .copied() + .filter(|&node| node != parent_node) + .collect::>(); + let node = choices[(random.next_u64() as usize) % choices.len()]; + let id = format!("topology-role-{vertex}"); + roles.push(case.processes.len()); + case.processes.push(ProcessProgram { + id: id.clone(), + logical_node_id: node, + access: AccessSpec::unrestricted(format!("{}-{id}", case.id)), + depends_on: Vec::new(), + actions: Vec::new(), + }); + } + if vertex == 0 { + continue; + } + let node = case.processes[roles[vertex]].logical_node_id; + let parents = (0..vertex) + .filter(|&source| { + outgoing[source] < 4 && case.processes[roles[source]].logical_node_id != node + }) + .collect::>(); + if parents.is_empty() { + return Err(format!("case {} has no live-node DAG parent", case.id)); + } + let source = parents[(random.next_u64() as usize) % parents.len()]; + edges.push((source, vertex)); + outgoing[source] += 1; + incoming[vertex] += 1; + } + let small_edges = edges[..2].to_vec(); + // Every role belongs to the random spanning tree; extra independently + // selected forward edges introduce joins without changing acyclicity. + let mut candidates = (0..vertices) + .flat_map(|source| (source + 1..vertices).map(move |destination| (source, destination))) + .filter(|&(source, destination)| { + case.processes[roles[source]].logical_node_id + != case.processes[roles[destination]].logical_node_id + && !edges.contains(&(source, destination)) + }) + .collect::>(); + for end in (1..candidates.len()).rev() { + let other = (random.next_u64() as usize) % (end + 1); + candidates.swap(end, other); + } + let extra_limit = (max_edges - edges.len()).min(4).min(candidates.len()); + let target = edges.len() + (random.next_u64() as usize) % (extra_limit + 1); + let mut copies = [0_usize; 20]; + for (source, destination) in candidates { + if edges.len() == target { + break; + } + if outgoing[source] == 4 || incoming[destination] == 4 { + continue; + } + edges.push((source, destination)); + // Bound concatenation growth by independently counting source payload + // contributions. Eight copies per edge keeps even large framed DAGs + // well below the existing per-case allocation/payload ceiling. + for vertex in 0..vertices { + copies[vertex] = edges + .iter() + .filter(|&&(_, destination)| destination == vertex) + .map(|&(source, _)| copies[source]) + .sum::() + .max(1); + } + if copies.iter().any(|&copies| copies > 8) { + edges.pop(); + continue; + } + outgoing[source] += 1; + incoming[destination] += 1; + } + edges.sort_unstable(); + if kinds.len() > 1 { + append_role_route(case, small_kind, &roles[..3], &small_edges); + } + append_role_route(case, large_kind, &roles, &edges); + Ok(()) +} + +fn append_role_route( + case: &mut BehaviorCase, + kind: DataKind, + roles: &[usize], + edges: &[(usize, usize)], +) { + let label = if kind == DataKind::Blob { + "topology-blob" + } else { + "topology-stream" + }; + let payload = boundary_payload( + case.seed ^ label.bytes().map(u64::from).sum::(), + if kind == DataKind::Blob { 257 } else { 4_097 }, + ); + let paths = edges + .iter() + .enumerate() + .map(|(edge, (source, destination))| { + format!( + "/cases/{}/routes/{label}/{edge}-role-{source}-{destination}", + case.id + ) + }) + .collect::>(); + let mut edge_payloads = vec![Vec::new(); edges.len()]; + for vertex in 0..roles.len() { + if !edges.iter().any(|&(source, _)| source == vertex) { + continue; + } + let inputs = edges + .iter() + .enumerate() + .filter(|&(_, &(_, destination))| destination == vertex) + .map(|(edge, _)| edge_payloads[edge].as_slice()) + .collect::>(); + let relay = !inputs.is_empty(); + let joined = if relay { + inputs.concat() + } else { + payload.clone() + }; + for (edge, &(source, _)) in edges.iter().enumerate() { + if source == vertex { + let mut output = joined.clone(); + if relay && !output.is_empty() { + let rotation = (edge + 1) % output.len(); + output.rotate_left(rotation); + } + edge_payloads[edge] = output; + } + } + } + let route_edges = edges + .iter() + .zip(&paths) + .map(|(&(source, destination), path)| { + let source = &case.processes[roles[source]]; + let destination = &case.processes[roles[destination]]; + DataEdge { + source: source.logical_node_id, + destination: destination.logical_node_id, + source_role: source.id.clone(), + destination_role: destination.id.clone(), + path: path.clone(), + } + }) + .collect(); + let mut join_inputs = Vec::new(); + for (vertex, &process) in roles.iter().enumerate() { + let incoming = edges + .iter() + .enumerate() + .filter(|&(_, &(_, destination))| destination == vertex) + .map(|(edge, _)| edge) + .collect::>(); + if incoming.len() > 1 { + join_inputs.extend(incoming.iter().map(|&edge| paths[edge].clone())); + } + let actions = &mut case.processes[process].actions; + for edge in incoming { + append_read_actions(actions, kind, &paths[edge], &edge_payloads[edge], edge); + } + for (edge, &(source, _)) in edges.iter().enumerate() { + if source == vertex { + append_write_action(actions, kind, &paths[edge], &edge_payloads[edge], edge); + } + } + } + case.routes.push(DataRoute { + id: format!("{}-{label}", case.id), + kind, + edges: route_edges, + join_inputs, + }); +} + +fn topology_edges(family: TopologyFamily, nodes: &[u64], index: usize) -> Vec<(u64, u64)> { + let mut rotated = nodes.to_vec(); + let length = rotated.len(); + rotated.rotate_left(index % length); + match family { + TopologyFamily::Fixed | TopologyFamily::Chain => { + rotated.windows(2).map(|pair| (pair[0], pair[1])).collect() + } + TopologyFamily::RingWalk => { + // Rings traverse multiple laps with a fresh data path per lap, so + // generation keeps a compact rotating ring: action bounds hold for + // every live-set size while rotation still spreads relay duty + // across all nodes over the campaign. + let ring = &rotated[..length.min(3)]; + ring.iter() + .copied() + .zip(ring.iter().copied().cycle().skip(1)) + .take(ring.len()) + .collect() + } + TopologyFamily::FanOut => rotated[1..] + .iter() + .map(|destination| (rotated[0], *destination)) + .collect(), + TopologyFamily::FanIn => rotated[..rotated.len() - 1] + .iter() + .map(|source| (*source, rotated[rotated.len() - 1])) + .collect(), + TopologyFamily::Diamond => vec![ + (rotated[0], rotated[1]), + (rotated[0], rotated[2]), + (rotated[1], rotated[3]), + (rotated[2], rotated[3]), + ], + TopologyFamily::RandomDag => unreachable!("random DAGs use independent process vertices"), + } +} + +fn append_executable_route( + case: &mut BehaviorCase, + kind: DataKind, + label: &str, + edges: &[(u64, u64)], + family: TopologyFamily, +) -> Result<(), String> { + if edges.is_empty() { + return Err(format!("case {} route {label} has no edges", case.id)); + } + if family == TopologyFamily::RingWalk && edges.len() > 1 { + append_ring_processes(case, kind, label, edges); + return Ok(()); + } + let payload_seed = case.seed ^ label.bytes().map(u64::from).sum::(); + let length = if family == TopologyFamily::Fixed { + let mut random = + XorShift64(payload_seed ^ edges[0].0.rotate_left(13) ^ edges[0].1.rotate_left(29)); + 258 + (random.next_u64() % (4_095 - 258)) as usize + } else if kind == DataKind::Blob { + 257 + } else { + 4_097 + }; + let payload = boundary_payload(payload_seed, length); + let paths = edges + .iter() + .enumerate() + .map(|(edge, (source, destination))| { + format!( + "/cases/{}/routes/{label}/{edge}-{source}-{destination}", + case.id + ) + }) + .collect::>(); + let roles = append_dag_processes(case, kind, label, edges, &paths, &payload)?; + let route_id = format!("{}-{label}", case.id); + case.routes.push(DataRoute { + id: route_id, + kind, + edges: edges + .iter() + .zip(&paths) + .map(|((source, destination), path)| DataEdge { + source: *source, + destination: *destination, + source_role: roles[source].clone(), + destination_role: roles[destination].clone(), + path: path.clone(), + }) + .collect(), + join_inputs: join_inputs(edges, &paths), + }); + Ok(()) +} + +fn join_inputs(edges: &[(u64, u64)], paths: &[String]) -> Vec { + let mut incoming = BTreeMap::>::new(); + for ((_, destination), path) in edges.iter().zip(paths) { + incoming.entry(*destination).or_default().push(path.clone()); + } + incoming + .into_values() + .filter(|paths| paths.len() > 1) + .flatten() + .collect() +} + +fn append_dag_processes( + case: &mut BehaviorCase, + kind: DataKind, + label: &str, + edges: &[(u64, u64)], + paths: &[String], + payload: &[u8], +) -> Result, String> { + let involved = edges + .iter() + .flat_map(|(source, destination)| [*source, *destination]) + .collect::>(); + // Derive expectations from the graph, separately from the generated + // binding program's actual received-byte transformations. + let mut edge_payloads = vec![None::>; edges.len()]; + let mut pending = involved.clone(); + while !pending.is_empty() { + let node = pending + .iter() + .copied() + .find(|node| { + edges.iter().enumerate().all(|(edge, (_, destination))| { + destination != node || edge_payloads[edge].is_some() + }) + }) + .ok_or_else(|| format!("case {} route {label} is cyclic", case.id))?; + let incoming = edges + .iter() + .enumerate() + .filter(|(_, (_, destination))| *destination == node) + .map(|(edge, _)| edge_payloads[edge].as_deref().expect("ready incoming edge")) + .collect::>(); + let joined = if incoming.is_empty() { + payload.to_vec() + } else { + incoming.concat() + }; + let relay = !incoming.is_empty(); + for (edge, (source, _)) in edges.iter().enumerate() { + if *source == node { + let mut output = joined.clone(); + if relay && !output.is_empty() { + let rotation = (edge + 1) % output.len(); + output.rotate_left(rotation); + } + edge_payloads[edge] = Some(output); + } + } + pending.remove(&node); + } + let mut roles = BTreeMap::new(); + for node in involved { + let mut actions = Vec::new(); + for (((_, destination), path), edge_index) in edges.iter().zip(paths).zip(0_usize..) { + if *destination == node { + append_read_actions( + &mut actions, + kind, + path, + edge_payloads[edge_index] + .as_deref() + .expect("derived route payload"), + edge_index, + ); + } + } + for (((source, _), path), edge_index) in edges.iter().zip(paths).zip(0_usize..) { + if *source == node { + append_write_action( + &mut actions, + kind, + path, + edge_payloads[edge_index] + .as_deref() + .expect("derived route payload"), + edge_index, + ); + } + } + let id = format!("{label}-node-{node}"); + if label == "topology-stream" { + let blob_id = format!("topology-blob-node-{node}"); + if let Some(program) = case + .processes + .iter_mut() + .find(|program| program.id == blob_id) + { + program.actions.extend(actions); + roles.insert(node, blob_id); + continue; + } + } + case.processes.push(ProcessProgram { + id: id.clone(), + logical_node_id: node, + access: AccessSpec::unrestricted(format!("{}-{label}-node-{node}", case.id)), + depends_on: Vec::new(), + actions, + }); + roles.insert(node, id); + } + Ok(roles) +} + +/// Lap count for generated rings; the plan requires at least two laps so a +/// ring proves repeated traversal rather than a single cycle. +const RING_LAPS: usize = 2; + +/// Labels a ring action with its lap token identity, lap, edge, and barrier +/// kind using the canonical evidence-hint keys shared with codegen. +fn tag_ring_action(action: &mut Action, token: &str, lap: usize, edge_index: usize, barrier: &str) { + action + .evidence_hints + .insert(EVIDENCE_HINT_TOKEN.to_owned(), token.to_owned()); + action + .evidence_hints + .insert(EVIDENCE_HINT_LAP.to_owned(), lap.to_string()); + action + .evidence_hints + .insert(EVIDENCE_HINT_EDGE_INDEX.to_owned(), edge_index.to_string()); + action + .evidence_hints + .insert(EVIDENCE_HINT_BARRIER.to_owned(), barrier.to_owned()); +} + +/// Appends a ring edge read and labels the consuming action with the lap's +/// token identity so observations prove the per-edge receive milestone. +fn append_ring_read( + actions: &mut Vec, + kind: DataKind, + path: &str, + token: &[u8], + edge_index: usize, + lap: usize, + barrier: &str, +) { + append_read_actions(actions, kind, path, token, edge_index); + tag_ring_action( + actions + .last_mut() + .expect("ring read appends a consuming action"), + &crate::codegen::digest(token), + lap, + edge_index, + barrier, + ); +} + +/// Appends a ring edge write and labels the forwarding action with the lap's +/// token identity so observations prove the per-edge forward milestone. +fn append_ring_write( + actions: &mut Vec, + kind: DataKind, + path: &str, + token: &[u8], + edge_index: usize, + lap: usize, + barrier: &str, +) { + append_write_action(actions, kind, path, token, edge_index); + tag_ring_action( + actions + .last_mut() + .expect("ring write appends a forwarding action"), + &crate::codegen::digest(token), + lap, + edge_index, + barrier, + ); +} + +fn append_ring_processes( + case: &mut BehaviorCase, + kind: DataKind, + label: &str, + edges: &[(u64, u64)], +) { + let role_label = if label == "topology-stream" + && case + .processes + .iter() + .any(|process| process.id == "topology-blob-origin-relay") + { + "topology-blob" + } else { + label + }; + // Ring execution (plan 10.2): every participant starts concurrently, each + // lap carries an explicit token identity, every non-origin relay reads + // before it forwards, the origin forwards its outgoing-edge token while a + // separate reader on the same node concurrently awaits the final lap's + // arrival, and the ring ends on clean EOF once the final token is + // accounted for. Each lap uses its own data path per edge, so no data + // edge is ever reused across laps. + let laps = RING_LAPS.max(2); + let hops = edges.len(); + let case_id = case.id.clone(); + let case_seed = case.seed; + let hop_path = |lap: usize, hop: usize| -> String { + let (source, destination) = edges[hop]; + format!("/cases/{case_id}/routes/{label}/lap-{lap}/hop-{hop}-{source}-{destination}") + }; + let token = |_lap: usize| -> Vec { + let length = if kind == DataKind::Blob { 257 } else { 4_097 }; + let body = boundary_payload( + case_seed ^ label.bytes().map(u64::from).sum::(), + length, + ); + let header = format!("ring-token/{label}"); + [header.as_bytes(), body.as_slice()].concat() + }; + case.routes.push(DataRoute { + id: format!("{case_id}-{label}"), + kind, + edges: (0..laps) + .flat_map(|lap| (0..hops).map(move |hop| (lap, hop))) + .map(|(lap, hop)| { + let (source, destination) = edges[hop]; + DataEdge { + source, + destination, + source_role: if hop == 0 { + format!("{role_label}-origin-relay") + } else { + format!("{role_label}-relay-{source}") + }, + destination_role: if hop + 1 < hops { + format!("{role_label}-relay-{destination}") + } else if lap + 1 < laps { + format!("{role_label}-origin-relay") + } else { + format!("{role_label}-origin-reader") + }, + path: hop_path(lap, hop), + } + }) + .collect(), + join_inputs: Vec::new(), + }); + let origin = edges[0].0; + let arrival = hops - 1; + let mut origin_actions = Vec::new(); + append_ring_write( + &mut origin_actions, + kind, + &hop_path(0, 0), + &token(0), + 0, + 0, + BARRIER_TOKEN_FORWARDED, + ); + for lap in 0..laps - 1 { + append_ring_read( + &mut origin_actions, + kind, + &hop_path(lap, arrival), + &token(lap), + lap * hops + arrival, + lap, + BARRIER_TOKEN_RECEIVED, + ); + append_ring_write( + &mut origin_actions, + kind, + &hop_path(lap + 1, 0), + &token(lap + 1), + (lap + 1) * hops, + lap + 1, + BARRIER_TOKEN_FORWARDED, + ); + } + append_ring_role( + case, + format!("{role_label}-origin-relay"), + origin, + origin_actions, + ); + let mut reader_actions = Vec::new(); + append_ring_read( + &mut reader_actions, + kind, + &hop_path(laps - 1, arrival), + &token(laps - 1), + (laps - 1) * hops + arrival, + laps - 1, + BARRIER_LAP_COMPLETED, + ); + append_ring_role( + case, + format!("{role_label}-origin-reader"), + origin, + reader_actions, + ); + for hop in 1..hops { + let node = edges[hop].0; + let mut actions = Vec::new(); + for lap in 0..laps { + append_ring_read( + &mut actions, + kind, + &hop_path(lap, hop - 1), + &token(lap), + lap * hops + hop - 1, + lap, + BARRIER_TOKEN_RECEIVED, + ); + append_ring_write( + &mut actions, + kind, + &hop_path(lap, hop), + &token(lap), + lap * hops + hop, + lap, + BARRIER_TOKEN_FORWARDED, + ); + } + append_ring_role(case, format!("{role_label}-relay-{node}"), node, actions); + } +} + +fn append_ring_role( + case: &mut BehaviorCase, + id: String, + logical_node_id: u64, + actions: Vec, +) { + if let Some(process) = case.processes.iter_mut().find(|process| process.id == id) { + process.actions.extend(actions); + return; + } + case.processes.push(ProcessProgram { + access: AccessSpec::unrestricted(format!("{}-{id}", case.id)), + id, + logical_node_id, + depends_on: Vec::new(), + actions, + }); +} + +fn append_read_actions( + actions: &mut Vec, + kind: DataKind, + path: &str, + payload: &[u8], + edge_index: usize, +) { + match kind { + DataKind::Blob => { + actions.push(Action::ok(ActionOp::AwaitEntry { + path: path.to_owned(), + expected_kind: "blob".to_owned(), + })); + actions.push(Action::ok(ActionOp::ReadBlob { + path: path.to_owned(), + expected: payload.to_vec(), + })); + } + DataKind::Stream => { + let operation = if edge_index % 2 == 0 { + ActionOp::StreamRead { + path: path.to_owned(), + expected: payload.to_vec(), + } + } else { + ActionOp::StreamReadInto { + path: path.to_owned(), + expected: payload.to_vec(), + buffer_sizes: vec![1, 255, 4_096, 17], + } + }; + actions.push(Action::ok(operation)); + } + } +} + +fn append_write_action( + actions: &mut Vec, + kind: DataKind, + path: &str, + payload: &[u8], + edge_index: usize, +) { + match kind { + DataKind::Blob => actions.push(Action::ok(ActionOp::PublishBlob { + path: path.to_owned(), + bytes: payload.to_vec(), + })), + DataKind::Stream => { + let split = (edge_index % payload.len().max(1)) + .max(1) + .min(payload.len()); + actions.push(Action::ok(ActionOp::StreamWrite { + path: path.to_owned(), + chunks: vec![payload[..split].to_vec(), payload[split..].to_vec()], + replace: false, + })); + } + } +} + +pub fn random_short_dags(seed: u64, count: usize, node_count: u8) -> Vec { + random_short_dags_iter(seed, count, node_count).collect() +} + +pub fn random_short_dags_budgeted( + seed: u64, + count: usize, + node_count: u8, + budget: &Budget, +) -> Result, String> { + let mut cases = Vec::new(); + let mut generated = random_short_dags_iter(seed, count, node_count); + for _ in 0..count { + budget.check("generate deterministic random DAG and boundary payload")?; + cases.push( + generated + .next() + .expect("generator emits one case per requested index"), + ); + } + Ok(cases) +} + +fn random_short_dags_iter( + seed: u64, + count: usize, + node_count: u8, +) -> impl Iterator { + let mut random = XorShift64(seed); + (0..count).map(move |index| { + let source_node = 1 + random.next_u64() % u64::from(node_count); + let mut destination_node = 1 + random.next_u64() % u64::from(node_count); + if destination_node == source_node { + destination_node = destination_node % u64::from(node_count) + 1; + } + let length = [0, 1, 4095, 4096, 4097, 65_537][(random.next_u64() as usize) % 6]; + let shape = random.next_u64() % 5; + let id = format!("random-{seed}-{index}"); + let path = format!("/cases/random/{seed}/{index}"); + let bytes = boundary_payload(random.next_u64(), length); + let processes = match shape { + 0 => vec![ + ProcessProgram { + id: "producer".to_owned(), + logical_node_id: source_node, + access: AccessSpec::unrestricted(format!("{id}-producer")), + depends_on: Vec::new(), + actions: vec![ + Action::ok(ActionOp::PublishBlob { + path: path.clone(), + bytes: bytes.clone(), + }), + Action::ok(ActionOp::Lookup { + path: path.clone(), + expected_kind: "blob".to_owned(), + }), + ], + }, + ProcessProgram { + id: "consumer".to_owned(), + logical_node_id: destination_node, + access: AccessSpec::unrestricted(format!("{id}-consumer")), + depends_on: vec!["producer".to_owned()], + actions: vec![Action::ok(ActionOp::ReadBlob { + path, + expected: bytes, + })], + }, + ], + 1 => { + let destination = format!("{path}-renamed"); + vec![ ProcessProgram { - id: "producer".to_owned(), + id: "mutator".to_owned(), logical_node_id: source_node, - access: AccessSpec::unrestricted(format!("{id}-producer")), + access: AccessSpec::unrestricted(format!("{id}-mutator")), depends_on: Vec::new(), actions: vec![ Action::ok(ActionOp::PublishBlob { path: path.clone(), bytes: bytes.clone(), }), + Action::ok(ActionOp::Rename { + source: path, + destination: destination.clone(), + replace: false, + }), Action::ok(ActionOp::Lookup { - path: path.clone(), + path: destination.clone(), expected_kind: "blob".to_owned(), }), ], @@ -1352,177 +3053,263 @@ pub fn random_short_dags(seed: u64, count: usize, node_count: u8) -> Vec { - let destination = format!("{path}-renamed"); - vec![ - ProcessProgram { - id: "mutator".to_owned(), - logical_node_id: source_node, - access: AccessSpec::unrestricted(format!("{id}-mutator")), - depends_on: Vec::new(), - actions: vec![ - Action::ok(ActionOp::PublishBlob { - path: path.clone(), - bytes: bytes.clone(), - }), - Action::ok(ActionOp::Rename { - source: path, - destination: destination.clone(), - replace: false, - }), - Action::ok(ActionOp::Lookup { - path: destination.clone(), - expected_kind: "blob".to_owned(), - }), - ], - }, - ProcessProgram { - id: "consumer".to_owned(), - logical_node_id: destination_node, - access: AccessSpec::unrestricted(format!("{id}-consumer")), - depends_on: vec!["mutator".to_owned()], - actions: vec![Action::ok(ActionOp::ReadBlob { - path: destination, - expected: bytes, - })], - }, - ] - } - 2 => { - let payload = if bytes.is_empty() { vec![0] } else { bytes }; - let write_method = match random.next_u64() % 3 { - 0 => DescriptorWriteMethod::Write, - 1 => DescriptorWriteMethod::WriteFrom, - _ => DescriptorWriteMethod::Mapping, - }; - let read_method = match random.next_u64() % 3 { - 0 => DescriptorReadMethod::Read, - 1 => DescriptorReadMethod::ReadInto, - _ => DescriptorReadMethod::Mapping, - }; - vec![ - ProcessProgram { - id: "descriptor-writer".to_owned(), - logical_node_id: source_node, - access: AccessSpec::unrestricted(format!("{id}-writer")), - depends_on: Vec::new(), - actions: vec![Action::ok(ActionOp::DescriptorWrite { - path: path.clone(), - flags: libc::O_WRONLY - | libc::O_CREAT - | libc::O_EXCL - | libc::O_TRUNC, - length: Some(payload.len() as u64), - bytes: payload.clone(), - method: write_method, - finish: DescriptorFinish::Close, - })], - }, - ProcessProgram { - id: "descriptor-reader".to_owned(), - logical_node_id: destination_node, - access: AccessSpec::unrestricted(format!("{id}-reader")), - depends_on: vec!["descriptor-writer".to_owned()], - actions: vec![Action::ok(ActionOp::DescriptorRead { - path, - flags: libc::O_RDONLY, - expected: payload, - method: read_method, - offset: 0, - finish: DescriptorFinish::Close, - })], - }, - ] - } - 3 => { - let payload = if bytes.is_empty() { vec![0] } else { bytes }; - let split = payload.len() / 2; - let chunks = vec![payload[..split].to_vec(), payload[split..].to_vec()]; - vec![ - ProcessProgram { - id: "stream-writer".to_owned(), - logical_node_id: source_node, - access: AccessSpec::unrestricted(format!("{id}-writer")), - depends_on: Vec::new(), - actions: vec![Action::ok(ActionOp::StreamWrite { - path: path.clone(), - chunks, - replace: false, - })], - }, - ProcessProgram { - id: "stream-reader".to_owned(), - logical_node_id: destination_node, - access: AccessSpec::unrestricted(format!("{id}-reader")), - depends_on: Vec::new(), - actions: vec![Action::ok(ActionOp::StreamRead { - path, - expected: payload, - })], - }, - ] - } - _ => { - let replacement = boundary_payload(random.next_u64(), length.saturating_add(1)); - vec![ - ProcessProgram { - id: "replacer".to_owned(), - logical_node_id: source_node, - access: AccessSpec::unrestricted(format!("{id}-replacer")), - depends_on: Vec::new(), - actions: vec![ - Action::ok(ActionOp::PublishBlob { - path: path.clone(), - bytes, - }), - Action::ok(ActionOp::DescriptorWrite { - path: path.clone(), - flags: libc::O_WRONLY | libc::O_TRUNC, - length: Some(replacement.len() as u64), - bytes: replacement.clone(), - method: DescriptorWriteMethod::WriteFrom, - finish: DescriptorFinish::Close, - }), - ], - }, - ProcessProgram { - id: "replacement-reader".to_owned(), - logical_node_id: destination_node, - access: AccessSpec::unrestricted(format!("{id}-reader")), - depends_on: vec!["replacer".to_owned()], - actions: vec![Action::ok(ActionOp::ReadBlob { - path, - expected: replacement, - })], - }, - ] - } - }; - BehaviorCase { - id, - seed: random.next_u64(), - node_count, - processes, - failure: FailureInjection::None, + ] } - }) - .collect() + 2 => { + let payload = if bytes.is_empty() { vec![0] } else { bytes }; + let write_method = match random.next_u64() % 3 { + 0 => DescriptorWriteMethod::Write, + 1 => DescriptorWriteMethod::WriteFrom, + _ => DescriptorWriteMethod::Mapping, + }; + let read_method = match random.next_u64() % 3 { + 0 => DescriptorReadMethod::Read, + 1 => DescriptorReadMethod::ReadInto, + _ => DescriptorReadMethod::Mapping, + }; + vec![ + ProcessProgram { + id: "descriptor-writer".to_owned(), + logical_node_id: source_node, + access: AccessSpec::unrestricted(format!("{id}-writer")), + depends_on: Vec::new(), + actions: vec![Action::ok(ActionOp::DescriptorWrite { + path: path.clone(), + flags: libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL | libc::O_TRUNC, + length: Some(payload.len() as u64), + bytes: payload.clone(), + method: write_method, + finish: DescriptorFinish::Close, + })], + }, + ProcessProgram { + id: "descriptor-reader".to_owned(), + logical_node_id: destination_node, + access: AccessSpec::unrestricted(format!("{id}-reader")), + depends_on: vec!["descriptor-writer".to_owned()], + actions: vec![Action::ok(ActionOp::DescriptorRead { + path, + flags: libc::O_RDONLY, + expected: payload, + method: read_method, + offset: 0, + finish: DescriptorFinish::Close, + })], + }, + ] + } + 3 => { + let payload = if bytes.is_empty() { vec![0] } else { bytes }; + let split = payload.len() / 2; + let chunks = vec![payload[..split].to_vec(), payload[split..].to_vec()]; + vec![ + ProcessProgram { + id: "stream-writer".to_owned(), + logical_node_id: source_node, + access: AccessSpec::unrestricted(format!("{id}-writer")), + depends_on: Vec::new(), + actions: vec![Action::ok(ActionOp::StreamWrite { + path: path.clone(), + chunks, + replace: false, + })], + }, + ProcessProgram { + id: "stream-reader".to_owned(), + logical_node_id: destination_node, + access: AccessSpec::unrestricted(format!("{id}-reader")), + depends_on: Vec::new(), + actions: vec![Action::ok(ActionOp::StreamRead { + path, + expected: payload, + })], + }, + ] + } + _ => { + let replacement = boundary_payload(random.next_u64(), length.saturating_add(1)); + vec![ + ProcessProgram { + id: "replacer".to_owned(), + logical_node_id: source_node, + access: AccessSpec::unrestricted(format!("{id}-replacer")), + depends_on: Vec::new(), + actions: vec![ + Action::ok(ActionOp::PublishBlob { + path: path.clone(), + bytes, + }), + Action::ok(ActionOp::DescriptorWrite { + path: path.clone(), + flags: libc::O_WRONLY | libc::O_TRUNC, + length: Some(replacement.len() as u64), + bytes: replacement.clone(), + method: DescriptorWriteMethod::WriteFrom, + finish: DescriptorFinish::Close, + }), + ], + }, + ProcessProgram { + id: "replacement-reader".to_owned(), + logical_node_id: destination_node, + access: AccessSpec::unrestricted(format!("{id}-reader")), + depends_on: vec!["replacer".to_owned()], + actions: vec![Action::ok(ActionOp::ReadBlob { + path, + expected: replacement, + })], + }, + ] + } + }; + BehaviorCase { + id, + seed: random.next_u64(), + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + live_nodes: crate::ir::contiguous_nodes(node_count), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), + processes, + failure: FailureInjection::None, + } + }) } struct XorShift64(u64); impl XorShift64 { fn next_u64(&mut self) -> u64 { + if self.0 == 0 { + self.0 = 0x9e37_79b9_7f4a_7c15; + } self.0 ^= self.0 << 13; self.0 ^= self.0 >> 7; self.0 ^= self.0 << 17; self.0 } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn generated_campaign_topologies_preserve_survivor_cases_and_all_pairs() { + for (nodes, count) in [(3, 32), (4, 32), (5, 128)] { + let cases = generated_campaign_cases(17, count, nodes).unwrap(); + assert_eq!(cases.len(), count); + let mut observed_pairs = BTreeSet::new(); + for case in &cases { + case.validate().unwrap(); + let resources = case.resource_summary().unwrap(); + assert!((16..=64).contains(&resources.action_count)); + assert!((2..=20).contains(&resources.process_count)); + if nodes == 3 { + assert_ne!(case.topology, TopologyFamily::Diamond); + } + for route in &case.routes { + observed_pairs.extend( + route + .edges + .iter() + .map(|edge| (edge.source, edge.destination, route.kind)), + ); + } + } + for case in &cases { + for kind in [DataKind::Blob, DataKind::Stream] { + assert!( + cases.iter().any(|candidate| { + candidate.topology == case.topology + && candidate.routes.iter().any(|route| { + route.kind == kind + && crate::coverage::route_has_topology( + route, + candidate.topology, + ) + }) + }), + "{nodes}-node campaign lacks {:?} {kind:?}", + case.topology + ); + } + } + for source in 1..=u64::from(nodes) { + for destination in 1..=u64::from(nodes) { + if source != destination { + for kind in [DataKind::Blob, DataKind::Stream] { + assert!( + observed_pairs.contains(&(source, destination, kind)), + "{nodes}-node campaign omitted {source}->{destination} {kind:?}" + ); + } + } + } + } + } + } + + #[test] + fn random_role_dags_cover_vertex_bounds_with_seed_varied_branching() { + let first = generated_campaign_cases(17, 128, 5).unwrap(); + let repeated = generated_campaign_cases(17, 128, 5).unwrap(); + let different = generated_campaign_cases(35, 128, 5).unwrap(); + assert_eq!(first, repeated); + let mut vertex_counts = BTreeSet::new(); + let mut changed_topologies = 0; + for (left, right) in first.iter().zip(&different) { + if left.topology != TopologyFamily::RandomDag { + continue; + } + let signature = |case: &BehaviorCase| { + case.routes + .iter() + .map(|route| { + ( + route.kind, + route + .edges + .iter() + .map(|edge| { + (edge.source_role.clone(), edge.destination_role.clone()) + }) + .collect::>(), + ) + }) + .collect::>() + }; + changed_topologies += usize::from(signature(left) != signature(right)); + for route in &left.routes { + let mut degrees = BTreeMap::<&str, (usize, usize)>::new(); + for edge in &route.edges { + degrees.entry(&edge.source_role).or_default().1 += 1; + degrees.entry(&edge.destination_role).or_default().0 += 1; + } + vertex_counts.insert(degrees.len()); + assert!( + degrees + .values() + .all(|&(input, output)| input <= 4 && output <= 4) + ); + assert!(crate::coverage::route_has_topology( + route, + TopologyFamily::RandomDag + )); + } + } + assert_eq!(vertex_counts, (3..=20).collect()); + // The seeds select identical vertex-count slots. Changes therefore + // prove actual graph variation, not relabeling or a size-only switch. + assert!(changed_topologies >= 8); + } +} diff --git a/tools/myelin-e2e-fuzz/src/coverage.rs b/tools/myelin-e2e-fuzz/src/coverage.rs new file mode 100644 index 0000000..ecb57d1 --- /dev/null +++ b/tools/myelin-e2e-fuzz/src/coverage.rs @@ -0,0 +1,4447 @@ +//! Planned and completed coverage ledgers for fixed campaign requirements. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::ir::{ + ActionClass, ActionObservation, ActionOp, BarrierObservation, BehaviorCase, CaseObservation, + CoverageScenario, DataKind, DataRoute, DescriptorFinish, DescriptorObservation, + DescriptorReadMethod, DescriptorTerminalResult, DescriptorWriteMethod, ExecutionObservation, + ExpectedOutcome, FailureInjection, ProcessProgram, TopologyFamily, +}; + +pub const COVERAGE_SCHEMA_VERSION: u32 = 5; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum NodeRole { + Source, + Sink, + Relay, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PayloadClass { + Empty, + Small, + FramingBoundary, + ChunkBoundary, + MultiChunk, + Randomized, +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum CoverageKey { + Topology { + family: TopologyFamily, + }, + NodeRole { + node: u64, + kind: DataKind, + role: NodeRole, + }, + OrderedEdge { + source: u64, + destination: u64, + kind: DataKind, + }, + ActionAdjacency { + before: ActionClass, + after: ActionClass, + }, + Payload { + class: PayloadClass, + }, + DescriptorMethod { + direction: String, + method: String, + }, + DescriptorTerminal { + direction: String, + finish: String, + }, + Outcome { + class: String, + }, + ConcurrentStart { + family: TopologyFamily, + }, + Scenario { + scenario: CoverageScenario, + }, +} + +/// Immutable environment that produced coverage. Changing any component +/// requires fresh evidence, even when the campaign seed and node set agree. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EvidenceIdentity { + pub plan_digest: String, + pub artifacts_digest: String, + pub deployment_generation: String, + pub fixture_mapping_digest: String, +} + +impl EvidenceIdentity { + fn validate(&self) -> Result<(), String> { + if !valid_digest(&self.plan_digest) + || !valid_digest(&self.artifacts_digest) + || !valid_digest(&self.fixture_mapping_digest) + || self.deployment_generation.trim().is_empty() + { + return Err("coverage evidence identity is incomplete".to_owned()); + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +struct PlannedCaseEvidence { + case_digest: String, + keys: BTreeSet, + execution_ids: BTreeMap, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +struct CompletedCaseEvidence { + planned_case_digest: String, + identity_digest: String, + executed_case_digest: String, + observation_digest: String, + attempt: Option<(u64, bool)>, + execution_ids: BTreeMap, + request_ids: BTreeMap, + keys: BTreeSet, +} + +fn valid_digest(digest: &str) -> bool { + digest.len() == 64 && digest.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +fn evidence_digest(value: &impl Serialize) -> Result { + struct DigestWriter(Sha256); + impl std::io::Write for DigestWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.update(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let mut writer = DigestWriter(Sha256::new()); + serde_json::to_writer(&mut writer, value) + .map_err(|error| format!("hash coverage evidence: {error}"))?; + Ok(format!("{:x}", writer.0.finalize())) +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct CoverageLedger { + pub schema_version: u32, + pub live_nodes: BTreeSet, + #[serde(with = "coverage_map")] + pub required: BTreeMap, + #[serde(with = "coverage_map")] + pub planned: BTreeMap, + #[serde(with = "coverage_map")] + pub observed: BTreeMap, + pub completed_cases: BTreeSet, + identity: Option, + planned_cases: BTreeMap, + completed_evidence: BTreeMap, + /// Deserialization never authorizes reuse; the live consumer must compare + /// the persisted identity and complete evidence closure with its own plan. + #[serde(skip)] + identity_validated: bool, +} +mod coverage_map { + use std::collections::BTreeMap; + + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + + use super::CoverageKey; + + pub fn serialize(map: &BTreeMap, serializer: S) -> Result + where + S: Serializer, + { + map.iter().collect::>().serialize(serializer) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let entries = Vec::<(CoverageKey, u32)>::deserialize(deserializer)?; + let mut map = BTreeMap::new(); + for (key, count) in entries { + if map.insert(key, count).is_some() { + return Err(serde::de::Error::custom("duplicate coverage key")); + } + } + Ok(map) + } +} + +impl CoverageLedger { + pub fn five_node(live_nodes: BTreeSet) -> Result { + if live_nodes.len() != 5 { + return Err("five-node coverage ledger requires exactly five nodes".to_owned()); + } + let mut ledger = Self::survivor(live_nodes)?; + for family in [ + TopologyFamily::Chain, + TopologyFamily::RingWalk, + TopologyFamily::FanOut, + TopologyFamily::FanIn, + TopologyFamily::Diamond, + TopologyFamily::RandomDag, + ] { + ledger.require(CoverageKey::Topology { family }, 8); + } + let classes = [ + ActionClass::PublishBlob, + ActionClass::ReadBlob, + ActionClass::StreamWrite, + ActionClass::StreamRead, + ActionClass::Lookup, + ActionClass::Rename, + ActionClass::Unlink, + ActionClass::Descriptor, + ]; + for before in classes { + for after in classes { + ledger.require(CoverageKey::ActionAdjacency { before, after }, 1); + } + } + for class in [ + PayloadClass::Empty, + PayloadClass::Small, + PayloadClass::FramingBoundary, + PayloadClass::ChunkBoundary, + PayloadClass::MultiChunk, + PayloadClass::Randomized, + ] { + ledger.require(CoverageKey::Payload { class }, 1); + } + for (direction, methods) in [ + ("write", ["write", "write_from", "mapping"]), + ("read", ["read", "read_into", "mapping"]), + ] { + for method in methods { + ledger.require( + CoverageKey::DescriptorMethod { + direction: direction.to_owned(), + method: method.to_owned(), + }, + 1, + ); + } + for finish in ["close", "abort", "drop", "close_twice", "abort_twice"] { + ledger.require( + CoverageKey::DescriptorTerminal { + direction: direction.to_owned(), + finish: finish.to_owned(), + }, + 1, + ); + } + } + for class in ["success", "modeled_error", "contextual_failure"] { + ledger.require( + CoverageKey::Outcome { + class: class.to_owned(), + }, + 1, + ); + } + for family in [ + TopologyFamily::Chain, + TopologyFamily::RingWalk, + TopologyFamily::FanOut, + TopologyFamily::FanIn, + TopologyFamily::Diamond, + ] { + ledger.require(CoverageKey::ConcurrentStart { family }, 1); + } + for scenario in [ + CoverageScenario::ConcurrentStartup, + CoverageScenario::RingCompletion, + CoverageScenario::HotColdFairness, + CoverageScenario::ProcessChurn, + CoverageScenario::ActiveStreamWriterAbort, + CoverageScenario::ActiveStreamReaderStop, + CoverageScenario::ActiveMutation, + CoverageScenario::QuiescentMutation, + CoverageScenario::FailureIsolation, + CoverageScenario::Authorization, + ] { + ledger.require(CoverageKey::Scenario { scenario }, 1); + } + Ok(ledger) + } + + pub fn survivor(live_nodes: BTreeSet) -> Result { + if live_nodes.len() < 3 || live_nodes.len() > 5 || live_nodes.contains(&0) { + return Err( + "survivor coverage ledger requires three to five valid exact nodes".to_owned(), + ); + } + let mut ledger = Self { + schema_version: COVERAGE_SCHEMA_VERSION, + live_nodes: live_nodes.clone(), + required: BTreeMap::new(), + planned: BTreeMap::new(), + observed: BTreeMap::new(), + completed_cases: BTreeSet::new(), + identity: None, + planned_cases: BTreeMap::new(), + completed_evidence: BTreeMap::new(), + identity_validated: false, + }; + for kind in [DataKind::Blob, DataKind::Stream] { + for &node in &live_nodes { + for role in [NodeRole::Source, NodeRole::Sink, NodeRole::Relay] { + ledger.require(CoverageKey::NodeRole { node, kind, role }, 1); + } + } + for &source in &live_nodes { + for &destination in &live_nodes { + if source != destination { + ledger.require( + CoverageKey::OrderedEdge { + source, + destination, + kind, + }, + 1, + ); + } + } + } + } + Ok(ledger) + } + + fn require(&mut self, key: CoverageKey, count: u32) { + self.required + .entry(key) + .and_modify(|required| *required = (*required).max(count)) + .or_insert(count); + } + + pub fn identity(&self) -> Option<&EvidenceIdentity> { + self.identity.as_ref() + } + + pub fn plan_case(&mut self, case: &BehaviorCase) -> Result<(), String> { + case.validate()?; + if case.live_nodes != self.live_nodes { + return Err(format!( + "coverage ledger live set {:?} differs from case {} live set {:?}", + self.live_nodes, case.id, case.live_nodes + )); + } + if self.identity.is_some() || self.planned_cases.contains_key(&case.id) { + return Err(format!( + "case {} is duplicate or the coverage plan is sealed", + case.id + )); + } + let keys = case_coverage(case); + let evidence = PlannedCaseEvidence { + case_digest: evidence_digest(case)?, + execution_ids: case + .processes + .iter() + .map(|process| (process.id.clone(), process.access.execution_id.clone())) + .collect(), + keys, + }; + for key in &evidence.keys { + *self.planned.entry(key.clone()).or_default() += 1; + } + self.planned_cases.insert(case.id.clone(), evidence); + Ok(()) + } + + pub fn bind_identity(&mut self, identity: EvidenceIdentity) -> Result<(), String> { + identity.validate()?; + if !self.completed_cases.is_empty() + || !self.completed_evidence.is_empty() + || !self.observed.is_empty() + { + return Err( + "completed coverage must be validated against the live plan, not rebound" + .to_owned(), + ); + } + if self + .identity + .as_ref() + .is_some_and(|bound| bound != &identity) + { + return Err("coverage is already bound to a different deployment".to_owned()); + } + self.validate_evidence_closure()?; + self.identity = Some(identity); + self.identity_validated = true; + Ok(()) + } + + pub(crate) fn validate_plan(&self, expected: &Self) -> Result<(), String> { + self.validate_evidence_closure()?; + expected.validate_evidence_closure()?; + if self.schema_version != expected.schema_version + || self.live_nodes != expected.live_nodes + || self.required != expected.required + || self.planned != expected.planned + || self.planned_cases != expected.planned_cases + { + return Err("coverage differs from the immutable campaign cases".to_owned()); + } + Ok(()) + } + + pub fn validate_resume(&mut self, expected: &Self) -> Result<(), String> { + self.identity_validated = false; + expected.require_validated_identity()?; + self.validate_plan(expected)?; + if self.identity != expected.identity { + return Err( + "persisted coverage differs from the immutable plan or deployment identity" + .to_owned(), + ); + } + self.identity_validated = true; + Ok(()) + } + + fn require_validated_identity(&self) -> Result<(), String> { + if !self.identity_validated { + return Err("coverage deployment identity has not been validated".to_owned()); + } + self.identity + .as_ref() + .ok_or_else(|| "coverage has no deployment identity".to_owned())? + .validate() + } + + fn validate_evidence_closure(&self) -> Result<(), String> { + if self.schema_version != COVERAGE_SCHEMA_VERSION { + return Err("coverage schema is incompatible".to_owned()); + } + let mut planned = BTreeMap::new(); + for evidence in self.planned_cases.values() { + if !valid_digest(&evidence.case_digest) { + return Err("coverage has an invalid planned case digest".to_owned()); + } + for key in &evidence.keys { + let count = planned.entry(key.clone()).or_insert(0_u32); + *count = count + .checked_add(1) + .ok_or("planned coverage count overflow")?; + } + } + if planned != self.planned + || self.completed_cases.len() != self.completed_evidence.len() + || self + .completed_cases + .iter() + .any(|id| !self.completed_evidence.contains_key(id)) + { + return Err("coverage planned/completed evidence closure is corrupt".to_owned()); + } + let mut observed = BTreeMap::new(); + let mut requests = BTreeSet::new(); + let mut executions = BTreeSet::new(); + let identity_digest = self.identity.as_ref().map(evidence_digest).transpose()?; + for (id, evidence) in &self.completed_evidence { + let plan = self + .planned_cases + .get(id) + .ok_or_else(|| format!("coverage completed unplanned case {id}"))?; + if evidence.planned_case_digest != plan.case_digest + || identity_digest.as_ref() != Some(&evidence.identity_digest) + || !plan.execution_ids.keys().eq(evidence.execution_ids.keys()) + || plan.execution_ids.iter().any(|(process, execution_id)| { + let expected = match evidence.attempt { + Some((attempt, _)) => format!("{execution_id}-attempt-{attempt}"), + None => execution_id.clone(), + }; + evidence.execution_ids.get(process) != Some(&expected) + }) + || !valid_digest(&evidence.executed_case_digest) + || !valid_digest(&evidence.observation_digest) + || evidence.request_ids.is_empty() + || !evidence + .request_ids + .keys() + .eq(evidence.execution_ids.keys()) + || evidence + .request_ids + .values() + .any(|id| id.is_empty() || !requests.insert(id)) + || evidence + .execution_ids + .values() + .any(|id| id.is_empty() || !executions.insert(id)) + { + return Err(format!( + "coverage attempt evidence for {id} is corrupt or reused" + )); + } + for key in &evidence.keys { + let count = observed.entry(key.clone()).or_insert(0_u32); + *count = count + .checked_add(1) + .ok_or("observed coverage count overflow")?; + } + } + if observed != self.observed { + return Err( + "coverage observed counts differ from completed attempt evidence".to_owned(), + ); + } + Ok(()) + } + + pub fn observe_case( + &mut self, + case: &BehaviorCase, + observation: &CaseObservation, + ) -> Result<(), String> { + self.observe_case_inner(case, case, observation, None, None) + } + + pub fn observe_case_with_budget( + &mut self, + case: &BehaviorCase, + observation: &CaseObservation, + budget: &crate::budget::Budget, + ) -> Result<(), String> { + self.observe_case_inner(case, case, observation, None, Some(budget)) + } + + /// Record only an execution derived from the exact immutable planned case. + pub fn observe_attempt( + &mut self, + template: &BehaviorCase, + executed: &BehaviorCase, + observation: &CaseObservation, + attempt: u64, + isolate_paths: bool, + budget: &crate::budget::Budget, + ) -> Result<(), String> { + budget.check("coverage attempt provenance")?; + if &template.for_attempt(attempt, isolate_paths) != executed { + return Err( + "coverage execution differs from its planned attempt transformation".to_owned(), + ); + } + self.observe_case_inner( + template, + executed, + observation, + Some((attempt, isolate_paths)), + Some(budget), + ) + } + + fn observe_case_inner( + &mut self, + template: &BehaviorCase, + case: &BehaviorCase, + observation: &CaseObservation, + attempt: Option<(u64, bool)>, + budget: Option<&crate::budget::Budget>, + ) -> Result<(), String> { + if let Some(budget) = budget { + budget.check("observed coverage validation")?; + } + self.require_validated_identity()?; + let planned = self + .planned_cases + .get(&case.id) + .ok_or_else(|| format!("coverage cannot complete unplanned case {}", case.id))?; + if template.id != case.id || evidence_digest(template)? != planned.case_digest { + return Err(format!( + "case {} differs from its immutable coverage plan", + case.id + )); + } + case.validate()?; + if case.live_nodes != self.live_nodes { + return Err(format!( + "case {} differs from the coverage live set", + case.id + )); + } + if self.completed_cases.contains(&case.id) { + return Err(format!("case {} was counted twice", case.id)); + } + let keys = observed_case_coverage(case, observation, budget)?; + let evidence = CompletedCaseEvidence { + planned_case_digest: planned.case_digest.clone(), + identity_digest: evidence_digest(self.identity.as_ref().expect("validated identity"))?, + attempt, + executed_case_digest: evidence_digest(case)?, + observation_digest: evidence_digest(observation)?, + execution_ids: case + .processes + .iter() + .map(|process| (process.id.clone(), process.access.execution_id.clone())) + .collect(), + request_ids: observation + .executions + .iter() + .map(|execution| (execution.process.clone(), execution.request_id.clone())) + .collect(), + keys, + }; + if evidence.request_ids.values().any(|id| { + self.completed_evidence + .values() + .any(|previous| previous.request_ids.values().any(|previous| previous == id)) + }) || evidence.execution_ids.values().any(|id| { + self.completed_evidence.values().any(|previous| { + previous + .execution_ids + .values() + .any(|previous| previous == id) + }) + }) { + return Err("coverage attempt reuses a completed execution identity".to_owned()); + } + if let Some(budget) = budget { + budget.check("observed coverage commit")?; + } + self.completed_cases.insert(case.id.clone()); + for key in &evidence.keys { + *self.observed.entry(key.clone()).or_default() += 1; + } + self.completed_evidence.insert(case.id.clone(), evidence); + Ok(()) + } + + pub fn assert_planned_closed(&self) -> Result<(), String> { + self.validate_evidence_closure()?; + self.assert_closed(&self.planned, "planned") + } + + pub fn assert_observed_closed(&self) -> Result<(), String> { + self.require_validated_identity()?; + self.validate_evidence_closure()?; + self.assert_closed(&self.observed, "observed") + } + + fn assert_closed( + &self, + actual: &BTreeMap, + label: &str, + ) -> Result<(), String> { + if self.schema_version != COVERAGE_SCHEMA_VERSION { + return Err("coverage schema is incompatible".to_owned()); + } + let missing = self + .required + .iter() + .filter_map(|(key, required)| { + let actual = actual.get(key).copied().unwrap_or_default(); + (actual < *required).then_some(format!("{key:?}: {actual}/{required}")) + }) + .collect::>(); + if missing.is_empty() { + Ok(()) + } else { + Err(format!( + "{label} coverage ledger is incomplete: {}", + missing.join(", ") + )) + } + } +} + +struct ProcessRecords<'a> { + program: &'a ProcessProgram, + execution: &'a ExecutionObservation, + terminals: Vec<&'a ActionObservation>, + steps: Vec>, + barriers: Vec<&'a BarrierObservation>, +} + +/// All values borrow original records. Per-step buckets retain multiplicity +/// and emission order; no sorting or last-write-wins lookup can hide ambiguity. +struct ObservationIndex<'a> { + observation: &'a CaseObservation, + processes: BTreeMap<&'a str, ProcessRecords<'a>>, + endpoints: BTreeMap<(u64, &'a str, &'a str), Vec<&'a ActionObservation>>, + paths: BTreeMap<&'a str, Vec<&'a ActionObservation>>, +} + +impl<'a> ObservationIndex<'a> { + fn new(case: &'a BehaviorCase, observation: &'a CaseObservation) -> Result { + let programs = case + .processes + .iter() + .map(|program| (program.id.as_str(), program)) + .collect::>(); + let mut index = Self { + observation, + processes: BTreeMap::new(), + endpoints: BTreeMap::new(), + paths: BTreeMap::new(), + }; + for execution in &observation.executions { + let program = *programs + .get(execution.process.as_str()) + .ok_or_else(|| format!("unplanned execution {:?}", execution.process))?; + let mut records = ProcessRecords { + program, + execution, + terminals: Vec::with_capacity(program.actions.len()), + steps: vec![Vec::new(); program.actions.len()], + barriers: Vec::new(), + }; + for result in &execution.results { + let action = program.actions.get(result.step).ok_or_else(|| { + format!("{} has an unplanned step {}", program.id, result.step) + })?; + if result.process != execution.process + || !crate::oracle::stream_record_action_matches(&action.operation, result) + || result.path != action.operation.path() + { + return Err(format!( + "{} step {} has a mismatched record identity", + program.id, result.step + )); + } + let step = &mut records.steps[result.step]; + if result.outcome == "barrier" { + let barrier = result.barrier.as_ref().ok_or_else(|| { + format!( + "{} step {} has an untyped milestone", + program.id, result.step + ) + })?; + let stream_milestone = matches!( + barrier, + BarrierObservation::StreamOpened { .. } + | BarrierObservation::StreamFirstFrame { .. } + | BarrierObservation::StreamFrame { .. } + | BarrierObservation::StreamEof { .. } + | BarrierObservation::ReleaseObserved { .. } + ); + if stream_milestone + && !matches!( + action.operation, + ActionOp::StreamWrite { .. } + | ActionOp::GatedStreamWrite { .. } + | ActionOp::StreamRead { .. } + | ActionOp::StreamReadWithRetry { .. } + | ActionOp::GatedStreamRead { .. } + | ActionOp::StreamRoundTrip { .. } + | ActionOp::StreamReadInto { .. } + ) + { + return Err(format!( + "{} step {} has stream evidence for a non-stream action", + program.id, result.step + )); + } + let precedes_terminal = stream_milestone + || matches!(barrier, BarrierObservation::MutationApplied { .. }); + let terminal_seen = step.iter().any(|record| record.outcome != "barrier"); + if precedes_terminal == terminal_seen { + return Err(format!( + "{} step {} has a reordered milestone", + program.id, result.step + )); + } + if !matches!(barrier, BarrierObservation::StreamFrame { .. }) + && step.iter().any(|previous| { + previous.barrier.as_ref().is_some_and(|previous| { + std::mem::discriminant(previous) == std::mem::discriminant(barrier) + }) + }) + { + return Err(format!( + "{} step {} has an ambiguous duplicate milestone", + program.id, result.step + )); + } + records.barriers.push(barrier); + } else { + if result.barrier.is_some() { + return Err(format!( + "{} step {} mixes a terminal outcome and milestone", + program.id, result.step + )); + } + records.terminals.push(result); + if execution.exit_success && result.outcome == "ok" { + index + .endpoints + .entry(( + execution.logical_node_id, + result.path.as_str(), + result.action.as_str(), + )) + .or_default() + .push(result); + } + } + step.push(result); + index + .paths + .entry(result.path.as_str()) + .or_default() + .push(result); + } + if index + .processes + .insert(execution.process.as_str(), records) + .is_some() + { + return Err(format!("duplicate execution {}", execution.process)); + } + } + Ok(index) + } +} + +/// Coverage is committed only after the complete observation has been checked. +/// The planned ledger is deliberately not an input to this derivation. +fn observed_case_coverage( + case: &BehaviorCase, + observation: &CaseObservation, + budget: Option<&crate::budget::Budget>, +) -> Result, String> { + let index = ObservationIndex::new(case, observation)?; + let mut requests = BTreeSet::new(); + let stream_route_paths = case + .routes + .iter() + .filter(|route| route.kind == DataKind::Stream) + .flat_map(|route| route.edges.iter().map(|edge| edge.path.as_str())) + .collect::>(); + let route_paths = case + .routes + .iter() + .flat_map(|route| route.edges.iter().map(|edge| edge.path.as_str())) + .collect::>(); + for execution in &observation.executions { + let program = index.processes[execution.process.as_str()].program; + if execution.request_id.is_empty() || !requests.insert(execution.request_id.as_str()) { + return Err(format!( + "duplicate or unidentified execution {}", + execution.process + )); + } + if execution.logical_node_id != program.logical_node_id { + return Err(format!( + "{} ran on node {}, expected {}", + program.id, execution.logical_node_id, program.logical_node_id + )); + } + if crate::oracle::process_failure_is_expected(case, program) { + // A causally stopped transfer can have an authentic partial + // prefix. The complete independent oracle below must validate + // that prefix and its terminal lifecycle before any credit. + if execution.exit_success { + return Err(format!( + "{} must prove its expected process failure", + program.id + )); + } + if program + .actions + .iter() + .any(|action| matches!(action.expected, ExpectedOutcome::Linearized { .. })) + { + return Err(format!( + "{} has an unobserved linearized operation in a failed execution", + program.id + )); + } + } else if !execution.exit_success { + return Err(format!("{} did not execute successfully", program.id)); + } + // Results are emitted in program order. Sorting by the claimed step would + // conceal reordered or duplicated records and invent adjacency evidence. + // Action results are emitted in program order; barrier milestone + // records ride along at their owning action's step (stream milestones + // precede the action record, hint barriers follow it). + let first_route_step = program.actions.iter().position(|action| { + route_paths.contains(action.operation.path()) + && (action_payload_length(&action.operation).is_some() + || matches!(&action.operation, ActionOp::AwaitEntry { path, .. } + if case.routes.iter().any(|route| route.edges.iter().any(|edge| + edge.destination_role == program.id && edge.path == *path)))) + }); + let mut next_action_step = 0_usize; + let mut last_action_step = None::; + for result in &execution.results { + if result.outcome == "barrier" { + if stream_route_paths.contains(result.path.as_str()) + && matches!( + result.barrier, + Some(BarrierObservation::StreamOpened { .. }) + ) + { + if first_route_step != Some(next_action_step) { + return Err(format!( + "{} prepared a route endpoint outside startup", + program.id + )); + } + continue; + } + if result.step != next_action_step && Some(result.step) != last_action_step { + return Err(format!( + "{} has a barrier record detached from its action step {}", + program.id, result.step + )); + } + continue; + } + if result.step != next_action_step { + return Err(format!( + "{} has a missing, duplicate, or reordered step {next_action_step}", + program.id + )); + } + if result.outcome == "ok" + && (result.errno.is_some() || result.error_type.is_some() || result.error.is_some()) + { + return Err(format!( + "{} step {next_action_step} claims both success and failure", + program.id + )); + } + last_action_step = Some(next_action_step); + next_action_step += 1; + } + } + // Reuse the behavioral checks for exact process/step/class/path, typed + // outcomes, independently computed payload digests, and lifecycle completion. + match budget { + Some(budget) => { + crate::oracle::BehaviorOracle::verify_with_budget(case, observation, budget) + } + None => crate::oracle::BehaviorOracle::verify(case, observation), + } + .map_err(|error| error.to_string())?; + + let mut keys = BTreeSet::new(); + let mut unsupported = BTreeSet::new(); + for records in index.processes.values() { + let program = records.program; + let execution = records.execution; + if !execution.exit_success { + keys.insert(CoverageKey::Outcome { + class: "contextual_failure".to_owned(), + }); + continue; + } + for pair in records.terminals.windows(2) { + keys.insert(CoverageKey::ActionAdjacency { + before: program.actions[pair[0].step].operation.class(), + after: program.actions[pair[1].step].operation.class(), + }); + } + for result in &records.terminals { + let operation = &program.actions[result.step].operation; + keys.insert(CoverageKey::Outcome { + class: if result.outcome == "ok" { + "success" + } else { + "modeled_error" + } + .to_owned(), + }); + if result.outcome == "ok" + && (action_payload_length(operation).is_some() + || matches!(operation, ActionOp::StreamRoundTrip { .. })) + { + // The oracle checked this actual length and digest against the + // independent expected bytes, not just the number of records. + for class in payload_classes(result.length.expect("payload verified above")) { + keys.insert(CoverageKey::Payload { class }); + } + } + observe_descriptor(operation, result, &mut keys)?; + } + } + for scenario in &case.scenarios { + if !observe_scenario(case, *scenario, &index, &mut keys)? { + unsupported.insert(format!( + "{scenario:?} lacks its required typed milestone evidence" + )); + } + } + let mut topology_proven = case.topology == TopologyFamily::Fixed; + for route in &case.routes { + observe_route(route, &index, &mut keys)?; + topology_proven |= route_has_topology(route, case.topology); + } + if !topology_proven { + return Err(format!( + "case {} has no completely observed route proving topology {:?}", + case.id, case.topology + )); + } + if !unsupported.is_empty() { + return Err(format!( + "case {} has unprovable coverage; observation schema extension required: {}", + case.id, + unsupported.into_iter().collect::>().join("; ") + )); + } + if case.topology != TopologyFamily::Fixed { + keys.insert(CoverageKey::Topology { + family: case.topology, + }); + } + Ok(keys) +} + +/// Proves one declared scenario from typed milestone records actually +/// emitted by the workload. Returns false when the required evidence is +/// absent; construction-level guarantees (dependency edges, failures) are +/// cross-checked against the case, never against poll order. +fn observe_scenario( + case: &BehaviorCase, + scenario: CoverageScenario, + index: &ObservationIndex<'_>, + keys: &mut BTreeSet, +) -> Result { + let observation = index.observation; + let execution_of = |process: &str| { + index + .processes + .get(process) + .map(|records| records.execution) + }; + let barriers = |process: &str| { + index + .processes + .get(process) + .map(|records| records.barriers.as_slice()) + .unwrap_or(&[]) + }; + match scenario { + CoverageScenario::ConcurrentStartup => { + // Startup participants are the case's mutually independent route + // processes; every stream participant opened its endpoint and + // every participant completed, proving progress that never + // depended on endpoint creation order. + let route_paths = case + .routes + .iter() + .flat_map(|route| route.edges.iter().map(|edge| edge.path.as_str())) + .collect::>(); + let participants = case + .processes + .iter() + .filter(|program| { + program + .actions + .iter() + .any(|action| route_paths.contains(action.operation.path())) + }) + .collect::>(); + if participants.len() < 2 + || participants + .iter() + .any(|program| !program.depends_on.is_empty()) + { + return Ok(false); + } + let stream_paths = case + .routes + .iter() + .filter(|route| route.kind == DataKind::Stream) + .flat_map(|route| route.edges.iter().map(|edge| edge.path.as_str())) + .collect::>(); + if stream_paths.is_empty() { + return Ok(false); + } + for program in &participants { + let Some(execution) = execution_of(&program.id) else { + return Ok(false); + }; + if !execution.exit_success { + return Ok(false); + } + let required_steps = program + .actions + .iter() + .enumerate() + .filter(|(_, action)| stream_paths.contains(action.operation.path())) + .map(|(step, _)| step) + .collect::>(); + let mut opened = BTreeSet::new(); + for result in &execution.results { + if !required_steps.contains(&result.step) { + continue; + } + if matches!( + result.barrier, + Some(BarrierObservation::StreamOpened { .. }) + ) { + opened.insert(result.step); + } else if (result.outcome != "barrier" + || matches!( + result.barrier, + Some( + BarrierObservation::StreamFirstFrame { .. } + | BarrierObservation::StreamFrame { .. } + | BarrierObservation::StreamEof { .. } + ) + )) + && opened != required_steps + { + return Ok(false); + } + } + if opened != required_steps { + return Ok(false); + } + } + keys.insert(CoverageKey::ConcurrentStart { + family: case.topology, + }); + keys.insert(CoverageKey::Scenario { + scenario: CoverageScenario::ConcurrentStartup, + }); + Ok(true) + } + CoverageScenario::RingCompletion => { + // A token label cannot manufacture a ring. Derive lap and edge + // identities from a closed, ordered, repeated physical route, then + // bind every milestone to that edge's successful endpoint record. + let mut proven = false; + for route in &case.routes { + let Some(hops) = returning_ring_hops(route) else { + continue; + }; + let mut token = None; + let mut previous_sink: Option<&ActionObservation> = None; + for (edge_index, edge) in route.edges.iter().enumerate() { + if edge_index % hops == 0 { + token = None; + } + let source = route_endpoint( + index, + route.kind, + NodeRole::Source, + edge.source, + &edge.path, + &edge.source_role, + )?; + let sink = route_endpoint( + index, + route.kind, + NodeRole::Sink, + edge.destination, + &edge.path, + &edge.destination_role, + )?; + let Some(digest) = source.digest.as_deref() else { + return Ok(false); + }; + if source.length.is_none() + || source.length != sink.length + || sink.digest.as_deref() != Some(digest) + || token.is_some_and(|known| known != digest) + || previous_sink.is_some_and(|previous| { + previous.process != source.process || previous.step >= source.step + }) + { + return Ok(false); + } + token = Some(digest); + let lap = (edge_index / hops) as u32; + let forwarded = BarrierObservation::TokenForwarded { + token: digest.to_owned(), + lap, + edge_index: edge_index as u32, + }; + let received = if edge_index + 1 == route.edges.len() { + BarrierObservation::LapCompleted { + token: digest.to_owned(), + lap, + } + } else { + BarrierObservation::TokenReceived { + token: digest.to_owned(), + lap, + edge_index: edge_index as u32, + } + }; + for (endpoint, expected) in [(source, forwarded), (sink, received)] { + let records = &index.processes[endpoint.process.as_str()]; + if records.steps[endpoint.step] + .iter() + .filter(|result| result.barrier.as_ref() == Some(&expected)) + .count() + != 1 + { + return Ok(false); + } + } + previous_sink = Some(sink); + } + proven = true; + } + if !proven { + return Ok(false); + } + keys.insert(CoverageKey::Scenario { + scenario: CoverageScenario::RingCompletion, + }); + Ok(true) + } + CoverageScenario::HotColdFairness => { + // Causal chain: hot reader observed its first frame, every cold + // flow completed, the release publisher (dependent on all colds) + // published release, the hot writer observed that release, and + // the hot pair completed. + let Some((hot_writer, hot_reader)) = + case.processes + .iter() + .find(|program| { + program.actions.iter().any(|action| { + matches!(action.operation, ActionOp::GatedStreamWrite { .. }) + }) + }) + .zip(case.processes.iter().find(|program| { + program.actions.iter().any(|action| { + matches!(action.operation, ActionOp::GatedStreamRead { .. }) + }) + })) + else { + return Ok(false); + }; + let (hot_path, release_path) = hot_writer + .actions + .iter() + .find_map(|action| match &action.operation { + ActionOp::GatedStreamWrite { + path, release_path, .. + } => Some((path.as_str(), release_path.as_str())), + _ => None, + }) + .expect("hot writer shape checked above"); + let Some(observed_path) = + hot_reader + .actions + .iter() + .find_map(|action| match &action.operation { + ActionOp::GatedStreamRead { + path, + observed_path, + .. + } if path == hot_path => Some(observed_path.as_str()), + _ => None, + }) + else { + return Ok(false); + }; + let is_cold = |program: &crate::ProcessProgram| { + program.id != hot_writer.id + && program.id != hot_reader.id + && program.actions.first().is_some_and(|action| { + action.expected == ExpectedOutcome::Ok + && matches!(&action.operation, ActionOp::AwaitEntry { path, expected_kind } + if path == observed_path && expected_kind == "blob") + }) + && program.actions.iter().skip(1).any(|action| { + matches!(action.operation, + ActionOp::PublishBlob { .. } | ActionOp::ReadBlob { .. } + | ActionOp::StreamWrite { .. } | ActionOp::StreamRead { .. } + | ActionOp::StreamRoundTrip { .. } | ActionOp::StreamReadInto { .. } + | ActionOp::DescriptorWrite { .. } | ActionOp::DescriptorRead { .. }) + }) + && !program.actions.iter().any(|action| { + matches!(action.operation, ActionOp::GatedStreamWrite { .. }) + || matches!( + &action.operation, + ActionOp::PublishBlob { path, .. } if path == release_path + ) + }) + }; + // The release publisher declares its cold set exactly; unrelated + // route participants that merely await entries are not colds. + let publisher = case.processes.iter().find(|program| { + !program.depends_on.is_empty() + && program.depends_on.iter().all(|dependency| { + index.processes.get(dependency.as_str()).is_some_and(|records| is_cold(records.program)) + }) + && program.actions.iter().any(|action| { + matches!(&action.operation, ActionOp::PublishBlob { path, .. } if path == release_path) + }) + }); + let Some(publisher) = publisher else { + return Ok(false); + }; + let colds = case + .processes + .iter() + .filter(|program| publisher.depends_on.contains(&program.id)) + .filter(|program| is_cold(program)) + .collect::>(); + if colds.len() < 2 { + return Ok(false); + } + for cold in &colds { + let Some(records) = index.processes.get(cold.id.as_str()) else { + return Ok(false); + }; + // Expected errors can make an execution successful without + // moving data. Credit only a verified payload operation after + // this cold's first-frame gate, not its planned action class. + let transferred = records.terminals.iter().any(|result| { + result.step > 0 + && successful_data_observation(&cold.actions[result.step].operation, result) + }); + if !records.execution.exit_success || !transferred { + return Ok(false); + } + } + if !execution_of(&publisher.id).is_some_and(|e| e.exit_success) { + return Ok(false); + } + let reader_first_frame = barriers(&hot_reader.id) + .iter() + .any(|barrier| matches!(barrier, BarrierObservation::StreamFirstFrame { .. })); + let writer_release = barriers(&hot_writer.id) + .iter() + .any(|barrier| { + matches!(barrier, BarrierObservation::ReleaseObserved { path } if path == release_path) + }); + let hot_completed = execution_of(&hot_writer.id).is_some_and(|e| e.exit_success) + && execution_of(&hot_reader.id).is_some_and(|e| e.exit_success); + if !reader_first_frame || !writer_release || !hot_completed { + return Ok(false); + } + keys.insert(CoverageKey::Scenario { + scenario: CoverageScenario::HotColdFairness, + }); + Ok(true) + } + CoverageScenario::ProcessChurn => { + // The paired route is already transferring when the target stops. + // Only its terminal stop unlocks the release publisher, and the + // gated writer subsequently observes release and completes. + let Some((pair, publisher)) = churn_stream_pair(case) else { + return Ok(false); + }; + let FailureInjection::StopProcess { process, .. } = &case.failure else { + return Ok(false); + }; + let Some(target) = execution_of(process) else { + return Ok(false); + }; + let ActionOp::GatedStreamWrite { release_path, .. } = + &pair.writer.actions[pair.write_step].operation + else { + return Ok(false); + }; + if target.exit_success + || !target.terminal + || observed_stream_attachment(&pair, index).is_none() + || observation + .executions + .iter() + .any(|execution| execution.process != *process && !execution.exit_success) + { + return Ok(false); + } + let writer = &index.processes[pair.writer.id.as_str()]; + let reader = &index.processes[pair.reader.id.as_str()]; + let release = &index.processes[publisher.id.as_str()]; + if !writer.steps[pair.write_step].iter().any(|result| { + successful_data_observation(&pair.writer.actions[pair.write_step].operation, result) + }) || !reader.steps[pair.read_step].iter().any(|result| { + successful_data_observation(&pair.reader.actions[pair.read_step].operation, result) + }) || !writer.steps[pair.write_step].iter().any(|result| { + matches!(&result.barrier, Some(BarrierObservation::ReleaseObserved { path }) if path == release_path) + }) || !release.terminals.iter().any(|result| { + result.outcome == "ok" + && matches!(&publisher.actions[result.step].operation, + ActionOp::PublishBlob { path, .. } if path == release_path) + }) { + return Ok(false); + } + keys.insert(CoverageKey::Scenario { + scenario: CoverageScenario::ProcessChurn, + }); + Ok(true) + } + CoverageScenario::ActiveStreamWriterAbort | CoverageScenario::ActiveStreamReaderStop => { + let Some(pair) = active_stream_fault_pair(case, scenario) else { + return Ok(false); + }; + let Some(incarnation) = observed_stream_attachment(&pair, index) else { + return Ok(false); + }; + let writer_abort = scenario == CoverageScenario::ActiveStreamWriterAbort; + let (target, survivor, survivor_step) = if writer_abort { + (pair.writer, pair.reader, pair.read_step) + } else { + (pair.reader, pair.writer, pair.write_step) + }; + let target_records = &index.processes[target.id.as_str()]; + let survivor_records = &index.processes[survivor.id.as_str()]; + let target_step = if writer_abort { + pair.write_step + } else { + pair.read_step + }; + if target_records.execution.exit_success + || !target_records.execution.terminal + || target_records.steps[target_step] + .iter() + .any(|result| result.outcome != "barrier") + || !survivor_records.execution.exit_success + || !survivor_records.steps[survivor_step].iter().any(|result| { + result.outcome == "expected_error" + && result.incarnation == Some(incarnation) + && matches!(result.errno, None | Some(libc::EPIPE | libc::ECONNRESET)) + && result.error_type.as_deref() == Some("StreamError") + }) + { + return Ok(false); + } + let path = pair.writer.actions[pair.write_step].operation.path(); + // Peer loss must release the same attached incarnation. A new + // replacement's inactivity cannot stand in for fault cleanup. + let cleaned = survivor_records.terminals.iter().any(|result| { + result.step > survivor_step + && result.outcome == "ok" + && result.kind.as_deref() == Some("stream") + && result.active == Some(false) + && result.revision == Some(incarnation) + && matches!( + &survivor.actions[result.step].operation, + ActionOp::WaitForQuiescent { path: cleanup_path } if cleanup_path == path + ) + }); + if cleaned { + keys.insert(CoverageKey::Scenario { scenario }); + } + Ok(cleaned) + } + CoverageScenario::ActiveMutation | CoverageScenario::QuiescentMutation => { + // Classify the displaced incarnation, not a fresh replacement's + // eventual EOF. Inactive namespace evidence retains the same + // revision; active displacement produces ESTALE on old handles. + let active = scenario == CoverageScenario::ActiveMutation; + let mut proven = false; + for program in &case.processes { + let Some(process_records) = index.processes.get(program.id.as_str()) else { + continue; + }; + for (step, action) in program.actions.iter().enumerate() { + let is_mutation = match &action.operation { + ActionOp::StreamWrite { replace, .. } + | ActionOp::GatedStreamWrite { replace, .. } => *replace, + ActionOp::Rename { .. } | ActionOp::Unlink { .. } => true, + _ => false, + }; + if !is_mutation { + continue; + } + let path = action.operation.path(); + let records = &process_records.steps[step]; + let transition = records + .iter() + .find_map(|result| match &result.barrier { + Some(BarrierObservation::MutationApplied { + from_revision: Some(from), + to_revision: Some(to), + }) if to > from => Some((*from, *to)), + _ => None, + }) + .or_else(|| { + if active + || !matches!( + action.operation, + ActionOp::StreamWrite { replace: true, .. } + | ActionOp::GatedStreamWrite { replace: true, .. } + ) + { + return None; + } + // A reader can create B before the writer's origin + // lookup, so its receipt legitimately reports B/B. + // That lookup is not an atomic predecessor receipt: + // prove A -> B from an endpoint's ordered evidence. + records.iter().find_map(|result| match &result.barrier { + Some(BarrierObservation::MutationApplied { + from_revision: Some(_), + to_revision: Some(replacement), + }) => quiescent_stream_origin(path, *replacement, index) + .map(|displaced| (displaced, *replacement)), + _ => None, + }) + }); + let Some((displaced, replacement)) = transition else { + continue; + }; + if !records.iter().any(|result| { + result.outcome == "ok" && result.incarnation == Some(replacement) + }) { + continue; + } + let displaced_records = index.paths.get(path).map(Vec::as_slice).unwrap_or(&[]); + let quiescence_observed = displaced_records.iter().any(|result| { + result.outcome == "ok" + && result.kind.as_deref() == Some("stream") + && result.revision == Some(displaced) + && result.active == Some(false) + }); + let active_displacement = displaced_records.iter().any(|result| { + result.outcome == "expected_error" + && result.errno == Some(libc::ESTALE) + && result.incarnation == Some(displaced) + }); + if (active && active_displacement && !quiescence_observed) + || (!active && quiescence_observed) + { + proven = true; + } + } + } + if proven { + keys.insert(CoverageKey::Scenario { scenario }); + } + Ok(proven) + } + CoverageScenario::FailureIsolation => { + // A failing branch is isolated, or an ordering-only slow branch + // parks until every healthy sibling completes. Case validation and + // the namespace oracle prove the latter from public marker actions. + let target = match &case.failure { + FailureInjection::StopProcess { process, .. } + | FailureInjection::LaunchFailure { process, .. } + | FailureInjection::SlowProcess { process, .. } => process.clone(), + FailureInjection::None => return Ok(false), + }; + let Some(failed) = execution_of(&target) else { + return Ok(false); + }; + let ordering_only = matches!(case.failure, FailureInjection::SlowProcess { .. }); + if failed.exit_success != ordering_only { + return Ok(false); + } + let siblings_ok = observation + .executions + .iter() + .filter(|execution| execution.process != target) + .all(|execution| execution.exit_success); + if !siblings_ok { + return Ok(false); + } + keys.insert(CoverageKey::Scenario { + scenario: CoverageScenario::FailureIsolation, + }); + Ok(true) + } + CoverageScenario::Authorization => { + // A restricted session observed both a denied operation (typed + // EACCES on a path outside its grants) and a successful + // operation inside its grants. + for program in &case.processes { + if program.access.read_prefixes.is_empty() + && program.access.write_prefixes.is_empty() + { + continue; + } + let Some(execution) = execution_of(&program.id) else { + continue; + }; + let mut denied = false; + let mut authorized = false; + for result in &execution.results { + if result.outcome == "expected_error" && result.errno == Some(libc::EACCES) { + denied = true; + } + if result.outcome == "ok" { + authorized = true; + } + } + if denied && authorized { + keys.insert(CoverageKey::Scenario { + scenario: CoverageScenario::Authorization, + }); + return Ok(true); + } + } + Ok(false) + } + } +} + +fn observe_descriptor( + operation: &ActionOp, + result: &ActionObservation, + keys: &mut BTreeSet, +) -> Result<(), String> { + if result.outcome != "ok" && result.descriptor.is_none() { + return if result.length.is_none() && result.digest.is_none() { + Ok(()) + } else { + Err(format!( + "{} step {} reports payload without an observed completed transfer", + result.process, result.step + )) + }; + } + let (direction, method, finish, terminals, bytes) = match ( + operation, + result.descriptor.as_ref(), + ) { + ( + ActionOp::DescriptorWrite { + method, + finish, + bytes, + .. + }, + Some(DescriptorObservation::Write { + method: observed_method, + finish: observed_finish, + terminal_results, + .. + }), + ) if method == observed_method && finish == observed_finish => ( + "write", + descriptor_write_method(*observed_method), + *observed_finish, + terminal_results, + bytes, + ), + ( + ActionOp::DescriptorRead { + method, + finish, + expected, + .. + }, + Some(DescriptorObservation::Read { + method: observed_method, + finish: observed_finish, + terminal_results, + }), + ) if method == observed_method && finish == observed_finish => ( + "read", + descriptor_read_method(*observed_method), + *observed_finish, + terminal_results, + expected, + ), + (ActionOp::DescriptorWrite { .. } | ActionOp::DescriptorRead { .. }, _) => { + return Err(format!( + "{} step {} lacks matching observed descriptor direction, method, and completed terminal operations", + result.process, result.step + )); + } + (_, Some(_)) => { + return Err(format!( + "{} step {} reports descriptor evidence for another action", + result.process, result.step + )); + } + (_, None) => return Ok(()), + }; + let dropped = finish == DescriptorFinish::Drop; + if dropped && (!terminals.is_empty() || result.outcome != "ok") { + return Err("descriptor drop must not claim an uncalled terminal API result".to_owned()); + } + let writer_dropped = direction == "write" && dropped; + if writer_dropped + && !matches!(&result.descriptor, + Some(DescriptorObservation::Write { + dropped: true, reservation_released: true, terminal_results, .. + }) if terminal_results.is_empty() && result.outcome == "ok") + { + return Err("writer drop lacks observed release and nonpublication evidence".to_owned()); + } + let terminal_count = if matches!( + finish, + DescriptorFinish::CloseTwice | DescriptorFinish::AbortTwice + ) { + 2 + } else { + 1 + }; + if !dropped + && (terminals.len() != terminal_count + || terminals[..terminal_count - 1] + .iter() + .any(|terminal| !matches!(terminal, DescriptorTerminalResult::Ok)) + || !match terminals.last() { + Some(DescriptorTerminalResult::Ok) => result.outcome == "ok", + Some(DescriptorTerminalResult::Error { errno, error_type }) => { + result.outcome == "expected_error" + && *errno == result.errno + && Some(error_type.as_str()) == result.error_type.as_deref() + } + None => false, + }) + { + return Err(format!( + "{} step {} has missing, reordered, or inconsistent descriptor terminal outcomes", + result.process, result.step + )); + } + if let Some(transfer) = &result.transfer { + if !transfer.complete + || transfer.length != bytes.len() + || transfer.digest != crate::codegen::digest(bytes) + { + return Err(format!( + "{} step {} changed transferred descriptor bytes", + result.process, result.step + )); + } + for class in payload_classes(transfer.length) { + keys.insert(CoverageKey::Payload { class }); + } + } else if result.outcome != "ok" && (result.length.is_some() || result.digest.is_some()) { + return Err( + "failed descriptor outcome lacks distinct completed transfer evidence".to_owned(), + ); + } + keys.insert(CoverageKey::DescriptorMethod { + direction: direction.to_owned(), + method: method.to_owned(), + }); + keys.insert(CoverageKey::DescriptorTerminal { + direction: direction.to_owned(), + finish: descriptor_finish(finish).to_owned(), + }); + Ok(()) +} + +fn route_endpoint<'a>( + index: &ObservationIndex<'a>, + kind: DataKind, + role: NodeRole, + node: u64, + path: &str, + process: &str, +) -> Result<&'a ActionObservation, String> { + let action = match (kind, role) { + (DataKind::Blob, NodeRole::Source) => "publish_blob", + (DataKind::Blob, NodeRole::Sink) => "read_blob", + (DataKind::Stream, NodeRole::Source) => "stream_write", + (DataKind::Stream, NodeRole::Sink) => "stream_read", + (_, NodeRole::Relay) => unreachable!("relay requires both endpoints"), + }; + let matches = index + .endpoints + .get(&(node, path, action)) + .map(Vec::as_slice) + .unwrap_or(&[]); + let result = matches.first().ok_or_else(|| { + format!("route has no successful {kind:?} {role:?} on node {node} at {path:?}") + })?; + if matches.len() != 1 || result.process != process { + return Err(format!( + "route has ambiguous {kind:?} {role:?} on node {node} at {path:?}; publication/stream incarnation identity is required" + )); + } + Ok(result) +} + +fn observe_route( + route: &DataRoute, + index: &ObservationIndex<'_>, + keys: &mut BTreeSet, +) -> Result<(), String> { + let mut incoming = BTreeMap::<&str, Vec<&ActionObservation>>::new(); + let mut outgoing = BTreeMap::<&str, Vec<&ActionObservation>>::new(); + let mut paths = BTreeSet::new(); + for edge in &route.edges { + if !paths.insert(edge.path.as_str()) { + return Err(format!( + "route {} reuses edge path {:?}", + route.id, edge.path + )); + } + let source = route_endpoint( + index, + route.kind, + NodeRole::Source, + edge.source, + &edge.path, + &edge.source_role, + )?; + let sink = route_endpoint( + index, + route.kind, + NodeRole::Sink, + edge.destination, + &edge.path, + &edge.destination_role, + )?; + if source.length.is_none() + || source.digest.is_none() + || source.length != sink.length + || source.digest != sink.digest + { + return Err(format!( + "route {} changed payload on {:?}", + route.id, edge.path + )); + } + outgoing.entry(&edge.source_role).or_default().push(source); + incoming + .entry(&edge.destination_role) + .or_default() + .push(sink); + keys.insert(CoverageKey::OrderedEdge { + source: edge.source, + destination: edge.destination, + kind: route.kind, + }); + keys.insert(CoverageKey::NodeRole { + node: edge.source, + kind: route.kind, + role: NodeRole::Source, + }); + keys.insert(CoverageKey::NodeRole { + node: edge.destination, + kind: route.kind, + role: NodeRole::Sink, + }); + } + for path in &route.join_inputs { + if !paths.contains(path.as_str()) { + return Err(format!( + "route {} has an unobserved join input {path:?}", + route.id + )); + } + } + for (role, inputs) in &incoming { + let distinct_sources = route + .edges + .iter() + .filter(|edge| edge.destination_role == *role) + .map(|edge| edge.source_role.as_str()) + .collect::>(); + // A fan-in join from several distinct upstream nodes must be consumed + // by one observed execution. Repeated edges from the same upstream + // node are sequential ring laps through this node, not a join: each + // lap may close in a different consumer of that node. + if distinct_sources.len() > 1 + && inputs + .iter() + .any(|input| input.process != inputs[0].process) + { + return Err(format!( + "route {} joins across executions at role {role}; observed cross-process input/join causal links are required", + route.id + )); + } + let Some(outputs) = outgoing.get(role) else { + continue; + }; + // Every consumed input is either relayed onward by the same + // execution at a later step, or ends its execution's walk as a + // terminal consumer. Per-edge payload integrity is checked above; + // graph-derived relay transformations can change the outgoing digest. + let mut witnessed_relay = false; + for input in inputs { + let relayed = outputs + .iter() + .any(|output| input.process == output.process && input.step < output.step); + witnessed_relay |= relayed; + let terminal = !outputs.iter().any(|output| input.process == output.process); + if !relayed && !terminal { + return Err(format!( + "route {} has no observed receive-before-forward relay at role {role}; cross-process causal links and payload provenance are required", + route.id + )); + } + } + if witnessed_relay { + keys.insert(CoverageKey::NodeRole { + node: index.processes[*role].program.logical_node_id, + kind: route.kind, + role: NodeRole::Relay, + }); + } + } + Ok(()) +} + +fn process_depends_on(case: &BehaviorCase, process: &str, dependency: &str) -> bool { + let mut pending = vec![process]; + let mut visited = BTreeSet::new(); + while let Some(process) = pending.pop() { + if !visited.insert(process) { + continue; + } + let Some(program) = case.processes.iter().find(|program| program.id == process) else { + continue; + }; + for predecessor in &program.depends_on { + if predecessor == dependency { + return true; + } + pending.push(predecessor); + } + } + false +} + +fn churn_stream_pair(case: &BehaviorCase) -> Option<(StreamPair<'_>, &ProcessProgram)> { + let FailureInjection::StopProcess { + process, + phase: crate::ir::ProcessStopPhase::AfterSiblingStreamFirstFrame, + .. + } = &case.failure + else { + return None; + }; + let target = case + .processes + .iter() + .find(|program| program.id == *process)?; + for route in &case.routes { + for edge in &route.edges { + if [&edge.source_role, &edge.destination_role] + .into_iter() + .any(|role| { + role == process + || process_depends_on(case, process, role) + || process_depends_on(case, role, process) + }) + || target + .actions + .iter() + .any(|action| action.operation.path() == edge.path) + { + return None; + } + } + } + for route in case + .routes + .iter() + .filter(|route| route.kind == DataKind::Stream) + { + for edge in &route.edges { + let Some(pair) = stream_pair(case, &edge.path) else { + continue; + }; + let ActionOp::GatedStreamWrite { release_path, .. } = + &pair.writer.actions[pair.write_step].operation + else { + continue; + }; + if !matches!( + pair.reader.actions[pair.read_step].operation, + ActionOp::GatedStreamRead { .. } + ) || pair.writer.id != edge.source_role + || pair.reader.id != edge.destination_role + { + continue; + } + if let Some(publisher) = case.processes.iter().find(|program| { + program.depends_on.contains(process) + && program.actions.iter().any(|action| { + action.expected == ExpectedOutcome::Ok + && matches!(&action.operation, ActionOp::PublishBlob { path, .. } if path == release_path) + }) + }) { + return Some((pair, publisher)); + } + } + } + None +} + +struct StreamPair<'a> { + writer: &'a ProcessProgram, + write_step: usize, + reader: &'a ProcessProgram, + read_step: usize, +} + +fn stream_pair<'a>(case: &'a BehaviorCase, path: &str) -> Option> { + let mut writer = None; + let mut reader = None; + for program in &case.processes { + for (step, action) in program.actions.iter().enumerate() { + if action.operation.path() != path { + continue; + } + let endpoint = match action.operation { + ActionOp::StreamWrite { .. } | ActionOp::GatedStreamWrite { .. } => &mut writer, + ActionOp::StreamRead { .. } + | ActionOp::StreamReadWithRetry { .. } + | ActionOp::GatedStreamRead { .. } + | ActionOp::StreamReadInto { .. } => &mut reader, + _ => continue, + }; + if endpoint.replace((program, step)).is_some() { + return None; + } + } + } + let (writer, write_step) = writer?; + let (reader, read_step) = reader?; + (writer.id != reader.id).then_some(StreamPair { + writer, + write_step, + reader, + read_step, + }) +} + +fn active_stream_fault_pair( + case: &BehaviorCase, + scenario: CoverageScenario, +) -> Option> { + let FailureInjection::StopProcess { + process, + phase: crate::ir::ProcessStopPhase::AfterStreamFirstFrame, + .. + } = &case.failure + else { + return None; + }; + let writer_abort = scenario == CoverageScenario::ActiveStreamWriterAbort; + let target = case + .processes + .iter() + .find(|program| program.id == *process)?; + for action in &target.actions { + let Some(pair) = stream_pair(case, action.operation.path()) else { + continue; + }; + let (faulted, survivor, survivor_step) = if writer_abort { + (pair.writer, pair.reader, pair.read_step) + } else { + (pair.reader, pair.writer, pair.write_step) + }; + let faulted_step = if writer_abort { + pair.write_step + } else { + pair.read_step + }; + let first_frame = match &pair.writer.actions[pair.write_step].operation { + ActionOp::StreamWrite { chunks, .. } => chunks.first(), + ActionOp::GatedStreamWrite { frames, .. } => frames.first(), + _ => None, + }; + if faulted.id != *process + || faulted.actions[faulted_step].expected != ExpectedOutcome::Ok + || first_frame.is_none_or(Vec::is_empty) + || survivor.actions[survivor_step].expected + != ExpectedOutcome::Exception(crate::ir::PythonException::StreamError) + { + continue; + } + let path = action.operation.path(); + if survivor.actions.iter().skip(survivor_step + 1).any(|action| { + action.expected == ExpectedOutcome::Ok + && matches!(&action.operation, ActionOp::WaitForQuiescent { path: cleanup } if cleanup == path) + }) { + return Some(pair); + } + } + None +} + +/// Public quiescence followed by a later same-process open proves an old-to-new +/// binding even when the opposite endpoint joined the new pending incarnation. +/// The oracle has already checked both identities against a legal history. +fn quiescent_stream_origin( + path: &str, + replacement: u64, + index: &ObservationIndex<'_>, +) -> Option { + for records in index.processes.values() { + for (position, result) in records.execution.results.iter().enumerate() { + let Some(displaced) = result.revision else { + continue; + }; + if result.path != path + || result.outcome != "ok" + || result.kind.as_deref() != Some("stream") + || result.active != Some(false) + || displaced >= replacement + || !matches!( + records.program.actions[result.step].operation, + ActionOp::WaitForQuiescent { .. } + ) + { + continue; + } + // Check emission order too: route endpoints may be prepared before + // earlier-numbered actions, so step numbers alone are insufficient. + if records.execution.results[position + 1..] + .iter() + .any(|opened| { + opened.step > result.step + && opened.path == path + && opened.barrier + == Some(BarrierObservation::StreamOpened { + incarnation: replacement, + }) + && records.steps[opened.step].iter().any(|terminal| { + terminal.outcome == "ok" && terminal.incarnation == Some(replacement) + }) + }) + { + return Some(displaced); + } + } + } + None +} + +fn observed_stream_attachment(pair: &StreamPair<'_>, index: &ObservationIndex<'_>) -> Option { + let writer = index.processes.get(pair.writer.id.as_str())?; + let reader = index.processes.get(pair.reader.id.as_str())?; + let writer_records = &writer.steps[pair.write_step]; + let reader_records = &reader.steps[pair.read_step]; + let first = writer_records + .iter() + .find_map(|result| match &result.barrier { + Some(BarrierObservation::StreamFrame { + incarnation, + index: 0, + length, + digest, + }) if *incarnation != 0 && *length > 0 => { + Some((*incarnation, *length, digest.as_str())) + } + _ => None, + })?; + for records in [writer_records, reader_records] { + if !records.iter().any(|result| { + result.barrier + == Some(BarrierObservation::StreamOpened { + incarnation: first.0, + }) + }) { + return None; + } + } + if !reader_records.iter().any(|result| { + matches!(&result.barrier, Some(BarrierObservation::StreamFrame { + incarnation, index: 0, length, digest, + }) if *incarnation == first.0 && *length == first.1 && digest == first.2) + }) || !reader_records.iter().any(|result| { + result.barrier + == Some(BarrierObservation::StreamFirstFrame { + incarnation: first.0, + }) + }) { + return None; + } + Some(first.0) +} + +fn successful_data_observation(operation: &ActionOp, result: &ActionObservation) -> bool { + result.outcome == "ok" + && result.length.is_some() + && result.digest.is_some() + && (action_payload_length(operation).is_some() + || matches!(operation, ActionOp::StreamRoundTrip { .. })) +} + +/// Fresh edge paths can revisit physical nodes and end in a separate reader +/// on the origin node, but every intermediate receive must feed the next +/// outgoing action in the same process. Hints never define this structure. +fn returning_ring_hops(route: &DataRoute) -> Option { + if !route_has_topology(route, TopologyFamily::RingWalk) || !route.join_inputs.is_empty() { + return None; + } + let hops = route + .edges + .iter() + .map(|edge| edge.source) + .collect::>() + .len(); + if route.edges.len() < 2 * hops || route.edges.len() % hops != 0 { + return None; + } + let origin = route.edges[0].source; + if route.edges.last()?.destination != origin + || route.edges.windows(2).any(|pair| { + pair[0].destination != pair[1].source || pair[0].destination_role != pair[1].source_role + }) + || route.edges.iter().enumerate().any(|(index, edge)| { + let first_lap = &route.edges[index % hops]; + edge.source != first_lap.source + || edge.destination != first_lap.destination + || ((index + 1) % hops == 0 && edge.destination != origin) + }) + { + return None; + } + Some(hops) +} + +/// Shared structural admission; observations must independently prove transfers. +pub(crate) fn route_has_topology(route: &DataRoute, family: TopologyFamily) -> bool { + #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] + enum Vertex<'a> { + Node(u64), + Role(&'a str), + } + let mut degrees = BTreeMap::, (usize, usize)>::new(); + let mut edges = BTreeSet::new(); + for edge in &route.edges { + // Ring walks revisit physical nodes over fresh per-lap roles. Every + // other topology, especially a DAG, is a graph of endpoint roles. + let (source, destination) = if family == TopologyFamily::RingWalk { + (Vertex::Node(edge.source), Vertex::Node(edge.destination)) + } else { + ( + Vertex::Role(&edge.source_role), + Vertex::Role(&edge.destination_role), + ) + }; + if edges.insert((source, destination)) { + degrees.entry(source).or_default().1 += 1; + degrees.entry(destination).or_default().0 += 1; + } + } + let Some(&first) = degrees.keys().next() else { + return false; + }; + let mut connected = BTreeSet::from([first]); + loop { + let previous = connected.len(); + for &(source, destination) in &edges { + if connected.contains(&source) || connected.contains(&destination) { + connected.insert(source); + connected.insert(destination); + } + } + if previous == connected.len() { + break; + } + } + if connected.len() != degrees.len() { + return false; + } + let count = |degree| degrees.values().filter(|&&actual| actual == degree).count(); + let nodes = degrees.len(); + match family { + TopologyFamily::Fixed => true, + TopologyFamily::Chain => { + let physical_nodes = route + .edges + .iter() + .flat_map(|edge| [edge.source, edge.destination]) + .collect::>(); + (3..=5).contains(&nodes) + && physical_nodes.len() == nodes + && count((0, 1)) == 1 + && count((1, 0)) == 1 + && count((1, 1)) == nodes - 2 + } + TopologyFamily::RingWalk => nodes >= 3 && count((1, 1)) == nodes, + TopologyFamily::FanOut => { + (3..=5).contains(&nodes) && count((0, nodes - 1)) == 1 && count((1, 0)) == nodes - 1 + } + TopologyFamily::FanIn => { + (3..=5).contains(&nodes) && count((nodes - 1, 0)) == 1 && count((0, 1)) == nodes - 1 + } + TopologyFamily::Diamond => { + nodes == 4 && count((0, 2)) == 1 && count((1, 1)) == 2 && count((2, 0)) == 1 + } + TopologyFamily::RandomDag => { + if degrees + .values() + .any(|&(incoming, outgoing)| incoming > 4 || outgoing > 4) + { + return false; + } + let mut ready = degrees + .iter() + .filter_map(|(&node, &(incoming, _))| (incoming == 0).then_some(node)) + .collect::>(); + let mut visited = 0; + while let Some(node) = ready.pop() { + visited += 1; + for &(_, destination) in edges.iter().filter(|&&(source, _)| source == node) { + let incoming = &mut degrees.get_mut(&destination).expect("edge node").0; + *incoming -= 1; + if *incoming == 0 { + ready.push(destination); + } + } + } + (3..=20).contains(&nodes) && visited == nodes + } + } +} + +pub fn case_coverage(case: &BehaviorCase) -> BTreeSet { + let mut keys = BTreeSet::new(); + if case.topology != TopologyFamily::Fixed { + keys.insert(CoverageKey::Topology { + family: case.topology, + }); + } + for scenario in &case.scenarios { + if matches!( + scenario, + CoverageScenario::ActiveStreamWriterAbort | CoverageScenario::ActiveStreamReaderStop + ) && active_stream_fault_pair(case, *scenario).is_none() + { + continue; + } + keys.insert(CoverageKey::Scenario { + scenario: *scenario, + }); + } + if case + .scenarios + .contains(&CoverageScenario::ConcurrentStartup) + { + keys.insert(CoverageKey::ConcurrentStart { + family: case.topology, + }); + } + for route in &case.routes { + let mut incoming = BTreeMap::<&str, u64>::new(); + let mut outgoing = BTreeSet::<&str>::new(); + for edge in &route.edges { + outgoing.insert(&edge.source_role); + incoming.insert(&edge.destination_role, edge.destination); + keys.insert(CoverageKey::OrderedEdge { + source: edge.source, + destination: edge.destination, + kind: route.kind, + }); + keys.insert(CoverageKey::NodeRole { + node: edge.source, + kind: route.kind, + role: NodeRole::Source, + }); + keys.insert(CoverageKey::NodeRole { + node: edge.destination, + kind: route.kind, + role: NodeRole::Sink, + }); + } + for (_, node) in incoming + .iter() + .filter(|(role, _)| outgoing.contains(**role)) + { + keys.insert(CoverageKey::NodeRole { + node: *node, + kind: route.kind, + role: NodeRole::Relay, + }); + } + } + for process in &case.processes { + for pair in process.actions.windows(2) { + keys.insert(CoverageKey::ActionAdjacency { + before: pair[0].operation.class(), + after: pair[1].operation.class(), + }); + } + for action in &process.actions { + if let Some(length) = action_payload_length(&action.operation) { + for class in payload_classes(length) { + keys.insert(CoverageKey::Payload { class }); + } + } + match &action.operation { + ActionOp::DescriptorWrite { method, finish, .. } => { + keys.insert(CoverageKey::DescriptorMethod { + direction: "write".to_owned(), + method: descriptor_write_method(*method).to_owned(), + }); + keys.insert(CoverageKey::DescriptorTerminal { + direction: "write".to_owned(), + finish: descriptor_finish(*finish).to_owned(), + }); + } + ActionOp::DescriptorRead { method, finish, .. } => { + keys.insert(CoverageKey::DescriptorMethod { + direction: "read".to_owned(), + method: descriptor_read_method(*method).to_owned(), + }); + keys.insert(CoverageKey::DescriptorTerminal { + direction: "read".to_owned(), + finish: descriptor_finish(*finish).to_owned(), + }); + } + _ => {} + } + let outcome = match action.expected { + ExpectedOutcome::Ok => "success", + ExpectedOutcome::Error(_) + | ExpectedOutcome::Exception(_) + | ExpectedOutcome::Linearized { .. } => "modeled_error", + }; + keys.insert(CoverageKey::Outcome { + class: outcome.to_owned(), + }); + } + } + if !matches!( + case.failure, + crate::ir::FailureInjection::None | crate::ir::FailureInjection::SlowProcess { .. } + ) { + keys.insert(CoverageKey::Outcome { + class: "contextual_failure".to_owned(), + }); + } + keys +} + +fn action_payload_length(operation: &ActionOp) -> Option { + match operation { + ActionOp::PublishBlob { bytes, .. } | ActionOp::DescriptorWrite { bytes, .. } => { + Some(bytes.len()) + } + ActionOp::ReadBlob { expected, .. } + | ActionOp::StreamRead { expected, .. } + | ActionOp::StreamReadWithRetry { expected, .. } + | ActionOp::GatedStreamRead { expected, .. } + | ActionOp::StreamReadInto { expected, .. } + | ActionOp::DescriptorRead { expected, .. } => Some(expected.len()), + ActionOp::StreamWrite { chunks, .. } | ActionOp::StreamRoundTrip { chunks, .. } => { + Some(chunks.iter().map(Vec::len).sum()) + } + ActionOp::GatedStreamWrite { frames, .. } => Some(frames.iter().map(Vec::len).sum()), + _ => None, + } +} + +fn payload_classes(length: usize) -> BTreeSet { + let mut classes = BTreeSet::new(); + match length { + 0 => { + classes.insert(PayloadClass::Empty); + } + 1..=64 => { + classes.insert(PayloadClass::Small); + } + 255..=257 => { + classes.insert(PayloadClass::FramingBoundary); + } + 4_095..=4_097 | 65_535..=65_537 => { + classes.insert(PayloadClass::ChunkBoundary); + } + 65_538.. => { + classes.insert(PayloadClass::MultiChunk); + } + _ => { + classes.insert(PayloadClass::Randomized); + } + } + classes +} + +const fn descriptor_write_method(method: DescriptorWriteMethod) -> &'static str { + match method { + DescriptorWriteMethod::Write => "write", + DescriptorWriteMethod::WriteFrom => "write_from", + DescriptorWriteMethod::Mapping => "mapping", + } +} + +const fn descriptor_read_method(method: DescriptorReadMethod) -> &'static str { + match method { + DescriptorReadMethod::Read => "read", + DescriptorReadMethod::ReadInto => "read_into", + DescriptorReadMethod::Mapping => "mapping", + } +} + +const fn descriptor_finish(finish: DescriptorFinish) -> &'static str { + match finish { + DescriptorFinish::Close => "close", + DescriptorFinish::Abort => "abort", + DescriptorFinish::Drop => "drop", + DescriptorFinish::CloseTwice => "close_twice", + DescriptorFinish::AbortTwice => "abort_twice", + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::corpus::ordered_pair_corpus; + + fn evidence_identity(generation: &str) -> EvidenceIdentity { + EvidenceIdentity { + plan_digest: crate::codegen::digest(b"test immutable plan"), + artifacts_digest: crate::codegen::digest(b"test artifact contents"), + deployment_generation: generation.to_owned(), + fixture_mapping_digest: crate::codegen::digest(b"test exact fixture mapping"), + } + } + + #[test] + fn survivor_ledger_requires_every_exact_pair_and_role() { + let nodes = BTreeSet::from([2, 4, 7]); + let ledger = CoverageLedger::survivor(nodes.clone()).unwrap(); + for kind in [DataKind::Blob, DataKind::Stream] { + for source in &nodes { + for destination in &nodes { + if source != destination { + assert!(ledger.required.contains_key(&CoverageKey::OrderedEdge { + source: *source, + destination: *destination, + kind, + })); + } + } + } + } + } + + #[test] + fn incomplete_planned_coverage_is_a_failure() { + let mut ledger = CoverageLedger::five_node(BTreeSet::from([1, 2, 3, 4, 5])).unwrap(); + for case in ordered_pair_corpus(5, 11) { + ledger.plan_case(&case).unwrap(); + } + assert!( + ledger + .assert_planned_closed() + .unwrap_err() + .contains("incomplete") + ); + } + + #[test] + fn expected_launch_failure_counts_without_inventing_actions() { + use crate::ir::{FailureInjection, LaunchFailureKind}; + + let mut case = ordered_pair_corpus(5, 11).remove(0); + case.id = "expected-launch-failure".to_owned(); + case.processes.truncate(1); + case.failure = FailureInjection::LaunchFailure { + process: case.processes[0].id.clone(), + kind: LaunchFailureKind::MissingExecutable, + }; + let mut ledger = CoverageLedger::five_node(BTreeSet::from([1, 2, 3, 4, 5])).unwrap(); + ledger.plan_case(&case).unwrap(); + ledger + .bind_identity(evidence_identity("generation-a")) + .unwrap(); + let observation = CaseObservation { + case_id: case.id.clone(), + executions: case + .processes + .iter() + .map(|process| crate::ir::ExecutionObservation { + process: process.id.clone(), + request_id: case.execution_request_id(process), + logical_node_id: process.logical_node_id, + lifecycle: vec!["spawned".to_owned(), "spawn_failed".to_owned()], + results: Vec::new(), + terminal: true, + exit_success: false, + exit_status: None, + stdout: String::new(), + stderr: String::new(), + }) + .collect(), + }; + + ledger.observe_case(&case, &observation).unwrap(); + assert!(ledger.completed_cases.contains("expected-launch-failure")); + assert_eq!( + ledger.observed, + BTreeMap::from([( + CoverageKey::Outcome { + class: "contextual_failure".to_owned(), + }, + 1 + )]) + ); + assert!(ledger.assert_observed_closed().is_err()); + } + + fn observed_blob_chain() -> (BehaviorCase, CaseObservation) { + use crate::ir::{AccessSpec, Action, DataEdge, ExecutionObservation, ProcessProgram}; + + let mut case = ordered_pair_corpus(3, 11).remove(0); + case.id = "observed-chain".to_owned(); + case.topology = TopologyFamily::Chain; + let payload = b"route-payload".to_vec(); + let mut relayed = payload.clone(); + let rotation = 2 % relayed.len(); + relayed.rotate_left(rotation); + let left = "/cases/observed-chain/left".to_owned(); + let right = "/cases/observed-chain/right".to_owned(); + case.routes = vec![DataRoute { + id: "chain".to_owned(), + kind: DataKind::Blob, + edges: vec![ + DataEdge { + source: 1, + destination: 2, + source_role: "source".to_owned(), + destination_role: "relay".to_owned(), + path: left.clone(), + }, + DataEdge { + source: 2, + destination: 3, + source_role: "relay".to_owned(), + destination_role: "sink".to_owned(), + path: right.clone(), + }, + ], + join_inputs: Vec::new(), + }]; + case.processes = vec![ + ProcessProgram { + id: "source".to_owned(), + logical_node_id: 1, + access: AccessSpec::unrestricted("source"), + depends_on: Vec::new(), + actions: vec![Action::ok(ActionOp::PublishBlob { + path: left.clone(), + bytes: payload.clone(), + })], + }, + ProcessProgram { + id: "relay".to_owned(), + logical_node_id: 2, + access: AccessSpec::unrestricted("relay"), + depends_on: Vec::new(), + actions: vec![ + Action::ok(ActionOp::ReadBlob { + path: left, + expected: payload.clone(), + }), + Action::ok(ActionOp::PublishBlob { + path: right.clone(), + bytes: relayed.clone(), + }), + ], + }, + ProcessProgram { + id: "sink".to_owned(), + logical_node_id: 3, + access: AccessSpec::unrestricted("sink"), + depends_on: Vec::new(), + actions: vec![Action::ok(ActionOp::ReadBlob { + path: right, + expected: relayed, + })], + }, + ]; + let observation = CaseObservation { + case_id: case.id.clone(), + executions: case + .processes + .iter() + .map(|process| ExecutionObservation { + process: process.id.clone(), + request_id: case.execution_request_id(process), + logical_node_id: process.logical_node_id, + lifecycle: ["process_started", "context_ready", "user_result", "exited"] + .into_iter() + .map(str::to_owned) + .collect(), + results: process + .actions + .iter() + .enumerate() + .map(|(step, action)| { + let payload = match &action.operation { + ActionOp::PublishBlob { bytes, .. } => bytes, + ActionOp::ReadBlob { expected, .. } => expected, + operation => panic!( + "unexpected observed blob-chain operation: {operation:?}" + ), + }; + ActionObservation { + process: process.id.clone(), + step, + action: action.operation.class().as_str().to_owned(), + path: action.operation.path().to_owned(), + outcome: "ok".to_owned(), + length: Some(payload.len()), + digest: Some(crate::codegen::digest(payload)), + kind: None, + revision: None, + active: None, + errno: None, + error_type: None, + error: None, + descriptor: None, + transfer: None, + barrier: None, + incarnation: None, + token: None, + lap: None, + } + }) + .collect(), + terminal: true, + exit_success: true, + exit_status: Some("success".to_owned()), + stdout: String::new(), + stderr: String::new(), + }) + .collect(), + }; + (case, observation) + } + + fn observed_fairness() -> (BehaviorCase, CaseObservation) { + use crate::ir::{AccessSpec, Action}; + + let (mut case, template) = observed_blob_chain(); + case.topology = TopologyFamily::Fixed; + case.routes.clear(); + case.scenarios = BTreeSet::from([CoverageScenario::HotColdFairness]); + let hot = "/cases/observed-chain/hot"; + let observed = "/cases/observed-chain/first-frame"; + let release = "/cases/observed-chain/release"; + let frames = vec![b"first".to_vec(), b"after-release".to_vec()]; + let program = |id: &str, node, actions, depends_on| ProcessProgram { + id: id.to_owned(), + logical_node_id: node, + access: AccessSpec::unrestricted(id), + actions, + depends_on, + }; + case.processes = vec![ + program( + "hot-writer", + 1, + vec![Action::ok(ActionOp::GatedStreamWrite { + path: hot.to_owned(), + replace: false, + frames: frames.clone(), + release_path: release.to_owned(), + })], + Vec::new(), + ), + program( + "hot-reader", + 2, + vec![Action::ok(ActionOp::GatedStreamRead { + path: hot.to_owned(), + expected: frames.concat(), + observed_path: observed.to_owned(), + retry_attach: false, + park_after_first_frame: false, + })], + Vec::new(), + ), + ]; + for id in ["cold-a", "cold-b"] { + let path = format!("/cases/observed-chain/{id}"); + case.processes.push(program( + id, + 3, + vec![ + Action::ok(ActionOp::AwaitEntry { + path: observed.to_owned(), + expected_kind: "blob".to_owned(), + }), + Action::ok(ActionOp::PublishBlob { + path: path.clone(), + bytes: b"cold".to_vec(), + }), + Action::ok(ActionOp::ReadBlob { + path, + expected: b"cold".to_vec(), + }), + ], + Vec::new(), + )); + } + case.processes.push(program( + "release", + 3, + vec![Action::ok(ActionOp::PublishBlob { + path: release.to_owned(), + bytes: Vec::new(), + })], + vec!["cold-a".to_owned(), "cold-b".to_owned()], + )); + let observation = CaseObservation { + case_id: case.id.clone(), + executions: case + .processes + .iter() + .map(|program| { + let mut execution = template.executions[0].clone(); + execution.process = program.id.clone(); + execution.request_id = case.execution_request_id(program); + execution.logical_node_id = program.logical_node_id; + execution.results.clear(); + for (step, action) in program.actions.iter().enumerate() { + let mut terminal = template.executions[0].results[0].clone(); + terminal.process = program.id.clone(); + terminal.step = step; + terminal.action = action.operation.class().as_str().to_owned(); + terminal.path = action.operation.path().to_owned(); + let payload = match &action.operation { + ActionOp::PublishBlob { bytes, .. } => Some(bytes.clone()), + ActionOp::ReadBlob { expected, .. } + | ActionOp::GatedStreamRead { expected, .. } => Some(expected.clone()), + ActionOp::GatedStreamWrite { frames, .. } => Some(frames.concat()), + ActionOp::AwaitEntry { .. } => { + terminal.kind = Some("blob".to_owned()); + terminal.revision = Some(2); + None + } + _ => unreachable!(), + }; + terminal.length = payload.as_ref().map(Vec::len); + terminal.digest = + payload.as_ref().map(|bytes| crate::codegen::digest(bytes)); + let write = matches!(action.operation, ActionOp::GatedStreamWrite { .. }); + let read = matches!(action.operation, ActionOp::GatedStreamRead { .. }); + if write || read { + terminal.incarnation = Some(1); + let mut emit = |barrier| { + let mut record = terminal.clone(); + record.outcome = "barrier".to_owned(); + record.length = None; + record.digest = None; + record.barrier = Some(barrier); + execution.results.push(record); + }; + emit(BarrierObservation::StreamOpened { incarnation: 1 }); + for (index, frame) in frames.iter().enumerate() { + emit(BarrierObservation::StreamFrame { + incarnation: 1, + index: index as u64, + length: frame.len() as u64, + digest: crate::codegen::digest(frame), + }); + if index == 0 { + emit(if write { + BarrierObservation::ReleaseObserved { + path: release.to_owned(), + } + } else { + BarrierObservation::StreamFirstFrame { incarnation: 1 } + }); + } + } + if read { + emit(BarrierObservation::StreamEof { incarnation: 1 }); + } + } + execution.results.push(terminal); + } + execution + }) + .collect(), + }; + (case, observation) + } + + fn observed_active_stream_fault(writer_abort: bool) -> (BehaviorCase, CaseObservation) { + let (mut case, mut observation) = observed_fairness(); + case.processes.truncate(2); + observation.executions.truncate(2); + let target = if writer_abort { 0 } else { 1 }; + let survivor = 1 - target; + let scenario = if writer_abort { + CoverageScenario::ActiveStreamWriterAbort + } else { + CoverageScenario::ActiveStreamReaderStop + }; + case.scenarios = BTreeSet::from([scenario]); + case.failure = FailureInjection::StopProcess { + process: case.processes[target].id.clone(), + phase: crate::ir::ProcessStopPhase::AfterStreamFirstFrame, + kill_after_ms: Some(100), + }; + if !writer_abort { + let ActionOp::GatedStreamWrite { path, frames, .. } = + case.processes[0].actions[0].operation.clone() + else { + unreachable!(); + }; + case.processes[0].actions[0].operation = ActionOp::StreamWrite { + path, + chunks: frames, + replace: false, + }; + let ActionOp::GatedStreamRead { + park_after_first_frame, + .. + } = &mut case.processes[1].actions[0].operation + else { + unreachable!(); + }; + *park_after_first_frame = true; + } + case.processes[survivor].actions[0].expected = + ExpectedOutcome::Exception(crate::ir::PythonException::StreamError); + let path = case.processes[survivor].actions[0] + .operation + .path() + .to_owned(); + case.processes[survivor] + .actions + .push(crate::ir::Action::ok(ActionOp::WaitForQuiescent { path })); + for (position, execution) in observation.executions.iter_mut().enumerate() { + let mut terminal = execution.results.last().unwrap().clone(); + execution.results.retain(|result| { + matches!( + result.barrier, + Some( + BarrierObservation::StreamOpened { .. } + | BarrierObservation::StreamFrame { index: 0, .. } + | BarrierObservation::StreamFirstFrame { .. } + ) + ) + }); + if position == target { + execution.exit_success = false; + execution.exit_status = Some("signal=15".to_owned()); + execution.lifecycle = [ + "process_started", + "context_ready", + "stop_accepted", + "exited", + ] + .into_iter() + .map(str::to_owned) + .collect(); + } else { + let mut cleanup = terminal.clone(); + cleanup.step = 1; + cleanup.action = "lookup".to_owned(); + cleanup.length = None; + cleanup.digest = None; + cleanup.incarnation = None; + cleanup.kind = Some("stream".to_owned()); + cleanup.revision = Some(1); + cleanup.active = Some(false); + terminal.outcome = "expected_error".to_owned(); + terminal.length = None; + terminal.digest = None; + terminal.error_type = Some("StreamError".to_owned()); + terminal.transfer = Some(crate::ir::TransferObservation { + length: b"first".len(), + digest: crate::codegen::digest(b"first"), + complete: false, + }); + execution.results.extend([terminal, cleanup]); + } + } + (case, observation) + } + + #[test] + fn active_stream_fault_credit_requires_attached_peer_loss_and_exact_cleanup() { + for writer_abort in [true, false] { + let (case, observation) = observed_active_stream_fault(writer_abort); + let scenario = *case.scenarios.first().unwrap(); + let mut complete = ledger_for_case(&case); + complete.observe_case(&case, &observation).unwrap(); + assert!( + complete + .observed + .contains_key(&CoverageKey::Scenario { scenario }) + ); + let survivor = usize::from(writer_abort); + for corruption in 0..3 { + let mut incomplete = observation.clone(); + match corruption { + 0 => incomplete.executions[1].results.retain(|result| { + !matches!( + result.barrier, + Some(BarrierObservation::StreamFirstFrame { .. }) + ) + }), + 1 => { + let error = incomplete.executions[survivor] + .results + .iter_mut() + .find(|result| result.outcome == "expected_error") + .unwrap(); + error.errno = Some(libc::ESTALE); + error.error_type = Some("OSError".to_owned()); + } + _ => { + incomplete.executions[survivor] + .results + .last_mut() + .unwrap() + .revision = Some(2) + } + } + let mut rejected = ledger_for_case(&case); + assert!(rejected.observe_case(&case, &incomplete).is_err()); + assert!(rejected.observed.is_empty()); + assert!(rejected.completed_cases.is_empty()); + } + } + } + + #[test] + fn descriptor_aborts_unmatched_stops_and_replacements_leave_stream_fault_obligations_open() { + let mut ledger = CoverageLedger::five_node(BTreeSet::from([1, 2, 3, 4, 5])).unwrap(); + let faults = [ + CoverageScenario::ActiveStreamWriterAbort, + CoverageScenario::ActiveStreamReaderStop, + ]; + for mut case in crate::corpus::stable_corpus(5, 11) + .into_iter() + .chain(crate::corpus::failure_corpus(5, 11)) + .filter(|case| { + matches!(case.failure, FailureInjection::StopProcess { .. }) + || case + .processes + .iter() + .flat_map(|program| &program.actions) + .any(|action| { + matches!( + action.operation, + ActionOp::DescriptorWrite { + finish: DescriptorFinish::Abort | DescriptorFinish::AbortTwice, + .. + } | ActionOp::DescriptorRead { + finish: DescriptorFinish::Abort | DescriptorFinish::AbortTwice, + .. + } | ActionOp::StreamWrite { replace: true, .. } + ) + }) + }) + { + // Even relabeling the old corpus cannot plan an attached fault. + case.scenarios.extend(faults); + ledger.plan_case(&case).unwrap(); + } + for scenario in faults { + let key = CoverageKey::Scenario { scenario }; + assert_eq!(ledger.required.get(&key), Some(&1)); + assert!(!ledger.planned.contains_key(&key)); + assert!(!ledger.observed.contains_key(&key)); + } + assert!(ledger.assert_planned_closed().is_err()); + } + + #[test] + fn error_only_cold_flows_cannot_close_fairness() { + let (case, observation) = observed_fairness(); + let mut complete = ledger_for_case(&case); + complete.observe_case(&case, &observation).unwrap(); + assert!(complete.observed.contains_key(&CoverageKey::Scenario { + scenario: CoverageScenario::HotColdFairness, + })); + // Retain the first-frame gate, release dependency, and all hot-stream + // evidence. Even one error-only cold must prevent fairness credit. + for cold in ["cold-a", "cold-b"] { + let mut denied_case = case.clone(); + let mut denied_observation = observation.clone(); + let program = denied_case + .processes + .iter_mut() + .find(|p| p.id == cold) + .unwrap(); + let gate_path = program.actions[0].operation.path().to_owned(); + program.access.read_prefixes = vec![gate_path]; + program.actions.truncate(1); + program.actions.push(crate::ir::Action::error( + ActionOp::ReadBlob { + path: format!("/cases/observed-chain/denied-{cold}"), + expected: b"cold".to_vec(), + }, + libc::EACCES, + )); + let execution = denied_observation + .executions + .iter_mut() + .find(|e| e.process == cold) + .unwrap(); + let mut denied = execution.results.last().unwrap().clone(); + denied.step = 1; + denied.path = program.actions[1].operation.path().to_owned(); + denied.outcome = "expected_error".to_owned(); + denied.errno = Some(libc::EACCES); + denied.length = None; + denied.digest = None; + execution.results.truncate(1); + execution.results.push(denied); + crate::oracle::BehaviorOracle::verify(&denied_case, &denied_observation).unwrap(); + let mut rejected = ledger_for_case(&denied_case); + assert!( + rejected + .observe_case(&denied_case, &denied_observation) + .is_err() + ); + assert!(rejected.observed.is_empty()); + } + } + + #[test] + fn churn_requires_route_continuation_released_by_the_stop() { + let (mut case, mut observation) = observed_fairness(); + case.processes + .retain(|program| !program.id.starts_with("cold-")); + observation + .executions + .retain(|execution| !execution.process.starts_with("cold-")); + let mut target = case.processes[0].clone(); + target.id = "churn-target".to_owned(); + target.access = crate::ir::AccessSpec::unrestricted("churn-target"); + target.actions = vec![crate::ir::Action::ok(ActionOp::StreamRead { + path: "/cases/observed-chain/parked".to_owned(), + expected: Vec::new(), + })]; + case.processes[2].depends_on = vec![target.id.clone()]; + let mut stopped = observation.executions[0].clone(); + stopped.process = target.id.clone(); + stopped.request_id = case.execution_request_id(&target); + stopped.results.clear(); + stopped.exit_success = false; + stopped.exit_status = Some("signal=15".to_owned()); + stopped.lifecycle = [ + "process_started", + "context_ready", + "stop_accepted", + "exited", + ] + .into_iter() + .map(str::to_owned) + .collect(); + case.failure = FailureInjection::StopProcess { + process: target.id.clone(), + phase: crate::ir::ProcessStopPhase::AfterSiblingStreamFirstFrame, + kill_after_ms: Some(100), + }; + case.processes.push(target); + observation.executions.push(stopped); + case.scenarios = BTreeSet::from([CoverageScenario::ProcessChurn]); + case.routes = vec![DataRoute { + id: "healthy-gated-route".to_owned(), + kind: DataKind::Stream, + edges: vec![crate::ir::DataEdge { + source: 1, + destination: 2, + source_role: case.processes[0].id.clone(), + destination_role: case.processes[1].id.clone(), + path: case.processes[0].actions[0].operation.path().to_owned(), + }], + join_inputs: Vec::new(), + }]; + let mut complete = ledger_for_case(&case); + complete.observe_case(&case, &observation).unwrap(); + assert!(complete.observed.contains_key(&CoverageKey::Scenario { + scenario: CoverageScenario::ProcessChurn, + })); + // Identical successful work is not proof that it crossed the stop + // when release can publish independently of the target's terminal. + case.processes[2].depends_on.clear(); + crate::oracle::BehaviorOracle::verify(&case, &observation).unwrap(); + let mut rejected = ledger_for_case(&case); + assert!(rejected.observe_case(&case, &observation).is_err()); + assert!(rejected.observed.is_empty()); + } + + #[test] + fn serialized_stop_after_healthy_routes_cannot_close_churn() { + let (mut case, mut observation) = observed_blob_chain(); + let mut target = case.processes[0].clone(); + target.id = "churn-target".to_owned(); + target.access = crate::ir::AccessSpec::unrestricted("churn-target"); + target.depends_on = case + .processes + .iter() + .map(|program| program.id.clone()) + .collect(); + target.actions = vec![crate::ir::Action::ok(ActionOp::StreamRead { + path: "/cases/observed-chain/parked".to_owned(), + expected: Vec::new(), + })]; + let mut stopped = observation.executions[0].clone(); + stopped.process = target.id.clone(); + stopped.request_id = case.execution_request_id(&target); + stopped.results.clear(); + stopped.exit_success = false; + stopped.exit_status = Some("signal=15".to_owned()); + stopped.lifecycle = [ + "process_started", + "context_ready", + "stop_accepted", + "exited", + ] + .into_iter() + .map(str::to_owned) + .collect(); + case.failure = FailureInjection::StopProcess { + process: target.id.clone(), + phase: crate::ir::ProcessStopPhase::AfterContextReady, + kill_after_ms: Some(100), + }; + case.processes.push(target); + observation.executions.push(stopped); + crate::oracle::BehaviorOracle::verify(&case, &observation).unwrap(); + case.scenarios.insert(CoverageScenario::ProcessChurn); + let mut rejected = ledger_for_case(&case); + assert!(rejected.observe_case(&case, &observation).is_err()); + assert!(rejected.observed.is_empty()); + } + + fn ledger_for_case(case: &BehaviorCase) -> CoverageLedger { + let mut ledger = CoverageLedger::survivor(case.live_nodes.clone()).unwrap(); + ledger.required = case_coverage(case) + .into_iter() + .map(|key| (key, 1)) + .collect(); + ledger.plan_case(case).unwrap(); + ledger + .bind_identity(evidence_identity("generation-a")) + .unwrap(); + ledger + } + + #[test] + fn coverage_rejects_forged_missing_and_swapped_records_atomically() { + let (case, observation) = observed_blob_chain(); + let mut valid = ledger_for_case(&case); + valid.observe_case(&case, &observation).unwrap(); + valid.assert_observed_closed().unwrap(); + + let corruptions: &[(&str, fn(&mut CaseObservation))] = &[ + ("mislocated execution", |seen| { + seen.executions[0].logical_node_id = 3 + }), + ("duplicate execution", |seen| { + seen.executions[2] = seen.executions[0].clone() + }), + ("duplicate request", |seen| { + seen.executions[2].request_id = seen.executions[0].request_id.clone() + }), + ("swapped process records", |seen| { + let source = seen.executions[0].results[0].clone(); + seen.executions[0].results[0] = seen.executions[2].results[0].clone(); + seen.executions[2].results[0] = source; + }), + ("duplicate action replacing missing step", |seen| { + seen.executions[1].results[1] = seen.executions[1].results[0].clone(); + }), + ("reordered actions", |seen| { + seen.executions[1].results.swap(0, 1) + }), + ("wrong action", |seen| { + seen.executions[0].results[0].action = "stream_write".to_owned() + }), + ("wrong path", |seen| { + seen.executions[0].results[0].path.push_str("-other") + }), + ("wrong payload length", |seen| { + seen.executions[0].results[0].length = Some(0) + }), + ("missing payload digest", |seen| { + seen.executions[0].results[0].digest = None + }), + ("wrong payload digest", |seen| { + seen.executions[0].results[0].digest = Some(crate::codegen::digest(b"forged")) + }), + ("wrong outcome", |seen| { + seen.executions[0].results[0].outcome = "expected_error".to_owned() + }), + ]; + for (label, corrupt) in corruptions { + let mut ledger = ledger_for_case(&case); + let mut forged = observation.clone(); + corrupt(&mut forged); + assert!(ledger.observe_case(&case, &forged).is_err(), "{label}"); + assert!(ledger.assert_observed_closed().is_err(), "{label}"); + assert!(ledger.observed.is_empty(), "{label}"); + assert!(!ledger.completed_cases.contains(&case.id), "{label}"); + // A rejected attempt cannot consume the completion identity. + ledger.observe_case(&case, &observation).unwrap(); + ledger.assert_observed_closed().unwrap(); + } + } + + #[test] + fn route_admission_requires_endpoint_roles_and_receive_before_forward() { + let (case, _) = observed_blob_chain(); + let mut wrong_route = case.clone(); + wrong_route.routes[0].edges[0].source = 3; + wrong_route.topology = TopologyFamily::Fixed; + let mut planned = CoverageLedger::survivor(case.live_nodes.clone()).unwrap(); + assert!(planned.plan_case(&wrong_route).is_err()); + + let mut wrong_topology = case.clone(); + wrong_topology.topology = TopologyFamily::Diamond; + assert!(planned.plan_case(&wrong_topology).is_err()); + + let mut no_relay = case; + no_relay.processes[1].actions.swap(0, 1); + assert!(planned.plan_case(&no_relay).is_err()); + } + + fn label_ring_observation( + case: &mut BehaviorCase, + observation: &mut CaseObservation, + hops: usize, + ) { + for (program, execution) in case.processes.iter_mut().zip(&mut observation.executions) { + let mut labeled = Vec::new(); + for mut result in execution.results.drain(..) { + let edge_index = case.routes[0] + .edges + .iter() + .position(|edge| edge.path == result.path) + .unwrap(); + let lap = (edge_index / hops) as u32; + let token = result.digest.clone().unwrap(); + let source = result.action == "publish_blob"; + let completed = !source && edge_index + 1 == case.routes[0].edges.len(); + let barrier_name = if source { + crate::ir::BARRIER_TOKEN_FORWARDED + } else if completed { + crate::ir::BARRIER_LAP_COMPLETED + } else { + crate::ir::BARRIER_TOKEN_RECEIVED + }; + program.actions[result.step].evidence_hints = BTreeMap::from([ + ("token".to_owned(), token.clone()), + ("lap".to_owned(), lap.to_string()), + ("edge_index".to_owned(), edge_index.to_string()), + ("barrier".to_owned(), barrier_name.to_owned()), + ]); + result.token = Some(token.clone()); + result.lap = Some(lap); + let mut milestone = result.clone(); + milestone.outcome = "barrier".to_owned(); + milestone.length = None; + milestone.digest = None; + milestone.barrier = Some(if source { + BarrierObservation::TokenForwarded { + token, + lap, + edge_index: edge_index as u32, + } + } else if completed { + BarrierObservation::LapCompleted { token, lap } + } else { + BarrierObservation::TokenReceived { + token, + lap, + edge_index: edge_index as u32, + } + }); + labeled.extend([result, milestone]); + } + execution.results = labeled; + } + case.scenarios.insert(CoverageScenario::RingCompletion); + } + + #[test] + fn labeled_open_chain_cannot_complete_a_ring() { + let (mut case, mut observation) = observed_blob_chain(); + label_ring_observation(&mut case, &mut observation, 2); + // All hinted transfers and their payloads really succeeded. Only the + // alleged return/multi-lap traversal is missing. + crate::oracle::BehaviorOracle::verify(&case, &observation).unwrap(); + let mut ledger = ledger_for_case(&case); + assert!(ledger.observe_case(&case, &observation).is_err()); + assert!(ledger.observed.is_empty()); + assert!(ledger.completed_cases.is_empty()); + } + + #[test] + fn ring_laps_through_one_node_close_coverage_across_consumers() { + use crate::ir::{AccessSpec, Action, DataEdge, ExecutionObservation, ProcessProgram}; + + let mut case = ordered_pair_corpus(3, 11).remove(0); + case.id = "observed-ring".to_owned(); + case.live_nodes = BTreeSet::from([2, 3, 4]); + case.topology = TopologyFamily::RingWalk; + let lap0 = b"ring-lap-0-payload".to_vec(); + let lap1 = b"ring-lap-1-payload".to_vec(); + let p = |hop: &str| format!("/cases/observed-ring/{hop}"); + let edge = |source: u64, + destination: u64, + source_role: &str, + destination_role: &str, + hop: &str| DataEdge { + source, + destination, + source_role: source_role.to_owned(), + destination_role: destination_role.to_owned(), + path: p(hop), + }; + case.routes = vec![DataRoute { + id: "ring".to_owned(), + kind: DataKind::Blob, + edges: vec![ + edge(2, 3, "origin-relay", "relay-3", "lap-0/2-3"), + edge(3, 4, "relay-3", "relay-4", "lap-0/3-4"), + edge(4, 2, "relay-4", "origin-relay", "lap-0/4-2"), + edge(2, 3, "origin-relay", "relay-3", "lap-1/2-3"), + edge(3, 4, "relay-3", "relay-4", "lap-1/3-4"), + edge(4, 2, "relay-4", "reader", "lap-1/4-2"), + ], + join_inputs: Vec::new(), + }]; + let publish = |hop: &str, payload: &Vec| { + Action::ok(ActionOp::PublishBlob { + path: p(hop), + bytes: payload.clone(), + }) + }; + let read = |hop: &str, payload: &Vec| { + Action::ok(ActionOp::ReadBlob { + path: p(hop), + expected: payload.clone(), + }) + }; + // The origin relay closes lap 0 and mints the lap-1 payload; a + // dedicated reader terminates lap 1. Both lap closures are edges + // from node 4 into node 2 but land in different executions. + case.processes = vec![ + ProcessProgram { + id: "origin-relay".to_owned(), + logical_node_id: 2, + access: AccessSpec::unrestricted("origin-relay"), + depends_on: Vec::new(), + actions: vec![ + publish("lap-0/2-3", &lap0), + read("lap-0/4-2", &lap0), + publish("lap-1/2-3", &lap1), + ], + }, + ProcessProgram { + id: "relay-3".to_owned(), + logical_node_id: 3, + access: AccessSpec::unrestricted("relay-3"), + depends_on: Vec::new(), + actions: vec![ + read("lap-0/2-3", &lap0), + publish("lap-0/3-4", &lap0), + read("lap-1/2-3", &lap1), + publish("lap-1/3-4", &lap1), + ], + }, + ProcessProgram { + id: "relay-4".to_owned(), + logical_node_id: 4, + access: AccessSpec::unrestricted("relay-4"), + depends_on: Vec::new(), + actions: vec![ + read("lap-0/3-4", &lap0), + publish("lap-0/4-2", &lap0), + read("lap-1/3-4", &lap1), + publish("lap-1/4-2", &lap1), + ], + }, + ProcessProgram { + id: "reader".to_owned(), + logical_node_id: 2, + access: AccessSpec::unrestricted("reader"), + depends_on: Vec::new(), + actions: vec![read("lap-1/4-2", &lap1)], + }, + ]; + let payload_for = |hop: &str| match hop.starts_with("lap-0") { + true => lap0.clone(), + false => lap1.clone(), + }; + let mut observation = CaseObservation { + case_id: case.id.clone(), + executions: case + .processes + .iter() + .map(|process| ExecutionObservation { + process: process.id.clone(), + request_id: case.execution_request_id(process), + logical_node_id: process.logical_node_id, + lifecycle: ["process_started", "context_ready", "user_result", "exited"] + .into_iter() + .map(str::to_owned) + .collect(), + results: process + .actions + .iter() + .enumerate() + .map(|(step, action)| { + let payload = payload_for( + action + .operation + .path() + .trim_start_matches("/cases/observed-ring/"), + ); + ActionObservation { + process: process.id.clone(), + step, + action: action.operation.class().as_str().to_owned(), + path: action.operation.path().to_owned(), + outcome: "ok".to_owned(), + length: Some(payload.len()), + digest: Some(crate::codegen::digest(&payload)), + kind: None, + revision: None, + active: None, + errno: None, + error_type: None, + error: None, + descriptor: None, + transfer: None, + barrier: None, + incarnation: None, + token: None, + lap: None, + } + }) + .collect(), + terminal: true, + exit_success: true, + exit_status: Some("success".to_owned()), + stdout: String::new(), + stderr: String::new(), + }) + .collect(), + }; + label_ring_observation(&mut case, &mut observation, 3); + let mut ledger = ledger_for_case(&case); + ledger.observe_case(&case, &observation).unwrap(); + ledger.assert_observed_closed().unwrap(); + assert!(ledger.observed.contains_key(&CoverageKey::Scenario { + scenario: CoverageScenario::RingCompletion, + })); + + // A genuine single closed lap still does not prove repeated traversal. + let mut single_lap = case.clone(); + single_lap.routes[0].edges.truncate(3); + let index = ObservationIndex::new(&single_lap, &observation).unwrap(); + let mut keys = BTreeSet::new(); + assert!( + !observe_scenario( + &single_lap, + CoverageScenario::RingCompletion, + &index, + &mut keys, + ) + .unwrap() + ); + assert!(!keys.contains(&CoverageKey::Scenario { + scenario: CoverageScenario::RingCompletion, + })); + + // A physical placement is not ownership: a claimed reader role must + // execute that edge's read itself, not another process on its node. + let mut fanin_case = case.clone(); + fanin_case.topology = TopologyFamily::Fixed; + fanin_case.routes[0].edges.push(DataEdge { + source: 3, + destination: 2, + source_role: "relay-3".to_owned(), + destination_role: "reader".to_owned(), + path: p("lap-1/3-2"), + }); + fanin_case.processes[0] + .actions + .push(read("lap-1/3-2", &lap1)); + fanin_case.processes[1] + .actions + .push(publish("lap-1/3-2", &lap1)); + let mut fanin_ledger = CoverageLedger::survivor(fanin_case.live_nodes.clone()).unwrap(); + assert!(fanin_ledger.plan_case(&fanin_case).is_err()); + } + + #[test] + fn labels_without_required_observation_metadata_do_not_close_coverage() { + let (mut case, observation) = observed_blob_chain(); + case.scenarios.insert(CoverageScenario::ConcurrentStartup); + let mut ledger = ledger_for_case(&case); + assert!(ledger.observe_case(&case, &observation).is_err()); + assert!(ledger.assert_observed_closed().is_err()); + assert!(ledger.observed.is_empty()); + + case.scenarios.clear(); + let ActionOp::PublishBlob { path, bytes } = case.processes[0].actions[0].operation.clone() + else { + unreachable!(); + }; + case.processes[0].actions[0].operation = ActionOp::DescriptorWrite { + path, + length: Some(bytes.len() as u64), + bytes, + flags: libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL | libc::O_TRUNC, + method: DescriptorWriteMethod::WriteFrom, + finish: DescriptorFinish::Abort, + }; + case.routes.clear(); + case.topology = TopologyFamily::Fixed; + case.processes.truncate(1); + let mut descriptor_observation = observation; + descriptor_observation.executions.truncate(1); + descriptor_observation.executions[0].results[0].action = "descriptor".to_owned(); + let mut ledger = ledger_for_case(&case); + assert!(ledger.observe_case(&case, &descriptor_observation).is_err()); + assert!(ledger.assert_observed_closed().is_err()); + assert!(ledger.observed.is_empty()); + descriptor_observation.executions[0].results[0].descriptor = + Some(DescriptorObservation::Write { + method: DescriptorWriteMethod::WriteFrom, + finish: DescriptorFinish::Close, + terminal_results: vec![DescriptorTerminalResult::Ok], + dropped: false, + reservation_released: false, + }); + assert!(ledger.observe_case(&case, &descriptor_observation).is_err()); + descriptor_observation.executions[0].results[0].descriptor = + Some(DescriptorObservation::Write { + method: DescriptorWriteMethod::WriteFrom, + finish: DescriptorFinish::Abort, + terminal_results: vec![DescriptorTerminalResult::Ok], + dropped: false, + reservation_released: false, + }); + ledger.observe_case(&case, &descriptor_observation).unwrap(); + ledger.assert_observed_closed().unwrap(); + } + + #[test] + fn modeled_error_does_not_credit_payload_that_was_not_transferred() { + let (mut case, mut observation) = observed_blob_chain(); + case.topology = TopologyFamily::Fixed; + case.routes.clear(); + case.processes.truncate(1); + // EACCES is only a legal modeled outcome when the session's write + // prefixes genuinely deny the publication path. + case.processes[0].access.write_prefixes = vec!["/runs".to_owned()]; + case.processes[0].actions[0].expected = ExpectedOutcome::Error(libc::EACCES); + observation.executions.truncate(1); + let result = &mut observation.executions[0].results[0]; + result.outcome = "expected_error".to_owned(); + result.errno = Some(libc::EACCES); + result.length = None; + result.digest = None; + let mut ledger = ledger_for_case(&case); + ledger.observe_case(&case, &observation).unwrap(); + assert_eq!( + ledger.observed, + BTreeMap::from([( + CoverageKey::Outcome { + class: "modeled_error".to_owned() + }, + 1 + )]) + ); + assert!(ledger.assert_observed_closed().is_err()); + } + + #[test] + fn repeated_descriptor_terminal_requires_both_observed_attempts() { + let (mut case, mut observation) = observed_blob_chain(); + case.topology = TopologyFamily::Fixed; + case.routes.clear(); + case.processes.truncate(1); + observation.executions.truncate(1); + let ActionOp::PublishBlob { path, bytes } = case.processes[0].actions[0].operation.clone() + else { + unreachable!(); + }; + case.processes[0].actions[0] = crate::ir::Action::error( + ActionOp::DescriptorWrite { + path, + length: Some(bytes.len() as u64), + bytes: bytes.clone(), + flags: libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL | libc::O_TRUNC, + method: DescriptorWriteMethod::Write, + finish: DescriptorFinish::CloseTwice, + }, + libc::EBADF, + ); + let terminal_error = DescriptorTerminalResult::Error { + errno: Some(libc::EBADF), + error_type: "OSError".to_owned(), + }; + let result = &mut observation.executions[0].results[0]; + result.action = "descriptor".to_owned(); + result.outcome = "expected_error".to_owned(); + result.errno = Some(libc::EBADF); + result.error_type = Some("OSError".to_owned()); + result.length = None; + result.digest = None; + result.descriptor = Some(DescriptorObservation::Write { + method: DescriptorWriteMethod::Write, + finish: DescriptorFinish::CloseTwice, + terminal_results: vec![DescriptorTerminalResult::Ok, terminal_error.clone()], + dropped: false, + reservation_released: false, + }); + let mut without_bytes = ledger_for_case(&case); + assert!(without_bytes.observe_case(&case, &observation).is_err()); + assert!(without_bytes.observed.is_empty()); + + observation.executions[0].results[0].transfer = Some(crate::ir::TransferObservation { + length: bytes.len(), + digest: crate::codegen::digest(&bytes), + complete: true, + }); + let mut complete = ledger_for_case(&case); + complete.observe_case(&case, &observation).unwrap(); + complete.assert_observed_closed().unwrap(); + + for terminal_results in [ + vec![terminal_error.clone()], + vec![terminal_error, DescriptorTerminalResult::Ok], + vec![ + DescriptorTerminalResult::Ok, + DescriptorTerminalResult::Error { + errno: Some(libc::ENOENT), + error_type: "OSError".to_owned(), + }, + ], + ] { + let mut forged = observation.clone(); + forged.executions[0].results[0].descriptor = Some(DescriptorObservation::Write { + method: DescriptorWriteMethod::Write, + finish: DescriptorFinish::CloseTwice, + terminal_results, + dropped: false, + reservation_released: false, + }); + let mut ledger = ledger_for_case(&case); + assert!(ledger.observe_case(&case, &forged).is_err()); + assert!(ledger.observed.is_empty()); + assert!(ledger.assert_observed_closed().is_err()); + } + } + + #[test] + fn legacy_planned_recount_ledgers_cannot_claim_observed_closure() { + let (case, observation) = observed_blob_chain(); + let mut ledger = ledger_for_case(&case); + ledger.observe_case(&case, &observation).unwrap(); + ledger.assert_observed_closed().unwrap(); + ledger.schema_version = 1; + assert!(ledger.assert_observed_closed().is_err()); + } + + fn observed_stream_replacement(reader_first: bool) -> (BehaviorCase, CaseObservation) { + let mut case = crate::corpus::stable_corpus(3, 11) + .into_iter() + .find(|case| case.id == "stream-replacement-11") + .unwrap(); + case.scenarios.insert(CoverageScenario::QuiescentMutation); + let (_, template) = observed_blob_chain(); + let mut observation = CaseObservation { + case_id: case.id.clone(), + executions: Vec::new(), + }; + // Public revisions: first stream=20, first-complete=21, quiescent=22, + // second stream=23. Reader-first creates 23 before the writer's lookup; + // writer-first observes 20 before creating 23. Neither uses poll order. + for program in &case.processes { + let mut execution = template.executions[0].clone(); + execution.process = program.id.clone(); + execution.request_id = case.execution_request_id(program); + execution.logical_node_id = program.logical_node_id; + execution.results.clear(); + for (step, action) in program.actions.iter().enumerate() { + let mut terminal = template.executions[0].results[0].clone(); + terminal.process = program.id.clone(); + terminal.step = step; + terminal.action = action.operation.class().as_str().to_owned(); + terminal.path = action.operation.path().to_owned(); + terminal.length = None; + terminal.digest = None; + let payload = match &action.operation { + ActionOp::StreamWrite { chunks, .. } => Some(chunks.concat()), + ActionOp::StreamRead { expected, .. } => Some(expected.clone()), + ActionOp::PublishBlob { bytes, .. } => Some(bytes.clone()), + ActionOp::AwaitEntry { path, .. } => { + terminal.kind = Some("blob".to_owned()); + terminal.revision = Some(if path.ends_with("-first-complete") { + 21 + } else { + 22 + }); + None + } + ActionOp::WaitForQuiescent { .. } => { + terminal.kind = Some("stream".to_owned()); + terminal.revision = Some(20); + terminal.active = Some(false); + None + } + _ => unreachable!(), + }; + if let Some(payload) = &payload { + terminal.length = Some(payload.len()); + terminal.digest = Some(crate::codegen::digest(payload)); + } + if matches!( + action.operation, + ActionOp::StreamWrite { .. } | ActionOp::StreamRead { .. } + ) { + let payload = payload.as_ref().unwrap(); + let incarnation = if payload == b"first" { 20 } else { 23 }; + terminal.incarnation = Some(incarnation); + let mut emit = |barrier| { + let mut record = terminal.clone(); + record.outcome = "barrier".to_owned(); + record.length = None; + record.digest = None; + record.incarnation = None; + record.barrier = Some(barrier); + execution.results.push(record); + }; + emit(BarrierObservation::StreamOpened { incarnation }); + emit(BarrierObservation::StreamFrame { + incarnation, + index: 0, + length: payload.len() as u64, + digest: crate::codegen::digest(payload), + }); + if matches!(action.operation, ActionOp::StreamRead { .. }) { + emit(BarrierObservation::StreamFirstFrame { incarnation }); + emit(BarrierObservation::StreamEof { incarnation }); + } else if matches!( + action.operation, + ActionOp::StreamWrite { replace: true, .. } + ) { + emit(BarrierObservation::MutationApplied { + from_revision: Some(if reader_first { 23 } else { 20 }), + to_revision: Some(23), + }); + } + } + execution.results.push(terminal); + } + observation.executions.push(execution); + } + (case, observation) + } + + #[test] + fn quiescent_replacement_closes_for_reader_and_writer_created_incarnations() { + for reader_first in [true, false] { + let (case, mut observation) = observed_stream_replacement(reader_first); + for _ in 0..2 { + let mut ledger = ledger_for_case(&case); + ledger.observe_case(&case, &observation).unwrap(); + assert_eq!( + ledger.observed.get(&CoverageKey::Scenario { + scenario: CoverageScenario::QuiescentMutation, + }), + Some(&1) + ); + ledger.assert_observed_closed().unwrap(); + observation.executions.reverse(); + } + } + } + + #[test] + fn quiescent_replacement_rejects_unchanged_and_stale_identity() { + let (case, observation) = observed_stream_replacement(true); + for unchanged in [true, false] { + let mut forged = observation.clone(); + for execution in &mut forged.executions { + for result in &mut execution.results { + if unchanged { + if result.incarnation == Some(23) { + result.incarnation = Some(20); + } + match &mut result.barrier { + Some( + BarrierObservation::StreamOpened { incarnation } + | BarrierObservation::StreamFrame { incarnation, .. } + | BarrierObservation::StreamFirstFrame { incarnation } + | BarrierObservation::StreamEof { incarnation }, + ) if *incarnation == 23 => *incarnation = 20, + Some(BarrierObservation::MutationApplied { + from_revision, + to_revision, + }) => { + *from_revision = Some(20); + *to_revision = Some(20); + } + _ => {} + } + } else if result.kind.as_deref() == Some("stream") { + // A revision older than the stream actually drained + // cannot stand in for the observed quiescent binding. + result.revision = Some(19); + } + } + } + if unchanged { + let index = ObservationIndex::new(&case, &forged).unwrap(); + let mut keys = BTreeSet::new(); + assert!( + !observe_scenario( + &case, + CoverageScenario::QuiescentMutation, + &index, + &mut keys, + ) + .unwrap() + ); + assert!(keys.is_empty()); + } + let mut ledger = ledger_for_case(&case); + assert!(ledger.observe_case(&case, &forged).is_err()); + assert!(ledger.observed.is_empty()); + assert!(ledger.completed_cases.is_empty()); + } + } + + #[test] + fn quiescent_replacement_does_not_credit_the_new_incarnations_eventual_eof() { + let (mut case, mut observation) = observed_stream_replacement(true); + let reader = case + .processes + .iter_mut() + .find(|process| process.id == "replacement-reader") + .unwrap(); + let probe = reader.actions.remove(2); + reader.actions.push(probe); + let execution = observation + .executions + .iter_mut() + .find(|execution| execution.process == reader.id) + .unwrap(); + let position = execution + .results + .iter() + .position(|result| result.step == 2) + .unwrap(); + let mut probe = execution.results.remove(position); + for result in &mut execution.results { + if result.step > 2 { + result.step -= 1; + } + } + probe.step = 4; + probe.revision = Some(23); + execution.results.push(probe); + // The revised program is legal, but only observes quiescence after B. + // Successful replacement I/O plus B/B is not old-to-new evidence. + crate::oracle::BehaviorOracle::verify(&case, &observation).unwrap(); + let mut ledger = ledger_for_case(&case); + assert!(ledger.observe_case(&case, &observation).is_err()); + assert!(ledger.observed.is_empty()); + assert!(ledger.completed_cases.is_empty()); + } + + fn observed_roundtrip() -> (BehaviorCase, CaseObservation) { + let (mut case, mut observation) = observed_blob_chain(); + case.topology = TopologyFamily::Fixed; + case.routes.clear(); + case.processes.truncate(1); + observation.executions.truncate(1); + let path = case.processes[0].actions[0].operation.path().to_owned(); + let chunks = vec![Vec::new(), b"a".to_vec(), Vec::new()]; + case.processes[0].actions[0] = crate::ir::Action::ok(ActionOp::StreamRoundTrip { + path, + chunks: chunks.clone(), + }); + let terminal = &mut observation.executions[0].results[0]; + terminal.action = ActionClass::StreamWrite.as_str().to_owned(); + terminal.length = Some(1); + terminal.digest = Some(crate::codegen::digest(b"a")); + terminal.incarnation = Some(20); + let terminal = terminal.clone(); + let mut records = Vec::new(); + let mut emit = |barrier, action: &str| { + let mut record = terminal.clone(); + record.outcome = "barrier".to_owned(); + record.length = None; + record.digest = None; + record.action = action.to_owned(); + record.barrier = Some(barrier); + records.push(record); + }; + emit( + BarrierObservation::StreamOpened { incarnation: 20 }, + "stream_write", + ); + for direction in ["stream_write", "stream_read"] { + for (index, chunk) in chunks.iter().enumerate() { + emit( + BarrierObservation::StreamFrame { + incarnation: 20, + index: index as u64, + length: chunk.len() as u64, + digest: crate::codegen::digest(chunk), + }, + direction, + ); + if direction == "stream_read" && index == 0 { + emit( + BarrierObservation::StreamFirstFrame { incarnation: 20 }, + "stream_write", + ); + } + } + } + emit( + BarrierObservation::StreamEof { incarnation: 20 }, + "stream_write", + ); + records.push(terminal); + observation.executions[0].results = records; + (case, observation) + } + + #[test] + fn stream_milestones_do_not_manufacture_errors_or_action_adjacency() { + let (case, observation) = observed_roundtrip(); + let mut ledger = ledger_for_case(&case); + ledger.observe_case(&case, &observation).unwrap(); + ledger.assert_observed_closed().unwrap(); + assert!(!ledger.observed.contains_key(&CoverageKey::Outcome { + class: "modeled_error".to_owned(), + })); + assert!( + !ledger.observed.contains_key(&CoverageKey::ActionAdjacency { + before: ActionClass::StreamWrite, + after: ActionClass::StreamWrite, + }) + ); + + for corruption in 0..3 { + let mut forged = observation.clone(); + let records = &mut forged.executions[0].results; + match corruption { + 0 => records.insert(0, records[0].clone()), + 1 => records.swap(0, 1), + _ => records.swap(2, 3), + } + let mut ledger = ledger_for_case(&case); + assert!(ledger.observe_case(&case, &forged).is_err()); + assert!(ledger.completed_cases.is_empty()); + assert!(ledger.observed.is_empty()); + } + } + + #[test] + fn framing_corruption_never_commits_aggregate_only_coverage() { + let (case, observation) = observed_roundtrip(); + for direction in ["stream_write", "stream_read"] { + for corruption in 0..3 { + let mut forged = observation.clone(); + let records = &mut forged.executions[0].results; + let last = records + .iter() + .rposition(|record| { + record.action == direction + && matches!( + record.barrier, + Some(BarrierObservation::StreamFrame { .. }) + ) + }) + .unwrap(); + match corruption { + 0 => { + records.remove(last); + } + 1 => { + let mut duplicate = records[last].clone(); + if let Some(BarrierObservation::StreamFrame { index, .. }) = + &mut duplicate.barrier + { + *index += 1; + } + records.insert(last + 1, duplicate); + } + _ => { + if let Some(BarrierObservation::StreamFrame { digest, .. }) = + &mut records[last].barrier + { + *digest = crate::codegen::digest(b"corrupt empty frame"); + } + } + } + let mut rejected = ledger_for_case(&case); + assert!(rejected.observe_case(&case, &forged).is_err()); + assert!(rejected.completed_cases.is_empty()); + assert!(rejected.observed.is_empty()); + } + } + } + + #[test] + fn dag_topology_counts_roles_not_revisited_physical_nodes() { + let mut route = DataRoute { + id: "roles".to_owned(), + kind: DataKind::Stream, + join_inputs: Vec::new(), + edges: vec![ + crate::ir::DataEdge { + source: 1, + destination: 2, + source_role: "a".to_owned(), + destination_role: "b".to_owned(), + path: "/cases/roles/a-b".to_owned(), + }, + crate::ir::DataEdge { + source: 2, + destination: 1, + source_role: "b".to_owned(), + destination_role: "c".to_owned(), + path: "/cases/roles/b-c".to_owned(), + }, + crate::ir::DataEdge { + source: 1, + destination: 3, + source_role: "c".to_owned(), + destination_role: "d".to_owned(), + path: "/cases/roles/c-d".to_owned(), + }, + ], + }; + assert!(route_has_topology(&route, TopologyFamily::RandomDag)); + route.edges[1].destination_role = "a".to_owned(); + assert!(!route_has_topology(&route, TopologyFamily::RandomDag)); + } + + #[test] + fn actual_consecutive_stream_actions_receive_adjacency_credit() { + let (mut case, mut observation) = observed_roundtrip(); + let mut second = case.processes[0].actions[0].clone(); + let second_path = format!("{}-second", second.operation.path()); + let ActionOp::StreamRoundTrip { path, .. } = &mut second.operation else { + unreachable!(); + }; + *path = second_path.clone(); + case.processes[0].actions.push(second); + let mut second_records = observation.executions[0].results.clone(); + for record in &mut second_records { + record.step = 1; + record.path = second_path.clone(); + record.incarnation = Some(21); + match &mut record.barrier { + Some( + BarrierObservation::StreamOpened { incarnation } + | BarrierObservation::StreamFirstFrame { incarnation } + | BarrierObservation::StreamFrame { incarnation, .. } + | BarrierObservation::StreamEof { incarnation }, + ) => *incarnation = 21, + _ => {} + } + } + observation.executions[0].results.extend(second_records); + let mut ledger = ledger_for_case(&case); + ledger.observe_case(&case, &observation).unwrap(); + assert_eq!( + ledger.observed.get(&CoverageKey::ActionAdjacency { + before: ActionClass::StreamWrite, + after: ActionClass::StreamWrite, + }), + Some(&1) + ); + assert!(!ledger.observed.contains_key(&CoverageKey::Outcome { + class: "modeled_error".to_owned(), + })); + // One remaining EOF cannot cover another completed reader. + for missing_step in 0..2 { + let mut incomplete = observation.clone(); + incomplete.executions[0].results.retain(|record| { + record.step != missing_step + || !matches!(record.barrier, Some(BarrierObservation::StreamEof { .. })) + }); + let mut rejected = ledger_for_case(&case); + assert!(rejected.observe_case(&case, &incomplete).is_err()); + assert!(rejected.completed_cases.is_empty()); + assert!(rejected.observed.is_empty()); + } + } + + #[test] + fn unrelated_terminal_reader_and_writer_do_not_receive_relay_credit() { + let (mut case, mut observation) = observed_blob_chain(); + let relay = CoverageKey::NodeRole { + node: 2, + kind: DataKind::Blob, + role: NodeRole::Relay, + }; + let mut valid = ledger_for_case(&case); + valid.observe_case(&case, &observation).unwrap(); + assert_eq!(valid.observed.get(&relay), Some(&1)); + + let root_payload = match &case.processes[0].actions[0].operation { + ActionOp::PublishBlob { bytes, .. } => bytes.clone(), + _ => unreachable!(), + }; + let mut writer = case.processes[1].clone(); + writer.id = "unrelated-writer".to_owned(); + writer.access = crate::ir::AccessSpec::unrestricted("unrelated-writer"); + writer.actions = vec![case.processes[1].actions.remove(1)]; + match &mut writer.actions[0].operation { + ActionOp::PublishBlob { bytes, .. } => *bytes = root_payload.clone(), + _ => unreachable!(), + } + match &mut case.processes[2].actions[0].operation { + ActionOp::ReadBlob { expected, .. } => *expected = root_payload.clone(), + _ => unreachable!(), + } + let sink = &case.processes[2]; + observation.executions[2].request_id = case.execution_request_id(sink); + observation.executions[2].results[0].length = Some(root_payload.len()); + observation.executions[2].results[0].digest = Some(crate::codegen::digest(&root_payload)); + let mut writer_execution = observation.executions[1].clone(); + writer_execution.process = writer.id.clone(); + writer_execution.request_id = case.execution_request_id(&writer); + let mut write = observation.executions[1].results.remove(1); + write.process = writer.id.clone(); + write.step = 0; + write.length = Some(root_payload.len()); + write.digest = Some(crate::codegen::digest(&root_payload)); + writer_execution.results = vec![write]; + case.topology = TopologyFamily::Fixed; + case.routes[0].edges[1].source_role = writer.id.clone(); + case.processes.push(writer); + observation.executions.push(writer_execution); + // Swapping global polling order does not invent process-local causality. + observation.executions.reverse(); + let mut split = ledger_for_case(&case); + split.observe_case(&case, &observation).unwrap(); + assert!(!split.observed.contains_key(&relay)); + split.assert_observed_closed().unwrap(); + } + + #[test] + fn persisted_coverage_requires_the_exact_live_plan_build_deployment_and_fixture() { + let (case, observation) = observed_blob_chain(); + let expected = ledger_for_case(&case); + let mut completed = expected.clone(); + completed.observe_case(&case, &observation).unwrap(); + let encoded = serde_json::to_vec(&completed).unwrap(); + let decode = || serde_json::from_slice::(&encoded).unwrap(); + let mut resumed = decode(); + assert!(resumed.assert_observed_closed().is_err()); + resumed.validate_resume(&expected).unwrap(); + resumed.assert_observed_closed().unwrap(); + + for changed in 0..4 { + let mut next = expected.clone(); + let identity = next.identity.as_mut().unwrap(); + match changed { + 0 => identity.deployment_generation = "redeployment-b".to_owned(), + 1 => identity.fixture_mapping_digest = crate::codegen::digest(b"recreated fixture"), + 2 => identity.artifacts_digest = crate::codegen::digest(b"different binaries"), + _ => identity.plan_digest = crate::codegen::digest(b"different immutable plan"), + } + let mut stale = decode(); + assert!(stale.validate_resume(&next).is_err()); + assert!(stale.assert_observed_closed().is_err()); + assert!(!next.completed_cases.contains(&case.id)); + next.observe_case(&case, &observation).unwrap(); + } + } + + #[test] + fn corrupt_completion_ids_counts_and_attempt_identity_cannot_resume() { + let (case, observation) = observed_blob_chain(); + let expected = ledger_for_case(&case); + let mut completed = expected.clone(); + completed.observe_case(&case, &observation).unwrap(); + let corruptions: &[fn(&mut CoverageLedger)] = &[ + |ledger| { + ledger.completed_cases.insert("unplanned".to_owned()); + }, + |ledger| { + *ledger.observed.values_mut().next().unwrap() += 1; + }, + |ledger| { + ledger.completed_evidence.clear(); + }, + |ledger| { + let evidence = ledger.completed_evidence.values_mut().next().unwrap(); + *evidence.execution_ids.values_mut().next().unwrap() = "another-attempt".to_owned(); + }, + |ledger| { + ledger + .completed_evidence + .values_mut() + .next() + .unwrap() + .identity_digest = crate::codegen::digest(b"another deployment"); + }, + |ledger| { + let evidence = ledger.completed_evidence.values_mut().next().unwrap(); + *evidence.request_ids.values_mut().next().unwrap() = String::new(); + }, + ]; + for corrupt in corruptions { + let mut forged: CoverageLedger = + serde_json::from_slice(&serde_json::to_vec(&completed).unwrap()).unwrap(); + corrupt(&mut forged); + assert!(forged.validate_resume(&expected).is_err()); + assert!(forged.assert_observed_closed().is_err()); + } + } + + #[test] + fn completion_proves_exact_template_to_attempt_transformation() { + let (case, mut observation) = observed_blob_chain(); + let executed = case.for_attempt(7, true); + for (execution, program) in observation.executions.iter_mut().zip(&executed.processes) { + execution.request_id = executed.execution_request_id(program); + for (result, action) in execution.results.iter_mut().zip(&program.actions) { + result.path = action.operation.path().to_owned(); + } + } + let expected = ledger_for_case(&case); + let mut ledger = expected.clone(); + let budget = crate::budget::Budget::new(std::time::Duration::from_secs(30)); + let mut changed = executed.clone(); + changed.processes[0] + .access + .execution_id + .push_str("-not-the-attempt"); + assert!( + ledger + .observe_attempt(&case, &changed, &observation, 7, true, &budget) + .is_err() + ); + ledger + .observe_attempt(&case, &executed, &observation, 7, true, &budget) + .unwrap(); + let mut restored: CoverageLedger = + serde_json::from_slice(&serde_json::to_vec(&ledger).unwrap()).unwrap(); + restored.validate_resume(&expected).unwrap(); + restored.assert_observed_closed().unwrap(); + } + + #[test] + fn authorization_requires_allowed_and_denied_results_in_the_same_restricted_process() { + let (mut case, mut observation) = observed_blob_chain(); + case.topology = TopologyFamily::Fixed; + case.routes.clear(); + case.processes.truncate(1); + observation.executions.truncate(1); + case.scenarios.insert(CoverageScenario::Authorization); + let allowed = case.processes[0].actions[0].operation.path().to_owned(); + case.processes[0].access.write_prefixes = vec![allowed]; + let denied_path = "/cases/observed-chain/denied".to_owned(); + case.processes[0].actions.push(crate::ir::Action::error( + ActionOp::PublishBlob { + path: denied_path.clone(), + bytes: b"denied".to_vec(), + }, + libc::EACCES, + )); + let mut denied = observation.executions[0].results[0].clone(); + denied.step = 1; + denied.path = denied_path; + denied.outcome = "expected_error".to_owned(); + denied.errno = Some(libc::EACCES); + denied.length = None; + denied.digest = None; + observation.executions[0].results.push(denied); + let mut complete = ledger_for_case(&case); + complete.observe_case(&case, &observation).unwrap(); + assert_eq!( + complete.observed.get(&CoverageKey::Scenario { + scenario: CoverageScenario::Authorization, + }), + Some(&1) + ); + + for missing in 0..2 { + let mut incomplete_case = case.clone(); + let mut incomplete_observation = observation.clone(); + incomplete_case.processes[0].actions.remove(missing); + incomplete_observation.executions[0].results.remove(missing); + incomplete_observation.executions[0].results[0].step = 0; + let mut incomplete = ledger_for_case(&incomplete_case); + assert!( + incomplete + .observe_case(&incomplete_case, &incomplete_observation) + .is_err() + ); + } + + let mut unrelated = case.processes[0].clone(); + unrelated.id = "unrelated-allowed".to_owned(); + unrelated.access = crate::ir::AccessSpec::unrestricted("unrelated-allowed"); + unrelated.actions = vec![case.processes[0].actions.remove(0)]; + let mut allowed_execution = observation.executions[0].clone(); + allowed_execution.process = unrelated.id.clone(); + allowed_execution.request_id = case.execution_request_id(&unrelated); + let mut allowed_result = observation.executions[0].results.remove(0); + allowed_result.process = unrelated.id.clone(); + allowed_execution.results = vec![allowed_result]; + observation.executions[0].results[0].step = 0; + case.processes.push(unrelated); + observation.executions.push(allowed_execution); + let mut split = ledger_for_case(&case); + assert!(split.observe_case(&case, &observation).is_err()); + assert!(split.observed.is_empty()); + } + + #[test] + fn reader_drop_credits_release_without_an_invented_terminal_call() { + let (mut case, mut observation) = observed_blob_chain(); + case.topology = TopologyFamily::Fixed; + case.routes.clear(); + case.processes.truncate(2); + observation.executions.truncate(2); + case.processes[1].actions.truncate(1); + observation.executions[1].results.truncate(1); + let ActionOp::ReadBlob { path, expected } = case.processes[1].actions[0].operation.clone() + else { + unreachable!(); + }; + case.processes[1].actions[0].operation = ActionOp::DescriptorRead { + path, + expected, + flags: libc::O_RDONLY, + offset: 0, + method: DescriptorReadMethod::Read, + finish: DescriptorFinish::Drop, + }; + let result = &mut observation.executions[1].results[0]; + result.action = ActionClass::Descriptor.as_str().to_owned(); + result.descriptor = Some(DescriptorObservation::Read { + method: DescriptorReadMethod::Read, + finish: DescriptorFinish::Drop, + terminal_results: Vec::new(), + }); + let mut ledger = ledger_for_case(&case); + ledger.observe_case(&case, &observation).unwrap(); + assert_eq!( + ledger.observed.get(&CoverageKey::DescriptorTerminal { + direction: "read".to_owned(), + finish: "drop".to_owned(), + }), + Some(&1) + ); + + let Some(DescriptorObservation::Read { + terminal_results, .. + }) = &mut observation.executions[1].results[0].descriptor + else { + unreachable!(); + }; + terminal_results.push(DescriptorTerminalResult::Ok); + let mut forged = ledger_for_case(&case); + assert!(forged.observe_case(&case, &observation).is_err()); + assert!(forged.observed.is_empty()); + } +} diff --git a/tools/myelin-e2e-fuzz/src/harness/control.rs b/tools/myelin-e2e-fuzz/src/harness/control.rs index e65dd90..30e564d 100644 --- a/tools/myelin-e2e-fuzz/src/harness/control.rs +++ b/tools/myelin-e2e-fuzz/src/harness/control.rs @@ -1,217 +1,984 @@ //! Contextual process control: case execution, spawn/stop, failure injection. -use std::collections::{BTreeMap, BTreeSet, VecDeque}; +use std::collections::{BTreeMap, BTreeSet}; use std::fs; -use std::io::Write as IoWrite; +use std::io::{BufWriter, Write as IoWrite}; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, mpsc}; use std::thread; -use std::time::Instant; +use std::time::{Duration, Instant}; use base64::Engine as _; -use serde_json::{Value, json}; - -use crate::codegen::{case_cleanup_paths, render_case_python, render_cleanup_python}; -use crate::harness::ClusterHarness; -use crate::ir::{ - AccessSpec, ActionObservation, BehaviorCase, CaseObservation, ExecutionObservation, - FailureInjection, LaunchFailureKind, ProcessProgram, ProcessStopPhase, +use myelin_control_contract::{ + ContextualControlReply, ContextualEventCursor, ContextualEventsAckRequest, + ContextualEventsRequest, ContextualExecutionView, ContextualExitStatus, + ContextualProcessEventKind, ContextualProcessSpec, ContextualSpawnRequest, + ContextualStopRequest, Versioned, }; -use crate::oracle::BehaviorOracle; -use crate::resources::{http_json, list_containers, list_workload_processes}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; -use super::{GENERATED_LAUNCHER, POLL_INTERVAL}; +use crate::budget::Budget; +use crate::campaign::{PersistedFixtureEntry, RecoveryCase}; +use crate::codegen::render_case_python; +use crate::harness::{ClusterHarness, FixturePathObservation, FixturePathSnapshot}; +use crate::ir::{ + AccessSpec, Action, ActionObservation, ActionOp, BehaviorCase, CaseObservation, + ExecutionObservation, FailureInjection, LaunchFailureKind, ProcessProgram, ProcessStopPhase, + TopologyFamily, +}; +use crate::oracle::{ + BehaviorOracle, FailureClass, FailureSignature, recorded_action_failure, stream_stop_ready, +}; +use crate::resources::http_json_budget; + +use super::{GENERATED_LAUNCHER, HarnessProvider, POLL_INTERVAL}; const GENERATED_SOURCE_ENV: &str = "MYELIN_E2E_PROGRAM_B64"; const BOOTSTRAP_HOLD_ENV: &str = "MYELIN_E2E_HOLD_BEFORE_BOOTSTRAP"; const MAX_CONTEXTUAL_SOURCE_ENV_BYTES: usize = 48 * 1024; +const EVENT_REQUEST_SLICE: Duration = Duration::from_secs(1); +const MIN_RECONCILE_INTERVAL: Duration = Duration::from_millis(5); -impl ClusterHarness { - pub fn run_case(&mut self, case: &BehaviorCase) -> Result { +/// The one immutable namespace and execution ownership map for an attempt. +/// Preparing it does not grant permission to remove any of these paths. +pub(super) struct CaseAttempt { + id: u64, + directory: PathBuf, + serialization_ns: u128, + pub(super) case: BehaviorCase, + pub(super) cleanup_paths: Vec, + pub(super) observation: std::sync::OnceLock, +} + +impl CaseAttempt { + fn prepare(id: u64, artifacts: &Path, case: BehaviorCase) -> Result { case.validate()?; - if case.node_count != self.config.node_count { - return Err(format!( - "case topology {} differs from harness topology {}", - case.node_count, self.config.node_count - )); - } - let case_dir = self.config.artifacts.join(&case.id); - fs::create_dir_all(&case_dir) + let cleanup_paths = case.owned_paths().into_iter().collect(); + let directory = artifacts.join(&case.id).join(format!("attempt-{id}")); + fs::create_dir_all(directory.parent().expect("attempt has case parent")) .map_err(|error| format!("create case artifact directory: {error}"))?; - fs::write( - case_dir.join("case.json"), - serde_json::to_vec_pretty(case) - .map_err(|error| format!("serialize case IR: {error}"))?, - ) - .map_err(|error| format!("write case IR: {error}"))?; - let sources = case - .processes - .iter() - .map(|program| (program.id.clone(), render_case_python(case, program))) - .collect::>(); - for (process, source) in &sources { - fs::write(case_dir.join(format!("{process}.py")), source) - .map_err(|error| format!("write generated program {process}: {error}"))?; - } - - let mut completed = BTreeSet::new(); - let mut pending = case - .processes - .iter() - .map(|process| process.id.clone()) - .collect::>(); - let mut observations = Vec::new(); - let mut failure_injected = false; - let mut running = VecDeque::new(); - while !pending.is_empty() || !running.is_empty() { - let ready = case - .processes - .iter() - .filter(|process| pending.contains(&process.id)) - .filter(|process| { - process - .depends_on - .iter() - .all(|dependency| completed.contains(dependency)) - }) - .collect::>(); - if ready.is_empty() && running.is_empty() { - return Err("case dependency graph made no progress".to_owned()); - } - let prioritize_new = !running.is_empty(); - let mut launched = Vec::new(); - for process in ready { - let request_id = format!("{}-{}-{}", case.id, process.id, case.seed); - let stop_injection = match &case.failure { - FailureInjection::StopProcess { - process: target, - phase, - kill_after_ms, - } if !failure_injected && target == &process.id => { - Some((*phase, *kill_after_ms)) - } - _ => None, - }; - let launch_failure = match &case.failure { - FailureInjection::LaunchFailure { - process: target, - kind, - } if target == &process.id => Some(*kind), - _ => None, - }; - let hold_before_bootstrap = stop_injection - .is_some_and(|(phase, _)| phase == ProcessStopPhase::DuringBootstrap); - self.spawn_program( - process, - &request_id, - &sources[&process.id], - hold_before_bootstrap, - launch_failure, - )?; - let deferred_stop = if let Some((phase, kill_after_ms)) = stop_injection { - failure_injected = true; - match phase { - ProcessStopPhase::AfterSpawn => { - self.stop_process(&request_id, kill_after_ms)?; - None - } - ProcessStopPhase::DuringBootstrap | ProcessStopPhase::AfterContextReady => { - Some((phase, kill_after_ms)) - } - } - } else { - None - }; - pending.remove(&process.id); - launched.push((process, request_id, deferred_stop)); - } - if prioritize_new { - for execution in launched.into_iter().rev() { - running.push_front(execution); - } - } else { - running.extend(launched); - } - if !failure_injected - && let FailureInjection::KillNode { logical_node_id } = case.failure - { - self.kill_node(logical_node_id)?; - failure_injected = true; - } - if !failure_injected && matches!(case.failure, FailureInjection::StopOrchestrator) { - self.stop_orchestrator_injection()?; - pending.clear(); - running.clear(); - break; - } - let Some((process, request_id, deferred_stop)) = running.pop_front() else { - continue; - }; - let observation = self.wait_execution(process, &request_id, deferred_stop)?; - if !observation.exit_success && matches!(case.failure, FailureInjection::None) { - return Err(format!( - "process {} failed with status {:?}\nstdout={}\nstderr={}", - process.id, observation.exit_status, observation.stdout, observation.stderr - )); - } - completed.insert(process.id.clone()); - observations.push(observation); - } - let observation = CaseObservation { - case_id: case.id.clone(), - executions: observations, - }; - fs::write( - case_dir.join("observed.json"), - serde_json::to_vec_pretty(&observation) - .map_err(|error| format!("serialize case observation: {error}"))?, - ) - .map_err(|error| format!("write case observation: {error}"))?; - if !matches!(case.failure, FailureInjection::StopOrchestrator) { - BehaviorOracle::verify(case, &observation).map_err(|error| error.to_string())?; - } - if matches!(case.failure, FailureInjection::None) { - self.cleanup_case(case)?; - } - Ok(observation) + fs::create_dir(&directory) + .map_err(|error| format!("exclusively claim attempt artifact directory: {error}"))?; + let started = Instant::now(); + write_json(&directory.join("case.json"), &case)?; + let serialization_ns = started.elapsed().as_nanos(); + Ok(Self { + id, + directory, + serialization_ns, + case, + cleanup_paths, + observation: std::sync::OnceLock::new(), + }) } - fn cleanup_case(&mut self, case: &BehaviorCase) -> Result<(), String> { - let paths = case_cleanup_paths(case); - if paths.is_empty() { - return Ok(()); + pub(super) fn persist_retained_blobs( + &self, + reply: &myelin_control_contract::RetainedBlobsReply, + ) -> Result<(), String> { + write_json(&self.directory.join("retained-blobs.json"), reply) + } +} + +pub(super) struct PendingAttempt { + attempt: Arc, + preconditions_verified: bool, +} + +impl ClusterHarness { + pub fn last_failure_signature(&self) -> Option<&FailureSignature> { + self.last_failure_signature.as_ref() + } + + pub fn last_attempt_id(&self) -> Option { + self.last_attempt_id + } + + fn allocate_attempt_id(&mut self) -> Result { + let budget = self.operation_budget().clone(); + reserve_attempt_identity( + &self.state_dir, + &self.config.artifacts, + &mut self.next_attempt_id, + &budget, + ) + } + + /// Allocate both recovery phases before any persistence/absence probe. + /// They share a scoped namespace but have distinct execution identities. + pub fn prepare_recovery_attempt( + &mut self, + recovery: &RecoveryCase, + ) -> Result { + self.cleanup_budget = None; + if let Some(reason) = self.quarantine_reason() { + return Err(format!("fixture is quarantined: {reason}")); } - let process = ProcessProgram { - id: format!("cleanup-{}", case.id), - logical_node_id: 1, - access: AccessSpec::unrestricted(format!("cleanup-{}", case.id)), - depends_on: Vec::new(), - actions: Vec::new(), - }; - let request_id = format!("{}-cleanup-{}", case.id, case.seed); - self.spawn_program( - &process, - &request_id, - &render_cleanup_python(&paths), - false, - None, - )?; - let observation = self.wait_execution(&process, &request_id, None)?; - if !observation.exit_success { - return Err(format!( - "case {} cleanup failed with status {:?}\nstdout={}\nstderr={}", - case.id, observation.exit_status, observation.stdout, observation.stderr + if recovery.pre_restart.id == recovery.post_restart.id { + return Err("recovery phases must have distinct case identities".to_owned()); + } + for case in [&recovery.pre_restart, &recovery.post_restart] { + if self.active_attempts.contains_key(&case.id) + || self.pending_attempts.contains_key(&case.id) + { + return Err(format!( + "case {} already has an owned or pending attempt", + case.id + )); + } + } + let namespace_id = self.allocate_attempt_id()?; + let mut scoped = recovery.for_attempt(namespace_id); + let pre = Arc::new(CaseAttempt::prepare( + namespace_id, + &self.config.artifacts, + scoped.pre_restart.clone(), + )?); + let post_id = self.allocate_attempt_id()?; + scoped.post_restart = scoped.post_restart.for_attempt(post_id, false); + let post = Arc::new(CaseAttempt::prepare( + post_id, + &self.config.artifacts, + scoped.post_restart.clone(), + )?); + self.pending_attempts.insert( + scoped.pre_restart.id.clone(), + PendingAttempt { + attempt: pre, + preconditions_verified: false, + }, + ); + self.pending_attempts.insert( + scoped.post_restart.id.clone(), + PendingAttempt { + attempt: post, + preconditions_verified: false, + }, + ); + Ok(scoped) + } + pub(crate) fn persistence_probe_case<'a>( + template: &BehaviorCase, + entries: &[PersistedFixtureEntry], + node_ids: &[u64], + absent_paths: impl IntoIterator, + probe_id: u64, + ) -> Result { + if entries.is_empty() { + return Err("recovery case did not declare persisted fixture entries".to_owned()); + } + let logical_node_id = node_ids + .first() + .copied() + .ok_or_else(|| "fixture has no live node for persistence probe".to_owned())?; + let mut probe = probe_case(template, format!("{}-persistence", template.id), node_ids); + probe.read_only_fixture_paths = entries + .iter() + .map(|entry| entry.path().to_owned()) + .collect(); + let mut actions = Vec::new(); + for entry in entries { + match entry { + PersistedFixtureEntry::Blob { path, expected } => { + actions.push(Action::ok(ActionOp::Lookup { + path: path.clone(), + expected_kind: "blob".to_owned(), + })); + actions.push(Action::ok(ActionOp::ReadBlob { + path: path.clone(), + expected: expected.clone(), + })); + } + PersistedFixtureEntry::QuiescentStream { path } => { + actions.push(Action::ok(ActionOp::Lookup { + path: path.clone(), + expected_kind: "stream".to_owned(), + })); + actions.push(Action::ok(ActionOp::WaitForQuiescent { + path: path.clone(), + })); + } + } + } + for path in absent_paths { + if !probe.read_only_fixture_paths.insert(path.clone()) { + return Err(format!( + "persisted fixture path {path} is also claimed by the next attempt" + )); + } + actions.push(Action::error( + ActionOp::Lookup { + path: path.clone(), + expected_kind: "absent".to_owned(), + }, + libc::ENOENT, )); } + probe.resource_bounds.max_actions = probe + .resource_bounds + .max_actions + .max(u32::try_from(actions.len()).unwrap_or(u32::MAX)); + probe.processes = vec![ProcessProgram { + id: "persisted-path-probe".to_owned(), + logical_node_id, + access: read_only_access( + format!("{}-persistence-attempt-{probe_id}", template.id), + &probe.read_only_fixture_paths, + ), + depends_on: Vec::new(), + actions, + }]; + probe.validate()?; + Ok(probe) + } + + pub fn snapshot_fixture_paths( + &mut self, + template: &BehaviorCase, + entries: &[PersistedFixtureEntry], + absent_case: Option<&BehaviorCase>, + ) -> Result { + let remaining = self + .execution_budget + .remaining("reserve recovery snapshot and namespace cleanup")?; + let total = remaining.min(self.config.deadline.saturating_mul(2)); + let reserve = self.config.deadline.min(total / 2); + let probe_budget = self.execution_budget.child(total.saturating_sub(reserve)); + self.cleanup_budget = Some(self.execution_budget.child(total)); + let absent_attempt = absent_case + .map(|case| { + let pending = self.pending_attempts.get(&case.id).ok_or_else(|| { + format!( + "case {} has no prepared attempt for fused preconditions", + case.id + ) + })?; + if pending.attempt.case != *case { + return Err(format!( + "case {} changed after attempt preparation", + case.id + )); + } + Ok(Arc::clone(&pending.attempt)) + }) + .transpose()?; + let probe_id = self.allocate_attempt_id()?; + let probe = Self::persistence_probe_case( + template, + entries, + &self.node_ids, + absent_attempt + .iter() + .flat_map(|attempt| &attempt.cleanup_paths), + probe_id, + )?; + let observation = self.run_read_only_probe(&probe, &probe_budget)?; + let snapshot = persisted_fixture_snapshot(entries, &observation)?; + probe_budget.check("complete recovery namespace snapshot")?; + if let Some(attempt) = absent_attempt { + self.pending_attempts + .get_mut(&attempt.case.id) + .expect("pending attempt remains installed during read-only probe") + .preconditions_verified = true; + } + Ok(snapshot) + } + /// Recover persisted facts from the verified actions of the retained + /// segment itself. This avoids launching a redundant observer process. + pub fn retained_fixture_snapshot( + &self, + case: &BehaviorCase, + entries: &[PersistedFixtureEntry], + ) -> Result { + let attempt = self + .active_attempts + .get(&case.id) + .filter(|attempt| attempt.case == *case) + .ok_or_else(|| format!("case {} has no exact retained attempt", case.id))?; + let observation = attempt + .observation + .get() + .ok_or_else(|| format!("case {} has no verified retained observation", case.id))?; + persisted_fixture_snapshot(entries, observation) + } + + pub fn abort_case_processes(&self, case: &BehaviorCase) { + let Some(owned_case) = self.active_attempts.get(&case.id) else { + return; + }; + let requests = owned_case + .case + .processes + .iter() + .map(|process| (&owned_case.case).execution_request_id(process)) + .collect::>(); + let errors = self.stop_owned_processes(&requests, self.operation_budget()); + if !errors.is_empty() { + let _ = write_json( + &self + .config + .artifacts + .join(&case.id) + .join("abort-errors.json"), + &json!({"schema_version": 1, "errors": errors}), + ); + } + } + + pub fn run_case(&mut self, case: &BehaviorCase) -> Result { + self.execute_with_failure_cleanup(case, true, true, true) + .map(|(_, observation)| observation) + } + + /// Return the exact attempt-scoped IR alongside its verified observation. + pub fn run_case_with_identity( + &mut self, + case: &BehaviorCase, + ) -> Result<(BehaviorCase, CaseObservation), String> { + self.execute_with_failure_cleanup(case, true, true, true) + .map(|(attempt, observation)| { + let case = Arc::try_unwrap(attempt) + .map_or_else(|attempt| attempt.case.clone(), |attempt| attempt.case); + (case, observation) + }) + } + + /// Run an unchanged phase from `prepare_recovery_attempt`, retaining its + /// namespace ownership and sealing its verified observation. + pub fn run_case_retained(&mut self, case: &BehaviorCase) -> Result<(), String> { + self.run_case_retained_inner(case, true) + } + + /// Retain one phase of a batched recovery campaign. The caller must seal + /// the aggregate retained baseline before restarting or running more work. + pub fn run_case_retained_deferred(&mut self, case: &BehaviorCase) -> Result<(), String> { + self.run_case_retained_inner(case, false) + } + + fn run_case_retained_inner( + &mut self, + case: &BehaviorCase, + seal_baseline: bool, + ) -> Result<(), String> { + let (attempt, observation) = + self.execute_with_failure_cleanup(case, false, false, seal_baseline)?; + let result = attempt + .observation + .set(observation) + .map_err(|_| "retained attempt observation was already sealed".to_owned()) + .and_then(|()| { + if seal_baseline { + self.seal_retained_attempts() + } else { + Ok(()) + } + }); + if let Err(error) = result { + let cleanup = self.cleanup_case(case); + return Err(match cleanup { + Ok(()) => error, + Err(cleanup) => format!("{error}; retained cleanup failed: {cleanup}"), + }); + } Ok(()) } - pub(super) fn spawn_program( + pub fn seal_retained_attempts(&self) -> Result<(), String> { + let baseline = self.restore_owned_baseline(None)?; + for attempt in self.active_attempts.values() { + if attempt.observation.get().is_some() { + write_json(&attempt.directory.join("retained-baseline.json"), &baseline)?; + } + } + Ok(()) + } + + /// Run an already prepared recovery phase, preserving its scoped paths. + /// Arbitrary templates are rejected; this is not a preflight bypass. + pub fn run_case_unscoped(&mut self, case: &BehaviorCase) -> Result { + self.execute_with_failure_cleanup(case, true, false, true) + .map(|(_, observation)| observation) + } + + fn execute_with_failure_cleanup( + &mut self, + case: &BehaviorCase, + cleanup: bool, + isolate_paths: bool, + restore_baseline: bool, + ) -> Result<(Arc, CaseObservation), String> { + self.last_failure_signature = None; + self.last_attempt_id = None; + self.cleanup_budget = None; + if let Some(reason) = self.quarantine_reason() { + return Err(format!("fixture is quarantined: {reason}")); + } + if self.active_attempts.contains_key(&case.id) { + return Err(format!("case {} still owns an uncleaned attempt", case.id)); + } + let prepared = if isolate_paths { + if self.pending_attempts.contains_key(&case.id) { + return Err(format!( + "case {} already has a prepared recovery attempt", + case.id + )); + } + let id = self.allocate_attempt_id()?; + PendingAttempt { + attempt: Arc::new(CaseAttempt::prepare( + id, + &self.config.artifacts, + case.for_attempt(id, true), + )?), + preconditions_verified: false, + } + } else { + let pending = self.pending_attempts.get(&case.id).ok_or_else(|| { + format!( + "case {} must be prepared as a recovery attempt before unscoped execution", + case.id + ) + })?; + if pending.attempt.case != *case { + return Err(format!( + "case {} changed after attempt preparation", + case.id + )); + } + self.pending_attempts + .remove(&case.id) + .expect("pending attempt checked") + }; + let attempt = prepared.attempt; + self.last_attempt_id = Some(attempt.id); + // Allocate once, before dispatch. Cancellation belongs to the narrower + // execution child, never to the parent that owns the cleanup reserve. + let parent = self.execution_budget.clone(); + let remaining = parent.remaining("reserve attempt execution and cleanup")?; + let total = remaining.min(self.config.deadline.saturating_mul(2)); + let reserve = self.config.deadline.min(total / 2); + let cleanup_budget = parent.child(total); + self.execution_budget = parent.child(total.saturating_sub(reserve)); + let started = Instant::now(); + let baseline = if restore_baseline { + self.take_or_restore_local_baseline().and_then(|baseline| { + write_json(&attempt.directory.join("resource-baseline.json"), &baseline) + }) + } else { + Ok(()) + }; + let result = baseline.and_then(|()| { + self.execute_case(&attempt, prepared.preconditions_verified, &cleanup_budget) + }); + if let Err(error) = &result { + if error.contains("fixture quarantine:") { + self.quarantine(error.clone()); + } + } + self.execution_budget.cancel(); + self.execution_budget = parent; + self.cleanup_budget = Some(cleanup_budget); + let cleanup_started = Instant::now(); + let cleanup_result = if cleanup || result.is_err() { + self.cleanup_case(case) + } else { + Ok(()) + }; + if cleanup_result.is_err() { + self.last_failure_signature = None; + } + let evidence = write_json( + &attempt.directory.join("attempt-timing.json"), + &json!({ + "schema_version": 1, + "attempt_id": attempt.id, + "elapsed_ns": started.elapsed().as_nanos(), + "cleanup_ns": cleanup_started.elapsed().as_nanos(), + "execution_allocation_ns": total.saturating_sub(reserve).as_nanos(), + "cleanup_reserved_ns": reserve.as_nanos(), + "execution_error": result.as_ref().err(), + "cleanup_error": cleanup_result.as_ref().err(), + }), + ); + match (result, cleanup_result, evidence) { + (Ok(observation), Ok(()), Ok(())) => Ok((attempt, observation)), + (result, cleanup, evidence) => { + let mut errors = Vec::new(); + if let Err(error) = result { + errors.push(error); + } + if let Err(error) = cleanup { + errors.push(format!( + "fixture quarantine: terminal cleanup failed: {error}" + )); + } + if let Err(error) = evidence { + self.last_failure_signature = None; + errors.push(error); + } + Err(errors.join("; ")) + } + } + } + + fn execute_case( + &mut self, + attempt: &Arc, + preconditions_verified: bool, + cleanup_budget: &Budget, + ) -> Result { + let started = Instant::now(); + let case = &attempt.case; + let case_dir = &attempt.directory; + let mut timings = BTreeMap::new(); + timings.insert("serialization_ns", attempt.serialization_ns); + let mut trackers = case + .processes + .iter() + .map(|process| { + ExecutionTracker::new( + process, + (case).execution_request_id(process), + stop_injection(case, process), + ) + }) + .collect::>(); + let mut event_log = BufWriter::new( + fs::File::create(case_dir.join("execution-events.jsonl")) + .map_err(|error| format!("create execution event log: {error}"))?, + ); + let execution = (|| { + self.operation_budget().check("validate attempt")?; + case.validate()?; + let fixture_nodes = self.node_ids.iter().copied().collect::>(); + if case.live_nodes != fixture_nodes { + return Err(format!( + "case live set {:?} differs from fixture live set {fixture_nodes:?}", + case.live_nodes + )); + } + let stage = Instant::now(); + if !preconditions_verified { + self.probe_absent_paths(attempt).map_err(|error| { + format!( + "fixture quarantine: attempt {} absence precondition failed: {error}", + attempt.id + ) + })?; + } + // Only a successful absence proof grants namespace cleanup + // ownership. In particular an unexpected existing path remains + // untouched even when the proof or its transport fails. + self.active_attempts + .insert(case.id.clone(), Arc::clone(attempt)); + timings.insert("precondition_ns", stage.elapsed().as_nanos()); + let stage = Instant::now(); + let mut sources = Vec::with_capacity(case.processes.len()); + for process in &case.processes { + let remaining = self + .operation_budget() + .remaining("prepare generated source")?; + let source = render_case_python(&case, process, remaining); + fs::write(case_dir.join(format!("{}.py", process.id)), &source) + .map_err(|error| format!("write generated program {}: {error}", process.id))?; + sources.push(source); + } + timings.insert("source_preparation_ns", stage.elapsed().as_nanos()); + let stage = Instant::now(); + let errors = self.schedule_case( + &case, + &sources, + &mut trackers, + &mut event_log, + started, + self.operation_budget(), + ); + timings.insert("spawn_and_collection_ns", stage.elapsed().as_nanos()); + if !errors.is_empty() { + return Err(errors.join("\n--- sibling failure ---\n")); + } + Ok(()) + })(); + // Every scoped transport worker has a cancellable owner and has joined + // before cleanup can touch namespace state. Drain original cursors under + // the reserved owner, retaining both failure and unwind evidence. + let mut errors = execution.err().into_iter().collect::>(); + if !errors.is_empty() { + let stage = Instant::now(); + let requests = trackers + .iter() + .filter(|state| state.dispatched && !state.terminal) + .map(|state| state.request_id.clone()) + .collect::>(); + errors.extend(self.stop_owned_processes(&requests, cleanup_budget)); + errors.extend(self.drain_owned_executions( + &mut trackers, + cleanup_budget, + &mut event_log, + started, + )); + timings.insert("failure_stop_and_drain_ns", stage.elapsed().as_nanos()); + if trackers + .iter() + .any(|state| state.dispatched && !state.terminal) + { + let reason = "fixture quarantine: owned executions lack terminal retraction proof" + .to_owned(); + self.quarantine(reason.clone()); + errors.push(reason); + } + } + let observation = CaseObservation { + case_id: case.id.clone(), + executions: trackers + .iter() + .filter(|state| state.dispatched) + .map(ExecutionTracker::partial_observation) + .collect(), + }; + if errors.is_empty() { + let stage = Instant::now(); + if let Err(error) = + BehaviorOracle::verify_with_budget(&case, &observation, self.operation_budget()) + { + errors.push(error.to_string()); + if !matches!( + error.signature.failure_class, + FailureClass::BudgetExceeded + | FailureClass::MissingEvidence + | FailureClass::InvalidEvidence + ) { + self.last_failure_signature = Some(error.signature); + } + } + if let Err(error) = self.operation_budget().check("complete behavioral oracle") { + self.last_failure_signature = None; + errors.push(error); + } + timings.insert("oracle_ns", stage.elapsed().as_nanos()); + } + if let Err(error) = event_log.flush() { + errors.push(format!("flush original execution records: {error}")); + self.last_failure_signature = None; + } + if let Err(error) = write_json(&case_dir.join("observed.json"), &observation) { + errors.push(error); + self.last_failure_signature = None; + } + if let Err(error) = write_json( + &case_dir.join("pending-executions.json"), + &json!({ + "schema_version": 1, + "errors": errors, + "executions": trackers.iter().map(ExecutionTracker::pending_evidence).collect::>(), + }), + ) { + errors.push(error); + self.last_failure_signature = None; + } + timings.insert("total_execution_ns", started.elapsed().as_nanos()); + if let Err(error) = write_json( + &case_dir.join("stage-timings.json"), + &json!({ + "schema_version": 2, "stages_ns": timings, + "executions": trackers.iter().map(ExecutionTracker::timing_evidence).collect::>(), + }), + ) { + errors.push(error); + self.last_failure_signature = None; + } + if !errors.is_empty() { + let fleet = http_json_budget( + "GET", + &format!("{}/api/control/fleet", self.base_url), + None, + &cleanup_budget.child(EVENT_REQUEST_SLICE), + ); + let telemetry = self + .telemetry_census + .lock() + .map(|census| census.snapshot()) + .map_err(|_| "telemetry census is poisoned"); + let diagnostics = self.orchestrator_diagnostics(); + if let Err(error) = write_json( + &case_dir.join("failure-state.json"), + &json!({ + "schema_version": 1, "attempt_id": attempt.id, + "model": "case.json", "processes": "pending-executions.json", + "fleet": fleet, "recent_telemetry": telemetry, + "orchestrator": diagnostics, + }), + ) { + errors.push(error); + self.last_failure_signature = None; + } + } + if errors.is_empty() { + Ok(observation) + } else { + Err(errors.join("\n--- sibling failure ---\n")) + } + } + + pub fn cleanup_case(&mut self, case: &BehaviorCase) -> Result<(), String> { + let result = self.cleanup_case_inner(case); + if let Err(error) = &result { + self.quarantine(format!("case {} cleanup failed: {error}", case.id)); + } + result + } + /// Atomically clean all attempt-scoped recovery namespaces, then prove one + /// exact fixture baseline after every path is absent. + pub fn cleanup_recovery_cases(&mut self, cases: &[&BehaviorCase]) -> Result<(), String> { + let result = self.cleanup_recovery_cases_inner(cases); + if let Err(error) = &result { + self.quarantine(format!("recovery campaign cleanup failed: {error}")); + } + result + } + + fn cleanup_recovery_cases_inner(&mut self, cases: &[&BehaviorCase]) -> Result<(), String> { + if cases.is_empty() { + return Err("recovery cleanup requires at least one owned case".to_owned()); + } + let mut attempts = Vec::with_capacity(cases.len()); + let mut owners = BTreeMap::::new(); + for case in cases { + self.pending_attempts.remove(&case.id); + let attempt = self + .active_attempts + .get(&case.id) + .filter(|attempt| attempt.case == **case) + .map(Arc::clone) + .ok_or_else(|| format!("recovery case {} has no exact active attempt", case.id))?; + for path in &attempt.cleanup_paths { + if let Some(prior) = owners.insert(path.clone(), case.id.clone()) { + return Err(format!( + "recovery cases {prior} and {} share cleanup path {path}", + case.id + )); + } + } + attempts.push(attempt); + } + let pending = self.pending_control_processes.lock().map_err(|_| { + "owned execution tracking is poisoned; refusing recovery cleanup".to_owned() + })?; + if !pending.is_empty() { + return Err(format!( + "owned executions lack terminal retraction proof; refusing recovery cleanup: {pending:?}" + )); + } + drop(pending); + + let paths = owners.keys().cloned().collect::>(); + let request_prefix = format!( + "recovery-campaign-cleanup-{}-{}", + self.config.seed, + self.next_observation_generation() + ); + let mut reply = myelin_control_contract::NamespaceCleanupReply { + schema_version: myelin_control_contract::SCHEMA_VERSION, + request_id: request_prefix.clone(), + paths: Vec::with_capacity(paths.len()), + }; + let requests = paths + .chunks(16) + .enumerate() + .map(|(chunk_index, chunk)| { + let request_id = format!("{request_prefix}-{chunk_index}"); + let request = myelin_control_contract::NamespaceCleanupRequest { + schema_version: myelin_control_contract::SCHEMA_VERSION, + request_id: request_id.clone(), + paths: chunk.to_vec(), + }; + serde_json::to_value(request) + .map(|value| (chunk_index, request_id, value)) + .map_err(|error| error.to_string()) + }) + .collect::, _>>()?; + let budget = self.operation_budget().clone(); + let cleanup_url = format!("{}/api/control/contextual/namespace-cleanup", self.base_url); + let chunk_results = requests + .iter() + .map(|(_, _, request)| { + http_json_budget("POST", &cleanup_url, Some(request.clone()), &budget).and_then( + |reply| { + serde_json::from_value(reply).map_err(|error| { + format!("decode batched namespace cleanup proof: {error}") + }) + }, + ) + }) + .collect::>(); + for ((chunk_index, request_id, _), chunk_result) in requests.iter().zip(chunk_results) { + let chunk_reply: myelin_control_contract::NamespaceCleanupReply = chunk_result?; + if chunk_reply.schema_version != myelin_control_contract::SCHEMA_VERSION + || chunk_reply.request_id != *request_id + { + return Err("batched namespace cleanup proof identity mismatch".to_owned()); + } + write_json( + &self + .config + .artifacts + .join(format!("recovery-namespace-cleanup-{chunk_index}.json")), + &chunk_reply, + )?; + reply.paths.extend(chunk_reply.paths); + } + let stdout = reply + .paths + .iter() + .map(serde_json::to_string) + .collect::, _>>() + .map_err(|error| format!("serialize batched namespace cleanup proof: {error}"))? + .join("\n"); + verify_cleanup_paths(&paths, &stdout)?; + self.mark_health_boundary()?; + let baseline = self.restore_health(&[])?; + + for attempt in &attempts { + let owned = attempt.cleanup_paths.iter().collect::>(); + let proof = myelin_control_contract::NamespaceCleanupReply { + schema_version: reply.schema_version, + request_id: reply.request_id.clone(), + paths: reply + .paths + .iter() + .filter(|record| owned.contains(&record.path)) + .cloned() + .collect(), + }; + write_json(&attempt.directory.join("namespace-cleanup.json"), &proof)?; + write_json(&attempt.directory.join("cleanup-baseline.json"), &baseline)?; + } + for case in cases { + self.active_attempts.remove(&case.id); + } + self.remember_local_baseline(&baseline); + Ok(()) + } + + fn take_or_restore_local_baseline(&mut self) -> Result { + let Some(baseline) = self.verified_local_baseline.take() else { + return self.restore_owned_baseline(None); + }; + if !matches!(self.config.provider, HarnessProvider::LocalMock) + || !self.active_attempts.is_empty() + || !self + .pending_control_processes + .lock() + .map_err(|_| "owned execution tracking is poisoned".to_owned())? + .is_empty() + { + return self.restore_owned_baseline(None); + } + let cursor = self.control_revision(self.operation_budget())?; + if baseline["control_revision"] == cursor { + Ok(baseline) + } else { + self.restore_owned_baseline(None) + } + } + + fn remember_local_baseline(&mut self, baseline: &Value) { + if matches!(self.config.provider, HarnessProvider::LocalMock) { + self.verified_local_baseline = Some(baseline.clone()); + } + } + + fn cleanup_case_inner(&mut self, case: &BehaviorCase) -> Result<(), String> { + // A prepared-but-unadmitted attempt owns no namespace resources. + self.pending_attempts.remove(&case.id); + // Ownership is removed only after successful cleanup. Repeated + // terminal-path cleanup must not launch another request for an + // already-cleaned attempt, or fall back to its unscoped template. + let Some(attempt) = self.active_attempts.get(&case.id).map(Arc::clone) else { + let baseline = self.restore_owned_baseline(None)?; + self.remember_local_baseline(&baseline); + return Ok(()); + }; + { + let pending = self.pending_control_processes.lock().map_err(|_| { + "owned execution tracking is poisoned; refusing namespace cleanup".to_owned() + })?; + if !pending.is_empty() { + return Err(format!( + "owned executions lack terminal retraction proof; refusing namespace unlink: {pending:?}" + )); + } + } + let paths = &attempt.cleanup_paths; + if paths.is_empty() { + self.mark_health_boundary()?; + let baseline = self.restore_owned_baseline(Some(&case.id))?; + write_json(&attempt.directory.join("cleanup-baseline.json"), &baseline)?; + self.active_attempts.remove(&case.id); + self.remember_local_baseline(&baseline); + return Ok(()); + } + let budget = self.operation_budget().clone(); + let request_id = format!( + "{}-cleanup-{}-{}", + case.id, + case.seed, + self.next_observation_generation() + ); + let request = myelin_control_contract::NamespaceCleanupRequest { + schema_version: myelin_control_contract::SCHEMA_VERSION, + request_id: request_id.clone(), + paths: paths.clone(), + }; + let reply: myelin_control_contract::NamespaceCleanupReply = + serde_json::from_value(http_json_budget( + "POST", + &format!("{}/api/control/contextual/namespace-cleanup", self.base_url), + Some(serde_json::to_value(&request).map_err(|error| error.to_string())?), + &budget, + )?) + .map_err(|error| format!("decode typed namespace cleanup proof: {error}"))?; + if reply.schema_version != myelin_control_contract::SCHEMA_VERSION + || reply.request_id != request_id + { + return Err("namespace cleanup proof identity mismatch".to_owned()); + } + let stdout = reply + .paths + .iter() + .map(serde_json::to_string) + .collect::, _>>() + .map_err(|error| format!("serialize namespace cleanup proof: {error}"))? + .join("\n"); + verify_cleanup_paths(paths, &stdout)?; + self.mark_health_boundary()?; + let baseline = self.restore_owned_baseline(Some(&case.id))?; + let (cleanup_evidence, baseline_evidence) = thread::scope(|scope| { + let cleanup = scope + .spawn(|| write_json(&attempt.directory.join("namespace-cleanup.json"), &reply)); + let health = scope + .spawn(|| write_json(&attempt.directory.join("cleanup-baseline.json"), &baseline)); + (cleanup.join(), health.join()) + }); + cleanup_evidence.map_err(|_| "cleanup evidence writer panicked".to_owned())??; + baseline_evidence.map_err(|_| "baseline evidence writer panicked".to_owned())??; + self.active_attempts.remove(&case.id); + self.remember_local_baseline(&baseline); + Ok(()) + } + + fn spawn_program_budget( &self, process: &ProcessProgram, request_id: &str, source: &str, hold_before_bootstrap: bool, launch_failure: Option, + budget: &Budget, ) -> Result<(), String> { + budget.check("compress contextual program")?; let mut compressor = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); compressor @@ -252,207 +1019,174 @@ impl ClusterHarness { } else { process.access.execution_id.clone() }; - let attach_timeout_ms = u64::try_from(self.config.deadline.as_millis()).unwrap_or(u64::MAX); - let reply = http_json( - "POST", - &format!("{}/api/control/contextual/spawn", self.base_url), - Some(json!({ - "logical_node_id": process.logical_node_id, - "request_id": request_id, - "command": command, - "args": args, - "env": env, - "working_dir": null, - "label": process.id, - "execution_id": execution_id, - "read_prefixes": process.access.read_prefixes, - "write_prefixes": process.access.write_prefixes, - "attach_timeout_ms": attach_timeout_ms, - })), - )?; - let event_type = reply - .pointer("/observation/event/type") - .and_then(Value::as_str); - if reply.get("type").and_then(Value::as_str) != Some("event") - || (event_type != Some("spawned") - && !(launch_failure.is_some() && event_type == Some("spawn_rejected"))) - { - return Err(format!("contextual spawn was not accepted: {reply}")); - } - Ok(()) - } - - fn stop_process(&self, request_id: &str, kill_after_ms: Option) -> Result<(), String> { - let control_request_id = format!("stop-{request_id}"); - let reply = http_json( - "POST", - &format!("{}/api/control/contextual/{request_id}/stop", self.base_url), - Some(json!({ - "control_request_id": control_request_id, - "kill_after_ms": kill_after_ms, - })), - )?; - if reply - .pointer("/observation/event/type") - .and_then(Value::as_str) - != Some("stop_accepted") - { - return Err(format!("contextual stop was not accepted: {reply}")); - } - Ok(()) - } - - pub(super) fn wait_execution( - &mut self, - process: &ProcessProgram, - request_id: &str, - stop_at: Option<(ProcessStopPhase, Option)>, - ) -> Result { - let deadline = Instant::now() + self.config.deadline; - let mut after_sequence = 0_u64; - let mut lifecycle = Vec::new(); - let mut stdout = Vec::new(); - let mut stderr = Vec::new(); - let mut terminal = false; - let mut exit_success = false; - let mut exit_status = None; - let mut stop_requested = false; - while !terminal { - self.ensure_orchestrator_live()?; - let reply = http_json( - "GET", - &format!( - "{}/api/control/contextual/{request_id}/events?after_sequence={after_sequence}", - self.base_url - ), - None, - )?; - let execution = reply - .get("execution") - .ok_or_else(|| format!("contextual events response is malformed: {reply}"))?; - let truncated_before = execution - .get("truncated_before") - .and_then(Value::as_u64) - .ok_or_else(|| { - format!("contextual events response has no truncation cursor: {reply}") - })?; - if truncated_before > after_sequence { - return Err(format!( - "contextual event history for {request_id} was truncated before sequence {truncated_before}; requested {after_sequence}" - )); - } - after_sequence = execution - .get("next_sequence") - .and_then(Value::as_u64) - .unwrap_or(after_sequence); - for record in execution - .get("events") - .and_then(Value::as_array) - .into_iter() - .flatten() - { - let event = record - .pointer("/observation/event") - .ok_or_else(|| format!("contextual event record is malformed: {record}"))?; - let event_type = event - .get("type") - .and_then(Value::as_str) - .ok_or_else(|| format!("contextual event type is missing: {record}"))?; - match event_type { - "spawned" => lifecycle.push("spawned".to_owned()), - "process_started" => lifecycle.push("process_started".to_owned()), - "context_ready" => lifecycle.push("context_ready".to_owned()), - "stdout" => { - append_json_bytes(event, &mut stdout)?; - lifecycle.push("user_result".to_owned()); - } - "stderr" => append_json_bytes(event, &mut stderr)?, - "exited" => { - lifecycle.push("exited".to_owned()); - terminal = true; - exit_status = event.get("status").map(Value::to_string); - exit_success = event.pointer("/status/kind").and_then(Value::as_str) - == Some("code") - && event.pointer("/status/value").and_then(Value::as_i64) == Some(0); - } - "spawn_failed" | "process_error" | "spawn_rejected" => { - lifecycle.push(event_type.to_owned()); - terminal = true; - } - "bootstrap_failed" | "stop_accepted" | "stop_rejected" => { - lifecycle.push(event_type.to_owned()); - } - "live_executions" => {} - other => return Err(format!("unknown contextual event {other:?}")), - } - let stop_event = match stop_at.map(|(phase, _)| phase) { - Some(ProcessStopPhase::DuringBootstrap) => Some("process_started"), - Some(ProcessStopPhase::AfterContextReady) => Some("context_ready"), - Some(ProcessStopPhase::AfterSpawn) | None => None, - }; - if !stop_requested && stop_event == Some(event_type) { - let kill_after_ms = stop_at - .expect("deferred stop retains its escalation policy") - .1; - self.stop_process(request_id, kill_after_ms)?; - stop_requested = true; - } - } - terminal |= execution - .get("terminal") - .and_then(Value::as_bool) - .unwrap_or(false); - if !terminal { - if Instant::now() >= deadline { - return Err(self.timeout_evidence(&format!( - "contextual process {} terminal; lifecycle={lifecycle:?}", - process.id - ))); - } - thread::sleep(POLL_INTERVAL); - } - } - if stop_at.is_some() && !stop_requested { - return Err(format!( - "contextual process {request_id} terminated before the requested stop phase" - )); - } - let stdout_text = String::from_utf8(stdout) - .map_err(|error| format!("{} stdout is not UTF-8: {error}", process.id))?; - let stderr_text = String::from_utf8_lossy(&stderr).into_owned(); - let results = stdout_text - .lines() - .filter(|line| !line.trim().is_empty()) - .map(|line| { - serde_json::from_str::(line) - .map_err(|error| format!("parse structured Python output {line:?}: {error}")) - }) - .collect::, _>>()?; - Ok(ExecutionObservation { - process: process.id.clone(), - request_id: request_id.to_owned(), + let attach_timeout_ms = + u64::try_from(budget.remaining("attach contextual program")?.as_millis()) + .unwrap_or(u64::MAX) + .max(1); + let spawn_url = format!("{}/api/control/contextual/spawn", self.base_url); + let spawn_body = serde_json::to_value(ContextualSpawnRequest { logical_node_id: process.logical_node_id, - lifecycle, - results, - terminal, - exit_success, - exit_status, - stdout: stdout_text, - stderr: stderr_text, + request_id: request_id.to_owned(), + spec: ContextualProcessSpec { + command, + args, + env, + working_dir: None, + label: Some(process.id.clone()), + execution_id, + read_prefixes: process.access.read_prefixes.clone(), + write_prefixes: process.access.write_prefixes.clone(), + attach_timeout_ms, + staged_program: None, + }, + }) + .map_err(|error| format!("serialize contextual spawn request: {error}"))?; + // A request is never retried: an ambiguous reply is reconciled through + // the already registered execution identity, not another spawn. + let reply = match http_json_budget("POST", &spawn_url, Some(spawn_body), budget) { + Ok(reply) => reply, + Err(_) + if self.contextual_execution_exists( + request_id, + process.logical_node_id, + budget, + ) => + { + return Ok(()); + } + Err(error) => return Err(error), + }; + let reply = serde_json::from_value::(reply) + .map_err(|error| format!("decode contextual spawn reply: {error}"))?; + let accepted = matches!( + &reply, + ContextualControlReply::Event { observation } + if observation.request_id == request_id + && observation.logical_node_id == process.logical_node_id + && (matches!( + &observation.event, + ContextualProcessEventKind::Spawned { .. } + ) || (launch_failure.is_some() + && matches!( + &observation.event, + ContextualProcessEventKind::SpawnRejected { .. } + ))) + ); + if !accepted { + return Err(format!("contextual spawn was not accepted: {reply:?}")); + } + Ok(()) + } + fn contextual_execution_exists( + &self, + request_id: &str, + logical_node_id: u64, + budget: &Budget, + ) -> bool { + let started = Instant::now(); + let Ok(reply) = http_json_budget( + "GET", + &format!( + "{}/api/control/contextual/{request_id}/events?after_sequence=0", + self.base_url + ), + None, + budget, + ) else { + return false; + }; + // This reconciliation GET may already contain terminal output. Retain + // its original reply too, even if the attempt expires before the main + // collector can request the same cursor again. + let Ok(mut log) = self.probe_event_log(request_id, "spawn-reconciliation") else { + return false; + }; + if write_record( + &mut log, + &WireEvidence { + elapsed_ns: started.elapsed().as_nanos(), + request_ns: started.elapsed().as_nanos(), + response: &reply, + }, + ) + .and_then(|()| log.flush().map_err(|error| error.to_string())) + .is_err() + { + return false; + } + serde_json::from_value::(reply).is_ok_and(|reply| { + matches!( + reply, + ContextualControlReply::Events { execution } + if execution.request_id == request_id + && execution.logical_node_id == logical_node_id + ) }) } + fn stop_process_budget( + &self, + request_id: &str, + kill_after_ms: Option, + budget: &Budget, + ) -> Result<(), String> { + let control_request_id = format!("stop-{request_id}"); + let request = serde_json::to_value(ContextualStopRequest { + control_request_id: control_request_id.clone(), + kill_after_ms, + }) + .map_err(|error| format!("serialize contextual stop request: {error}"))?; + let reply = http_json_budget( + "POST", + &format!("{}/api/control/contextual/{request_id}/stop", self.base_url), + Some(request), + &budget.child(self.config.deadline), + )?; + let reply = serde_json::from_value::(reply) + .map_err(|error| format!("decode contextual stop reply: {error}"))?; + let resolved = match &reply { + ContextualControlReply::Event { observation } + if observation.request_id == control_request_id => + { + match &observation.event { + ContextualProcessEventKind::StopAccepted { .. } => true, + // The target can retire before its stop reaches the node. + // Its retained terminal cursor still provides cleanup proof. + ContextualProcessEventKind::StopRejected { error } + if error == "contextual process is not live on this node" => + { + true + } + _ => false, + } + } + _ => false, + }; + if !resolved { + return Err(format!("contextual stop was not accepted: {reply:?}")); + } + Ok(()) + } + pub(super) fn kill_node(&self, logical_node_id: u64) -> Result<(), String> { - http_json( + let budget = self.operation_budget().child(self.config.deadline); + http_json_budget( "POST", &format!("{}/api/control/nodes/{logical_node_id}/kill", self.base_url), Some(json!({ "command_id": format!("e2e-kill-{}-{logical_node_id}", self.config.seed), })), + &budget, )?; - let deadline = Instant::now() + self.config.deadline; loop { - let fleet = http_json("GET", &format!("{}/api/control/fleet", self.base_url), None)?; + budget.check(&format!("node {logical_node_id} stopped"))?; + self.ensure_orchestrator_live()?; + let fleet = http_json_budget( + "GET", + &format!("{}/api/control/fleet", self.base_url), + None, + &budget, + )?; let phase = fleet .pointer("/FleetStatus/nodes") .and_then(Value::as_array) @@ -468,85 +1202,2395 @@ impl ClusterHarness { Some("stop_failed") => { return Err(format!("node {logical_node_id} failed to stop: {fleet}")); } - _ if Instant::now() >= deadline => { - return Err(self.timeout_evidence(&format!( - "node {logical_node_id} stopped; fleet={fleet}" - ))); - } - _ => thread::sleep(POLL_INTERVAL), + _ => budget.wait( + POLL_INTERVAL, + &format!("node {logical_node_id} stopped; last phase {phase:?}"), + )?, } } } - fn stop_orchestrator_injection(&mut self) -> Result<(), String> { - // Record the worker containers that exist before the injection: - // earlier failure cases may legally have removed containers (e.g. a - // killed node whose replacement never arrived), so the post-mortem - // census must compare against this snapshot, not the configured node - // count. Orchestrator loss itself must not add or remove any. - self.pre_stop_containers = list_containers(&self.container_prefix)?; - self.orchestrator - .kill() - .map_err(|error| format!("stop orchestrator injection: {error}"))?; - let deadline = Instant::now() + self.config.deadline; - loop { - if self - .orchestrator - .try_wait() - .map_err(|error| format!("observe stopped orchestrator: {error}"))? - .is_some() - { - self.orchestrator_stopped = true; + fn schedule_case( + &self, + case: &BehaviorCase, + sources: &[impl AsRef], + trackers: &mut [ExecutionTracker], + log: &mut BufWriter, + started: Instant, + owner: &Budget, + ) -> Vec { + let budget = owner.child(self.config.deadline); + let (mut errors, receiver) = thread::scope(|scope| { + let _cancel_before_join = CancelOnDrop(budget.clone()); + let (sender, receiver) = mpsc::channel::(); + let mut controls = 0; + let mut errors = Vec::new(); + loop { + while let Ok(done) = receiver.try_recv() { + controls -= 1; + if let Err(error) = trackers[done.index].control_completed(done) { + errors.push(error); + } + } + if !errors.is_empty() { + break; + } + if let Err(error) = budget.check("DAG ready set and execution terminal cursors") { + errors.push(error); + break; + } + if let FailureInjection::StopProcess { process, phase, .. } = &case.failure + && matches!( + phase, + ProcessStopPhase::AfterStreamFirstFrame + | ProcessStopPhase::AfterSiblingStreamFirstFrame + ) + && stream_stop_ready( + case, + process, + *phase, + trackers.iter().map(|state| { + ( + state.process.as_str(), + state.results.as_slice(), + state.lifecycle.as_slice(), + ) + }), + ) + && let Some(state) = trackers.iter_mut().find(|state| state.process == *process) + { + state.stop_due = true; + } + // Stop controls have reserved capacity independent of blocked + // spawn acknowledgements, so bootstrap holds can be released. + for (index, state) in trackers.iter_mut().enumerate() { + if !state.stop_due || state.stop_requested { + continue; + } + state.stop_requested = true; + state.stop_pending = true; + let request = state.request_id.clone(); + let grace = state.stop_at.and_then(|(_, grace)| grace); + let sender = sender.clone(); + let worker_budget = budget.child(self.config.deadline); + state.stop_budget = Some(worker_budget.clone()); + controls += 1; + scope.spawn(move || { + let begin = Instant::now(); + let result = catch_unwind(AssertUnwindSafe(|| { + self.stop_process_budget(&request, grace, &worker_budget) + })) + .map_err(|_| "stop worker panicked".to_owned()) + .and_then(std::convert::identity); + let _ = sender.send(ControlCompletion { + index, + kind: ControlKind::Stop, + result, + elapsed: begin.elapsed(), + }); + }); + } + let completed = trackers + .iter() + .filter(|state| state.completed) + .map(|state| state.process.as_str()) + .collect::>(); + let ready = case + .processes + .iter() + .enumerate() + .filter(|(index, process)| { + !trackers[*index].dispatched + && process + .depends_on + .iter() + .all(|dependency| completed.contains(dependency.as_str())) + }) + .map(|(index, _)| index) + .collect::>(); + for index in ready { + let state = &mut trackers[index]; + // Install the identity before dispatch. Event collection starts + // after spawn acceptance; retained cursors make that lossless. + match self.pending_control_processes.lock() { + Ok(mut pending) => { + pending.insert(state.request_id.clone()); + } + Err(_) => { + errors.push( + "owned execution tracking is poisoned before dispatch".to_owned(), + ); + break; + } + } + state.dispatched = true; + state.spawn_pending = true; + state.dispatched_ns = Some(started.elapsed().as_nanos()); + let request = state.request_id.clone(); + let process = &case.processes[index]; + let source = sources[index].as_ref(); + let hold = state + .stop_at + .is_some_and(|(phase, _)| phase == ProcessStopPhase::DuringBootstrap); + let launch_failure = match &case.failure { + FailureInjection::LaunchFailure { + process: target, + kind, + } if target == &process.id => Some(*kind), + _ => None, + }; + let sender = sender.clone(); + let worker_budget = budget.child(self.config.deadline); + state.spawn_budget = Some(worker_budget.clone()); + controls += 1; + scope.spawn(move || { + let begin = Instant::now(); + let result = catch_unwind(AssertUnwindSafe(|| { + self.spawn_program_budget( + process, + &request, + source, + hold, + launch_failure, + &worker_budget, + ) + })) + .map_err(|_| "spawn worker panicked".to_owned()) + .and_then(std::convert::identity); + let _ = sender.send(ControlCompletion { + index, + kind: ControlKind::Spawn, + result, + elapsed: begin.elapsed(), + }); + }); + } + let progress = match self.collect_executions(trackers, &budget, log, started) { + Ok(progress) => progress, + Err(error) => { + errors.push(error); + break; + } + }; + for state in trackers + .iter_mut() + .filter(|state| state.terminal && !state.completed) + { + if state.spawn_pending || state.stop_pending { + continue; + } + if let Err(error) = state.validate_terminal() { + errors.push(error); + } else if !state.exit_success + && !expected_process_failure(case, &state.process) + && !case.processes.iter().any(|program| { + program.id == state.process + && recorded_action_failure(program, &state.partial_observation()) + .is_some() + }) + { + errors.push(format!( + "process {} failed with status {:?}\nstdout={}\nstderr={}", + state.process, + state.exit_status, + String::from_utf8_lossy(&state.stdout), + String::from_utf8_lossy(&state.stderr), + )); + } else { + // Expected failures and complete typed action aborts + // release dependencies without canceling healthy siblings. + state.completed = true; + } + } + if !errors.is_empty() + || (trackers.iter().all(|state| state.completed && state.acked) + && controls == 0) + { + break; + } + if controls == 0 + && !trackers + .iter() + .any(|state| state.dispatched && !state.completed) + && trackers.iter().any(|state| !state.dispatched) + { + if case.processes.iter().enumerate().any(|(index, process)| { + !trackers[index].dispatched + && process.depends_on.iter().all(|dependency| { + trackers + .iter() + .any(|state| state.process == *dependency && state.completed) + }) + }) { + continue; + } + errors.push("case dependency graph made no progress".to_owned()); + break; + } + if progress { + thread::yield_now(); + } + // The batch request parks on real event revisions. Only an + // outstanding spawn or ACK needs short bounded reconciliation. + if !progress + && (trackers.iter().any(|state| state.spawn_pending) + || trackers + .iter() + .all(|state| !state.dispatched || state.terminal)) + && let Err(error) = budget.wait( + MIN_RECONCILE_INTERVAL, + "spawn reply or terminal acknowledgement", + ) + { + errors.push(error); + break; + } + } + // HTTP checks shared cancellation during connect/write/read. The + // scope therefore joins owned workers; it never detaches a stalled + // request and calls that request canceled. + budget.cancel(); + (errors, receiver) + }); + // The receiver outlives scoped joins, so final acceptance/stop outcomes + // cannot disappear when cancellation races a worker's final send. + for done in receiver.try_iter() { + if let Err(error) = trackers[done.index].control_completed(done) { + errors.push(error); + } + } + // No scheduler worker can dispatch these queued controls after scope + // exit. Keep requested controls pending until their native outcome. + for state in trackers.iter_mut().filter(|state| !state.stop_requested) { + state.stop_due = false; + } + errors + } + + fn collect_executions( + &self, + trackers: &mut [ExecutionTracker], + budget: &Budget, + log: &mut BufWriter, + started: Instant, + ) -> Result { + self.ensure_orchestrator_live()?; + budget.check("collect contextual event cursors")?; + let cursors = trackers + .iter() + .filter(|state| { + state.dispatched + && !state.spawn_pending + && !state.acked + && !(state.terminal && state.gap.is_some()) + }) + .map(|state| ContextualEventCursor { + request_id: state.request_id.clone(), + after_sequence: state.cursor, + }) + .collect::>(); + if cursors.is_empty() { + return Ok(false); + } + let wait_ms = if trackers + .iter() + .all(|state| !state.dispatched || state.terminal) + { + 0 + } else { + 100 + }; + let body = + serde_json::to_value(Versioned::new(ContextualEventsRequest { cursors, wait_ms })) + .map_err(|error| format!("serialize event cursors: {error}"))?; + let begin = Instant::now(); + let reply = http_json_budget( + "POST", + &format!("{}/api/control/contextual/events", self.base_url), + Some(body), + &budget.child(EVENT_REQUEST_SLICE), + ); + let elapsed = begin.elapsed().as_nanos(); + for state in trackers + .iter_mut() + .filter(|state| state.dispatched && !state.acked) + { + state.requests += 1; + state.request_ns += elapsed; + } + let reply = match reply { + Ok(reply) => reply, + Err(error) => { + let idle = wait_ms != 0 && error.contains("status code 504"); + for state in trackers + .iter_mut() + .filter(|state| state.dispatched && !state.acked) + { + if !idle { + state.retries += 1; + state.last_transport_error = Some(error.clone()); + } + } + write_record( + log, + &json!({ + "elapsed_ns": started.elapsed().as_nanos(), "request_ns": elapsed, + "retry": error, "predicate": "contextual event cursor reconciliation", + }), + )?; + budget.check("reconcile failed event transport")?; + if !idle { + let retries = trackers + .iter() + .map(|state| state.retries) + .max() + .unwrap_or(0) + .min(5); + let backoff = (MIN_RECONCILE_INTERVAL * (1_u32 << retries)).min(POLL_INTERVAL); + budget.wait(backoff, "bounded event transport reconciliation")?; + } + return Ok(false); + } + }; + // Persist the original wire records, including duplicates, unknown + // events and malformed output, before interpretation can fail. + write_record( + log, + &WireEvidence { + elapsed_ns: started.elapsed().as_nanos(), + request_ns: elapsed, + response: &reply, + }, + )?; + let reply = serde_json::from_value::>(reply) + .map_err(|error| format!("decode versioned batch events: {error}"))? + .into_payload()?; + let ContextualControlReply::EventsBatch(batch) = reply else { + return Err(format!( + "contextual event batch returned the wrong reply type: {reply:?}" + )); + }; + let executions = &batch.executions; + let missing = &batch.missing; + let mut answered = BTreeSet::new(); + let mut errors = Vec::new(); + let mut progress = false; + for execution in executions { + let request = execution.request_id.as_str(); + let Some(state) = trackers + .iter_mut() + .find(|state| state.dispatched && state.request_id == request) + else { + errors.push(format!("event batch returned unowned request {request}")); + continue; + }; + if !answered.insert(request) { + errors.push(format!("event batch repeated request {request}")); + continue; + } + let previous_cursor = state.cursor; + let drain_started = Instant::now(); + let ingestion = state.ingest(execution, drain_started.duration_since(started)); + state.event_drain_ns += drain_started.elapsed().as_nanos(); + match ingestion { + Ok(changed) => { + progress |= changed; + if changed { + state.max_event_backlog_records = state + .max_event_backlog_records + .max(state.cursor - previous_cursor); + // A complete prior snapshot excludes these records at + // some instant after its request began. This bounds + // their drain lag using only the runner's clock. + if let Some(earliest) = + state.last_snapshot_request_ns.or(state.dispatched_ns) + { + let bound = started.elapsed().as_nanos().saturating_sub(earliest); + state.event_drain_lag_upper_bound_ns = + Some(state.event_drain_lag_upper_bound_ns.unwrap_or(0).max(bound)); + } + } + state.last_snapshot_request_ns = Some(begin.duration_since(started).as_nanos()); + } + Err(error) => { + state.gap = Some(error.clone()); + errors.push(error); + } + } + if state.terminal { + self.pending_control_processes + .lock() + .map_err(|_| "owned execution tracking is poisoned after terminal".to_owned())? + .remove(&state.request_id); + } + } + for request in missing { + let request = request.as_str(); + if !answered.insert(request) { + errors.push(format!("event batch repeated missing request {request}")); + continue; + } + match trackers.iter().find(|state| state.request_id == request) { + Some(state) if state.terminal || state.spawn_pending || !state.accepted => {} + Some(_) => errors.push(format!("accepted contextual execution {request} disappeared before terminal reconciliation")), + None => errors.push(format!("event batch reported unowned missing request {request}")), + } + } + for state in trackers.iter().filter(|state| { + state.dispatched + && !state.spawn_pending + && !state.acked + && !(state.terminal && state.gap.is_some()) + }) { + if !answered.contains(state.request_id.as_str()) { + errors.push(format!( + "event batch omitted requested cursor {}", + state.request_id + )); + } + } + if trackers.iter().any(ExecutionTracker::ack_ready) { + log.flush() + .map_err(|error| format!("flush terminal execution records: {error}"))?; + log.get_ref() + .sync_data() + .map_err(|error| format!("persist terminal execution records: {error}"))?; + // A stalled terminal ACK cannot serialize all siblings' event + // drains behind one request slice per completed execution. + let acknowledgements = thread::scope(|scope| { + let jobs = trackers + .iter() + .enumerate() + .filter(|(_, state)| state.ack_ready()) + .map(|(index, state)| { + scope.spawn(move || (index, self.acknowledge_execution(state, budget))) + }) + .collect::>(); + jobs.into_iter().map(|job| job.join()).collect::>() + }); + for acknowledgement in acknowledgements { + match acknowledgement { + Ok((index, Ok(()))) => trackers[index].acked = true, + Ok((index, Err(error))) => { + trackers[index].retries += 1; + trackers[index].last_transport_error = Some(error); + } + Err(_) => errors.push("terminal acknowledgement worker panicked".to_owned()), + } + } + } + if errors.is_empty() { + Ok(progress) + } else { + Err(errors.join("; ")) + } + } + + fn acknowledge_execution( + &self, + state: &ExecutionTracker, + budget: &Budget, + ) -> Result<(), String> { + let incarnation = state + .execution_incarnation + .as_ref() + .ok_or_else(|| format!("terminal execution {} lacks incarnation", state.request_id))?; + let reply = http_json_budget( + "POST", + &format!( + "{}/api/control/contextual/{}/ack", + self.base_url, state.request_id + ), + Some( + serde_json::to_value(Versioned::new(ContextualEventsAckRequest { + through_sequence: state.cursor, + execution_incarnation: incarnation.clone(), + })) + .map_err(|error| format!("serialize terminal acknowledgement: {error}"))?, + ), + &budget.child(EVENT_REQUEST_SLICE), + )?; + let reply = serde_json::from_value::>(reply) + .map_err(|error| format!("decode terminal acknowledgement: {error}"))? + .into_payload()?; + let ContextualControlReply::Acknowledged(acknowledgement) = &reply else { + return Err(format!( + "terminal acknowledgement returned the wrong reply type: {reply:?}" + )); + }; + if acknowledgement.request_id != state.request_id + || acknowledgement.execution_incarnation != *incarnation + || acknowledgement.through_sequence != state.cursor + { + return Err(format!( + "terminal acknowledgement did not cover owned cursor: {reply:?}" + )); + } + Ok(()) + } + + fn stop_owned_processes(&self, requests: &[String], budget: &Budget) -> Vec { + // The IR admits at most twenty processes. Launch every independent stop + // before waiting for any reply; leave half the remaining reserve for + // terminal/retraction collection and namespace cleanup. + let stop_cap = budget + .remaining("stop all owned executions") + .unwrap_or_default() + / 2; + let stop_budget = budget.child(stop_cap.min(self.config.deadline)); + thread::scope(|scope| { + let jobs = requests + .iter() + .map(|request| { + let budget = &stop_budget; + scope.spawn(move || { + self.stop_process_budget(request, Some(0), budget) + .map_err(|error| format!("stop owned execution {request}: {error}")) + }) + }) + .collect::>(); + jobs.into_iter() + .filter_map(|job| match job.join() { + Ok(Ok(())) => None, + // A durable terminal ACK already retired this identity. + Ok(Err(error)) if error.contains("does not exist") => None, + Ok(Err(error)) => Some(error), + Err(_) => Some("owned execution stop worker panicked".to_owned()), + }) + .collect() + }) + } + + fn drain_owned_executions( + &self, + trackers: &mut [ExecutionTracker], + budget: &Budget, + log: &mut BufWriter, + started: Instant, + ) -> Vec { + let mut errors = Vec::new(); + let mut pause = MIN_RECONCILE_INTERVAL; + while trackers.iter().any(|state| { + state.dispatched && (!state.terminal || (!state.acked && state.gap.is_none())) + }) { + if let Err(error) = budget.check("terminal retraction of all owned executions") { + errors.push(error); break; } - if Instant::now() >= deadline { - return Err(self.timeout_evidence("orchestrator failure injection")); + match self.collect_executions(trackers, budget, log, started) { + Ok(progress) => { + pause = if progress { + MIN_RECONCILE_INTERVAL + } else { + (pause * 2).min(POLL_INTERVAL) + }; + } + Err(error) => { + // A gap remains failure and is NEVER acknowledged away. + // Continue draining every other owned execution. + if !errors.contains(&error) { + errors.push(error); + } + } + } + if !trackers.iter().any(|state| { + state.dispatched && (!state.terminal || (!state.acked && state.gap.is_none())) + }) { + break; + } + if let Err(error) = budget.wait(pause, "owned execution terminal/retraction progress") { + errors.push(error); + break; } - thread::sleep(POLL_INTERVAL); } - self.wait_for_no_workload_processes() + errors } - pub(super) fn wait_for_no_workload_processes(&self) -> Result<(), String> { - let deadline = Instant::now() + self.config.deadline; - loop { - match list_workload_processes(&self.container_prefix)? { - Some(processes) if processes.is_empty() => return Ok(()), - Some(processes) => { - if Instant::now() >= deadline { - return Err(format!( - "deadline waiting for contextual native processes to exit: {processes:?}" - )); - } - } - // `docker top` cannot introspect a container that is not - // running right now (stopped, paused, or mid-transition), - // which hosts no workload anyway. Keep polling: Docker - // either starts introspecting it again or it disappears. - None => { - if Instant::now() >= deadline { - return Err( - "deadline waiting to inspect harness workload containers".to_owned() - ); - } - } + fn probe_event_log( + &self, + request_id: &str, + owner: &str, + ) -> Result, String> { + let directory = self.config.artifacts.join("control-executions"); + fs::create_dir_all(&directory) + .map_err(|error| format!("create probe evidence directory: {error}"))?; + let digest = format!("{:x}", Sha256::digest(request_id.as_bytes())); + let file = fs::OpenOptions::new() + .create(true) + .append(true) + .open(directory.join(format!("{digest}.{owner}.jsonl"))) + .map_err(|error| format!("open probe evidence for {request_id}: {error}"))?; + Ok(BufWriter::new(file)) + } + + fn run_owned_probe( + &self, + process: &ProcessProgram, + request_id: &str, + source: &str, + owner: &Budget, + work: &Budget, + ) -> Result { + let mut tracker = ExecutionTracker::new(process, request_id.to_owned(), None); + let mut log = self.probe_event_log(request_id, "collector")?; + write_record(&mut log, &ProbeSourceEvidence { request_id, source })?; + let started = Instant::now(); + let probe = BehaviorCase { + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + id: process.id.clone(), + seed: self.config.seed, + live_nodes: self.node_ids.iter().copied().collect(), + topology: TopologyFamily::Fixed, + scenarios: BTreeSet::new(), + routes: Vec::new(), + read_only_fixture_paths: BTreeSet::new(), + resource_bounds: Default::default(), + processes: vec![process.clone()], + failure: FailureInjection::None, + }; + let errors = self.schedule_case( + &probe, + &[source], + std::slice::from_mut(&mut tracker), + &mut log, + started, + work, + ); + let result = if errors.is_empty() { + tracker + .validate_terminal() + .map(|()| tracker.partial_observation()) + } else { + Err(errors.join("; ")) + }; + if result.is_err() { + work.cancel(); + let mut errors = self.stop_owned_processes(&[request_id.to_owned()], owner); + errors.extend(self.drain_owned_executions( + std::slice::from_mut(&mut tracker), + owner, + &mut log, + started, + )); + write_record( + &mut log, + &json!({ + "pending": tracker.pending_evidence(), "error": result.as_ref().err(), "cleanup_errors": errors, + }), + )?; + if !errors.is_empty() || !tracker.terminal { + log.flush() + .map_err(|error| format!("flush pending probe evidence: {error}"))?; + return Err(format!( + "{}; fixture quarantine: probe terminal cleanup failed: {}", + result.expect_err("failure branch"), + errors.join("; "), + )); } - thread::sleep(POLL_INTERVAL); } + log.flush() + .map_err(|error| format!("flush fresh probe evidence: {error}"))?; + result + } + + fn run_read_only_probe( + &self, + probe: &BehaviorCase, + budget: &Budget, + ) -> Result { + probe.validate()?; + if !probe.owned_paths().is_empty() + || probe.processes.len() != 1 + || !probe.processes[0].access.write_prefixes.is_empty() + { + return Err( + "read-only probe cannot acquire namespace write or cleanup ownership".to_owned(), + ); + } + let process = &probe.processes[0]; + let request = (probe).execution_request_id(process); + let work = reserve_probe_cleanup(budget)?; + let source = + render_case_python(probe, process, work.remaining("generate read-only probe")?); + let observed = self.run_owned_probe(process, &request, &source, budget, &work)?; + let observation = CaseObservation { + case_id: probe.id.clone(), + executions: vec![observed], + }; + BehaviorOracle::verify_with_budget(probe, &observation, &work) + .map_err(|error| error.to_string())?; + Ok(observation) + } + + fn probe_absent_paths(&self, attempt: &CaseAttempt) -> Result<(), String> { + if attempt.cleanup_paths.is_empty() { + return Ok(()); + } + let reply = + self.query_retained_blobs(attempt.cleanup_paths.clone(), self.operation_budget())?; + verify_typed_absence(&reply)?; + write_json(&attempt.directory.join("precondition-absence.json"), &reply) } } -fn append_json_bytes(event: &Value, output: &mut Vec) -> Result<(), String> { - let bytes = event - .get("bytes") - .and_then(Value::as_array) - .ok_or_else(|| format!("contextual byte event is malformed: {event}"))?; - for byte in bytes { - let byte = byte - .as_u64() - .and_then(|byte| u8::try_from(byte).ok()) - .ok_or_else(|| format!("contextual byte is outside u8: {byte}"))?; - output.push(byte); +fn verify_typed_absence(reply: &myelin_control_contract::RetainedBlobsReply) -> Result<(), String> { + let non_absent = reply + .non_blobs + .iter() + .filter_map(|(path, state)| { + (!matches!(state, myelin_control_contract::RetainedNonBlob::Absent)).then_some(path) + }) + .cloned() + .collect::>(); + if !reply.blobs.is_empty() || !non_absent.is_empty() { + return Err(format!( + "typed precondition found owned paths: blobs={:?}, non_absent={non_absent:?}", + reply.blobs.keys().collect::>() + )); } Ok(()) } + +fn persisted_fixture_snapshot( + entries: &[PersistedFixtureEntry], + observation: &CaseObservation, +) -> Result { + let mut snapshots = BTreeMap::new(); + for entry in entries { + let path = entry.path(); + let results = observation + .executions + .iter() + .flat_map(|execution| &execution.results) + .filter(|result| result.path == path) + .collect::>(); + let namespace = results + .iter() + .rev() + .find(|result| result.kind.is_some() && result.revision.is_some()) + .ok_or_else(|| format!("persistence probe omitted namespace facts for {path}"))?; + let payload = results.iter().find(|result| result.action == "read_blob"); + let snapshot = FixturePathObservation { + kind: namespace.kind.clone().expect("namespace kind checked"), + revision: namespace.revision.expect("namespace revision checked"), + active: namespace.active.ok_or_else(|| { + format!("persistence probe omitted active-state evidence for {path}") + })?, + length: payload.and_then(|result| result.length), + digest: payload.and_then(|result| result.digest.clone()), + }; + match entry { + PersistedFixtureEntry::Blob { .. } + if snapshot.kind != "blob" + || snapshot.length.is_none() + || snapshot.digest.is_none() => + { + return Err(format!("incomplete persisted blob facts for {path}")); + } + PersistedFixtureEntry::QuiescentStream { .. } + if snapshot.kind != "stream" || snapshot.active => + { + return Err(format!("persisted stream {path} is not quiescent")); + } + _ => {} + } + snapshots.insert(path.to_owned(), snapshot); + } + Ok(FixturePathSnapshot { entries: snapshots }) +} + +fn reserve_attempt_identity( + state: &Path, + artifacts: &Path, + cursor: &mut u64, + budget: &Budget, +) -> Result { + // Claim in both durable fixture state and the evidence root. A retained + // fixture with a new evidence directory, or a fresh local fixture reusing + // its evidence directory, must never restart execution/path identities. + let roots = [ + state.join("owned-attempt-identities"), + artifacts.join("attempt-identities"), + ]; + for root in &roots { + fs::create_dir_all(root) + .map_err(|error| format!("create attempt identity ledger: {error}"))?; + } + 'candidate: loop { + budget.check("reserve durable attempt identity")?; + *cursor = cursor + .checked_add(1) + .ok_or_else(|| "case attempt identity exhausted".to_owned())?; + for root in &roots { + let claim = root.join(format!("attempt-{cursor}")); + match fs::create_dir(&claim) { + Ok(()) => { + fs::File::open(root) + .and_then(|directory| directory.sync_all()) + .map_err(|error| format!("persist attempt identity claim: {error}"))?; + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + continue 'candidate; + } + Err(error) => return Err(format!("exclusively claim attempt identity: {error}")), + } + } + return Ok(*cursor); + } +} + +fn read_only_access(execution_id: String, paths: &BTreeSet) -> AccessSpec { + AccessSpec { + execution_id, + read_prefixes: paths.iter().cloned().collect(), + write_prefixes: Vec::new(), + } +} + +fn reserve_probe_cleanup(owner: &Budget) -> Result { + let remaining = owner.remaining("reserve fresh probe cleanup")?; + Ok(owner.child(remaining.saturating_sub((remaining / 4).min(Duration::from_secs(5))))) +} + +#[derive(serde::Serialize)] +struct ProbeSourceEvidence<'a> { + request_id: &'a str, + source: &'a str, +} + +#[derive(serde::Serialize)] +struct WireEvidence<'a> { + elapsed_ns: u128, + request_ns: u128, + response: &'a Value, +} + +struct CancelOnDrop(Budget); + +impl Drop for CancelOnDrop { + fn drop(&mut self) { + self.0.cancel(); + } +} + +enum ControlKind { + Spawn, + Stop, +} + +struct ControlCompletion { + index: usize, + kind: ControlKind, + result: Result<(), String>, + elapsed: Duration, +} + +/// One cursor owns each execution's byte stream. Arrival order between cursors +/// is deliberately never used as causal order; the Python action records carry +/// the causal identities the oracle consumes. +struct ExecutionTracker { + process: String, + request_id: String, + logical_node_id: u64, + execution_incarnation: Option, + dependencies: Vec, + dispatched: bool, + spawn_pending: bool, + spawn_budget: Option, + accepted: bool, + completed: bool, + cursor: u64, + terminal: bool, + acked: bool, + exit_success: bool, + exit_status: Option, + lifecycle: Vec, + stdout: Vec, + stderr: Vec, + results: Vec, + parsed_stdout: usize, + output_error: Option, + gap: Option, + stop_at: Option<(ProcessStopPhase, Option)>, + stop_due: bool, + stop_requested: bool, + stop_pending: bool, + stop_budget: Option, + dispatched_ns: Option, + spawn_ns: Option, + attach_ns: Option, + first_output_ns: Option, + terminal_ns: Option, + stop_ns: Option, + requests: u64, + retries: u64, + request_ns: u128, + last_snapshot_request_ns: Option, + max_event_backlog_records: u64, + event_drain_ns: u128, + event_drain_lag_upper_bound_ns: Option, + last_transport_error: Option, +} + +impl ExecutionTracker { + fn new( + process: &ProcessProgram, + request_id: String, + stop_at: Option<(ProcessStopPhase, Option)>, + ) -> Self { + Self { + process: process.id.clone(), + request_id, + logical_node_id: process.logical_node_id, + dispatched: false, + spawn_pending: false, + accepted: false, + completed: false, + cursor: 0, + terminal: false, + acked: false, + exit_success: false, + exit_status: None, + lifecycle: Vec::new(), + stdout: Vec::new(), + stderr: Vec::new(), + results: Vec::new(), + parsed_stdout: 0, + output_error: None, + gap: None, + stop_at, + stop_due: false, + stop_requested: false, + stop_pending: false, + dispatched_ns: None, + spawn_ns: None, + attach_ns: None, + first_output_ns: None, + terminal_ns: None, + stop_ns: None, + requests: 0, + retries: 0, + request_ns: 0, + last_snapshot_request_ns: None, + max_event_backlog_records: 0, + event_drain_ns: 0, + event_drain_lag_upper_bound_ns: None, + last_transport_error: None, + spawn_budget: None, + stop_budget: None, + execution_incarnation: None, + dependencies: process.depends_on.clone(), + } + } + + fn ack_ready(&self) -> bool { + self.terminal + && !self.acked + && self.gap.is_none() + && !self.stop_pending + && !(self.stop_due && !self.stop_requested) + } + + fn control_completed(&mut self, done: ControlCompletion) -> Result<(), String> { + match done.kind { + ControlKind::Spawn => { + self.spawn_pending = false; + self.spawn_budget = None; + self.spawn_ns = Some(done.elapsed.as_nanos()); + match done.result { + Ok(()) => { + self.accepted = true; + if self + .stop_at + .is_some_and(|(phase, _)| phase == ProcessStopPhase::AfterSpawn) + { + self.stop_due = true; + } + } + // Cursor evidence reconciles an ambiguous spawn response; + // it never authorizes redelivery of the spawn request. + Err(error) if self.cursor != 0 => { + self.last_transport_error = Some(error); + self.accepted = true; + } + Err(error) => return Err(format!("spawn process {}: {error}", self.process)), + } + } + ControlKind::Stop => { + self.stop_pending = false; + self.stop_ns = Some(done.elapsed.as_nanos()); + self.stop_budget = None; + if let Err(error) = done.result { + if self.lifecycle.iter().any(|event| event == "stop_accepted") { + self.last_transport_error = Some(error); + } else { + return Err(format!("stop process {}: {error}", self.process)); + } + } + } + } + Ok(()) + } + + fn ingest( + &mut self, + execution: &ContextualExecutionView, + elapsed: Duration, + ) -> Result { + if execution.request_id != self.request_id + || execution.logical_node_id != self.logical_node_id + { + return Err(format!( + "event identity differs from owned execution {}: {execution:?}", + self.request_id + )); + } + let incarnation = execution.execution_incarnation.as_str(); + if incarnation.is_empty() { + return Err(format!( + "{} events lack execution incarnation", + self.request_id + )); + } + if self + .execution_incarnation + .as_deref() + .is_some_and(|owned| owned != incarnation) + { + return Err(format!( + "execution incarnation changed for {}; retained cursor belongs to {:?}", + self.request_id, self.execution_incarnation + )); + } + self.execution_incarnation + .get_or_insert_with(|| incarnation.to_owned()); + let truncated = execution.truncated_before; + let next = execution.next_sequence; + let declared_terminal = execution.terminal; + let before = self.cursor; + if truncated > self.cursor { + self.gap.get_or_insert_with(|| format!( + "contextual event history for {} was truncated before sequence {truncated}; requested {}", + self.request_id, self.cursor, + )); + } + if next < self.cursor { + self.gap.get_or_insert_with(|| { + format!( + "contextual cursor for {} regressed from {} to {next}", + self.request_id, self.cursor, + ) + }); + } + for record in &execution.events { + let sequence = record.sequence; + if sequence < self.cursor { + // A reconnect may replay an acknowledged prefix. Its original + // envelope remains in the wire log, but bytes are decoded once. + continue; + } + if sequence != self.cursor { + self.gap.get_or_insert_with(|| { + format!( + "contextual event gap for {}: expected {}, received {sequence}", + self.request_id, self.cursor, + ) + }); + } + if sequence >= next { + self.gap.get_or_insert_with(|| { + format!( + "contextual event {} sequence {sequence} is outside next cursor {next}", + self.request_id, + ) + }); + } + if record.observation.request_id != self.request_id + || record.observation.logical_node_id != self.logical_node_id + { + return Err(format!( + "contextual event record identity mismatch for {}: {record:?}", + self.request_id + )); + } + let event = &record.observation.event; + let kind = match event { + ContextualProcessEventKind::Spawned { .. } => "spawned", + ContextualProcessEventKind::SpawnRejected { .. } => "spawn_rejected", + ContextualProcessEventKind::ProcessStarted { .. } => "process_started", + ContextualProcessEventKind::ContextReady => "context_ready", + ContextualProcessEventKind::Stdout { .. } => "stdout", + ContextualProcessEventKind::Stderr { .. } => "stderr", + ContextualProcessEventKind::SpawnFailed { .. } => "spawn_failed", + ContextualProcessEventKind::BootstrapFailed { .. } => "bootstrap_failed", + ContextualProcessEventKind::Exited { .. } => "exited", + ContextualProcessEventKind::ProcessError { .. } => "process_error", + ContextualProcessEventKind::StopAccepted { .. } => "stop_accepted", + ContextualProcessEventKind::StopRejected { .. } => "stop_rejected", + ContextualProcessEventKind::LiveExecutions { .. } => "live_executions", + ContextualProcessEventKind::ControlUnavailable { .. } => "control_unavailable", + }; + match event { + ContextualProcessEventKind::Spawned { .. } + | ContextualProcessEventKind::ProcessStarted { .. } + | ContextualProcessEventKind::ContextReady => { + self.lifecycle.push(kind.to_owned()); + if matches!(event, ContextualProcessEventKind::ContextReady) { + self.attach_ns.get_or_insert(elapsed.as_nanos()); + } + } + ContextualProcessEventKind::Stdout { bytes } => { + self.stdout.extend_from_slice(bytes); + self.lifecycle.push("user_result".to_owned()); + self.first_output_ns.get_or_insert(elapsed.as_nanos()); + self.parse_output(false); + } + ContextualProcessEventKind::Stderr { bytes } => { + self.stderr.extend_from_slice(bytes); + } + ContextualProcessEventKind::Exited { status } => { + self.lifecycle.push(kind.to_owned()); + self.terminal = true; + self.terminal_ns = Some(elapsed.as_nanos()); + self.exit_status = Some( + serde_json::to_string(status) + .map_err(|error| format!("encode contextual exit status: {error}"))?, + ); + self.exit_success = matches!(status, &ContextualExitStatus::Code(0)); + } + ContextualProcessEventKind::SpawnFailed { .. } + | ContextualProcessEventKind::ProcessError { .. } + | ContextualProcessEventKind::SpawnRejected { .. } => { + self.lifecycle.push(kind.to_owned()); + self.terminal = true; + self.terminal_ns = Some(elapsed.as_nanos()); + } + // Bootstrap failure is followed by a real process exit. It is + // not sufficient proof of reaping or namespace retraction. + ContextualProcessEventKind::BootstrapFailed { .. } + | ContextualProcessEventKind::StopAccepted { .. } + | ContextualProcessEventKind::StopRejected { .. } => { + self.lifecycle.push(kind.to_owned()); + } + ContextualProcessEventKind::LiveExecutions { .. } => {} + ContextualProcessEventKind::ControlUnavailable { error } => { + return Err(format!( + "contextual control became unavailable for {}: {error}", + self.request_id + )); + } + } + let trigger = match self.stop_at.map(|(phase, _)| phase) { + Some(ProcessStopPhase::AfterSpawn) => Some("spawned"), + Some(ProcessStopPhase::DuringBootstrap) => Some("process_started"), + Some(ProcessStopPhase::AfterContextReady) => Some("context_ready"), + Some( + ProcessStopPhase::AfterStreamFirstFrame + | ProcessStopPhase::AfterSiblingStreamFirstFrame, + ) + | None => None, + }; + if trigger == Some(kind) { + self.stop_due = true; + } + self.cursor = sequence + .checked_add(1) + .ok_or_else(|| "contextual event sequence overflow".to_owned())?; + } + if self.cursor != next { + self.gap.get_or_insert_with(|| { + format!( + "contextual event gap for {}: drained {}, advertised next {next}", + self.request_id, self.cursor, + ) + }); + } + if declared_terminal != self.terminal { + self.gap.get_or_insert_with(|| format!( + "contextual terminal reconciliation gap for {}: terminal flag {declared_terminal}, terminal record {}", + self.request_id, self.terminal, + )); + } + if self.terminal { + self.parse_output(true); + } + // Once the cursor proves acceptance, a withheld POST response no + // longer owns progress. Cancel only that transport child, preserving + // both the execution collector and every sibling's budget. + if self.cursor != 0 { + if let Some(budget) = &self.spawn_budget { + budget.cancel(); + } + } + if self.lifecycle.iter().any(|event| event == "stop_accepted") { + if let Some(budget) = &self.stop_budget { + budget.cancel(); + } + } + if let Some(error) = &self.gap { + Err(error.clone()) + } else { + Ok(before != self.cursor) + } + } + + fn parse_output(&mut self, terminal: bool) { + while self.parsed_stdout < self.stdout.len() { + let remaining = &self.stdout[self.parsed_stdout..]; + let Some(length) = remaining + .iter() + .position(|byte| *byte == b'\n') + .map(|position| position + 1) + .or_else(|| terminal.then_some(remaining.len())) + else { + break; + }; + let line = &remaining[..length]; + self.parsed_stdout += length; + if line.iter().all(u8::is_ascii_whitespace) { + continue; + } + let parsed = serde_json::from_slice::(line).and_then(|value| { + if value.get("type").and_then(Value::as_str) == Some("path_cleanup") { + // Auxiliary cleanup facts are not IR actions. Keep their + // original stdout and validate the separate protocol. + if value.get("absent").and_then(Value::as_bool) != Some(true) + || !value.get("error").is_some_and(Value::is_null) + { + self.output_error + .get_or_insert_with(|| format!("path cleanup failed: {value}")); + } + Ok(None) + } else { + serde_json::from_value::(value).map(Some) + } + }); + match parsed { + Ok(Some(result)) => self.results.push(result), + Ok(None) => {} + Err(error) => { + self.output_error.get_or_insert_with(|| format!( + "parse structured Python output for {}: {error}; original bytes retained", self.process, + )); + } + } + } + } + + fn validate_terminal(&self) -> Result<(), String> { + if let Some(error) = self.gap.as_ref().or(self.output_error.as_ref()) { + return Err(error.clone()); + } + if !self.terminal { + return Err(format!( + "{} has no observed terminal record at cursor {}", + self.request_id, self.cursor + )); + } + if self.stop_at.is_some() && !self.stop_requested { + return Err(format!( + "contextual process {} terminated before the requested stop phase", + self.request_id + )); + } + if std::str::from_utf8(&self.stdout).is_err() { + return Err(format!( + "{} stdout is not UTF-8; original bytes retained", + self.process + )); + } + Ok(()) + } + + fn partial_observation(&self) -> ExecutionObservation { + ExecutionObservation { + process: self.process.clone(), + request_id: self.request_id.clone(), + logical_node_id: self.logical_node_id, + lifecycle: self.lifecycle.clone(), + results: self.results.clone(), + terminal: self.terminal, + exit_success: self.exit_success, + exit_status: self.exit_status.clone(), + stdout: String::from_utf8_lossy(&self.stdout).into_owned(), + stderr: String::from_utf8_lossy(&self.stderr).into_owned(), + } + } + + fn pending_evidence(&self) -> Value { + json!({ + "process": self.process, "request_id": self.request_id, "logical_node_id": self.logical_node_id, + "execution_incarnation": self.execution_incarnation, "dependencies": self.dependencies, + "dispatched": self.dispatched, "accepted": self.accepted, "spawn_pending": self.spawn_pending, + "next_sequence": self.cursor, "terminal": self.terminal, "acknowledged": self.acked, + "stop_due": self.stop_due, "stop_requested": self.stop_requested, "stop_pending": self.stop_pending, + "gap": self.gap, "output_error": self.output_error, "last_transport_error": self.last_transport_error, + "pending_predicate": if !self.dispatched { "DAG terminal dependencies" } + else if !self.terminal { "terminal execution record and retraction" } + else if !self.acked { "durable terminal cursor acknowledgement" } else { "complete" }, + }) + } + + fn timing_evidence(&self) -> Value { + json!({ + "request_id": self.request_id, + "dispatch_elapsed_ns": self.dispatched_ns, "spawn_acceptance_ns": self.spawn_ns, + "context_ready_observed_elapsed_ns": self.attach_ns, + "first_output_observed_elapsed_ns": self.first_output_ns, + "terminal_observed_elapsed_ns": self.terminal_ns, "stop_request_ns": self.stop_ns, + "event_request_count": self.requests, "event_request_total_ns": self.request_ns, + "transport_retry_count": self.retries, "event_records": self.cursor, + "max_event_backlog_records": self.max_event_backlog_records, + "event_drain_ns": self.event_drain_ns, + "event_drain_lag_upper_bound_ns": self.event_drain_lag_upper_bound_ns, + "stdout_bytes": self.stdout.len(), "stderr_bytes": self.stderr.len(), + "action_records": self.results.len(), "unparsed_stdout_bytes": self.stdout.len() - self.parsed_stdout, + }) + } +} + +fn write_record(writer: &mut impl IoWrite, value: &T) -> Result<(), String> { + serde_json::to_writer(&mut *writer, value) + .map_err(|error| format!("serialize execution evidence: {error}"))?; + writer + .write_all(b"\n") + .map_err(|error| format!("write execution evidence: {error}")) +} + +fn write_json(path: &std::path::Path, value: &T) -> Result<(), String> { + let mut writer = BufWriter::new( + fs::File::create(path).map_err(|error| format!("create {}: {error}", path.display()))?, + ); + serde_json::to_writer(&mut writer, value) + .map_err(|error| format!("serialize {}: {error}", path.display()))?; + writer + .flush() + .map_err(|error| format!("flush {}: {error}", path.display())) +} + +fn stop_injection( + case: &BehaviorCase, + process: &ProcessProgram, +) -> Option<(ProcessStopPhase, Option)> { + match &case.failure { + FailureInjection::StopProcess { + process: target, + phase, + kill_after_ms, + } if target == &process.id => Some((*phase, *kill_after_ms)), + _ => None, + } +} + +fn expected_process_failure(case: &BehaviorCase, process: &str) -> bool { + match &case.failure { + FailureInjection::StopProcess { + process: target, .. + } + | FailureInjection::LaunchFailure { + process: target, .. + } => target == process, + FailureInjection::None | FailureInjection::SlowProcess { .. } => false, + } +} + +fn probe_case(template: &BehaviorCase, id: String, node_ids: &[u64]) -> BehaviorCase { + // Do not clone the workload's payloads/routes merely to replace every + // process. Probe source contains only the boundary facts it actually uses. + BehaviorCase { + schema_version: template.schema_version, + generator_version: template.generator_version, + id, + seed: template.seed, + live_nodes: node_ids.iter().copied().collect(), + topology: TopologyFamily::Fixed, + scenarios: BTreeSet::new(), + routes: Vec::new(), + read_only_fixture_paths: BTreeSet::new(), + resource_bounds: template.resource_bounds.clone(), + processes: Vec::new(), + failure: FailureInjection::None, + } +} + +fn verify_cleanup_paths(paths: &[String], stdout: &str) -> Result<(), String> { + let expected = paths.iter().map(String::as_str).collect::>(); + let mut observed = BTreeSet::new(); + for line in stdout.lines().filter(|line| !line.trim().is_empty()) { + let record: Value = serde_json::from_str(line) + .map_err(|error| format!("decode cleanup absence evidence: {error}"))?; + let path = record + .get("path") + .and_then(Value::as_str) + .ok_or_else(|| format!("cleanup record has no path: {record}"))?; + if record.get("type").and_then(Value::as_str) != Some("path_cleanup") + || !expected.contains(path) + || !observed.insert(path.to_owned()) + { + return Err(format!( + "cleanup returned duplicate, unowned, or unknown path evidence: {record}" + )); + } + if record.get("attempted").and_then(Value::as_bool) != Some(true) + || record.get("absent").and_then(Value::as_bool) != Some(true) + || !record.get("error").is_some_and(Value::is_null) + { + return Err(format!("cleanup did not prove path absence: {record}")); + } + if let Some(quiescence) = record.get("quiescence").filter(|value| !value.is_null()) { + if quiescence.get("active").and_then(Value::as_bool) != Some(false) + || quiescence.get("revision").and_then(Value::as_u64).is_none() + { + return Err(format!( + "cleanup stream lacks exact inactive revision: {record}" + )); + } + } + } + let missing = expected + .into_iter() + .filter(|path| !observed.contains(*path)) + .collect::>(); + if !missing.is_empty() { + return Err(format!( + "cleanup omitted owned path absence evidence: {missing:?}" + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use myelin_control_contract::{ + ContextualEventRecord, ContextualEventsAcknowledgement, ContextualEventsBatch, + ContextualProcessEvent, ExecutionIdentity, + }; + + #[test] + fn persisted_stream_snapshot_requires_explicit_quiescence() { + let path = "/cases/persisted/stream"; + let entries = [PersistedFixtureEntry::QuiescentStream { + path: path.to_owned(), + }]; + let mut observation: CaseObservation = serde_json::from_value(json!({ + "case_id": "persistence", + "executions": [{ + "process": "persisted-path-probe", + "request_id": "persistence-probe", + "logical_node_id": 3, + "lifecycle": [ + "spawned", "process_started", "context_ready", "user_result", "exited" + ], + "results": [ + { + "process": "persisted-path-probe", + "step": 0, + "action": "lookup", + "path": path, + "outcome": "ok", + "kind": "stream", + "revision": 7 + }, + { + "process": "persisted-path-probe", + "step": 1, + "action": "wait_for_quiescent", + "path": path, + "outcome": "ok", + "kind": "stream", + "revision": 7 + } + ], + "terminal": true, + "exit_success": true, + "stdout": "", + "stderr": "" + }] + })) + .unwrap(); + + assert!( + persisted_fixture_snapshot(&entries, &observation).is_err(), + "successful wait without active-state evidence must not prove quiescence" + ); + for result in &mut observation.executions[0].results { + result.active = Some(false); + } + persisted_fixture_snapshot(&entries, &observation) + .expect("explicit inactive stream must be accepted"); + for result in &mut observation.executions[0].results { + result.active = Some(true); + } + assert!( + persisted_fixture_snapshot(&entries, &observation).is_err(), + "active stream must not be accepted as a persisted quiescent stream" + ); + } + + fn attempt_template() -> BehaviorCase { + let mut writer = process("writer"); + writer.actions.push(Action::ok(ActionOp::PublishBlob { + path: "/cases/owned/value".to_owned(), + bytes: vec![1], + })); + BehaviorCase { + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + id: "attempt-isolation".to_owned(), + seed: 7, + live_nodes: BTreeSet::from([1, 3]), + topology: TopologyFamily::Fixed, + scenarios: BTreeSet::new(), + routes: Vec::new(), + read_only_fixture_paths: BTreeSet::new(), + resource_bounds: Default::default(), + processes: vec![writer], + failure: FailureInjection::None, + } + } + + #[test] + fn restarted_attempts_never_reuse_paths_or_execution_identity() { + // Exercise restart persistence without coupling the deadline to host disk flushes. + let root = tempfile::tempdir_in("/dev/shm").expect("Linux tmpfs fixture storage"); + let state = root.path().join("state"); + let evidence = root.path().join("evidence"); + let mut cursor = 0; + // Each simulated runner owns a fresh deadline; only its durable claims survive. + let reserve = |state: &Path, evidence: &Path, cursor: &mut u64| { + let budget = Budget::new(Duration::from_secs(5)); + reserve_attempt_identity(state, evidence, cursor, &budget).unwrap() + }; + let first = reserve(&state, &evidence, &mut cursor); + // The runner restarts with an empty in-memory counter. + cursor = 0; + let restarted = reserve(&state, &evidence, &mut cursor); + // Retained fixture, new evidence directory. + cursor = 0; + let relocated = reserve(&state, &root.path().join("new-evidence"), &mut cursor); + // New local fixture, reused evidence directory. + cursor = 0; + let recreated = reserve(&root.path().join("new-state"), &evidence, &mut cursor); + let template = attempt_template(); + for lineage in [[first, restarted, relocated], [first, restarted, recreated]] { + let attempts = lineage.map(|id| template.for_attempt(id, true)); + let paths = attempts + .iter() + .flat_map(BehaviorCase::owned_paths) + .collect::>(); + let requests = attempts + .iter() + .map(|case| (case).execution_request_id(&case.processes[0])) + .collect::>(); + assert_eq!( + paths.len(), + attempts.len(), + "restarted attempts must not share writable paths" + ); + assert_eq!( + requests.len(), + attempts.len(), + "restarted executions must not alias retired requests" + ); + } + } + + #[test] + fn typed_absence_rejects_non_absent_namespace_entries() { + use myelin_control_contract::{RetainedBlobsReply, RetainedNonBlob, SCHEMA_VERSION}; + + let mut reply = RetainedBlobsReply { + schema_version: SCHEMA_VERSION, + request_id: "precondition".to_owned(), + blobs: BTreeMap::new(), + non_blobs: BTreeMap::from([("/cases/absent".to_owned(), RetainedNonBlob::Absent)]), + }; + verify_typed_absence(&reply).unwrap(); + reply.non_blobs.insert( + "/cases/live".to_owned(), + RetainedNonBlob::QuiescentStream { revision: 7 }, + ); + assert!(verify_typed_absence(&reply).is_err()); + } + + fn process(id: &str) -> ProcessProgram { + ProcessProgram { + id: id.to_owned(), + logical_node_id: 3, + access: AccessSpec::unrestricted(format!("execution-{id}")), + depends_on: Vec::new(), + actions: Vec::new(), + } + } + + fn record( + request: &str, + sequence: u64, + event: ContextualProcessEventKind, + ) -> ContextualEventRecord { + ContextualEventRecord { + sequence, + observation: ContextualProcessEvent { + request_id: request.to_owned(), + logical_node_id: 3, + event, + }, + } + } + + fn view( + request: &str, + next: u64, + terminal: bool, + events: Vec, + ) -> ContextualExecutionView { + ContextualExecutionView { + request_id: request.to_owned(), + execution_incarnation: format!("incarnation-{request}"), + logical_node_id: 3, + process: None, + terminal, + truncated_before: 0, + next_sequence: next, + events, + } + } + + fn exited() -> ContextualProcessEventKind { + ContextualProcessEventKind::Exited { + status: ContextualExitStatus::Code(0), + } + } + + fn spawned() -> ContextualProcessEventKind { + ContextualProcessEventKind::Spawned { + process: "actor".to_owned(), + identity: ExecutionIdentity { + execution_id: 1, + generation: 1, + }, + } + } + + #[test] + fn reconnect_replays_prefix_without_duplicating_split_causal_record() { + let mut state = ExecutionTracker::new(&process("reader"), "request".to_owned(), None); + let output = b"{\"process\":\"reader\",\"step\":0,\"action\":\"lookup\",\"path\":\"/cases/x\",\"outcome\":\"ok\"}\n"; + let first = record( + "request", + 0, + ContextualProcessEventKind::Stdout { + bytes: output[..23].to_vec(), + }, + ); + state + .ingest( + &view("request", 1, false, vec![first.clone()]), + Duration::ZERO, + ) + .unwrap(); + assert!( + state.partial_observation().results.is_empty(), + "a torn line is not a causal record" + ); + state + .ingest( + &view( + "request", + 3, + true, + vec![ + first, + record( + "request", + 1, + ContextualProcessEventKind::Stdout { + bytes: output[23..].to_vec(), + }, + ), + record("request", 2, exited()), + ], + ), + Duration::from_millis(1), + ) + .unwrap(); + state.validate_terminal().unwrap(); + let observed = state.partial_observation(); + assert_eq!(observed.stdout.as_bytes(), output); + assert_eq!(observed.results.len(), 1); + assert_eq!(observed.results[0].path, "/cases/x"); + } + + #[test] + fn reused_request_with_equal_cursor_cannot_replace_owned_incarnation() { + let mut state = ExecutionTracker::new(&process("reader"), "request".to_owned(), None); + state + .ingest( + &view("request", 1, true, vec![record("request", 0, exited())]), + Duration::ZERO, + ) + .unwrap(); + let mut replacement = view("request", 1, true, vec![record("request", 0, exited())]); + replacement.execution_incarnation = "replacement".to_owned(); + assert!(state.ingest(&replacement, Duration::ZERO).is_err()); + assert_eq!( + state.execution_incarnation.as_deref(), + Some("incarnation-request") + ); + assert!(!state.acked); + } + + #[test] + fn terminal_flag_cannot_replace_a_missing_terminal_record() { + let mut state = ExecutionTracker::new(&process("reader"), "request".to_owned(), None); + assert!( + state + .ingest(&view("request", 1, true, vec![]), Duration::ZERO) + .is_err() + ); + assert!(!state.partial_observation().terminal); + assert!(state.validate_terminal().is_err()); + } + + #[test] + fn sequence_gap_retains_later_output_but_cannot_acknowledge_success() { + let mut state = ExecutionTracker::new(&process("reader"), "request".to_owned(), None); + let output = b"{\"process\":\"reader\",\"step\":2,\"action\":\"lookup\",\"path\":\"/cases/later\",\"outcome\":\"ok\"}\n"; + assert!( + state + .ingest( + &view( + "request", + 4, + true, + vec![ + record( + "request", + 2, + ContextualProcessEventKind::Stdout { + bytes: output.to_vec(), + } + ), + record("request", 3, exited()), + ] + ), + Duration::ZERO + ) + .is_err() + ); + assert_eq!(state.partial_observation().results[0].path, "/cases/later"); + assert_eq!(state.partial_observation().stdout.as_bytes(), output); + assert!(state.partial_observation().terminal); + assert!(state.validate_terminal().is_err()); + assert!(!state.acked); + } + + #[test] + fn bootstrap_failure_waits_for_reaping_and_preserves_deferred_stop() { + let mut state = ExecutionTracker::new( + &process("held"), + "request".to_owned(), + Some((ProcessStopPhase::DuringBootstrap, Some(0))), + ); + state + .ingest( + &view( + "request", + 2, + false, + vec![ + record( + "request", + 0, + ContextualProcessEventKind::ProcessStarted { pid: 1 }, + ), + record( + "request", + 1, + ContextualProcessEventKind::BootstrapFailed { + error: "injected test failure".to_owned(), + }, + ), + ], + ), + Duration::ZERO, + ) + .unwrap(); + assert!(state.stop_due); + assert!( + !state.terminal, + "bootstrap failure does not prove the child was reaped" + ); + state.stop_requested = true; + state + .ingest( + &view("request", 3, true, vec![record("request", 2, exited())]), + Duration::ZERO, + ) + .unwrap(); + state.validate_terminal().unwrap(); + } + + #[test] + fn observed_acceptance_cancels_only_the_stalled_spawn_reply() { + let owner = Budget::new(Duration::from_secs(2)); + let spawn = owner.child(Duration::from_secs(1)); + let sibling = owner.child(Duration::from_secs(1)); + let mut state = ExecutionTracker::new(&process("accepted"), "request".to_owned(), None); + state.spawn_pending = true; + state.spawn_budget = Some(spawn.clone()); + state + .ingest( + &view("request", 1, false, vec![record("request", 0, spawned())]), + Duration::ZERO, + ) + .unwrap(); + assert!(spawn.check("withheld spawn response").is_err()); + owner.check("attempt collector").unwrap(); + sibling.check("independent healthy sibling").unwrap(); + state + .control_completed(ControlCompletion { + index: 0, + kind: ControlKind::Spawn, + result: Err("canceled reply after observed acceptance".to_owned()), + elapsed: Duration::ZERO, + }) + .unwrap(); + assert!(state.accepted); + assert!(!state.spawn_pending); + } + + #[test] + fn cleanup_requires_each_unique_owned_path_and_inactive_stream_revision() { + let paths = vec!["/cases/a".to_owned(), "/cases/b".to_owned()]; + let record = |path: &str, quiescence: Value| { + json!({ + "type": "path_cleanup", "path": path, "attempted": true, + "absent": true, "error": null, "quiescence": quiescence, + }) + .to_string() + }; + let a = record("/cases/a", Value::Null); + let b = record("/cases/b", json!({"revision": 7, "active": false})); + assert!(verify_cleanup_paths(&paths, &a).is_err()); + assert!(verify_cleanup_paths(&paths, &format!("{a}\n{a}\n{b}")).is_err()); + assert!( + verify_cleanup_paths( + &paths, + &format!( + "{a}\n{}", + record("/cases/b", json!({"revision": 7, "active": true})) + ) + ) + .is_err() + ); + verify_cleanup_paths(&paths, &format!("{a}\n{b}")).unwrap(); + } + + #[test] + fn expected_branch_failure_does_not_mask_failed_sibling() { + let failed = process("failed"); + let healthy = process("healthy"); + let case = BehaviorCase { + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + id: "branch".to_owned(), + seed: 1, + live_nodes: BTreeSet::from([1, 3]), + topology: TopologyFamily::Fixed, + scenarios: BTreeSet::new(), + routes: Vec::new(), + read_only_fixture_paths: BTreeSet::new(), + resource_bounds: Default::default(), + processes: vec![failed.clone(), healthy.clone()], + failure: FailureInjection::LaunchFailure { + process: failed.id.clone(), + kind: LaunchFailureKind::MissingExecutable, + }, + }; + assert!(expected_process_failure(&case, &failed.id)); + assert!(!expected_process_failure(&case, &healthy.id)); + let mut failed_state = ExecutionTracker::new(&failed, "failed".to_owned(), None); + failed_state + .ingest( + &view( + "failed", + 1, + true, + vec![record( + "failed", + 0, + ContextualProcessEventKind::SpawnFailed { + error: "injected test failure".to_owned(), + }, + )], + ), + Duration::ZERO, + ) + .unwrap(); + let mut sibling = ExecutionTracker::new(&healthy, "healthy".to_owned(), None); + sibling + .ingest( + &view("healthy", 1, true, vec![record("healthy", 0, exited())]), + Duration::ZERO, + ) + .unwrap(); + sibling.validate_terminal().unwrap(); + assert!(sibling.partial_observation().exit_success); + assert!(!failed_state.partial_observation().exit_success); + } + + #[test] + fn canceled_stalled_http_joins_and_preserves_final_channel_outcome() { + let listener = std::net::TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let address = listener.local_addr().unwrap(); + let (accepted, accepted_rx) = mpsc::channel(); + let (release, release_rx) = mpsc::channel(); + let server = thread::spawn(move || { + let (_socket, _) = listener.accept().unwrap(); + accepted.send(()).unwrap(); + let _ = release_rx.recv_timeout(Duration::from_secs(2)); + }); + let budget = Budget::new(Duration::from_secs(2)); + let receiver = thread::scope(|scope| { + let cancel_before_join = CancelOnDrop(budget.clone()); + let (sender, receiver) = mpsc::channel(); + let worker_budget = &budget; + scope.spawn(move || { + let result = http_json_budget( + "POST", + &format!("http://{address}/stalled-spawn"), + None, + worker_budget, + ); + sender.send(result).unwrap(); + }); + accepted_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + drop(cancel_before_join); + receiver + }); + assert!( + receiver + .recv_timeout(Duration::from_millis(100)) + .unwrap() + .is_err() + ); + release.send(()).unwrap(); + server.join().unwrap(); + } + + struct PeerChild(std::process::Child); + + impl Drop for PeerChild { + fn drop(&mut self) { + let _ = self.0.kill(); + self.0.wait().expect("reap owned loopback child"); + } + } + + fn collector_peer_harness(artifacts: &Path, address: std::net::SocketAddr) -> ClusterHarness { + use std::os::fd::FromRawFd; + use std::sync::{Mutex, atomic::AtomicU64}; + + let mut orchestrator = std::process::Command::new("sleep") + .arg("30") + .spawn() + .unwrap(); + let fd = unsafe { libc::syscall(libc::SYS_pidfd_open, orchestrator.id(), 0) as i32 }; + if fd < 0 { + let _ = orchestrator.kill(); + orchestrator.wait().unwrap(); + panic!("pin collector peer process"); + } + ClusterHarness { + config: super::super::ClusterHarnessConfig { + workspace: artifacts.to_owned(), + artifacts: artifacts.to_owned(), + node_count: 2, + seed: 1, + image: None, + build_image: false, + deadline: Duration::from_secs(1), + provider: super::super::HarnessProvider::LocalMock, + selected_offer_ids: Vec::new(), + state_dir: None, + reset_state: false, + adopt_only: false, + offer_search_id: None, + provision: false, + relay_url: None, + run_id: 1, + }, + execution_budget: Budget::new(Duration::from_secs(1)), + cleanup_budget: None, + fixture_cleanup_budget: Budget::new(Duration::from_secs(10)), + binaries: crate::resources::resource_deadline_tests::unused_binaries_for_control_peer(), + provider_client: None, + lifecycle_started: Instant::now(), + observation_generation: AtomicU64::new(0), + health_boundary: AtomicU64::new(0), + base_url: format!("http://{address}"), + state_dir: artifacts.to_owned(), + dashboard_port: address.port(), + image: String::new(), + container_prefix: String::new(), + telemetry: artifacts.join("telemetry"), + telemetry_census: Mutex::new(Default::default()), + provider_baseline: BTreeSet::new(), + fixture_mapping: BTreeMap::new(), + stopped_nodes: BTreeSet::new(), + provider_accounting_baseline: None, + node_generation_baseline: BTreeMap::new(), + fixture_blob_baseline: None, + fixture_actor_baseline: BTreeSet::new(), + verified_local_baseline: None, + orchestrator, + orchestrator_pidfd: unsafe { std::os::fd::OwnedFd::from_raw_fd(fd) }, + stdout: Default::default(), + stderr: Default::default(), + node_ids: vec![1, 3], + next_attempt_id: 0, + last_attempt_id: None, + pending_attempts: BTreeMap::new(), + active_attempts: BTreeMap::new(), + pending_control_processes: Mutex::new(BTreeSet::new()), + last_failure_signature: None, + quarantine_reason: Mutex::new(None), + // No provider resources exist; Drop still kills and reaps the child. + torn_down: true, + } + } + + #[test] + fn remote_retention_reaps_after_injected_exit_or_execution_cancellation() { + for already_exited in [true, false] { + let root = tempfile::tempdir().unwrap(); + let mut harness = collector_peer_harness(root.path(), ([127, 0, 0, 1], 9).into()); + harness.config.provider = super::super::HarnessProvider::StaticSsh { + manifest: root.path().join("unused-manifest"), + identity: root.path().join("unused-identity"), + bundle: root.path().join("unused-bundle"), + }; + let pid = harness.orchestrator.id() as libc::pid_t; + if already_exited { + harness.orchestrator.kill().unwrap(); + super::super::wait_for_child( + &mut harness.orchestrator, + &harness.fixture_cleanup_budget, + "observe injected orchestrator exit", + ) + .unwrap(); + } else { + harness.execution_budget.cancel(); + } + harness.retain_remote_fixture(true).unwrap(); + let mut status = 0; + assert_eq!( + unsafe { libc::waitpid(pid, &mut status, libc::WNOHANG) }, + -1 + ); + assert_eq!( + std::io::Error::last_os_error().raw_os_error(), + Some(libc::ECHILD) + ); + } + } + + #[test] + fn withheld_terminal_event_expires_after_healthy_sibling_and_drains_remaining_stops() { + use std::io::Read; + use std::net::TcpListener; + use std::process::{Command, Stdio}; + + // Keep real evidence fsyncs, but isolate control deadlines from host disk latency. + let root = tempfile::tempdir_in("/dev/shm").expect("Linux tmpfs fixture storage"); + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let address = listener.local_addr().unwrap(); + listener.set_nonblocking(true).unwrap(); + let mut case = attempt_template(); + case.processes = ["withheld", "healthy", "cleanup"] + .map(|name| { + let mut program = process(name); + program.actions.push(Action::ok(ActionOp::Lookup { + path: "/cases/read-only".to_owned(), + expected_kind: "blob".to_owned(), + })); + program + }) + .to_vec(); + case.read_only_fixture_paths + .insert("/cases/read-only".to_owned()); + let requests = case + .processes + .iter() + .map(|process| (process.id.clone(), (&case).execution_request_id(process))) + .collect::>(); + let attempt = Arc::new(CaseAttempt::prepare(1, root.path(), case).unwrap()); + let mut harness = collector_peer_harness(root.path(), address); + let server_budget = Budget::new(Duration::from_secs(8)); + let started = Instant::now(); + let (result, stops, acknowledgements, reaped) = thread::scope(|scope| { + let owner = &server_budget; + let server = scope.spawn(move || { + let mut children = BTreeMap::::new(); + let mut stops = BTreeSet::new(); + let mut acknowledgements = BTreeSet::new(); + let mut reaped = BTreeSet::new(); + 'requests: while owner.check("loopback collector peer").is_ok() { + let (mut socket, _) = match listener.accept() { + Ok(connection) => connection, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + let _ = owner.wait(Duration::from_millis(1), "collector request"); + continue; + } + Err(error) => panic!("accept collector request: {error}"), + }; + socket + .set_read_timeout(Some(Duration::from_secs(1))) + .unwrap(); + socket + .set_write_timeout(Some(Duration::from_secs(1))) + .unwrap(); + let mut header = Vec::new(); + let mut byte = [0_u8; 1]; + while !header.ends_with(b"\r\n\r\n") { + if socket.read_exact(&mut byte).is_err() { + continue 'requests; + } + header.push(byte[0]); + assert!(header.len() < 65536, "bounded HTTP header"); + } + let header = String::from_utf8(header).unwrap(); + let path = header.split_whitespace().nth(1).unwrap(); + let length = header + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().unwrap()) + }) + .unwrap_or(0); + assert!(length < 1024 * 1024, "bounded HTTP body"); + let mut body = vec![0; length]; + if socket.read_exact(&mut body).is_err() { + continue 'requests; + } + let body: Value = serde_json::from_slice(&body).unwrap(); + let reply = if path.ends_with("/spawn") { + let request = body["request_id"].as_str().unwrap().to_owned(); + let label = body["label"].as_str().unwrap().to_owned(); + let child = if label == "healthy" { + Command::new("true").spawn().unwrap() + } else { + Command::new("sleep") + .arg("30") + .stdin(Stdio::null()) + .spawn() + .unwrap() + }; + assert!( + children + .insert(request.clone(), (label.clone(), PeerChild(child))) + .is_none() + ); + serde_json::to_value(ContextualControlReply::Event { + observation: ContextualProcessEvent { + request_id: request, + logical_node_id: 3, + event: ContextualProcessEventKind::Spawned { + process: label, + identity: ExecutionIdentity { + execution_id: 1, + generation: 1, + }, + }, + }, + }) + .unwrap() + } else if path.ends_with("/events") { + let mut executions = Vec::new(); + let mut missing = Vec::new(); + for cursor in body["payload"]["cursors"].as_array().unwrap() { + let request = cursor["request_id"].as_str().unwrap(); + let Some((label, child)) = children.get_mut(request) else { + missing.push(request.to_owned()); + continue; + }; + let status = child.0.try_wait().unwrap(); + if status.is_some() { + reaped.insert(request.to_owned()); + } + // The child really exits/reaps on stop, but this peer + // independently withholds its terminal cursor forever. + let terminal = label != "withheld" && status.is_some(); + let mut events = vec![record(request, 0, spawned())]; + if terminal { + use std::os::unix::process::ExitStatusExt; + let status = status.unwrap(); + let status = match status.code() { + Some(code) => ContextualExitStatus::Code(code), + None => status.signal().map_or( + ContextualExitStatus::Unknown, + ContextualExitStatus::Signal, + ), + }; + events.push(record( + request, + 1, + ContextualProcessEventKind::Exited { status }, + )); + } + let next = events.len() as u64; + let after = cursor["after_sequence"].as_u64().unwrap(); + events.retain(|record| record.sequence >= after); + executions.push(view(request, next, terminal, events)); + } + serde_json::to_value(Versioned::new(ContextualControlReply::EventsBatch( + ContextualEventsBatch { + executions, + missing, + }, + ))) + .unwrap() + } else if path.ends_with("/stop") { + let request = path + .trim_start_matches("/api/control/contextual/") + .trim_end_matches("/stop"); + let (label, child) = children.get_mut(request).unwrap(); + child.0.kill().unwrap(); + child.0.wait().unwrap(); + reaped.insert(request.to_owned()); + stops.insert(request.to_owned()); + serde_json::to_value(ContextualControlReply::Event { + observation: ContextualProcessEvent { + request_id: request.to_owned(), + logical_node_id: 3, + event: ContextualProcessEventKind::StopAccepted { + process: label.clone(), + }, + }, + }) + .unwrap() + } else if path.ends_with("/ack") { + acknowledgements.insert( + path.trim_start_matches("/api/control/contextual/") + .trim_end_matches("/ack") + .to_owned(), + ); + serde_json::to_value(Versioned::new(ContextualControlReply::Acknowledged( + ContextualEventsAcknowledgement { + request_id: path + .trim_start_matches("/api/control/contextual/") + .trim_end_matches("/ack") + .to_owned(), + execution_incarnation: body["payload"]["execution_incarnation"] + .as_str() + .unwrap() + .to_owned(), + through_sequence: body["payload"]["through_sequence"] + .as_u64() + .unwrap(), + }, + ))) + .unwrap() + } else { + panic!("unexpected collector request: {path}"); + }; + let reply = reply.to_string(); + let _ = write!( + socket, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n{reply}", + reply.len() + ); + } + // PeerChild reaps all children even if the client panics. + (stops, acknowledgements, reaped) + }); + let cancel_before_join = CancelOnDrop(server_budget.clone()); + let cleanup = Budget::new(Duration::from_secs(3)); + let result = harness.execute_case(&attempt, true, &cleanup); + drop(cancel_before_join); + let (stops, acknowledgements, reaped) = server.join().unwrap(); + (result, stops, acknowledgements, reaped) + }); + let elapsed = started.elapsed(); + let error = result.unwrap_err(); + assert!(elapsed < Duration::from_secs(5), "{elapsed:?}: {error}"); + assert!(error.contains("budget exhausted"), "{error}"); + assert!(harness.is_quarantined()); + assert!(harness.last_failure_signature().is_none()); + assert_eq!( + stops, + BTreeSet::from([requests["withheld"].clone(), requests["cleanup"].clone()]) + ); + assert_eq!( + acknowledgements, + BTreeSet::from([requests["healthy"].clone(), requests["cleanup"].clone()]) + ); + assert_eq!(reaped, requests.values().cloned().collect::>()); + assert_eq!( + *harness.pending_control_processes.lock().unwrap(), + BTreeSet::from([requests["withheld"].clone()]) + ); + let pending: Value = serde_json::from_slice( + &fs::read(attempt.directory.join("pending-executions.json")).unwrap(), + ) + .unwrap(); + let withheld = pending["executions"] + .as_array() + .unwrap() + .iter() + .find(|execution| execution["process"] == "withheld") + .unwrap(); + assert_eq!(withheld["request_id"], requests["withheld"]); + assert_eq!(withheld["next_sequence"], 1); + assert_eq!(withheld["terminal"], false); + assert_eq!(withheld["acknowledged"], false); + assert_eq!(withheld["gap"], Value::Null); + let wire = fs::read_to_string(attempt.directory.join("execution-events.jsonl")).unwrap(); + let reconciliations = wire + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .filter(|record| { + record + .pointer("/response/payload/executions") + .and_then(Value::as_array) + .is_some_and(|executions| { + executions.iter().any(|execution| { + execution["request_id"] == requests["withheld"] + && execution["terminal"] == false + }) + }) + }) + .count(); + assert!( + reconciliations >= 2, + "the collector must reconcile a live cursor repeatedly, not fail one stalled HTTP request" + ); + let observed: CaseObservation = + serde_json::from_slice(&fs::read(attempt.directory.join("observed.json")).unwrap()) + .unwrap(); + assert!( + observed + .executions + .iter() + .find(|execution| execution.process == "healthy") + .unwrap() + .exit_success + ); + let orchestrator_pid = harness.orchestrator.id() as libc::pid_t; + drop(harness); + let mut status = 0; + assert_eq!( + unsafe { libc::waitpid(orchestrator_pid, &mut status, libc::WNOHANG) }, + -1 + ); + assert_eq!( + std::io::Error::last_os_error().raw_os_error(), + Some(libc::ECHILD), + "the existing harness owner must reap its child, not just signal it" + ); + } +} diff --git a/tools/myelin-e2e-fuzz/src/harness/convergence.rs b/tools/myelin-e2e-fuzz/src/harness/convergence.rs index 3a0f623..f4ca914 100644 --- a/tools/myelin-e2e-fuzz/src/harness/convergence.rs +++ b/tools/myelin-e2e-fuzz/src/harness/convergence.rs @@ -1,143 +1,284 @@ //! Cluster convergence: dashboard, provisioning, and pairwise functional proof. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::thread; -use std::time::{Instant, SystemTime, UNIX_EPOCH}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use serde_json::{Value, json}; -use crate::codegen::{digest, render_python}; +use crate::budget::Budget; use crate::harness::ClusterHarness; -use crate::ir::{AccessSpec, Action, ActionOp, ExecutionObservation, ProcessProgram}; -use crate::oracle::BehaviorOracle; -use crate::resources::{http_json, transient_actor_identities}; +use crate::ir::{ + AccessSpec, Action, ActionOp, BehaviorCase, CASE_SCHEMA_VERSION, CaseResourceBounds, + FailureInjection, GENERATOR_VERSION, ProcessProgram, TopologyFamily, +}; +use crate::resources::http_json_budget; use super::POLL_INTERVAL; impl ClusterHarness { pub(super) fn wait_for_dashboard(&mut self) -> Result<(), String> { - let deadline = Instant::now() + self.config.deadline; - loop { - self.ensure_orchestrator_live()?; - if let Ok(status) = http_json( - "GET", - &format!("{}/api/control/status", self.base_url), - None, - ) { - let configured_image = status - .pointer("/Status/provider/runtime_image") - .and_then(Value::as_str) - .ok_or_else(|| { - format!("control status omitted configured runtime image: {status}") - })?; - if configured_image != self.image { - return Err(format!( - "control status runtime image {configured_image:?} does not match harness image {:?}", - self.image - )); + let started = Instant::now(); + let budget = self.operation_budget().child(self.config.deadline); + let result = (|| { + let mut pending = "dashboard control status and actors".to_owned(); + loop { + budget.check(&pending)?; + self.ensure_orchestrator_live()?; + let request_budget = budget.child(Duration::from_secs(2)); + match http_json_budget( + "GET", + &format!("{}/api/control/status", self.base_url), + None, + &request_budget, + ) { + Ok(status) => { + let provider = status + .pointer("/Status/provider") + .ok_or_else(|| format!("control status omitted provider: {status}"))?; + let configured_image = provider + .get("runtime_image") + .and_then(Value::as_str) + .ok_or_else(|| { + format!("control status omitted configured runtime image: {status}") + })?; + if configured_image != self.image { + return Err(format!( + "control runtime image {configured_image:?} does not match {:?}", + self.image + )); + } + let readiness = + provider + .get("kind") + .and_then(Value::as_str) + .ok_or_else(|| { + format!("control status omitted provider readiness: {status}") + })?; + if readiness != "ready" { + pending = format!( + "provider readiness {readiness}: {}", + provider + .get("error") + .and_then(Value::as_str) + .unwrap_or("validation pending") + ); + } else { + match http_json_budget( + "GET", + &format!("{}/api/control/actors", self.base_url), + None, + &request_budget, + ) { + Ok(_) => return Ok(()), + Err(error) => pending = format!("dashboard actor control: {error}"), + } + } + } + Err(error) => pending = format!("dashboard status: {error}"), } - return Ok(()); + // Startup has no fleet-event subscription until the dashboard binds. + budget.wait(POLL_INTERVAL, &pending)?; } - if Instant::now() >= deadline { - return Err(self.timeout_evidence("dashboard readiness")); - } - thread::sleep(POLL_INTERVAL); - } + })(); + self.record_lifecycle("dashboard_readiness", started, &result)?; + result } - pub(super) fn provision_nodes(&mut self) -> Result<(), String> { - let command_id = format!("e2e-provision-{}", self.config.seed); - let selected_offer_ids = (1..=u64::from(self.config.node_count)).collect::>(); - http_json( + pub(super) fn dispatch_provision(&self) -> Result<(), String> { + let response = http_json_budget( "POST", &format!("{}/api/control/provision", self.base_url), Some(json!({ - "command_id": command_id, + "command_id": format!("e2e-provision-{}", self.config.seed), "count": self.config.node_count, - "selected_offer_ids": selected_offer_ids, + "selected_offer_ids": self.config.selected_offer_ids, + "search_id": self.config.offer_search_id, })), + &self.operation_budget().child(self.config.deadline), )?; - let deadline = Instant::now() + self.config.deadline; - loop { - self.ensure_orchestrator_live()?; - let fleet = http_json("GET", &format!("{}/api/control/fleet", self.base_url), None)?; - let nodes = fleet - .get("FleetStatus") - .and_then(|fleet| fleet.get("nodes")) - .and_then(Value::as_array); - let running = nodes - .into_iter() - .flatten() - .filter(|node| node.get("phase").and_then(Value::as_str) == Some("running")) - .filter_map(|node| node.get("logical_node_id").and_then(Value::as_u64)) - .collect::>(); - if running.len() == usize::from(self.config.node_count) { - self.node_ids = running; - self.node_ids.sort_unstable(); - return Ok(()); - } - if nodes.is_some_and(|nodes| { - nodes.len() == usize::from(self.config.node_count) - && nodes.iter().all(|node| { - node.get("phase").and_then(Value::as_str) == Some("stopped") - && !node.get("last_error").is_none_or(Value::is_null) - }) - }) { - return Err(format!("worker provisioning failed: {fleet}")); - } - if Instant::now() >= deadline { - return Err(self.timeout_evidence(&format!( - "{} workers running; fleet={fleet}", - self.config.node_count - ))); - } - - thread::sleep(POLL_INTERVAL); + let admitted = response + .pointer("/Accepted/node_ids") + .and_then(Value::as_array) + .is_some_and(|nodes| nodes.len() == usize::from(self.config.node_count)); + if !admitted { + return Err(format!( + "provision command did not durably admit the exact node count: {response}" + )); } + Ok(()) + } + + pub(super) fn provision_nodes(&mut self, provision: bool) -> Result<(), String> { + let started = Instant::now(); + let budget = self.operation_budget().child(self.config.deadline); + let result = (|| { + if provision { + self.dispatch_provision()?; + } + let expected = if self.node_ids.is_empty() { + (1..=u64::from(self.config.node_count)).collect() + } else { + self.node_ids.clone() + }; + loop { + budget.check(&format!("exact running nodes {expected:?}"))?; + self.ensure_orchestrator_live()?; + let cursor = self.control_revision(&budget)?; + let fleet = http_json_budget( + "GET", + &format!("{}/api/control/fleet", self.base_url), + None, + &budget, + )?; + let nodes = fleet + .pointer("/FleetStatus/nodes") + .and_then(Value::as_array) + .ok_or_else(|| format!("fleet omitted node census: {fleet}"))?; + let expected_running = expected.iter().copied().collect::>(); + if expected_running.len() != expected.len() || expected_running.contains(&0) { + return Err(format!("invalid expected logical-node set {expected:?}")); + } + let mut seen = BTreeSet::new(); + let mut running = Vec::new(); + let mut stopped = BTreeSet::new(); + for node in nodes { + let node_id = node["logical_node_id"] + .as_u64() + .filter(|node_id| *node_id != 0) + .ok_or_else(|| format!("fleet node omitted valid identity: {node}"))?; + if !seen.insert(node_id) { + return Err(format!("fleet duplicated logical node {node_id}")); + } + if !expected_running.contains(&node_id) + && !self.stopped_nodes.contains(&node_id) + { + return Err(format!( + "unexpected replacement or bootstrap logical node {node_id}: {fleet}" + )); + } + if self.stopped_nodes.contains(&node_id) { + if node["phase"] == "stopped" { + stopped.insert(node_id); + } else { + return Err(format!( + "removed logical node {node_id} left stopped phase: {node}" + )); + } + } else if node["phase"] == "running" { + running.push(node_id); + } else if provisioning_failed(std::slice::from_ref(node)) { + return Err(format!("worker provisioning failed: {fleet}")); + } + } + running.sort_unstable(); + if running == expected && stopped == self.stopped_nodes { + self.node_ids = running; + return Ok(()); + } + self.wait_for_control_change( + &cursor, + &budget, + &format!("running nodes {expected:?}; observed={fleet}"), + )?; + } + })(); + self.record_lifecycle("fleet_convergence", started, &result)?; + result } pub fn verify_workload_convergence(&mut self) -> Result<(), String> { self.wait_contextual_control()?; - if self.resource_baseline.is_empty() { - let baseline = self.health_snapshot()?; - self.resource_baseline = transient_actor_identities(&baseline); + if self.provider_baseline.is_empty() { + self.record_health_baseline()?; } - self.verify_ordered_pair_communication() + self.verify_ordered_pair_communication()?; + // The reusable fixture begins only after the readiness workload and + // every owned readiness resource have been cleaned and re-observed. + self.record_health_baseline() } - fn wait_contextual_control(&mut self) -> Result<(), String> { - for node in self.node_ids.clone() { - let deadline = Instant::now() + self.config.deadline; - let mut attempt = 0_u64; + pub(super) fn wait_contextual_control(&mut self) -> Result<(), String> { + let started = Instant::now(); + let budget = self.operation_budget().child(self.config.deadline); + let result = (|| { loop { + budget.check("exact concurrent contextual readiness")?; self.ensure_orchestrator_live()?; - attempt = attempt.saturating_add(1); - let control_request_id = - format!("convergence-{}-{node}-{attempt}", self.config.seed); - let response = http_json( - "GET", - &format!( - "{}/api/control/contextual/nodes/{node}?control_request_id={control_request_id}", - self.base_url - ), - None, - ); - if response.as_ref().is_ok_and(|reply| { - reply - .pointer("/observation/event/type") - .and_then(Value::as_str) - == Some("live_executions") - }) { - break; + let cursor = self.control_revision(&budget)?; + match self.query_contextual_nodes(&self.node_ids, &budget) { + Ok(_) => return Ok(()), + Err(error) => self.wait_for_control_change(&cursor, &budget, &error)?, } - if Instant::now() >= deadline { - return Err(self.timeout_evidence(&format!( - "contextual process control reachable on node {node}; last={response:?}" - ))); - } - thread::sleep(POLL_INTERVAL); } + })(); + self.record_lifecycle("contextual_readiness", started, &result)?; + result + } + + pub(super) fn query_contextual_nodes( + &self, + nodes: &[u64], + budget: &Budget, + ) -> Result, String> { + let expected = nodes.iter().copied().collect::>(); + if nodes.is_empty() || expected.len() != nodes.len() || expected.contains(&0) { + return Err(format!( + "contextual readiness requires exact, unique live logical nodes; observed {nodes:?}" + )); } - Ok(()) + let generation = self.next_observation_generation(); + thread::scope(|scope| { + let requests = nodes + .iter() + .map(|&node| { + let base_url = &self.base_url; + scope.spawn(move || { + let mut attempt = 0_u64; + loop { + budget.check(&format!("contextual health for node {node}"))?; + let request_id = + format!("health-{}-{generation}-{node}-{attempt}", self.config.seed); + let request_budget = budget.child(Duration::from_secs(5)); + match http_json_budget( + "GET", + &format!( + "{base_url}/api/control/contextual/nodes/{node}?control_request_id={request_id}" + ), + None, + &request_budget, + ) { + Ok(response) => { + return validate_contextual_reply(&response, node, &request_id); + } + Err(error) => { + attempt = attempt.saturating_add(1); + budget.wait( + POLL_INTERVAL, + &format!( + "contextual health for node {node}; retry after {error}" + ), + )?; + } + } + } + }) + }) + .collect::>(); + let mut responses = Vec::with_capacity(requests.len()); + let mut errors = Vec::new(); + for request in requests { + match request.join() { + Ok(Ok(response)) => responses.push(response), + Ok(Err(error)) => errors.push(error), + Err(_) => errors.push("contextual readiness collector panicked".to_owned()), + } + } + if errors.is_empty() { + Ok(responses) + } else { + Err(errors.join("; ")) + } + }) } /// Prove functional communication across every ordered node pair. @@ -153,29 +294,80 @@ impl ClusterHarness { .duration_since(UNIX_EPOCH) .map_err(|error| format!("system clock precedes epoch: {error}"))? .as_millis(); - let nodes = self.node_ids.clone(); - let pair_path = |source: u64, destination: u64| { - format!("/cases/convergence/{run_nonce}/{source}-to-{destination}") + let case = Self::convergence_case(&self.node_ids, run_nonce)?; + self.run_case(&case).map(|_| ()) + } + + pub(crate) fn convergence_case(nodes: &[u64], run_nonce: u128) -> Result { + if nodes.is_empty() { + return Err("convergence requires live nodes".to_owned()); + } + let blob_path = |source: u64, destination: u64| { + format!("/cases/convergence/{run_nonce}/blob-{source}-to-{destination}") }; - let mut payloads = BTreeMap::new(); + let stream_path = |source: u64, destination: u64| { + format!("/cases/convergence/{run_nonce}/stream-{source}-to-{destination}") + }; + let ready_path = |process: &str| format!("/cases/convergence/{run_nonce}/ready/{process}"); + let release_path = format!("/cases/convergence/{run_nonce}/release"); + let boundary_lengths = [0_usize, 1, 4_095, 4_096, 4_097, 65_537]; + let mut blob_payloads = BTreeMap::new(); + let mut stream_payloads = BTreeMap::new(); let mut programs = Vec::new(); - for &source in &nodes { - let mut actions = Vec::new(); - for &destination in &nodes { - if source == destination { - continue; - } - let path = pair_path(source, destination); - let payload = format!("myelin-e2e-convergence:{run_nonce}:{source}:{destination}") - .into_bytes(); + for (source_index, &source) in nodes.iter().enumerate() { + let process_id = format!("convergence-writer-{source}"); + let mut actions = vec![ + Action::ok(ActionOp::PublishBlob { + path: ready_path(&process_id), + bytes: Vec::new(), + }), + Action::ok(ActionOp::AwaitEntry { + path: release_path.clone(), + expected_kind: "blob".to_owned(), + }), + ]; + for round in 1..nodes.len() { + let destination = nodes[(source_index + round) % nodes.len()]; + let pair_index = source_index * (nodes.len() - 1) + round - 1; + let length = boundary_lengths[pair_index % boundary_lengths.len()]; + let body = (0..length) + .map(|offset| { + (u128::from(source) * 17 + u128::from(destination) * 31 + offset as u128) + as u8 + }) + .collect::>(); + let blob = blob_path(source, destination); actions.push(Action::ok(ActionOp::PublishBlob { - path: path.clone(), - bytes: payload.clone(), + path: blob.clone(), + bytes: body.clone(), })); - payloads.insert(path, payload); + blob_payloads.insert(blob, body.clone()); + + let mut framed = Vec::with_capacity(body.len() + 8); + let split = body.len() / 2; + for frame in [&body[..split], &body[split..]] { + framed.extend_from_slice(&(frame.len() as u32).to_be_bytes()); + framed.extend_from_slice(frame); + } + let stream = stream_path(source, destination); + let mut cuts = vec![0, 1, 3, 257, framed.len().saturating_sub(1), framed.len()]; + cuts.retain(|cut| *cut <= framed.len()); + cuts.sort_unstable(); + cuts.dedup(); + let chunks = cuts + .windows(2) + .filter(|range| range[0] != range[1]) + .map(|range| framed[range[0]..range[1]].to_vec()) + .collect(); + actions.push(Action::ok(ActionOp::StreamWrite { + path: stream.clone(), + chunks, + replace: false, + })); + stream_payloads.insert(stream, framed); } programs.push(ProcessProgram { - id: format!("convergence-writer-{source}"), + id: process_id, logical_node_id: source, access: AccessSpec::unrestricted(format!( "convergence-writer-{run_nonce}-{source}" @@ -184,21 +376,43 @@ impl ClusterHarness { actions, }); } - for &destination in &nodes { - let mut actions = Vec::new(); - for &source in &nodes { - if source == destination { - continue; - } - let path = pair_path(source, destination); - actions.push(Action::ok(ActionOp::ReadBlob { - path: path.clone(), - expected: payloads[&path].clone(), + for (destination_index, &destination) in nodes.iter().enumerate() { + let process_id = format!("convergence-reader-{destination}"); + let mut actions = vec![ + Action::ok(ActionOp::PublishBlob { + path: ready_path(&process_id), + bytes: Vec::new(), + }), + Action::ok(ActionOp::AwaitEntry { + path: release_path.clone(), + expected_kind: "blob".to_owned(), + }), + ]; + for round in 1..nodes.len() { + let source = nodes[(destination_index + nodes.len() - round) % nodes.len()]; + let blob = blob_path(source, destination); + actions.push(Action::ok(ActionOp::AwaitEntry { + path: blob.clone(), + expected_kind: "blob".to_owned(), })); - actions.push(Action::ok(ActionOp::Unlink { path })); + actions.push(Action::ok(ActionOp::ReadBlob { + path: blob.clone(), + expected: blob_payloads[&blob].clone(), + })); + actions.push(Action::ok(ActionOp::Unlink { path: blob })); + + let stream = stream_path(source, destination); + actions.push(Action::ok(ActionOp::StreamRead { + path: stream.clone(), + expected: stream_payloads[&stream].clone(), + })); + actions.push(Action::ok(ActionOp::WaitForQuiescent { + path: stream.clone(), + })); + actions.push(Action::ok(ActionOp::Unlink { path: stream })); } programs.push(ProcessProgram { - id: format!("convergence-reader-{destination}"), + id: process_id, logical_node_id: destination, access: AccessSpec::unrestricted(format!( "convergence-reader-{run_nonce}-{destination}" @@ -207,80 +421,103 @@ impl ClusterHarness { actions, }); } - for program in &programs { - let request_id = format!("convergence-{run_nonce}-{}", program.id); - self.spawn_program(program, &request_id, &render_python(program), false, None)?; - let observation = self.wait_execution(program, &request_id, None)?; - verify_convergence_execution(program, &observation) - .map_err(|error| format!("ordered-pair convergence proof: {error}"))?; - } - // Convergence processes must be fully reclaimed before the resource - // baseline is captured, otherwise their transient actors would be - // grandfathered into every later health assertion. - self.assert_healthy()?; - Ok(()) + let participant_ready_paths = programs + .iter() + .map(|program| ready_path(&program.id)) + .collect::>(); + programs.push(ProcessProgram { + id: "zz-convergence-release".to_owned(), + logical_node_id: nodes[0], + access: AccessSpec::unrestricted(format!("convergence-release-{run_nonce}")), + depends_on: Vec::new(), + actions: participant_ready_paths + .into_iter() + .map(|path| { + Action::ok(ActionOp::AwaitEntry { + path, + expected_kind: "blob".to_owned(), + }) + }) + .chain(std::iter::once(Action::ok(ActionOp::PublishBlob { + path: release_path, + bytes: Vec::new(), + }))) + .collect(), + }); + let case = BehaviorCase { + schema_version: CASE_SCHEMA_VERSION, + generator_version: GENERATOR_VERSION, + id: format!("convergence-{run_nonce}"), + seed: u64::try_from(run_nonce).unwrap_or(u64::MAX), + live_nodes: nodes.iter().copied().collect(), + topology: TopologyFamily::Fixed, + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: CaseResourceBounds { + max_actions: 256, + max_processes: 20, + max_payload_bytes: 4 * 1024 * 1024, + max_allocation_bytes: 64 * 1024 * 1024, + max_race_states: 65_536, + }, + processes: programs, + failure: FailureInjection::None, + }; + case.validate()?; + Ok(case) } } -fn verify_convergence_execution( - program: &ProcessProgram, - observation: &ExecutionObservation, -) -> Result<(), String> { - BehaviorOracle::verify_execution(observation).map_err(|error| error.to_string())?; - if !observation.exit_success { - return Err(format!( - "process {} on node {} failed with status {:?}\nstdout={}\nstderr={}", - program.id, - program.logical_node_id, - observation.exit_status, - observation.stdout, - observation.stderr - )); - } - if observation.results.len() != program.actions.len() { - return Err(format!( - "process {} expected {} action results, observed {}", - program.id, - program.actions.len(), - observation.results.len() - )); - } - for (step, action) in program.actions.iter().enumerate() { - let result = observation - .results - .iter() - .find(|result| result.step == step) - .ok_or_else(|| format!("process {} omitted step {step}", program.id))?; - if result.outcome != "ok" { - return Err(format!( - "process {} step {step} on node {} reported outcome {:?} (errno {:?}, error {:?})", - program.id, program.logical_node_id, result.outcome, result.errno, result.error - )); - } - if result.path != action.operation.path() { - return Err(format!( - "process {} step {step} reported path {:?} instead of {:?}", - program.id, - result.path, - action.operation.path() - )); - } - let expected_bytes = match &action.operation { - ActionOp::PublishBlob { bytes, .. } - | ActionOp::ReadBlob { - expected: bytes, .. - } => Some(bytes.as_slice()), - _ => None, - }; - if let Some(expected) = expected_bytes - && (result.length != Some(expected.len()) - || result.digest.as_deref() != Some(&digest(expected))) - { - return Err(format!( - "process {} step {step} returned the wrong payload length or digest", - program.id - )); - } - } - Ok(()) +fn validate_contextual_reply( + reply: &Value, + node: u64, + request_id: &str, +) -> Result { + let reply: myelin_control_contract::ContextualHealthReply = + serde_json::from_value(reply.clone()) + .map_err(|error| format!("invalid contextual health reply for node {node}: {error}"))?; + reply.validate(node, request_id)?; + Ok(reply) +} + +fn provisioning_failed(nodes: &[Value]) -> bool { + nodes.iter().any(|node| { + matches!( + node.get("phase").and_then(Value::as_str), + Some("kill_requested" | "stopping" | "stop_failed" | "stopped" | "orphan") + ) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn readiness_rejects_stale_or_wrong_node_replies() { + let reply = crate::resources::resource_deadline_tests::health_reply("fresh-3", 3, 1, 0); + assert!(validate_contextual_reply(&reply, 3, "fresh-3").is_ok()); + assert!(validate_contextual_reply(&reply, 4, "fresh-3").is_err()); + assert!(validate_contextual_reply(&reply, 3, "new-3").is_err()); + assert!(validate_contextual_reply(&json!({}), 3, "fresh-3").is_err()); + let mut unversioned = reply.clone(); + unversioned + .as_object_mut() + .unwrap() + .remove("schema_version"); + assert!(validate_contextual_reply(&unversioned, 3, "fresh-3").is_err()); + let mut rejected = reply.clone(); + rejected["type"] = json!("rejected"); + assert!(validate_contextual_reply(&rejected, 3, "fresh-3").is_err()); + } + + #[test] + fn terminal_node_fails_a_partially_converged_fleet() { + let nodes = vec![ + json!({"logical_node_id": 1, "phase": "joining"}), + json!({"logical_node_id": 2, "phase": "stopped", "last_error": "substrate lost"}), + ]; + assert!(provisioning_failed(&nodes)); + } } diff --git a/tools/myelin-e2e-fuzz/src/harness/mod.rs b/tools/myelin-e2e-fuzz/src/harness/mod.rs index 58687f1..a39b68b 100644 --- a/tools/myelin-e2e-fuzz/src/harness/mod.rs +++ b/tools/myelin-e2e-fuzz/src/harness/mod.rs @@ -2,27 +2,55 @@ mod control; mod convergence; +pub mod raw_fleet; -use std::collections::{BTreeSet, VecDeque}; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::fs; +use std::io::Write; +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::thread; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use crate::budget::Budget; +use crate::ir::BehaviorCase; use serde_json::{Value, json}; +use swactor_vastai::{BlockingVastClient, VastClient}; use crate::resources::{ - TelemetryResourceCensus, build_myelin_binaries, build_workload_image, capture_lines, - contains_poison, http_json, list_containers, pending_resource_cleanup, remove_container, - remove_containers, reserve_port, transient_actor_identities, + BuiltBinaries, TelemetryResourceCensus, build_workload_image, capture_lines, container_census, + contains_poison, docker_image_identity, http_json_budget, list_containers, + pending_resource_cleanup, remove_container, remove_containers, reserve_port, + resolve_myelin_binaries, }; const POLL_INTERVAL: Duration = Duration::from_millis(100); +const TELEMETRY_REFRESH_INTERVAL: Duration = Duration::from_millis(10); pub(crate) const GENERATED_LAUNCHER: &str = "/usr/local/bin/myelin-e2e-python"; +const BUILTIN_FIXTURE_PATH: &str = "/models/tiny-linear/weights"; type CapturedLines = Arc>>; -type SpawnedOrchestrator = (Child, CapturedLines, CapturedLines); +type SpawnedOrchestrator = (Child, OwnedFd, CapturedLines, CapturedLines); + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum HarnessProvider { + LocalMock, + VastAiReal { + ssh_identity: PathBuf, + api_key_env: String, + admission: PathBuf, + /// Optional deployment bundle: when set, the orchestrator refreshes + /// retained nodes through the SSH artifact bootstrap transaction. + bundle: Option, + }, + StaticSsh { + manifest: PathBuf, + identity: PathBuf, + bundle: PathBuf, + }, +} #[derive(Clone, Debug)] pub struct ClusterHarnessConfig { @@ -33,10 +61,60 @@ pub struct ClusterHarnessConfig { pub image: Option, pub build_image: bool, pub deadline: Duration, + pub provider: HarnessProvider, + pub selected_offer_ids: Vec, + pub state_dir: Option, + pub reset_state: bool, + /// Wait for nothing at startup: adopt the retained fixture's nodes and + /// let the caller repair them (in-place agent relaunch) before any + /// running-state wait happens. + pub adopt_only: bool, + /// Orchestrator run identity. Derives the provider labels and container + /// names; a fresh orchestrator state does not persist `cluster.json` + /// until its first command, so the harness passes the run id explicitly + /// instead of reading it back from state that may not exist yet. + pub offer_search_id: Option, + pub provision: bool, + /// Operator-managed iroh relay for providers whose nodes rely on relay + /// transport (static-ssh fixtures isolate node networks). Mirrors the + /// remote deployment's relay configuration. + pub relay_url: Option, + /// Orchestrator run identity. Derives the provider labels and container + /// names; a fresh orchestrator state does not persist `cluster.json` + /// until its first command, so the harness passes the run id explicitly + /// instead of reading it back from state that may not exist yet. + pub run_id: u64, +} +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FixturePathObservation { + pub kind: String, + pub revision: u64, + pub active: bool, + pub length: Option, + pub digest: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FixturePathSnapshot { + pub entries: BTreeMap, +} + +/// An unforgeable handle to one still-owned retained attempt, not arbitrary +/// observed actor identities. Every checkpoint resolves its resources afresh. +pub struct RetainedCaseActors { + attempt: Arc, } pub struct ClusterHarness { config: ClusterHarnessConfig, + pub(crate) execution_budget: Budget, + pub(crate) cleanup_budget: Option, + fixture_cleanup_budget: Budget, + binaries: BuiltBinaries, + provider_client: Option, + lifecycle_started: Instant, + observation_generation: AtomicU64, + health_boundary: AtomicU64, base_url: String, state_dir: PathBuf, dashboard_port: u16, @@ -44,26 +122,41 @@ pub struct ClusterHarness { container_prefix: String, telemetry: PathBuf, telemetry_census: Mutex, - resource_baseline: BTreeSet, + provider_baseline: BTreeSet, + fixture_mapping: BTreeMap, + stopped_nodes: BTreeSet, + provider_accounting_baseline: Option, + node_generation_baseline: BTreeMap, + fixture_blob_baseline: Option<(u64, u64)>, + fixture_actor_baseline: BTreeSet, + verified_local_baseline: Option, orchestrator: Child, + orchestrator_pidfd: OwnedFd, stdout: CapturedLines, stderr: CapturedLines, node_ids: Vec, - orchestrator_stopped: bool, - pre_stop_containers: Vec, + next_attempt_id: u64, + last_attempt_id: Option, + pending_attempts: BTreeMap, + active_attempts: BTreeMap>, + pub(crate) pending_control_processes: Mutex>, + last_failure_signature: Option, + quarantine_reason: Mutex>, torn_down: bool, } fn spawn_orchestrator( config: &ClusterHarnessConfig, + binary: &Path, dashboard_port: u16, state_dir: &Path, image: &str, container_prefix: &str, telemetry: &Path, + budget: &Budget, reset_state: bool, ) -> Result { - let binary = config.workspace.join("target/release/myelin-orchestrator"); + budget.check("spawn retained orchestrator process")?; if !binary.is_file() { return Err(format!( "orchestrator binary is missing: {}", @@ -73,16 +166,87 @@ fn spawn_orchestrator( let mut command = Command::new(binary); command .current_dir(&config.workspace) - .args([ - "--provider", - "vastai", - "--vastai-provisioning", - "mock", - "--image", - image, - "--state-dir", - ]) - .arg(state_dir); + .args(["--provider", "vastai", "--image", image, "--state-dir"]) + .arg(state_dir) + .arg("--run-id") + .arg(config.run_id.to_string()); + match &config.provider { + HarnessProvider::LocalMock => { + command + .args(["--vastai-provisioning", "mock"]) + .env("MYELIN_MOCK_VASTAI_CONTAINER_PREFIX", container_prefix) + .env("MYELIN_DOCKER_GPUS", ""); + } + HarnessProvider::StaticSsh { + manifest, + identity, + bundle, + } => { + if let Some(relay_url) = &config.relay_url { + command.env("MYELIN_IROH_RELAY_URL", relay_url); + } + command + .args(["--provider", "static-ssh"]) + .arg("--static-ssh-manifest") + .arg(manifest) + .arg("--ssh-identity") + .arg(identity) + .arg("--deployment-bundle") + .arg(bundle); + } + HarnessProvider::VastAiReal { + ssh_identity, + api_key_env, + bundle, + admission, + } => { + if let Some(bundle) = bundle { + command.arg("--deployment-bundle").arg(bundle); + } + let api_key = std::env::var(api_key_env).map_err(|_| { + format!("required VastAI credential environment {api_key_env} is unset") + })?; + if api_key.trim().is_empty() { + return Err(format!( + "required VastAI credential environment {api_key_env} is empty" + )); + } + command + .args([ + "--vastai-provisioning", + "real", + "--vastai-require-verified", + "--vastai-bootstrap-command", + "exec /usr/local/bin/myelin-node", + "--vastai-ssh-identity", + ]) + .arg(ssh_identity) + // The app reads VAST_API_KEY before either legacy alias. + .env("VAST_API_KEY", api_key) + .env_remove("MYELIN_VASTAI_API_KEY") + .env_remove("VASTAI_API_KEY") + .env("MYELIN_PAID_FIXTURE_ADMISSION", admission); + } + } + let mut monotonic = libc::timespec { + tv_sec: 0, + tv_nsec: 0, + }; + if unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut monotonic) } != 0 { + return Err(format!( + "read monotonic subprocess deadline: {}", + std::io::Error::last_os_error() + )); + } + let now_ms = (monotonic.tv_sec as u128) * 1_000 + (monotonic.tv_nsec as u128) / 1_000_000; + let deadline_ms = now_ms + + budget + .remaining("orchestrator subprocess ownership")? + .as_millis(); + command.env( + "MYELIN_E2E_EXECUTION_DEADLINE_MONOTONIC_MS", + deadline_ms.to_string(), + ); if reset_state { command.arg("--reset-state"); } @@ -90,14 +254,25 @@ fn spawn_orchestrator( .arg("--telemetry-frame-log") .arg(telemetry) .env("MYELIN_DASHBOARD_PORT", dashboard_port.to_string()) - .env("MYELIN_MOCK_VASTAI_CONTAINER_PREFIX", container_prefix) - .env("MYELIN_DOCKER_GPUS", "") + .env("MYELIN_IROH_BIND_PORT", dashboard_port.to_string()) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); let mut orchestrator = command .spawn() .map_err(|error| format!("start real Myelin orchestrator: {error}"))?; + let pidfd = unsafe { libc::syscall(libc::SYS_pidfd_open, orchestrator.id(), 0) }; + if pidfd < 0 { + let error = std::io::Error::last_os_error(); + let _ = orchestrator.kill(); + let _ = wait_for_child( + &mut orchestrator, + budget, + "reap child without process identity", + ); + return Err(format!("open owned orchestrator process identity: {error}")); + } + let pidfd = unsafe { OwnedFd::from_raw_fd(pidfd as i32) }; let stdout = Arc::new(Mutex::new(VecDeque::new())); let stderr = Arc::new(Mutex::new(VecDeque::new())); capture_lines( @@ -114,13 +289,79 @@ fn spawn_orchestrator( .expect("captured orchestrator stderr"), Arc::clone(&stderr), ); - Ok((orchestrator, stdout, stderr)) + Ok((orchestrator, pidfd, stdout, stderr)) } impl ClusterHarness { pub fn start(config: ClusterHarnessConfig) -> Result { - if config.node_count < 2 || config.node_count > 3 { - return Err("cluster harness requires two or three workers".to_owned()); + let execution = Budget::new(config.deadline); + let cleanup = Budget::new( + config.deadline + Duration::from_secs(crate::campaign::CLEANUP_DEADLINE_SECS), + ); + Self::start_with_budgets(config, execution, cleanup) + } + + pub fn start_with_budgets( + config: ClusterHarnessConfig, + execution: Budget, + cleanup: Budget, + ) -> Result { + Self::start_internal(config, execution, cleanup, true) + } + + /// Dispatch real deployment while keeping boundary injection on the caller's owner. + pub fn start_deployment_unobserved( + config: ClusterHarnessConfig, + budget: Budget, + cleanup: Budget, + ) -> Result { + Self::start_internal(config, budget, cleanup, false) + } + + fn start_internal( + config: ClusterHarnessConfig, + budget: Budget, + fixture_cleanup_budget: Budget, + observe_deployment: bool, + ) -> Result { + let lifecycle_started = Instant::now(); + budget.check("fixture construction")?; + match &config.provider { + HarnessProvider::LocalMock | HarnessProvider::StaticSsh { .. } + if !(2..=5).contains(&config.node_count) => + { + return Err("local cluster harness requires two to five workers".to_owned()); + } + HarnessProvider::VastAiReal { .. } + if config.node_count != 5 + && (config.provision + || config.reset_state + || !(3..=4).contains(&config.node_count)) => + { + return Err( + "paid acquisition requires five workers; retained recovery requires three to five" + .to_owned(), + ); + } + _ => {} + } + if config.provision + && !matches!(&config.provider, HarnessProvider::StaticSsh { .. }) + && config.selected_offer_ids.len() != usize::from(config.node_count) + { + return Err("provisioning requires one exact selected offer per worker".to_owned()); + } + if matches!(&config.provider, HarnessProvider::VastAiReal { .. }) && config.image.is_none() + { + return Err( + "paid VastAI campaign requires an explicit remote runtime image".to_owned(), + ); + } + if matches!(&config.provider, HarnessProvider::StaticSsh { .. }) && config.build_image { + return Err( + "static-ssh fixtures deploy a bundle; docker image building is not applicable" + .to_owned(), + ); } fs::create_dir_all(&config.artifacts) .map_err(|error| format!("create artifact directory: {error}"))?; @@ -128,33 +369,84 @@ impl ClusterHarness { .duration_since(UNIX_EPOCH) .map_err(|error| format!("system clock precedes epoch: {error}"))? .as_millis(); - let state_dir = config + let state_dir = match &config.state_dir { + Some(path) => path.clone(), + None => crate::resources::private_fixture_dir(&config.artifacts)? + .join(format!("state-{}-{nonce}", std::process::id())), + }; + { + use std::os::unix::fs::DirBuilderExt; + fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(&state_dir) + .map_err(|error| format!("create harness state directory: {error}"))?; + } + let state_dir = state_dir + .canonicalize() + .map_err(|error| format!("resolve harness state directory: {error}"))?; + let artifact_dir = config .artifacts - .join(format!("state-{}-{nonce}", std::process::id())); - fs::create_dir_all(&state_dir) - .map_err(|error| format!("create harness state directory: {error}"))?; + .canonicalize() + .map_err(|error| format!("resolve harness artifact directory: {error}"))?; + if state_dir.starts_with(&artifact_dir) { + return Err("runtime identity state cannot reside in publishable artifacts".to_owned()); + } let image = config .image .clone() .unwrap_or_else(|| format!("myelin-e2e:{}-{nonce}", std::process::id())); - build_myelin_binaries(&config.workspace)?; + let binaries = resolve_myelin_binaries(&config.workspace, &budget)?; if config.build_image { - build_workload_image(&config.workspace, &image)?; + if !matches!( + &config.provider, + HarnessProvider::LocalMock | HarnessProvider::StaticSsh { .. } + ) { + return Err( + "remote runtime images must be built and published before paid execution" + .to_owned(), + ); + } + build_workload_image(&config.workspace, &image, &budget)?; + } + if !config.build_image && matches!(config.provider, HarnessProvider::LocalMock) { + docker_image_identity(&image, &budget)?; } let port = reserve_port()?; let base_url = format!("http://127.0.0.1:{port}"); let container_prefix = format!("myelin-e2e-{}-{nonce}", std::process::id()); let telemetry = config.artifacts.join(format!("telemetry-{nonce}.jsonl")); - let (orchestrator, stdout, stderr) = spawn_orchestrator( + let provider_client = match &config.provider { + HarnessProvider::VastAiReal { api_key_env, .. } => { + let key = std::env::var(api_key_env).map_err(|_| { + format!("required VastAI credential environment {api_key_env} is unset") + })?; + Some(BlockingVastClient::new(VastClient::new(key))?) + } + _ => None, + }; + let (orchestrator, orchestrator_pidfd, stdout, stderr) = spawn_orchestrator( &config, + &binaries.orchestrator, port, &state_dir, &image, &container_prefix, &telemetry, - true, + &fixture_cleanup_budget, + config.reset_state, )?; + let retain_on_start_error = + !matches!(config.provider, HarnessProvider::LocalMock) && !config.reset_state; let mut harness = Self { + execution_budget: budget, + cleanup_budget: None, + fixture_cleanup_budget, + binaries, + provider_client, + lifecycle_started, + health_boundary: AtomicU64::new(0), + observation_generation: AtomicU64::new(0), config, base_url, dashboard_port: port, @@ -163,22 +455,298 @@ impl ClusterHarness { container_prefix, telemetry, telemetry_census: Mutex::new(TelemetryResourceCensus::default()), - resource_baseline: BTreeSet::new(), + provider_baseline: BTreeSet::new(), + fixture_mapping: BTreeMap::new(), + stopped_nodes: BTreeSet::new(), + provider_accounting_baseline: None, + node_generation_baseline: BTreeMap::new(), + fixture_blob_baseline: None, + fixture_actor_baseline: BTreeSet::new(), + verified_local_baseline: None, orchestrator, + orchestrator_pidfd, stdout, stderr, node_ids: Vec::new(), - orchestrator_stopped: false, - pre_stop_containers: Vec::new(), - torn_down: false, + next_attempt_id: 0, + active_attempts: BTreeMap::new(), + pending_control_processes: Mutex::new(BTreeSet::new()), + last_failure_signature: None, + last_attempt_id: None, + pending_attempts: BTreeMap::new(), + quarantine_reason: Mutex::new(None), + torn_down: retain_on_start_error, }; + if !harness.config.reset_state { + if let HarnessProvider::VastAiReal { admission, .. } = &harness.config.provider { + let binding = provisioning::PaidFixtureAdmission::open(admission) + .and_then(|ownership| ownership.bind_orchestrator(harness.orchestrator.id())); + if let Err(error) = binding { + let _ = harness.orchestrator.kill(); + let _ = wait_for_child( + &mut harness.orchestrator, + &harness.fixture_cleanup_budget, + "reap unbound retained orchestrator", + ); + return Err(error); + } + } + } harness.wait_for_dashboard()?; - harness.provision_nodes()?; - let baseline = harness.health_snapshot()?; - harness.resource_baseline = transient_actor_identities(&baseline); + if !harness.config.reset_state { + harness.adopt_retained_nodes()?; + } + if !observe_deployment { + if harness.config.provision { + harness.dispatch_provision()?; + } + } else if harness.config.provision { + harness.complete_deployment_after_dispatch(true)?; + } else if !harness.config.reset_state && !harness.config.adopt_only { + harness + .provision_nodes(false) + .map_err(|error| format!("{error}; {}", harness.orchestrator_diagnostics()))?; + harness.record_health_baseline()?; + // An adopt-only resume snapshots its baseline after in-place + // repair: the retained agents may be down and their contextual + // control is restored by the repair itself. + } + harness.torn_down = false; + harness.record_lifecycle("fixture_start", lifecycle_started, &Ok(()))?; Ok(harness) } + pub fn set_execution_budget(&mut self, budget: Budget) { + self.execution_budget = budget; + self.cleanup_budget = None; + } + + pub fn execution_budget(&self) -> &Budget { + &self.execution_budget + } + + pub fn set_cleanup_budget(&mut self, budget: Budget) { + self.fixture_cleanup_budget = budget; + } + + pub(crate) fn operation_budget(&self) -> &Budget { + self.cleanup_budget + .as_ref() + .unwrap_or(&self.execution_budget) + } + + pub fn complete_deployment(&mut self) -> Result<(), String> { + self.complete_deployment_after_dispatch(false) + } + + fn complete_deployment_after_dispatch(&mut self, dispatch: bool) -> Result<(), String> { + self.provision_nodes(dispatch)?; + self.wait_contextual_control()?; + self.record_health_baseline() + } + + fn record_lifecycle( + &self, + stage: &str, + started: Instant, + result: &Result, + ) -> Result<(), String> { + let record = json!({ + "schema_version": 2, + "run_id": self.config.run_id, + "orchestrator_pid": self.orchestrator.id(), + "stage": stage, + "started_monotonic_ns": started.duration_since(self.lifecycle_started).as_nanos(), + "elapsed_ns": started.elapsed().as_nanos(), + "pending_predicate": result.as_ref().err(), + "succeeded": result.is_ok(), + "observation": result.as_ref().ok(), + }); + let mut file = fs::OpenOptions::new() + .create(true) + .append(true) + .open(self.config.artifacts.join("lifecycle-timings.jsonl")) + .map_err(|error| format!("open lifecycle evidence: {error}"))?; + serde_json::to_writer(&mut file, &record) + .map_err(|error| format!("write lifecycle evidence: {error}"))?; + writeln!(file).map_err(|error| format!("finish lifecycle evidence: {error}")) + } + + pub fn is_quarantined(&self) -> bool { + self.quarantine_reason().is_some() + } + + fn quarantine_reason(&self) -> Option { + self.quarantine_reason + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clone() + } + + fn quarantine(&self, reason: String) { + self.quarantine_reason + .lock() + .unwrap_or_else(|error| error.into_inner()) + .get_or_insert(reason); + } + + pub(super) fn next_observation_generation(&self) -> u64 { + self.observation_generation.fetch_add(1, Ordering::Relaxed) + 1 + } + + fn bounded_lifecycle( + &mut self, + stage: &str, + run: impl FnOnce(&mut Self) -> Result<(), String>, + ) -> Result<(), String> { + let started = Instant::now(); + let budget = self.operation_budget().child(self.config.deadline); + let previous = self.cleanup_budget.replace(budget); + let result = run(self); + self.cleanup_budget = previous; + self.record_lifecycle(stage, started, &result)?; + result + } + + pub(super) fn control_revision(&self, budget: &Budget) -> Result { + let reply = http_json_budget( + "POST", + &format!("{}/api/control/changes", self.base_url), + Some(json!({"schema_version": 1, "payload": { + "generation": null, "after_revision": null, "wait_ms": 0 + }})), + budget, + )?; + validate_control_revision(&reply)?; + Ok(reply["payload"].clone()) + } + + pub(super) fn wait_for_control_change( + &self, + cursor: &Value, + budget: &Budget, + predicate: &str, + ) -> Result<(), String> { + self.ensure_orchestrator_live()?; + let wait_ms = budget.remaining(predicate)?.as_millis().min(1_000) as u64; + let reply = http_json_budget( + "POST", + &format!("{}/api/control/changes", self.base_url), + Some(json!({"schema_version": 1, "payload": { + "generation": cursor["generation"], "after_revision": cursor["revision"], "wait_ms": wait_ms + }})), + budget, + ); + match reply { + Ok(reply) => validate_control_revision(&reply)?, + Err(error) if error.contains("504") => {} + Err(error) => { + budget.wait( + POLL_INTERVAL, + &format!("{predicate}; control observation reconnect: {error}"), + )?; + } + } + budget.check(predicate)?; + self.ensure_orchestrator_live() + } + + /// Resolve the exact retained live-node set without initiating provisioning + /// or accepting a replacement logical identity. + fn retained_live_node_ids(&self) -> Result, String> { + let mut adopted = self + .retained_node_specs()? + .into_iter() + .map(|(node_id, _, _)| node_id) + .collect::>(); + adopted.sort_unstable(); + if adopted.contains(&0) + || adopted.windows(2).any(|pair| pair[0] == pair[1]) + || adopted.len() != usize::from(self.config.node_count) + { + return Err(format!( + "retained fixture must contain {} exact, unique live logical nodes; observed {adopted:?}", + self.config.node_count + )); + } + Ok(adopted) + } + fn retained_stopped_node_ids(&self) -> Result, String> { + let snapshot: Value = serde_json::from_slice( + &fs::read(self.state_dir.join("cluster.json")) + .map_err(|error| format!("read retained node phases: {error}"))?, + ) + .map_err(|error| format!("parse retained node phases: {error}"))?; + let mut seen = BTreeSet::new(); + let mut stopped = BTreeSet::new(); + for node in snapshot["nodes"] + .as_array() + .ok_or("retained cluster snapshot omitted nodes")? + { + let node_id = node["logical_node_id"] + .as_u64() + .filter(|node_id| *node_id != 0) + .ok_or_else(|| format!("retained node omitted valid identity: {node}"))?; + if !seen.insert(node_id) { + return Err(format!( + "retained cluster duplicated logical node {node_id}" + )); + } + match node["phase"].as_str() { + Some("stopped") => { + stopped.insert(node_id); + } + Some("orphan") => { + return Err(format!( + "retained logical node {node_id} is orphaned and cannot be adopted" + )); + } + Some(_) => {} + None => return Err(format!("retained node {node_id} omitted phase")), + } + } + Ok(stopped) + } + + /// Adopt the retained fixture without waiting for any node state: fill + /// the live-node list from the durable snapshot so in-place repair can + /// relaunch agents before running-state waits happen. + fn adopt_retained_nodes(&mut self) -> Result<(), String> { + let live = self.retained_live_node_ids()?; + let stopped = self.retained_stopped_node_ids()?; + if live.iter().any(|node| stopped.contains(node)) { + return Err("retained live and stopped logical-node sets overlap".to_owned()); + } + self.node_ids = live; + self.stopped_nodes = stopped; + Ok(()) + } + + /// Captured orchestrator process output for failure diagnostics. + pub fn orchestrator_diagnostics(&mut self) -> String { + let exit = match self.orchestrator.try_wait() { + Ok(Some(status)) => format!("exited: {status}"), + Ok(None) => "running".to_owned(), + Err(error) => format!("try_wait failed: {error}"), + }; + let stdout = self + .stdout + .lock() + .unwrap_or_else(|error| error.into_inner()) + .iter() + .cloned() + .collect::>() + .join("\n"); + let stderr = self + .stderr + .lock() + .unwrap_or_else(|error| error.into_inner()) + .iter() + .cloned() + .collect::>() + .join("\n"); + format!("orchestrator {exit}; stdout tail:\n{stdout}\norchestrator stderr tail:\n{stderr}") + } pub fn node_ids(&self) -> &[u64] { &self.node_ids } @@ -190,325 +758,2094 @@ impl ClusterHarness { pub fn state_dir(&self) -> &Path { &self.state_dir } + pub fn fixture_run_id(&self) -> Result { + let snapshot = serde_json::from_slice::( + &fs::read(self.state_dir.join("cluster.json")) + .map_err(|error| format!("read cluster run id: {error}"))?, + ) + .map_err(|error| format!("parse cluster run id: {error}"))?; + snapshot + .get("run_id") + .and_then(Value::as_u64) + .filter(|run_id| *run_id != 0) + .ok_or_else(|| "cluster snapshot omitted nonzero run_id".to_owned()) + } + pub fn search_offers(&self, request: Value) -> Result { + http_json_budget( + "POST", + &format!("{}/api/control/offers", self.base_url), + Some(request), + &self.operation_budget().child(self.config.deadline), + ) + } + + pub fn provision_selected( + &mut self, + selected_offer_ids: Vec, + search_id: u64, + ) -> Result<(), String> { + if !self.node_ids.is_empty() { + return Err("fixture is already provisioned".to_owned()); + } + if selected_offer_ids.len() != usize::from(self.config.node_count) { + return Err("exact offer count differs from fixture node count".to_owned()); + } + self.config.selected_offer_ids = selected_offer_ids; + self.config.offer_search_id = Some(search_id); + self.complete_deployment_after_dispatch(true) + } + + /// Authorizes the exact five-node paid acquisition this fixture will run. + pub fn authorize_paid_admission( + &mut self, + run_id: u64, + nodes: Vec, + deadline_unix_ms: u64, + ) -> Result<(), String> { + let admission_path = match &self.config.provider { + HarnessProvider::VastAiReal { admission, .. } => admission.clone(), + _ => return Err("paid admission requires the real VastAI provider".to_owned()), + }; + provisioning::PaidFixtureAdmission::create( + &admission_path, + run_id, + nodes, + deadline_unix_ms, + ) + .map_err(|error| format!("authorize paid admission: {error}"))?; + Ok(()) + } + + /// Seals preparation once five exact contracts and bootstrap sessions exist. + pub fn seal_paid_admission(&self) -> Result<(), String> { + let admission_path = match &self.config.provider { + HarnessProvider::VastAiReal { admission, .. } => admission, + _ => return Err("paid admission requires the real VastAI provider".to_owned()), + }; + let admission = provisioning::PaidFixtureAdmission::open(admission_path) + .map_err(|error| format!("open paid admission: {error}"))?; + admission + .seal_prepared() + .map_err(|error| format!("seal paid admission: {error}"))?; + let snapshot = admission + .snapshot() + .map_err(|error| format!("read paid admission: {error}"))?; + match snapshot.mode { + provisioning::PaidAdmissionMode::Prepared => Ok(()), + _ => Err("paid fixture preparation is incomplete or closed".to_owned()), + } + } + + fn record_health_baseline(&mut self) -> Result<(), String> { + self.record_health_baseline_snapshot(true).map(drop) + } + + fn record_health_baseline_snapshot( + &mut self, + require_stable_fixture_sample: bool, + ) -> Result { + if !self.config.reset_state + && matches!(self.config.provider, HarnessProvider::VastAiReal { .. }) + { + self.stopped_nodes = self + .paid_admission()? + .nodes + .iter() + .map(|node| node.node_id) + .filter(|node| !self.node_ids.contains(node)) + .collect(); + } + let budget = self.operation_budget().child(self.config.deadline); + let mut prior_fixture_actors = None; + let (baseline, fixture_actor_baseline) = loop { + let cursor = self.control_revision(&budget)?; + let mut health = self.health_snapshot_for_nodes(&self.node_ids, &budget)?; + let fixture = self.builtin_fixture_blobs(&budget)?; + let fixture_actors = retained_actor_identities(&health, &fixture)?; + let retained = self + .active_attempts + .values() + .map(|attempt| RetainedCaseActors { + attempt: Arc::clone(attempt), + }) + .collect::>(); + let mut resource_baseline = fixture_actors.clone(); + for attempt in &retained { + resource_baseline.extend(self.retained_blob_exemptions(attempt, &health, &budget)?); + } + health["builtin_fixture"] = + serde_json::to_value(&fixture).map_err(|error| error.to_string())?; + if let Some(reason) = pending_resource_cleanup(&health, &resource_baseline) { + self.wait_for_control_change( + &cursor, + &budget, + &format!("clean initial fixture: {reason}"), + )?; + continue; + } + if require_stable_fixture_sample + && prior_fixture_actors.as_ref() != Some(&fixture_actors) + { + prior_fixture_actors = Some(fixture_actors); + budget.wait(POLL_INTERVAL, "stable built-in fixture actor identities")?; + continue; + } + let blob = &fixture[BUILTIN_FIXTURE_PATH]; + let observed_blob = (blob.revision, blob.length); + if self + .fixture_blob_baseline + .is_some_and(|expected| expected != observed_blob) + { + return Err(format!( + "built-in fixture blob baseline changed: expected={:?}, observed={observed_blob:?}", + self.fixture_blob_baseline + )); + } + self.fixture_blob_baseline.get_or_insert(observed_blob); + break (health, fixture_actors); + }; + self.fixture_actor_baseline = fixture_actor_baseline; + self.provider_accounting_baseline = baseline + .get("provider_accounting") + .filter(|value| !value.is_null()) + .cloned(); + self.provider_baseline = serde_json::from_value( + baseline + .get("provider_resources") + .cloned() + .ok_or_else(|| "health snapshot omitted provider resources".to_owned())?, + ) + .map_err(|error| format!("invalid provider resource census: {error}"))?; + self.fixture_mapping = + fixture_node_mapping(&baseline, &self.node_ids, &self.stopped_nodes)?; + validate_fixture_census( + &baseline, + &self.provider_baseline, + &self.node_ids, + &self.fixture_mapping, + &self.stopped_nodes, + )?; + self.node_generation_baseline = baseline["nodes"] + .as_array() + .into_iter() + .flatten() + .map(|node| { + let observation = &node["observation"]; + let id = observation["logical_node_id"] + .as_u64() + .ok_or_else(|| "baseline contextual node omitted identity".to_owned())?; + Ok(( + id, + resource_runtime_identity(&observation["event"]["resources"]), + )) + }) + .collect::>()?; + self.persist_fixture_identity(&baseline)?; + Ok(baseline) + } + + pub fn fixture_identity_digest(&self) -> Result { + use sha2::Digest; + if self.fixture_mapping.is_empty() { + return Err("fixture identity is unavailable before validated readiness".to_owned()); + } + let identity = json!({ + "schema_version": 1, "node_contract_hosts": self.fixture_mapping, + "admission": self.provider_accounting_baseline, + }); + let bytes = serde_json::to_vec(&identity).map_err(|error| error.to_string())?; + Ok(format!("{:x}", sha2::Sha256::digest(bytes))) + } + + fn persist_fixture_identity(&self, health: &Value) -> Result<(), String> { + let stem = self + .telemetry + .file_stem() + .and_then(|value| value.to_str()) + .ok_or("telemetry identity")?; + let generation = health["generation"] + .as_u64() + .ok_or("fixture observation generation")?; + let proof = json!({ + "schema_version": 1, "fixture_identity_digest": self.fixture_identity_digest()?, + "node_contract_hosts": self.fixture_mapping, "admission": self.provider_accounting_baseline, + "running_nodes": self.node_ids, "stopped_nodes": self.stopped_nodes, + "provider_resources": health["provider_resources"], "provider_hosts": health["provider_hosts"], + "control_revision": health["control_revision"], "generation": generation, + "contextual_nodes": health["nodes"], "live_resource_baseline": health["resources"], + "fixture_actor_baseline": self.fixture_actor_baseline, + "builtin_fixture": health["builtin_fixture"], + }); + let path = self + .config + .artifacts + .join(format!("fixture-identity-{stem}.jsonl")); + let created = !path.exists(); + let mut file = fs::OpenOptions::new() + .create(true) + .append(true) + .open(&path) + .map_err(|error| format!("open fixture identity evidence: {error}"))?; + serde_json::to_writer(&mut file, &proof).map_err(|error| error.to_string())?; + file.write_all(b"\n") + .map_err(|error| format!("finish fixture identity record: {error}"))?; + file.sync_data() + .map_err(|error| format!("sync fixture identity evidence: {error}"))?; + if created { + fs::File::open(&self.config.artifacts) + .and_then(|directory| directory.sync_all()) + .map_err(|error| format!("sync fixture identity directory: {error}"))?; + } + Ok(()) + } + pub fn owned_provider_contracts(&self) -> Result, String> { + match &self.config.provider { + HarnessProvider::LocalMock => { + Err("local fixtures do not own VastAI contracts".to_owned()) + } + HarnessProvider::StaticSsh { .. } => Ok(BTreeSet::new()), + HarnessProvider::VastAiReal { .. } => self.remote_contracts(), + } + } + + pub fn owned_provider_labels(&self) -> Result, String> { + match &self.config.provider { + HarnessProvider::LocalMock => Err("local fixtures do not own VastAI labels".to_owned()), + HarnessProvider::StaticSsh { .. } => Ok(BTreeSet::new()), + HarnessProvider::VastAiReal { .. } => self.remote_labels(), + } + } + + /// Per-node agent launch facts (node id, environment, argv) from the + /// retained cluster snapshot, used to relaunch agents in place. + fn retained_node_specs( + &self, + ) -> Result, Vec)>, String> { + let snapshot = serde_json::from_slice::( + &fs::read(self.state_dir.join("cluster.json")) + .map_err(|error| format!("read cluster snapshot: {error}"))?, + ) + .map_err(|error| format!("parse cluster snapshot: {error}"))?; + snapshot + .get("nodes") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .filter(|node| { + node.get("phase") + .and_then(serde_json::Value::as_str) + .is_some_and(|phase| phase != "stopped" && phase != "orphan") + }) + .map(|node| { + let node_id = node + .get("logical_node_id") + .and_then(serde_json::Value::as_u64) + .ok_or_else(|| "cluster node omitted logical_node_id".to_owned())?; + let spec = node + .get("spec") + .ok_or_else(|| format!("cluster node {node_id} omitted its spec"))?; + let env = spec + .get("env") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .map(|pair| { + let pair = pair + .as_array() + .ok_or_else(|| "node spec env entry is not a pair".to_owned())?; + let key = pair + .first() + .and_then(serde_json::Value::as_str) + .ok_or_else(|| "node spec env pair omitted key".to_owned())?; + let value = pair + .get(1) + .and_then(serde_json::Value::as_str) + .ok_or_else(|| "node spec env pair omitted value".to_owned())?; + Ok((key.to_owned(), value.to_owned())) + }) + .collect::, String>>()?; + let args = spec + .get("args") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .filter_map(serde_json::Value::as_str) + .map(str::to_owned) + .collect::>(); + Ok((node_id, env, args)) + }) + .collect() + } + pub fn retain_remote_fixture(&mut self, development_authorized: bool) -> Result<(), String> { + if !self.active_attempts.is_empty() + || !self.pending_attempts.is_empty() + || !self + .pending_control_processes + .lock() + .map_err(|_| "retained fixture control ownership lock poisoned".to_owned())? + .is_empty() + { + return Err( + "fixture retention requires quiescent workload and control ownership".to_owned(), + ); + } + if matches!(self.config.provider, HarnessProvider::LocalMock) { + return Err("fixture retention requires a remote-node provider".to_owned()); + } + if let HarnessProvider::VastAiReal { admission, .. } = &self.config.provider { + provisioning::PaidFixtureAdmission::open(admission)? + .retain_development_fixture(development_authorized)?; + } else if !development_authorized { + return Err("fixture retention requires explicit development authorization".to_owned()); + } + self.torn_down = true; + // Retention releases the local owner, including after injected death + // or execution expiry. It is cleanup, not a new crash injection. + let signal = self.orchestrator.kill(); + wait_for_child( + &mut self.orchestrator, + &self.fixture_cleanup_budget, + "retained orchestrator reap", + ) + .map_err(|error| match signal { + Ok(()) => error, + Err(signal) => format!("stop retained orchestrator: {signal}; {error}"), + }) + } } impl ClusterHarness { + fn vastai_client(&self) -> Result<&BlockingVastClient, String> { + self.provider_client + .as_ref() + .ok_or_else(|| "VastAI client requested for local fixture".to_owned()) + } + + fn provider_host_mapping( + &self, + fleet: &Value, + containers: Option<&BTreeMap>, + ) -> Result, String> { + let nodes = fleet["FleetStatus"]["nodes"] + .as_array() + .ok_or("fleet omitted node mapping")?; + match &self.config.provider { + HarnessProvider::VastAiReal { .. } => Ok(self + .paid_admission()? + .nodes + .into_iter() + .map(|node| (node.node_id, json!(node.host_id))) + .collect()), + HarnessProvider::StaticSsh { manifest, .. } => { + let manifest: Value = serde_json::from_slice( + &fs::read(manifest) + .map_err(|error| format!("read raw fixture host mapping: {error}"))?, + ) + .map_err(|error| format!("decode raw fixture host mapping: {error}"))?; + let mut hosts = BTreeMap::new(); + let mut resources = BTreeSet::new(); + for node in manifest["nodes"] + .as_array() + .ok_or("raw manifest omitted nodes")? + { + let id = node["slot"] + .as_u64() + .and_then(|slot| slot.checked_add(1)) + .ok_or("raw slot identity")?; + let resource = node["container"] + .as_str() + .filter(|value| !value.is_empty()) + .ok_or("raw resource identity")?; + let host = node["ssh_host"] + .as_str() + .filter(|value| !value.is_empty()) + .ok_or("raw SSH host")?; + let port = node["ssh_port"] + .as_u64() + .filter(|port| (1..=65535).contains(port)) + .ok_or("raw SSH port")?; + if hosts + .insert( + id, + json!({"container": resource, "ssh_host": host, "ssh_port": port}), + ) + .is_some() + || !resources.insert(resource) + { + return Err("duplicate raw node/resource host mapping".to_owned()); + } + } + for node in nodes.iter().filter(|node| node["phase"] == "running") { + let id = node["logical_node_id"] + .as_u64() + .ok_or("raw running node identity")?; + let expected = hosts + .get(&id) + .ok_or_else(|| format!("node {id} has no raw host admission"))?; + if node["provider_ref"] + != format!( + "static-ssh:{}", + expected["container"].as_str().ok_or("raw resource")? + ) + { + return Err(format!( + "raw node {id} is associated with the wrong container/SSH host" + )); + } + } + Ok(hosts) + } + HarnessProvider::LocalMock => nodes + .iter() + .filter(|node| node["phase"] == "running") + .map(|node| { + let name = node["provider_ref"] + .as_str() + .ok_or("local provider reference")?; + let identity = + containers + .and_then(|census| census.get(name)) + .ok_or_else(|| { + format!( + "local provider reference {name} is absent from Docker census" + ) + })?; + Ok(( + node["logical_node_id"] + .as_u64() + .ok_or("local node identity")?, + json!({"container": name, "container_id": identity}), + )) + }) + .collect(), + } + } + + fn paid_admission(&self) -> Result { + let HarnessProvider::VastAiReal { admission, .. } = &self.config.provider else { + return Err("paid admission requested for local fixture".to_owned()); + }; + provisioning::PaidFixtureAdmission::open(admission)?.snapshot() + } + + fn remote_labels(&self) -> Result, String> { + // Admission reserves labels before create, including responses lost before + // cluster.json could record a lease. It remains authoritative after stop. + Ok(self + .paid_admission()? + .nodes + .into_iter() + .map(|node| node.label) + .collect()) + } + + fn remote_contracts(&self) -> Result, String> { + self.remote_contracts_budget(&self.operation_budget().child(self.config.deadline)) + } + + fn remote_contracts_budget(&self, budget: &Budget) -> Result, String> { + let started = Instant::now(); + let admission = self.paid_admission()?; + let labels = admission.cleanup_labels(); + let known = admission.known_contract_ids(); + let census = self.vastai_client()?.owned_census( + &labels, + &known, + started, + budget.remaining("fresh exact owned provider census")?, + )?; + for instance in &census.instances { + let node = admission + .contracts + .iter() + .find_map(|(node, contract)| (*contract == instance.contract_id).then_some(*node)) + .and_then(|id| admission.nodes.iter().find(|node| node.node_id == id)) + .ok_or_else(|| format!("unadmitted provider contract {}", instance.contract_id))?; + if instance.host_id != Some(node.host_id) { + return Err(format!( + "node {} contract {} moved hosts: expected {}, observed {:?}", + node.node_id, instance.contract_id, node.host_id, instance.host_id + )); + } + } + self.record_lifecycle("provider_census", started, &Ok(()))?; + let record = json!({ + "schema_version": 1, "generation": census.generation, + "elapsed_ns": started.elapsed().as_nanos(), + "request_started_ns": census.request_started.saturating_duration_since(self.lifecycle_started).as_nanos(), + "observed_ns": census.observed_at.saturating_duration_since(self.lifecycle_started).as_nanos(), + "counters": census.counters, + "contracts": census.instances.iter().map(|instance| instance.contract_id).collect::>(), + "missing_contract_ids": census.missing_contract_ids, + }); + let mut file = fs::OpenOptions::new() + .create(true) + .append(true) + .open(self.config.artifacts.join("provider-census.jsonl")) + .map_err(|error| format!("open scoped provider census evidence: {error}"))?; + serde_json::to_writer(&mut file, &record) + .map_err(|error| format!("write scoped provider census evidence: {error}"))?; + writeln!(file) + .map_err(|error| format!("finish scoped provider census evidence: {error}"))?; + Ok(census + .instances + .into_iter() + .map(|instance| instance.contract_id) + .collect()) + } + + fn provider_resources(&self) -> Result, String> { + self.provider_resources_budget(&self.operation_budget().child(self.config.deadline)) + } + + fn provider_resources_budget(&self, budget: &Budget) -> Result, String> { + match &self.config.provider { + HarnessProvider::LocalMock => Ok(list_containers(&self.container_prefix, budget)? + .into_iter() + .collect()), + HarnessProvider::StaticSsh { manifest, .. } => { + Ok(raw_fleet::read_manifest_containers(manifest, budget)? + .into_iter() + .collect()) + } + HarnessProvider::VastAiReal { .. } => Ok(self + .remote_contracts_budget(budget)? + .into_iter() + .map(|contract| format!("vastai:{contract}")) + .collect()), + } + } + fn crash_orchestrator(&mut self) -> Result<(), String> { + self.operation_budget() + .check("orchestrator crash and reap")?; self.ensure_orchestrator_live()?; self.orchestrator .kill() .map_err(|error| format!("kill orchestrator for recovery: {error}"))?; - self.orchestrator - .wait() - .map_err(|error| format!("wait for crashed orchestrator: {error}"))?; + let budget = self.operation_budget().clone(); + wait_for_child(&mut self.orchestrator, &budget, "reap crashed orchestrator")?; + let mut census = self + .telemetry_census + .lock() + .map_err(|_| "telemetry census is poisoned".to_owned())?; + // Drain the stopped producer before resetting its sequence. Otherwise + // unread pre-crash frames can repopulate the retired incarnation. + census.update(&self.telemetry, &budget)?; + census.forget_stream(&format!("myelin-orchestrator#{}", self.config.run_id)); Ok(()) } fn restart_orchestrator(&mut self) -> Result<(), String> { - let (orchestrator, stdout, stderr) = spawn_orchestrator( + self.operation_budget() + .check("restart retained orchestrator")?; + let retained_nodes = self.retained_live_node_ids()?; + let retained_stopped = self.retained_stopped_node_ids()?; + if retained_nodes != self.node_ids || retained_stopped != self.stopped_nodes { + return Err(format!( + "orchestrator restart must adopt exact logical nodes running={:?}, stopped={:?}; retained snapshot has running={retained_nodes:?}, stopped={retained_stopped:?}", + self.node_ids, self.stopped_nodes + )); + } + let (orchestrator, orchestrator_pidfd, stdout, stderr) = spawn_orchestrator( &self.config, + &self.binaries.orchestrator, self.dashboard_port, &self.state_dir, &self.image, &self.container_prefix, &self.telemetry, + &self.fixture_cleanup_budget, false, )?; self.orchestrator = orchestrator; + self.orchestrator_pidfd = orchestrator_pidfd; self.stdout = stdout; self.stderr = stderr; - self.orchestrator_stopped = false; + if let HarnessProvider::VastAiReal { admission, .. } = &self.config.provider { + provisioning::PaidFixtureAdmission::open(admission)? + .bind_orchestrator(self.orchestrator.id())?; + } self.wait_for_dashboard() } pub fn verify_running_recovery(&mut self) -> Result<(), String> { - let expected_containers = list_containers(&self.container_prefix)? - .into_iter() - .collect::>(); - let expected_nodes = self.node_ids.clone(); - self.crash_orchestrator()?; - self.restart_orchestrator()?; - self.provision_nodes()?; - let recovered_containers = list_containers(&self.container_prefix)? - .into_iter() - .collect::>(); - if recovered_containers != expected_containers { - return Err(format!( - "recovery changed provider resources: before={expected_containers:?}, \ - after={recovered_containers:?}" - )); - } - if self.node_ids != expected_nodes { - return Err(format!( - "recovery changed logical nodes: before={expected_nodes:?}, after={:?}", - self.node_ids - )); - } - Ok(()) + self.bounded_lifecycle("orchestrator_restart_rejoin", |harness| { + if let Some(reason) = harness.quarantine_reason() { + return Err(format!("fixture is quarantined: {reason}")); + } + harness.restore_owned_baseline(None)?; + let expected_resources = harness.provider_baseline.clone(); + let expected_nodes = harness.node_ids.clone(); + let expected_mapping = harness.fixture_mapping.clone(); + let previous_pid = harness.orchestrator.id(); + if let HarnessProvider::VastAiReal { admission, .. } = &harness.config.provider { + provisioning::PaidFixtureAdmission::open(admission)? + .begin_orchestrator_recovery()?; + } + harness.crash_orchestrator()?; + harness.restart_orchestrator()?; + if harness.orchestrator.id() == previous_pid { + return Err("orchestrator restart did not create a fresh process".to_owned()); + } + harness.provision_nodes(false)?; + harness.wait_contextual_control()?; + // Actor addresses are process-incarnation identities. Re-establish + // that census after replacement. Contextual readiness already + // proves the retained fixture is stable, so one exact post-restart + // snapshot is sufficient and is reused for recovery validation. + let recovered_health = harness.record_health_baseline_snapshot(false)?; + validate_fixture_census( + &recovered_health, + &expected_resources, + &expected_nodes, + &expected_mapping, + &harness.stopped_nodes, + )?; + Ok(()) + }) } pub fn verify_missing_resource_recovery(&mut self) -> Result<(), String> { - if self.orchestrator_stopped { - return Ok(()); - } - let mut expected_containers = list_containers(&self.container_prefix)? - .into_iter() - .collect::>(); - let removed = expected_containers - .pop_last() - .ok_or_else(|| "missing-resource recovery requires a provider resource".to_owned())?; - self.crash_orchestrator()?; - remove_container(&removed)?; - self.restart_orchestrator()?; + self.bounded_lifecycle("missing_resource_recovery", |harness| { + if !matches!(harness.config.provider, HarnessProvider::LocalMock) { + return Err("missing-resource mock recovery requires local Docker ownership".to_owned()); + } + let mut expected_resources = harness.provider_resources()?; + let removed_node = *harness.node_ids.last() + .ok_or_else(|| "missing-resource recovery requires a live node".to_owned())?; + let removed = harness.resource_for_node(removed_node)?; + if !expected_resources.remove(&removed) { + return Err(format!("missing node {removed_node} did not own resource {removed}")); + } + let expected_nodes = harness.node_ids.iter().copied() + .filter(|&node| node != removed_node).collect::>(); + let mut stopped_nodes = harness.stopped_nodes.clone(); + stopped_nodes.insert(removed_node); + harness.crash_orchestrator()?; + remove_container(&removed, harness.operation_budget())?; + harness.restart_orchestrator()?; + loop { + harness.operation_budget().check(&format!("resource {removed} absent and node {removed_node} terminal"))?; + harness.ensure_orchestrator_live()?; + let cursor = harness.control_revision(harness.operation_budget())?; + let resources = harness.provider_resources()?; + if resources != expected_resources { + return Err(format!("missing provider resource was recreated: expected={expected_resources:?}, observed={resources:?}")); + } + let status = http_json_budget("GET", &format!("{}/api/control/status", harness.base_url), + None, harness.operation_budget())?; + let nodes = status.pointer("/Status/nodes").and_then(Value::as_array) + .ok_or_else(|| format!("recovery status omitted nodes: {status}"))?; + let running = nodes.iter().filter(|node| node["phase"] == "running") + .filter_map(|node| node["logical_node_id"].as_u64()).collect::>(); + let missing_is_terminal = nodes.iter().find(|node| node["logical_node_id"] == removed_node) + .is_some_and(|node| node["phase"] == "stopped" + && node["last_error"].as_str().is_some_and(|error| + error.contains("absent") || error.contains("no provider resource"))); + if running == expected_nodes.iter().copied().collect() && missing_is_terminal { + let health = harness.health_snapshot_for_nodes(&expected_nodes, harness.operation_budget())?; + validate_fixture_census(&health, &expected_resources, &expected_nodes, + &harness.fixture_mapping, &stopped_nodes)?; + harness.node_ids = expected_nodes; + harness.provider_baseline = expected_resources; + harness.stopped_nodes = stopped_nodes; + harness.record_health_baseline()?; + return Ok(()); + } + harness.wait_for_control_change(&cursor, harness.operation_budget(), + &format!("missing-resource terminal state; observed={status}"))?; + } + }) + } - let deadline = Instant::now() + self.config.deadline; - loop { - self.ensure_orchestrator_live()?; - let containers = list_containers(&self.container_prefix)? - .into_iter() - .collect::>(); - if containers != expected_containers { + pub fn remove_node(&mut self, logical_node_id: u64) -> Result<(), String> { + self.bounded_lifecycle("destructive_transition", |harness| { + if let Some(reason) = harness.quarantine_reason() { + return Err(format!("fixture is quarantined: {reason}")); + } + if !harness.active_attempts.is_empty() || !harness.pending_attempts.is_empty() { + return Err("node removal requires complete attempt cleanup".to_owned()); + } + harness.assert_healthy()?; + if !harness.node_ids.contains(&logical_node_id) { + return Err(format!("logical node {logical_node_id} is not live")); + } + if harness.node_ids.len() <= 3 { + return Err( + "survivor campaign cannot remove a node below three survivors".to_owned(), + ); + } + let before = harness.health_snapshot()?; + validate_fixture_census( + &before, + &harness.provider_baseline, + &harness.node_ids, + &harness.fixture_mapping, + &harness.stopped_nodes, + )?; + let mut expected_resources = harness.provider_resources()?; + let target = harness.resource_for_node(logical_node_id)?; + if !expected_resources.remove(&target) { return Err(format!( - "missing provider resource was recreated: expected={expected_containers:?}, \ - observed={containers:?}" + "target node {logical_node_id} resource {target} is absent" )); } - let status = http_json( - "GET", - &format!("{}/api/control/status", self.base_url), - None, - )?; - let nodes = status - .pointer("/Status/nodes") - .and_then(Value::as_array) - .cloned() - .unwrap_or_default(); - let running = nodes + let expected_nodes = harness + .node_ids .iter() - .filter(|node| node.get("phase").and_then(Value::as_str) == Some("running")) - .count(); - let missing_is_terminal = nodes.iter().any(|node| { - node.get("phase").and_then(Value::as_str) == Some("stopped") - && node - .get("last_error") - .and_then(Value::as_str) - .is_some_and(|error| { - error.contains("absent") || error.contains("no provider resource") - }) - }); - if running == expected_containers.len() && missing_is_terminal { - return Ok(()); + .copied() + .filter(|node| *node != logical_node_id) + .collect::>(); + let mut stopped_nodes = harness.stopped_nodes.clone(); + stopped_nodes.insert(logical_node_id); + harness.kill_node(logical_node_id)?; + if matches!(harness.config.provider, HarnessProvider::StaticSsh { .. }) { + // The static provider stops the exact container but deliberately + // retains it for deployment gates. A destructive tail owns removal. + remove_container(&target, harness.operation_budget())?; } - if Instant::now() >= deadline { - return Err(self.timeout_evidence(&format!( - "missing resource to become terminal without replacement; status={status}" - ))); + loop { + harness.operation_budget().check(&format!( + "exact survivors {expected_nodes:?} and resources {expected_resources:?}" + ))?; + harness.ensure_orchestrator_live()?; + let cursor = harness.control_revision(harness.operation_budget())?; + let health = harness + .health_snapshot_for_nodes(&expected_nodes, harness.operation_budget())?; + let stopped = health + .pointer("/fleet/FleetStatus/nodes") + .and_then(Value::as_array) + .into_iter() + .flatten() + .find(|node| node["logical_node_id"] == logical_node_id) + .is_some_and(|node| node["phase"] == "stopped"); + if stopped + && validate_fixture_census( + &health, + &expected_resources, + &expected_nodes, + &harness.fixture_mapping, + &stopped_nodes, + ) + .is_ok() + { + harness.node_ids = expected_nodes; + harness.provider_baseline = expected_resources; + harness.stopped_nodes = stopped_nodes; + harness.record_health_baseline()?; + return Ok(()); + } + harness.wait_for_control_change( + &cursor, + harness.operation_budget(), + &format!("exact survivor census: {health}"), + )?; } - thread::sleep(POLL_INTERVAL); + }) + } + + fn resource_for_node(&self, logical_node_id: u64) -> Result { + if matches!(self.config.provider, HarnessProvider::VastAiReal { .. }) { + return self + .paid_admission()? + .contracts + .get(&logical_node_id) + .map(|contract| format!("vastai:{contract}")) + .ok_or_else(|| { + format!("node {logical_node_id} has no admitted provider contract") + }); } + if matches!(self.config.provider, HarnessProvider::LocalMock) { + return self + .fixture_mapping + .get(&logical_node_id) + .and_then(|identity| identity["provider_host"]["container_id"].as_str()) + .map(str::to_owned) + .ok_or_else(|| format!("node {logical_node_id} has no validated Docker identity")); + } + let snapshot: Value = serde_json::from_slice( + &fs::read(self.state_dir.join("cluster.json")) + .map_err(|error| format!("read node resource mapping: {error}"))?, + ) + .map_err(|error| format!("decode node resource mapping: {error}"))?; + let resource = snapshot["nodes"] + .as_array() + .into_iter() + .flatten() + .find(|node| node["logical_node_id"] == logical_node_id) + .and_then(|node| node["provider_ref"].as_str()) + .ok_or_else(|| format!("node {logical_node_id} has no retained provider identity"))?; + Ok(resource + .strip_prefix("static-ssh:") + .unwrap_or(resource) + .to_owned()) } pub fn health_snapshot(&self) -> Result { - let actors = http_json( - "GET", - &format!("{}/api/control/actors", self.base_url), - None, - )?; - let fleet = http_json("GET", &format!("{}/api/control/fleet", self.base_url), None)?; - let running_nodes = fleet - .pointer("/FleetStatus/nodes") - .and_then(Value::as_array) - .into_iter() - .flatten() - .filter(|node| node.get("phase").and_then(Value::as_str) == Some("running")) - .filter_map(|node| node.get("logical_node_id").and_then(Value::as_u64)) - .collect::>(); - let mut nodes = Vec::new(); - for node in &running_nodes { - let control_request_id = format!("health-{}-{node}", self.config.seed); - let live = http_json( - "GET", - &format!( - "{}/api/control/contextual/nodes/{node}?control_request_id={control_request_id}", - self.base_url - ), - None, - )?; - nodes.push(live); + self.health_snapshot_for_nodes( + &self.node_ids, + &self.operation_budget().child(self.config.deadline), + ) + } + + pub(crate) fn mark_health_boundary(&self) -> Result<(), String> { + self.operation_budget() + .check("post-terminal resource observation boundary")?; + let generation = self.next_observation_generation(); + self.health_boundary.store(generation, Ordering::Release); + self.record_lifecycle("post_terminal_resource_boundary", Instant::now(), &Ok(())) + } + + fn health_snapshot_for_nodes( + &self, + expected_nodes: &[u64], + budget: &Budget, + ) -> Result { + let started = Instant::now(); + let result = (|| { + loop { + budget.check("fresh consistent actor, execution, arena and provider health")?; + self.ensure_orchestrator_live()?; + let cursor = self.control_revision(budget)?; + let generation = self.next_observation_generation(); + let actor_request_id = format!("health-actors-{}-{generation}", self.config.seed); + let (fleet, nodes, provider_resources) = thread::scope(|scope| { + let fleet = scope.spawn(|| { + http_json_budget( + "GET", + &format!("{}/api/control/fleet", self.base_url), + None, + budget, + ) + }); + let nodes = scope.spawn(|| self.query_contextual_nodes(expected_nodes, budget)); + let provider = scope.spawn(|| { + if matches!(self.config.provider, HarnessProvider::LocalMock) { + let census = container_census(&self.container_prefix, budget)?; + Ok((census.values().cloned().collect(), Some(census))) + } else { + self.provider_resources_budget(budget) + .map(|resources| (resources, None)) + } + }); + // All bounded operations are joined even if the first result fails. + (fleet.join(), nodes.join(), provider.join()) + }); + let fleet = fleet.map_err(|_| "fleet census collector panicked".to_owned())??; + let nodes = + nodes.map_err(|_| "contextual census collector panicked".to_owned())??; + let (provider_resources, containers) = provider_resources + .map_err(|_| "provider census collector panicked".to_owned())??; + // Query observers must finish before the actor census. They + // are not fixture actors eligible for a baseline exemption. + let actors = http_json_budget( + "GET", + &format!( + "{}/api/control/actors/snapshot?request_id={actor_request_id}", + self.base_url + ), + None, + budget, + )?; + let after = self.control_revision(budget)?; + if cursor != after { + continue; + } + if generation <= self.health_boundary.load(Ordering::Acquire) { + return Err( + "health snapshot crossed a newer terminal/retraction boundary".to_owned(), + ); + } + let running_nodes = fleet + .pointer("/FleetStatus/nodes") + .and_then(Value::as_array) + .ok_or_else(|| format!("health fleet omitted nodes: {fleet}"))? + .iter() + .filter(|node| node["phase"] == "running") + .map(|node| { + node["logical_node_id"] + .as_u64() + .ok_or_else(|| format!("running node omitted identity: {node}")) + }) + .collect::, _>>()?; + let resources = { + let mut census = self + .telemetry_census + .lock() + .map_err(|_| "telemetry census is poisoned".to_owned())?; + census.update(&self.telemetry, budget)?; + census.record_orchestrator_snapshot( + &format!("myelin-orchestrator#{}", self.config.run_id), + &actor_request_id, + &actors, + )?; + for node in &nodes { + let observation = &node.observation; + let node_id = observation.logical_node_id; + let myelin_control_contract::ContextualHealthEvent::LiveExecutions { + resources, + .. + } = &observation.event; + let snapshot = serde_json::to_value(resources) + .map_err(|error| format!("encode typed resource snapshot: {error}"))?; + census.record_resource_snapshot( + &observation.request_id, + node_id, + &snapshot, + )?; + if let Some(expected) = self.node_generation_baseline.get(&node_id) { + let observed = resource_runtime_identity(&snapshot); + if &observed != expected { + return Err(format!( + "retained node {node_id} process/deployment identity changed: expected={expected}, observed={observed}" + )); + } + } + } + census.snapshot_for_nodes(&expected_nodes.iter().copied().collect()) + }; + let provider_hosts = self.provider_host_mapping(&fleet, containers.as_ref())?; + let provider_accounting = match &self.config.provider { + HarnessProvider::VastAiReal { .. } => { + let admission = self.paid_admission()?; + let admitted = admission + .nodes + .iter() + .map(|node| node.node_id) + .collect::>(); + if admitted.len() != 5 + || admission.contracts.keys().copied().collect::>() + != admitted + || admission + .create_reservations + .keys() + .copied() + .collect::>() + != admitted + || admission + .initial_bootstraps + .keys() + .copied() + .collect::>() + != admitted + || !admission.accounting_errors.is_empty() + { + return Err("paid readiness requires five exact observed creations and initial bootstrap admissions".to_owned()); + } + for node in fleet["FleetStatus"]["nodes"] + .as_array() + .into_iter() + .flatten() + { + if node["phase"] != "running" { + continue; + } + let id = node["logical_node_id"] + .as_u64() + .ok_or("fleet node identity")?; + let specification = admission + .nodes + .iter() + .find(|node| node.node_id == id) + .ok_or_else(|| format!("unadmitted running node {id}"))?; + let contract = admission.contracts[&id]; + if node["provider_ref"] != format!("vastai:{contract}") + || node["selected_offer_id"].as_u64() + != Some(specification.offer_id) + || node["runtime"]["run_id"].as_u64() != Some(admission.run_id) + || node["runtime"]["attempt_id"].as_u64() + != Some(admission.create_reservations[&id].attempt_id) + { + return Err(format!( + "node {id} does not match admitted contract/offer/bootstrap identity" + )); + } + } + Some(json!({ + "nodes": admission.nodes, + "contracts": admission.contracts, + "create_reservations": admission.create_reservations, + "initial_bootstraps": admission.initial_bootstraps, + })) + } + _ => None, + }; + if self + .provider_accounting_baseline + .as_ref() + .is_some_and(|expected| provider_accounting.as_ref() != Some(expected)) + { + return Err(format!( + "paid contract/create/bootstrap accounting changed: before={:?}, after={provider_accounting:?}", + self.provider_accounting_baseline + )); + } + return Ok(json!({ + "schema_version": 2, "generation": generation, + "control_revision": cursor, "health_boundary": self.health_boundary.load(Ordering::Acquire), + "elapsed_ns": started.elapsed().as_nanos(), + "actors": actors, "fleet": fleet, "running_nodes": running_nodes, + "nodes": nodes, "provider_resources": provider_resources, "resources": resources, + "provider_accounting": provider_accounting, + "provider_hosts": provider_hosts, + })); + } + })(); + #[derive(serde::Serialize)] + struct HealthTiming<'a> { + control_revision: &'a Value, + namespace_commits: Option<&'a Value>, } - let containers = list_containers(&self.container_prefix)?; - let resources = self - .telemetry_census - .lock() - .unwrap_or_else(|error| error.into_inner()) - .update(&self.telemetry)?; - Ok(json!({ - "actors": actors, - "fleet": fleet, - "running_nodes": running_nodes, - "nodes": nodes, - "containers": containers, - "resources": resources, - })) + self.record_lifecycle( + "fresh_health_snapshot", + started, + &result + .as_ref() + .map(|health| HealthTiming { + control_revision: &health["control_revision"], + namespace_commits: health["actors"].get("namespace_commits"), + }) + .map_err(Clone::clone), + )?; + result } pub fn assert_healthy(&self) -> Result<(), String> { - if self.orchestrator_stopped { - let containers = list_containers(&self.container_prefix)?; - if containers != self.pre_stop_containers { - return Err(format!( - "orchestrator failure changed worker container census: {containers:?}" - )); + self.restore_health(&[]).map(|_| ()) + } + + pub fn assert_healthy_against(&self, retained: &RetainedCaseActors) -> Result<(), String> { + self.restore_health(std::slice::from_ref(retained)) + .map(|_| ()) + } + + fn restore_owned_baseline(&self, excluding: Option<&str>) -> Result { + let retained = self + .active_attempts + .iter() + .filter(|(id, _)| Some(id.as_str()) != excluding) + .map(|(_, attempt)| RetainedCaseActors { + attempt: Arc::clone(attempt), + }) + .collect::>(); + self.restore_health(&retained) + } + + fn restore_health(&self, retained: &[RetainedCaseActors]) -> Result { + let started = Instant::now(); + let budget = self.operation_budget().child(self.config.deadline); + let result = (|| { + let pending = self + .pending_control_processes + .lock() + .map_err(|_| "owned execution tracking is poisoned".to_owned())?; + if !pending.is_empty() { + return Err(format!("owned executions lack terminal proof: {pending:?}")); } - return self.wait_for_no_workload_processes(); - } - let deadline = Instant::now() + self.config.deadline; - loop { - let health = self.health_snapshot()?; - if contains_poison(&health) { - return Err(format!("actor poison or worker panic detected: {health}")); - } - let containers = health - .get("containers") - .and_then(Value::as_array) - .map_or(0, Vec::len); - let running_nodes = health - .get("running_nodes") - .and_then(Value::as_array) - .map_or(0, Vec::len); - if containers != running_nodes { - return Err(format!( - "container census mismatch: expected {running_nodes}, observed {containers}; {health}" - )); - } - if let Some(reason) = pending_resource_cleanup(&health, &self.resource_baseline) { - if Instant::now() >= deadline { - return Err(format!("deadline waiting for {reason}; health={health}")); + drop(pending); + loop { + budget.check("fixture health restoration")?; + let (health, fixture) = thread::scope(|scope| { + let health = + scope.spawn(|| self.health_snapshot_for_nodes(&self.node_ids, &budget)); + let fixture = scope.spawn(|| self.builtin_fixture_blobs(&budget)); + (health.join(), fixture.join()) + }); + let mut health = + health.map_err(|_| "health snapshot collector panicked".to_owned())??; + let fixture = + fixture.map_err(|_| "fixture ownership collector panicked".to_owned())??; + if contains_poison(&health) { + return Err(format!("actor poison or worker panic detected: {health}")); } - thread::sleep(POLL_INTERVAL); - continue; + validate_fixture_census( + &health, + &self.provider_baseline, + &self.node_ids, + &self.fixture_mapping, + &self.stopped_nodes, + )?; + let fixture_actors = retained_actor_identities(&health, &fixture)?; + if fixture_actors != self.fixture_actor_baseline { + return Err(format!( + "built-in fixture actor baseline changed: expected={:?}, observed={fixture_actors:?}", + self.fixture_actor_baseline + )); + } + let mut baseline = fixture_actors; + health["builtin_fixture"] = + serde_json::to_value(&fixture).map_err(|error| error.to_string())?; + for retained in retained { + baseline.extend(self.retained_blob_exemptions(retained, &health, &budget)?); + } + if let Some(reason) = pending_resource_cleanup(&health, &baseline) { + // This predicate advances when the append-only telemetry + // archive changes, not when manual-control state changes. + budget.wait(TELEMETRY_REFRESH_INTERVAL, &reason)?; + continue; + } + self.persist_fixture_identity(&health)?; + return Ok(health); } + })(); + if let Err(error) = &result { + self.quarantine(format!("fixture baseline restoration failed: {error}")); + } + self.record_lifecycle("health_restoration", started, &result)?; + result + } + + pub fn retained_case_actors(&self, case: &BehaviorCase) -> Result { + let attempt = self + .active_attempts + .get(&case.id) + .filter(|attempt| attempt.case == *case && attempt.observation.get().is_some()) + .ok_or_else(|| format!("case {} has no verified retained ownership", case.id))?; + Ok(RetainedCaseActors { + attempt: Arc::clone(attempt), + }) + } + + fn retained_blob_exemptions( + &self, + retained: &RetainedCaseActors, + health: &Value, + budget: &Budget, + ) -> Result, String> { + let attempt = &retained.attempt; + if self + .active_attempts + .get(&attempt.case.id) + .is_none_or(|active| !Arc::ptr_eq(active, attempt)) + { + return Err("retained resource exemption expired with modeled case cleanup".to_owned()); + } + let reply = self.query_retained_blobs(attempt.cleanup_paths.clone(), budget)?; + attempt.persist_retained_blobs(&reply)?; + let observation = attempt + .observation + .get() + .ok_or("retained case omitted verified execution evidence")?; + let blobs = reply + .blobs + .iter() + .map(|(path, blob)| (path.clone(), (blob.revision, blob.length))) + .collect(); + crate::oracle::BehaviorOracle::verify_retained_blobs_with_budget( + &attempt.case, + observation, + &blobs, + budget, + ) + .map_err(|error| format!("retained namespace ownership disagrees with model: {error}"))?; + retained_actor_identities(health, &reply.blobs) + } + + fn builtin_fixture_blobs( + &self, + budget: &Budget, + ) -> Result, String> { + let reply = self.query_retained_blobs(vec![BUILTIN_FIXTURE_PATH.to_owned()], budget)?; + let blob = reply + .blobs + .get(BUILTIN_FIXTURE_PATH) + .ok_or("built-in read-only fixture is no longer a blob")?; + if self + .fixture_blob_baseline + .is_some_and(|expected| expected != (blob.revision, blob.length)) + { + return Err("built-in read-only fixture revision or length changed".to_owned()); + } + Ok(reply.blobs) + } + + fn query_retained_blobs( + &self, + paths: Vec, + budget: &Budget, + ) -> Result { + use myelin_control_contract::{RetainedBlobsReply, RetainedBlobsRequest, SCHEMA_VERSION}; + let request_id = format!( + "retained-{}-{}", + self.config.seed, + self.next_observation_generation() + ); + let request = RetainedBlobsRequest { + schema_version: SCHEMA_VERSION, + request_id: request_id.clone(), + paths, + }; + let reply: RetainedBlobsReply = serde_json::from_value(http_json_budget( + "POST", + &format!("{}/api/control/contextual/retained-blobs", self.base_url), + Some(serde_json::to_value(&request).map_err(|error| error.to_string())?), + budget, + )?) + .map_err(|error| format!("typed retained ownership proof: {error}"))?; + if reply.schema_version != SCHEMA_VERSION + || reply.request_id != request_id + || reply + .blobs + .keys() + .any(|path| reply.non_blobs.contains_key(path)) + || reply + .blobs + .keys() + .chain(reply.non_blobs.keys()) + .cloned() + .collect::>() + != request.paths.iter().cloned().collect() + { + return Err("retained namespace proof omitted or added an owned path".to_owned()); + } + Ok(reply) + } + + /// Orchestrator process id, for caller-side child-process assertions. + pub fn orchestrator_pid(&self) -> u32 { + self.orchestrator.id() + } + + /// Raw fleet status from the control plane. + pub fn fleet_snapshot(&self) -> Result { + http_json_budget( + "GET", + &format!("{}/api/control/fleet", self.base_url), + None, + &self.operation_budget().child(self.config.deadline), + ) + } + + fn ensure_orchestrator_live(&self) -> Result<(), String> { + let mut poll = libc::pollfd { + fd: self.orchestrator_pidfd.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }; + let ready = unsafe { libc::poll(&mut poll, 1, 0) }; + if ready == 0 { return Ok(()); } - } - - fn ensure_orchestrator_live(&mut self) -> Result<(), String> { - match self - .orchestrator - .try_wait() - .map_err(|error| format!("observe orchestrator process: {error}"))? - { - Some(status) => Err(format!( - "orchestrator exited {status}; stdout={:?}; stderr={:?}", - self.stdout - .lock() - .unwrap_or_else(|error| error.into_inner()), - self.stderr - .lock() - .unwrap_or_else(|error| error.into_inner()) - )), - None => Ok(()), - } - } - - fn timeout_evidence(&self, predicate: &str) -> String { - format!( - "deadline waiting for {predicate}; stdout={:?}; stderr={:?}", + Err(format!( + "owned orchestrator incarnation exited or cannot be observed (poll={ready}, events={}): stdout={:?}; stderr={:?}", + poll.revents, self.stdout .lock() .unwrap_or_else(|error| error.into_inner()), self.stderr .lock() .unwrap_or_else(|error| error.into_inner()) - ) + )) + } + fn stop_orchestrator_process_for_cleanup(&mut self, budget: &Budget) -> Result<(), String> { + let mut errors = Vec::new(); + let live = self.orchestrator.try_wait(); + if let Err(error) = &live { + errors.push(format!("observe paid runner: {error}")); + } + if !matches!(live, Ok(Some(_))) { + if let Err(error) = self.orchestrator.kill() { + errors.push(format!("stop paid runner: {error}")); + } + let reap_budget = budget.child( + budget + .remaining("reserve paid runner reap") + .unwrap_or_default() + / 4, + ); + if let Err(error) = + wait_for_child(&mut self.orchestrator, &reap_budget, "reap paid runner") + { + errors.push(error); + } + } + if errors.is_empty() { + Ok(()) + } else { + Err(errors.join("; ")) + } + } + + /// Stop the exact paid orchestrator incarnation before releasing the + /// independent admission owner to perform provider deletion. + pub fn stop_paid_orchestrator_for_cleanup(&mut self, budget: &Budget) -> Result<(), String> { + if !matches!(self.config.provider, HarnessProvider::VastAiReal { .. }) { + return Err("paid cleanup stop phase requires the real provider".to_owned()); + } + let started = Instant::now(); + let result = self.stop_orchestrator_process_for_cleanup(budget); + self.record_lifecycle("paid_orchestrator_stop", started, &result)?; + result + } + + pub fn teardown_with_budget(&mut self, budget: Budget) -> Result<(), String> { + self.fixture_cleanup_budget = budget; + self.teardown() } pub fn teardown(&mut self) -> Result<(), String> { if self.torn_down { return Ok(()); } - let orchestrator_live = self - .orchestrator - .try_wait() - .map_err(|error| format!("observe orchestrator during teardown: {error}"))? - .is_none(); + let started = Instant::now(); + let budget = self.fixture_cleanup_budget.clone(); + let previous = self.cleanup_budget.replace(budget.clone()); + if matches!(&self.config.provider, HarnessProvider::VastAiReal { .. }) { + let admission = match &self.config.provider { + HarnessProvider::VastAiReal { admission, .. } => admission.clone(), + _ => unreachable!("provider checked"), + }; + let mut errors = Vec::new(); + if let Err(error) = self.stop_orchestrator_process_for_cleanup(&budget) { + errors.push(error); + } + // The local runner is no longer allowed to perform provider work + // before the independent durable owner is released to delete. + let admission_result: Result<(), String> = (|| { + if admission.try_exists().map_err(|error| error.to_string())? { + provisioning::PaidFixtureAdmission::open(admission)?.enter_cleanup_only()?; + } + Ok(()) + })(); + if let Err(error) = admission_result { + errors.push(format!("seal paid cleanup ownership: {error}")); + } + // Discovery, exact absence and durable accounting remain required + // even when sealing, signaling or reaping failed. + let resources = self.cleanup_provider_resources(&budget); + self.torn_down = + resources.is_ok() && self.orchestrator.try_wait().ok().flatten().is_some(); + if let Err(error) = resources { + errors.push(error); + } + self.cleanup_budget = previous; + let result = if errors.is_empty() { + Ok(()) + } else { + Err(errors.join("; ")) + }; + self.record_lifecycle("fixture_teardown", started, &result)?; + return result; + } + let mut errors = Vec::new(); + let remaining = budget.remaining("fixture teardown").unwrap_or_default(); + // Stop requests and graceful shutdown must leave time for the independent + // resource owner and its final absence proof. These are allocations, not + // scaled product protocol timers. + let stop_budget = budget.child((remaining / 4).min(self.config.deadline)); let mut recovery_complete = false; - if orchestrator_live { - for node in self.node_ids.clone() { - let _ = self.kill_node(node); - } - let deadline = Instant::now() + self.config.deadline; - while Instant::now() < deadline { - if self.orchestrator.try_wait().ok().flatten().is_some() { - break; - } - let status = http_json( - "GET", - &format!("{}/api/control/status", self.base_url), - None, - ); - if status.as_ref().is_ok_and(|status| { - status - .pointer("/Status/nodes") - .and_then(Value::as_array) - .into_iter() - .flatten() - .all(|node| { - matches!( - node.get("phase").and_then(Value::as_str), - Some("stopped" | "orphan") - ) + let orchestrator_live = self.orchestrator.try_wait().map(|status| status.is_none()); + match orchestrator_live { + Ok(true) => { + let stop_nodes = if self.node_ids.is_empty() { + (1..=u64::from(self.config.node_count)).collect::>() + } else { + self.node_ids.clone() + }; + let stop_errors = thread::scope(|scope| { + let requests = stop_nodes + .iter() + .map(|&node| { + let base_url = &self.base_url; + let seed = self.config.seed; + let request_budget = stop_budget.child(Duration::from_secs(2)); + scope.spawn(move || { + http_json_budget( + "POST", + &format!("{base_url}/api/control/nodes/{node}/kill"), + Some(json!({"command_id": format!("e2e-kill-{seed}-{node}")})), + &request_budget, + ) + .map(|_| ()) + .map_err(|error| format!("stop node {node}: {error}")) + }) }) - }) { - recovery_complete = true; - break; + .collect::>(); + requests + .into_iter() + .filter_map(|request| match request.join() { + Ok(Ok(())) => None, + Ok(Err(error)) => Some(error), + Err(_) => Some("node stop collector panicked".to_owned()), + }) + .collect::>() + }); + errors.extend(stop_errors); + let stopped = (|| { + loop { + let cursor = self.control_revision(&stop_budget)?; + stop_budget + .check("all managed nodes terminal before orchestrator shutdown")?; + if self + .orchestrator + .try_wait() + .map_err(|error| format!("observe stop owner: {error}"))? + .is_some() + { + return Err("orchestrator exited before managed-node terminal census" + .to_owned()); + } + let status = http_json_budget( + "GET", + &format!("{}/api/control/status", self.base_url), + None, + &stop_budget, + )?; + let nodes = status + .pointer("/Status/nodes") + .and_then(Value::as_array) + .ok_or_else(|| format!("shutdown status omitted nodes: {status}"))?; + if nodes.iter().all(|node| { + matches!(node["phase"].as_str(), Some("stopped" | "orphan")) + }) { + return Ok(()); + } + self.wait_for_control_change( + &cursor, + &stop_budget, + &format!("managed-node terminal census: {status}"), + )?; + } + })(); + recovery_complete = stopped.is_ok(); + if let Err(error) = stopped { + errors.push(error); } - thread::sleep(POLL_INTERVAL); } + Ok(false) => {} + Err(error) => errors.push(format!("observe orchestrator during teardown: {error}")), } let mut graceful_shutdown = false; if self.orchestrator.try_wait().ok().flatten().is_none() { - let pid = i32::try_from(self.orchestrator.id()) - .map_err(|_| "orchestrator pid exceeds platform range".to_owned())?; - let signal_result = unsafe { libc::kill(pid, libc::SIGTERM) }; - if signal_result == 0 { - let deadline = Instant::now() + self.config.deadline; - while Instant::now() < deadline { - if self.orchestrator.try_wait().ok().flatten().is_some() { - graceful_shutdown = true; - break; + match i32::try_from(self.orchestrator.id()) { + Ok(pid) => { + if unsafe { libc::kill(pid, libc::SIGTERM) } == 0 { + let grace = budget.child( + (budget + .remaining("orchestrator graceful shutdown") + .unwrap_or_default() + / 4) + .min(Duration::from_secs(10)), + ); + match wait_for_child( + &mut self.orchestrator, + &grace, + "graceful orchestrator reap", + ) { + Ok(()) => graceful_shutdown = true, + Err(error) => errors.push(error), + } + } else { + errors.push(format!( + "signal orchestrator shutdown: {}", + std::io::Error::last_os_error() + )); } - thread::sleep(POLL_INTERVAL); } + Err(_) => errors.push("orchestrator pid exceeds platform range".to_owned()), } } if self.orchestrator.try_wait().ok().flatten().is_none() { - let _ = self.orchestrator.kill(); - let _ = self.orchestrator.wait(); + if let Err(error) = self.orchestrator.kill() { + errors.push(format!("force stop orchestrator: {error}")); + } + if let Err(error) = + wait_for_child(&mut self.orchestrator, &budget, "forced orchestrator reap") + { + errors.push(error); + } } if graceful_shutdown { let checkpoint_exists = self.state_dir.join("cluster.json").exists(); if recovery_complete && checkpoint_exists { - return Err("graceful shutdown retained an empty recovery checkpoint".to_owned()); - } - if !recovery_complete && !checkpoint_exists { - return Err("graceful shutdown discarded an active recovery checkpoint".to_owned()); + errors.push("graceful shutdown retained an empty recovery checkpoint".to_owned()); + } else if !recovery_complete && !checkpoint_exists { + errors.push("graceful shutdown discarded an active recovery checkpoint".to_owned()); } } - remove_containers(&self.container_prefix)?; - let remaining = list_containers(&self.container_prefix)?; - if !remaining.is_empty() { - return Err(format!( - "harness containers remained after teardown: {remaining:?}" - )); + + // Never short-circuit this owner because a stop, reap, or discovery failed. + let resources = self.cleanup_provider_resources(&budget); + self.torn_down = resources.is_ok() && self.orchestrator.try_wait().ok().flatten().is_some(); + if let Err(error) = resources { + errors.push(error); + } + self.cleanup_budget = previous; + let result = if errors.is_empty() { + Ok(()) + } else { + Err(errors.join("; ")) + }; + self.record_lifecycle("fixture_teardown", started, &result)?; + result + } + + fn cleanup_provider_resources(&self, budget: &Budget) -> Result<(), String> { + match &self.config.provider { + HarnessProvider::LocalMock => { + let removal = remove_containers(&self.container_prefix, budget); + let absence = + list_containers(&self.container_prefix, budget).and_then(|remaining| { + if remaining.is_empty() { + Ok(()) + } else { + Err(format!( + "harness containers remained after teardown: {remaining:?}" + )) + } + }); + combine_cleanup_results([removal, absence]) + } + HarnessProvider::StaticSsh { manifest, .. } => { + // Read ownership, not a pre-deletion Docker query: one failed + // inspect must not prevent attempts on any other manifest entry. + let manifest_value: Value = serde_json::from_slice( + &fs::read(manifest) + .map_err(|error| format!("read raw ownership manifest: {error}"))?, + ) + .map_err(|error| format!("decode raw ownership manifest: {error}"))?; + let containers = manifest_value["nodes"] + .as_array() + .ok_or_else(|| "raw ownership manifest omitted nodes".to_owned())? + .iter() + .map(|node| { + node["container"] + .as_str() + .map(str::to_owned) + .ok_or_else(|| { + "raw ownership manifest node omitted container".to_owned() + }) + }) + .collect::, _>>()?; + let removals = thread::scope(|scope| { + let requests = containers + .iter() + .map(|container| scope.spawn(move || remove_container(container, budget))) + .collect::>(); + requests + .into_iter() + .map(|request| { + request.join().unwrap_or_else(|_| { + Err("raw container removal owner panicked".to_owned()) + }) + }) + .collect::>() + }); + let absence = + raw_fleet::read_manifest_containers(manifest, budget).and_then(|remaining| { + if remaining.is_empty() { + Ok(()) + } else { + Err(format!( + "raw fleet containers remained after teardown: {remaining:?}" + )) + } + }); + combine_cleanup_results(removals.into_iter().chain([absence])) + } + HarnessProvider::VastAiReal { + admission: admission_path, + .. + } => { + let admission_absent = !admission_path + .try_exists() + .map_err(|error| format!("inspect paid admission ownership: {error}"))?; + if admission_absent + && !self.config.provision + && self.config.selected_offer_ids.is_empty() + && self.node_ids.is_empty() + && self.provider_baseline.is_empty() + { + let proof = json!({ + "schema_version": 1, "ownership_unstarted": true, + "absent_contract_ids": [], "remaining_contract_ids": [], + "discovered_contract_labels": {}, + "observed_monotonic_ns": self.lifecycle_started.elapsed().as_nanos(), + }); + fs::write( + self.config.artifacts.join("provider-absence.json"), + serde_json::to_vec(&proof).map_err(|error| { + format!("encode unstarted ownership evidence: {error}") + })?, + ) + .map_err(|error| format!("persist unstarted ownership evidence: {error}"))?; + return Ok(()); + } + let owner = provisioning::PaidFixtureAdmission::open(admission_path)?; + let admission = owner.snapshot()?; + if admission.cleanup.is_none() { + return Err("paid cleanup requires the durable independent cleanup owner; recover ownership first".to_owned()); + } + owner.enter_cleanup_only()?; + loop { + budget.check("independent owner exact provider absence")?; + let admission = owner.snapshot()?; + let cleanup = admission + .cleanup + .as_ref() + .ok_or("paid cleanup ownership disappeared")?; + if cleanup.spending_stopped && !cleanup.complete { + return Err(format!( + "paid resources absent but accounting evidence rejected: {:?}", + admission.accounting_errors + )); + } + if cleanup.complete { + let proof = json!({ + "schema_version": 1, "cleanup_owner": cleanup, + "absent_contract_ids": admission.known_contract_ids(), + "remaining_contract_ids": [], + "observed_monotonic_ns": self.lifecycle_started.elapsed().as_nanos(), + }); + fs::write( + self.config.artifacts.join("provider-absence.json"), + serde_json::to_vec(&proof).map_err(|error| error.to_string())?, + ) + .map_err(|error| format!("persist owner absence evidence: {error}"))?; + return Ok(()); + } + budget.wait(POLL_INTERVAL, "independent owner exact provider absence")?; + } + } } - self.torn_down = true; - Ok(()) } } impl Drop for ClusterHarness { fn drop(&mut self) { - let _ = self.teardown(); + // A retained remote fixture still relinquishes its local process owner. + // Failed cleanup reuses the original reserve; Drop cannot extend it. + if self.torn_down { + let _ = self.orchestrator.kill(); + let _ = wait_for_child( + &mut self.orchestrator, + &self.fixture_cleanup_budget, + "retained orchestrator reap", + ); + } else { + let _ = self.teardown(); + } + } +} + +fn validate_control_revision(reply: &Value) -> Result<(), String> { + if reply["schema_version"] != 1 + || reply.pointer("/payload/type").and_then(Value::as_str) != Some("control_revision") + || reply + .pointer("/payload/generation") + .and_then(Value::as_str) + .is_none() + || reply + .pointer("/payload/revision") + .and_then(Value::as_u64) + .is_none() + { + return Err(format!("invalid versioned control revision: {reply}")); + } + Ok(()) +} + +fn combine_cleanup_results( + results: impl IntoIterator>, +) -> Result<(), String> { + let errors = results + .into_iter() + .filter_map(Result::err) + .collect::>(); + if errors.is_empty() { + Ok(()) + } else { + Err(errors.join("; ")) + } +} + +fn wait_for_child(child: &mut Child, budget: &Budget, predicate: &str) -> Result<(), String> { + #[cfg(target_os = "linux")] + let pidfd = { + use std::os::fd::{FromRawFd, OwnedFd}; + let fd = unsafe { libc::syscall(libc::SYS_pidfd_open, child.id(), 0) } as i32; + if fd >= 0 { + Some(unsafe { OwnedFd::from_raw_fd(fd) }) + } else { + None + } + }; + loop { + if child + .try_wait() + .map_err(|error| format!("{predicate}: {error}"))? + .is_some() + { + return Ok(()); + } + let remaining = budget.remaining(predicate)?; + #[cfg(target_os = "linux")] + if let Some(fd) = &pidfd { + use std::os::fd::AsRawFd; + let mut poll = libc::pollfd { + fd: fd.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }; + let milliseconds = remaining.min(POLL_INTERVAL).as_millis().max(1) as i32; + if unsafe { libc::poll(&mut poll, 1, milliseconds) } < 0 { + let error = std::io::Error::last_os_error(); + if error.kind() != std::io::ErrorKind::Interrupted { + return Err(format!( + "{predicate}: child-exit notification failed: {error}" + )); + } + } + continue; + } + budget.wait(remaining.min(POLL_INTERVAL), predicate)?; + } +} + +fn resource_runtime_identity(snapshot: &Value) -> Value { + json!({ + "generation": snapshot["generation"], + "iroh_node_id": snapshot["iroh_node_id"], + "artifact_digest": snapshot["artifact_digest"], + "deployment_generation": snapshot["deployment_generation"], + }) +} + +fn validate_fixture_census( + health: &Value, + expected_resources: &BTreeSet, + expected_nodes: &[u64], + expected_mapping: &BTreeMap, + stopped_nodes: &BTreeSet, +) -> Result<(), String> { + let resources: BTreeSet = serde_json::from_value( + health + .get("provider_resources") + .cloned() + .ok_or_else(|| "health snapshot omitted provider resources".to_owned())?, + ) + .map_err(|error| format!("invalid provider resource census: {error}"))?; + let nodes: Vec = serde_json::from_value( + health + .get("running_nodes") + .cloned() + .ok_or_else(|| "health snapshot omitted running nodes".to_owned())?, + ) + .map_err(|error| format!("invalid running node census: {error}"))?; + let expected_nodes: BTreeSet = expected_nodes.iter().copied().collect(); + let observed_nodes: BTreeSet = nodes.iter().copied().collect(); + if &resources != expected_resources + || observed_nodes != expected_nodes + || nodes.len() != observed_nodes.len() + || resources.len() != nodes.len() + { + return Err(format!( + "fixture identity changed: expected resources={expected_resources:?}, \ + nodes={expected_nodes:?}; observed resources={resources:?}, nodes={nodes:?}" + )); + } + let mapping = fixture_node_mapping(health, &nodes, stopped_nodes)?; + for (node, identity) in &mapping { + if expected_mapping.get(node) != Some(identity) { + return Err(format!( + "node {node} contract/host/runtime association changed: expected {:?}, observed {identity}", + expected_mapping.get(node) + )); + } + } + let mapped_resources = mapping + .values() + .filter_map(|identity| { + identity["provider_host"]["container_id"] + .as_str() + .or_else(|| identity["provider_ref"].as_str()) + }) + .map(|resource| { + resource + .strip_prefix("static-ssh:") + .unwrap_or(resource) + .to_owned() + }) + .collect::>(); + if mapped_resources != resources { + return Err(format!( + "fleet mapping does not own exact provider resources: {mapped_resources:?} != {resources:?}" + )); + } + Ok(()) +} + +fn fixture_node_mapping( + health: &Value, + running: &[u64], + stopped: &BTreeSet, +) -> Result, String> { + use myelin_control_contract::{NodePhase, NodeStatus}; + let nodes: Vec = + serde_json::from_value(health["fleet"]["FleetStatus"]["nodes"].clone()) + .map_err(|error| format!("invalid typed fleet node census: {error}"))?; + let mut seen = BTreeSet::new(); + let mut mapping = BTreeMap::new(); + for node in nodes { + let id = node.logical_node_id; + if !seen.insert(id) { + return Err(format!("duplicate fleet node {id}")); + } + if stopped.contains(&id) { + if node.phase != NodePhase::Stopped { + return Err(format!( + "removed node {id} is not stopped: {:?}", + node.phase + )); + } + } else if !running.contains(&id) || node.phase != NodePhase::Running { + return Err(format!( + "unexpected node or replacement/bootstrap intent for node {id}: {:?}", + node.phase + )); + } else { + let provider_ref = node + .provider_ref + .filter(|value| !value.is_empty()) + .ok_or_else(|| format!("node {id} omitted provider resource"))?; + let runtime = node + .runtime + .ok_or_else(|| format!("node {id} omitted live runtime identity"))?; + mapping.insert( + id, + json!({ + "provider_ref": provider_ref, "selected_offer_id": node.selected_offer_id, + "provider_host": health["provider_hosts"][id.to_string()].clone(), + "run_id": runtime.run_id, "attempt_id": runtime.attempt_id, + "endpoint": runtime.endpoint, "node_actor": runtime.node_actor, + "swim_node_id": runtime.swim_node_id, "readiness_id": runtime.readiness_id, + }), + ); + } + } + if seen + != running + .iter() + .copied() + .chain(stopped.iter().copied()) + .collect() + { + return Err("fleet omitted an expected running or stopped node".to_owned()); + } + Ok(mapping) +} + +fn retained_actor_identities( + health: &Value, + blobs: &BTreeMap, +) -> Result, String> { + let actors = health + .pointer("/resources/active_actors") + .and_then(Value::as_array) + .ok_or("retained ownership requires a live actor census")?; + let mut identities = BTreeSet::new(); + for blob in blobs.values() { + for (address, expected_type) in + std::iter::once((&blob.source, "data_plane::source::FileBlobSourceActor")).chain( + blob.binding + .as_ref() + .map(|binding| (binding, "data_plane::host::HostBlobBindingActor")), + ) + { + if address.len() != 64 || !address.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err("retained source proof omitted a complete actor identity".to_owned()); + } + let mut matches = actors.iter().filter(|actor| { + actor["identity"] + .as_str() + .and_then(|identity| identity.rsplit_once('/')) + .is_some_and(|(_, observed)| observed == address) + }); + let actor = matches + .next() + .ok_or_else(|| format!("retained {expected_type} {address} is absent"))?; + if matches.next().is_some() || actor["type"] != expected_type { + return Err(format!( + "retained actor {address} is duplicated or has the wrong type" + )); + } + identities.insert( + actor["identity"] + .as_str() + .ok_or("retained actor identity")? + .to_owned(), + ); + } + } + Ok(identities) +} + +#[cfg(test)] +mod fixture_census_tests { + use super::*; + + #[test] + fn expired_child_wait_preserves_owner_for_reserved_cleanup() { + let owner = Budget::new(Duration::from_secs(2)); + let cleanup = owner.child(Duration::from_secs(2)); + let execution = owner.child(Duration::ZERO); + let mut child = Command::new("sleep") + .arg("30") + .spawn() + .expect("start parked child"); + let expired = wait_for_child(&mut child, &execution, "withheld process exit"); + let killed = child.kill(); + let reaped = wait_for_child(&mut child, &cleanup, "reserved process reap"); + assert!(expired.is_err()); + assert!(killed.is_ok()); + assert!(reaped.is_ok()); + assert!(child.try_wait().expect("observe reaped child").is_some()); + } + + #[test] + fn control_reconciliation_rejects_unversioned_and_incomplete_cursors() { + assert!(validate_control_revision(&json!({ + "schema_version": 1, "payload": {"type": "control_revision", "generation": "boot-a", "revision": 7}, + })).is_ok()); + assert!(validate_control_revision(&json!({ + "schema_version": 2, "payload": {"type": "control_revision", "generation": "boot-a", "revision": 7}, + })).is_err()); + assert!( + validate_control_revision(&json!({ + "schema_version": 1, "payload": {"type": "control_revision", "revision": 7}, + })) + .is_err() + ); + } + + fn fleet_node(id: u64, resource: u64, phase: &str) -> Value { + json!({ + "logical_node_id": id, "provider_ref": format!("vastai:{resource}"), + "selected_offer_id": id+1000, "phase": phase, "last_error": null, "last_seen_unix_ms": 1, + "runtime": {"run_id": 9, "attempt_id": 1, "endpoint": format!("host-{id}"), + "node_actor": vec![id as u8; 32], "swim_node_id": vec![id as u8; 32], + "stage_index": 0, "readiness_id": id}, + }) + } + + fn original_fleet() -> Value { + json!({"provider_resources": ["vastai:101", "vastai:102"], + "running_nodes": [1, 3], + "fleet": {"FleetStatus": {"nodes": [fleet_node(1, 101, "running"), fleet_node(3, 102, "running")]}}}) + } + + #[test] + fn docker_census_resolves_names_without_accepting_swaps_or_replacements() { + let mut original = original_fleet(); + let first = "a".repeat(64); + let second = "b".repeat(64); + original["provider_resources"] = json!([first, second]); + original["fleet"]["FleetStatus"]["nodes"][0]["provider_ref"] = json!("node-one"); + original["fleet"]["FleetStatus"]["nodes"][1]["provider_ref"] = json!("node-three"); + original["provider_hosts"] = json!({ + "1": {"container": "node-one", "container_id": first}, + "3": {"container": "node-three", "container_id": second}, + }); + let resources = BTreeSet::from([first.clone(), second.clone()]); + let mapping = fixture_node_mapping(&original, &[1, 3], &BTreeSet::new()).unwrap(); + let validate = |health: &Value| { + validate_fixture_census(health, &resources, &[1, 3], &mapping, &BTreeSet::new()) + }; + assert!(validate(&original).is_ok()); + let mut swapped = original.clone(); + swapped["provider_hosts"]["1"]["container_id"] = json!(second); + swapped["provider_hosts"]["3"]["container_id"] = json!(first); + assert!(validate(&swapped).is_err()); + let replacement = "c".repeat(64); + original["provider_hosts"]["1"]["container_id"] = json!(replacement); + original["provider_resources"] = json!([replacement, second]); + assert!(validate(&original).is_err()); + } + + #[test] + fn retained_fixture_rejects_swapped_mapping_and_new_bootstrap_identity() { + let original = original_fleet(); + let resources = BTreeSet::from(["vastai:101".to_owned(), "vastai:102".to_owned()]); + let mapping = fixture_node_mapping(&original, &[1, 3], &BTreeSet::new()).unwrap(); + let validate = |health: &Value| { + validate_fixture_census(health, &resources, &[1, 3], &mapping, &BTreeSet::new()) + }; + assert!(validate(&original).is_ok()); + let mut swapped = original.clone(); + swapped["fleet"]["FleetStatus"]["nodes"][0]["provider_ref"] = json!("vastai:102"); + swapped["fleet"]["FleetStatus"]["nodes"][1]["provider_ref"] = json!("vastai:101"); + assert!(validate(&swapped).is_err()); + let mut bootstrap = original.clone(); + bootstrap["fleet"]["FleetStatus"]["nodes"][0]["runtime"]["attempt_id"] = json!(2); + assert!(validate(&bootstrap).is_err()); + let mut replacement = original; + replacement["fleet"]["FleetStatus"]["nodes"] + .as_array_mut() + .unwrap() + .push(fleet_node(4, 103, "bootstrapping")); + assert!(validate(&replacement).is_err()); + } + + #[test] + fn destructive_census_requires_exact_target_stopped_and_survivors_unchanged() { + let original = original_fleet(); + let mapping = fixture_node_mapping(&original, &[1, 3], &BTreeSet::new()).unwrap(); + let resources = BTreeSet::from(["vastai:102".to_owned()]); + let stopped = BTreeSet::from([1]); + let validate = + |health: &Value| validate_fixture_census(health, &resources, &[3], &mapping, &stopped); + let mut removed = original; + removed["provider_resources"] = json!(["vastai:102"]); + removed["running_nodes"] = json!([3]); + removed["fleet"]["FleetStatus"]["nodes"][0]["phase"] = json!("stopped"); + assert!(validate(&removed).is_ok()); + let mut wrong = removed.clone(); + wrong["provider_resources"] = json!(["vastai:101"]); + assert!(validate(&wrong).is_err()); + let mut orphan = removed.clone(); + orphan["fleet"]["FleetStatus"]["nodes"][0]["phase"] = json!("orphan"); + assert!(validate(&orphan).is_err()); + let mut changed_survivor = removed; + changed_survivor["fleet"]["FleetStatus"]["nodes"][1]["runtime"]["endpoint"] = + json!("different-host"); + assert!(validate(&changed_survivor).is_err()); + } + + #[test] + fn retained_blob_exempts_only_exact_owned_source_and_binding() { + let source = "a".repeat(64); + let binding = "b".repeat(64); + let leaked = "c".repeat(64); + let blobs = BTreeMap::from([( + "/cases/owned/blob".to_owned(), + myelin_control_contract::RetainedBlob { + source: source.clone(), + binding: Some(binding.clone()), + source_node: [0; 32], + length: 3, + revision: 8, + }, + )]); + let mut reply = + crate::resources::resource_deadline_tests::health_reply("retained", 1, 1, 0); + reply["observation"]["event"]["resources"]["actors"] = json!([ + {"address": source, "actor_type": "data_plane::source::FileBlobSourceActor", "worker_id": 0, "mailbox_depth": 0, "poisoned": false, "stopping": false}, + {"address": binding, "actor_type": "data_plane::host::HostBlobBindingActor", "worker_id": 0, "mailbox_depth": 0, "poisoned": false, "stopping": false} + ]); + let mut census = TelemetryResourceCensus::default(); + census + .record_resource_snapshot("retained", 1, &reply["observation"]["event"]["resources"]) + .unwrap(); + let mut health = json!({"schema_version": 2, "generation": 2, "health_boundary": 1, + "running_nodes": [1], "nodes": [reply], "resources": census.snapshot()}); + let exempt = retained_actor_identities(&health, &blobs).unwrap(); + assert_eq!(pending_resource_cleanup(&health, &exempt), None); + assert!(pending_resource_cleanup(&health, &BTreeSet::new()).is_some()); + health["resources"]["active_actors"].as_array_mut().unwrap().push(json!({ + "identity": format!("1#2/{leaked}"), "type": "data_plane::source::FileBlobSourceActor", + })); + let exempt = retained_actor_identities(&health, &blobs).unwrap(); + assert!(pending_resource_cleanup(&health, &exempt).is_some()); } } diff --git a/tools/myelin-e2e-fuzz/src/harness/raw_fleet.rs b/tools/myelin-e2e-fuzz/src/harness/raw_fleet.rs new file mode 100644 index 0000000..b627b48 --- /dev/null +++ b/tools/myelin-e2e-fuzz/src/harness/raw_fleet.rs @@ -0,0 +1,2402 @@ +//! Raw Docker fleet fixture for the deployment E2E. +//! +//! Blank nodes: each container is created from the production base image +//! (`apps/myelin/node-image/Dockerfile.base`) with no Myelin binaries, no +//! Swactor wheel, and no E2E launcher. Each node lives on its own Docker +//! bridge, so no shared Docker LAN exists between nodes and cross-bridge +//! forwarding is blocked by Docker's inter-network isolation. The only way +//! in is the published per-node SSH port on loopback. +//! +//! The fixture owns node lifecycle: `ensure` creates or rediscovers the +//! containers (stable fixture/slot labels survive harness crashes), +//! `teardown` removes them. Refresh means destroy-and-recreate, never +//! incremental repair of half-booted state. The durable ownership manifest is +//! authoritative across runner restarts; names alone never establish ownership. + +use std::collections::{BTreeMap, BTreeSet}; +use std::io::{Read, Seek}; +use std::net::TcpStream; +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; +use std::os::unix::process::CommandExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output, Stdio}; +use std::sync::{Mutex, OnceLock}; +use std::time::{Duration, Instant}; + +use serde::{Deserialize, Serialize}; + +use crate::budget::{Budget, record_execution_stage}; + +pub(crate) const BASE_IMAGE: &str = "myelin-node-base:cuda12.6"; + +const SSH_BANNER_TIMEOUT: Duration = Duration::from_secs(30); +const SSH_BANNER_POLL: Duration = Duration::from_millis(200); + +#[derive(Clone, Debug)] +pub struct RawNode { + pub slot: u32, + pub container: String, + pub network: String, + pub ssh_host: String, + pub ssh_port: u16, + container_claim: RawResourceClaim, + network_claim: RawResourceClaim, +} + +pub struct RawDockerFleet { + pub nodes: Vec, + pub identity: PathBuf, + pub prefix: String, + prior_processes: Mutex>>, + budget: Budget, + cleanup_budget: Budget, + manifest: RawManifestStore, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RawFixtureNodeIdentity { + pub logical_node_id: u64, + pub slot: u32, + pub container: String, + pub container_id: String, + pub network: String, + pub network_id: String, + pub ssh_host: String, + pub ssh_port: u16, +} + +#[derive(Clone, Debug, Default)] +struct RawResourceClaim { + // A create reservation permits label-based reconciliation, not name-based + // deletion. Once observed, the immutable provider ID can never be replaced. + creation: Option, + id: OnceLock, +} + +#[derive(Clone, Copy, Debug)] +enum RawResourceKind { + Container, + Network, +} + +#[derive(Debug)] +struct RawResourceObservation { + id: String, + name: String, + labels: BTreeMap, +} + +impl RawResourceClaim { + fn reserve_create(&mut self) -> Result<&str, String> { + let mut nonce = [0_u8; 16]; + std::fs::File::open("/dev/urandom") + .and_then(|mut file| file.read_exact(&mut nonce)) + .map_err(|error| format!("reserve raw resource creation identity: {error}"))?; + self.creation = Some(format!("{:032x}", u128::from_ne_bytes(nonce))); + Ok(self + .creation + .as_deref() + .expect("creation identity just reserved")) + } + + fn verify( + &self, + observed: &RawResourceObservation, + fixture: &str, + slot: u32, + ) -> Result<(), String> { + if observed.id.len() != 64 || !observed.id.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(format!( + "raw resource {} has an invalid immutable ID", + observed.name + )); + } + if observed + .labels + .get("myelin.raw-fixture") + .map(String::as_str) + != Some(fixture) + || observed.labels.get("myelin.raw-slot") != Some(&slot.to_string()) + || self.creation.as_ref().is_some_and(|creation| { + observed.labels.get("myelin.raw-creation") != Some(creation) + }) + { + return Err(format!( + "raw resource {} ({}) does not belong to fixture {fixture} slot {slot}", + observed.name, observed.id + )); + } + if self.id.get().is_some_and(|id| id != &observed.id) { + return Err(format!( + "raw resource {} was replaced: retained {:?}, observed {}", + observed.name, + self.id.get(), + observed.id + )); + } + Ok(()) + } + + fn adopt( + &self, + observed: &RawResourceObservation, + fixture: &str, + slot: u32, + ) -> Result<&str, String> { + self.verify(observed, fixture, slot)?; + // Cleanup and construction may both reconcile a reserved creation. + // Never allow a losing observation to replace the first accepted ID. + self.id.get_or_init(|| observed.id.clone()); + self.verify(observed, fixture, slot)?; + self.retained_id() + } + + fn retained_id(&self) -> Result<&str, String> { + self.id + .get() + .map(String::as_str) + .ok_or_else(|| "raw resource has no verified immutable identity".to_owned()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RawFleetSnapshot { + pub nodes: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RawProcessFact { + pub pid: u32, + pub start_ticks: u64, + pub parent_pid: u32, + pub process_group: u32, + pub command: String, + pub executable_digest: Option, + pub artifact_digest: Option, + pub deployment_generation: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RawNodeCensus { + pub logical_node_id: u64, + pub container: String, + pub active_deployment_raw: Option, + pub active_link_target: Option, + pub active_executable_digest: Option, + pub agent_pid: Option, + pub worker_processes: Vec, + pub worker_group_processes: Vec, + pub prior_processes: Vec, + pub surviving_prior_processes: Vec, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RawPriorState { + NoWorkerOrTrustworthyMetadata, + HealthyStaleWorker, + DeadWorkerWithStaleMetadata, + MultipleStaleWorkersAndDescendants, + InterruptedTransferAndPartialInstall, + CorruptActiveBinary, + CorruptActivationPointer, + CorruptDeploymentDescriptor, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DeploymentBoundary { + TransferStarted, + BeforeLaunch, + AfterLaunch, + BeforeReceipt, +} + +impl DeploymentBoundary { + pub fn as_remote_name(self) -> &'static str { + match self { + Self::TransferStarted => "transfer_started", + Self::BeforeLaunch => "before_launch", + Self::AfterLaunch => "after_launch", + Self::BeforeReceipt => "before_receipt", + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct ManifestClaim { + creation: String, + attempted: bool, + id: Option, +} + +impl ManifestClaim { + fn new() -> Result { + let mut claim = RawResourceClaim::default(); + Ok(Self { + creation: claim.reserve_create()?.to_owned(), + attempted: false, + id: None, + }) + } + + fn retained(&self) -> RawResourceClaim { + RawResourceClaim { + creation: self.attempted.then(|| self.creation.clone()), + id: self.id.clone().map(OnceLock::from).unwrap_or_default(), + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct ManifestNode { + slot: u32, + container: String, + network: String, + ssh_host: String, + ssh_port: u16, + container_claim: ManifestClaim, + network_claim: ManifestClaim, +} + +impl ManifestNode { + fn retained(&self) -> RawNode { + RawNode { + slot: self.slot, + container: self.container.clone(), + network: self.network.clone(), + ssh_host: self.ssh_host.clone(), + ssh_port: self.ssh_port, + container_claim: self.container_claim.retained(), + network_claim: self.network_claim.retained(), + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Manifest { + schema_version: u32, + fixture: String, + base_image: String, + image_id: String, + public_key: String, + ready: bool, + retired: bool, + nodes: Vec, +} + +impl Manifest { + fn fresh(node_count: u32, image_id: String, public_key: String) -> Result { + let fixture = format!("myelin-e2e-raw-{}", ManifestClaim::new()?.creation); + let nodes = (0..node_count) + .map(|slot| { + Ok(ManifestNode { + slot, + container: format!("{fixture}-node-{slot}"), + network: format!("{fixture}-net-{slot}"), + ssh_host: "127.0.0.1".to_owned(), + ssh_port: 0, + container_claim: ManifestClaim::new()?, + network_claim: ManifestClaim::new()?, + }) + }) + .collect::>()?; + let manifest = Self { + schema_version: 1, + fixture, + base_image: BASE_IMAGE.to_owned(), + image_id, + public_key, + ready: false, + retired: false, + nodes, + }; + manifest.validate()?; + Ok(manifest) + } + + fn load(path: &Path) -> Result, String> { + let text = match std::fs::read_to_string(path) { + Ok(text) => text, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(format!( + "read raw-fleet manifest {}: {error}", + path.display() + )); + } + }; + let manifest: Self = serde_json::from_str(&text).map_err(|error| format!( + "unsupported or ambiguous raw-fleet manifest {}; retained without replacement: {error}", + path.display(), + ))?; + manifest.validate()?; + Ok(Some(manifest)) + } + + fn validate(&self) -> Result<(), String> { + let hex = |value: &str, len| { + value.len() == len && value.bytes().all(|byte| byte.is_ascii_hexdigit()) + }; + if self.schema_version != 1 + || !self + .fixture + .strip_prefix("myelin-e2e-raw-") + .is_some_and(|nonce| hex(nonce, 32)) + || self.base_image != BASE_IMAGE + || !self + .image_id + .strip_prefix("sha256:") + .is_some_and(|id| hex(id, 64)) + || self.public_key.is_empty() + || self.nodes.is_empty() + { + return Err( + "unsupported raw-fleet identity/configuration; manifest retained".to_owned(), + ); + } + let mut nonces = BTreeSet::new(); + let mut ids = BTreeSet::new(); + for (slot, node) in self.nodes.iter().enumerate() { + if node.slot as usize != slot + || node.container != format!("{}-node-{slot}", self.fixture) + || node.network != format!("{}-net-{slot}", self.fixture) + || node.ssh_host != "127.0.0.1" + || (self.ready && node.ssh_port == 0) + || (node.ssh_port != 0 && node.container_claim.id.is_none()) + { + return Err( + "raw-fleet slot/endpoint configuration changed; manifest retained".to_owned(), + ); + } + for claim in [&node.container_claim, &node.network_claim] { + if !hex(&claim.creation, 32) + || !nonces.insert(&claim.creation) + || (claim.id.is_some() && !claim.attempted) + || (self.ready && claim.id.is_none()) + || claim + .id + .as_ref() + .is_some_and(|id| !hex(id, 64) || !ids.insert(id)) + { + return Err( + "ambiguous raw-fleet resource ownership; manifest retained".to_owned() + ); + } + } + } + Ok(()) + } + + fn require_configuration(&self, node_count: u32, public_key: &str) -> Result<(), String> { + if self.nodes.len() != node_count as usize || self.public_key != public_key { + return Err( + "retained raw-fleet node count or SSH identity differs; manifest retained" + .to_owned(), + ); + } + Ok(()) + } + + fn persist(&self, path: &Path) -> Result<(), String> { + let parent = path + .parent() + .ok_or_else(|| "raw-fleet manifest has no parent".to_owned())?; + let mut temporary = tempfile::NamedTempFile::new_in(parent) + .map_err(|error| format!("create raw-fleet manifest temporary file: {error}"))?; + serde_json::to_writer_pretty(temporary.as_file_mut(), self) + .map_err(|error| format!("serialize raw-fleet manifest: {error}"))?; + temporary + .as_file() + .sync_all() + .map_err(|error| format!("sync raw-fleet manifest: {error}"))?; + temporary + .persist(path) + .map_err(|error| format!("commit raw-fleet manifest: {error}"))?; + std::fs::File::open(parent) + .and_then(|directory| directory.sync_all()) + .map_err(|error| format!("sync raw-fleet manifest directory: {error}")) + } +} + +struct RawManifestStore { + path: PathBuf, + state: Mutex, + // The open description owns the flock until this fleet is dropped. + _lock: std::fs::File, +} + +impl RawManifestStore { + fn lock(artifacts: &Path) -> Result { + let lock = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(artifacts.join("raw-fleet.lock")) + .map_err(|error| format!("open raw-fleet ownership lock: {error}"))?; + if unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } != 0 { + return Err(format!( + "raw-fleet artifacts already owned or unavailable: {}", + std::io::Error::last_os_error() + )); + } + Ok(lock) + } + + fn update( + &self, + change: impl FnOnce(&mut Manifest) -> Result<(), String>, + ) -> Result<(), String> { + let mut manifest = self + .state + .lock() + .map_err(|_| "raw-fleet manifest lock poisoned".to_owned())?; + change(&mut manifest)?; + manifest.validate()?; + manifest.persist(&self.path) + } + + fn begin_creation(&self, slot: u32, kind: RawResourceKind) -> Result { + let mut nonce = String::new(); + self.update(|manifest| { + let node = &mut manifest.nodes[slot as usize]; + let claim = match kind { + RawResourceKind::Container => &mut node.container_claim, + RawResourceKind::Network => &mut node.network_claim, + }; + if claim.attempted || claim.id.is_some() { + return Err("refusing to replace a previously attempted raw resource".to_owned()); + } + claim.attempted = true; + nonce.clone_from(&claim.creation); + Ok(()) + })?; + Ok(nonce) + } + + fn record(&self, node: &RawNode) -> Result<(), String> { + self.update(|manifest| { + let retained = &mut manifest.nodes[node.slot as usize]; + for (saved, claim) in [ + (&mut retained.container_claim, &node.container_claim), + (&mut retained.network_claim, &node.network_claim), + ] { + if let Some(id) = claim.id.get() { + if !saved.attempted + || claim.creation.as_ref() != Some(&saved.creation) + || saved.id.as_ref().is_some_and(|saved| saved != id) + { + return Err( + "refusing to overwrite retained raw resource ownership".to_owned() + ); + } + saved.id = Some(id.clone()); + } + } + retained.ssh_port = node.ssh_port; + Ok(()) + }) + } +} + +/// The owner retains the child until it has been killed and reaped. Output +/// goes to anonymous files, so a descendant cannot strand a pipe-reader join. +fn command_output( + command: &mut Command, + budget: &Budget, + predicate: &str, +) -> Result { + let started = Instant::now(); + budget.check(predicate)?; + let mut stdout = tempfile::tempfile().map_err(|error| format!("{predicate}: {error}"))?; + let mut stderr = tempfile::tempfile().map_err(|error| format!("{predicate}: {error}"))?; + command.process_group(0); + command.stdout(Stdio::from( + stdout.try_clone().map_err(|error| error.to_string())?, + )); + command.stderr(Stdio::from( + stderr.try_clone().map_err(|error| error.to_string())?, + )); + let mut child = command + .spawn() + .map_err(|error| format!("{predicate}: {error}"))?; + let fd = unsafe { libc::syscall(libc::SYS_pidfd_open, child.id(), 0) as i32 }; + let exit = (fd >= 0).then(|| unsafe { OwnedFd::from_raw_fd(fd) }); + let status = loop { + match child.try_wait() { + Ok(Some(status)) => break Ok(status), + Ok(None) => {} + Err(error) => break Err(format!("{predicate}: wait: {error}")), + } + let remaining = match budget.remaining(predicate) { + Ok(remaining) => remaining, + Err(error) => break Err(error), + }; + let delay = remaining.min(Duration::from_millis(50)); + if let Some(exit) = &exit { + let mut descriptor = libc::pollfd { + fd: exit.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }; + unsafe { + libc::poll(&mut descriptor, 1, delay.as_millis().max(1) as i32); + } + } else if let Err(error) = budget.wait(delay, predicate) { + break Err(error); + } + }; + if status.is_err() { + unsafe { + libc::kill(-(child.id() as i32), libc::SIGKILL); + } + let _ = child.kill(); + child + .wait() + .map_err(|error| format!("{predicate}: reap: {error}"))?; + } + if status.is_err() { + record_execution_stage(predicate, started.elapsed(), 0, 1); + } + let status = status?; + stdout.rewind().map_err(|error| error.to_string())?; + stderr.rewind().map_err(|error| error.to_string())?; + let mut output = Output { + status, + stdout: Vec::new(), + stderr: Vec::new(), + }; + stdout + .read_to_end(&mut output.stdout) + .map_err(|error| error.to_string())?; + stderr + .read_to_end(&mut output.stderr) + .map_err(|error| error.to_string())?; + record_execution_stage( + predicate, + started.elapsed(), + (output.stdout.len() + output.stderr.len()) as u64, + 1, + ); + budget.check(predicate)?; + Ok(output) +} + +fn run_checked(command: &mut Command, predicate: &str, budget: &Budget) -> Result<(), String> { + let output = command_output(command, budget, predicate)?; + if output.status.success() { + Ok(()) + } else { + Err(format!( + "{predicate}: {}: {}", + output.status, + String::from_utf8_lossy(&output.stderr) + )) + } +} + +fn docker(args: &[&str], budget: &Budget) -> Result { + let output = command_output( + Command::new("docker").args(args), + budget, + &format!("raw.docker.{}", args.first().copied().unwrap_or("command")), + )?; + if output.status.success() { + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) + } else { + Err(format!( + "docker {:?} exited {}: {}", + args, + output.status, + String::from_utf8_lossy(&output.stderr).trim() + )) + } +} + +fn docker_shell(container: &str, script: &str, budget: &Budget) -> Result { + // Docker killing its client does not terminate an exec inside a container. + // The remote timeout therefore owns every helper independently as well. + let seconds = budget + .remaining("raw Docker exec")? + .as_secs_f64() + .to_string(); + docker( + &[ + "exec", container, "timeout", "-s", "KILL", &seconds, "sh", "-lc", script, + ], + budget, + ) + .map_err(|error| format!("raw node {container}: {error}")) +} + +fn parse_optional_u32(value: &str, description: &str) -> Result, String> { + let value = value.trim(); + if value.is_empty() { + return Ok(None); + } + value + .parse::() + .map(Some) + .map_err(|error| format!("parse {description} {value:?}: {error}")) +} + +fn validate_remote_token(value: &str, description: &str) -> Result<(), String> { + if value.is_empty() + || !value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b':')) + { + return Err(format!("{description} {value:?} is not a safe token")); + } + Ok(()) +} + +fn stop_worker_trees_script() -> &'static str { + r#"python3 - <<'PY' || exit $? +import os, select, signal, time +from pathlib import Path +def stat(pid): + try: + text = Path('/proc', str(pid), 'stat').read_text() + name, fields = text[text.index('(') + 1:].rsplit(')', 1) + fields = fields.split() + return int(fields[19]), int(fields[1]), int(fields[2]), name, fields[0] + except (FileNotFoundError, ProcessLookupError): + return None +captured = {} +deadline = time.monotonic() + 10 +try: + while True: + before = len(captured) + processes = {int(p.name): stat(int(p.name)) for p in Path('/proc').iterdir() if p.name.isdecimal()} + groups = {v[2] for v in processes.values() if v and v[3] == 'myelin-worker'} + for pid, value in processes.items(): + if not value or pid <= 1 or pid == os.getpid(): + continue + if value[3] == 'myelin-worker' or value[2] in groups or value[1] in {p for p, _ in captured}: + key = pid, value[0] + if key not in captured: + try: + fd = os.pidfd_open(pid) + except ProcessLookupError: + continue + current = stat(pid) + if current is None or current[0] != value[0]: + os.close(fd) + continue + captured[key] = fd + try: + signal.pidfd_send_signal(fd, signal.SIGSTOP) + except ProcessLookupError: + pass + stopped = all((current := stat(pid)) is None or current[0] != ticks + or current[4] in ('T', 't', 'Z', 'X') for pid, ticks in captured) + if len(captured) == before and stopped: + break + if time.monotonic() >= deadline: + raise TimeoutError('stale worker tree capture') + for fd in captured.values(): + try: + signal.pidfd_send_signal(fd, signal.SIGKILL) + except ProcessLookupError: + pass + pending = set(captured) + while pending: + pending = {key for key in pending if (value := stat(key[0])) and value[0] == key[1]} + if not pending: + break + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError('stale process incarnations remain: %r' % sorted(pending)) + # pidfds wake on exit; /proc reconciliation additionally proves reaping. + poll = select.poll() + for key in pending: + poll.register(captured[key], select.POLLIN) + if poll.poll(min(remaining, .05) * 1000): + time.sleep(min(remaining, .005)) +finally: + for fd in captured.values(): + try: + signal.pidfd_send_signal(fd, signal.SIGKILL) + except ProcessLookupError: + pass + os.close(fd) +PY +:"# +} + +fn process_census(container: &str, budget: &Budget) -> Result, String> { + let output = docker_shell( + container, + r#"python3 - <<'PY' +import hashlib +import json +from pathlib import Path + +processes = [] +for directory in Path("/proc").iterdir(): + if not directory.name.isdecimal(): + continue + try: + stat = (directory / "stat").read_text() + fields = stat.rsplit(")", 1)[1].split() + command = stat.split("(", 1)[1].rsplit(")", 1)[0] + executable_digest = None + artifact_digest = None + deployment_generation = None + if command == "myelin-worker": + try: + with (directory / "exe").open("rb") as executable: + executable_digest = "sha256:" + hashlib.file_digest(executable, "sha256").hexdigest() + except (FileNotFoundError, ProcessLookupError): + # A zombie still has an incarnation even though exe is gone. + pass + try: + for entry in (directory / "environ").read_bytes().split(b'\0'): + if entry.startswith(b'MYELIN_ARTIFACT_DIGEST=sha256:') and len(entry) > 30: + artifact_digest = entry.split(b'=', 1)[1].decode('utf-8', errors='replace') + elif entry.startswith(b'MYELIN_DEPLOYMENT_GENERATION=') and len(entry) > 29: + deployment_generation = entry.split(b'=', 1)[1].decode('utf-8', errors='replace') + except (FileNotFoundError, ProcessLookupError, PermissionError): + pass + current = (directory / "stat").read_text().rsplit(")", 1)[1].split() + if fields[19] != current[19]: + continue + processes.append({ + "pid": int(directory.name), + "start_ticks": int(fields[19]), + "parent_pid": int(fields[1]), + "process_group": int(fields[2]), + "command": command, + "executable_digest": executable_digest, + "artifact_digest": artifact_digest, + "deployment_generation": deployment_generation, + }) + except (FileNotFoundError, ProcessLookupError): + continue +print(json.dumps(processes)) +PY"#, + budget, + )?; + serde_json::from_str(&output) + .map_err(|error| format!("parse raw process census for {container}: {error}")) +} + +fn related_worker_processes( + processes: &[RawProcessFact], + prior: &[RawProcessFact], +) -> Vec { + let mut groups = processes + .iter() + .filter(|process| { + matches!(process.command.as_str(), "myelin-worker" | "swactor") + || (process.artifact_digest.is_some() && process.deployment_generation.is_some()) + }) + .map(|process| process.process_group) + .collect::>(); + for old in prior { + let leader = processes + .iter() + .find(|process| process.pid == old.process_group); + if leader.is_none_or(|leader| { + prior.iter().any(|original| { + original.pid == leader.pid && original.start_ticks == leader.start_ticks + }) + }) { + groups.insert(old.process_group); + } + } + let mut related = processes + .iter() + .filter(|process| { + groups.contains(&process.process_group) + || prior + .iter() + .any(|old| old.pid == process.pid && old.start_ticks == process.start_ticks) + }) + .map(|process| process.pid) + .collect::>(); + loop { + let mut changed = false; + for process in processes { + if related.contains(&process.parent_pid) && related.insert(process.pid) { + changed = true; + } + } + if !changed { + break; + } + } + let mut related = processes + .iter() + .filter(|process| related.contains(&process.pid)) + .cloned() + .collect::>(); + related.sort_by_key(|process| (process.pid, process.start_ticks)); + related +} + +fn container_running(container: &str, budget: &Budget) -> Result { + let output = command_output( + Command::new("docker").args(["inspect", "-f", "{{.State.Running}}", container]), + budget, + "inspect raw container running", + )?; + if !output.status.success() { + return Err(format!( + "docker inspect {container} exited {}", + output.status + )); + } + Ok(String::from_utf8_lossy(&output.stdout).trim() == "true") +} + +fn container_has_init(container: &str, budget: &Budget) -> Result { + docker( + &["inspect", "-f", "{{.HostConfig.Init}}", container], + budget, + ) + .map(|value| value.trim() == "true") +} + +fn published_ssh_port(container: &str, budget: &Budget) -> Result { + let text = docker(&["port", container, "22"], budget)?; + let mut endpoints = text.lines(); + let port = endpoints + .next() + .and_then(|line| line.strip_prefix("127.0.0.1:")) + .and_then(|port| port.parse::().ok()) + .filter(|port| *port != 0); + match (port, endpoints.next()) { + (Some(port), None) => Ok(port), + _ => Err(format!( + "container {container} has no exclusive loopback SSH endpoint" + )), + } +} + +fn resource_census( + kind: RawResourceKind, + creation: Option<&str>, + budget: &Budget, +) -> Result, String> { + let mut args = match kind { + RawResourceKind::Container => vec!["ps", "-a", "--no-trunc"], + RawResourceKind::Network => vec!["network", "ls", "--no-trunc"], + }; + let filter = creation.map(|nonce| format!("label=myelin.raw-creation={nonce}")); + if let Some(filter) = &filter { + args.extend(["--filter", filter]); + } + args.extend([ + "--format", + match kind { + RawResourceKind::Container => "{{.ID}}\t{{.Names}}", + RawResourceKind::Network => "{{.ID}}\t{{.Name}}", + }, + ]); + docker(&args, budget)? + .lines() + .map(|line| { + let (id, name) = line + .split_once('\t') + .ok_or_else(|| format!("invalid raw {kind:?} census row: {line:?}"))?; + if id.len() != 64 || !id.bytes().all(|byte| byte.is_ascii_hexdigit()) || name.is_empty() + { + return Err(format!("invalid raw {kind:?} census identity: {line:?}")); + } + Ok((id.to_owned(), name.to_owned())) + }) + .collect() +} + +fn inspect_resource( + kind: RawResourceKind, + reference: &str, + budget: &Budget, +) -> Result, String> { + let output = match kind { + RawResourceKind::Container => docker(&["container", "inspect", reference], budget), + RawResourceKind::Network => docker(&["network", "inspect", reference], budget), + }; + let output = match output { + Ok(output) => output, + Err(error) => { + // Only a successful exact-ID/name census proves absence. In + // particular, a timeout or daemon error must not erase ownership. + return match resource_census(kind, None, budget) { + Ok(resources) + if !resources + .iter() + .any(|(id, name)| id == reference || name == reference) => + { + Ok(None) + } + Ok(_) => Err(error), + Err(census) => Err(format!("{error}; {census}")), + }; + } + }; + let mut values: Vec = serde_json::from_str(&output) + .map_err(|error| format!("decode raw {kind:?} identity: {error}"))?; + if values.len() != 1 { + return Err(format!( + "raw {kind:?} {reference} has ambiguous inspection results" + )); + } + let value = values.pop().expect("exactly one inspected resource"); + let id = value["Id"] + .as_str() + .ok_or_else(|| format!("raw {kind:?} {reference} has no provider ID"))? + .to_owned(); + let name = value["Name"] + .as_str() + .ok_or_else(|| format!("raw {kind:?} {reference} has no provider name"))? + .trim_start_matches('/') + .to_owned(); + if reference != id && reference != name { + return Err(format!( + "raw {kind:?} {reference} resolved to a different resource {id} ({name})" + )); + } + let labels = match kind { + RawResourceKind::Container => &value["Config"]["Labels"], + RawResourceKind::Network => &value["Labels"], + }; + let labels = serde_json::from_value::>>(labels.clone()) + .map_err(|error| format!("decode raw {kind:?} ownership labels: {error}"))? + .unwrap_or_default(); + Ok(Some(RawResourceObservation { id, name, labels })) +} + +fn owned_resource( + kind: RawResourceKind, + claim: &RawResourceClaim, + fixture: &str, + slot: u32, + budget: &Budget, +) -> Result, String> { + let observed = if let Some(id) = claim.id.get() { + inspect_resource(kind, id, budget)? + } else if let Some(creation) = &claim.creation { + let candidates = resource_census(kind, Some(creation), budget)?; + // Another cleanup owner may have reconciled and removed the resource + // while this label census was running. Its pinned ID remains decisive. + if let Some(id) = claim.id.get() { + inspect_resource(kind, id, budget)? + } else if candidates.len() == 1 { + inspect_resource(kind, &candidates[0].0, budget)? + } else { + return Err(format!( + "unresolved raw {kind:?} creation {creation} for {fixture} slot {slot}: \ + expected one owned identity, found {}", + candidates.len() + )); + } + } else { + // A planned name that failed adoption is not a cleanup capability. + return Ok(None); + }; + if let Some(observed) = &observed { + claim.adopt(observed, fixture, slot)?; + } else if claim.id.get().is_none() { + return Err(format!( + "unresolved raw {kind:?} creation {:?} for {fixture} slot {slot}", + claim.creation + )); + } + Ok(observed) +} + +fn remove_resource(kind: RawResourceKind, id: &str, budget: &Budget) -> Result<(), String> { + let removal = match kind { + RawResourceKind::Container => docker(&["rm", "-f", id], budget), + RawResourceKind::Network => docker(&["network", "rm", id], budget), + }; + if let Err(error) = removal { + match resource_census(kind, None, budget) { + Ok(resources) if !resources.iter().any(|(observed, _)| observed == id) => {} + Ok(_) => return Err(error), + Err(census) => return Err(format!("{error}; {census}")), + } + } + Ok(()) +} + +fn container_ip_on_network(container: &str, budget: &Budget) -> Result { + let text = docker( + &[ + "inspect", + "-f", + "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", + container, + ], + budget, + )?; + let ip = text.trim().to_owned(); + if ip.is_empty() { + return Err(format!("container {container} has no network address")); + } + Ok(ip) +} + +fn ensure_base_image(workspace: &Path, budget: &Budget) -> Result<(), String> { + // Always present the build to Docker. Layer caching keeps this cheap and + // avoids silently reusing a tag built from an older production base. + run_checked( + Command::new("docker").current_dir(workspace).args([ + "build", + "-f", + "apps/myelin/node-image/Dockerfile.base", + "-t", + BASE_IMAGE, + ".", + ]), + "build raw-fleet base image", + budget, + ) +} + +fn ensure_identity(artifacts: &Path, budget: &Budget, required: bool) -> Result { + let identity = crate::resources::private_fixture_dir(artifacts)?.join("raw-fleet-id_ed25519"); + if identity.is_file() { + return Ok(identity); + } + if required { + return Err(format!( + "retained raw-fleet SSH identity {} is missing; manifest retained", + identity.display() + )); + } + run_checked( + Command::new("ssh-keygen") + .args(["-t", "ed25519", "-N", "", "-C", "myelin-raw-fleet", "-f"]) + .arg(&identity), + "generate raw-fleet SSH identity", + budget, + )?; + Ok(identity) +} + +fn public_key(identity: &Path) -> Result { + let text = std::fs::read_to_string(identity.with_extension("pub")) + .map_err(|error| format!("read raw-fleet public key: {error}"))?; + Ok(text.trim().to_owned()) +} + +/// Read the SSH identification banner over a raw TCP connection. The fixture +/// deliberately avoids invoking `ssh` itself: the only SSH client in the +/// deployment E2E belongs to the orchestrator's bootstrap transport. +fn wait_for_ssh_banner(host: &str, port: u16, budget: &Budget) -> Result<(), String> { + let budget = budget.child(SSH_BANNER_TIMEOUT); + let predicate = format!("SSH banner at {host}:{port}"); + let address = format!("{host}:{port}") + .parse() + .map_err(|error| format!("{predicate}: invalid address: {error}"))?; + loop { + let remaining = budget.remaining(&predicate)?; + if let Ok(mut stream) = TcpStream::connect_timeout(&address, remaining.min(SSH_BANNER_POLL)) + { + stream + .set_read_timeout(Some(budget.remaining(&predicate)?.min(SSH_BANNER_POLL))) + .map_err(|error| format!("{predicate}: {error}"))?; + let mut banner = [0_u8; 64]; + if let Ok(count) = stream.read(&mut banner) { + let text = String::from_utf8_lossy(&banner[..count]); + if text.starts_with("SSH-2.0-") || text.starts_with("SSH-1.99-") { + return Ok(()); + } + } + } + budget.wait(SSH_BANNER_POLL, &predicate)?; + } +} + +fn reserve_port() -> Result { + let listener = std::net::TcpListener::bind(("127.0.0.1", 0)) + .map_err(|error| format!("reserve raw-fleet SSH port: {error}"))?; + listener + .local_addr() + .map(|address| address.port()) + .map_err(|error| format!("read reserved raw-fleet SSH port: {error}")) +} + +/// Verified owned containers still present; legacy name-only manifests cannot +/// establish either ownership or absence. +pub(crate) fn read_manifest_containers( + manifest: &Path, + budget: &Budget, +) -> Result, String> { + let parsed = Manifest::load(manifest)? + .ok_or_else(|| format!("raw-fleet manifest {} is absent", manifest.display()))?; + let mut existing = Vec::new(); + for node in &parsed.nodes { + if let Some(observed) = owned_resource( + RawResourceKind::Container, + &node.container_claim.retained(), + &parsed.fixture, + node.slot, + budget, + )? { + existing.push(observed.name); + } + } + Ok(existing) +} + +fn recover_resource( + kind: RawResourceKind, + claim: &RawResourceClaim, + name: &str, + fixture: &str, + slot: u32, + budget: &Budget, +) -> Result, String> { + if claim.creation.is_some() || claim.id.get().is_some() { + let observed = owned_resource(kind, claim, fixture, slot, budget)?.ok_or_else(|| { + format!("retained raw {kind:?} {name} disappeared; refusing replacement") + })?; + if observed.name != name { + return Err(format!( + "retained raw {kind:?} {name} was renamed to {}; manifest retained", + observed.name + )); + } + Ok(Some(observed)) + } else if inspect_resource(kind, name, budget)?.is_some() { + Err(format!( + "unclaimed raw {kind:?} name collision {name}; refusing adoption" + )) + } else { + Ok(None) + } +} + +impl RawDockerFleet { + /// Creates or rediscovers `node_count` blank nodes and writes the static + /// fleet manifest. Newly created nodes are asserted to carry no Myelin + /// artifacts. + pub fn ensure( + workspace: &Path, + artifacts: &Path, + node_count: u32, + build_image: bool, + budget: &Budget, + cleanup_budget: &Budget, + ) -> Result { + std::fs::create_dir_all(artifacts) + .map_err(|error| format!("create raw-fleet artifacts dir: {error}"))?; + let lock = RawManifestStore::lock(artifacts)?; + let path = Self::manifest_path(artifacts); + let previous = Manifest::load(&path)?; + if let Some(previous) = previous.as_ref().filter(|manifest| manifest.retired) { + // A completed cleanup is a tombstone, not permission to forget + // ownership without rechecking the original immutable identities. + for node in &previous.nodes { + for (kind, claim) in [ + (RawResourceKind::Container, &node.container_claim), + (RawResourceKind::Network, &node.network_claim), + ] { + if owned_resource( + kind, + &claim.retained(), + &previous.fixture, + node.slot, + budget, + )? + .is_some() + { + return Err( + "retired raw-fleet still owns resources; manifest retained".to_owned() + ); + } + } + } + } + let recovered = previous.as_ref().is_some_and(|manifest| !manifest.retired); + if let Some(previous) = previous.as_ref().filter(|_| recovered) { + if previous.nodes.len() != node_count as usize { + return Err("retained raw-fleet node count differs; manifest retained".to_owned()); + } + } else if build_image { + ensure_base_image(workspace, budget)?; + } + let identity = ensure_identity(artifacts, budget, recovered)?; + let public_key = public_key(&identity)?; + let manifest = if let Some(previous) = previous.filter(|_| recovered) { + previous.require_configuration(node_count, &public_key)?; + previous + } else { + let image_id = docker(&["image", "inspect", "-f", "{{.Id}}", BASE_IMAGE], budget)?; + let manifest = + Manifest::fresh(node_count, image_id.trim().to_owned(), public_key.clone())?; + // All names and nonce reservations become durable before a single + // network/container creation can be issued. + manifest.persist(&path)?; + manifest + }; + let image_id = manifest.image_id.clone(); + let mut fleet = Self { + nodes: manifest.nodes.iter().map(ManifestNode::retained).collect(), + identity, + prefix: manifest.fixture.clone(), + prior_processes: Mutex::new(BTreeMap::new()), + budget: budget.clone(), + cleanup_budget: cleanup_budget.clone(), + manifest: RawManifestStore { + path, + state: Mutex::new(manifest), + _lock: lock, + }, + }; + let prepared = (|| { + let prefix = fleet.prefix.as_str(); + let manifest = &fleet.manifest; + std::thread::scope(|scope| { + // Five construction owners at most, irrespective of manifest size. + // Each owner is cancellable; scope cannot strand a Docker child. + let mut errors = Vec::new(); + for batch in fleet.nodes.chunks_mut(5) { + let mut pending = Vec::new(); + for node in batch { + let public_key = &public_key; + let image_id = &image_id; + pending.push(scope.spawn(move || { + let started = Instant::now(); + let fixture_budget = budget; + let node_budget = budget.child(Duration::from_secs(60)); + let budget = &node_budget; + let result = (|| { + let slot = node.slot; + let container = &node.container; + let network = &node.network; + let fixture_label = format!("myelin.raw-fixture={prefix}"); + let slot_label = format!("myelin.raw-slot={slot}"); + let existing_network = recover_resource( + RawResourceKind::Network, &node.network_claim, + network, prefix, slot, budget, + )?; + if let Some(observed) = existing_network { + node.network_claim.adopt(&observed, prefix, slot)?; + } else { + let nonce = manifest.begin_creation(slot, RawResourceKind::Network)?; + node.network_claim.creation = Some(nonce.clone()); + let creation_label = format!("myelin.raw-creation={nonce}"); + let id = docker(&[ + "network", "create", + "--label", &fixture_label, + "--label", &slot_label, + "--label", &creation_label, + network, + ], budget)?; + let observed = inspect_resource( + RawResourceKind::Network, id.trim(), budget, + )?.ok_or_else(|| format!("created raw network {network} disappeared"))?; + node.network_claim.adopt(&observed, prefix, slot)?; + } + manifest.record(node)?; + let network_id = node.network_claim.retained_id()?; + let existing_container = recover_resource( + RawResourceKind::Container, &node.container_claim, + container, prefix, slot, budget, + )?; + let created_now = existing_container.is_none(); + if let Some(observed) = existing_container { + node.container_claim.adopt(&observed, prefix, slot)?; + } else { + let port = reserve_port()?; + let nonce = manifest.begin_creation(slot, RawResourceKind::Container)?; + node.container_claim.creation = Some(nonce.clone()); + let creation_label = format!("myelin.raw-creation={nonce}"); + let id = docker(&[ + "run", "-d", "--init", + "--name", container, + "--label", &fixture_label, + "--label", &slot_label, + "--label", &creation_label, + "--network", network_id, + "--publish", &format!("127.0.0.1:{port}:22"), + "--env", &format!("PUBLIC_KEY={public_key}"), + image_id, + ], budget)?; + let observed = inspect_resource( + RawResourceKind::Container, id.trim(), budget, + )?.ok_or_else(|| format!("created raw node {container} disappeared"))?; + node.container_claim.adopt(&observed, prefix, slot)?; + } + manifest.record(node)?; + let container_id = node.container_claim.retained_id()?; + let configuration = docker(&[ + "inspect", "-f", "{{json .}}", container_id, + ], budget)?; + let configuration: serde_json::Value = serde_json::from_str(&configuration) + .map_err(|error| format!("decode retained raw node configuration: {error}"))?; + let expected_key = format!("PUBLIC_KEY={public_key}"); + if configuration["Image"].as_str() != Some(image_id.as_str()) + || !configuration["Config"]["Env"].as_array().is_some_and(|env| { + env.iter().filter_map(serde_json::Value::as_str) + .filter(|value| value.starts_with("PUBLIC_KEY=")) + .eq(std::iter::once(expected_key.as_str())) + }) + { + return Err(format!("retained raw node {container} image or SSH key changed")); + } + if created_now { + // Blank-node guarantee: the base image must not carry + // Myelin deployment state. + docker(&["exec", container_id, "test", "!", "-e", "/opt/myelin"], budget) + .map_err(|error| format!("fresh raw node {container} is not blank: {error}"))?; + docker(&[ + "exec", container_id, "test", "!", "-e", + "/usr/local/bin/myelin-node", + ], budget).map_err(|error| { + format!("fresh raw node {container} ships myelin-node: {error}") + })?; + } else if !container_running(container_id, budget)? { + docker(&["start", container_id], budget)?; + } + if !container_has_init(container_id, budget)? { + return Err(format!( + "raw node {container} has no init reaper; refresh the fixture" + )); + } + let attached = docker(&[ + "inspect", "-f", + "{{range .NetworkSettings.Networks}}{{.NetworkID}}{{end}}", + container_id, + ], budget)?; + if attached.trim() != network_id { + return Err(format!( + "raw node {container} is not attached exclusively to retained network {network_id}" + )); + } + let ssh_port = published_ssh_port(container_id, budget)?; + if node.ssh_port != 0 && node.ssh_port != ssh_port { + return Err(format!("retained raw node {container} SSH endpoint changed")); + } + wait_for_ssh_banner("127.0.0.1", ssh_port, budget)?; + node.ssh_port = ssh_port; + manifest.record(node)?; + Ok::<(), String>(()) + })(); + record_execution_stage("raw_fleet.construct_node", started.elapsed(), 0, 1); + if result.is_err() { + fixture_budget.cancel(); + } + result + })); + } + for owner in pending { + match owner.join() { + Ok(Ok(())) => {} + Ok(Err(error)) => errors.push(error), + Err(_) => { + budget.cancel(); + errors.push("raw construction owner panicked".to_owned()); + } + } + } + if !errors.is_empty() { + break; + } + } + if errors.is_empty() { + Ok(()) + } else { + Err(errors.join("; ")) + } + })?; + fleet.assert_isolated()?; + fleet.snapshot()?; + fleet.manifest.update(|manifest| { + manifest.ready = true; + Ok(()) + })?; + Ok::<(), String>(()) + })(); + if let Err(error) = prepared { + if recovered { + return Err(format!( + "recover raw fixture: {error}; original ownership manifest retained at {}", + fleet.manifest.path.display(), + )); + } + let teardown = fleet.teardown(); + let remaining = fleet.remaining_resources_with_budget(cleanup_budget); + return Err(format!( + "prepare raw fixture: {error}; teardown={teardown:?}; remaining={remaining:?}" + )); + } + Ok(fleet) + } + + pub fn set_execution_budget(&mut self, budget: Budget) { + self.budget = budget; + } + + /// Gateway address of node 0's bridge: a host IP reachable from every + /// node's isolated network and from the host itself. + pub fn host_gateway(&self) -> Result { + let budget = &self.budget; + let Some(node) = self.nodes.first() else { + return Err("raw fleet has no nodes".to_owned()); + }; + let gateway = docker( + &[ + "inspect", + "-f", + "{{range .NetworkSettings.Networks}}{{.Gateway}}{{end}}", + node.container_claim.retained_id()?, + ], + budget, + )?; + let gateway = gateway.trim().to_owned(); + if gateway.is_empty() { + return Err(format!( + "raw fleet network for {} has no gateway", + node.container + )); + } + Ok(gateway) + } + + pub fn manifest_path(artifacts: &Path) -> PathBuf { + artifacts.join("raw-fleet.json") + } + + fn node(&self, slot: u32) -> Result<&RawNode, String> { + self.nodes + .iter() + .find(|node| node.slot == slot) + .ok_or_else(|| format!("raw fleet has no slot {slot}")) + } + + /// Immutable provider-resource and logical mapping evidence. + pub fn snapshot(&self) -> Result { + let budget = &self.budget; + let mut nodes = Vec::with_capacity(self.nodes.len()); + for node in &self.nodes { + let container = inspect_resource(RawResourceKind::Container, &node.container, budget)? + .ok_or_else(|| format!("retained raw container {} disappeared", node.container))?; + let network = inspect_resource(RawResourceKind::Network, &node.network, budget)? + .ok_or_else(|| format!("retained raw network {} disappeared", node.network))?; + // Snapshot must compare to acquisition identities, not establish a + // new baseline after a same-name resource has been replaced. + node.container_claim + .verify(&container, &self.prefix, node.slot)?; + node.network_claim + .verify(&network, &self.prefix, node.slot)?; + let container_id = node.container_claim.retained_id()?.to_owned(); + let network_id = node.network_claim.retained_id()?.to_owned(); + nodes.push(RawFixtureNodeIdentity { + logical_node_id: u64::from(node.slot) + 1, + slot: node.slot, + container: node.container.clone(), + container_id, + network: node.network.clone(), + network_id, + ssh_host: node.ssh_host.clone(), + ssh_port: published_ssh_port(node.container_claim.retained_id()?, budget)?, + }); + } + nodes.sort_by_key(|node| node.slot); + Ok(RawFleetSnapshot { nodes }) + } + + pub fn assert_unchanged(&self, expected: &RawFleetSnapshot) -> Result<(), String> { + let observed = self.snapshot()?; + if observed == *expected { + Ok(()) + } else { + Err(format!( + "raw fixture identity changed: expected {expected:?}, observed {observed:?}" + )) + } + } + + /// Retains process identities on the host before a new generation starts. + /// Node reset cannot erase this evidence by removing deployment metadata. + pub fn record_prior_processes(&self) -> Result<(), String> { + for node in &self.nodes { + self.record_node_processes(node)?; + } + Ok(()) + } + + fn record_node_processes(&self, node: &RawNode) -> Result<(), String> { + let budget = &self.budget; + let processes = process_census(node.container_claim.retained_id()?, budget)?; + let mut prior = self + .prior_processes + .lock() + .map_err(|_| "raw prior-process census lock poisoned".to_owned())?; + let recorded = prior.entry(node.slot).or_default(); + let related = related_worker_processes(&processes, recorded); + for process in related { + if !recorded + .iter() + .any(|old| old.pid == process.pid && old.start_ticks == process.start_ticks) + { + recorded.push(process); + } + } + recorded.sort_by_key(|process| (process.pid, process.start_ticks)); + Ok(()) + } + + /// Docker is used only as a census mechanism. It does not launch or + /// install the tested worker. + pub fn census(&self) -> Result, String> { + let budget = &self.budget; + let mut census = Vec::with_capacity(self.nodes.len()); + for node in &self.nodes { + let processes = process_census(node.container_claim.retained_id()?, budget)?; + let prior_processes = self + .prior_processes + .lock() + .map_err(|_| "raw prior-process census lock poisoned".to_owned())? + .get(&node.slot) + .cloned() + .unwrap_or_default(); + let surviving_prior_processes = processes + .iter() + .filter(|process| { + prior_processes.iter().any(|prior| { + prior.pid == process.pid && prior.start_ticks == process.start_ticks + }) + }) + .cloned() + .collect::>(); + let mut worker_processes = processes + .iter() + .filter(|process| process.command == "myelin-worker") + .cloned() + .collect::>(); + worker_processes.sort_by_key(|process| process.pid); + let worker_group_processes = related_worker_processes(&processes, &prior_processes); + let agent_pid = parse_optional_u32( + &docker_shell( + node.container_claim.retained_id()?, + "cat /opt/myelin/agent.pid 2>/dev/null || true", + budget, + )?, + "raw node agent pid", + )?; + let active_link_target = docker_shell( + node.container_claim.retained_id()?, + "readlink /opt/myelin/current 2>/dev/null || true", + budget, + )?; + let active_link_target = (!active_link_target.trim().is_empty()) + .then(|| active_link_target.trim().to_owned()); + let active_digest = docker_shell( + node.container_claim.retained_id()?, + "sha256sum /opt/myelin/current/bin/myelin-worker 2>/dev/null || true", + budget, + )?; + let active_executable_digest = active_digest + .split_whitespace() + .next() + .filter(|digest| digest.len() == 64) + .map(|digest| format!("sha256:{digest}")); + let active_deployment = docker_shell( + node.container_claim.retained_id()?, + "cat /opt/myelin/active-deployment.json 2>/dev/null || true", + budget, + )?; + let active_deployment_raw = + (!active_deployment.trim().is_empty()).then(|| active_deployment.trim().to_owned()); + census.push(RawNodeCensus { + logical_node_id: u64::from(node.slot) + 1, + container: node.container.clone(), + active_deployment_raw, + agent_pid, + active_link_target, + active_executable_digest, + worker_processes, + worker_group_processes, + prior_processes, + surviving_prior_processes, + }); + } + Ok(census) + } + + /// Constructs a prior state without installing or launching the tested + /// binaries. The production SSH deployment transaction remains unaware + /// of which state was selected. + pub fn inject_prior_state(&self, slot: u32, state: RawPriorState) -> Result<(), String> { + let budget = &self.budget; + let node = self.node(slot)?; + let script = match state { + RawPriorState::NoWorkerOrTrustworthyMetadata => format!( + "{}; rm -rf /opt/myelin/agent.pid /opt/myelin/runtime \ + /opt/myelin/bootstrap.lock /opt/myelin/bootstrap.sock \ + /opt/myelin/current /opt/myelin/active-deployment.json \ + /opt/myelin/transactions", + stop_worker_trees_script() + ), + RawPriorState::HealthyStaleWorker => ":".to_owned(), + RawPriorState::DeadWorkerWithStaleMetadata => format!( + "{}; mkdir -p /opt/myelin/runtime /opt/myelin/bootstrap.lock; \ + printf '999999\\n' > /opt/myelin/agent.pid; \ + printf '999998\\n' > /opt/myelin/runtime/stale.pid; \ + printf '999997\\n' > /opt/myelin/bootstrap.lock/pid; \ + touch /opt/myelin/bootstrap.lock/complete /opt/myelin/bootstrap.sock", + stop_worker_trees_script() + ), + RawPriorState::MultipleStaleWorkersAndDescendants => r#"python3 - <<'PY' +import os, select, shutil, signal, subprocess, time +shutil.copy('/bin/sh', '/tmp/myelin-worker') +os.chmod('/tmp/myelin-worker', 0o755) +read, write = os.pipe() +workers = [] +try: + for index in range(3): + with open('/tmp/myelin-stale-%s.log' % index, 'wb') as log: + workers.append(subprocess.Popen( + ['/tmp/myelin-worker', '-c', + '(trap "" TERM; printf r >&%s; exec sleep 600) & wait' % write], + stdin=subprocess.DEVNULL, stdout=log, stderr=log, + pass_fds=(write,), start_new_session=True)) + os.close(write) + write = None + deadline = time.monotonic() + 2.5 + ready = b'' + while len(ready) < 3: + remaining = deadline - time.monotonic() + if remaining <= 0 or not select.select([read], [], [], remaining)[0]: + raise TimeoutError('TERM-resistant descendants did not acknowledge readiness') + data = os.read(read, 3 - len(ready)) + if not data: + raise RuntimeError('stale worker exited before its descendant was ready') + ready += data +except BaseException: + for worker in workers: + try: + os.killpg(worker.pid, signal.SIGKILL) + except ProcessLookupError: + pass + worker.wait() + raise +finally: + os.close(read) + if write is not None: + os.close(write) +PY"# + .to_owned(), + RawPriorState::InterruptedTransferAndPartialInstall => { + "mkdir -p /opt/myelin/staging/interrupted \ + /opt/myelin/releases/interrupted.staged.1/bin \ + /opt/myelin/transactions/interrupted; \ + printf 'partial-bundle' > /opt/myelin/staging/interrupted/bundle.tar; \ + printf 'partial-worker' > \ + /opt/myelin/releases/interrupted.staged.1/bin/myelin-worker; \ + printf '{\"state\":\"interrupted\"}' > \ + /opt/myelin/transactions/interrupted/deployment.json" + .to_owned() + } + RawPriorState::CorruptActiveBinary => { + "worker=/opt/myelin/current/bin/myelin-worker; test -e \"$worker\"; \ + corrupt=$(mktemp /opt/myelin/current/bin/.myelin-worker.corrupt.XXXXXX); \ + printf 'corrupt-worker' > \"$corrupt\"; chmod 0755 \"$corrupt\"; \ + mv -f \"$corrupt\" \"$worker\"" + .to_owned() + } + RawPriorState::CorruptActivationPointer => "rm -rf /opt/myelin/current; \ + ln -s /opt/myelin/releases/missing /opt/myelin/current" + .to_owned(), + RawPriorState::CorruptDeploymentDescriptor => { + "mkdir -p /opt/myelin/transactions/corrupt; \ + printf 'not-json' > /opt/myelin/active-deployment.json; \ + printf '{\"artifact_digest\":\"stale\"}' > \ + /opt/myelin/transactions/corrupt/deployment.json" + .to_owned() + } + }; + docker_shell(node.container_claim.retained_id()?, &script, budget)?; + self.record_node_processes(node) + } + + pub fn hold_boundary( + &self, + generation: &str, + boundary: DeploymentBoundary, + ) -> Result<(), String> { + let budget = &self.budget; + validate_remote_token(generation, "deployment generation")?; + let boundary = boundary.as_remote_name(); + for node in &self.nodes { + docker_shell( + node.container_claim.retained_id()?, + &format!( + "mkdir -p /opt/myelin/deployment-boundaries; \ + touch /opt/myelin/deployment-boundaries/{generation}.{boundary}.hold" + ), + budget, + )?; + } + Ok(()) + } + + pub fn release_boundary( + &self, + generation: &str, + boundary: DeploymentBoundary, + ) -> Result<(), String> { + self.release_boundary_with_budget(generation, boundary, &self.budget) + } + + pub fn release_boundary_with_budget( + &self, + generation: &str, + boundary: DeploymentBoundary, + budget: &Budget, + ) -> Result<(), String> { + validate_remote_token(generation, "deployment generation")?; + let boundary = boundary.as_remote_name(); + let mut errors = Vec::new(); + for node in &self.nodes { + if let Err(error) = node.container_claim.retained_id().and_then(|container| { + docker_shell( + container, + &format!( + "rm -f /opt/myelin/deployment-boundaries/{generation}.{boundary}.hold" + ), + budget, + ) + }) { + errors.push(format!("release boundary on {}: {error}", node.container)); + } + } + if errors.is_empty() { + Ok(()) + } else { + Err(errors.join("; ")) + } + } + + pub fn wait_for_boundary( + &self, + slot: u32, + generation: &str, + boundary: DeploymentBoundary, + timeout: Duration, + ) -> Result<(), String> { + validate_remote_token(generation, "deployment generation")?; + let node = self.node(slot)?; + let marker = format!( + "/opt/myelin/deployment-boundaries/{}.{}.reached", + generation, + boundary.as_remote_name() + ); + let budget = self.budget.child(timeout); + let seconds = budget + .remaining(&format!("deployment boundary {marker}"))? + .as_secs_f64(); + // Subscribe before checking the marker: neither an early boundary nor + // a boundary created between check and sleep can be missed. + let script = format!( + r#"python3 - {marker} {seconds} <<'PY' +import ctypes, os, select, sys, time +from pathlib import Path +path = Path(sys.argv[1]) +deadline = time.monotonic() + float(sys.argv[2]) +path.parent.mkdir(parents=True, exist_ok=True) +libc = ctypes.CDLL(None, use_errno=True) +fd = libc.inotify_init1(os.O_CLOEXEC | os.O_NONBLOCK) +if fd < 0 or libc.inotify_add_watch(fd, os.fsencode(path.parent), 0x100 | 0x80 | 0x8) < 0: + raise OSError(ctypes.get_errno(), 'subscribe deployment boundary') +try: + while not path.exists(): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError('pending deployment boundary: ' + str(path)) + if select.select([fd], [], [], remaining)[0]: + os.read(fd, 65536) +finally: + os.close(fd) +PY"# + ); + docker_shell(node.container_claim.retained_id()?, &script, &budget) + .map(|_| ()) + .map_err(|error| { + format!( + "pending deployment boundary {marker} on {}: {error}", + node.container + ) + }) + } + + pub fn set_node_reachable(&self, slot: u32, reachable: bool) -> Result<(), String> { + self.set_node_reachable_with_budget(slot, reachable, &self.budget) + } + + pub fn set_node_reachable_with_budget( + &self, + slot: u32, + reachable: bool, + budget: &Budget, + ) -> Result<(), String> { + let node = self.node(slot)?; + if reachable { + docker(&["unpause", node.container_claim.retained_id()?], budget)?; + wait_for_ssh_banner(&node.ssh_host, node.ssh_port, budget) + } else { + docker(&["pause", node.container_claim.retained_id()?], budget).map(|_| ()) + } + } + + /// Nodes must sit on exactly one network each, and no node may open a TCP + /// connection to another node's address. + pub fn assert_isolated(&self) -> Result<(), String> { + let budget = &self.budget; + let started = Instant::now(); + #[derive(Deserialize)] + #[serde(rename_all = "snake_case")] + enum Outcome { + Unreachable, + Reachable, + Error, + } + #[derive(Deserialize)] + struct Observation { + address: String, + outcome: Outcome, + error: Option, + } + const PROBE: &str = r#" +import concurrent.futures, errno, json, socket, sys +def probe(address): + try: + with socket.socket() as stream: + stream.settimeout(2.8) + stream.connect((address, 22)) + except TimeoutError: + return dict(address=address, outcome="unreachable") + except OSError as error: + if error.errno in (errno.ECONNREFUSED, errno.EHOSTUNREACH, errno.ENETUNREACH): + return dict(address=address, outcome="unreachable") + return dict(address=address, outcome="error", error=str(error)) + return dict(address=address, outcome="reachable") +with concurrent.futures.ThreadPoolExecutor(max_workers=min(4, len(sys.argv) - 1)) as workers: + print(json.dumps(list(workers.map(probe, sys.argv[1:])))) +"#; + let mut addresses = Vec::with_capacity(self.nodes.len()); + for node in &self.nodes { + let networks = docker( + &[ + "inspect", + "-f", + "{{len .NetworkSettings.Networks}}", + node.container_claim.retained_id()?, + ], + budget, + )?; + if networks.trim() != "1" { + return Err(format!( + "raw node {} is attached to {} networks; expected exactly 1", + node.container, + networks.trim() + )); + } + addresses.push(container_ip_on_network( + node.container_claim.retained_id()?, + budget, + )?); + } + if addresses + .iter() + .enumerate() + .any(|(index, address)| addresses[..index].contains(address)) + { + return Err("raw fixture peers share an isolation probe address".to_owned()); + } + let mut errors = Vec::new(); + // One interpreter per source, with four independent socket probes. + // All 20 directions remain concurrent for five nodes. Startup is inside + // the command budget, not a competing 3s fuse around a 2.8s socket wait. + for batch in self.nodes.chunks(5) { + let results = std::thread::scope(|scope| { + let pending = batch + .iter() + .map(|source| { + let targets = self + .nodes + .iter() + .zip(&addresses) + .filter(|(target, _)| target.slot != source.slot) + .map(|(target, address)| (address.as_str(), target)) + .collect::>(); + scope.spawn(move || { + if targets.is_empty() { + return Ok(()); + } + let probe = command_output( + Command::new("docker") + .args(["exec", source.container_claim.retained_id()?, "python3", "-c", PROBE]) + .args(targets.keys()), + &budget.child(Duration::from_secs(10)), + "directed raw isolation probes", + )?; + if !probe.status.success() { + return Err(format!( + "isolation probes from {} failed without absence proof: {}: {}", + source.container, + probe.status, + String::from_utf8_lossy(&probe.stderr), + )); + } + let observations: Vec = + serde_json::from_slice(&probe.stdout).map_err(|error| { + format!("decode isolation probes from {}: {error}", source.container) + })?; + if observations.len() != targets.len() { + return Err(format!("incomplete isolation probes from {}", source.container)); + } + let mut pending = targets; + let mut failures = Vec::new(); + for observation in observations { + let target = pending.remove(observation.address.as_str()).ok_or_else(|| { + format!("unexpected or duplicate isolation address {:?} from {}", + observation.address, source.container) + })?; + match observation.outcome { + Outcome::Unreachable => {} + Outcome::Reachable => failures.push(format!( + "raw node {} reached {} at {}:22", + source.container, target.container, observation.address, + )), + Outcome::Error => failures.push(format!( + "isolation probe {} -> {} failed without absence proof: {}", + source.container, target.container, + observation.error.as_deref().unwrap_or("missing probe error"), + )), + } + } + if failures.is_empty() { Ok(()) } else { Err(failures.join("; ")) } + }) + }) + .collect::>(); + pending + .into_iter() + .map(|owner| { + owner + .join() + .unwrap_or_else(|_| Err("raw isolation owner panicked".to_owned())) + }) + .collect::>() + }); + errors.extend(results.into_iter().filter_map(Result::err)); + } + record_execution_stage( + "raw_fleet.isolation", + started.elapsed(), + 0, + (self.nodes.len() * self.nodes.len().saturating_sub(1)) as u64, + ); + if errors.is_empty() { + Ok(()) + } else { + Err(errors.join("; ")) + } + } + + pub fn remaining_resources(&self) -> Result, String> { + self.remaining_resources_with_budget(&self.budget) + } + + pub fn remaining_resources_with_budget(&self, budget: &Budget) -> Result, String> { + let results = std::thread::scope(|scope| { + self.nodes + .iter() + .flat_map(|node| { + [ + (RawResourceKind::Container, &node.container_claim), + (RawResourceKind::Network, &node.network_claim), + ] + .map(|(kind, claim)| { + scope.spawn(move || { + owned_resource(kind, claim, &self.prefix, node.slot, budget).map( + |resource| { + resource.map(|resource| { + let kind = match kind { + RawResourceKind::Container => "container", + RawResourceKind::Network => "network", + }; + format!("{kind}:{}:{}", resource.name, resource.id) + }) + }, + ) + }) + }) + }) + .collect::>() + .into_iter() + .map(|owner| { + owner + .join() + .unwrap_or_else(|_| Err("raw ownership census owner panicked".to_owned())) + }) + .collect::>() + }); + let mut remaining = Vec::new(); + let mut errors = Vec::new(); + for result in results { + match result { + Ok(Some(resource)) => remaining.push(resource), + Ok(None) => {} + Err(error) => errors.push(error), + } + } + if !errors.is_empty() { + return Err(errors.join("; ")); + } + remaining.sort(); + Ok(remaining) + } + + /// Preserve bounded worker failure evidence before destroying the local fixture. + pub fn failure_diagnostics(&self, budget: &Budget) -> serde_json::Value { + let nodes = std::thread::scope(|scope| { + self.nodes + .iter() + .map(|node| { + scope.spawn(move || { + let log = node + .container_claim + .retained_id() + .and_then(|container| { + docker_shell( + container, + r#"python3 -c 'import json, pathlib +p = pathlib.Path("/var/log/myelin-node.log") +with p.open("rb") as f: + f.seek(0, 2) + size = f.tell() + f.seek(max(0, size - 65536)) + print(json.dumps({"log_bytes": size, "log_tail": f.read().decode("utf-8", errors="replace")})) +'"#, + budget, + ) + }) + .and_then(|text| { + serde_json::from_str::(&text) + .map_err(|error| format!("decode worker diagnostics: {error}")) + }); + serde_json::json!({ + "slot": node.slot, "container": node.container, + "worker_log": log.as_ref().ok(), + "error": log.as_ref().err(), + }) + }) + }) + .collect::>() + .into_iter() + .map(|owner| { + owner.join().unwrap_or_else( + |_| serde_json::json!({"error": "worker diagnostic collector panicked"}), + ) + }) + .collect::>() + }); + serde_json::json!({"schema_version": 1, "nodes": nodes}) + } + + /// Removes every container and network owned by this fixture. + pub fn teardown(&self) -> Result<(), String> { + self.teardown_with_budget(&self.cleanup_budget) + } + + pub fn teardown_with_budget(&self, budget: &Budget) -> Result<(), String> { + let mut errors = std::thread::scope(|scope| { + let pending = self + .nodes + .iter() + .flat_map(|node| { + let container = scope.spawn(move || { + let result = owned_resource( + RawResourceKind::Container, + &node.container_claim, + &self.prefix, + node.slot, + budget, + ) + .and_then(|resource| match resource { + Some(resource) => { + remove_resource(RawResourceKind::Container, &resource.id, budget) + } + None => Ok(()), + }); + result.err().into_iter().collect::>() + }); + let network = scope.spawn(move || { + // Endpoint detachment, not container process reaping, is + // the network removal dependency. Own it independently + // so a slow rm cannot consume the network's whole reserve. + // Resolve each endpoint to a verified immutable identity; + // an unrelated same-name container is never detached. + let result = (|| { + let Some(network) = owned_resource( + RawResourceKind::Network, + &node.network_claim, + &self.prefix, + node.slot, + budget, + )? + else { + return Ok(()); + }; + let disconnect = owned_resource( + RawResourceKind::Container, + &node.container_claim, + &self.prefix, + node.slot, + budget, + ) + .and_then(|container| match container { + Some(container) => docker( + &["network", "disconnect", "-f", &network.id, &container.id], + budget, + ) + .map(|_| ()), + None => Ok(()), + }); + if let Err(error) = + remove_resource(RawResourceKind::Network, &network.id, budget) + { + return Err(match disconnect { + Ok(()) => error, + Err(disconnect) => format!("{error}; {disconnect}"), + }); + } + // Successful removal also proves endpoint absence + // when concurrent container removal won the race. + Ok::<(), String>(()) + })(); + result.err().into_iter().collect::>() + }); + [container, network] + }) + .collect::>(); + pending + .into_iter() + .flat_map(|owner| { + owner + .join() + .unwrap_or_else(|_| vec!["raw teardown owner panicked".to_owned()]) + }) + .collect::>() + }); + match self.remaining_resources_with_budget(budget) { + Ok(remaining) if remaining.is_empty() => { + // Preserve IDs recovered during teardown before recording the + // terminal state. Never replace this manifest merely because a + // new runner has a different PID. + for node in &self.nodes { + self.manifest.record(node)?; + } + self.manifest.update(|manifest| { + manifest.retired = true; + Ok(()) + })?; + for path in [&self.identity, &self.identity.with_extension("pub")] { + if let Err(error) = std::fs::remove_file(path) { + if error.kind() != std::io::ErrorKind::NotFound { + errors.push(format!( + "remove private fixture identity {}: {error}", + path.display() + )); + } + } + } + } + Ok(remaining) => { + errors.push(format!("raw fixture resources remain: {remaining:?}")); + } + Err(error) => errors.push(format!("prove raw fixture absence: {error}")), + } + if errors.is_empty() { + Ok(()) + } else { + Err(errors.join("; ")) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn observation(id: char, fixture: &str, slot: u32) -> RawResourceObservation { + RawResourceObservation { + id: id.to_string().repeat(64), + name: format!("fixture-node-{slot}"), + labels: BTreeMap::from([ + ("myelin.raw-fixture".to_owned(), fixture.to_owned()), + ("myelin.raw-slot".to_owned(), slot.to_string()), + ]), + } + } + + #[test] + fn unrelated_name_collisions_never_become_cleanup_capabilities() { + let mut unlabeled = observation('a', "fixture", 0); + unlabeled.labels.clear(); + for observed in [ + unlabeled, + observation('a', "another-fixture", 0), + observation('a', "fixture", 1), + ] { + let claim = RawResourceClaim::default(); + assert!(claim.adopt(&observed, "fixture", 0).is_err()); + // Even with no command budget, unwind of a rejected adoption + // succeeds without inspecting, detaching or deleting the name. + for kind in [RawResourceKind::Container, RawResourceKind::Network] { + assert!( + owned_resource(kind, &claim, "fixture", 0, &Budget::new(Duration::ZERO),) + .unwrap() + .is_none() + ); + } + } + } + + #[test] + fn retained_resource_identity_survives_rename_but_rejects_replacement() { + let claim = RawResourceClaim::default(); + let mut retained = observation('a', "fixture", 0); + assert_eq!(claim.adopt(&retained, "fixture", 0).unwrap(), retained.id); + retained.name = "renamed-owned-resource".to_owned(); + assert_eq!(claim.adopt(&retained, "fixture", 0).unwrap(), retained.id); + let replacement = observation('b', "fixture", 0); + assert!(claim.adopt(&replacement, "fixture", 0).is_err()); + assert_eq!(claim.retained_id().unwrap(), retained.id); + } + + #[test] + fn ambiguous_creation_requires_its_exact_nonce_then_pins_one_id() { + let claim = RawResourceClaim { + creation: Some("attempt-a".to_owned()), + ..RawResourceClaim::default() + }; + let mut partial = observation('a', "fixture", 0); + assert!(claim.adopt(&partial, "fixture", 0).is_err()); + partial + .labels + .insert("myelin.raw-creation".to_owned(), "attempt-b".to_owned()); + assert!(claim.adopt(&partial, "fixture", 0).is_err()); + partial + .labels + .insert("myelin.raw-creation".to_owned(), "attempt-a".to_owned()); + assert_eq!(claim.adopt(&partial, "fixture", 0).unwrap(), partial.id); + partial.id = "b".repeat(64); + assert!(claim.adopt(&partial, "fixture", 0).is_err()); + assert_eq!(claim.retained_id().unwrap(), "a".repeat(64)); + } + + #[test] + fn interrupted_manifest_recovers_exact_fixture_and_creation_ownership() { + let directory = tempfile::tempdir().unwrap(); + let path = RawDockerFleet::manifest_path(directory.path()); + let manifest = Manifest::fresh( + 1, + format!("sha256:{}", "a".repeat(64)), + "ssh-ed25519 fixture-key".to_owned(), + ) + .unwrap(); + let fixture = manifest.fixture.clone(); + manifest.persist(&path).unwrap(); + let store = RawManifestStore { + path: path.clone(), + state: Mutex::new(manifest), + _lock: RawManifestStore::lock(directory.path()).unwrap(), + }; + let nonce = store.begin_creation(0, RawResourceKind::Network).unwrap(); + // Simulate runner loss after Docker accepted creation, before its ID + // reached the journal. A different runner must use this exact nonce. + drop(store); + let recovered = Manifest::load(&path).unwrap().unwrap(); + assert_eq!(recovered.fixture, fixture); + let node = recovered.nodes[0].retained(); + let mut observed = observation('b', &fixture, 0); + observed.name.clone_from(&node.network); + observed + .labels + .insert("myelin.raw-creation".to_owned(), nonce.clone()); + assert_eq!( + node.network_claim.adopt(&observed, &fixture, 0).unwrap(), + observed.id + ); + let store = RawManifestStore { + path: path.clone(), + state: Mutex::new(recovered), + _lock: RawManifestStore::lock(directory.path()).unwrap(), + }; + store.record(&node).unwrap(); + assert!(store.begin_creation(0, RawResourceKind::Network).is_err()); + drop(store); + let retained = Manifest::load(&path).unwrap().unwrap(); + assert_eq!(retained.fixture, fixture); + assert_eq!(retained.nodes[0].network_claim.creation, nonce); + let claim = retained.nodes[0].network_claim.retained(); + assert_eq!(claim.retained_id().unwrap(), observed.id); + observed.id = "c".repeat(64); + assert!(claim.adopt(&observed, &fixture, 0).is_err()); + observed.id = "b".repeat(64); + observed + .labels + .insert("myelin.raw-creation".to_owned(), "d".repeat(32)); + assert!(claim.adopt(&observed, &fixture, 0).is_err()); + } + + #[test] + fn legacy_or_changed_fixture_configuration_is_retained_without_execution() { + let directory = tempfile::tempdir().unwrap(); + let path = RawDockerFleet::manifest_path(directory.path()); + let legacy = r#"{"nodes":[{"slot":0,"container":"old-pid-node-0","ssh_host":"127.0.0.1","ssh_port":32000}]}"#; + std::fs::write(&path, legacy).unwrap(); + let budget = Budget::new(Duration::ZERO); + assert!( + RawDockerFleet::ensure( + directory.path(), + directory.path(), + 1, + false, + &budget, + &budget, + ) + .is_err() + ); + assert_eq!(std::fs::read_to_string(&path).unwrap(), legacy); + let manifest = Manifest::fresh( + 1, + format!("sha256:{}", "a".repeat(64)), + "ssh-ed25519 fixture-key".to_owned(), + ) + .unwrap(); + manifest.persist(&path).unwrap(); + let original = std::fs::read(&path).unwrap(); + assert!( + RawDockerFleet::ensure( + directory.path(), + directory.path(), + 2, + false, + &budget, + &budget, + ) + .is_err() + ); + assert!( + manifest + .require_configuration(1, "ssh-ed25519 another-key") + .is_err() + ); + assert_eq!(std::fs::read(&path).unwrap(), original); + } +} diff --git a/tools/myelin-e2e-fuzz/src/ir.rs b/tools/myelin-e2e-fuzz/src/ir.rs index 53f64f7..5bbea3f 100644 --- a/tools/myelin-e2e-fuzz/src/ir.rs +++ b/tools/myelin-e2e-fuzz/src/ir.rs @@ -1,6 +1,6 @@ //! Typed intermediate representation of generated behavioral programs. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use serde::{Deserialize, Serialize}; @@ -103,10 +103,37 @@ pub enum ActionOp { chunks: Vec>, replace: bool, }, + StreamRoundTrip { + path: String, + chunks: Vec>, + }, + GatedStreamWrite { + path: String, + #[serde(default)] + replace: bool, + frames: Vec>, + release_path: String, + }, StreamRead { path: String, expected: Vec, }, + /// A stream read that keeps retrying its attach while the stream is + /// mid-replacement (duplicate role, stale incarnation or absence are + /// transient) until it attaches to the live generation. + StreamReadWithRetry { + path: String, + expected: Vec, + }, + GatedStreamRead { + path: String, + expected: Vec, + observed_path: String, + #[serde(default)] + retry_attach: bool, + #[serde(default)] + park_after_first_frame: bool, + }, StreamReadInto { path: String, expected: Vec, @@ -158,8 +185,13 @@ impl ActionOp { match self { Self::PublishBlob { .. } => ActionClass::PublishBlob, Self::ReadBlob { .. } => ActionClass::ReadBlob, - Self::StreamWrite { .. } => ActionClass::StreamWrite, - Self::StreamRead { .. } | Self::StreamReadInto { .. } => ActionClass::StreamRead, + Self::StreamWrite { .. } + | Self::StreamRoundTrip { .. } + | Self::GatedStreamWrite { .. } => ActionClass::StreamWrite, + Self::StreamRead { .. } + | Self::StreamReadWithRetry { .. } + | Self::GatedStreamRead { .. } + | Self::StreamReadInto { .. } => ActionClass::StreamRead, Self::Lookup { .. } | Self::AwaitEntry { .. } | Self::WaitForQuiescent { .. } => { ActionClass::Lookup } @@ -176,7 +208,99 @@ impl ActionOp { Self::PublishBlob { path, .. } | Self::ReadBlob { path, .. } | Self::StreamWrite { path, .. } + | Self::StreamRoundTrip { path, .. } + | Self::GatedStreamWrite { path, .. } | Self::StreamRead { path, .. } + | Self::StreamReadWithRetry { path, .. } + | Self::GatedStreamRead { path, .. } + | Self::StreamReadInto { path, .. } + | Self::Lookup { path, .. } + | Self::AwaitEntry { path, .. } + | Self::WaitForQuiescent { path } + | Self::Unlink { path } + | Self::DescriptorWrite { path, .. } + | Self::DescriptorRead { path, .. } + | Self::MappingExportClose { path, .. } => path, + Self::Rename { source, .. } => source, + } + } +} +impl ActionOp { + pub(crate) fn map_paths(&mut self, mut map: impl FnMut(&mut String)) { + match self { + Self::Rename { + source, + destination, + .. + } => { + map(source); + map(destination); + } + Self::GatedStreamWrite { + path, release_path, .. + } => { + map(path); + map(release_path); + } + Self::GatedStreamRead { + path, + observed_path, + .. + } => { + map(path); + map(observed_path); + } + operation => map(operation.path_mut()), + } + } + + pub(crate) fn paths(&self) -> [Option<&str>; 2] { + let extra = match self { + Self::Rename { destination, .. } => Some(destination.as_str()), + Self::GatedStreamWrite { release_path, .. } => Some(release_path.as_str()), + Self::GatedStreamRead { observed_path, .. } => Some(observed_path.as_str()), + _ => None, + }; + [Some(self.path()), extra] + } + + pub(crate) fn mutating_paths(&self) -> [Option<&str>; 2] { + match self { + Self::ReadBlob { .. } + | Self::Lookup { .. } + | Self::AwaitEntry { .. } + | Self::WaitForQuiescent { .. } + | Self::MappingExportClose { .. } => [None, None], + Self::DescriptorRead { flags, path, .. } => [ + (matches!(*flags & libc::O_ACCMODE, libc::O_WRONLY | libc::O_RDWR) + || *flags & (libc::O_CREAT | libc::O_TRUNC) != 0) + .then_some(path.as_str()), + None, + ], + Self::GatedStreamWrite { path, .. } => [Some(path), None], + Self::PublishBlob { .. } + | Self::StreamWrite { .. } + | Self::StreamRoundTrip { .. } + | Self::StreamRead { .. } + | Self::StreamReadWithRetry { .. } + | Self::GatedStreamRead { .. } + | Self::StreamReadInto { .. } + | Self::Rename { .. } + | Self::Unlink { .. } + | Self::DescriptorWrite { .. } => self.paths(), + } + } + + fn path_mut(&mut self) -> &mut String { + match self { + Self::PublishBlob { path, .. } + | Self::ReadBlob { path, .. } + | Self::StreamRoundTrip { path, .. } + | Self::StreamWrite { path, .. } + | Self::GatedStreamWrite { path, .. } + | Self::StreamRead { path, .. } + | Self::StreamReadWithRetry { path, .. } + | Self::GatedStreamRead { path, .. } | Self::StreamReadInto { path, .. } | Self::Lookup { path, .. } | Self::AwaitEntry { path, .. } @@ -207,6 +331,15 @@ pub enum ExpectedOutcome { pub struct Action { pub operation: ActionOp, pub expected: ExpectedOutcome, + /// Opaque corpus-authored evidence labels threaded into rendered emits. + /// + /// Codegen never interprets hint semantics; it only forwards the known + /// keys ([`EVIDENCE_HINT_TOKEN`], [`EVIDENCE_HINT_LAP`], + /// [`EVIDENCE_HINT_EDGE_INDEX`], [`EVIDENCE_HINT_BARRIER`]) into the + /// generated Python emit calls so scenarios can label token-ring traffic + /// without codegen parsing corpus semantics. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub evidence_hints: BTreeMap, } impl Action { @@ -214,6 +347,7 @@ impl Action { Self { operation, expected: ExpectedOutcome::Ok, + evidence_hints: BTreeMap::new(), } } @@ -221,6 +355,7 @@ impl Action { Self { operation, expected: ExpectedOutcome::Error(errno), + evidence_hints: BTreeMap::new(), } } @@ -228,6 +363,7 @@ impl Action { Self { operation, expected: ExpectedOutcome::Exception(exception), + evidence_hints: BTreeMap::new(), } } @@ -239,8 +375,15 @@ impl Action { successes, error, }, + evidence_hints: BTreeMap::new(), } } + + /// Attaches opaque evidence labels to this action. + pub fn with_evidence_hints(mut self, evidence_hints: BTreeMap) -> Self { + self.evidence_hints = evidence_hints; + self + } } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -280,6 +423,8 @@ pub enum ProcessStopPhase { AfterSpawn, DuringBootstrap, AfterContextReady, + AfterStreamFirstFrame, + AfterSiblingStreamFirstFrame, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -292,6 +437,126 @@ pub enum LaunchFailureKind { PythonRuntime, } +pub const CASE_SCHEMA_VERSION: u32 = 1; +pub const GENERATOR_VERSION: u32 = 6; + +/// Hard admission cap, including zero-length logical frames. +pub const MAX_STREAM_FRAMES: usize = 65_536; + +// The public data-plane host opens every stream endpoint with this capacity +// (crates/data-plane/src/host.rs::STREAM_RING_CAPACITY). Python read() also +// allocates a capacity-sized Rust Vec and copies it into Python bytes. +const STREAM_ENDPOINT_CAPACITY: u64 = 256 * 1024; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TopologyFamily { + #[default] + Fixed, + Chain, + RingWalk, + FanOut, + FanIn, + Diamond, + RandomDag, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CoverageScenario { + ConcurrentStartup, + RingCompletion, + HotColdFairness, + ProcessChurn, + ActiveMutation, + QuiescentMutation, + ActiveStreamWriterAbort, + ActiveStreamReaderStop, + FailureIsolation, + Authorization, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DataKind { + Blob, + Stream, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DataEdge { + pub source: u64, + pub destination: u64, + /// Process identities owning the two logical vertices, independent of + /// physical placement so a route may revisit a live node. + pub source_role: String, + pub destination_role: String, + pub path: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DataRoute { + pub id: String, + pub kind: DataKind, + pub edges: Vec, + #[serde(default)] + pub join_inputs: Vec, +} + +/// Checked aggregate admission charges, not measured RSS. Allocation bytes sum +/// every action's requested data/mapping capacity, endpoint ring and public +/// binding buffers, conservatively charging sequential actions as concurrently live. +/// Observation count is an upper bound including action milestones and hints. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct CaseResources { + pub process_count: u64, + pub action_count: u64, + pub payload_bytes: u64, + pub allocation_bytes: u64, + pub observation_count: u64, +} + +impl CaseResources { + pub(crate) fn checked_add(self, other: Self) -> Result { + let add = |left: u64, right: u64| { + left.checked_add(right) + .ok_or_else(|| "case resource count overflowed".to_owned()) + }; + Ok(Self { + process_count: add(self.process_count, other.process_count)?, + action_count: add(self.action_count, other.action_count)?, + payload_bytes: add(self.payload_bytes, other.payload_bytes)?, + allocation_bytes: add(self.allocation_bytes, other.allocation_bytes)?, + observation_count: add(self.observation_count, other.observation_count)?, + }) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct CaseResourceBounds { + pub max_actions: u32, + pub max_processes: u8, + pub max_payload_bytes: u64, + #[serde(default = "default_max_allocation_bytes")] + pub max_allocation_bytes: u64, + pub max_race_states: u32, +} + +fn default_max_allocation_bytes() -> u64 { + 64 * 1024 * 1024 +} + +impl Default for CaseResourceBounds { + fn default() -> Self { + Self { + max_actions: 64, + max_processes: 20, + max_payload_bytes: 8 * 1024 * 1024, + max_allocation_bytes: default_max_allocation_bytes(), + max_race_states: 256, + } + } +} + #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "type", rename_all = "snake_case")] pub enum FailureInjection { @@ -306,48 +571,575 @@ pub enum FailureInjection { process: String, kind: LaunchFailureKind, }, - KillNode { - logical_node_id: u64, + /// Ordering-only fault. Explicit ordinary actions publish the park marker + /// and await release; the namespace oracle judges their causal history. + /// Legacy `delay_ms` cases are rejected by the required marker fields. + SlowProcess { + process: String, + parked_path: String, + release_path: String, }, - StopOrchestrator, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct BehaviorCase { + pub schema_version: u32, + pub generator_version: u32, pub id: String, pub seed: u64, - pub node_count: u8, + pub live_nodes: BTreeSet, + #[serde(default)] + pub topology: TopologyFamily, + #[serde(default)] + pub scenarios: BTreeSet, + #[serde(default)] + pub routes: Vec, + #[serde(default)] + pub read_only_fixture_paths: BTreeSet, + #[serde(default)] + pub resource_bounds: CaseResourceBounds, pub processes: Vec, #[serde(default)] pub failure: FailureInjection, } +pub(crate) fn contiguous_nodes(node_count: u8) -> BTreeSet { + (1..=u64::from(node_count)).collect() +} + +fn path_is_within(path: &str, prefix: &str) -> bool { + path == prefix + || path + .strip_prefix(prefix) + .is_some_and(|suffix| suffix.starts_with('/')) +} + +pub(crate) fn validate_artifact_id(id: &str, kind: &str) -> Result<(), String> { + if !id.as_bytes().first().is_some_and(u8::is_ascii_alphanumeric) + || !id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) + { + return Err(format!( + "{kind} ID {id:?} must be a safe single ASCII artifact component" + )); + } + Ok(()) +} + +fn validate_data_path(path: &str) -> Result<(), String> { + if !path.starts_with('/') + || path[1..].split('/').any(|part| { + part.is_empty() || matches!(part, "." | "..") || part.contains(['\0', '\\']) + }) + { + return Err(format!("noncanonical data path {path:?}")); + } + Ok(()) +} + +/// A single immutable namespace transformation, shared by recovery phases. +/// Fixtures owned by another phase move with that phase; external fixtures do not. +pub(crate) struct AttemptPaths { + attempt: u64, + read_only: BTreeSet, + owners: BTreeSet, +} + +impl AttemptPaths { + pub(crate) fn new(cases: &[&BehaviorCase], attempt: u64) -> Self { + let owned = cases + .iter() + .flat_map(|case| case.owned_paths()) + .collect::>(); + let read_only = cases + .iter() + .flat_map(|case| &case.read_only_fixture_paths) + .filter(|path| !owned.contains(*path)) + .cloned() + .collect(); + let owners = owned + .iter() + .filter_map(|path| path.strip_prefix("/cases/")) + .map(|suffix| suffix.split('/').next().expect("nonempty owner").to_owned()) + .collect(); + Self { + attempt, + read_only, + owners, + } + } + + pub(crate) fn map(&self, path: &str) -> String { + if self.read_only.contains(path) { + return path.to_owned(); + } + let Some(suffix) = path.strip_prefix("/cases/") else { + return path.to_owned(); + }; + let (owner, rest) = suffix.split_once('/').map_or((suffix, ""), |parts| parts); + if rest.is_empty() { + format!("/cases/{owner}/attempt-{}", self.attempt) + } else { + format!("/cases/{owner}/attempt-{}/{rest}", self.attempt) + } + } + + fn grants(&self, prefixes: &[String], write: bool) -> Vec { + let mut grants = BTreeSet::new(); + for prefix in prefixes { + if prefix == "/cases" { + grants.extend( + self.owners + .iter() + .map(|owner| self.map(&format!("/cases/{owner}"))), + ); + } else if !self.read_only.contains(prefix) { + grants.insert(self.map(prefix)); + } + if !write { + grants.extend( + self.read_only + .iter() + .filter(|path| path_is_within(path, prefix)) + .cloned(), + ); + } + } + grants.into_iter().collect() + } +} + impl BehaviorCase { - pub fn validate(&self) -> Result<(), String> { - if self.node_count < 2 || self.node_count > 3 { - return Err("behavior case topology must contain two or three nodes".to_owned()); + pub fn node_count(&self) -> u8 { + u8::try_from(self.live_nodes.len()).unwrap_or(u8::MAX) + } + + /// Bind observations to an exact scoped execution without delimiter ambiguity. + pub fn execution_request_id(&self, process: &ProcessProgram) -> String { + use sha2::{Digest, Sha256}; + + let mut digest = Sha256::new(); + digest.update(b"myelin-e2e-execution-v1\0"); + for field in [&self.id, &process.id, &process.access.execution_id] { + digest.update((field.len() as u64).to_le_bytes()); + digest.update(field.as_bytes()); } - if self.processes.is_empty() || self.processes.len() > 3 { - return Err("behavior case must contain one to three processes".to_owned()); + digest.update(self.seed.to_le_bytes()); + digest.update(process.logical_node_id.to_le_bytes()); + format!("e2e-{:x}", digest.finalize()) + } + + pub(crate) fn for_attempt(&self, attempt: u64, isolate_paths: bool) -> Self { + let paths = isolate_paths.then(|| AttemptPaths::new(&[self], attempt)); + self.with_attempt_paths(attempt, paths.as_ref()) + } + + pub(crate) fn with_attempt_paths(&self, attempt: u64, paths: Option<&AttemptPaths>) -> Self { + let mut scoped = self.clone(); + for process in &mut scoped.processes { + let old_execution = format!("/runs/{}", process.access.execution_id); + process.access.execution_id = + format!("{}-attempt-{attempt}", process.access.execution_id); + let new_execution = format!("/runs/{}", process.access.execution_id); + if let Some(paths) = paths { + process.access.read_prefixes = paths.grants(&process.access.read_prefixes, false); + process.access.write_prefixes = paths.grants(&process.access.write_prefixes, true); + for action in &mut process.actions { + action.operation.map_paths(|path| *path = paths.map(path)); + } + } + // Grants are evaluated against resolved paths, not the /runs/self alias. + for prefix in process + .access + .read_prefixes + .iter_mut() + .chain(&mut process.access.write_prefixes) + { + if prefix == "/runs" { + *prefix = new_execution.clone(); + } else if path_is_within(prefix, "/runs/self") { + *prefix = format!("{new_execution}{}", &prefix["/runs/self".len()..]); + } else if path_is_within(prefix, &old_execution) { + *prefix = format!("{new_execution}{}", &prefix[old_execution.len()..]); + } + } } - let mut ids = BTreeSet::new(); + if let Some(paths) = paths { + scoped.read_only_fixture_paths = scoped + .read_only_fixture_paths + .iter() + .map(|path| paths.map(path)) + .collect(); + if let FailureInjection::SlowProcess { + parked_path, + release_path, + .. + } = &mut scoped.failure + { + *parked_path = paths.map(parked_path); + *release_path = paths.map(release_path); + } + for route in &mut scoped.routes { + for edge in &mut route.edges { + edge.path = paths.map(&edge.path); + } + for path in &mut route.join_inputs { + *path = paths.map(path); + } + } + } + scoped + } + + pub(crate) fn owned_paths(&self) -> BTreeSet { + let mut owned = BTreeSet::new(); for process in &self.processes { + for path in process + .actions + .iter() + .flat_map(|action| action.operation.paths()) + .flatten() + { + if self.read_only_fixture_paths.contains(path) { + continue; + } + if path.starts_with("/cases/") { + owned.insert(path.to_owned()); + } else if path_is_within(path, "/runs/self") { + owned.insert(format!( + "/runs/{}{}", + process.access.execution_id, + &path["/runs/self".len()..] + )); + } + } + } + owned + } + + pub(crate) fn stream_frame_limit(&self) -> usize { + self.processes + .iter() + .flat_map(|process| &process.actions) + .filter_map(|action| match &action.operation { + ActionOp::StreamWrite { chunks, .. } | ActionOp::StreamRoundTrip { chunks, .. } => { + Some(chunks.len()) + } + ActionOp::GatedStreamWrite { frames, .. } => Some(frames.len()), + _ => None, + }) + .max() + .unwrap_or(0) + .max(1) + } + + pub fn resource_summary(&self) -> Result { + let size = |value: usize| { + u64::try_from(value).map_err(|_| "case resource size overflowed".to_owned()) + }; + let chunks_size = |chunks: &[Vec]| { + if chunks.len() > MAX_STREAM_FRAMES { + return Err("stream logical frame count exceeds hard admission bound".to_owned()); + } + chunks.iter().try_fold(0_u64, |sum, chunk| { + sum.checked_add(size(chunk.len())?) + .ok_or_else(|| "case chunk byte count overflowed".to_owned()) + }) + }; + let mut total = CaseResources { + process_count: size(self.processes.len())?, + ..Default::default() + }; + let read_frames = size(self.stream_frame_limit())?; + let buffered_allocation = |capacity: u64, payload: u64, copies: u64| { + payload + .checked_mul(copies) + .and_then(|bytes| capacity.checked_add(bytes)) + .ok_or_else(|| "data buffer allocation overflowed".to_owned()) + }; + let framed_allocation = |length: u64, copies: u64, headers: u64, rings: u64| { + length + .checked_mul(copies) + .and_then(|bytes| bytes.checked_add(headers * 16)) + .and_then(|bytes| bytes.checked_add(rings * STREAM_ENDPOINT_CAPACITY)) + .ok_or_else(|| "framed stream allocation overflowed".to_owned()) + }; + for action in self.processes.iter().flat_map(|process| &process.actions) { + let (payload_bytes, allocation_bytes, observations) = match &action.operation { + ActionOp::PublishBlob { bytes, .. } + | ActionOp::ReadBlob { + expected: bytes, .. + } => { + let length = size(bytes.len())?; + (length, buffered_allocation(length, length, 1)?, 1) + } + ActionOp::StreamWrite { + chunks, replace, .. + } => { + let length = chunks_size(chunks)?; + ( + length, + framed_allocation(length, 2, 2, 1)?, + 2 + u64::from(*replace) + size(chunks.len())?, + ) + } + ActionOp::StreamRoundTrip { chunks, .. } => { + let length = chunks_size(chunks)?; + ( + length, + framed_allocation(length, 3, 2, 4)?, + 5 + size(chunks.len())? * 2, + ) + } + ActionOp::GatedStreamWrite { frames, .. } => { + if frames.is_empty() { + return Err("gated stream writer requires a first logical frame".to_owned()); + } + let length = chunks_size(frames)?; + ( + length, + framed_allocation(length, 2, 2, 1)?, + 3 + size(frames.len())?, + ) + } + ActionOp::StreamRead { expected, .. } + | ActionOp::StreamReadWithRetry { expected, .. } + | ActionOp::GatedStreamRead { expected, .. } => { + let length = size(expected.len())?; + (length, framed_allocation(length, 2, 1, 3)?, 4 + read_frames) + } + ActionOp::StreamReadInto { + expected, + buffer_sizes, + .. + } => { + if buffer_sizes.is_empty() || buffer_sizes.contains(&0) { + return Err( + "stream readinto requires nonempty, nonzero buffer sizes".to_owned() + ); + } + let buffer = size(*buffer_sizes.iter().max().expect("nonempty buffers"))?; + let length = size(expected.len())?; + ( + length, + framed_allocation(length, 2, 1, 1)? + .checked_add(buffer) + .ok_or("readinto allocation overflowed")?, + 4 + read_frames, + ) + } + ActionOp::DescriptorWrite { + bytes, + length, + method, + .. + } => { + let payload = size(bytes.len())?; + // The reservation and Python payload coexist. write() + // copies into Rust; writefrom() and map() borrow the bytes. + let copies = 1 + u64::from(*method == DescriptorWriteMethod::Write); + ( + payload, + buffered_allocation( + length.unwrap_or(payload).max(payload), + payload, + copies, + )?, + 1, + ) + } + ActionOp::DescriptorRead { + expected, + offset, + method, + .. + } => { + let length = size(expected.len())?; + if *method == DescriptorReadMethod::Mapping { + offset + .checked_add(length) + .ok_or("descriptor mapping range overflowed")?; + } + let copies = if *method == DescriptorReadMethod::Read { + 2 + } else { + 1 + }; + (length, buffered_allocation(0, length, copies)?, 1) + } + ActionOp::MappingExportClose { length, .. } => (0, *length, 1), + ActionOp::Lookup { .. } + | ActionOp::AwaitEntry { .. } + | ActionOp::WaitForQuiescent { .. } => (0, 0, 1), + ActionOp::Rename { .. } | ActionOp::Unlink { .. } => (0, 0, 2), + }; + total = total.checked_add(CaseResources { + process_count: 0, + action_count: 1, + payload_bytes, + allocation_bytes, + observation_count: observations + + u64::from(action.evidence_hints.contains_key(EVIDENCE_HINT_BARRIER)), + })?; + } + Ok(total) + } + + pub fn validate(&self) -> Result<(), String> { + validate_artifact_id(&self.id, "case")?; + if self.schema_version != CASE_SCHEMA_VERSION { + return Err(format!( + "case schema {} is incompatible with supported schema {CASE_SCHEMA_VERSION}", + self.schema_version + )); + } + if self.generator_version == 0 || self.generator_version > GENERATOR_VERSION { + return Err(format!( + "case generator {} is incompatible with supported generator {GENERATOR_VERSION}", + self.generator_version + )); + } + if !self.routes.is_empty() && self.generator_version != GENERATOR_VERSION { + return Err( + "route case generator predates validated concurrent relay semantics".to_owned(), + ); + } + if self.live_nodes.len() < 2 || self.live_nodes.len() > 5 { + return Err( + "behavior case topology must contain two to five exact live nodes".to_owned(), + ); + } + if self.live_nodes.contains(&0) { + return Err("logical node zero is invalid".to_owned()); + } + let resources = self.resource_summary()?; + if resources.process_count == 0 + || resources.process_count > u64::from(self.resource_bounds.max_processes) + || resources.process_count > 20 + { + return Err("behavior case must contain one to twenty bounded processes".to_owned()); + } + if resources.action_count > u64::from(self.resource_bounds.max_actions) { + return Err(format!( + "case has {} actions above configured bound {}", + resources.action_count, self.resource_bounds.max_actions + )); + } + for (kind, bytes, configured, ceiling) in [ + ( + "payload", + resources.payload_bytes, + self.resource_bounds.max_payload_bytes, + 8 * 1024 * 1024, + ), + ( + "allocation", + resources.allocation_bytes, + self.resource_bounds.max_allocation_bytes, + 64 * 1024 * 1024, + ), + ] { + let bound = configured.min(ceiling); + if bytes > bound { + return Err(format!( + "case has {bytes} {kind} bytes above configured bound {bound}" + )); + } + } + for path in &self.read_only_fixture_paths { + validate_data_path(path)?; + } + for operation in self + .processes + .iter() + .flat_map(|process| process.actions.iter().map(|action| &action.operation)) + { + for path in operation.paths().into_iter().flatten() { + validate_data_path(path)?; + } + for path in operation.mutating_paths().into_iter().flatten() { + if self.read_only_fixture_paths.contains(path) { + return Err(format!( + "case mutates declared read-only fixture path {path:?}" + )); + } + if !path.starts_with("/cases/") && !path_is_within(path, "/runs/self") { + return Err(format!( + "writable path {path:?} has no attempt-local ownership" + )); + } + } + } + if self.resource_bounds.max_race_states == 0 { + return Err("case race-state bound must be nonzero".to_owned()); + } + + let mut ids = BTreeSet::new(); + let mut execution_ids = BTreeSet::new(); + for process in &self.processes { + validate_artifact_id(&process.id, "process")?; + validate_artifact_id(&process.access.execution_id, "execution")?; + if !execution_ids.insert(&process.access.execution_id) { + return Err( + "contextual processes must have distinct execution identities".to_owned(), + ); + } + for prefix in process + .access + .read_prefixes + .iter() + .chain(&process.access.write_prefixes) + { + validate_data_path(prefix)?; + } + let execution_root = format!("/runs/{}", process.access.execution_id); + for prefix in &process.access.write_prefixes { + if !path_is_within(prefix, "/cases") + && prefix != "/runs" + && !path_is_within(prefix, "/runs/self") + && !path_is_within(prefix, &execution_root) + { + return Err(format!( + "write grant {prefix:?} has no attempt-local ownership" + )); + } + } if process.id.is_empty() || !ids.insert(process.id.clone()) { return Err(format!( "process ID {:?} is empty or duplicated", process.id )); } - if process.logical_node_id == 0 || process.logical_node_id > u64::from(self.node_count) - { + if !self.live_nodes.contains(&process.logical_node_id) { return Err(format!( - "process {} targets node {} outside topology 1..={}", - process.id, process.logical_node_id, self.node_count + "process {} targets node {} outside exact live set {:?}", + process.id, process.logical_node_id, self.live_nodes )); } if process.actions.is_empty() { return Err(format!("process {} has no actions", process.id)); } + for action in &process.actions { + if matches!( + action.operation, + ActionOp::GatedStreamRead { + park_after_first_frame: true, + .. + } + ) && !matches!( + &self.failure, + FailureInjection::StopProcess { + process: target, + phase: ProcessStopPhase::AfterStreamFirstFrame, + .. + } if target == &process.id + ) { + return Err( + "a parked stream reader must be the first-frame stop target".to_owned() + ); + } + } } for process in &self.processes { for dependency in &process.depends_on { @@ -375,30 +1167,510 @@ impl BehaviorCase { return Err("process dependency graph contains a cycle".to_owned()); } } - match &self.failure { - FailureInjection::None | FailureInjection::StopOrchestrator => {} - FailureInjection::StopProcess { process, .. } - | FailureInjection::LaunchFailure { process, .. } - if ids.contains(process) => {} - FailureInjection::StopProcess { process, .. } - | FailureInjection::LaunchFailure { process, .. } => { + let mut race_groups = std::collections::BTreeMap::::new(); + for action in self.processes.iter().flat_map(|process| &process.actions) { + let ExpectedOutcome::Linearized { + group, successes, .. + } = action.expected + else { + continue; + }; + let entry = race_groups.entry(group).or_insert((successes, 0)); + if entry.0 != successes { return Err(format!( - "stop injection targets unknown process {process:?}" + "race group {group} declares conflicting legal success counts" )); } - FailureInjection::KillNode { logical_node_id } - if (1..=u64::from(self.node_count)).contains(logical_node_id) => {} - FailureInjection::KillNode { logical_node_id } => { + entry.1 = entry + .1 + .checked_add(1) + .ok_or_else(|| "race group member count overflowed".to_owned())?; + } + let mut legal_states = 1_u64; + for (group, (successes, members)) in race_groups { + if members < 2 { return Err(format!( - "node-loss injection targets node {logical_node_id} outside topology 1..={}", - self.node_count + "race group {group} requires at least two operations" )); } + if u32::from(successes) > members { + return Err(format!( + "race group {group} requires {successes} successes from {members} operations" + )); + } + let choose = bounded_binomial(members, u32::from(successes), legal_states) + .ok_or_else(|| "race legal-state count overflowed".to_owned())?; + legal_states = legal_states + .checked_mul(choose) + .ok_or_else(|| "race legal-state count overflowed".to_owned())?; + if legal_states > u64::from(self.resource_bounds.max_race_states) { + return Err(format!( + "case has {legal_states} legal race states above configured bound {}", + self.resource_bounds.max_race_states + )); + } + } + for action in self.processes.iter().flat_map(|process| &process.actions) { + for key in action.evidence_hints.keys() { + if !matches!( + key.as_str(), + EVIDENCE_HINT_TOKEN + | EVIDENCE_HINT_LAP + | EVIDENCE_HINT_EDGE_INDEX + | EVIDENCE_HINT_BARRIER + ) { + return Err(format!( + "action evidence hint {key:?} is not a supported hint key" + )); + } + } + for key in [EVIDENCE_HINT_LAP, EVIDENCE_HINT_EDGE_INDEX] { + if let Some(value) = action.evidence_hints.get(key) { + if value.parse::().is_err() { + return Err(format!( + "action evidence hint {key:?} must be a decimal u32, observed {value:?}" + )); + } + } + } + if let Some(barrier) = action.evidence_hints.get(EVIDENCE_HINT_BARRIER) { + if !matches!( + barrier.as_str(), + BARRIER_TOKEN_RECEIVED | BARRIER_TOKEN_FORWARDED | BARRIER_LAP_COMPLETED + ) { + return Err(format!( + "action evidence hint barrier {barrier:?} is not a supported barrier kind" + )); + } + if !action.evidence_hints.contains_key(EVIDENCE_HINT_TOKEN) { + return Err(format!( + "action evidence hint barrier {barrier:?} requires a {EVIDENCE_HINT_TOKEN:?} hint" + )); + } + } + } + let mut route_ids = BTreeSet::new(); + let mut route_paths = BTreeSet::new(); + for route in &self.routes { + if route.id.is_empty() || !route_ids.insert(&route.id) || route.edges.is_empty() { + return Err( + "data routes require a non-empty identity and at least one edge".to_owned(), + ); + } + for edge in &route.edges { + if !route_paths.insert(edge.path.as_str()) { + return Err("data routes must not share writable edge paths".to_owned()); + } + if edge.source == edge.destination + || edge.source_role == edge.destination_role + || !self.live_nodes.contains(&edge.source) + || !self.live_nodes.contains(&edge.destination) + || edge.path.is_empty() + { + return Err(format!("route {} contains invalid edge {edge:?}", route.id)); + } + for (node, role, source) in [ + (edge.source, &edge.source_role, true), + (edge.destination, &edge.destination_role, false), + ] { + let witnesses = self + .processes + .iter() + .flat_map(|process| { + process.actions.iter().map(move |action| (process, action)) + }) + .filter(|(_, action)| action.operation.path() == edge.path) + .filter( + |(_, action)| match (route.kind, source, &action.operation) { + (DataKind::Blob, true, ActionOp::PublishBlob { .. }) + | (DataKind::Blob, false, ActionOp::ReadBlob { .. }) + | ( + DataKind::Stream, + true, + ActionOp::StreamWrite { .. } + | ActionOp::GatedStreamWrite { .. }, + ) + | ( + DataKind::Stream, + false, + ActionOp::StreamRead { .. } + | ActionOp::StreamReadInto { .. } + | ActionOp::StreamReadWithRetry { .. } + | ActionOp::GatedStreamRead { .. }, + ) => true, + _ => false, + }, + ) + .map(|(process, _)| process); + let mut witnesses = witnesses; + if !witnesses.next().is_some_and(|process| { + process.logical_node_id == node && &process.id == role + }) || witnesses.next().is_some() + { + return Err(format!( + "route {} requires exactly one endpoint owned by {role:?} on node {node} for {:?}", + route.id, edge.path + )); + } + } + } + if self.topology == TopologyFamily::RandomDag { + let vertices = route + .edges + .iter() + .flat_map(|edge| [edge.source_role.as_str(), edge.destination_role.as_str()]) + .collect::>(); + let mut ready = BTreeSet::new(); + while ready.len() < vertices.len() { + let before = ready.len(); + for vertex in &vertices { + if route + .edges + .iter() + .filter(|edge| edge.destination_role == *vertex) + .all(|edge| ready.contains(edge.source_role.as_str())) + { + ready.insert(*vertex); + } + } + if ready.len() == before { + return Err(format!("route {} contains a logical-role cycle", route.id)); + } + } + } + let ring = self.topology == TopologyFamily::RingWalk && route.edges.len() > 1; + let (read_class, write_class) = match route.kind { + DataKind::Blob => (ActionClass::ReadBlob, ActionClass::PublishBlob), + DataKind::Stream => (ActionClass::StreamRead, ActionClass::StreamWrite), + }; + for (edge_index, edge) in route.edges.iter().enumerate() { + let (writer, step) = self + .processes + .iter() + .find_map(|program| { + (program.logical_node_id == edge.source && program.id == edge.source_role) + .then(|| { + program + .actions + .iter() + .position(|action| { + action.operation.path() == edge.path + && action.operation.class() == write_class + }) + .map(|step| (program, step)) + }) + .flatten() + }) + .expect("unique route source checked above"); + if self + .scenarios + .contains(&CoverageScenario::ConcurrentStartup) + && !writer.depends_on.is_empty() + { + return Err( + "concurrent route sources must start without completion dependencies" + .to_owned(), + ); + } + for (input_index, input) in route.edges.iter().enumerate() { + let required = if ring { + edge_index > 0 && input_index == edge_index - 1 + } else { + input.destination_role == edge.source_role + }; + if required + && (input.destination_role != edge.source_role + || !writer.actions[..step].iter().any(|action| { + action.operation.path() == input.path + && action.operation.class() == read_class + })) + { + return Err(format!( + "route {} forwards before consuming its required input {:?}", + route.id, input.path + )); + } + } + } + for path in route + .edges + .iter() + .map(|edge| edge.path.as_str()) + .chain(route.join_inputs.iter().map(String::as_str)) + { + validate_data_path(path)?; + } + if route.join_inputs.iter().collect::>().len() != route.join_inputs.len() + || route + .join_inputs + .iter() + .any(|path| !route.edges.iter().any(|edge| edge.path == *path)) + { + return Err(format!( + "route {} has duplicate or absent join inputs", + route.id + )); + } + } + if self.topology != TopologyFamily::Fixed { + if self.routes.is_empty() { + return Err("case topology requires a structural route witness".to_owned()); + } + if !self + .routes + .iter() + .any(|route| crate::coverage::route_has_topology(route, self.topology)) + { + return Err(format!( + "case lacks a structurally valid {:?} route", + self.topology + )); + } + } + match &self.failure { + FailureInjection::None => {} + FailureInjection::StopProcess { process, .. } + | FailureInjection::LaunchFailure { process, .. } + | FailureInjection::SlowProcess { process, .. } + if ids.contains(process) => {} + FailureInjection::StopProcess { process, .. } + | FailureInjection::LaunchFailure { process, .. } + | FailureInjection::SlowProcess { process, .. } => { + return Err(format!( + "fault injection targets unknown process {process:?}" + )); + } + } + if let FailureInjection::SlowProcess { + process, + parked_path, + release_path, + } = &self.failure + { + self.validate_slow_process(process, parked_path, release_path)?; + } + Ok(()) + } + + fn validate_slow_process( + &self, + target: &str, + parked_path: &str, + release_path: &str, + ) -> Result<(), String> { + let invalid = || { + "slow process requires owned park -> healthy completion -> release actions".to_owned() + }; + if parked_path == release_path + || !parked_path.starts_with("/cases/") + || !release_path.starts_with("/cases/") + || self.read_only_fixture_paths.contains(parked_path) + || self.read_only_fixture_paths.contains(release_path) + { + return Err(invalid()); + } + let awaits = |action: &Action, path: &str| { + action.expected == ExpectedOutcome::Ok + && matches!(&action.operation, ActionOp::AwaitEntry { path: observed, expected_kind } + if observed == path && expected_kind == "blob") + }; + let publishes = |action: &Action, path: &str| { + action.expected == ExpectedOutcome::Ok + && matches!(&action.operation, ActionOp::PublishBlob { path: observed, bytes } + if observed == path && bytes.is_empty()) + }; + let slow = self + .processes + .iter() + .find(|program| program.id == target) + .ok_or_else(invalid)?; + if !slow.depends_on.is_empty() + || slow.actions.len() < 3 + || !publishes(&slow.actions[0], parked_path) + || !awaits(&slow.actions[1], release_path) + { + return Err(invalid()); + } + let releasers = self + .processes + .iter() + .filter(|program| { + program + .actions + .last() + .is_some_and(|action| publishes(action, release_path)) + }) + .collect::>(); + let [releaser] = releasers.as_slice() else { + return Err(invalid()); + }; + let healthy = self + .processes + .iter() + .filter(|program| program.id != target && program.id != releaser.id) + .collect::>(); + if healthy.is_empty() + || releaser.actions.len() != 2 + || !awaits(&releaser.actions[0], parked_path) + || releaser.depends_on.iter().collect::>() + != healthy + .iter() + .map(|program| &program.id) + .collect::>() + || healthy.iter().any(|program| { + program + .depends_on + .iter() + .any(|dependency| dependency == target || dependency == &releaser.id) + || (program.depends_on.is_empty() + && !program + .actions + .first() + .is_some_and(|action| awaits(action, parked_path))) + }) + { + return Err(invalid()); + } + // Marker publishers are unique, and no mutation may spoof/retract a + // marker between the observations establishing the causal chain. + for program in &self.processes { + for (step, action) in program.actions.iter().enumerate() { + let allowed = (program.id == target && step == 0) + || (program.id == releaser.id && step == 1) + || matches!(action.operation, ActionOp::AwaitEntry { .. }); + if !allowed + && (action.operation.path() == parked_path + || action.operation.path() == release_path + || matches!(&action.operation, + ActionOp::Rename { destination, .. } + | ActionOp::GatedStreamRead { observed_path: destination, .. } + if destination == parked_path || destination == release_path)) + { + return Err(invalid()); + } + } } Ok(()) } } +fn bounded_binomial(n: u32, k: u32, current_product: u64) -> Option { + let k = k.min(n - k); + let mut result = 1_u64; + for divisor in 1..=k { + result = result.checked_mul(u64::from(n - k + divisor))?; + result /= u64::from(divisor); + if result.checked_mul(current_product).is_none() { + return None; + } + } + Some(result) +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "outcome", rename_all = "snake_case")] +pub enum DescriptorTerminalResult { + Ok, + Error { + errno: Option, + error_type: String, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "direction", rename_all = "snake_case")] +pub enum DescriptorObservation { + Write { + method: DescriptorWriteMethod, + finish: DescriptorFinish, + terminal_results: Vec, + #[serde(default)] + dropped: bool, + #[serde(default)] + reservation_released: bool, + }, + Read { + method: DescriptorReadMethod, + finish: DescriptorFinish, + terminal_results: Vec, + }, +} +/// Evidence-hint keys understood by codegen. Values are opaque corpus +/// labels; `lap`/`edge_index` values must be decimal `u32` text. +pub const EVIDENCE_HINT_TOKEN: &str = "token"; +pub const EVIDENCE_HINT_LAP: &str = "lap"; +pub const EVIDENCE_HINT_EDGE_INDEX: &str = "edge_index"; +pub const EVIDENCE_HINT_BARRIER: &str = "barrier"; + +/// Barrier kinds an action may request via the `barrier` evidence hint. +pub const BARRIER_TOKEN_RECEIVED: &str = "token_received"; +pub const BARRIER_TOKEN_FORWARDED: &str = "token_forwarded"; +pub const BARRIER_LAP_COMPLETED: &str = "lap_completed"; + +/// A causal milestone record emitted by a binding operation that actually +/// occurred. Barrier records reuse [`ActionObservation`] with +/// `outcome == "barrier"`, the owning action's `step`/`action`/`path`, and +/// the milestone payload in [`ActionObservation::barrier`]. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum BarrierObservation { + /// A `write_stream` context was entered or `read_stream` returned a + /// reader. `incarnation` is the binding-assigned nonzero namespace + /// revision identifying the attached stream, not a pre-open lookup. + StreamOpened { incarnation: u64 }, + /// One completely sent or decoded logical frame. The observation action + /// identifies stream_write versus stream_read, including round trips. + StreamFrame { + incarnation: u64, + index: u64, + length: u64, + digest: String, + }, + /// The first frame of a stream arrived (for gated reads this doubles as + /// the release-observed milestone). + StreamFirstFrame { incarnation: u64 }, + /// The stream reader observed end-of-stream. + StreamEof { incarnation: u64 }, + /// The gated writer observed its release path appear in the namespace. + ReleaseObserved { path: String }, + /// A labeled ring token arrived at this action. + TokenReceived { + token: String, + lap: u32, + edge_index: u32, + }, + /// A labeled ring token was forwarded past this action. + TokenForwarded { + token: String, + lap: u32, + edge_index: u32, + }, + /// A labeled ring lap completed at this action. + LapCompleted { token: String, lap: u32 }, + /// Reserved for harness-side lifecycle evidence (not emitted by codegen). + ProcessStopped { process: String }, + /// Reserved for harness-side lifecycle evidence (not emitted by codegen). + ProcessSpawned { process: String }, + /// Reserved for harness-side lifecycle evidence (not emitted by codegen). + FailureObserved { process: String }, + /// A namespace mutation committed: `to_revision` is the revision the + /// binding returned; `from_revision` is the pre-mutation lookup revision + /// when the entry already existed. + MutationApplied { + from_revision: Option, + to_revision: Option, + }, +} + +/// Bytes actually transferred before the terminal outcome. An absent value +/// means no transfer completed; incomplete values describe the observed prefix. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TransferObservation { + pub length: usize, + pub digest: String, + pub complete: bool, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct ActionObservation { pub process: String, @@ -414,6 +1686,21 @@ pub struct ActionObservation { pub errno: Option, pub error_type: Option, pub error: Option, + pub descriptor: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub transfer: Option, + /// Causal milestone payload when `outcome == "barrier"`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub barrier: Option, + /// Binding-assigned nonzero identity of the attached stream incarnation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub incarnation: Option, + /// Corpus-labeled ring token this action participated in. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub token: Option, + /// Corpus-labeled ring lap this action participated in. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub lap: Option, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -436,3 +1723,399 @@ pub struct CaseObservation { pub case_id: String, pub executions: Vec, } + +#[cfg(test)] +mod tests { + use super::*; + + fn case(operation: ActionOp) -> BehaviorCase { + BehaviorCase { + schema_version: CASE_SCHEMA_VERSION, + generator_version: GENERATOR_VERSION, + id: "resource-case".to_owned(), + seed: 1, + live_nodes: BTreeSet::from([1, 2]), + topology: TopologyFamily::Fixed, + scenarios: BTreeSet::new(), + routes: Vec::new(), + read_only_fixture_paths: BTreeSet::new(), + resource_bounds: CaseResourceBounds::default(), + processes: vec![ProcessProgram { + id: "worker".to_owned(), + logical_node_id: 1, + access: AccessSpec::unrestricted("worker"), + depends_on: Vec::new(), + actions: vec![Action::ok(operation)], + }], + failure: FailureInjection::None, + } + } + + #[test] + fn configured_resource_bounds_cannot_raise_hard_case_ceilings() { + let mut input = case(ActionOp::MappingExportClose { + path: "/models/fixture".to_owned(), + length: 64 * 1024 * 1024, + }); + input.resource_bounds.max_allocation_bytes = u64::MAX; + input.validate().unwrap(); + if let ActionOp::MappingExportClose { length, .. } = + &mut input.processes[0].actions[0].operation + { + *length += 1; + } + assert!(input.validate().is_err()); + input.processes[0].actions[0].operation = ActionOp::PublishBlob { + path: "/cases/bounds/blob".to_owned(), + bytes: vec![0; 8 * 1024 * 1024 + 1], + }; + input.resource_bounds.max_payload_bytes = u64::MAX; + assert!(input.validate().is_err()); + } + + #[test] + fn singleton_race_group_is_rejected_before_execution() { + let mut input = case(ActionOp::Unlink { + path: "/cases/race/missing".to_owned(), + }); + input.processes[0].actions[0].expected = ExpectedOutcome::Linearized { + group: 1, + successes: 0, + error: libc::ENOENT, + }; + assert!(input.validate().is_err()); + let peer = input.processes[0].actions[0].clone(); + input.processes[0].actions.push(peer); + input.validate().unwrap(); + } + + #[test] + fn request_identities_distinguish_delimiters_and_attempts() { + let mut template = case(ActionOp::PublishBlob { + path: "/runs/self/output".to_owned(), + bytes: vec![], + }); + template.processes[0].id = "a-1-b".to_owned(); + template.processes[0].access = AccessSpec::unrestricted("c"); + let mut sibling = template.processes[0].clone(); + sibling.id = "a".to_owned(); + sibling.access = AccessSpec::unrestricted("b-1-c"); + template.processes.push(sibling); + template.validate().unwrap(); + let first = template.for_attempt(7, true); + let second = template.for_attempt(8, true); + assert_ne!( + first.execution_request_id(&first.processes[0]), + first.execution_request_id(&first.processes[1]), + ); + assert_ne!( + first.execution_request_id(&first.processes[0]), + second.execution_request_id(&second.processes[0]), + ); + } + + #[test] + fn attempt_scoping_preserves_restricted_grants_and_fixture_reads() { + let allowed = "/cases/group/allowed/source"; + let denied = "/cases/group/denied/entry"; + let fixture = "/cases/fixture/blob"; + let mut template = case(ActionOp::PublishBlob { + path: allowed.to_owned(), + bytes: vec![1], + }); + template.read_only_fixture_paths.insert(fixture.to_owned()); + template.processes[0].access.read_prefixes = + vec!["/cases/group/allowed".to_owned(), fixture.to_owned()]; + template.processes[0].access.write_prefixes = vec!["/cases/group/allowed".to_owned()]; + template.processes[0].actions.extend([ + Action::error( + ActionOp::Rename { + source: denied.to_owned(), + destination: allowed.to_owned(), + replace: false, + }, + libc::EACCES, + ), + Action::error( + ActionOp::Rename { + source: allowed.to_owned(), + destination: denied.to_owned(), + replace: false, + }, + libc::EACCES, + ), + Action::ok(ActionOp::ReadBlob { + path: fixture.to_owned(), + expected: vec![2], + }), + ]); + template.processes.push(ProcessProgram { + id: "reader".to_owned(), + logical_node_id: 2, + access: AccessSpec::unrestricted("reader"), + depends_on: Vec::new(), + actions: vec![Action::ok(ActionOp::ReadBlob { + path: allowed.to_owned(), + expected: vec![1], + })], + }); + template.routes.push(DataRoute { + id: "allowed-route".to_owned(), + kind: DataKind::Blob, + edges: vec![DataEdge { + source: 1, + destination: 2, + source_role: "worker".to_owned(), + destination_role: "reader".to_owned(), + path: allowed.to_owned(), + }], + join_inputs: vec![allowed.to_owned()], + }); + template.validate().unwrap(); + let first = template.for_attempt(3, true); + let second = template.for_attempt(4, true); + first.validate().unwrap(); + second.validate().unwrap(); + let granted = |prefixes: &[String], path: &str| { + prefixes.iter().any(|prefix| path_is_within(path, prefix)) + }; + for (before, after) in template.processes[0] + .actions + .iter() + .zip(&first.processes[0].actions) + { + assert_eq!(before.expected, after.expected); + for (old_path, new_path) in before + .operation + .paths() + .into_iter() + .flatten() + .zip(after.operation.paths().into_iter().flatten()) + { + assert_eq!( + granted(&template.processes[0].access.read_prefixes, old_path), + granted(&first.processes[0].access.read_prefixes, new_path) + ); + assert_eq!( + granted(&template.processes[0].access.write_prefixes, old_path), + granted(&first.processes[0].access.write_prefixes, new_path) + ); + } + } + assert_eq!( + first.routes[0].edges[0].path, + first.processes[0].actions[0].operation.path() + ); + assert_eq!( + first.routes[0].join_inputs[0], + first.routes[0].edges[0].path + ); + assert_eq!(first.processes[0].actions[3].operation.path(), fixture); + assert!(!first.owned_paths().contains(fixture)); + assert!(first.owned_paths().is_disjoint(&second.owned_paths())); + for path in second.owned_paths() { + assert!(!granted(&first.processes[0].access.write_prefixes, &path)); + } + } + + #[test] + fn attempts_resolve_self_grants_without_aliasing_and_reject_fixed_writes() { + let template = case(ActionOp::PublishBlob { + path: "/runs/self/output".to_owned(), + bytes: vec![], + }); + let first = template.for_attempt(1, true); + let second = first.for_attempt(2, false); + first.validate().unwrap(); + second.validate().unwrap(); + assert_eq!( + first.processes[0].actions[0].operation.path(), + "/runs/self/output" + ); + assert!(first.owned_paths().is_disjoint(&second.owned_paths())); + for path in first.owned_paths() { + assert!( + first.processes[0] + .access + .write_prefixes + .iter() + .any(|prefix| path_is_within(&path, prefix)) + ); + assert!( + !second.processes[0] + .access + .write_prefixes + .iter() + .any(|prefix| path_is_within(&path, prefix)) + ); + } + assert!( + case(ActionOp::PublishBlob { + path: "/runs/fixed/shared".to_owned(), + bytes: vec![] + }) + .validate() + .is_err() + ); + assert!( + case(ActionOp::PublishBlob { + path: "/runs/selfish/shared".to_owned(), + bytes: vec![] + }) + .validate() + .is_err() + ); + } + + #[test] + fn artifact_components_reject_traversal_and_normalization_aliases() { + let template = case(ActionOp::Lookup { + path: "/cases/test/missing".to_owned(), + expected_kind: "blob".to_owned(), + }); + for id in [ + "", + ".", + "..", + "../escape", + "/absolute", + "nested/file", + "nested\\file", + "trailing.", + "space ", + "a\0b", + "café", + ] { + let mut invalid = template.clone(); + invalid.id = id.to_owned(); + assert!(invalid.validate().is_err(), "case {id:?}"); + invalid = template.clone(); + invalid.processes[0].id = id.to_owned(); + assert!(invalid.validate().is_err(), "process {id:?}"); + invalid = template.clone(); + invalid.processes[0].access.execution_id = id.to_owned(); + assert!(invalid.validate().is_err(), "execution {id:?}"); + } + let decoded: BehaviorCase = + serde_json::from_str(&serde_json::to_string(&template).unwrap()).unwrap(); + decoded.validate().unwrap(); + assert_eq!(decoded, template); + } + + #[test] + fn requested_allocations_and_read_buffers_are_bounded_before_generation() { + let mut input = case(ActionOp::StreamReadInto { + path: "/cases/test/stream".to_owned(), + expected: vec![], + buffer_sizes: vec![1], + }); + input.resource_bounds.max_allocation_bytes = STREAM_ENDPOINT_CAPACITY + 17; + input.validate().unwrap(); + input.resource_bounds.max_allocation_bytes -= 1; + assert!(input.validate().is_err()); + input.resource_bounds.max_allocation_bytes += 1; + for sizes in [vec![], vec![0], vec![1, 0], vec![1_usize << 40]] { + if let ActionOp::StreamReadInto { buffer_sizes, .. } = + &mut input.processes[0].actions[0].operation + { + *buffer_sizes = sizes; + } + assert!(input.validate().is_err()); + } + input.processes[0].actions[0].operation = ActionOp::DescriptorWrite { + path: "/cases/test/blob".to_owned(), + flags: libc::O_WRONLY | libc::O_CREAT, + length: Some(1 << 40), + bytes: vec![], + method: DescriptorWriteMethod::Write, + finish: DescriptorFinish::Abort, + }; + assert!(input.validate().is_err()); + input.processes[0].actions[0].operation = ActionOp::MappingExportClose { + path: "/cases/test/blob".to_owned(), + length: 1 << 40, + }; + assert!(input.validate().is_err()); + } + + #[test] + fn descriptor_copying_methods_require_their_additional_buffers() { + let mut input = case(ActionOp::DescriptorRead { + path: "/cases/test/blob".to_owned(), + flags: libc::O_RDONLY, + expected: vec![1; 8], + method: DescriptorReadMethod::ReadInto, + offset: 0, + finish: DescriptorFinish::Close, + }); + input.resource_bounds.max_allocation_bytes = 8; + input.validate().unwrap(); + if let ActionOp::DescriptorRead { method, .. } = + &mut input.processes[0].actions[0].operation + { + *method = DescriptorReadMethod::Read; + } + assert!(input.validate().is_err()); + input.resource_bounds.max_allocation_bytes = 16; + input.validate().unwrap(); + + input.processes[0].actions[0].operation = ActionOp::DescriptorWrite { + path: "/cases/test/blob".to_owned(), + flags: libc::O_WRONLY | libc::O_CREAT, + length: Some(16), + bytes: vec![1; 8], + method: DescriptorWriteMethod::WriteFrom, + finish: DescriptorFinish::Close, + }; + input.resource_bounds.max_allocation_bytes = 24; + input.validate().unwrap(); + if let ActionOp::DescriptorWrite { method, .. } = + &mut input.processes[0].actions[0].operation + { + *method = DescriptorWriteMethod::Write; + } + assert!(input.validate().is_err()); + input.resource_bounds.max_allocation_bytes = 32; + input.validate().unwrap(); + } + + #[test] + fn resource_summary_counts_roundtrip_retry_and_checked_descriptor_ranges() { + let mut input = case(ActionOp::StreamRoundTrip { + path: "/cases/test/stream".to_owned(), + chunks: vec![vec![], vec![1, 2], vec![3]], + }); + input.processes[0] + .actions + .push(Action::ok(ActionOp::StreamReadWithRetry { + path: "/cases/test/retry".to_owned(), + expected: vec![4, 5], + })); + let resources = input.resource_summary().unwrap(); + assert_eq!(resources.payload_bytes, 5); + assert_eq!( + resources.allocation_bytes, + 61 + 7 * STREAM_ENDPOINT_CAPACITY + ); + assert_eq!(resources.observation_count, 18); + input.processes[0] + .actions + .push(Action::ok(ActionOp::DescriptorRead { + path: "/cases/test/blob".to_owned(), + flags: libc::O_RDONLY, + expected: vec![1], + method: DescriptorReadMethod::Mapping, + offset: u64::MAX, + finish: DescriptorFinish::Close, + })); + assert!(input.validate().is_err()); + assert!( + CaseResources { + payload_bytes: u64::MAX, + ..Default::default() + } + .checked_add(resources) + .is_err() + ); + } +} diff --git a/tools/myelin-e2e-fuzz/src/lib.rs b/tools/myelin-e2e-fuzz/src/lib.rs index 4e5d9b1..3c9f25f 100644 --- a/tools/myelin-e2e-fuzz/src/lib.rs +++ b/tools/myelin-e2e-fuzz/src/lib.rs @@ -3,64 +3,77 @@ //! The crate is organized as a module tree; this root is a thin facade that //! preserves the historical flat API used by `main.rs`. +mod budget; +mod campaign; mod codegen; mod corpus; +mod coverage; mod harness; mod ir; mod oracle; +mod remote; mod resources; mod shrink; +pub use budget::{Budget, execution_evidence, record_execution_stage}; +pub use campaign::{ + ArtifactEnvelope, CAMPAIGN_DEADLINE_SECS, CLEANUP_DEADLINE_SECS, CampaignConfig, + CampaignLimits, CampaignPlan, DIAGNOSTIC_DEADLINE_SECS, LOCAL_CLEANUP_RESERVE_SECS, + OfferPolicy, PREPARATION_DEADLINE_SECS, PersistedFixtureEntry, ProviderMode, RecoveryCase, + WORKLOAD_DEADLINE_SECS, +}; pub use codegen::{render_case_python, render_python}; -pub use corpus::{failure_corpus, ordered_pair_corpus, random_short_dags, stable_corpus}; -pub use harness::{ClusterHarness, ClusterHarnessConfig}; +pub use corpus::{ + failure_corpus, ordered_pair_corpus, random_short_dags, random_short_dags_budgeted, + stable_corpus, +}; +pub use coverage::{ + CoverageKey, CoverageLedger, EvidenceIdentity, NodeRole, PayloadClass, case_coverage, +}; +pub use harness::raw_fleet::{ + DeploymentBoundary, RawDockerFleet, RawFleetSnapshot, RawNodeCensus, RawPriorState, +}; +pub use harness::{ + ClusterHarness, ClusterHarnessConfig, FixturePathObservation, FixturePathSnapshot, + HarnessProvider, +}; pub use ir::{ AccessSpec, Action, ActionClass, ActionObservation, ActionOp, BehaviorCase, CaseObservation, - DescriptorFinish, DescriptorReadMethod, DescriptorWriteMethod, ExecutionObservation, - ExpectedOutcome, FailureInjection, LaunchFailureKind, ProcessProgram, ProcessStopPhase, - PythonException, + CaseResourceBounds, CaseResources, CoverageScenario, DataEdge, DataKind, DataRoute, + DescriptorFinish, DescriptorObservation, DescriptorReadMethod, DescriptorTerminalResult, + DescriptorWriteMethod, ExecutionObservation, ExpectedOutcome, FailureInjection, + LaunchFailureKind, ProcessProgram, ProcessStopPhase, PythonException, TopologyFamily, + TransferObservation, }; -pub use oracle::{BehaviorOracle, BlobPublicationTrace, OracleViolation, StreamIncarnationTrace}; -pub use shrink::shrink_failure; +pub use oracle::{ + BehaviorOracle, BlobPublicationTrace, CausalRole, FailureClass, FailureSignature, + ObservedOutcome, OracleViolation, OutcomeExpectation, SemanticAction, StreamIncarnationTrace, +}; +pub use remote::{ + PaidCampaignPhase, PaidCampaignState, authorized_cleanup_limits, await_cleanup_owner, + cleanup_paid_ownership, conservative_selected_cost, decode_offer_results, offer_search_request, + read_campaign_plan, read_coverage_ledger, read_paid_state, recover_cleanup_owner, + run_cleanup_owner, run_cleanup_supervisor, run_scripted_retained_lifecycle, + scan_artifacts_for_secret, select_exact_offers, selected_cleanup_limits, start_cleanup_owner, + write_campaign_plan, write_coverage_ledger, write_paid_state, +}; +pub use resources::{ + BuiltBinaries, DeploymentBundle, GateImageIdentity, PreparedArtifactIdentity, + assemble_deployment_bundle, distinguish_deployment_payload, immutable_registry_reference, + private_fixture_dir, require_prepared_artifacts, resolve_myelin_binaries, + runtime_image_identity, stage_deployment_payload, +}; +pub use shrink::{shrink_failure, try_shrink_failure, try_shrink_failure_budgeted}; #[cfg(test)] mod tests { use super::*; use std::collections::BTreeSet; - - use serde_json::{Value, json}; + use std::process::{Command, Stdio}; use crate::codegen::digest; use crate::corpus::SEEDED_MODEL_PATH; - use crate::resources::{TelemetryResourceCensus, pending_resource_cleanup}; - #[test] - fn generated_barriers_poll_by_deadline_instead_of_attempt_counts() { - let program = ProcessProgram { - id: "barriers".to_owned(), - logical_node_id: 1, - access: AccessSpec::unrestricted("barriers"), - depends_on: Vec::new(), - actions: vec![ - Action::ok(ActionOp::AwaitEntry { - path: "/cases/1/entry".to_owned(), - expected_kind: "blob".to_owned(), - }), - Action::ok(ActionOp::WaitForQuiescent { - path: "/cases/1/stream".to_owned(), - }), - ], - }; - let source = render_python(&program); - assert!(source.contains("deadline = loop.time() + ")); - assert!(source.contains("if loop.time() >= deadline:")); - assert!(!source.contains("range(")); - - let cleanup = crate::codegen::render_cleanup_python(&["/cases/1/entry".to_owned()]); - assert!(cleanup.contains("deadline = loop.time() + ")); - assert!(cleanup.contains("if loop.time() >= deadline:")); - assert!(!cleanup.contains("range(")); - } fn valid_execution() -> ExecutionObservation { ExecutionObservation { process: "p".to_owned(), @@ -81,12 +94,18 @@ mod tests { outcome: "ok".to_owned(), length: None, digest: None, + transfer: None, kind: Some("blob".to_owned()), revision: Some(1), errno: None, active: None, error_type: None, error: None, + descriptor: None, + barrier: None, + incarnation: None, + token: None, + lap: None, }], terminal: true, exit_success: true, @@ -97,21 +116,110 @@ mod tests { } #[test] - fn generated_program_uses_only_public_binding_operations() { - let program = ProcessProgram { - id: "generated".to_owned(), - logical_node_id: 1, - access: AccessSpec::unrestricted("generated"), - depends_on: Vec::new(), - actions: vec![Action::ok(ActionOp::Lookup { - path: SEEDED_MODEL_PATH.to_owned(), - expected_kind: "blob".to_owned(), - })], + fn complete_campaign_python_is_syntactically_valid() { + let cases = crate::corpus::generated_campaign_cases(17, 128, 5).unwrap(); + let sources = cases + .iter() + .flat_map(|case| { + case.processes.iter().map(|process| { + render_case_python(case, process, std::time::Duration::from_secs(120)) + }) + }) + .collect::>(); + let mut child = Command::new("python3") + .args([ + "-c", + "import json,sys\nfor i,source in enumerate(json.load(sys.stdin)):\n compile(source, f'', 'exec')", + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn Python syntax checker"); + serde_json::to_writer( + child.stdin.as_mut().expect("Python syntax checker stdin"), + &sources, + ) + .unwrap(); + drop(child.stdin.take()); + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "generated Python syntax failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + #[test] + fn attempts_receive_fresh_process_and_namespace_ownership() { + let case = stable_corpus(2, 19) + .into_iter() + .find(|case| { + case.processes + .iter() + .flat_map(|process| &process.actions) + .any(|action| action.operation.path().starts_with("/cases/")) + }) + .unwrap(); + let first = case.for_attempt(1, true); + let second = case.for_attempt(2, true); + assert_eq!(first.id, case.id); + assert_ne!( + first.processes[0].access.execution_id, + second.processes[0].access.execution_id + ); + let first_paths = first + .processes + .iter() + .flat_map(|process| &process.actions) + .map(|action| action.operation.path()) + .filter(|path| path.starts_with("/cases/")) + .collect::>(); + let second_paths = second + .processes + .iter() + .flat_map(|process| &process.actions) + .map(|action| action.operation.path()) + .filter(|path| path.starts_with("/cases/")) + .collect::>(); + assert!(first_paths.iter().all(|path| path.contains("/attempt-1/"))); + assert!(second_paths.iter().all(|path| path.contains("/attempt-2/"))); + assert_ne!(first_paths, second_paths); + } + #[test] + fn read_only_fixture_paths_are_neither_scoped_nor_cleaned() { + let path = "/cases/recovery/persisted".to_owned(); + let case = BehaviorCase { + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + id: "read-only-fixture".to_owned(), + seed: 1, + live_nodes: BTreeSet::from([1, 2]), + topology: TopologyFamily::Fixed, + scenarios: BTreeSet::new(), + routes: Vec::new(), + read_only_fixture_paths: BTreeSet::from([path.clone()]), + resource_bounds: CaseResourceBounds::default(), + processes: vec![ProcessProgram { + id: "reader".to_owned(), + logical_node_id: 1, + access: AccessSpec::unrestricted("reader"), + depends_on: Vec::new(), + actions: vec![Action::ok(ActionOp::Lookup { + path: path.clone(), + expected_kind: "blob".to_owned(), + })], + }], + failure: FailureInjection::None, }; - let source = render_python(&program); - assert!(source.contains("swactor.run(main)")); - assert!(source.contains("await data.lookup")); - assert!(!source.contains("data_plane::")); + case.validate().unwrap(); + let scoped = case.for_attempt(7, true); + assert_eq!(scoped.processes[0].actions[0].operation.path(), path); + assert!(scoped.owned_paths().is_empty()); + + let mut invalid = case; + invalid.processes[0].actions = vec![Action::ok(ActionOp::Unlink { path })]; + assert!(invalid.validate().unwrap_err().contains("read-only")); } #[test] @@ -192,9 +300,16 @@ mod tests { let mut torn = valid.clone(); torn.results[0].kind = Some("stream".to_owned()); let case = BehaviorCase { + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, id: "case".to_owned(), seed: 1, - node_count: 2, + live_nodes: crate::ir::contiguous_nodes(2), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: Default::default(), + resource_bounds: Default::default(), processes: vec![ProcessProgram { id: "p".to_owned(), logical_node_id: 1, @@ -207,6 +322,7 @@ mod tests { }], failure: FailureInjection::None, }; + torn.request_id = case.execution_request_id(&case.processes[0]); assert_eq!( BehaviorOracle::verify( &case, @@ -220,20 +336,15 @@ mod tests { "namespace_kind" ); - let mut duplicated = valid.clone(); - duplicated.results.push(duplicated.results[0].clone()); - assert_eq!( - BehaviorOracle::verify( - &case, - &CaseObservation { - case_id: "case".to_owned(), - executions: vec![duplicated], - }, - ) - .unwrap_err() - .invariant, - "duplicate_action" - ); + let mut observation = CaseObservation { + case_id: case.id.clone(), + executions: vec![valid.clone()], + }; + observation.executions[0].request_id = case.execution_request_id(&case.processes[0]); + BehaviorOracle::verify(&case, &observation).unwrap(); + let duplicate = observation.executions[0].results[0].clone(); + observation.executions[0].results.push(duplicate); + assert!(BehaviorOracle::verify(&case, &observation).is_err()); assert_eq!( BehaviorOracle::verify_blob_publication(&BlobPublicationTrace { @@ -260,6 +371,103 @@ mod tests { "stale_incarnation" ); } + + #[test] + fn typed_action_abort_replays_and_shrinks_without_inventing_a_suffix() { + let action = Action::ok(ActionOp::Lookup { + path: SEEDED_MODEL_PATH.to_owned(), + expected_kind: "blob".to_owned(), + }); + let case = BehaviorCase { + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + id: "typed-abort".to_owned(), + seed: 1, + live_nodes: crate::ir::contiguous_nodes(2), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: BTreeSet::from([SEEDED_MODEL_PATH.to_owned()]), + resource_bounds: Default::default(), + processes: vec![ProcessProgram { + id: "p".to_owned(), + logical_node_id: 1, + access: AccessSpec::unrestricted("p"), + depends_on: Vec::new(), + actions: vec![action.clone(), action], + }], + failure: FailureInjection::None, + }; + let mut failed = valid_execution(); + failed.request_id = case.execution_request_id(&case.processes[0]); + failed.exit_success = false; + failed.exit_status = Some(r#"{"kind":"code","value":1}"#.to_owned()); + failed.results[0].outcome = "error".to_owned(); + failed.results[0].errno = Some(libc::ENOENT); + failed.results[0].error_type = Some("FileNotFoundError".to_owned()); + failed.results[0].error = Some("binding lookup failed".to_owned()); + let observe = |candidate: &BehaviorCase| CaseObservation { + case_id: candidate.id.clone(), + executions: vec![failed.clone()], + }; + let signature = BehaviorOracle::verify(&case, &observe(&case)) + .unwrap_err() + .signature; + assert_eq!(signature.invariant, "unexpected_error"); + assert_eq!(signature.failure_class, FailureClass::OutcomeMismatch); + assert_eq!( + signature.causal_role, + CausalRole::Action(SemanticAction::Lookup) + ); + let minimized = try_shrink_failure_budgeted( + case.clone(), + &Budget::new(std::time::Duration::from_secs(1)), + |candidate| { + Ok(BehaviorOracle::verify(candidate, &observe(candidate)) + .is_err_and(|error| error.signature == signature)) + }, + ) + .unwrap(); + assert_eq!(minimized.processes[0].actions.len(), 1); + assert_eq!( + BehaviorOracle::verify(&minimized, &observe(&minimized)) + .unwrap_err() + .signature, + signature, + ); + + let mut canceled = failed.clone(); + canceled.results[0].error_type = Some("TimeoutError".to_owned()); + assert!(crate::oracle::recorded_action_failure(&case.processes[0], &canceled).is_none()); + let mut missing_prefix = failed.clone(); + missing_prefix.results[0].step = 1; + assert!( + crate::oracle::recorded_action_failure(&case.processes[0], &missing_prefix).is_none() + ); + let mut signaled = failed.clone(); + signaled.exit_status = Some(r#"{"kind":"signal","value":9}"#.to_owned()); + assert!(crate::oracle::recorded_action_failure(&case.processes[0], &signaled).is_none()); + + let mut with_sibling = case.clone(); + let mut sibling = case.processes[0].clone(); + sibling.id = "missing-output".to_owned(); + sibling.access.execution_id = "missing-output".to_owned(); + with_sibling.processes.push(sibling); + let mut observation = observe(&with_sibling); + let mut missing = valid_execution(); + missing.process = "missing-output".to_owned(); + missing.request_id = with_sibling.execution_request_id(&with_sibling.processes[1]); + missing.results.clear(); + observation.executions.push(missing); + assert_eq!( + BehaviorOracle::verify(&with_sibling, &observation) + .unwrap_err() + .signature + .failure_class, + FailureClass::MissingEvidence, + ); + } + #[test] fn linearization_groups_reject_impossible_success_counts() { let operation = || ActionOp::Lookup { @@ -280,12 +488,23 @@ mod tests { second.process = "second".to_owned(); second.results[0].process = "second".to_owned(); let case = BehaviorCase { + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, id: "linearized".to_owned(), seed: 1, - node_count: 2, + live_nodes: crate::ir::contiguous_nodes(2), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: std::collections::BTreeSet::from([ + SEEDED_MODEL_PATH.to_owned() + ]), + resource_bounds: Default::default(), processes: vec![program("first"), program("second")], failure: FailureInjection::None, }; + first.request_id = case.execution_request_id(&case.processes[0]); + second.request_id = case.execution_request_id(&case.processes[1]); let error = BehaviorOracle::verify( &case, &CaseObservation { @@ -295,12 +514,20 @@ mod tests { ) .unwrap_err(); assert_eq!(error.invariant, "linearizability"); + let mut over_bound = case; + over_bound.resource_bounds.max_race_states = 1; + assert!( + over_bound + .validate() + .unwrap_err() + .contains("legal race states") + ); } #[test] fn failure_injections_validate_real_targets() { let corpus = failure_corpus(2, 11); - assert_eq!(corpus.len(), 10); + assert_eq!(corpus.len(), 8); assert!(corpus.iter().all(|case| case.validate().is_ok())); let mut invalid = corpus[0].clone(); @@ -311,85 +538,208 @@ mod tests { }; assert!(invalid.validate().is_err()); - invalid.failure = FailureInjection::KillNode { logical_node_id: 3 }; + invalid.failure = FailureInjection::SlowProcess { + process: "missing".to_owned(), + parked_path: "/cases/slow/parked".to_owned(), + release_path: "/cases/slow/release".to_owned(), + }; assert!(invalid.validate().is_err()); } #[test] - fn telemetry_census_tracks_transient_resources_to_zero() { - let frame = |channel: &str, payload: Value| { - json!({ - "channel": channel, - "stream": "1#1", - "payload": { - "encoding": "utf8", - "value": payload.to_string(), - }, - }) - .to_string() + fn typed_read_abort_shrinking_preserves_publication_and_causal_order() { + let path = "/cases/shrink/required"; + let publication = Action::ok(ActionOp::PublishBlob { + path: path.to_owned(), + bytes: b"payload".to_vec(), + }); + let read = Action::ok(ActionOp::ReadBlob { + path: path.to_owned(), + expected: b"payload".to_vec(), + }); + let unrelated = Action::ok(ActionOp::PublishBlob { + path: "/cases/shrink/unrelated".to_owned(), + bytes: b"unrelated".to_vec(), + }); + let program = |id: &str, actions, depends_on| ProcessProgram { + id: id.to_owned(), + logical_node_id: 1, + access: AccessSpec::unrestricted(id), + depends_on, + actions, }; - let started = frame( - "runtime.actors", - json!({ - "event": "started", - "actor": { - "address": "process", - "actor_type": "swactor_process::actor::ProcessActor", - "poisoned": false, + // The same ENOENT is a runtime defect after publication, but correct + // behavior without publication. A typed signature alone cannot tell. + let observe = |case: &BehaviorCase| CaseObservation { + case_id: case.id.clone(), + executions: case + .processes + .iter() + .map(|process| { + let mut execution = valid_execution(); + execution.process = process.id.clone(); + execution.request_id = case.execution_request_id(process); + execution.logical_node_id = process.logical_node_id; + execution.results.clear(); + for (step, action) in process.actions.iter().enumerate() { + let mut result = valid_execution().results.remove(0); + result.process = process.id.clone(); + result.step = step; + result.path = action.operation.path().to_owned(); + match &action.operation { + ActionOp::PublishBlob { bytes, .. } => { + result.action = "publish_blob".to_owned(); + result.length = Some(bytes.len()); + result.digest = Some(digest(bytes)); + result.transfer = Some(TransferObservation { + length: bytes.len(), + digest: digest(bytes), + complete: true, + }); + } + ActionOp::ReadBlob { .. } => { + result.action = "read_blob".to_owned(); + result.outcome = "error".to_owned(); + result.errno = Some(libc::ENOENT); + result.error_type = Some("FileNotFoundError".to_owned()); + result.error = Some("blob is absent".to_owned()); + result.kind = None; + result.revision = None; + execution.exit_success = false; + execution.exit_status = + Some(r#"{"kind":"code","value":1}"#.to_owned()); + } + ActionOp::AwaitEntry { .. } => { + result.action = "lookup".to_owned(); + } + _ => unreachable!(), + } + execution.results.push(result); + if !execution.exit_success { + break; + } + } + execution + }) + .collect(), + }; + // Cover local publication, dependency-ordered publication, and a + // concurrent publisher ordered by the reader's explicit entry wait. + for (separate_processes, wait_for_entry) in [(false, false), (true, false), (true, true)] { + let processes = if separate_processes { + vec![ + program( + "setup", + vec![publication.clone(), unrelated.clone()], + Vec::new(), + ), + program( + "reader", + if wait_for_entry { + vec![ + Action::ok(ActionOp::AwaitEntry { + path: path.to_owned(), + expected_kind: "blob".to_owned(), + }), + read.clone(), + ] + } else { + vec![read.clone()] + }, + if wait_for_entry { + Vec::new() + } else { + vec!["setup".to_owned()] + }, + ), + program("independent", vec![unrelated.clone()], Vec::new()), + ] + } else { + vec![program( + "reader", + vec![publication.clone(), unrelated.clone(), read.clone()], + Vec::new(), + )] + }; + let case = BehaviorCase { + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + id: "shrink-read-abort".to_owned(), + seed: 1, + live_nodes: crate::ir::contiguous_nodes(2), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: BTreeSet::new(), + resource_bounds: Default::default(), + processes, + failure: FailureInjection::None, + }; + case.validate().unwrap(); + let signature = BehaviorOracle::verify(&case, &observe(&case)) + .unwrap_err() + .signature; + assert_eq!(signature.invariant, "unexpected_error"); + assert_eq!(signature.failure_class, FailureClass::OutcomeMismatch); + assert_eq!( + signature.causal_role, + CausalRole::Action(SemanticAction::ReadBlob) + ); + let mut unjustified = case.clone(); + unjustified.processes = vec![program("reader", vec![read.clone()], Vec::new())]; + unjustified.validate().unwrap(); + assert_eq!( + BehaviorOracle::verify(&unjustified, &observe(&unjustified)) + .unwrap_err() + .signature, + signature, + ); + let minimized = try_shrink_failure_budgeted( + case, + &Budget::new(std::time::Duration::from_secs(1)), + |candidate| { + if !candidate + .processes + .iter() + .flat_map(|process| &process.actions) + .any(|action| matches!(action.operation, ActionOp::ReadBlob { .. })) + { + return Ok(false); + } + Ok(BehaviorOracle::verify(candidate, &observe(candidate)) + .is_err_and(|error| error.signature == signature)) }, - }), - ); - let live_arena = frame( - "mvp.arena", - json!({"live_bytes": 64, "active_leases": 1, "pending_leases": 0}), - ); - let mut census = TelemetryResourceCensus::default(); - census - .ingest(&format!("{started}\n{live_arena}\n")) + ) .unwrap(); - let live = census.snapshot(); - let health = json!({ - "nodes": [{"observation": {"event": {"executions": []}}}], - "running_nodes": [1], - "resources": live, - }); - assert!(pending_resource_cleanup(&health, &BTreeSet::new()).is_some()); - let lost_node_health = json!({ - "nodes": [], - "running_nodes": [], - "resources": health["resources"].clone(), - }); - assert_eq!( - pending_resource_cleanup(&lost_node_health, &BTreeSet::new()), - None - ); - - let stopped = frame( - "runtime.actors", - json!({ - "event": "stopped", - "actor": { - "address": "process", - "actor_type": "swactor_process::actor::ProcessActor", - "poisoned": false, - }, - }), - ); - let empty_arena = frame( - "mvp.arena", - json!({"live_bytes": 0, "active_leases": 0, "pending_leases": 0}), - ); - census - .ingest(&format!("{stopped}\n{empty_arena}\n")) - .unwrap(); - let clean = census.snapshot(); - let health = json!({ - "nodes": [{"observation": {"event": {"executions": []}}}], - "running_nodes": [1], - "resources": clean, - }); - assert_eq!(pending_resource_cleanup(&health, &BTreeSet::new()), None); + let actions = minimized + .processes + .iter() + .flat_map(|process| &process.actions) + .collect::>(); + assert_eq!(actions.len(), 2 + usize::from(wait_for_entry)); + assert!(matches!(&actions[0].operation, + ActionOp::PublishBlob { path: actual, bytes } if actual == path && bytes.is_empty())); + assert!(matches!(&actions.last().unwrap().operation, + ActionOp::ReadBlob { path: actual, expected } if actual == path && expected.is_empty())); + assert_eq!( + minimized.processes.len(), + if separate_processes { 2 } else { 1 } + ); + if separate_processes && !wait_for_entry { + assert_eq!(minimized.processes[1].depends_on, ["setup"]); + } + if wait_for_entry { + assert!(matches!(actions[1].operation, ActionOp::AwaitEntry { .. })); + } + assert_eq!( + BehaviorOracle::verify(&minimized, &observe(&minimized)) + .unwrap_err() + .signature, + signature, + ); + } } + #[test] fn shrinker_removes_unrelated_work_and_reduces_payloads() { let case = random_short_dags(19, 1, 3).remove(0); @@ -400,6 +750,36 @@ mod tests { &minimized.processes[0].actions[0].operation, ActionOp::PublishBlob { bytes, .. } if bytes.is_empty() )); - assert_eq!(minimized.node_count, 2); + assert_eq!(minimized.node_count(), 2); + } + #[test] + fn fallible_shrinker_stops_on_restoration_error() { + let case = random_short_dags(23, 1, 3).remove(0); + let mut attempts = 0; + let error = try_shrink_failure(case, |_| { + attempts += 1; + Err::("fixture restoration failed") + }) + .unwrap_err(); + assert_eq!(error, "fixture restoration failed"); + assert_eq!(attempts, 1); + } + + #[test] + fn cancelled_diagnosis_cannot_accept_a_partial_minimization() { + let case = random_short_dags(23, 1, 3).remove(0); + let budget = Budget::new(std::time::Duration::from_secs(30)); + let mut attempts = 0; + let error = try_shrink_failure_budgeted(case, &budget, |_| { + attempts += 1; + budget.cancel(); + Ok(true) + }) + .unwrap_err(); + assert!(error.contains("budget cancelled"), "{error}"); + assert_eq!( + attempts, 1, + "cancelled diagnosis must not launch another replay" + ); } } diff --git a/tools/myelin-e2e-fuzz/src/main.rs b/tools/myelin-e2e-fuzz/src/main.rs index d71fb09..9d26d9d 100644 --- a/tools/myelin-e2e-fuzz/src/main.rs +++ b/tools/myelin-e2e-fuzz/src/main.rs @@ -1,11 +1,24 @@ +use std::collections::{BTreeMap, BTreeSet}; use std::fs; -use std::path::PathBuf; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::io::{BufRead as _, Write as _}; +use std::path::{Path, PathBuf}; +use std::sync::{LazyLock, Mutex}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use myelin_e2e_fuzz::{ - BehaviorCase, ClusterHarness, ClusterHarnessConfig, failure_corpus, random_short_dags, - render_case_python, shrink_failure, stable_corpus, + BehaviorCase, Budget, CAMPAIGN_DEADLINE_SECS, CLEANUP_DEADLINE_SECS, CampaignConfig, + CampaignLimits, CampaignPlan, ClusterHarness, ClusterHarnessConfig, CoverageLedger, + DIAGNOSTIC_DEADLINE_SECS, DeploymentBoundary, DeploymentBundle, FixturePathSnapshot, + HarnessProvider, LOCAL_CLEANUP_RESERVE_SECS, OfferPolicy, PREPARATION_DEADLINE_SECS, + PaidCampaignPhase, PaidCampaignState, ProviderMode, RawDockerFleet, RawPriorState, + RecoveryCase, WORKLOAD_DEADLINE_SECS, authorized_cleanup_limits, conservative_selected_cost, + decode_offer_results, failure_corpus, offer_search_request, random_short_dags_budgeted, + read_campaign_plan, read_coverage_ledger, read_paid_state, render_case_python, + scan_artifacts_for_secret, select_exact_offers, stable_corpus, write_campaign_plan, + write_coverage_ledger, write_paid_state, }; +use myelin_e2e_fuzz::{GateImageIdentity, immutable_registry_reference, runtime_image_identity}; +use sha2::{Digest as _, Sha256}; struct Options { seed: u64, @@ -21,6 +34,26 @@ struct Options { replay: Option, shrink: bool, deadline: Duration, + campaign: bool, + deployment_e2e: bool, + prepare_artifacts: bool, + deployment_nodes: u8, + paid_vastai: bool, + authorize_paid_vastai: bool, + retain_paid_fixture: bool, + gate_attestation: Option, + scripted_provider: bool, + scripted_campaign_resource_overflow: bool, + cleanup_only: Option, + resume_paid: Option, + paid_cleanup_owner: Option, + paid_cleanup_supervisor: Option, + scripted_retained_lifecycle: Option, + fixture_lifetime_secs: Option, + max_total_cost_usd: Option, + max_hourly_cost_usd: Option, + ssh_identity: Option, + api_key_env: String, } impl Options { @@ -32,9 +65,9 @@ impl Options { let mut options = Self { seed, nodes: 2, - failure_cases: false, random_cases: 16, max_cases: None, + failure_cases: false, smoke_only: false, recovery_only: false, artifacts: PathBuf::from("target/e2e-behavioral-fuzz"), @@ -43,6 +76,26 @@ impl Options { replay: None, shrink: true, deadline: Duration::ZERO, + campaign: false, + deployment_e2e: false, + prepare_artifacts: false, + deployment_nodes: 5, + paid_vastai: false, + authorize_paid_vastai: false, + retain_paid_fixture: false, + gate_attestation: None, + scripted_provider: false, + scripted_campaign_resource_overflow: false, + cleanup_only: None, + resume_paid: None, + paid_cleanup_owner: None, + paid_cleanup_supervisor: None, + scripted_retained_lifecycle: None, + fixture_lifetime_secs: None, + max_total_cost_usd: None, + max_hourly_cost_usd: None, + ssh_identity: None, + api_key_env: "VASTAI_API_KEY".to_owned(), }; let mut args = std::env::args().skip(1); while let Some(argument) = args.next() { @@ -51,6 +104,17 @@ impl Options { .ok_or_else(|| format!("{name} requires a value")) }; match argument.as_str() { + "--paid-cleanup-owner" => { + options.paid_cleanup_owner = Some(PathBuf::from(next("--paid-cleanup-owner")?)) + } + "--paid-cleanup-supervisor" => { + options.paid_cleanup_supervisor = + Some(PathBuf::from(next("--paid-cleanup-supervisor")?)) + } + "--scripted-retained-lifecycle" => { + options.scripted_retained_lifecycle = + Some(PathBuf::from(next("--scripted-retained-lifecycle")?)) + } "--seed" => { options.seed = next("--seed")? .parse() @@ -81,6 +145,57 @@ impl Options { "--failure-cases" => options.failure_cases = true, "--recovery-only" => options.recovery_only = true, "--no-shrink" => options.shrink = false, + "--campaign" => options.campaign = true, + "--deployment-e2e" => options.deployment_e2e = true, + "--prepare-artifacts" => options.prepare_artifacts = true, + "--deployment-nodes" => { + options.deployment_nodes = next("--deployment-nodes")? + .parse() + .map_err(|error| format!("invalid --deployment-nodes: {error}"))?; + } + "--paid-vastai" => { + options.campaign = true; + options.paid_vastai = true; + } + "--authorize-paid-vastai" => options.authorize_paid_vastai = true, + "--retain-paid-fixture" => options.retain_paid_fixture = true, + "--gate-attestation" => { + options.gate_attestation = Some(PathBuf::from(next("--gate-attestation")?)) + } + "--scripted-provider" => options.scripted_provider = true, + "--scripted-campaign-resource-overflow" => { + options.scripted_campaign_resource_overflow = true + } + "--cleanup-only" => { + options.cleanup_only = Some(PathBuf::from(next("--cleanup-only")?)) + } + "--resume-paid" => { + options.resume_paid = Some(PathBuf::from(next("--resume-paid")?)) + } + "--fixture-lifetime-secs" => { + options.fixture_lifetime_secs = + Some(next("--fixture-lifetime-secs")?.parse().map_err(|error| { + format!("invalid --fixture-lifetime-secs: {error}") + })?); + } + "--max-total-cost-usd" => { + options.max_total_cost_usd = Some( + next("--max-total-cost-usd")? + .parse() + .map_err(|error| format!("invalid --max-total-cost-usd: {error}"))?, + ); + } + "--max-hourly-cost-usd" => { + options.max_hourly_cost_usd = Some( + next("--max-hourly-cost-usd")? + .parse() + .map_err(|error| format!("invalid --max-hourly-cost-usd: {error}"))?, + ); + } + "--ssh-identity" => { + options.ssh_identity = Some(PathBuf::from(next("--ssh-identity")?)) + } + "--api-key-env" => options.api_key_env = next("--api-key-env")?, "--deadline-secs" => { options.deadline = Duration::from_secs( next("--deadline-secs")? @@ -90,22 +205,117 @@ impl Options { } "--help" | "-h" => { println!( - "myelin-e2e-fuzz [--seed N] [--nodes 2|3] [--random-cases N] \ - [--max-cases N] [--smoke-only] [--failure-cases] [--recovery-only] \ - [--artifacts PATH] [--image TAG] [--no-build-image] [--no-shrink] \ - [--replay CASE.json] [--deadline-secs N]" + "myelin-e2e-fuzz [--campaign [--paid-vastai --authorize-paid-vastai]] \ + [--resume-paid PAID_STATE.json --authorize-paid-vastai] \ + [--fixture-lifetime-secs N] [--max-total-cost-usd N] \ + [--max-hourly-cost-usd N] [--ssh-identity PATH] [--api-key-env NAME] \ + [--gate-attestation ordered-acceptance.json] [--retain-paid-fixture] \ + [--scripted-provider [--scripted-campaign-resource-overflow]] \ + [--cleanup-only PAID_STATE.json] [--deployment-e2e \ + [--deployment-nodes 2..5]] [--seed N] [--nodes 2..5] \ + [--random-cases N] [--max-cases N] [--smoke-only] [--failure-cases] \ + [--recovery-only] [--artifacts PATH] [--image TAG] [--no-build-image] \ + [--prepare-artifacts] [--no-shrink] [--replay CASE.json] --deadline-secs N" ); std::process::exit(0); } other => return Err(format!("unknown option {other:?}")), } } - if !(2..=3).contains(&options.nodes) { - return Err("--nodes must be 2 or 3".to_owned()); + if options.scripted_campaign_resource_overflow + && (!options.paid_vastai + || !options.scripted_provider + || options.cleanup_only.is_some() + || options.resume_paid.is_some() + || options.paid_cleanup_owner.is_some() + || options.paid_cleanup_supervisor.is_some() + || options.scripted_retained_lifecycle.is_some() + || options.deployment_e2e + || options.prepare_artifacts) + { + return Err( + "--scripted-campaign-resource-overflow requires only the scripted paid campaign mode" + .to_owned(), + ); + } + if options.retain_paid_fixture + && ((!options.paid_vastai && options.resume_paid.is_none()) + || options.cleanup_only.is_some() + || options.paid_cleanup_owner.is_some() + || options.paid_cleanup_supervisor.is_some() + || options.scripted_retained_lifecycle.is_some() + || options.deployment_e2e + || options.prepare_artifacts) + { + return Err( + "--retain-paid-fixture requires paid preparation or explicit paid resume" + .to_owned(), + ); + } + if options.prepare_artifacts + && (options.campaign + || options.deployment_e2e + || options.paid_vastai + || options.resume_paid.is_some() + || options.cleanup_only.is_some() + || options.paid_cleanup_owner.is_some() + || options.paid_cleanup_supervisor.is_some() + || options.scripted_retained_lifecycle.is_some() + || options.failure_cases + || options.smoke_only + || options.recovery_only + || options.replay.is_some()) + { + return Err( + "--prepare-artifacts cannot be combined with execution or cleanup modes".to_owned(), + ); + } + if options.paid_cleanup_owner.is_some() + || options.paid_cleanup_supervisor.is_some() + || options.scripted_retained_lifecycle.is_some() + { + return Ok(options); } if options.deadline.is_zero() { return Err("--deadline-secs is required and must be nonzero".to_owned()); } + if options.deployment_e2e && !(2..=5).contains(&options.deployment_nodes) { + return Err("--deployment-nodes must be between 2 and 5".to_owned()); + } + if options.cleanup_only.is_some() { + return Ok(options); + } + if options.resume_paid.is_some() { + if !options.authorize_paid_vastai { + return Err("paid fixture resume requires --authorize-paid-vastai".to_owned()); + } + if options.image.is_none() || options.ssh_identity.is_none() { + return Err("paid fixture resume requires --image and --ssh-identity".to_owned()); + } + return Ok(options); + } + if options.campaign { + if options.fixture_lifetime_secs.is_none() { + return Err("--campaign requires --fixture-lifetime-secs".to_owned()); + } + if options.paid_vastai { + if !options.authorize_paid_vastai { + return Err("paid execution requires --authorize-paid-vastai".to_owned()); + } + if options.image.is_none() + || options.max_total_cost_usd.is_none() + || options.max_hourly_cost_usd.is_none() + || options.ssh_identity.is_none() + { + return Err( + "paid execution requires --image, both cost ceilings, and --ssh-identity" + .to_owned(), + ); + } + } + } else if !(2..=5).contains(&options.nodes) { + return Err("--nodes must be between 2 and 5".to_owned()); + } Ok(options) } } @@ -117,10 +327,218 @@ fn main() { } } +static TIMING_ORIGIN: LazyLock = LazyLock::new(Instant::now); +static TIMING_SPANS: Mutex> = Mutex::new(Vec::new()); + +struct PhaseTimer { + name: String, + started: Instant, +} + +impl PhaseTimer { + fn new(name: impl Into) -> Self { + Self { + name: name.into(), + started: Instant::now(), + } + } +} + +impl Drop for PhaseTimer { + fn drop(&mut self) { + let elapsed = self.started.elapsed(); + myelin_e2e_fuzz::record_execution_stage(&self.name, elapsed, 0, 0); + if let Ok(mut spans) = TIMING_SPANS.lock() { + spans.push(serde_json::json!({ + "stage": self.name, + "start_us": self.started.duration_since(*TIMING_ORIGIN).as_micros(), + "elapsed_us": elapsed.as_micros(), + })); + } + } +} + fn run() -> Result<(), String> { + let started = *TIMING_ORIGIN; let options = Options::parse()?; + if let Some(admission) = &options.scripted_retained_lifecycle { + return myelin_e2e_fuzz::run_scripted_retained_lifecycle(admission, &options.api_key_env); + } + if let Some(admission) = &options.paid_cleanup_supervisor { + return myelin_e2e_fuzz::run_cleanup_supervisor(admission, &options.api_key_env); + } + if let Some(admission) = &options.paid_cleanup_owner { + return myelin_e2e_fuzz::run_cleanup_owner(admission, &options.api_key_env); + } + let result = run_options(&options); + let stamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|error| format!("timing wall clock: {error}"))? + .as_millis(); + let mut evidence = serde_json::json!({ + "schema_version": 1, + "clock": "monotonic", + "elapsed_us": started.elapsed().as_micros(), + "passed": result.is_ok(), + "pending_predicate": result.as_ref().err(), + "seed": options.seed, + "machine": fs::read_to_string("/proc/sys/kernel/hostname").ok(), + "storage": storage_evidence(), + "cpu": fs::read_to_string("/proc/cpuinfo").ok().and_then(|cpu| cpu.lines() + .find(|line| line.starts_with("model name")).map(str::to_owned)), + "memory": fs::read_to_string("/proc/self/status").ok().map(|status| status.lines() + .filter(|line| line.starts_with("VmHWM:") || line.starts_with("VmPeak:")) + .map(str::to_owned).collect::>()), + "configuration": { + "case_deadline_secs": options.deadline.as_secs(), + "campaign_secs": CAMPAIGN_DEADLINE_SECS, + "workload_secs": WORKLOAD_DEADLINE_SECS, + "diagnosis_secs": DIAGNOSTIC_DEADLINE_SECS, + "local_cleanup_reserve_secs": LOCAL_CLEANUP_RESERVE_SECS, + "paid": options.paid_vastai || options.resume_paid.is_some(), + }, + "spans": TIMING_SPANS.lock().map_err(|_| "timing spans poisoned".to_owned())?.as_slice(), + "resources": myelin_e2e_fuzz::execution_evidence(), + }); + if let Ok(secret) = std::env::var(&options.api_key_env) { + if !secret.is_empty() { + redact_evidence(&mut evidence, &secret); + } + } + let timing = durable_json( + &options + .artifacts + .join("timing-runs") + .join(format!("{stamp}-{}.json", std::process::id())), + &evidence, + ); + let final_scan = (|| { + if !(options.paid_vastai || options.resume_paid.is_some() || options.cleanup_only.is_some()) + { + return Ok(()); + } + let secret = std::env::var(&options.api_key_env) + .map_err(|_| "paid credential environment is unset".to_owned())?; + let mut roots = vec![options.artifacts.clone()]; + if let Some(state_path) = options + .resume_paid + .as_ref() + .or(options.cleanup_only.as_ref()) + { + roots.push( + state_path + .parent() + .ok_or("paid state parent missing")? + .to_path_buf(), + ); + roots.push(read_paid_state(state_path)?.state_dir); + } else { + for entry in fs::read_dir(&options.artifacts) + .map_err(|error| format!("scan paid state roots: {error}"))? + { + let path = entry + .map_err(|error| format!("scan paid state entry: {error}"))? + .path() + .join("paid-state.json"); + if path.is_file() { + roots.push(read_paid_state(&path)?.state_dir); + } + } + } + let mut failures = Vec::new(); + for root in roots { + if root.exists() { + if let Err(error) = scan_artifacts_for_secret(&root, secret.as_bytes()) { + failures.push(error); + } + } + } + if failures.is_empty() { + Ok(()) + } else { + Err(failures.join("; ")) + } + })(); + merge_execution_cleanup(result, merge_execution_cleanup(timing, final_scan)) +} + +fn redact_evidence(value: &mut serde_json::Value, secret: &str) { + match value { + serde_json::Value::String(text) => *text = text.replace(secret, "[redacted]"), + serde_json::Value::Array(values) => { + for value in values { + redact_evidence(value, secret); + } + } + serde_json::Value::Object(fields) => { + let original = std::mem::take(fields); + for (key, mut value) in original { + redact_evidence(&mut value, secret); + fields.insert(key.replace(secret, "[redacted]"), value); + } + } + _ => {} + } +} + +fn storage_evidence() -> serde_json::Value { + let mut status = std::mem::MaybeUninit::::uninit(); + if unsafe { libc::statvfs(c".".as_ptr(), status.as_mut_ptr()) } != 0 { + return serde_json::json!({"error": std::io::Error::last_os_error().to_string()}); + } + let status = unsafe { status.assume_init() }; + serde_json::json!({ + "filesystem_id": status.f_fsid, + "block_size": status.f_bsize, + "fragment_size": status.f_frsize, + "blocks": status.f_blocks, + "available_blocks": status.f_bavail, + "available_inodes": status.f_favail, + }) +} + +fn run_options(options: &Options) -> Result<(), String> { let workspace = std::env::current_dir() .map_err(|error| format!("read current workspace directory: {error}"))?; + if options.cleanup_only.is_none() { + if let Some(path) = std::env::var_os("MYELIN_E2E_ARTIFACT_IDENTITY") { + let expected: myelin_e2e_fuzz::PreparedArtifactIdentity = serde_json::from_reader( + fs::File::open(path) + .map_err(|error| format!("open required artifact identity: {error}"))?, + ) + .map_err(|error| format!("decode required artifact identity: {error}"))?; + myelin_e2e_fuzz::require_prepared_artifacts( + &workspace, + &expected, + &Budget::new(options.deadline), + )?; + } + } + if options.prepare_artifacts { + let budget = Budget::new(options.deadline); + let _phase = PhaseTimer::new("artifact-preparation"); + myelin_e2e_fuzz::stage_deployment_payload(&workspace, &options.artifacts, &budget)?; + return budget.check("complete immutable binary and wheel preparation"); + } + if let Some(state_path) = &options.cleanup_only { + return run_cleanup_only(state_path, &options.api_key_env, options.deadline); + } + if options.paid_vastai || options.resume_paid.is_some() { + validate_paid_gates(options, &workspace)?; + } + if let Some(state_path) = &options.resume_paid { + return run_resume_paid(state_path, &options, workspace); + } + if options.deployment_e2e { + return run_deployment_e2e(&options, workspace); + } + if options.campaign { + return run_campaign(&options, workspace); + } + let total = Budget::new(Duration::from_secs(CAMPAIGN_DEADLINE_SECS)); + let execution = total.child(Duration::from_secs( + CAMPAIGN_DEADLINE_SECS - LOCAL_CLEANUP_RESERVE_SECS, + )); let mut cases = if let Some(replay) = &options.replay { vec![ serde_json::from_slice::( @@ -135,11 +553,12 @@ fn run() -> Result<(), String> { if options.smoke_only { stable.truncate(usize::from(options.nodes) * (usize::from(options.nodes) - 1) * 2); } else { - stable.extend(random_short_dags( + stable.extend(random_short_dags_budgeted( options.seed.rotate_left(17), options.random_cases, options.nodes, - )); + &execution, + )?); } stable }; @@ -153,149 +572,4239 @@ fn run() -> Result<(), String> { return Err("no behavioral cases selected".to_owned()); } + for case in &cases { + case.validate()?; + } fs::create_dir_all(&options.artifacts) .map_err(|error| format!("create artifact root: {error}"))?; + let fixture_node_count = if options.replay.is_some() { + cases + .first() + .map(BehaviorCase::node_count) + .ok_or_else(|| "replay case list is empty".to_owned())? + } else { + options.nodes + }; let config = ClusterHarnessConfig { workspace, artifacts: options.artifacts.clone(), - node_count: options.nodes, + node_count: fixture_node_count, seed: options.seed, - image: options.image, + image: options.image.clone(), build_image: options.build_image, deadline: options.deadline, + provider: HarnessProvider::LocalMock, + selected_offer_ids: (1..=u64::from(fixture_node_count)).collect(), + state_dir: None, + reset_state: true, + offer_search_id: None, + provision: true, + adopt_only: false, + relay_url: None, + run_id: 1, }; - let mut harness = ClusterHarness::start(config.clone())?; - harness.verify_running_recovery()?; - if !options.recovery_only { - harness.verify_workload_convergence()?; - } - println!( - "cluster ready: nodes={:?} image={} cases={} seed={}", - harness.node_ids(), - harness.image(), - cases.len(), - options.seed - ); + let mut harness = ClusterHarness::start_with_budgets(config, execution.clone(), total.clone())?; + let result = (|| { + let preparation = PhaseTimer::new("fixture-preparation"); + harness.verify_running_recovery()?; + if !options.recovery_only { + harness.verify_workload_convergence()?; + } + drop(preparation); + let workload = execution.child( + execution + .remaining("standalone workload allocation")? + .saturating_sub(Duration::from_secs(DIAGNOSTIC_DEADLINE_SECS)) + .min(Duration::from_secs(WORKLOAD_DEADLINE_SECS)), + ); + let diagnosis = execution.clone(); + harness.set_execution_budget(workload.clone()); + for (index, case) in cases.iter().enumerate() { + workload.check(&format!("admit case {}", case.id))?; + println!("case {}/{}: {}", index + 1, cases.len(), case.id); + run_campaign_case( + &mut harness, + case, + &options.artifacts, + &options.artifacts, + options.shrink, + &diagnosis, + )?; + } + harness.verify_missing_resource_recovery()?; + workload.check("complete standalone workload") + })(); + let _cleanup = PhaseTimer::new("fixture-cleanup"); + let cleanup_budget = total.child(Duration::from_secs(LOCAL_CLEANUP_RESERVE_SECS)); + let teardown = harness.teardown_with_budget(cleanup_budget); + merge_execution_cleanup(result, teardown)?; + total.check("complete standalone run and cleanup") +} - for (index, case) in cases.iter().enumerate() { - println!("case {}/{}: {}", index + 1, cases.len(), case.id); - let result = harness - .run_case(case) - .and_then(|_| harness.assert_healthy()); - if let Err(error) = result { - let image = harness.image().to_owned(); - let _ = harness.teardown(); - persist_regression(&options.artifacts, case, &error)?; - if !options.shrink { +fn durable_write(path: &Path, contents: &[u8]) -> Result<(), String> { + let parent = path + .parent() + .ok_or_else(|| format!("artifact path {} has no parent", path.display()))?; + fs::create_dir_all(parent).map_err(|error| { + format!( + "create durable artifact parent {}: {error}", + parent.display() + ) + })?; + let mut temp = tempfile::NamedTempFile::new_in(parent) + .map_err(|error| format!("create durable artifact temporary file: {error}"))?; + temp.write_all(contents) + .map_err(|error| format!("write durable artifact: {error}"))?; + temp.as_file_mut() + .sync_all() + .map_err(|error| format!("sync durable artifact: {error}"))?; + temp.persist(path).map_err(|error| { + format!( + "persist durable artifact {}: {}", + path.display(), + error.error + ) + })?; + fs::File::open(parent) + .and_then(|directory| directory.sync_all()) + .map_err(|error| { + format!( + "sync durable artifact directory {}: {error}", + parent.display() + ) + }) +} + +fn durable_json(path: &Path, value: &impl serde::Serialize) -> Result<(), String> { + let parent = path + .parent() + .ok_or_else(|| format!("artifact {} has no parent", path.display()))?; + fs::create_dir_all(parent).map_err(|error| format!("create JSON artifact parent: {error}"))?; + let mut temp = tempfile::NamedTempFile::new_in(parent) + .map_err(|error| format!("create JSON temporary file: {error}"))?; + { + let mut writer = std::io::BufWriter::new(temp.as_file_mut()); + serde_json::to_writer(&mut writer, value) + .map_err(|error| format!("stream JSON artifact: {error}"))?; + writer + .flush() + .map_err(|error| format!("flush JSON artifact: {error}"))?; + } + temp.as_file() + .sync_all() + .map_err(|error| format!("sync JSON artifact: {error}"))?; + temp.persist(path) + .map_err(|error| format!("persist JSON artifact: {}", error.error))?; + fs::File::open(parent) + .and_then(|directory| directory.sync_all()) + .map_err(|error| format!("sync JSON artifact directory: {error}")) +} + +fn claim_campaign_directory(path: &Path, paid: bool) -> Result<(), String> { + if !paid { + return fs::create_dir_all(path) + .map_err(|error| format!("create campaign directory: {error}")); + } + let parent = path + .parent() + .ok_or_else(|| "campaign directory has no parent".to_owned())?; + fs::create_dir_all(parent).map_err(|error| format!("create campaign parent: {error}"))?; + // Never open or replace any ownership/plan file until this exclusive claim succeeds. + fs::create_dir(path).map_err(|error| format!( + "refuse existing or unclaimable paid campaign {}: {error}; use cleanup-only or explicit resume", + path.display()))?; + fs::File::open(parent) + .and_then(|directory| directory.sync_all()) + .map_err(|error| format!("sync paid campaign claim: {error}")) +} + +#[derive(serde::Serialize, serde::Deserialize)] +struct OrderedGateAttestation { + schema_version: u32, + state: String, + completed_unix_ms: u64, + expires_unix_ms: u64, + build_artifacts: BTreeMap, + deployment_artifacts: myelin_e2e_fuzz::PreparedArtifactIdentity, + source_digest: String, + image_identities: BTreeMap, + configuration: GateConfiguration, + stages: Vec, + runs: Vec, + build_elapsed_secs: f64, + elapsed_secs: f64, +} + +#[derive(serde::Serialize, serde::Deserialize)] +struct GateConfiguration { + warm_runs: usize, + image: String, + seed: u64, + case_deadline_secs: u64, + build_images: bool, + workspace: PathBuf, + artifacts: PathBuf, +} + +#[derive(serde::Serialize, serde::Deserialize)] +struct GateStage { + name: String, + state: String, + exit_code: Option, + started_unix_ms: u64, + completed_unix_ms: u64, + command: Vec, + elapsed_secs: f64, +} + +#[derive(serde::Serialize, serde::Deserialize)] +struct GateRun { + index: usize, + state: String, + gate_a: String, + inside_target: bool, + campaign_elapsed_secs: f64, + elapsed_secs: f64, +} + +fn expected_gate_stages(evidence: &OrderedGateAttestation) -> Vec<(String, Vec)> { + let config = &evidence.configuration; + let binary = config + .workspace + .join("target/release/myelin-e2e-fuzz") + .to_string_lossy() + .into_owned(); + let strings = |values: &[&str]| { + values + .iter() + .map(|value| (*value).to_owned()) + .collect::>() + }; + let artifact = |name: &str| config.artifacts.join(name).to_string_lossy().into_owned(); + let mut tests = strings(&["cargo", "test"]); + for package in [ + "swactor", + "myelin-e2e-fuzz", + "swactor-vastai", + "myelin-control-contract", + "provisioning", + "myelin", + "data-plane", + "iroh-driver", + "distribution", + "swactor-process", + "swactor-process-context", + ] { + tests.extend(strings(&["-p", package])); + } + tests.push("--tests".to_owned()); + let mut no_run = tests.clone(); + no_run.push("--no-run".to_owned()); + let mut stages = vec![ + ( + "build".to_owned(), + strings(&[ + "cargo", + "build", + "--release", + "-p", + "myelin", + "--bins", + "-p", + "myelin-e2e-fuzz", + ]), + ), + ("build-test-binaries".to_owned(), no_run), + ( + "build-deployment-artifacts".to_owned(), + strings(&[ + &binary, + "--prepare-artifacts", + "--deadline-secs", + "3600", + "--artifacts", + &artifact("build-deployment-artifacts"), + ]), + ), + ]; + if config.build_images { + let mut parent: Option<&str> = None; + for (role, suffix, image) in [ + ("base", ".base", "myelin-node-base:cuda12.6"), + ("node", "", "myelin-node:latest"), + ("e2e", ".e2e", config.image.as_str()), + ] { + let mut command = strings(&[ + "docker", + "build", + "-f", + &format!("apps/myelin/node-image/Dockerfile{suffix}"), + "-t", + image, + ]); + if let Some(parent) = parent { + let tag = format!("myelin-e2e-parent:{}", parent.trim_start_matches("sha256:")); + stages.push(( + format!("build-image-{role}-parent"), + strings(&["docker", "tag", parent, &tag]), + )); + command.extend(strings(&["--build-arg", &format!("BASE_IMAGE={tag}")])); + } + for (key, value) in [ + ("provenance-version", "1"), + ( + "source-build-input-digest", + evidence + .deployment_artifacts + .source_build_input_digest + .as_str(), + ), + ("image-role", role), + ] { + command.extend(strings(&[ + "--label", + &format!("org.swactor.myelin.e2e.{key}={value}"), + ])); + } + if let Some(parent) = parent { + command.extend(strings(&[ + "--label", + &format!("org.swactor.myelin.e2e.parent-image-id={parent}"), + ])); + } + command.push(".".to_owned()); + stages.push((format!("build-image-{role}"), command)); + parent = evidence + .image_identities + .get(image) + .map(|identity| identity.id.as_str()); + } + } + for index in 1..=config.warm_runs { + let common = strings(&[ + &binary, + "--seed", + &config.seed.to_string(), + "--deadline-secs", + &config.case_deadline_secs.to_string(), + "--no-build-image", + "--image", + &config.image, + ]); + for (suffix, flags) in [ + ( + "gate-a", + strings(&["--deployment-e2e", "--deployment-nodes", "5"]), + ), + ( + "campaign", + strings(&["--campaign", "--fixture-lifetime-secs", "43200"]), + ), + ( + "failure-cases", + strings(&["--nodes", "5", "--failure-cases"]), + ), + ] { + let mut command = common.clone(); + command.extend(flags); + command.extend(strings(&[ + "--artifacts", + &artifact(&format!("warm-{index}/{suffix}")), + ])); + stages.push((format!("warm-{index}-{suffix}"), command)); + } + stages.push((format!("warm-{index}-contract-model-safety"), tests.clone())); + stages.push(( + format!("warm-{index}-scripted-provider"), + strings(&[ + "bash", + "tools/myelin-e2e-fuzz/scripted_safety_gate.sh", + &artifact(&format!("warm-{index}/scripted-provider")), + ]), + )); + } + stages.push(( + "verify-qualified-deployment-artifacts".to_owned(), + strings(&[ + &binary, + "--prepare-artifacts", + "--deadline-secs", + &config.case_deadline_secs.to_string(), + "--artifacts", + &artifact("verify-qualified-deployment-artifacts"), + ]), + )); + stages +} + +fn valid_gate_duration(seconds: f64) -> bool { + seconds.is_finite() && seconds >= 0.0 +} + +fn validate_gate_attestation(evidence: &OrderedGateAttestation, now: u64) -> Result<(), String> { + if evidence.schema_version != 5 + || evidence.state != "passed" + || evidence.completed_unix_ms > now + || evidence.expires_unix_ms <= now + || evidence.expires_unix_ms <= evidence.completed_unix_ms + || evidence.expires_unix_ms - evidence.completed_unix_ms > 86_400_000 + || evidence.configuration.warm_runs < 3 + || evidence.runs.len() != evidence.configuration.warm_runs + || evidence.configuration.case_deadline_secs == 0 + || !evidence.configuration.workspace.is_absolute() + || !evidence.configuration.artifacts.is_absolute() + || !valid_gate_duration(evidence.build_elapsed_secs) + || !valid_gate_duration(evidence.elapsed_secs) + { + return Err( + "paid gate attestation is missing, incomplete, stale, or incompatible".to_owned(), + ); + } + let expected_images = [ + "myelin-node-base:cuda12.6", + "myelin-node:latest", + evidence.configuration.image.as_str(), + ] + .into_iter() + .collect::>(); + if evidence + .image_identities + .keys() + .map(String::as_str) + .collect::>() + != expected_images + || expected_images.len() != 3 + || evidence.image_identities.values().any(|identity| { + identity.os.is_empty() + || identity.architecture.is_empty() + || identity.provenance_version != 1 + || identity.source_build_input_digest + != evidence.deployment_artifacts.source_build_input_digest + || identity.id.strip_prefix("sha256:").is_none_or(|digest| { + digest.len() != 64 || !digest.bytes().all(|byte| byte.is_ascii_hexdigit()) + }) + }) + { + return Err("paid gate evidence omitted an exact deployment runtime image".to_owned()); + } + let source_input = &evidence.deployment_artifacts.source_build_input_digest; + if source_input.len() != 64 || !source_input.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err("qualified source/build input digest is missing or malformed".to_owned()); + } + let base = &evidence.image_identities["myelin-node-base:cuda12.6"]; + let node = &evidence.image_identities["myelin-node:latest"]; + let runtime = &evidence.image_identities[&evidence.configuration.image]; + if base.image_role != "base" + || base.parent_image_id.is_some() + || node.image_role != "node" + || node.parent_image_id.as_ref() != Some(&base.id) + || runtime.image_role != "e2e" + || runtime.parent_image_id.as_ref() != Some(&node.id) + { + return Err( + "runtime images lack the qualified build-time source and parent provenance".to_owned(), + ); + } + let expected = expected_gate_stages(evidence); + if evidence.stages.len() != expected.len() { + return Err( + "paid authorization requires every build, warm gate and final verification stage" + .to_owned(), + ); + } + let mut previous = 0; + for (stage, (name, command)) in evidence.stages.iter().zip(expected) { + if stage.name != name { + return Err(format!( + "missing, duplicate or out-of-order required gate {name}" + )); + } + if stage.command != command { + return Err(format!( + "required gate {name} changed its qualified command or case selection" + )); + } + if stage.state != "passed" + || stage.exit_code != Some(0) + || !valid_gate_duration(stage.elapsed_secs) + || stage.started_unix_ms < previous + || stage.completed_unix_ms < stage.started_unix_ms + || stage.completed_unix_ms > evidence.completed_unix_ms + { + return Err("paid gate stages failed or were not executed in order".to_owned()); + } + previous = stage.completed_unix_ms; + } + let build_count = if evidence.configuration.build_images { + 8 + } else { + 3 + }; + if evidence.build_elapsed_secs + < evidence.stages[..build_count] + .iter() + .map(|stage| stage.elapsed_secs) + .sum::() + || evidence.elapsed_secs + < evidence + .stages + .iter() + .map(|stage| stage.elapsed_secs) + .sum::() + || evidence.elapsed_secs + < evidence.build_elapsed_secs + + evidence + .runs + .iter() + .map(|run| run.elapsed_secs) + .sum::() + { + return Err("paid gate timing omits required phase execution".to_owned()); + } + for (offset, run) in evidence.runs.iter().enumerate() { + let phases = &evidence.stages[build_count + offset * 5..build_count + (offset + 1) * 5]; + if run.index != offset + 1 + || run.state != "passed" + || run.gate_a != "passed" + || !run.inside_target + || !valid_gate_duration(run.campaign_elapsed_secs) + || !valid_gate_duration(run.elapsed_secs) + || run.campaign_elapsed_secs > 300.0 + || run.elapsed_secs > 600.0 + || run.campaign_elapsed_secs != phases[1].elapsed_secs + || run.elapsed_secs < phases.iter().map(|stage| stage.elapsed_secs).sum::() + { + return Err("paid authorization requires every complete local run inside measured campaign and total ceilings".to_owned()); + } + } + Ok(()) +} + +fn hash_file(path: &Path) -> Result { + let mut source = fs::File::open(path) + .map_err(|error| format!("open artifact {}: {error}", path.display()))?; + let mut digest = Sha256::new(); + std::io::copy(&mut source, &mut digest) + .map_err(|error| format!("hash artifact {}: {error}", path.display()))?; + Ok(format!("{:x}", digest.finalize())) +} + +fn artifact_identity(workspace: &Path) -> Result, String> { + ["myelin-e2e-fuzz", "myelin-orchestrator", "myelin-worker"] + .into_iter() + .map(|name| { + Ok(( + name.to_owned(), + hash_file(&workspace.join("target/release").join(name))?, + )) + }) + .collect() +} + +fn source_identity(workspace: &Path) -> Result { + fn collect(root: &Path, paths: &mut Vec) -> Result<(), String> { + for entry in + fs::read_dir(root).map_err(|error| format!("read source directory: {error}"))? + { + let entry = entry.map_err(|error| format!("read source entry: {error}"))?; + let kind = entry + .file_type() + .map_err(|error| format!("inspect source: {error}"))?; + if kind.is_symlink() { return Err(format!( - "case {} failed: {error}; replayable regression persisted", - case.id + "source identity rejects symlink {}", + entry.path().display() )); } - let shrink_root = options.artifacts.join("shrink"); - let expected_failure = failure_signature(&error); - let minimized = shrink_failure(case.clone(), |candidate| { - let attempt = ClusterHarnessConfig { - workspace: config.workspace.clone(), - artifacts: shrink_root.join(format!("attempt-{}", candidate.processes.len())), - node_count: candidate.node_count, - seed: candidate.seed, - image: Some(image.clone()), - build_image: false, - deadline: config.deadline, - }; - let Ok(mut harness) = ClusterHarness::start(attempt) else { - return false; - }; - let result = harness - .run_case(candidate) - .and_then(|_| harness.assert_healthy()); - let _ = harness.teardown(); - matches!( - result, - Err(candidate_error) - if failure_signature(&candidate_error) == expected_failure - ) - }); - let final_attempt = ClusterHarnessConfig { - workspace: config.workspace.clone(), - artifacts: shrink_root.join("final"), - node_count: minimized.node_count, - seed: minimized.seed, - image: Some(image), - build_image: false, - deadline: config.deadline, - }; - let mut final_harness = ClusterHarness::start(final_attempt)?; - let final_result = final_harness - .run_case(&minimized) - .and_then(|_| final_harness.assert_healthy()); - let _ = final_harness.teardown(); - let minimized_error = final_result.map_or_else( - |candidate_error| { - if failure_signature(&candidate_error) == expected_failure { - Ok(candidate_error) - } else { - Err(format!( - "minimized case changed failure from {expected_failure:?} to {:?}", - failure_signature(&candidate_error) - )) - } - }, - |()| Err("minimized case no longer reproduces the failure".to_owned()), - )?; - persist_regression(&options.artifacts, &minimized, &minimized_error)?; + if kind.is_dir() { + if !matches!( + entry.file_name().to_str(), + Some("target" | ".git" | ".venv" | "__pycache__" | "node_modules") + ) { + collect(&entry.path(), paths)?; + } + } else if kind.is_file() { + paths.push(entry.path()); + } + } + Ok(()) + } + let mut paths = [ + "Cargo.toml", + "Cargo.lock", + "rust-toolchain.toml", + ".dockerignore", + "clippy.toml", + ] + .into_iter() + .map(|name| workspace.join(name)) + .collect::>(); + for directory in [ + ".cargo", + "src", + "tests", + "crates", + "xtask", + "apps/myelin", + "tools/myelin-e2e-fuzz", + "tools/vastai", + "tools/actor-control-flow-lint", + ] { + collect(&workspace.join(directory), &mut paths)?; + } + paths.sort(); + let mut digest = Sha256::new(); + for path in paths { + digest.update( + path.strip_prefix(workspace) + .map_err(|error| error.to_string())? + .as_os_str() + .as_encoded_bytes(), + ); + digest.update([0]); + digest.update(hash_file(&path)?.as_bytes()); + digest.update([0]); + } + Ok(format!("{:x}", digest.finalize())) +} + +fn validate_paid_gates(options: &Options, workspace: &Path) -> Result<(), String> { + if options.scripted_provider { + // A test mode is not paid permission: only a literal loopback endpoint is permitted. + let endpoint = std::env::var("VASTAI_BASE_URL").unwrap_or_default(); + let port = endpoint + .strip_prefix("http://127.0.0.1:") + .and_then(|port| port.parse::().ok()) + .filter(|port| *port != 0); + return if port.is_some() { + Ok(()) + } else { + Err("scripted provider requires an explicit http://127.0.0.1:PORT endpoint".to_owned()) + }; + } + let path = options.gate_attestation.as_ref() + .ok_or_else(|| "paid execution requires --gate-attestation from ordered_acceptance.py; operator authorization is not gate evidence".to_owned())?; + let evidence: OrderedGateAttestation = serde_json::from_reader( + fs::File::open(path).map_err(|error| format!("open paid gate attestation: {error}"))?, + ) + .map_err(|error| format!("decode paid gate attestation: {error}"))?; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|error| format!("gate clock: {error}"))? + .as_millis() as u64; + validate_gate_attestation(&evidence, now)?; + if !immutable_registry_reference(&evidence.configuration.image) { + return Err("paid execution requires a qualified repository@sha256:manifest_digest image, not a mutable tag".to_owned()); + } + if evidence.build_artifacts != artifact_identity(workspace)? + || evidence.source_digest != source_identity(workspace)? + || evidence.configuration.workspace + != workspace + .canonicalize() + .map_err(|error| format!("resolve qualified workspace: {error}"))? + || hash_file(&std::env::current_exe().map_err(|error| error.to_string())?)? + != *evidence + .build_artifacts + .get("myelin-e2e-fuzz") + .ok_or("missing tested harness digest")? + || options.image.as_deref() != Some(evidence.configuration.image.as_str()) + { + return Err( + "paid gate evidence does not attest the current binaries, sources and image".to_owned(), + ); + } + let budget = Budget::new(options.deadline); + for (name, expected) in &evidence.image_identities { + if runtime_image_identity(name, &budget)? != *expected { return Err(format!( - "case {} failed: {error}; minimized regression persisted", - case.id + "runtime image {name} changed since ordered local gates" )); } } - harness.verify_missing_resource_recovery()?; - harness.teardown()?; - println!("behavioral fuzz harness completed {} cases", cases.len()); + let runtime = &evidence.image_identities[&evidence.configuration.image]; + if !runtime.repo_digests.contains(&evidence.configuration.image) { + return Err( + "qualified runtime image is not bound to its registry manifest digest".to_owned(), + ); + } + myelin_e2e_fuzz::require_prepared_artifacts( + workspace, + &evidence.deployment_artifacts, + &budget, + )?; Ok(()) } -fn failure_signature(error: &str) -> String { - let first_line = error.lines().next().unwrap_or(error); - for separator in ["; lifecycle=", "; stdout=", "; health="] { - if let Some((signature, _)) = first_line.split_once(separator) { - return signature.to_owned(); - } - } - if let Some((invariant, _)) = first_line.split_once(": ") - && invariant - .chars() - .all(|character| character.is_ascii_lowercase() || character == '_') - { - return invariant.to_owned(); - } - first_line.to_owned() +#[derive(serde::Serialize, serde::Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct RegressionWitness { + schema_version: u32, + case_id: String, + replay_verified: bool, + signature: Option, + artifacts: BTreeMap, } fn persist_regression( - root: &std::path::Path, + root: &Path, case: &BehaviorCase, error: &str, + signature: Option<&myelin_e2e_fuzz::FailureSignature>, + replay_verified: bool, ) -> Result<(), String> { - let directory = root.join("regressions").join(&case.id); + case.validate()?; + if replay_verified && signature.is_none() { + return Err("reusable regression requires a typed replay signature".to_owned()); + } + let directory = root.join("regressions"); fs::create_dir_all(&directory) - .map_err(|write_error| format!("create regression directory: {write_error}"))?; - fs::write( - directory.join("case.json"), - serde_json::to_vec_pretty(case) - .map_err(|write_error| format!("serialize minimized case: {write_error}"))?, - ) - .map_err(|write_error| format!("write minimized case: {write_error}"))?; - fs::write(directory.join("failure.txt"), error) - .map_err(|write_error| format!("write minimized failure: {write_error}"))?; + .map_err(|error| format!("create regression directory: {error}"))?; + // Pending artifacts never appear in the reusable corpus. Rename commits the complete + // directory only after every file and the versioned manifest have reached durable storage. + let pending = tempfile::Builder::new() + .prefix(".pending-regression-") + .tempdir_in(root) + .map_err(|error| format!("stage regression witness: {error}"))?; + let mut artifacts = BTreeMap::new(); + durable_json(&pending.path().join("case.json"), case)?; + durable_write(&pending.path().join("failure.txt"), error.as_bytes())?; + durable_json(&pending.path().join("signature.json"), &signature)?; for process in &case.processes { - fs::write( - directory.join(format!("{}.py", process.id)), - render_case_python(case, process), + durable_write( + &pending.path().join(format!("{}.py", process.id)), + render_case_python(case, process, Duration::from_secs(WORKLOAD_DEADLINE_SECS)) + .as_bytes(), + )?; + } + for entry in fs::read_dir(pending.path()).map_err(|error| error.to_string())? { + let entry = entry.map_err(|error| error.to_string())?; + artifacts.insert( + entry.file_name().to_string_lossy().into_owned(), + hash_file(&entry.path())?, + ); + } + let witness = RegressionWitness { + schema_version: 1, + case_id: case.id.clone(), + replay_verified, + signature: signature.cloned(), + artifacts, + }; + durable_json(&pending.path().join("witness.json"), &witness)?; + let target = directory.join(&case.id); + if target.exists() { + let existing = read_regression_witness(&target, replay_verified)?; + if existing.1 == witness { + return Ok(()); + } + return Err(format!( + "refuse to overwrite committed regression witness {}", + target.display() + )); + } + fs::rename(pending.path(), &target) + .map_err(|error| format!("commit regression witness: {error}"))?; + fs::File::open(&directory) + .and_then(|directory| directory.sync_all()) + .map_err(|error| format!("sync committed regression directory: {error}"))?; + fs::File::open(root) + .and_then(|directory| directory.sync_all()) + .map_err(|error| format!("sync regression staging directory: {error}")) +} + +fn read_regression_witness( + directory: &Path, + require_replay: bool, +) -> Result<(BehaviorCase, RegressionWitness), String> { + let read_json = |name: &str| -> Result { + let path = directory.join(name); + if !fs::symlink_metadata(&path) + .map_err(|error| format!("incomplete regression {}: {error}", path.display()))? + .is_file() + { + return Err(format!( + "regression artifact is not a regular file: {}", + path.display() + )); + } + serde_json::from_reader(fs::File::open(&path).map_err(|error| error.to_string())?) + .map_err(|error| format!("invalid regression artifact {}: {error}", path.display())) + }; + let witness: RegressionWitness = serde_json::from_value(read_json("witness.json")?) + .map_err(|error| format!("incompatible regression manifest: {error}"))?; + if witness.schema_version != 1 + || (require_replay && (!witness.replay_verified || witness.signature.is_none())) + || witness + .signature + .as_ref() + .is_some_and(|signature| signature.invariant.is_empty()) + { + return Err(format!( + "incomplete or incompatible regression witness {}", + directory.display() + )); + } + let case: BehaviorCase = serde_json::from_value(read_json("case.json")?) + .map_err(|error| format!("invalid regression case: {error}"))?; + case.validate()?; + let signature: Option = + serde_json::from_value(read_json("signature.json")?) + .map_err(|error| format!("invalid regression signature: {error}"))?; + if case.id != witness.case_id || signature != witness.signature { + return Err("regression case/signature does not match committed witness".to_owned()); + } + let mut required = BTreeSet::from([ + "case.json".to_owned(), + "failure.txt".to_owned(), + "signature.json".to_owned(), + ]); + required.extend( + case.processes + .iter() + .map(|process| format!("{}.py", process.id)), + ); + if witness.artifacts.keys().cloned().collect::>() != required { + return Err( + "regression manifest does not enumerate every required witness artifact".to_owned(), + ); + } + for (name, expected) in &witness.artifacts { + let path = directory.join(name); + if !fs::symlink_metadata(&path) + .map_err(|error| format!("incomplete regression {}: {error}", path.display()))? + .is_file() + || hash_file(&path)? != *expected + { + return Err(format!( + "regression artifact hash mismatch: {}", + path.display() + )); + } + } + for process in &case.processes { + let generated = + render_case_python(&case, process, Duration::from_secs(WORKLOAD_DEADLINE_SECS)); + if witness.artifacts[&format!("{}.py", process.id)] + != format!("{:x}", Sha256::digest(generated.as_bytes())) + { + return Err( + "regression generated program is incompatible with the current renderer".to_owned(), + ); + } + } + required.insert("witness.json".to_owned()); + let actual = fs::read_dir(directory) + .map_err(|error| error.to_string())? + .map(|entry| entry.map(|entry| entry.file_name().to_string_lossy().into_owned())) + .collect::, _>>() + .map_err(|error| error.to_string())?; + if actual != required { + return Err("regression witness contains uncommitted artifacts".to_owned()); + } + Ok((case, witness)) +} + +fn run_cleanup_only( + state_path: &Path, + api_key_env: &str, + requested_deadline: Duration, +) -> Result<(), String> { + let mut state = read_paid_state(state_path)?; + state.enter_cleanup_only(); + let initial_persistence = write_paid_state(state_path, &state); + let deadline = if requested_deadline.is_zero() { + Duration::from_secs(5 * 60) + } else { + requested_deadline + }; + let cleanup = myelin_e2e_fuzz::cleanup_paid_ownership( + &mut state, + state_path, + &std::env::current_exe().map_err(|error| error.to_string())?, + api_key_env, + deadline, + ); + let persistence = write_paid_state(state_path, &state); + merge_execution_cleanup( + initial_persistence, + merge_execution_cleanup(cleanup, persistence), + ) +} + +fn run_campaign(options: &Options, workspace: PathBuf) -> Result<(), String> { + let _campaign = PhaseTimer::new("campaign-total"); + let total = Budget::new(Duration::from_secs(if options.paid_vastai { + PREPARATION_DEADLINE_SECS + CAMPAIGN_DEADLINE_SECS + CLEANUP_DEADLINE_SECS + } else { + CAMPAIGN_DEADLINE_SECS + })); + let provider = if options.paid_vastai { + ProviderMode::Real + } else { + ProviderMode::Mock + }; + let runtime_image = options + .image + .clone() + .unwrap_or_else(|| format!("myelin-e2e-campaign:{}", options.seed)); + let limits = CampaignLimits { + fixture_lifetime_secs: options + .fixture_lifetime_secs + .expect("campaign lifetime validated by parser"), + total_cost_usd: options.max_total_cost_usd.unwrap_or(0.0), + total_hourly_price_usd: options.max_hourly_cost_usd.unwrap_or(0.0), + case_deadline_secs: options.deadline.as_secs(), + max_campaign_payload_bytes: 512 * 1024 * 1024, + max_campaign_allocation_bytes: 8 * 1024 * 1024 * 1024, + max_case_race_states: 262_144, + }; + let config = CampaignConfig { + provider, + seed: options.seed, + runtime_image: runtime_image.clone(), + limits, + offers: OfferPolicy { + gpu_model: None, + min_gpu_ram_mb: None, + min_compute_cap: Some(700), + min_reliability: Some(0.95), + min_download_mbps: Some(100.0), + min_upload_mbps: None, + max_hourly_price_per_node: options.max_hourly_cost_usd.map(|ceiling| ceiling / 5.0), + blacklist_hosts: vec![59017], + }, + }; + let generation = PhaseTimer::new("campaign-generation-and-oracle"); + let regressions = load_regressions(&options.artifacts)?; + let mut plan = CampaignPlan::build_with_budget(&config, regressions, &total)?; + if options.scripted_campaign_resource_overflow { + plan.inject_scripted_resource_overflow()?; + return match plan.validate() { + Err(error) => Err(error), + Ok(()) => { + Err("scripted campaign resource overflow was unexpectedly admitted".to_owned()) + } + }; + } + plan.validate()?; + drop(generation); + total.check("campaign generation and resource admission")?; + let campaign_root = options.artifacts.join(&plan.campaign_id); + claim_campaign_directory(&campaign_root, options.paid_vastai)?; + let serialization = PhaseTimer::new("campaign-plan-persistence"); + write_campaign_plan(&campaign_root.join("campaign-plan.json"), &plan)?; + drop(serialization); + durable_json( + &campaign_root.join("campaign-resource-plan.json"), + &plan.resources, + )?; + total.check("campaign plan persistence")?; + + if options.paid_vastai { + durable_json( + &campaign_root.join("paid-authorization.json"), + &serde_json::json!({ + "schema_version": 1, + "operator_authorized": options.authorize_paid_vastai, + "scripted_loopback_only": options.scripted_provider, + "retain_development_fixture": options.retain_paid_fixture, + "ordered_attestation_digest": options.gate_attestation.as_ref().map(|path| hash_file(path)).transpose()?, + "plan_digest": hash_file(&campaign_root.join("campaign-plan.json"))?, + "limits": config.limits, + }), + )?; + run_paid_campaign(options, workspace, config, plan, campaign_root, &total) + } else { + run_mock_campaign(options, workspace, config, plan, campaign_root, &total) + } +} + +/// A local `iroh-relay --dev` process. Isolated fixture nodes reach each +/// other only through a relay, exactly like remote nodes behind NAT; running +/// one on the host (reachable from every node bridge via its gateway) keeps +/// the E2E self-contained and gives relay-path traffic LAN latency. +struct LocalRelay { + url: String, + child: std::process::Child, +} + +impl LocalRelay { + fn start(fleet: &RawDockerFleet, budget: &Budget) -> Result, String> { + if std::env::var("MYELIN_E2E_RELAY_URL").is_ok() { + // An explicit relay overrides the local one. + return Ok(None); + } + if which("iroh-relay").is_none() { + return Ok(None); + } + let gateway = fleet.host_gateway()?; + budget.check("launch local relay")?; + let address = format!("{gateway}:3340") + .parse::() + .map_err(|error| format!("relay address: {error}"))?; + let mut child = std::process::Command::new("iroh-relay") + .arg("--dev") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .map_err(|error| format!("spawn local iroh relay: {error}"))?; + let url = format!("http://{gateway}:3340"); + let readiness = budget.child(Duration::from_secs(30)); + let result = (|| { + loop { + let remaining = readiness.remaining("local relay socket ready or child exit")?; + if let Some(status) = child + .try_wait() + .map_err(|error| format!("relay child status: {error}"))? + { + return Err(format!("local relay exited before readiness: {status}")); + } + if std::net::TcpStream::connect_timeout( + &address, + remaining.min(Duration::from_millis(100)), + ) + .is_ok() + { + return Ok(()); + } + // The relay has no readiness notification contract. Reconcile + // its socket while waking immediately for its actual pidfd exit. + wait_process_event(child.id(), remaining.min(Duration::from_millis(25)))?; + } + })(); + if let Err(error) = result { + let relay = Self { url, child }; + return merge_execution_cleanup(Err(error), relay.stop(budget)); + } + Ok(Some(Self { url, child })) + } + + fn stop(mut self, budget: &Budget) -> Result<(), String> { + if self + .child + .try_wait() + .map_err(|error| format!("observe local relay: {error}"))? + .is_some() + { + return Ok(()); + } + let signal = self + .child + .kill() + .map_err(|error| format!("stop local relay: {error}")); + let exit = wait_process_exit(self.child.id(), budget); + // Always try to reap, even when the inherited deadline has expired. + let reap = self + .child + .try_wait() + .map_err(|error| format!("reap local relay: {error}")) + .and_then(|status| status.ok_or_else(|| "local relay remains unreaped".to_owned())) + .map(|_| ()); + merge_execution_cleanup(merge_execution_cleanup(signal, exit), reap) + } +} + +impl Drop for LocalRelay { + fn drop(&mut self) { + // Retain Child ownership through forced reaping, including startup + // failure paths. No new cleanup deadline is created by this fallback. + if !matches!(self.child.try_wait(), Ok(Some(_))) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } + } +} + +fn which(program: &str) -> Option { + let path = std::env::var_os("PATH")?; + std::env::split_paths(&path) + .map(|directory| directory.join(program)) + .find(|candidate| candidate.is_file()) +} + +fn deployment_relay_url(local: Option<&LocalRelay>) -> Option { + if let Some(relay) = local { + return Some(relay.url.clone()); + } + std::env::var("MYELIN_E2E_RELAY_URL").ok() +} + +/// Deployment E2E over the raw SSH fleet fixture. +/// +/// Proves the properties the remote path needs: blank nodes converge through +/// one SSH bootstrap transaction per node, workers report the deployment +/// identity, no SSH child survives bootstrap, and a second generation +/// refreshes the same containers through reconciliation alone. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum DeploymentInterruption { + SshClientLoss, + OrchestratorLoss, +} + +const DEPLOYMENT_GATE_EVIDENCE_SCHEMA_VERSION: u32 = 4; + +#[derive(serde::Serialize)] +struct DeploymentGateEvidence { + schema_version: u32, + gate: String, + run_id: u64, + node_count: u8, + initial_fixture: myelin_e2e_fuzz::RawFleetSnapshot, + rounds: Vec, + cleanup: DeploymentCleanupEvidence, + passed: bool, + pending_prior_census: Vec, + pending_deployment_generation: Option, + failure: Option, +} + +#[derive(serde::Serialize)] +struct DeploymentRoundEvidence { + deployment_generation: String, + artifact_digest: String, + executable_digest: String, + prior_states: BTreeMap, + interruption: Option, + temporarily_unreachable_node: Option, + fixture: myelin_e2e_fuzz::RawFleetSnapshot, + raw_census: Vec, + fleet_status: serde_json::Value, + telemetry: DeploymentTelemetryEvidence, + matching_receipt_required_for_runtime_admission: bool, + stale_processes_and_descendants_absent: bool, + ssh_session_independent: bool, + fresh_membership_routing_readiness_telemetry: bool, + directed_all_pairs_behavior_passed: bool, +} + +#[derive(serde::Serialize)] +struct DeploymentTelemetryEvidence { + archives: BTreeSet, + provider_receipts: BTreeMap, + creating_event_count: usize, + creating_events: Vec, + bootstrapping_events: Vec, +} + +#[derive(serde::Serialize)] +struct DeploymentInterruptionEvidence { + kind: String, + boundary: String, +} + +#[derive(serde::Serialize)] +struct LocalProcessIdentity { + pid: u32, + start_ticks: u64, + #[serde(skip)] + pidfd: std::os::fd::OwnedFd, +} + +#[derive(serde::Serialize)] +struct DeploymentCleanupEvidence { + attempted: bool, + complete: bool, + remaining_resources: Option>, + orphan_processes: Vec, + error: Option, +} + +fn run_deployment_e2e(options: &Options, workspace: PathBuf) -> Result<(), String> { + let _gate_timer = PhaseTimer::new("deployment-gate-total"); + // The fixture spans every sequential redeployment, not one workload campaign. + // Retain each operation's deadline and reserve cleanup outside all rounds. + let rounds = 10 + 2 * 5_u32.div_ceil(u32::from(options.deployment_nodes)); + let execution_duration = options + .deadline + .saturating_mul(rounds) + .saturating_add(Duration::from_secs(PREPARATION_DEADLINE_SECS)); + let total = + Budget::new(execution_duration.saturating_add(Duration::from_secs(CLEANUP_DEADLINE_SECS))); + let execution = total.child(execution_duration); + // Crashed orchestrators must not hand orphaned SSH clients to an external + // init process where the fixture can no longer account for or reap them. + if unsafe { libc::prctl(libc::PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0) } != 0 { + return Err(format!( + "enable deployment child reaping: {}", + std::io::Error::last_os_error() + )); + } + let root = options.artifacts.join("deployment-e2e"); + fs::create_dir_all(&root).map_err(|error| format!("create deployment artifacts: {error}"))?; + let fixture_dir = root.join("fixture"); + fs::create_dir_all(&fixture_dir) + .map_err(|error| format!("create fixture artifacts: {error}"))?; + let state_dir = myelin_e2e_fuzz::private_fixture_dir(&root)?; + let run_id = options.seed | (1 << 62); + + let node_count = options.deployment_nodes; + let first_prior_states = [ + RawPriorState::NoWorkerOrTrustworthyMetadata, + RawPriorState::HealthyStaleWorker, + RawPriorState::DeadWorkerWithStaleMetadata, + RawPriorState::MultipleStaleWorkersAndDescendants, + RawPriorState::InterruptedTransferAndPartialInstall, + ]; + let second_prior_states = [ + RawPriorState::CorruptActiveBinary, + RawPriorState::CorruptActivationPointer, + RawPriorState::CorruptDeploymentDescriptor, + RawPriorState::DeadWorkerWithStaleMetadata, + RawPriorState::MultipleStaleWorkersAndDescendants, + ]; + let payload = myelin_e2e_fuzz::stage_deployment_payload(&workspace, &root, &execution)?; + let generation_one = format!("deploy-{run_id}-a"); + let generation_two = format!("deploy-{run_id}-b"); + let generation_three = format!("deploy-{run_id}-c"); + let bundle_one = + myelin_e2e_fuzz::assemble_deployment_bundle(&root, &payload, &generation_one, &execution)?; + myelin_e2e_fuzz::distinguish_deployment_payload(&payload, &generation_two)?; + let mut prior_state_bundles = Vec::new(); + for (generation, states) in [ + (&generation_two, first_prior_states.as_slice()), + (&generation_three, second_prior_states.as_slice()), + ] { + if generation == &generation_three { + myelin_e2e_fuzz::distinguish_deployment_payload(&payload, generation)?; + } + for (index, chunk) in states.chunks(usize::from(node_count)).enumerate() { + let identity = if index == 0 { + generation.clone() + } else { + format!("{generation}-prior-state-{index}") + }; + let mut assigned_states = chunk.to_vec(); + assigned_states.resize(usize::from(node_count), RawPriorState::HealthyStaleWorker); + prior_state_bundles.push(( + myelin_e2e_fuzz::assemble_deployment_bundle( + &root, &payload, &identity, &execution, + )?, + assigned_states, + )); + } + } + let bundle_two = &prior_state_bundles[0].0; + let bundle_three = + &prior_state_bundles[first_prior_states.len().div_ceil(usize::from(node_count))].0; + let mut interruption_bundles = Vec::new(); + for interruption in [ + DeploymentInterruption::SshClientLoss, + DeploymentInterruption::OrchestratorLoss, + ] { + let interruption_name = match interruption { + DeploymentInterruption::SshClientLoss => "ssh", + DeploymentInterruption::OrchestratorLoss => "orchestrator", + }; + for boundary in [ + DeploymentBoundary::TransferStarted, + DeploymentBoundary::BeforeLaunch, + DeploymentBoundary::AfterLaunch, + DeploymentBoundary::BeforeReceipt, + ] { + let generation = format!( + "deploy-{run_id}-fault-{interruption_name}-{}", + boundary.as_remote_name() + ); + interruption_bundles.push(( + interruption, + boundary, + myelin_e2e_fuzz::assemble_deployment_bundle( + &root, + &payload, + &generation, + &execution, + )?, + )); + } + } + let unreachable_bundle = myelin_e2e_fuzz::assemble_deployment_bundle( + &root, + &payload, + &format!("deploy-{run_id}-temporarily-unreachable"), + &execution, + )?; + if bundle_one.artifact_digest == bundle_two.artifact_digest + || bundle_one.executable_digest == bundle_two.executable_digest + || bundle_one.artifact_digest == bundle_three.artifact_digest + || bundle_one.executable_digest == bundle_three.executable_digest + || bundle_two.artifact_digest == bundle_three.artifact_digest + || bundle_two.executable_digest == bundle_three.executable_digest + || interruption_bundles.iter().any(|(_, _, bundle)| { + bundle.artifact_digest != bundle_three.artifact_digest + || bundle.executable_digest != bundle_three.executable_digest + }) + || unreachable_bundle.artifact_digest != bundle_three.artifact_digest + || unreachable_bundle.executable_digest != bundle_three.executable_digest + { + return Err( + "generations A, B, and C must have distinct payloads; interruption rounds must reuse C bytes".to_owned(), + ); + } + + let fixture_timer = PhaseTimer::new("raw-fixture-construction"); + let fleet = RawDockerFleet::ensure( + &workspace, + &fixture_dir, + u32::from(node_count), + options.build_image, + &execution, + &total, + )?; + drop(fixture_timer); + let immutable_fixture = match fleet.snapshot() { + Ok(snapshot) => snapshot, + Err(error) => return merge_execution_cleanup(Err(error), fleet.teardown()), + }; + let mut evidence = DeploymentGateEvidence { + schema_version: DEPLOYMENT_GATE_EVIDENCE_SCHEMA_VERSION, + gate: "retained-node-redeployment".to_owned(), + run_id, + node_count, + initial_fixture: immutable_fixture.clone(), + rounds: Vec::new(), + cleanup: DeploymentCleanupEvidence { + attempted: false, + complete: false, + remaining_resources: None, + orphan_processes: Vec::new(), + error: None, + }, + passed: false, + pending_prior_census: Vec::new(), + pending_deployment_generation: Some(generation_one.clone()), + failure: None, + }; + let evidence_path = root.join("gate-a-evidence.json"); + let mut relay = None; + let manifest = RawDockerFleet::manifest_path(&fixture_dir); + let result = (|| { + write_deployment_gate_evidence(&evidence_path, &evidence)?; + relay = LocalRelay::start(&fleet, &execution)?; + let deployment = DeploymentRoundContext { + options, + workspace: &workspace, + fixture_dir: &fixture_dir, + state_dir: &state_dir, + manifest: &manifest, + fleet: &fleet, + relay: relay.as_ref(), + run_id, + node_count, + immutable_fixture: &immutable_fixture, + cleanup: &total, + }; + let mut telemetry_before = deployment_telemetry_files(&fixture_dir)?; + let mut harness = + start_deployment_round(&deployment, &bundle_one, true, true, &execution, false)?; + evidence.rounds.push(capture_deployment_round( + &deployment, + &harness, + &bundle_one, + &telemetry_before, + DeploymentRoundObservation { + allow_creating: true, + prior_states: uniform_prior_states( + &fleet, + "no_worker_or_trustworthy_deployment_metadata", + ), + interruption: None, + temporarily_unreachable_node: None, + }, + )?); + evidence.pending_deployment_generation = None; + write_deployment_gate_evidence(&evidence_path, &evidence)?; + harness.retain_remote_fixture(true)?; + telemetry_before = deployment_telemetry_files(&fixture_dir)?; + + for (bundle, prior_states) in &prior_state_bundles { + let _round_timer = + PhaseTimer::new(format!("deployment-round-{}", bundle.deployment_generation)); + fleet.record_prior_processes()?; + evidence.pending_prior_census = fleet.census()?; + evidence.pending_deployment_generation = Some(bundle.deployment_generation.clone()); + write_deployment_gate_evidence(&evidence_path, &evidence)?; + inject_prior_states(&fleet, prior_states)?; + evidence.pending_prior_census = fleet.census()?; + write_deployment_gate_evidence(&evidence_path, &evidence)?; + let mut refreshed = + start_deployment_round(&deployment, bundle, false, false, &execution, false)?; + evidence.rounds.push(capture_deployment_round( + &deployment, + &refreshed, + bundle, + &telemetry_before, + DeploymentRoundObservation { + allow_creating: false, + prior_states: enumerated_prior_states(&fleet, prior_states)?, + interruption: None, + temporarily_unreachable_node: None, + }, + )?); + evidence.pending_prior_census.clear(); + evidence.pending_deployment_generation = None; + write_deployment_gate_evidence(&evidence_path, &evidence)?; + refreshed.retain_remote_fixture(true)?; + telemetry_before = deployment_telemetry_files(&fixture_dir)?; + } + + for (interruption, boundary, bundle) in &interruption_bundles { + let _round_timer = + PhaseTimer::new(format!("deployment-round-{}", bundle.deployment_generation)); + fleet.record_prior_processes()?; + evidence.pending_prior_census = fleet.census()?; + evidence.pending_deployment_generation = Some(bundle.deployment_generation.clone()); + write_deployment_gate_evidence(&evidence_path, &evidence)?; + let mut interrupted = run_interrupted_deployment_round( + &deployment, + bundle, + *boundary, + *interruption, + &execution, + )?; + evidence.rounds.push(capture_deployment_round( + &deployment, + &interrupted, + bundle, + &telemetry_before, + DeploymentRoundObservation { + allow_creating: false, + prior_states: uniform_prior_states(&fleet, "healthy_stale_worker"), + interruption: Some(DeploymentInterruptionEvidence { + kind: match interruption { + DeploymentInterruption::SshClientLoss => "ssh_client_loss", + DeploymentInterruption::OrchestratorLoss => "orchestrator_loss", + } + .to_owned(), + boundary: boundary.as_remote_name().to_owned(), + }), + temporarily_unreachable_node: None, + }, + )?); + evidence.pending_prior_census.clear(); + evidence.pending_deployment_generation = None; + write_deployment_gate_evidence(&evidence_path, &evidence)?; + interrupted.retain_remote_fixture(true)?; + telemetry_before = deployment_telemetry_files(&fixture_dir)?; + } + + fleet.record_prior_processes()?; + evidence.pending_deployment_generation = + Some(unreachable_bundle.deployment_generation.clone()); + evidence.pending_prior_census = fleet.census()?; + write_deployment_gate_evidence(&evidence_path, &evidence)?; + let mut recovered = + run_temporarily_unreachable_round(&deployment, &unreachable_bundle, &execution)?; + evidence.rounds.push(capture_deployment_round( + &deployment, + &recovered, + &unreachable_bundle, + &telemetry_before, + DeploymentRoundObservation { + allow_creating: false, + prior_states: uniform_prior_states( + &fleet, + "temporarily_unreachable_with_prior_state_intact", + ), + interruption: None, + temporarily_unreachable_node: Some(1), + }, + )?); + evidence.pending_prior_census.clear(); + evidence.pending_deployment_generation = None; + write_deployment_gate_evidence(&evidence_path, &evidence)?; + execution.check("complete all Gate A rounds")?; + recovered.retain_remote_fixture(true) + })(); + evidence.failure = result.as_ref().err().cloned(); + evidence.cleanup.attempted = true; + let before_cleanup = write_deployment_gate_evidence(&evidence_path, &evidence); + let diagnostics = if result.is_err() { + durable_json( + &root.join("failure-diagnostics.json"), + &fleet.failure_diagnostics(&execution.child(Duration::from_secs(5))), ) - .map_err(|write_error| format!("write minimized Python source: {write_error}"))?; + } else { + Ok(()) + }; + let _cleanup_timer = PhaseTimer::new("deployment-fixture-cleanup"); + let cleanup_budget = total.child(Duration::from_secs(CLEANUP_DEADLINE_SECS)); + // Capture orphan identities before spawning cleanup commands. The orphan + // owner must never discover and kill a live Docker client or reap the relay + // out from under its Child owner. + let orphan_snapshot = capture_local_deployment_processes( + relay.as_ref().map(|relay| relay.child.id()), + &mut evidence.cleanup.orphan_processes, + &cleanup_budget, + ); + let (relay_cleanup, fleet_teardown, raw_remaining, local_cleanup) = + std::thread::scope(|scope| { + let raw = scope.spawn(|| { + let teardown = fleet.teardown_with_budget(&cleanup_budget); + let remaining = fleet.remaining_resources_with_budget(&cleanup_budget); + (teardown, remaining) + }); + let relay = scope.spawn(|| relay.map_or(Ok(()), |relay| relay.stop(&cleanup_budget))); + let local_cleanup = merge_execution_cleanup( + orphan_snapshot, + cleanup_local_deployment_processes( + &mut evidence.cleanup.orphan_processes, + &cleanup_budget, + ), + ); + let relay_cleanup = relay + .join() + .unwrap_or_else(|_| Err("relay cleanup owner panicked".to_owned())); + let (fleet_teardown, remaining) = raw.join().unwrap_or_else(|_| { + ( + Err("raw cleanup owner panicked".to_owned()), + Err("raw cleanup census unavailable after owner panic".to_owned()), + ) + }); + (relay_cleanup, fleet_teardown, remaining, local_cleanup) + }); + // All managed cleanup children are now relinquished. Capture any late + // orphan (including a failed relay reap) and reconcile without a new budget. + let final_snapshot = capture_local_deployment_processes( + None, + &mut evidence.cleanup.orphan_processes, + &cleanup_budget, + ); + let final_local_cleanup = merge_execution_cleanup( + final_snapshot, + cleanup_local_deployment_processes(&mut evidence.cleanup.orphan_processes, &cleanup_budget), + ); + let local_remaining = process_descendants(std::process::id()).map(|pids| { + pids.into_iter() + .map(|pid| format!("process:{pid}")) + .collect::>() + }); + let remaining_resources = match (raw_remaining, local_remaining) { + (Ok(mut raw), Ok(local)) => { + raw.extend(local); + Ok(raw) + } + (Err(raw), Err(local)) => Err(format!("{raw}; {local}")), + (Err(error), _) | (_, Err(error)) => Err(error), + }; + evidence.cleanup.remaining_resources = remaining_resources.as_ref().ok().cloned(); + let mut cleanup_errors = Vec::new(); + for result in [ + &relay_cleanup, + &fleet_teardown, + &local_cleanup, + &final_local_cleanup, + ] { + if let Err(error) = result { + cleanup_errors.push(error.clone()); + } + } + match &remaining_resources { + Ok(remaining) if !remaining.is_empty() => { + cleanup_errors.push(format!( + "fixture resources remain after cleanup: {remaining:?}" + )); + } + Err(error) => cleanup_errors.push(format!("prove fixture cleanup: {error}")), + Ok(_) => {} + } + evidence.cleanup.complete = cleanup_errors.is_empty(); + evidence.cleanup.error = (!cleanup_errors.is_empty()).then(|| cleanup_errors.join("; ")); + + let mut errors = Vec::new(); + if let Err(error) = diagnostics { + errors.push(format!("persist worker failure diagnostics: {error}")); + } + errors.extend(cleanup_errors); + if let Err(error) = total.check("complete Gate A deployment and cleanup") { + errors.push(error); + } + if let Err(error) = result { + errors.push(error); + } + if let Err(error) = before_cleanup { + errors.push(error); + } + evidence.passed = errors.is_empty() && evidence.cleanup.complete; + evidence.failure = (!errors.is_empty()).then(|| errors.join("; ")); + if let Err(error) = write_deployment_gate_evidence(&evidence_path, &evidence) { + errors.push(error); + } + if errors.is_empty() { + Ok(()) + } else { + Err(errors.join("; ")) + } +} + +struct LocalProcessStat { + start_ticks: u64, + parent_pid: u32, + state: char, +} + +fn local_process_stat(pid: u32) -> Result, String> { + let stat = match fs::read_to_string(format!("/proc/{pid}/stat")) { + Ok(stat) => stat, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(format!("inspect local deployment process {pid}: {error}")), + }; + let mut fields = stat + .rsplit_once(')') + .ok_or_else(|| format!("process {pid} stat omitted fields"))? + .1 + .split_whitespace(); + let state = fields + .next() + .and_then(|state| state.chars().next()) + .ok_or_else(|| format!("process {pid} stat omitted state"))?; + let parent_pid = fields + .next() + .ok_or_else(|| format!("process {pid} stat omitted parent"))? + .parse::() + .map_err(|error| format!("parse process {pid} parent: {error}"))?; + let start_ticks = fields + .nth(17) + .ok_or_else(|| format!("process {pid} stat omitted start ticks"))? + .parse::() + .map_err(|error| format!("parse process {pid} start ticks: {error}"))?; + Ok(Some(LocalProcessStat { + start_ticks, + parent_pid, + state, + })) +} + +fn pin_local_process( + pid: u32, + expected_start_ticks: u64, +) -> Result, String> { + let Some(pidfd) = open_process_exit_fd(pid)? else { + return Ok(None); + }; + if !local_process_stat(pid)?.is_some_and(|stat| stat.start_ticks == expected_start_ticks) { + return Ok(None); + } + // The descriptor, not a later /proc lookup, owns signal and reap authority. + Ok(Some(LocalProcessIdentity { + pid, + start_ticks: expected_start_ticks, + pidfd, + })) +} + +fn signal_local_process(process: &LocalProcessIdentity, signal: i32) -> Result<(), String> { + let result = unsafe { + libc::syscall( + libc::SYS_pidfd_send_signal, + std::os::fd::AsRawFd::as_raw_fd(&process.pidfd), + signal, + std::ptr::null::(), + 0, + ) + }; + if result == 0 { + return Ok(()); + } + let error = std::io::Error::last_os_error(); + if error.raw_os_error() == Some(libc::ESRCH) { + Ok(()) + } else { + Err(format!( + "signal owned deployment process {}:{}: {error}", + process.pid, process.start_ticks + )) + } +} + +fn capture_local_children( + parent: &LocalProcessIdentity, + excluded_pid: Option, + observed: &[LocalProcessIdentity], +) -> Result, String> { + let mut captured = Vec::new(); + for pid in process_children(parent.pid)? { + if Some(pid) == excluded_pid { + continue; + } + let Some(stat) = local_process_stat(pid)? else { + continue; + }; + if stat.parent_pid != parent.pid { + continue; + } + let mut already_owned = false; + for process in observed { + if process.pid == pid + && process.start_ticks == stat.start_ticks + && process_fd_events(&process.pidfd, process.pid, Duration::ZERO)? & libc::POLLHUP + == 0 + { + already_owned = true; + break; + } + } + if already_owned { + continue; + } + let Some(process) = pin_local_process(pid, stat.start_ticks)? else { + continue; + }; + // Both sides of the parent-child edge must still name the pinned + // incarnations. A recycled parent PID grants no discovery authority. + if !local_process_stat(pid)?.is_some_and(|current| { + current.start_ticks == stat.start_ticks && current.parent_pid == parent.pid + }) || process_fd_events(&parent.pidfd, parent.pid, Duration::ZERO)? != 0 + { + continue; + } + captured.push(process); + } + Ok(captured) +} + +fn capture_local_deployment_processes( + excluded_pid: Option, + observed: &mut Vec, + budget: &Budget, +) -> Result<(), String> { + let pid = std::process::id(); + let stat = local_process_stat(pid)? + .ok_or_else(|| "local deployment cleanup owner disappeared".to_owned())?; + let owner = pin_local_process(pid, stat.start_ticks)? + .ok_or_else(|| "local deployment cleanup owner changed identity".to_owned())?; + let mut settled = false; + loop { + budget.check("capture stopped local deployment descendants")?; + let before = observed.len(); + let children = capture_local_children(&owner, excluded_pid, observed)?; + observed.extend(children); + let mut all_stopped = true; + let mut index = 0; + while index < observed.len() { + let process = &observed[index]; + if process_fd_events(&process.pidfd, process.pid, Duration::ZERO)? == 0 { + if local_process_stat(process.pid)?.is_some_and(|stat| { + stat.start_ticks == process.start_ticks + && !matches!(stat.state, 'T' | 't' | 'Z' | 'X') + }) { + signal_local_process(process, libc::SIGSTOP)?; + all_stopped = false; + } + let children = capture_local_children(process, excluded_pid, observed)?; + observed.extend(children); + } + index += 1; + } + // Two settled scans include children reparented while the first scan + // was stopping their parent. No owned process can fork after capture. + if before == observed.len() && all_stopped { + if settled { + return Ok(()); + } + settled = true; + } else { + settled = false; + } + if !all_stopped { + // pidfds notify exit, not job-control stops; reconcile stop state. + budget.wait( + Duration::from_millis(1), + "capture stopped local deployment descendants", + )?; + } + } +} + +fn cleanup_local_deployment_processes( + observed: &mut Vec, + budget: &Budget, +) -> Result<(), String> { + let mut errors = Vec::new(); + for process in observed.iter() { + if let Err(error) = signal_local_process(process, libc::SIGKILL) { + errors.push(error); + } + } + loop { + let mut pending = Vec::new(); + let mut wait_on = None; + for process in observed.iter() { + // Non-direct descendants can become our children only after their + // parent exits. P_PIDFD never reaps a replacement numeric PID. + let mut status = unsafe { std::mem::zeroed::() }; + let result = unsafe { + libc::waitid( + libc::P_PIDFD, + std::os::fd::AsRawFd::as_raw_fd(&process.pidfd) as libc::id_t, + &mut status, + libc::WEXITED | libc::WNOHANG, + ) + }; + if result != 0 { + let error = std::io::Error::last_os_error(); + if !matches!( + error.raw_os_error(), + Some(libc::ECHILD | libc::EINTR | libc::ESRCH) + ) { + errors.push(format!( + "reap owned deployment process {}:{}: {error}", + process.pid, process.start_ticks + )); + } + } + let events = process_fd_events(&process.pidfd, process.pid, Duration::ZERO)?; + // POLLIN proves exit; POLLHUP additionally proves this exact + // incarnation was reaped, even by its original non-direct parent. + if events & libc::POLLHUP == 0 { + pending.push(format!("process:{}:{}", process.pid, process.start_ticks)); + if events & libc::POLLIN == 0 { + wait_on = Some(process); + } + } + } + if !errors.is_empty() { + return Err(errors.join("; ")); + } + if pending.is_empty() { + return Ok(()); + } + let predicate = format!("reap deployment descendants {pending:?}"); + let remaining = budget.remaining(&predicate)?; + if let Some(process) = wait_on { + process_fd_events( + &process.pidfd, + process.pid, + remaining.min(Duration::from_millis(100)), + )?; + } else { + // Exit-ready pidfds otherwise spin while an ancestor finishes + // reparenting/reaping. All authority stays on the retained fds. + budget.wait(Duration::from_millis(5), &predicate)?; + } + } +} + +#[cfg(test)] +mod local_process_tests { + use super::*; + + fn blocked_child() -> std::process::Child { + std::process::Command::new("sh") + .args(["-c", "read line; exit 23"]) + .stdin(std::process::Stdio::piped()) + .spawn() + .unwrap() + } + + #[test] + fn pinning_rejects_a_changed_start_identity() { + let mut child = blocked_child(); + let stat = local_process_stat(child.id()).unwrap().unwrap(); + let pinned = pin_local_process(child.id(), stat.start_ticks + 1); + child.stdin.take().unwrap().write_all(b"finish\n").unwrap(); + let status = child.wait().unwrap(); + assert!(pinned.unwrap().is_none()); + assert_eq!(status.code(), Some(23)); + } + + #[test] + fn cleanup_never_adopts_a_replacement_numeric_pid() { + let mut original = blocked_child(); + let stat = local_process_stat(original.id()).unwrap().unwrap(); + let mut captured = pin_local_process(original.id(), stat.start_ticks) + .unwrap() + .unwrap(); + original.kill().unwrap(); + original.wait().unwrap(); + + let mut replacement = blocked_child(); + // Model PID recycling without relying on allocator timing: numeric + // metadata now names another child, but the retained kernel handle + // still names the original, already-reaped incarnation. + captured.pid = replacement.id(); + captured.start_ticks = local_process_stat(replacement.id()) + .unwrap() + .unwrap() + .start_ticks; + let cleanup = cleanup_local_deployment_processes( + &mut vec![captured], + &Budget::new(Duration::from_secs(5)), + ); + let released = replacement.stdin.take().unwrap().write_all(b"finish\n"); + let status = replacement.wait(); + cleanup.unwrap(); + released.unwrap(); + assert_eq!(status.unwrap().code(), Some(23)); + } + + #[test] + fn cleanup_reaps_stopped_non_direct_descendants() { + const CHILD_TEST: &str = "MYELIN_LOCAL_DESCENDANT_CLEANUP_TEST"; + if std::env::var_os(CHILD_TEST).is_none() { + let status = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "local_process_tests::cleanup_reaps_stopped_non_direct_descendants", + "--nocapture", + ]) + .env(CHILD_TEST, "1") + .status() + .unwrap(); + assert!(status.success()); + return; + } + // Re-execution confines the process-wide subreaper setting and the + // self-descendant census to this test, even under parallel cargo test. + assert_eq!( + unsafe { libc::prctl(libc::PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0) }, + 0 + ); + let mut parent = std::process::Command::new("sh") + .args(["-c", "sleep 60 & echo $!; kill -STOP $$; wait"]) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + let parent_pid = parent.id(); + let mut child_pid = String::new(); + std::io::BufReader::new(parent.stdout.take().unwrap()) + .read_line(&mut child_pid) + .unwrap(); + let child_pid = child_pid.trim().parse::().unwrap(); + let non_direct = local_process_stat(child_pid) + .unwrap() + .is_some_and(|stat| stat.parent_pid == parent_pid); + let mut managed = blocked_child(); + let mut observed = Vec::new(); + let budget = Budget::new(Duration::from_secs(5)); + let capture = + capture_local_deployment_processes(Some(managed.id()), &mut observed, &budget); + // Child ownership is explicitly relinquished to the captured pidfds. + drop(parent); + let captured_both = [parent_pid, child_pid] + .iter() + .all(|pid| observed.iter().any(|process| process.pid == *pid)); + let cleanup = cleanup_local_deployment_processes(&mut observed, &budget); + let released = managed.stdin.take().unwrap().write_all(b"finish\n"); + let managed_status = managed.wait(); + capture.unwrap(); + cleanup.unwrap(); + released.unwrap(); + assert_eq!(managed_status.unwrap().code(), Some(23)); + assert!(non_direct && captured_both); + for pid in [parent_pid, child_pid] { + assert!(local_process_stat(pid).unwrap().is_none()); + } + assert!(process_descendants(std::process::id()).unwrap().is_empty()); + } +} + +struct DeploymentRoundContext<'a> { + options: &'a Options, + workspace: &'a Path, + fixture_dir: &'a Path, + state_dir: &'a Path, + manifest: &'a Path, + fleet: &'a RawDockerFleet, + relay: Option<&'a LocalRelay>, + run_id: u64, + node_count: u8, + immutable_fixture: &'a myelin_e2e_fuzz::RawFleetSnapshot, + cleanup: &'a Budget, +} + +fn start_deployment_round( + context: &DeploymentRoundContext<'_>, + bundle: &DeploymentBundle, + reset_state: bool, + provision: bool, + budget: &Budget, + unobserved: bool, +) -> Result { + let config = ClusterHarnessConfig { + workspace: context.workspace.to_path_buf(), + artifacts: context.fixture_dir.to_path_buf(), + node_count: context.node_count, + seed: context.options.seed, + image: Some("myelin-raw-deploy".to_owned()), + build_image: false, + deadline: context.options.deadline, + provider: HarnessProvider::StaticSsh { + manifest: context.manifest.to_path_buf(), + identity: context.fleet.identity.clone(), + bundle: bundle.tar_path.clone(), + }, + selected_offer_ids: Vec::new(), + state_dir: Some(context.state_dir.to_path_buf()), + reset_state, + offer_search_id: None, + provision, + relay_url: deployment_relay_url(context.relay), + adopt_only: false, + run_id: context.run_id, + }; + let harness = ClusterHarness::start_deployment_unobserved( + config, + budget.clone(), + context.cleanup.clone(), + )?; + if unobserved { + Ok(harness) + } else { + complete_deployment_round(harness, context.fleet, bundle, context.immutable_fixture) + } +} + +fn complete_deployment_round( + mut harness: ClusterHarness, + fleet: &RawDockerFleet, + bundle: &DeploymentBundle, + immutable_fixture: &myelin_e2e_fuzz::RawFleetSnapshot, +) -> Result { + let _round_timer = PhaseTimer::new(format!( + "deployment-convergence-{}", + bundle.deployment_generation + )); + let result = (|| { + harness.complete_deployment()?; + harness.verify_workload_convergence()?; + let census = assert_raw_deployment(fleet, bundle)?; + assert_fleet_identity(&harness.fleet_snapshot()?, &census, bundle)?; + fleet.assert_unchanged(immutable_fixture)?; + assert_no_ssh_children(harness.orchestrator_pid())?; + harness + .execution_budget() + .check("deployment generation fully proved") + })(); + match result { + Ok(()) => Ok(harness), + Err(error) => merge_execution_cleanup(Err(error), harness.retain_remote_fixture(true)), + } +} + +fn run_interrupted_deployment_round( + context: &DeploymentRoundContext<'_>, + bundle: &DeploymentBundle, + boundary: DeploymentBoundary, + interruption: DeploymentInterruption, + budget: &Budget, +) -> Result { + let round = budget.child(budget.remaining("admit deployment interruption")?); + context + .fleet + .hold_boundary(&bundle.deployment_generation, boundary)?; + // The real orchestrator owns the concurrent SSH deployment task. The + // runner owns its Child directly: no scoped threads or unbounded joins. + let launched = start_deployment_round(context, bundle, false, false, &round, true); + let mut harness = match launched { + Ok(harness) => harness, + Err(error) => { + return merge_execution_cleanup( + Err(error), + context.fleet.release_boundary_with_budget( + &bundle.deployment_generation, + boundary, + context.cleanup, + ), + ); + } + }; + let injection = (|| { + context.fleet.wait_for_boundary( + 0, + &bundle.deployment_generation, + boundary, + round + .remaining("SSH interruption boundary")? + .min(Duration::from_secs(60)), + )?; + match interruption { + DeploymentInterruption::SshClientLoss => kill_ssh_children(harness.orchestrator_pid()), + DeploymentInterruption::OrchestratorLoss => kill_process( + harness.orchestrator_pid(), + "orchestrator at deployment boundary", + ), + } + })(); + let release = context.fleet.release_boundary_with_budget( + &bundle.deployment_generation, + boundary, + context.cleanup, + ); + if let Err(error) = merge_execution_cleanup(injection, release) { + round.cancel(); + return merge_execution_cleanup(Err(error), harness.retain_remote_fixture(true)); + } + match interruption { + DeploymentInterruption::SshClientLoss => { + complete_deployment_round(harness, context.fleet, bundle, context.immutable_fixture) + } + DeploymentInterruption::OrchestratorLoss => { + // Only the injected loss permits this explicit retained restart; + // elapsed budget exhaustion is never a product-terminal success. + if harness.complete_deployment().is_ok() { + return merge_execution_cleanup( + Err(format!( + "deployment unexpectedly completed after orchestrator loss at {boundary:?}" + )), + harness.retain_remote_fixture(true), + ); + } + harness.retain_remote_fixture(true)?; + round.check("restart retained deployment after injected orchestrator loss")?; + start_deployment_round(context, bundle, false, false, &round, false) + } + } +} +fn run_temporarily_unreachable_round( + context: &DeploymentRoundContext<'_>, + bundle: &DeploymentBundle, + budget: &Budget, +) -> Result { + let _round_timer = PhaseTimer::new("deployment-round-temporarily-unreachable"); + let round = budget.child(budget.remaining("admit temporary node unreachability")?); + context.fleet.set_node_reachable(0, false)?; + let launched = start_deployment_round(context, bundle, false, false, &round, true); + let mut harness = match launched { + Ok(harness) => harness, + Err(error) => { + return merge_execution_cleanup( + Err(error), + context + .fleet + .set_node_reachable_with_budget(0, true, context.cleanup), + ); + } + }; + let observation = round + .remaining("healthy peer transfer while one node unreachable") + .and_then(|remaining| { + context.fleet.wait_for_boundary( + 1, + &bundle.deployment_generation, + DeploymentBoundary::TransferStarted, + remaining.min(Duration::from_secs(60)), + ) + }); + let reachable = context + .fleet + .set_node_reachable_with_budget(0, true, context.cleanup); + if let Err(error) = merge_execution_cleanup(observation, reachable) { + round.cancel(); + return merge_execution_cleanup(Err(error), harness.retain_remote_fixture(true)); + } + complete_deployment_round(harness, context.fleet, bundle, context.immutable_fixture) +} + +fn open_process_exit_fd(pid: u32) -> Result, String> { + use std::os::fd::FromRawFd as _; + let fd = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0) } as i32; + if fd < 0 { + let error = std::io::Error::last_os_error(); + return if error.raw_os_error() == Some(libc::ESRCH) { + Ok(None) + } else { + Err(format!("open exit notification for process {pid}: {error}")) + }; + } + Ok(Some(unsafe { std::os::fd::OwnedFd::from_raw_fd(fd) })) +} + +fn process_fd_events( + fd: &std::os::fd::OwnedFd, + pid: u32, + duration: Duration, +) -> Result { + use std::os::fd::AsRawFd as _; + let mut descriptor = libc::pollfd { + fd: fd.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }; + let started = Instant::now(); + let ready = loop { + let remaining = duration.saturating_sub(started.elapsed()); + let timeout = remaining.as_millis().min(i32::MAX as u128) as i32; + let ready = unsafe { libc::poll(&mut descriptor, 1, timeout) }; + if ready >= 0 || std::io::Error::last_os_error().kind() != std::io::ErrorKind::Interrupted { + break ready; + } + // An interrupted zero-time probe must not grant discovery authority + // by pretending an exited parent is still alive. + }; + if ready < 0 { + return Err(format!( + "wait process {pid} exit: {}", + std::io::Error::last_os_error() + )); + } + if ready <= 0 { + return Ok(0); + } + if descriptor.revents & (libc::POLLERR | libc::POLLNVAL) != 0 { + return Err(format!("invalid exit notification for process {pid}")); + } + Ok(descriptor.revents & (libc::POLLIN | libc::POLLHUP)) +} + +fn wait_process_event(pid: u32, duration: Duration) -> Result { + let Some(fd) = open_process_exit_fd(pid)? else { + return Ok(true); + }; + process_fd_events(&fd, pid, duration).map(|events| events != 0) +} + +fn wait_process_exit(pid: u32, budget: &Budget) -> Result<(), String> { + loop { + let remaining = budget.remaining(&format!("process {pid} exit and reap"))?; + if wait_process_event(pid, remaining.min(Duration::from_millis(100)))? { + return Ok(()); + } + } +} + +fn process_children(parent_pid: u32) -> Result, String> { + let mut children = BTreeSet::new(); + let task_dir = format!("/proc/{parent_pid}/task"); + let tasks = match std::fs::read_dir(&task_dir) { + Ok(tasks) => tasks, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(format!("read process tasks {task_dir}: {error}")), + }; + for task in tasks { + let task = task.map_err(|error| format!("inspect process task: {error}"))?; + let task_children = match std::fs::read_to_string(task.path().join("children")) { + Ok(children) => children, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => { + return Err(format!( + "read child processes for task {}: {error}", + task.path().display() + )); + } + }; + children.extend( + task_children + .split_whitespace() + .filter_map(|pid| pid.parse::().ok()), + ); + } + Ok(children.into_iter().collect()) +} + +fn process_descendants(root_pid: u32) -> Result, String> { + let mut descendants = BTreeSet::new(); + let mut pending = vec![root_pid]; + while let Some(parent_pid) = pending.pop() { + for child in process_children(parent_pid)? { + if descendants.insert(child) { + pending.push(child); + } + } + } + Ok(descendants.into_iter().collect()) +} + +fn kill_ssh_children(orchestrator_pid: u32) -> Result<(), String> { + let ssh_children = process_descendants(orchestrator_pid)? + .into_iter() + .filter(|pid| { + std::fs::read_to_string(format!("/proc/{pid}/comm")) + .is_ok_and(|comm| matches!(comm.trim(), "ssh" | "scp")) + }) + .collect::>(); + if ssh_children.is_empty() { + return Err(format!( + "orchestrator {orchestrator_pid} had no SSH descendant at the deployment boundary" + )); + } + for pid in ssh_children { + kill_process(pid, "SSH deployment client")?; } Ok(()) } + +fn kill_process(pid: u32, description: &str) -> Result<(), String> { + if unsafe { libc::kill(pid as i32, libc::SIGKILL) } == 0 + || std::io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH) + { + Ok(()) + } else { + Err(format!( + "kill {description} {pid}: {}", + std::io::Error::last_os_error() + )) + } +} + +fn uniform_prior_states(fleet: &RawDockerFleet, state: &str) -> BTreeMap { + fleet + .nodes + .iter() + .map(|node| (u64::from(node.slot) + 1, state.to_owned())) + .collect() +} + +fn enumerated_prior_states( + fleet: &RawDockerFleet, + states: &[RawPriorState], +) -> Result, String> { + if states.len() != fleet.nodes.len() { + return Err(format!( + "{} prior states cannot cover {} raw nodes", + states.len(), + fleet.nodes.len() + )); + } + fleet + .nodes + .iter() + .zip(states) + .map(|(node, state)| { + let value = serde_json::to_value(state) + .map_err(|error| format!("serialize injected prior state: {error}"))?; + let state = value + .as_str() + .ok_or_else(|| format!("injected prior state is not a string: {value}"))?; + Ok((u64::from(node.slot) + 1, state.to_owned())) + }) + .collect() +} + +struct DeploymentRoundObservation { + allow_creating: bool, + prior_states: BTreeMap, + interruption: Option, + temporarily_unreachable_node: Option, +} + +fn capture_deployment_round( + context: &DeploymentRoundContext<'_>, + harness: &ClusterHarness, + bundle: &DeploymentBundle, + telemetry_before: &BTreeSet, + observation: DeploymentRoundObservation, +) -> Result { + let raw_census = assert_raw_deployment(context.fleet, bundle)?; + let fleet_status = harness.fleet_snapshot()?; + assert_fleet_identity(&fleet_status, &raw_census, bundle)?; + assert_no_ssh_children(harness.orchestrator_pid())?; + let archives = deployment_telemetry_files(context.fixture_dir)? + .difference(telemetry_before) + .cloned() + .collect(); + let telemetry = capture_deployment_telemetry(archives, &raw_census, bundle)?; + if !observation.allow_creating && telemetry.creating_event_count != 0 { + return Err(format!( + "retained deployment {} attempted {} acquisitions: {:?}; archives={:?}", + bundle.deployment_generation, + telemetry.creating_event_count, + telemetry.creating_events, + telemetry.archives + )); + } + Ok(DeploymentRoundEvidence { + deployment_generation: bundle.deployment_generation.clone(), + artifact_digest: bundle.artifact_digest.clone(), + executable_digest: bundle.executable_digest.clone(), + prior_states: observation.prior_states, + interruption: observation.interruption, + temporarily_unreachable_node: observation.temporarily_unreachable_node, + fixture: context.fleet.snapshot()?, + raw_census, + fleet_status, + telemetry, + matching_receipt_required_for_runtime_admission: true, + stale_processes_and_descendants_absent: true, + ssh_session_independent: true, + fresh_membership_routing_readiness_telemetry: true, + directed_all_pairs_behavior_passed: true, + }) +} + +fn write_deployment_gate_evidence( + path: &Path, + evidence: &DeploymentGateEvidence, +) -> Result<(), String> { + let bytes = serde_json::to_vec_pretty(evidence) + .map_err(|error| format!("serialize deployment gate evidence: {error}"))?; + let temporary = path.with_extension("json.tmp"); + let mut file = std::fs::OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&temporary) + .map_err(|error| format!("create deployment gate evidence: {error}"))?; + file.write_all(&bytes) + .map_err(|error| format!("write deployment gate evidence: {error}"))?; + file.sync_all() + .map_err(|error| format!("sync deployment gate evidence: {error}"))?; + std::fs::rename(&temporary, path) + .map_err(|error| format!("commit deployment gate evidence: {error}"))?; + let parent = path + .parent() + .ok_or_else(|| format!("deployment evidence path {} has no parent", path.display()))?; + std::fs::File::open(parent) + .and_then(|directory| directory.sync_all()) + .map_err(|error| format!("sync deployment evidence directory: {error}")) +} + +fn inject_prior_states(fleet: &RawDockerFleet, states: &[RawPriorState]) -> Result<(), String> { + if states.len() != fleet.nodes.len() { + return Err(format!( + "{} prior states do not exactly cover {} raw nodes", + states.len(), + fleet.nodes.len() + )); + } + for (node, state) in fleet.nodes.iter().zip(states.iter().copied()) { + fleet.inject_prior_state(node.slot, state)?; + } + Ok(()) +} + +fn assert_raw_deployment( + fleet: &RawDockerFleet, + expected: &DeploymentBundle, +) -> Result, String> { + let census = fleet.census()?; + if census.len() != fleet.nodes.len() { + return Err(format!( + "raw census returned {} nodes for a {}-node fixture", + census.len(), + fleet.nodes.len() + )); + } + let expected_descriptor = serde_json::json!({ + "artifact_digest": expected.artifact_digest, + "deployment_generation": expected.deployment_generation, + "executable_digest": expected.executable_digest, + }); + let release_suffix = expected + .artifact_digest + .strip_prefix("sha256:") + .ok_or_else(|| "deployment bundle artifact digest is not sha256-prefixed".to_owned())?; + for node in &census { + if !node.surviving_prior_processes.is_empty() { + return Err(format!( + "node {} retained prior process incarnations: {:?}", + node.logical_node_id, node.surviving_prior_processes + )); + } + if node.worker_processes.len() != 1 { + return Err(format!( + "node {} has {} worker processes, expected exactly one: {:?}", + node.logical_node_id, + node.worker_processes.len(), + node.worker_processes + )); + } + let worker = &node.worker_processes[0]; + if node.agent_pid != Some(worker.pid) { + return Err(format!( + "node {} agent pid {:?} differs from sole worker {}", + node.logical_node_id, node.agent_pid, worker.pid + )); + } + if worker.executable_digest.as_deref() != Some(&expected.executable_digest) + || node.active_executable_digest.as_deref() != Some(&expected.executable_digest) + { + return Err(format!( + "node {} executable identity mismatch: worker={:?}, active={:?}, expected={}", + node.logical_node_id, + worker.executable_digest, + node.active_executable_digest, + expected.executable_digest + )); + } + if node.worker_group_processes.len() != 1 + || node.worker_group_processes[0].pid != worker.pid + { + return Err(format!( + "node {} retained stale worker descendants or group members: {:?}", + node.logical_node_id, node.worker_group_processes + )); + } + let link = node.active_link_target.as_deref().unwrap_or_default(); + if !link.ends_with(&format!("/releases/{release_suffix}")) { + return Err(format!( + "node {} active link {link:?} does not select artifact {}", + node.logical_node_id, expected.artifact_digest + )); + } + let descriptor = node + .active_deployment_raw + .as_deref() + .ok_or_else(|| { + format!( + "node {} has no active deployment descriptor", + node.logical_node_id + ) + }) + .and_then(|raw| { + serde_json::from_str::(raw).map_err(|error| { + format!( + "node {} active deployment descriptor is invalid: {error}", + node.logical_node_id + ) + }) + })?; + if descriptor != expected_descriptor { + return Err(format!( + "node {} active descriptor {descriptor} differs from {expected_descriptor}", + node.logical_node_id + )); + } + } + Ok(census) +} +fn assert_fleet_identity( + fleet: &serde_json::Value, + census: &[myelin_e2e_fuzz::RawNodeCensus], + expected: &DeploymentBundle, +) -> Result<(), String> { + let nodes = fleet + .pointer("/FleetStatus/nodes") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| format!("fleet status has no nodes: {fleet}"))?; + if nodes.len() != census.len() { + return Err(format!( + "expected {} fleet nodes, saw {}", + census.len(), + nodes.len() + )); + } + let mut observed = BTreeSet::new(); + for node in nodes { + let node_id = node + .get("logical_node_id") + .and_then(serde_json::Value::as_u64) + .ok_or_else(|| format!("fleet node has no logical identity: {node}"))?; + let raw = census + .iter() + .find(|raw| raw.logical_node_id == node_id) + .ok_or_else(|| format!("fleet node {node_id} has no matching raw node"))?; + if !observed.insert(node_id) { + return Err(format!( + "fleet reports logical node {node_id} more than once" + )); + } + let phase = node.get("phase").and_then(serde_json::Value::as_str); + if phase != Some("running") { + return Err(format!( + "node {node_id} is {phase:?}, expected running; node={node}" + )); + } + let digest = node + .pointer("/runtime/artifact_digest") + .and_then(serde_json::Value::as_str); + let generation = node + .pointer("/runtime/deployment_generation") + .and_then(serde_json::Value::as_str); + if digest != Some(&expected.artifact_digest) + || generation != Some(&expected.deployment_generation) + { + return Err(format!( + "node {node_id} reports deployment ({digest:?}, {generation:?}), expected ({}, {})", + expected.artifact_digest, expected.deployment_generation + )); + } + let worker = &raw.worker_processes[0]; + let incarnation = format!( + "{}:{}:{}", + worker.pid, worker.start_ticks, expected.deployment_generation + ); + let hash = Sha256::digest(incarnation.as_bytes()); + let mut identity_bytes = [0_u8; 8]; + identity_bytes.copy_from_slice(&hash[..8]); + let readiness_id = u64::from_be_bytes(identity_bytes); + if node + .pointer("/runtime/readiness_id") + .and_then(serde_json::Value::as_u64) + != Some(readiness_id) + { + return Err(format!( + "node {node_id} runtime readiness is not bound to incarnation {incarnation}" + )); + } + } + Ok(()) +} + +fn deployment_telemetry_files(fixture_dir: &Path) -> Result, String> { + let mut files = BTreeSet::new(); + for entry in fs::read_dir(fixture_dir) + .map_err(|error| format!("read deployment telemetry directory: {error}"))? + { + let entry = entry.map_err(|error| format!("inspect deployment telemetry: {error}"))?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + if name.starts_with("telemetry-") && name.ends_with(".jsonl") { + files.insert(entry.path()); + } + } + Ok(files) +} + +fn capture_deployment_telemetry( + archives: BTreeSet, + census: &[myelin_e2e_fuzz::RawNodeCensus], + expected: &DeploymentBundle, +) -> Result { + let mut receipts = BTreeMap::new(); + let mut creating_events = Vec::new(); + let mut bootstrapping_events = Vec::new(); + for archive in &archives { + let file = fs::File::open(archive) + .map_err(|error| format!("open deployment telemetry: {error}"))?; + let mut reader = std::io::BufReader::new(file); + let mut line = String::new(); + loop { + line.clear(); + if reader + .read_line(&mut line) + .map_err(|error| format!("read deployment telemetry: {error}"))? + == 0 + { + break; + } + // A live archive can end in an uncommitted partial frame. + if !line.ends_with('\n') || !line.contains("myelin.provisioning.") { + continue; + } + let frame: serde_json::Value = serde_json::from_str(&line) + .map_err(|error| format!("decode deployment telemetry frame: {error}"))?; + let channel = frame + .get("channel") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + if channel != "myelin.provisioning.events" + && !(channel.starts_with("myelin.provisioning.logs.node.") + && channel.ends_with(".provider")) + { + continue; + } + let archived = frame + .pointer("/payload/value") + .ok_or_else(|| "deployment telemetry frame has no decoded payload".to_owned())?; + let payload = match archived { + serde_json::Value::String(text) => serde_json::from_str::(text) + .map_err(|error| format!("decode deployment telemetry payload: {error}"))?, + value => value.clone(), + }; + if channel == "myelin.provisioning.events" { + match payload + .pointer("/event/kind") + .and_then(serde_json::Value::as_str) + { + Some("Creating") => creating_events.push(payload), + Some("Bootstrapping") => bootstrapping_events.push(payload), + _ => {} + } + continue; + } + let Some(provider_line) = payload + .pointer("/line/line") + .and_then(serde_json::Value::as_str) + else { + continue; + }; + if !provider_line.contains("MyelinBootstrapDispatched") { + continue; + } + let dispatched: serde_json::Value = serde_json::from_str(provider_line) + .map_err(|error| format!("decode deployment provider line: {error}"))?; + if dispatched.get("type").and_then(serde_json::Value::as_str) + != Some("MyelinBootstrapDispatched") + || dispatched + .get("deployment_generation") + .and_then(serde_json::Value::as_str) + != Some(&expected.deployment_generation) + { + continue; + } + let Some(node_id) = dispatched + .get("node_id") + .and_then(serde_json::Value::as_u64) + else { + continue; + }; + let Some(node) = census.iter().find(|node| node.logical_node_id == node_id) else { + continue; + }; + let worker = &node.worker_processes[0]; + let incarnation = format!( + "{}:{}:{}", + worker.pid, worker.start_ticks, expected.deployment_generation + ); + let receipt = dispatched + .get("receipt") + .ok_or_else(|| format!("node {node_id} dispatched without a receipt"))?; + let expected_receipt = serde_json::json!({ + "type": "MyelinBootstrapReceipt", + "artifact_digest": expected.artifact_digest, + "deployment_generation": expected.deployment_generation, + "executable_digest": expected.executable_digest, + "pid": worker.pid, + "process_start_ticks": worker.start_ticks, + "incarnation": incarnation, + "worker_count": 1, + }); + if receipt == &expected_receipt { + receipts.insert(node_id, receipt.clone()); + } + } + } + for node in census { + if !bootstrapping_events.iter().any(|event| { + event + .pointer("/event/node_id") + .and_then(serde_json::Value::as_u64) + == Some(node.logical_node_id) + }) { + return Err(format!( + "node {} has no Bootstrapping phase telemetry for this deployment round", + node.logical_node_id + )); + } + if !receipts.contains_key(&node.logical_node_id) { + return Err(format!( + "node {} has no matching provider launch receipt for generation {} and its live process incarnation", + node.logical_node_id, expected.deployment_generation + )); + } + } + Ok(DeploymentTelemetryEvidence { + archives, + provider_receipts: receipts, + creating_event_count: creating_events.len(), + creating_events, + bootstrapping_events, + }) +} + +fn assert_no_ssh_children(orchestrator_pid: u32) -> Result<(), String> { + for pid in process_descendants(orchestrator_pid)? { + let comm = std::fs::read_to_string(format!("/proc/{pid}/comm")) + .unwrap_or_default() + .trim() + .to_owned(); + if comm == "ssh" || comm == "scp" { + return Err(format!( + "orchestrator {orchestrator_pid} still runs {comm} descendant {pid} after bootstrap" + )); + } + } + Ok(()) +} + +fn run_mock_campaign( + options: &Options, + workspace: PathBuf, + config: CampaignConfig, + plan: CampaignPlan, + campaign_root: PathBuf, + total: &Budget, +) -> Result<(), String> { + let fixture = ClusterHarnessConfig { + workspace, + artifacts: campaign_root.join("fixture"), + node_count: 5, + seed: plan.seed, + image: Some(config.runtime_image), + build_image: options.build_image, + deadline: options.deadline, + provider: HarnessProvider::LocalMock, + selected_offer_ids: (1..=5).collect(), + state_dir: Some(myelin_e2e_fuzz::private_fixture_dir(&campaign_root)?), + reset_state: true, + offer_search_id: None, + provision: true, + adopt_only: false, + relay_url: None, + run_id: 1, + }; + let preparation = PhaseTimer::new("fixture-preparation"); + let execution = total.child( + total + .remaining("local fixture preparation")? + .saturating_sub(Duration::from_secs(LOCAL_CLEANUP_RESERVE_SECS)), + ); + let mut harness = + ClusterHarness::start_with_budgets(fixture, execution.clone(), total.clone())?; + let result = harness.verify_workload_convergence().and_then(|()| { + execution.check("local prepared-fixture commit")?; + drop(preparation); + execute_campaign( + &mut harness, + &plan, + &campaign_root, + &options.artifacts, + options.shrink, + &execution, + false, + ) + }); + let _cleanup = PhaseTimer::new("fixture-cleanup"); + let cleanup = total.child(Duration::from_secs(LOCAL_CLEANUP_RESERVE_SECS)); + let teardown = harness.teardown_with_budget(cleanup); + merge_execution_cleanup(result, teardown)?; + total.check("complete local campaign and evidence") +} + +fn run_paid_campaign( + options: &Options, + workspace: PathBuf, + config: CampaignConfig, + plan: CampaignPlan, + campaign_root: PathBuf, + total: &Budget, +) -> Result<(), String> { + let state_path = campaign_root.join("paid-state.json"); + let state_dir = myelin_e2e_fuzz::private_fixture_dir(&campaign_root)?; + // Model seeds are deliberately repeatable; paid ownership identities are not. + // Reusing a seed in another artifact root must never reuse provider labels. + let mut entropy = [0_u8; 8]; + fs::File::open("/dev/urandom") + .and_then(|mut source| std::io::Read::read_exact(&mut source, &mut entropy)) + .map_err(|error| format!("allocate paid fixture identity: {error}"))?; + let run_id = u64::from_le_bytes(entropy) | (1 << 63); + let mut paid = PaidCampaignState::planned(plan.campaign_id.clone(), state_dir.clone()); + write_paid_state(&state_path, &paid)?; + let execution = total.child( + total + .remaining("paid invocation admission")? + .saturating_sub(Duration::from_secs(CLEANUP_DEADLINE_SECS)), + ); + // Preparation starts before the orchestrator/provider can do any work, + // including both searches, readiness cleanup and the durable fixture commit. + let preparation = execution.child(Duration::from_secs(PREPARATION_DEADLINE_SECS)); + let preparation_timer = PhaseTimer::new("paid-preparation"); + let now_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|error| format!("system clock precedes epoch: {error}"))? + .as_millis() as u64; + let admission_expiry = now_ms + .checked_add(config.limits.fixture_lifetime_secs.saturating_mul(1000)) + .ok_or_else(|| "paid lifetime expiry overflowed".to_owned())?; + let cleanup_limits = authorized_cleanup_limits(&config.limits)?; + let admission_path = state_dir.join("paid-admission.json"); + let admission = provisioning::PaidFixtureAdmission::create_cleanup_only( + &admission_path, + run_id, + admission_expiry, + provisioning::PaidProcessIdentity::current()?, + cleanup_limits.clone(), + )?; + myelin_e2e_fuzz::start_cleanup_owner( + &std::env::current_exe().map_err(|error| error.to_string())?, + &admission_path, + &options.api_key_env, + cleanup_limits, + preparation.remaining("start independent paid cleanup owner before provider search")?, + )?; + let fixture = ClusterHarnessConfig { + workspace, + artifacts: campaign_root.join("fixture"), + node_count: 5, + seed: plan.seed, + image: Some(config.runtime_image.clone()), + build_image: false, + deadline: options.deadline, + provider: HarnessProvider::VastAiReal { + ssh_identity: options + .ssh_identity + .clone() + .expect("paid identity validated"), + api_key_env: options.api_key_env.clone(), + bundle: None, + admission: admission_path, + }, + selected_offer_ids: Vec::new(), + state_dir: Some(state_dir.clone()), + reset_state: true, + offer_search_id: None, + provision: false, + adopt_only: false, + relay_url: None, + run_id, + }; + let mut harness = + ClusterHarness::start_with_budgets(fixture, preparation.clone(), total.clone())?; + admission.bind_orchestrator(harness.orchestrator_pid())?; + let result = (|| { + let search_timer = PhaseTimer::new("offer-selection"); + let request = offer_search_request(&config); + request.validate()?; + let payload = serde_json::to_value(&request) + .map_err(|error| format!("serialize paid offer search: {error}"))?; + let search = preparation.child(Duration::from_secs(120)); + harness.set_execution_budget(search.clone()); + let observe = || { + loop { + search.check("observe eligible cheapest distinct-host offers")?; + match harness.search_offers(payload.clone()) { + Ok(response) => break decode_offer_results(response), + Err(error) => search.wait( + Duration::from_millis(250), + &format!("provider search readiness/retry: {error}"), + )?, + } + } + }; + let first = observe()?; + let first_selected = select_exact_offers(&first, &config, &plan)?; + // Both actual observations are independently eligible and cheapest. + // Provider HTTP/429 policy controls rate; elapsed dwell is not stability. + let confirmed = observe()?; + let selected = select_exact_offers(&confirmed, &config, &plan)?; + if first_selected + .iter() + .map(|offer| (offer.offer_id, offer.host_id, offer.hourly_price)) + .ne(selected + .iter() + .map(|offer| (offer.offer_id, offer.host_id, offer.hourly_price))) + { + return Err( + "eligible cheapest distinct-host selection changed between observations".to_owned(), + ); + } + durable_json( + &campaign_root.join("offer-stability.json"), + &serde_json::json!({ + "schema_version": 1, "first": first_selected, "confirmed": selected, + "first_search_id": first.search_id, "confirmed_search_id": confirmed.search_id, + }), + )?; + drop(search_timer); + harness.set_execution_budget(preparation.clone()); + let worst_case_cost = conservative_selected_cost(&selected, &config.limits)?; + myelin_e2e_fuzz::selected_cleanup_limits(&selected, &config.limits)?; + let offer_ids = selected + .iter() + .map(|offer| offer.offer_id) + .collect::>(); + if harness.fixture_run_id()? != run_id { + return Err("paid orchestrator changed the allocated fixture identity".to_owned()); + } + let labels = (1..=5) + .map(|node| format!("myelin-{run_id}-{node}-attempt-0")) + .collect::>(); + let admission_nodes = selected + .iter() + .zip(1_u64..) + .map(|(offer, node_id)| provisioning::PaidNodeAdmission { + node_id, + offer_id: offer.offer_id, + host_id: offer.host_id.expect("selection requires exact host"), + label: format!("myelin-{run_id}-{node_id}-attempt-0"), + }) + .collect(); + + // Durable cleanup ownership precedes the capability authorizing a create. + paid.begin_acquisition(offer_ids.clone(), labels)?; + write_paid_state(&state_path, &paid)?; + preparation.check("authorize five exact paid creates")?; + admission.bind_selected_nodes(admission_nodes)?; + paid.reconcile_admission(&admission.snapshot()?)?; + write_paid_state(&state_path, &paid)?; + harness.provision_selected(offer_ids, confirmed.search_id)?; + harness.verify_workload_convergence()?; + harness.seal_paid_admission()?; + let contracts = harness.owned_provider_contracts()?; + let actual_labels = harness.owned_provider_labels()?; + paid.reconcile_prepared( + &provisioning::PaidFixtureAdmission::open(paid.state_dir.join("paid-admission.json"))? + .snapshot()?, + contracts, + actual_labels, + )?; + write_paid_state(&state_path, &paid)?; + preparation.check("paid prepared-fixture commit and readiness cleanup")?; + drop(preparation_timer); + println!( + "paid fixture prepared: campaign={} contracts={} worst_case_cost=${worst_case_cost:.6}", + plan.campaign_id, + paid.owned_contract_ids.len() + ); + if options.retain_paid_fixture { + return retain_paid_harness(&mut harness, &state_path, &mut paid); + } + let campaign = execution.child(Duration::from_secs(CAMPAIGN_DEADLINE_SECS)); + execute_campaign( + &mut harness, + &plan, + &campaign_root, + &options.artifacts, + options.shrink, + &campaign, + false, + ) + })(); + if options.retain_paid_fixture && result.is_ok() { + println!( + "development fixture retained at {}; campaign acceptance not attempted", + state_path.display() + ); + return result; + } + let _cleanup_timer = PhaseTimer::new("paid-fixture-cleanup"); + let cleanup = total.child(Duration::from_secs(CLEANUP_DEADLINE_SECS)); + let cleaned = cleanup_paid_harness( + &mut harness, + &state_path, + &mut paid, + options, + &campaign_root, + &cleanup, + ); + let secret_scan = std::env::var(&options.api_key_env) + .map_err(|_| "paid credential environment is unset".to_owned()) + .and_then(|secret| scan_artifacts_for_secret(&campaign_root, secret.as_bytes())); + merge_execution_cleanup(result, merge_execution_cleanup(cleaned, secret_scan))?; + total.check("complete paid campaign and cleanup") +} + +fn retain_paid_harness( + harness: &mut ClusterHarness, + state_path: &Path, + paid: &mut PaidCampaignState, +) -> Result<(), String> { + paid.quarantine( + "explicitly retained development fixture; fresh binary redeployment and readiness required", + ); + write_paid_state(state_path, paid)?; + harness.retain_remote_fixture(true)?; + paid.reconcile_admission( + &provisioning::PaidFixtureAdmission::open(paid.state_dir.join("paid-admission.json"))? + .snapshot()?, + )?; + write_paid_state(state_path, paid) +} + +#[derive(serde::Serialize)] +#[serde(deny_unknown_fields)] +struct PaidCleanupProof<'a> { + schema_version: u32, + kind: &'static str, + complete: bool, + paid_state: &'a PaidCampaignState, + admission: Option<&'a provisioning::PaidAdmissionSnapshot>, + errors: &'a [String], +} + +fn cleanup_paid_harness( + harness: &mut ClusterHarness, + state_path: &Path, + paid: &mut PaidCampaignState, + options: &Options, + campaign_root: &Path, + cleanup: &Budget, +) -> Result<(), String> { + paid.enter_cleanup_only(); + // Both durable records deny work before shutdown begins. The admission + // barrier prevents its cleanup owner from deleting provider resources until + // the exact orchestrator incarnation has been stopped and reaped. + let persisted_state = write_paid_state(state_path, paid); + let admission_path = paid.state_dir.join("paid-admission.json"); + let shutdown_barrier = if admission_path.is_file() { + provisioning::PaidFixtureAdmission::open(&admission_path) + .and_then(|admission| admission.begin_cleanup_shutdown()) + } else { + Ok(()) + }; + let orchestrator_result = harness.stop_paid_orchestrator_for_cleanup(cleanup); + let cleanup_release = if admission_path.is_file() { + provisioning::PaidFixtureAdmission::open(&admission_path) + .and_then(|admission| admission.release_cleanup_after_orchestrator_stopped()) + } else { + Ok(()) + }; + let sealed_admission = seal_cleanup_admission(paid); + let state_result = merge_execution_cleanup( + persisted_state, + merge_execution_cleanup( + shutdown_barrier, + merge_execution_cleanup( + orchestrator_result, + merge_execution_cleanup(cleanup_release, sealed_admission), + ), + ), + ); + let provider_result = cleanup + .remaining("independent spending-stop cleanup") + .and_then(|remaining| { + myelin_e2e_fuzz::cleanup_paid_ownership( + paid, + state_path, + &std::env::current_exe().map_err(|error| error.to_string())?, + &options.api_key_env, + remaining, + ) + }); + // The final harness phase only observes the completed durable owner receipt + // and persists its lifecycle/absence evidence; it cannot race deletion. + let harness_result = harness.teardown_with_budget(cleanup.clone()); + let mut errors = [&state_result, &provider_result, &harness_result] + .into_iter() + .filter_map(|result| result.as_ref().err().cloned()) + .collect::>(); + if let Err(error) = write_paid_state(state_path, paid) { + errors.push(error); + } + let admission_path = paid.state_dir.join("paid-admission.json"); + let admission = if admission_path.is_file() { + match provisioning::PaidFixtureAdmission::open(&admission_path) + .and_then(|admission| admission.snapshot()) + { + Ok(admission) => Some(admission), + Err(error) => { + errors.push(format!("read final paid cleanup accounting: {error}")); + None + } + } + } else { + None + }; + let complete = errors.is_empty() + && paid.phase == PaidCampaignPhase::Complete + && paid.owned_contract_ids.is_empty() + && admission.as_ref().is_none_or(|admission| { + admission.accounting_errors.is_empty() + && admission + .cleanup + .as_ref() + .is_some_and(|cleanup| cleanup.complete && cleanup.spending_stopped) + }); + let receipt = durable_json( + &campaign_root.join("cleanup-proof.json"), + &PaidCleanupProof { + schema_version: 2, + kind: "paid-cleanup-proof", + complete, + paid_state: paid, + admission: admission.as_ref(), + errors: &errors, + }, + ); + if let Err(error) = receipt { + errors.push(error); + } + if errors.is_empty() { + Ok(()) + } else { + Err(errors.join("; ")) + } +} + +fn seal_cleanup_admission(paid: &PaidCampaignState) -> Result<(), String> { + let path = paid.state_dir.join("paid-admission.json"); + if path.is_file() { + provisioning::PaidFixtureAdmission::open(path)?.enter_cleanup_only()?; + } + Ok(()) +} + +fn run_resume_paid(state_path: &Path, options: &Options, workspace: PathBuf) -> Result<(), String> { + let _resume_timer = PhaseTimer::new("paid-retained-redeployment"); + let campaign_root = state_path + .parent() + .ok_or_else(|| "paid state path has no campaign directory".to_owned())?; + let mut paid = read_paid_state(state_path)?; + if paid.phase != PaidCampaignPhase::Quarantined { + return Err(format!( + "paid fixture resume requires quarantined state, found {:?}", + paid.phase + )); + } + if paid.acquisition_count != 5 || paid.bootstrap_count != 5 { + return Err( + "incomplete preparation permits cleanup-only, never resumed bootstrap".to_owned(), + ); + } + let admission_owner = + provisioning::PaidFixtureAdmission::open(paid.state_dir.join("paid-admission.json"))?; + admission_owner.require_live_cleanup_owner()?; + let admission = admission_owner.snapshot()?; + if admission.mode != provisioning::PaidAdmissionMode::Prepared { + return Err("cleanup-only admission cannot authorize retained redeployment".to_owned()); + } + let cleanup_owner = admission + .cleanup + .as_ref() + .filter(|owner| owner.retained_development_fixture) + .ok_or("resume requires an explicitly authorized retained development fixture")?; + paid.reconcile_admission(&admission)?; + let now_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|error| format!("retained lifetime clock: {error}"))? + .as_millis() as u64; + let lifetime = Duration::from_millis( + cleanup_owner + .stop_unix_ms + .saturating_add(provisioning::PAID_CLEANUP_RESERVE_MS) + .min(admission.deadline_unix_ms) + .saturating_sub(now_ms), + ); + let total = Budget::new(lifetime.min(Duration::from_secs( + PREPARATION_DEADLINE_SECS + CAMPAIGN_DEADLINE_SECS + CLEANUP_DEADLINE_SECS, + ))); + let execution = total.child( + total + .remaining("retained fixture lifetime")? + .saturating_sub(Duration::from_secs(CLEANUP_DEADLINE_SECS)), + ); + let preparation = execution.child(Duration::from_secs(PREPARATION_DEADLINE_SECS)); + let checkpoint_path = campaign_root.join("campaign-progress.json"); + if checkpoint_path.is_file() { + let checkpoint: CampaignCheckpoint = serde_json::from_reader( + fs::File::open(&checkpoint_path) + .map_err(|error| format!("read recovery checkpoint: {error}"))?, + ) + .map_err(|error| format!("parse recovery checkpoint: {error}"))?; + if checkpoint.pending_recovery.is_some() { + return Err( + "failed formal recovery permits cleanup-only; prior coverage cannot resume" + .to_owned(), + ); + } + } + let plan = read_campaign_plan(&campaign_root.join("campaign-plan.json"))?; + if plan.campaign_id != paid.campaign_id { + return Err("paid state and campaign plan identities differ".to_owned()); + } + let snapshot = serde_json::from_slice::( + &fs::read(paid.state_dir.join("cluster.json")) + .map_err(|error| format!("read retained cluster snapshot: {error}"))?, + ) + .map_err(|error| format!("parse retained cluster snapshot: {error}"))?; + if snapshot.get("run_id").and_then(serde_json::Value::as_u64) != Some(admission.run_id) { + return Err( + "retained cluster snapshot and paid admission run identities differ".to_owned(), + ); + } + let live_nodes = snapshot + .get("nodes") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .filter(|node| { + node.get("phase") + .and_then(serde_json::Value::as_str) + .is_some_and(|phase| phase != "stopped" && phase != "orphan") + }) + .filter_map(|node| { + node.get("logical_node_id") + .and_then(serde_json::Value::as_u64) + }) + .collect::>(); + if live_nodes != plan.expected_initial_nodes { + return Err(format!( + "binary redeployment invalidates prior coverage; retained set {live_nodes:?} cannot rerun the full five-node campaign without forbidden replacement" + )); + } + let node_count = u8::try_from(live_nodes.len()) + .map_err(|_| "retained fixture node count exceeds u8".to_owned())?; + // Refresh retained paid nodes through the orchestrator's normal + // reconciliation: a rebuilt deployment bundle with a fresh generation + // demotes stale nodes to Bootstrapping and re-runs the SSH artifact + // transaction. The harness never runs per-node SCP/SSH orchestration. + let bundle_artifacts = campaign_root.join("bundle"); + std::fs::create_dir_all(&bundle_artifacts) + .map_err(|error| format!("create bundle directory: {error}"))?; + let payload = + myelin_e2e_fuzz::stage_deployment_payload(&workspace, &bundle_artifacts, &preparation)?; + let generation = format!( + "resume-{}", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|error| format!("system clock precedes epoch: {error}"))? + .as_millis() + ); + let bundle = myelin_e2e_fuzz::assemble_deployment_bundle( + &bundle_artifacts, + &payload, + &generation, + &preparation, + )?; + admission_owner.authorize_retained_deployment( + true, + provisioning::plugin::DeploymentIdentity { + artifact_digest: bundle.artifact_digest.clone(), + deployment_generation: bundle.deployment_generation.clone(), + }, + )?; + let fixture = ClusterHarnessConfig { + workspace, + artifacts: campaign_root.join("fixture"), + node_count, + seed: plan.seed, + image: options.image.clone(), + build_image: false, + deadline: options.deadline, + provider: HarnessProvider::VastAiReal { + admission: paid.state_dir.join("paid-admission.json"), + ssh_identity: options + .ssh_identity + .clone() + .expect("resume SSH identity validated by parser"), + api_key_env: options.api_key_env.clone(), + bundle: Some(bundle.tar_path.clone()), + }, + selected_offer_ids: paid.selected_offer_ids.clone(), + state_dir: Some(paid.state_dir.clone()), + reset_state: false, + offer_search_id: None, + provision: false, + adopt_only: false, + relay_url: None, + run_id: admission.run_id, + }; + let mut harness = + ClusterHarness::start_with_budgets(fixture, preparation.clone(), total.clone())?; + let repair = (|| { + let before = harness.owned_provider_contracts()?; + if before.len() != live_nodes.len() { + return Err(format!( + "retained fixture has {} live nodes but {} attributed contracts", + live_nodes.len(), + before.len() + )); + } + if !paid.owned_contract_ids.is_empty() && !before.is_subset(&paid.owned_contract_ids) { + return Err(format!( + "retained contracts {before:?} are not a subset of durable ownership {:?}", + paid.owned_contract_ids + )); + } + harness.verify_workload_convergence()?; + let after = harness.owned_provider_contracts()?; + if after != before { + return Err(format!( + "in-place deployment changed contracts: before={before:?}, after={after:?}" + )); + } + paid.resume_deny_only( + &provisioning::PaidFixtureAdmission::open(paid.state_dir.join("paid-admission.json"))? + .snapshot()?, + after, + )?; + write_paid_state(state_path, &paid)?; + preparation.check("retained redeployment prepared commit") + })(); + if let Err(error) = repair { + let diagnostics = harness.orchestrator_diagnostics(); + let failure = format!("retained paid fixture repair failed: {error}; {diagnostics}"); + paid.quarantine(format!( + "{failure}; exact contracts retained; cases paused; only explicit in-place retry or cleanup is permitted" + )); + let persistence = write_paid_state(state_path, &paid); + let retention = harness.retain_remote_fixture(true); + let reconciliation = admission_owner + .snapshot() + .and_then(|admission| paid.reconcile_admission(&admission)) + .and_then(|()| write_paid_state(state_path, &paid)); + return merge_execution_cleanup( + Err(failure), + merge_execution_cleanup( + persistence, + merge_execution_cleanup(retention, reconciliation), + ), + ); + } + let campaign = execution.child(Duration::from_secs(CAMPAIGN_DEADLINE_SECS)); + let execution = if options.retain_paid_fixture { + retain_paid_harness(&mut harness, state_path, &mut paid) + } else { + execute_campaign( + &mut harness, + &plan, + campaign_root, + campaign_root + .parent() + .ok_or("campaign corpus root missing")?, + options.shrink, + &campaign, + false, + ) + }; + if options.retain_paid_fixture && execution.is_ok() { + println!( + "development fixture retained at {}; campaign acceptance not attempted", + state_path.display() + ); + return execution; + } + let _cleanup_timer = PhaseTimer::new("paid-fixture-cleanup"); + let cleanup_budget = total.child(Duration::from_secs(CLEANUP_DEADLINE_SECS)); + let cleanup = cleanup_paid_harness( + &mut harness, + state_path, + &mut paid, + options, + campaign_root, + &cleanup_budget, + ); + merge_execution_cleanup(execution, cleanup)?; + total.check("complete retained deployment campaign and cleanup") +} + +fn execute_campaign( + harness: &mut ClusterHarness, + plan: &CampaignPlan, + campaign_root: &Path, + corpus_root: &Path, + shrink: bool, + campaign: &Budget, + allow_resume: bool, +) -> Result<(), String> { + let _workload_timer = PhaseTimer::new("campaign-workload"); + let workload = campaign.child( + campaign + .remaining("campaign workload allocation")? + .saturating_sub(Duration::from_secs(DIAGNOSTIC_DEADLINE_SECS)) + .min(Duration::from_secs(WORKLOAD_DEADLINE_SECS)), + ); + harness.set_execution_budget(workload.clone()); + let fleet = harness.fleet_snapshot()?; + let nodes = fleet + .pointer("/FleetStatus/nodes") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| "campaign identity census omitted nodes".to_owned())?; + let mut identity = nodes.clone(); + identity.sort_by_key(|node| { + node.get("logical_node_id") + .and_then(serde_json::Value::as_u64) + }); + let serialized_identity = serde_json::to_vec(&identity).map_err(|error| error.to_string())?; + let fixture_digest = format!("{:x}", Sha256::digest(&serialized_identity)); + let binaries = artifact_identity(&std::env::current_dir().map_err(|error| error.to_string())?)?; + let evidence_identity = myelin_e2e_fuzz::EvidenceIdentity { + plan_digest: hash_file(&campaign_root.join("campaign-plan.json"))?, + artifacts_digest: format!( + "{:x}", + Sha256::digest(serde_json::to_vec(&binaries).map_err(|error| error.to_string())?) + ), + deployment_generation: fixture_digest, + fixture_mapping_digest: harness.fixture_identity_digest()?, + }; + let checkpoint_path = campaign_root.join("campaign-progress.json"); + let mut checkpoint = CampaignCheckpoint { + schema_version: 2, + campaign_id: plan.campaign_id.clone(), + deployment_identity: serde_json::to_value(identity).map_err(|error| error.to_string())?, + evidence_identity: evidence_identity.clone(), + completed_regressions: BTreeSet::new(), + completed_recoveries: BTreeSet::new(), + pending_recovery: None, + }; + if allow_resume && checkpoint_path.is_file() { + let prior: CampaignCheckpoint = serde_json::from_reader( + fs::File::open(&checkpoint_path) + .map_err(|error| format!("read campaign checkpoint: {error}"))?, + ) + .map_err(|error| format!("decode campaign checkpoint: {error}"))?; + if prior.schema_version != checkpoint.schema_version + || prior.campaign_id != checkpoint.campaign_id + || prior.deployment_identity != checkpoint.deployment_identity + || prior.evidence_identity != checkpoint.evidence_identity + || prior.pending_recovery.is_some() + || !prior.completed_regressions.is_subset( + &plan + .regressions + .iter() + .map(|case| case.id.clone()) + .collect(), + ) + || !prior + .completed_recoveries + .is_subset(&plan.recovery.iter().map(|case| case.id.clone()).collect()) + { + return Err("campaign checkpoint is not a fully cleaned boundary on this exact deployment; failed formal recovery cannot resume".to_owned()); + } + checkpoint = prior; + } + durable_json(&checkpoint_path, &checkpoint)?; + if !allow_resume { + // A fresh fixture or binary redeployment invalidates every old ledger. + for (phase, ledger) in [ + ("normal", &plan.normal_coverage), + ("four-node", &plan.four_node_coverage), + ("three-node", &plan.three_node_coverage), + ] { + let mut ledger = ledger.clone(); + ledger.bind_identity(evidence_identity.clone())?; + write_coverage_ledger( + &campaign_root.join(format!("{phase}-coverage.json")), + &plan.campaign_id, + phase, + &ledger, + )?; + } + } + let mut live = harness.node_ids().iter().copied().collect::>(); + if live != plan.expected_initial_nodes + && live != plan.first_survivors + && live != plan.second_survivors + { + return Err(format!( + "fixture nodes {live:?} do not match any persisted campaign segment" + )); + } + + if live == plan.expected_initial_nodes { + for case in &plan.regressions { + if checkpoint.completed_regressions.contains(&case.id) { + continue; + } + workload.check(&format!("admit regression {}", case.id))?; + run_campaign_case(harness, case, campaign_root, corpus_root, shrink, campaign)?; + checkpoint.completed_regressions.insert(case.id.clone()); + durable_json(&checkpoint_path, &checkpoint)?; + } + let mut normal_coverage = plan.normal_coverage.clone(); + normal_coverage.bind_identity(evidence_identity.clone())?; + run_covered_phase( + harness, + &plan.normal, + &mut normal_coverage, + campaign_root, + corpus_root, + &plan.campaign_id, + "normal", + shrink, + campaign, + )?; + let recoveries = plan + .recovery + .iter() + .filter(|recovery| !checkpoint.completed_recoveries.contains(&recovery.id)) + .collect::>(); + if let Some(first) = recoveries.first() { + workload.check("admit formal recovery campaign")?; + checkpoint.pending_recovery = Some(first.id.clone()); + durable_json(&checkpoint_path, &checkpoint)?; + run_recovery_campaign(harness, &recoveries)?; + workload.check("commit fully cleaned recovery campaign")?; + checkpoint + .completed_recoveries + .extend(recoveries.iter().map(|recovery| recovery.id.clone())); + checkpoint.pending_recovery = None; + durable_json(&checkpoint_path, &checkpoint)?; + } + + let removed = plan + .expected_initial_nodes + .difference(&plan.first_survivors) + .copied() + .collect::>(); + if removed.len() != 1 { + return Err("four-node survivor plan does not remove exactly one node".to_owned()); + } + let _transition = PhaseTimer::new("destructive-transition-five-to-four"); + workload.check("admit five-to-four destructive transition")?; + harness.remove_node(removed[0])?; + live = harness.node_ids().iter().copied().collect(); + if live != plan.first_survivors { + return Err("four-node survivor set differs from the persisted plan".to_owned()); + } + } + + if live == plan.first_survivors { + let mut four_coverage = plan.four_node_coverage.clone(); + four_coverage.bind_identity(evidence_identity.clone())?; + run_covered_phase( + harness, + &plan.four_node, + &mut four_coverage, + campaign_root, + corpus_root, + &plan.campaign_id, + "four-node", + shrink, + campaign, + )?; + let removed = plan + .first_survivors + .difference(&plan.second_survivors) + .copied() + .collect::>(); + if removed.len() != 1 { + return Err("three-node survivor plan does not remove exactly one node".to_owned()); + } + let _transition = PhaseTimer::new("destructive-transition-four-to-three"); + workload.check("admit four-to-three destructive transition")?; + harness.remove_node(removed[0])?; + live = harness.node_ids().iter().copied().collect(); + if live != plan.second_survivors { + return Err("three-node survivor set differs from the persisted plan".to_owned()); + } + } + + let mut three_coverage = plan.three_node_coverage.clone(); + three_coverage.bind_identity(evidence_identity)?; + run_covered_phase( + harness, + &plan.three_node, + &mut three_coverage, + campaign_root, + corpus_root, + &plan.campaign_id, + "three-node", + shrink, + campaign, + )?; + if checkpoint.completed_regressions + != plan + .regressions + .iter() + .map(|case| case.id.clone()) + .collect() + || checkpoint.completed_recoveries + != plan.recovery.iter().map(|case| case.id.clone()).collect() + { + return Err( + "campaign completed survivor phase without full regression/recovery coverage" + .to_owned(), + ); + } + workload.check("complete 208 logical cases / 224 workload segments plus regressions") +} + +#[derive(serde::Serialize, serde::Deserialize)] +struct CampaignCheckpoint { + schema_version: u32, + campaign_id: String, + deployment_identity: serde_json::Value, + evidence_identity: myelin_e2e_fuzz::EvidenceIdentity, + completed_regressions: BTreeSet, + completed_recoveries: BTreeSet, + pending_recovery: Option, +} + +#[derive(serde::Serialize, serde::Deserialize)] +struct AcceptedAttempt { + identity: myelin_e2e_fuzz::EvidenceIdentity, + attempt: u64, + executed_case: BehaviorCase, + observation: myelin_e2e_fuzz::CaseObservation, +} + +fn run_covered_phase( + harness: &mut ClusterHarness, + cases: &[BehaviorCase], + ledger: &mut CoverageLedger, + campaign_root: &Path, + corpus_root: &Path, + campaign_id: &str, + phase: &str, + shrink: bool, + diagnosis: &Budget, +) -> Result<(), String> { + let _phase_timer = PhaseTimer::new(format!("coverage-phase-{phase}")); + let path = campaign_root.join(format!("{phase}-coverage.json")); + let proofs = campaign_root.join("coverage-evidence").join(phase); + if path.is_file() { + let mut resumed = read_coverage_ledger(&path, campaign_id, phase)?; + resumed.validate_resume(ledger)?; + let mut reconstructed = ledger.clone(); + for case in cases + .iter() + .filter(|case| resumed.completed_cases.contains(&case.id)) + { + let proof_path = proofs.join(format!("{}.json", case.id)); + let proof: AcceptedAttempt = + serde_json::from_reader(fs::File::open(&proof_path).map_err(|error| { + format!( + "read accepted attempt proof {}: {error}", + proof_path.display() + ) + })?) + .map_err(|error| format!("decode accepted attempt proof: {error}"))?; + if reconstructed.identity() != Some(&proof.identity) { + return Err("accepted attempt belongs to another deployment".to_owned()); + } + reconstructed.observe_attempt( + case, + &proof.executed_case, + &proof.observation, + proof.attempt, + true, + harness.execution_budget(), + )?; + } + if serde_json::to_value(&reconstructed).map_err(|error| error.to_string())? + != serde_json::to_value(&resumed).map_err(|error| error.to_string())? + { + return Err( + "persisted coverage is not the closure of its accepted attempt evidence".to_owned(), + ); + } + *ledger = reconstructed; + } else { + write_coverage_ledger(&path, campaign_id, phase, ledger)?; + } + for case in cases { + if ledger.completed_cases.contains(&case.id) { + continue; + } + harness + .execution_budget() + .check(&format!("admit {phase} case {}", case.id))?; + let (executed_case, observation) = + run_campaign_case(harness, case, campaign_root, corpus_root, shrink, diagnosis)?; + let attempt = harness + .last_attempt_id() + .ok_or("coverage missing immutable attempt identity")?; + ledger.observe_attempt( + case, + &executed_case, + &observation, + attempt, + true, + harness.execution_budget(), + )?; + durable_json( + &proofs.join(format!("{}.json", case.id)), + &AcceptedAttempt { + identity: ledger + .identity() + .cloned() + .ok_or("coverage deployment identity missing")?, + attempt, + executed_case, + observation, + }, + )?; + write_coverage_ledger(&path, campaign_id, phase, ledger)?; + } + ledger.assert_observed_closed() +} + +fn run_campaign_case( + harness: &mut ClusterHarness, + case: &BehaviorCase, + campaign_root: &Path, + corpus_root: &Path, + shrink: bool, + diagnostic_parent: &Budget, +) -> Result<(BehaviorCase, myelin_e2e_fuzz::CaseObservation), String> { + let _attempt_timer = PhaseTimer::new(format!("attempt-{}", case.id)); + harness + .execution_budget() + .check(&format!("admit workload {}", case.id))?; + let result = harness.run_case_with_identity(case); + // A successful attempt already persisted fresh post-cleanup health proof. + // Failed admission paths still need the explicit check before diagnosis. + if result.is_ok() { + return result; + } + let signature = harness.last_failure_signature().cloned(); + let health = harness.assert_healthy(); + let baseline_restored = health.is_ok(); + let result = merge_execution_cleanup(result, health); + let Err(error) = result else { return result }; + let original = format!("case {} failed: {error}", case.id); + if let Err(persist_error) = persist_regression( + &campaign_root.join("failures"), + case, + &error, + signature.as_ref(), + false, + ) { + return Err(format!( + "{original}; persist original regression failed: {persist_error}" + )); + } + if !shrink || !baseline_restored || harness.is_quarantined() { + return Err(original); + } + let Some(signature) = signature else { + return Err(format!( + "{original}; no typed behavioral signature; infrastructure failure is not shrinkable" + )); + }; + let _diagnosis_timer = PhaseTimer::new(format!("diagnosis-{}", case.id)); + let workload = harness.execution_budget().clone(); + let diagnostic = diagnostic_parent.child(Duration::from_secs(DIAGNOSTIC_DEADLINE_SECS)); + harness.set_execution_budget(diagnostic.clone()); + let diagnosis = (|| { + let mut replay = |candidate: &BehaviorCase| -> Result< + Option<(String, myelin_e2e_fuzz::FailureSignature)>, + String, + > { + diagnostic.check(&format!("diagnostic replay {}", candidate.id))?; + if harness.is_quarantined() { + return Err("fixture quarantined; diagnosis prohibited".to_owned()); + } + let attempt = harness.run_case(candidate); + let observed_signature = harness.last_failure_signature().cloned(); + let restored = harness.assert_healthy(); + if let Err(restoration) = restored { + return Err(format!( + "diagnostic fixture restoration failed: {restoration}" + )); + } + if harness.is_quarantined() { + return Err("diagnostic attempt quarantined fixture".to_owned()); + } + diagnostic.check("diagnostic candidate complete and baseline restored")?; + attempt + .err() + .map(|error| { + observed_signature + .map(|signature| (error, signature)) + .ok_or_else(|| { + "replay failed without a typed behavioral signature".to_owned() + }) + }) + .transpose() + }; + let exact = replay(case)?.ok_or_else(|| "exact replay did not reproduce".to_owned())?; + if exact.1 != signature { + return Err("exact replay changed typed failure signature".to_owned()); + } + let minimized = + myelin_e2e_fuzz::try_shrink_failure_budgeted(case.clone(), &diagnostic, |candidate| { + Ok(replay(candidate)?.is_some_and(|(_, observed)| observed == signature)) + })?; + let minimized_error = replay(&minimized)? + .ok_or_else(|| "final minimized replay did not reproduce".to_owned())?; + if minimized_error.1 != signature { + return Err("minimized replay changed typed failure signature".to_owned()); + } + // Never overwrite the original witness with a candidate. + persist_regression( + &campaign_root.join("minimized"), + &minimized, + &minimized_error.0, + Some(&signature), + true, + )?; + // Only a cleaned, exactly replayed, same-signature witness enters the reusable corpus. + persist_regression( + corpus_root, + &minimized, + &minimized_error.0, + Some(&signature), + true, + ) + })(); + harness.set_execution_budget(workload); + let status = match diagnosis { + Ok(()) => "exact replay and verified minimized witness persisted".to_owned(), + Err(reason) => format!("diagnosis incomplete: {reason}; original witness retained"), + }; + let persistence = durable_json( + &campaign_root + .join("diagnosis") + .join(format!("{}.json", case.id)), + &serde_json::json!({"schema_version": 2, "original_failure": error, "signature": signature, "status": status}), + ); + Err(format!( + "{original}; {status}{}", + persistence + .err() + .map(|error| format!("; diagnostic evidence failed: {error}")) + .unwrap_or_default() + )) +} + +struct PreparedRecoveryAttempt { + recovery: RecoveryCase, + before: FixturePathSnapshot, +} + +fn run_recovery_campaign( + harness: &mut ClusterHarness, + recoveries: &[&RecoveryCase], +) -> Result<(), String> { + let _campaign_timer = PhaseTimer::new("recovery-campaign"); + let mut prepared = Vec::::with_capacity(recoveries.len()); + let result = (|| { + for recovery in recoveries { + let _pre_timer = PhaseTimer::new(format!("recovery-pre-{}", recovery.id)); + let recovery = harness.prepare_recovery_attempt(recovery)?; + harness.run_case_retained_deferred(&recovery.pre_restart)?; + let before = harness + .retained_fixture_snapshot(&recovery.pre_restart, &recovery.persisted_entries)?; + prepared.push(PreparedRecoveryAttempt { recovery, before }); + } + harness.seal_retained_attempts()?; + + // Every pre-restart segment is durable before the single formal + // orchestrator replacement. All paths are attempt-scoped, and the + // retained-attempt census admits only their modeled persistent actors. + let _restart_timer = PhaseTimer::new("restart-rejoin-recovery-campaign"); + harness.verify_running_recovery()?; + drop(_restart_timer); + + for attempt in &prepared { + let recovery = &attempt.recovery; + let _post_timer = PhaseTimer::new(format!("recovery-post-{}", recovery.id)); + harness.run_case_retained_deferred(&recovery.post_restart)?; + let after = harness + .retained_fixture_snapshot(&recovery.post_restart, &recovery.persisted_entries)?; + if after != attempt.before { + return Err(format!( + "recovery {} changed persisted namespace facts: before={:?}, after={after:?}", + recovery.id, attempt.before + )); + } + } + harness.seal_retained_attempts()?; + let cases = prepared + .iter() + .flat_map(|attempt| { + [ + &attempt.recovery.pre_restart, + &attempt.recovery.post_restart, + ] + }) + .collect::>(); + harness.cleanup_recovery_cases(&cases)?; + Ok(()) + })(); + + if result.is_ok() { + return Ok(()); + } + let mut cleanup = Ok(()); + for attempt in &prepared { + let recovery = &attempt.recovery; + harness.abort_case_processes(&recovery.pre_restart); + harness.abort_case_processes(&recovery.post_restart); + cleanup = merge_execution_cleanup(cleanup, harness.cleanup_case(&recovery.pre_restart)); + cleanup = merge_execution_cleanup(cleanup, harness.cleanup_case(&recovery.post_restart)); + } + cleanup = merge_execution_cleanup(cleanup, harness.assert_healthy()); + merge_execution_cleanup(result, cleanup) +} + +fn load_regressions(root: &Path) -> Result, String> { + let directory = root.join("regressions"); + let entries = match fs::read_dir(&directory) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(format!("read regression directory: {error}")), + }; + let mut paths = Vec::new(); + for entry in entries { + let entry = entry.map_err(|error| format!("read regression directory entry: {error}"))?; + if !entry + .file_type() + .map_err(|error| format!("inspect regression entry: {error}"))? + .is_dir() + { + return Err(format!( + "unexpected regression entry {}", + entry.path().display() + )); + } + paths.push(entry.path()); + } + paths.sort(); + paths + .into_iter() + .map(|path| read_regression_witness(&path, true).map(|(case, _)| case)) + .collect() +} + +fn merge_execution_cleanup( + execution: Result, + cleanup: Result<(), String>, +) -> Result { + match (execution, cleanup) { + (Ok(value), Ok(())) => Ok(value), + (Err(error), Ok(())) => Err(error), + (Ok(_), Err(cleanup_error)) => Err(format!("cleanup failed: {cleanup_error}")), + (Err(error), Err(cleanup_error)) => { + Err(format!("{error}; cleanup also failed: {cleanup_error}")) + } + } +} + +#[cfg(test)] +mod audit_tests { + use super::*; + + fn ordered_attestation() -> OrderedGateAttestation { + let mut evidence = OrderedGateAttestation { + schema_version: 5, + state: "passed".to_owned(), + completed_unix_ms: 100, + expires_unix_ms: 200, + build_artifacts: BTreeMap::new(), + deployment_artifacts: myelin_e2e_fuzz::PreparedArtifactIdentity { + schema_version: 2, + source_build_input_digest: "a".repeat(64), + executables: BTreeMap::new(), + wheel_input_digest: String::new(), + wheels: BTreeMap::new(), + payload: BTreeMap::new(), + }, + source_digest: String::new(), + image_identities: [ + ("myelin-node-base:cuda12.6", "base"), + ("myelin-node:latest", "node"), + ("myelin-e2e:tested", "e2e"), + ] + .into_iter() + .enumerate() + .map(|(index, (name, role))| { + ( + name.to_owned(), + GateImageIdentity { + id: format!("sha256:{}", index.to_string().repeat(64)), + os: "linux".to_owned(), + architecture: "amd64".to_owned(), + variant: String::new(), + repo_digests: BTreeSet::new(), + provenance_version: 1, + source_build_input_digest: "a".repeat(64), + image_role: role.to_owned(), + parent_image_id: index + .checked_sub(1) + .map(|parent| format!("sha256:{}", parent.to_string().repeat(64))), + }, + ) + }) + .collect(), + configuration: GateConfiguration { + warm_runs: 3, + image: "myelin-e2e:tested".to_owned(), + seed: 17, + case_deadline_secs: 120, + build_images: false, + workspace: PathBuf::from("/qualified/workspace"), + artifacts: PathBuf::from("/qualified/evidence"), + }, + stages: Vec::new(), + runs: (1..=3) + .map(|index| GateRun { + index, + state: "passed".to_owned(), + gate_a: "passed".to_owned(), + inside_target: true, + campaign_elapsed_secs: 0.001, + elapsed_secs: 0.01, + }) + .collect(), + build_elapsed_secs: 0.02, + elapsed_secs: 0.1, + }; + evidence.stages = expected_gate_stages(&evidence) + .into_iter() + .enumerate() + .map(|(index, (name, command))| GateStage { + name, + command, + state: "passed".to_owned(), + exit_code: Some(0), + started_unix_ms: index as u64 * 2 + 1, + completed_unix_ms: index as u64 * 2 + 2, + elapsed_secs: 0.001, + }) + .collect(); + evidence + } + + #[test] + fn ordered_gates_reject_partial_stale_failed_and_reordered_evidence() { + assert!(validate_gate_attestation(&ordered_attestation(), 150).is_ok()); + assert!(validate_gate_attestation(&ordered_attestation(), 200).is_err()); + assert!(validate_gate_attestation(&ordered_attestation(), 99).is_err()); + let mut partial = ordered_attestation(); + partial.stages.remove(1); + assert!(validate_gate_attestation(&partial, 150).is_err()); + let mut failed = ordered_attestation(); + failed.stages[1].exit_code = Some(1); + assert!(validate_gate_attestation(&failed, 150).is_err()); + let mut reordered = ordered_attestation(); + reordered.stages[0].name = "warm-1-campaign".to_owned(); + reordered.stages[1].name = "warm-1-gate-a".to_owned(); + assert!(validate_gate_attestation(&reordered, 150).is_err()); + let mut isolated = ordered_attestation(); + isolated.runs.truncate(1); + assert!(validate_gate_attestation(&isolated, 150).is_err()); + let mut missing_base = ordered_attestation(); + missing_base + .image_identities + .remove("myelin-node-base:cuda12.6"); + assert!(validate_gate_attestation(&missing_base, 150).is_err()); + } + + #[test] + fn ordered_gates_require_all_phases_and_exact_case_selection() { + for name in [ + "build", + "build-test-binaries", + "build-deployment-artifacts", + "verify-qualified-deployment-artifacts", + ] { + let mut evidence = ordered_attestation(); + evidence.stages.retain(|stage| stage.name != name); + assert!(validate_gate_attestation(&evidence, 150).is_err(), "{name}"); + } + let mut reduced = ordered_attestation(); + let gate_a = reduced + .stages + .iter_mut() + .find(|stage| stage.name == "warm-1-gate-a") + .unwrap(); + let nodes = gate_a + .command + .iter() + .position(|argument| argument == "--deployment-nodes") + .unwrap(); + gate_a.command[nodes + 1] = "2".to_owned(); + assert!(validate_gate_attestation(&reduced, 150).is_err()); + let mut incomplete_build = ordered_attestation(); + incomplete_build.configuration.build_images = true; + assert!(validate_gate_attestation(&incomplete_build, 150).is_err()); + incomplete_build.stages = expected_gate_stages(&incomplete_build) + .into_iter() + .enumerate() + .map(|(index, (name, command))| GateStage { + name, + command, + state: "passed".to_owned(), + exit_code: Some(0), + started_unix_ms: index as u64 * 2 + 1, + completed_unix_ms: index as u64 * 2 + 2, + elapsed_secs: 0.001, + }) + .collect(); + assert!(validate_gate_attestation(&incomplete_build, 150).is_ok()); + incomplete_build + .stages + .retain(|stage| stage.name != "build-image-node-parent"); + assert!(validate_gate_attestation(&incomplete_build, 150).is_err()); + } + + #[test] + fn ordered_gates_enforce_measured_ceilings_and_phase_accounting() { + let mut boundary = ordered_attestation(); + boundary.runs[0].campaign_elapsed_secs = 300.0; + boundary.stages[4].elapsed_secs = 300.0; + boundary.runs[0].elapsed_secs = 600.0; + boundary.elapsed_secs = 1000.0; + assert!(validate_gate_attestation(&boundary, 150).is_ok()); + boundary.runs[0].elapsed_secs = 600.001; + assert!(validate_gate_attestation(&boundary, 150).is_err()); + boundary.runs[0].elapsed_secs = 600.0; + boundary.runs[0].campaign_elapsed_secs = 300.001; + boundary.stages[4].elapsed_secs = 300.001; + assert!(validate_gate_attestation(&boundary, 150).is_err()); + for seconds in [-1.0, f64::NAN, f64::INFINITY] { + let mut invalid = ordered_attestation(); + invalid.runs[0].elapsed_secs = seconds; + assert!(validate_gate_attestation(&invalid, 150).is_err()); + let mut invalid = ordered_attestation(); + invalid.stages[0].elapsed_secs = seconds; + assert!(validate_gate_attestation(&invalid, 150).is_err()); + } + let mut omitted = ordered_attestation(); + omitted.runs[0].elapsed_secs = 0.001; + assert!(validate_gate_attestation(&omitted, 150).is_err()); + let mut understated = ordered_attestation(); + understated.stages[4].elapsed_secs = 301.0; + understated.runs[0].elapsed_secs = 500.0; + understated.elapsed_secs = 1000.0; + assert!(validate_gate_attestation(&understated, 150).is_err()); + let mut outside = ordered_attestation(); + outside.runs[0].inside_target = false; + assert!(validate_gate_attestation(&outside, 150).is_err()); + } + + #[test] + fn ordered_gates_reject_legacy_missing_timing_and_stale_image_provenance() { + let mut legacy = ordered_attestation(); + legacy.schema_version = 4; + assert!(validate_gate_attestation(&legacy, 150).is_err()); + for field in ["inside_target", "campaign_elapsed_secs", "elapsed_secs"] { + let mut missing = serde_json::to_value(ordered_attestation()).unwrap(); + missing["runs"][0].as_object_mut().unwrap().remove(field); + assert!( + serde_json::from_value::(missing).is_err(), + "{field}" + ); + } + let mut stale = ordered_attestation(); + stale + .image_identities + .get_mut("myelin-e2e:tested") + .unwrap() + .source_build_input_digest = "b".repeat(64); + assert!(validate_gate_attestation(&stale, 150).is_err()); + let mut unrelated = ordered_attestation(); + unrelated + .image_identities + .get_mut("myelin-e2e:tested") + .unwrap() + .parent_image_id = Some(format!("sha256:{}", "f".repeat(64))); + assert!(validate_gate_attestation(&unrelated, 150).is_err()); + let mut unproven = serde_json::to_value(ordered_attestation()).unwrap(); + unproven["image_identities"]["myelin-e2e:tested"] + .as_object_mut() + .unwrap() + .remove("provenance_version"); + assert!(serde_json::from_value::(unproven).is_err()); + } + + #[test] + fn repeat_paid_claim_preserves_all_durable_ownership_bytes() { + let temporary = tempfile::tempdir().unwrap(); + let run = temporary.path().join("campaign"); + claim_campaign_directory(&run, true).unwrap(); + let ownership = br#"{"labels":["myelin-1-1-attempt-0"],"contracts":[42],"unresolved":[2]}"#; + durable_write(&run.join("paid-state.json"), ownership).unwrap(); + durable_write(&run.join("campaign-plan.json"), b"immutable-plan").unwrap(); + assert!(claim_campaign_directory(&run, true).is_err()); + assert_eq!(fs::read(run.join("paid-state.json")).unwrap(), ownership); + assert_eq!( + fs::read(run.join("campaign-plan.json")).unwrap(), + b"immutable-plan" + ); + } + + #[test] + fn promoted_regression_is_loaded_from_shared_corpus_and_corruption_fails() { + let temporary = tempfile::tempdir().unwrap(); + let case = stable_corpus(2, 17).remove(0); + let partial = tempfile::tempdir().unwrap(); + durable_json( + &partial + .path() + .join("regressions") + .join(&case.id) + .join("case.json"), + &case, + ) + .unwrap(); + assert!(load_regressions(partial.path()).is_err()); + let signature = myelin_e2e_fuzz::FailureSignature { + invariant: "payload".to_owned(), + failure_class: myelin_e2e_fuzz::FailureClass::PayloadMismatch, + causal_role: myelin_e2e_fuzz::CausalRole::Action( + myelin_e2e_fuzz::SemanticAction::ReadBlob, + ), + observed_outcome: None, + }; + persist_regression( + temporary.path(), + &case, + "typed failure evidence", + Some(&signature), + true, + ) + .unwrap(); + let loaded = load_regressions(temporary.path()).unwrap(); + assert_eq!( + serde_json::to_value(&loaded).unwrap(), + serde_json::json!([case]) + ); + let witness = temporary.path().join("regressions").join(&case.id); + durable_write(&witness.join("signature.json"), b"null").unwrap(); + assert!(load_regressions(temporary.path()).is_err()); + } +} diff --git a/tools/myelin-e2e-fuzz/src/oracle.rs b/tools/myelin-e2e-fuzz/src/oracle.rs index d6fa498..44b46f3 100644 --- a/tools/myelin-e2e-fuzz/src/oracle.rs +++ b/tools/myelin-e2e-fuzz/src/oracle.rs @@ -1,19 +1,208 @@ //! Behavioral invariants checked against observed program output. +use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet}; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; -use crate::codegen::digest; +use crate::budget::Budget; use crate::ir::{ - ActionObservation, ActionOp, BehaviorCase, CaseObservation, ExecutionObservation, - ExpectedOutcome, FailureInjection, LaunchFailureKind, ProcessProgram, ProcessStopPhase, + ActionObservation, ActionOp, BarrierObservation, BehaviorCase, CaseObservation, DataKind, + DescriptorFinish, DescriptorObservation, DescriptorReadMethod, DescriptorTerminalResult, + ExecutionObservation, ExpectedOutcome, FailureInjection, LaunchFailureKind, ProcessProgram, + ProcessStopPhase, }; +/// Durable behavioral identity, independent of request, process, path and attempt IDs. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct FailureSignature { + pub invariant: String, + pub failure_class: FailureClass, + pub causal_role: CausalRole, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observed_outcome: Option, +} + +/// Typed outcome evidence without diagnostic text or race-group identities. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ObservedOutcome { + pub expected: OutcomeExpectation, + pub outcome: String, + pub errno: Option, + pub error_type: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", content = "value", rename_all = "snake_case")] +pub enum OutcomeExpectation { + Ok, + Error(i32), + Exception(String), + Linearized { successes: u8, error: i32 }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum FailureClass { + ContractViolation, + BudgetExceeded, + MissingEvidence, + InvalidEvidence, + MissingIncarnation, + InvalidIncarnation, + ConflictingIncarnation, + MilestoneOrder, + MissingStreamSource, + StreamSourceContent, + ReservationNotReleased, + MissingTransfer, + PayloadMismatch, + OutcomeMismatch, + NamespaceHistoryConflict, + RetainedBlobMismatch, + MissingStreamEof, + FrameCountMismatch, + FrameOrderMismatch, + FrameContentMismatch, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "role", content = "operation", rename_all = "snake_case")] +pub enum CausalRole { + Case, + Lifecycle, + Namespace, + Action(SemanticAction), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SemanticAction { + PublishBlob, + ReadBlob, + StreamWrite, + StreamRead, + StreamReadWithRetry, + StreamRoundTrip, + GatedStreamWrite, + GatedStreamRead, + StreamReadInto, + Lookup, + AwaitEntry, + WaitForQuiescent, + Rename, + Unlink, + DescriptorWrite, + DescriptorRead, + MappingExportClose, +} + +impl From<&ActionOp> for SemanticAction { + fn from(action: &ActionOp) -> Self { + match action { + ActionOp::PublishBlob { .. } => Self::PublishBlob, + ActionOp::ReadBlob { .. } => Self::ReadBlob, + ActionOp::StreamWrite { .. } => Self::StreamWrite, + ActionOp::StreamRead { .. } => Self::StreamRead, + ActionOp::StreamReadWithRetry { .. } => Self::StreamReadWithRetry, + ActionOp::StreamRoundTrip { .. } => Self::StreamRoundTrip, + ActionOp::GatedStreamWrite { .. } => Self::GatedStreamWrite, + ActionOp::GatedStreamRead { .. } => Self::GatedStreamRead, + ActionOp::StreamReadInto { .. } => Self::StreamReadInto, + ActionOp::Lookup { .. } => Self::Lookup, + ActionOp::AwaitEntry { .. } => Self::AwaitEntry, + ActionOp::WaitForQuiescent { .. } => Self::WaitForQuiescent, + ActionOp::Rename { .. } => Self::Rename, + ActionOp::Unlink { .. } => Self::Unlink, + ActionOp::DescriptorWrite { .. } => Self::DescriptorWrite, + ActionOp::DescriptorRead { .. } => Self::DescriptorRead, + ActionOp::MappingExportClose { .. } => Self::MappingExportClose, + } + } +} + #[derive(Clone, Debug, PartialEq, Eq)] pub struct OracleViolation { pub invariant: &'static str, pub detail: String, + pub signature: FailureSignature, +} + +impl OracleViolation { + fn new(invariant: &'static str, detail: impl Into) -> Self { + let failure_class = match invariant { + "execution_budget" => FailureClass::BudgetExceeded, + "missing_execution" | "missing_action" | "namespace_model_evidence" => { + FailureClass::MissingEvidence + } + "payload_integrity" | "stream_bytes" | "blob_publication" => { + FailureClass::PayloadMismatch + } + "unexpected_error" | "typed_error" | "typed_exception" => FailureClass::OutcomeMismatch, + "namespace_linearizability" => FailureClass::NamespaceHistoryConflict, + _ => FailureClass::ContractViolation, + }; + let causal_role = if invariant.starts_with("namespace_") { + CausalRole::Namespace + } else if matches!( + invariant, + "duplicate_lifecycle" + | "bootstrap_resolution" + | "context_order" + | "missing_context" + | "user_before_ready" + | "terminal_resolution" + | "output_after_terminal" + | "process_failure" + | "stop_phase" + | "launch_failure_kind" + ) { + CausalRole::Lifecycle + } else { + CausalRole::Case + }; + Self { + invariant, + detail: detail.into(), + signature: FailureSignature { + invariant: invariant.to_owned(), + failure_class, + causal_role, + observed_outcome: None, + }, + } + } + + fn classified(mut self, failure_class: FailureClass, causal_role: CausalRole) -> Self { + self.signature.failure_class = failure_class; + self.signature.causal_role = causal_role; + self + } + + fn at_action(mut self, action: &ActionOp) -> Self { + self.signature.causal_role = CausalRole::Action(action.into()); + self + } + + fn with_outcome(mut self, expected: ExpectedOutcome, result: &ActionObservation) -> Self { + self.signature.observed_outcome = Some(ObservedOutcome { + expected: match expected { + ExpectedOutcome::Ok => OutcomeExpectation::Ok, + ExpectedOutcome::Error(errno) => OutcomeExpectation::Error(errno), + ExpectedOutcome::Exception(exception) => { + OutcomeExpectation::Exception(exception.as_str().to_owned()) + } + ExpectedOutcome::Linearized { + successes, error, .. + } => OutcomeExpectation::Linearized { successes, error }, + }, + outcome: result.outcome.clone(), + errno: result.errno, + error_type: result.error_type.clone(), + }); + self + } } impl std::fmt::Display for OracleViolation { @@ -26,11 +215,247 @@ impl std::error::Error for OracleViolation {} pub struct BehaviorOracle; +fn is_barrier_record(result: &ActionObservation) -> bool { + result.outcome == "barrier" +} + +fn check_budget(budget: Option<&Budget>, predicate: &str) -> Result<(), OracleViolation> { + if let Some(budget) = budget { + budget + .check(predicate) + .map_err(|detail| OracleViolation::new("execution_budget", detail))?; + } + Ok(()) +} + +/// An action exception explains an unexecuted suffix only when the original +/// cursor ends in a normal nonzero Python exit, not cancellation or a failed +/// transport/bootstrap. The collector separately proves cursor durability. +fn terminal_action_error(execution: &ExecutionObservation) -> Option<&ActionObservation> { + if execution.exit_success || !execution.terminal { + return None; + } + let status: serde_json::Value = serde_json::from_str(execution.exit_status.as_deref()?).ok()?; + if status.get("kind")?.as_str()? != "code" + || status.get("value")?.as_i64()? == 0 + || execution.lifecycle.iter().any(|event| { + matches!( + event.as_str(), + "bootstrap_failed" + | "spawn_failed" + | "spawn_rejected" + | "process_error" + | "stop_accepted" + | "stop_rejected" + ) + }) + { + return None; + } + let result = execution.results.last()?; + let error_type = result.error_type.as_deref()?; + if result.outcome != "error" + || error_type.is_empty() + || matches!( + error_type, + "TimeoutError" | "CancelledError" | "KeyboardInterrupt" | "SystemExit" | "MemoryError" + ) + { + return None; + } + Some(result) +} + +/// Recognize a complete, causally identified action prefix ending in an +/// unexpected typed exception. Never fill in the suffix the program did not run. +pub(crate) fn recorded_action_failure( + program: &ProcessProgram, + execution: &ExecutionObservation, +) -> Option { + let failure = terminal_action_error(execution)?; + if execution.process != program.id + || execution.logical_node_id != program.logical_node_id + || execution.request_id.is_empty() + || BehaviorOracle::verify_execution(execution).is_err() + { + return None; + } + let mut next = 0; + for result in &execution.results { + let action = program.actions.get(result.step)?; + if result.process != program.id + || !stream_record_action_matches(&action.operation, result) + || result.path != action.operation.path() + { + return None; + } + if is_barrier_record(result) { + continue; + } + if result.step != next { + return None; + } + next += 1; + if result.step != failure.step + && !matches!(result.outcome.as_str(), "ok" | "expected_error") + { + return None; + } + } + (next == failure.step + 1).then_some(failure.step) +} + +/// The controller and oracle share only this observation predicate, not the +/// namespace model: a stop is released by public, matching endpoint milestones. +pub(crate) fn stream_stop_ready<'a>( + case: &BehaviorCase, + target: &str, + phase: ProcessStopPhase, + executions: impl Iterator + Clone, +) -> bool { + let Some((_, target_results, lifecycle)) = executions + .clone() + .find(|(process, _, _)| *process == target) + else { + return false; + }; + if !lifecycle.iter().any(|event| event == "context_ready") { + return false; + } + let endpoint = |program: &ProcessProgram, + results: &[ActionObservation], + step: usize, + incarnation: u64, + writer: bool| { + let Some(action) = program.actions.get(step) else { + return false; + }; + let valid = |record: &ActionObservation| { + record.step == step + && record.process == program.id + && record.path == action.operation.path() + && stream_record_action_matches(&action.operation, record) + && is_barrier_record(record) + }; + let opened = results.iter().position(|record| valid(record) + && matches!(record.barrier, Some(BarrierObservation::StreamOpened { incarnation: observed }) + if observed == incarnation)); + let progress = results.iter().position(|record| valid(record) && if writer { + record.action == "stream_write" + && matches!(record.barrier, Some(BarrierObservation::StreamFrame { incarnation: observed, index: 0, .. }) + if observed == incarnation) + } else { + matches!(record.barrier, Some(BarrierObservation::StreamFirstFrame { incarnation: observed }) + if observed == incarnation) + }); + incarnation != 0 + && opened + .zip(progress) + .is_some_and(|(opened, progress)| opened < progress) + }; + let matched_peer = |program: &ProcessProgram, step: usize, incarnation: u64, writer: bool| { + let path = namespace_path(program, program.actions[step].operation.path()); + executions.clone().any(|(peer, results, _)| { + let Some(peer_program) = case.processes.iter().find(|program| program.id == peer) + else { + return false; + }; + peer_program + .actions + .iter() + .enumerate() + .any(|(step, action)| { + let Some((peer_writer, _, _)) = stream_parameters(&action.operation) else { + return false; + }; + (peer_writer != writer + || matches!(action.operation, ActionOp::StreamRoundTrip { .. })) + && namespace_path(peer_program, action.operation.path()) == path + && endpoint(peer_program, results, step, incarnation, !writer) + }) + }) + }; + if phase == ProcessStopPhase::AfterSiblingStreamFirstFrame { + return executions.clone().any(|(process, results, lifecycle)| { + process != target + && lifecycle.iter().any(|event| event == "context_ready") + && case + .processes + .iter() + .find(|program| program.id == process) + .is_some_and(|program| { + results.iter().any(|record| { + let Some(BarrierObservation::StreamFirstFrame { incarnation }) = + record.barrier + else { + return false; + }; + program.actions.get(record.step).is_some_and(|action| { + matches!(action.operation, ActionOp::GatedStreamRead { .. }) + }) && endpoint(program, results, record.step, incarnation, false) + && matched_peer(program, record.step, incarnation, false) + }) + }) + }); + } + let Some(program) = case.processes.iter().find(|program| program.id == target) else { + return false; + }; + target_results.iter().any(|record| { + let Some(BarrierObservation::StreamOpened { incarnation }) = record.barrier else { + return false; + }; + let Some(action) = program.actions.get(record.step) else { + return false; + }; + let Some((writer, _, _)) = stream_parameters(&action.operation) else { + return false; + }; + if !endpoint(program, target_results, record.step, incarnation, writer) { + return false; + } + matched_peer(program, record.step, incarnation, writer) + }) +} + impl BehaviorOracle { pub fn verify( case: &BehaviorCase, observation: &CaseObservation, ) -> Result<(), OracleViolation> { + Self::verify_inner(case, observation, None, None) + } + + pub fn verify_with_budget( + case: &BehaviorCase, + observation: &CaseObservation, + budget: &Budget, + ) -> Result<(), OracleViolation> { + Self::verify_inner(case, observation, Some(budget), None) + } + + /// The caller supplies an authoritative, complete final snapshot of owned + /// blobs as path -> (namespace revision, byte length). This constrains the + /// same history search; it never picks an arbitrary race winner for health. + pub fn verify_retained_blobs_with_budget( + case: &BehaviorCase, + observation: &CaseObservation, + retained: &BTreeMap, + budget: &Budget, + ) -> Result<(), OracleViolation> { + Self::verify_inner(case, observation, Some(budget), Some(retained)) + } + + fn verify_inner( + case: &BehaviorCase, + observation: &CaseObservation, + budget: Option<&Budget>, + retained: Option<&BTreeMap>, + ) -> Result<(), OracleViolation> { + check_budget(budget, "oracle model validation")?; + case.validate() + .map_err(|detail| OracleViolation::new("invalid_model", detail))?; + verify_route_expected_values(case)?; if observation.case_id != case.id { return violation("case_identity", "observation belongs to a different case"); } @@ -45,6 +470,7 @@ impl BehaviorOracle { ); } let mut execution_ids = BTreeSet::new(); + let mut request_ids = BTreeSet::new(); for execution in &observation.executions { if !execution_ids.insert(&execution.process) { return violation( @@ -52,17 +478,187 @@ impl BehaviorOracle { format!("process {} was observed more than once", execution.process), ); } + if execution.request_id.is_empty() || !request_ids.insert(&execution.request_id) { + return violation( + "execution_identity", + "execution request identity is empty or reused", + ); + } + let program = case + .processes + .iter() + .find(|program| program.id == execution.process) + .ok_or_else(|| OracleViolation::new("execution_identity", "unknown process"))?; + if execution.request_id != case.execution_request_id(program) { + return violation( + "execution_identity", + format!( + "{} has an execution request from another attempt", + program.id + ), + ); + } + if execution.logical_node_id != program.logical_node_id { + return violation( + "execution_placement", + format!("{} executed outside its assigned logical node", program.id), + ); + } + let mut next_step = 0; + for result in &execution.results { + let action = program.actions.get(result.step).ok_or_else(|| { + OracleViolation::new( + "action_identity", + format!("{} reported unplanned step {}", program.id, result.step), + ) + })?; + if result.process != program.id + || !stream_record_action_matches(&action.operation, result) + { + return violation( + "action_identity", + format!( + "{} step {} belongs to another action", + program.id, result.step + ), + ); + } + if result.path != action.operation.path() { + return violation( + "path_consistency", + format!( + "{} step {} reported the wrong path", + program.id, result.step + ), + ); + } + // Endpoint-preparation barriers may arrive for later actions, + // but terminal action records follow this process's serial IR. + // Do not reconstruct a different local history by sorting steps. + if !is_barrier_record(result) { + if result.step < next_step { + return violation( + "duplicate_action", + format!("{} emitted step {} twice", program.id, result.step), + ); + } + if result.step != next_step { + return Err(OracleViolation::new( + "action_order", + format!( + "{} expected step {next_step}, observed {}", + program.id, result.step + ), + ) + .classified( + FailureClass::MilestoneOrder, + CausalRole::Action((&action.operation).into()), + )); + } + next_step += 1; + } + } + } + // A typed failure must not hide missing evidence in a later sibling. + // Successful runs retain the ordinary invariant diagnostic ordering. + if observation + .executions + .iter() + .any(|execution| !execution.exit_success) + { + for program in &case.processes { + check_budget(budget, "terminal action-prefix evidence")?; + let execution = observation + .executions + .iter() + .find(|execution| execution.process == program.id) + .ok_or_else(|| OracleViolation::new("missing_execution", &program.id))?; + if process_failure_is_expected(case, program) { + let mut lifecycle_only = execution.clone(); + lifecycle_only.results.clear(); + Self::verify_execution(&lifecycle_only)?; + Self::verify_expected_process_failure(case, program, execution)?; + continue; + } + Self::verify_execution(execution)?; + let complete = if execution.exit_success { + let mut next = 0; + execution.results.iter().all(|result| { + let Some(action) = program.actions.get(result.step) else { + return false; + }; + if result.process != program.id + || !stream_record_action_matches(&action.operation, result) + || result.path != action.operation.path() + { + return false; + } + if is_barrier_record(result) { + return true; + } + if result.step != next { + return false; + } + next += 1; + true + }) && next == program.actions.len() + } else { + recorded_action_failure(program, execution).is_some() + }; + if !complete { + return Err(OracleViolation::new( + "missing_action", + format!( + "{} lacks a complete action history or typed terminal prefix", + program.id + ), + )); + } + } + if let FailureInjection::StopProcess { process, phase, .. } = &case.failure + && matches!( + phase, + ProcessStopPhase::AfterStreamFirstFrame + | ProcessStopPhase::AfterSiblingStreamFirstFrame + ) + && !stream_stop_ready( + case, + process, + *phase, + observation.executions.iter().map(|execution| { + ( + execution.process.as_str(), + execution.results.as_slice(), + execution.lifecycle.as_slice(), + ) + }), + ) + { + return violation( + "stop_phase", + "stream stop lacks matching first-frame milestones", + ); + } } for program in &case.processes { + check_budget(budget, "oracle process and action evidence")?; let execution = observation .executions .iter() .find(|execution| execution.process == program.id) - .ok_or_else(|| OracleViolation { - invariant: "missing_execution", - detail: format!("process {} has no execution observation", program.id), + .ok_or_else(|| { + OracleViolation::new( + "missing_execution", + format!("process {} has no execution observation", program.id), + ) })?; - if process_failure_is_expected(case, program) && !execution.exit_success { + if process_failure_is_expected(case, program) { + if execution.exit_success { + return violation( + "process_failure", + "expected failed process exited successfully", + ); + } let mut lifecycle_only = execution.clone(); lifecycle_only.results.clear(); Self::verify_execution(&lifecycle_only)?; @@ -71,7 +667,11 @@ impl BehaviorOracle { } Self::verify_execution(execution)?; let mut seen = BTreeSet::new(); - for result in &execution.results { + for result in execution + .results + .iter() + .filter(|result| !is_barrier_record(result)) + { if !seen.insert(result.step) { return violation( "duplicate_action", @@ -79,14 +679,21 @@ impl BehaviorOracle { ); } } - if execution.results.len() != program.actions.len() { + if execution + .results + .iter() + .filter(|result| !is_barrier_record(result)) + .count() + != program.actions.len() + && recorded_action_failure(program, execution).is_none() + { return violation( "action_count", format!( "{} expected {} action results, observed {}", program.id, program.actions.len(), - execution.results.len() + execution.results.len(), ), ); } @@ -96,45 +703,35 @@ impl BehaviorOracle { let result = execution .results .iter() - .find(|result| result.step == step) - .ok_or_else(|| OracleViolation { - invariant: "missing_action", - detail: format!("{} omitted step {step}", program.id), + .find(|result| result.step == step && !is_barrier_record(result)) + .ok_or_else(|| { + OracleViolation::new( + "missing_action", + format!("{} omitted step {step}", program.id), + ) + .at_action(&action.operation) })?; - if result.process != program.id - || result.action != action.operation.class().as_str() - { - return violation( - "action_identity", - format!( - "{} step {step} reported process {:?} and action {:?}", - program.id, result.process, result.action - ), - ); - } - if result.path != action.operation.path() { - return violation( - "path_consistency", - format!("{} step {step} reported the wrong path", program.id), - ); - } match action.expected { ExpectedOutcome::Ok if result.outcome != "ok" => { - return violation( + return Err(OracleViolation::new( "unexpected_error", format!("{} step {step}: {:?}", program.id, result.error), - ); + ) + .at_action(&action.operation) + .with_outcome(action.expected, result)); } ExpectedOutcome::Error(expected) if result.outcome != "expected_error" || result.errno != Some(expected) => { - return violation( + return Err(OracleViolation::new( "typed_error", format!( "{} step {step} expected errno {expected}, observed {:?}", program.id, result.errno ), - ); + ) + .at_action(&action.operation) + .with_outcome(action.expected, result)); } ExpectedOutcome::Linearized { error: expected, .. @@ -142,19 +739,17 @@ impl BehaviorOracle { && (result.outcome != "expected_error" || result.errno != Some(expected)) => { - return violation( + return Err(OracleViolation::new( "typed_error", - format!( - "{} step {step} expected success or errno {expected}, observed {:?}", - program.id, result.errno - ), - ); + format!("{} step {step} expected success or errno {expected}, observed {:?}", + program.id, result.errno), + ).at_action(&action.operation).with_outcome(action.expected, result)); } ExpectedOutcome::Exception(expected) if result.outcome != "expected_error" || result.error_type.as_deref() != Some(expected.as_str()) => { - return violation( + return Err(OracleViolation::new( "typed_exception", format!( "{} step {step} expected {}, observed {:?}", @@ -162,13 +757,14 @@ impl BehaviorOracle { expected.as_str(), result.error_type ), - ); + ) + .at_action(&action.operation) + .with_outcome(action.expected, result)); } _ => {} } - if result.outcome == "ok" { - verify_action_payload(&program.id, step, &action.operation, result)?; - } + verify_action_payload(&program.id, step, &action.operation, result) + .map_err(|error| error.at_action(&action.operation))?; if let Some(revision) = result.revision { if matches!( action.operation, @@ -203,7 +799,9 @@ impl BehaviorOracle { } } } + verify_namespace_histories(case, observation, budget, retained)?; Self::verify_linearized_groups(case, observation)?; + check_budget(budget, "oracle completion")?; Ok(()) } @@ -225,11 +823,16 @@ impl BehaviorOracle { else { continue; }; - let result = execution + let Some(result) = execution .results .iter() - .find(|result| result.step == step) - .expect("action presence checked before linearization"); + .find(|result| result.step == step && !is_barrier_record(result)) + else { + return violation( + "namespace_model_evidence", + format!("{} step {step} has no race result", program.id), + ); + }; let entry = groups.entry(group).or_insert((successes, 0, 0)); if entry.0 != successes { return violation( @@ -344,7 +947,10 @@ impl BehaviorOracle { "user output followed process termination", ); } - if !execution.exit_success && !execution.results.is_empty() { + if !execution.exit_success + && !execution.results.is_empty() + && terminal_action_error(execution).is_none() + { return violation( "process_failure", "process failed after emitting action results", @@ -384,6 +990,18 @@ impl BehaviorOracle { ); } } + FailureInjection::StopProcess { process, phase, .. } + if process == &program.id + && matches!( + phase, + ProcessStopPhase::AfterStreamFirstFrame + | ProcessStopPhase::AfterSiblingStreamFirstFrame + ) => + { + if !has("context_ready") || !has("stop_accepted") { + return violation("stop_phase", "stream stop lacks readiness or acceptance"); + } + } FailureInjection::LaunchFailure { process, kind } if process == &program.id => { let expected = match kind { LaunchFailureKind::EmptyCommand => has("spawn_rejected"), @@ -433,9 +1051,11 @@ pub enum StreamIncarnationTrace { impl BehaviorOracle { pub fn verify_blob_publication(trace: &BlobPublicationTrace) -> Result<(), OracleViolation> { - if trace.observed_bytes.len() != trace.declared_length - || digest(&trace.observed_bytes) != trace.declared_digest - { + if !publication_matches( + Some(trace.declared_length), + Some(&trace.declared_digest), + &trace.observed_bytes, + ) { return violation( "torn_publication", "published blob length or digest differs from committed metadata", @@ -490,7 +1110,24 @@ fn verify_action_payload( action: &ActionOp, result: &ActionObservation, ) -> Result<(), OracleViolation> { - let expected_bytes = match action { + let success = result.outcome == "ok"; + if descriptor_first_terminal_succeeded(result) + && !success + && result + .transfer + .as_ref() + .is_none_or(|transfer| !transfer.complete) + { + return Err(OracleViolation::new( + "payload_integrity", + format!("{process} step {step} omitted completed transfer before terminal error"), + ) + .classified( + FailureClass::MissingTransfer, + CausalRole::Action(action.into()), + )); + } + match action { ActionOp::PublishBlob { bytes, .. } | ActionOp::ReadBlob { expected: bytes, .. @@ -498,88 +1135,6466 @@ fn verify_action_payload( | ActionOp::StreamRead { expected: bytes, .. } + | ActionOp::StreamReadWithRetry { + expected: bytes, .. + } + | ActionOp::GatedStreamRead { + expected: bytes, .. + } | ActionOp::StreamReadInto { expected: bytes, .. - } => Some(bytes.as_slice()), - ActionOp::StreamWrite { chunks, .. } => { - let concatenated = chunks.concat(); - if result.length != Some(concatenated.len()) - || result.digest.as_deref() != Some(&digest(&concatenated)) - { - return violation( - "stream_bytes", - format!("{process} step {step} changed stream bytes"), - ); - } - None + } + | ActionOp::DescriptorWrite { bytes, .. } + | ActionOp::DescriptorRead { + expected: bytes, .. + } => { + verify_transferred_payload(process, step, result, std::iter::once(bytes.as_slice()))?; + } + ActionOp::StreamWrite { chunks, .. } + | ActionOp::StreamRoundTrip { chunks, .. } + | ActionOp::GatedStreamWrite { frames: chunks, .. } => { + verify_transferred_payload(process, step, result, chunks.iter().map(Vec::as_slice))?; } ActionOp::Lookup { expected_kind, .. } | ActionOp::AwaitEntry { expected_kind, .. } => { - if result.kind.as_deref() != Some(expected_kind) || result.revision.is_none() { + if success + && (result.kind.as_deref() != Some(expected_kind) || result.revision.is_none()) + { return violation( "namespace_kind", format!("{process} step {step} returned an inconsistent namespace node"), ); } - None } ActionOp::WaitForQuiescent { .. } => { - if result.kind.as_deref() != Some("stream") - || result.revision.is_none() - || result.active != Some(false) + if success + && (result.kind.as_deref() != Some("stream") + || result.revision.is_none() + || result.active != Some(false)) { return violation( "stream_quiescence", format!("{process} step {step} did not observe a quiescent stream node"), ); } - None } - ActionOp::DescriptorWrite { bytes, .. } - | ActionOp::DescriptorRead { - expected: bytes, .. - } => Some(bytes.as_slice()), - ActionOp::MappingExportClose { .. } => None, + ActionOp::MappingExportClose { .. } => {} ActionOp::Rename { .. } | ActionOp::Unlink { .. } => { - if result.revision.is_none() { + if success && result.revision.is_none() { return violation( "namespace_revision", format!("{process} step {step} omitted mutation revision"), ); } - None } - }; - if let Some(expected) = expected_bytes - && (result.length != Some(expected.len()) - || result.digest.as_deref() != Some(&digest(expected))) - { - return violation( - "payload_integrity", - format!("{process} step {step} returned the wrong length or digest"), - ); } Ok(()) } -fn violation(invariant: &'static str, detail: impl Into) -> Result { - Err(OracleViolation { - invariant, - detail: detail.into(), +fn digest(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} + +#[derive(Clone, Copy)] +enum RoutePayload<'a> { + Bytes(&'a [u8]), + Frames(&'a [Vec]), +} + +impl RoutePayload<'_> { + fn checked_len(self) -> Option { + match self { + Self::Bytes(bytes) => Some(bytes.len()), + Self::Frames(frames) => frames + .iter() + .try_fold(0_usize, |length, frame| length.checked_add(frame.len())), + } + } + + fn update_range(self, hasher: &mut Sha256, start: usize, end: usize) -> Option<()> { + if start > end || end > self.checked_len()? { + return None; + } + match self { + Self::Bytes(bytes) => hasher.update(&bytes[start..end]), + Self::Frames(frames) => { + let mut offset = 0_usize; + for frame in frames { + let frame_end = offset.checked_add(frame.len())?; + let overlap_start = start.max(offset); + let overlap_end = end.min(frame_end); + if overlap_start < overlap_end { + hasher.update(&frame[overlap_start - offset..overlap_end - offset]); + } + offset = frame_end; + if offset >= end { + break; + } + } + } + } + Some(()) + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +struct PayloadFingerprint { + length: usize, + digest: [u8; 32], +} + +fn route_payload_fingerprint(payload: RoutePayload<'_>) -> Option { + let length = payload.checked_len()?; + let mut hasher = Sha256::new(); + payload.update_range(&mut hasher, 0, length)?; + Some(PayloadFingerprint { + length, + digest: hasher.finalize().into(), }) } -fn process_failure_is_expected(case: &BehaviorCase, process: &ProcessProgram) -> bool { - match &case.failure { - FailureInjection::None => false, - FailureInjection::StopProcess { - process: target, .. - } => target == &process.id, - FailureInjection::LaunchFailure { - process: target, .. - } => target == &process.id, - FailureInjection::KillNode { logical_node_id } => { - *logical_node_id == process.logical_node_id +fn update_route_payload_range( + hasher: &mut Sha256, + payloads: &[RoutePayload<'_>], + start: usize, + end: usize, +) -> Option<()> { + let mut offset = 0_usize; + for payload in payloads { + let length = payload.checked_len()?; + let payload_end = offset.checked_add(length)?; + let overlap_start = start.max(offset); + let overlap_end = end.min(payload_end); + if overlap_start < overlap_end { + payload.update_range(hasher, overlap_start - offset, overlap_end - offset)?; } - FailureInjection::StopOrchestrator => true, + offset = payload_end; + if offset >= end { + break; + } + } + Some(()) +} + +fn transformed_route_fingerprint( + inputs: &[RoutePayload<'_>], + rotation: usize, +) -> Option { + let length = inputs.iter().try_fold(0_usize, |length, payload| { + length.checked_add(payload.checked_len()?) + })?; + let rotation = if length == 0 { 0 } else { rotation % length }; + let mut hasher = Sha256::new(); + update_route_payload_range(&mut hasher, inputs, rotation, length)?; + update_route_payload_range(&mut hasher, inputs, 0, rotation)?; + Some(PayloadFingerprint { + length, + digest: hasher.finalize().into(), + }) +} + +fn invalid_route_value(route: &str, path: &str, detail: &str) -> OracleViolation { + OracleViolation::new( + "route_expected_value", + format!("route {route} path {path}: {detail}"), + ) + .classified(FailureClass::InvalidEvidence, CausalRole::Case) +} + +fn route_endpoint_payload<'a>( + case: &'a BehaviorCase, + route: &str, + role: &str, + path: &str, + kind: DataKind, + source: bool, +) -> Result, OracleViolation> { + let process = case + .processes + .iter() + .find(|process| process.id == role) + .ok_or_else(|| invalid_route_value(route, path, "endpoint role is absent"))?; + let mut found = None; + for action in &process.actions { + if action.operation.path() != path { + continue; + } + let payload = match (&action.operation, kind, source) { + (ActionOp::PublishBlob { bytes, .. }, DataKind::Blob, true) + | ( + ActionOp::ReadBlob { + expected: bytes, .. + }, + DataKind::Blob, + false, + ) + | ( + ActionOp::StreamRead { + expected: bytes, .. + } + | ActionOp::StreamReadWithRetry { + expected: bytes, .. + } + | ActionOp::GatedStreamRead { + expected: bytes, .. + } + | ActionOp::StreamReadInto { + expected: bytes, .. + }, + DataKind::Stream, + false, + ) => RoutePayload::Bytes(bytes), + ( + ActionOp::StreamWrite { chunks, .. } + | ActionOp::GatedStreamWrite { frames: chunks, .. }, + DataKind::Stream, + true, + ) => RoutePayload::Frames(chunks), + _ => continue, + }; + if found.replace(payload).is_some() { + return Err(invalid_route_value( + route, + path, + "endpoint role has more than one data action", + )); + } + } + found.ok_or_else(|| invalid_route_value(route, path, "endpoint data action is absent")) +} + +/// Derive every relay/join output from the route graph and root payload rather +/// than trusting the reader and writer operands to agree with each other. +fn verify_route_expected_values(case: &BehaviorCase) -> Result<(), OracleViolation> { + for route in &case.routes { + if case.topology == crate::ir::TopologyFamily::RingWalk { + let mut previous = None; + let first_source = route.edges.first().map(|edge| edge.source_role.as_str()); + for (index, edge) in route.edges.iter().enumerate() { + if index > 0 && Some(edge.source_role.as_str()) == first_source { + previous = None; + } + let writer = route_endpoint_payload( + case, + &route.id, + &edge.source_role, + &edge.path, + route.kind, + true, + )?; + let reader = route_endpoint_payload( + case, + &route.id, + &edge.destination_role, + &edge.path, + route.kind, + false, + )?; + let writer = route_payload_fingerprint(writer).ok_or_else(|| { + invalid_route_value(&route.id, &edge.path, "payload length overflow") + })?; + let reader = route_payload_fingerprint(reader).ok_or_else(|| { + invalid_route_value(&route.id, &edge.path, "payload length overflow") + })?; + if previous.is_some_and(|expected| expected != writer) { + return Err(invalid_route_value( + &route.id, + &edge.path, + "relay changed the lap token", + )); + } + if reader != writer { + return Err(invalid_route_value( + &route.id, + &edge.path, + "reader expectation differs from the independently derived token", + )); + } + previous = Some(writer); + } + continue; + } + + let mut completed = vec![false; route.edges.len()]; + let mut pending = route + .edges + .iter() + .map(|edge| edge.source_role.as_str()) + .collect::>(); + let mut root = None; + while !pending.is_empty() { + let Some(role) = pending.iter().copied().find(|role| { + route + .edges + .iter() + .enumerate() + .all(|(index, edge)| edge.destination_role != *role || completed[index]) + }) else { + return Err(invalid_route_value( + &route.id, + "", + "route dependencies are cyclic", + )); + }; + let inputs = route + .edges + .iter() + .enumerate() + .filter(|(_, edge)| edge.destination_role == role) + .map(|(_, edge)| { + route_endpoint_payload( + case, + &route.id, + &edge.source_role, + &edge.path, + route.kind, + true, + ) + }) + .collect::, _>>()?; + for (index, edge) in route.edges.iter().enumerate() { + if edge.source_role != role { + continue; + } + let writer = + route_endpoint_payload(case, &route.id, role, &edge.path, route.kind, true)?; + let reader = route_endpoint_payload( + case, + &route.id, + &edge.destination_role, + &edge.path, + route.kind, + false, + )?; + let writer = route_payload_fingerprint(writer).ok_or_else(|| { + invalid_route_value(&route.id, &edge.path, "payload length overflow") + })?; + let reader = route_payload_fingerprint(reader).ok_or_else(|| { + invalid_route_value(&route.id, &edge.path, "payload length overflow") + })?; + let expected = if inputs.is_empty() { + *root.get_or_insert(writer) + } else { + transformed_route_fingerprint(&inputs, index + 1).ok_or_else(|| { + invalid_route_value(&route.id, &edge.path, "joined payload length overflow") + })? + }; + if writer != expected { + return Err(invalid_route_value( + &route.id, + &edge.path, + "writer payload differs from the independently derived relay output", + )); + } + if reader != expected { + return Err(invalid_route_value( + &route.id, + &edge.path, + "reader expectation differs from the independently derived relay output", + )); + } + completed[index] = true; + } + pending.remove(role); + } + if completed.iter().any(|complete| !complete) { + return Err(invalid_route_value( + &route.id, + "", + "route edge has no derivable source", + )); + } + } + Ok(()) +} + +/// Hash ordered immutable IR chunks, retaining their separate framing evidence. +/// `prefix` limits hashing, not checked total-length accounting. +fn payload_summary<'a>( + chunks: impl Iterator, + prefix: Option, +) -> Option<(usize, String)> { + let mut length = 0_usize; + let mut remaining = prefix.unwrap_or(usize::MAX); + let mut hasher = Sha256::new(); + for chunk in chunks { + length = length.checked_add(chunk.len())?; + let take = remaining.min(chunk.len()); + hasher.update(&chunk[..take]); + remaining -= take; + } + Some((length, format!("{:x}", hasher.finalize()))) +} + +fn verify_transferred_payload<'a>( + process: &str, + step: usize, + result: &ActionObservation, + chunks: impl Iterator + Clone, +) -> Result<(), OracleViolation> { + let mut completed_summary = None; + if let Some(transfer) = &result.transfer { + let (length, expected_digest) = payload_summary(chunks.clone(), Some(transfer.length)) + .ok_or_else(|| { + OracleViolation::new("payload_integrity", "IR payload length overflow") + })?; + if transfer.length > length + || (transfer.complete && transfer.length != length) + || (result.outcome == "ok" && !transfer.complete) + || transfer.digest != expected_digest + { + return violation( + "payload_integrity", + format!("{process} step {step} changed the observed transfer or prefix"), + ); + } + if transfer.complete { + completed_summary = Some((length, expected_digest)); + } + } + // Terminal-error records may contain only a proven prefix. Legacy aggregate + // fields still describe a completed transfer and must not contradict it. + if result.outcome == "ok" || result.length.is_some() || result.digest.is_some() { + let (length, expected_digest) = completed_summary + .or_else(|| payload_summary(chunks, None)) + .ok_or_else(|| { + OracleViolation::new("payload_integrity", "IR payload length overflow") + })?; + if result.length != Some(length) || result.digest.as_deref() != Some(&expected_digest) { + return violation( + "payload_integrity", + format!("{process} step {step} returned the wrong length or digest"), + ); + } + } + Ok(()) +} + +fn descriptor_first_terminal_succeeded(result: &ActionObservation) -> bool { + let terminal_results = match &result.descriptor { + Some(DescriptorObservation::Write { + terminal_results, .. + }) + | Some(DescriptorObservation::Read { + terminal_results, .. + }) => terminal_results, + _ => return false, + }; + matches!(terminal_results.first(), Some(DescriptorTerminalResult::Ok)) +} + +fn descriptor_completion( + action: &ActionOp, + result: &ActionObservation, +) -> Result { + let success = result.outcome == "ok" && result.errno.is_none(); + if let ActionOp::DescriptorWrite { + method, + finish: DescriptorFinish::Drop, + .. + } = action + { + if !success { + return Ok(false); + } + if !matches!(&result.descriptor, + Some(DescriptorObservation::Write { + method: actual_method, finish: DescriptorFinish::Drop, + terminal_results, dropped: true, reservation_released: true, + }) if actual_method == method && terminal_results.is_empty()) + { + return Err(OracleViolation::new( + "namespace_model_evidence", + "writer drop did not prove absence and exclusive reservation reuse", + ) + .classified( + FailureClass::ReservationNotReleased, + CausalRole::Action(action.into()), + )); + } + return Ok(true); + } + if success { + return Ok(true); + } + if !namespace_errno(result, libc::EBADF) { + return Ok(false); + } + let (finish, terminals) = match (action, &result.descriptor) { + ( + ActionOp::DescriptorWrite { method, finish, .. }, + Some(DescriptorObservation::Write { + method: observed_method, + finish: observed_finish, + terminal_results, + .. + }), + ) if method == observed_method && finish == observed_finish => (finish, terminal_results), + ( + ActionOp::DescriptorRead { method, finish, .. }, + Some(DescriptorObservation::Read { + method: observed_method, + finish: observed_finish, + terminal_results, + }), + ) if method == observed_method && finish == observed_finish => (finish, terminal_results), + _ => return Ok(false), + }; + if !matches!( + finish, + DescriptorFinish::CloseTwice | DescriptorFinish::AbortTwice + ) { + return Ok(false); + } + if !matches!(terminals.as_slice(), [ + DescriptorTerminalResult::Ok, + DescriptorTerminalResult::Error { errno: Some(libc::EBADF), error_type } + ] if error_type == "OSError") + { + return Err(OracleViolation::new( + "namespace_model_evidence", + "repeated descriptor finish omitted first-terminal success and second-terminal EBADF", + ) + .classified( + FailureClass::InvalidEvidence, + CausalRole::Action(action.into()), + )); + } + Ok(true) +} + +fn publication_matches(length: Option, content_digest: Option<&str>, bytes: &[u8]) -> bool { + length == Some(bytes.len()) && content_digest == Some(digest(bytes).as_str()) +} + +// Publications separate lookup/open, exclusive reservation, and commit. +// Streams separate pending creation, matched attachment, byte/EOF progress, +// completion, and asynchronous namespace teardown. Every creation/mutation +// consumes one authority revision slot; attached identities constrain that +// exact slot. Names may be replaced independently of old handles. +// Neither execution-vector order nor collector receipt order is causal. +#[derive(Clone, Debug)] +enum NamespaceOperation<'a> { + Publish { + path: String, + bytes: &'a [u8], + commit: bool, + exclusive: bool, + require_existing: bool, + }, + Read { + path: String, + expected: &'a [u8], + offset: usize, + whole: bool, + }, + Lookup { + path: String, + }, + MutationOrigin { + path: String, + revision: u64, + }, + Rename { + source: String, + destination: String, + replace: bool, + }, + Unlink { + path: String, + }, + StreamOpen { + path: String, + incarnation: Option, + source: bool, + replace: bool, + chunks: Option<&'a [Vec]>, + gated: bool, + }, + StreamAttach { + path: String, + incarnation: u64, + roundtrip: bool, + }, + StreamTransfer { + path: String, + incarnation: u64, + }, + StreamFirstFrame { + path: String, + incarnation: u64, + source: bool, + }, + StreamComplete { + path: String, + incarnation: u64, + source: bool, + expected: Option<&'a [u8]>, + frame_sources: Vec<&'a [Vec]>, + frames_observed: bool, + }, + StreamPendingFailure { + path: String, + }, + StreamStop { + path: String, + incarnation: Option, + source: bool, + replace: bool, + roundtrip: bool, + chunks: Option<&'a [Vec]>, + gated: bool, + first_frame: bool, + eof: bool, + frame_sources: Vec<&'a [Vec]>, + stop_gates: Vec<(String, u64, Option)>, + }, + StreamProbe { + path: String, + }, + GatePublish { + path: String, + }, + GateLookup { + path: String, + }, + Ignore, +} + +impl NamespaceOperation<'_> { + fn path(&self) -> Option<&str> { + match self { + Self::Publish { path, .. } + | Self::Read { path, .. } + | Self::Lookup { path } + | Self::MutationOrigin { path, .. } + | Self::Unlink { path } + | Self::StreamOpen { path, .. } + | Self::StreamAttach { path, .. } + | Self::StreamTransfer { path, .. } + | Self::StreamFirstFrame { path, .. } + | Self::StreamComplete { path, .. } + | Self::StreamPendingFailure { path } + | Self::StreamStop { path, .. } + | Self::StreamProbe { path } + | Self::GatePublish { path } + | Self::GateLookup { path } => Some(path), + Self::Rename { source, .. } => Some(source), + Self::Ignore => None, + } + } + + fn writes(&self) -> bool { + matches!( + self, + Self::Publish { .. } + | Self::Rename { .. } + | Self::Unlink { .. } + | Self::StreamOpen { source: true, .. } + | Self::StreamFirstFrame { source: true, .. } + | Self::StreamComplete { source: true, .. } + | Self::GatePublish { .. } + ) } } + +struct NamespaceAction<'a> { + step: usize, + // Preparations and transfers share an endpoint owner, not a scheduler slot. + stream_owner: Option, + preparation: bool, + operation: NamespaceOperation<'a>, + role: SemanticAction, + result: Option<&'a ActionObservation>, + denied: bool, + publication_finished: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +struct NamespaceBinding<'a> { + bytes: &'a [u8], + revision: usize, +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +struct NamespaceHistory<'a> { + positions: Vec, + publications: BTreeSet, + publication_lookups: BTreeSet, + reservations: BTreeMap, + streams: BTreeMap, + active_streams: BTreeMap, + sessions: BTreeMap>, + open_streams: BTreeMap, // endpoint owner -> incarnation slot + entries: BTreeMap>, + initial_revision: u64, + // Each slot is a committed mutation, in authority order. A publication + // without a receipt has an unknown revision until a lookup constrains it. + revisions: Vec>, +} + +// Namespace entries and attached handles are deliberately separate. A name +// may be replaced while its old participants still have unfinished actions. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +struct StreamSession<'a> { + path: String, + source: Option, + sink: Option, + chunks: Option<&'a [Vec]>, + source_attached: bool, + first_frame_permitted: bool, + first_frame: bool, + eof: bool, + aborted: bool, +} + +impl StreamSession<'_> { + fn matched(&self) -> bool { + self.source.is_some() && self.sink.is_some() + } +} + +impl NamespaceHistory<'_> { + fn constrain_revision(&mut self, slot: usize, revision: u64) -> bool { + if self.revisions[slot].is_some_and(|known| known != revision) { + return false; + } + self.revisions[slot] = Some(revision); + self.revisions_fit() + } + + fn revisions_fit(&self) -> bool { + let mut previous = self.initial_revision; + let mut distance = 0; + for known in &self.revisions { + distance += 1; + if let Some(known) = known { + if known.checked_sub(previous).is_none_or(|gap| gap < distance) { + return false; + } + previous = *known; + distance = 0; + } + } + previous.checked_add(distance).is_some() + } + + fn commit_revision(&mut self, revision: Option) -> Option { + let slot = self.revisions.len(); + self.revisions.push(None); + if let Some(revision) = revision + && !self.constrain_revision(slot, revision) + { + return None; + } + self.revisions_fit().then_some(slot) + } +} + +fn namespace_path(program: &ProcessProgram, path: &str) -> String { + if let Some(suffix) = path.strip_prefix("/runs/self") + && (suffix.is_empty() || suffix.starts_with('/')) + { + format!("/runs/{}{suffix}", program.access.execution_id) + } else { + path.to_owned() + } +} + +fn namespace_prefix_allows(prefixes: &[String], path: &str) -> bool { + prefixes.iter().any(|prefix| { + path == prefix + || path + .strip_prefix(prefix) + .is_some_and(|suffix| suffix.starts_with('/')) + }) +} + +fn namespace_errno(result: &ActionObservation, errno: i32) -> bool { + result.outcome == "expected_error" && result.errno == Some(errno) +} +fn namespace_binding_error(result: &ActionObservation, errno: i32, raw: bool) -> bool { + if raw { + return namespace_errno(result, errno); + } + let exception = match errno { + libc::ENXIO | libc::ECONNRESET | libc::EPIPE => Some("StreamError"), + libc::EEXIST | libc::EIO => Some("SessionError"), + _ => None, + }; + match exception { + Some(exception) => { + result.outcome == "expected_error" + && result.errno.is_none() + && result.error_type.as_deref() == Some(exception) + } + None => namespace_errno(result, errno), + } +} + +fn initial_fixture_revision( + case: &BehaviorCase, + observation: &CaseObservation, +) -> Result { + let mut revisions = BTreeMap::new(); + for execution in &observation.executions { + for result in &execution.results { + if !case.read_only_fixture_paths.contains(&result.path) || result.outcome != "ok" { + continue; + } + if let Some(revision) = result.revision { + if revision == 0 + || revisions + .insert(&result.path, revision) + .is_some_and(|known| known != revision) + { + return violation( + "namespace_linearizability", + format!("immutable fixture {} changed revision", result.path), + ); + } + } + } + } + Ok(revisions.values().copied().max().unwrap_or(0)) +} + +type StreamFrameSources<'a> = BTreeMap<(&'a str, usize), Vec<&'a [Vec]>>; + +struct StreamFrameEvidence<'a> { + program: &'a ProcessProgram, + step: usize, + incarnation: u64, + sent: Vec<(u64, &'a str)>, + received: Vec<(u64, &'a str)>, + complete: bool, +} + +/// Hash each immutable writer frame once, outside the bounded history search. +/// Candidate membership is subsequently constrained by the source actually +/// attached in that history; equal aggregate bytes never equate different +/// logical frame boundaries. Cross-process collector order is irrelevant. +fn verify_stream_frames<'a>( + case: &'a BehaviorCase, + observation: &'a CaseObservation, + budget: Option<&Budget>, +) -> Result, OracleViolation> { + let mut evidence = Vec::new(); + let mut sources = Vec::new(); + for program in &case.processes { + let execution = observation + .executions + .iter() + .find(|execution| execution.process == program.id) + .expect("execution identities checked"); + for record in &execution.results { + if matches!(record.barrier, Some(BarrierObservation::StreamFrame { .. })) + && program + .actions + .get(record.step) + .is_none_or(|action| stream_parameters(&action.operation).is_none()) + { + return Err(OracleViolation::new( + "stream_frames", + "frame evidence has no stream action", + ) + .classified(FailureClass::InvalidEvidence, CausalRole::Namespace)); + } + } + for (step, action) in program.actions.iter().enumerate() { + let Some((source, _, chunks)) = stream_parameters(&action.operation) else { + continue; + }; + check_budget(budget, "logical stream frame evidence")?; + let records = || { + execution + .results + .iter() + .filter(|record| record.step == step) + }; + if !records().any(|_| true) { + if let Some(chunks) = chunks + && matches!(&case.failure, FailureInjection::StopProcess { process, .. } + if process == &program.id) + && execution.terminal + && !execution.exit_success + && execution + .lifecycle + .iter() + .any(|event| event == "context_ready") + { + // Native rendezvous may attach the reader before the + // stopped writer can emit its Python opened milestone. + // This is only a candidate: the history search must still + // attach this exact IR source to the observed incarnation. + sources.push(( + namespace_path(program, action.operation.path()), + None, + chunks, + chunks + .iter() + .map(|chunk| (chunk.len() as u64, digest(chunk))) + .collect(), + )); + } + continue; + } + let Some(incarnation) = + stream_incarnation_evidence(execution, step, &program.id, &action.operation)? + else { + continue; + }; + let result = records().find(|record| !is_barrier_record(record)); + let complete = result.is_some_and(|result| { + result.outcome == "ok" + || result + .transfer + .as_ref() + .is_some_and(|transfer| transfer.complete) + }) || records() + .any(|record| matches!(record.barrier, Some(BarrierObservation::StreamEof { .. }))); + let mut frames = StreamFrameEvidence { + program, + step, + incarnation, + sent: Vec::new(), + received: Vec::new(), + complete, + }; + for record in records() { + if let Some(BarrierObservation::StreamFrame { length, digest, .. }) = + &record.barrier + { + if record.action == "stream_write" { + frames.sent.push((*length, digest.as_str())); + } else { + frames.received.push((*length, digest.as_str())); + } + } + } + let progress = + if source && !matches!(action.operation, ActionOp::StreamRoundTrip { .. }) { + &frames.sent + } else { + &frames.received + }; + let length = progress + .iter() + .try_fold(0_u64, |total, frame| total.checked_add(frame.0)) + .ok_or_else(|| { + OracleViolation::new("stream_frames", "logical frame lengths overflow") + .classified( + FailureClass::FrameContentMismatch, + CausalRole::Action((&action.operation).into()), + ) + })?; + if result + .and_then(|result| result.transfer.as_ref()) + .is_some_and(|transfer| u64::try_from(transfer.length).ok() != Some(length)) + { + return Err(OracleViolation::new( + "stream_frames", + "logical frames disagree with transferred bytes", + ) + .classified( + FailureClass::FrameCountMismatch, + CausalRole::Action((&action.operation).into()), + )); + } + if let Some(chunks) = chunks { + let expected = chunks + .iter() + .map(|chunk| (chunk.len() as u64, digest(chunk))) + .collect::>(); + check_logical_frames(&frames.sent, &expected, complete).map_err(|class| { + OracleViolation::new( + "stream_frames", + format!("{} step {step}: sent frames differ from IR", program.id), + ) + .classified(class, CausalRole::Action((&action.operation).into())) + })?; + sources.push(( + namespace_path(program, action.operation.path()), + Some(incarnation), + chunks, + expected, + )); + } + evidence.push(frames); + } + } + let mut constraints = BTreeMap::new(); + for frames in evidence { + let action = &frames.program.actions[frames.step].operation; + let (source, _, own_chunks) = stream_parameters(action).expect("stream evidence"); + if source && !matches!(action, ActionOp::StreamRoundTrip { .. }) { + constraints.insert( + (frames.program.id.as_str(), frames.step), + vec![own_chunks.expect("writer chunks")], + ); + continue; + } + let path = namespace_path(frames.program, action.path()); + let mut same_path = false; + let mut same_incarnation = false; + let mut frame_failure = None; + let mut matching = Vec::new(); + for (source_path, incarnation, chunks, expected) in &sources { + if *source_path != path { + continue; + } + same_path = true; + if incarnation.is_some_and(|incarnation| incarnation != frames.incarnation) + || own_chunks.is_some_and(|own| !std::ptr::eq(own, *chunks)) + { + continue; + } + same_incarnation = true; + match check_logical_frames(&frames.received, expected, frames.complete) { + Ok(()) => matching.push(*chunks), + Err(class) => frame_failure = Some(class), + } + } + if matching.is_empty() { + let class = if !same_path { + FailureClass::MissingStreamSource + } else if !same_incarnation { + FailureClass::ConflictingIncarnation + } else { + frame_failure.unwrap_or(FailureClass::FrameContentMismatch) + }; + return Err(OracleViolation::new( + "namespace_linearizability", + format!( + "{} step {}: no matching logical frame source", + frames.program.id, frames.step + ), + ) + .classified(class, CausalRole::Action(action.into()))); + } + constraints.insert((frames.program.id.as_str(), frames.step), matching); + } + Ok(constraints) +} + +fn check_logical_frames( + observed: &[(u64, &str)], + expected: &[(u64, String)], + complete: bool, +) -> Result<(), FailureClass> { + if observed.len() > expected.len() || (complete && observed.len() != expected.len()) { + return Err(FailureClass::FrameCountMismatch); + } + if observed.iter().zip(expected).any( + |((length, digest), (expected_length, expected_digest))| { + length != expected_length || *digest != expected_digest.as_str() + }, + ) { + return Err(FailureClass::FrameContentMismatch); + } + Ok(()) +} + +fn verify_namespace_histories( + case: &BehaviorCase, + observation: &CaseObservation, + budget: Option<&Budget>, + retained: Option<&BTreeMap>, +) -> Result<(), OracleViolation> { + let frame_sources = verify_stream_frames(case, observation, budget)?; + let mut paths = BTreeSet::new(); + for program in &case.processes { + for action in &program.actions { + let path = namespace_path(program, action.operation.path()); + match &action.operation { + ActionOp::PublishBlob { .. } + | ActionOp::DescriptorWrite { .. } + | ActionOp::Unlink { .. } => { + paths.insert(path); + } + ActionOp::Rename { destination, .. } => { + paths.insert(path); + paths.insert(namespace_path(program, destination)); + } + ActionOp::StreamWrite { .. } + | ActionOp::StreamRoundTrip { .. } + | ActionOp::StreamRead { .. } + | ActionOp::StreamReadWithRetry { .. } + | ActionOp::StreamReadInto { .. } => { + paths.insert(path); + } + ActionOp::WaitForQuiescent { .. } + if !case.read_only_fixture_paths.contains(&path) => + { + paths.insert(path); + } + ActionOp::GatedStreamWrite { release_path, .. } => { + paths.insert(path); + paths.insert(namespace_path(program, release_path)); + } + ActionOp::GatedStreamRead { observed_path, .. } => { + paths.insert(path); + paths.insert(namespace_path(program, observed_path)); + } + ActionOp::ReadBlob { .. } + | ActionOp::DescriptorRead { .. } + | ActionOp::Lookup { .. } + | ActionOp::AwaitEntry { .. } + | ActionOp::WaitForQuiescent { .. } + | ActionOp::MappingExportClose { .. } => {} + } + // A read alone cannot turn an initially absent attempt-owned name + // into a fixture. Model its absence even when no writer was planned. + let path = namespace_path(program, action.operation.path()); + if !case.read_only_fixture_paths.contains(&path) + && (path.starts_with("/cases/") || path.starts_with("/runs/")) + && matches!( + action.operation, + ActionOp::ReadBlob { .. } + | ActionOp::DescriptorRead { .. } + | ActionOp::Lookup { .. } + | ActionOp::AwaitEntry { .. } + ) + { + paths.insert(path); + } + // A linearized expectation on a declared read-only fixture path + // races nothing (validate forbids mutating it); its outcome is + // judged by the legacy success-count check on declared facts. + if matches!(action.expected, ExpectedOutcome::Linearized { .. }) + && !case + .read_only_fixture_paths + .contains(&namespace_path(program, action.operation.path())) + { + paths.insert(namespace_path(program, action.operation.path())); + } + } + } + // Declared read-only fixtures cannot be mutated (case.validate enforces + // this), so their IR kind/content assertions remain initial facts checked + // by the action oracle. They do not participate in namespace races. + // In particular a read-only snapshot probe must not recursively require + // another snapshot merely to validate its own observations. + // Hints order constructive attempts only, never prune the frontier. + // A path can be replaced or renamed: its smallest observed revision is + // not necessarily the receipt of the currently enabled mutation. + let mut revision_hints = BTreeMap::::new(); + for execution in &observation.executions { + let program = case + .processes + .iter() + .find(|program| program.id == execution.process) + .expect("execution identity checked before namespace verification"); + for result in &execution.results { + if result.outcome == "ok" { + if let Some(revision) = result.revision { + revision_hints + .entry(namespace_path(program, &result.path)) + .and_modify(|known| *known = (*known).min(revision)) + .or_insert(revision); + } + } + } + } + if let Some(retained) = retained { + for (path, (revision, _)) in retained { + revision_hints + .entry(path.clone()) + .and_modify(|known| *known = (*known).min(*revision)) + .or_insert(*revision); + } + } + let mut programs = Vec::with_capacity(case.processes.len()); + let mut dependencies = Vec::with_capacity(case.processes.len()); + for program in &case.processes { + check_budget(budget, "namespace evidence preparation")?; + let execution = observation + .executions + .iter() + .find(|execution| execution.process == program.id) + .expect("execution presence checked before namespace verification"); + let stopped = matches!(&case.failure, FailureInjection::StopProcess { process, .. } if process == &program.id) + && !execution.exit_success + && execution.terminal + && execution + .lifecycle + .iter() + .any(|event| event == "context_ready"); + // A launch failure produces no action results: nothing executed, so + // the process contributes no namespace actions at all. Requiring + // per-step evidence here would conflate "never ran" with + // "interrupted mid-action". + if execution.results.is_empty() && !stopped { + programs.push(Vec::new()); + dependencies.push(Vec::new()); + continue; + } + let mut actions = Vec::with_capacity(program.actions.len()); + for (step, action) in program.actions.iter().enumerate() { + let path = namespace_path(program, action.operation.path()); + let relevant = paths.contains(&path); + if stopped + && !execution + .results + .iter() + .any(|record| record.step == step && !is_barrier_record(record)) + { + if let Some(stopped_action) = namespace_stopped_stream( + case, + observation, + program, + execution, + step, + &action.operation, + &frame_sources, + )? { + if let ActionOp::GatedStreamRead { observed_path, .. } = &action.operation + && let Some(record) = execution.results.iter().find(|record| { + record.step == step + && matches!( + record.barrier, + Some(BarrierObservation::StreamFirstFrame { .. }) + ) + }) + && let Some(BarrierObservation::StreamFirstFrame { incarnation }) = + record.barrier + { + // The generated reader commits its marker before this + // milestone. A stop cannot erase that namespace effect. + let marker = namespace_path(program, observed_path); + for operation in [ + NamespaceOperation::StreamOpen { + path: path.clone(), + incarnation: Some(incarnation), + source: false, + replace: false, + chunks: None, + gated: false, + }, + NamespaceOperation::StreamAttach { + path: path.clone(), + incarnation, + roundtrip: false, + }, + NamespaceOperation::StreamFirstFrame { + path: path.clone(), + incarnation, + source: false, + }, + NamespaceOperation::GatePublish { path: marker }, + ] { + let prefixes = if operation.writes() { + &program.access.write_prefixes + } else { + &program.access.read_prefixes + }; + actions.push(NamespaceAction { + step, + stream_owner: None, + preparation: false, + denied: operation + .path() + .is_some_and(|path| !namespace_prefix_allows(prefixes, path)), + operation, + role: (&action.operation).into(), + result: Some(record), + publication_finished: true, + }); + } + } + actions.push(stopped_action); + } + // Only the first uncompleted transfer can have started. + // Route preparations for later steps are added independently. + break; + } + let operation = match &action.operation { + ActionOp::PublishBlob { bytes, .. } => NamespaceOperation::Publish { + path, + bytes, + commit: true, + exclusive: false, + require_existing: false, + }, + ActionOp::DescriptorWrite { + flags, + length, + bytes, + finish, + .. + } if relevant => { + let exclusive = + *flags == (libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL | libc::O_TRUNC); + let require_existing = *flags == (libc::O_WRONLY | libc::O_TRUNC); + if (!exclusive && !require_existing) || *length != Some(bytes.len() as u64) { + return violation( + "namespace_model_evidence", + format!( + "{} step {step}: descriptor publication needs canonical flags/allocation", + program.id, + ), + ); + } + NamespaceOperation::Publish { + path, + bytes, + commit: matches!( + finish, + DescriptorFinish::Close | DescriptorFinish::CloseTwice + ), + exclusive, + require_existing, + } + } + ActionOp::ReadBlob { expected, .. } if relevant => NamespaceOperation::Read { + path, + expected, + offset: 0, + whole: true, + }, + ActionOp::DescriptorRead { + flags, + expected, + method, + offset, + .. + } if relevant => { + if *flags != libc::O_RDONLY { + return violation( + "namespace_model_evidence", + format!( + "{} step {step}: namespace descriptor read needs modeled flags", + program.id, + ), + ); + } + NamespaceOperation::Read { + path, + expected, + offset: if *method == DescriptorReadMethod::Mapping { + usize::try_from(*offset).map_err(|_| { + OracleViolation::new( + "namespace_model_evidence", + "descriptor mapping offset exceeds model address space", + ) + .at_action(&action.operation) + })? + } else { + 0 + }, + whole: false, + } + } + ActionOp::Lookup { .. } | ActionOp::AwaitEntry { .. } if relevant => { + NamespaceOperation::Lookup { path } + } + ActionOp::Rename { + destination, + replace, + .. + } => NamespaceOperation::Rename { + source: path, + destination: namespace_path(program, destination), + replace: *replace, + }, + ActionOp::Unlink { .. } => NamespaceOperation::Unlink { path }, + ActionOp::StreamWrite { + replace, chunks, .. + } => NamespaceOperation::StreamOpen { + path, + incarnation: stream_incarnation_evidence( + execution, + step, + &program.id, + &action.operation, + )?, + source: true, + replace: *replace, + chunks: Some(chunks), + gated: false, + }, + ActionOp::StreamRoundTrip { chunks, .. } => NamespaceOperation::StreamOpen { + path, + incarnation: stream_incarnation_evidence( + execution, + step, + &program.id, + &action.operation, + )?, + source: true, + replace: false, + chunks: Some(chunks), + gated: false, + }, + ActionOp::GatedStreamWrite { + frames, replace, .. + } => NamespaceOperation::StreamOpen { + path, + incarnation: stream_incarnation_evidence( + execution, + step, + &program.id, + &action.operation, + )?, + source: true, + replace: *replace, + chunks: Some(frames), + gated: true, + }, + ActionOp::StreamRead { .. } + | ActionOp::StreamReadWithRetry { .. } + | ActionOp::GatedStreamRead { .. } + | ActionOp::StreamReadInto { .. } => NamespaceOperation::StreamOpen { + path, + incarnation: stream_incarnation_evidence( + execution, + step, + &program.id, + &action.operation, + )?, + source: false, + replace: false, + chunks: None, + gated: false, + }, + ActionOp::WaitForQuiescent { .. } if relevant => { + NamespaceOperation::StreamProbe { path } + } + ActionOp::MappingExportClose { .. } if relevant => { + return violation( + "namespace_model_evidence", + format!( + "{} step {step}: mapped descriptor namespace effects are not observed", + program.id, + ), + ); + } + ActionOp::ReadBlob { .. } + | ActionOp::DescriptorRead { .. } + | ActionOp::DescriptorWrite { .. } + | ActionOp::Lookup { .. } + | ActionOp::AwaitEntry { .. } + | ActionOp::WaitForQuiescent { .. } + | ActionOp::MappingExportClose { .. } => NamespaceOperation::Ignore, + }; + let Some(result) = execution + .results + .iter() + .find(|result| result.step == step && !is_barrier_record(result)) + else { + if operation.path().is_some() { + return violation( + "namespace_model_evidence", + format!( + "{} step {step}: interrupted namespace action has no commit/reservation evidence", + program.id, + ), + ); + } + continue; + }; + let mut mutation_seen = false; + for record in execution + .results + .iter() + .filter(|record| record.step == step) + { + let Some(BarrierObservation::MutationApplied { + from_revision, + to_revision, + }) = &record.barrier + else { + continue; + }; + let receipt = match &operation { + NamespaceOperation::Rename { .. } | NamespaceOperation::Unlink { .. } => { + result.revision + } + NamespaceOperation::StreamOpen { + source: true, + replace: true, + incarnation, + .. + } => *incarnation, + _ => None, + }; + if mutation_seen + || result.outcome != "ok" + || result.errno.is_some() + || !is_barrier_record(record) + || record.process != program.id + || record.path != action.operation.path() + || record.action != action.operation.class().as_str() + || receipt.is_none() + || *to_revision != receipt + || execution + .results + .iter() + .position(|seen| std::ptr::eq(seen, record)) + >= execution + .results + .iter() + .position(|seen| std::ptr::eq(seen, result)) + { + return Err(OracleViolation::new( + "namespace_model_evidence", + format!("{} step {step}: inconsistent mutation receipt", program.id), + ) + .classified( + FailureClass::InvalidEvidence, + CausalRole::Action((&action.operation).into()), + )); + } + mutation_seen = true; + if let Some(revision) = from_revision { + let path = namespace_path(program, action.operation.path()); + actions.push(NamespaceAction { + step, + stream_owner: None, + preparation: false, + denied: !namespace_prefix_allows(&program.access.read_prefixes, &path), + operation: NamespaceOperation::MutationOrigin { + path, + revision: *revision, + }, + role: (&action.operation).into(), + result: None, + publication_finished: true, + }); + } + } + let publication_finished = descriptor_completion(&action.operation, result) + .map_err(|error| error.at_action(&action.operation))?; + let prefixes = if operation.writes() { + &program.access.write_prefixes + } else { + &program.access.read_prefixes + }; + let denied = operation + .path() + .is_some_and(|path| !namespace_prefix_allows(prefixes, path)) + || matches!(&operation, NamespaceOperation::Rename { destination, .. } + if !namespace_prefix_allows(prefixes, destination)); + if !denied { + for path in operation.path().into_iter().chain(match &operation { + NamespaceOperation::Rename { destination, .. } => Some(destination.as_str()), + _ => None, + }) { + // Attempt-owned paths are cleaned before execution. Fixture + // names alone do not prove their initial kind or contents. + if !case.read_only_fixture_paths.contains(path) + && !(path.starts_with("/cases/") || path.starts_with("/runs/")) + { + return violation( + "namespace_model_evidence", + format!( + "initial namespace snapshot missing for {path}: need kind, revision, active, blob length and digest", + ), + ); + } + } + } + let stream = match &operation { + NamespaceOperation::StreamOpen { + path, + incarnation, + source, + .. + } if !denied + && (incarnation.is_some() || namespace_errno(result, libc::ESTALE)) => + { + Some((path.clone(), *incarnation, *source)) + } + _ => None, + }; + actions.push(NamespaceAction { + step, + stream_owner: None, + preparation: false, + role: (&action.operation).into(), + operation, + result: Some(result), + denied, + publication_finished, + }); + if let Some((path, incarnation, source)) = stream { + let mut push = |operation| { + actions.push(NamespaceAction { + step, + stream_owner: None, + preparation: false, + role: (&action.operation).into(), + operation, + result: Some(result), + denied: false, + publication_finished: true, + }) + }; + let Some(incarnation) = incarnation else { + push(NamespaceOperation::StreamPendingFailure { path }); + continue; + }; + push(NamespaceOperation::StreamAttach { + path: path.clone(), + incarnation, + roundtrip: matches!(action.operation, ActionOp::StreamRoundTrip { .. }), + }); + match &action.operation { + ActionOp::GatedStreamWrite { release_path, .. } + if result.outcome == "ok" + || execution.results.iter().any(|record| { + record.step == step + && matches!( + &record.barrier, + Some(BarrierObservation::ReleaseObserved { .. }) + ) + }) => + { + push(NamespaceOperation::StreamFirstFrame { + path: path.clone(), + incarnation, + source, + }); + let release_path = namespace_path(program, release_path); + if execution.results.iter().any(|record| record.step == step + && matches!(&record.barrier, Some(BarrierObservation::ReleaseObserved { path: observed }) + if namespace_path(program, observed) != release_path)) + { + return violation("namespace_model_evidence", "gated release milestone names the wrong path"); + } + if !namespace_prefix_allows(&program.access.read_prefixes, &release_path) { + return violation( + "namespace_model_evidence", + "gated release lacks read authority", + ); + } + push(NamespaceOperation::GateLookup { path: release_path }); + } + ActionOp::GatedStreamRead { observed_path, .. } + if result.outcome == "ok" + || execution.results.iter().any(|record| { + record.step == step + && matches!( + &record.barrier, + Some(BarrierObservation::StreamFirstFrame { .. }) + ) + }) => + { + push(NamespaceOperation::StreamFirstFrame { + path: path.clone(), + incarnation, + source, + }); + let observed_path = namespace_path(program, observed_path); + if !namespace_prefix_allows(&program.access.write_prefixes, &observed_path) + { + return violation( + "namespace_model_evidence", + "gated publication lacks write authority", + ); + } + push(NamespaceOperation::GatePublish { + path: observed_path, + }); + } + _ => {} + } + let expected = match &action.operation { + ActionOp::StreamRead { expected, .. } + | ActionOp::StreamReadWithRetry { expected, .. } + | ActionOp::GatedStreamRead { expected, .. } + | ActionOp::StreamReadInto { expected, .. } => Some(expected.as_slice()), + _ => None, + }; + push(NamespaceOperation::StreamComplete { + path, + incarnation, + source, + expected, + frame_sources: frame_sources + .get(&(program.id.as_str(), step)) + .cloned() + .unwrap_or_default(), + frames_observed: execution.results.iter().any(|record| { + record.step == step + && matches!( + record.barrier, + Some(BarrierObservation::StreamFrame { .. }) + ) + }), + }); + } + } + programs.push(actions); + dependencies.push( + program + .depends_on + .iter() + .map(|dependency| { + case.processes + .iter() + .position(|candidate| &candidate.id == dependency) + .expect("dependencies validated before namespace verification") + }) + .collect::>(), + ); + } + prepare_namespace_routes( + case, + observation, + &mut programs, + &mut dependencies, + &frame_sources, + )?; + let initial = NamespaceHistory { + positions: vec![0; programs.len()], + publications: BTreeSet::new(), + publication_lookups: BTreeSet::new(), + reservations: BTreeMap::new(), + streams: BTreeMap::new(), + active_streams: BTreeMap::new(), + open_streams: BTreeMap::new(), + sessions: BTreeMap::new(), + entries: BTreeMap::new(), + initial_revision: initial_fixture_revision(case, observation)?, + revisions: Vec::new(), + }; + // Greedy constructive replay: on success it IS a legal history (sound + // acceptance without any interleaving search). On failure the full + // frontier exploration below remains the deciding authority. + match verify_namespace_constructively( + &initial, + &programs, + &dependencies, + &revision_hints, + budget, + retained, + ) { + Ok(()) => return Ok(()), + Err(error) if error.invariant == "execution_budget" => return Err(error), + Err(_) => {} + } + // Depth-first exploration retains each discovered legal state once. The + // visited and queued sets together are the bounded legal-state frontier; + // exceeding the bound is a model explosion, never permission to select a + // convenient winner. + let mut visited = BTreeSet::new(); + let mut queued = BTreeSet::from([initial.clone()]); + let mut stack = Vec::from([initial]); + let mut path_keys = BTreeMap::<&str, u64>::new(); + for program in &programs { + for action in program { + let observed_value = action + .result + .and_then(|result| result.revision.or(result.incarnation)) + .or_else(|| match action.operation { + NamespaceOperation::MutationOrigin { revision, .. } => Some(revision), + _ => None, + }) + .filter(|value| *value != 0); + if let (Some(path), Some(value)) = (action.operation.path(), observed_value) { + path_keys + .entry(path) + .and_modify(|known| *known = (*known).min(value)) + .or_insert(value); + } + } + } + let mut dead_end_role: Option<(usize, SemanticAction)> = None; + let mut retained_conflict = false; + while let Some(mut state) = stack.pop() { + queued.remove(&state); + check_budget(budget, "namespace interleaving search")?; + // Invisible, unrelated operations commute; collapse them without + // charging the race bound. + loop { + let mut advanced = false; + for process in 0..programs.len() { + if dependencies[process] + .iter() + .any(|dependency| state.positions[*dependency] < programs[*dependency].len()) + { + continue; + } + while programs[process] + .get(state.positions[process]) + .is_some_and(|action| matches!(action.operation, NamespaceOperation::Ignore)) + { + state.positions[process] += 1; + advanced = true; + } + } + if !advanced { + break; + } + } + if !visited.insert(state.clone()) { + continue; + } + if visited.len() > case.resource_bounds.max_race_states as usize { + return violation( + "namespace_model_explosion", + format!( + "namespace search exceeds {} legal states", + case.resource_bounds.max_race_states + ), + ); + } + // Pinned-value feasibility: the next committed slot's value can no + // longer fall below initial + slots + 1, so any pending operation + // whose observed revision or incarnation is pinned below that floor + // can never commit (renames and unlinks always allocate; a stream + // open can only survive as an attach to a live session already + // carrying that value). Prune such states instead of exploring the + // interleavings that strand them. + let mut previous = state.initial_revision; + let mut distance = 0_u64; + for known in &state.revisions { + distance += 1; + if let Some(known) = known { + previous = *known; + distance = 0; + } + } + let next_floor = previous + .checked_add(distance) + .and_then(|value| value.checked_add(1)); + // These causal checks are unconditional. Environment or traversal + // order must never change which legal histories the oracle retains. + let mut doomed = false; + for process in 0..programs.len() { + for action in &programs[process][state.positions[process]..] { + let Some(value) = action + .result + .filter(|result| result.outcome == "ok") + .and_then(|result| result.revision.or(result.incarnation)) + else { + continue; + }; + if next_floor.is_some_and(|floor| value >= floor) { + continue; + } + match &action.operation { + NamespaceOperation::Rename { .. } | NamespaceOperation::Unlink { .. } => { + doomed = true; + } + NamespaceOperation::Lookup { path } => { + // A lookup observes the path's value at lookup time. + // If the entry already carries a different pinned + // value it can never return to this one (values only + // grow), and an absent entry can only be created at + // or beyond the floor. An unconstrained slot may + // still legally take this value when the lookup + let entry = state.entries.get(path); + let entry_value = entry.and_then(|binding| { + state.revisions.get(binding.revision).copied().flatten() + }); + let stream_slot = state.streams.get(path); + let stream_value = stream_slot + .and_then(|slot| state.revisions.get(*slot).copied().flatten()); + let doomed_entry = matches!(entry_value, Some(pinned) if pinned != value); + let doomed_stream = matches!(stream_value, Some(pinned) if pinned != value); + let absent = entry.is_none() + && stream_slot.is_none() + && !state.reservations.contains_key(path); + if doomed_entry || doomed_stream || absent { + doomed = true; + } + } + NamespaceOperation::StreamOpen { .. } => { + let attachable = state.sessions.keys().any(|slot| { + state.revisions.get(*slot).copied().flatten() == Some(value) + }); + if !attachable { + doomed = true; + } + } + _ => {} + } + if doomed { + break; + } + } + if doomed { + break; + } + } + if doomed { + continue; + } + if namespace_history_complete(&state, &programs) { + if namespace_retained_matches(&state, retained) { + return Ok(()); + } + retained_conflict = true; + continue; + } + // Evidence-ordered exploration: successors whose observed revision + // or incarnation evidence orders them first (the same candidate key + // the constructive replay uses) are expanded before unconstrained + // commits, and slot-free asynchronous stream events lead. Blind + // process-order search drowns in the interleavings of independent + // mutations even when every observation pins a legal history. + let mut successors = Vec::<(u64, NamespaceHistory)>::new(); + for process in 0..programs.len() { + let Some(action) = ready_action(&state, &programs, &dependencies, process) else { + continue; + }; + let key = action + .result + .and_then(|result| result.revision.or(result.incarnation)) + .or_else(|| match action.operation { + NamespaceOperation::StreamOpen { incarnation, .. } => incarnation, + _ => None, + }) + .or_else(|| { + operation_paths(&action.operation).find_map(|path| path_keys.get(path).copied()) + }) + .unwrap_or(u64::MAX); + if let Some(next_state) = namespace_transition(&state, process, action) { + successors.push((key, next_state)); + } + if let Some(next_state) = namespace_interrupted_open(&state, process, action) { + successors.push((key, next_state)); + } + } + for path in state.active_streams.keys() { + if let Some(next_state) = namespace_quiescence(&state, path) { + successors.push((0, next_state)); + } + if let Some(next_state) = namespace_first_frame(&state, path) { + successors.push((0, next_state)); + } + } + if successors.is_empty() { + let progress = state.positions.iter().sum(); + let role = (0..programs.len()) + .filter_map(|process| ready_action(&state, &programs, &dependencies, process)) + .map(|action| action.role) + .min(); + if let Some(role) = role + && dead_end_role.is_none_or(|(previous, previous_role)| { + progress > previous || (progress == previous && role < previous_role) + }) + { + dead_end_role = Some((progress, role)); + } + } + successors.sort_by(|left, right| left.0.cmp(&right.0)); + for (_, next_state) in successors.into_iter().rev() { + if visited.contains(&next_state) || !queued.insert(next_state.clone()) { + continue; + } + if visited.len() + queued.len() > case.resource_bounds.max_race_states as usize { + return violation( + "namespace_model_explosion", + format!( + "namespace search exceeds {} legal states", + case.resource_bounds.max_race_states + ), + ); + } + stack.push(next_state); + } + } + if retained_conflict { + return Err(OracleViolation::new( + "namespace_linearizability", + "final retained blob snapshot has no matching history", + ) + .classified(FailureClass::RetainedBlobMismatch, CausalRole::Namespace)); + } + Err(namespace_history_failure( + case, + observation, + dead_end_role.map(|(_, role)| role), + )) +} + +fn namespace_retained_matches( + state: &NamespaceHistory<'_>, + retained: Option<&BTreeMap>, +) -> bool { + let Some(retained) = retained else { + return true; + }; + if state.entries.len() != retained.len() + || state.entries.iter().any(|(path, binding)| { + retained.get(path).is_none_or(|(_, length)| { + usize::try_from(*length).ok() != Some(binding.bytes.len()) + }) + }) + { + return false; + } + let mut constrained = state.clone(); + retained.iter().all(|(path, (revision, _))| { + constrained.constrain_revision(state.entries[path].revision, *revision) + }) +} +/// Diagnose an already-rejected history from typed facts, never from its +/// formatted diagnostic. This does not prune or alter the successor set. +fn namespace_history_failure( + case: &BehaviorCase, + observation: &CaseObservation, + role: Option, +) -> OracleViolation { + let failure = || { + OracleViolation::new( + "namespace_linearizability", + "no namespace history satisfies action results, process order, dependencies and revisions", + ).classified(FailureClass::NamespaceHistoryConflict, role.map_or(CausalRole::Namespace, CausalRole::Action)) + }; + for program in &case.processes { + let Some(execution) = observation + .executions + .iter() + .find(|entry| entry.process == program.id) + else { + continue; + }; + for (step, action) in program.actions.iter().enumerate() { + let expected = match &action.operation { + ActionOp::StreamRead { expected, .. } + | ActionOp::StreamReadWithRetry { expected, .. } + | ActionOp::GatedStreamRead { expected, .. } + | ActionOp::StreamReadInto { expected, .. } => expected, + _ => continue, + }; + let Some(result) = execution + .results + .iter() + .find(|record| record.step == step && !is_barrier_record(record)) + else { + continue; + }; + let Some(incarnation) = result.incarnation else { + continue; + }; + let path = namespace_path(program, action.operation.path()); + let mut same_path = false; + let mut same_incarnation = false; + let mut matching_content = false; + for producer in &case.processes { + let Some(producer_execution) = observation + .executions + .iter() + .find(|entry| entry.process == producer.id) + else { + continue; + }; + for (producer_step, producer_action) in producer.actions.iter().enumerate() { + let chunks = match &producer_action.operation { + ActionOp::StreamWrite { chunks, .. } + | ActionOp::StreamRoundTrip { chunks, .. } + | ActionOp::GatedStreamWrite { frames: chunks, .. } => chunks, + _ => continue, + }; + if namespace_path(producer, producer_action.operation.path()) != path { + continue; + } + same_path = true; + if !producer_execution.results.iter().any(|record| record.step == producer_step + && (record.incarnation == Some(incarnation) + || matches!(record.barrier, Some(BarrierObservation::StreamOpened { incarnation: observed }) + if observed == incarnation))) + { continue; } + same_incarnation = true; + matching_content |= if let Some(transfer) = &result.transfer { + payload_summary(chunks.iter().map(Vec::as_slice), Some(transfer.length)) + .is_some_and(|(length, digest)| { + transfer.length <= length + && (!transfer.complete || transfer.length == length) + && transfer.digest == digest + }) + } else { + chunks.iter().flatten().eq(expected.iter()) + }; + } + } + let class = if !same_path { + FailureClass::MissingStreamSource + } else if !same_incarnation { + FailureClass::ConflictingIncarnation + } else if !matching_content { + FailureClass::StreamSourceContent + } else { + continue; + }; + return failure().classified(class, CausalRole::Action((&action.operation).into())); + } + } + failure() +} + +fn stream_parameters(operation: &ActionOp) -> Option<(bool, bool, Option<&[Vec]>)> { + match operation { + ActionOp::StreamWrite { + chunks, replace, .. + } => Some((true, *replace, Some(chunks))), + ActionOp::StreamRoundTrip { chunks, .. } => Some((true, false, Some(chunks))), + ActionOp::GatedStreamWrite { + frames, replace, .. + } => Some((true, *replace, Some(frames))), + ActionOp::StreamRead { .. } + | ActionOp::StreamReadWithRetry { .. } + | ActionOp::GatedStreamRead { .. } + | ActionOp::StreamReadInto { .. } => Some((false, false, None)), + _ => None, + } +} + +fn namespace_stopped_stream<'a>( + case: &BehaviorCase, + observation: &CaseObservation, + program: &ProcessProgram, + execution: &ExecutionObservation, + step: usize, + operation: &'a ActionOp, + frame_sources: &StreamFrameSources<'a>, +) -> Result>, OracleViolation> { + let Some((source, replace, chunks)) = stream_parameters(operation) else { + return Ok(None); + }; + let records = || { + execution + .results + .iter() + .filter(move |record| record.step == step) + }; + if records().any(|record| { + record.process != program.id + || record.path != operation.path() + || !stream_record_action_matches(operation, record) + }) { + return violation( + "namespace_model_evidence", + "stopped stream milestone belongs to another action", + ); + } + let incarnation = if records().any(|record| { + matches!( + record.barrier, + Some( + BarrierObservation::StreamOpened { .. } + | BarrierObservation::StreamFirstFrame { .. } + | BarrierObservation::StreamFrame { .. } + | BarrierObservation::StreamEof { .. } + ) + ) + }) { + stream_incarnation_evidence(execution, step, &program.id, operation)? + } else { + None + }; + let prefixes = if source { + &program.access.write_prefixes + } else { + &program.access.read_prefixes + }; + let path = namespace_path(program, operation.path()); + let mut stop_gates = Vec::new(); + if let FailureInjection::StopProcess { phase, .. } = &case.failure + && matches!( + phase, + ProcessStopPhase::AfterStreamFirstFrame + | ProcessStopPhase::AfterSiblingStreamFirstFrame + ) + { + for peer in &case.processes { + let Some(execution) = observation + .executions + .iter() + .find(|execution| execution.process == peer.id) + else { + continue; + }; + for record in &execution.results { + let Some(BarrierObservation::StreamFirstFrame { + incarnation: observed, + }) = record.barrier + else { + continue; + }; + let Some(action) = peer.actions.get(record.step) else { + continue; + }; + let peer_path = namespace_path(peer, action.operation.path()); + let marker = match &action.operation { + ActionOp::GatedStreamRead { observed_path, .. } => { + Some(namespace_path(peer, observed_path)) + } + _ => None, + }; + let relevant = if *phase == ProcessStopPhase::AfterSiblingStreamFirstFrame { + peer.id != program.id && marker.is_some() + } else { + peer_path == path && incarnation == Some(observed) + }; + if relevant { + stop_gates.push((peer_path, observed, marker)); + } + } + } + if stop_gates.is_empty() { + return violation( + "namespace_model_evidence", + "stream stop omitted its causal first-frame gate", + ); + } + } + Ok(Some(NamespaceAction { + step, + stream_owner: None, + preparation: false, + role: operation.into(), + denied: !namespace_prefix_allows(prefixes, &path), + operation: NamespaceOperation::StreamStop { + path, + incarnation, + source, + replace, + roundtrip: matches!(operation, ActionOp::StreamRoundTrip { .. }), + chunks, + gated: matches!(operation, ActionOp::GatedStreamWrite { .. }), + first_frame: records().any(|record| { + matches!( + record.barrier, + Some( + BarrierObservation::StreamFirstFrame { .. } + | BarrierObservation::StreamFrame { .. } + ) + ) + }), + eof: records() + .any(|record| matches!(record.barrier, Some(BarrierObservation::StreamEof { .. }))), + frame_sources: frame_sources + .get(&(program.id.as_str(), step)) + .cloned() + .unwrap_or_default(), + stop_gates, + }, + result: None, + publication_finished: false, + })) +} + +// Endpoint preparation starts beside the serial prefix so a prefix blocked on +// an upstream route cannot deadlock a downstream stream rendezvous. Every open +// must still precede the original scheduler slot's transfer suffix; receipt +// order never orders different opens. +fn prepare_namespace_routes<'a>( + case: &'a BehaviorCase, + observation: &'a CaseObservation, + programs: &mut Vec>>, + dependencies: &mut Vec>, + frame_sources: &StreamFrameSources<'a>, +) -> Result<(), OracleViolation> { + let route_paths: BTreeSet<_> = case + .routes + .iter() + .flat_map(|route| route.edges.iter().map(|edge| edge.path.as_str())) + .collect(); + for (process, program) in case.processes.iter().enumerate() { + let stream_steps: BTreeSet<_> = program + .actions + .iter() + .enumerate() + .filter(|(_, action)| { + route_paths.contains(action.operation.path()) + && stream_parameters(&action.operation).is_some() + }) + .map(|(step, _)| step) + .collect(); + if stream_steps.is_empty() { + continue; + } + let first_step = program + .actions + .iter() + .position(|action| { + route_paths.contains(action.operation.path()) + && (stream_parameters(&action.operation).is_some() + || matches!( + action.operation, + ActionOp::PublishBlob { .. } | ActionOp::ReadBlob { .. } + )) + }) + .expect("a stream route endpoint is a route transfer"); + let execution = observation + .executions + .iter() + .find(|execution| execution.process == program.id) + .expect("execution presence checked before namespace verification"); + let evidence_failure = |class, step: Option, detail: String| { + let role = step + .and_then(|step| program.actions.get(step)) + .map_or(CausalRole::Namespace, |action| { + CausalRole::Action((&action.operation).into()) + }); + OracleViolation::new("namespace_model_evidence", detail).classified(class, role) + }; + let stopped = matches!(&case.failure, FailureInjection::StopProcess { process, .. } + if process == &program.id) + && execution + .lifecycle + .iter() + .any(|event| event == "context_ready"); + if execution.results.is_empty() && !stopped { + continue; + } + let mut opened = BTreeMap::new(); + let mut next_terminal = 0; + let mut last_step = None; + let mut transferring = false; + for record in &execution.results { + let Some(action) = program.actions.get(record.step) else { + return Err(evidence_failure( + FailureClass::InvalidEvidence, + None, + "route record names an unknown step".to_owned(), + )); + }; + if record.process != program.id + || record.path != action.operation.path() + || !stream_record_action_matches(&action.operation, record) + { + return Err(evidence_failure( + FailureClass::InvalidEvidence, + Some(record.step), + "route record belongs to another action".to_owned(), + )); + } + if stream_steps.contains(&record.step) + && matches!( + record.barrier, + Some(BarrierObservation::StreamOpened { .. }) + ) + { + let already_opened = opened.insert(record.step, record).is_some(); + if !is_barrier_record(record) || transferring || already_opened { + return Err(evidence_failure( + FailureClass::MilestoneOrder, + Some(record.step), + format!( + "{} step {}: route preparation violates the transfer fence \ + (transferring={transferring}, already_opened={already_opened})", + program.id, record.step + ), + )); + } + continue; + } + if last_step.is_some_and(|step| record.step < step) + || record.step > next_terminal + || (!is_barrier_record(record) && record.step != next_terminal) + { + return Err(evidence_failure( + FailureClass::MilestoneOrder, + Some(record.step), + "route data results violate action order".to_owned(), + )); + } + last_step = Some(record.step); + if record.step >= first_step { + transferring = true; + if opened.len() != stream_steps.len() { + return Err(evidence_failure( + FailureClass::MissingEvidence, + Some(record.step), + "route transfer precedes complete endpoint preparation".to_owned(), + )); + } + } + if !is_barrier_record(record) { + next_terminal += 1; + } + } + // A stopped prefix never reached the common preparation phase. + if next_terminal < first_step { + continue; + } + if stopped { + for &step in &stream_steps { + if !programs[process].iter().any(|action| action.step == step) + && let Some(action) = namespace_stopped_stream( + case, + observation, + program, + execution, + step, + &program.actions[step].operation, + frame_sources, + )? + { + programs[process].push(action); + } + } + } + let mut prefix = std::mem::take(&mut programs[process]); + let prefix_end = prefix.partition_point(|action| action.step < first_step); + let actions = prefix.split_off(prefix_end); + let original_dependencies = std::mem::take(&mut dependencies[process]); + let mut transfer_dependencies = Vec::new(); + if !prefix.is_empty() { + let prefix_process = programs.len(); + programs.push(prefix); + dependencies.push(original_dependencies.clone()); + transfer_dependencies.push(prefix_process); + } + let mut owners = BTreeMap::new(); + for &step in &stream_steps { + let owner = programs.len(); + owners.insert(step, owner); + programs.push(Vec::new()); + dependencies.push(original_dependencies.clone()); + transfer_dependencies.push(owner); + } + dependencies[process] = transfer_dependencies; + let all_prepared = opened.len() == stream_steps.len(); + for mut action in actions { + let Some(&owner) = owners.get(&action.step) else { + programs[process].push(action); + continue; + }; + action.stream_owner = Some(owner); + match &action.operation { + NamespaceOperation::MutationOrigin { .. } + | NamespaceOperation::StreamOpen { .. } => { + action.preparation = true; + programs[owner].push(action); + } + NamespaceOperation::StreamAttach { + path, incarnation, .. + } => { + programs[process].push(NamespaceAction { + step: action.step, + stream_owner: Some(owner), + preparation: false, + operation: NamespaceOperation::StreamTransfer { + path: path.clone(), + incarnation: *incarnation, + }, + role: action.role, + result: action.result, + denied: false, + publication_finished: true, + }); + action.preparation = true; + programs[owner].push(action); + } + NamespaceOperation::StreamStop { + path, + incarnation: Some(incarnation), + source, + replace, + chunks, + gated, + roundtrip, + .. + } => { + let result = opened.get(&action.step).copied().ok_or_else(|| { + evidence_failure( + FailureClass::MissingEvidence, + Some(action.step), + "stopped route endpoint omitted its preparation".to_owned(), + ) + })?; + if !programs[owner].iter().any(|prepared| { + matches!(prepared.operation, NamespaceOperation::StreamAttach { .. }) + }) { + for operation in [ + NamespaceOperation::StreamOpen { + path: path.clone(), + incarnation: Some(*incarnation), + source: *source, + replace: *replace, + chunks: *chunks, + gated: *gated, + }, + NamespaceOperation::StreamAttach { + path: path.clone(), + incarnation: *incarnation, + roundtrip: *roundtrip, + }, + ] { + programs[owner].push(NamespaceAction { + step: action.step, + stream_owner: Some(owner), + preparation: true, + operation, + role: action.role, + result: Some(result), + denied: action.denied, + publication_finished: true, + }); + } + } + if all_prepared { + if action.step == next_terminal { + programs[process].push(NamespaceAction { + step: action.step, + stream_owner: Some(owner), + preparation: false, + operation: NamespaceOperation::StreamTransfer { + path: path.clone(), + incarnation: *incarnation, + }, + role: action.role, + result: Some(result), + denied: false, + publication_finished: true, + }); + } + programs[process].push(action); + } else { + action.preparation = true; + programs[owner].push(action); + } + } + NamespaceOperation::StreamStop { .. } => { + action.preparation = true; + programs[owner].push(action); + } + _ => programs[process].push(action), + } + } + if programs[process].is_empty() { + // Empty scheduler slots bypass dependencies. Keep a completion + // fence even when cancellation left no transfer to execute. + programs[process].push(NamespaceAction { + step: first_step, + stream_owner: None, + preparation: false, + operation: NamespaceOperation::Ignore, + role: (&program.actions[first_step].operation).into(), + result: None, + denied: false, + publication_finished: true, + }); + } + } + Ok(()) +} + +fn ready_action<'a>( + state: &NamespaceHistory<'_>, + programs: &'a [Vec>], + dependencies: &[Vec], + process: usize, +) -> Option<&'a NamespaceAction<'a>> { + if dependencies[process] + .iter() + .any(|dependency| state.positions[*dependency] < programs[*dependency].len()) + { + return None; + } + programs[process].get(state.positions[process]) +} + +fn operation_paths<'a>(operation: &'a NamespaceOperation<'_>) -> impl Iterator { + let first = operation.path(); + let second = match operation { + NamespaceOperation::Rename { destination, .. } => Some(destination.as_str()), + _ => None, + }; + first.into_iter().chain(second) +} + +fn namespace_action_key( + action: &NamespaceAction<'_>, + revision_hints: &BTreeMap, +) -> u64 { + action + .result + .and_then(|result| result.revision.or(result.incarnation)) + .or_else(|| match action.operation { + NamespaceOperation::StreamOpen { incarnation, .. } => incarnation, + _ => None, + }) + .or_else(|| { + operation_paths(&action.operation).find_map(|path| revision_hints.get(path).copied()) + }) + .unwrap_or(u64::MAX) +} + +fn namespace_commit_key( + action: &NamespaceAction<'_>, + revision_hints: &BTreeMap, +) -> Option { + if action.denied { + return None; + } + let result = action.result; + let success = result.is_some_and(|result| result.outcome == "ok" && result.errno.is_none()); + let path_hint = || { + action + .operation + .path() + .and_then(|path| revision_hints.get(path).copied()) + }; + match action.operation { + NamespaceOperation::StreamOpen { + incarnation: Some(incarnation), + .. + } => Some(incarnation), + NamespaceOperation::Publish { commit: true, .. } + | NamespaceOperation::Rename { .. } + | NamespaceOperation::Unlink { .. } + if success => + { + result.and_then(|result| result.revision).or_else(path_hint) + } + NamespaceOperation::GatePublish { .. } => path_hint(), + _ => None, + } +} +pub(crate) fn stream_record_action_matches(action: &ActionOp, record: &ActionObservation) -> bool { + record.action == action.class().as_str() + || (matches!(action, ActionOp::StreamRoundTrip { .. }) + && record.action == "stream_read" + && is_barrier_record(record) + && matches!(record.barrier, Some(BarrierObservation::StreamFrame { .. }))) +} + +fn stream_incarnation_evidence( + execution: &ExecutionObservation, + step: usize, + process: &str, + action: &ActionOp, +) -> Result, OracleViolation> { + let fail = |class, detail| { + OracleViolation::new("namespace_model_evidence", detail) + .classified(class, CausalRole::Action(action.into())) + }; + let records = || { + execution + .results + .iter() + .filter(move |result| result.step == step) + }; + let result = records().find(|record| !is_barrier_record(record)); + let observed = result.and_then(|result| result.incarnation); + let opened = records().find_map(|record| match record.barrier.as_ref() { + Some(BarrierObservation::StreamOpened { incarnation }) => Some(*incarnation), + _ => None, + }); + // Only typed open failures can lack attachment identity. The transition + // model still requires the exact collision/displacement state; a generic + // StreamError is not itself proof of any legal transition. + let Some(incarnation) = observed.or(opened) else { + if result.is_some_and(|result| { + [ + libc::EACCES, + libc::ENXIO, + libc::EEXIST, + libc::ESTALE, + libc::EIO, + ] + .into_iter() + .any(|errno| namespace_binding_error(result, errno, false)) + && result.transfer.is_none() + }) && !records().any(|record| { + matches!( + &record.barrier, + Some( + BarrierObservation::StreamFirstFrame { .. } + | BarrierObservation::StreamFrame { .. } + | BarrierObservation::StreamEof { .. } + ) + ) + }) { + return Ok(None); + } + return Err(fail( + FailureClass::MissingIncarnation, + format!("{process} step {step}: stream action omitted its incarnation identity"), + )); + }; + if incarnation == 0 { + return Err(fail( + FailureClass::InvalidIncarnation, + format!("{process} step {step}: zero stream incarnation"), + )); + } + let mut attached = false; + let mut first_frame = false; + let mut eof = false; + let mut terminal = false; + let mut sent = 0_u64; + let mut received = 0_u64; + let reader = matches!( + action, + ActionOp::StreamRoundTrip { .. } + | ActionOp::StreamRead { .. } + | ActionOp::StreamReadWithRetry { .. } + | ActionOp::GatedStreamRead { .. } + | ActionOp::StreamReadInto { .. } + ); + let bound = match action { + ActionOp::StreamWrite { chunks, .. } + | ActionOp::StreamRoundTrip { chunks, .. } + | ActionOp::GatedStreamWrite { frames: chunks, .. } => chunks.iter().map(Vec::len).sum(), + ActionOp::StreamRead { expected, .. } + | ActionOp::StreamReadWithRetry { expected, .. } + | ActionOp::GatedStreamRead { expected, .. } + | ActionOp::StreamReadInto { expected, .. } => expected.len(), + _ => 0, + }; + for record in records() { + let barrier_incarnation = match &record.barrier { + Some(BarrierObservation::StreamOpened { incarnation }) + | Some(BarrierObservation::StreamFirstFrame { incarnation }) + | Some(BarrierObservation::StreamFrame { incarnation, .. }) + | Some(BarrierObservation::StreamEof { incarnation }) => Some(*incarnation), + _ => None, + }; + if barrier_incarnation.is_some_and(|identity| identity != incarnation) + || record + .incarnation + .is_some_and(|identity| identity != incarnation) + { + return Err(fail( + FailureClass::ConflictingIncarnation, + format!( + "{process} step {step}: stream milestone disagrees with the action incarnation" + ), + )); + } + let stream_lifecycle = matches!( + &record.barrier, + Some( + BarrierObservation::StreamOpened { .. } + | BarrierObservation::StreamFirstFrame { .. } + | BarrierObservation::StreamFrame { .. } + | BarrierObservation::StreamEof { .. } + | BarrierObservation::ReleaseObserved { .. } + ) + ); + if is_barrier_record(record) && !stream_lifecycle { + continue; + } + if !is_barrier_record(record) { + if stream_lifecycle { + return Err(fail( + FailureClass::InvalidEvidence, + format!("{process} step {step}: terminal record contains stream milestone"), + )); + } + terminal = true; + continue; + } + if record.process != process + || record.path != action.path() + || !stream_record_action_matches(action, record) + { + return Err(fail( + FailureClass::InvalidEvidence, + format!("{process} step {step}: stream milestone belongs to another action"), + )); + } + let legal = !terminal + && match &record.barrier { + Some(BarrierObservation::StreamOpened { .. }) => { + let legal = !attached && !eof; + attached = true; + legal + } + Some(BarrierObservation::StreamFrame { index, length, .. }) => { + let next = if record.action == "stream_write" { + &mut sent + } else { + &mut received + }; + if *index != *next { + return Err(fail( + FailureClass::FrameOrderMismatch, + format!( + "{process} step {step}: missing, repeated, or reordered logical frame index" + ), + )); + } + *next = next.checked_add(1).ok_or_else(|| { + fail( + FailureClass::FrameOrderMismatch, + format!("{process} step {step}: logical frame index overflow"), + ) + })?; + if *length > bound as u64 { + return Err(fail( + FailureClass::FrameContentMismatch, + format!("{process} step {step}: frame exceeds its IR payload bound"), + )); + } + attached && !eof + } + Some(BarrierObservation::StreamFirstFrame { .. }) => { + let legal = attached && !first_frame && !eof && received == 1; + first_frame = true; + legal + } + Some(BarrierObservation::StreamEof { .. }) => { + let legal = attached && !eof && reader; + eof = true; + legal + } + Some(BarrierObservation::ReleaseObserved { .. }) => attached && !eof, + _ => true, + }; + if !legal { + return Err(fail( + FailureClass::MilestoneOrder, + format!("{process} step {step}: stream milestones violate local causal order"), + )); + } + } + if !attached { + return Err(fail( + FailureClass::MissingEvidence, + format!("{process} step {step}: stream omitted its opened milestone"), + )); + } + if reader + && result.is_some_and(|result| { + result.outcome == "ok" + || result + .transfer + .as_ref() + .is_some_and(|transfer| transfer.complete) + }) + && !eof + { + return Err(fail( + FailureClass::MissingStreamEof, + format!("{process} step {step}: completed reader omitted clean EOF"), + )); + } + if let Some(result) = result + && result.outcome != "ok" + && (first_frame || eof || received != 0 || (!reader && sent != 0)) + && result + .transfer + .as_ref() + .is_none_or(|transfer| eof && !transfer.complete) + { + return Err(fail( + FailureClass::MissingTransfer, + format!("{process} step {step}: stream progress omitted transferred-prefix evidence"), + )); + } + Ok(Some(incarnation)) +} + +fn namespace_history_complete( + state: &NamespaceHistory<'_>, + programs: &[Vec>], +) -> bool { + state + .positions + .iter() + .zip(programs) + .all(|(position, actions)| *position == actions.len()) + && state.publications.is_empty() + && state.publication_lookups.is_empty() + && state.reservations.is_empty() + && state.open_streams.is_empty() +} + +fn stream_initial_frame(chunks: Option<&[Vec]>, _gated: bool) -> bool { + chunks.is_some_and(|chunks| !chunks.is_empty()) +} + +// A first frame can be delivered before the source's aggregate action has +// completed. This matters when a consuming process is stopped mid-stream. +fn namespace_first_frame<'a>( + state: &NamespaceHistory<'a>, + path: &str, +) -> Option> { + let revision = state.active_streams.get(path)?; + let session = state.sessions.get(revision)?; + if !session.matched() + || !session.source_attached + || session.first_frame + || session.aborted + || !session.first_frame_permitted + { + return None; + } + let mut next = state.clone(); + next.sessions.get_mut(revision)?.first_frame = true; + Some(next) +} + +// A stop with no action record can occur before invocation or while the +// namespace open is pending. These are distinct legal branches, not a +// fabricated successful action. Any attached milestone constrains the same +// commit slot as every other participant before the stop can complete. +fn namespace_interrupted_open<'a>( + state: &NamespaceHistory<'a>, + process: usize, + action: &NamespaceAction<'a>, +) -> Option> { + let owner = action.stream_owner.unwrap_or(process); + let NamespaceOperation::StreamStop { + path, + incarnation, + source, + replace, + roundtrip, + chunks, + gated, + eof, + .. + } = &action.operation + else { + return None; + }; + if action.denied + || state.open_streams.contains_key(&owner) + || state.reservations.contains_key(path) + { + return None; + } + if !(path.starts_with("/cases/") || path.starts_with("/runs/")) { + return None; + } + let pending = state.active_streams.get(path).and_then(|revision| { + state + .sessions + .get(revision) + .map(|session| (*revision, session)) + }); + let compatible = pending.is_some_and(|(_, session)| { + !session.matched() + && if *source { + session.source.is_none() + } else { + session.sink.is_none() + } + }); + if (pending.is_some() && !compatible && !replace) + || (state.entries.contains_key(path) && !replace) + || (compatible && *roundtrip) + { + return None; + } + let mut next = state.clone(); + let revision = if compatible { + let (revision, _) = pending?; + if let Some(incarnation) = incarnation + && !next.constrain_revision(revision, *incarnation) + { + return None; + } + let session = next.sessions.get_mut(&revision)?; + if *source { + session.source = Some(owner); + session.chunks = *chunks; + session.source_attached = incarnation.is_some() && !action.preparation; + session.first_frame_permitted = stream_initial_frame(*chunks, *gated); + } else { + session.sink = Some(owner); + } + revision + } else { + let revision = next.commit_revision(*incarnation)?; + next.entries.remove(path); + next.streams.insert(path.clone(), revision); + next.active_streams.insert(path.clone(), revision); + next.sessions.insert( + revision, + StreamSession { + path: path.clone(), + source: source.then_some(owner), + sink: (!source || *roundtrip).then_some(owner), + chunks: *chunks, + source_attached: *source && incarnation.is_some() && !action.preparation, + first_frame_permitted: stream_initial_frame(*chunks, *gated), + first_frame: *roundtrip && *eof && chunks.is_some_and(|chunks| !chunks.is_empty()), + eof: *roundtrip && *eof, + aborted: false, + }, + ); + revision + }; + next.open_streams.insert(owner, revision); + Some(next) +} + +// Endpoint teardown sends an asynchronous namespace close. It can become +// visible before or after the action result, and consumes no revision. +fn namespace_quiescence<'a>( + state: &NamespaceHistory<'a>, + path: &str, +) -> Option> { + let revision = state.active_streams.get(path)?; + if !state + .sessions + .get(revision) + .is_some_and(|session| session.eof || session.aborted) + { + return None; + } + let mut next = state.clone(); + next.active_streams.remove(path); + Some(next) +} + +fn namespace_transition<'a>( + state: &NamespaceHistory<'a>, + process: usize, + action: &NamespaceAction<'a>, +) -> Option> { + let owner = action.stream_owner.unwrap_or(process); + if let NamespaceOperation::StreamStop { + path, + incarnation, + first_frame, + eof, + frame_sources, + stop_gates, + .. + } = &action.operation + { + if !stop_gates.is_empty() + && !stop_gates.iter().any(|(path, incarnation, marker)| { + marker + .as_ref() + .is_none_or(|marker| state.entries.contains_key(marker)) + && state.sessions.iter().any(|(revision, session)| { + session.path == *path + && session.matched() + && session.first_frame + && !session.aborted + && state.revisions[*revision] == Some(*incarnation) + }) + }) + { + return None; + } + let mut next = Cow::Borrowed(state); + if let Some(revision) = state.open_streams.get(&owner) { + let session = state.sessions.get(revision)?; + if session.path != *path + || incarnation.is_some_and(|identity| { + state.revisions[*revision] != Some(identity) || !session.matched() + }) + || (*first_frame && !session.first_frame) + || (*eof && !session.eof) + || (incarnation.is_some() + && !frame_sources.iter().any(|chunks| { + session + .chunks + .is_some_and(|source| std::ptr::eq(*chunks, source)) + })) + { + return None; + } + next.to_mut().sessions.get_mut(revision)?.aborted = true; + next.to_mut().open_streams.remove(&owner); + } else if incarnation.is_some() || *first_frame || *eof { + return None; + } + next.to_mut().positions[process] += 1; + return Some(next.into_owned()); + } + if let NamespaceOperation::MutationOrigin { path, revision } = &action.operation { + if action.denied || state.reservations.contains_key(path) { + return None; + } + let slot = state + .entries + .get(path) + .map(|binding| binding.revision) + .or_else(|| state.streams.get(path).copied())?; + let mut next = state.clone(); + if !next.constrain_revision(slot, *revision) { + return None; + } + next.positions[process] += 1; + return Some(next); + } + if matches!(action.operation, NamespaceOperation::Ignore) { + let mut next = state.clone(); + next.positions[process] += 1; + return Some(next); + } + let result = action.result?; + let mut next = Cow::Borrowed(state); + if action.denied { + if !namespace_errno(result, libc::EACCES) + || matches!( + action.operation, + NamespaceOperation::StreamOpen { + incarnation: Some(_), + .. + } + ) + { + return None; + } + next.to_mut().positions[process] += 1; + return Some(next.into_owned()); + } + let success = result.outcome == "ok" && result.errno.is_none(); + let mut complete = true; + match &action.operation { + NamespaceOperation::Publish { + path, + bytes, + commit, + exclusive, + require_existing, + } => { + if state.publications.contains(&process) { + if !action.publication_finished + || state + .reservations + .get(path) + .is_some_and(|owner| *owner != process) + { + return None; + } + next.to_mut().publications.remove(&process); + next.to_mut().reservations.remove(path); + if *commit { + let revision = next.to_mut().commit_revision(result.revision)?; + next.to_mut().streams.remove(path); + next.to_mut().active_streams.remove(path); + next.to_mut().entries.insert( + path.clone(), + NamespaceBinding { + bytes: *bytes, + revision, + }, + ); + } + } else if *exclusive && state.publication_lookups.contains(&process) { + if state.entries.contains_key(path) + || state.streams.contains_key(path) + || state.reservations.contains_key(path) + { + if !namespace_errno(result, libc::EEXIST) { + return None; + } + } else { + if !action.publication_finished { + return None; + } + next.to_mut().publications.insert(process); + next.to_mut().reservations.insert(path.clone(), process); + complete = false; + } + next.to_mut().publication_lookups.remove(&process); + } else if (*exclusive && state.entries.contains_key(path)) + || state.reservations.contains_key(path) + { + if !namespace_binding_error(result, libc::EEXIST, *exclusive || *require_existing) { + return None; + } + } else if state.streams.contains_key(path) { + // Staged-blob opens reject a stream seen by their lookup. + // Only an already-open publication can replace one. + if !namespace_binding_error(result, libc::ENXIO, *exclusive || *require_existing) { + return None; + } + } else if *require_existing && !state.entries.contains_key(path) { + if !namespace_errno(result, libc::ENOENT) { + return None; + } + } else if *exclusive { + // A stream may appear between lookup and reservation: + // EEXIST at reservation is distinct from ENXIO at lookup. + next.to_mut().publication_lookups.insert(process); + complete = false; + } else { + if !action.publication_finished { + return None; + } + next.to_mut().publications.insert(process); + complete = false; + } + } + NamespaceOperation::Read { + path, + expected, + offset, + whole, + } => { + if state.reservations.contains_key(path) { + if !namespace_binding_error(result, libc::EEXIST, !*whole) { + return None; + } + } else if let Some(binding) = state.entries.get(path) { + if !action.publication_finished { + return None; + } + let bytes = if *whole { + binding.bytes + } else { + let end = offset.checked_add(expected.len())?; + binding.bytes.get(*offset..end)? + }; + if bytes != *expected { + return None; + } + if let Some(revision) = result.revision + && !next.to_mut().constrain_revision(binding.revision, revision) + { + return None; + } + } else if state.streams.contains_key(path) { + if !namespace_binding_error(result, libc::ENXIO, !*whole) { + return None; + } + } else if !namespace_errno(result, libc::ENOENT) { + return None; + } + } + NamespaceOperation::Lookup { path } => { + if state.reservations.contains_key(path) { + if !namespace_errno(result, libc::EEXIST) { + return None; + } + } else if let Some(binding) = state.entries.get(path) { + if !success + || result.kind.as_deref() != Some("blob") + || result.active == Some(true) + || !next + .to_mut() + .constrain_revision(binding.revision, result.revision?) + { + return None; + } + } else if let Some(revision) = state.streams.get(path) { + if !success + || result.kind.as_deref() != Some("stream") + || result.active != Some(state.active_streams.contains_key(path)) + || !next + .to_mut() + .constrain_revision(*revision, result.revision?) + { + return None; + } + } else if !namespace_errno(result, libc::ENOENT) { + return None; + } + } + NamespaceOperation::Rename { + source, + destination, + replace, + } => { + if !state.entries.contains_key(source) && !state.streams.contains_key(source) { + if !namespace_errno(result, libc::ENOENT) { + return None; + } + } else if state.reservations.contains_key(source) + || state.reservations.contains_key(destination) + { + if !namespace_errno(result, libc::EEXIST) { + return None; + } + } else if state.active_streams.contains_key(source) + || state.active_streams.contains_key(destination) + { + if !namespace_errno(result, libc::EBUSY) { + return None; + } + } else if source != destination + && (state.entries.contains_key(destination) + || state.streams.contains_key(destination)) + && !replace + { + if !namespace_errno(result, libc::EEXIST) { + return None; + } + } else { + if !success { + return None; + } + let binding = next.to_mut().entries.remove(source); + let stream = next.to_mut().streams.remove(source); + let revision = next.to_mut().commit_revision(result.revision)?; + next.to_mut().entries.remove(destination); + next.to_mut().streams.remove(destination); + if let Some(mut binding) = binding { + binding.revision = revision; + next.to_mut().entries.insert(destination.clone(), binding); + } else if stream.is_some() { + next.to_mut().streams.insert(destination.clone(), revision); + } + } + } + NamespaceOperation::Unlink { path } => { + if state.active_streams.contains_key(path) { + if !namespace_errno(result, libc::ENXIO) { + return None; + } + } else if state.reservations.contains_key(path) { + if !namespace_errno(result, libc::EEXIST) { + return None; + } + } else if state.entries.contains_key(path) || state.streams.contains_key(path) { + if !success { + return None; + } + next.to_mut().entries.remove(path); + next.to_mut().streams.remove(path); + next.to_mut().commit_revision(result.revision)?; + } else if !namespace_errno(result, libc::ENOENT) { + return None; + } + } + NamespaceOperation::StreamOpen { + path, + incarnation, + source, + replace, + chunks, + gated, + } => { + if state.open_streams.contains_key(&owner) { + return None; + } + let pending = state.active_streams.get(path).and_then(|revision| { + state + .sessions + .get(revision) + .map(|session| (*revision, session)) + }); + let compatible = pending.is_some_and(|(_, session)| { + !session.matched() + && if *source { + session.source.is_none() + } else { + session.sink.is_none() + } + }); + let error = if state.reservations.contains_key(path) { + Some(libc::EEXIST) + } else if pending.is_some() && !compatible && !replace { + // DuplicateStreamRole maps to SessionFailed/EIO, not EEXIST. + Some(libc::EIO) + } else if state.entries.contains_key(path) && !replace { + Some(libc::ENXIO) + } else { + None + }; + if let Some(errno) = error { + if incarnation.is_some() || !namespace_binding_error(result, errno, false) { + return None; + } + } else { + if incarnation.is_none() && !namespace_errno(result, libc::ESTALE) { + return None; + } + if compatible { + let (revision, _) = pending?; + if let Some(incarnation) = incarnation + && !next.to_mut().constrain_revision(revision, *incarnation) + { + return None; + } + let session = next.to_mut().sessions.get_mut(&revision)?; + if *source { + session.first_frame_permitted = stream_initial_frame(*chunks, *gated); + session.source = Some(owner); + session.chunks = *chunks; + } else { + session.sink = Some(owner); + } + next.to_mut().open_streams.insert(owner, revision); + } else { + // Pending creation commits a fresh revision. An open + // displaced before attachment has no handle identity; + // its symbolic slot is constrained by later evidence, + // never filled with a guessed incarnation. + let revision = next.to_mut().commit_revision(*incarnation)?; + next.to_mut().entries.remove(path); + next.to_mut().streams.insert(path.clone(), revision); + next.to_mut().active_streams.insert(path.clone(), revision); + next.to_mut().open_streams.insert(owner, revision); + next.to_mut().sessions.insert( + revision, + StreamSession { + path: path.clone(), + source: source.then_some(owner), + sink: (!source).then_some(owner), + chunks: *chunks, + source_attached: false, + first_frame_permitted: stream_initial_frame(*chunks, *gated), + first_frame: false, + eof: false, + aborted: false, + }, + ); + } + } + } + NamespaceOperation::StreamAttach { + path, + incarnation, + roundtrip, + } => { + let Some(&revision) = state.open_streams.get(&owner) else { + return None; + }; + let Some(session) = state.sessions.get(&revision) else { + return None; + }; + if session.path != *path { + return None; + } + if state.revisions[revision] != Some(*incarnation) { + return None; + } + if *roundtrip { + if session.source != Some(owner) + || session.sink.is_some() + || state.active_streams.get(path) != Some(&revision) + { + return None; + } + } else if !session.matched() { + return None; + } + let session = next.to_mut().sessions.get_mut(&revision)?; + if *roundtrip { + session.sink = Some(owner); + } + if session.source == Some(owner) && !action.preparation { + session.source_attached = true; + } + } + NamespaceOperation::StreamTransfer { path, incarnation } => { + let revision = *state.open_streams.get(&owner)?; + let session = state.sessions.get(&revision)?; + if session.path != *path + || state.revisions[revision] != Some(*incarnation) + || !session.matched() + || (session.source != Some(owner) && session.sink != Some(owner)) + { + return None; + } + if session.source == Some(owner) { + next.to_mut().sessions.get_mut(&revision)?.source_attached = true; + } + } + NamespaceOperation::StreamFirstFrame { + path, + incarnation, + source, + } => { + let revision = *state.open_streams.get(&owner)?; + let session = state.sessions.get(&revision)?; + if state.active_streams.get(path) != Some(&revision) + || !session.matched() + || state.revisions[revision] != Some(*incarnation) + { + return None; + } + if *source { + let session = next.to_mut().sessions.get_mut(&revision)?; + session.first_frame |= session.first_frame_permitted; + } else if !session.first_frame { + return None; + } + } + NamespaceOperation::StreamComplete { + path, + incarnation, + source, + expected, + frame_sources, + frames_observed, + } => { + let revision = *state.open_streams.get(&owner)?; + let session = state.sessions.get(&revision)?; + if !session.matched() + || session.path != *path + || state.revisions[revision] != Some(*incarnation) + || !frame_sources.iter().any(|chunks| { + session + .chunks + .is_some_and(|source| std::ptr::eq(*chunks, source)) + }) + || (!source && *frames_observed && !session.first_frame) + || if *source { + session.source != Some(owner) + } else { + session.sink != Some(owner) + } + { + return None; + } + if !*source && let Some(transfer) = &result.transfer { + let (length, expected_digest) = payload_summary( + session.chunks?.iter().map(Vec::as_slice), + Some(transfer.length), + )?; + if transfer.length > length + || transfer.digest != expected_digest + || (transfer.complete && transfer.length != length) + || (transfer.complete && !session.eof) + { + return None; + } + } + let current = state.streams.get(path) == Some(&revision); + if !current { + if !namespace_errno(result, libc::ESTALE) { + return None; + } + } else if session.aborted { + if !namespace_binding_error(result, libc::ECONNRESET, false) + && !namespace_binding_error(result, libc::EPIPE, false) + { + return None; + } + } else if !success { + return None; + } else if *source { + let session = next.to_mut().sessions.get_mut(&revision)?; + session.first_frame = !session.chunks?.is_empty(); + session.eof = true; + } else { + if !session.eof { + return None; + } + if !session + .chunks? + .iter() + .flat_map(|chunk| chunk.iter()) + .eq((*expected)?.iter()) + { + return None; + } + } + next.to_mut().open_streams.remove(&owner); + } + NamespaceOperation::StreamPendingFailure { path } => { + let revision = *state.open_streams.get(&owner)?; + let session = state.sessions.get(&revision)?; + if session.path != *path + || session.matched() + || state.active_streams.get(path) == Some(&revision) + || !namespace_errno(result, libc::ESTALE) + { + return None; + } + next.to_mut().open_streams.remove(&owner); + } + NamespaceOperation::StreamProbe { path } => { + let revision = *state.streams.get(path)?; + if !success + || result.kind.as_deref() != Some("stream") + || result.active != Some(false) + || state.active_streams.contains_key(path) + || !next.to_mut().constrain_revision(revision, result.revision?) + { + return None; + } + } + NamespaceOperation::GatePublish { path } => { + if state.reservations.contains_key(path) { + return None; + } + if state.publications.contains(&process) { + next.to_mut().publications.remove(&process); + let revision = next.to_mut().commit_revision(None)?; + next.to_mut().streams.remove(path); + next.to_mut().active_streams.remove(path); + next.to_mut().entries.insert( + path.clone(), + NamespaceBinding { + bytes: &[], + revision, + }, + ); + } else { + if state.streams.contains_key(path) { + return None; + } + next.to_mut().publications.insert(process); + complete = false; + } + } + NamespaceOperation::GateLookup { path } => { + if state.reservations.contains_key(path) + || (!state.entries.contains_key(path) && !state.streams.contains_key(path)) + { + return None; + } + let revision = state.open_streams.get(&owner)?; + let session = next.to_mut().sessions.get_mut(revision)?; + session.first_frame_permitted = stream_initial_frame(session.chunks, false); + } + NamespaceOperation::StreamStop { .. } | NamespaceOperation::MutationOrigin { .. } => { + return None; + } + NamespaceOperation::Ignore => {} + } + if complete { + next.to_mut().positions[process] += 1; + } + Some(next.into_owned()) +} + +fn violation(invariant: &'static str, detail: impl Into) -> Result { + Err(OracleViolation::new(invariant, detail)) +} + +pub(crate) fn process_failure_is_expected(case: &BehaviorCase, process: &ProcessProgram) -> bool { + match &case.failure { + FailureInjection::None | FailureInjection::SlowProcess { .. } => false, + FailureInjection::StopProcess { + process: target, .. + } + | FailureInjection::LaunchFailure { + process: target, .. + } => target == &process.id, + } +} + +#[cfg(test)] +mod namespace_tests { + use super::*; + use crate::ir::{ + AccessSpec, Action, DataEdge, DataRoute, DescriptorObservation, DescriptorTerminalResult, + DescriptorWriteMethod, PythonException, TransferObservation, + }; + + fn program(id: &str, actions: Vec, dependencies: &[&str]) -> ProcessProgram { + ProcessProgram { + id: id.to_owned(), + logical_node_id: 1, + access: AccessSpec::unrestricted(id), + depends_on: dependencies + .iter() + .map(|dependency| (*dependency).to_owned()) + .collect(), + actions, + } + } + + fn case(processes: Vec) -> BehaviorCase { + BehaviorCase { + schema_version: crate::ir::CASE_SCHEMA_VERSION, + generator_version: crate::ir::GENERATOR_VERSION, + id: "namespace-model".to_owned(), + seed: 1, + live_nodes: BTreeSet::from([1, 2]), + topology: Default::default(), + scenarios: Default::default(), + routes: Vec::new(), + read_only_fixture_paths: BTreeSet::new(), + resource_bounds: Default::default(), + processes, + failure: FailureInjection::None, + } + } + + fn publication(path: &str, bytes: &[u8]) -> ActionOp { + ActionOp::PublishBlob { + path: path.to_owned(), + bytes: bytes.to_vec(), + } + } + + fn exclusive(path: &str, bytes: &[u8]) -> ActionOp { + ActionOp::DescriptorWrite { + path: path.to_owned(), + flags: libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL | libc::O_TRUNC, + length: Some(bytes.len() as u64), + bytes: bytes.to_vec(), + method: DescriptorWriteMethod::Write, + finish: DescriptorFinish::Close, + } + } + + fn observe(case: &BehaviorCase) -> CaseObservation { + CaseObservation { + case_id: case.id.clone(), + executions: case + .processes + .iter() + .map(|program| ExecutionObservation { + process: program.id.clone(), + request_id: case.execution_request_id(program), + logical_node_id: program.logical_node_id, + lifecycle: ["process_started", "context_ready", "user_result", "exited"] + .into_iter() + .map(str::to_owned) + .collect(), + results: program + .actions + .iter() + .enumerate() + .map(|(step, action)| { + let bytes = match &action.operation { + ActionOp::PublishBlob { bytes, .. } + | ActionOp::DescriptorWrite { bytes, .. } + | ActionOp::ReadBlob { + expected: bytes, .. + } => Some(bytes), + _ => None, + }; + let errno = match action.expected { + ExpectedOutcome::Error(errno) => Some(errno), + _ => None, + }; + ActionObservation { + process: program.id.clone(), + step, + action: action.operation.class().as_str().to_owned(), + path: action.operation.path().to_owned(), + outcome: if errno.is_some() + || matches!(action.expected, ExpectedOutcome::Exception(_)) + { + "expected_error" + } else { + "ok" + } + .to_owned(), + length: bytes.map(Vec::len), + digest: bytes.map(|bytes| digest(bytes)), + kind: match &action.operation { + ActionOp::Lookup { expected_kind, .. } => { + Some(expected_kind.clone()) + } + _ => None, + }, + revision: matches!( + action.operation, + ActionOp::Rename { .. } + | ActionOp::Unlink { .. } + | ActionOp::Lookup { .. } + ) + .then_some(10 + step as u64), + active: None, + errno, + error_type: match action.expected { + ExpectedOutcome::Exception(exception) => { + Some(exception.as_str().to_owned()) + } + ExpectedOutcome::Error(_) => Some("OSError".to_owned()), + _ => None, + }, + error: None, + descriptor: match &action.operation { + ActionOp::DescriptorWrite { method, finish, .. } + if errno.is_none() => + { + Some(DescriptorObservation::Write { + method: *method, + finish: *finish, + terminal_results: vec![DescriptorTerminalResult::Ok], + dropped: false, + reservation_released: false, + }) + } + _ => None, + }, + barrier: None, + incarnation: None, + token: None, + lap: None, + transfer: None, + } + }) + .collect(), + terminal: true, + exit_success: true, + exit_status: None, + stdout: String::new(), + stderr: String::new(), + }) + .collect(), + } + } + + #[test] + fn successful_actions_require_process_local_order() { + let case = case(vec![program( + "owner", + vec![ + Action::ok(publication("/cases/order/blob", b"value")), + Action::ok(ActionOp::ReadBlob { + path: "/cases/order/blob".to_owned(), + expected: b"value".to_vec(), + }), + ], + &[], + )]); + let mut observation = observe(&case); + BehaviorOracle::verify(&case, &observation).unwrap(); + observation.executions[0].results.reverse(); + assert_eq!( + BehaviorOracle::verify(&case, &observation) + .unwrap_err() + .signature + .failure_class, + FailureClass::MilestoneOrder, + ); + } + + #[test] + fn owned_reads_cannot_invent_an_initial_fixture() { + let path = "/cases/absent/blob"; + let mut case = case(vec![program( + "reader", + vec![Action::ok(ActionOp::ReadBlob { + path: path.to_owned(), + expected: b"value".to_vec(), + })], + &[], + )]); + let observation = observe(&case); + assert_eq!( + BehaviorOracle::verify(&case, &observation) + .unwrap_err() + .invariant, + "namespace_linearizability", + ); + case.read_only_fixture_paths.insert(path.to_owned()); + BehaviorOracle::verify(&case, &observation).unwrap(); + } + + #[test] + fn exhausted_revision_space_rejects_without_overflowing_search() { + let fixture = "/models/revision-limit"; + let mut case = case(vec![program( + "owner", + vec![ + Action::ok(ActionOp::Lookup { + path: fixture.to_owned(), + expected_kind: "blob".to_owned(), + }), + Action::ok(publication("/cases/revision/blob", b"value")), + ], + &[], + )]); + case.read_only_fixture_paths.insert(fixture.to_owned()); + let mut observation = observe(&case); + observation.executions[0].results[0].revision = Some(u64::MAX); + assert_eq!( + BehaviorOracle::verify(&case, &observation) + .unwrap_err() + .invariant, + "namespace_linearizability", + ); + } + + #[test] + fn exclusive_winner_permutations_ignore_observation_vector_order() { + let case = case(vec![ + program( + "a", + vec![Action::linearized( + exclusive("/cases/race/name", b"a"), + 1, + 1, + libc::EEXIST, + )], + &[], + ), + program( + "b", + vec![Action::linearized( + exclusive("/cases/race/name", b"b"), + 1, + 1, + libc::EEXIST, + )], + &[], + ), + ]); + for winner in 0..2 { + let mut observation = observe(&case); + let loser = &mut observation.executions[1 - winner].results[0]; + loser.outcome = "expected_error".to_owned(); + loser.errno = Some(libc::EEXIST); + loser.descriptor = None; + assert!(BehaviorOracle::verify(&case, &observation).is_ok()); + observation.executions.reverse(); + assert!(BehaviorOracle::verify(&case, &observation).is_ok()); + } + } + + #[test] + fn declared_success_count_cannot_legalize_double_exclusive_publication() { + let case = case(vec![ + program( + "a", + vec![Action::linearized( + exclusive("/cases/race/name", b"a"), + 1, + 2, + libc::EEXIST, + )], + &[], + ), + program( + "b", + vec![Action::linearized( + exclusive("/cases/race/name", b"b"), + 1, + 2, + libc::EEXIST, + )], + &[], + ), + ]); + assert_eq!( + BehaviorOracle::verify(&case, &observe(&case)) + .unwrap_err() + .invariant, + "namespace_linearizability" + ); + } + + #[test] + fn replacement_cannot_leave_displaced_blob_visible() { + let mut case = case(vec![program( + "p", + vec![ + Action::ok(publication("/cases/race/source", b"new")), + Action::ok(publication("/cases/race/destination", b"old")), + Action::ok(ActionOp::Rename { + source: "/cases/race/source".to_owned(), + destination: "/cases/race/destination".to_owned(), + replace: true, + }), + Action::ok(ActionOp::ReadBlob { + path: "/cases/race/destination".to_owned(), + expected: b"old".to_vec(), + }), + ], + &[], + )]); + assert_eq!( + BehaviorOracle::verify(&case, &observe(&case)) + .unwrap_err() + .invariant, + "namespace_linearizability" + ); + case.processes[0].actions[3] = Action::ok(ActionOp::ReadBlob { + path: "/cases/race/destination".to_owned(), + expected: b"new".to_vec(), + }); + BehaviorOracle::verify(&case, &observe(&case)).unwrap(); + } + + #[test] + fn truncating_descriptor_replacement_requires_an_existing_blob() { + let path = "/cases/truncate/blob"; + let replacement = Action::ok(ActionOp::DescriptorWrite { + path: path.to_owned(), + flags: libc::O_WRONLY | libc::O_TRUNC, + length: Some(3), + bytes: b"new".to_vec(), + method: DescriptorWriteMethod::Write, + finish: DescriptorFinish::Close, + }); + let mut case = case(vec![ + program( + "writer", + vec![Action::ok(publication(path, b"old")), replacement], + &[], + ), + program( + "reader", + vec![Action::ok(ActionOp::ReadBlob { + path: path.to_owned(), + expected: b"new".to_vec(), + })], + &["writer"], + ), + ]); + BehaviorOracle::verify(&case, &observe(&case)).unwrap(); + case.processes[0].actions.remove(0); + assert!(BehaviorOracle::verify(&case, &observe(&case)).is_err()); + case.processes[0].actions[0].expected = ExpectedOutcome::Error(libc::ENOENT); + case.processes.pop(); + BehaviorOracle::verify(&case, &observe(&case)).unwrap(); + } + + #[test] + fn mutation_origin_is_a_causal_lookup_and_destination_matches_receipt() { + let case = case(vec![program( + "owner", + vec![ + Action::ok(publication("/cases/mutation/source", b"value")), + Action::ok(ActionOp::Lookup { + path: "/cases/mutation/source".to_owned(), + expected_kind: "blob".to_owned(), + }), + Action::ok(ActionOp::Rename { + source: "/cases/mutation/source".to_owned(), + destination: "/cases/mutation/destination".to_owned(), + replace: false, + }), + ], + &[], + )]); + let mut observation = observe(&case); + observation.executions[0].results[1].revision = Some(20); + observation.executions[0].results[2].revision = Some(21); + let mut mutation = observation.executions[0].results[2].clone(); + mutation.outcome = "barrier".to_owned(); + mutation.barrier = Some(BarrierObservation::MutationApplied { + from_revision: Some(20), + to_revision: Some(21), + }); + observation.executions[0].results.insert(2, mutation); + BehaviorOracle::verify(&case, &observation).unwrap(); + let mut errored_receipt = observation.clone(); + errored_receipt.executions[0].results[3].errno = Some(libc::EIO); + assert_eq!( + BehaviorOracle::verify(&case, &errored_receipt) + .unwrap_err() + .signature + .failure_class, + FailureClass::InvalidEvidence + ); + observation.executions[0].results[2].barrier = Some(BarrierObservation::MutationApplied { + from_revision: Some(19), + to_revision: Some(21), + }); + assert_eq!( + BehaviorOracle::verify(&case, &observation) + .unwrap_err() + .invariant, + "namespace_linearizability" + ); + observation.executions[0].results[2].barrier = Some(BarrierObservation::MutationApplied { + from_revision: Some(20), + to_revision: Some(22), + }); + assert_eq!( + BehaviorOracle::verify(&case, &observation) + .unwrap_err() + .signature + .failure_class, + FailureClass::InvalidEvidence + ); + } + + #[test] + fn mutation_origin_orders_its_unobserved_publication_among_unrelated_paths() { + let race = "/cases/origin-order/race"; + let paths = [ + "/cases/origin-order/prelude", + "/cases/origin-order/a", + "/cases/origin-order/b", + "/cases/origin-order/c", + ]; + let mut case = case(vec![ + program( + "prelude-writer", + vec![Action::ok(publication(paths[0], b"p"))], + &[], + ), + program( + "prelude-reader", + vec![Action::ok(ActionOp::Lookup { + path: paths[0].to_owned(), + expected_kind: "blob".to_owned(), + })], + &["prelude-writer"], + ), + program("setup", vec![Action::ok(publication(race, b"race"))], &[]), + program( + "winner", + vec![Action::linearized( + ActionOp::Unlink { + path: race.to_owned(), + }, + 7, + 1, + libc::ENOENT, + )], + &["setup"], + ), + program( + "loser", + vec![Action::linearized( + ActionOp::Unlink { + path: race.to_owned(), + }, + 7, + 1, + libc::ENOENT, + )], + &["setup"], + ), + program( + "topology-writer", + paths[1..] + .iter() + .map(|path| Action::ok(publication(path, path.as_bytes()))) + .collect(), + &[], + ), + program( + "topology-reader", + paths[1..] + .iter() + .map(|path| { + Action::ok(ActionOp::Lookup { + path: (*path).to_owned(), + expected_kind: "blob".to_owned(), + }) + }) + .collect(), + &["topology-writer"], + ), + ]); + case.resource_bounds.max_race_states = 64; + let mut observation = observe(&case); + observation.executions[1].results[0].revision = Some(100); + for (result, revision) in observation.executions[6].results.iter_mut().zip(102..=104) { + result.revision = Some(revision); + } + observation.executions[3].results[0].revision = Some(105); + let mut receipt = observation.executions[3].results[0].clone(); + receipt.outcome = "barrier".to_owned(); + receipt.barrier = Some(BarrierObservation::MutationApplied { + from_revision: Some(101), + to_revision: Some(105), + }); + observation.executions[3].results.insert(0, receipt); + let loser = &mut observation.executions[4].results[0]; + loser.outcome = "expected_error".to_owned(); + loser.revision = None; + loser.errno = Some(libc::ENOENT); + loser.error_type = Some("OSError".to_owned()); + BehaviorOracle::verify(&case, &observation).unwrap(); + } + + #[test] + fn failed_rename_cannot_claim_a_mutation_receipt() { + let case = case(vec![program( + "owner", + vec![ + Action::ok(publication("/cases/mutation/source", b"source")), + Action::ok(ActionOp::Lookup { + path: "/cases/mutation/source".to_owned(), + expected_kind: "blob".to_owned(), + }), + Action::ok(publication("/cases/mutation/destination", b"destination")), + Action::error( + ActionOp::Rename { + source: "/cases/mutation/source".to_owned(), + destination: "/cases/mutation/destination".to_owned(), + replace: false, + }, + libc::EEXIST, + ), + ], + &[], + )]); + let mut observation = observe(&case); + observation.executions[0].results[1].revision = Some(20); + observation.executions[0].results[3].revision = Some(22); + BehaviorOracle::verify(&case, &observation).unwrap(); + let mut receipt = observation.executions[0].results[3].clone(); + receipt.outcome = "barrier".to_owned(); + receipt.errno = None; + receipt.error_type = None; + receipt.barrier = Some(BarrierObservation::MutationApplied { + from_revision: Some(20), + to_revision: Some(22), + }); + observation.executions[0].results.insert(3, receipt); + assert_eq!( + BehaviorOracle::verify(&case, &observation) + .unwrap_err() + .signature + .failure_class, + FailureClass::InvalidEvidence + ); + } + + #[test] + fn auxiliary_barriers_require_the_owning_action_identity() { + let case = case(vec![program( + "owner", + vec![Action::ok(publication("/cases/barrier/blob", b"value"))], + &[], + )]); + let mut observation = observe(&case); + let mut marker = observation.executions[0].results[0].clone(); + marker.outcome = "barrier".to_owned(); + marker.barrier = Some(BarrierObservation::LapCompleted { + token: "token".to_owned(), + lap: 1, + }); + observation.executions[0].results.push(marker); + BehaviorOracle::verify(&case, &observation).unwrap(); + for (field, invariant) in [ + ("process", "action_identity"), + ("step", "action_identity"), + ("action", "action_identity"), + ("path", "path_consistency"), + ] { + let mut foreign = observation.clone(); + let marker = foreign.executions[0].results.last_mut().unwrap(); + match field { + "process" => marker.process = "foreign".to_owned(), + "step" => marker.step = 1, + "action" => marker.action = "lookup".to_owned(), + "path" => marker.path = "/cases/barrier/foreign".to_owned(), + _ => unreachable!(), + } + assert_eq!( + BehaviorOracle::verify(&case, &foreign) + .unwrap_err() + .invariant, + invariant + ); + } + } + + #[test] + fn dependent_double_unlink_has_no_history_even_with_matching_count() { + let case = case(vec![ + program( + "setup", + vec![Action::ok(publication("/cases/race/name", b"value"))], + &[], + ), + program( + "a", + vec![Action::linearized( + ActionOp::Unlink { + path: "/cases/race/name".to_owned(), + }, + 1, + 2, + libc::ENOENT, + )], + &["setup"], + ), + program( + "b", + vec![Action::linearized( + ActionOp::Unlink { + path: "/cases/race/name".to_owned(), + }, + 1, + 2, + libc::ENOENT, + )], + &["a"], + ), + ]); + let mut observation = observe(&case); + observation.executions[2].results[0].revision = Some(11); + assert_eq!( + BehaviorOracle::verify(&case, &observation) + .unwrap_err() + .invariant, + "namespace_linearizability" + ); + } + + #[test] + fn lookup_revision_constrains_the_winning_rename() { + let mut case = case(vec![ + program( + "setup", + vec![Action::ok(publication("/cases/race/source", b"value"))], + &[], + ), + program( + "a", + vec![Action::linearized( + ActionOp::Rename { + source: "/cases/race/source".to_owned(), + destination: "/cases/race/a".to_owned(), + replace: false, + }, + 1, + 1, + libc::ENOENT, + )], + &["setup"], + ), + program( + "b", + vec![Action::linearized( + ActionOp::Rename { + source: "/cases/race/source".to_owned(), + destination: "/cases/race/b".to_owned(), + replace: false, + }, + 1, + 1, + libc::ENOENT, + )], + &["setup"], + ), + program( + "reader", + vec![Action::ok(ActionOp::Lookup { + path: "/cases/race/a".to_owned(), + expected_kind: "blob".to_owned(), + })], + &["a", "b"], + ), + ]); + for (winner, destination) in [(1, "/cases/race/a"), (2, "/cases/race/b")] { + case.processes[3].actions[0] = Action::ok(ActionOp::Lookup { + path: destination.to_owned(), + expected_kind: "blob".to_owned(), + }); + let mut observation = observe(&case); + let loser = &mut observation.executions[3 - winner].results[0]; + loser.outcome = "expected_error".to_owned(); + loser.errno = Some(libc::ENOENT); + loser.revision = None; + assert!(BehaviorOracle::verify(&case, &observation).is_ok()); + observation.executions.reverse(); + assert!(BehaviorOracle::verify(&case, &observation).is_ok()); + observation + .executions + .iter_mut() + .find(|execution| execution.process == "reader") + .unwrap() + .results[0] + .revision = Some(9); + assert_eq!( + BehaviorOracle::verify(&case, &observation) + .unwrap_err() + .invariant, + "namespace_linearizability" + ); + } + } + + #[test] + fn namespace_frontier_limit_is_inclusive_and_never_selects_a_winner() { + let mut races = case(vec![ + program( + "a", + vec![Action::error( + ActionOp::Unlink { + path: "/cases/race/missing".to_owned(), + }, + libc::ENOENT, + )], + &[], + ), + program( + "b", + vec![Action::error( + ActionOp::Unlink { + path: "/cases/race/missing".to_owned(), + }, + libc::ENOENT, + )], + &[], + ), + ]); + races.resource_bounds.max_race_states = 2; + let mut observed = observe(&races); + // Both orders verify: acceptance never depends on observation + // vector order, and a complete greedy legal history needs no + // interleaving search at any bound. + assert!(BehaviorOracle::verify(&races, &observed).is_ok()); + observed.executions.reverse(); + assert!(BehaviorOracle::verify(&races, &observed).is_ok()); + + // A case no greedy history completes falls back to frontier + // exploration; an inclusive bound of one legal state still allows + // exactly one frontier member, and exceeding it is an explicit + // model failure rather than an arbitrary winner. + let mut impossible = case(vec![ + program( + "setup", + vec![Action::ok(publication("/cases/race/name", b"value"))], + &[], + ), + program( + "a", + vec![Action::ok(ActionOp::Unlink { + path: "/cases/race/name".to_owned(), + })], + &["setup"], + ), + program( + "b", + vec![Action::ok(ActionOp::Unlink { + path: "/cases/race/name".to_owned(), + })], + &["setup"], + ), + ]); + impossible.resource_bounds.max_race_states = 1; + assert_eq!( + BehaviorOracle::verify(&impossible, &observe(&impossible)) + .unwrap_err() + .invariant, + "namespace_model_explosion" + ); + impossible.resource_bounds.max_race_states = 8; + assert_eq!( + BehaviorOracle::verify(&impossible, &observe(&impossible)) + .unwrap_err() + .invariant, + "namespace_linearizability" + ); + } + + #[test] + fn slow_branch_release_is_proved_causally_not_by_observation_arrival() { + let parked = "/cases/slow/parked"; + let release = "/cases/slow/release"; + let await_blob = |path: &str| { + Action::ok(ActionOp::AwaitEntry { + path: path.to_owned(), + expected_kind: "blob".to_owned(), + }) + }; + let mut case = case(vec![ + program( + "slow", + vec![ + Action::ok(publication(parked, b"")), + await_blob(release), + Action::ok(publication("/cases/slow/completed", b"released")), + ], + &[], + ), + program( + "healthy", + vec![ + await_blob(parked), + Action::ok(publication("/cases/slow/healthy", b"work")), + ], + &[], + ), + program( + "release", + vec![await_blob(parked), Action::ok(publication(release, b""))], + &["healthy"], + ), + ]); + case.failure = FailureInjection::SlowProcess { + process: "slow".to_owned(), + parked_path: parked.to_owned(), + release_path: release.to_owned(), + }; + let mut observation = observe(&case); + for execution in &mut observation.executions { + for result in &mut execution.results { + if result.action == "lookup" { + result.kind = Some("blob".to_owned()); + result.revision = Some(if result.path == parked { 100 } else { 102 }); + result.active = Some(false); + } + } + } + BehaviorOracle::verify(&case, &observation).unwrap(); + observation.executions.reverse(); + BehaviorOracle::verify(&case, &observation).unwrap(); + let slow = observation + .executions + .iter_mut() + .find(|execution| execution.process == "slow") + .unwrap(); + slow.results[1].revision = Some(99); + assert_eq!( + BehaviorOracle::verify(&case, &observation) + .unwrap_err() + .invariant, + "namespace_linearizability" + ); + let scoped = case.for_attempt(7, true); + scoped.validate().unwrap(); + assert!(matches!(&scoped.failure, + FailureInjection::SlowProcess { parked_path, release_path, .. } + if parked_path == scoped.processes[0].actions[0].operation.path() + && release_path == scoped.processes[0].actions[1].operation.path() + && parked_path != parked + )); + case.processes[2].depends_on.push("slow".to_owned()); + assert!( + case.validate().is_err(), + "release cannot depend on the parked process" + ); + assert!( + serde_json::from_value::(serde_json::json!({ + "type": "slow_process", "process": "slow", "delay_ms": 1000 + })) + .is_err(), + "elapsed-only legacy cases must not silently change meaning" + ); + } + + #[test] + fn ordinary_publication_replacement_accepts_either_last_commit() { + for bytes in [b"a".as_slice(), b"b".as_slice()] { + let case = case(vec![ + program( + "a", + vec![Action::ok(publication("/cases/race/name", b"a"))], + &[], + ), + program( + "b", + vec![Action::ok(publication("/cases/race/name", b"b"))], + &[], + ), + program( + "reader", + vec![Action::ok(ActionOp::ReadBlob { + path: "/cases/race/name".to_owned(), + expected: bytes.to_vec(), + })], + &["a", "b"], + ), + ]); + let mut observation = observe(&case); + assert!(BehaviorOracle::verify(&case, &observation).is_ok()); + observation.executions.reverse(); + assert!(BehaviorOracle::verify(&case, &observation).is_ok()); + } + } + + #[test] + fn unobserved_stream_incarnations_cannot_pass_namespace_race_verification() { + let case = case(vec![ + program( + "a", + vec![Action::ok(ActionOp::StreamWrite { + path: "/cases/race/stream".to_owned(), + chunks: vec![b"a".to_vec()], + replace: false, + })], + &[], + ), + program( + "b", + vec![Action::ok(ActionOp::StreamWrite { + path: "/cases/race/stream".to_owned(), + chunks: vec![b"a".to_vec()], + replace: true, + })], + &[], + ), + ]); + let mut observation = observe(&case); + for execution in &mut observation.executions { + execution.results[0].length = Some(1); + execution.results[0].digest = Some(digest(b"a")); + } + assert_eq!( + BehaviorOracle::verify(&case, &observation) + .unwrap_err() + .invariant, + "namespace_model_evidence" + ); + } + + fn stream_observation( + case: &BehaviorCase, + identities: &[(&str, usize, u64)], + ) -> CaseObservation { + let mut observation = observe(case); + for execution in &mut observation.executions { + let program = case + .processes + .iter() + .find(|program| program.id == execution.process) + .unwrap(); + let mut records = Vec::new(); + for mut result in std::mem::take(&mut execution.results) { + let action = &program.actions[result.step].operation; + let Some((_, _, own_chunks)) = stream_parameters(action) else { + records.push(result); + continue; + }; + result.incarnation = identities + .iter() + .find(|(process, step, _)| { + *process == execution.process && *step == result.step + }) + .map(|(_, _, incarnation)| *incarnation); + let expected = match action { + ActionOp::StreamRead { expected, .. } + | ActionOp::StreamReadWithRetry { expected, .. } + | ActionOp::GatedStreamRead { expected, .. } + | ActionOp::StreamReadInto { expected, .. } => expected.clone(), + _ => own_chunks.unwrap().concat(), + }; + let chunks = own_chunks.or_else(|| { + case.processes + .iter() + .flat_map(|source| { + source + .actions + .iter() + .enumerate() + .map(move |(step, action)| (source, step, action)) + }) + .filter_map(|(source, step, source_action)| { + let (_, _, chunks) = stream_parameters(&source_action.operation)?; + let chunks = chunks?; + if namespace_path(source, source_action.operation.path()) + != namespace_path(program, action.path()) + { + return None; + } + let identity = identities + .iter() + .find(|(id, known_step, _)| *id == source.id && *known_step == step) + .map(|(_, _, incarnation)| *incarnation); + Some((identity != result.incarnation, chunks)) + }) + .min_by_key(|(different, _)| *different) + .map(|(_, chunks)| chunks) + }); + let fallback = vec![expected.clone()]; + let complete = result.outcome == "ok"; + if complete { + result.length = Some(expected.len()); + result.digest = Some(digest(&expected)); + } else { + result.length = None; + result.digest = None; + } + let frames = if complete { + chunks.unwrap_or(&fallback) + } else { + &[] + }; + records.extend(framed_records(result, action, frames, complete)); + } + execution.results = records; + } + observation + } + + fn framed_records( + mut terminal: ActionObservation, + action: &ActionOp, + chunks: &[Vec], + complete: bool, + ) -> Vec { + let Some(incarnation) = terminal.incarnation else { + return vec![terminal]; + }; + if !complete && !chunks.is_empty() { + terminal.transfer = Some(TransferObservation { + length: chunks.iter().map(Vec::len).sum(), + digest: digest(&chunks.concat()), + complete: false, + }); + } + let mut records = Vec::new(); + let mut emit = |barrier: BarrierObservation, direction: Option<&str>| { + let mut record = terminal.clone(); + record.outcome = "barrier".to_owned(); + record.errno = None; + record.error_type = None; + record.length = None; + record.digest = None; + record.transfer = None; + record.barrier = Some(barrier); + if let Some(direction) = direction { + record.action = direction.to_owned(); + } + records.push(record); + }; + emit(BarrierObservation::StreamOpened { incarnation }, None); + let (source, _, _) = stream_parameters(action).unwrap(); + if source { + for (index, chunk) in chunks.iter().enumerate() { + emit( + BarrierObservation::StreamFrame { + incarnation, + index: index as u64, + length: chunk.len() as u64, + digest: digest(chunk), + }, + Some("stream_write"), + ); + if index == 0 + && let ActionOp::GatedStreamWrite { release_path, .. } = action + { + emit( + BarrierObservation::ReleaseObserved { + path: release_path.clone(), + }, + None, + ); + } + } + } + if !source || matches!(action, ActionOp::StreamRoundTrip { .. }) { + for (index, chunk) in chunks.iter().enumerate() { + emit( + BarrierObservation::StreamFrame { + incarnation, + index: index as u64, + length: chunk.len() as u64, + digest: digest(chunk), + }, + Some("stream_read"), + ); + if index == 0 { + emit(BarrierObservation::StreamFirstFrame { incarnation }, None); + } + } + if complete { + emit(BarrierObservation::StreamEof { incarnation }, None); + } + } + records.push(terminal); + records + } + + fn stream_result(execution: &mut ExecutionObservation, step: usize) -> &mut ActionObservation { + execution + .results + .iter_mut() + .find(|result| result.step == step && !is_barrier_record(result)) + .unwrap() + } + + fn retag_stream(execution: &mut ExecutionObservation, step: usize, identity: u64) { + for record in execution + .results + .iter_mut() + .filter(|record| record.step == step) + { + record.incarnation = Some(identity); + match &mut record.barrier { + Some( + BarrierObservation::StreamOpened { incarnation } + | BarrierObservation::StreamFrame { incarnation, .. } + | BarrierObservation::StreamFirstFrame { incarnation } + | BarrierObservation::StreamEof { incarnation }, + ) => *incarnation = identity, + _ => {} + } + } + } + + fn stream_roundtrip(path: &str) -> Action { + Action::ok(ActionOp::StreamRoundTrip { + path: path.to_owned(), + chunks: vec![b"a".to_vec(), b"b".to_vec()], + }) + } + + #[test] + fn logical_frame_loss_duplication_order_and_content_are_independent_of_aggregate_bytes() { + let chunks = vec![ + Vec::new(), + b"a".to_vec(), + Vec::new(), + b"bc".to_vec(), + Vec::new(), + ]; + let case = case(vec![ + program( + "writer", + vec![Action::ok(ActionOp::StreamWrite { + path: "/cases/frames/stream".to_owned(), + chunks, + replace: false, + })], + &[], + ), + program( + "reader", + vec![Action::ok(ActionOp::StreamReadInto { + path: "/cases/frames/stream".to_owned(), + expected: b"abc".to_vec(), + buffer_sizes: vec![1, 17, 2], + })], + &[], + ), + ]); + let observed = stream_observation(&case, &[("writer", 0, 20), ("reader", 0, 20)]); + BehaviorOracle::verify(&case, &observed).unwrap(); + let mut reversed = observed.clone(); + reversed.executions.reverse(); + BehaviorOracle::verify(&case, &reversed).unwrap(); + for process in 0..2 { + for corruption in 0..6 { + let mut forged = observed.clone(); + let records = &mut forged.executions[process].results; + let positions = records + .iter() + .enumerate() + .filter_map(|(position, record)| { + matches!(record.barrier, Some(BarrierObservation::StreamFrame { .. })) + .then_some(position) + }) + .collect::>(); + let class = match corruption { + 0 => { + // Empty trailing frames cannot disappear behind the same byte digest. + records.remove(positions[4]); + FailureClass::FrameCountMismatch + } + 1 => { + let mut repeated = records[positions[4]].clone(); + if let Some(BarrierObservation::StreamFrame { index, .. }) = + &mut repeated.barrier + { + *index = 5; + } + records.insert(positions[4] + 1, repeated); + FailureClass::FrameCountMismatch + } + 2 => { + records.swap(positions[1], positions[3]); + FailureClass::FrameOrderMismatch + } + 3 => { + if let Some(BarrierObservation::StreamFrame { digest, .. }) = + &mut records[positions[1]].barrier + { + *digest = super::digest(b"x"); + } + FailureClass::FrameContentMismatch + } + 4 => { + if let Some(BarrierObservation::StreamFrame { length, .. }) = + &mut records[positions[1]].barrier + { + *length = 2; + } + if let Some(BarrierObservation::StreamFrame { length, .. }) = + &mut records[positions[3]].barrier + { + *length = 1; + } + FailureClass::FrameContentMismatch + } + _ => { + if let Some(BarrierObservation::StreamFrame { digest, .. }) = + &mut records[positions[2]].barrier + { + *digest = super::digest(b"not empty"); + } + FailureClass::FrameContentMismatch + } + }; + // Terminal length/digest remain the correct aggregate "abc". + let failure = BehaviorOracle::verify(&case, &forged).unwrap_err(); + assert_eq!(failure.signature.failure_class, class); + } + } + } + + #[test] + fn every_successful_reader_and_roundtrip_requires_its_own_clean_eof() { + let case = case(vec![ + program( + "writer", + vec![Action::ok(ActionOp::StreamWrite { + path: "/cases/eof/stream".to_owned(), + chunks: vec![Vec::new()], + replace: false, + })], + &[], + ), + program( + "reader", + vec![Action::ok(ActionOp::StreamRead { + path: "/cases/eof/stream".to_owned(), + expected: Vec::new(), + })], + &[], + ), + program( + "roundtrip", + vec![stream_roundtrip("/cases/eof/roundtrip")], + &[], + ), + ]); + let observed = stream_observation( + &case, + &[("writer", 0, 20), ("reader", 0, 20), ("roundtrip", 0, 21)], + ); + BehaviorOracle::verify(&case, &observed).unwrap(); + for process in [1, 2] { + let mut incomplete = observed.clone(); + incomplete.executions[process].results.retain(|record| { + !matches!(record.barrier, Some(BarrierObservation::StreamEof { .. })) + }); + assert_eq!( + BehaviorOracle::verify(&case, &incomplete) + .unwrap_err() + .signature + .failure_class, + FailureClass::MissingStreamEof + ); + } + } + + #[test] + fn stream_consumers_require_the_exact_created_identity_and_payload() { + let mut case = case(vec![ + program( + "writer", + vec![Action::ok(ActionOp::StreamWrite { + path: "/cases/race/stream".to_owned(), + chunks: vec![b"a".to_vec(), b"b".to_vec()], + replace: false, + })], + &[], + ), + program( + "reader", + vec![Action::ok(ActionOp::StreamRead { + path: "/cases/race/stream".to_owned(), + expected: b"ab".to_vec(), + })], + &[], + ), + ]); + let observed = stream_observation(&case, &[("writer", 0, 20), ("reader", 0, 20)]); + assert!(BehaviorOracle::verify(&case, &observed).is_ok()); + let expired = Budget::new(std::time::Duration::ZERO); + assert_eq!( + BehaviorOracle::verify_with_budget(&case, &observed, &expired) + .unwrap_err() + .invariant, + "execution_budget", + ); + let invented = stream_observation(&case, &[("writer", 0, 20), ("reader", 0, 21)]); + assert_eq!( + BehaviorOracle::verify(&case, &invented) + .unwrap_err() + .invariant, + "namespace_linearizability" + ); + // Both observations satisfy their own payload assertions, but the + // consumer must still receive the bytes of its matched source. + if let ActionOp::StreamRead { expected, .. } = &mut case.processes[1].actions[0].operation { + *expected = b"ba".to_vec(); + } + let wrong_bytes = stream_observation(&case, &[("writer", 0, 20), ("reader", 0, 20)]); + assert_eq!( + BehaviorOracle::verify(&case, &wrong_bytes) + .unwrap_err() + .invariant, + "namespace_linearizability" + ); + } + + #[test] + fn ring_token_barriers_after_stream_results_do_not_violate_causal_order() { + let case = case(vec![program( + "relay", + vec![Action::ok(ActionOp::StreamRoundTrip { + path: "/cases/race/stream".to_owned(), + chunks: vec![b"a".to_vec(), b"b".to_vec()], + })], + &[], + )]); + let mut observed = stream_observation(&case, &[("relay", 0, 20)]); + assert!(BehaviorOracle::verify(&case, &observed).is_ok()); + for execution in &mut observed.executions { + let mut forwarded = execution.results[0].clone(); + forwarded.outcome = "barrier".to_owned(); + forwarded.errno = None; + forwarded.incarnation = None; + forwarded.barrier = Some(BarrierObservation::TokenForwarded { + token: "ring-token".to_owned(), + lap: 0, + edge_index: 0, + }); + let mut lap = forwarded.clone(); + execution.results.push(forwarded); + lap.barrier = Some(BarrierObservation::LapCompleted { + token: "ring-token".to_owned(), + lap: 1, + }); + execution.results.push(lap); + } + assert!(BehaviorOracle::verify(&case, &observed).is_ok()); + } + + #[test] + fn stream_reopen_and_blob_mutations_share_one_revision_order() { + let case = case(vec![program( + "owner", + vec![ + stream_roundtrip("/cases/race/stream"), + Action::ok(publication("/cases/race/blob", b"blob")), + Action::ok(ActionOp::Lookup { + path: "/cases/race/blob".to_owned(), + expected_kind: "blob".to_owned(), + }), + stream_roundtrip("/cases/race/stream"), + ], + &[], + )]); + let mut observed = stream_observation(&case, &[("owner", 0, 20), ("owner", 3, 22)]); + stream_result(&mut observed.executions[0], 2).revision = Some(21); + assert!(BehaviorOracle::verify(&case, &observed).is_ok()); + // Reusing either a prior incarnation or the intervening blob's + // revision cannot be legalized by generation counts or inequalities. + for invalid in [20, 21] { + retag_stream(&mut observed.executions[0], 3, invalid); + assert_eq!( + BehaviorOracle::verify(&case, &observed) + .unwrap_err() + .invariant, + "namespace_linearizability" + ); + } + } + + #[test] + fn gated_displacement_requires_stale_old_handles_not_new_identities() { + let mut case = case(vec![ + program( + "writer", + vec![Action::error( + ActionOp::GatedStreamWrite { + path: "/cases/race/stream".to_owned(), + replace: false, + frames: vec![b"a".to_vec(), b"b".to_vec()], + release_path: "/cases/race/release".to_owned(), + }, + libc::ESTALE, + )], + &[], + ), + program( + "reader", + vec![Action::error( + ActionOp::GatedStreamRead { + path: "/cases/race/stream".to_owned(), + expected: b"ab".to_vec(), + observed_path: "/cases/race/ready".to_owned(), + retry_attach: false, + park_after_first_frame: false, + }, + libc::ESTALE, + )], + &[], + ), + program( + "replacer", + vec![ + Action::ok(publication("/cases/race/stream", b"replacement")), + Action::ok(publication("/cases/race/release", b"")), + ], + &[], + ), + ]); + let mut observed = stream_observation(&case, &[("writer", 0, 20), ("reader", 0, 20)]); + // Both displaced endpoints prove only the first complete logical frame. + for process in 0..2 { + let terminal = stream_result(&mut observed.executions[process], 0).clone(); + observed.executions[process].results = framed_records( + terminal, + &case.processes[process].actions[0].operation, + &[b"a".to_vec()], + false, + ); + } + assert!(BehaviorOracle::verify(&case, &observed).is_ok()); + let mut corrupt_prefix = observed.clone(); + let transferred = corrupt_prefix.executions[1] + .results + .iter_mut() + .find(|result| !is_barrier_record(result)) + .unwrap(); + transferred.transfer.as_mut().unwrap().digest = digest(b"X"); + assert_eq!( + BehaviorOracle::verify(&case, &corrupt_prefix) + .unwrap_err() + .signature + .failure_class, + FailureClass::PayloadMismatch + ); + for process in 0..2 { + let mut corrupted = observed.clone(); + let frame = corrupted.executions[process] + .results + .iter_mut() + .find_map(|record| match &mut record.barrier { + Some(BarrierObservation::StreamFrame { digest, .. }) => Some(digest), + _ => None, + }) + .unwrap(); + *frame = digest(b"X"); + assert_eq!( + BehaviorOracle::verify(&case, &corrupted) + .unwrap_err() + .signature + .failure_class, + FailureClass::FrameContentMismatch + ); + } + observed.executions.reverse(); + assert!(BehaviorOracle::verify(&case, &observed).is_ok()); + observed.executions.reverse(); + // Release is causally after blob replacement. An old writer cannot + // finish successfully by pretending its whole action preceded it. + case.processes[0].actions[0].expected = ExpectedOutcome::Ok; + let mut terminal = stream_result(&mut observed.executions[0], 0).clone(); + terminal.outcome = "ok".to_owned(); + terminal.errno = None; + terminal.error_type = None; + terminal.length = Some(2); + terminal.digest = Some(digest(b"ab")); + terminal.transfer = None; + observed.executions[0].results = framed_records( + terminal, + &case.processes[0].actions[0].operation, + &[b"a".to_vec(), b"b".to_vec()], + true, + ); + assert_eq!( + BehaviorOracle::verify(&case, &observed) + .unwrap_err() + .invariant, + "namespace_linearizability" + ); + } + + #[test] + fn pending_stream_rename_is_busy_and_quiescent_rename_changes_revision() { + let mut case = case(vec![ + program( + "writer", + vec![Action::ok(ActionOp::GatedStreamWrite { + path: "/cases/race/stream".to_owned(), + replace: false, + frames: vec![b"a".to_vec(), b"b".to_vec()], + release_path: "/cases/race/release".to_owned(), + })], + &[], + ), + program( + "reader", + vec![Action::ok(ActionOp::GatedStreamRead { + path: "/cases/race/stream".to_owned(), + expected: b"ab".to_vec(), + observed_path: "/cases/race/ready".to_owned(), + retry_attach: false, + park_after_first_frame: false, + })], + &[], + ), + program( + "renamer", + vec![ + Action::ok(ActionOp::Lookup { + path: "/cases/race/ready".to_owned(), + expected_kind: "blob".to_owned(), + }), + Action::error( + ActionOp::Rename { + source: "/cases/race/stream".to_owned(), + destination: "/cases/race/moved".to_owned(), + replace: false, + }, + libc::EBUSY, + ), + Action::ok(publication("/cases/race/release", b"")), + Action::ok(ActionOp::WaitForQuiescent { + path: "/cases/race/stream".to_owned(), + }), + Action::ok(ActionOp::Rename { + source: "/cases/race/stream".to_owned(), + destination: "/cases/race/moved".to_owned(), + replace: false, + }), + Action::ok(ActionOp::Lookup { + path: "/cases/race/moved".to_owned(), + expected_kind: "stream".to_owned(), + }), + ], + &[], + ), + ]); + // Global receipts: stream=20, ready=21, release=22, rename=23. + let mut observed = stream_observation(&case, &[("writer", 0, 20), ("reader", 0, 20)]); + let results = &mut observed.executions[2].results; + results[0].revision = Some(21); + results[1].revision = None; + results[3].revision = Some(20); + results[3].kind = Some("stream".to_owned()); + results[3].active = Some(false); + results[4].revision = Some(23); + results[5].revision = Some(23); + results[5].active = Some(false); + assert!(BehaviorOracle::verify(&case, &observed).is_ok()); + // A renamed stream node is a new namespace revision, not its old + // transport incarnation. + observed.executions[2].results[5].revision = Some(20); + assert_eq!( + BehaviorOracle::verify(&case, &observed) + .unwrap_err() + .invariant, + "namespace_linearizability" + ); + // Keep this assertion about behavior rather than a specific error + // string or the private representation of runtime endpoints. + case.processes[2].actions[5].operation = ActionOp::Lookup { + path: "/cases/race/stream".to_owned(), + expected_kind: "stream".to_owned(), + }; + observed.executions[2].results[5].path = "/cases/race/stream".to_owned(); + assert_eq!( + BehaviorOracle::verify(&case, &observed) + .unwrap_err() + .invariant, + "namespace_linearizability" + ); + } + + #[test] + fn stream_type_errors_and_quiescent_unlink_preserve_namespace_state() { + let case = case(vec![program( + "owner", + vec![ + stream_roundtrip("/cases/race/stream"), + Action::error(exclusive("/cases/race/stream", b"x"), libc::ENXIO), + Action::error( + ActionOp::Unlink { + path: "/cases/race/stream".to_owned(), + }, + libc::ENXIO, + ), + Action::ok(ActionOp::WaitForQuiescent { + path: "/cases/race/stream".to_owned(), + }), + Action::ok(ActionOp::Unlink { + path: "/cases/race/stream".to_owned(), + }), + Action::ok(publication("/cases/race/stream", b"blob")), + Action::ok(ActionOp::Lookup { + path: "/cases/race/stream".to_owned(), + expected_kind: "blob".to_owned(), + }), + Action::exception( + ActionOp::StreamWrite { + path: "/cases/race/stream".to_owned(), + chunks: vec![b"x".to_vec()], + replace: false, + }, + PythonException::StreamError, + ), + ], + &[], + )]); + let mut observed = stream_observation(&case, &[("owner", 0, 20)]); + stream_result(&mut observed.executions[0], 2).revision = None; + let probe = stream_result(&mut observed.executions[0], 3); + probe.revision = Some(20); + probe.kind = Some("stream".to_owned()); + probe.active = Some(false); + stream_result(&mut observed.executions[0], 4).revision = Some(21); + stream_result(&mut observed.executions[0], 6).revision = Some(22); + BehaviorOracle::verify(&case, &observed).unwrap(); + // A pre-open type error must not claim it attached to the old stream. + let terminal = stream_result(&mut observed.executions[0], 7); + terminal.incarnation = Some(20); + assert_eq!( + BehaviorOracle::verify(&case, &observed) + .unwrap_err() + .invariant, + "namespace_model_evidence" + ); + } + + #[test] + fn blocked_disjoint_lookup_cannot_prune_the_history_that_enables_it() { + let case = case(vec![ + program( + "waiter", + vec![Action::ok(ActionOp::Lookup { + path: "/cases/race/ready".to_owned(), + expected_kind: "blob".to_owned(), + })], + &[], + ), + program( + "a", + vec![Action::ok(publication("/cases/race/name", b"a"))], + &[], + ), + program( + "b", + vec![Action::ok(publication("/cases/race/name", b"b"))], + &[], + ), + program( + "reader", + vec![ + Action::ok(ActionOp::ReadBlob { + path: "/cases/race/name".to_owned(), + expected: b"a".to_vec(), + }), + Action::ok(publication("/cases/race/ready", b"")), + ], + &["a", "b"], + ), + ]); + // Greedy a-before-b dead-ends. Full replay must explore b-before-a, + // then the reader which enables the initially blocked waiter. Those + // two actions are path-disjoint, but cannot be pruned as commuting. + let mut observed = observe(&case); + observed.executions[0].results[0].revision = Some(30); + assert!(BehaviorOracle::verify(&case, &observed).is_ok()); + observed.executions.reverse(); + assert!(BehaviorOracle::verify(&case, &observed).is_ok()); + } + + #[test] + fn replacement_displaces_pending_open_without_inventing_a_handle_identity() { + let case = case(vec![ + program( + "pending", + vec![Action::error( + ActionOp::StreamWrite { + path: "/cases/race/stream".to_owned(), + chunks: vec![b"old".to_vec()], + replace: false, + }, + libc::ESTALE, + )], + &[], + ), + program( + "replacement", + vec![Action::ok(ActionOp::StreamWrite { + path: "/cases/race/stream".to_owned(), + chunks: vec![b"new".to_vec()], + replace: true, + })], + &[], + ), + program( + "reader", + vec![Action::ok(ActionOp::StreamReadWithRetry { + path: "/cases/race/stream".to_owned(), + expected: b"new".to_vec(), + })], + &[], + ), + ]); + let mut observed = stream_observation(&case, &[("replacement", 0, 20), ("reader", 0, 20)]); + assert!(BehaviorOracle::verify(&case, &observed).is_ok()); + // The pending open must commit before its replacement. Revision one + // leaves no possible earlier nonzero revision for that creation. + retag_stream(&mut observed.executions[1], 0, 1); + retag_stream(&mut observed.executions[2], 0, 1); + assert_eq!( + BehaviorOracle::verify(&case, &observed) + .unwrap_err() + .invariant, + "namespace_linearizability" + ); + } + + #[test] + fn stream_replacement_cannot_reuse_an_attached_incarnation() { + let case = case(vec![ + program( + "old-writer", + vec![Action::error( + ActionOp::StreamWrite { + path: "/cases/race/stream".to_owned(), + chunks: vec![b"old".to_vec()], + replace: false, + }, + libc::ESTALE, + )], + &[], + ), + program( + "old-reader", + vec![Action::error( + ActionOp::StreamRead { + path: "/cases/race/stream".to_owned(), + expected: b"old".to_vec(), + }, + libc::ESTALE, + )], + &[], + ), + program( + "new-writer", + vec![Action::ok(ActionOp::StreamWrite { + path: "/cases/race/stream".to_owned(), + chunks: vec![b"new".to_vec()], + replace: true, + })], + &[], + ), + program( + "new-reader", + vec![Action::ok(ActionOp::StreamRead { + path: "/cases/race/stream".to_owned(), + expected: b"new".to_vec(), + })], + &[], + ), + ]); + let mut observed = stream_observation( + &case, + &[ + ("old-writer", 0, 20), + ("old-reader", 0, 20), + ("new-writer", 0, 21), + ("new-reader", 0, 21), + ], + ); + assert!(BehaviorOracle::verify(&case, &observed).is_ok()); + retag_stream(&mut observed.executions[2], 0, 20); + retag_stream(&mut observed.executions[3], 0, 20); + assert_eq!( + BehaviorOracle::verify(&case, &observed) + .unwrap_err() + .invariant, + "namespace_linearizability" + ); + } + + #[test] + fn stopped_pending_reader_can_cancel_without_an_attached_identity() { + let mut case = case(vec![ + program( + "stopped", + vec![ + Action::ok(ActionOp::StreamRead { + path: "/cases/race/pending".to_owned(), + expected: b"never".to_vec(), + }), + Action::ok(publication("/cases/race/unexecuted", b"never")), + ], + &[], + ), + program( + "observer", + vec![ + Action::ok(ActionOp::Lookup { + path: "/cases/race/pending".to_owned(), + expected_kind: "stream".to_owned(), + }), + Action::error( + ActionOp::Lookup { + path: "/cases/race/unexecuted".to_owned(), + expected_kind: "blob".to_owned(), + }, + libc::ENOENT, + ), + ], + &["stopped"], + ), + ]); + case.failure = FailureInjection::StopProcess { + process: "stopped".to_owned(), + phase: ProcessStopPhase::AfterContextReady, + kill_after_ms: Some(10), + }; + let mut observed = observe(&case); + observed.executions[0].results.clear(); + observed.executions[0].exit_success = false; + observed.executions[0] + .lifecycle + .retain(|event| event != "user_result"); + observed.executions[1].results[0].revision = Some(20); + observed.executions[1].results[0].active = Some(false); + observed.executions[1].results[1].revision = None; + assert!(BehaviorOracle::verify(&case, &observed).is_ok()); + // Cancellation leaves the committed stream name but cannot execute + // the stopped process's subsequent publication. + case.processes[1].actions[1].expected = ExpectedOutcome::Ok; + observed.executions[1].results[1].outcome = "ok".to_owned(); + observed.executions[1].results[1].errno = None; + observed.executions[1].results[1].revision = Some(21); + assert_eq!( + BehaviorOracle::verify(&case, &observed) + .unwrap_err() + .invariant, + "namespace_linearizability" + ); + } + + #[test] + fn empty_logical_frame_releases_the_gate_before_the_tail() { + let case = case(vec![ + program( + "writer", + vec![Action::ok(ActionOp::GatedStreamWrite { + path: "/cases/race/stream".to_owned(), + replace: false, + frames: vec![Vec::new(), b"tail".to_vec()], + release_path: "/cases/race/ready".to_owned(), + })], + &[], + ), + program( + "reader", + vec![Action::ok(ActionOp::GatedStreamRead { + path: "/cases/race/stream".to_owned(), + expected: b"tail".to_vec(), + observed_path: "/cases/race/ready".to_owned(), + retry_attach: false, + park_after_first_frame: false, + })], + &[], + ), + ]); + let observed = stream_observation(&case, &[("writer", 0, 20), ("reader", 0, 20)]); + BehaviorOracle::verify(&case, &observed).unwrap(); + } + + #[test] + fn retry_read_requires_a_real_matching_source_and_current_incarnation() { + let path = "/cases/retry/stream"; + let reader = || { + program( + "reader", + vec![Action::ok(ActionOp::StreamReadWithRetry { + path: path.to_owned(), + expected: b"ab".to_vec(), + })], + &[], + ) + }; + let phantom = case(vec![reader()]); + let no_identity = stream_observation(&phantom, &[]); + assert_eq!( + BehaviorOracle::verify(&phantom, &no_identity) + .unwrap_err() + .signature + .failure_class, + FailureClass::MissingIncarnation + ); + let invented = stream_observation(&phantom, &[("reader", 0, 20)]); + assert_eq!( + BehaviorOracle::verify(&phantom, &invented) + .unwrap_err() + .signature + .failure_class, + FailureClass::MissingStreamSource + ); + + let mut exchange = case(vec![ + program( + "writer", + vec![Action::ok(ActionOp::StreamWrite { + path: path.to_owned(), + chunks: vec![vec![], b"a".to_vec(), vec![], b"b".to_vec()], + replace: false, + })], + &[], + ), + reader(), + ]); + let matching = stream_observation(&exchange, &[("writer", 0, 20), ("reader", 0, 20)]); + BehaviorOracle::verify(&exchange, &matching).unwrap(); + let stale = stream_observation(&exchange, &[("writer", 0, 20), ("reader", 0, 19)]); + let stale_failure = BehaviorOracle::verify(&exchange, &stale).unwrap_err(); + assert_eq!( + stale_failure.signature.failure_class, + FailureClass::ConflictingIncarnation + ); + if let ActionOp::StreamReadWithRetry { path, .. } = + &mut exchange.processes[1].actions[0].operation + { + *path = "/cases/retry/unrelated".to_owned(); + } + let unrelated = stream_observation(&exchange, &[("writer", 0, 20), ("reader", 0, 20)]); + assert_eq!( + BehaviorOracle::verify(&exchange, &unrelated) + .unwrap_err() + .signature + .failure_class, + FailureClass::MissingStreamSource + ); + exchange.processes[1].actions[0].operation = ActionOp::StreamReadWithRetry { + path: path.to_owned(), + expected: b"ba".to_vec(), + }; + let wrong_source = stream_observation(&exchange, &[("writer", 0, 20), ("reader", 0, 20)]); + let content_failure = BehaviorOracle::verify(&exchange, &wrong_source).unwrap_err(); + assert_eq!( + content_failure.signature.failure_class, + FailureClass::StreamSourceContent + ); + assert_eq!(content_failure.invariant, stale_failure.invariant); + assert_ne!(content_failure.signature, stale_failure.signature); + } + + #[test] + fn binding_collision_exceptions_require_the_operation_specific_namespace_state() { + let path = "/cases/collision/name"; + let stream_collision = Action::exception( + ActionOp::StreamWrite { + path: path.to_owned(), + chunks: vec![b"stream".to_vec()], + replace: false, + }, + PythonException::StreamError, + ); + let blob_first = case(vec![program( + "owner", + vec![ + Action::ok(publication(path, b"blob")), + stream_collision.clone(), + ], + &[], + )]); + let mut observed = observe(&blob_first); + BehaviorOracle::verify(&blob_first, &observed).unwrap(); + observed.executions[0].results[1].error_type = Some("SessionError".to_owned()); + assert!(BehaviorOracle::verify(&blob_first, &observed).is_err()); + let absent = case(vec![program("owner", vec![stream_collision], &[])]); + assert_eq!( + BehaviorOracle::verify(&absent, &observe(&absent)) + .unwrap_err() + .invariant, + "namespace_linearizability" + ); + + let stream_first = case(vec![program( + "owner", + vec![ + stream_roundtrip(path), + Action::exception(publication(path, b"blob"), PythonException::StreamError), + Action::error(exclusive(path, b"raw"), libc::ENXIO), + ], + &[], + )]); + let mut observed = stream_observation(&stream_first, &[("owner", 0, 20)]); + BehaviorOracle::verify(&stream_first, &observed).unwrap(); + // A high-level exception cannot masquerade as the raw descriptor errno. + stream_result(&mut observed.executions[0], 2).errno = None; + stream_result(&mut observed.executions[0], 2).error_type = Some("StreamError".to_owned()); + assert!(BehaviorOracle::verify(&stream_first, &observed).is_err()); + } + + #[test] + fn writer_drop_requires_release_without_commit_and_allows_exclusive_reuse() { + let path = "/cases/drop/name"; + let mut drop = exclusive(path, b"uncommitted"); + if let ActionOp::DescriptorWrite { finish, .. } = &mut drop { + *finish = DescriptorFinish::Drop; + } + let mut case = case(vec![program( + "writer", + vec![ + Action::ok(drop), + Action::error( + ActionOp::Lookup { + path: path.to_owned(), + expected_kind: "blob".to_owned(), + }, + libc::ENOENT, + ), + Action::ok(exclusive(path, b"reused")), + Action::ok(ActionOp::ReadBlob { + path: path.to_owned(), + expected: b"reused".to_vec(), + }), + ], + &[], + )]); + let mut observed = observe(&case); + observed.executions[0].results[0].descriptor = Some(DescriptorObservation::Write { + method: DescriptorWriteMethod::Write, + finish: DescriptorFinish::Drop, + terminal_results: vec![], + dropped: true, + reservation_released: true, + }); + BehaviorOracle::verify(&case, &observed).unwrap(); + if let Some(DescriptorObservation::Write { + reservation_released, + .. + }) = &mut observed.executions[0].results[0].descriptor + { + *reservation_released = false; + } + assert_eq!( + BehaviorOracle::verify(&case, &observed) + .unwrap_err() + .signature + .failure_class, + FailureClass::ReservationNotReleased + ); + if let Some(DescriptorObservation::Write { + reservation_released, + .. + }) = &mut observed.executions[0].results[0].descriptor + { + *reservation_released = true; + } + // Even internally consistent claims of accidental publication are illegal. + case.processes[0].actions[1].expected = ExpectedOutcome::Ok; + let published = &mut observed.executions[0].results[1]; + published.outcome = "ok".to_owned(); + published.errno = None; + assert_eq!( + BehaviorOracle::verify(&case, &observed) + .unwrap_err() + .invariant, + "namespace_linearizability" + ); + } + + #[test] + fn descriptor_terminal_error_cannot_hide_a_completed_corrupt_fixture_read() { + let path = "/models/fixture"; + let mut case = case(vec![program( + "reader", + vec![Action::error( + ActionOp::DescriptorRead { + path: path.to_owned(), + flags: libc::O_RDONLY, + expected: b"a".to_vec(), + method: DescriptorReadMethod::Read, + offset: 0, + finish: DescriptorFinish::CloseTwice, + }, + libc::EBADF, + )], + &[], + )]); + case.read_only_fixture_paths.insert(path.to_owned()); + let mut observed = observe(&case); + let result = &mut observed.executions[0].results[0]; + result.descriptor = Some(DescriptorObservation::Read { + method: DescriptorReadMethod::Read, + finish: DescriptorFinish::CloseTwice, + terminal_results: vec![ + DescriptorTerminalResult::Ok, + DescriptorTerminalResult::Error { + errno: Some(libc::EBADF), + error_type: "OSError".to_owned(), + }, + ], + }); + result.transfer = Some(TransferObservation { + length: 1, + digest: digest(b"a"), + complete: true, + }); + BehaviorOracle::verify(&case, &observed).unwrap(); + observed.executions[0].results[0] + .transfer + .as_mut() + .unwrap() + .digest = digest(b"X"); + assert_eq!( + BehaviorOracle::verify(&case, &observed) + .unwrap_err() + .signature + .failure_class, + FailureClass::PayloadMismatch + ); + observed.executions[0].results[0].transfer = None; + assert_eq!( + BehaviorOracle::verify(&case, &observed) + .unwrap_err() + .signature + .failure_class, + FailureClass::MissingTransfer + ); + // A failure before open/read has neither terminal success nor invented payload. + observed.executions[0].results[0].descriptor = None; + BehaviorOracle::verify(&case, &observed).unwrap(); + } + + #[test] + fn typed_failure_signature_ignores_attempt_identity_but_distinguishes_evidence_failures() { + let mut case = case(vec![program( + "reader", + vec![Action::ok(ActionOp::StreamReadWithRetry { + path: "/cases/original/stream".to_owned(), + expected: b"a".to_vec(), + })], + &[], + )]); + let original = BehaviorOracle::verify(&case, &stream_observation(&case, &[])).unwrap_err(); + case.id = "fresh-attempt".to_owned(); + case.processes[0].id = "fresh-process".to_owned(); + if let ActionOp::StreamReadWithRetry { path, .. } = + &mut case.processes[0].actions[0].operation + { + *path = "/cases/fresh/stream".to_owned(); + } + let mut fresh_observation = stream_observation(&case, &[]); + let fresh = BehaviorOracle::verify(&case, &fresh_observation).unwrap_err(); + assert_eq!(original.signature, fresh.signature); + fresh_observation.executions[0].results[0].incarnation = Some(0); + let zero = BehaviorOracle::verify(&case, &fresh_observation).unwrap_err(); + assert_eq!(original.invariant, zero.invariant); + assert_ne!(original.signature, zero.signature); + let serialized = serde_json::to_vec(&fresh.signature).unwrap(); + assert_eq!( + serde_json::from_slice::(&serialized).unwrap(), + fresh.signature + ); + } + + #[test] + fn typed_action_abort_signatures_preserve_outcomes_not_attempt_identities() { + let mut case = case(vec![program( + "owner", + vec![Action::ok(ActionOp::Lookup { + path: "/cases/abort/name".to_owned(), + expected_kind: "blob".to_owned(), + })], + &[], + )]); + let abort = |case: &BehaviorCase, errno: i32, error_type: &str| { + let mut observation = observe(case); + let execution = &mut observation.executions[0]; + execution.exit_success = false; + execution.exit_status = Some(r#"{"kind":"code","value":1}"#.to_owned()); + let result = &mut execution.results[0]; + result.outcome = "error".to_owned(); + result.errno = Some(errno); + result.error_type = Some(error_type.to_owned()); + result.error = Some(format!("failure in attempt {}", case.id)); + BehaviorOracle::verify(case, &observation) + .unwrap_err() + .signature + }; + let missing = abort(&case, libc::ENOENT, "FileNotFoundError"); + let denied = abort(&case, libc::EACCES, "PermissionError"); + assert_eq!(missing.failure_class, FailureClass::OutcomeMismatch); + assert_ne!(missing, denied); + assert_ne!(missing, abort(&case, libc::ENOENT, "OSError")); + assert_ne!(missing, abort(&case, libc::EACCES, "FileNotFoundError")); + case.id = "another-attempt".to_owned(); + case.processes[0].id = "another-process".to_owned(); + case.processes[0].access.execution_id = "another-execution".to_owned(); + assert_eq!(missing, abort(&case, libc::ENOENT, "FileNotFoundError")); + case.processes[0].actions[0].expected = ExpectedOutcome::Error(libc::ENOENT); + let expected_missing = abort(&case, libc::EIO, "OSError"); + case.processes[0].actions[0].expected = ExpectedOutcome::Error(libc::EACCES); + assert_ne!(expected_missing, abort(&case, libc::EIO, "OSError")); + let encoded = serde_json::to_vec(&missing).unwrap(); + assert_eq!( + serde_json::from_slice::(&encoded).unwrap(), + missing + ); + } + + #[test] + fn typed_action_abort_cannot_hide_a_misplaced_later_sibling() { + let case = case(vec![ + program( + "failed", + vec![Action::ok(ActionOp::Lookup { + path: "/cases/abort/name".to_owned(), + expected_kind: "blob".to_owned(), + })], + &[], + ), + program( + "sibling", + vec![Action::ok(publication("/cases/abort/sibling", b"value"))], + &[], + ), + ]); + let mut observation = observe(&case); + let execution = &mut observation.executions[0]; + execution.exit_success = false; + execution.exit_status = Some(r#"{"kind":"code","value":1}"#.to_owned()); + execution.results[0].outcome = "error".to_owned(); + execution.results[0].errno = Some(libc::ENOENT); + execution.results[0].error_type = Some("FileNotFoundError".to_owned()); + assert_eq!( + BehaviorOracle::verify(&case, &observation) + .unwrap_err() + .signature + .failure_class, + FailureClass::OutcomeMismatch + ); + observation.executions[1].logical_node_id = 2; + assert_eq!( + BehaviorOracle::verify(&case, &observation) + .unwrap_err() + .invariant, + "execution_placement" + ); + } + + #[test] + fn self_scoped_observation_cannot_be_replayed_from_another_attempt() { + let template = case(vec![program( + "owner", + vec![Action::ok(publication("/runs/self/blob", b"value"))], + &[], + )]); + let earlier = template.for_attempt(7, true); + let later = template.for_attempt(8, true); + let stale = observe(&earlier); + BehaviorOracle::verify(&earlier, &stale).unwrap(); + BehaviorOracle::verify(&later, &observe(&later)).unwrap(); + assert_eq!( + BehaviorOracle::verify(&later, &stale) + .unwrap_err() + .invariant, + "execution_identity" + ); + } + + #[test] + fn typed_peer_loss_requires_an_observed_stopped_attached_retry_reader() { + let path = "/cases/peer/stream"; + let mut case = case(vec![ + program( + "writer", + vec![Action::exception( + ActionOp::StreamWrite { + path: path.to_owned(), + chunks: vec![b"a".to_vec(), b"b".to_vec()], + replace: false, + }, + PythonException::StreamError, + )], + &[], + ), + program( + "reader", + vec![Action::ok(ActionOp::StreamReadWithRetry { + path: path.to_owned(), + expected: b"ab".to_vec(), + })], + &[], + ), + ]); + case.failure = FailureInjection::StopProcess { + process: "reader".to_owned(), + phase: ProcessStopPhase::AfterContextReady, + kill_after_ms: Some(10), + }; + let mut observed = stream_observation(&case, &[("writer", 0, 20), ("reader", 0, 20)]); + let writer_terminal = observed.executions[0].results.pop().unwrap(); + observed.executions[0].results = framed_records( + writer_terminal, + &case.processes[0].actions[0].operation, + &[b"a".to_vec()], + false, + ); + let stopped = &mut observed.executions[1]; + let first_frame = stopped + .results + .iter() + .position(|record| { + matches!( + record.barrier, + Some(BarrierObservation::StreamFirstFrame { .. }) + ) + }) + .unwrap(); + stopped.results.truncate(first_frame + 1); + stopped.exit_success = false; + BehaviorOracle::verify(&case, &observed).unwrap(); + + case.failure = FailureInjection::None; + let mut impossible = stream_observation(&case, &[("writer", 0, 20), ("reader", 0, 20)]); + let writer_terminal = stream_result(&mut impossible.executions[0], 0).clone(); + // Matching sent/received frames cannot justify peer loss without a stopped reader. + impossible.executions[0].results = framed_records( + writer_terminal, + &case.processes[0].actions[0].operation, + &[b"a".to_vec(), b"b".to_vec()], + false, + ); + let violation = BehaviorOracle::verify(&case, &impossible).unwrap_err(); + assert_eq!(violation.invariant, "namespace_linearizability"); + assert_eq!( + violation.signature.failure_class, + FailureClass::NamespaceHistoryConflict + ); + } + + #[test] + fn zero_frame_peer_loss_allows_a_writer_stopped_before_its_opened_record() { + let path = "/cases/peer/zero-frame"; + let mut case = case(vec![ + program( + "writer", + vec![Action::ok(ActionOp::StreamWrite { + path: path.to_owned(), + chunks: vec![b"unsent".to_vec()], + replace: false, + })], + &[], + ), + program( + "reader", + vec![Action::exception( + ActionOp::StreamRead { + path: path.to_owned(), + expected: b"unsent".to_vec(), + }, + PythonException::StreamError, + )], + &[], + ), + ]); + case.failure = FailureInjection::StopProcess { + process: "writer".to_owned(), + phase: ProcessStopPhase::AfterContextReady, + kill_after_ms: Some(0), + }; + let mut observed = stream_observation(&case, &[("reader", 0, 20)]); + observed.executions[0].results.clear(); + observed.executions[0].exit_success = false; + BehaviorOracle::verify(&case, &observed).unwrap(); + observed.executions.reverse(); + BehaviorOracle::verify(&case, &observed).unwrap(); + case.failure = FailureInjection::None; + assert!(BehaviorOracle::verify(&case, &observed).is_err()); + } + + #[test] + fn first_frame_stop_waits_for_both_matched_endpoint_milestones() { + let path = "/cases/stop-gate/stream"; + let case = case(vec![ + program( + "writer", + vec![Action::ok(ActionOp::StreamWrite { + path: path.to_owned(), + chunks: vec![b"first".to_vec()], + replace: false, + })], + &[], + ), + program( + "reader", + vec![Action::ok(ActionOp::StreamRead { + path: path.to_owned(), + expected: b"first".to_vec(), + })], + &[], + ), + ]); + let observed = stream_observation(&case, &[("writer", 0, 20), ("reader", 0, 20)]); + let ready = |observation: &CaseObservation| { + stream_stop_ready( + &case, + "writer", + ProcessStopPhase::AfterStreamFirstFrame, + observation.executions.iter().map(|execution| { + ( + execution.process.as_str(), + execution.results.as_slice(), + execution.lifecycle.as_slice(), + ) + }), + ) + }; + assert!(ready(&observed)); + let mut writer_only = observed.clone(); + writer_only.executions[1].results.retain(|record| { + !matches!( + record.barrier, + Some(BarrierObservation::StreamFirstFrame { .. }) + ) + }); + assert!(!ready(&writer_only)); + let mut different_session = observed.clone(); + retag_stream(&mut different_session.executions[1], 0, 21); + assert!(!ready(&different_session)); + let mut reader_only = observed; + reader_only.executions[0].results.retain(|record| { + !matches!(record.barrier, Some(BarrierObservation::StreamFrame { .. })) + }); + assert!(!ready(&reader_only)); + } + + #[test] + fn sibling_first_frame_stop_matches_the_gated_step_not_an_earlier_stream() { + let prior = "/cases/stop-gate/prior"; + let gated = "/cases/stop-gate/gated"; + let case = case(vec![ + program( + "target", + vec![Action::ok(publication( + "/cases/stop-gate/target", + b"target", + ))], + &[], + ), + program( + "writer", + [prior, gated] + .into_iter() + .map(|path| { + Action::ok(ActionOp::StreamWrite { + path: path.to_owned(), + chunks: vec![b"first".to_vec()], + replace: false, + }) + }) + .collect(), + &[], + ), + program( + "reader", + vec![ + Action::ok(ActionOp::StreamRead { + path: prior.to_owned(), + expected: b"first".to_vec(), + }), + Action::ok(ActionOp::GatedStreamRead { + path: gated.to_owned(), + expected: b"first".to_vec(), + observed_path: "/cases/stop-gate/observed".to_owned(), + retry_attach: false, + park_after_first_frame: false, + }), + ], + &[], + ), + ]); + let observed = stream_observation( + &case, + &[ + ("writer", 0, 20), + ("reader", 0, 20), + ("writer", 1, 21), + ("reader", 1, 21), + ], + ); + let ready = |observation: &CaseObservation| { + stream_stop_ready( + &case, + "target", + ProcessStopPhase::AfterSiblingStreamFirstFrame, + observation.executions.iter().map(|execution| { + ( + execution.process.as_str(), + execution.results.as_slice(), + execution.lifecycle.as_slice(), + ) + }), + ) + }; + assert!(ready(&observed)); + let mut undrained_writer = observed.clone(); + undrained_writer.executions[1].results.retain(|record| { + record.step != 1 + || matches!( + record.barrier, + Some(BarrierObservation::StreamOpened { .. }) + ) + }); + assert!(!ready(&undrained_writer)); + let mut wrong_incarnation = observed.clone(); + retag_stream(&mut wrong_incarnation.executions[1], 1, 22); + assert!(!ready(&wrong_incarnation)); + let mut wrong_path = observed; + for record in &mut wrong_path.executions[1].results { + if record.step == 1 { + record.path = prior.to_owned(); + } + } + assert!(!ready(&wrong_path)); + } + + #[test] + fn retained_blob_snapshot_constrains_any_legal_race_winner_without_guessing_revisions() { + let path = "/cases/retained/name"; + let case = case(vec![ + program("short", vec![Action::ok(publication(path, b"a"))], &[]), + program("long", vec![Action::ok(publication(path, b"long"))], &[]), + ]); + let observed = observe(&case); + let verify = |retained: BTreeMap| { + BehaviorOracle::verify_retained_blobs_with_budget( + &case, + &observed, + &retained, + &Budget::new(std::time::Duration::from_secs(60)), + ) + }; + verify(BTreeMap::from([(path.to_owned(), (20, 1))])).unwrap(); + verify(BTreeMap::from([(path.to_owned(), (20, 4))])).unwrap(); + assert!(verify(BTreeMap::new()).is_err()); + assert!(verify(BTreeMap::from([(path.to_owned(), (20, 2))])).is_err()); + assert!(verify(BTreeMap::from([(path.to_owned(), (1, 1))])).is_err()); + assert!( + verify(BTreeMap::from([( + "/cases/retained/unrelated".to_owned(), + (20, 1) + )])) + .is_err() + ); + } + + #[test] + fn retained_revisions_order_commits_hidden_behind_stream_completion() { + let stream = "/cases/retained/stream"; + let early = "/cases/retained/early"; + let late = "/cases/retained/late"; + let mut case = case(vec![ + program( + "writer", + vec![ + Action::ok(ActionOp::StreamWrite { + path: stream.to_owned(), + chunks: vec![b"frame".to_vec()], + replace: false, + }), + Action::ok(publication(early, b"a")), + ], + &[], + ), + program( + "reader", + vec![Action::ok(ActionOp::StreamRead { + path: stream.to_owned(), + expected: b"frame".to_vec(), + })], + &[], + ), + program("late", vec![Action::ok(publication(late, b"b"))], &[]), + ]); + case.processes[1].logical_node_id = 2; + case.routes.push(DataRoute { + id: "stream-route".to_owned(), + kind: DataKind::Stream, + edges: vec![DataEdge { + source: 1, + destination: 2, + source_role: "writer".to_owned(), + destination_role: "reader".to_owned(), + path: stream.to_owned(), + }], + join_inputs: Vec::new(), + }); + let observed = stream_observation(&case, &[("writer", 0, 20), ("reader", 0, 20)]); + BehaviorOracle::verify_retained_blobs_with_budget( + &case, + &observed, + &BTreeMap::from([(early.to_owned(), (21, 1)), (late.to_owned(), (22, 1))]), + &Budget::new(std::time::Duration::from_secs(5)), + ) + .unwrap(); + } + + #[test] + fn incremental_chunk_digest_checks_prefix_across_empty_frames() { + let chunks = [vec![], b"ab".to_vec(), vec![], b"cd".to_vec(), vec![]]; + assert_eq!( + payload_summary(chunks.iter().map(Vec::as_slice), None), + Some((4, digest(b"abcd"))) + ); + assert_eq!( + payload_summary(chunks.iter().map(Vec::as_slice), Some(3)), + Some((4, digest(b"abc"))) + ); + assert_eq!( + payload_summary(chunks.iter().map(Vec::as_slice), Some(0)), + Some((4, digest(b""))) + ); + } + + #[test] + fn read_only_quiescent_stream_probe_is_an_initial_fact() { + let path = "/cases/recovery/persisted-stream"; + let mut case = case(vec![program( + "persisted-path-probe", + vec![ + Action::ok(ActionOp::Lookup { + path: path.to_owned(), + expected_kind: "stream".to_owned(), + }), + Action::ok(ActionOp::WaitForQuiescent { + path: path.to_owned(), + }), + ], + &[], + )]); + case.read_only_fixture_paths.insert(path.to_owned()); + let mut observed = observe(&case); + observed.executions[0].results[0].active = Some(false); + let mut quiescent = observed.executions[0].results[0].clone(); + quiescent.step = 1; + observed.executions[0].results[1] = quiescent; + BehaviorOracle::verify(&case, &observed).unwrap(); + } +} + +fn verify_namespace_constructively( + initial: &NamespaceHistory<'_>, + programs: &[Vec>], + dependencies: &[Vec], + revision_hints: &BTreeMap, + budget: Option<&Budget>, + retained: Option<&BTreeMap>, +) -> Result<(), OracleViolation> { + let mut state = initial.clone(); + while !namespace_history_complete(&state, programs) { + check_budget(budget, "namespace constructive replay")?; + // An exact revision anywhere behind a process-local or dependency + // fence is a lower bound for every commit that can run now. Crossing + // it would permanently make that observed authority order impossible. + // Looking only at the current action misses a publish hidden behind a + // read or stream completion in the same process. + let pending_commit_key = programs + .iter() + .enumerate() + .flat_map(|(process, program)| program[state.positions[process]..].iter()) + .filter_map(|action| namespace_commit_key(action, revision_hints)) + .min(); + // Try observed namespace revisions/incarnations first, never collector + // receipt order. An unsuccessful construction falls back to all states. + let mut candidates: Vec<(u64, usize, usize)> = Vec::new(); + for process in 0..programs.len() { + let Some(action) = ready_action(&state, programs, dependencies, process) else { + continue; + }; + candidates.push(( + namespace_action_key(action, revision_hints), + process, + state.positions[process], + )); + } + let commit_floor = pending_commit_key; + if candidates.is_empty() { + if let Some(next) = state.active_streams.keys().find_map(|path| { + namespace_first_frame(&state, path).or_else(|| namespace_quiescence(&state, path)) + }) { + state = next; + continue; + } + return violation( + "namespace_linearizability", + "no namespace history satisfies action results, process order, dependencies and revisions", + ); + } + candidates.sort_unstable(); + let mut progressed = false; + for &(key, process, _) in &candidates { + let action = &programs[process][state.positions[process]]; + let Some(next) = namespace_transition(&state, process, action) + .or_else(|| namespace_interrupted_open(&state, process, action)) + else { + continue; + }; + let appended_key = next + .revisions + .get(state.revisions.len()) + .and_then(|revision| *revision) + .or_else(|| namespace_commit_key(action, revision_hints)) + .unwrap_or(key); + if next.revisions.len() > state.revisions.len() + && commit_floor.is_some_and(|floor| floor < appended_key) + { + continue; + } + // Appending a revision slot advances the shared authority + // counter past every slot committed after it, so a commit must + // never overtake a candidate ordered earlier by its evidence + // that only an asynchronous event holds back. Two-party + // rendezvous blocks (an attach whose counterpart has not + // joined) consume no revision slot and are legitimately + // overtaken; teardown starvation is not, because it strands + // the pinned commit behind the probe that waits for it. + state = if next.revisions.len() > state.revisions.len() + && let Some(yielded) = namespace_async_yield(&state, &candidates, programs, key) + { + yielded + } else { + next + }; + progressed = true; + break; + } + // When every ready operation is waiting on an asynchronous stream + // event, advance the event that makes the earliest evidence-ordered + // operation legal. Arbitrarily advancing the first active path can + // strand a valid history and force an exponential frontier search. + if !progressed + && let Some(next) = namespace_async_yield(&state, &candidates, programs, u64::MAX) + { + state = next; + progressed = true; + } + if !progressed + && let Some(next) = state.active_streams.keys().find_map(|path| { + namespace_first_frame(&state, path).or_else(|| namespace_quiescence(&state, path)) + }) + { + state = next; + progressed = true; + } + if !progressed { + return violation( + "namespace_linearizability", + "no namespace history satisfies action results, process order, dependencies and revisions", + ); + } + } + if namespace_retained_matches(&state, retained) { + Ok(()) + } else { + violation( + "namespace_linearizability", + "final retained blob snapshot has no matching history", + ) + } +} + +// Asynchronous namespace events (endpoint teardown, first-frame delivery) +// consume no revision slot, so deferring one can never help another +// candidate — but it can permanently strand one: a probe that waits for +// teardown holds its process back while unrelated candidates append ever +// higher revision slots, until the commit behind the probe (typically an +// unlink pinned to its observed revision) can no longer fit the counter. +// Before a candidate appends a revision slot, yield instead to any +// available event that demonstrably unblocks a candidate whose observed +// evidence orders it before that commit. +fn namespace_async_yield<'a>( + state: &NamespaceHistory<'a>, + candidates: &[(u64, usize, usize)], + programs: &[Vec>], + commit_key: u64, +) -> Option> { + for &(key, process, _) in candidates { + if key >= commit_key { + break; + } + let action = &programs[process][state.positions[process]]; + if namespace_transition(state, process, action) + .or_else(|| namespace_interrupted_open(state, process, action)) + .is_some() + { + continue; + } + for path in state.active_streams.keys() { + for event in [ + namespace_quiescence(state, path), + namespace_first_frame(state, path), + ] + .into_iter() + .flatten() + { + if namespace_transition(&event, process, action) + .or_else(|| namespace_interrupted_open(&event, process, action)) + .is_some() + { + return Some(event); + } + } + } + } + None +} diff --git a/tools/myelin-e2e-fuzz/src/remote.rs b/tools/myelin-e2e-fuzz/src/remote.rs new file mode 100644 index 0000000..d045bb1 --- /dev/null +++ b/tools/myelin-e2e-fuzz/src/remote.rs @@ -0,0 +1,1744 @@ +//! Paid VastAI campaign admission, durable ownership, and deny-only state. + +use std::collections::BTreeSet; +use std::fs::{self, File}; +use std::io::Write as _; +use std::path::{Path, PathBuf}; +use std::time::Duration; +use std::time::{SystemTime, UNIX_EPOCH}; + +use swactor_vastai::{BlockingVastClient, VastClient}; + +use myelin_control_contract::{ControlReply, Offer, OfferSearchRequest, OfferSearchResults}; +use provisioning::{ + PaidAdmissionMode, PaidAdmissionSnapshot, PaidCleanupLimits, PaidCleanupOwnership, + PaidFixtureAdmission, PaidProcessIdentity, +}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::campaign::{ + ARTIFACT_SCHEMA_VERSION, ArtifactEnvelope, CampaignConfig, CampaignLimits, CampaignPlan, +}; + +pub const PAID_STATE_SCHEMA_VERSION: u32 = 1; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PaidCampaignPhase { + Planned, + Acquiring, + Prepared, + DenyOnly, + Quarantined, + CleanupOnly, + Complete, +} + +impl PaidCampaignPhase { + pub const fn permits_acquisition(self) -> bool { + matches!(self, Self::Planned | Self::Acquiring) + } + + pub const fn permits_workload(self) -> bool { + matches!(self, Self::DenyOnly) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct AcquisitionRecord { + pub sequence: u32, + pub operation: String, + pub expected_offer_ids: Vec, + pub expected_labels: BTreeSet, + pub observed_contract_ids: BTreeSet, + pub completed: bool, + pub written_unix_ms: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PaidCampaignState { + pub schema_version: u32, + pub campaign_id: String, + pub phase: PaidCampaignPhase, + pub state_dir: PathBuf, + pub selected_offer_ids: Vec, + pub owned_labels: BTreeSet, + pub owned_contract_ids: BTreeSet, + pub acquisition_count: u32, + pub bootstrap_count: u32, + pub records: Vec, + pub quarantine_reason: Option, + #[serde(default)] + pub cleanup_ownership: Option, +} + +impl PaidCampaignState { + pub fn planned(campaign_id: impl Into, state_dir: PathBuf) -> Self { + Self { + schema_version: PAID_STATE_SCHEMA_VERSION, + campaign_id: campaign_id.into(), + phase: PaidCampaignPhase::Planned, + state_dir, + selected_offer_ids: Vec::new(), + owned_labels: BTreeSet::new(), + owned_contract_ids: BTreeSet::new(), + acquisition_count: 0, + bootstrap_count: 0, + records: Vec::new(), + quarantine_reason: None, + cleanup_ownership: None, + } + } + + pub fn validate(&self) -> Result<(), String> { + if self.schema_version != PAID_STATE_SCHEMA_VERSION { + return Err(format!( + "paid state schema {} is unsupported; expected {}", + self.schema_version, PAID_STATE_SCHEMA_VERSION + )); + } + if self.campaign_id.is_empty() { + return Err("paid state campaign id is empty".to_owned()); + } + if self.acquisition_count > 5 || self.bootstrap_count > 5 { + return Err( + "paid state exceeds the five-node acquisition/bootstrap ceiling".to_owned(), + ); + } + if !self.phase.permits_acquisition() + && self.records.iter().any(|record| !record.completed) + && !matches!( + self.phase, + PaidCampaignPhase::CleanupOnly | PaidCampaignPhase::Quarantined + ) + { + return Err("deny-only state retains an unresolved acquisition record".to_owned()); + } + if matches!( + self.phase, + PaidCampaignPhase::Prepared | PaidCampaignPhase::DenyOnly + ) && (self.acquisition_count != 5 + || self.bootstrap_count != 5 + || !(3..=5).contains(&self.owned_contract_ids.len()) + || self.owned_labels.len() != 5) + { + return Err( + "prepared fixture does not own three to five surviving contracts and five labels" + .to_owned(), + ); + } + Ok(()) + } + + pub fn begin_acquisition( + &mut self, + offer_ids: Vec, + labels: BTreeSet, + ) -> Result<(), String> { + if !self.phase.permits_acquisition() { + return Err(format!( + "phase {:?} denies provider acquisition", + self.phase + )); + } + if offer_ids.len() != 5 || labels.len() != 5 { + return Err("paid acquisition requires five exact offers and labels".to_owned()); + } + if !self.records.is_empty() { + return Err("paid acquisition may be attempted only once".to_owned()); + } + self.phase = PaidCampaignPhase::Acquiring; + self.selected_offer_ids = offer_ids.clone(); + self.owned_labels = labels.clone(); + self.records.push(AcquisitionRecord { + sequence: 1, + operation: "create-five-exact-contracts".to_owned(), + expected_offer_ids: offer_ids, + expected_labels: labels, + observed_contract_ids: BTreeSet::new(), + completed: false, + written_unix_ms: unix_ms()?, + }); + Ok(()) + } + + pub fn reconcile_prepared( + &mut self, + admission: &PaidAdmissionSnapshot, + contracts: BTreeSet, + labels: BTreeSet, + ) -> Result<(), String> { + if self.phase != PaidCampaignPhase::Acquiring { + return Err("only an acquiring campaign can become prepared".to_owned()); + } + if contracts.len() != 5 || labels != self.owned_labels { + return Err(format!( + "acquisition reconciliation expected five exact labels/contracts: labels={labels:?}, contracts={contracts:?}" + )); + } + self.reconcile_admission(admission)?; + if admission.mode != PaidAdmissionMode::Prepared + || admission + .contracts + .values() + .copied() + .collect::>() + != contracts + || admission + .nodes + .iter() + .map(|node| node.label.clone()) + .collect::>() + != labels + || admission.create_reservations.len() != 5 + || admission.initial_bootstraps.len() != 5 + || !admission.rejected_creates.is_empty() + || !admission.accounting_errors.is_empty() + { + return Err( + "prepared fixture lacks actual five-creation/five-bootstrap admission evidence" + .to_owned(), + ); + } + let record = self + .records + .last_mut() + .ok_or_else(|| "acquisition record is missing".to_owned())?; + record.observed_contract_ids = contracts.clone(); + record.completed = true; + record.written_unix_ms = unix_ms()?; + self.owned_contract_ids = contracts; + self.acquisition_count = admission.create_reservations.len() as u32; + self.bootstrap_count = admission.initial_bootstraps.len() as u32; + // The provider admission is already permanently sealed. Commit the + // runner's prepared accounting directly as deny-only so no crash can + // leave a durable workload state with acquisition still in transition. + self.phase = PaidCampaignPhase::DenyOnly; + self.validate() + } + + pub fn quarantine(&mut self, reason: impl Into) { + self.phase = PaidCampaignPhase::Quarantined; + self.quarantine_reason = Some(reason.into()); + } + pub fn resume_deny_only( + &mut self, + admission: &PaidAdmissionSnapshot, + contracts: BTreeSet, + ) -> Result<(), String> { + if self.phase != PaidCampaignPhase::Quarantined { + return Err("only a quarantined paid fixture can be resumed".to_owned()); + } + if !(3..=5).contains(&contracts.len()) { + return Err(format!( + "resumed fixture must retain three to five contracts, found {}", + contracts.len() + )); + } + self.reconcile_admission(admission)?; + if admission.mode != PaidAdmissionMode::Prepared + || !contracts.is_subset(&admission.contracts.values().copied().collect()) + || admission.create_reservations.len() != 5 + || admission.initial_bootstraps.len() != 5 + || !admission.accounting_errors.is_empty() + { + return Err( + "resume requires original five-creation/five-bootstrap admission evidence" + .to_owned(), + ); + } + self.acquisition_count = admission.create_reservations.len() as u32; + self.bootstrap_count = admission.initial_bootstraps.len() as u32; + self.owned_contract_ids = contracts; + self.phase = PaidCampaignPhase::DenyOnly; + self.quarantine_reason = None; + self.validate() + } + + /// Reconcile rather than replace either durable ownership source. Exact IDs + /// survive runner loss, including discoveries that no longer carry labels. + pub fn reconcile_admission(&mut self, admission: &PaidAdmissionSnapshot) -> Result<(), String> { + let labels = admission + .nodes + .iter() + .map(|node| node.label.clone()) + .collect::>(); + if !self.owned_labels.is_empty() && self.owned_labels != labels { + return Err("runner ownership labels disagree with durable admission".to_owned()); + } + self.owned_labels.extend(labels); + self.owned_contract_ids + .extend(admission.known_contract_ids()); + self.cleanup_ownership = admission.cleanup.clone(); + Ok(()) + } + + pub fn enter_cleanup_only(&mut self) { + self.phase = PaidCampaignPhase::CleanupOnly; + } + + pub fn complete_cleanup(&mut self) -> Result<(), String> { + if !self.owned_contract_ids.is_empty() { + return Err("cannot complete cleanup while attributed contracts remain".to_owned()); + } + if self.records.iter().any(|record| !record.completed) { + return Err( + "cannot complete cleanup with unresolved acquisition accounting".to_owned(), + ); + } + self.phase = PaidCampaignPhase::Complete; + self.validate() + } +} + +pub fn offer_search_request(config: &CampaignConfig) -> OfferSearchRequest { + OfferSearchRequest { + gpu_model: config.offers.gpu_model.clone(), + min_gpu_ram_mb: config.offers.min_gpu_ram_mb, + min_compute_cap: config.offers.min_compute_cap, + min_reliability: config.offers.min_reliability, + require_verified: Some(true), + min_download_mbps: config.offers.min_download_mbps, + min_upload_mbps: config.offers.min_upload_mbps, + max_hourly_price: config.offers.max_hourly_price_per_node, + blacklist_hosts: config.offers.blacklist_hosts.clone(), + count: Some(8), + } +} + +pub fn decode_offer_results(response: Value) -> Result { + match serde_json::from_value::(response) + .map_err(|error| format!("decode orchestrator offer reply: {error}"))? + { + ControlReply::Offers(results) => Ok(results), + ControlReply::Rejected(error) => Err(format!("offer search rejected: {error}")), + other => Err(format!("unexpected offer search reply: {other:?}")), + } +} + +pub fn select_exact_offers( + results: &OfferSearchResults, + config: &CampaignConfig, + plan: &CampaignPlan, +) -> Result, String> { + let mut offers = results.offers.clone(); + offers.sort_by(|left, right| { + left.hourly_price + .total_cmp(&right.hourly_price) + .then_with(|| left.offer_id.cmp(&right.offer_id)) + }); + let mut selected = Vec::with_capacity(5); + let mut hosts = BTreeSet::new(); + for offer in offers { + if !offer_is_eligible(&offer, config)? { + continue; + } + let Some(host) = offer.host_id else { + continue; + }; + if hosts.insert(host) { + selected.push(offer); + } + if selected.len() == 5 { + break; + } + } + if selected.len() != 5 { + return Err(format!( + "offer search {} returned only {} eligible distinct verified hosts", + results.search_id, + selected.len() + )); + } + let hourly = selected.iter().map(|offer| offer.hourly_price).sum(); + plan.admit_selected_hourly_cost(&config.limits, hourly)?; + Ok(selected) +} + +fn offer_is_eligible(offer: &Offer, config: &CampaignConfig) -> Result { + for (name, value) in [ + ("hourly_price", Some(offer.hourly_price)), + ("download_cost_per_tb", Some(offer.download_cost_per_tb)), + ("upload_cost_per_tb", Some(offer.upload_cost_per_tb)), + ("gpu_ram_mb", offer.gpu_ram_mb), + ("reliability", offer.reliability), + ("download_mbps", offer.download_mbps), + ("upload_mbps", offer.upload_mbps), + ] { + if value.is_some_and(|value| !value.is_finite() || value < 0.0) { + return Err(format!("offer {} has malformed {name}", offer.offer_id)); + } + } + let Some(host_id) = offer.host_id else { + return Ok(false); + }; + if offer.verification.as_deref() != Some("verified") + || config.offers.blacklist_hosts.contains(&host_id) + || config.offers.gpu_model.as_ref().is_some_and(|required| { + !offer + .gpu_model + .to_ascii_lowercase() + .contains(&required.to_ascii_lowercase()) + }) + || config.offers.min_gpu_ram_mb.is_some_and(|minimum| { + !offer + .gpu_ram_mb + .is_some_and(|value| value >= minimum as f64) + }) + || config + .offers + .min_compute_cap + .is_some_and(|minimum| offer.compute_cap < minimum) + || config + .offers + .min_reliability + .is_some_and(|minimum| !offer.reliability.is_some_and(|value| value >= minimum)) + || config + .offers + .min_download_mbps + .is_some_and(|minimum| !offer.download_mbps.is_some_and(|value| value >= minimum)) + || config + .offers + .min_upload_mbps + .is_some_and(|minimum| !offer.upload_mbps.is_some_and(|value| value >= minimum)) + || config + .offers + .max_hourly_price_per_node + .is_some_and(|maximum| offer.hourly_price > maximum) + { + return Ok(false); + } + Ok(true) +} + +pub fn write_paid_state(path: &Path, state: &PaidCampaignState) -> Result<(), String> { + state.validate()?; + durable_json(path, state) +} + +pub fn read_paid_state(path: &Path) -> Result { + let state = serde_json::from_slice::( + &fs::read(path).map_err(|error| format!("read paid campaign state: {error}"))?, + ) + .map_err(|error| format!("parse paid campaign state: {error}"))?; + state.validate()?; + Ok(state) +} + +pub fn write_campaign_plan(path: &Path, plan: &CampaignPlan) -> Result<(), String> { + plan.validate()?; + let envelope = ArtifactEnvelope::new("campaign-plan", plan.campaign_id.clone(), plan)?; + if envelope.artifact_schema_version != ARTIFACT_SCHEMA_VERSION { + return Err("campaign artifact schema mismatch".to_owned()); + } + durable_json(path, &envelope) +} +pub fn read_campaign_plan(path: &Path) -> Result { + let envelope = + serde_json::from_reader::<_, ArtifactEnvelope>(std::io::BufReader::new( + File::open(path).map_err(|error| format!("read campaign plan: {error}"))?, + )) + .map_err(|error| format!("parse campaign plan: {error}"))?; + envelope.validate("campaign-plan")?; + envelope.payload.validate()?; + if envelope.campaign_id != envelope.payload.campaign_id { + return Err("campaign plan envelope identity differs from its payload".to_owned()); + } + Ok(envelope.payload) +} + +pub fn read_coverage_ledger( + path: &Path, + campaign_id: &str, + phase: &str, +) -> Result { + let envelope = serde_json::from_slice::>( + &fs::read(path).map_err(|error| format!("read {phase} coverage ledger: {error}"))?, + ) + .map_err(|error| format!("parse {phase} coverage ledger: {error}"))?; + envelope.validate(&format!("{phase}-coverage"))?; + if envelope.campaign_id != campaign_id { + return Err(format!( + "{phase} coverage campaign {:?} differs from expected {campaign_id:?}", + envelope.campaign_id + )); + } + Ok(envelope.payload) +} + +pub fn write_coverage_ledger( + path: &Path, + campaign_id: &str, + phase: &str, + ledger: &crate::coverage::CoverageLedger, +) -> Result<(), String> { + let envelope = + ArtifactEnvelope::new(format!("{phase}-coverage"), campaign_id.to_owned(), ledger)?; + durable_json(path, &envelope) +} + +/// Convert the operator ceiling and an hourly price conservatively. +fn cleanup_limits(hourly: f64, limits: &CampaignLimits) -> Result { + let maximum = (limits.total_cost_usd * 1_000_000.0).floor(); + let hourly_micros = (hourly * 1_000_000.0).ceil(); + if !maximum.is_finite() + || !hourly_micros.is_finite() + || maximum < 1.0 + || hourly_micros < 1.0 + || maximum >= u64::MAX as f64 + || hourly_micros >= u64::MAX as f64 + || hourly > limits.total_hourly_price_usd + { + return Err("paid cleanup financial limits are invalid or exceed authorization".to_owned()); + } + Ok(PaidCleanupLimits { + maximum_cost_microusd: maximum as u64, + hourly_price_microusd: hourly_micros as u64, + }) +} + +/// Install cleanup ownership before offer search using the operator's full +/// authorized hourly ceiling. Exact selected prices are checked separately. +pub fn authorized_cleanup_limits(limits: &CampaignLimits) -> Result { + cleanup_limits(limits.total_hourly_price_usd, limits) +} + +/// Convert the operator ceiling and selected rental prices conservatively. +/// Nonzero metered network prices cannot be bounded by rental lifetime alone. +pub fn selected_cleanup_limits( + offers: &[Offer], + limits: &CampaignLimits, +) -> Result { + if offers + .iter() + .any(|offer| offer.download_cost_per_tb != 0.0 || offer.upload_cost_per_tb != 0.0) + { + return Err( + "paid spending ceiling requires zero metered upload/download charges".to_owned(), + ); + } + cleanup_limits(offers.iter().map(|offer| offer.hourly_price).sum(), limits) +} + +/// Install detached restart supervision before provision_selected. Credentials +/// are inherited through the environment, never process arguments. +pub fn start_cleanup_owner( + executable: &Path, + admission_path: &Path, + api_key_env: &str, + limits: PaidCleanupLimits, + preparation_remaining: Duration, +) -> Result<(), String> { + let deadline = std::time::Instant::now() + preparation_remaining.min(Duration::from_secs(10)); + let admission = PaidFixtureAdmission::open(admission_path)?; + let runner = PaidProcessIdentity::current()?; + match admission.snapshot()?.cleanup { + None => admission.install_cleanup_ownership(runner, limits)?, + Some(ownership) if ownership.runner == runner && ownership.limits == limits => {} + Some(_) => { + return Err( + "paid cleanup ownership differs from the immutable run authorization".to_owned(), + ); + } + } + spawn_cleanup_supervisor(executable, admission_path, api_key_env, false, deadline) +} + +/// Recovery is always deny-only. It cannot extend limits or create reservations. +pub fn recover_cleanup_owner( + executable: &Path, + admission_path: &Path, + api_key_env: &str, +) -> Result<(), String> { + let deadline = std::time::Instant::now() + Duration::from_secs(10); + let admission = PaidFixtureAdmission::open(admission_path)?; + admission.enter_cleanup_only()?; + spawn_cleanup_supervisor(executable, admission_path, api_key_env, true, deadline) +} + +fn spawn_cleanup_supervisor( + executable: &Path, + admission_path: &Path, + api_key_env: &str, + recovery: bool, + deadline: std::time::Instant, +) -> Result<(), String> { + use std::process::{Command, Stdio}; + let admission = PaidFixtureAdmission::open(admission_path)?; + let spawn = || -> Result { + let mut command = Command::new(executable); + command + .arg("--paid-cleanup-supervisor") + .arg(admission_path) + .arg("--api-key-env") + .arg(api_key_env) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + // Neither runner exit nor terminal loss takes down supervision. + unsafe { + command.pre_exec(|| { + if libc::setsid() < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + } + #[cfg(not(unix))] + return Err("independent paid cleanup requires Unix process supervision".to_owned()); + command + .spawn() + .map_err(|error| format!("start independent paid cleanup supervisor: {error}")) + }; + let mut child = if admission.cleanup_supervisor_active()? { + None + } else { + Some(spawn()?) + }; + let result = (|| loop { + let state = admission.snapshot()?; + if std::time::Instant::now() < deadline + && recovery + && state + .cleanup + .as_ref() + .is_some_and(|owner| owner.spending_stopped) + { + return Ok(()); + } + let exited = child + .as_mut() + .map(|child| child.try_wait()) + .transpose() + .map_err(|error| format!("observe cleanup supervisor: {error}"))? + .flatten() + .is_some(); + if exited { + child = None; + } + // A previously observed supervisor can finish after renewed cleanup + // invalidates its old absence receipt. Its lock remains the arbiter. + if child.is_none() + && std::time::Instant::now() < deadline + && !admission.cleanup_supervisor_active()? + { + child = Some(spawn()?); + } + let ready = admission.cleanup_supervisor_active()? + && admission.cleanup_owner_active()? + && state.cleanup.as_ref().is_some_and(|owner| { + owner + .supervisor + .as_ref() + .is_some_and(PaidProcessIdentity::is_live) + && owner + .owner + .as_ref() + .is_some_and(PaidProcessIdentity::is_live) + }); + let expired = std::time::Instant::now() >= deadline; + if ready || expired { + if !ready || expired { + admission.enter_cleanup_only()?; + return Err( + "cleanup supervision failed readiness; acquisition remains denied".to_owned(), + ); + } + return if recovery { + Ok(()) + } else { + admission.require_live_cleanup_owner() + }; + } + std::thread::sleep( + Duration::from_millis(25) + .min(deadline.saturating_duration_since(std::time::Instant::now())), + ); + })(); + if let Some(mut child) = child { + // Detach cleanup responsibility, but retain reaping on every return. + std::thread::spawn(move || { + let _ = child.wait(); + }); + } + result +} + +/// Supervises an exact ledger until typed absence is durable. A worker crash +/// restarts cleanup-only immediately, without new limits or reservation slots. +/// This protects owner-process loss, not host loss or loss of both processes. +pub fn run_cleanup_supervisor(admission_path: &Path, api_key_env: &str) -> Result<(), String> { + use std::process::{Command, Stdio}; + let admission = PaidFixtureAdmission::open(admission_path)?; + let _supervisor_lock = admission.claim_cleanup_supervisor()?; + let executable = std::env::current_exe().map_err(|error| error.to_string())?; + let mut first = true; + loop { + let state = admission.snapshot()?; + if state + .cleanup + .as_ref() + .is_some_and(|owner| owner.spending_stopped) + { + return Ok(()); + } + if !first + || state + .cleanup + .as_ref() + .is_some_and(|owner| owner.owner.is_some()) + { + admission.enter_cleanup_only()?; + } + first = false; + // Adopting an already active worker never introduces a rival deletion + // loop; its OS-held lock remains the authoritative exclusion boundary. + while admission.cleanup_owner_active()? { + if admission + .snapshot()? + .cleanup + .as_ref() + .is_some_and(|owner| owner.spending_stopped) + { + return Ok(()); + } + std::thread::sleep(Duration::from_millis(100)); + } + let result = Command::new(&executable) + .arg("--paid-cleanup-owner") + .arg(admission_path) + .arg("--api-key-env") + .arg(api_key_env) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn(); + if let Ok(mut child) = result { + let _ = child.wait(); + } + // Spawn errors and worker exits retain the duty and original ceilings. + // A short bounded backoff prevents a broken executable spinning. + admission.enter_cleanup_only()?; + std::thread::sleep(Duration::from_millis(100)); + } +} + +/// Cleanup worker entrypoint. Only the supervisor restarts this exact duty; +/// ambiguous creates and query failures never discharge ownership. +pub fn run_cleanup_owner(admission_path: &Path, api_key_env: &str) -> Result<(), String> { + let admission = PaidFixtureAdmission::open(admission_path)?; + let owner = admission.claim_cleanup_owner()?; + let api_key = std::env::var(api_key_env) + .map_err(|_| format!("required VastAI credential environment {api_key_env} is unset"))?; + let client = BlockingVastClient::new(VastClient::new(api_key))?; + loop { + let stopping = owner.tick()?; + let state = admission.snapshot()?; + if state + .cleanup + .as_ref() + .is_some_and(|ownership| ownership.complete) + { + return Ok(()); + } + if stopping { + let labels = state.cleanup_labels(); + let known_ids = state.known_contract_ids(); + let unresolved = state.unresolved_create_labels(); + let result = client.cleanup_owned_reconciled( + &labels, + &known_ids, + &unresolved, + Duration::from_secs(20), + &mut |discovered| admission.reconcile_cleanup_discovery(discovered), + ); + match result { + Ok(census) => { + admission.complete_cleanup(&census.missing_contract_ids)?; + return Ok(()); + } + Err(error) => { + if error.accounting_failed { + admission.record_cleanup_accounting_error(error.detail.clone())?; + } + if let Some(census) = error.absence { + admission.record_cleanup_absence(&census.missing_contract_ids)?; + return Err(error.detail); + } + } + } + } + std::thread::sleep(Duration::from_millis(250)); + } +} + +/// Loopback-only lifecycle fixture: the scripted provider supplies existing +/// contracts, while real admission and detached processes exercise retention. +/// This does not claim SSH/runtime readiness or perform provider acquisition. +pub fn run_scripted_retained_lifecycle( + admission_path: &Path, + api_key_env: &str, +) -> Result<(), String> { + let endpoint = std::env::var("VASTAI_BASE_URL").unwrap_or_default(); + if endpoint + .strip_prefix("http://127.0.0.1:") + .and_then(|port| port.parse::().ok()) + .is_none_or(|port| port == 0) + { + return Err("scripted retained lifecycle requires a literal loopback endpoint".to_owned()); + } + let generation = provisioning::plugin::DeploymentIdentity { + artifact_digest: format!("sha256:{}", "a".repeat(64)), + deployment_generation: "scripted-retained-generation".to_owned(), + }; + let admission = if admission_path + .try_exists() + .map_err(|error| error.to_string())? + { + let admission = PaidFixtureAdmission::open(admission_path)?; + let before = admission.snapshot()?; + if admission + .authorize_retained_deployment(false, generation.clone()) + .is_ok() + || admission + .reserve_retained_bootstrap(42, 1, 9000, &generation) + .is_ok() + { + return Err("unauthorized scripted retained bootstrap was admitted".to_owned()); + } + admission.authorize_retained_deployment(true, generation.clone())?; + admission.bind_orchestrator(std::process::id())?; + for node in &before.nodes { + admission.reserve_retained_bootstrap( + 42, + node.node_id, + before.contracts[&node.node_id], + &generation, + )?; + if admission + .reserve_create(42, node.node_id, 0, node.offer_id, &node.label) + .is_ok() + || admission.reserve_bootstrap(42, node.node_id).is_ok() + || admission + .reserve_retained_bootstrap( + 42, + node.node_id, + before.contracts[&node.node_id], + &generation, + ) + .is_ok() + { + return Err("retained deployment reopened a consumed slot".to_owned()); + } + } + let after = admission.snapshot()?; + if before.contracts != after.contracts + || before.create_reservations != after.create_reservations + || before.initial_bootstraps != after.initial_bootstraps + { + return Err("retained deployment changed initial ownership accounting".to_owned()); + } + admission + } else { + let nodes = (1..=5) + .map(|node_id| provisioning::PaidNodeAdmission { + node_id, + offer_id: 1000 + node_id, + host_id: 2000 + node_id, + label: format!("scripted-retained-42-{node_id}-attempt-0"), + }) + .collect(); + let admission = PaidFixtureAdmission::create( + admission_path, + 42, + nodes, + unix_ms()? + provisioning::PAID_CLEANUP_RESERVE_MS + 45_000, + )?; + start_cleanup_owner( + &std::env::current_exe().map_err(|error| error.to_string())?, + admission_path, + api_key_env, + PaidCleanupLimits { + maximum_cost_microusd: 1_000_000, + hourly_price_microusd: 1_000_000, + }, + Duration::from_secs(10), + )?; + admission.bind_orchestrator(std::process::id())?; + for node in admission.snapshot()?.nodes { + admission.reserve_create(42, node.node_id, 0, node.offer_id, &node.label)?; + admission.record_contract(node.node_id, 8999 + node.node_id)?; + admission.reserve_bootstrap(42, node.node_id)?; + } + admission.seal_prepared()?; + admission + }; + admission.retain_development_fixture(true)?; + println!( + "{}", + serde_json::to_string(&admission.snapshot()?).map_err(|error| error.to_string())? + ); + Ok(()) +} + +/// Foreground cleanup requests the existing supervisor, never starts a rival +/// deletion loop. The caller may time out; the independent duty continues. +pub fn await_cleanup_owner(admission_path: &Path, deadline: Duration) -> Result<(), String> { + let admission = PaidFixtureAdmission::open(admission_path)?; + admission.enter_cleanup_only()?; + let until = std::time::Instant::now() + deadline; + loop { + let state = admission.snapshot()?; + if state.cleanup.as_ref().is_some_and(|owner| owner.complete) { + return Ok(()); + } + if state + .cleanup + .as_ref() + .is_some_and(|owner| owner.spending_stopped) + { + return Err(format!( + "paid resources absent but accounting failed: {:?}", + state.accounting_errors + )); + } + if std::time::Instant::now() >= until { + return Err(format!( + "independent cleanup remains active/recoverable; unresolved={:?}; accounting_errors={:?}", + state.unresolved_create_labels(), + state.accounting_errors + )); + } + std::thread::sleep(Duration::from_millis(100)); + } +} + +/// The only runner-level cleanup integration: reconcile durable sources before +/// touching the provider, delegate to the owner, and retain identities on error. +pub fn cleanup_paid_ownership( + paid: &mut PaidCampaignState, + state_path: &Path, + executable: &Path, + api_key_env: &str, + deadline: Duration, +) -> Result<(), String> { + let until = std::time::Instant::now() + deadline; + paid.enter_cleanup_only(); + let mut errors = Vec::new(); + // Persist deny-only runner state independently of admission sealing: failure + // of either durable source must not suppress the other cleanup boundary. + if let Err(error) = write_paid_state(state_path, paid) { + errors.push(error); + } + let result = (|| -> Result<(), String> { + let path = paid.state_dir.join("paid-admission.json"); + if !path + .try_exists() + .map_err(|error| format!("inspect paid admission: {error}"))? + { + if paid.owned_labels.is_empty() + && paid.owned_contract_ids.is_empty() + && paid.records.is_empty() + { + return paid.complete_cleanup(); + } + // Legacy runner-only state has no admission capability to reopen. + // Every incomplete record stays ambiguous until its exact labels appear. + let api_key = std::env::var(api_key_env).map_err(|_| { + format!("required VastAI credential environment {api_key_env} is unset") + })?; + let client = BlockingVastClient::new(VastClient::new(api_key))?; + let unresolved = paid + .records + .iter() + .filter(|record| !record.completed) + .flat_map(|record| record.expected_labels.iter().cloned()) + .collect(); + let labels = paid.owned_labels.clone(); + let ids = paid.owned_contract_ids.clone(); + let census = client + .cleanup_owned_reconciled( + &labels, + &ids, + &unresolved, + until.saturating_duration_since(std::time::Instant::now()), + &mut |discovered| { + paid.owned_contract_ids.extend(discovered.keys().copied()); + let observed_labels = discovered.values().cloned().collect::>(); + for record in &mut paid.records { + record + .observed_contract_ids + .extend(discovered.iter().filter_map(|(&contract, label)| { + record.expected_labels.contains(label).then_some(contract) + })); + if record.expected_labels.is_subset(&observed_labels) { + record.completed = true; + } + } + write_paid_state(state_path, paid) + }, + ) + .map_err(|error| error.detail)?; + for record in &mut paid.records { + if !record.completed { + let observed_labels = census + .discovered_contract_labels + .values() + .cloned() + .collect::>(); + if !record.expected_labels.is_subset(&observed_labels) { + return Err("legacy paid acquisition remains unresolved".to_owned()); + } + record + .observed_contract_ids + .extend(census.discovered_contract_labels.keys().copied()); + record.completed = true; + } + } + paid.owned_contract_ids.clear(); + return paid.complete_cleanup(); + } + let admission = PaidFixtureAdmission::open(&path)?; + let ownership = + admission.include_cleanup_ownership(&paid.owned_contract_ids, &paid.owned_labels); + let requested = admission.request_cleanup(); + for result in [ownership, requested] { + if let Err(error) = result { + if let Err(persist_error) = admission.record_cleanup_accounting_error(error.clone()) + { + errors.push(persist_error); + } + errors.push(error); + } + } + let initial = admission.snapshot()?; + if let Err(error) = paid.reconcile_admission(&initial) { + if let Err(persist_error) = admission.record_cleanup_accounting_error(error.clone()) { + errors.push(persist_error); + } + errors.push(error); + } + let result = if initial.cleanup.is_some() { + loop { + let state = admission.snapshot()?; + if state.cleanup.as_ref().is_some_and(|owner| owner.complete) { + break Ok(()); + } + if state + .cleanup + .as_ref() + .is_some_and(|owner| owner.spending_stopped) + { + break Err(format!( + "paid spending stopped but cleanup accounting failed: {:?}", + state.accounting_errors + )); + } + if std::time::Instant::now() >= until { + break Err(format!( + "paid cleanup remains recoverable; unresolved={:?}; accounting_errors={:?}", + state.unresolved_create_labels(), + state.accounting_errors + )); + } + if !admission.cleanup_owner_active()? { + if let Err(error) = spawn_cleanup_supervisor( + executable, + &path, + api_key_env, + true, + until.min(std::time::Instant::now() + Duration::from_secs(10)), + ) { + break Err(error); + } + } + std::thread::sleep(Duration::from_millis(100)); + } + } else { + // Legacy cleanup uses the same exclusive owner lock and the same + // bounded engine; it cannot manufacture missing historical limits. + let _owner = admission.claim_cleanup_owner()?; + let api_key = std::env::var(api_key_env).map_err(|_| { + format!("required VastAI credential environment {api_key_env} is unset") + })?; + let client = BlockingVastClient::new(VastClient::new(api_key))?; + client + .cleanup_owned_reconciled( + &initial.cleanup_labels(), + &initial.known_contract_ids(), + &initial.unresolved_create_labels(), + until.saturating_duration_since(std::time::Instant::now()), + &mut |discovered| admission.reconcile_cleanup_discovery(discovered), + ) + .map_err(|error| error.detail) + .and_then(|census| admission.complete_cleanup(&census.missing_contract_ids)) + }; + let final_state = admission.snapshot()?; + paid.owned_contract_ids + .extend(final_state.known_contract_ids()); + paid.owned_labels.extend(final_state.cleanup_labels()); + paid.cleanup_ownership = final_state.cleanup.clone(); + result?; + for record in &mut paid.records { + record + .observed_contract_ids + .extend(final_state.known_contract_ids()); + record.completed = true; + } + paid.acquisition_count = final_state.create_reservations.len() as u32; + paid.bootstrap_count = final_state.initial_bootstraps.len() as u32; + paid.owned_contract_ids.clear(); + paid.complete_cleanup() + })(); + if let Err(error) = result { + errors.push(error); + } + if !errors.is_empty() { + paid.enter_cleanup_only(); + let cleanup_error = format!("paid cleanup failed: {}", errors.join("; ")); + paid.quarantine_reason = Some(match paid.quarantine_reason.take() { + Some(reason) => format!("{reason}; {cleanup_error}"), + None => cleanup_error, + }); + } + if let Err(error) = write_paid_state(state_path, paid) { + errors.push(error); + } + if errors.is_empty() { + Ok(()) + } else { + Err(errors.join("; ")) + } +} + +fn durable_json(path: &Path, value: &impl Serialize) -> Result<(), String> { + let parent = path + .parent() + .ok_or_else(|| format!("artifact path {} has no parent", path.display()))?; + fs::create_dir_all(parent) + .map_err(|error| format!("create artifact parent {}: {error}", parent.display()))?; + let mut temp = tempfile::NamedTempFile::new_in(parent) + .map_err(|error| format!("create temporary artifact: {error}"))?; + { + let mut writer = std::io::BufWriter::new(&mut temp); + serde_json::to_writer(&mut writer, value) + .map_err(|error| format!("serialize durable artifact: {error}"))?; + writer + .write_all(b"\n") + .and_then(|()| writer.flush()) + .map_err(|error| format!("finish durable artifact: {error}"))?; + } + temp.as_file_mut() + .sync_all() + .map_err(|error| format!("sync durable artifact: {error}"))?; + temp.persist(path) + .map_err(|error| format!("persist durable artifact {}: {error}", path.display()))?; + File::open(parent) + .and_then(|directory| directory.sync_all()) + .map_err(|error| format!("sync artifact directory {}: {error}", parent.display()))?; + Ok(()) +} + +fn unix_ms() -> Result { + let value = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|error| format!("system clock precedes epoch: {error}"))? + .as_millis(); + u64::try_from(value).map_err(|_| "unix timestamp exceeds u64".to_owned()) +} + +pub fn conservative_selected_cost( + offers: &[Offer], + limits: &CampaignLimits, +) -> Result { + let hourly: f64 = offers.iter().map(|offer| offer.hourly_price).sum(); + let lifetime = hourly * limits.fixture_lifetime_secs as f64 / 3_600.0; + if !lifetime.is_finite() { + return Err("selected worst-case cost overflowed".to_owned()); + } + Ok(lifetime) +} + +pub fn scan_artifacts_for_secret(root: &Path, secret: &[u8]) -> Result<(), String> { + if secret.is_empty() { + return Err("credential scan marker must not be empty".to_owned()); + } + let mut pending = vec![root.to_owned()]; + while let Some(path) = pending.pop() { + let metadata = fs::symlink_metadata(&path) + .map_err(|error| format!("inspect artifact {}: {error}", path.display()))?; + if metadata.file_type().is_symlink() { + return Err(format!( + "credential scan cannot certify symlink {}", + path.display() + )); + } + if metadata.is_dir() { + for entry in fs::read_dir(&path) + .map_err(|error| format!("read artifact directory {}: {error}", path.display()))? + { + pending.push( + entry + .map_err(|error| format!("read artifact entry: {error}"))? + .path(), + ); + } + continue; + } + if !metadata.is_file() { + return Err(format!( + "credential scan cannot certify special file {}", + path.display() + )); + } + let bytes = fs::read(&path) + .map_err(|error| format!("scan artifact {}: {error}", path.display()))?; + if bytes.windows(secret.len()).any(|window| window == secret) { + return Err(format!( + "credential material was persisted in artifact {}", + path.display() + )); + } + } + Ok(()) +} +#[cfg(test)] +mod tests { + + use super::*; + use crate::campaign::{CampaignLimits, OfferPolicy, ProviderMode}; + + fn prepared_admission() -> PaidAdmissionSnapshot { + PaidAdmissionSnapshot { + schema_version: 3, + run_id: 7, + nodes: (1..=5) + .map(|id| provisioning::PaidNodeAdmission { + node_id: id, + offer_id: id, + host_id: id + 100, + label: format!("myelin-7-{id}-attempt-0"), + }) + .collect(), + contracts: (1..=5).map(|id| (id, id + 10)).collect(), + create_reservations: (1..=5) + .map(|id| { + ( + id, + provisioning::PaidCreateReservation { + attempt_id: 0, + reserved_unix_ms: 1, + }, + ) + }) + .collect(), + rejected_creates: BTreeSet::new(), + initial_bootstraps: (1..=5) + .map(|id| { + ( + id, + provisioning::PaidBootstrapReservation { + reserved_unix_ms: 2, + }, + ) + }) + .collect(), + retained_deployments: Default::default(), + mode: PaidAdmissionMode::Prepared, + deadline_unix_ms: u64::MAX, + cleanup: None, + discovered_contracts: Default::default(), + accounting_errors: BTreeSet::new(), + recovery_contract_ids: BTreeSet::new(), + recovery_labels: BTreeSet::new(), + } + } + + #[test] + fn matching_labels_and_contracts_cannot_manufacture_bootstrap_evidence() { + let mut admission = prepared_admission(); + admission.initial_bootstraps.remove(&5); + let labels = admission.cleanup_labels(); + let mut state = PaidCampaignState::planned("campaign", PathBuf::from("state")); + state + .begin_acquisition((1..=5).collect(), labels.clone()) + .unwrap(); + assert!( + state + .reconcile_prepared(&admission, (11..=15).collect(), labels) + .is_err() + ); + assert_eq!(state.acquisition_count, 0); + assert_eq!(state.bootstrap_count, 0); + assert!(!state.records[0].completed); + } + + #[test] + fn metered_transfer_prices_cannot_hide_outside_operator_ceiling() { + let mut offers = (0..5).map(offer).collect::>(); + assert!(selected_cleanup_limits(&offers, &config().limits).is_ok()); + offers[0].download_cost_per_tb = 0.001; + assert!(selected_cleanup_limits(&offers, &config().limits).is_err()); + offers[0].download_cost_per_tb = 0.0; + offers[0].upload_cost_per_tb = 0.001; + assert!(selected_cleanup_limits(&offers, &config().limits).is_err()); + } + + fn config() -> CampaignConfig { + CampaignConfig { + provider: ProviderMode::Real, + seed: 9, + runtime_image: "registry.example/myelin:test".to_owned(), + limits: CampaignLimits { + fixture_lifetime_secs: 24 * 60 * 60, + total_cost_usd: 20.0, + total_hourly_price_usd: 1.0, + case_deadline_secs: 30, + max_campaign_allocation_bytes: 8 * 1024 * 1024 * 1024, + max_campaign_payload_bytes: 512 * 1024 * 1024, + max_case_race_states: 256, + }, + offers: OfferPolicy { + gpu_model: None, + min_gpu_ram_mb: None, + min_compute_cap: Some(700), + min_reliability: Some(0.95), + min_download_mbps: Some(100.0), + min_upload_mbps: None, + max_hourly_price_per_node: Some(0.2), + blacklist_hosts: Vec::new(), + }, + } + } + + fn offer(index: u64) -> Offer { + Offer { + offer_id: 100 + index, + host_id: Some(200 + index), + gpu_model: "test".to_owned(), + gpu_ram_mb: Some(8_192.0), + compute_cap: 800, + verification: Some("verified".to_owned()), + reliability: Some(0.99), + download_mbps: Some(1_000.0), + upload_mbps: Some(1_000.0), + location: None, + hourly_price: 0.1, + download_cost_per_tb: 0.0, + upload_cost_per_tb: 0.0, + } + } + + #[test] + fn prepared_state_permanently_denies_more_acquisition() { + let labels = (1..=5) + .map(|node| format!("myelin-7-{node}-attempt-0")) + .collect::>(); + let mut state = PaidCampaignState::planned("campaign", PathBuf::from("state")); + state + .begin_acquisition(vec![1, 2, 3, 4, 5], labels.clone()) + .unwrap(); + state + .reconcile_prepared(&prepared_admission(), (11..=15).collect(), labels) + .unwrap(); + assert!( + state + .begin_acquisition(vec![1, 2, 3, 4, 5], BTreeSet::new()) + .is_err() + ); + } + + #[test] + fn quarantined_fixture_resumes_without_another_acquisition() { + let labels = (1..=5) + .map(|node| format!("myelin-7-{node}-attempt-0")) + .collect::>(); + let mut state = PaidCampaignState::planned("campaign", PathBuf::from("state")); + state + .begin_acquisition(vec![1, 2, 3, 4, 5], labels.clone()) + .unwrap(); + state + .reconcile_prepared(&prepared_admission(), (11..=15).collect(), labels) + .unwrap(); + state.quarantine("candidate failed"); + state + .resume_deny_only(&prepared_admission(), (13..=15).collect()) + .unwrap(); + assert_eq!(state.phase, PaidCampaignPhase::DenyOnly); + assert_eq!(state.acquisition_count, 5); + assert_eq!(state.bootstrap_count, 5); + assert_eq!(state.owned_contract_ids, (13..=15).collect()); + assert!(state.quarantine_reason.is_none()); + } + + #[test] + fn cleanup_recovery_identity_cannot_become_a_fixture_survivor() { + let mut admission = prepared_admission(); + let mut state = PaidCampaignState::planned("campaign", PathBuf::from("state")); + state + .begin_acquisition((1..=5).collect(), admission.cleanup_labels()) + .unwrap(); + state + .reconcile_prepared(&admission, (11..=15).collect(), admission.cleanup_labels()) + .unwrap(); + state.quarantine("retained deployment"); + admission.recovery_contract_ids.insert(99); + assert!( + state + .resume_deny_only(&admission, BTreeSet::from([13, 14, 99])) + .is_err() + ); + assert_eq!(state.phase, PaidCampaignPhase::Quarantined); + } + + #[test] + fn empty_census_does_not_settle_an_ambiguous_acquisition() { + let labels = (1..=5) + .map(|node| format!("myelin-7-{node}-attempt-0")) + .collect::>(); + let mut state = PaidCampaignState::planned("campaign", PathBuf::from("state")); + state + .begin_acquisition(vec![1, 2, 3, 4, 5], labels) + .unwrap(); + state.enter_cleanup_only(); + assert!(state.complete_cleanup().is_err()); + assert_eq!(state.phase, PaidCampaignPhase::CleanupOnly); + assert!(!state.phase.permits_acquisition()); + let restored: PaidCampaignState = + serde_json::from_str(&serde_json::to_string(&state).unwrap()).unwrap(); + assert!(!restored.records[0].completed); + assert!(restored.validate().is_ok()); + } + + #[test] + fn selection_requires_five_verified_distinct_hosts_within_cost() { + let config = config(); + let plan = CampaignPlan::build(&config, Vec::new()).unwrap(); + let results = OfferSearchResults { + search_id: 4, + offers: (0..5).map(offer).collect(), + }; + assert_eq!( + select_exact_offers(&results, &config, &plan).unwrap().len(), + 5 + ); + + let mut duplicate_hosts = results.clone(); + duplicate_hosts.offers[4].host_id = duplicate_hosts.offers[0].host_id; + assert!( + select_exact_offers(&duplicate_hosts, &config, &plan) + .unwrap_err() + .contains("distinct") + ); + + let mut unverified = results; + unverified.offers[0].verification = Some("unverified".to_owned()); + assert!( + select_exact_offers(&unverified, &config, &plan) + .unwrap_err() + .contains("eligible distinct verified") + ); + } + + #[test] + fn credential_scan_rejects_nested_artifact_leaks() { + let temp = tempfile::tempdir().unwrap(); + fs::create_dir(temp.path().join("nested")).unwrap(); + fs::write(temp.path().join("safe.json"), b"{\"safe\":true}").unwrap(); + let marker = b"rare-credential-marker-6e8f"; + scan_artifacts_for_secret(temp.path(), marker).unwrap(); + fs::write(temp.path().join("nested/leak.txt"), marker).unwrap(); + assert!( + scan_artifacts_for_secret(temp.path(), marker) + .unwrap_err() + .contains("credential material") + ); + } + + #[cfg(unix)] + #[test] + fn credential_scan_rejects_symlinked_artifacts() { + let root = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + let marker = b"private-secret-behind-symlink"; + fs::write(outside.path().join("secret"), marker).unwrap(); + std::os::unix::fs::symlink(outside.path().join("secret"), root.path().join("linked")) + .unwrap(); + assert!(scan_artifacts_for_secret(root.path(), marker).is_err()); + } + + #[cfg(target_os = "linux")] + mod supervisor_tests { + use super::*; + use std::io::Read; + use std::os::fd::{AsFd, FromRawFd, OwnedFd}; + use std::os::unix::ffi::OsStrExt; + use std::os::unix::fs::PermissionsExt; + use std::os::unix::net::UnixStream; + use std::process::{Child, Command, Stdio}; + use std::time::Instant; + + const CHILD_ROLE: &str = "MYELIN_SUPERVISOR_REGRESSION_ROLE"; + const CHILD_PATH: &str = "MYELIN_SUPERVISOR_REGRESSION_PATH"; + const RECOVERY_TEST: &str = + "remote::tests::supervisor_tests::exiting_observed_supervisor_restarts_renewed_cleanup"; + + struct FixtureProcess(Child); + + impl Drop for FixtureProcess { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + + fn admission(path: &Path) -> PaidFixtureAdmission { + let admission = PaidFixtureAdmission::create( + path, + 7, + prepared_admission().nodes, + unix_ms().unwrap() + 3_600_000, + ) + .unwrap(); + admission + .install_cleanup_ownership( + PaidProcessIdentity::current().unwrap(), + PaidCleanupLimits { + maximum_cost_microusd: 1_000_000, + hourly_price_microusd: 1_000_000, + }, + ) + .unwrap(); + admission + } + + fn release_after_supervisor_probe(mut events: File, mut release: UnixStream) { + let until = Instant::now() + Duration::from_secs(10); + let mut buffer = [0; 1024]; + loop { + match events.read(&mut buffer) { + Ok(count) if count > 0 => { + release.write_all(&[1]).unwrap(); + return; + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {} + result => panic!("observe supervisor lock probe: {result:?}"), + } + assert!( + Instant::now() < until, + "startup never observed the held lock" + ); + std::thread::sleep(Duration::from_millis(1)); + } + } + + #[test] + fn exiting_observed_supervisor_restarts_renewed_cleanup() { + if let Some(role) = std::env::var_os(CHILD_ROLE) { + let path = PathBuf::from(std::env::var_os(CHILD_PATH).unwrap()); + let admission = PaidFixtureAdmission::open(&path).unwrap(); + let supervisor = admission.claim_cleanup_supervisor().unwrap(); + if role == "observed" { + admission.enter_cleanup_only().unwrap(); + // No create was reserved: the original empty duty is settled. + admission.complete_cleanup(&BTreeSet::new()).unwrap(); + let mut release = + UnixStream::from(std::io::stdin().as_fd().try_clone_to_owned().unwrap()); + release + .set_read_timeout(Some(Duration::from_secs(10))) + .unwrap(); + release.write_all(&[1]).unwrap(); + release.read_exact(&mut [0]).unwrap(); + // Exit based on the old receipt, after the caller renewed it. + drop(supervisor); + } else { + assert_eq!(role, "replacement"); + let renewed = admission.snapshot().unwrap(); + assert!(!renewed.cleanup.as_ref().unwrap().spending_stopped); + assert!(renewed.create_reservations.is_empty()); + let _owner = admission.claim_cleanup_owner().unwrap(); + let completion = + File::open(path.parent().unwrap().join("allow-completion")).unwrap(); + completion.lock().unwrap(); + admission.complete_cleanup(&BTreeSet::new()).unwrap(); + } + return; + } + + let directory = tempfile::tempdir().unwrap(); + fs::set_permissions(directory.path(), fs::Permissions::from_mode(0o700)).unwrap(); + let path = directory.path().join("admission.json"); + let admission = admission(&path); + let executable = std::env::current_exe().unwrap(); + let launcher = directory.path().join("replacement-supervisor"); + fs::write( + &launcher, + format!( + "#!/bin/sh\n{CHILD_PATH}=\"$2\" {CHILD_ROLE}=replacement exec '{}' --exact '{RECOVERY_TEST}' --nocapture\n", + executable.to_str().unwrap().replace('\'', "'\\''"), + ), + ) + .unwrap(); + fs::set_permissions(&launcher, fs::Permissions::from_mode(0o700)).unwrap(); + let completion = File::create(directory.path().join("allow-completion")).unwrap(); + completion.lock().unwrap(); + + let (mut release, child_socket) = UnixStream::pair().unwrap(); + release + .set_read_timeout(Some(Duration::from_secs(10))) + .unwrap(); + let mut observed = FixtureProcess( + Command::new(&executable) + .args(["--exact", RECOVERY_TEST, "--nocapture"]) + .env(CHILD_ROLE, "observed") + .env(CHILD_PATH, &path) + .stdin(Stdio::from(OwnedFd::from(child_socket))) + .stdout(Stdio::null()) + .spawn() + .unwrap(), + ); + release.read_exact(&mut [0]).unwrap(); + let settled = admission.snapshot().unwrap(); + let old_identity = settled + .cleanup + .as_ref() + .unwrap() + .supervisor + .clone() + .unwrap(); + assert_eq!(old_identity.pid, observed.0.id()); + assert!(old_identity.is_live()); + assert!(settled.cleanup.as_ref().unwrap().spending_stopped); + admission.request_cleanup().unwrap(); + let renewed = admission.snapshot().unwrap(); + assert!(!renewed.cleanup.as_ref().unwrap().spending_stopped); + + // IN_CLOSE_WRITE arrives only after the active-lock probe has + // finished with its independently opened fd. Releasing before that + // probe would test an initially absent supervisor, not this race. + let fd = unsafe { libc::inotify_init1(libc::IN_CLOEXEC | libc::IN_NONBLOCK) }; + assert!(fd >= 0, "{}", std::io::Error::last_os_error()); + let events = unsafe { File::from_raw_fd(fd) }; + let lock_path = std::ffi::CString::new( + directory + .path() + .join("admission.json.supervisor.lock") + .as_os_str() + .as_bytes(), + ) + .unwrap(); + assert!( + unsafe { libc::inotify_add_watch(fd, lock_path.as_ptr(), libc::IN_CLOSE_WRITE) } + >= 0, + "{}", + std::io::Error::last_os_error(), + ); + let releasing = std::thread::spawn(move || { + release_after_supervisor_probe(events, release); + }); + let recovered = spawn_cleanup_supervisor( + &launcher, + &path, + "MYELIN_SUPERVISOR_REGRESSION_UNUSED_CREDENTIAL", + true, + Instant::now() + Duration::from_secs(5), + ); + releasing.join().unwrap(); + assert!(observed.0.wait().unwrap().success()); + assert!(!old_identity.is_live()); + recovered.unwrap(); + let ready = admission.snapshot().unwrap(); + let replacement = ready.cleanup.as_ref().unwrap(); + assert!(admission.cleanup_supervisor_active().unwrap()); + assert!(admission.cleanup_owner_active().unwrap()); + assert!(replacement.supervisor.as_ref().unwrap().is_live()); + assert_eq!(replacement.owner, replacement.supervisor); + assert_ne!(replacement.supervisor.as_ref(), Some(&old_identity)); + assert!(!replacement.spending_stopped); + drop(completion); + await_cleanup_owner(&path, Duration::from_secs(5)).unwrap(); + + let completed = admission.snapshot().unwrap(); + let ownership = completed.cleanup.as_ref().unwrap(); + assert!(ownership.complete); + assert!(ownership.spending_stopped); + assert_ne!(ownership.supervisor.as_ref(), Some(&old_identity)); + assert_eq!(completed.mode, PaidAdmissionMode::CleanupOnly); + assert_eq!(completed.create_reservations, renewed.create_reservations); + assert_eq!(ownership.limits, renewed.cleanup.as_ref().unwrap().limits); + assert_eq!( + ownership.stop_unix_ms, + renewed.cleanup.as_ref().unwrap().stop_unix_ms, + ); + assert!( + admission + .reserve_create(7, 1, 0, 1, "myelin-7-1-attempt-0") + .is_err() + ); + } + + #[test] + fn expired_readiness_cannot_authorize_acquisition() { + let directory = tempfile::tempdir().unwrap(); + fs::set_permissions(directory.path(), fs::Permissions::from_mode(0o700)).unwrap(); + let path = directory.path().join("admission.json"); + let admission = admission(&path); + let _supervisor = admission.claim_cleanup_supervisor().unwrap(); + let _owner = admission.claim_cleanup_owner().unwrap(); + admission.bind_orchestrator(std::process::id()).unwrap(); + admission.require_live_cleanup_owner().unwrap(); + let before = admission.snapshot().unwrap(); + + let result = spawn_cleanup_supervisor( + &directory.path().join("must-not-spawn"), + &path, + "MYELIN_SUPERVISOR_REGRESSION_UNUSED_CREDENTIAL", + false, + Instant::now(), + ); + assert!(result.is_err(), "live ownership overrode an expired caller"); + assert!(admission.cleanup_supervisor_active().unwrap()); + assert!(admission.cleanup_owner_active().unwrap()); + assert!(admission.require_live_cleanup_owner().is_err()); + assert!( + admission + .reserve_create(7, 1, 0, 1, "myelin-7-1-attempt-0") + .is_err() + ); + let after = admission.snapshot().unwrap(); + assert_eq!(after.mode, PaidAdmissionMode::CleanupOnly); + assert_eq!(after.create_reservations, before.create_reservations); + assert_eq!(after.cleanup, before.cleanup); + } + + #[test] + fn expired_recovery_cannot_accept_a_completed_receipt() { + let directory = tempfile::tempdir().unwrap(); + fs::set_permissions(directory.path(), fs::Permissions::from_mode(0o700)).unwrap(); + let path = directory.path().join("admission.json"); + let admission = admission(&path); + let _supervisor = admission.claim_cleanup_supervisor().unwrap(); + admission.enter_cleanup_only().unwrap(); + admission.complete_cleanup(&BTreeSet::new()).unwrap(); + let before = admission.snapshot().unwrap(); + + let result = spawn_cleanup_supervisor( + &directory.path().join("must-not-spawn"), + &path, + "MYELIN_SUPERVISOR_REGRESSION_UNUSED_CREDENTIAL", + true, + Instant::now(), + ); + assert!(result.is_err(), "old completion overrode an expired caller"); + let after = admission.snapshot().unwrap(); + assert_eq!(after.mode, PaidAdmissionMode::CleanupOnly); + assert_eq!(after.cleanup, before.cleanup); + assert!(admission.cleanup_supervisor_active().unwrap()); + } + } +} diff --git a/tools/myelin-e2e-fuzz/src/resources.rs b/tools/myelin-e2e-fuzz/src/resources.rs index 7eaac46..f9c603e 100644 --- a/tools/myelin-e2e-fuzz/src/resources.rs +++ b/tools/myelin-e2e-fuzz/src/resources.rs @@ -2,85 +2,958 @@ use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::fs; -use std::io::{BufRead, BufReader, Read as IoRead, Seek, SeekFrom}; -use std::net::TcpListener; -use std::path::Path; -use std::process::Command; -use std::sync::{Arc, Mutex}; +use std::io::{BufRead, BufReader, Read as IoRead, Seek, SeekFrom, Write as IoWrite}; +use std::net::{SocketAddr, TcpListener, TcpStream}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::{Arc, LazyLock, Mutex}; use std::thread; -use std::time::Duration; +use std::time::{Duration, Instant}; use serde_json::{Value, json}; -use crate::harness::GENERATED_LAUNCHER; +use crate::budget::{Budget, record_execution_identity, record_execution_stage}; -const TRANSIENT_ACTOR_TYPES: [&str; 6] = [ +const TRANSIENT_ACTOR_TYPES: [&str; 5] = [ "swactor_process::", "swactor_process_context::", "myelin::contextual_process::ContextualOutputRelay", - "myelin::orchestration::control::ContextualReplyObserver", "data_plane::host::", "data_plane::source::FileBlobSourceActor", ]; -pub(crate) fn build_myelin_binaries(workspace: &Path) -> Result<(), String> { - run_checked( - Command::new("cargo").current_dir(workspace).args([ - "build", - "--release", - "-p", - "myelin", - "--bins", - ]), - "build Myelin binaries", - ) -} +const LIVE_TRANSPORT_FIELDS: [&str; 6] = [ + "pending_sinks", + "pending_inbound", + "active_controls", + "source_probes", + "sink_probes", + "local_incarnations", +]; -pub(crate) fn build_workload_image(workspace: &Path, image: &str) -> Result<(), String> { - run_checked( - Command::new("docker").current_dir(workspace).args([ - "build", - "-f", - "apps/myelin/node-image/Dockerfile.base", - "-t", - "myelin-node-base:cuda12.6", - ".", - ]), - "build Myelin base image", - )?; - run_checked( - Command::new("docker").current_dir(workspace).args([ - "build", - "-f", - "apps/myelin/node-image/Dockerfile", - "-t", - "myelin-node:latest", - ".", - ]), - "build Myelin node image", - )?; - run_checked( - Command::new("docker").current_dir(workspace).args([ - "build", - "-f", - "apps/myelin/node-image/Dockerfile.e2e", - "-t", - image, - ".", - ]), - "build Myelin E2E workload image", - ) -} +/// Durable private fixture state is separate from publishable evidence. +pub fn private_fixture_dir(artifacts: &Path) -> Result { + use sha2::{Digest, Sha256}; + use std::os::unix::ffi::OsStrExt; + use std::os::unix::fs::{DirBuilderExt, MetadataExt}; -fn run_checked(command: &mut Command, label: &str) -> Result<(), String> { - let status = command - .status() - .map_err(|error| format!("{label}: {error}"))?; - if status.success() { - Ok(()) - } else { - Err(format!("{label} exited {status}")) + let artifacts = artifacts + .canonicalize() + .map_err(|error| format!("resolve artifact directory: {error}"))?; + let base = + match std::env::var_os("XDG_STATE_HOME") { + Some(path) => std::path::PathBuf::from(path), + None => std::path::PathBuf::from(std::env::var_os("HOME").ok_or_else(|| { + "private fixture state requires HOME or XDG_STATE_HOME".to_owned() + })?) + .join(".local/state"), + }; + if !base.is_absolute() { + return Err("private fixture state root must be absolute".to_owned()); } + let key = format!("{:x}", Sha256::digest(artifacts.as_os_str().as_bytes())); + let directory = base.join("myelin-e2e").join(key); + fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(&directory) + .map_err(|error| format!("create private fixture state: {error}"))?; + let metadata = fs::symlink_metadata(&directory) + .map_err(|error| format!("inspect private fixture state: {error}"))?; + if !metadata.is_dir() + || metadata.uid() != unsafe { libc::geteuid() } + || metadata.mode() & 0o077 != 0 + { + return Err( + "private fixture state must be an owned, non-symlink directory with mode 0700" + .to_owned(), + ); + } + let directory = directory + .canonicalize() + .map_err(|error| format!("resolve private fixture state: {error}"))?; + if directory.starts_with(&artifacts) { + return Err("private fixture state cannot be inside publishable artifacts".to_owned()); + } + Ok(directory) +} + +/// Invocation-stable executable references. Never launch a mutable Cargo output +/// after accepting this identity. +#[derive(Clone, Debug)] +pub struct BuiltBinaries { + pub orchestrator: PathBuf, + pub worker: PathBuf, + pub input_digest: String, + pub executable_digests: BTreeMap, + verified_files: BTreeMap, +} + +/// The exact resolver-selected inputs and bytes qualified by ordered local gates. +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PreparedArtifactIdentity { + pub schema_version: u32, + pub source_build_input_digest: String, + pub executables: BTreeMap, + pub wheel_input_digest: String, + pub wheels: BTreeMap, + pub payload: BTreeMap, +} + +static REQUIRED_ARTIFACTS: LazyLock>> = + LazyLock::new(Mutex::default); + +fn required_artifacts(budget: &Budget) -> Result, String> { + Ok(budget_lock(&REQUIRED_ARTIFACTS, budget, "qualified artifact identity")?.clone()) +} + +/// Fail closed without building or installing anything, then pin every later resolver call. +pub fn require_prepared_artifacts( + workspace: &Path, + expected: &PreparedArtifactIdentity, + budget: &Budget, +) -> Result<(), String> { + if expected.schema_version != 2 || expected.wheels.is_empty() { + return Err("qualified deployment artifacts are incomplete or incompatible".to_owned()); + } + { + let mut required = budget_lock(&REQUIRED_ARTIFACTS, budget, "pin qualified artifacts")?; + if required.as_ref().is_some_and(|current| current != expected) { + return Err("attempted to change qualified deployment artifacts".to_owned()); + } + *required = Some(expected.clone()); + } + let binaries = resolve_myelin_binaries(workspace, budget)?; + let cache_root = workspace.join("target/myelin-e2e-cache"); + let key = wheel_input_digest( + &binaries, + &cache_root.join("maturin-venv/bin/maturin"), + budget, + )?; + if key != expected.wheel_input_digest { + return Err( + "wheel compiler, interpreter or build inputs changed since qualification".to_owned(), + ); + } + let wheel_dir = cache_root.join("wheels").join(&key); + for (name, digest) in &expected.wheels { + validate_artifact_filename(name, ".whl")?; + if sha256_hex_budget(&wheel_dir.join(name), budget)? != *digest { + return Err(format!("qualified wheel changed: {name}")); + } + } + let actual = + prepared_artifact_identity(workspace, &binaries, key, expected.wheels.clone(), budget)?; + if &actual != expected { + return Err("qualified deployment payload changed".to_owned()); + } + Ok(()) +} + +fn validate_artifact_filename(name: &str, suffix: &str) -> Result<(), String> { + if !matches!( + Path::new(name).components().next(), + Some(std::path::Component::Normal(_)) + ) || Path::new(name).components().count() != 1 + || !name.ends_with(suffix) + { + return Err(format!("invalid artifact filename: {name}")); + } + Ok(()) +} + +fn prepared_artifact_identity( + workspace: &Path, + binaries: &BuiltBinaries, + wheel_input_digest: String, + wheels: BTreeMap, + budget: &Budget, +) -> Result { + let mut payload = BTreeMap::from([ + ( + "bin/myelin-worker".to_owned(), + binaries.executable_digests["myelin-worker"].clone(), + ), + ( + "bin/myelin-e2e-python".to_owned(), + sha256_hex_budget( + &workspace.join("apps/myelin/node-image/e2e_python_launcher.py"), + budget, + )?, + ), + ]); + payload.extend( + wheels + .iter() + .map(|(name, digest)| (format!("python/{name}"), digest.clone())), + ); + Ok(PreparedArtifactIdentity { + schema_version: 2, + source_build_input_digest: binaries.input_digest.clone(), + executables: binaries.executable_digests.clone(), + wheel_input_digest, + wheels, + payload, + }) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct FileIdentity { + device: u64, + inode: u64, + length: u64, + modified: (i64, i64), + changed: (i64, i64), +} + +fn file_identity(path: &Path) -> Result { + use std::os::unix::fs::MetadataExt; + let metadata = fs::symlink_metadata(path).map_err(|error| error.to_string())?; + if !metadata.is_file() { + return Err(format!( + "immutable executable is not a regular file: {}", + path.display() + )); + } + Ok(FileIdentity { + device: metadata.dev(), + inode: metadata.ino(), + length: metadata.len(), + modified: (metadata.mtime(), metadata.mtime_nsec()), + changed: (metadata.ctime(), metadata.ctime_nsec()), + }) +} + +static BINARIES: LazyLock>> = LazyLock::new(Mutex::default); + +pub fn resolve_myelin_binaries(workspace: &Path, budget: &Budget) -> Result { + let started = Instant::now(); + let workspace = workspace + .canonicalize() + .map_err(|error| error.to_string())?; + let mut cache = budget_lock(&BINARIES, budget, "immutable binary cache")?; + let required = required_artifacts(budget)?; + if let Some(binaries) = cache.get_mut(&workspace) { + if required.as_ref().is_some_and(|expected| { + expected.source_build_input_digest != binaries.input_digest + || expected.executables != binaries.executable_digests + }) { + return Err("cached executable identity differs from qualified artifacts".to_owned()); + } + verify_binaries(binaries, budget)?; + record_execution_stage("build.binary.cache_hit", started.elapsed(), 0, 1); + return Ok(binaries.clone()); + } + // Freeze source inputs once for this invocation; later fixture rounds use + // these accepted executable references, never a mutable Cargo output. + let input_digest = build_input_digest(&workspace, budget)?; + if required + .as_ref() + .is_some_and(|expected| expected.source_build_input_digest != input_digest) + { + return Err("compiler, toolchain or build inputs changed since qualification".to_owned()); + } + let directory = workspace + .join("target/myelin-e2e-cache/binaries") + .join(&input_digest); + fs::create_dir_all(&directory).map_err(|error| format!("create binary cache: {error}"))?; + let manifest = directory.join("digests.json"); + let mut binaries = BuiltBinaries { + orchestrator: directory.join("myelin-orchestrator"), + worker: directory.join("myelin-worker"), + input_digest: input_digest.clone(), + executable_digests: BTreeMap::new(), + verified_files: BTreeMap::new(), + }; + if let Ok(file) = fs::File::open(&manifest) { + if let Ok(digests) = serde_json::from_reader(BufReader::new(file)) { + binaries.executable_digests = digests; + if required + .as_ref() + .is_some_and(|expected| expected.executables != binaries.executable_digests) + { + return Err( + "resolver-selected executables differ from qualified artifacts".to_owned(), + ); + } + if verify_binaries(&mut binaries, budget).is_ok() { + record_binary_identity(&binaries, "disk_hit"); + cache.insert(workspace, binaries.clone()); + record_execution_stage("build.binary.disk_cache_hit", started.elapsed(), 0, 1); + return Ok(binaries); + } + } + } + if required.is_some() { + return Err( + "qualified executable cache is missing or corrupt; rebuilding is prohibited".to_owned(), + ); + } + run_checked( + Command::new("cargo") + .current_dir(&workspace) + .env("CARGO_TARGET_DIR", workspace.join("target")) + .args(["build", "--release", "--locked", "-p", "myelin", "--bins"]), + "build Myelin binaries", + budget, + )?; + for (name, destination) in [ + ("myelin-orchestrator", &binaries.orchestrator), + ("myelin-worker", &binaries.worker), + ] { + budget.check("stage immutable executable")?; + copy_atomic(&workspace.join("target/release").join(name), destination)?; + set_executable(destination)?; + let mut permissions = fs::metadata(destination) + .map_err(|error| error.to_string())? + .permissions(); + permissions.set_readonly(true); + fs::set_permissions(destination, permissions).map_err(|error| error.to_string())?; + binaries + .executable_digests + .insert(name.to_owned(), sha256_hex_budget(destination, budget)?); + } + write_json_atomic(&manifest, &binaries.executable_digests)?; + verify_binaries(&mut binaries, budget)?; + record_binary_identity(&binaries, "built"); + cache.insert(workspace, binaries.clone()); + record_execution_stage("build.binary.miss", started.elapsed(), 0, 1); + Ok(binaries) +} + +fn record_binary_identity(binaries: &BuiltBinaries, cache_state: &str) { + record_execution_identity( + &format!("binaries:{}", binaries.input_digest), + json!({ + "schema_version": 1, "source_build_input_digest": binaries.input_digest, + "executables": binaries.executable_digests, "cache_state": cache_state, + }), + ); +} + +fn verify_binaries(binaries: &mut BuiltBinaries, budget: &Budget) -> Result<(), String> { + for (name, path) in [ + ("myelin-orchestrator", &binaries.orchestrator), + ("myelin-worker", &binaries.worker), + ] { + budget.check("verify immutable executable identity")?; + let identity = file_identity(path)?; + if binaries.verified_files.get(name) == Some(&identity) { + continue; + } + let actual = sha256_hex_budget(path, budget)?; + if binaries.executable_digests.get(name) != Some(&actual) { + return Err(format!("immutable executable digest mismatch: {name}")); + } + binaries.verified_files.insert(name.to_owned(), identity); + } + Ok(()) +} + +fn budget_lock<'a, T>( + mutex: &'a Mutex, + budget: &Budget, + predicate: &str, +) -> Result, String> { + loop { + budget.check(predicate)?; + match mutex.try_lock() { + Ok(guard) => return Ok(guard), + Err(std::sync::TryLockError::Poisoned(error)) => return Ok(error.into_inner()), + Err(std::sync::TryLockError::WouldBlock) => { + budget.wait(Duration::from_millis(10), predicate)? + } + } + } +} + +fn build_input_digest(workspace: &Path, budget: &Budget) -> Result { + use sha2::{Digest, Sha256}; + let started = Instant::now(); + let mut digest = Sha256::new(); + digest.update(b"myelin-e2e-build-v2\0release\0locked\0myelin-bins\0"); + for name in [ + "Cargo.toml", + "Cargo.lock", + "rust-toolchain.toml", + "rust-toolchain", + ".dockerignore", + "clippy.toml", + ".cargo", + "src", + "tests", + "apps", + "crates", + "tools", + "xtask", + ] { + let path = workspace.join(name); + if path.exists() { + digest.update(name.as_bytes()); + digest.update([0]); + hash_tree(&path, &path, &mut digest, budget)?; + } + } + let compiler = run_output_budget( + Command::new("rustc").arg("-Vv").current_dir(workspace), + "identify Rust compiler", + budget, + )?; + digest.update(&compiler.stdout); + let sysroot = run_output_budget( + Command::new("rustc") + .args(["--print", "sysroot"]) + .current_dir(workspace), + "identify Rust toolchain", + budget, + )?; + let sysroot = PathBuf::from( + String::from_utf8(sysroot.stdout) + .map_err(|error| error.to_string())? + .trim(), + ); + for name in ["rustc", "cargo"] { + digest.update(name.as_bytes()); + digest.update(sha256_hex_budget(&sysroot.join("bin").join(name), budget)?.as_bytes()); + } + let cargo_home = std::env::var_os("CARGO_HOME") + .map(PathBuf::from) + .or_else(|| std::env::var_os("HOME").map(|home| PathBuf::from(home).join(".cargo"))); + if let Some(home) = cargo_home { + for name in ["config", "config.toml"] { + let path = home.join(name); + if path.exists() { + digest.update(name.as_bytes()); + digest.update(sha256_hex_budget(&path, budget)?.as_bytes()); + } + } + } + for name in ["cc", "c++", "ar", "ld"] { + if let Some(path) = resolve_tool(name) { + digest.update(name.as_bytes()); + digest.update(sha256_hex_budget(&path, budget)?.as_bytes()); + } + } + let build_env = std::env::vars_os() + .filter(|(name, _)| { + let name = name.to_string_lossy(); + name.starts_with("CARGO_") + || name.starts_with("RUST") + || name.starts_with("CC") + || name.starts_with("CXX") + || name.starts_with("AR") + || name.starts_with("LD") + || name.starts_with("PYO3_") + || name.starts_with("PYTHON") + || name.starts_with("MATURIN_") + || name.starts_with("PKG_CONFIG") + || matches!( + name.as_ref(), + "PATH" + | "CFLAGS" + | "CPPFLAGS" + | "CXXFLAGS" + | "LDFLAGS" + | "LIBRARY_PATH" + | "CPATH" + | "C_INCLUDE_PATH" + | "CPLUS_INCLUDE_PATH" + | "VIRTUAL_ENV" + | "CONDA_PREFIX" + ) + }) + .collect::>(); + for (name, value) in build_env { + digest.update(name.as_encoded_bytes()); + digest.update([0]); + digest.update(value.as_encoded_bytes()); + digest.update([0]); + // Tool overrides may keep the same command string while the selected compiler + // or wrapper changes on disk. Bind those executable bytes, not just the flag. + let name = name.to_string_lossy(); + if name.starts_with("CC") + || name.starts_with("CXX") + || name.starts_with("AR") + || name.starts_with("LD") + || name.starts_with("RUSTC") + || name.ends_with("_LINKER") + || name == "PYO3_PYTHON" + { + for argument in value.to_string_lossy().split_whitespace() { + let path = Path::new(argument); + let path = if path.is_file() { + Some(path.to_path_buf()) + } else { + resolve_tool(argument) + }; + if let Some(path) = path { + digest.update(sha256_hex_budget(&path, budget)?.as_bytes()); + } + } + } + } + let value = format!("{:x}", digest.finalize()); + record_execution_stage("build.input_digest", started.elapsed(), 0, 1); + Ok(value) +} + +fn resolve_tool(name: &str) -> Option { + std::env::split_paths(&std::env::var_os("PATH")?) + .map(|directory| directory.join(name)) + .find(|path| path.is_file()) +} + +fn wheel_input_digest( + binaries: &BuiltBinaries, + maturin: &Path, + budget: &Budget, +) -> Result { + use sha2::{Digest, Sha256}; + let python = run_output_budget( + Command::new("python3").args([ + "-c", + "import sys; print(sys.version); print(sys.executable)", + ]), + "identify Python interpreter", + budget, + )?; + let python_output = String::from_utf8(python.stdout).map_err(|error| error.to_string())?; + let interpreter = python_output + .lines() + .last() + .ok_or("Python omitted interpreter path")?; + let builder = run_output_budget( + Command::new(maturin).arg("--version"), + "identify wheel builder", + budget, + )?; + let mut digest = Sha256::new(); + digest.update(b"myelin-e2e-wheel-v2\0"); + digest.update(&binaries.input_digest); + digest.update(python_output.as_bytes()); + digest.update(sha256_hex_budget(Path::new(interpreter), budget)?.as_bytes()); + digest.update(builder.stdout); + digest.update(sha256_hex_budget(maturin, budget)?.as_bytes()); + Ok(format!("{:x}", digest.finalize())) +} + +fn hash_tree( + root: &Path, + path: &Path, + digest: &mut sha2::Sha256, + budget: &Budget, +) -> Result<(), String> { + use sha2::Digest; + use std::os::unix::fs::MetadataExt; + budget.check("hash immutable artifact inputs")?; + let mut metadata = fs::symlink_metadata(path) + .map_err(|error| format!("inspect {}: {error}", path.display()))?; + digest.update( + path.strip_prefix(root) + .unwrap_or(path) + .as_os_str() + .as_encoded_bytes(), + ); + digest.update([0]); + if metadata.is_symlink() { + let target = path + .canonicalize() + .map_err(|error| format!("resolve {}: {error}", path.display()))?; + for ancestor in path.ancestors().skip(1) { + if ancestor.canonicalize().ok().as_ref() == Some(&target) { + return Err(format!( + "cyclic immutable artifact input: {}", + path.display() + )); + } + } + digest.update(b"symlink\0"); + digest.update( + fs::read_link(path) + .map_err(|error| error.to_string())? + .as_os_str() + .as_encoded_bytes(), + ); + digest.update([0]); + metadata = fs::metadata(path).map_err(|error| error.to_string())?; + } + if metadata.is_dir() { + digest.update(b"directory\0"); + let mut children = fs::read_dir(path) + .map_err(|error| error.to_string())? + .map(|entry| entry.map(|entry| entry.path())) + .collect::, _>>() + .map_err(|error| error.to_string())?; + children.sort(); + for child in children { + if matches!( + child.file_name().and_then(|name| name.to_str()), + Some("target" | ".git" | ".venv" | "__pycache__" | "node_modules") + ) { + continue; + } + hash_tree(root, &child, digest, budget)?; + } + } else if metadata.is_file() { + digest.update(b"file\0"); + digest.update((metadata.mode() & 0o777).to_le_bytes()); + digest.update(metadata.len().to_le_bytes()); + let mut file = fs::File::open(path).map_err(|error| error.to_string())?; + let mut buffer = [0_u8; 65536]; + loop { + budget.check("hash immutable artifact inputs")?; + let count = file.read(&mut buffer).map_err(|error| error.to_string())?; + if count == 0 { + break; + } + digest.update(&buffer[..count]); + } + } else { + return Err(format!( + "unsupported immutable artifact input: {}", + path.display() + )); + } + Ok(()) +} + +fn copy_atomic(source: &Path, destination: &Path) -> Result<(), String> { + let parent = destination + .parent() + .ok_or_else(|| "cache destination has no parent".to_owned())?; + let mut temporary = + tempfile::NamedTempFile::new_in(parent).map_err(|error| error.to_string())?; + let mut source = fs::File::open(source).map_err(|error| error.to_string())?; + std::io::copy(&mut source, &mut temporary).map_err(|error| error.to_string())?; + temporary + .persist(destination) + .map_err(|error| error.to_string())?; + Ok(()) +} + +fn write_json_atomic(path: &Path, value: &impl serde::Serialize) -> Result<(), String> { + let mut temporary = tempfile::NamedTempFile::new_in(path.parent().unwrap_or(Path::new("."))) + .map_err(|error| error.to_string())?; + serde_json::to_writer(temporary.as_file_mut(), value).map_err(|error| error.to_string())?; + temporary.persist(path).map_err(|error| error.to_string())?; + Ok(()) +} + +static WORKLOAD_IMAGES: LazyLock>> = LazyLock::new(Mutex::default); + +pub(crate) fn docker_image_identity(image: &str, budget: &Budget) -> Result { + let output = run_output_budget( + Command::new("docker").args(["image", "inspect", "--format", "{{.Id}}", image]), + "identify Docker image", + budget, + )?; + let identity = String::from_utf8(output.stdout).map_err(|error| error.to_string())?; + let identity = identity.trim(); + if !identity.starts_with("sha256:") { + return Err("Docker image inspection omitted immutable identity".to_owned()); + } + record_execution_identity( + &format!("image-reference:{image}"), + json!({ + "schema_version": 1, "image_reference": image, "image_digest": identity, + }), + ); + Ok(identity.to_owned()) +} + +pub(crate) fn build_workload_image( + workspace: &Path, + image: &str, + budget: &Budget, +) -> Result<(), String> { + let started = Instant::now(); + let source = resolve_myelin_binaries(workspace, budget)?.input_digest; + let mut cache = budget_lock(&WORKLOAD_IMAGES, budget, "workload image cache")?; + let base = docker_image_identity("nvidia/cuda:12.6.3-runtime-ubuntu24.04", budget).ok(); + let key = format!("{source}/{}", base.as_deref().unwrap_or("unresolved")); + if let Some(identity) = cache.get(&key) { + if docker_image_identity(identity, budget).as_ref() == Ok(identity) { + run_checked( + Command::new("docker").args(["tag", identity, image]), + "tag cached workload image", + budget, + )?; + record_execution_identity( + &format!("image:{identity}"), + json!({ + "schema_version": 1, "source_build_input_digest": source, + "image_digest": identity, "base_image_digest": base, "cache_state": "invocation_hit", + }), + ); + record_execution_stage("build.image.cache_hit", started.elapsed(), 0, 1); + return Ok(()); + } + } + let mut parent: Option = None; + for (role, dockerfile, name) in [ + ( + "base", + "apps/myelin/node-image/Dockerfile.base", + "myelin-node-base:cuda12.6", + ), + ( + "node", + "apps/myelin/node-image/Dockerfile", + "myelin-node:latest", + ), + ("e2e", "apps/myelin/node-image/Dockerfile.e2e", image), + ] { + let parent_tag = parent.as_ref().map(|identity| { + format!( + "myelin-e2e-parent:{}", + identity.trim_start_matches("sha256:") + ) + }); + if let (Some(parent), Some(tag)) = (&parent, &parent_tag) { + run_checked( + Command::new("docker").args(["tag", parent, tag]), + "pin Myelin parent image", + budget, + )?; + } + let mut command = Command::new("docker"); + command + .current_dir(workspace) + .args(["build", "-f", dockerfile, "-t", name]); + // These labels are part of the real build's immutable config, never a + // qualification-time assertion attached to an existing runtime image. + command.args(["--label", "org.swactor.myelin.e2e.provenance-version=1"]); + command.args([ + "--label", + &format!("org.swactor.myelin.e2e.source-build-input-digest={source}"), + ]); + command.args([ + "--label", + &format!("org.swactor.myelin.e2e.image-role={role}"), + ]); + if let (Some(parent), Some(tag)) = (&parent, &parent_tag) { + command.args(["--build-arg", &format!("BASE_IMAGE={tag}")]); + command.args([ + "--label", + &format!("org.swactor.myelin.e2e.parent-image-id={parent}"), + ]); + } + run_checked( + command.arg("."), + &format!("build Myelin {role} image"), + budget, + )?; + if let (Some(parent), Some(tag)) = (&parent, &parent_tag) { + if docker_image_identity(tag, budget)? != *parent { + return Err("runtime image parent changed during build".to_owned()); + } + } + parent = Some(docker_image_identity(name, budget)?); + } + if build_input_digest(workspace, budget)? != source { + return Err("source/build inputs changed while constructing runtime images".to_owned()); + } + let identity = docker_image_identity(image, budget)?; + let base = docker_image_identity("nvidia/cuda:12.6.3-runtime-ubuntu24.04", budget)?; + record_execution_identity( + &format!("image:{identity}"), + json!({ + "schema_version": 1, "source_build_input_digest": source, + "image_digest": identity, "base_image_digest": base, "cache_state": "built", + }), + ); + cache.insert(format!("{source}/{base}"), identity); + record_execution_stage("build.image.miss", started.elapsed(), 0, 1); + Ok(()) +} + +pub(crate) fn run_checked( + command: &mut Command, + label: &str, + budget: &Budget, +) -> Result<(), String> { + run_output_budget(command, label, budget).map(|_| ()) +} + +#[derive(serde::Serialize, serde::Deserialize, PartialEq, Eq)] +pub struct GateImageIdentity { + pub id: String, + pub os: String, + pub architecture: String, + pub variant: String, + pub repo_digests: BTreeSet, + pub provenance_version: u32, + pub source_build_input_digest: String, + pub image_role: String, + pub parent_image_id: Option, +} + +pub fn runtime_image_identity(name: &str, budget: &Budget) -> Result { + let image = run_output_budget( + Command::new("docker").args(["image", "inspect", name]), + "inspect qualified local runtime image", + budget, + )?; + let values: Vec = serde_json::from_slice(&image.stdout) + .map_err(|error| format!("decode runtime image identity: {error}"))?; + let value = values + .first() + .ok_or("Docker omitted runtime image identity")?; + if immutable_registry_reference(name) { + let manifest = run_output_budget( + Command::new("docker").args(["manifest", "inspect", name]), + "inspect qualified registry manifest", + budget, + )?; + let manifest: serde_json::Value = serde_json::from_slice(&manifest.stdout) + .map_err(|error| format!("decode qualified registry manifest: {error}"))?; + if manifest["schemaVersion"].as_u64() != Some(2) + || manifest.get("manifests").is_some() + || manifest["config"]["digest"].as_str() != value["Id"].as_str() + { + return Err("paid runtime requires a platform-specific registry manifest whose config digest matches the tested local image".to_owned()); + } + } + let field = |name: &str| value[name].as_str().unwrap_or_default().to_owned(); + let labels = &value["Config"]["Labels"]; + if labels["org.swactor.myelin.e2e.provenance-version"].as_str() != Some("1") { + return Err(format!( + "runtime image {name} lacks build-time source provenance; rebuild it" + )); + } + Ok(GateImageIdentity { + id: field("Id"), + os: field("Os"), + architecture: field("Architecture"), + variant: field("Variant"), + repo_digests: value["RepoDigests"] + .as_array() + .into_iter() + .flatten() + .filter_map(|value| value.as_str().map(str::to_owned)) + .collect(), + provenance_version: 1, + source_build_input_digest: labels["org.swactor.myelin.e2e.source-build-input-digest"] + .as_str() + .unwrap_or_default() + .to_owned(), + image_role: labels["org.swactor.myelin.e2e.image-role"] + .as_str() + .unwrap_or_default() + .to_owned(), + parent_image_id: labels["org.swactor.myelin.e2e.parent-image-id"] + .as_str() + .map(str::to_owned), + }) +} + +pub fn immutable_registry_reference(reference: &str) -> bool { + reference + .split_once("@sha256:") + .is_some_and(|(repository, digest)| { + !repository.is_empty() + && !repository.contains('@') + && !repository.bytes().any(|byte| byte.is_ascii_whitespace()) + && digest.len() == 64 + && digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + }) +} + +/// Capture into files, not pipes: no reader thread can outlive the owner, and a +/// grandchild holding stdout open cannot prevent process reaping on expiry. +fn run_output_budget( + command: &mut Command, + label: &str, + budget: &Budget, +) -> Result { + use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; + use std::os::unix::process::CommandExt; + let started = Instant::now(); + budget.check(label)?; + let mut stdout = tempfile::tempfile().map_err(|error| format!("{label}: {error}"))?; + let mut stderr = tempfile::tempfile().map_err(|error| format!("{label}: {error}"))?; + command + .process_group(0) + .stdin(Stdio::null()) + .stdout(Stdio::from( + stdout.try_clone().map_err(|error| error.to_string())?, + )) + .stderr(Stdio::from( + stderr.try_clone().map_err(|error| error.to_string())?, + )); + let mut child = command + .spawn() + .map_err(|error| format!("{label}: {error}"))?; + let pid = child.id() as libc::pid_t; + let descriptor = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0) } as libc::c_int; + let pidfd = (descriptor >= 0).then(|| unsafe { OwnedFd::from_raw_fd(descriptor) }); + let outcome = loop { + match child.try_wait() { + Ok(Some(status)) => break Ok(status), + Ok(None) => {} + Err(error) => break Err(format!("{label}: observe child: {error}")), + } + let remaining = match budget.remaining(label) { + Ok(remaining) => remaining, + Err(error) => break Err(error), + }; + if let Some(pidfd) = &pidfd { + let mut poll = libc::pollfd { + fd: pidfd.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }; + let timeout = + remaining.min(Duration::from_millis(50)).as_millis().max(1) as libc::c_int; + if unsafe { libc::poll(&mut poll, 1, timeout) } < 0 { + let error = std::io::Error::last_os_error(); + if error.kind() != std::io::ErrorKind::Interrupted { + break Err(format!("{label}: await child exit: {error}")); + } + } + } else if let Err(error) = budget.wait(Duration::from_millis(10), label) { + // Compatibility for kernels without pidfd_open; ownership and + // cancellation remain bounded, unlike a blocking child.wait(). + break Err(error); + } + }; + if outcome.is_err() { + // SIGKILL the whole owned group, then reap the actual child. There are + // no detached helper threads or inherited-output pipe joins. + unsafe { + libc::kill(-pid, libc::SIGKILL); + } + let _ = child.kill(); + child + .wait() + .map_err(|error| format!("{label}: reap cancelled child: {error}"))?; + } + record_execution_stage(&format!("subprocess:{label}"), started.elapsed(), 0, 1); + let status = outcome?; + stdout.rewind().map_err(|error| error.to_string())?; + stderr.rewind().map_err(|error| error.to_string())?; + let mut out = Vec::new(); + let mut err = Vec::new(); + stdout + .read_to_end(&mut out) + .map_err(|error| error.to_string())?; + stderr + .read_to_end(&mut err) + .map_err(|error| error.to_string())?; + if !status.success() { + return Err(format!( + "{label} exited {status}: {}", + String::from_utf8_lossy(&err) + )); + } + Ok(std::process::Output { + status, + stdout: out, + stderr: err, + }) } pub(crate) fn reserve_port() -> Result { @@ -107,29 +980,337 @@ pub(crate) fn capture_lines( }); } -pub(crate) fn http_json(method: &str, url: &str, body: Option) -> Result { - let response = match (method, body) { - ("GET", None) => ureq::get(url).timeout(Duration::from_secs(5)).call(), - ("POST", Some(body)) => ureq::post(url) - .timeout(Duration::from_secs(5)) - .set("content-type", "application/json") - .send_string(&body.to_string()), - _ => return Err(format!("unsupported harness HTTP request {method} {url}")), +/// One bounded control transaction. Each attempt uses a fresh socket so a +/// restarted orchestrator cannot inherit a pooled connection's stale stream. +/// GET, HEAD, and the explicitly read-only retained-resource POST are retried. +/// Mutating POSTs are never duplicated by the transport. +pub(crate) fn http_json_budget( + method: &str, + url: &str, + body: Option, + budget: &Budget, +) -> Result { + let started = Instant::now(); + let rest = url + .strip_prefix("http://") + .ok_or_else(|| "unsupported harness HTTP scheme".to_owned())?; + let (authority, path) = rest + .split_once('/') + .map(|(authority, path)| (authority, format!("/{path}"))) + .unwrap_or((rest, "/".to_owned())); + if method.bytes().any(|byte| !byte.is_ascii_uppercase()) + || path.contains(['\r', '\n']) + || authority.contains(['\r', '\n', '@']) + { + return Err("invalid harness HTTP request target".to_owned()); } - .map_err(|error| format!("{method} {url}: {error}"))?; - let mut bytes = Vec::new(); - response - .into_reader() - .read_to_end(&mut bytes) - .map_err(|error| format!("read {method} {url} response: {error}"))?; - if bytes.is_empty() { - Ok(Value::Null) + // Harness control URLs are numeric loopback addresses. Do not introduce an + // uninterruptible DNS resolver into a deadline-bounded request. + let address: SocketAddr = authority.parse().map_err(|_| { + "harness control HTTP requires an explicit numeric address and port".to_owned() + })?; + if !address.ip().is_loopback() { + return Err("harness control HTTP requires a loopback address".to_owned()); + } + let payload = body.map(|body| body.to_string()).unwrap_or_default(); + let request = format!( + "{method} {path} HTTP/1.1\r\nHost: {authority}\r\nConnection: close\r\n\ + Content-Length: {}\r\nContent-Type: application/json\r\n\r\n{payload}", + payload.len() + ); + let retryable_request = matches!(method, "GET" | "HEAD") + || (method == "POST" && path == "/api/control/contextual/retained-blobs"); + // Exclude query/request identities and bodies from metric keys. + let endpoint = path.split('?').next().unwrap_or("/"); + let endpoint = if endpoint.starts_with("/api/control/contextual/") { + "/api/control/contextual" } else { - serde_json::from_slice(&bytes) - .map_err(|error| format!("decode {method} {url} response: {error}")) + endpoint + }; + let predicate = format!("HTTP {method} {endpoint}"); + let mut attempts = 0_u64; + let result = (|| loop { + budget.check(&predicate)?; + attempts += 1; + match http_json_once(address, request.as_bytes(), budget, &predicate) { + Ok(value) => break Ok(value), + Err((error, retryable)) => { + if !retryable || !retryable_request || attempts == 3 { + break Err(error); + } + record_execution_stage("http.retry.transport", Duration::ZERO, 0, 1); + budget.wait(Duration::from_millis(20 * attempts), &predicate)?; + } + } + })(); + record_execution_stage( + &format!("http.total:{method}:{endpoint}"), + started.elapsed(), + 0, + attempts, + ); + result +} + +type HttpResult = Result; + +struct HttpAttemptSpan(Instant); + +impl Drop for HttpAttemptSpan { + fn drop(&mut self) { + record_execution_stage("http.attempt", self.0.elapsed(), 0, 1); } } +fn http_remaining(budget: &Budget, predicate: &str) -> HttpResult { + budget.remaining(predicate).map_err(|error| (error, false)) +} + +fn http_json_once( + address: SocketAddr, + request: &[u8], + budget: &Budget, + predicate: &str, +) -> HttpResult { + let started = Instant::now(); + let _span = HttpAttemptSpan(started); + let connect_budget = budget.child(Duration::from_secs(2)); + let mut stream = loop { + let timeout = http_remaining(&connect_budget, predicate)?.min(Duration::from_millis(100)); + match TcpStream::connect_timeout(&address, timeout) { + Ok(stream) => break stream, + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::TimedOut | std::io::ErrorKind::Interrupted + ) => {} + Err(error) => return Err((format!("{predicate}: Connect error: {error}"), true)), + } + }; + record_execution_stage("http.connect", started.elapsed(), 0, 1); + let write_started = Instant::now(); + let write_budget = budget.child(Duration::from_secs(10)); + let mut written = 0; + while written < request.len() { + let timeout = http_remaining(&write_budget, predicate)?.min(Duration::from_millis(50)); + stream + .set_write_timeout(Some(timeout)) + .map_err(|error| (error.to_string(), true))?; + match stream.write(&request[written..]) { + Ok(0) => return Err((format!("{predicate}: write returned zero"), true)), + Ok(count) => written += count, + Err(error) if transient_io(&error) => {} + Err(error) => return Err((format!("{predicate}: write request: {error}"), true)), + } + } + record_execution_stage("http.write", write_started.elapsed(), written as u64, 1); + let read_started = Instant::now(); + let mut response = Vec::with_capacity(4096); + let header_end = loop { + if let Some(position) = response.windows(4).position(|window| window == b"\r\n\r\n") { + break position + 4; + } + if response.len() >= 65536 { + return Err(( + format!("{predicate}: response headers exceed 64 KiB"), + false, + )); + } + if !http_read(&mut stream, &mut response, budget, predicate)? { + return Err((format!("{predicate}: response ended before headers"), true)); + } + }; + let header = std::str::from_utf8(&response[..header_end]).map_err(|error| { + ( + format!("{predicate}: invalid response header: {error}"), + false, + ) + })?; + let mut lines = header.split("\r\n"); + let status = lines + .next() + .and_then(|line| line.split_ascii_whitespace().nth(1)) + .and_then(|code| code.parse::().ok()) + .ok_or_else(|| (format!("{predicate}: invalid response status"), false))?; + let mut length = None; + let mut chunked = false; + for line in lines.filter(|line| !line.is_empty()) { + let (name, value) = line + .split_once(':') + .ok_or_else(|| (format!("{predicate}: malformed HTTP header"), false))?; + if name.eq_ignore_ascii_case("content-length") { + let parsed = value.trim().parse::().map_err(|error| { + ( + format!("{predicate}: invalid content length: {error}"), + false, + ) + })?; + if length + .replace(parsed) + .is_some_and(|previous| previous != parsed) + { + return Err((format!("{predicate}: conflicting content lengths"), false)); + } + } else if name.eq_ignore_ascii_case("transfer-encoding") { + if !value.trim().eq_ignore_ascii_case("chunked") { + return Err((format!("{predicate}: unsupported transfer encoding"), false)); + } + chunked = true; + } + } + if chunked && length.is_some() { + return Err(( + format!("{predicate}: ambiguous HTTP response framing"), + false, + )); + } + let body = if chunked { + read_chunked(&mut stream, &mut response, header_end, budget, predicate)? + } else if let Some(length) = length { + let end = header_end + .checked_add(length) + .ok_or_else(|| (format!("{predicate}: content length overflow"), false))?; + while response.len() < end { + if !http_read(&mut stream, &mut response, budget, predicate)? { + return Err((format!("{predicate}: truncated response body"), true)); + } + } + response[header_end..end].to_vec() + } else if matches!(status, 204 | 304) { + Vec::new() + } else { + // HTTP/1.0/close-delimited compatibility remains bounded by the same + // absolute budget, including a peer sending one byte per timeout. + while http_read(&mut stream, &mut response, budget, predicate)? {} + response.split_off(header_end) + }; + record_execution_stage( + "http.read", + read_started.elapsed(), + response.len() as u64, + 1, + ); + http_remaining(budget, predicate)?; + if !(200..300).contains(&status) { + return Err(( + format!( + "{predicate}: status code {status}: {}", + String::from_utf8_lossy(&body) + ), + matches!(status, 502 | 503 | 504), + )); + } + if body.is_empty() { + return Ok(Value::Null); + } + let decode_started = Instant::now(); + let result = serde_json::from_slice(&body) + .map_err(|error| (format!("{predicate}: decode response: {error}"), false)); + record_execution_stage( + "http.decode", + decode_started.elapsed(), + body.len() as u64, + 1, + ); + http_remaining(budget, predicate)?; + result +} + +fn transient_io(error: &std::io::Error) -> bool { + matches!( + error.kind(), + std::io::ErrorKind::TimedOut + | std::io::ErrorKind::WouldBlock + | std::io::ErrorKind::Interrupted + ) +} + +fn http_read( + stream: &mut TcpStream, + bytes: &mut Vec, + budget: &Budget, + predicate: &str, +) -> HttpResult { + let mut buffer = [0_u8; 8192]; + let started = Instant::now(); + loop { + let timeout = http_remaining(budget, predicate)?.min(Duration::from_millis(50)); + stream + .set_read_timeout(Some(timeout)) + .map_err(|error| (error.to_string(), true))?; + match stream.read(&mut buffer) { + Ok(0) => return Ok(false), + Ok(count) => { + bytes.extend_from_slice(&buffer[..count]); + record_execution_stage("http.received", started.elapsed(), count as u64, 1); + return Ok(true); + } + Err(error) if transient_io(&error) => {} + Err(error) => return Err((format!("{predicate}: read response: {error}"), true)), + } + } +} + +fn read_chunked( + stream: &mut TcpStream, + bytes: &mut Vec, + mut cursor: usize, + budget: &Budget, + predicate: &str, +) -> HttpResult> { + let mut body = Vec::new(); + loop { + http_remaining(budget, predicate)?; + let line_end = loop { + if let Some(position) = bytes[cursor..] + .windows(2) + .position(|window| window == b"\r\n") + { + break cursor + position; + } + if bytes.len() - cursor > 65536 { + return Err((format!("{predicate}: oversized chunk header"), false)); + } + if !http_read(stream, bytes, budget, predicate)? { + return Err((format!("{predicate}: truncated chunk header"), true)); + } + }; + let size = std::str::from_utf8(&bytes[cursor..line_end]) + .ok() + .and_then(|line| usize::from_str_radix(line.split(';').next()?.trim(), 16).ok()) + .ok_or_else(|| (format!("{predicate}: invalid chunk size"), false))?; + cursor = line_end + 2; + if size == 0 { + // Consume the terminating trailer section, not the connection EOF. + loop { + if bytes[cursor..].starts_with(b"\r\n") + || bytes[cursor..] + .windows(4) + .any(|window| window == b"\r\n\r\n") + { + return Ok(body); + } + if bytes.len() - cursor > 65536 || !http_read(stream, bytes, budget, predicate)? { + return Err((format!("{predicate}: invalid chunk trailer"), false)); + } + } + } + let end = cursor + .checked_add(size) + .and_then(|end| end.checked_add(2)) + .ok_or_else(|| (format!("{predicate}: chunk size overflow"), false))?; + while bytes.len() < end { + if !http_read(stream, bytes, budget, predicate)? { + return Err((format!("{predicate}: truncated chunk body"), true)); + } + } + if &bytes[end - 2..end] != b"\r\n" { + return Err((format!("{predicate}: missing chunk terminator"), false)); + } + body.extend_from_slice(&bytes[cursor..end - 2]); + cursor = end; + } +} #[derive(Default)] pub(crate) struct TelemetryResourceCensus { offset: u64, @@ -138,40 +1319,207 @@ pub(crate) struct TelemetryResourceCensus { active: BTreeMap, arenas: BTreeMap, poisoned: Vec, + file_identity: Option<(u64, u64)>, + source_positions: BTreeMap, + lifecycle_sequences: BTreeMap, + telemetry_gaps: Vec, + resource_snapshots: BTreeMap, + orchestrator_snapshot: Option<(String, Value)>, } impl TelemetryResourceCensus { - pub(crate) fn update(&mut self, path: &Path) -> Result { + pub(crate) fn update(&mut self, path: &Path, budget: &Budget) -> Result<(), String> { + use std::os::unix::fs::MetadataExt; + let started = Instant::now(); + budget.check("ingest telemetry resource census")?; let mut file = fs::File::open(path).map_err(|error| { format!("open telemetry resource census {}: {error}", path.display()) })?; - let length = file - .metadata() - .map_err(|error| format!("inspect telemetry resource census: {error}"))? - .len(); - if length < self.offset { - *self = Self::default(); + let metadata = file.metadata().map_err(|error| error.to_string())?; + let identity = (metadata.dev(), metadata.ino()); + let length = metadata.len(); + if length < self.offset + || self + .file_identity + .is_some_and(|previous| previous != identity) + { + return Err("telemetry archive identity changed or was truncated; lifecycle proof is incomplete".to_owned()); } + self.file_identity = Some(identity); file.seek(SeekFrom::Start(self.offset)) - .map_err(|error| format!("seek telemetry resource census: {error}"))?; - let mut appended = Vec::new(); - file.read_to_end(&mut appended) - .map_err(|error| format!("read telemetry resource census: {error}"))?; - let appended_len = u64::try_from(appended.len()) - .map_err(|_| "telemetry resource census exceeded u64".to_owned())?; - self.offset = self - .offset - .checked_add(appended_len) - .ok_or_else(|| "telemetry resource census offset overflowed".to_owned())?; - self.carry.extend_from_slice(&appended); - if let Some(last_newline) = self.carry.iter().rposition(|byte| *byte == b'\n') { - let remainder = self.carry.split_off(last_newline + 1); - let complete = std::mem::replace(&mut self.carry, remainder); - let text = std::str::from_utf8(&complete) - .map_err(|error| format!("telemetry archive is not UTF-8: {error}"))?; - self.ingest(text)?; + .map_err(|error| error.to_string())?; + let mut reader = BufReader::new(file.take(length - self.offset)); + let mut line = std::mem::take(&mut self.carry); + let initial_offset = self.offset; + let initial_records = self.next_line; + loop { + budget.check("ingest telemetry resource census")?; + let count = reader + .read_until(b'\n', &mut line) + .map_err(|error| error.to_string())?; + self.offset = self + .offset + .checked_add(count as u64) + .ok_or_else(|| "telemetry cursor overflow".to_owned())?; + if count == 0 || !line.ends_with(b"\n") { + self.carry = line; + break; + } + let text = std::str::from_utf8(&line).map_err(|error| { + let message = format!("telemetry archive is not UTF-8: {error}"); + self.telemetry_gaps.push(message.clone()); + message + })?; + self.ingest(text).map_err(|error| { + self.telemetry_gaps.push(error.clone()); + error + })?; + line.clear(); } - Ok(self.snapshot()) + record_execution_stage( + "telemetry.ingest", + started.elapsed(), + self.offset - initial_offset, + self.next_line.saturating_sub(initial_records) as u64, + ); + Ok(()) + } + + /// Only a request-tagged producer snapshot may establish fresh cleanup. + /// Archive arrival order is deliberately not a freshness predicate. + pub(crate) fn record_resource_snapshot( + &mut self, + request_id: &str, + expected_node: u64, + resources: &Value, + ) -> Result<(), String> { + if resources.get("schema_version").and_then(Value::as_u64) != Some(1) + || resources.get("request_id").and_then(Value::as_str) != Some(request_id) + || resources.get("logical_node_id").and_then(Value::as_u64) != Some(expected_node) + { + return Err(format!( + "fresh resource snapshot identity mismatch for node {expected_node}" + )); + } + let generation = resources + .get("generation") + .and_then(Value::as_u64) + .ok_or_else(|| "resource snapshot omitted generation".to_owned())?; + let sequence = resources + .get("sample_sequence") + .and_then(Value::as_u64) + .ok_or_else(|| "resource snapshot omitted sample sequence".to_owned())?; + if resources + .get("iroh_node_id") + .and_then(Value::as_str) + .is_none_or(str::is_empty) + { + return Err("resource snapshot omitted runtime identity".to_owned()); + } + if let Some(previous) = self.resource_snapshots.get(&expected_node) { + let previous_generation = previous["generation"].as_u64().unwrap_or(u64::MAX); + if generation < previous_generation + || (generation == previous_generation + && sequence <= previous["sample_sequence"].as_u64().unwrap_or(u64::MAX)) + { + return Err(format!( + "stale resource snapshot for node {expected_node}: generation {generation}, sequence {sequence}" + )); + } + if generation == previous_generation + && previous["iroh_node_id"] != resources["iroh_node_id"] + { + return Err(format!( + "resource snapshot changed runtime identity within generation for node {expected_node}" + )); + } + } + let actors = resources + .get("actors") + .and_then(Value::as_array) + .ok_or_else(|| "resource snapshot omitted actor census".to_owned())?; + for actor in actors { + if actor.get("address").and_then(Value::as_str).is_none() + || actor.get("actor_type").and_then(Value::as_str).is_none() + || actor.get("poisoned").and_then(Value::as_bool).is_none() + || actor.get("stopping").and_then(Value::as_bool).is_none() + || actor.get("mailbox_depth").and_then(Value::as_u64).is_none() + || actor.get("worker_id").and_then(Value::as_u64).is_none() + { + return Err("resource snapshot has an unidentified actor".to_owned()); + } + } + for field in ["live_bytes", "active_leases", "pending_leases"] { + if resources + .get("arena") + .and_then(|arena| arena.get(field)) + .and_then(Value::as_u64) + .is_none() + { + return Err(format!("resource snapshot omitted arena {field}")); + } + } + for field in LIVE_TRANSPORT_FIELDS { + if resources + .get("transport") + .and_then(|transport| transport.get(field)) + .and_then(Value::as_u64) + .is_none() + { + return Err(format!("resource snapshot omitted transport {field}")); + } + } + if contains_poison(resources) { + self.poisoned.push( + json!({"stream": format!("{expected_node}#{generation}"), "payload": resources}), + ); + } + self.resource_snapshots + .insert(expected_node, resources.clone()); + Ok(()) + } + + pub(crate) fn record_orchestrator_snapshot( + &mut self, + stream: &str, + request_id: &str, + reply: &Value, + ) -> Result<(), String> { + if reply.get("schema_version").and_then(Value::as_u64) != Some(1) + || reply.get("request_id").and_then(Value::as_str) != Some(request_id) + { + return Err("fresh orchestrator actor snapshot identity mismatch".to_owned()); + } + let actors = reply + .get("actors") + .and_then(Value::as_array) + .ok_or_else(|| "orchestrator snapshot omitted actors".to_owned())?; + for actor in actors { + if actor.get("address").and_then(Value::as_str).is_none() + || actor.get("actor_type").and_then(Value::as_str).is_none() + || actor.get("poisoned").and_then(Value::as_bool).is_none() + || actor.get("stopping").and_then(Value::as_bool).is_none() + || actor.get("mailbox_depth").and_then(Value::as_u64).is_none() + || actor.get("worker_id").and_then(Value::as_u64).is_none() + { + return Err("orchestrator snapshot contains an unidentified actor".to_owned()); + } + } + if self + .orchestrator_snapshot + .as_ref() + .is_some_and(|(_, previous)| { + previous.get("request_id").and_then(Value::as_str) == Some(request_id) + }) + { + return Err("orchestrator snapshot request identity was reused".to_owned()); + } + if contains_poison(reply) { + self.poisoned + .push(json!({"stream": stream, "payload": reply})); + } + self.orchestrator_snapshot = Some((stream.to_owned(), reply.clone())); + Ok(()) } pub(crate) fn ingest(&mut self, text: &str) -> Result<(), String> { @@ -190,15 +1538,50 @@ impl TelemetryResourceCensus { .get("stream") .and_then(Value::as_str) .unwrap_or("unknown"); - let payload = frame - .pointer("/payload/value") - .and_then(Value::as_str) - .ok_or_else(|| format!("telemetry frame {} has no UTF-8 payload", self.next_line)) - .and_then(|payload| { + let archived_payload = frame.pointer("/payload/value").ok_or_else(|| { + format!("telemetry frame {} has no decoded payload", self.next_line) + })?; + let payload = match archived_payload { + Value::String(payload) => { serde_json::from_str::(payload).map_err(|error| { format!("decode telemetry payload {}: {error}", self.next_line) - }) - })?; + })? + } + payload => payload.clone(), + }; + if channel == "runtime.actors" { + let generation = payload + .get("generation") + .and_then(Value::as_u64) + .ok_or_else(|| format!("actor telemetry {stream} omitted generation"))?; + let sequence = payload + .get("sequence") + .and_then(Value::as_u64) + .ok_or_else(|| format!("actor telemetry {stream} omitted sequence"))?; + if let Some(&(previous_generation, previous_sequence)) = + self.lifecycle_sequences.get(stream) + { + if generation != previous_generation + || previous_sequence.checked_add(1) != Some(sequence) + { + self.telemetry_gaps.push(format!("actor telemetry {stream}: ({previous_generation}, {previous_sequence}) -> ({generation}, {sequence})")); + } + } else if sequence != 1 { + self.telemetry_gaps.push(format!("actor telemetry {stream} begins after missing sequence 1 (observed {sequence})")); + } + self.lifecycle_sequences + .insert(stream.to_owned(), (generation, sequence)); + } + self.source_positions.insert( + format!("{stream}/{channel}"), + json!({ + "stream": stream, + "generation": payload.get("generation"), + "sequence": payload.get("sequence"), + "archive_position": frame.get("position"), + "archive_line": self.next_line, + }), + ); if contains_poison(&payload) { self.poisoned .push(json!({"stream": stream, "payload": payload})); @@ -231,118 +1614,224 @@ impl TelemetryResourceCensus { Ok(()) } + pub(crate) fn forget_stream(&mut self, stream: &str) { + let prefix = format!("{stream}/"); + self.active + .retain(|identity, _| !identity.starts_with(&prefix)); + self.arenas.remove(stream); + self.source_positions + .retain(|identity, _| !identity.starts_with(&prefix)); + self.lifecycle_sequences.remove(stream); + if self + .orchestrator_snapshot + .as_ref() + .is_some_and(|(current, _)| current == stream) + { + self.orchestrator_snapshot = None; + } + } + pub(crate) fn snapshot(&self) -> Value { - let active_actors = self + self.snapshot_filtered(None) + } + + pub(crate) fn snapshot_for_nodes(&self, live_nodes: &BTreeSet) -> Value { + self.snapshot_filtered(Some(live_nodes)) + } + + fn snapshot_filtered(&self, live_nodes: Option<&BTreeSet>) -> Value { + let started = Instant::now(); + let include_stream = |stream: &str| { + live_nodes.is_none_or(|live| { + stream + .split_once('#') + .and_then(|(node, _)| node.parse::().ok()) + .is_none_or(|node| live.contains(&node)) + }) + }; + let superseded_node = |stream: &str| { + self.orchestrator_snapshot + .as_ref() + .is_some_and(|(current, _)| current == stream) + || stream + .split_once('#') + .and_then(|(node, _)| node.parse::().ok()) + .is_some_and(|node| self.resource_snapshots.contains_key(&node)) + }; + let transient = |actor_type: &str| { + TRANSIENT_ACTOR_TYPES + .iter() + .any(|prefix| actor_type.contains(prefix)) + }; + let mut active_actors = self .active .iter() - .filter(|(_, actor_type)| { - TRANSIENT_ACTOR_TYPES - .iter() - .any(|prefix| actor_type.contains(prefix)) + .filter(|(identity, actor_type)| { + let stream = identity.split('/').next().unwrap_or(identity); + include_stream(stream) && !superseded_node(stream) && transient(actor_type) }) .map(|(identity, actor_type)| json!({"identity": identity, "type": actor_type})) .collect::>(); - json!({ + let mut arenas = self + .arenas + .iter() + .filter(|(stream, _)| include_stream(stream) && !superseded_node(stream)) + .map(|(stream, arena)| (stream.clone(), arena.clone())) + .collect::>(); + let mut resource_snapshots = BTreeMap::new(); + for (node, snapshot) in &self.resource_snapshots { + if live_nodes.is_some_and(|live| !live.contains(node)) { + continue; + } + resource_snapshots.insert(node, snapshot); + let stream = format!("{node}#{}", snapshot["generation"]); + arenas.insert(stream.clone(), snapshot["arena"].clone()); + for actor in snapshot["actors"].as_array().into_iter().flatten() { + if let (Some(address), Some(actor_type)) = + (actor["address"].as_str(), actor["actor_type"].as_str()) + { + if transient(actor_type) { + active_actors.push( + json!({"identity": format!("{stream}/{address}"), "type": actor_type}), + ); + } + } + } + } + if let Some((stream, snapshot)) = &self.orchestrator_snapshot { + for actor in snapshot["actors"].as_array().into_iter().flatten() { + if let (Some(address), Some(actor_type)) = + (actor["address"].as_str(), actor["actor_type"].as_str()) + { + if transient(actor_type) { + active_actors.push( + json!({"identity": format!("{stream}/{address}"), "type": actor_type}), + ); + } + } + } + } + let value = json!({ + "schema_version": 2, "active_actors": active_actors, - "arenas": &self.arenas, + "arenas": arenas, "poisoned": &self.poisoned, - }) + "source_positions": &self.source_positions, + "resource_snapshots": resource_snapshots, + "orchestrator_stream": self.orchestrator_snapshot.as_ref().map(|(stream, _)| stream), + "orchestrator_snapshot": self.orchestrator_snapshot.as_ref().map(|(_, snapshot)| snapshot), + "telemetry_gaps": &self.telemetry_gaps, + "archive_offset": self.offset, + "archive_records": self.next_line, + }); + record_execution_stage("telemetry.snapshot", started.elapsed(), 0, 1); + value } } -pub(crate) fn transient_actor_identities(health: &Value) -> BTreeSet { - health - .pointer("/resources/active_actors") - .and_then(Value::as_array) - .into_iter() - .flatten() - .filter_map(|actor| actor.get("identity").and_then(Value::as_str)) - .map(str::to_owned) - .collect() -} - pub(crate) fn pending_resource_cleanup( health: &Value, resource_baseline: &BTreeSet, ) -> Option { - for node in health - .get("nodes") - .and_then(Value::as_array) - .into_iter() - .flatten() - { - let live = node - .pointer("/observation/event/executions") - .and_then(Value::as_array) - .map_or(0, Vec::len); - if live != 0 { - return Some(format!("{live} contextual processes to exit")); + use myelin_control_contract::{ContextualHealthEvent, ContextualHealthReply}; + let inspect = || -> Result<(), String> { + if health.get("schema_version").and_then(Value::as_u64) != Some(2) { + return Err("versioned health snapshot".to_owned()); } - } - let running_nodes = health - .get("running_nodes") - .and_then(Value::as_array) - .into_iter() - .flatten() - .filter_map(Value::as_u64) - .collect::>(); - let active = health - .pointer("/resources/active_actors") - .and_then(Value::as_array) - .into_iter() - .flatten() - .filter(|actor| { - let Some(identity) = actor.get("identity").and_then(Value::as_str) else { - return true; - }; - if resource_baseline.contains(identity) { - return false; + let generation = health["generation"] + .as_u64() + .ok_or("health observation generation")?; + let boundary = health["health_boundary"] + .as_u64() + .ok_or("terminal health boundary")?; + if generation <= boundary { + return Err("resource observation newer than terminal health boundary".to_owned()); + } + let running: Vec = serde_json::from_value(health["running_nodes"].clone()) + .map_err(|error| format!("exact running node identities: {error}"))?; + let running_nodes = running.iter().copied().collect::>(); + if running_nodes.is_empty() || running_nodes.len() != running.len() { + return Err("nonempty unique running node identities".to_owned()); + } + let nodes: Vec = serde_json::from_value(health["nodes"].clone()) + .map_err(|error| format!("typed successful contextual replies: {error}"))?; + let mut observed = BTreeSet::new(); + for reply in &nodes { + let observation = &reply.observation; + let node = observation.logical_node_id; + reply.validate(node, &observation.request_id)?; + if !running_nodes.contains(&node) || !observed.insert(node) { + return Err(format!("unexpected or duplicate contextual node {node}")); } - identity - .split_once('/') - .and_then(|(stream, _)| stream.split_once('#')) - .and_then(|(logical_node, _)| logical_node.parse::().ok()) - .is_none_or(|logical_node| running_nodes.contains(&logical_node)) - }) - .count(); - if active != 0 { - return Some(format!("{active} transient actors to stop")); - } - let Some(arenas) = health - .pointer("/resources/arenas") - .and_then(Value::as_object) - else { - return Some("telemetry arena census".to_owned()); - }; - for node in &running_nodes { - let arena = arenas - .iter() - .filter(|(stream, _)| { - stream - .split_once('#') - .and_then(|(logical_node, _)| logical_node.parse::().ok()) - == Some(*node) - }) - .max_by_key(|(stream, _)| { - stream - .split_once('#') - .and_then(|(_, generation)| generation.parse::().ok()) - .unwrap_or(0) - }) - .map(|(_, arena)| arena); - let Some(arena) = arena else { - return Some(format!("arena census for node {node}")); - }; - for field in ["live_bytes", "active_leases", "pending_leases"] { - let count = arena.get(field).and_then(Value::as_u64).unwrap_or(u64::MAX); - if count != 0 { - return Some(format!( - "node {node} arena {field} to reach zero (observed {count})" + let ContextualHealthEvent::LiveExecutions { + executions, + resources, + } = &observation.event; + if !executions.is_empty() { + return Err(format!( + "{} contextual processes on node {node} to exit", + executions.len() )); } + let recorded = health + .pointer(&format!("/resources/resource_snapshots/{node}")) + .ok_or_else(|| format!("fresh producer resource snapshot for node {node}"))?; + if serde_json::to_value(resources).map_err(|error| error.to_string())? != *recorded { + return Err(format!( + "resource snapshot does not match contextual barrier for node {node}" + )); + } + for (field, count) in [ + ("live_bytes", resources.arena.live_bytes), + ("active_leases", resources.arena.active_leases), + ("pending_leases", resources.arena.pending_leases), + ("pending_sinks", resources.transport.pending_sinks), + ("pending_inbound", resources.transport.pending_inbound), + ("active_controls", resources.transport.active_controls), + ("source_probes", resources.transport.source_probes), + ("sink_probes", resources.transport.sink_probes), + ("local_incarnations", resources.transport.local_incarnations), + ] { + if count != 0 { + return Err(format!( + "node {node} {field} to reach zero (observed {count})" + )); + } + } } - } - None + if observed != running_nodes { + return Err(format!( + "contextual replies for every exact running node: expected {running_nodes:?}, observed {observed:?}" + )); + } + let gaps = health + .pointer("/resources/telemetry_gaps") + .and_then(Value::as_array) + .ok_or("telemetry continuity proof")?; + if !gaps.is_empty() { + return Err(format!("telemetry lifecycle gaps: {gaps:?}")); + } + let actors = health + .pointer("/resources/active_actors") + .and_then(Value::as_array) + .ok_or("fresh transient actor census")?; + for actor in actors { + let identity = actor["identity"].as_str().ok_or("actor identity")?; + actor["type"] + .as_str() + .filter(|value| !value.is_empty()) + .ok_or("actor type")?; + if resource_baseline.contains(identity) { + continue; + } + return Err(format!("transient actor {identity} to stop")); + } + if contains_poison(health) { + return Err("poisoned actor or worker panic".to_owned()); + } + Ok(()) + }; + inspect().err() } pub(crate) fn contains_poison(value: &Value) -> bool { @@ -357,78 +1846,893 @@ pub(crate) fn contains_poison(value: &Value) -> bool { } } -pub(crate) fn list_containers(prefix: &str) -> Result, String> { - let output = Command::new("docker") - .args(["ps", "-aq", "--filter", &format!("name=^{prefix}")]) - .output() - .map_err(|error| format!("list harness containers: {error}"))?; - if !output.status.success() { - return Err(format!( - "list harness containers exited {}: {}", - output.status, - String::from_utf8_lossy(&output.stderr) - )); - } - Ok(String::from_utf8_lossy(&output.stdout) - .split_whitespace() - .map(str::to_owned) - .collect()) +pub(crate) fn list_containers(prefix: &str, budget: &Budget) -> Result, String> { + Ok(container_census(prefix, budget)?.into_values().collect()) } -/// List generated-launcher command lines across harness containers. -/// -/// Returns `Ok(None)` when `docker top` cannot introspect some container — -/// it is stopped, paused, or mid-transition, so it hosts no workload right -/// now. Callers treat that as a retry signal rather than a failure, since -/// failure-injection cases routinely query containers the orchestrator is -/// concurrently tearing down. -pub(crate) fn list_workload_processes(prefix: &str) -> Result>, String> { - let mut processes = Vec::new(); - for container in list_containers(prefix)? { - let output = Command::new("docker") - .args(["top", &container, "-eo", "pid,args"]) - .output() - .map_err(|error| format!("inspect harness container {container}: {error}"))?; - if !output.status.success() { - return Ok(None); - } - for command in String::from_utf8_lossy(&output.stdout).lines().skip(1) { - if command.contains(GENERATED_LAUNCHER) { - processes.push(format!("{container}: {command}")); - } +pub(crate) fn container_census( + prefix: &str, + budget: &Budget, +) -> Result, String> { + let output = run_output_budget( + Command::new("docker").args([ + "ps", + "-a", + "--no-trunc", + "--filter", + &format!("name=^{prefix}"), + "--format", + "{{.Names}}\t{{.ID}}", + ]), + "list harness containers", + budget, + )?; + let mut census = BTreeMap::new(); + let mut identities = BTreeSet::new(); + for line in String::from_utf8_lossy(&output.stdout).lines() { + let (name, id) = line.split_once('\t').ok_or("invalid Docker census row")?; + if name.is_empty() + || id.len() != 64 + || !id.bytes().all(|byte| byte.is_ascii_hexdigit()) + || !identities.insert(id.to_owned()) + || census.insert(name.to_owned(), id.to_owned()).is_some() + { + return Err( + "Docker census omitted or repeated a complete container identity".to_owned(), + ); } } - Ok(Some(processes)) + Ok(census) } -pub(crate) fn remove_container(container: &str) -> Result<(), String> { - let status = Command::new("docker") - .args(["rm", "-f", container]) - .status() - .map_err(|error| format!("remove harness container {container}: {error}"))?; - if status.success() { - Ok(()) - } else { - Err(format!( - "remove harness container {container} exited {status}" - )) - } +pub(crate) fn remove_container(container: &str, budget: &Budget) -> Result<(), String> { + run_checked( + Command::new("docker").args(["rm", "-f", container]), + "remove harness container", + budget, + ) } -pub(crate) fn remove_containers(prefix: &str) -> Result<(), String> { - let ids = list_containers(prefix)?; +pub(crate) fn remove_containers(prefix: &str, budget: &Budget) -> Result<(), String> { + let ids = list_containers(prefix, budget)?; if ids.is_empty() { return Ok(()); } - let status = Command::new("docker") - .arg("rm") - .arg("-f") - .args(ids) - .status() - .map_err(|error| format!("remove harness containers: {error}"))?; - if status.success() { - Ok(()) + run_checked( + Command::new("docker").arg("rm").arg("-f").args(ids), + "remove harness containers", + budget, + ) +} + +/// A versioned deployment bundle: outer tar path plus the deployment +/// identity it declares. +#[derive(Clone, Debug)] +pub struct DeploymentBundle { + pub tar_path: std::path::PathBuf, + pub artifact_digest: String, + pub executable_digest: String, + pub deployment_generation: String, +} + +fn sha256_hex_budget(path: &Path, budget: &Budget) -> Result { + use sha2::{Digest, Sha256}; + let started = Instant::now(); + let mut file = + fs::File::open(path).map_err(|error| format!("hash {}: {error}", path.display()))?; + let mut digest = Sha256::new(); + let mut buffer = [0_u8; 65536]; + let mut bytes = 0_u64; + loop { + budget.check("verify immutable artifact digest")?; + let count = file.read(&mut buffer).map_err(|error| error.to_string())?; + if count == 0 { + break; + } + digest.update(&buffer[..count]); + bytes += count as u64; + } + record_execution_stage("artifact.digest", started.elapsed(), bytes, 1); + Ok(format!("{:x}", digest.finalize())) +} + +/// One-time payload staging: release binaries plus the Swactor wheel. The +/// same payload backs multiple bundle generations; only the descriptor +/// changes between them. +pub fn stage_deployment_payload( + workspace: &Path, + artifacts: &Path, + budget: &Budget, +) -> Result { + let required = required_artifacts(budget)?; + let started = Instant::now(); + let binaries = resolve_myelin_binaries(workspace, budget)?; + let cache_root = workspace + .canonicalize() + .map_err(|error| error.to_string())? + .join("target/myelin-e2e-cache"); + let venv = cache_root.join("maturin-venv"); + let maturin = venv.join("bin/maturin"); + if !maturin.is_file() { + if required.is_some() { + return Err( + "qualified wheel builder is missing; installation is prohibited".to_owned(), + ); + } + run_checked( + Command::new("python3").args(["-m", "venv"]).arg(&venv), + "create maturin virtualenv", + budget, + )?; + run_checked( + Command::new(venv.join("bin/pip")) + .args(["install", "--quiet", "maturin>=1.7,<2"]) + .current_dir(workspace), + "install maturin", + budget, + )?; + } + let key = wheel_input_digest(&binaries, &maturin, budget)?; + if required + .as_ref() + .is_some_and(|expected| expected.wheel_input_digest != key) + { + return Err("qualified wheel build inputs changed; rebuilding is prohibited".to_owned()); + } + let wheel_dir = cache_root.join("wheels").join(&key); + fs::create_dir_all(&wheel_dir).map_err(|error| format!("create wheel cache: {error}"))?; + let manifest = wheel_dir.join("digests.json"); + let cached: Option> = fs::File::open(&manifest) + .ok() + .and_then(|file| serde_json::from_reader(BufReader::new(file)).ok()); + let mut wheels = BTreeMap::new(); + if let Some(cached) = cached { + for (name, digest) in cached { + validate_artifact_filename(&name, ".whl")?; + if sha256_hex_budget(&wheel_dir.join(&name), budget) + .ok() + .as_ref() + == Some(&digest) + { + wheels.insert(name, digest); + } else { + wheels.clear(); + break; + } + } + } + if wheels.is_empty() { + if required.is_some() { + return Err( + "qualified wheel cache is missing or corrupt; rebuilding is prohibited".to_owned(), + ); + } + let temporary = tempfile::tempdir_in(&wheel_dir).map_err(|error| error.to_string())?; + run_checked( + Command::new(&maturin) + .args([ + "build", + "--release", + "--locked", + "--interpreter", + "python3", + "--manifest-path", + "crates/bindings/python/Cargo.toml", + "--out", + ]) + .arg(temporary.path()) + .current_dir(workspace), + "build Swactor wheel", + budget, + )?; + for entry in fs::read_dir(temporary.path()).map_err(|error| error.to_string())? { + let entry = entry.map_err(|error| error.to_string())?; + let name = entry.file_name().to_string_lossy().into_owned(); + if name.ends_with(".whl") { + let digest = sha256_hex_budget(&entry.path(), budget)?; + copy_atomic(&entry.path(), &wheel_dir.join(&name))?; + wheels.insert(name, digest); + } + } + if wheels.is_empty() { + return Err("maturin produced no wheel".to_owned()); + } + write_json_atomic(&manifest, &wheels)?; + record_execution_stage( + "build.wheel.miss", + started.elapsed(), + 0, + wheels.len() as u64, + ); } else { - Err(format!("remove harness containers exited {status}")) + record_execution_stage( + "build.wheel.cache_hit", + started.elapsed(), + 0, + wheels.len() as u64, + ); + } + let identity = + prepared_artifact_identity(workspace, &binaries, key.clone(), wheels.clone(), budget)?; + if required + .as_ref() + .is_some_and(|expected| expected != &identity) + { + return Err("staged payload differs from qualified artifacts".to_owned()); + } + record_execution_identity( + &format!("wheel:{key}"), + json!({ + "schema_version": 1, "source_build_input_digest": binaries.input_digest, + "wheel_input_digest": key, "wheels": wheels, + }), + ); + // Each caller receives an independent mutable staging tree: generation + // marker appends must never mutate the immutable cache or another round. + fs::create_dir_all(artifacts).map_err(|error| error.to_string())?; + let payload = tempfile::Builder::new() + .prefix("payload-") + .tempdir_in(artifacts) + .map_err(|error| error.to_string())?; + let payload_bin = payload.path().join("bin"); + let payload_python = payload.path().join("python"); + fs::create_dir_all(&payload_bin).map_err(|error| error.to_string())?; + fs::create_dir_all(&payload_python).map_err(|error| error.to_string())?; + copy_atomic(&binaries.worker, &payload_bin.join("myelin-worker"))?; + set_executable(&payload_bin.join("myelin-worker"))?; + copy_atomic( + &workspace.join("apps/myelin/node-image/e2e_python_launcher.py"), + &payload_bin.join("myelin-e2e-python"), + )?; + set_executable(&payload_bin.join("myelin-e2e-python"))?; + for name in wheels.keys() { + budget.check("stage Swactor wheel")?; + copy_atomic(&wheel_dir.join(name), &payload_python.join(name))?; + } + // Hash the copied bytes as well: cache/source changes during staging cannot pass. + for (name, expected) in &identity.payload { + if sha256_hex_budget(&payload.path().join(name), budget)? != *expected { + return Err(format!("deployment payload changed during staging: {name}")); + } + } + write_json_atomic(&artifacts.join("build-identity.json"), &identity)?; + record_execution_stage("deployment.stage", started.elapsed(), 0, 1); + Ok(payload.keep()) +} + +/// Produces a byte-distinct but executable-compatible worker artifact. ELF +/// loaders ignore trailing bytes; the marker is therefore part of the tested +/// executable digest without introducing a second build configuration. +pub fn distinguish_deployment_payload( + payload: &Path, + deployment_generation: &str, +) -> Result<(), String> { + let worker = payload.join("bin").join("myelin-worker"); + let mut file = std::fs::OpenOptions::new() + .append(true) + .open(&worker) + .map_err(|error| format!("open {} for deployment marker: {error}", worker.display()))?; + file.write_all(format!("\nMYELIN_DEPLOYMENT_ARTIFACT={deployment_generation}\n").as_bytes()) + .map_err(|error| format!("mark {} as distinct deployment: {error}", worker.display()))?; + file.sync_all() + .map_err(|error| format!("sync deployment marker in {}: {error}", worker.display())) +} + +#[cfg(unix)] +fn set_executable(path: &Path) -> Result<(), String> { + use std::os::unix::fs::PermissionsExt; + let metadata = + std::fs::metadata(path).map_err(|error| format!("inspect {}: {error}", path.display()))?; + let mut permissions = metadata.permissions(); + permissions.set_mode(0o755); + std::fs::set_permissions(path, permissions) + .map_err(|error| format!("mark {} executable: {error}", path.display())) +} + +#[cfg(not(unix))] +fn set_executable(_path: &Path) -> Result<(), String> { + Ok(()) +} + +/// Deterministic payload tar: identical payload bytes hash identically +/// across invocations. +fn write_payload_tar( + artifacts: &Path, + payload: &Path, + budget: &Budget, +) -> Result<(PathBuf, String), String> { + use sha2::{Digest, Sha256}; + let started = Instant::now(); + let mut key = Sha256::new(); + key.update(b"myelin-e2e-payload-ustar-v1\0"); + for name in ["bin", "python"] { + hash_tree(payload, &payload.join(name), &mut key, budget)?; + } + let key = format!("{:x}", key.finalize()); + let cache = artifacts + .canonicalize() + .map_err(|error| error.to_string())? + .join("deployment-cache") + .join(key); + fs::create_dir_all(&cache).map_err(|error| error.to_string())?; + let payload_tar = cache.join("payload.tar"); + let manifest = cache.join("digest.json"); + if let Ok(file) = fs::File::open(&manifest) { + if let Ok(expected) = serde_json::from_reader::<_, String>(BufReader::new(file)) { + if sha256_hex_budget(&payload_tar, budget).ok().as_ref() == Some(&expected) { + record_execution_stage("deployment.payload.cache_hit", started.elapsed(), 0, 1); + return Ok((payload_tar, expected)); + } + } + } + let temporary = tempfile::NamedTempFile::new_in(&cache).map_err(|error| error.to_string())?; + run_checked( + Command::new("tar") + .args([ + "--format=ustar", + "--sort=name", + "--mtime=@0", + "--owner=0", + "--group=0", + "--numeric-owner", + "-cf", + ]) + .arg(temporary.path()) + .args(["bin", "python"]) + .current_dir(payload), + "pack deployment payload", + budget, + )?; + let digest = sha256_hex_budget(temporary.path(), budget)?; + temporary + .persist(&payload_tar) + .map_err(|error| error.to_string())?; + write_json_atomic(&manifest, &digest)?; + record_execution_stage( + "deployment.payload.miss", + started.elapsed(), + fs::metadata(&payload_tar) + .map_err(|error| error.to_string())? + .len(), + 1, + ); + Ok((payload_tar, digest)) +} + +/// Assembles `deployment.tar` (descriptor + payload) for one generation. +/// The digest covers the payload only, so a new generation with identical +/// binaries keeps the digest and still forces a node refresh. +pub fn assemble_deployment_bundle( + artifacts: &Path, + payload: &Path, + deployment_generation: &str, + budget: &Budget, +) -> Result { + use sha2::{Digest, Sha256}; + let started = Instant::now(); + if deployment_generation.is_empty() + || !deployment_generation + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.')) + { + return Err("deployment generation must be a nonempty filename-safe identity".to_owned()); + } + let (payload_tar, digest) = write_payload_tar(artifacts, payload, budget)?; + let executable_digest = format!( + "sha256:{}", + sha256_hex_budget(&payload.join("bin/myelin-worker"), budget)? + ); + let descriptor = json!({ + "artifact_digest": format!("sha256:{digest}"), + "deployment_generation": deployment_generation, + "executable_digest": executable_digest, + }); + let descriptor_bytes = serde_json::to_vec(&descriptor).map_err(|error| error.to_string())?; + let descriptor_digest = format!("{:x}", Sha256::digest(&descriptor_bytes)); + let artifacts = artifacts + .canonicalize() + .map_err(|error| error.to_string())?; + let bundle_tar = artifacts.join(format!( + "deployment-{deployment_generation}-{descriptor_digest}.tar" + )); + let manifest = bundle_tar.with_extension("digest.json"); + let valid_cached = fs::File::open(&manifest) + .ok() + .and_then(|file| serde_json::from_reader::<_, String>(BufReader::new(file)).ok()) + .is_some_and(|expected| { + sha256_hex_budget(&bundle_tar, budget).ok().as_ref() == Some(&expected) + }); + if !valid_cached { + let descriptor_dir = tempfile::tempdir_in(&artifacts).map_err(|error| error.to_string())?; + fs::write( + descriptor_dir.path().join("deployment.json"), + descriptor_bytes, + ) + .map_err(|error| error.to_string())?; + let temporary = + tempfile::NamedTempFile::new_in(&artifacts).map_err(|error| error.to_string())?; + run_checked( + Command::new("tar") + .args([ + "--format=ustar", + "--sort=name", + "--mtime=@0", + "--owner=0", + "--group=0", + "--numeric-owner", + "-cf", + ]) + .arg(temporary.path()) + .arg("-C") + .arg(descriptor_dir.path()) + .arg("deployment.json") + .arg("-C") + .arg( + payload_tar + .parent() + .ok_or_else(|| "payload cache has no parent".to_owned())?, + ) + .arg("payload.tar"), + "pack deployment bundle", + budget, + )?; + let bundle_digest = sha256_hex_budget(temporary.path(), budget)?; + temporary + .persist(&bundle_tar) + .map_err(|error| error.to_string())?; + write_json_atomic(&manifest, &bundle_digest)?; + } + record_execution_stage( + if valid_cached { + "deployment.bundle.cache_hit" + } else { + "deployment.bundle.miss" + }, + started.elapsed(), + fs::metadata(&bundle_tar) + .map_err(|error| error.to_string())? + .len(), + 1, + ); + Ok(DeploymentBundle { + tar_path: bundle_tar, + artifact_digest: format!("sha256:{digest}"), + executable_digest, + deployment_generation: deployment_generation.to_owned(), + }) +} + +#[cfg(test)] +pub(crate) mod resource_deadline_tests { + use super::*; + use std::sync::mpsc; + + pub(crate) fn unused_binaries_for_control_peer() -> BuiltBinaries { + // Collector tests own a real loopback peer, but never build or relaunch + // a fixture. These paths must remain unusable if that contract changes. + BuiltBinaries { + orchestrator: PathBuf::new(), + worker: PathBuf::new(), + input_digest: String::new(), + executable_digests: BTreeMap::new(), + verified_files: BTreeMap::new(), + } + } + + fn accept_with_deadline(listener: &TcpListener) -> (TcpStream, SocketAddr) { + listener.set_nonblocking(true).unwrap(); + let budget = Budget::new(Duration::from_secs(2)); + loop { + match listener.accept() { + Ok((stream, address)) => { + stream.set_nonblocking(false).unwrap(); + return (stream, address); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + budget + .wait(Duration::from_millis(1), "test client connects") + .unwrap(); + } + Err(error) => panic!("accept test connection: {error}"), + } + } + } + + fn read_request(stream: &mut TcpStream) { + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut request = Vec::new(); + let mut byte = [0_u8; 1]; + while !request.ends_with(b"\r\n\r\n") { + stream.read_exact(&mut byte).unwrap(); + request.push(byte[0]); + } + } + + #[test] + fn withheld_dashboard_response_expires_and_releases_connection() { + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let address = listener.local_addr().unwrap(); + let (stop, stopped) = mpsc::channel(); + let server = thread::spawn(move || { + let (mut stream, _) = accept_with_deadline(&listener); + read_request(&mut stream); + let _ = stopped.recv_timeout(Duration::from_secs(3)); + }); + let started = Instant::now(); + let error = http_json_budget( + "GET", + &format!("http://{address}/withheld"), + None, + &Budget::new(Duration::from_millis(150)), + ) + .unwrap_err(); + let elapsed = started.elapsed(); + let _ = stop.send(()); + server.join().unwrap(); + assert!( + error.contains("pending predicate: HTTP GET /withheld"), + "{error}" + ); + assert!(elapsed < Duration::from_secs(1), "{elapsed:?}"); + } + + #[test] + fn trickle_response_cannot_renew_total_deadline() { + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let address = listener.local_addr().unwrap(); + let (stop, stopped) = mpsc::channel(); + let server = thread::spawn(move || { + let (mut stream, _) = accept_with_deadline(&listener); + read_request(&mut stream); + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 10000\r\n\r\n") + .unwrap(); + for _ in 0..200 { + if stream.write_all(b" ").is_err() { + break; + } + if stopped.recv_timeout(Duration::from_millis(10)).is_ok() { + break; + } + } + }); + let started = Instant::now(); + let error = http_json_budget( + "GET", + &format!("http://{address}/trickle"), + None, + &Budget::new(Duration::from_millis(150)), + ) + .unwrap_err(); + let elapsed = started.elapsed(); + let _ = stop.send(()); + server.join().unwrap(); + assert!( + error.contains("pending predicate: HTTP GET /trickle"), + "{error}" + ); + assert!(elapsed < Duration::from_secs(1), "{elapsed:?}"); + } + + #[test] + fn framed_response_finishes_without_waiting_for_peer_close() { + for response in [ + b"HTTP/1.1 200 OK\r\nContent-Length: 11\r\n\r\n{\"ok\":true}".as_slice(), + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\nb\r\n{\"ok\":true}\r\n0\r\n\r\n" + .as_slice(), + ] { + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let address = listener.local_addr().unwrap(); + let (stop, stopped) = mpsc::channel(); + let server = thread::spawn(move || { + let (mut stream, _) = accept_with_deadline(&listener); + read_request(&mut stream); + stream.write_all(response).unwrap(); + let _ = stopped.recv_timeout(Duration::from_secs(3)); + }); + let reply = http_json_budget( + "GET", + &format!("http://{address}/framed"), + None, + &Budget::new(Duration::from_millis(500)), + ); + let _ = stop.send(()); + server.join().unwrap(); + assert_eq!(reply.unwrap(), json!({"ok": true})); + } + } + + #[test] + fn control_restart_reconciles_with_new_connection() { + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let address = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut disconnected, _) = accept_with_deadline(&listener); + read_request(&mut disconnected); + drop(disconnected); + for generation in [2, 3] { + let (mut stream, _) = accept_with_deadline(&listener); + read_request(&mut stream); + let body = json!({"generation": generation}).to_string(); + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ) + .unwrap(); + } + }); + let budget = Budget::new(Duration::from_secs(2)); + assert_eq!( + http_json_budget("GET", &format!("http://{address}/status"), None, &budget).unwrap(), + json!({"generation": 2}) + ); + assert_eq!( + http_json_budget("GET", &format!("http://{address}/status"), None, &budget).unwrap(), + json!({"generation": 3}) + ); + server.join().unwrap(); + } + + #[test] + fn ambiguous_post_is_not_duplicated_by_transport() { + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let address = listener.local_addr().unwrap(); + let (stop, stopped) = mpsc::channel(); + let server = thread::spawn(move || { + let (mut stream, _) = accept_with_deadline(&listener); + read_request(&mut stream); + drop(stream); + let _ = stopped.recv_timeout(Duration::from_secs(2)); + listener.set_nonblocking(true).unwrap(); + assert!(listener.accept().is_err(), "ambiguous mutation was retried"); + }); + let result = http_json_budget( + "POST", + &format!("http://{address}/spawn"), + Some(json!({})), + &Budget::new(Duration::from_millis(500)), + ); + let _ = stop.send(()); + server.join().unwrap(); + assert!(result.is_err()); + } + + #[test] + fn retained_resource_post_retries_gateway_timeout() { + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let address = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + for (status, body) in [ + ("504 Gateway Timeout", json!({"error": "deadline"})), + ("200 OK", json!({"ok": true})), + ] { + let (mut stream, _) = accept_with_deadline(&listener); + read_request(&mut stream); + let body = body.to_string(); + write!( + stream, + "HTTP/1.1 {status}\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ) + .unwrap(); + } + }); + let result = http_json_budget( + "POST", + &format!("http://{address}/api/control/contextual/retained-blobs"), + None, + &Budget::new(Duration::from_secs(2)), + ); + server.join().unwrap(); + assert_eq!(result.unwrap(), json!({"ok": true})); + } + + fn snapshot(request: &str, sequence: u64, live_bytes: u64) -> Value { + json!({ + "schema_version": 1, "request_id": request, "logical_node_id": 1, + "generation": 2, "sample_sequence": sequence, "iroh_node_id": "node-1", + "artifact_digest": null, "deployment_generation": null, "collection_elapsed_us": 1, + "actors": [], "arena": { + "live_bytes": live_bytes, "active_leases": 0, "pending_leases": 0, + "seq": sequence, "sample_unix_ms": 1, "capacity_bytes": 1024, "free_bytes": 1024-live_bytes, + "largest_free_range_bytes": 1024-live_bytes, "allocation_failures_total": 0, "release_failures_total": 0, + }, + "transport": { + "pending_sinks": 0, "pending_inbound": 0, "active_controls": 0, + "source_probes": 0, "sink_probes": 0, "local_incarnations": 0, + "retired_incarnations": 7, + }, + }) + } + + pub(crate) fn health_reply(request: &str, node: u64, sequence: u64, live_bytes: u64) -> Value { + let mut resources = snapshot(request, sequence, live_bytes); + resources["logical_node_id"] = json!(node); + json!({"schema_version": 1, "type": "event", "observation": { + "logical_node_id": node, "request_id": request, + "event": {"type": "live_executions", "executions": [], "resources": resources}, + }}) + } + + fn health(census: &TelemetryResourceCensus) -> Value { + let resources = census.snapshot(); + let snapshot = &resources["resource_snapshots"]["1"]; + json!({ + "schema_version": 2, "generation": 3, "health_boundary": 2, + "nodes": [{"schema_version": 1, "type": "event", "observation": { + "logical_node_id": 1, "request_id": snapshot["request_id"], + "event": {"type": "live_executions", "executions": [], "resources": snapshot}, + }}], + "running_nodes": [1], "resources": resources, + }) + } + + #[test] + fn stale_archive_zero_cannot_replace_live_producer_snapshot() { + let mut census = TelemetryResourceCensus::default(); + census + .record_resource_snapshot("cleanup-1", 1, &snapshot("cleanup-1", 1, 64)) + .unwrap(); + census.ingest(&json!({ + "channel": "mvp.arena", "stream": "1#2", + "payload": {"value": {"live_bytes": 0, "active_leases": 0, "pending_leases": 0}}, + }).to_string()).unwrap(); + let health = health(&census); + assert!(pending_resource_cleanup(&health, &BTreeSet::new()).is_some()); + assert!( + census + .record_resource_snapshot("cleanup-2", 1, &snapshot("cleanup-1", 2, 0)) + .is_err() + ); + assert!( + census + .record_resource_snapshot("cleanup-2", 1, &snapshot("cleanup-2", 1, 0)) + .is_err() + ); + census + .record_resource_snapshot("cleanup-2", 1, &snapshot("cleanup-2", 2, 0)) + .unwrap(); + let health = self::health(&census); + assert_eq!(pending_resource_cleanup(&health, &BTreeSet::new()), None); + } + + #[test] + fn pending_transport_cannot_pass_an_empty_arena_cleanup_gate() { + let mut census = TelemetryResourceCensus::default(); + let mut pending = snapshot("cleanup-transport", 1, 0); + pending["transport"]["pending_inbound"] = json!(1); + census + .record_resource_snapshot("cleanup-transport", 1, &pending) + .unwrap(); + let health = health(&census); + assert!(pending_resource_cleanup(&health, &BTreeSet::new()).is_some()); + census + .record_resource_snapshot("cleanup-retracted", 1, &snapshot("cleanup-retracted", 2, 0)) + .unwrap(); + let health = self::health(&census); + assert_eq!(pending_resource_cleanup(&health, &BTreeSet::new()), None); + } + + #[test] + fn malformed_missing_wrong_node_and_boundary_stale_health_fail_closed() { + let mut census = TelemetryResourceCensus::default(); + census + .record_resource_snapshot("fresh", 1, &snapshot("fresh", 1, 0)) + .unwrap(); + let clean = health(&census); + assert_eq!(pending_resource_cleanup(&clean, &BTreeSet::new()), None); + let mut malformed = clean.clone(); + malformed["nodes"] = json!([{"error": "control failed"}]); + assert!(pending_resource_cleanup(&malformed, &BTreeSet::new()).is_some()); + let mut missing = clean.clone(); + missing["nodes"] = json!([]); + assert!(pending_resource_cleanup(&missing, &BTreeSet::new()).is_some()); + let mut wrong_node = clean.clone(); + wrong_node["nodes"][0]["observation"]["logical_node_id"] = json!(2); + assert!(pending_resource_cleanup(&wrong_node, &BTreeSet::new()).is_some()); + let mut stale = clean; + stale["health_boundary"] = stale["generation"].clone(); + assert!(pending_resource_cleanup(&stale, &BTreeSet::new()).is_some()); + } + + #[test] + fn dropped_lifecycle_event_cannot_be_hidden_by_fresh_zero_snapshot() { + let mut census = TelemetryResourceCensus::default(); + for sequence in [1, 3] { + census.ingest(&json!({ + "stream": "1#2", "channel": "runtime.actors", + "payload": {"value": {"generation": 2, "sequence": sequence, "event": "census", "actors": []}}, + }).to_string()).unwrap(); + } + census + .record_resource_snapshot("fresh", 1, &snapshot("fresh", 1, 0)) + .unwrap(); + assert!(pending_resource_cleanup(&health(&census), &BTreeSet::new()).is_some()); + } + + #[test] + fn orchestrator_snapshot_and_restart_use_the_exact_nondefault_stream() { + let mut census = TelemetryResourceCensus::default(); + let started = |stream: &str, address: &str| { + json!({ + "stream": stream, "channel": "runtime.actors", + "payload": {"value": { + "generation": 7, "sequence": 1, "event": "started", + "actor": {"address": address, "actor_type": "data_plane::host::DataPlaneHost"}, + }}, + }) + .to_string() + }; + census + .ingest(&started("myelin-orchestrator#7", "old")) + .unwrap(); + census.ingest(&started("1#7", "worker")).unwrap(); + census + .record_orchestrator_snapshot( + "myelin-orchestrator#7", + "fresh", + &json!({"schema_version": 1, "request_id": "fresh", "actors": []}), + ) + .unwrap(); + assert_eq!( + census.snapshot()["active_actors"], + json!([{ + "identity": "1#7/worker", "type": "data_plane::host::DataPlaneHost", + }]) + ); + + census.forget_stream("myelin-orchestrator#7"); + census + .ingest(&started("myelin-orchestrator#7", "replacement")) + .unwrap(); + let restored = census.snapshot(); + let identities = restored["active_actors"] + .as_array() + .unwrap() + .iter() + .map(|actor| actor["identity"].as_str().unwrap()) + .collect::>(); + assert_eq!( + identities, + BTreeSet::from(["1#7/worker", "myelin-orchestrator#7/replacement"]) + ); + assert_eq!(restored["telemetry_gaps"], json!([])); + } + + #[test] + fn payload_cache_preserves_generation_and_executable_identity() { + let artifacts = tempfile::tempdir().unwrap(); + let payload = artifacts.path().join("payload"); + fs::create_dir_all(payload.join("bin")).unwrap(); + fs::create_dir_all(payload.join("python")).unwrap(); + fs::write(payload.join("bin/myelin-worker"), b"executable-a").unwrap(); + let budget = Budget::new(Duration::from_secs(10)); + let first = assemble_deployment_bundle(artifacts.path(), &payload, "generation-a", &budget) + .unwrap(); + let second = + assemble_deployment_bundle(artifacts.path(), &payload, "generation-b", &budget) + .unwrap(); + assert_eq!(first.artifact_digest, second.artifact_digest); + assert_ne!(first.tar_path, second.tar_path); + distinguish_deployment_payload(&payload, "generation-c").unwrap(); + let third = assemble_deployment_bundle(artifacts.path(), &payload, "generation-c", &budget) + .unwrap(); + assert_ne!(first.artifact_digest, third.artifact_digest); + assert_ne!(first.executable_digest, third.executable_digest); + let original = run_output_budget( + Command::new("tar") + .arg("-xOf") + .arg(&first.tar_path) + .arg("deployment.json"), + "inspect first immutable descriptor", + &budget, + ) + .unwrap(); + let descriptor: Value = serde_json::from_slice(&original.stdout).unwrap(); + assert_eq!(descriptor["deployment_generation"], "generation-a"); + assert_eq!(descriptor["artifact_digest"], first.artifact_digest); } } diff --git a/tools/myelin-e2e-fuzz/src/shrink.rs b/tools/myelin-e2e-fuzz/src/shrink.rs index 9ab2ffb..9c66615 100644 --- a/tools/myelin-e2e-fuzz/src/shrink.rs +++ b/tools/myelin-e2e-fuzz/src/shrink.rs @@ -1,15 +1,209 @@ //! Failure-case reduction for persisted regressions. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeSet; -use crate::ir::{ActionOp, BehaviorCase, FailureInjection}; +use crate::budget::Budget; +use crate::ir::{ + ActionOp, BehaviorCase, ExpectedOutcome, FailureInjection, ProcessProgram, TopologyFamily, +}; + +struct CleanupScope { + live_nodes: BTreeSet, + read_only_paths: BTreeSet, + owned_paths: BTreeSet, + exact_live_nodes: bool, +} + +impl CleanupScope { + fn new(case: &BehaviorCase, exact_live_nodes: bool) -> Self { + Self { + live_nodes: case.live_nodes.clone(), + read_only_paths: case.read_only_fixture_paths.clone(), + owned_paths: case.owned_paths(), + exact_live_nodes, + } + } + + fn admits(&self, candidate: &BehaviorCase) -> bool { + candidate.read_only_fixture_paths == self.read_only_paths + && (if self.exact_live_nodes { + candidate.live_nodes == self.live_nodes + } else { + candidate.live_nodes.is_subset(&self.live_nodes) + }) + && candidate.owned_paths().is_subset(&self.owned_paths) + } +} pub fn shrink_failure( - mut case: BehaviorCase, + case: BehaviorCase, mut still_fails: impl FnMut(&BehaviorCase) -> bool, ) -> BehaviorCase { + match try_shrink_failure(case, |candidate| { + Ok::(still_fails(candidate)) + }) { + Ok(case) => case, + Err(never) => match never {}, + } +} + +pub fn try_shrink_failure( + case: BehaviorCase, + still_fails: impl FnMut(&BehaviorCase) -> Result, +) -> Result { + shrink_checked(case, still_fails, || Ok(()), true) +} + +/// Diagnosis stays on the accepted fixture; minimizing the live topology +/// would require a different baseline and is not an admissible candidate. +pub fn try_shrink_failure_budgeted( + case: BehaviorCase, + budget: &Budget, + still_fails: impl FnMut(&BehaviorCase) -> Result, +) -> Result { + shrink_checked( + case, + still_fails, + || budget.check("shrink candidate generation and oracle"), + false, + ) +} + +fn shrink_checked( + mut case: BehaviorCase, + mut still_fails: impl FnMut(&BehaviorCase) -> Result, + mut check: impl FnMut() -> Result<(), E>, + reduce_live_nodes: bool, +) -> Result { + let cleanup_scope = CleanupScope::new(&case, !reduce_live_nodes); + check()?; + if !matches!(case.failure, FailureInjection::None) { + let mut candidate = case.clone(); + candidate.failure = FailureInjection::None; + if cleanup_scope.admits(&candidate) + && candidate.validate().is_ok() + && still_fails(&candidate)? + { + case = candidate; + } + } + for route_index in (0..case.routes.len()).rev() { + check()?; + let mut candidate = case.clone(); + let paths = candidate + .routes + .remove(route_index) + .edges + .into_iter() + .map(|edge| edge.path) + .collect::>(); + if candidate + .processes + .iter() + .flat_map(|process| &process.actions) + .any(|action| { + paths.contains(action.operation.path()) + && matches!( + action.operation, + ActionOp::GatedStreamRead { .. } | ActionOp::GatedStreamWrite { .. } + ) + }) + { + continue; + } + for process in &mut candidate.processes { + process + .actions + .retain(|action| !paths.contains(action.operation.path())); + } + candidate + .processes + .retain(|process| !process.actions.is_empty()); + let remaining = candidate + .processes + .iter() + .map(|process| process.id.clone()) + .collect::>(); + for process in &mut candidate.processes { + process + .depends_on + .retain(|dependency| remaining.contains(dependency)); + } + if candidate.routes.is_empty() { + candidate.topology = TopologyFamily::Fixed; + candidate + .scenarios + .remove(&crate::ir::CoverageScenario::ConcurrentStartup); + candidate + .scenarios + .remove(&crate::ir::CoverageScenario::RingCompletion); + } + if structural_candidate_is_admissible(&cleanup_scope, &case, &candidate) + && still_fails(&candidate)? + { + case = candidate; + } + } + // Same-path mutations can be mutual prerequisites across processes. + // Try removing their namespace work together before individual deletions, + // rather than weakening the guard or getting stuck on an unrelated cycle. + // This adds at most one candidate per distinct, execution-resolved path. + let paths = case + .processes + .iter() + .flat_map(|process| { + process.actions.iter().flat_map(move |action| { + action + .operation + .paths() + .into_iter() + .flatten() + .map(move |path| prerequisite_path(process, path).into_owned()) + }) + }) + .collect::>(); + for path in paths { + check()?; + let mut candidate = case.clone(); + let mut removed = false; + for (process_index, process) in candidate.processes.iter_mut().enumerate() { + let original = &case.processes[process_index]; + process.actions.retain(|action| { + let keep = !action + .operation + .paths() + .into_iter() + .flatten() + .any(|other| prerequisite_path(original, other).as_ref() == path.as_str()); + removed |= !keep; + keep + }); + } + if !removed { + continue; + } + candidate + .processes + .retain(|process| !process.actions.is_empty()); + let remaining = candidate + .processes + .iter() + .map(|process| process.id.clone()) + .collect::>(); + for process in &mut candidate.processes { + process + .depends_on + .retain(|dependency| remaining.contains(dependency)); + } + if structural_candidate_is_admissible(&cleanup_scope, &case, &candidate) + && still_fails(&candidate)? + { + case = candidate; + } + } let mut index = case.processes.len(); - while index > 1 { + while index > 0 && case.processes.len() > 1 { + check()?; index -= 1; let mut candidate = case.clone(); let removed = candidate.processes.remove(index).id; @@ -18,44 +212,45 @@ pub fn shrink_failure( .depends_on .retain(|dependency| dependency != &removed); } - if candidate.validate().is_ok() && still_fails(&candidate) { + if structural_candidate_is_admissible(&cleanup_scope, &case, &candidate) + && still_fails(&candidate)? + { case = candidate; } } for process_index in 0..case.processes.len() { let mut action_index = case.processes[process_index].actions.len(); - while action_index > 1 { + while action_index > 0 && case.processes[process_index].actions.len() > 1 { + check()?; action_index -= 1; let mut candidate = case.clone(); candidate.processes[process_index] .actions .remove(action_index); - if candidate.validate().is_ok() && still_fails(&candidate) { + if structural_candidate_is_admissible(&cleanup_scope, &case, &candidate) + && still_fails(&candidate)? + { case = candidate; } } } - for process_index in 1..case.processes.len() { - for predecessor_index in (0..process_index).rev() { - let predecessor = case.processes[predecessor_index].id.clone(); - if case.processes[process_index] - .depends_on - .contains(&predecessor) - { - continue; - } + for process_index in 0..case.processes.len() { + for dependency_index in (0..case.processes[process_index].depends_on.len()).rev() { + check()?; let mut candidate = case.clone(); candidate.processes[process_index] .depends_on - .push(predecessor); - if candidate.validate().is_ok() && still_fails(&candidate) { + .remove(dependency_index); + if structural_candidate_is_admissible(&cleanup_scope, &case, &candidate) + && still_fails(&candidate)? + { case = candidate; - break; } } } for process_index in 0..case.processes.len() { for action_index in 0..case.processes[process_index].actions.len() { + check()?; let stream = match &case.processes[process_index].actions[action_index].operation { ActionOp::StreamWrite { path, chunks, .. } if chunks.len() > 1 => { Some((path.clone(), chunks.concat())) @@ -65,39 +260,37 @@ pub fn shrink_failure( let Some((path, payload)) = stream else { continue; }; + if case.routes.iter().any(|route| { + route.edges.len() > 1 && route.edges.iter().any(|edge| edge.path == path) + }) { + continue; + } let mut candidate = case.clone(); if let ActionOp::StreamWrite { chunks, .. } = &mut candidate.processes[process_index].actions[action_index].operation { - *chunks = vec![payload.clone()]; + *chunks = vec![payload]; } - for process in &mut candidate.processes { - for action in &mut process.actions { - if let ActionOp::StreamRead { - path: reader_path, - expected, - } - | ActionOp::StreamReadInto { - path: reader_path, - expected, - .. - } = &mut action.operation - && reader_path == &path - { - *expected = payload.clone(); - } - } - } - if candidate.validate().is_ok() && still_fails(&candidate) { + if cleanup_scope.admits(&candidate) + && candidate.validate().is_ok() + && still_fails(&candidate)? + { case = candidate; } } } for process_index in 0..case.processes.len() { for action_index in 0..case.processes[process_index].actions.len() { - while let Some(candidate) = shrink_payload_candidate(&case, process_index, action_index) - { - if candidate.validate().is_ok() && still_fails(&candidate) { + loop { + check()?; + let Some(candidate) = shrink_payload_candidate(&case, process_index, action_index) + else { + break; + }; + if cleanup_scope.admits(&candidate) + && candidate.validate().is_ok() + && still_fails(&candidate)? + { case = candidate; } else { break; @@ -105,38 +298,164 @@ pub fn shrink_failure( } } } + check()?; + if !reduce_live_nodes { + return Ok(case); + } let mut used_nodes = case .processes .iter() .map(|process| process.logical_node_id) + .chain( + case.routes + .iter() + .flat_map(|route| &route.edges) + .flat_map(|edge| [edge.source, edge.destination]), + ) .collect::>(); - if let FailureInjection::KillNode { logical_node_id } = &case.failure { - used_nodes.insert(*logical_node_id); - } - let node_mapping = used_nodes - .into_iter() - .enumerate() - .map(|(index, logical_node_id)| (logical_node_id, index as u64 + 1)) - .collect::>(); - let reduced_nodes = node_mapping.len().max(2) as u8; - let topology_changed = reduced_nodes != case.node_count - || node_mapping + if used_nodes.len() == 1 + && let Some(spare) = case + .live_nodes .iter() - .any(|(logical_node_id, replacement)| logical_node_id != replacement); - if topology_changed { + .copied() + .find(|node| !used_nodes.contains(node)) + { + used_nodes.insert(spare); + } + check()?; + if used_nodes != case.live_nodes { let mut candidate = case.clone(); - candidate.node_count = reduced_nodes; - for process in &mut candidate.processes { - process.logical_node_id = node_mapping[&process.logical_node_id]; - } - if let FailureInjection::KillNode { logical_node_id } = &mut candidate.failure { - *logical_node_id = node_mapping[logical_node_id]; - } - if candidate.validate().is_ok() && still_fails(&candidate) { + candidate.live_nodes = used_nodes; + if cleanup_scope.admits(&candidate) + && candidate.validate().is_ok() + && still_fails(&candidate)? + { case = candidate; } } - case + check()?; + Ok(case) +} + +/// A typed abort can match even when deleting its setup makes the error correct. +/// Preserve the original namespace prerequisites instead of asking that failed +/// replay to prove its own expected-success contract. This deliberately keeps +/// all potentially relevant mutations, not just a guessed winning publication. +fn structural_candidate_is_admissible( + cleanup_scope: &CleanupScope, + original: &BehaviorCase, + candidate: &BehaviorCase, +) -> bool { + if !cleanup_scope.admits(candidate) || candidate.validate().is_err() { + return false; + } + let retained = original + .processes + .iter() + .map(|process| { + let mut actions = candidate + .processes + .iter() + .find(|remaining| remaining.id == process.id) + .into_iter() + .flat_map(|remaining| &remaining.actions) + .peekable(); + let retained = process + .actions + .iter() + .map(|action| { + if actions.peek().is_some_and(|next| *next == action) { + actions.next(); + true + } else { + false + } + }) + .collect::>(); + debug_assert!( + actions.next().is_none(), + "structural candidates only delete actions" + ); + retained + }) + .collect::>(); + for (process_index, process) in original.processes.iter().enumerate() { + for (action_index, action) in process.actions.iter().enumerate() { + if retained[process_index][action_index] { + continue; + } + for path in action.operation.paths().into_iter().flatten() { + // Successful local probes and waits also establish a prefix + // on which a later success relies. Across processes, retain + // mutations and synchronization, not unrelated pure readers. + let affects_peers = action.operation.mutating_paths().contains(&Some(path)) + || matches!( + action.operation, + ActionOp::AwaitEntry { .. } | ActionOp::WaitForQuiescent { .. } + ); + let path = prerequisite_path(process, path); + for (consumer_index, consumer) in original.processes.iter().enumerate() { + for (step, required) in consumer.actions.iter().enumerate() { + if !retained[consumer_index][step] + || (process_index == consumer_index && step <= action_index) + || (process_index != consumer_index && !affects_peers) + || !matches!( + required.expected, + ExpectedOutcome::Ok + | ExpectedOutcome::Linearized { successes: 1.., .. } + ) + { + continue; + } + if required + .operation + .paths() + .into_iter() + .flatten() + .any(|required| prerequisite_path(consumer, required) == path) + { + return false; + } + } + } + } + } + } + // A retained producer is insufficient if removing a dependency (or an + // intermediate process) lets its consumer run before publication. Keep + // transitive ordering, while permitting removal of redundant direct edges. + candidate.processes.iter().all(|process| { + let before = dependency_ancestors(original, process); + let after = dependency_ancestors(candidate, process); + candidate.processes.iter().all(|ancestor| { + !before.contains(ancestor.id.as_str()) || after.contains(ancestor.id.as_str()) + }) + }) +} + +fn prerequisite_path<'a>(process: &ProcessProgram, path: &'a str) -> std::borrow::Cow<'a, str> { + if let Some(suffix) = path.strip_prefix("/runs/self") + && (suffix.is_empty() || suffix.starts_with('/')) + { + format!("/runs/{}{suffix}", process.access.execution_id).into() + } else { + path.into() + } +} + +fn dependency_ancestors<'a>(case: &'a BehaviorCase, process: &ProcessProgram) -> BTreeSet<&'a str> { + let mut ancestors = BTreeSet::new(); + let mut pending = vec![process.id.as_str()]; + while let Some(id) = pending.pop() { + if let Some(program) = case.processes.iter().find(|program| program.id == id) { + for dependency in &program.depends_on { + if ancestors.insert(dependency.as_str()) { + pending.push(dependency); + } + } + } + } + ancestors } fn shrink_payload_candidate( @@ -150,66 +469,169 @@ fn shrink_payload_candidate( .actions .get(action_index)? .operation; - let mut candidate = case.clone(); - match operation { - ActionOp::PublishBlob { path, bytes } if !bytes.is_empty() => { - let reduced = bytes[..bytes.len() / 2].to_vec(); - if let ActionOp::PublishBlob { bytes, .. } = - &mut candidate.processes[process_index].actions[action_index].operation - { - *bytes = reduced.clone(); - } - for process in &mut candidate.processes { - for action in &mut process.actions { - if let ActionOp::ReadBlob { - path: reader_path, - expected, - } = &mut action.operation - && reader_path == path - { - *expected = reduced.clone(); - } - } - } + // A multi-edge relay's payload is coupled to downstream transformations. + // Keep that behavioral graph intact rather than manufacturing a mismatch + // by changing only one edge's writer/reader expectations. + if case.routes.iter().any(|route| { + route.edges.len() > 1 && route.edges.iter().any(|edge| edge.path == operation.path()) + }) { + return None; + } + let (stream, payload) = match operation { + ActionOp::PublishBlob { bytes, .. } => { + (false, std::borrow::Cow::Borrowed(bytes.as_slice())) } - ActionOp::StreamWrite { path, chunks, .. } - if chunks.iter().map(Vec::len).sum::() != 0 => - { - let payload = chunks.concat(); - let reduced = payload[..payload.len() / 2].to_vec(); - if let ActionOp::StreamWrite { chunks, .. } = - &mut candidate.processes[process_index].actions[action_index].operation - { - *chunks = vec![reduced.clone()]; - } - for process in &mut candidate.processes { - for action in &mut process.actions { - if let ActionOp::StreamRead { - path: reader_path, - expected, - } - | ActionOp::StreamReadInto { - path: reader_path, - expected, - .. - } = &mut action.operation - && reader_path == path - { - *expected = reduced.clone(); - } - } - } - } - ActionOp::DescriptorWrite { bytes, .. } if !bytes.is_empty() => { - if let ActionOp::DescriptorWrite { - bytes: candidate_bytes, - .. - } = &mut candidate.processes[process_index].actions[action_index].operation - { - candidate_bytes.truncate(bytes.len() / 2); - } + ActionOp::DescriptorWrite { bytes, length, .. } if *length == Some(bytes.len() as u64) => { + (false, std::borrow::Cow::Borrowed(bytes.as_slice())) } + ActionOp::StreamWrite { chunks, .. } => (true, std::borrow::Cow::Owned(chunks.concat())), _ => return None, + }; + if payload.is_empty() { + return None; + } + let path = prerequisite_path(&case.processes[process_index], operation.path()); + let reduced = &payload[..payload.len() / 2]; + let mut candidate = case.clone(); + for (peer_index, process) in candidate.processes.iter_mut().enumerate() { + for (step, action) in process.actions.iter_mut().enumerate() { + if peer_index == process_index && step == action_index { + continue; + } + let original_process = &case.processes[peer_index]; + if !action + .operation + .paths() + .into_iter() + .flatten() + .any(|other| prerequisite_path(original_process, other) == path) + { + continue; + } + // Coupled replacements, renames and partial descriptor ranges need + // a different reduction. Never create a new payload error by + // rewriting just one possible source or leaving its reader stale. + let expected = match &mut action.operation { + ActionOp::ReadBlob { expected, .. } if !stream => expected, + ActionOp::DescriptorRead { + expected, + flags, + offset, + .. + } if !stream && *flags == libc::O_RDONLY && *offset == 0 => expected, + ActionOp::StreamRead { expected, .. } + | ActionOp::StreamReadWithRetry { expected, .. } + | ActionOp::StreamReadInto { expected, .. } + if stream => + { + expected + } + ActionOp::Lookup { .. } + | ActionOp::AwaitEntry { .. } + | ActionOp::WaitForQuiescent { .. } + | ActionOp::Unlink { .. } => continue, + _ => return None, + }; + if expected.as_slice() != payload.as_ref() { + return None; + } + *expected = reduced.to_vec(); + } + } + match &mut candidate.processes[process_index].actions[action_index].operation { + ActionOp::PublishBlob { bytes, .. } => bytes.truncate(reduced.len()), + ActionOp::DescriptorWrite { bytes, length, .. } => { + bytes.truncate(reduced.len()); + *length = Some(reduced.len() as u64); + } + ActionOp::StreamWrite { chunks, .. } => *chunks = vec![reduced.to_vec()], + _ => unreachable!("supported payload operation checked before cloning"), } Some(candidate) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::ir::Action; + + #[test] + fn payload_candidates_keep_halving_order_and_matching_reader_contract() { + let mut case = crate::corpus::stable_corpus(2, 1).remove(0); + let path = "/cases/shrink/payload".to_owned(); + case.processes[0].actions = vec![Action::ok(ActionOp::PublishBlob { + path: path.clone(), + bytes: b"abcdef".to_vec(), + })]; + case.processes[1].actions = vec![ + Action::ok(ActionOp::ReadBlob { + path: path.clone(), + expected: b"abcdef".to_vec(), + }), + Action::ok(ActionOp::ReadBlob { + path: "/cases/shrink/other".to_owned(), + expected: b"unchanged".to_vec(), + }), + ]; + for expected in [b"abc".as_slice(), b"a".as_slice(), b"".as_slice()] { + case = shrink_payload_candidate(&case, 0, 0).unwrap(); + assert!(matches!(&case.processes[0].actions[0].operation, + ActionOp::PublishBlob { bytes, .. } if bytes == expected)); + assert!(matches!(&case.processes[1].actions[0].operation, + ActionOp::ReadBlob { expected: bytes, .. } if bytes == expected)); + assert!(matches!(&case.processes[1].actions[1].operation, + ActionOp::ReadBlob { expected, .. } if expected == b"unchanged")); + } + assert!(shrink_payload_candidate(&case, 0, 0).is_none()); + assert!(shrink_payload_candidate(&case, 1, 0).is_none()); + case.processes[0].actions[0].operation = ActionOp::StreamWrite { + path, + chunks: vec![Vec::new(), Vec::new()], + replace: false, + }; + assert!(shrink_payload_candidate(&case, 0, 0).is_none()); + } + + #[test] + fn descriptor_payload_reduction_preserves_allocation_and_scoped_readers() { + use crate::ir::{DescriptorFinish, DescriptorReadMethod, DescriptorWriteMethod}; + + let mut case = crate::corpus::stable_corpus(2, 1).remove(0); + let path = "/runs/self/blob"; + case.processes[0].actions = vec![ + Action::ok(ActionOp::DescriptorWrite { + path: path.to_owned(), + flags: libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL | libc::O_TRUNC, + length: Some(6), + bytes: b"abcdef".to_vec(), + method: DescriptorWriteMethod::Write, + finish: DescriptorFinish::Close, + }), + Action::ok(ActionOp::DescriptorRead { + path: path.to_owned(), + flags: libc::O_RDONLY, + expected: b"abcdef".to_vec(), + method: DescriptorReadMethod::Mapping, + offset: 0, + finish: DescriptorFinish::Close, + }), + ]; + case.processes[1].actions = vec![Action::ok(ActionOp::ReadBlob { + path: path.to_owned(), + expected: b"unrelated".to_vec(), + })]; + let candidate = shrink_payload_candidate(&case, 0, 0).unwrap(); + assert!(matches!(&candidate.processes[0].actions[0].operation, + ActionOp::DescriptorWrite { bytes, length: Some(3), .. } if bytes == b"abc")); + assert!(matches!(&candidate.processes[0].actions[1].operation, + ActionOp::DescriptorRead { expected, .. } if expected == b"abc")); + assert_eq!(candidate.processes[1], case.processes[1]); + + case.processes[0].actions.push(Action::ok(ActionOp::Rename { + source: path.to_owned(), + destination: "/runs/self/moved".to_owned(), + replace: false, + })); + assert!(shrink_payload_candidate(&case, 0, 0).is_none()); + } +}