fuzz: checkpoint reusable fleet campaigns and ordered qualification
Extend the existing IR, generated Python workloads, independent oracle, regression corpus, resource accounting, and shrinker for the five-node normal/recovery/survivor campaign. Retain exact fixture identities across attempts and persist coverage, failure, timing, and cleanup evidence. Add raw Docker/SSH redeployment fixtures, bounded campaign execution, durable paid-run cleanup state, ordered gate attestation, and scripted-provider safety scenarios. Include the offline oracle reproduction utility for focused diagnosis. Preserve paid access guards; no paid acceptance is claimed. Review verification: 169 harness library tests and 4 ordered-attestation tests passed. The retained ordered run passed Gate A and the full local campaign, then exceeded the complete workflow timing ceiling before later safety checks.
This commit is contained in:
parent
5bbdfb041e
commit
6b8b1de897
21 changed files with 39575 additions and 2224 deletions
|
|
@ -4,6 +4,7 @@ version = "0.1.0"
|
|||
edition = "2024"
|
||||
license = "AGPL-3.0-only"
|
||||
publish = false
|
||||
default-run = "myelin-e2e-fuzz"
|
||||
|
||||
[dependencies]
|
||||
base64 = "0.22"
|
||||
|
|
@ -13,6 +14,10 @@ serde = { version = "1", features = ["derive"] }
|
|||
serde_json = "1"
|
||||
sha2 = "0.10"
|
||||
ureq = "2"
|
||||
myelin-control-contract = { path = "../../crates/myelin-control-contract" }
|
||||
provisioning = { path = "../../crates/provisioning" }
|
||||
swactor-vastai = { path = "../vastai" }
|
||||
tempfile = "3"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
|
|
|||
344
tools/myelin-e2e-fuzz/ordered_acceptance.py
Executable file
344
tools/myelin-e2e-fuzz/ordered_acceptance.py
Executable file
|
|
@ -0,0 +1,344 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Run the complete local gates in order; retain every timing, including failures.
|
||||
|
||||
Build artifacts first. Warm runs still time fixture creation, deployment,
|
||||
readiness, workload, recovery, fault injection, evidence and cleanup. This
|
||||
runner never acquires paid resources or removes caches to manufacture a cold run.
|
||||
Use --build-images to record source/build provenance during the real Docker
|
||||
builds. Without it, every cached image must already prove the same qualified
|
||||
inputs and parent image identities. Container workers/wheels are ABI-specific:
|
||||
qualification binds their build inputs, not equality with host artifact bytes.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
BINARY = ROOT / "target/release/myelin-e2e-fuzz"
|
||||
|
||||
|
||||
TEST_PACKAGES = [
|
||||
"swactor", "myelin-e2e-fuzz", "swactor-vastai", "myelin-control-contract", "provisioning",
|
||||
"myelin", "data-plane", "iroh-driver", "distribution", "swactor-process",
|
||||
"swactor-process-context",
|
||||
]
|
||||
TEST_COMMAND = ["cargo", "test"] + [
|
||||
argument for package in TEST_PACKAGES for argument in ["-p", package]
|
||||
] + ["--tests"]
|
||||
def digest(path):
|
||||
with path.open("rb") as source:
|
||||
return hashlib.file_digest(source, "sha256").hexdigest()
|
||||
|
||||
|
||||
def source_digest():
|
||||
paths = [ROOT / name for name in (
|
||||
"Cargo.toml", "Cargo.lock", "rust-toolchain.toml", ".dockerignore", "clippy.toml")]
|
||||
excluded = {"target", ".git", ".venv", "__pycache__", "node_modules"}
|
||||
for name in (".cargo", "src", "tests", "crates", "xtask", "apps/myelin", "tools/myelin-e2e-fuzz",
|
||||
"tools/vastai", "tools/actor-control-flow-lint"):
|
||||
for directory, directories, files in os.walk(ROOT / name, followlinks=False):
|
||||
for entry in directories + files:
|
||||
if (Path(directory) / entry).is_symlink():
|
||||
raise RuntimeError(f"source identity rejects symlink {directory}/{entry}")
|
||||
directories[:] = [entry for entry in directories if entry not in excluded]
|
||||
paths.extend(Path(directory) / entry for entry in files)
|
||||
result = hashlib.sha256()
|
||||
for path in sorted(paths):
|
||||
result.update(os.fsencode(path.relative_to(ROOT)))
|
||||
result.update(b"\0")
|
||||
result.update(digest(path).encode())
|
||||
result.update(b"\0")
|
||||
return result.hexdigest()
|
||||
|
||||
|
||||
def persist(path, evidence):
|
||||
temporary = path.with_suffix(".tmp")
|
||||
with temporary.open("w") as output:
|
||||
json.dump(evidence, output, separators=(",", ":"))
|
||||
output.flush()
|
||||
os.fsync(output.fileno())
|
||||
temporary.replace(path)
|
||||
parent = os.open(path.parent, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(parent)
|
||||
finally:
|
||||
os.close(parent)
|
||||
|
||||
|
||||
def probe(command):
|
||||
result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True, timeout=30)
|
||||
if result.returncode:
|
||||
raise RuntimeError(f"metadata command failed: {command[0]}: {result.stderr}")
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def runtime_image_identities(image):
|
||||
roles = {"myelin-node-base:cuda12.6": "base", "myelin-node:latest": "node", image: "e2e"}
|
||||
identities = {}
|
||||
for name in sorted({"myelin-node-base:cuda12.6", "myelin-node:latest", image}):
|
||||
value = json.loads(probe(["docker", "image", "inspect", name]))[0]
|
||||
if "@sha256:" in name:
|
||||
manifest = json.loads(probe(["docker", "manifest", "inspect", name]))
|
||||
if (manifest.get("schemaVersion") != 2 or "manifests" in manifest
|
||||
or manifest.get("config", {}).get("digest") != value["Id"]):
|
||||
raise RuntimeError("runtime image requires a platform-specific registry manifest "
|
||||
"whose config digest matches the tested local image")
|
||||
labels = value.get("Config", {}).get("Labels") or {}
|
||||
prefix = "org.swactor.myelin.e2e."
|
||||
if labels.get(prefix + "provenance-version") != "1":
|
||||
raise RuntimeError(f"runtime image {name} lacks build-time source provenance; rebuild it")
|
||||
identities[name] = {
|
||||
"id": value["Id"], "os": value["Os"], "architecture": value["Architecture"],
|
||||
"variant": value.get("Variant", ""),
|
||||
"repo_digests": sorted(value.get("RepoDigests") or []),
|
||||
"provenance_version": 1,
|
||||
"source_build_input_digest": labels.get(prefix + "source-build-input-digest", ""),
|
||||
"image_role": labels.get(prefix + "image-role", ""),
|
||||
"parent_image_id": labels.get(prefix + "parent-image-id"),
|
||||
}
|
||||
if identities[name]["image_role"] != roles[name]:
|
||||
raise RuntimeError(f"runtime image {name} has the wrong build provenance role")
|
||||
return identities
|
||||
|
||||
|
||||
def verify_image_provenance(identities, image, source_build_input_digest):
|
||||
if len(identities) != 3 or any(
|
||||
value["source_build_input_digest"] != source_build_input_digest
|
||||
for value in identities.values()):
|
||||
raise RuntimeError("runtime images were not built from the qualified source/build inputs")
|
||||
base = identities["myelin-node-base:cuda12.6"]
|
||||
node = identities["myelin-node:latest"]
|
||||
runtime = identities[image]
|
||||
if (base["parent_image_id"] is not None
|
||||
or node["parent_image_id"] != base["id"]
|
||||
or runtime["parent_image_id"] != node["id"]):
|
||||
raise RuntimeError("runtime image provenance does not match the qualified parent images")
|
||||
|
||||
|
||||
def image_build_command(role, image, source_build_input_digest, parent_image_id=None):
|
||||
suffix = {"base": ".base", "node": "", "e2e": ".e2e"}[role]
|
||||
command = ["docker", "build", "-f", "apps/myelin/node-image/Dockerfile" + suffix,
|
||||
"-t", image]
|
||||
labels = {
|
||||
"provenance-version": "1",
|
||||
"source-build-input-digest": source_build_input_digest,
|
||||
"image-role": role,
|
||||
}
|
||||
if parent_image_id is not None:
|
||||
labels["parent-image-id"] = parent_image_id
|
||||
command += ["--build-arg", "BASE_IMAGE=myelin-e2e-parent:" + parent_image_id.removeprefix("sha256:")]
|
||||
for key, value in labels.items():
|
||||
command += ["--label", "org.swactor.myelin.e2e." + key + "=" + value]
|
||||
return command + ["."]
|
||||
|
||||
|
||||
def verify_qualified_inputs(evidence):
|
||||
identity_path = Path(evidence["artifact_identity_path"])
|
||||
if json.loads(identity_path.read_text()) != evidence["deployment_artifacts"]:
|
||||
raise RuntimeError("qualified artifact manifest changed during ordered gates")
|
||||
if evidence["source_digest"] != source_digest() or any(
|
||||
digest(ROOT / "target/release" / name) != expected
|
||||
for name, expected in evidence["build_artifacts"].items()):
|
||||
raise RuntimeError("tested source or binaries changed during ordered gates")
|
||||
if evidence["image_identities"] != runtime_image_identities(evidence["configuration"]["image"]):
|
||||
raise RuntimeError("deployment runtime images changed during ordered gates")
|
||||
|
||||
|
||||
def execute(name, command, directory, evidence, evidence_path, env=None, warm_started=None):
|
||||
started = time.monotonic()
|
||||
env = dict(os.environ if env is None else env,
|
||||
CARGO_TARGET_DIR=str(ROOT / "target"))
|
||||
log = directory / f"{name}.log"
|
||||
stage = {"name": name, "command": [str(part) for part in command],
|
||||
"log": str(log), "state": "running",
|
||||
"started_unix_ms": time.time_ns() // 1_000_000}
|
||||
evidence["stages"].append(stage)
|
||||
persist(evidence_path, evidence)
|
||||
print(f"starting {name}: {log}", flush=True)
|
||||
try:
|
||||
if warm_started is not None and time.monotonic() - warm_started >= 600:
|
||||
raise RuntimeError("complete warm workflow exhausted its timing ceiling; later gates not run")
|
||||
if "image_identities" in evidence:
|
||||
verify_qualified_inputs(evidence)
|
||||
if "deployment_artifacts" in evidence:
|
||||
identity_path = Path(evidence["artifact_identity_path"])
|
||||
if json.loads(identity_path.read_text()) != evidence["deployment_artifacts"]:
|
||||
raise RuntimeError("qualified artifact manifest changed during ordered gates")
|
||||
env["MYELIN_E2E_ARTIFACT_IDENTITY"] = str(identity_path)
|
||||
with log.open("wb") as output:
|
||||
result = subprocess.run(command, cwd=ROOT, env=env, stdout=output,
|
||||
stderr=subprocess.STDOUT)
|
||||
stage["exit_code"] = result.returncode
|
||||
stage["state"] = "passed" if result.returncode == 0 else "failed"
|
||||
except BaseException as error:
|
||||
stage["state"] = "interrupted"
|
||||
stage["error"] = str(error)
|
||||
raise
|
||||
finally:
|
||||
stage["elapsed_secs"] = time.monotonic() - started
|
||||
stage["completed_unix_ms"] = time.time_ns() // 1_000_000
|
||||
persist(evidence_path, evidence)
|
||||
if result.returncode:
|
||||
raise RuntimeError(f"{name} failed ({result.returncode}); see {log}; later gates not run")
|
||||
if name.endswith("-campaign") and stage["elapsed_secs"] > 300:
|
||||
raise RuntimeError(f"{name} exceeded the complete campaign timing ceiling; later gates not run")
|
||||
if warm_started is not None and time.monotonic() - warm_started > 600:
|
||||
raise RuntimeError("complete warm workflow exceeded its timing ceiling; later gates not run")
|
||||
print(f"passed {name}: {stage['elapsed_secs']:.3f}s", flush=True)
|
||||
return stage["elapsed_secs"]
|
||||
|
||||
|
||||
def run(args):
|
||||
root = args.artifacts.resolve()
|
||||
# Do not reuse coverage or overwrite evidence from an earlier invocation.
|
||||
root.mkdir(parents=True, exist_ok=False)
|
||||
path = root / "ordered-acceptance.json"
|
||||
evidence = {"schema_version": 5, "state": "running", "stages": [], "runs": [],
|
||||
"machine": {"platform": platform.platform(),
|
||||
"cpu_count": os.cpu_count(),
|
||||
"cpuinfo": Path("/proc/cpuinfo").read_text(),
|
||||
"memory": Path("/proc/meminfo").read_text()},
|
||||
"configuration": {"seed": args.seed, "warm_runs": args.warm_runs,
|
||||
"case_deadline_secs": args.deadline_secs,
|
||||
"image": args.image, "build_images": args.build_images,
|
||||
"workspace": str(ROOT), "artifacts": str(root)},
|
||||
"paid_execution": "not_attempted"}
|
||||
started = time.monotonic()
|
||||
try:
|
||||
evidence["source_digest"] = source_digest()
|
||||
evidence["storage"] = probe(["df", "-T", str(root)])
|
||||
evidence["images_before_build"] = probe(
|
||||
["docker", "image", "list", "--format", "{{.Repository}}:{{.Tag}} {{.ID}}"])
|
||||
execute("build", ["cargo", "build", "--release", "-p", "myelin",
|
||||
"--bins", "-p", "myelin-e2e-fuzz"], root, evidence, path)
|
||||
execute("build-test-binaries", TEST_COMMAND + ["--no-run"], root, evidence, path)
|
||||
execute("build-deployment-artifacts",
|
||||
[str(BINARY), "--prepare-artifacts", "--deadline-secs", "3600",
|
||||
"--artifacts", str(root / "build-deployment-artifacts")],
|
||||
root, evidence, path)
|
||||
identity_path = root / "build-deployment-artifacts/build-identity.json"
|
||||
evidence["deployment_artifacts"] = json.loads(identity_path.read_text())
|
||||
evidence["artifact_identity_path"] = str(identity_path)
|
||||
if args.build_images:
|
||||
parent = None
|
||||
for role, image in [("base", "myelin-node-base:cuda12.6"),
|
||||
("node", "myelin-node:latest"), ("e2e", args.image)]:
|
||||
if parent is not None:
|
||||
parent_tag = "myelin-e2e-parent:" + parent.removeprefix("sha256:")
|
||||
execute(f"build-image-{role}-parent", ["docker", "tag", parent, parent_tag],
|
||||
root, evidence, path)
|
||||
execute(f"build-image-{role}", image_build_command(
|
||||
role, image, evidence["deployment_artifacts"]["source_build_input_digest"],
|
||||
parent), root, evidence, path)
|
||||
if parent is not None and json.loads(
|
||||
probe(["docker", "image", "inspect", parent_tag]))[0]["Id"] != parent:
|
||||
raise RuntimeError("runtime image parent changed during build")
|
||||
parent = json.loads(probe(["docker", "image", "inspect", image]))[0]["Id"]
|
||||
evidence["build_artifacts"] = {
|
||||
name: digest(ROOT / "target/release" / name)
|
||||
for name in ["myelin-e2e-fuzz", "myelin-orchestrator", "myelin-worker"]}
|
||||
if evidence["source_digest"] != source_digest():
|
||||
raise RuntimeError("source changed while building acceptance artifacts")
|
||||
evidence["image_identities"] = runtime_image_identities(args.image)
|
||||
verify_image_provenance(evidence["image_identities"], args.image,
|
||||
evidence["deployment_artifacts"]["source_build_input_digest"])
|
||||
if "@sha256:" in args.image and args.image not in evidence["image_identities"][args.image]["repo_digests"]:
|
||||
raise RuntimeError("runtime image is not bound to the requested registry manifest")
|
||||
evidence["build_elapsed_secs"] = time.monotonic() - started
|
||||
# Building with a pre-existing cache is not a cold benchmark.
|
||||
evidence["cold_measurement"] = "not_claimed; existing cache retained"
|
||||
for index in range(args.warm_runs):
|
||||
directory = root / f"warm-{index + 1}"
|
||||
directory.mkdir()
|
||||
record = {"index": index + 1, "state": "running"}
|
||||
evidence["runs"].append(record)
|
||||
run_start = time.monotonic()
|
||||
common = [str(BINARY), "--seed", str(args.seed), "--deadline-secs",
|
||||
str(args.deadline_secs), "--no-build-image", "--image", args.image]
|
||||
try:
|
||||
execute(f"warm-{index + 1}-gate-a",
|
||||
common + ["--deployment-e2e", "--deployment-nodes", "5",
|
||||
"--artifacts", str(directory / "gate-a")],
|
||||
directory, evidence, path, warm_started=run_start)
|
||||
record["gate_a"] = "passed"
|
||||
campaign_secs = execute(f"warm-{index + 1}-campaign",
|
||||
common + ["--campaign", "--fixture-lifetime-secs", "43200",
|
||||
"--artifacts", str(directory / "campaign")],
|
||||
directory, evidence, path, warm_started=run_start)
|
||||
record["campaign_elapsed_secs"] = campaign_secs
|
||||
execute(f"warm-{index + 1}-failure-cases",
|
||||
common + ["--nodes", "5", "--failure-cases", "--artifacts",
|
||||
str(directory / "failure-cases")], directory, evidence, path,
|
||||
warm_started=run_start)
|
||||
execute(f"warm-{index + 1}-contract-model-safety",
|
||||
TEST_COMMAND, directory, evidence, path, warm_started=run_start)
|
||||
execute(f"warm-{index + 1}-scripted-provider",
|
||||
["bash", "tools/myelin-e2e-fuzz/scripted_safety_gate.sh",
|
||||
str(directory / "scripted-provider")], directory, evidence, path,
|
||||
env=dict(os.environ, SAFETY_GATE_ROOT=str(ROOT),
|
||||
SCRIPTED_HARNESS_BINARY=str(BINARY)), warm_started=run_start)
|
||||
record["state"] = "passed"
|
||||
except BaseException:
|
||||
record["state"] = "failed"
|
||||
raise
|
||||
finally:
|
||||
record["elapsed_secs"] = time.monotonic() - run_start
|
||||
record["inside_target"] = (record["state"] == "passed"
|
||||
and record.get("campaign_elapsed_secs", float("inf")) <= 300
|
||||
and record["elapsed_secs"] <= 600)
|
||||
persist(path, evidence)
|
||||
if not record["inside_target"]:
|
||||
raise RuntimeError(f"warm run {index + 1} passed behavior but exceeded timing target")
|
||||
execute("verify-qualified-deployment-artifacts",
|
||||
[str(BINARY), "--prepare-artifacts", "--deadline-secs", str(args.deadline_secs),
|
||||
"--artifacts", str(root / "verify-qualified-deployment-artifacts")],
|
||||
root, evidence, path)
|
||||
if json.loads((root / "verify-qualified-deployment-artifacts/build-identity.json").read_text()) != evidence["deployment_artifacts"]:
|
||||
raise RuntimeError("resolver-selected deployment artifacts changed during ordered gates")
|
||||
verify_qualified_inputs(evidence)
|
||||
evidence["state"] = "passed"
|
||||
except BaseException as error:
|
||||
evidence["state"] = "failed"
|
||||
evidence["error"] = str(error)
|
||||
raise
|
||||
finally:
|
||||
evidence["elapsed_secs"] = time.monotonic() - started
|
||||
evidence["completed_unix_ms"] = time.time_ns() // 1_000_000
|
||||
evidence["expires_unix_ms"] = evidence["completed_unix_ms"] + 86_400_000
|
||||
persist(path, evidence)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--artifacts", type=Path, required=True)
|
||||
parser.add_argument("--image", default="myelin-e2e-speed:local")
|
||||
parser.add_argument("--build-images", action="store_true")
|
||||
parser.add_argument("--seed", type=int, default=20260910)
|
||||
parser.add_argument("--deadline-secs", type=int, default=120)
|
||||
parser.add_argument("--warm-runs", type=int, default=3)
|
||||
args = parser.parse_args()
|
||||
if args.warm_runs < 3 or args.deadline_secs <= 0:
|
||||
parser.error("at least three warm runs and a positive operation deadline are required")
|
||||
if args.image in {"myelin-node-base:cuda12.6", "myelin-node:latest"}:
|
||||
parser.error("the workload image must be distinct from its base and node images")
|
||||
if args.build_images and "@sha256:" in args.image:
|
||||
parser.error("build with a mutable image tag, publish it, then qualify its immutable "
|
||||
"registry reference without --build-images")
|
||||
try:
|
||||
run(args)
|
||||
except (OSError, RuntimeError, subprocess.SubprocessError) as error:
|
||||
print(error, file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
1068
tools/myelin-e2e-fuzz/scripted_provider.py
Executable file
1068
tools/myelin-e2e-fuzz/scripted_provider.py
Executable file
File diff suppressed because it is too large
Load diff
16
tools/myelin-e2e-fuzz/scripted_safety_gate.sh
Executable file
16
tools/myelin-e2e-fuzz/scripted_safety_gate.sh
Executable file
|
|
@ -0,0 +1,16 @@
|
|||
#!/usr/bin/env bash
|
||||
# Gate B only: run after Gate A. Every scenario invokes the real paid CLI
|
||||
# against an independently owned loopback HTTP provider and request ledger.
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
OUT="${1:-$ROOT/target/vastai-gate-b-scripted-1}"
|
||||
cd "$ROOT"
|
||||
TARGET="${CARGO_TARGET_DIR:-$ROOT/target}"
|
||||
# An ordered owner can supply its already-built immutable binary. Standalone
|
||||
# invocation builds both required binaries once, never once per scenario.
|
||||
if [ -z "${SCRIPTED_HARNESS_BINARY:-}" ]; then
|
||||
cargo build --release -q -p myelin --bins -p myelin-e2e-fuzz
|
||||
SCRIPTED_HARNESS_BINARY="$TARGET/release/myelin-e2e-fuzz"
|
||||
fi
|
||||
exec python3 -E -B "$ROOT/tools/myelin-e2e-fuzz/scripted_provider.py" \
|
||||
--gate "$ROOT" "$OUT" "$SCRIPTED_HARNESS_BINARY"
|
||||
35
tools/myelin-e2e-fuzz/src/bin/oracle_repro.rs
Normal file
35
tools/myelin-e2e-fuzz/src/bin/oracle_repro.rs
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
use myelin_e2e_fuzz::{BehaviorCase, BehaviorOracle, CaseObservation};
|
||||
use std::env;
|
||||
use std::fs;
|
||||
use std::time::Instant;
|
||||
|
||||
fn main() {
|
||||
let args: Vec<String> = env::args().collect();
|
||||
if args.len() != 3 {
|
||||
eprintln!("usage: oracle_repro <case.json> <observed.json>");
|
||||
std::process::exit(2);
|
||||
}
|
||||
let mut case: BehaviorCase =
|
||||
serde_json::from_slice(&fs::read(&args[1]).expect("read case")).expect("parse case");
|
||||
let observation: CaseObservation =
|
||||
serde_json::from_slice(&fs::read(&args[2]).expect("read observation"))
|
||||
.expect("parse observation");
|
||||
if let Ok(budget) = std::env::var("REPRO_RACE_BUDGET") {
|
||||
case.resource_bounds.max_race_states = budget.parse().expect("budget integer");
|
||||
}
|
||||
let started = Instant::now();
|
||||
match BehaviorOracle::verify(&case, &observation) {
|
||||
Ok(()) => {
|
||||
println!("ORACLE OK in {:?}", started.elapsed());
|
||||
}
|
||||
Err(violation) => {
|
||||
println!(
|
||||
"ORACLE VIOLATION {} {} in {:?}",
|
||||
violation.invariant,
|
||||
violation.detail,
|
||||
started.elapsed()
|
||||
);
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
247
tools/myelin-e2e-fuzz/src/budget.rs
Normal file
247
tools/myelin-e2e-fuzz/src/budget.rs
Normal file
|
|
@ -0,0 +1,247 @@
|
|||
//! Monotonic, hierarchical execution ownership and out-of-band timing evidence.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Condvar, LazyLock, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use serde::Serialize;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Cancellation {
|
||||
cancelled: AtomicBool,
|
||||
parent: Option<Arc<Cancellation>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct Wake {
|
||||
revision: Mutex<u64>,
|
||||
changed: Condvar,
|
||||
}
|
||||
|
||||
/// Clones share cancellation and an absolute deadline. Child cancellation is
|
||||
/// isolated; parent cancellation wakes and stops every descendant.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Budget {
|
||||
deadline: Instant,
|
||||
cancellation: Arc<Cancellation>,
|
||||
wake: Arc<Wake>,
|
||||
}
|
||||
|
||||
impl Budget {
|
||||
pub fn new(duration: Duration) -> Self {
|
||||
let now = Instant::now();
|
||||
Self {
|
||||
// An unrepresentable allocation must fail closed, not be unbounded.
|
||||
deadline: now.checked_add(duration).unwrap_or(now),
|
||||
cancellation: Arc::new(Cancellation {
|
||||
cancelled: AtomicBool::new(false),
|
||||
parent: None,
|
||||
}),
|
||||
wake: Arc::new(Wake::default()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn child(&self, duration: Duration) -> Self {
|
||||
let now = Instant::now();
|
||||
Self {
|
||||
deadline: self.deadline.min(now.checked_add(duration).unwrap_or(now)),
|
||||
cancellation: Arc::new(Cancellation {
|
||||
cancelled: AtomicBool::new(false),
|
||||
parent: Some(Arc::clone(&self.cancellation)),
|
||||
}),
|
||||
wake: Arc::clone(&self.wake),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn remaining(&self, predicate: &str) -> Result<Duration, String> {
|
||||
let mut cancellation = Some(self.cancellation.as_ref());
|
||||
while let Some(state) = cancellation {
|
||||
if state.cancelled.load(Ordering::Acquire) {
|
||||
record_pending(predicate, "cancelled");
|
||||
return Err(format!("budget cancelled; pending predicate: {predicate}"));
|
||||
}
|
||||
cancellation = state.parent.as_deref();
|
||||
}
|
||||
let remaining = self.deadline.saturating_duration_since(Instant::now());
|
||||
if remaining.is_zero() {
|
||||
record_pending(predicate, "deadline_exhausted");
|
||||
return Err(format!("budget exhausted; pending predicate: {predicate}"));
|
||||
}
|
||||
Ok(remaining)
|
||||
}
|
||||
|
||||
pub fn check(&self, predicate: &str) -> Result<(), String> {
|
||||
self.remaining(predicate).map(|_| ())
|
||||
}
|
||||
|
||||
/// Interruptible pacing/reconciliation wait; never extends its owner.
|
||||
pub fn wait(&self, duration: Duration, predicate: &str) -> Result<(), String> {
|
||||
let started = Instant::now();
|
||||
let target = started.checked_add(duration).unwrap_or(self.deadline);
|
||||
let mut revision = self
|
||||
.wake
|
||||
.revision
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner());
|
||||
let mut wake_count = 0_u64;
|
||||
let result = loop {
|
||||
let remaining = match self.remaining(predicate) {
|
||||
Ok(remaining) => remaining,
|
||||
Err(error) => break Err(error),
|
||||
};
|
||||
let until_target = target.saturating_duration_since(Instant::now());
|
||||
if until_target.is_zero() {
|
||||
break Ok(());
|
||||
}
|
||||
let (guard, _) = self
|
||||
.wake
|
||||
.changed
|
||||
.wait_timeout(revision, remaining.min(until_target))
|
||||
.unwrap_or_else(|error| error.into_inner());
|
||||
revision = guard;
|
||||
wake_count = wake_count.saturating_add(1);
|
||||
};
|
||||
drop(revision);
|
||||
record_stage(
|
||||
&format!("wait:{predicate}"),
|
||||
started.elapsed(),
|
||||
0,
|
||||
0,
|
||||
wake_count,
|
||||
);
|
||||
result
|
||||
}
|
||||
|
||||
pub fn cancel(&self) {
|
||||
let mut revision = self
|
||||
.wake
|
||||
.revision
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner());
|
||||
self.cancellation.cancelled.store(true, Ordering::Release);
|
||||
*revision = revision.wrapping_add(1);
|
||||
self.wake.changed.notify_all();
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Serialize)]
|
||||
struct StageEvidence {
|
||||
count: u64,
|
||||
elapsed_ns: u128,
|
||||
max_elapsed_ns: u128,
|
||||
bytes: u64,
|
||||
records: u64,
|
||||
wake_count: u64,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ExecutionEvidence {
|
||||
stages: BTreeMap<String, StageEvidence>,
|
||||
pending: BTreeMap<String, BTreeMap<String, u64>>,
|
||||
immutable_artifacts: BTreeMap<String, Value>,
|
||||
}
|
||||
|
||||
static EVIDENCE: LazyLock<Mutex<ExecutionEvidence>> = LazyLock::new(Mutex::default);
|
||||
|
||||
fn evidence() -> &'static Mutex<ExecutionEvidence> {
|
||||
&EVIDENCE
|
||||
}
|
||||
|
||||
/// Record runner work without changing the causal workload stdout protocol.
|
||||
pub fn record_execution_stage(stage: &str, elapsed: Duration, bytes: u64, records: u64) {
|
||||
record_stage(stage, elapsed, bytes, records, 0);
|
||||
}
|
||||
|
||||
fn record_stage(stage: &str, elapsed: Duration, bytes: u64, records: u64, wakes: u64) {
|
||||
let mut evidence = evidence().lock().unwrap_or_else(|error| error.into_inner());
|
||||
let entry = evidence.stages.entry(stage.to_owned()).or_default();
|
||||
entry.count = entry.count.saturating_add(1);
|
||||
entry.elapsed_ns = entry.elapsed_ns.saturating_add(elapsed.as_nanos());
|
||||
entry.max_elapsed_ns = entry.max_elapsed_ns.max(elapsed.as_nanos());
|
||||
entry.bytes = entry.bytes.saturating_add(bytes);
|
||||
entry.records = entry.records.saturating_add(records);
|
||||
entry.wake_count = entry.wake_count.saturating_add(wakes);
|
||||
}
|
||||
|
||||
fn record_pending(predicate: &str, reason: &str) {
|
||||
let mut evidence = evidence().lock().unwrap_or_else(|error| error.into_inner());
|
||||
let count = evidence
|
||||
.pending
|
||||
.entry(predicate.to_owned())
|
||||
.or_default()
|
||||
.entry(reason.to_owned())
|
||||
.or_default();
|
||||
*count = count.saturating_add(1);
|
||||
}
|
||||
|
||||
/// Retain verified source/build/image identities alongside timing evidence.
|
||||
pub(crate) fn record_execution_identity(name: &str, identity: Value) {
|
||||
evidence()
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner())
|
||||
.immutable_artifacts
|
||||
.insert(name.to_owned(), identity);
|
||||
}
|
||||
|
||||
/// A versioned cumulative snapshot persisted at phase and failure boundaries,
|
||||
/// separately from observations used to establish causal ordering.
|
||||
pub fn execution_evidence() -> Value {
|
||||
let evidence = evidence().lock().unwrap_or_else(|error| error.into_inner());
|
||||
json!({
|
||||
"schema_version": 1,
|
||||
"clock": "monotonic",
|
||||
"stages": evidence.stages,
|
||||
"pending_predicates": evidence.pending,
|
||||
"immutable_artifacts": evidence.immutable_artifacts,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn child_cancellation_does_not_cancel_parent_or_sibling() {
|
||||
let parent = Budget::new(Duration::from_secs(5));
|
||||
let child = parent.child(Duration::from_secs(2));
|
||||
let sibling = parent.child(Duration::from_secs(2));
|
||||
child.clone().cancel();
|
||||
assert!(child.check("child").is_err());
|
||||
parent.check("parent").unwrap();
|
||||
sibling.check("sibling").unwrap();
|
||||
parent.cancel();
|
||||
assert!(sibling.check("parent cancelled").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ancestor_cancellation_interrupts_descendant_wait() {
|
||||
let parent = Budget::new(Duration::from_secs(30));
|
||||
let child = parent.child(Duration::from_secs(30));
|
||||
let started = Instant::now();
|
||||
let waiter =
|
||||
std::thread::spawn(move || child.wait(Duration::from_secs(20), "withheld event"));
|
||||
parent.cancel();
|
||||
assert!(
|
||||
waiter
|
||||
.join()
|
||||
.unwrap()
|
||||
.unwrap_err()
|
||||
.contains("withheld event")
|
||||
);
|
||||
assert!(started.elapsed() < Duration::from_secs(2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn child_never_renews_expired_parent() {
|
||||
let parent = Budget::new(Duration::ZERO);
|
||||
let child = parent.child(Duration::from_secs(30));
|
||||
assert!(
|
||||
child
|
||||
.remaining("nested operation")
|
||||
.unwrap_err()
|
||||
.contains("nested operation")
|
||||
);
|
||||
}
|
||||
}
|
||||
1318
tools/myelin-e2e-fuzz/src/campaign.rs
Normal file
1318
tools/myelin-e2e-fuzz/src/campaign.rs
Normal file
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
4447
tools/myelin-e2e-fuzz/src/coverage.rs
Normal file
4447
tools/myelin-e2e-fuzz/src/coverage.rs
Normal file
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
|
|
@ -1,143 +1,284 @@
|
|||
//! Cluster convergence: dashboard, provisioning, and pairwise functional proof.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::thread;
|
||||
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::codegen::{digest, render_python};
|
||||
use crate::budget::Budget;
|
||||
use crate::harness::ClusterHarness;
|
||||
use crate::ir::{AccessSpec, Action, ActionOp, ExecutionObservation, ProcessProgram};
|
||||
use crate::oracle::BehaviorOracle;
|
||||
use crate::resources::{http_json, transient_actor_identities};
|
||||
use crate::ir::{
|
||||
AccessSpec, Action, ActionOp, BehaviorCase, CASE_SCHEMA_VERSION, CaseResourceBounds,
|
||||
FailureInjection, GENERATOR_VERSION, ProcessProgram, TopologyFamily,
|
||||
};
|
||||
use crate::resources::http_json_budget;
|
||||
|
||||
use super::POLL_INTERVAL;
|
||||
|
||||
impl ClusterHarness {
|
||||
pub(super) fn wait_for_dashboard(&mut self) -> Result<(), String> {
|
||||
let deadline = Instant::now() + self.config.deadline;
|
||||
let started = Instant::now();
|
||||
let budget = self.operation_budget().child(self.config.deadline);
|
||||
let result = (|| {
|
||||
let mut pending = "dashboard control status and actors".to_owned();
|
||||
loop {
|
||||
budget.check(&pending)?;
|
||||
self.ensure_orchestrator_live()?;
|
||||
if let Ok(status) = http_json(
|
||||
let request_budget = budget.child(Duration::from_secs(2));
|
||||
match http_json_budget(
|
||||
"GET",
|
||||
&format!("{}/api/control/status", self.base_url),
|
||||
None,
|
||||
&request_budget,
|
||||
) {
|
||||
let configured_image = status
|
||||
.pointer("/Status/provider/runtime_image")
|
||||
Ok(status) => {
|
||||
let provider = status
|
||||
.pointer("/Status/provider")
|
||||
.ok_or_else(|| format!("control status omitted provider: {status}"))?;
|
||||
let configured_image = provider
|
||||
.get("runtime_image")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or_else(|| {
|
||||
format!("control status omitted configured runtime image: {status}")
|
||||
})?;
|
||||
if configured_image != self.image {
|
||||
return Err(format!(
|
||||
"control status runtime image {configured_image:?} does not match harness image {:?}",
|
||||
"control runtime image {configured_image:?} does not match {:?}",
|
||||
self.image
|
||||
));
|
||||
}
|
||||
return Ok(());
|
||||
let readiness =
|
||||
provider
|
||||
.get("kind")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or_else(|| {
|
||||
format!("control status omitted provider readiness: {status}")
|
||||
})?;
|
||||
if readiness != "ready" {
|
||||
pending = format!(
|
||||
"provider readiness {readiness}: {}",
|
||||
provider
|
||||
.get("error")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("validation pending")
|
||||
);
|
||||
} else {
|
||||
match http_json_budget(
|
||||
"GET",
|
||||
&format!("{}/api/control/actors", self.base_url),
|
||||
None,
|
||||
&request_budget,
|
||||
) {
|
||||
Ok(_) => return Ok(()),
|
||||
Err(error) => pending = format!("dashboard actor control: {error}"),
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(self.timeout_evidence("dashboard readiness"));
|
||||
}
|
||||
thread::sleep(POLL_INTERVAL);
|
||||
}
|
||||
Err(error) => pending = format!("dashboard status: {error}"),
|
||||
}
|
||||
// Startup has no fleet-event subscription until the dashboard binds.
|
||||
budget.wait(POLL_INTERVAL, &pending)?;
|
||||
}
|
||||
})();
|
||||
self.record_lifecycle("dashboard_readiness", started, &result)?;
|
||||
result
|
||||
}
|
||||
|
||||
pub(super) fn provision_nodes(&mut self) -> Result<(), String> {
|
||||
let command_id = format!("e2e-provision-{}", self.config.seed);
|
||||
let selected_offer_ids = (1..=u64::from(self.config.node_count)).collect::<Vec<_>>();
|
||||
http_json(
|
||||
pub(super) fn dispatch_provision(&self) -> Result<(), String> {
|
||||
let response = http_json_budget(
|
||||
"POST",
|
||||
&format!("{}/api/control/provision", self.base_url),
|
||||
Some(json!({
|
||||
"command_id": command_id,
|
||||
"command_id": format!("e2e-provision-{}", self.config.seed),
|
||||
"count": self.config.node_count,
|
||||
"selected_offer_ids": selected_offer_ids,
|
||||
"selected_offer_ids": self.config.selected_offer_ids,
|
||||
"search_id": self.config.offer_search_id,
|
||||
})),
|
||||
&self.operation_budget().child(self.config.deadline),
|
||||
)?;
|
||||
let deadline = Instant::now() + self.config.deadline;
|
||||
loop {
|
||||
self.ensure_orchestrator_live()?;
|
||||
let fleet = http_json("GET", &format!("{}/api/control/fleet", self.base_url), None)?;
|
||||
let nodes = fleet
|
||||
.get("FleetStatus")
|
||||
.and_then(|fleet| fleet.get("nodes"))
|
||||
.and_then(Value::as_array);
|
||||
let running = nodes
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter(|node| node.get("phase").and_then(Value::as_str) == Some("running"))
|
||||
.filter_map(|node| node.get("logical_node_id").and_then(Value::as_u64))
|
||||
.collect::<Vec<_>>();
|
||||
if running.len() == usize::from(self.config.node_count) {
|
||||
self.node_ids = running;
|
||||
self.node_ids.sort_unstable();
|
||||
return Ok(());
|
||||
let admitted = response
|
||||
.pointer("/Accepted/node_ids")
|
||||
.and_then(Value::as_array)
|
||||
.is_some_and(|nodes| nodes.len() == usize::from(self.config.node_count));
|
||||
if !admitted {
|
||||
return Err(format!(
|
||||
"provision command did not durably admit the exact node count: {response}"
|
||||
));
|
||||
}
|
||||
if nodes.is_some_and(|nodes| {
|
||||
nodes.len() == usize::from(self.config.node_count)
|
||||
&& nodes.iter().all(|node| {
|
||||
node.get("phase").and_then(Value::as_str) == Some("stopped")
|
||||
&& !node.get("last_error").is_none_or(Value::is_null)
|
||||
})
|
||||
}) {
|
||||
return Err(format!("worker provisioning failed: {fleet}"));
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(self.timeout_evidence(&format!(
|
||||
"{} workers running; fleet={fleet}",
|
||||
self.config.node_count
|
||||
)));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
thread::sleep(POLL_INTERVAL);
|
||||
pub(super) fn provision_nodes(&mut self, provision: bool) -> Result<(), String> {
|
||||
let started = Instant::now();
|
||||
let budget = self.operation_budget().child(self.config.deadline);
|
||||
let result = (|| {
|
||||
if provision {
|
||||
self.dispatch_provision()?;
|
||||
}
|
||||
let expected = if self.node_ids.is_empty() {
|
||||
(1..=u64::from(self.config.node_count)).collect()
|
||||
} else {
|
||||
self.node_ids.clone()
|
||||
};
|
||||
loop {
|
||||
budget.check(&format!("exact running nodes {expected:?}"))?;
|
||||
self.ensure_orchestrator_live()?;
|
||||
let cursor = self.control_revision(&budget)?;
|
||||
let fleet = http_json_budget(
|
||||
"GET",
|
||||
&format!("{}/api/control/fleet", self.base_url),
|
||||
None,
|
||||
&budget,
|
||||
)?;
|
||||
let nodes = fleet
|
||||
.pointer("/FleetStatus/nodes")
|
||||
.and_then(Value::as_array)
|
||||
.ok_or_else(|| format!("fleet omitted node census: {fleet}"))?;
|
||||
let expected_running = expected.iter().copied().collect::<BTreeSet<_>>();
|
||||
if expected_running.len() != expected.len() || expected_running.contains(&0) {
|
||||
return Err(format!("invalid expected logical-node set {expected:?}"));
|
||||
}
|
||||
let mut seen = BTreeSet::new();
|
||||
let mut running = Vec::new();
|
||||
let mut stopped = BTreeSet::new();
|
||||
for node in nodes {
|
||||
let node_id = node["logical_node_id"]
|
||||
.as_u64()
|
||||
.filter(|node_id| *node_id != 0)
|
||||
.ok_or_else(|| format!("fleet node omitted valid identity: {node}"))?;
|
||||
if !seen.insert(node_id) {
|
||||
return Err(format!("fleet duplicated logical node {node_id}"));
|
||||
}
|
||||
if !expected_running.contains(&node_id)
|
||||
&& !self.stopped_nodes.contains(&node_id)
|
||||
{
|
||||
return Err(format!(
|
||||
"unexpected replacement or bootstrap logical node {node_id}: {fleet}"
|
||||
));
|
||||
}
|
||||
if self.stopped_nodes.contains(&node_id) {
|
||||
if node["phase"] == "stopped" {
|
||||
stopped.insert(node_id);
|
||||
} else {
|
||||
return Err(format!(
|
||||
"removed logical node {node_id} left stopped phase: {node}"
|
||||
));
|
||||
}
|
||||
} else if node["phase"] == "running" {
|
||||
running.push(node_id);
|
||||
} else if provisioning_failed(std::slice::from_ref(node)) {
|
||||
return Err(format!("worker provisioning failed: {fleet}"));
|
||||
}
|
||||
}
|
||||
running.sort_unstable();
|
||||
if running == expected && stopped == self.stopped_nodes {
|
||||
self.node_ids = running;
|
||||
return Ok(());
|
||||
}
|
||||
self.wait_for_control_change(
|
||||
&cursor,
|
||||
&budget,
|
||||
&format!("running nodes {expected:?}; observed={fleet}"),
|
||||
)?;
|
||||
}
|
||||
})();
|
||||
self.record_lifecycle("fleet_convergence", started, &result)?;
|
||||
result
|
||||
}
|
||||
pub fn verify_workload_convergence(&mut self) -> Result<(), String> {
|
||||
self.wait_contextual_control()?;
|
||||
if self.resource_baseline.is_empty() {
|
||||
let baseline = self.health_snapshot()?;
|
||||
self.resource_baseline = transient_actor_identities(&baseline);
|
||||
if self.provider_baseline.is_empty() {
|
||||
self.record_health_baseline()?;
|
||||
}
|
||||
self.verify_ordered_pair_communication()
|
||||
self.verify_ordered_pair_communication()?;
|
||||
// The reusable fixture begins only after the readiness workload and
|
||||
// every owned readiness resource have been cleaned and re-observed.
|
||||
self.record_health_baseline()
|
||||
}
|
||||
|
||||
fn wait_contextual_control(&mut self) -> Result<(), String> {
|
||||
for node in self.node_ids.clone() {
|
||||
let deadline = Instant::now() + self.config.deadline;
|
||||
pub(super) fn wait_contextual_control(&mut self) -> Result<(), String> {
|
||||
let started = Instant::now();
|
||||
let budget = self.operation_budget().child(self.config.deadline);
|
||||
let result = (|| {
|
||||
loop {
|
||||
budget.check("exact concurrent contextual readiness")?;
|
||||
self.ensure_orchestrator_live()?;
|
||||
let cursor = self.control_revision(&budget)?;
|
||||
match self.query_contextual_nodes(&self.node_ids, &budget) {
|
||||
Ok(_) => return Ok(()),
|
||||
Err(error) => self.wait_for_control_change(&cursor, &budget, &error)?,
|
||||
}
|
||||
}
|
||||
})();
|
||||
self.record_lifecycle("contextual_readiness", started, &result)?;
|
||||
result
|
||||
}
|
||||
|
||||
pub(super) fn query_contextual_nodes(
|
||||
&self,
|
||||
nodes: &[u64],
|
||||
budget: &Budget,
|
||||
) -> Result<Vec<myelin_control_contract::ContextualHealthReply>, String> {
|
||||
let expected = nodes.iter().copied().collect::<BTreeSet<_>>();
|
||||
if nodes.is_empty() || expected.len() != nodes.len() || expected.contains(&0) {
|
||||
return Err(format!(
|
||||
"contextual readiness requires exact, unique live logical nodes; observed {nodes:?}"
|
||||
));
|
||||
}
|
||||
let generation = self.next_observation_generation();
|
||||
thread::scope(|scope| {
|
||||
let requests = nodes
|
||||
.iter()
|
||||
.map(|&node| {
|
||||
let base_url = &self.base_url;
|
||||
scope.spawn(move || {
|
||||
let mut attempt = 0_u64;
|
||||
loop {
|
||||
self.ensure_orchestrator_live()?;
|
||||
attempt = attempt.saturating_add(1);
|
||||
let control_request_id =
|
||||
format!("convergence-{}-{node}-{attempt}", self.config.seed);
|
||||
let response = http_json(
|
||||
budget.check(&format!("contextual health for node {node}"))?;
|
||||
let request_id =
|
||||
format!("health-{}-{generation}-{node}-{attempt}", self.config.seed);
|
||||
let request_budget = budget.child(Duration::from_secs(5));
|
||||
match http_json_budget(
|
||||
"GET",
|
||||
&format!(
|
||||
"{}/api/control/contextual/nodes/{node}?control_request_id={control_request_id}",
|
||||
self.base_url
|
||||
"{base_url}/api/control/contextual/nodes/{node}?control_request_id={request_id}"
|
||||
),
|
||||
None,
|
||||
);
|
||||
if response.as_ref().is_ok_and(|reply| {
|
||||
reply
|
||||
.pointer("/observation/event/type")
|
||||
.and_then(Value::as_str)
|
||||
== Some("live_executions")
|
||||
}) {
|
||||
break;
|
||||
&request_budget,
|
||||
) {
|
||||
Ok(response) => {
|
||||
return validate_contextual_reply(&response, node, &request_id);
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(self.timeout_evidence(&format!(
|
||||
"contextual process control reachable on node {node}; last={response:?}"
|
||||
)));
|
||||
}
|
||||
thread::sleep(POLL_INTERVAL);
|
||||
Err(error) => {
|
||||
attempt = attempt.saturating_add(1);
|
||||
budget.wait(
|
||||
POLL_INTERVAL,
|
||||
&format!(
|
||||
"contextual health for node {node}; retry after {error}"
|
||||
),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let mut responses = Vec::with_capacity(requests.len());
|
||||
let mut errors = Vec::new();
|
||||
for request in requests {
|
||||
match request.join() {
|
||||
Ok(Ok(response)) => responses.push(response),
|
||||
Ok(Err(error)) => errors.push(error),
|
||||
Err(_) => errors.push("contextual readiness collector panicked".to_owned()),
|
||||
}
|
||||
}
|
||||
if errors.is_empty() {
|
||||
Ok(responses)
|
||||
} else {
|
||||
Err(errors.join("; "))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Prove functional communication across every ordered node pair.
|
||||
|
|
@ -153,29 +294,80 @@ impl ClusterHarness {
|
|||
.duration_since(UNIX_EPOCH)
|
||||
.map_err(|error| format!("system clock precedes epoch: {error}"))?
|
||||
.as_millis();
|
||||
let nodes = self.node_ids.clone();
|
||||
let pair_path = |source: u64, destination: u64| {
|
||||
format!("/cases/convergence/{run_nonce}/{source}-to-{destination}")
|
||||
};
|
||||
let mut payloads = BTreeMap::new();
|
||||
let mut programs = Vec::new();
|
||||
for &source in &nodes {
|
||||
let mut actions = Vec::new();
|
||||
for &destination in &nodes {
|
||||
if source == destination {
|
||||
continue;
|
||||
let case = Self::convergence_case(&self.node_ids, run_nonce)?;
|
||||
self.run_case(&case).map(|_| ())
|
||||
}
|
||||
let path = pair_path(source, destination);
|
||||
let payload = format!("myelin-e2e-convergence:{run_nonce}:{source}:{destination}")
|
||||
.into_bytes();
|
||||
|
||||
pub(crate) fn convergence_case(nodes: &[u64], run_nonce: u128) -> Result<BehaviorCase, String> {
|
||||
if nodes.is_empty() {
|
||||
return Err("convergence requires live nodes".to_owned());
|
||||
}
|
||||
let blob_path = |source: u64, destination: u64| {
|
||||
format!("/cases/convergence/{run_nonce}/blob-{source}-to-{destination}")
|
||||
};
|
||||
let stream_path = |source: u64, destination: u64| {
|
||||
format!("/cases/convergence/{run_nonce}/stream-{source}-to-{destination}")
|
||||
};
|
||||
let ready_path = |process: &str| format!("/cases/convergence/{run_nonce}/ready/{process}");
|
||||
let release_path = format!("/cases/convergence/{run_nonce}/release");
|
||||
let boundary_lengths = [0_usize, 1, 4_095, 4_096, 4_097, 65_537];
|
||||
let mut blob_payloads = BTreeMap::new();
|
||||
let mut stream_payloads = BTreeMap::new();
|
||||
let mut programs = Vec::new();
|
||||
for (source_index, &source) in nodes.iter().enumerate() {
|
||||
let process_id = format!("convergence-writer-{source}");
|
||||
let mut actions = vec![
|
||||
Action::ok(ActionOp::PublishBlob {
|
||||
path: ready_path(&process_id),
|
||||
bytes: Vec::new(),
|
||||
}),
|
||||
Action::ok(ActionOp::AwaitEntry {
|
||||
path: release_path.clone(),
|
||||
expected_kind: "blob".to_owned(),
|
||||
}),
|
||||
];
|
||||
for round in 1..nodes.len() {
|
||||
let destination = nodes[(source_index + round) % nodes.len()];
|
||||
let pair_index = source_index * (nodes.len() - 1) + round - 1;
|
||||
let length = boundary_lengths[pair_index % boundary_lengths.len()];
|
||||
let body = (0..length)
|
||||
.map(|offset| {
|
||||
(u128::from(source) * 17 + u128::from(destination) * 31 + offset as u128)
|
||||
as u8
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let blob = blob_path(source, destination);
|
||||
actions.push(Action::ok(ActionOp::PublishBlob {
|
||||
path: path.clone(),
|
||||
bytes: payload.clone(),
|
||||
path: blob.clone(),
|
||||
bytes: body.clone(),
|
||||
}));
|
||||
payloads.insert(path, payload);
|
||||
blob_payloads.insert(blob, body.clone());
|
||||
|
||||
let mut framed = Vec::with_capacity(body.len() + 8);
|
||||
let split = body.len() / 2;
|
||||
for frame in [&body[..split], &body[split..]] {
|
||||
framed.extend_from_slice(&(frame.len() as u32).to_be_bytes());
|
||||
framed.extend_from_slice(frame);
|
||||
}
|
||||
let stream = stream_path(source, destination);
|
||||
let mut cuts = vec![0, 1, 3, 257, framed.len().saturating_sub(1), framed.len()];
|
||||
cuts.retain(|cut| *cut <= framed.len());
|
||||
cuts.sort_unstable();
|
||||
cuts.dedup();
|
||||
let chunks = cuts
|
||||
.windows(2)
|
||||
.filter(|range| range[0] != range[1])
|
||||
.map(|range| framed[range[0]..range[1]].to_vec())
|
||||
.collect();
|
||||
actions.push(Action::ok(ActionOp::StreamWrite {
|
||||
path: stream.clone(),
|
||||
chunks,
|
||||
replace: false,
|
||||
}));
|
||||
stream_payloads.insert(stream, framed);
|
||||
}
|
||||
programs.push(ProcessProgram {
|
||||
id: format!("convergence-writer-{source}"),
|
||||
id: process_id,
|
||||
logical_node_id: source,
|
||||
access: AccessSpec::unrestricted(format!(
|
||||
"convergence-writer-{run_nonce}-{source}"
|
||||
|
|
@ -184,21 +376,43 @@ impl ClusterHarness {
|
|||
actions,
|
||||
});
|
||||
}
|
||||
for &destination in &nodes {
|
||||
let mut actions = Vec::new();
|
||||
for &source in &nodes {
|
||||
if source == destination {
|
||||
continue;
|
||||
}
|
||||
let path = pair_path(source, destination);
|
||||
actions.push(Action::ok(ActionOp::ReadBlob {
|
||||
path: path.clone(),
|
||||
expected: payloads[&path].clone(),
|
||||
for (destination_index, &destination) in nodes.iter().enumerate() {
|
||||
let process_id = format!("convergence-reader-{destination}");
|
||||
let mut actions = vec![
|
||||
Action::ok(ActionOp::PublishBlob {
|
||||
path: ready_path(&process_id),
|
||||
bytes: Vec::new(),
|
||||
}),
|
||||
Action::ok(ActionOp::AwaitEntry {
|
||||
path: release_path.clone(),
|
||||
expected_kind: "blob".to_owned(),
|
||||
}),
|
||||
];
|
||||
for round in 1..nodes.len() {
|
||||
let source = nodes[(destination_index + nodes.len() - round) % nodes.len()];
|
||||
let blob = blob_path(source, destination);
|
||||
actions.push(Action::ok(ActionOp::AwaitEntry {
|
||||
path: blob.clone(),
|
||||
expected_kind: "blob".to_owned(),
|
||||
}));
|
||||
actions.push(Action::ok(ActionOp::Unlink { path }));
|
||||
actions.push(Action::ok(ActionOp::ReadBlob {
|
||||
path: blob.clone(),
|
||||
expected: blob_payloads[&blob].clone(),
|
||||
}));
|
||||
actions.push(Action::ok(ActionOp::Unlink { path: blob }));
|
||||
|
||||
let stream = stream_path(source, destination);
|
||||
actions.push(Action::ok(ActionOp::StreamRead {
|
||||
path: stream.clone(),
|
||||
expected: stream_payloads[&stream].clone(),
|
||||
}));
|
||||
actions.push(Action::ok(ActionOp::WaitForQuiescent {
|
||||
path: stream.clone(),
|
||||
}));
|
||||
actions.push(Action::ok(ActionOp::Unlink { path: stream }));
|
||||
}
|
||||
programs.push(ProcessProgram {
|
||||
id: format!("convergence-reader-{destination}"),
|
||||
id: process_id,
|
||||
logical_node_id: destination,
|
||||
access: AccessSpec::unrestricted(format!(
|
||||
"convergence-reader-{run_nonce}-{destination}"
|
||||
|
|
@ -207,80 +421,103 @@ impl ClusterHarness {
|
|||
actions,
|
||||
});
|
||||
}
|
||||
for program in &programs {
|
||||
let request_id = format!("convergence-{run_nonce}-{}", program.id);
|
||||
self.spawn_program(program, &request_id, &render_python(program), false, None)?;
|
||||
let observation = self.wait_execution(program, &request_id, None)?;
|
||||
verify_convergence_execution(program, &observation)
|
||||
.map_err(|error| format!("ordered-pair convergence proof: {error}"))?;
|
||||
}
|
||||
// Convergence processes must be fully reclaimed before the resource
|
||||
// baseline is captured, otherwise their transient actors would be
|
||||
// grandfathered into every later health assertion.
|
||||
self.assert_healthy()?;
|
||||
Ok(())
|
||||
let participant_ready_paths = programs
|
||||
.iter()
|
||||
.map(|program| ready_path(&program.id))
|
||||
.collect::<Vec<_>>();
|
||||
programs.push(ProcessProgram {
|
||||
id: "zz-convergence-release".to_owned(),
|
||||
logical_node_id: nodes[0],
|
||||
access: AccessSpec::unrestricted(format!("convergence-release-{run_nonce}")),
|
||||
depends_on: Vec::new(),
|
||||
actions: participant_ready_paths
|
||||
.into_iter()
|
||||
.map(|path| {
|
||||
Action::ok(ActionOp::AwaitEntry {
|
||||
path,
|
||||
expected_kind: "blob".to_owned(),
|
||||
})
|
||||
})
|
||||
.chain(std::iter::once(Action::ok(ActionOp::PublishBlob {
|
||||
path: release_path,
|
||||
bytes: Vec::new(),
|
||||
})))
|
||||
.collect(),
|
||||
});
|
||||
let case = BehaviorCase {
|
||||
schema_version: CASE_SCHEMA_VERSION,
|
||||
generator_version: GENERATOR_VERSION,
|
||||
id: format!("convergence-{run_nonce}"),
|
||||
seed: u64::try_from(run_nonce).unwrap_or(u64::MAX),
|
||||
live_nodes: nodes.iter().copied().collect(),
|
||||
topology: TopologyFamily::Fixed,
|
||||
scenarios: Default::default(),
|
||||
routes: Vec::new(),
|
||||
read_only_fixture_paths: Default::default(),
|
||||
resource_bounds: CaseResourceBounds {
|
||||
max_actions: 256,
|
||||
max_processes: 20,
|
||||
max_payload_bytes: 4 * 1024 * 1024,
|
||||
max_allocation_bytes: 64 * 1024 * 1024,
|
||||
max_race_states: 65_536,
|
||||
},
|
||||
processes: programs,
|
||||
failure: FailureInjection::None,
|
||||
};
|
||||
case.validate()?;
|
||||
Ok(case)
|
||||
}
|
||||
}
|
||||
|
||||
fn verify_convergence_execution(
|
||||
program: &ProcessProgram,
|
||||
observation: &ExecutionObservation,
|
||||
) -> Result<(), String> {
|
||||
BehaviorOracle::verify_execution(observation).map_err(|error| error.to_string())?;
|
||||
if !observation.exit_success {
|
||||
return Err(format!(
|
||||
"process {} on node {} failed with status {:?}\nstdout={}\nstderr={}",
|
||||
program.id,
|
||||
program.logical_node_id,
|
||||
observation.exit_status,
|
||||
observation.stdout,
|
||||
observation.stderr
|
||||
));
|
||||
fn validate_contextual_reply(
|
||||
reply: &Value,
|
||||
node: u64,
|
||||
request_id: &str,
|
||||
) -> Result<myelin_control_contract::ContextualHealthReply, String> {
|
||||
let reply: myelin_control_contract::ContextualHealthReply =
|
||||
serde_json::from_value(reply.clone())
|
||||
.map_err(|error| format!("invalid contextual health reply for node {node}: {error}"))?;
|
||||
reply.validate(node, request_id)?;
|
||||
Ok(reply)
|
||||
}
|
||||
if observation.results.len() != program.actions.len() {
|
||||
return Err(format!(
|
||||
"process {} expected {} action results, observed {}",
|
||||
program.id,
|
||||
program.actions.len(),
|
||||
observation.results.len()
|
||||
));
|
||||
|
||||
fn provisioning_failed(nodes: &[Value]) -> bool {
|
||||
nodes.iter().any(|node| {
|
||||
matches!(
|
||||
node.get("phase").and_then(Value::as_str),
|
||||
Some("kill_requested" | "stopping" | "stop_failed" | "stopped" | "orphan")
|
||||
)
|
||||
})
|
||||
}
|
||||
for (step, action) in program.actions.iter().enumerate() {
|
||||
let result = observation
|
||||
.results
|
||||
.iter()
|
||||
.find(|result| result.step == step)
|
||||
.ok_or_else(|| format!("process {} omitted step {step}", program.id))?;
|
||||
if result.outcome != "ok" {
|
||||
return Err(format!(
|
||||
"process {} step {step} on node {} reported outcome {:?} (errno {:?}, error {:?})",
|
||||
program.id, program.logical_node_id, result.outcome, result.errno, result.error
|
||||
));
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn readiness_rejects_stale_or_wrong_node_replies() {
|
||||
let reply = crate::resources::resource_deadline_tests::health_reply("fresh-3", 3, 1, 0);
|
||||
assert!(validate_contextual_reply(&reply, 3, "fresh-3").is_ok());
|
||||
assert!(validate_contextual_reply(&reply, 4, "fresh-3").is_err());
|
||||
assert!(validate_contextual_reply(&reply, 3, "new-3").is_err());
|
||||
assert!(validate_contextual_reply(&json!({}), 3, "fresh-3").is_err());
|
||||
let mut unversioned = reply.clone();
|
||||
unversioned
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.remove("schema_version");
|
||||
assert!(validate_contextual_reply(&unversioned, 3, "fresh-3").is_err());
|
||||
let mut rejected = reply.clone();
|
||||
rejected["type"] = json!("rejected");
|
||||
assert!(validate_contextual_reply(&rejected, 3, "fresh-3").is_err());
|
||||
}
|
||||
if result.path != action.operation.path() {
|
||||
return Err(format!(
|
||||
"process {} step {step} reported path {:?} instead of {:?}",
|
||||
program.id,
|
||||
result.path,
|
||||
action.operation.path()
|
||||
));
|
||||
}
|
||||
let expected_bytes = match &action.operation {
|
||||
ActionOp::PublishBlob { bytes, .. }
|
||||
| ActionOp::ReadBlob {
|
||||
expected: bytes, ..
|
||||
} => Some(bytes.as_slice()),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(expected) = expected_bytes
|
||||
&& (result.length != Some(expected.len())
|
||||
|| result.digest.as_deref() != Some(&digest(expected)))
|
||||
{
|
||||
return Err(format!(
|
||||
"process {} step {step} returned the wrong payload length or digest",
|
||||
program.id
|
||||
));
|
||||
|
||||
#[test]
|
||||
fn terminal_node_fails_a_partially_converged_fleet() {
|
||||
let nodes = vec![
|
||||
json!({"logical_node_id": 1, "phase": "joining"}),
|
||||
json!({"logical_node_id": 2, "phase": "stopped", "last_error": "substrate lost"}),
|
||||
];
|
||||
assert!(provisioning_failed(&nodes));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
2402
tools/myelin-e2e-fuzz/src/harness/raw_fleet.rs
Normal file
2402
tools/myelin-e2e-fuzz/src/harness/raw_fleet.rs
Normal file
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
|
|
@ -3,64 +3,77 @@
|
|||
//! The crate is organized as a module tree; this root is a thin facade that
|
||||
//! preserves the historical flat API used by `main.rs`.
|
||||
|
||||
mod budget;
|
||||
mod campaign;
|
||||
mod codegen;
|
||||
mod corpus;
|
||||
mod coverage;
|
||||
mod harness;
|
||||
mod ir;
|
||||
mod oracle;
|
||||
mod remote;
|
||||
mod resources;
|
||||
mod shrink;
|
||||
|
||||
pub use budget::{Budget, execution_evidence, record_execution_stage};
|
||||
pub use campaign::{
|
||||
ArtifactEnvelope, CAMPAIGN_DEADLINE_SECS, CLEANUP_DEADLINE_SECS, CampaignConfig,
|
||||
CampaignLimits, CampaignPlan, DIAGNOSTIC_DEADLINE_SECS, LOCAL_CLEANUP_RESERVE_SECS,
|
||||
OfferPolicy, PREPARATION_DEADLINE_SECS, PersistedFixtureEntry, ProviderMode, RecoveryCase,
|
||||
WORKLOAD_DEADLINE_SECS,
|
||||
};
|
||||
pub use codegen::{render_case_python, render_python};
|
||||
pub use corpus::{failure_corpus, ordered_pair_corpus, random_short_dags, stable_corpus};
|
||||
pub use harness::{ClusterHarness, ClusterHarnessConfig};
|
||||
pub use corpus::{
|
||||
failure_corpus, ordered_pair_corpus, random_short_dags, random_short_dags_budgeted,
|
||||
stable_corpus,
|
||||
};
|
||||
pub use coverage::{
|
||||
CoverageKey, CoverageLedger, EvidenceIdentity, NodeRole, PayloadClass, case_coverage,
|
||||
};
|
||||
pub use harness::raw_fleet::{
|
||||
DeploymentBoundary, RawDockerFleet, RawFleetSnapshot, RawNodeCensus, RawPriorState,
|
||||
};
|
||||
pub use harness::{
|
||||
ClusterHarness, ClusterHarnessConfig, FixturePathObservation, FixturePathSnapshot,
|
||||
HarnessProvider,
|
||||
};
|
||||
pub use ir::{
|
||||
AccessSpec, Action, ActionClass, ActionObservation, ActionOp, BehaviorCase, CaseObservation,
|
||||
DescriptorFinish, DescriptorReadMethod, DescriptorWriteMethod, ExecutionObservation,
|
||||
ExpectedOutcome, FailureInjection, LaunchFailureKind, ProcessProgram, ProcessStopPhase,
|
||||
PythonException,
|
||||
CaseResourceBounds, CaseResources, CoverageScenario, DataEdge, DataKind, DataRoute,
|
||||
DescriptorFinish, DescriptorObservation, DescriptorReadMethod, DescriptorTerminalResult,
|
||||
DescriptorWriteMethod, ExecutionObservation, ExpectedOutcome, FailureInjection,
|
||||
LaunchFailureKind, ProcessProgram, ProcessStopPhase, PythonException, TopologyFamily,
|
||||
TransferObservation,
|
||||
};
|
||||
pub use oracle::{BehaviorOracle, BlobPublicationTrace, OracleViolation, StreamIncarnationTrace};
|
||||
pub use shrink::shrink_failure;
|
||||
pub use oracle::{
|
||||
BehaviorOracle, BlobPublicationTrace, CausalRole, FailureClass, FailureSignature,
|
||||
ObservedOutcome, OracleViolation, OutcomeExpectation, SemanticAction, StreamIncarnationTrace,
|
||||
};
|
||||
pub use remote::{
|
||||
PaidCampaignPhase, PaidCampaignState, authorized_cleanup_limits, await_cleanup_owner,
|
||||
cleanup_paid_ownership, conservative_selected_cost, decode_offer_results, offer_search_request,
|
||||
read_campaign_plan, read_coverage_ledger, read_paid_state, recover_cleanup_owner,
|
||||
run_cleanup_owner, run_cleanup_supervisor, run_scripted_retained_lifecycle,
|
||||
scan_artifacts_for_secret, select_exact_offers, selected_cleanup_limits, start_cleanup_owner,
|
||||
write_campaign_plan, write_coverage_ledger, write_paid_state,
|
||||
};
|
||||
pub use resources::{
|
||||
BuiltBinaries, DeploymentBundle, GateImageIdentity, PreparedArtifactIdentity,
|
||||
assemble_deployment_bundle, distinguish_deployment_payload, immutable_registry_reference,
|
||||
private_fixture_dir, require_prepared_artifacts, resolve_myelin_binaries,
|
||||
runtime_image_identity, stage_deployment_payload,
|
||||
};
|
||||
pub use shrink::{shrink_failure, try_shrink_failure, try_shrink_failure_budgeted};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use serde_json::{Value, json};
|
||||
use std::process::{Command, Stdio};
|
||||
|
||||
use crate::codegen::digest;
|
||||
use crate::corpus::SEEDED_MODEL_PATH;
|
||||
use crate::resources::{TelemetryResourceCensus, pending_resource_cleanup};
|
||||
|
||||
#[test]
|
||||
fn generated_barriers_poll_by_deadline_instead_of_attempt_counts() {
|
||||
let program = ProcessProgram {
|
||||
id: "barriers".to_owned(),
|
||||
logical_node_id: 1,
|
||||
access: AccessSpec::unrestricted("barriers"),
|
||||
depends_on: Vec::new(),
|
||||
actions: vec![
|
||||
Action::ok(ActionOp::AwaitEntry {
|
||||
path: "/cases/1/entry".to_owned(),
|
||||
expected_kind: "blob".to_owned(),
|
||||
}),
|
||||
Action::ok(ActionOp::WaitForQuiescent {
|
||||
path: "/cases/1/stream".to_owned(),
|
||||
}),
|
||||
],
|
||||
};
|
||||
let source = render_python(&program);
|
||||
assert!(source.contains("deadline = loop.time() + "));
|
||||
assert!(source.contains("if loop.time() >= deadline:"));
|
||||
assert!(!source.contains("range("));
|
||||
|
||||
let cleanup = crate::codegen::render_cleanup_python(&["/cases/1/entry".to_owned()]);
|
||||
assert!(cleanup.contains("deadline = loop.time() + "));
|
||||
assert!(cleanup.contains("if loop.time() >= deadline:"));
|
||||
assert!(!cleanup.contains("range("));
|
||||
}
|
||||
fn valid_execution() -> ExecutionObservation {
|
||||
ExecutionObservation {
|
||||
process: "p".to_owned(),
|
||||
|
|
@ -81,12 +94,18 @@ mod tests {
|
|||
outcome: "ok".to_owned(),
|
||||
length: None,
|
||||
digest: None,
|
||||
transfer: None,
|
||||
kind: Some("blob".to_owned()),
|
||||
revision: Some(1),
|
||||
errno: None,
|
||||
active: None,
|
||||
error_type: None,
|
||||
error: None,
|
||||
descriptor: None,
|
||||
barrier: None,
|
||||
incarnation: None,
|
||||
token: None,
|
||||
lap: None,
|
||||
}],
|
||||
terminal: true,
|
||||
exit_success: true,
|
||||
|
|
@ -97,21 +116,110 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn generated_program_uses_only_public_binding_operations() {
|
||||
let program = ProcessProgram {
|
||||
id: "generated".to_owned(),
|
||||
fn complete_campaign_python_is_syntactically_valid() {
|
||||
let cases = crate::corpus::generated_campaign_cases(17, 128, 5).unwrap();
|
||||
let sources = cases
|
||||
.iter()
|
||||
.flat_map(|case| {
|
||||
case.processes.iter().map(|process| {
|
||||
render_case_python(case, process, std::time::Duration::from_secs(120))
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let mut child = Command::new("python3")
|
||||
.args([
|
||||
"-c",
|
||||
"import json,sys\nfor i,source in enumerate(json.load(sys.stdin)):\n compile(source, f'<generated-{i}>', 'exec')",
|
||||
])
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.expect("spawn Python syntax checker");
|
||||
serde_json::to_writer(
|
||||
child.stdin.as_mut().expect("Python syntax checker stdin"),
|
||||
&sources,
|
||||
)
|
||||
.unwrap();
|
||||
drop(child.stdin.take());
|
||||
let output = child.wait_with_output().unwrap();
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"generated Python syntax failed: {}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attempts_receive_fresh_process_and_namespace_ownership() {
|
||||
let case = stable_corpus(2, 19)
|
||||
.into_iter()
|
||||
.find(|case| {
|
||||
case.processes
|
||||
.iter()
|
||||
.flat_map(|process| &process.actions)
|
||||
.any(|action| action.operation.path().starts_with("/cases/"))
|
||||
})
|
||||
.unwrap();
|
||||
let first = case.for_attempt(1, true);
|
||||
let second = case.for_attempt(2, true);
|
||||
assert_eq!(first.id, case.id);
|
||||
assert_ne!(
|
||||
first.processes[0].access.execution_id,
|
||||
second.processes[0].access.execution_id
|
||||
);
|
||||
let first_paths = first
|
||||
.processes
|
||||
.iter()
|
||||
.flat_map(|process| &process.actions)
|
||||
.map(|action| action.operation.path())
|
||||
.filter(|path| path.starts_with("/cases/"))
|
||||
.collect::<Vec<_>>();
|
||||
let second_paths = second
|
||||
.processes
|
||||
.iter()
|
||||
.flat_map(|process| &process.actions)
|
||||
.map(|action| action.operation.path())
|
||||
.filter(|path| path.starts_with("/cases/"))
|
||||
.collect::<Vec<_>>();
|
||||
assert!(first_paths.iter().all(|path| path.contains("/attempt-1/")));
|
||||
assert!(second_paths.iter().all(|path| path.contains("/attempt-2/")));
|
||||
assert_ne!(first_paths, second_paths);
|
||||
}
|
||||
#[test]
|
||||
fn read_only_fixture_paths_are_neither_scoped_nor_cleaned() {
|
||||
let path = "/cases/recovery/persisted".to_owned();
|
||||
let case = BehaviorCase {
|
||||
schema_version: crate::ir::CASE_SCHEMA_VERSION,
|
||||
generator_version: crate::ir::GENERATOR_VERSION,
|
||||
id: "read-only-fixture".to_owned(),
|
||||
seed: 1,
|
||||
live_nodes: BTreeSet::from([1, 2]),
|
||||
topology: TopologyFamily::Fixed,
|
||||
scenarios: BTreeSet::new(),
|
||||
routes: Vec::new(),
|
||||
read_only_fixture_paths: BTreeSet::from([path.clone()]),
|
||||
resource_bounds: CaseResourceBounds::default(),
|
||||
processes: vec![ProcessProgram {
|
||||
id: "reader".to_owned(),
|
||||
logical_node_id: 1,
|
||||
access: AccessSpec::unrestricted("generated"),
|
||||
access: AccessSpec::unrestricted("reader"),
|
||||
depends_on: Vec::new(),
|
||||
actions: vec![Action::ok(ActionOp::Lookup {
|
||||
path: SEEDED_MODEL_PATH.to_owned(),
|
||||
path: path.clone(),
|
||||
expected_kind: "blob".to_owned(),
|
||||
})],
|
||||
}],
|
||||
failure: FailureInjection::None,
|
||||
};
|
||||
let source = render_python(&program);
|
||||
assert!(source.contains("swactor.run(main)"));
|
||||
assert!(source.contains("await data.lookup"));
|
||||
assert!(!source.contains("data_plane::"));
|
||||
case.validate().unwrap();
|
||||
let scoped = case.for_attempt(7, true);
|
||||
assert_eq!(scoped.processes[0].actions[0].operation.path(), path);
|
||||
assert!(scoped.owned_paths().is_empty());
|
||||
|
||||
let mut invalid = case;
|
||||
invalid.processes[0].actions = vec![Action::ok(ActionOp::Unlink { path })];
|
||||
assert!(invalid.validate().unwrap_err().contains("read-only"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -192,9 +300,16 @@ mod tests {
|
|||
let mut torn = valid.clone();
|
||||
torn.results[0].kind = Some("stream".to_owned());
|
||||
let case = BehaviorCase {
|
||||
schema_version: crate::ir::CASE_SCHEMA_VERSION,
|
||||
generator_version: crate::ir::GENERATOR_VERSION,
|
||||
id: "case".to_owned(),
|
||||
seed: 1,
|
||||
node_count: 2,
|
||||
live_nodes: crate::ir::contiguous_nodes(2),
|
||||
topology: Default::default(),
|
||||
scenarios: Default::default(),
|
||||
routes: Vec::new(),
|
||||
read_only_fixture_paths: Default::default(),
|
||||
resource_bounds: Default::default(),
|
||||
processes: vec![ProcessProgram {
|
||||
id: "p".to_owned(),
|
||||
logical_node_id: 1,
|
||||
|
|
@ -207,6 +322,7 @@ mod tests {
|
|||
}],
|
||||
failure: FailureInjection::None,
|
||||
};
|
||||
torn.request_id = case.execution_request_id(&case.processes[0]);
|
||||
assert_eq!(
|
||||
BehaviorOracle::verify(
|
||||
&case,
|
||||
|
|
@ -220,20 +336,15 @@ mod tests {
|
|||
"namespace_kind"
|
||||
);
|
||||
|
||||
let mut duplicated = valid.clone();
|
||||
duplicated.results.push(duplicated.results[0].clone());
|
||||
assert_eq!(
|
||||
BehaviorOracle::verify(
|
||||
&case,
|
||||
&CaseObservation {
|
||||
case_id: "case".to_owned(),
|
||||
executions: vec![duplicated],
|
||||
},
|
||||
)
|
||||
.unwrap_err()
|
||||
.invariant,
|
||||
"duplicate_action"
|
||||
);
|
||||
let mut observation = CaseObservation {
|
||||
case_id: case.id.clone(),
|
||||
executions: vec![valid.clone()],
|
||||
};
|
||||
observation.executions[0].request_id = case.execution_request_id(&case.processes[0]);
|
||||
BehaviorOracle::verify(&case, &observation).unwrap();
|
||||
let duplicate = observation.executions[0].results[0].clone();
|
||||
observation.executions[0].results.push(duplicate);
|
||||
assert!(BehaviorOracle::verify(&case, &observation).is_err());
|
||||
|
||||
assert_eq!(
|
||||
BehaviorOracle::verify_blob_publication(&BlobPublicationTrace {
|
||||
|
|
@ -260,6 +371,103 @@ mod tests {
|
|||
"stale_incarnation"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn typed_action_abort_replays_and_shrinks_without_inventing_a_suffix() {
|
||||
let action = Action::ok(ActionOp::Lookup {
|
||||
path: SEEDED_MODEL_PATH.to_owned(),
|
||||
expected_kind: "blob".to_owned(),
|
||||
});
|
||||
let case = BehaviorCase {
|
||||
schema_version: crate::ir::CASE_SCHEMA_VERSION,
|
||||
generator_version: crate::ir::GENERATOR_VERSION,
|
||||
id: "typed-abort".to_owned(),
|
||||
seed: 1,
|
||||
live_nodes: crate::ir::contiguous_nodes(2),
|
||||
topology: Default::default(),
|
||||
scenarios: Default::default(),
|
||||
routes: Vec::new(),
|
||||
read_only_fixture_paths: BTreeSet::from([SEEDED_MODEL_PATH.to_owned()]),
|
||||
resource_bounds: Default::default(),
|
||||
processes: vec![ProcessProgram {
|
||||
id: "p".to_owned(),
|
||||
logical_node_id: 1,
|
||||
access: AccessSpec::unrestricted("p"),
|
||||
depends_on: Vec::new(),
|
||||
actions: vec![action.clone(), action],
|
||||
}],
|
||||
failure: FailureInjection::None,
|
||||
};
|
||||
let mut failed = valid_execution();
|
||||
failed.request_id = case.execution_request_id(&case.processes[0]);
|
||||
failed.exit_success = false;
|
||||
failed.exit_status = Some(r#"{"kind":"code","value":1}"#.to_owned());
|
||||
failed.results[0].outcome = "error".to_owned();
|
||||
failed.results[0].errno = Some(libc::ENOENT);
|
||||
failed.results[0].error_type = Some("FileNotFoundError".to_owned());
|
||||
failed.results[0].error = Some("binding lookup failed".to_owned());
|
||||
let observe = |candidate: &BehaviorCase| CaseObservation {
|
||||
case_id: candidate.id.clone(),
|
||||
executions: vec![failed.clone()],
|
||||
};
|
||||
let signature = BehaviorOracle::verify(&case, &observe(&case))
|
||||
.unwrap_err()
|
||||
.signature;
|
||||
assert_eq!(signature.invariant, "unexpected_error");
|
||||
assert_eq!(signature.failure_class, FailureClass::OutcomeMismatch);
|
||||
assert_eq!(
|
||||
signature.causal_role,
|
||||
CausalRole::Action(SemanticAction::Lookup)
|
||||
);
|
||||
let minimized = try_shrink_failure_budgeted(
|
||||
case.clone(),
|
||||
&Budget::new(std::time::Duration::from_secs(1)),
|
||||
|candidate| {
|
||||
Ok(BehaviorOracle::verify(candidate, &observe(candidate))
|
||||
.is_err_and(|error| error.signature == signature))
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(minimized.processes[0].actions.len(), 1);
|
||||
assert_eq!(
|
||||
BehaviorOracle::verify(&minimized, &observe(&minimized))
|
||||
.unwrap_err()
|
||||
.signature,
|
||||
signature,
|
||||
);
|
||||
|
||||
let mut canceled = failed.clone();
|
||||
canceled.results[0].error_type = Some("TimeoutError".to_owned());
|
||||
assert!(crate::oracle::recorded_action_failure(&case.processes[0], &canceled).is_none());
|
||||
let mut missing_prefix = failed.clone();
|
||||
missing_prefix.results[0].step = 1;
|
||||
assert!(
|
||||
crate::oracle::recorded_action_failure(&case.processes[0], &missing_prefix).is_none()
|
||||
);
|
||||
let mut signaled = failed.clone();
|
||||
signaled.exit_status = Some(r#"{"kind":"signal","value":9}"#.to_owned());
|
||||
assert!(crate::oracle::recorded_action_failure(&case.processes[0], &signaled).is_none());
|
||||
|
||||
let mut with_sibling = case.clone();
|
||||
let mut sibling = case.processes[0].clone();
|
||||
sibling.id = "missing-output".to_owned();
|
||||
sibling.access.execution_id = "missing-output".to_owned();
|
||||
with_sibling.processes.push(sibling);
|
||||
let mut observation = observe(&with_sibling);
|
||||
let mut missing = valid_execution();
|
||||
missing.process = "missing-output".to_owned();
|
||||
missing.request_id = with_sibling.execution_request_id(&with_sibling.processes[1]);
|
||||
missing.results.clear();
|
||||
observation.executions.push(missing);
|
||||
assert_eq!(
|
||||
BehaviorOracle::verify(&with_sibling, &observation)
|
||||
.unwrap_err()
|
||||
.signature
|
||||
.failure_class,
|
||||
FailureClass::MissingEvidence,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn linearization_groups_reject_impossible_success_counts() {
|
||||
let operation = || ActionOp::Lookup {
|
||||
|
|
@ -280,12 +488,23 @@ mod tests {
|
|||
second.process = "second".to_owned();
|
||||
second.results[0].process = "second".to_owned();
|
||||
let case = BehaviorCase {
|
||||
schema_version: crate::ir::CASE_SCHEMA_VERSION,
|
||||
generator_version: crate::ir::GENERATOR_VERSION,
|
||||
id: "linearized".to_owned(),
|
||||
seed: 1,
|
||||
node_count: 2,
|
||||
live_nodes: crate::ir::contiguous_nodes(2),
|
||||
topology: Default::default(),
|
||||
scenarios: Default::default(),
|
||||
routes: Vec::new(),
|
||||
read_only_fixture_paths: std::collections::BTreeSet::from([
|
||||
SEEDED_MODEL_PATH.to_owned()
|
||||
]),
|
||||
resource_bounds: Default::default(),
|
||||
processes: vec![program("first"), program("second")],
|
||||
failure: FailureInjection::None,
|
||||
};
|
||||
first.request_id = case.execution_request_id(&case.processes[0]);
|
||||
second.request_id = case.execution_request_id(&case.processes[1]);
|
||||
let error = BehaviorOracle::verify(
|
||||
&case,
|
||||
&CaseObservation {
|
||||
|
|
@ -295,12 +514,20 @@ mod tests {
|
|||
)
|
||||
.unwrap_err();
|
||||
assert_eq!(error.invariant, "linearizability");
|
||||
let mut over_bound = case;
|
||||
over_bound.resource_bounds.max_race_states = 1;
|
||||
assert!(
|
||||
over_bound
|
||||
.validate()
|
||||
.unwrap_err()
|
||||
.contains("legal race states")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failure_injections_validate_real_targets() {
|
||||
let corpus = failure_corpus(2, 11);
|
||||
assert_eq!(corpus.len(), 10);
|
||||
assert_eq!(corpus.len(), 8);
|
||||
assert!(corpus.iter().all(|case| case.validate().is_ok()));
|
||||
|
||||
let mut invalid = corpus[0].clone();
|
||||
|
|
@ -311,85 +538,208 @@ mod tests {
|
|||
};
|
||||
assert!(invalid.validate().is_err());
|
||||
|
||||
invalid.failure = FailureInjection::KillNode { logical_node_id: 3 };
|
||||
invalid.failure = FailureInjection::SlowProcess {
|
||||
process: "missing".to_owned(),
|
||||
parked_path: "/cases/slow/parked".to_owned(),
|
||||
release_path: "/cases/slow/release".to_owned(),
|
||||
};
|
||||
assert!(invalid.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn telemetry_census_tracks_transient_resources_to_zero() {
|
||||
let frame = |channel: &str, payload: Value| {
|
||||
json!({
|
||||
"channel": channel,
|
||||
"stream": "1#1",
|
||||
"payload": {
|
||||
"encoding": "utf8",
|
||||
"value": payload.to_string(),
|
||||
},
|
||||
})
|
||||
.to_string()
|
||||
fn typed_read_abort_shrinking_preserves_publication_and_causal_order() {
|
||||
let path = "/cases/shrink/required";
|
||||
let publication = Action::ok(ActionOp::PublishBlob {
|
||||
path: path.to_owned(),
|
||||
bytes: b"payload".to_vec(),
|
||||
});
|
||||
let read = Action::ok(ActionOp::ReadBlob {
|
||||
path: path.to_owned(),
|
||||
expected: b"payload".to_vec(),
|
||||
});
|
||||
let unrelated = Action::ok(ActionOp::PublishBlob {
|
||||
path: "/cases/shrink/unrelated".to_owned(),
|
||||
bytes: b"unrelated".to_vec(),
|
||||
});
|
||||
let program = |id: &str, actions, depends_on| ProcessProgram {
|
||||
id: id.to_owned(),
|
||||
logical_node_id: 1,
|
||||
access: AccessSpec::unrestricted(id),
|
||||
depends_on,
|
||||
actions,
|
||||
};
|
||||
let started = frame(
|
||||
"runtime.actors",
|
||||
json!({
|
||||
"event": "started",
|
||||
"actor": {
|
||||
"address": "process",
|
||||
"actor_type": "swactor_process::actor::ProcessActor",
|
||||
"poisoned": false,
|
||||
},
|
||||
}),
|
||||
);
|
||||
let live_arena = frame(
|
||||
"mvp.arena",
|
||||
json!({"live_bytes": 64, "active_leases": 1, "pending_leases": 0}),
|
||||
);
|
||||
let mut census = TelemetryResourceCensus::default();
|
||||
census
|
||||
.ingest(&format!("{started}\n{live_arena}\n"))
|
||||
.unwrap();
|
||||
let live = census.snapshot();
|
||||
let health = json!({
|
||||
"nodes": [{"observation": {"event": {"executions": []}}}],
|
||||
"running_nodes": [1],
|
||||
"resources": live,
|
||||
// The same ENOENT is a runtime defect after publication, but correct
|
||||
// behavior without publication. A typed signature alone cannot tell.
|
||||
let observe = |case: &BehaviorCase| CaseObservation {
|
||||
case_id: case.id.clone(),
|
||||
executions: case
|
||||
.processes
|
||||
.iter()
|
||||
.map(|process| {
|
||||
let mut execution = valid_execution();
|
||||
execution.process = process.id.clone();
|
||||
execution.request_id = case.execution_request_id(process);
|
||||
execution.logical_node_id = process.logical_node_id;
|
||||
execution.results.clear();
|
||||
for (step, action) in process.actions.iter().enumerate() {
|
||||
let mut result = valid_execution().results.remove(0);
|
||||
result.process = process.id.clone();
|
||||
result.step = step;
|
||||
result.path = action.operation.path().to_owned();
|
||||
match &action.operation {
|
||||
ActionOp::PublishBlob { bytes, .. } => {
|
||||
result.action = "publish_blob".to_owned();
|
||||
result.length = Some(bytes.len());
|
||||
result.digest = Some(digest(bytes));
|
||||
result.transfer = Some(TransferObservation {
|
||||
length: bytes.len(),
|
||||
digest: digest(bytes),
|
||||
complete: true,
|
||||
});
|
||||
assert!(pending_resource_cleanup(&health, &BTreeSet::new()).is_some());
|
||||
let lost_node_health = json!({
|
||||
"nodes": [],
|
||||
"running_nodes": [],
|
||||
"resources": health["resources"].clone(),
|
||||
});
|
||||
assert_eq!(
|
||||
pending_resource_cleanup(&lost_node_health, &BTreeSet::new()),
|
||||
None
|
||||
);
|
||||
|
||||
let stopped = frame(
|
||||
"runtime.actors",
|
||||
json!({
|
||||
"event": "stopped",
|
||||
"actor": {
|
||||
"address": "process",
|
||||
"actor_type": "swactor_process::actor::ProcessActor",
|
||||
"poisoned": false,
|
||||
},
|
||||
}),
|
||||
);
|
||||
let empty_arena = frame(
|
||||
"mvp.arena",
|
||||
json!({"live_bytes": 0, "active_leases": 0, "pending_leases": 0}),
|
||||
);
|
||||
census
|
||||
.ingest(&format!("{stopped}\n{empty_arena}\n"))
|
||||
.unwrap();
|
||||
let clean = census.snapshot();
|
||||
let health = json!({
|
||||
"nodes": [{"observation": {"event": {"executions": []}}}],
|
||||
"running_nodes": [1],
|
||||
"resources": clean,
|
||||
});
|
||||
assert_eq!(pending_resource_cleanup(&health, &BTreeSet::new()), None);
|
||||
}
|
||||
ActionOp::ReadBlob { .. } => {
|
||||
result.action = "read_blob".to_owned();
|
||||
result.outcome = "error".to_owned();
|
||||
result.errno = Some(libc::ENOENT);
|
||||
result.error_type = Some("FileNotFoundError".to_owned());
|
||||
result.error = Some("blob is absent".to_owned());
|
||||
result.kind = None;
|
||||
result.revision = None;
|
||||
execution.exit_success = false;
|
||||
execution.exit_status =
|
||||
Some(r#"{"kind":"code","value":1}"#.to_owned());
|
||||
}
|
||||
ActionOp::AwaitEntry { .. } => {
|
||||
result.action = "lookup".to_owned();
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
execution.results.push(result);
|
||||
if !execution.exit_success {
|
||||
break;
|
||||
}
|
||||
}
|
||||
execution
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
// Cover local publication, dependency-ordered publication, and a
|
||||
// concurrent publisher ordered by the reader's explicit entry wait.
|
||||
for (separate_processes, wait_for_entry) in [(false, false), (true, false), (true, true)] {
|
||||
let processes = if separate_processes {
|
||||
vec![
|
||||
program(
|
||||
"setup",
|
||||
vec![publication.clone(), unrelated.clone()],
|
||||
Vec::new(),
|
||||
),
|
||||
program(
|
||||
"reader",
|
||||
if wait_for_entry {
|
||||
vec![
|
||||
Action::ok(ActionOp::AwaitEntry {
|
||||
path: path.to_owned(),
|
||||
expected_kind: "blob".to_owned(),
|
||||
}),
|
||||
read.clone(),
|
||||
]
|
||||
} else {
|
||||
vec![read.clone()]
|
||||
},
|
||||
if wait_for_entry {
|
||||
Vec::new()
|
||||
} else {
|
||||
vec!["setup".to_owned()]
|
||||
},
|
||||
),
|
||||
program("independent", vec![unrelated.clone()], Vec::new()),
|
||||
]
|
||||
} else {
|
||||
vec![program(
|
||||
"reader",
|
||||
vec![publication.clone(), unrelated.clone(), read.clone()],
|
||||
Vec::new(),
|
||||
)]
|
||||
};
|
||||
let case = BehaviorCase {
|
||||
schema_version: crate::ir::CASE_SCHEMA_VERSION,
|
||||
generator_version: crate::ir::GENERATOR_VERSION,
|
||||
id: "shrink-read-abort".to_owned(),
|
||||
seed: 1,
|
||||
live_nodes: crate::ir::contiguous_nodes(2),
|
||||
topology: Default::default(),
|
||||
scenarios: Default::default(),
|
||||
routes: Vec::new(),
|
||||
read_only_fixture_paths: BTreeSet::new(),
|
||||
resource_bounds: Default::default(),
|
||||
processes,
|
||||
failure: FailureInjection::None,
|
||||
};
|
||||
case.validate().unwrap();
|
||||
let signature = BehaviorOracle::verify(&case, &observe(&case))
|
||||
.unwrap_err()
|
||||
.signature;
|
||||
assert_eq!(signature.invariant, "unexpected_error");
|
||||
assert_eq!(signature.failure_class, FailureClass::OutcomeMismatch);
|
||||
assert_eq!(
|
||||
signature.causal_role,
|
||||
CausalRole::Action(SemanticAction::ReadBlob)
|
||||
);
|
||||
let mut unjustified = case.clone();
|
||||
unjustified.processes = vec![program("reader", vec![read.clone()], Vec::new())];
|
||||
unjustified.validate().unwrap();
|
||||
assert_eq!(
|
||||
BehaviorOracle::verify(&unjustified, &observe(&unjustified))
|
||||
.unwrap_err()
|
||||
.signature,
|
||||
signature,
|
||||
);
|
||||
let minimized = try_shrink_failure_budgeted(
|
||||
case,
|
||||
&Budget::new(std::time::Duration::from_secs(1)),
|
||||
|candidate| {
|
||||
if !candidate
|
||||
.processes
|
||||
.iter()
|
||||
.flat_map(|process| &process.actions)
|
||||
.any(|action| matches!(action.operation, ActionOp::ReadBlob { .. }))
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
Ok(BehaviorOracle::verify(candidate, &observe(candidate))
|
||||
.is_err_and(|error| error.signature == signature))
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let actions = minimized
|
||||
.processes
|
||||
.iter()
|
||||
.flat_map(|process| &process.actions)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(actions.len(), 2 + usize::from(wait_for_entry));
|
||||
assert!(matches!(&actions[0].operation,
|
||||
ActionOp::PublishBlob { path: actual, bytes } if actual == path && bytes.is_empty()));
|
||||
assert!(matches!(&actions.last().unwrap().operation,
|
||||
ActionOp::ReadBlob { path: actual, expected } if actual == path && expected.is_empty()));
|
||||
assert_eq!(
|
||||
minimized.processes.len(),
|
||||
if separate_processes { 2 } else { 1 }
|
||||
);
|
||||
if separate_processes && !wait_for_entry {
|
||||
assert_eq!(minimized.processes[1].depends_on, ["setup"]);
|
||||
}
|
||||
if wait_for_entry {
|
||||
assert!(matches!(actions[1].operation, ActionOp::AwaitEntry { .. }));
|
||||
}
|
||||
assert_eq!(
|
||||
BehaviorOracle::verify(&minimized, &observe(&minimized))
|
||||
.unwrap_err()
|
||||
.signature,
|
||||
signature,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shrinker_removes_unrelated_work_and_reduces_payloads() {
|
||||
let case = random_short_dags(19, 1, 3).remove(0);
|
||||
|
|
@ -400,6 +750,36 @@ mod tests {
|
|||
&minimized.processes[0].actions[0].operation,
|
||||
ActionOp::PublishBlob { bytes, .. } if bytes.is_empty()
|
||||
));
|
||||
assert_eq!(minimized.node_count, 2);
|
||||
assert_eq!(minimized.node_count(), 2);
|
||||
}
|
||||
#[test]
|
||||
fn fallible_shrinker_stops_on_restoration_error() {
|
||||
let case = random_short_dags(23, 1, 3).remove(0);
|
||||
let mut attempts = 0;
|
||||
let error = try_shrink_failure(case, |_| {
|
||||
attempts += 1;
|
||||
Err::<bool, _>("fixture restoration failed")
|
||||
})
|
||||
.unwrap_err();
|
||||
assert_eq!(error, "fixture restoration failed");
|
||||
assert_eq!(attempts, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancelled_diagnosis_cannot_accept_a_partial_minimization() {
|
||||
let case = random_short_dags(23, 1, 3).remove(0);
|
||||
let budget = Budget::new(std::time::Duration::from_secs(30));
|
||||
let mut attempts = 0;
|
||||
let error = try_shrink_failure_budgeted(case, &budget, |_| {
|
||||
attempts += 1;
|
||||
budget.cancel();
|
||||
Ok(true)
|
||||
})
|
||||
.unwrap_err();
|
||||
assert!(error.contains("budget cancelled"), "{error}");
|
||||
assert_eq!(
|
||||
attempts, 1,
|
||||
"cancelled diagnosis must not launch another replay"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
1744
tools/myelin-e2e-fuzz/src/remote.rs
Normal file
1744
tools/myelin-e2e-fuzz/src/remote.rs
Normal file
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
|
|
@ -1,15 +1,209 @@
|
|||
//! Failure-case reduction for persisted regressions.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use crate::ir::{ActionOp, BehaviorCase, FailureInjection};
|
||||
use crate::budget::Budget;
|
||||
use crate::ir::{
|
||||
ActionOp, BehaviorCase, ExpectedOutcome, FailureInjection, ProcessProgram, TopologyFamily,
|
||||
};
|
||||
|
||||
struct CleanupScope {
|
||||
live_nodes: BTreeSet<u64>,
|
||||
read_only_paths: BTreeSet<String>,
|
||||
owned_paths: BTreeSet<String>,
|
||||
exact_live_nodes: bool,
|
||||
}
|
||||
|
||||
impl CleanupScope {
|
||||
fn new(case: &BehaviorCase, exact_live_nodes: bool) -> Self {
|
||||
Self {
|
||||
live_nodes: case.live_nodes.clone(),
|
||||
read_only_paths: case.read_only_fixture_paths.clone(),
|
||||
owned_paths: case.owned_paths(),
|
||||
exact_live_nodes,
|
||||
}
|
||||
}
|
||||
|
||||
fn admits(&self, candidate: &BehaviorCase) -> bool {
|
||||
candidate.read_only_fixture_paths == self.read_only_paths
|
||||
&& (if self.exact_live_nodes {
|
||||
candidate.live_nodes == self.live_nodes
|
||||
} else {
|
||||
candidate.live_nodes.is_subset(&self.live_nodes)
|
||||
})
|
||||
&& candidate.owned_paths().is_subset(&self.owned_paths)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn shrink_failure(
|
||||
mut case: BehaviorCase,
|
||||
case: BehaviorCase,
|
||||
mut still_fails: impl FnMut(&BehaviorCase) -> bool,
|
||||
) -> BehaviorCase {
|
||||
match try_shrink_failure(case, |candidate| {
|
||||
Ok::<bool, std::convert::Infallible>(still_fails(candidate))
|
||||
}) {
|
||||
Ok(case) => case,
|
||||
Err(never) => match never {},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn try_shrink_failure<E>(
|
||||
case: BehaviorCase,
|
||||
still_fails: impl FnMut(&BehaviorCase) -> Result<bool, E>,
|
||||
) -> Result<BehaviorCase, E> {
|
||||
shrink_checked(case, still_fails, || Ok(()), true)
|
||||
}
|
||||
|
||||
/// Diagnosis stays on the accepted fixture; minimizing the live topology
|
||||
/// would require a different baseline and is not an admissible candidate.
|
||||
pub fn try_shrink_failure_budgeted(
|
||||
case: BehaviorCase,
|
||||
budget: &Budget,
|
||||
still_fails: impl FnMut(&BehaviorCase) -> Result<bool, String>,
|
||||
) -> Result<BehaviorCase, String> {
|
||||
shrink_checked(
|
||||
case,
|
||||
still_fails,
|
||||
|| budget.check("shrink candidate generation and oracle"),
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
fn shrink_checked<E>(
|
||||
mut case: BehaviorCase,
|
||||
mut still_fails: impl FnMut(&BehaviorCase) -> Result<bool, E>,
|
||||
mut check: impl FnMut() -> Result<(), E>,
|
||||
reduce_live_nodes: bool,
|
||||
) -> Result<BehaviorCase, E> {
|
||||
let cleanup_scope = CleanupScope::new(&case, !reduce_live_nodes);
|
||||
check()?;
|
||||
if !matches!(case.failure, FailureInjection::None) {
|
||||
let mut candidate = case.clone();
|
||||
candidate.failure = FailureInjection::None;
|
||||
if cleanup_scope.admits(&candidate)
|
||||
&& candidate.validate().is_ok()
|
||||
&& still_fails(&candidate)?
|
||||
{
|
||||
case = candidate;
|
||||
}
|
||||
}
|
||||
for route_index in (0..case.routes.len()).rev() {
|
||||
check()?;
|
||||
let mut candidate = case.clone();
|
||||
let paths = candidate
|
||||
.routes
|
||||
.remove(route_index)
|
||||
.edges
|
||||
.into_iter()
|
||||
.map(|edge| edge.path)
|
||||
.collect::<BTreeSet<_>>();
|
||||
if candidate
|
||||
.processes
|
||||
.iter()
|
||||
.flat_map(|process| &process.actions)
|
||||
.any(|action| {
|
||||
paths.contains(action.operation.path())
|
||||
&& matches!(
|
||||
action.operation,
|
||||
ActionOp::GatedStreamRead { .. } | ActionOp::GatedStreamWrite { .. }
|
||||
)
|
||||
})
|
||||
{
|
||||
continue;
|
||||
}
|
||||
for process in &mut candidate.processes {
|
||||
process
|
||||
.actions
|
||||
.retain(|action| !paths.contains(action.operation.path()));
|
||||
}
|
||||
candidate
|
||||
.processes
|
||||
.retain(|process| !process.actions.is_empty());
|
||||
let remaining = candidate
|
||||
.processes
|
||||
.iter()
|
||||
.map(|process| process.id.clone())
|
||||
.collect::<BTreeSet<_>>();
|
||||
for process in &mut candidate.processes {
|
||||
process
|
||||
.depends_on
|
||||
.retain(|dependency| remaining.contains(dependency));
|
||||
}
|
||||
if candidate.routes.is_empty() {
|
||||
candidate.topology = TopologyFamily::Fixed;
|
||||
candidate
|
||||
.scenarios
|
||||
.remove(&crate::ir::CoverageScenario::ConcurrentStartup);
|
||||
candidate
|
||||
.scenarios
|
||||
.remove(&crate::ir::CoverageScenario::RingCompletion);
|
||||
}
|
||||
if structural_candidate_is_admissible(&cleanup_scope, &case, &candidate)
|
||||
&& still_fails(&candidate)?
|
||||
{
|
||||
case = candidate;
|
||||
}
|
||||
}
|
||||
// Same-path mutations can be mutual prerequisites across processes.
|
||||
// Try removing their namespace work together before individual deletions,
|
||||
// rather than weakening the guard or getting stuck on an unrelated cycle.
|
||||
// This adds at most one candidate per distinct, execution-resolved path.
|
||||
let paths = case
|
||||
.processes
|
||||
.iter()
|
||||
.flat_map(|process| {
|
||||
process.actions.iter().flat_map(move |action| {
|
||||
action
|
||||
.operation
|
||||
.paths()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(move |path| prerequisite_path(process, path).into_owned())
|
||||
})
|
||||
})
|
||||
.collect::<BTreeSet<_>>();
|
||||
for path in paths {
|
||||
check()?;
|
||||
let mut candidate = case.clone();
|
||||
let mut removed = false;
|
||||
for (process_index, process) in candidate.processes.iter_mut().enumerate() {
|
||||
let original = &case.processes[process_index];
|
||||
process.actions.retain(|action| {
|
||||
let keep = !action
|
||||
.operation
|
||||
.paths()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.any(|other| prerequisite_path(original, other).as_ref() == path.as_str());
|
||||
removed |= !keep;
|
||||
keep
|
||||
});
|
||||
}
|
||||
if !removed {
|
||||
continue;
|
||||
}
|
||||
candidate
|
||||
.processes
|
||||
.retain(|process| !process.actions.is_empty());
|
||||
let remaining = candidate
|
||||
.processes
|
||||
.iter()
|
||||
.map(|process| process.id.clone())
|
||||
.collect::<BTreeSet<_>>();
|
||||
for process in &mut candidate.processes {
|
||||
process
|
||||
.depends_on
|
||||
.retain(|dependency| remaining.contains(dependency));
|
||||
}
|
||||
if structural_candidate_is_admissible(&cleanup_scope, &case, &candidate)
|
||||
&& still_fails(&candidate)?
|
||||
{
|
||||
case = candidate;
|
||||
}
|
||||
}
|
||||
let mut index = case.processes.len();
|
||||
while index > 1 {
|
||||
while index > 0 && case.processes.len() > 1 {
|
||||
check()?;
|
||||
index -= 1;
|
||||
let mut candidate = case.clone();
|
||||
let removed = candidate.processes.remove(index).id;
|
||||
|
|
@ -18,44 +212,45 @@ pub fn shrink_failure(
|
|||
.depends_on
|
||||
.retain(|dependency| dependency != &removed);
|
||||
}
|
||||
if candidate.validate().is_ok() && still_fails(&candidate) {
|
||||
if structural_candidate_is_admissible(&cleanup_scope, &case, &candidate)
|
||||
&& still_fails(&candidate)?
|
||||
{
|
||||
case = candidate;
|
||||
}
|
||||
}
|
||||
for process_index in 0..case.processes.len() {
|
||||
let mut action_index = case.processes[process_index].actions.len();
|
||||
while action_index > 1 {
|
||||
while action_index > 0 && case.processes[process_index].actions.len() > 1 {
|
||||
check()?;
|
||||
action_index -= 1;
|
||||
let mut candidate = case.clone();
|
||||
candidate.processes[process_index]
|
||||
.actions
|
||||
.remove(action_index);
|
||||
if candidate.validate().is_ok() && still_fails(&candidate) {
|
||||
if structural_candidate_is_admissible(&cleanup_scope, &case, &candidate)
|
||||
&& still_fails(&candidate)?
|
||||
{
|
||||
case = candidate;
|
||||
}
|
||||
}
|
||||
}
|
||||
for process_index in 1..case.processes.len() {
|
||||
for predecessor_index in (0..process_index).rev() {
|
||||
let predecessor = case.processes[predecessor_index].id.clone();
|
||||
if case.processes[process_index]
|
||||
.depends_on
|
||||
.contains(&predecessor)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
for process_index in 0..case.processes.len() {
|
||||
for dependency_index in (0..case.processes[process_index].depends_on.len()).rev() {
|
||||
check()?;
|
||||
let mut candidate = case.clone();
|
||||
candidate.processes[process_index]
|
||||
.depends_on
|
||||
.push(predecessor);
|
||||
if candidate.validate().is_ok() && still_fails(&candidate) {
|
||||
.remove(dependency_index);
|
||||
if structural_candidate_is_admissible(&cleanup_scope, &case, &candidate)
|
||||
&& still_fails(&candidate)?
|
||||
{
|
||||
case = candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
for process_index in 0..case.processes.len() {
|
||||
for action_index in 0..case.processes[process_index].actions.len() {
|
||||
check()?;
|
||||
let stream = match &case.processes[process_index].actions[action_index].operation {
|
||||
ActionOp::StreamWrite { path, chunks, .. } if chunks.len() > 1 => {
|
||||
Some((path.clone(), chunks.concat()))
|
||||
|
|
@ -65,39 +260,37 @@ pub fn shrink_failure(
|
|||
let Some((path, payload)) = stream else {
|
||||
continue;
|
||||
};
|
||||
if case.routes.iter().any(|route| {
|
||||
route.edges.len() > 1 && route.edges.iter().any(|edge| edge.path == path)
|
||||
}) {
|
||||
continue;
|
||||
}
|
||||
let mut candidate = case.clone();
|
||||
if let ActionOp::StreamWrite { chunks, .. } =
|
||||
&mut candidate.processes[process_index].actions[action_index].operation
|
||||
{
|
||||
*chunks = vec![payload.clone()];
|
||||
*chunks = vec![payload];
|
||||
}
|
||||
for process in &mut candidate.processes {
|
||||
for action in &mut process.actions {
|
||||
if let ActionOp::StreamRead {
|
||||
path: reader_path,
|
||||
expected,
|
||||
}
|
||||
| ActionOp::StreamReadInto {
|
||||
path: reader_path,
|
||||
expected,
|
||||
..
|
||||
} = &mut action.operation
|
||||
&& reader_path == &path
|
||||
if cleanup_scope.admits(&candidate)
|
||||
&& candidate.validate().is_ok()
|
||||
&& still_fails(&candidate)?
|
||||
{
|
||||
*expected = payload.clone();
|
||||
}
|
||||
}
|
||||
}
|
||||
if candidate.validate().is_ok() && still_fails(&candidate) {
|
||||
case = candidate;
|
||||
}
|
||||
}
|
||||
}
|
||||
for process_index in 0..case.processes.len() {
|
||||
for action_index in 0..case.processes[process_index].actions.len() {
|
||||
while let Some(candidate) = shrink_payload_candidate(&case, process_index, action_index)
|
||||
loop {
|
||||
check()?;
|
||||
let Some(candidate) = shrink_payload_candidate(&case, process_index, action_index)
|
||||
else {
|
||||
break;
|
||||
};
|
||||
if cleanup_scope.admits(&candidate)
|
||||
&& candidate.validate().is_ok()
|
||||
&& still_fails(&candidate)?
|
||||
{
|
||||
if candidate.validate().is_ok() && still_fails(&candidate) {
|
||||
case = candidate;
|
||||
} else {
|
||||
break;
|
||||
|
|
@ -105,38 +298,164 @@ pub fn shrink_failure(
|
|||
}
|
||||
}
|
||||
}
|
||||
check()?;
|
||||
if !reduce_live_nodes {
|
||||
return Ok(case);
|
||||
}
|
||||
let mut used_nodes = case
|
||||
.processes
|
||||
.iter()
|
||||
.map(|process| process.logical_node_id)
|
||||
.collect::<BTreeSet<_>>();
|
||||
if let FailureInjection::KillNode { logical_node_id } = &case.failure {
|
||||
used_nodes.insert(*logical_node_id);
|
||||
}
|
||||
let node_mapping = used_nodes
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(index, logical_node_id)| (logical_node_id, index as u64 + 1))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let reduced_nodes = node_mapping.len().max(2) as u8;
|
||||
let topology_changed = reduced_nodes != case.node_count
|
||||
|| node_mapping
|
||||
.chain(
|
||||
case.routes
|
||||
.iter()
|
||||
.any(|(logical_node_id, replacement)| logical_node_id != replacement);
|
||||
if topology_changed {
|
||||
.flat_map(|route| &route.edges)
|
||||
.flat_map(|edge| [edge.source, edge.destination]),
|
||||
)
|
||||
.collect::<BTreeSet<_>>();
|
||||
if used_nodes.len() == 1
|
||||
&& let Some(spare) = case
|
||||
.live_nodes
|
||||
.iter()
|
||||
.copied()
|
||||
.find(|node| !used_nodes.contains(node))
|
||||
{
|
||||
used_nodes.insert(spare);
|
||||
}
|
||||
check()?;
|
||||
if used_nodes != case.live_nodes {
|
||||
let mut candidate = case.clone();
|
||||
candidate.node_count = reduced_nodes;
|
||||
for process in &mut candidate.processes {
|
||||
process.logical_node_id = node_mapping[&process.logical_node_id];
|
||||
}
|
||||
if let FailureInjection::KillNode { logical_node_id } = &mut candidate.failure {
|
||||
*logical_node_id = node_mapping[logical_node_id];
|
||||
}
|
||||
if candidate.validate().is_ok() && still_fails(&candidate) {
|
||||
candidate.live_nodes = used_nodes;
|
||||
if cleanup_scope.admits(&candidate)
|
||||
&& candidate.validate().is_ok()
|
||||
&& still_fails(&candidate)?
|
||||
{
|
||||
case = candidate;
|
||||
}
|
||||
}
|
||||
case
|
||||
check()?;
|
||||
Ok(case)
|
||||
}
|
||||
|
||||
/// A typed abort can match even when deleting its setup makes the error correct.
|
||||
/// Preserve the original namespace prerequisites instead of asking that failed
|
||||
/// replay to prove its own expected-success contract. This deliberately keeps
|
||||
/// all potentially relevant mutations, not just a guessed winning publication.
|
||||
fn structural_candidate_is_admissible(
|
||||
cleanup_scope: &CleanupScope,
|
||||
original: &BehaviorCase,
|
||||
candidate: &BehaviorCase,
|
||||
) -> bool {
|
||||
if !cleanup_scope.admits(candidate) || candidate.validate().is_err() {
|
||||
return false;
|
||||
}
|
||||
let retained = original
|
||||
.processes
|
||||
.iter()
|
||||
.map(|process| {
|
||||
let mut actions = candidate
|
||||
.processes
|
||||
.iter()
|
||||
.find(|remaining| remaining.id == process.id)
|
||||
.into_iter()
|
||||
.flat_map(|remaining| &remaining.actions)
|
||||
.peekable();
|
||||
let retained = process
|
||||
.actions
|
||||
.iter()
|
||||
.map(|action| {
|
||||
if actions.peek().is_some_and(|next| *next == action) {
|
||||
actions.next();
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
debug_assert!(
|
||||
actions.next().is_none(),
|
||||
"structural candidates only delete actions"
|
||||
);
|
||||
retained
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
for (process_index, process) in original.processes.iter().enumerate() {
|
||||
for (action_index, action) in process.actions.iter().enumerate() {
|
||||
if retained[process_index][action_index] {
|
||||
continue;
|
||||
}
|
||||
for path in action.operation.paths().into_iter().flatten() {
|
||||
// Successful local probes and waits also establish a prefix
|
||||
// on which a later success relies. Across processes, retain
|
||||
// mutations and synchronization, not unrelated pure readers.
|
||||
let affects_peers = action.operation.mutating_paths().contains(&Some(path))
|
||||
|| matches!(
|
||||
action.operation,
|
||||
ActionOp::AwaitEntry { .. } | ActionOp::WaitForQuiescent { .. }
|
||||
);
|
||||
let path = prerequisite_path(process, path);
|
||||
for (consumer_index, consumer) in original.processes.iter().enumerate() {
|
||||
for (step, required) in consumer.actions.iter().enumerate() {
|
||||
if !retained[consumer_index][step]
|
||||
|| (process_index == consumer_index && step <= action_index)
|
||||
|| (process_index != consumer_index && !affects_peers)
|
||||
|| !matches!(
|
||||
required.expected,
|
||||
ExpectedOutcome::Ok
|
||||
| ExpectedOutcome::Linearized { successes: 1.., .. }
|
||||
)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if required
|
||||
.operation
|
||||
.paths()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.any(|required| prerequisite_path(consumer, required) == path)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// A retained producer is insufficient if removing a dependency (or an
|
||||
// intermediate process) lets its consumer run before publication. Keep
|
||||
// transitive ordering, while permitting removal of redundant direct edges.
|
||||
candidate.processes.iter().all(|process| {
|
||||
let before = dependency_ancestors(original, process);
|
||||
let after = dependency_ancestors(candidate, process);
|
||||
candidate.processes.iter().all(|ancestor| {
|
||||
!before.contains(ancestor.id.as_str()) || after.contains(ancestor.id.as_str())
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn prerequisite_path<'a>(process: &ProcessProgram, path: &'a str) -> std::borrow::Cow<'a, str> {
|
||||
if let Some(suffix) = path.strip_prefix("/runs/self")
|
||||
&& (suffix.is_empty() || suffix.starts_with('/'))
|
||||
{
|
||||
format!("/runs/{}{suffix}", process.access.execution_id).into()
|
||||
} else {
|
||||
path.into()
|
||||
}
|
||||
}
|
||||
|
||||
fn dependency_ancestors<'a>(case: &'a BehaviorCase, process: &ProcessProgram) -> BTreeSet<&'a str> {
|
||||
let mut ancestors = BTreeSet::new();
|
||||
let mut pending = vec![process.id.as_str()];
|
||||
while let Some(id) = pending.pop() {
|
||||
if let Some(program) = case.processes.iter().find(|program| program.id == id) {
|
||||
for dependency in &program.depends_on {
|
||||
if ancestors.insert(dependency.as_str()) {
|
||||
pending.push(dependency);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ancestors
|
||||
}
|
||||
|
||||
fn shrink_payload_candidate(
|
||||
|
|
@ -150,66 +469,169 @@ fn shrink_payload_candidate(
|
|||
.actions
|
||||
.get(action_index)?
|
||||
.operation;
|
||||
let mut candidate = case.clone();
|
||||
match operation {
|
||||
ActionOp::PublishBlob { path, bytes } if !bytes.is_empty() => {
|
||||
let reduced = bytes[..bytes.len() / 2].to_vec();
|
||||
if let ActionOp::PublishBlob { bytes, .. } =
|
||||
&mut candidate.processes[process_index].actions[action_index].operation
|
||||
{
|
||||
*bytes = reduced.clone();
|
||||
// A multi-edge relay's payload is coupled to downstream transformations.
|
||||
// Keep that behavioral graph intact rather than manufacturing a mismatch
|
||||
// by changing only one edge's writer/reader expectations.
|
||||
if case.routes.iter().any(|route| {
|
||||
route.edges.len() > 1 && route.edges.iter().any(|edge| edge.path == operation.path())
|
||||
}) {
|
||||
return None;
|
||||
}
|
||||
for process in &mut candidate.processes {
|
||||
for action in &mut process.actions {
|
||||
if let ActionOp::ReadBlob {
|
||||
path: reader_path,
|
||||
expected,
|
||||
} = &mut action.operation
|
||||
&& reader_path == path
|
||||
{
|
||||
*expected = reduced.clone();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ActionOp::StreamWrite { path, chunks, .. }
|
||||
if chunks.iter().map(Vec::len).sum::<usize>() != 0 =>
|
||||
{
|
||||
let payload = chunks.concat();
|
||||
let reduced = payload[..payload.len() / 2].to_vec();
|
||||
if let ActionOp::StreamWrite { chunks, .. } =
|
||||
&mut candidate.processes[process_index].actions[action_index].operation
|
||||
{
|
||||
*chunks = vec![reduced.clone()];
|
||||
}
|
||||
for process in &mut candidate.processes {
|
||||
for action in &mut process.actions {
|
||||
if let ActionOp::StreamRead {
|
||||
path: reader_path,
|
||||
expected,
|
||||
}
|
||||
| ActionOp::StreamReadInto {
|
||||
path: reader_path,
|
||||
expected,
|
||||
..
|
||||
} = &mut action.operation
|
||||
&& reader_path == path
|
||||
{
|
||||
*expected = reduced.clone();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ActionOp::DescriptorWrite { bytes, .. } if !bytes.is_empty() => {
|
||||
if let ActionOp::DescriptorWrite {
|
||||
bytes: candidate_bytes,
|
||||
..
|
||||
} = &mut candidate.processes[process_index].actions[action_index].operation
|
||||
{
|
||||
candidate_bytes.truncate(bytes.len() / 2);
|
||||
let (stream, payload) = match operation {
|
||||
ActionOp::PublishBlob { bytes, .. } => {
|
||||
(false, std::borrow::Cow::Borrowed(bytes.as_slice()))
|
||||
}
|
||||
ActionOp::DescriptorWrite { bytes, length, .. } if *length == Some(bytes.len() as u64) => {
|
||||
(false, std::borrow::Cow::Borrowed(bytes.as_slice()))
|
||||
}
|
||||
ActionOp::StreamWrite { chunks, .. } => (true, std::borrow::Cow::Owned(chunks.concat())),
|
||||
_ => return None,
|
||||
};
|
||||
if payload.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let path = prerequisite_path(&case.processes[process_index], operation.path());
|
||||
let reduced = &payload[..payload.len() / 2];
|
||||
let mut candidate = case.clone();
|
||||
for (peer_index, process) in candidate.processes.iter_mut().enumerate() {
|
||||
for (step, action) in process.actions.iter_mut().enumerate() {
|
||||
if peer_index == process_index && step == action_index {
|
||||
continue;
|
||||
}
|
||||
let original_process = &case.processes[peer_index];
|
||||
if !action
|
||||
.operation
|
||||
.paths()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.any(|other| prerequisite_path(original_process, other) == path)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
// Coupled replacements, renames and partial descriptor ranges need
|
||||
// a different reduction. Never create a new payload error by
|
||||
// rewriting just one possible source or leaving its reader stale.
|
||||
let expected = match &mut action.operation {
|
||||
ActionOp::ReadBlob { expected, .. } if !stream => expected,
|
||||
ActionOp::DescriptorRead {
|
||||
expected,
|
||||
flags,
|
||||
offset,
|
||||
..
|
||||
} if !stream && *flags == libc::O_RDONLY && *offset == 0 => expected,
|
||||
ActionOp::StreamRead { expected, .. }
|
||||
| ActionOp::StreamReadWithRetry { expected, .. }
|
||||
| ActionOp::StreamReadInto { expected, .. }
|
||||
if stream =>
|
||||
{
|
||||
expected
|
||||
}
|
||||
ActionOp::Lookup { .. }
|
||||
| ActionOp::AwaitEntry { .. }
|
||||
| ActionOp::WaitForQuiescent { .. }
|
||||
| ActionOp::Unlink { .. } => continue,
|
||||
_ => return None,
|
||||
};
|
||||
if expected.as_slice() != payload.as_ref() {
|
||||
return None;
|
||||
}
|
||||
*expected = reduced.to_vec();
|
||||
}
|
||||
}
|
||||
match &mut candidate.processes[process_index].actions[action_index].operation {
|
||||
ActionOp::PublishBlob { bytes, .. } => bytes.truncate(reduced.len()),
|
||||
ActionOp::DescriptorWrite { bytes, length, .. } => {
|
||||
bytes.truncate(reduced.len());
|
||||
*length = Some(reduced.len() as u64);
|
||||
}
|
||||
ActionOp::StreamWrite { chunks, .. } => *chunks = vec![reduced.to_vec()],
|
||||
_ => unreachable!("supported payload operation checked before cloning"),
|
||||
}
|
||||
Some(candidate)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::ir::Action;
|
||||
|
||||
#[test]
|
||||
fn payload_candidates_keep_halving_order_and_matching_reader_contract() {
|
||||
let mut case = crate::corpus::stable_corpus(2, 1).remove(0);
|
||||
let path = "/cases/shrink/payload".to_owned();
|
||||
case.processes[0].actions = vec![Action::ok(ActionOp::PublishBlob {
|
||||
path: path.clone(),
|
||||
bytes: b"abcdef".to_vec(),
|
||||
})];
|
||||
case.processes[1].actions = vec![
|
||||
Action::ok(ActionOp::ReadBlob {
|
||||
path: path.clone(),
|
||||
expected: b"abcdef".to_vec(),
|
||||
}),
|
||||
Action::ok(ActionOp::ReadBlob {
|
||||
path: "/cases/shrink/other".to_owned(),
|
||||
expected: b"unchanged".to_vec(),
|
||||
}),
|
||||
];
|
||||
for expected in [b"abc".as_slice(), b"a".as_slice(), b"".as_slice()] {
|
||||
case = shrink_payload_candidate(&case, 0, 0).unwrap();
|
||||
assert!(matches!(&case.processes[0].actions[0].operation,
|
||||
ActionOp::PublishBlob { bytes, .. } if bytes == expected));
|
||||
assert!(matches!(&case.processes[1].actions[0].operation,
|
||||
ActionOp::ReadBlob { expected: bytes, .. } if bytes == expected));
|
||||
assert!(matches!(&case.processes[1].actions[1].operation,
|
||||
ActionOp::ReadBlob { expected, .. } if expected == b"unchanged"));
|
||||
}
|
||||
assert!(shrink_payload_candidate(&case, 0, 0).is_none());
|
||||
assert!(shrink_payload_candidate(&case, 1, 0).is_none());
|
||||
case.processes[0].actions[0].operation = ActionOp::StreamWrite {
|
||||
path,
|
||||
chunks: vec![Vec::new(), Vec::new()],
|
||||
replace: false,
|
||||
};
|
||||
assert!(shrink_payload_candidate(&case, 0, 0).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn descriptor_payload_reduction_preserves_allocation_and_scoped_readers() {
|
||||
use crate::ir::{DescriptorFinish, DescriptorReadMethod, DescriptorWriteMethod};
|
||||
|
||||
let mut case = crate::corpus::stable_corpus(2, 1).remove(0);
|
||||
let path = "/runs/self/blob";
|
||||
case.processes[0].actions = vec![
|
||||
Action::ok(ActionOp::DescriptorWrite {
|
||||
path: path.to_owned(),
|
||||
flags: libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL | libc::O_TRUNC,
|
||||
length: Some(6),
|
||||
bytes: b"abcdef".to_vec(),
|
||||
method: DescriptorWriteMethod::Write,
|
||||
finish: DescriptorFinish::Close,
|
||||
}),
|
||||
Action::ok(ActionOp::DescriptorRead {
|
||||
path: path.to_owned(),
|
||||
flags: libc::O_RDONLY,
|
||||
expected: b"abcdef".to_vec(),
|
||||
method: DescriptorReadMethod::Mapping,
|
||||
offset: 0,
|
||||
finish: DescriptorFinish::Close,
|
||||
}),
|
||||
];
|
||||
case.processes[1].actions = vec![Action::ok(ActionOp::ReadBlob {
|
||||
path: path.to_owned(),
|
||||
expected: b"unrelated".to_vec(),
|
||||
})];
|
||||
let candidate = shrink_payload_candidate(&case, 0, 0).unwrap();
|
||||
assert!(matches!(&candidate.processes[0].actions[0].operation,
|
||||
ActionOp::DescriptorWrite { bytes, length: Some(3), .. } if bytes == b"abc"));
|
||||
assert!(matches!(&candidate.processes[0].actions[1].operation,
|
||||
ActionOp::DescriptorRead { expected, .. } if expected == b"abc"));
|
||||
assert_eq!(candidate.processes[1], case.processes[1]);
|
||||
|
||||
case.processes[0].actions.push(Action::ok(ActionOp::Rename {
|
||||
source: path.to_owned(),
|
||||
destination: "/runs/self/moved".to_owned(),
|
||||
replace: false,
|
||||
}));
|
||||
assert!(shrink_payload_candidate(&case, 0, 0).is_none());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue