Zachery Aaron Shores-Chmielewski
6b8b1de897
Extend the existing IR, generated Python workloads, independent oracle, regression corpus, resource accounting, and shrinker for the five-node normal/recovery/survivor campaign. Retain exact fixture identities across attempts and persist coverage, failure, timing, and cleanup evidence. Add raw Docker/SSH redeployment fixtures, bounded campaign execution, durable paid-run cleanup state, ordered gate attestation, and scripted-provider safety scenarios. Include the offline oracle reproduction utility for focused diagnosis. Preserve paid access guards; no paid acceptance is claimed. Review verification: 169 harness library tests and 4 ordered-attestation tests passed. The retained ordered run passed Gate A and the full local campaign, then exceeded the complete workflow timing ceiling before later safety checks.
1068 lines
61 KiB
Python
Executable file
1068 lines
61 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Local VastAI wire fixture and independent real-CLI safety scenarios.
|
|
|
|
Server: scripted_provider.py PORT CONTROL_JSON REQUEST_LOG [READY_FD]
|
|
Gate: scripted_provider.py --gate ROOT OUTPUT HARNESS_BINARY
|
|
|
|
Each server owns its contracts, request ledger, and fault controls. The ready
|
|
pipe is written only after HTTP bind/listen succeeds. No credential is logged.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import select
|
|
import secrets
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.request
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
from urllib.parse import urlparse
|
|
|
|
|
|
UNRELATED_ID = 800000
|
|
UNRELATED_LABEL = "unrelated-account-resource-marker"
|
|
UNRELATED_CONTRACTS = {UNRELATED_ID: UNRELATED_LABEL}
|
|
UNRELATED_WIRE_CONTRACTS = {str(UNRELATED_ID): UNRELATED_LABEL}
|
|
KEY_ENV = "MYELIN_SCRIPTED_VASTAI_KEY"
|
|
CREDENTIAL_ENVS = (KEY_ENV, "VAST_API_KEY", "MYELIN_VASTAI_API_KEY", "VASTAI_API_KEY")
|
|
|
|
|
|
def credential_values(env):
|
|
return tuple(env[name].encode() for name in CREDENTIAL_ENVS if env.get(name))
|
|
|
|
|
|
def require_credential_absence(content, env, surface):
|
|
if any(value in content for value in credential_values(env)):
|
|
raise AssertionError(f"credential leaked into {surface}")
|
|
|
|
|
|
def scan_credentials(directory, env):
|
|
overlap = max(map(len, credential_values(env)), default=1) - 1
|
|
files = 0
|
|
total_bytes = 0
|
|
for parent, directories, names in os.walk(directory, followlinks=False):
|
|
for name in directories + names:
|
|
artifact = Path(parent) / name
|
|
if artifact.is_symlink():
|
|
raise AssertionError("credential scan cannot attest a symlinked artifact")
|
|
for name in names:
|
|
artifact = Path(parent) / name
|
|
require_credential_absence(os.fsencode(artifact), env, "artifact path")
|
|
with artifact.open("rb") as source:
|
|
carry = b""
|
|
while chunk := source.read(1024 * 1024):
|
|
total_bytes += len(chunk)
|
|
require_credential_absence(carry + chunk, env, "durable artifact")
|
|
carry = (carry + chunk)[-overlap:] if overlap else b""
|
|
files += 1
|
|
return {"files": files, "bytes": total_bytes, "injected_credentials": len(credential_values(env))}
|
|
|
|
|
|
def default_offers():
|
|
return [
|
|
{"id": 1000 + index, "host_id": 2000 + index, "gpu_name": "RTX A2000",
|
|
"dph_total": 0.05, "gpu_ram": 24000, "compute_cap": 860,
|
|
"verification": "verified", "reliability2": 0.99,
|
|
"inet_down": 1000.0, "inet_up": 1000.0,
|
|
"internet_down_cost_per_tb": 0.0, "internet_up_cost_per_tb": 0.0}
|
|
for index in range(1, 6)
|
|
]
|
|
|
|
|
|
def instance_status(status_mode, contract, label, ssh_endpoint=None):
|
|
host, port = ssh_endpoint or ("", 0)
|
|
return {"id": contract, "label": label, "actual_status": status_mode,
|
|
"intended_status": "running", "public_ipaddr": host, "ssh_port": port,
|
|
"status_msg": "scripted provider failed" if status_mode == "error" else "",
|
|
"disk_usage": 0.0}
|
|
|
|
|
|
class State:
|
|
def __init__(self, control_path, log_path):
|
|
self.lock = threading.RLock()
|
|
self.control_path = Path(control_path)
|
|
self.log_path = Path(log_path)
|
|
self.contracts = dict(UNRELATED_CONTRACTS)
|
|
self.created = {}
|
|
existing = self.control().get("initial_contracts", {})
|
|
self.contracts.update({int(contract): label for contract, label in existing.items()})
|
|
self.created.update({int(contract): label for contract, label in existing.items()})
|
|
self.ssh_keys = []
|
|
self.destroyed = set()
|
|
self.pending = {}
|
|
self.next_contract = 9000
|
|
self.create_count = 0
|
|
self.list_count = 0
|
|
self.offer_count = 0
|
|
self.next_listing = 0.0
|
|
self.late_inserted = False
|
|
self.delete_release = threading.Event()
|
|
self.create_release = threading.Event()
|
|
self.offer_release = threading.Event()
|
|
self.sequence = 0
|
|
|
|
def control(self):
|
|
try:
|
|
return json.loads(self.control_path.read_text())
|
|
except FileNotFoundError:
|
|
return {}
|
|
|
|
def log(self, method, path, authorized, **fields):
|
|
with self.lock:
|
|
self.sequence += 1
|
|
entry = {"sequence": self.sequence, "method": method, "path": path,
|
|
"authorized": authorized, "elapsed_ns": time.monotonic_ns(), **fields}
|
|
with self.log_path.open("a") as handle:
|
|
handle.write(json.dumps(entry) + "\n")
|
|
return self.sequence
|
|
|
|
def reveal(self, control):
|
|
for contract, (label, after_list, after_time) in list(self.pending.items()):
|
|
if self.list_count >= after_list and time.monotonic() >= after_time:
|
|
self.contracts[contract] = label
|
|
del self.pending[contract]
|
|
self.log("EVENT", "contract-visible", True, contract_id=contract, label=label)
|
|
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
state = None
|
|
|
|
def log_message(self, *_args):
|
|
pass
|
|
|
|
def parse_request(self):
|
|
if self.raw_requestline.startswith(b"SSH-"):
|
|
self.state.log("SSH", "bootstrap-connection", True)
|
|
self.state.create_release.wait()
|
|
return False
|
|
if not super().parse_request():
|
|
return False
|
|
if any(value in self.path.encode() for value in credential_values(os.environ)):
|
|
self.state.log("LEAK", "request-target", False)
|
|
self._reply(400, {"error": "credential in request target"})
|
|
return False
|
|
return True
|
|
|
|
@property
|
|
def authorized(self):
|
|
expected = os.environ.get(KEY_ENV)
|
|
return bool(expected) and self.headers.get("Authorization") == f"Bearer {expected}"
|
|
|
|
def _reply(self, code, payload, **headers):
|
|
if code >= 400 and self.state.control().get("credential_echo"):
|
|
payload = dict(payload, diagnostic=os.environ[KEY_ENV])
|
|
self.state.log("EVENT", "credential-echo", True, status=code)
|
|
body = json.dumps(payload).encode()
|
|
try:
|
|
self.send_response(code)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
for name, value in headers.items():
|
|
self.send_header(name.replace("_", "-"), str(value))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
except (BrokenPipeError, ConnectionResetError):
|
|
# Deliberately cancelled requests are part of the cleanup scenario.
|
|
pass
|
|
|
|
def _body(self):
|
|
body = self.rfile.read(int(self.headers.get("Content-Length", "0"))) or b"{}"
|
|
require_credential_absence(body, os.environ, "provider request body")
|
|
return json.loads(body)
|
|
|
|
def do_GET(self):
|
|
state = self.state
|
|
control = state.control()
|
|
path = urlparse(self.path).path
|
|
if not self.authorized:
|
|
state.log("GET", path, False)
|
|
return self._reply(401, {"error": "unauthorized"})
|
|
if path == "/api/v0/bundles/":
|
|
with state.lock:
|
|
state.offer_count += 1
|
|
offer_count = state.offer_count
|
|
state.log("GET", path, True, offer_count=offer_count)
|
|
offers = control.get("offers", default_offers())
|
|
if offer_count > control.get("withhold_offer_responses_after", sys.maxsize):
|
|
state.offer_release.wait()
|
|
return self._reply(200, {"offers": offers})
|
|
with state.lock:
|
|
if path == "/api/v0/ssh/":
|
|
state.log("GET", path, True)
|
|
return self._reply(200, {"ssh_keys": [
|
|
{"id": index + 1, "public_key": key}
|
|
for index, key in enumerate(state.ssh_keys)]})
|
|
if path == "/__scripted__/state":
|
|
state.log("GET", path, True)
|
|
return self._reply(200, {"contracts": state.contracts, "created": state.created,
|
|
"pending": list(state.pending), "destroyed": sorted(state.destroyed)})
|
|
if path == "/api/v0/instances/":
|
|
state.list_count += 1
|
|
now = time.monotonic()
|
|
rate_limited = (
|
|
state.list_count <= control.get("rate_limit_listings", 0)
|
|
or now < state.next_listing
|
|
)
|
|
if rate_limited:
|
|
retry_after = control.get("retry_after", 1)
|
|
state.next_listing = max(state.next_listing, now + retry_after)
|
|
state.log("GET", path, True, status=429, retry_after=retry_after)
|
|
return self._reply(429, {"retry_after": retry_after}, Retry_After=retry_after)
|
|
endpoint = ("127.0.0.1", self.server.server_port) if control.get("ssh_endpoint") else None
|
|
instances = [instance_status(control.get("status_mode", "error"), contract, label, endpoint)
|
|
for contract, label in sorted(state.contracts.items())]
|
|
state.log("GET", path, True, status=200,
|
|
returned_ids=sorted(state.contracts))
|
|
return self._reply(200, {"instances": instances})
|
|
parts = path.strip("/").split("/")
|
|
try:
|
|
contract = int(parts[3])
|
|
except (IndexError, ValueError):
|
|
state.log("GET", path, True, status=404)
|
|
return self._reply(404, {"error": "unknown path"})
|
|
state.reveal(control)
|
|
label = state.contracts.get(contract)
|
|
state.log("GET", path, True, status=200 if label else 404)
|
|
if label is None:
|
|
return self._reply(404, {"error": "not found"})
|
|
endpoint = ("127.0.0.1", self.server.server_port) if control.get("ssh_endpoint") else None
|
|
return self._reply(200, {"instances": instance_status(
|
|
control.get("status_mode", "error"), contract, label, endpoint)})
|
|
|
|
def do_PUT(self):
|
|
state = self.state
|
|
control = state.control()
|
|
path = urlparse(self.path).path
|
|
body = self._body()
|
|
parts = path.strip("/").split("/")
|
|
label = body.get("label")
|
|
state.log("PUT", path, self.authorized, label=label,
|
|
image=body.get("image"), disk=body.get("disk"))
|
|
if not self.authorized:
|
|
return self._reply(401, {"error": "unauthorized"})
|
|
if len(parts) != 4 or parts[2] != "asks":
|
|
return self._reply(404, {"error": "unknown path"})
|
|
mode = control.get("create_mode", "ok")
|
|
if mode == "withheld-acceptance":
|
|
state.create_release.wait()
|
|
with state.lock:
|
|
state.create_count += 1
|
|
mode = control.get("create_mode", "ok")
|
|
if mode == "reject":
|
|
return self._reply(400, {"error": "no_such_ask"})
|
|
contract = state.next_contract
|
|
state.next_contract += 1
|
|
state.created[contract] = label
|
|
if mode == "ambiguous":
|
|
state.pending[contract] = (label, state.list_count + control.get("reveal_after_listings", 2),
|
|
time.monotonic() + control.get("create_visibility_delay", 0.05))
|
|
return self._reply(503, {"error": "create reply lost after acceptance"})
|
|
if mode == "withheld-response":
|
|
state.pending[contract] = (label, state.list_count + 2, time.monotonic() + 0.1)
|
|
else:
|
|
state.contracts[contract] = label
|
|
if state.create_count == 1:
|
|
for offset in range(control.get("extra_owned_contracts", 0)):
|
|
extra = 9500 + offset
|
|
state.contracts[extra] = label
|
|
state.created[extra] = label
|
|
state.log("EVENT", "create-accepted-before-response", True, contract_id=contract, label=label)
|
|
if mode == "withheld-response":
|
|
state.create_release.wait()
|
|
return self._reply(200, {"new_contract": contract})
|
|
|
|
def do_DELETE(self):
|
|
state = self.state
|
|
control = state.control()
|
|
path = urlparse(self.path).path
|
|
request_sequence = state.log("DELETE", path, self.authorized)
|
|
if not self.authorized:
|
|
return self._reply(401, {"error": "unauthorized"})
|
|
try:
|
|
contract = int(path.strip("/").split("/")[3])
|
|
except (IndexError, ValueError):
|
|
return self._reply(404, {"error": "unknown contract"})
|
|
if contract == control.get("withhold_delete"):
|
|
state.delete_release.wait()
|
|
if contract == control.get("slow_delete"):
|
|
time.sleep(control.get("slow_delete_seconds", 1.5))
|
|
with state.lock:
|
|
label = state.contracts.pop(contract, None)
|
|
state.pending.pop(contract, None)
|
|
state.destroyed.add(contract)
|
|
if label and control.get("late_contract_after_delete") and not state.late_inserted:
|
|
state.late_inserted = True
|
|
late = control["late_contract_after_delete"]
|
|
state.contracts[late] = label
|
|
state.created[late] = label
|
|
state.log("EVENT", "delete-completed", True, contract_id=contract,
|
|
request_sequence=request_sequence, existed=label is not None)
|
|
return self._reply(200 if label is not None else 404, {"deleted": contract})
|
|
|
|
def do_POST(self):
|
|
state = self.state
|
|
path = urlparse(self.path).path
|
|
state.log("POST", path, self.authorized)
|
|
if not self.authorized:
|
|
return self._reply(401, {"error": "unauthorized"})
|
|
body = self._body()
|
|
with state.lock:
|
|
if path == "/api/v0/ssh/":
|
|
key = body["ssh_key"]
|
|
if key not in state.ssh_keys:
|
|
state.ssh_keys.append(key)
|
|
return self._reply(200, {"success": True, "key": {
|
|
"id": state.ssh_keys.index(key) + 1, "public_key": key}})
|
|
if path == "/__scripted__/restore":
|
|
for contract, label in body["contracts"].items():
|
|
contract = int(contract)
|
|
state.contracts[contract] = label
|
|
state.destroyed.discard(contract)
|
|
state.delete_release.clear()
|
|
elif path == "/__scripted__/release-deletes":
|
|
state.delete_release.set()
|
|
else:
|
|
return self._reply(404, {"error": "unknown fault control"})
|
|
return self._reply(200, {"applied": True})
|
|
|
|
|
|
def serve():
|
|
Handler.state = State(sys.argv[2], sys.argv[3])
|
|
server = ThreadingHTTPServer(("127.0.0.1", int(sys.argv[1])), Handler)
|
|
server.daemon_threads = True
|
|
if len(sys.argv) > 4:
|
|
with os.fdopen(int(sys.argv[4]), "w") as ready:
|
|
ready.write(json.dumps({"port": server.server_port, "pid": os.getpid()}) + "\n")
|
|
ready.flush()
|
|
server.serve_forever()
|
|
|
|
|
|
def run_process(command, directory, env, stdout_path, stderr_path, timeout=150):
|
|
require_credential_absence(b"\0".join(os.fsencode(part) for part in command), env, "process arguments")
|
|
started = time.monotonic()
|
|
with stdout_path.open("w") as stdout, stderr_path.open("w") as stderr:
|
|
process = subprocess.Popen(command, cwd=directory, env=env, stdout=stdout,
|
|
stderr=stderr, start_new_session=True)
|
|
try:
|
|
status = process.wait(timeout=timeout)
|
|
except BaseException:
|
|
os.killpg(process.pid, signal.SIGTERM)
|
|
try:
|
|
process.wait(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
os.killpg(process.pid, signal.SIGKILL)
|
|
process.wait()
|
|
raise
|
|
return {"exit_status": status, "elapsed_seconds": time.monotonic() - started}
|
|
|
|
|
|
def crash_runner(command, root, env, directory, request_path, failure, boundary):
|
|
require_credential_absence(b"\0".join(os.fsencode(part) for part in command), env, "process arguments")
|
|
started = time.monotonic()
|
|
with (directory / "stdout.log").open("w") as stdout, (directory / "stderr.log").open("w") as stderr:
|
|
process = subprocess.Popen(command, cwd=root, env=env, stdout=stdout,
|
|
stderr=stderr, start_new_session=True)
|
|
try:
|
|
until = time.monotonic() + 90
|
|
while time.monotonic() < until:
|
|
rows = [json.loads(line) for line in request_path.read_text().splitlines()] if request_path.exists() else []
|
|
state_paths = list((directory / "campaign").glob("*/paid-state.json"))
|
|
admission = None
|
|
if len(state_paths) == 1:
|
|
paid = json.loads(state_paths[0].read_text())
|
|
admission_path = Path(paid["state_dir"]) / "paid-admission.json"
|
|
if admission_path.exists():
|
|
admission = json.loads(admission_path.read_text())
|
|
creates = [row for row in rows
|
|
if row["method"] == "PUT" and row["path"].startswith("/api/v0/asks/")]
|
|
accepted = sum(row["path"] == "create-accepted-before-response" for row in rows)
|
|
offers = sum(row["path"] == "/api/v0/bundles/" for row in rows)
|
|
reached = admission is not None and {
|
|
"admission-before-create-reservation":
|
|
offers >= 3 and not admission["create_reservations"] and not admission["contracts"],
|
|
"create-reserved-before-provider-acceptance":
|
|
len(creates) == 5 and len(admission["create_reservations"]) == 5
|
|
and accepted == 0 and not admission["contracts"],
|
|
"provider-accepted-before-response-accounting":
|
|
accepted == 5 and len(admission["create_reservations"]) == 5
|
|
and not admission["contracts"],
|
|
"contract-accounted-before-bootstrap":
|
|
len(admission["contracts"]) == 5 and not admission["initial_bootstraps"],
|
|
"bootstrap-reserved-before-prepared-commit":
|
|
len(admission["initial_bootstraps"]) == 5
|
|
and admission["mode"] == "preparing",
|
|
}[boundary]
|
|
if reached:
|
|
break
|
|
assert process.poll() is None, f"runner exited before {boundary}"
|
|
time.sleep(0.02)
|
|
else:
|
|
raise AssertionError(f"{boundary} was not observed before runner kill")
|
|
assert len(state_paths) == 1 and admission is not None
|
|
for role in ("runner", "orchestrator", "owner", "supervisor"):
|
|
identity = admission["cleanup"][role]
|
|
assert identity, f"missing admitted {role} identity"
|
|
require_credential_absence(
|
|
Path(f"/proc/{identity['pid']}/cmdline").read_bytes(), env, f"{role} arguments")
|
|
if failure == "workstation-loss":
|
|
# The provider is outside the failed workstation. Stop its local
|
|
# supervision first so no owner can be restarted during the crash.
|
|
signal_cleanup_process(admission_path, "supervisor", signal.SIGKILL)
|
|
if failure in ("owner-loss-automatic", "workstation-loss"):
|
|
signal_cleanup_process(admission_path, "owner", signal.SIGKILL)
|
|
if failure in ("orchestrator-loss", "workstation-loss"):
|
|
signal_cleanup_process(admission_path, "orchestrator", signal.SIGKILL)
|
|
if failure == "orchestrator-loss":
|
|
process.wait(timeout=60)
|
|
else:
|
|
os.killpg(process.pid, signal.SIGKILL)
|
|
process.wait(timeout=5)
|
|
before = state_paths[0].read_bytes()
|
|
repeated = run_process(command, root, env, directory / "repeat.stdout.log", directory / "repeat.stderr.log")
|
|
assert repeated["exit_status"] != 0, "repeated paid campaign overwrote ownership"
|
|
assert state_paths[0].read_bytes() == before, "refusal modified original paid ownership"
|
|
finally:
|
|
if process.poll() is None:
|
|
os.killpg(process.pid, signal.SIGKILL)
|
|
process.wait()
|
|
return {"exit_status": process.returncode, "elapsed_seconds": time.monotonic() - started,
|
|
"crash_boundary": boundary,
|
|
"runner_killed_before_accounting": failure != "orchestrator-loss",
|
|
"cleanup_owner_killed": failure in ("owner-loss-automatic", "workstation-loss"),
|
|
"original_cleanup": admission["cleanup"],
|
|
"original_contracts": admission["contracts"],
|
|
"original_initial_bootstraps": admission["initial_bootstraps"],
|
|
"original_create_reservations": admission["create_reservations"],
|
|
"post_crash_sequence": rows[-1]["sequence"]}
|
|
|
|
|
|
def signal_cleanup_process(admission_path, role, sig):
|
|
identity = json.loads(admission_path.read_text())["cleanup"][role]
|
|
assert identity, f"missing cleanup {role} identity"
|
|
pidfd = os.pidfd_open(identity["pid"])
|
|
try:
|
|
proc = Path(f"/proc/{identity['pid']}")
|
|
fields = (proc / "stat").read_text().rsplit(") ", 1)[1].split()
|
|
command = (proc / "cmdline").read_bytes().split(b"\0")
|
|
assert int(fields[19]) == identity["start_ticks"], "cleanup process incarnation changed"
|
|
assert Path("/proc/sys/kernel/random/boot_id").read_text().strip() == identity["boot_id"]
|
|
if role == "orchestrator":
|
|
assert Path(os.fsdecode(command[0])).name == "myelin-orchestrator"
|
|
assert str(admission_path.parent).encode() in command
|
|
else:
|
|
assert f"--paid-cleanup-{role}".encode() in command and str(admission_path).encode() in command
|
|
signal.pidfd_send_signal(pidfd, sig)
|
|
assert select.select([pidfd], [], [], 5)[0], f"{role} crash did not stop the admitted incarnation"
|
|
finally:
|
|
os.close(pidfd)
|
|
return identity
|
|
|
|
|
|
def stop_scripted_cleanup_owners(directory):
|
|
paths = [directory / "private" / "paid-admission.json"]
|
|
for paid_path in (directory / "campaign").glob("*/paid-state.json"):
|
|
paid = json.loads(paid_path.read_text())
|
|
paths.append(Path(paid["state_dir"]) / "paid-admission.json")
|
|
for admission_path in paths:
|
|
if not admission_path.exists():
|
|
continue
|
|
cleanup = json.loads(admission_path.read_text()).get("cleanup") or {}
|
|
spending_stopped = cleanup.get("spending_stopped", False)
|
|
# A completed worker still belongs to its supervisor's child.wait().
|
|
# Let that supervisor reap and exit naturally before forced teardown.
|
|
roles = ("owner", "supervisor") if spending_stopped else ("supervisor", "owner")
|
|
for role in roles:
|
|
identity = (json.loads(admission_path.read_text()).get("cleanup") or {}).get(role)
|
|
if not identity:
|
|
continue
|
|
pidfd = None
|
|
try:
|
|
pidfd = os.pidfd_open(identity["pid"])
|
|
if select.select([pidfd], [], [], 0)[0]:
|
|
continue
|
|
proc = Path(f"/proc/{identity['pid']}")
|
|
fields = (proc / "stat").read_text().rsplit(") ", 1)[1].split()
|
|
if fields[0] in ("Z", "X"):
|
|
continue
|
|
assert int(fields[19]) == identity["start_ticks"], "cleanup process incarnation changed"
|
|
assert Path("/proc/sys/kernel/random/boot_id").read_text().strip() == identity["boot_id"]
|
|
if spending_stopped and select.select([pidfd], [], [], 5)[0]:
|
|
continue
|
|
command = (proc / "cmdline").read_bytes().split(b"\0")
|
|
# Exit can race both the stat and cmdline reads. Empty argv
|
|
# alone is not proof of exit and must never authorize a signal.
|
|
if select.select([pidfd], [], [], 0)[0]:
|
|
continue
|
|
fields = (proc / "stat").read_text().rsplit(") ", 1)[1].split()
|
|
if fields[0] in ("Z", "X"):
|
|
continue
|
|
if not any(command) and select.select([pidfd], [], [], 5)[0]:
|
|
continue
|
|
assert int(fields[19]) == identity["start_ticks"], "cleanup process incarnation changed"
|
|
assert f"--paid-cleanup-{role}".encode() in command and str(admission_path).encode() in command
|
|
signal.pidfd_send_signal(pidfd, signal.SIGTERM)
|
|
assert select.select([pidfd], [], [], 5)[0], f"cleanup {role} did not stop"
|
|
except (FileNotFoundError, ProcessLookupError):
|
|
pass
|
|
finally:
|
|
if pidfd is not None:
|
|
os.close(pidfd)
|
|
|
|
|
|
def retained_owner_loss(binary, root, env, directory, request, ledger):
|
|
private = directory / "private"
|
|
private.mkdir(mode=0o700)
|
|
admission_path = private / "paid-admission.json"
|
|
command = [str(binary), "--scripted-retained-lifecycle", str(admission_path),
|
|
"--api-key-env", KEY_ENV]
|
|
initial = run_process(command, root, env, directory / "prepare.stdout.log", directory / "prepare.stderr.log")
|
|
assert initial["exit_status"] == 0, "scripted retained admission did not prepare"
|
|
before = json.loads(admission_path.read_text())
|
|
assert before["mode"] == "prepared" and before["cleanup"]["retained_development_fixture"]
|
|
resumed = run_process(command, root, env, directory / "redeploy.stdout.log", directory / "redeploy.stderr.log")
|
|
assert resumed["exit_status"] == 0, "explicit retained generation was not admitted"
|
|
retained = json.loads(admission_path.read_text())
|
|
grant = retained["retained_deployments"]["scripted-retained-generation"]
|
|
assert set(grant["bootstraps"]) == set(retained["contracts"]), "generation did not cover exact contracts"
|
|
for field in ("contracts", "create_reservations", "initial_bootstraps"):
|
|
assert retained[field] == before[field], f"redeployment changed {field}"
|
|
for field in ("limits", "started_unix_ms", "stop_unix_ms", "supervisor"):
|
|
assert retained["cleanup"][field] == before["cleanup"][field], f"redeployment changed original {field}"
|
|
# Both foreground processes have exited; no orchestrator/runner can issue
|
|
# manual cleanup. Only the independently supervised worker can delete.
|
|
for role in ("runner", "orchestrator"):
|
|
process = retained["cleanup"][role]
|
|
proc = Path(f"/proc/{process['pid']}/stat")
|
|
if proc.exists():
|
|
fields = proc.read_text().rsplit(") ", 1)[1].split()
|
|
assert int(fields[19]) != process["start_ticks"] or fields[0] in ("Z", "X")
|
|
prior_sequence = ledger()[-1]["sequence"] if ledger() else 0
|
|
killed = signal_cleanup_process(admission_path, "owner", signal.SIGKILL)
|
|
until = time.monotonic() + 30
|
|
while time.monotonic() < until:
|
|
after = json.loads(admission_path.read_text())
|
|
state = request("/__scripted__/state")
|
|
if (after["cleanup"]["complete"] and after["cleanup"]["spending_stopped"]
|
|
and state["contracts"] == UNRELATED_WIRE_CONTRACTS and not state["pending"]):
|
|
break
|
|
time.sleep(0.05)
|
|
else:
|
|
raise AssertionError("owner loss did not automatically stop retained spending")
|
|
assert after["cleanup"]["owner"] != killed, "owner was not restarted"
|
|
assert after["mode"] == "cleanup_only", "restart reopened paid permission"
|
|
for field in ("limits", "started_unix_ms", "stop_unix_ms", "supervisor"):
|
|
assert after["cleanup"][field] == before["cleanup"][field], f"owner restart changed {field}"
|
|
for field in ("contracts", "create_reservations", "initial_bootstraps", "retained_deployments"):
|
|
assert after[field] == retained[field], f"owner restart changed {field}"
|
|
ceiling_ms = min(before["deadline_unix_ms"], before["cleanup"]["stop_unix_ms"] + 300_000)
|
|
assert time.time_ns() // 1_000_000 < ceiling_ms, "original ceiling was exceeded"
|
|
rows = ledger()
|
|
assert all(row["authorized"] for row in rows), "unauthenticated lifecycle request"
|
|
assert not any(row["method"] == "PUT" or row["path"] == "/api/v0/bundles/" for row in rows), "retained lifecycle searched or acquired"
|
|
assert not any(row["method"] == "DELETE"
|
|
and int(row["path"].rstrip("/").split("/")[-1]) in UNRELATED_CONTRACTS
|
|
for row in rows if row["path"].startswith("/api/v0/instances/"))
|
|
known = set(map(int, retained["contracts"].values()))
|
|
listings = [row for row in rows if row["sequence"] > prior_sequence
|
|
and row["path"] == "/api/v0/instances/" and row.get("status") == 200]
|
|
assert listings and not known.intersection(listings[-1]["returned_ids"]), "missing typed exact absence after owner loss"
|
|
assert known <= set(state["destroyed"]), "cleanup omitted an exact retained identity"
|
|
(directory / "provider-final-state.json").write_text(json.dumps(state, indent=2))
|
|
return {"exit_status": 0, "creates": 0, "retained_redeployment_admitted": True,
|
|
"crash_boundary": "prepared-commit-cleanup-owner-loss",
|
|
"initial_reservations_unchanged": True, "automatic_owner_restart": True,
|
|
"acquisition_slots": len(after["create_reservations"]),
|
|
"initial_bootstrap_slots": len(after["initial_bootstraps"]),
|
|
"manual_cleanup_commands": 0, "original_ceilings_preserved": True,
|
|
"created_contract_ids": [], "existing_contract_ids": sorted(known),
|
|
"destroyed_contract_ids": sorted(state["destroyed"]), "exact_absence": True,
|
|
"unrelated_preserved": True, "scope": "admission-and-real-cleanup-processes"}
|
|
|
|
|
|
def gate(root, output, binary):
|
|
if not __debug__:
|
|
raise RuntimeError("scripted safety proofs require Python assertions; optimization is forbidden")
|
|
gate_started = time.monotonic()
|
|
root, output, binary = Path(root).resolve(), Path(output).resolve(), Path(binary).resolve()
|
|
from ordered_acceptance import ROOT, TEST_COMMAND, digest, source_digest
|
|
|
|
assert root == ROOT, "gate fixtures must hash the harness workspace"
|
|
rejection_build_artifacts = {
|
|
name: digest(root / "target/release" / name)
|
|
for name in ("myelin-e2e-fuzz", "myelin-orchestrator", "myelin-worker")
|
|
}
|
|
assert digest(binary) == rejection_build_artifacts["myelin-e2e-fuzz"], "scripted gate binary is not the qualified executable"
|
|
rejection_source_digest = source_digest()
|
|
rejection_image = "registry.invalid/myelin-node@sha256:" + "0" * 64
|
|
rejected_gates = {
|
|
"missing-gates": "paid execution requires --gate-attestation from ordered_acceptance.py; operator authorization is not gate evidence",
|
|
"stale-gates": "paid gate attestation is missing, incomplete, stale, or incompatible",
|
|
"failed-gates": "paid gate stages failed or were not executed in order",
|
|
"out-of-order-gates": "missing, duplicate or out-of-order required gate warm-1-campaign",
|
|
"changed-binaries": "paid gate evidence does not attest the current binaries, sources and image",
|
|
}
|
|
output.mkdir(parents=True, exist_ok=False)
|
|
env = dict(os.environ, **{name: "scripted-" + secrets.token_hex(24) for name in CREDENTIAL_ENVS})
|
|
marker = env[KEY_ENV]
|
|
key = output / "dummy_ed25519"
|
|
subprocess.run(["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(key),
|
|
"-C", "scripted-safety"], check=True)
|
|
duplicate_offers = default_offers()
|
|
for offer in duplicate_offers:
|
|
offer["host_id"] = 2001
|
|
malformed_offers = default_offers()
|
|
malformed_offers[0]["dph_total"] = -1.0
|
|
priced_offers = list(reversed(default_offers()))
|
|
priced_offers += [
|
|
dict(default_offers()[0], id=1101, dph_total=0.01),
|
|
dict(default_offers()[0], id=1102, host_id=2102, dph_total=0.001, verification="unverified"),
|
|
dict(default_offers()[0], id=1103, host_id=59017, dph_total=0.001),
|
|
dict(default_offers()[0], id=1104, host_id=2104, dph_total=0.08),
|
|
]
|
|
preflight = {
|
|
"zero-cost": ("--max-total-cost-usd", "0", "explicit positive paid-run ceiling"),
|
|
"invalid-cost": ("--max-hourly-cost-usd", "NaN", "must be finite and non-negative"),
|
|
"zero-lifetime": ("--fixture-lifetime-secs", "0", "resource bounds must be nonzero"),
|
|
"short-lifetime": ("--fixture-lifetime-secs", "1199", "planned campaign requires"),
|
|
"empty-image": ("--image", " ", "runtime image must not be empty"),
|
|
"missing-credential": (None, None, "credential environment"),
|
|
}
|
|
provider_admissions = {
|
|
"selected-cost-over-budget": ("--max-total-cost-usd", "1", "selected worst-case cost"),
|
|
}
|
|
crash_scenarios = {
|
|
"runner-loss": ("runner-loss", "provider-accepted-before-response-accounting"),
|
|
"owner-loss-automatic": ("owner-loss-automatic", "provider-accepted-before-response-accounting"),
|
|
"orchestrator-loss": ("orchestrator-loss", "provider-accepted-before-response-accounting"),
|
|
"workstation-loss": ("workstation-loss", "provider-accepted-before-response-accounting"),
|
|
"crash-before-create-reservation": ("runner-loss", "admission-before-create-reservation"),
|
|
"crash-after-create-reservation": ("runner-loss", "create-reserved-before-provider-acceptance"),
|
|
"crash-after-contract-accounting": ("runner-loss", "contract-accounted-before-bootstrap"),
|
|
"crash-after-bootstrap-reservation": ("runner-loss", "bootstrap-reserved-before-prepared-commit"),
|
|
}
|
|
scenarios = {
|
|
"missing-gates": {},
|
|
"stale-gates": {},
|
|
"failed-gates": {},
|
|
"out-of-order-gates": {},
|
|
"changed-binaries": {},
|
|
**{name: {} for name in preflight},
|
|
"campaign-resource-overflow": {},
|
|
**{name: {} for name in provider_admissions},
|
|
"runner-loss": {"create_mode": "withheld-response"},
|
|
"owner-loss-automatic": {"create_mode": "withheld-response"},
|
|
"orchestrator-loss": {"create_mode": "withheld-response"},
|
|
"workstation-loss": {"create_mode": "withheld-response"},
|
|
"crash-before-create-reservation": {"withhold_offer_responses_after": 2},
|
|
"crash-after-create-reservation": {"create_mode": "withheld-acceptance"},
|
|
"crash-after-contract-accounting": {"status_mode": "loading"},
|
|
"crash-after-bootstrap-reservation": {"status_mode": "running", "ssh_endpoint": True},
|
|
"retained-owner-loss": {"initial_contracts": {
|
|
str(8999 + node): f"scripted-retained-42-{node}-attempt-0" for node in range(1, 6)}},
|
|
"duplicate-hosts": {"offers": duplicate_offers},
|
|
"insufficient-offers": {"offers": []},
|
|
"malformed-offers": {"offers": malformed_offers},
|
|
"cheapest-distinct-verified": {"offers": priced_offers},
|
|
"create-rejected": {"create_mode": "reject", "credential_echo": True},
|
|
"rate-limited-listing": {"create_mode": "reject", "rate_limit_listings": 1, "retry_after": 1},
|
|
"delayed-ambiguous-create": {"create_mode": "ambiguous", "reveal_after_listings": 2,
|
|
"create_visibility_delay": 0},
|
|
"late-contract": {"status_mode": "running", "late_contract_after_delete": 9700,
|
|
"extra_owned_contracts": 2},
|
|
"slow-delete": {"slow_delete": 9000, "slow_delete_seconds": 1.5, "extra_owned_contracts": 2,
|
|
"late_contract_after_delete": 9700},
|
|
"cleanup-only-recovery": {"extra_owned_contracts": 2},
|
|
}
|
|
summary = {"schema_version": 3, "state": "running", "scenarios": {},
|
|
"provider": "loopback-scripted", "paid_resources": 0}
|
|
for name, control in scenarios.items():
|
|
directory = output / name
|
|
directory.mkdir(parents=True, exist_ok=False)
|
|
control_path = directory / "control.json"
|
|
control_path.write_text(json.dumps(control))
|
|
request_path = directory / "requests.jsonl"
|
|
ready_read, ready_write = os.pipe()
|
|
provider_log = (directory / "provider.log").open("w")
|
|
provider = subprocess.Popen([sys.executable, "-E", "-B", __file__, "0", str(control_path), str(request_path),
|
|
str(ready_write)], env=env, pass_fds=(ready_write,),
|
|
stdout=provider_log, stderr=provider_log)
|
|
os.close(ready_write)
|
|
try:
|
|
assert select.select([ready_read], [], [], 10)[0], "provider did not signal readiness"
|
|
ready = json.loads(os.read(ready_read, 4096))
|
|
assert ready["pid"] == provider.pid and provider.poll() is None
|
|
url = f"http://127.0.0.1:{ready['port']}"
|
|
scenario_env = dict(env, VASTAI_BASE_URL=url,
|
|
XDG_STATE_HOME=str(directory / "private-state"))
|
|
|
|
def request(path, body=None):
|
|
data = None if body is None else json.dumps(body).encode()
|
|
req = urllib.request.Request(url + path, data=data,
|
|
headers={"Authorization": f"Bearer {marker}", "Content-Type": "application/json"})
|
|
with urllib.request.urlopen(req, timeout=5) as response:
|
|
return json.load(response)
|
|
|
|
def ledger():
|
|
return [json.loads(line) for line in request_path.read_text().splitlines()] if request_path.exists() else []
|
|
|
|
if name == "retained-owner-loss":
|
|
summary["scenarios"][name] = retained_owner_loss(
|
|
binary, root, scenario_env, directory, request, ledger)
|
|
(output / "scripted-safety-summary.json").write_text(json.dumps(summary, indent=2))
|
|
continue
|
|
|
|
command = [str(binary), "--paid-vastai", "--authorize-paid-vastai", "--scripted-provider",
|
|
"--seed", "20260910", "--fixture-lifetime-secs", "43200", "--max-total-cost-usd", "5",
|
|
"--max-hourly-cost-usd", "0.5", "--ssh-identity", str(key),
|
|
"--image", "myelin-node:latest", "--api-key-env", KEY_ENV,
|
|
"--deadline-secs", "5", "--no-build-image", "--artifacts", str(directory / "campaign")]
|
|
# These fixtures reject before runtime-image or deployment-artifact inspection.
|
|
if name in rejected_gates:
|
|
command.remove("--scripted-provider")
|
|
command[command.index("--image") + 1] = rejection_image
|
|
if name != "missing-gates":
|
|
now = time.time_ns() // 1_000_000
|
|
stages = []
|
|
qualified_binary = str(root / "target/release/myelin-e2e-fuzz")
|
|
qualified_artifacts = directory / "qualified-local-gates"
|
|
|
|
def stage(stage_name, argv):
|
|
stages.append({"name": stage_name, "command": argv,
|
|
"state": "passed", "exit_code": 0, "elapsed_secs": 0.001,
|
|
"started_unix_ms": now - 1000 + len(stages) * 2,
|
|
"completed_unix_ms": now - 999 + len(stages) * 2})
|
|
|
|
stage("build", ["cargo", "build", "--release", "-p", "myelin",
|
|
"--bins", "-p", "myelin-e2e-fuzz"])
|
|
stage("build-test-binaries", TEST_COMMAND + ["--no-run"])
|
|
stage("build-deployment-artifacts",
|
|
[qualified_binary, "--prepare-artifacts", "--deadline-secs", "3600",
|
|
"--artifacts", str(qualified_artifacts / "build-deployment-artifacts")])
|
|
for index in range(1, 4):
|
|
common = [qualified_binary, "--seed", "20260910", "--deadline-secs",
|
|
"120", "--no-build-image", "--image", rejection_image]
|
|
for suffix, flags in [
|
|
("gate-a", ["--deployment-e2e", "--deployment-nodes", "5"]),
|
|
("campaign", ["--campaign", "--fixture-lifetime-secs", "43200"]),
|
|
("failure-cases", ["--nodes", "5", "--failure-cases"])]:
|
|
stage(f"warm-{index}-{suffix}", common + flags + ["--artifacts",
|
|
str(qualified_artifacts / f"warm-{index}" / suffix)])
|
|
stage(f"warm-{index}-contract-model-safety", TEST_COMMAND)
|
|
stage(f"warm-{index}-scripted-provider",
|
|
["bash", "tools/myelin-e2e-fuzz/scripted_safety_gate.sh",
|
|
str(qualified_artifacts / f"warm-{index}" / "scripted-provider")])
|
|
stage("verify-qualified-deployment-artifacts",
|
|
[qualified_binary, "--prepare-artifacts", "--deadline-secs", "120",
|
|
"--artifacts", str(qualified_artifacts / "verify-qualified-deployment-artifacts")])
|
|
# Synthetic image/deployment identities satisfy the schema, not paid
|
|
# qualification. Only the intended guard below may reject each fixture.
|
|
evidence = {"schema_version": 5, "state": "passed", "completed_unix_ms": now - 2,
|
|
"expires_unix_ms": now + 60_000,
|
|
"build_elapsed_secs": 0.02, "elapsed_secs": 0.1,
|
|
"build_artifacts": dict(rejection_build_artifacts),
|
|
"deployment_artifacts": {
|
|
"schema_version": 2, "source_build_input_digest": "0" * 64,
|
|
"executables": dict(rejection_build_artifacts),
|
|
"wheel_input_digest": "0" * 64, "wheels": {}, "payload": {}},
|
|
"source_digest": rejection_source_digest,
|
|
"image_identities": {
|
|
image: {"id": "sha256:" + str(index) * 64, "os": "linux",
|
|
"architecture": "amd64", "variant": "",
|
|
"repo_digests": [rejection_image],
|
|
"provenance_version": 1,
|
|
"source_build_input_digest": "0" * 64,
|
|
"image_role": role,
|
|
"parent_image_id": None if index == 0 else "sha256:" + str(index - 1) * 64}
|
|
for index, (image, role) in enumerate([
|
|
("myelin-node-base:cuda12.6", "base"),
|
|
("myelin-node:latest", "node"), (rejection_image, "e2e")])},
|
|
"configuration": {"warm_runs": 3, "image": rejection_image,
|
|
"seed": 20260910, "case_deadline_secs": 120,
|
|
"build_images": False, "workspace": str(root),
|
|
"artifacts": str(qualified_artifacts)},
|
|
"stages": stages, "runs": [{"index": index, "state": "passed", "gate_a": "passed",
|
|
"inside_target": True, "campaign_elapsed_secs": 0.001,
|
|
"elapsed_secs": 0.01} for index in range(1, 4)]}
|
|
if name == "stale-gates":
|
|
evidence["expires_unix_ms"] = now - 1
|
|
elif name == "failed-gates":
|
|
stages[1]["state"], stages[1]["exit_code"] = "failed", 1
|
|
elif name == "out-of-order-gates":
|
|
stages[4]["name"], stages[5]["name"] = stages[5]["name"], stages[4]["name"]
|
|
elif name == "changed-binaries":
|
|
tested = evidence["build_artifacts"]["myelin-e2e-fuzz"]
|
|
evidence["build_artifacts"]["myelin-e2e-fuzz"] = (
|
|
("1" if tested[0] == "0" else "0") + tested[1:])
|
|
attestation_path = directory / "rejected-attestation.json"
|
|
attestation_path.write_text(json.dumps(evidence))
|
|
command += ["--gate-attestation", str(attestation_path)]
|
|
if name in preflight:
|
|
flag, value, _ = preflight[name]
|
|
if flag is not None:
|
|
command[command.index(flag) + 1] = value
|
|
else:
|
|
scenario_env.pop(KEY_ENV)
|
|
if name in provider_admissions:
|
|
flag, value, _ = provider_admissions[name]
|
|
command[command.index(flag) + 1] = value
|
|
if name == "campaign-resource-overflow":
|
|
command.append("--scripted-campaign-resource-overflow")
|
|
if name in crash_scenarios:
|
|
failure, boundary = crash_scenarios[name]
|
|
result = crash_runner(
|
|
command, root, scenario_env, directory, request_path, failure, boundary)
|
|
else:
|
|
result = run_process(command, root, scenario_env, directory / "stdout.log", directory / "stderr.log")
|
|
assert result["exit_status"] != 0, f"{name}: fault scenario incorrectly passed"
|
|
rows = ledger()
|
|
creates = [row for row in rows
|
|
if row["method"] == "PUT" and row["path"].startswith("/api/v0/asks/")]
|
|
assert all(row["authorized"] for row in rows), f"{name}: unauthenticated provider work"
|
|
assert len(creates) <= 5, f"{name}: exceeded five admitted acquisition attempts"
|
|
assert len({row["path"] for row in creates}) == len(creates), f"{name}: create retried for an admitted offer"
|
|
if (name in rejected_gates or name in preflight or name in provider_admissions
|
|
or name == "campaign-resource-overflow"
|
|
or name in ("duplicate-hosts", "insufficient-offers", "malformed-offers",
|
|
"crash-before-create-reservation")):
|
|
assert not creates, f"{name}: rejected or pre-create scenario acquired contracts"
|
|
else:
|
|
assert creates, f"{name}: scenario never exercised admitted acquisition"
|
|
if name in rejected_gates:
|
|
assert not rows, "rejected gate evidence reached the provider"
|
|
expected_guard = rejected_gates[name]
|
|
stderr = (directory / "stderr.log").read_text()
|
|
assert f"myelin-e2e-behavioral-fuzz: {expected_guard}" in stderr.splitlines(), (
|
|
f"{name}: did not reach intended rejection guard {expected_guard!r}")
|
|
result.update({"rejection_guard": expected_guard, "provider_requests_before_rejection": 0})
|
|
if name in preflight:
|
|
assert not rows, f"{name}: admission failure reached provider"
|
|
assert preflight[name][2] in (directory / "stderr.log").read_text(), f"{name}: wrong preflight failure"
|
|
result["provider_requests_before_rejection"] = 0
|
|
if name in provider_admissions:
|
|
assert any(row["path"] == "/api/v0/bundles/" for row in rows), f"{name}: offer-cost admission was not exercised"
|
|
assert provider_admissions[name][2] in (directory / "stderr.log").read_text(), f"{name}: wrong provider admission failure"
|
|
result["provider_requests_before_rejection"] = len(rows)
|
|
if name == "campaign-resource-overflow":
|
|
assert not rows, f"{name}: campaign resource admission reached provider"
|
|
assert "campaign resources exceed the hard admission ceilings" in (
|
|
directory / "stderr.log").read_text(), f"{name}: wrong campaign resource admission failure"
|
|
result["provider_requests_before_rejection"] = 0
|
|
if name == "malformed-offers":
|
|
assert any(row["path"] == "/api/v0/bundles/" for row in rows), "malformed offer response was not exercised"
|
|
if control.get("credential_echo"):
|
|
assert any(row["path"] == "credential-echo" for row in rows), "credential echo was not exercised"
|
|
state_paths = list((directory / "campaign").rglob("paid-state.json"))
|
|
paid_path = state_paths[0] if state_paths else None
|
|
if creates or name in crash_scenarios:
|
|
assert len(state_paths) == 1, f"{name}: missing unique durable cleanup state"
|
|
paid = json.loads(paid_path.read_text())
|
|
if name not in crash_scenarios:
|
|
assert paid["phase"] not in ("planned", "acquiring", "prepared"), f"{name}: acquisition remained authorized"
|
|
assert all(row["label"] in paid["owned_labels"] for row in creates), f"{name}: unauthorized owned label"
|
|
assert all(int(row["path"].rstrip("/").split("/")[-1]) in paid["selected_offer_ids"]
|
|
for row in creates), f"{name}: unselected offer acquired"
|
|
if name == "cheapest-distinct-verified":
|
|
assert paid["selected_offer_ids"] == [1101, 1002, 1003, 1004, 1005], "price/verification/distinct-host policy selected the wrong offers"
|
|
result["selected_offer_ids"] = paid["selected_offer_ids"]
|
|
|
|
def cleanup(suffix, seconds=15):
|
|
before = ledger()[-1]["sequence"] if ledger() else 0
|
|
outcome = run_process([str(binary), "--cleanup-only", str(paid_path),
|
|
"--deadline-secs", str(seconds), "--api-key-env", KEY_ENV,
|
|
"--artifacts", str(directory / "cleanup-artifacts")],
|
|
root, scenario_env, directory / f"{suffix}.stdout.log",
|
|
directory / f"{suffix}.stderr.log", timeout=seconds + 15)
|
|
new_rows = [row for row in ledger() if row["sequence"] > before]
|
|
assert not any(row["method"] == "PUT" or row["path"] == "/api/v0/bundles/"
|
|
for row in new_rows), f"{name}: cleanup-only searched or acquired"
|
|
return outcome, new_rows
|
|
|
|
if name == "workstation-loss":
|
|
recovered, _ = cleanup("workstation-recovery")
|
|
assert recovered["exit_status"] == 0, "workstation cleanup-only recovery failed"
|
|
recovered_admission = json.loads((Path(paid["state_dir"]) / "paid-admission.json").read_text())
|
|
assert recovered_admission["mode"] == "cleanup_only"
|
|
for field in ("limits", "started_unix_ms", "stop_unix_ms"):
|
|
assert recovered_admission["cleanup"][field] == result["original_cleanup"][field], "workstation recovery renewed cleanup ceilings"
|
|
result["cleanup_only"] = recovered
|
|
result["manual_cleanup_commands"] = 1
|
|
|
|
if name in set(crash_scenarios) - {"workstation-loss"}:
|
|
admission_path = Path(paid["state_dir"]) / "paid-admission.json"
|
|
until = time.monotonic() + 30
|
|
while time.monotonic() < until:
|
|
state = request("/__scripted__/state")
|
|
admission = json.loads(admission_path.read_text())
|
|
if (state["contracts"] == UNRELATED_WIRE_CONTRACTS
|
|
and not state["pending"] and admission["cleanup"]["complete"]):
|
|
break
|
|
time.sleep(0.05)
|
|
else:
|
|
raise AssertionError("independent supervision did not automatically stop spending")
|
|
for field in ("limits", "started_unix_ms", "stop_unix_ms", "supervisor"):
|
|
assert admission["cleanup"][field] == result["original_cleanup"][field]
|
|
assert admission["mode"] == "cleanup_only"
|
|
if name == "owner-loss-automatic":
|
|
assert admission["cleanup"]["owner"] != result["original_cleanup"]["owner"]
|
|
result["automatic_cleanup"] = True
|
|
result["manual_cleanup_commands"] = 0
|
|
|
|
if name in crash_scenarios:
|
|
restored = json.loads((Path(paid["state_dir"]) / "paid-admission.json").read_text())
|
|
assert restored["initial_bootstraps"] == result["original_initial_bootstraps"], "crash recovery resumed initial bootstrap"
|
|
assert restored["create_reservations"] == result["original_create_reservations"], "crash recovery changed acquisition reservations"
|
|
assert restored["contracts"] == result["original_contracts"], "crash recovery changed contract accounting"
|
|
result["acquisition_slots"] = len(restored["create_reservations"])
|
|
result["initial_bootstrap_slots"] = len(restored["initial_bootstraps"])
|
|
assert not any(row["sequence"] > result["post_crash_sequence"]
|
|
and (row["method"] == "PUT" or row["path"] == "/api/v0/bundles/")
|
|
for row in ledger()), "crash recovery searched or acquired"
|
|
|
|
if name == "delayed-ambiguous-create":
|
|
initial = request("/__scripted__/state")
|
|
if initial["pending"] or len(initial["contracts"]) > 1:
|
|
assert json.loads(paid_path.read_text())["phase"] == "cleanup_only", "late create obligation was abandoned"
|
|
recovery, _ = cleanup("ambiguous-recovery")
|
|
result["cleanup_only"] = recovery
|
|
assert recovery["exit_status"] == 0, "observed ambiguous identities failed cleanup-only recovery"
|
|
assert any(row["path"] == "contract-visible" for row in ledger()), "delayed discovery was not exercised"
|
|
|
|
if name == "cleanup-only-recovery":
|
|
prior = request("/__scripted__/state")
|
|
assert len(prior["created"]) >= 3, "independent cleanup siblings were not constructed"
|
|
control["withhold_delete"] = 9000
|
|
control_path.write_text(json.dumps(control))
|
|
request("/__scripted__/restore", {"contracts": prior["created"]})
|
|
failed, cleanup_rows = cleanup("withheld-delete", seconds=1)
|
|
assert failed["exit_status"] != 0 and failed["elapsed_seconds"] < 5, "withheld deletion did not yield bounded failure"
|
|
assert json.loads(paid_path.read_text())["phase"] == "cleanup_only", "foreground expiry abandoned durable cleanup"
|
|
pending = request("/__scripted__/state")
|
|
assert set(map(int, pending["contracts"])) == set(UNRELATED_CONTRACTS) | {9000}, "withheld delete blocked healthy cleanup"
|
|
assert any(row["path"] == "delete-completed" and row["contract_id"] != 9000 for row in cleanup_rows)
|
|
request("/__scripted__/release-deletes", {})
|
|
recovered, recovery_rows = cleanup("cleanup-recovery")
|
|
assert recovered["exit_status"] != 0, "duplicate-label accounting must remain a failure after spending stops"
|
|
assert not any(row["method"] == "PUT" or row["path"] == "/api/v0/bundles/"
|
|
for row in recovery_rows), "cleanup-only recovery reopened acquisition"
|
|
result["withheld_delete"] = failed
|
|
result["cleanup_only"] = recovered
|
|
|
|
final = request("/__scripted__/state")
|
|
(directory / "provider-final-state.json").write_text(json.dumps(final, indent=2))
|
|
assert final["contracts"] == UNRELATED_WIRE_CONTRACTS and not final["pending"], f"{name}: exact attributable absence not proved"
|
|
rows = ledger()
|
|
assert not any(row["method"] == "DELETE"
|
|
and int(row["path"].rstrip("/").split("/")[-1]) in UNRELATED_CONTRACTS
|
|
for row in rows if row["path"].startswith("/api/v0/instances/")), f"{name}: unrelated resource touched"
|
|
created_ids = set(map(int, final["created"]))
|
|
if created_ids:
|
|
successful_listings = [row for row in rows if row["path"] == "/api/v0/instances/" and row.get("status") == 200]
|
|
assert successful_listings and not created_ids.intersection(successful_listings[-1]["returned_ids"]), "missing final typed provider absence census"
|
|
assert created_ids <= set(final["destroyed"]), "destroy accounting omitted an attributable identity"
|
|
if name == "rate-limited-listing":
|
|
rate_limited = [row for row in rows if row.get("status") == 429]
|
|
assert rate_limited, "listing rate limit not exercised"
|
|
for limited in rate_limited:
|
|
subsequent = next(row for row in rows if row["sequence"] > limited["sequence"]
|
|
and row["path"] == "/api/v0/instances/")
|
|
assert subsequent["elapsed_ns"] - limited["elapsed_ns"] >= limited["retry_after"] * 1e9, "Retry-After was violated"
|
|
if name == "slow-delete":
|
|
completions = [row for row in rows if row["path"] == "delete-completed"]
|
|
slow = next(row for row in completions if row["contract_id"] == 9000)
|
|
healthy = [row for row in completions if row["contract_id"] in (9500, 9501)]
|
|
assert len(healthy) == 2 and all(row["elapsed_ns"] < slow["elapsed_ns"] for row in healthy), "slow delete serialized healthy deletion"
|
|
late = next(row for row in completions if row["contract_id"] == 9700)
|
|
assert late["elapsed_ns"] < slow["elapsed_ns"], "pending delete blocked late-contract discovery"
|
|
for application_root in (directory / "campaign", directory / "private-state", directory / "private"):
|
|
for artifact in application_root.rglob("*"):
|
|
if artifact.is_file():
|
|
content = artifact.read_bytes()
|
|
assert not any(label.encode() in content for label in UNRELATED_CONTRACTS.values()), "application persisted unrelated account data"
|
|
result.update({"creates": len(creates), "created_contract_ids": sorted(created_ids),
|
|
"destroyed_contract_ids": sorted(final["destroyed"]), "exact_absence": True,
|
|
"unrelated_preserved": True,
|
|
"listing_requests": sum(row["path"] == "/api/v0/instances/" for row in rows)})
|
|
summary["scenarios"][name] = result
|
|
(output / "scripted-safety-summary.json").write_text(json.dumps(summary, indent=2))
|
|
except BaseException as error:
|
|
summary["state"] = "failed"
|
|
summary["failed_scenario"] = name
|
|
detail = str(error)
|
|
for value in credential_values(env):
|
|
detail = detail.replace(value.decode(), "[REDACTED]")
|
|
summary["error"] = detail
|
|
raise
|
|
finally:
|
|
try:
|
|
stop_scripted_cleanup_owners(directory)
|
|
finally:
|
|
os.close(ready_read)
|
|
provider.terminate()
|
|
try:
|
|
provider.wait(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
provider.kill()
|
|
provider.wait()
|
|
provider_log.close()
|
|
try:
|
|
scanned = scan_credentials(directory, env)
|
|
summary["scenarios"].setdefault(name, {})["credential_scan"] = scanned
|
|
except BaseException:
|
|
summary["state"] = "failed"
|
|
summary["failed_scenario"] = name
|
|
summary["credential_absence"] = False
|
|
raise
|
|
finally:
|
|
summary["elapsed_seconds"] = time.monotonic() - gate_started
|
|
(output / "scripted-safety-summary.json").write_text(json.dumps(summary, indent=2))
|
|
summary["credential_scan"] = scan_credentials(output, env)
|
|
summary["gate_b_coverage"] = {
|
|
"4_offer_selection": [
|
|
"duplicate-hosts", "insufficient-offers", "malformed-offers",
|
|
"cheapest-distinct-verified",
|
|
],
|
|
"5_admission": [
|
|
*preflight, *provider_admissions, "campaign-resource-overflow",
|
|
],
|
|
"6_shared_acquisition_bound": [
|
|
*crash_scenarios, "create-rejected", "delayed-ambiguous-create",
|
|
"retained-owner-loss",
|
|
],
|
|
"7_bootstrap_and_redeployment_bound": [
|
|
"crash-after-bootstrap-reservation", "retained-owner-loss",
|
|
],
|
|
"8_preparation_crash_accounting": [
|
|
"crash-before-create-reservation", "crash-after-create-reservation",
|
|
"runner-loss", "crash-after-contract-accounting",
|
|
"crash-after-bootstrap-reservation", "retained-owner-loss",
|
|
],
|
|
"9_cleanup_only_failure_recovery": [
|
|
"runner-loss", "owner-loss-automatic", "orchestrator-loss",
|
|
"workstation-loss", "retained-owner-loss", "cleanup-only-recovery",
|
|
],
|
|
"15_concurrent_exact_cleanup": [
|
|
"late-contract", "slow-delete", "cleanup-only-recovery",
|
|
],
|
|
"16_credential_redaction": ["create-rejected", "all-scenario-artifact-scans"],
|
|
}
|
|
summary["state"] = "passed"
|
|
summary["credential_absence"] = True
|
|
summary["elapsed_seconds"] = time.monotonic() - gate_started
|
|
(output / "scripted-safety-summary.json").write_text(json.dumps(summary, indent=2))
|
|
(output.parent / "vastai-gate-b-scripted.json").write_text(json.dumps(summary, indent=2))
|
|
print(json.dumps(summary, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if sys.argv[1] == "--gate":
|
|
try:
|
|
gate(*sys.argv[2:])
|
|
except Exception as error:
|
|
print(f"scripted safety gate failed ({type(error).__name__}); inspect its summary and logs", file=sys.stderr)
|
|
sys.exit(1)
|
|
else:
|
|
serve()
|