From df72b4ff60167e7a4e0d9a67b1d9a7bc37ba08df Mon Sep 17 00:00:00 2001 From: Zachery Aaron Shores-Chmielewski Date: Sat, 22 Aug 2026 21:18:25 +0400 Subject: [PATCH] chore(test): enforce complete local test barrier Deny workspace warnings and lint suppressions, consolidate Rust and Python coverage under cargo xtask test with 60-second per-test limits, and remove stale flaky, stateful, Docker, and orphaned test artifacts. --- .cargo/config.toml | 2 +- .config/nextest.toml | 2 + .github/workflows/myelin-properties.yml | 119 - Cargo.lock | 6 + Cargo.toml | 11 + apps/myelin/node-image/local_docker_e2e.sh | 54 - .../proptest-regressions/tests/e2e_vastai.txt | 5 - .../tests/tinygrad_device_bridge_backend.py | 305 --- apps/myelin/tests/stateful_vastai.rs | 2319 ----------------- crates/bindings/python/pyproject.toml | 2 +- crates/bindings/python/uv.lock | 41 + crates/bindings/wasm-runtime/Cargo.toml | 3 + crates/bindings/wasm-runtime/src/lib.rs | 6 + .../tests/namespace_host_read_guarantees.rs | 0 crates/engine/Cargo.toml | 3 + crates/engine/src/engine.rs | 8 +- crates/engine/tests/common/mod.rs | 9 +- crates/engine/tests/engine_contract.rs | 2 +- crates/engine/tests/engine_unit.rs | 7 +- crates/transport/Cargo.toml | 3 + crates/transport/src/codec.rs | 2 +- crates/transport/src/identity.rs | 2 +- crates/transport/tests/transport_api.rs | 4 +- src/actor.rs | 1 - src/admin.rs | 12 +- src/worker.rs | 9 +- tests/actor_lifecycle.rs | 2 +- tests/common/mod.rs | 2 - tests/core_extension_seams.rs | 4 - tests/lint_policy.rs | 205 ++ tests/message_delivery.rs | 2 +- tests/multicore.rs | 9 +- tests/proptest_runtime.rs | 2 +- tests/runtime_admin.rs | 2 +- tests/runtime_stress.rs | 2 +- tests/std_extension.rs | 2 +- tools/actor-control-flow-lint/Cargo.toml | 3 + .../fail-domain-capabilities/Cargo.lock | 7 + .../fail-domain-capabilities/src/lib.rs | 1 - .../fixtures/pass-actor-domain/Cargo.lock | 7 + tools/vastai/Cargo.toml | 3 + tools/vastai/src/config.rs | 75 +- tools/vastai/src/filters.rs | 4 +- tools/vastai/src/lease.rs | 83 +- tools/vastai/src/search.rs | 8 +- xtask/Cargo.toml | 3 + xtask/src/demo/control.rs | 7 +- xtask/src/demo/edge.rs | 3 +- xtask/src/demo/feed.rs | 86 +- xtask/src/demo/mod.rs | 74 +- xtask/src/demo/node.rs | 19 +- xtask/src/demo/provider.rs | 17 +- xtask/src/main.rs | 174 +- 53 files changed, 683 insertions(+), 3060 deletions(-) create mode 100644 .config/nextest.toml delete mode 100644 .github/workflows/myelin-properties.yml delete mode 100755 apps/myelin/node-image/local_docker_e2e.sh delete mode 100644 apps/myelin/proptest-regressions/tests/e2e_vastai.txt delete mode 100755 apps/myelin/src/tests/tinygrad_device_bridge_backend.py delete mode 100644 apps/myelin/tests/stateful_vastai.rs mode change 100755 => 100644 crates/data-plane/tests/namespace_host_read_guarantees.rs create mode 100644 tests/lint_policy.rs diff --git a/.cargo/config.toml b/.cargo/config.toml index 9ef77e8..304616c 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -3,4 +3,4 @@ rustc-workspace-wrapper = "tools/actor-control-flow-lint/rustc-wrapper.py" [alias] xtask = "run --package xtask --" -myelin-chat = "run --package xtask -- myelin-chat" +lint = "clippy --workspace --all-targets --all-features" diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 0000000..f6f2b4e --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,2 @@ +[profile.default] +slow-timeout = { period = "60s", terminate-after = 1 } diff --git a/.github/workflows/myelin-properties.yml b/.github/workflows/myelin-properties.yml deleted file mode 100644 index 3a83e8a..0000000 --- a/.github/workflows/myelin-properties.yml +++ /dev/null @@ -1,119 +0,0 @@ -name: Myelin properties - -on: - pull_request: - schedule: - - cron: "17 3 * * *" - workflow_dispatch: - -permissions: - contents: read - -jobs: - component-properties: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Install pinned Rust toolchain - run: rustup toolchain install nightly-2026-02-07 --profile minimal --component rustfmt --component rustc-dev --component rust-src --component llvm-tools-preview - - name: Check formatting - run: cargo fmt --all -- --check - - name: Check default workspace members - run: cargo check - - name: Check compiler-policy contracts - run: cargo test -p actor-control-flow-lint-tests --test contracts compiler_policy_contracts -- --exact - - name: Engine scheduling properties - run: | - cargo test -p swactor-engine --test engine_unit generated_actor_timers_and_completion_are_bounded -- --exact - cargo test -p swactor-engine --test engine_unit lifecycle_invariant_detects_injected_duplicate_completion -- --exact - cargo test -p swactor-engine --test engine_unit lifecycle_invariant_detects_injected_uncancelled_periodic_timer -- --exact - - name: Process adapter properties - run: | - cargo test -p swactor-process --lib operations::properties::generated_stream_observations_close_once_and_stay_closed -- --exact - cargo test -p swactor-process --lib operations::properties::generated_lifecycle_actions_make_stop_idempotent_and_exit_terminal -- --exact - cargo test -p swactor-process --lib operations::properties::generated_stop_notifications_are_delivered_at_most_once -- --exact - cargo test -p swactor-process --lib operations::properties::generated_stdin_commands_and_eof_notify_once -- --exact - cargo test -p swactor-process --lib operations::properties::property_invariants_reject_controlled_defects -- --exact - cargo test -p swactor-process --lib operations::properties::trivial_real_child_exit_has_a_hard_timeout -- --exact - - name: Myelin component properties (exclude job and reconciler tests) - run: | - cargo test -p myelin --lib orchestration::manual_control::tests::aggressive_random_event_stream_preserves_control_invariants -- --exact - cargo test -p myelin --lib orchestration::manual_control::tests::rental_free_end_to_end_sequences_converge -- --exact - cargo test -p myelin --lib orchestration::manual_control::tests::manual_actor_generated_public_actions_and_callbacks_are_bounded -- --exact - cargo test -p myelin --lib orchestration::manual_control::tests::fixed_helper_cardinality_invariant_detects_controlled_extra_spawn -- --exact - cargo test -p myelin --lib orchestration::manual_control::tests::callback_panic_reports_typed_failure_without_poisoning_work_actor -- --exact - cargo test -p myelin --lib orchestration::manual_control::tests::callback_panic_invariant_detects_controlled_unguarded_panic -- --exact - cargo test -p myelin --lib provisioning::tests::mock_vastai_handle_state_survives_random_create_and_stop_sequences -- --exact - cargo test -p myelin --lib provisioning::tests::docker_generated_attempt_lifecycles_are_idempotent_and_bounded -- --exact - cargo test -p myelin --lib provisioning::tests::docker_duplicate_resource_detector_rejects_controlled_fault -- --exact - cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::offer_status_classes_are_offers_or_typed_rejections -- --exact - cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::malformed_offer_bodies_are_typed_rejections -- --exact - cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::wrong_or_missing_offer_fields_are_typed_rejections -- --exact - cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::duplicate_offer_records_remain_explicit_values -- --exact - cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::provider_monitor_preserves_contract_identity_and_cardinality -- --exact - cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::provider_monitor_terminal_polling_stops_after_one_typed_outcome -- --exact - cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::provider_monitor_poll_stop_orderings_cease_polling -- --exact - cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::duplicate_terminal_detector_rejects_controlled_fault -- --exact - cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_output_lines_preserve_stream_and_protocol -- --exact - cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_malformed_protocol_is_data_not_poison -- --exact - cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_eof_orderings_stop_relay_and_actor -- --exact - cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_child_failures_have_typed_attempt_outcomes -- --exact - cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_timeout_is_typed_and_stops_polling -- --exact - cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_stop_orderings_emit_one_terminal_and_stop_all_actors -- --exact - cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::runtime_actors_generated_transitions_complete_once_on_one_worker -- --exact - cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::runtime_invariant_checker_rejects_duplicate_readiness_publication -- --exact - cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::helper_wait_generated_terminal_sequences_complete_once_on_one_worker -- --exact - cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::helper_invariant_checker_rejects_expected_output_after_terminal_error -- --exact - cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::stage_fetch_generated_observations_complete_once_on_one_worker -- --exact - cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::stage_invariant_checker_rejects_wrong_terminal_classification -- --exact - cargo test -p myelin --lib orchestration::app::serve_cluster_properties::serve_cluster_production_transitions_converge_once_without_growth -- --exact - cargo test -p myelin --lib orchestration::app::serve_cluster_properties::serve_cluster_lifecycle_invariants_reject_injected_duplicate_and_growth -- --exact - cargo test -p myelin --lib orchestration::control::properties::generated_http_bridge_sequences_terminate_without_control_actor_growth -- --exact - cargo test -p myelin --lib orchestration::control::properties::generated_duplicate_control_replies_deliver_first_once_and_remove_observer -- --exact - cargo test -p myelin --lib orchestration::control::properties::reply_observer_disappearance_returns_a_bounded_terminal_http_response -- --exact - cargo test -p myelin --lib orchestration::control::properties::http_bridge_invariant_rejects_a_controlled_duplicate_forward -- --exact - - name: Controlled E2E oracle contract - run: cargo test -p myelin --features test-support --test stateful_vastai e2e_oracle_rejects_controlled_lifecycle_faults -- --exact - - name: Dashboard bridge properties - run: | - cargo test -p dashboard --features demo-control --lib control::properties::generated_concurrent_bridge_commands_forward_once_and_shutdown -- --exact - cargo test -p dashboard --features demo-control --lib control::properties::bridge_invariant_rejects_a_controlled_duplicate_delivery -- --exact - cargo test -p dashboard --features demo-control --lib server::tests::generated_control_http_sequences_are_bounded_and_typed -- --exact - cargo test -p dashboard --features demo-control --lib server::tests::control_http_invariant_rejects_a_controlled_server_error -- --exact - - name: Demo actor properties - run: | - cargo test -p xtask --bin xtask demo::control::properties::generated_control_commands_forward_only_after_supervisor_registration -- --exact - cargo test -p xtask --bin xtask demo::control::properties::control_transition_oracle_rejects_duplicate_forwarding -- --exact - cargo test -p xtask --bin xtask demo::feed::properties::generated_supervisor_transitions_are_once_only_nonblocking_and_clean -- --exact - cargo test -p xtask --bin xtask demo::feed::properties::supervisor_transition_oracle_rejects_duplicate_identity_resources -- --exact - cargo test -p xtask --bin xtask demo::node::properties::generated_node_runtime_transitions_emit_heartbeats_and_stop_once -- --exact - cargo test -p xtask --bin xtask demo::node::properties::node_transition_oracle_rejects_duplicate_resources -- --exact - cargo test -p xtask --bin xtask demo::provider::properties::generated_process_reports_complete_exit_watchers_once_and_preserve_last_state -- --exact - cargo test -p xtask --bin xtask demo::provider::properties::process_relay_oracle_rejects_lost_exit -- --exact - cargo test -p xtask --bin xtask demo::properties::direct_binary_signal_smoke_has_a_hard_timeout -- --exact - - process-e2e: - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' - needs: component-properties - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@v4 - - name: Install pinned Rust toolchain - run: rustup toolchain install nightly-2026-02-07 --profile minimal --component rustc-dev --component rust-src --component llvm-tools-preview - - name: Stateful VastAI process E2E - env: - PROPTEST_CASES: "4" - run: | - set -o pipefail - mkdir -p artifacts - cargo test -p myelin --features test-support --test stateful_vastai stateful_vastai_dashboard_control_survives_restarts -- --ignored --exact --nocapture 2>&1 | tee artifacts/stateful-vastai.log - - name: Preserve E2E failure artifacts - if: failure() - uses: actions/upload-artifact@v4 - with: - name: stateful-vastai-failure-${{ github.run_id }} - path: | - artifacts/stateful-vastai.log - apps/myelin/proptest-regressions/tests/e2e_vastai.txt - if-no-files-found: warn diff --git a/Cargo.lock b/Cargo.lock index 2c0dcab..e4c37d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -962,7 +962,9 @@ dependencies = [ "futures-lite", "libc", "parking_lot", + "proptest", "serde", + "serde_json", "sha2 0.10.9", "swactor", "swactor-engine", @@ -4335,9 +4337,11 @@ dependencies = [ "getrandom 0.2.17", "loom", "parking_lot", + "proc-macro2", "proptest", "proptest-state-machine", "serde", + "syn", "tracing", "web-time 0.2.4", ] @@ -4489,11 +4493,13 @@ name = "telemetry" version = "0.1.0" dependencies = [ "crossbeam-channel", + "futures-channel", "iroh", "libc", "serde", "serde_json", "swactor", + "swactor-engine", "swactor-transport", ] diff --git a/Cargo.toml b/Cargo.toml index ef71ae1..10cdcad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,4 +75,15 @@ criterion = { version = "0.5", features = ["html_reports"] } proptest = "1" proptest-state-machine = "0.3" loom = "0.7" +proc-macro2 = "1" +syn = { version = "2", features = ["full", "visit"] } + +[workspace.lints.rust] +warnings = "deny" + +[workspace.lints.clippy] +allow_attributes = "forbid" + +[lints] +workspace = true diff --git a/apps/myelin/node-image/local_docker_e2e.sh b/apps/myelin/node-image/local_docker_e2e.sh deleted file mode 100755 index bb3631b..0000000 --- a/apps/myelin/node-image/local_docker_e2e.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -BASE_IMAGE=${BASE_IMAGE:-myelin-node-base:cuda12.6} -IMAGE=${IMAGE:-myelin-node:latest} -CONTAINER=${CONTAINER:-myelin-node-e2e-$$} -GPUS=${MYELIN_CUDA_GPUS:-all} -PROMPT=${MYELIN_NODE_SELF_TEST_PROMPT:-ping} -TIMEOUT_SECS=${MYELIN_NODE_E2E_TIMEOUT_SECS:-1800} -FRAME_LOG=${MYELIN_TELEMETRY_FRAME_LOG:-/var/log/myelin-telemetry.ndjson} - -cleanup() { - docker rm -f "$CONTAINER" >/dev/null 2>&1 || true -} -trap cleanup EXIT - -cargo build --release -p myelin --bin myelin-worker -docker build -f apps/myelin/node-image/Dockerfile.base -t "$BASE_IMAGE" . -docker build -f apps/myelin/node-image/Dockerfile --build-arg BASE_IMAGE="$BASE_IMAGE" -t "$IMAGE" . - -docker run -d \ - --name "$CONTAINER" \ - --gpus "$GPUS" \ - -e MYELIN_NODE_SELF_TEST_PROMPT="$PROMPT" \ - -e MYELIN_NODE_MAX_RUNTIME_SECS=1 \ - -e MYELIN_SELF_TEST_MAX_TOKENS="${MYELIN_SELF_TEST_MAX_TOKENS:-1}" \ - -e MYELIN_MODEL_CACHE_DIR=/var/cache/myelin-models \ - -e MYELIN_TELEMETRY_FRAME_LOG="$FRAME_LOG" \ - ${HF_TOKEN:+-e HF_TOKEN="$HF_TOKEN"} \ - "$IMAGE" >/dev/null - -deadline=$((SECONDS + TIMEOUT_SECS)) -while (( SECONDS < deadline )); do - logs=$(docker logs "$CONTAINER" 2>&1 || true) - if grep -q '"type":"ready"' <<<"$logs" && grep -q '"type":"self_test_completed"' <<<"$logs"; then - frames=$(docker exec "$CONTAINER" cat "$FRAME_LOG" 2>/dev/null || true) - if grep -q '"channel":"myelin.node.ready"' <<<"$frames" && - grep -q '"channel":"myelin.worker.weights"' <<<"$frames" && - grep -q '"channel":"myelin.worker.prompt"' <<<"$frames"; then - printf '%s\n' "$logs" - printf '%s\n' "$frames" - exit 0 - fi - fi - if grep -q 'WorkerFatal\|myelin-node: .*failed\|ModelLoadFailed\|GgufDownloadFailed' <<<"$logs"; then - printf '%s\n' "$logs" >&2 - exit 1 - fi - sleep 5 -done - -docker logs "$CONTAINER" 2>&1 || true -echo "myelin-node Docker E2E timed out after ${TIMEOUT_SECS}s" >&2 -exit 1 diff --git a/apps/myelin/proptest-regressions/tests/e2e_vastai.txt b/apps/myelin/proptest-regressions/tests/e2e_vastai.txt deleted file mode 100644 index 511ca75..0000000 --- a/apps/myelin/proptest-regressions/tests/e2e_vastai.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Seeds for stateful VastAI E2E failures. Proptest replays these before generated cases. -cc 26c3cd5944cb25de70a984389373e7380a0c6e6ba1c8f3e6eff5a0f78dbef168 # current-strategy replay; E2eCase is derived from e2e_case() -cc 5efb0a6c5c348475c859a2c4267d5177aaecd5adefcc5e8b7fc2e6d728deecd1 # shrinks to case = E2eCase { seed: 7099259626237328177, node_seed: 245, kill_mask: 24, offer_offset: 3, actions: [ConcurrentQueries, Search { count: 2 }, Restart { mode: FlushSafeAbrupt }, Query, Kill { node_slot: 218, command_slot: 5 }, Query, Provision { command_slot: 5, use_searched_offers: true }, EndpointProbe { node_slot: 217 }, Restart { mode: Graceful }, Flush, Kill { node_slot: 217, command_slot: 5 }] } -cc 6d005687858520ed65af20ec5b2de056efcda15771b6420d7f67e3cb6c246d5b # shrinks to case = E2eCase { seed: 17649392557414661864, node_seed: 249, kill_mask: 159, offer_offset: 5, actions: [Kill { node_slot: 88, command_slot: 242 }, Flush, ConcurrentQueries, EndpointProbe { node_slot: 88 }, Query, Search { count: 2 }, Provision { command_slot: 242, use_searched_offers: true }, Restart { mode: FlushSafeAbrupt }, Kill { node_slot: 89, command_slot: 242 }, Query, Restart { mode: Graceful }] } -cc 2e36f51e7cb347b571ac838f81638748f8414a32ec91cb9f4b2730363d72be61 # shrinks to case = E2eCase { seed: 13301618846512983428, node_seed: 99, kill_mask: 30, offer_offset: 2, actions: [Query, Restart { mode: FlushSafeAbrupt }, Kill { node_slot: 136, command_slot: 187 }, ConcurrentQueries, Restart { mode: Graceful }, Query, EndpointProbe { node_slot: 136 }, Kill { node_slot: 137, command_slot: 187 }, Provision { command_slot: 187, use_searched_offers: false }, Flush, Search { count: 0 }] } diff --git a/apps/myelin/src/tests/tinygrad_device_bridge_backend.py b/apps/myelin/src/tests/tinygrad_device_bridge_backend.py deleted file mode 100755 index 79666c3..0000000 --- a/apps/myelin/src/tests/tinygrad_device_bridge_backend.py +++ /dev/null @@ -1,305 +0,0 @@ -#!/usr/bin/env python3 -"""Tinygrad-backed device bridge probe for Rust Myelin bridge tests. - -Line-delimited JSON control only. Payload bytes live in the arena file whose -path Rust passes during initialize. -""" - -from __future__ import annotations - -import json -import os -import struct -import sys -from dataclasses import dataclass -from typing import Any - -try: - from tinygrad import Tensor # type: ignore -except Exception as exc: # pragma: no cover - exercised from Rust process tests - print( - json.dumps( - { - "type": "worker_fatal", - "reason": "tinygrad_unavailable", - "message": str(exc), - } - ), - flush=True, - ) - raise SystemExit(2) - - -@dataclass -class DeviceObject: - dtype: str - shape: str - extent: int - values: list[int] - tensor: Any - - -@dataclass -class PendingCopy: - kind: str - copy_id: int - handle_id: int - host_offset: int - device_offset: int - length: int - - -arena_path: str | None = None -arena_bytes = 0 -generation = 0 -objects: dict[int, DeviceObject] = {} -pending_copies: dict[int, PendingCopy] = {} -fail_next: str | None = None - - -def emit(obj: dict[str, Any]) -> None: - print(json.dumps(obj, separators=(",", ":")), flush=True) - - -def backend_error(reason: str) -> None: - emit({"type": "backend_error", "reason": reason}) - - -def fatal(reason: str, message: str) -> None: - emit({"type": "worker_fatal", "reason": reason, "message": message}) - - -def require_arena() -> str: - if arena_path is None: - raise RuntimeError("arena is not initialized") - return arena_path - - -def consume_failure(reason: str) -> bool: - global fail_next - if fail_next == reason: - fail_next = None - backend_error(reason) - return True - return False - - -def check_u32_range(offset: int, length: int) -> None: - if offset < 0 or length < 0 or offset % 4 != 0 or length % 4 != 0: - raise ValueError("u32 ranges must be non-negative and 4-byte aligned") - - -def tensor_from_values(values: list[int]) -> Any: - try: - return Tensor(values, dtype="uint32").realize() - except Exception: - return Tensor(values, dtype="int32").realize() - - -def allocate_tensor(dtype: str, extent: int) -> tuple[list[int], Any]: - if dtype == "u32": - if extent % 4 != 0: - raise ValueError("u32 extent must be 4-byte aligned") - values = [0] * (extent // 4) - return values, tensor_from_values(values) - if dtype == "f16": - values = [0] * (extent // 2) - return values, Tensor(values, dtype="float16").realize() - raise ValueError(f"unsupported dtype {dtype!r}") - - -def update_tensor(obj: DeviceObject) -> None: - if obj.dtype == "u32": - obj.tensor = tensor_from_values(obj.values) - elif obj.dtype == "f16": - obj.tensor = Tensor(obj.values, dtype="float16").realize() - else: - raise ValueError(f"unsupported dtype {obj.dtype!r}") - - -def materialized_values(obj: DeviceObject) -> list[int]: - return [int(v) for v in obj.tensor.tolist()] - - -def read_arena(offset: int, length: int) -> bytes: - path = require_arena() - with open(path, "rb", buffering=0) as f: - f.seek(offset) - data = f.read(length) - if len(data) != length: - raise EOFError("short arena read") - return data - - -def write_arena(offset: int, data: bytes) -> None: - path = require_arena() - with open(path, "r+b", buffering=0) as f: - f.seek(offset) - f.write(data) - f.flush() - - -def perform_host_to_device(copy: PendingCopy) -> None: - obj = objects[copy.handle_id] - if obj.dtype != "u32": - raise ValueError("payload copy is implemented for u32 test objects only") - check_u32_range(copy.device_offset, copy.length) - payload = read_arena(copy.host_offset, copy.length) - words = list(struct.unpack("<" + "I" * (copy.length // 4), payload)) - start = copy.device_offset // 4 - end = start + len(words) - if end > len(obj.values): - raise ValueError("device range out of bounds") - obj.values[start:end] = words - update_tensor(obj) - - -def perform_device_to_host(copy: PendingCopy) -> None: - obj = objects[copy.handle_id] - if obj.dtype != "u32": - raise ValueError("payload copy is implemented for u32 test objects only") - check_u32_range(copy.device_offset, copy.length) - values = materialized_values(obj) - start = copy.device_offset // 4 - end = start + (copy.length // 4) - if end > len(values): - raise ValueError("device range out of bounds") - payload = struct.pack("<" + "I" * (end - start), *values[start:end]) - write_arena(copy.host_offset, payload) - - -def perform_copy(copy: PendingCopy) -> None: - if copy.kind == "host_to_device": - perform_host_to_device(copy) - elif copy.kind == "device_to_host": - perform_device_to_host(copy) - else: - raise ValueError(f"unknown copy kind {copy.kind!r}") - - -def handle(req: dict[str, Any]) -> bool: - global arena_path, arena_bytes, generation, fail_next - - typ = req.get("type") - if typ == "initialize": - arena_path = str(req["arena_path"]) - arena_bytes = int(req["arena_bytes"]) - generation = int(req["generation"]) - with open(arena_path, "r+b", buffering=0) as f: - f.truncate(arena_bytes) - emit({"type": "worker_ready", "generation": generation}) - return True - - if typ == "alloc": - if consume_failure("allocation_failed"): - return True - handle_id = int(req["handle_id"]) - dtype = str(req["dtype"]) - shape = str(req["shape"]) - extent = int(req["extent"]) - values, tensor = allocate_tensor(dtype, extent) - objects[handle_id] = DeviceObject(dtype=dtype, shape=shape, extent=extent, values=values, tensor=tensor) - emit({"type": "allocated", "handle_id": handle_id}) - return True - - if typ in ("host_to_device", "device_to_host"): - if consume_failure("copy_failed"): - return True - copy = PendingCopy( - kind=typ, - copy_id=int(req["copy_id"]), - handle_id=int(req["handle_id"]), - host_offset=int(req["host_offset"]), - device_offset=int(req["device_offset"]), - length=int(req["len"]), - ) - if copy.handle_id not in objects: - backend_error("copy_failed") - return True - if bool(req.get("defer", False)): - pending_copies[copy.copy_id] = copy - emit({"type": "copy_started", "copy_id": copy.copy_id}) - else: - perform_copy(copy) - emit({"type": "copy_completed", "copy_id": copy.copy_id}) - return True - - if typ == "complete_copy": - copy_id = int(req["copy_id"]) - copy = pending_copies.pop(copy_id, None) - if copy is None: - backend_error("copy_failed") - return True - perform_copy(copy) - emit({"type": "copy_completed", "copy_id": copy_id}) - return True - - if typ == "wrap_for_tinygrad": - if consume_failure("invalid_view"): - return True - handle_id = int(req["handle_id"]) - obj = objects.get(handle_id) - if obj is None: - backend_error("invalid_view") - return True - dtype = str(req["dtype"]) - shape = str(req["shape"]) - if obj.dtype != dtype or obj.shape != shape: - backend_error("invalid_view") - return True - # Force materialization at view time so success depends on live tensor state. - _ = obj.tensor.tolist() - emit({"type": "view", "handle_id": handle_id, "dtype": dtype, "shape": shape}) - return True - - if typ == "free": - handle_id = int(req["handle_id"]) - if handle_id not in objects: - backend_error("invalid_view") - return True - del objects[handle_id] - emit({"type": "freed", "handle_id": handle_id}) - return True - - if typ == "restart": - generation = int(req["generation"]) - objects.clear() - pending_copies.clear() - fail_next = None - emit({"type": "worker_ready", "generation": generation}) - return True - - if typ == "fail_next": - fail_next = str(req["failure"]) - emit({"type": "ok"}) - return True - - if typ == "shutdown": - emit({"type": "worker_stopped"}) - return False - - fatal("protocol_error", f"unknown command type {typ!r}") - return False - - -def main() -> int: - for line in sys.stdin: - line = line.strip() - if not line: - continue - try: - req = json.loads(line) - if not isinstance(req, dict): - raise ValueError("request must be an object") - if not handle(req): - return 0 - except SystemExit: - raise - except Exception as exc: - fatal("backend_exception", str(exc)) - return 2 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/apps/myelin/tests/stateful_vastai.rs b/apps/myelin/tests/stateful_vastai.rs deleted file mode 100644 index d365e30..0000000 --- a/apps/myelin/tests/stateful_vastai.rs +++ /dev/null @@ -1,2319 +0,0 @@ -#![cfg(target_os = "linux")] - -use std::collections::{BTreeMap, BTreeSet}; -use std::fs::OpenOptions; -use std::io::{BufRead, BufReader, Read, Write}; -use std::net::{TcpListener, TcpStream}; -use std::os::unix::net::UnixStream; -use std::path::PathBuf; -use std::process::{Child, Command, Stdio}; -use std::sync::OnceLock; -use std::time::{Duration, Instant}; - -use proptest::prelude::*; -use proptest::test_runner::FileFailurePersistence; -use serde_json::{Value, json}; -use swactor_process::{ - child_kill, child_try_wait, child_wait, command_spawn, find_process_identities_by_environment, - request_child_termination, terminate_process_group, -}; -use swactor_vastai::test_http::{TestHttpRoute, TestHttpServer}; - -const CASE_DEADLINE: Duration = Duration::from_secs(10); -const SUITE_DEADLINE: Duration = Duration::from_secs(30); -const POLL: Duration = Duration::from_millis(25); -const CENSUS_STABILITY: Duration = Duration::from_millis(25); -const ACTORS_PER_NODE_LIMIT: u64 = 32; -const OFFER_COUNT: usize = 8; - -static SUITE_STARTED: OnceLock = OnceLock::new(); -std::thread_local! { - static OBSERVED_HTTP_FAILURES: std::cell::RefCell> = - const { std::cell::RefCell::new(Vec::new()) }; -} - -fn http_failure_checkpoint() -> usize { - OBSERVED_HTTP_FAILURES.with(|failures| failures.borrow().len()) -} - -fn discard_http_failures_since(checkpoint: usize) { - OBSERVED_HTTP_FAILURES.with(|failures| failures.borrow_mut().truncate(checkpoint)); -} - -fn record_http_failure(failure: String) { - OBSERVED_HTTP_FAILURES.with(|failures| failures.borrow_mut().push(failure)); -} - -fn observed_http_failures() -> Vec { - OBSERVED_HTTP_FAILURES.with(|failures| failures.borrow().clone()) -} - -#[derive(Clone, Copy, Debug)] -enum RestartMode { - Graceful, - FlushSafeAbrupt, -} - -#[derive(Clone, Debug)] -enum ExternalAction { - Search { - count: usize, - }, - Provision { - command_slot: u8, - use_searched_offers: bool, - }, - Query, - Kill { - node_slot: u8, - command_slot: u8, - }, - Flush, - Restart { - mode: RestartMode, - }, - EndpointProbe { - node_slot: u8, - }, - ConcurrentQueries, -} - -#[derive(Clone, Debug)] -struct E2eCase { - seed: u64, - node_seed: u8, - kill_mask: u8, - offer_offset: usize, - actions: Vec, -} - -fn e2e_case() -> impl Strategy { - ( - any::(), - any::(), - any::(), - 0_usize..OFFER_COUNT, - any::(), - any::(), - any::(), - any::(), - proptest::collection::vec(any::(), 11), - ) - .prop_map( - |( - seed, - node_seed, - kill_mask, - offer_offset, - invalid_search, - use_searched_offers, - node_slot, - command_slot, - ordering, - )| { - let mut keyed_actions = vec![ - ExternalAction::Search { - count: if invalid_search { 0 } else { 2 }, - }, - ExternalAction::Provision { - command_slot, - use_searched_offers, - }, - ExternalAction::Query, - ExternalAction::Kill { - node_slot, - command_slot, - }, - ExternalAction::Flush, - ExternalAction::Restart { - mode: RestartMode::Graceful, - }, - ExternalAction::EndpointProbe { node_slot }, - ExternalAction::ConcurrentQueries, - ExternalAction::Query, - ExternalAction::Kill { - node_slot: node_slot.wrapping_add(1), - command_slot, - }, - ExternalAction::Restart { - mode: RestartMode::FlushSafeAbrupt, - }, - ] - .into_iter() - .enumerate() - .map(|(index, action)| ((ordering[index], index), action)) - .collect::>(); - keyed_actions.sort_by_key(|(key, _)| *key); - E2eCase { - seed, - node_seed, - kill_mask, - offer_offset, - actions: keyed_actions - .into_iter() - .map(|(_, action)| action) - .collect(), - } - }, - ) -} - -fn e2e_proptest_config() -> ProptestConfig { - let has_case_override = std::env::var_os("PROPTEST_CASES").is_some(); - let mut config = ProptestConfig::default(); - if !has_case_override { - config.cases = 4; - } - config.failure_persistence = Some(Box::new(FileFailurePersistence::Direct(concat!( - env!("CARGO_MANIFEST_DIR"), - "/proptest-regressions/tests/e2e_vastai.txt" - )))); - config.max_shrink_iters = 0; - config -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -struct ActorCensus { - actors: u64, - orchestrator_actors: u64, - worker_actors: u64, - poisoned: u64, - type_mismatches: u64, - mailbox_depth: u64, - orchestrator_by_type: BTreeMap, - workers_by_stream: BTreeMap>, -} - -#[derive(Clone, Debug)] -struct OracleObservation { - expected_survivors: BTreeSet, - expected_stopped: BTreeSet, - running_after_restart: BTreeSet, - stopped_after_restart: BTreeSet, - reachable_nodes: BTreeSet, - provider_resources: Vec, - provider_created: BTreeSet, - provider_destroyed: BTreeSet, - selected_offers: BTreeMap, - searched_offers: BTreeSet, - actor_before_replays: ActorCensus, - actor_after_replays: ActorCensus, - actor_initial: ActorCensus, - actor_after_teardown: ActorCensus, - terminal_record_counts: BTreeMap, - unexpected_process_exits: Vec, - http_failures: Vec, - teardown_resources: Vec, - labeled_temp_resources: Vec, -} - -fn validate_oracle(observation: &OracleObservation) -> Result<(), String> { - if observation - .terminal_record_counts - .values() - .any(|count| *count != 1) - { - return Err(format!( - "accepted reply did not have exactly one terminal record: {:?}", - observation.terminal_record_counts - )); - } - if !observation.unexpected_process_exits.is_empty() { - return Err(format!( - "unexpected process exit: {:?}", - observation.unexpected_process_exits - )); - } - if !observation.http_failures.is_empty() { - return Err(format!( - "HTTP failure or disconnect: {:?}", - observation.http_failures - )); - } - if observation.running_after_restart != observation.expected_survivors { - return Err(format!( - "restart-state loss: expected running {:?}, observed {:?}", - observation.expected_survivors, observation.running_after_restart - )); - } - if observation.stopped_after_restart != observation.expected_stopped { - return Err(format!( - "destroy set mismatch: expected stopped {:?}, observed {:?}", - observation.expected_stopped, observation.stopped_after_restart - )); - } - if observation.reachable_nodes != observation.expected_survivors { - return Err(format!( - "survivor reachability mismatch: expected {:?}, observed {:?}", - observation.expected_survivors, observation.reachable_nodes - )); - } - let provider_set = observation - .provider_resources - .iter() - .copied() - .collect::>(); - if provider_set.len() != observation.provider_resources.len() { - return Err(format!( - "duplicate provider resources: {:?}", - observation.provider_resources - )); - } - if provider_set != observation.expected_survivors { - return Err(format!( - "provider ledger mismatch: expected {:?}, observed {:?}", - observation.expected_survivors, provider_set - )); - } - let expected_all = observation - .expected_survivors - .union(&observation.expected_stopped) - .copied() - .collect::>(); - if observation.provider_created != expected_all { - return Err(format!( - "provider create ledger mismatch: expected {:?}, observed {:?}", - expected_all, observation.provider_created - )); - } - if observation.provider_destroyed != observation.expected_stopped { - return Err(format!( - "provider destroy ledger mismatch: expected {:?}, observed {:?}", - observation.expected_stopped, observation.provider_destroyed - )); - } - if observation.selected_offers.len() - != observation - .expected_survivors - .len() - .saturating_add(observation.expected_stopped.len()) - || observation - .selected_offers - .values() - .any(|offer| !observation.searched_offers.contains(offer)) - { - return Err(format!( - "selected offer did not come from search response: selected={:?}, searched={:?}", - observation.selected_offers, observation.searched_offers - )); - } - if observation.actor_initial.poisoned != 0 - || observation.actor_before_replays.poisoned != 0 - || observation.actor_after_replays.poisoned != 0 - || observation.actor_after_teardown.poisoned != 0 - { - return Err(format!( - "actor poisoning observed: initial={:?}, before={:?}, after={:?}, teardown={:?}", - observation.actor_initial, - observation.actor_before_replays, - observation.actor_after_replays, - observation.actor_after_teardown, - )); - } - if observation.actor_initial.type_mismatches != 0 - || observation.actor_before_replays.type_mismatches != 0 - || observation.actor_after_replays.type_mismatches != 0 - || observation.actor_after_teardown.type_mismatches != 0 - { - return Err(format!( - "actor message type mismatch observed: initial={:?}, before={:?}, after={:?}, teardown={:?}", - observation.actor_initial, - observation.actor_before_replays, - observation.actor_after_replays, - observation.actor_after_teardown, - )); - } - for (phase, census) in [ - ("before replay", &observation.actor_before_replays), - ("after replay", &observation.actor_after_replays), - ] { - let typed_worker_count = census - .workers_by_stream - .values() - .flat_map(|types| types.iter()) - .map(|(actor_type, count)| { - if actor_type == "" { - Err(format!( - "{phase} worker census contains an unknown actor type" - )) - } else { - Ok(*count) - } - }) - .collect::, _>>()? - .into_iter() - .sum::(); - if census.workers_by_stream.len() != observation.expected_survivors.len() - || typed_worker_count != census.worker_actors - { - return Err(format!( - "{phase} worker census is incomplete: expected_nodes={}, census={census:?}", - observation.expected_survivors.len() - )); - } - } - let mut before_types = observation - .actor_before_replays - .orchestrator_by_type - .clone(); - let mut after_types = observation.actor_after_replays.orchestrator_by_type.clone(); - before_types.remove("myelin::orchestration::control::ControlReplyObserver"); - after_types.remove("myelin::orchestration::control::ControlReplyObserver"); - if observation.actor_before_replays.actors != observation.actor_after_replays.actors - || observation.actor_before_replays.orchestrator_actors - != observation.actor_after_replays.orchestrator_actors - || observation.actor_before_replays.worker_actors - != observation.actor_after_replays.worker_actors - || observation.actor_before_replays.mailbox_depth - != observation.actor_after_replays.mailbox_depth - || observation.actor_before_replays.workers_by_stream - != observation.actor_after_replays.workers_by_stream - || before_types != after_types - { - return Err(format!( - "steady-state actor census changed under replay: before={:?}, after={:?}", - observation.actor_before_replays, observation.actor_after_replays - )); - } - let live_node_count = observation.expected_survivors.len() as u64; - if observation.actor_before_replays.worker_actors - > live_node_count.saturating_mul(ACTORS_PER_NODE_LIMIT) - { - return Err(format!( - "per-node actor bound exceeded: live_nodes={live_node_count}, census={:?}", - observation.actor_before_replays - )); - } - if observation.actor_after_teardown.orchestrator_actors - != observation.actor_initial.orchestrator_actors - || observation.actor_after_teardown.mailbox_depth != 0 - { - return Err(format!( - "actor census did not return to baseline: initial={:?}, teardown={:?}", - observation.actor_initial, observation.actor_after_teardown - )); - } - if !observation.teardown_resources.is_empty() || !observation.labeled_temp_resources.is_empty() - { - return Err(format!( - "teardown leak: provider={:?}, labeled_temp={:?}", - observation.teardown_resources, observation.labeled_temp_resources - )); - } - Ok(()) -} - -struct ScenarioHarness { - state_dir: PathBuf, - provider_url: String, - dashboard_port: u16, - run_id: u64, - provider_ledger_path: PathBuf, - restart: usize, - child: Option, - log_paths: Vec, - unexpected_exits: Vec, -} - -impl ScenarioHarness { - fn new(state_dir: PathBuf, provider_url: String, dashboard_port: u16, run_id: u64) -> Self { - let provider_ledger_path = state_dir.join("mock-vastai-lifecycle.jsonl"); - Self { - state_dir, - provider_url, - dashboard_port, - run_id, - provider_ledger_path, - restart: 0, - child: None, - log_paths: Vec::new(), - unexpected_exits: Vec::new(), - } - } - - fn start(&mut self) -> Result<(), String> { - if self.child.is_some() { - return Err("orchestrator is already running".to_owned()); - } - let log_path = self - .state_dir - .join(format!("orchestrator-{}.log", self.restart)); - let stdout = OpenOptions::new() - .create(true) - .append(true) - .open(&log_path) - .map_err(|error| format!("open {}: {error}", log_path.display()))?; - let stderr = stdout - .try_clone() - .map_err(|error| format!("clone {}: {error}", log_path.display()))?; - let mut command = Command::new(env!("CARGO_BIN_EXE_myelin-orchestrator")); - command - .current_dir(&self.state_dir) - .args([ - "--provider", - "vastai", - "--vastai-provisioning", - "mock", - "--dashboard", - "--state-dir", - ]) - .arg(&self.state_dir) - .args(["--run-id", &self.run_id.to_string()]) - .env("VASTAI_BASE_URL", &self.provider_url) - .env("VAST_API_KEY", "stateful-e2e-secret") - .env("MYELIN_DASHBOARD_PORT", self.dashboard_port.to_string()) - .env("MYELIN_MOCK_VASTAI_LEDGER_PATH", &self.provider_ledger_path) - .env("MYELIN_VASTAI_POLL_INTERVAL_SECS", "1") - .env("MYELIN_IROH_RELAY_MODE", "disabled") - .env("RUST_BACKTRACE", "1") - .stdin(Stdio::null()) - .stdout(Stdio::from(stdout)) - .stderr(Stdio::from(stderr)); - let child = command_spawn(&mut command) - .map_err(|error| format!("spawn Cargo-built Myelin orchestrator: {error}"))?; - self.child = Some(child); - self.log_paths.push(log_path); - self.restart += 1; - Ok(()) - } - fn ensure_running(&mut self) -> Result<(), String> { - let child = self - .child - .as_mut() - .ok_or_else(|| "orchestrator process is absent".to_owned())?; - if let Some(status) = child_try_wait(child) - .map_err(|error| format!("inspect live orchestrator process: {error}"))? - { - let failure = format!("orchestrator exited unexpectedly with {status}"); - self.unexpected_exits.push(failure.clone()); - return Err(failure); - } - Ok(()) - } - - fn restart_gracefully(&mut self, deadline: Instant) -> Result<(), String> { - self.stop_child_gracefully(deadline)?; - self.start() - } - - fn restart_abruptly(&mut self) -> Result<(), String> { - self.stop_child_abruptly()?; - self.start() - } - - fn stop_child_gracefully(&mut self, deadline: Instant) -> Result<(), String> { - let Some(mut child) = self.child.take() else { - return Ok(()); - }; - if child_try_wait(&mut child) - .map_err(|error| format!("inspect orchestrator process: {error}"))? - .is_none() - { - request_child_termination(&child) - .map_err(|error| format!("request graceful orchestrator stop: {error}"))?; - } - loop { - if let Some(status) = child_try_wait(&mut child) - .map_err(|error| format!("inspect graceful orchestrator stop: {error}"))? - { - if status.success() { - return Ok(()); - } - return Err(format!( - "graceful orchestrator stop exited with {status}; logs={}", - self.logs() - )); - } - if Instant::now() >= deadline { - let _ = child_kill(&mut child); - let _ = child_wait(&mut child); - return Err(format!( - "graceful orchestrator stop exceeded case deadline; logs={}", - self.logs() - )); - } - std::thread::sleep(POLL); - } - } - - fn stop_child_abruptly(&mut self) -> Result<(), String> { - let Some(mut child) = self.child.take() else { - return Ok(()); - }; - if child_try_wait(&mut child) - .map_err(|error| format!("inspect orchestrator process: {error}"))? - .is_none() - { - child_kill(&mut child) - .map_err(|error| format!("kill orchestrator process: {error}"))?; - } - child_wait(&mut child) - .map_err(|error| format!("wait for orchestrator process: {error}"))?; - Ok(()) - } - - fn base_url(&self) -> String { - format!("http://127.0.0.1:{}", self.dashboard_port) - } - - fn logs(&self) -> String { - let mut paths = self.log_paths.clone(); - if let Ok(entries) = std::fs::read_dir(self.state_dir.join("process-output")) { - paths.extend(entries.filter_map(|entry| entry.ok().map(|entry| entry.path()))); - } - paths.sort(); - paths - .iter() - .map(|path| { - std::fs::read_to_string(path) - .map(|text| format!("\n--- {} ---\n{text}", path.display())) - .unwrap_or_else(|error| { - format!("\n--- {} unavailable: {error} ---", path.display()) - }) - }) - .collect() - } - - fn cleanup_workers(&self) { - let environment = [("MYELIN_RUN_ID".to_owned(), self.run_id.to_string())]; - let Ok(processes) = find_process_identities_by_environment(&environment, true) else { - return; - }; - for process in processes { - let _ = terminate_process_group( - &process, - Duration::from_secs(2), - Duration::from_millis(20), - ); - } - } - fn remove_worker_sockets(&self, nodes: &BTreeSet) -> Result<(), String> { - for node_id in nodes { - let path = node_socket(self.run_id, *node_id); - match std::fs::remove_file(&path) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => { - return Err(format!( - "remove labeled worker socket {}: {error}", - path.display() - )); - } - } - } - Ok(()) - } -} - -impl Drop for ScenarioHarness { - fn drop(&mut self) { - let _ = self.stop_child_abruptly(); - self.cleanup_workers(); - let possible_nodes = BTreeSet::from([1, 2]); - let _ = self.remove_worker_sockets(&possible_nodes); - } -} - -fn production_offers() -> Value { - let offers = (0..OFFER_COUNT) - .map(|index| { - json!({ - "id": 8_675_300_u64 + index as u64, - "gpu_name": "RTX 4090", - "num_gpus": 1, - "gpu_ram": 24_576.0, - "dph_total": 0.20 + index as f64 / 100.0, - "host_id": 90_000_u64 + index as u64, - "compute_cap": 890, - "verification": "verified", - "reliability2": 0.995, - "inet_down": 1_000.0, - "inet_up": 800.0, - "internet_down_cost_per_tb": 1.5, - "internet_up_cost_per_tb": 2.5, - "geolocation": "US", - "disk_bw": 2_000.0, - "duration": 86_400.0, - "rentable": true - }) - }) - .collect::>(); - json!({"offers": offers}) -} - -fn reserve_port() -> Result { - let listener = TcpListener::bind(("127.0.0.1", 0)) - .map_err(|error| format!("reserve dashboard port: {error}"))?; - listener - .local_addr() - .map(|address| address.port()) - .map_err(|error| format!("read reserved dashboard address: {error}")) -} - -fn check_deadlines(case_deadline: Instant, trace: &[String]) -> Result<(), String> { - let now = Instant::now(); - let suite_started = *SUITE_STARTED.get_or_init(Instant::now); - if now >= case_deadline { - return Err(format!( - "{}-second E2E case deadline exceeded; trace={trace:#?}", - CASE_DEADLINE.as_secs() - )); - } - if now.duration_since(suite_started) >= SUITE_DEADLINE { - return Err(format!( - "{}-second E2E suite deadline exceeded; trace={trace:#?}", - SUITE_DEADLINE.as_secs() - )); - } - Ok(()) -} - -fn wait_for( - case_deadline: Instant, - trace: &[String], - description: &str, - mut probe: impl FnMut() -> Result, String>, -) -> Result { - let mut last_error = None; - loop { - check_deadlines(case_deadline, trace)?; - match probe() { - Ok(Some(value)) => return Ok(value), - Ok(None) => {} - Err(error) => last_error = Some(error), - } - std::thread::sleep(POLL); - if Instant::now() >= case_deadline { - return Err(format!( - "timed out waiting for {description}; last_error={last_error:?}; trace={trace:#?}" - )); - } - } -} - -fn request_json( - method: &str, - url: &str, - body: Option<&Value>, -) -> Result<(u16, Option), String> { - let agent = ureq::AgentBuilder::new() - .timeout_connect(Duration::from_millis(250)) - .timeout_read(Duration::from_secs(3)) - .timeout_write(Duration::from_secs(1)) - .build(); - let request = agent.request(method, url).set("Accept", "application/json"); - let result = match body { - Some(body) => request - .set("Content-Type", "application/json") - .send_string(&body.to_string()), - None => request.call(), - }; - let response = match result { - Ok(response) => response, - Err(ureq::Error::Status(_, response)) => response, - Err(error) => { - let failure = format!("{method} {url} disconnected: {error}"); - record_http_failure(failure.clone()); - return Err(failure); - } - }; - let status = response.status(); - if status >= 500 { - record_http_failure(format!("{method} {url} returned HTTP {status}")); - } - let raw = response.into_string().map_err(|error| { - let failure = format!("read {method} {url} response: {error}"); - record_http_failure(failure.clone()); - failure - })?; - let json = if raw.trim().is_empty() { - None - } else { - Some(serde_json::from_str(&raw).map_err(|error| { - let failure = format!("parse {method} {url} response {raw:?}: {error}"); - record_http_failure(failure.clone()); - failure - })?) - }; - Ok((status, json)) -} - -fn get_status(base_url: &str) -> Result { - let url = format!("{base_url}/api/control/status"); - let (status, body) = request_json("GET", &url, None)?; - if status != 200 { - return Err(format!("status endpoint returned {status}: {body:?}")); - } - body.and_then(|body| body.get("Status").cloned()) - .ok_or_else(|| "status response has no Status model".to_owned()) -} - -fn search_offers(base_url: &str, count: usize) -> Result, String> { - let url = format!("{base_url}/api/control/offers"); - let request = json!({ - "gpu_model": "RTX 4090", - "min_gpu_ram_mb": 20_000, - "min_compute_cap": 800, - "min_reliability": 0.99, - "require_verified": true, - "min_download_mbps": 500.0, - "min_upload_mbps": 400.0, - "max_hourly_price": 1.0, - "blacklist_hosts": [], - "count": count - }); - let (status, body) = request_json("POST", &url, Some(&request))?; - if status != 200 { - return Err(format!("offer search returned {status}: {body:?}")); - } - body.and_then(|body| body.get("Offers").cloned()) - .and_then(|offers| offers.as_array().cloned()) - .ok_or_else(|| "offer response has no Offers array".to_owned())? - .into_iter() - .map(|offer| { - offer - .get("offer_id") - .and_then(Value::as_u64) - .ok_or_else(|| format!("offer has no offer_id: {offer}")) - }) - .collect() -} -fn assert_typed_prerequisite_rejection(base_url: &str) -> Result { - match search_offers(base_url, 0) { - Err(error) if error.contains("409") && error.contains("error") => Ok(error), - result => Err(format!( - "prerequisite-invalid action did not produce a typed rejection: {result:?}" - )), - } -} - -#[derive(Clone, Debug)] -struct AcceptedCommand { - path: String, - submissions: Vec, - terminal: Option, -} - -#[derive(Clone, Debug, Default)] -struct ExternalReplyCollector { - accepted: BTreeMap, - public_replies: Vec, - http_failures: Vec, -} - -impl ExternalReplyCollector { - fn submit(&mut self, base_url: &str, path: &str, body: &Value) -> Result<(), String> { - let command_id = body - .get("command_id") - .and_then(Value::as_str) - .ok_or_else(|| format!("external mutation has no command_id: {body}"))? - .to_owned(); - let url = format!("{base_url}{path}"); - let response = request_json("POST", &url, Some(body)); - let (status, response_body) = match response { - Ok(response) => response, - Err(error) => { - self.http_failures - .push(format!("POST {path} disconnected: {error}")); - return Err(error); - } - }; - if status >= 500 { - self.http_failures - .push(format!("POST {path} returned {status}: {response_body:?}")); - } - if status != 202 { - return Err(format!("POST {path} returned {status}: {response_body:?}")); - } - self.public_replies - .push(format!("POST {path} command_id={command_id} status=202")); - self.accepted - .entry(command_id) - .or_insert_with(|| AcceptedCommand { - path: path.to_owned(), - submissions: Vec::new(), - terminal: None, - }) - .submissions - .push(body.clone()); - Ok(()) - } - - fn observe_status(&mut self, status: &Value) -> Result<(), String> { - let commands = status - .get("commands") - .and_then(Value::as_array) - .ok_or_else(|| format!("status has no command ledger: {status}"))?; - for (command_id, accepted) in &mut self.accepted { - let matches = commands - .iter() - .filter(|command| { - command.get("command_id").and_then(Value::as_str) == Some(command_id.as_str()) - }) - .collect::>(); - if matches.len() > 1 { - return Err(format!( - "accepted command {command_id} has {} public records: {matches:?}", - matches.len() - )); - } - let Some(record) = matches.first() else { - continue; - }; - if record - .get("state") - .and_then(Value::as_str) - .is_some_and(|state| matches!(state, "succeeded" | "failed")) - { - if accepted - .terminal - .as_ref() - .is_some_and(|terminal| terminal != *record) - { - return Err(format!( - "accepted command {command_id} changed terminal reply: before={:?}, after={record}", - accepted.terminal - )); - } - accepted.terminal = Some((*record).clone()); - } - } - Ok(()) - } - - fn all_terminal(&self) -> bool { - self.accepted - .values() - .all(|accepted| accepted.terminal.is_some()) - } - - fn terminal(&self, command_id: &str) -> Option<&Value> { - self.accepted - .get(command_id) - .and_then(|accepted| accepted.terminal.as_ref()) - } -} - -fn flush_control(base_url: &str) -> Result<(), String> { - let url = format!("{base_url}/api/control/flush"); - let (status, response) = request_json("POST", &url, None)?; - if status != 200 || response != Some(Value::String("Flushed".to_owned())) { - return Err(format!("control flush returned {status}: {response:?}")); - } - Ok(()) -} - -fn wait_for_accepted_replies( - harness: &ScenarioHarness, - case_deadline: Instant, - trace: &[String], - replies: &mut ExternalReplyCollector, -) -> Result { - wait_for(case_deadline, trace, "all accepted command replies", || { - let status = get_status(&harness.base_url())?; - replies.observe_status(&status)?; - Ok(replies.all_terminal().then_some(status)) - }) -} - -fn concurrent_status_requests(port: u16) -> Result<(), String> { - let request = - b"GET /api/control/status HTTP/1.1\r\nHost: 127.0.0.1\r\nAccept: application/json\r\nConnection: close\r\n\r\n"; - let mut streams = Vec::new(); - for _ in 0..2 { - let mut stream = TcpStream::connect(("127.0.0.1", port)) - .map_err(|error| format!("connect concurrent dashboard request: {error}"))?; - stream - .set_read_timeout(Some(Duration::from_secs(1))) - .map_err(|error| format!("set concurrent dashboard read timeout: {error}"))?; - stream - .write_all(request) - .map_err(|error| format!("write concurrent dashboard request: {error}"))?; - stream - .flush() - .map_err(|error| format!("flush concurrent dashboard request: {error}"))?; - streams.push(stream); - } - for mut stream in streams { - let mut response = String::new(); - stream - .read_to_string(&mut response) - .map_err(|error| format!("read concurrent dashboard response: {error}"))?; - let status = response - .lines() - .next() - .and_then(|line| line.split_whitespace().nth(1)) - .and_then(|status| status.parse::().ok()) - .ok_or_else(|| format!("malformed concurrent dashboard response: {response:?}"))?; - if status != 200 { - return Err(format!( - "concurrent dashboard request returned {status}: {response:?}" - )); - } - } - Ok(()) -} - -fn node_sets(status: &Value) -> Result<(BTreeSet, BTreeSet), String> { - let nodes = status - .get("nodes") - .and_then(Value::as_array) - .ok_or_else(|| format!("status has no nodes: {status}"))?; - let mut running = BTreeSet::new(); - let mut stopped = BTreeSet::new(); - for node in nodes { - let id = node - .get("logical_node_id") - .and_then(Value::as_u64) - .ok_or_else(|| format!("node has no logical_node_id: {node}"))?; - match node.get("phase").and_then(Value::as_str) { - Some("running") => { - if node.get("runtime").is_none_or(Value::is_null) { - return Err(format!("running node {id} has no runtime readiness facts")); - } - running.insert(id); - } - Some("stopped") => { - stopped.insert(id); - } - _ => {} - } - } - Ok((running, stopped)) -} - -fn selected_offer_map(status: &Value) -> Result, String> { - status - .get("nodes") - .and_then(Value::as_array) - .ok_or_else(|| format!("status has no nodes: {status}"))? - .iter() - .map(|node| { - let node_id = node - .get("logical_node_id") - .and_then(Value::as_u64) - .ok_or_else(|| format!("node has no logical_node_id: {node}"))?; - let offer_id = node - .get("selected_offer_id") - .and_then(Value::as_u64) - .ok_or_else(|| format!("node {node_id} has no selected_offer_id"))?; - Ok((node_id, offer_id)) - }) - .collect() -} - -#[derive(Clone, Debug, Default)] -struct ProviderSnapshot { - live: Vec, - created: Vec, - destroyed: Vec, - events: Vec, -} - -fn provider_ledger_snapshot(path: &PathBuf, run_id: u64) -> Result { - let contents = match std::fs::read_to_string(path) { - Ok(contents) => contents, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => String::new(), - Err(error) => { - return Err(format!( - "read independent provider ledger {}: {error}", - path.display() - )); - } - }; - let mut resources = BTreeMap::::new(); - let mut snapshot = ProviderSnapshot::default(); - for (index, line) in contents.lines().enumerate() { - let event: Value = serde_json::from_str(line).map_err(|error| { - format!( - "parse independent provider ledger {} line {}: {error}; line={line:?}", - path.display(), - index + 1 - ) - })?; - if event.get("run_id").and_then(Value::as_u64) != Some(run_id) { - return Err(format!( - "independent provider ledger contains a foreign run: {event}" - )); - } - let event_type = event - .get("event") - .and_then(Value::as_str) - .ok_or_else(|| format!("provider lifecycle observation has no event: {event}"))?; - let node_id = event - .get("node_id") - .and_then(Value::as_u64) - .ok_or_else(|| format!("provider lifecycle observation has no node_id: {event}"))?; - let provider_ref = event - .get("provider_ref") - .and_then(Value::as_str) - .ok_or_else(|| { - format!("provider lifecycle observation has no provider_ref: {event}") - })?; - match event_type { - "created" => { - if event - .get("selected_offer_id") - .and_then(Value::as_u64) - .is_none_or(|offer_id| offer_id == 0) - { - return Err(format!( - "provider create did not preserve a selected offer: {event}" - )); - } - snapshot.created.push(node_id); - if let Some(previous_node) = resources.insert(provider_ref.to_owned(), node_id) { - return Err(format!( - "duplicate provider resource {provider_ref}: previous node {previous_node}, event={event}" - )); - } - } - "adopted" | "recreated" => match resources.get(provider_ref) { - Some(resource_node) if *resource_node == node_id => {} - _ => { - return Err(format!( - "provider recovered a resource absent from the independent ledger: {event}; live={resources:?}" - )); - } - }, - "destroyed" => { - snapshot.destroyed.push(node_id); - if resources.remove(provider_ref) != Some(node_id) { - return Err(format!( - "provider destroyed a resource absent from the independent ledger: {event}; live={resources:?}" - )); - } - } - other => { - return Err(format!( - "unknown independent provider lifecycle event {other:?}: {event}" - )); - } - } - snapshot.events.push(event); - } - snapshot.live = resources.into_values().collect(); - Ok(snapshot) -} - -fn node_socket(run_id: u64, node_id: u64) -> PathBuf { - std::env::temp_dir().join(format!("myelin-node-debug-join-{run_id}-{node_id}.sock")) -} - -fn process_count(run_id: u64, node_id: u64) -> Result { - find_process_identities_by_environment( - &[ - ("MYELIN_RUN_ID".to_owned(), run_id.to_string()), - ("MYELIN_LOGICAL_NODE_ID".to_owned(), node_id.to_string()), - ], - true, - ) - .map(|matches| matches.len()) - .map_err(|error| format!("find worker process for node {node_id}: {error}")) -} - -fn reachable_nodes(run_id: u64, all_nodes: &BTreeSet) -> Result, String> { - let mut reachable = BTreeSet::new(); - for &node_id in all_nodes { - let count = process_count(run_id, node_id)?; - if count > 1 { - return Err(format!("node {node_id} owns {count} worker processes")); - } - if count == 1 && UnixStream::connect(node_socket(run_id, node_id)).is_ok() { - reachable.insert(node_id); - } - } - Ok(reachable) -} - -fn node_endpoint_request(run_id: u64, node_id: u64, request: &str) -> Result { - let mut stream = UnixStream::connect(node_socket(run_id, node_id)) - .map_err(|error| format!("connect node {node_id} debug endpoint: {error}"))?; - stream - .set_read_timeout(Some(Duration::from_secs(1))) - .map_err(|error| format!("set node {node_id} endpoint read timeout: {error}"))?; - stream - .set_write_timeout(Some(Duration::from_secs(1))) - .map_err(|error| format!("set node {node_id} endpoint write timeout: {error}"))?; - stream - .write_all(request.as_bytes()) - .map_err(|error| format!("write node {node_id} endpoint request: {error}"))?; - stream - .flush() - .map_err(|error| format!("flush node {node_id} endpoint request: {error}"))?; - let mut response = String::new(); - BufReader::new(stream) - .read_line(&mut response) - .map_err(|error| format!("read node {node_id} endpoint response: {error}"))?; - serde_json::from_str(&response) - .map_err(|error| format!("parse node {node_id} endpoint response: {error}")) -} - -fn probe_node_endpoint(run_id: u64, node_id: u64, status: &Value) -> Result<(), String> { - let node = status - .get("nodes") - .and_then(Value::as_array) - .and_then(|nodes| { - nodes - .iter() - .find(|node| node.get("logical_node_id").and_then(Value::as_u64) == Some(node_id)) - }) - .ok_or_else(|| format!("status has no node {node_id}: {status}"))?; - let endpoint_json = node - .pointer("/runtime/endpoint") - .and_then(Value::as_str) - .ok_or_else(|| format!("running node {node_id} has no endpoint: {node}"))?; - let _: Value = serde_json::from_str(endpoint_json) - .map_err(|error| format!("parse node {node_id} endpoint: {error}"))?; - let response = node_endpoint_request(run_id, node_id, "{}\n")?; - if response.get("type").and_then(Value::as_str) != Some("JoinRejected") - || response.get("error").and_then(Value::as_str) != Some("MalformedCommand") - { - return Err(format!( - "node {node_id} endpoint did not complete a typed readiness round trip: {response}" - )); - } - Ok(()) -} - -fn actor_census( - harness: &ScenarioHarness, - worker_nodes: &BTreeSet, -) -> Result { - let base_url = harness.base_url(); - let actor_url = format!("{base_url}/api/control/actors"); - let (actor_status, actor_body) = request_json("GET", &actor_url, None)?; - if actor_status != 200 { - return Err(format!( - "live orchestrator actor census returned {actor_status}: {actor_body:?}" - )); - } - let actor_body = - actor_body.ok_or_else(|| "live orchestrator actor census returned no body".to_owned())?; - let mut census = ActorCensus::default(); - census.orchestrator_actors = actor_body - .get("actors") - .and_then(Value::as_array) - .map_or(0, |actors| actors.len() as u64); - census.actors = census.orchestrator_actors; - if let Some(workers) = actor_body.get("workers").and_then(Value::as_array) { - for worker in workers { - census.mailbox_depth = census.mailbox_depth.saturating_add( - worker - .get("mailbox_depth") - .and_then(Value::as_u64) - .unwrap_or(0), - ); - census.poisoned = census - .poisoned - .saturating_add(worker.get("panics").and_then(Value::as_u64).unwrap_or(0)); - census.type_mismatches = census.type_mismatches.saturating_add( - worker - .get("type_mismatches") - .and_then(Value::as_u64) - .unwrap_or(0), - ); - } - } - - let url = format!("{base_url}/api/view/fleet"); - let (status, body) = request_json("GET", &url, None)?; - if status != 200 { - return Err(format!("fleet actor census returned {status}: {body:?}")); - } - let body = body.ok_or_else(|| "fleet actor census returned no body".to_owned())?; - let live = body - .get("live") - .and_then(Value::as_array) - .ok_or_else(|| format!("fleet actor census has no live array: {body}"))?; - if live.is_empty() { - return Err("fleet actor census has no live streams".to_owned()); - } - for node in live { - let Some(summary) = node.get("actor_summary") else { - continue; - }; - let _actors = summary.get("actors").and_then(Value::as_u64).unwrap_or(0); - let orchestrator = node - .pointer("/stream/origin") - .and_then(Value::as_str) - .is_some_and(|origin| origin == "orchestrator"); - let mut by_type = BTreeMap::new(); - if let Some(roster) = node.get("roster").and_then(Value::as_array) { - for actor in roster { - let actor_type = actor - .get("actor_type") - .and_then(Value::as_str) - .unwrap_or("") - .to_owned(); - *by_type.entry(actor_type).or_insert(0) += 1; - } - } - if orchestrator { - for (actor_type, count) in by_type { - *census.orchestrator_by_type.entry(actor_type).or_insert(0) += count; - } - } - } - for &node_id in worker_nodes { - let response = - node_endpoint_request(harness.run_id, node_id, "{\"type\":\"RuntimeStats\"}\n")?; - if response.get("type").and_then(Value::as_str) != Some("RuntimeStats") { - return Err(format!( - "node {node_id} rejected runtime census request: {response}" - )); - } - let stats = response - .get("stats") - .ok_or_else(|| format!("node {node_id} runtime census has no stats: {response}"))?; - let details = stats - .get("actors") - .and_then(Value::as_array) - .ok_or_else(|| format!("node {node_id} runtime census has no actors: {stats}"))?; - if details.is_empty() { - return Err(format!( - "node {node_id} runtime census is not populated yet" - )); - } - let actors = details.len() as u64; - census.worker_actors = census.worker_actors.saturating_add(actors); - census.actors = census.actors.saturating_add(actors); - census.poisoned = census.poisoned.saturating_add( - details - .iter() - .filter(|actor| actor.get("poisoned").and_then(Value::as_bool) == Some(true)) - .count() as u64, - ); - census.type_mismatches = census.type_mismatches.saturating_add( - stats - .get("workers") - .and_then(Value::as_array) - .into_iter() - .flatten() - .filter_map(|worker| worker.get("type_mismatches").and_then(Value::as_u64)) - .sum::(), - ); - census.mailbox_depth = census.mailbox_depth.saturating_add( - details - .iter() - .filter_map(|actor| actor.get("mailbox_depth").and_then(Value::as_u64)) - .sum(), - ); - let mut by_type = BTreeMap::new(); - for actor in details { - let actor_type = actor - .get("actor_type") - .and_then(Value::as_str) - .unwrap_or("") - .to_owned(); - *by_type.entry(actor_type).or_insert(0) += 1; - } - census - .workers_by_stream - .insert(node_id.to_string(), by_type); - } - let typed_orchestrator_actors = census.orchestrator_by_type.values().sum::(); - if typed_orchestrator_actors == 0 && !worker_nodes.is_empty() { - return Err(format!( - "live orchestrator roster is not populated yet: fleet={body}" - )); - } - if typed_orchestrator_actors != 0 && typed_orchestrator_actors != census.orchestrator_actors { - return Err(format!( - "orchestrator actor roster disagrees with the live runtime: \ - roster={typed_orchestrator_actors}, runtime={}; fleet={body}; runtime_stats={actor_body}", - census.orchestrator_actors - )); - } - if census.orchestrator_actors == 0 { - return Err(format!( - "live orchestrator actor census has no actors: {actor_body}" - )); - } - Ok(census) -} - -fn wait_for_stable_actor_census( - harness: &ScenarioHarness, - case_deadline: Instant, - trace: &[String], - worker_nodes: &BTreeSet, -) -> Result { - let mut candidate: Option<(ActorCensus, Instant)> = None; - wait_for(case_deadline, trace, "stable actor census", || { - let census = actor_census(harness, worker_nodes)?; - if census.mailbox_depth != 0 { - candidate = None; - return Ok(None); - } - match &candidate { - Some((previous, since)) if previous == &census => { - Ok((since.elapsed() >= CENSUS_STABILITY).then_some(census)) - } - _ => { - candidate = Some((census, Instant::now())); - Ok(None) - } - } - }) -} - -fn wait_for_baseline_actor_census( - harness: &ScenarioHarness, - case_deadline: Instant, - trace: &[String], - baseline: &ActorCensus, -) -> Result { - wait_for(case_deadline, trace, "baseline actor census", || { - let census = actor_census(harness, &BTreeSet::new())?; - Ok((census.actors == baseline.actors - && census.orchestrator_actors == baseline.orchestrator_actors - && census.worker_actors == baseline.worker_actors - && census.poisoned == baseline.poisoned - && census.type_mismatches == baseline.type_mismatches - && census.mailbox_depth == baseline.mailbox_depth) - .then_some(census)) - }) -} - -fn wait_for_state( - harness: &ScenarioHarness, - case_deadline: Instant, - trace: &[String], - expected_running: &BTreeSet, - expected_stopped: &BTreeSet, -) -> Result { - wait_for(case_deadline, trace, "durable node state", || { - let status = get_status(&harness.base_url())?; - let (running, stopped) = node_sets(&status)?; - Ok((running == *expected_running && stopped == *expected_stopped).then_some(status)) - }) -} - -fn wait_for_reachability( - harness: &ScenarioHarness, - case_deadline: Instant, - trace: &[String], - all_nodes: &BTreeSet, - expected: &BTreeSet, -) -> Result, String> { - wait_for(case_deadline, trace, "worker reachability", || { - let reachable = reachable_nodes(harness.run_id, all_nodes)?; - Ok((reachable == *expected).then_some(reachable)) - }) -} - -fn wait_for_provider_ledger( - harness: &ScenarioHarness, - case_deadline: Instant, - trace: &[String], - expected_created: &BTreeSet, - expected_destroyed: &BTreeSet, - expected_live: &BTreeSet, -) -> Result { - wait_for( - case_deadline, - trace, - "independent mock-provider lifecycle ledger", - || { - let snapshot = provider_ledger_snapshot(&harness.provider_ledger_path, harness.run_id)?; - let live_set = snapshot.live.iter().copied().collect::>(); - let created_set = snapshot.created.iter().copied().collect::>(); - let destroyed_set = snapshot.destroyed.iter().copied().collect::>(); - Ok((created_set == *expected_created - && snapshot.created.len() == expected_created.len() - && destroyed_set == *expected_destroyed - && snapshot.destroyed.len() == expected_destroyed.len() - && live_set == *expected_live - && snapshot.live.len() == expected_live.len()) - .then_some(snapshot)) - }, - ) -} -#[derive(Clone, Debug)] -struct LifecyclePlan { - selected_offers: Vec, - all_nodes: BTreeSet, - stopped: BTreeSet, - survivors: BTreeSet, -} - -fn lifecycle_plan(case: &E2eCase, searched: &[u64]) -> Result { - let usable = searched.len().min(2); - if usable == 0 { - return Err("cannot derive lifecycle plan without a usable search result".to_owned()); - } - let node_count = 1 + usize::from(case.node_seed) % usable; - let selected_offers = (0..node_count) - .map(|offset| searched[(case.offer_offset + offset) % searched.len()]) - .collect::>(); - let all_nodes = (1..=node_count as u64).collect::>(); - let protected_survivor = 1 + case.seed % node_count as u64; - let mut stopped = BTreeSet::new(); - for node_id in &all_nodes { - if *node_id != protected_survivor - && case.kill_mask & (1_u8 << ((*node_id as usize - 1) % 8)) != 0 - { - stopped.insert(*node_id); - } - } - let survivors = all_nodes - .difference(&stopped) - .copied() - .collect::>(); - Ok(LifecyclePlan { - selected_offers, - all_nodes, - stopped, - survivors, - }) -} - -fn labeled_temp_resources(harness: &ScenarioHarness, all_nodes: &BTreeSet) -> Vec { - let mut leaked = all_nodes - .iter() - .map(|node_id| node_socket(harness.run_id, *node_id)) - .filter(|path| path.exists()) - .collect::>(); - let registry_path = harness.state_dir.join("process-nodes.json"); - let registry_leaked = match std::fs::read_to_string(®istry_path) { - Err(error) if error.kind() == std::io::ErrorKind::NotFound => false, - Ok(contents) => !matches!( - serde_json::from_str::(&contents), - Ok(Value::Object(registry)) if registry.is_empty() - ), - Err(_) => true, - }; - if registry_leaked { - leaked.push(registry_path); - } - leaked -} - -fn setup_failure_with_diagnostics(error: String, case: &E2eCase, trace: &[String]) -> String { - format!( - "{error}; seed={}; generated_actions={:#?}; trace={trace:#?}; \ - public_replies=[]; http_failures=[]; process_logs=; \ - census=; reachability=; provider_ledger=", - case.seed, case.actions - ) -} - -fn failure_with_diagnostics( - error: String, - case: &E2eCase, - trace: &[String], - replies: &ExternalReplyCollector, - census_snapshots: &[ActorCensus], - harness: &ScenarioHarness, - provider: &TestHttpServer, -) -> String { - let status = get_status(&harness.base_url()); - let node_sets = status - .as_ref() - .ok() - .and_then(|status| node_sets(status).ok()); - let observed_nodes = node_sets - .as_ref() - .map(|(running, stopped)| running.union(stopped).copied().collect()) - .unwrap_or_default(); - let reachability = reachable_nodes(harness.run_id, &observed_nodes); - let census = node_sets - .as_ref() - .map(|(running, _)| actor_census(harness, running)) - .transpose(); - let provider_ledger = provider_ledger_snapshot(&harness.provider_ledger_path, harness.run_id); - let worker_processes = find_process_identities_by_environment( - &[("MYELIN_RUN_ID".to_owned(), harness.run_id.to_string())], - true, - ); - let worker_sockets = observed_nodes - .iter() - .map(|node_id| (*node_id, node_socket(harness.run_id, *node_id).exists())) - .collect::>(); - format!( - "{error}; seed={}; generated_actions={:#?}; trace={trace:#?}; \ - public_replies={:#?}; http_failures={:#?}; observed_http_failures={:#?}; \ - unexpected_process_exits={:#?}; \ - status={status:#?}; census_snapshots={census_snapshots:#?}; current_census={census:#?}; \ - reachability={reachability:#?}; worker_processes={worker_processes:#?}; \ - worker_sockets={worker_sockets:#?}; provider_ledger={provider_ledger:#?}; \ - provider_http_requests={:#?}; process_logs={}", - case.seed, - case.actions, - replies.public_replies, - replies.http_failures, - observed_http_failures(), - harness.unexpected_exits, - provider.requests(), - harness.logs(), - ) -} - -fn run_stateful_case(case: E2eCase) -> Result<(), String> { - let case_started = Instant::now(); - let case_deadline = case_started + CASE_DEADLINE; - let mut trace = vec![format!( - "seed={} generated_actions={:#?}", - case.seed, case.actions - )]; - discard_http_failures_since(0); - check_deadlines(case_deadline, &trace) - .map_err(|error| setup_failure_with_diagnostics(error, &case, &trace))?; - - let provider = TestHttpServer::start(vec![TestHttpRoute::json( - "GET", - "/api/v0/bundles/", - 200, - production_offers(), - )]) - .map_err(|error| setup_failure_with_diagnostics(error, &case, &trace))?; - let temp = tempfile::tempdir().map_err(|error| { - setup_failure_with_diagnostics(format!("create E2E state dir: {error}"), &case, &trace) - })?; - let run_id = 7_000_000_u64 + case.seed % 1_000_000; - let dashboard_port = - reserve_port().map_err(|error| setup_failure_with_diagnostics(error, &case, &trace))?; - let mut harness = ScenarioHarness::new( - temp.path().to_path_buf(), - provider.uri(), - dashboard_port, - run_id, - ); - let mut replies = ExternalReplyCollector::default(); - let mut census_snapshots = Vec::new(); - - let execution = (|| -> Result { - harness.start()?; - trace.push("launch direct Cargo-built Myelin orchestrator".to_owned()); - let readiness_http_checkpoint = http_failure_checkpoint(); - wait_for(case_deadline, &trace, "dashboard readiness", || { - get_status(&harness.base_url()).map(Some) - })?; - discard_http_failures_since(readiness_http_checkpoint); - harness.ensure_running()?; - let mut generated_restart_mode = None; - concurrent_status_requests(harness.dashboard_port)?; - trace.push("two concurrent independent status requests completed".to_owned()); - let actor_initial = - wait_for_stable_actor_census(&harness, case_deadline, &trace, &BTreeSet::new())?; - census_snapshots.push(actor_initial.clone()); - - let mut searched = Vec::new(); - let mut generated_restarts = 0_usize; - let mut typed_rejections = 0_usize; - for (action_index, action) in case.actions.clone().into_iter().enumerate() { - check_deadlines(case_deadline, &trace)?; - harness.ensure_running()?; - trace.push(format!("generated action {action_index}: {action:?}")); - match action { - ExternalAction::Search { count } => match search_offers(&harness.base_url(), count) - { - Ok(offers) if count > 0 => { - searched = offers; - trace.push(format!("search returned usable offers {searched:?}")); - } - Ok(offers) => { - return Err(format!( - "prerequisite-invalid search unexpectedly succeeded: {offers:?}" - )); - } - Err(error) - if count == 0 && error.contains("409") && error.contains("error") => - { - typed_rejections += 1; - trace.push(format!("typed invalid-search rejection: {error}")); - } - Err(error) => { - replies - .http_failures - .push(format!("offer search failed: {error}")); - return Err(error); - } - }, - ExternalAction::Provision { - command_slot, - use_searched_offers, - } => { - let Some(plan) = use_searched_offers - .then(|| lifecycle_plan(&case, &searched).ok()) - .flatten() - else { - let rejection = assert_typed_prerequisite_rejection(&harness.base_url())?; - typed_rejections += 1; - trace.push(format!( - "provision rejected before usable search selection: {rejection}" - )); - continue; - }; - let request = json!({ - "command_id": format!("generated-provision-{command_slot}"), - "count": plan.selected_offers.len(), - "selected_offer_ids": plan.selected_offers, - }); - replies.submit(&harness.base_url(), "/api/control/provision", &request)?; - } - ExternalAction::Query => { - let status = get_status(&harness.base_url()).map_err(|error| { - replies - .http_failures - .push(format!("status query failed: {error}")); - error - })?; - replies.observe_status(&status)?; - } - ExternalAction::Kill { - node_slot, - command_slot, - } => { - let Some(plan) = lifecycle_plan(&case, &searched).ok() else { - let rejection = assert_typed_prerequisite_rejection(&harness.base_url())?; - typed_rejections += 1; - trace.push(format!( - "kill rejected before usable search selection: {rejection}" - )); - continue; - }; - let Some(target) = (!plan.stopped.is_empty()).then(|| { - let index = usize::from(node_slot) % plan.stopped.len(); - *plan.stopped.iter().nth(index).expect("bounded kill index") - }) else { - let rejection = assert_typed_prerequisite_rejection(&harness.base_url())?; - typed_rejections += 1; - trace.push(format!( - "kill rejected because its generated subset is empty: {rejection}" - )); - continue; - }; - let status = get_status(&harness.base_url())?; - replies.observe_status(&status)?; - let (running, stopped) = node_sets(&status)?; - if !running.contains(&target) && !stopped.contains(&target) { - let rejection = assert_typed_prerequisite_rejection(&harness.base_url())?; - typed_rejections += 1; - trace.push(format!( - "kill of node {target} rejected before provision: {rejection}" - )); - continue; - } - let request = json!({ - "command_id": format!("generated-kill-{command_slot}"), - "logical_node_id": target, - }); - replies.submit(&harness.base_url(), "/api/control/kill", &request)?; - } - ExternalAction::Flush => { - flush_control(&harness.base_url()).map_err(|error| { - replies - .http_failures - .push(format!("control flush failed: {error}")); - error - })?; - replies - .public_replies - .push("POST /api/control/flush reply=Flushed".to_owned()); - } - ExternalAction::Restart { mode } => { - if replies.accepted.is_empty() { - trace.push(format!( - "restart {mode:?} rejected by external model before any acknowledged mutation" - )); - continue; - } - if generated_restarts == 1 { - trace.push(format!( - "restart {mode:?} rejected by the bounded external model after one generated restart" - )); - continue; - } - flush_control(&harness.base_url())?; - replies.public_replies.push(format!( - "POST /api/control/flush reply=Flushed before {mode:?}" - )); - match mode { - RestartMode::Graceful => { - harness.restart_gracefully(case_deadline)?; - } - RestartMode::FlushSafeAbrupt => { - harness.restart_abruptly()?; - } - } - generated_restarts += 1; - generated_restart_mode = Some(mode); - let readiness_http_checkpoint = http_failure_checkpoint(); - wait_for( - case_deadline, - &trace, - "dashboard after generated restart", - || get_status(&harness.base_url()).map(Some), - )?; - discard_http_failures_since(readiness_http_checkpoint); - trace.push(format!( - "completed generated {mode:?} restart between acknowledged operations" - )); - } - ExternalAction::EndpointProbe { node_slot } => { - let status = get_status(&harness.base_url())?; - replies.observe_status(&status)?; - let (running, _) = node_sets(&status)?; - if !running.is_empty() { - let index = usize::from(node_slot) % running.len(); - let node_id = *running.iter().nth(index).expect("bounded probe index"); - probe_node_endpoint(run_id, node_id, &status)?; - typed_rejections += 1; - trace.push(format!( - "node {node_id} returned typed MalformedCommand rejection" - )); - } else { - trace.push( - "endpoint probe rejected by external model before provision".to_owned(), - ); - } - } - ExternalAction::ConcurrentQueries => { - concurrent_status_requests(harness.dashboard_port).map_err(|error| { - replies - .http_failures - .push(format!("concurrent status requests failed: {error}")); - error - })?; - } - } - } - - if searched.is_empty() { - searched = search_offers(&harness.base_url(), 2)?; - trace.push(format!( - "convergence search returned usable offers {searched:?}" - )); - } - let plan = lifecycle_plan(&case, &searched)?; - if plan.selected_offers.len() > 2 || plan.survivors.is_empty() { - return Err(format!("invalid bounded lifecycle plan: {plan:?}")); - } - - let mut status = if replies.accepted.is_empty() { - get_status(&harness.base_url())? - } else { - wait_for_accepted_replies(&harness, case_deadline, &trace, &mut replies)? - }; - for (command_id, accepted) in &replies.accepted { - if let Some(terminal) = &accepted.terminal - && terminal.get("state").and_then(Value::as_str) == Some("failed") - { - if terminal.get("error").and_then(Value::as_str).is_none() { - return Err(format!( - "failed command {command_id} has no typed rejection: {terminal}" - )); - } - typed_rejections += 1; - } - } - - let (running, stopped) = node_sets(&status)?; - let observed = running.union(&stopped).copied().collect::>(); - if observed.is_empty() { - let request = json!({ - "command_id": format!("converge-provision-{}", case.seed), - "count": plan.selected_offers.len(), - "selected_offer_ids": plan.selected_offers, - }); - replies.submit(&harness.base_url(), "/api/control/provision", &request)?; - wait_for_accepted_replies(&harness, case_deadline, &trace, &mut replies)?; - let terminal = replies - .terminal(request["command_id"].as_str().expect("command id")) - .expect("accepted command has terminal reply"); - if terminal.get("state").and_then(Value::as_str) != Some("succeeded") { - return Err(format!("convergence provision failed: {terminal}")); - } - } else if observed != plan.all_nodes { - return Err(format!( - "accepted replies produced an unexpected node set: expected {:?}, observed {observed:?}, status={status}", - plan.all_nodes - )); - } - - status = wait_for( - case_deadline, - &trace, - "provisioned state from accepted replies", - || { - let status = get_status(&harness.base_url())?; - let (running, stopped) = node_sets(&status)?; - let observed = running.union(&stopped).copied().collect::>(); - Ok((observed == plan.all_nodes).then_some(status)) - }, - )?; - let (_, already_stopped) = node_sets(&status)?; - if !already_stopped.is_subset(&plan.stopped) { - return Err(format!( - "generated successful kills exceeded the survivor-preserving subset: desired={:?}, observed={already_stopped:?}", - plan.stopped - )); - } - for node_id in plan.stopped.difference(&already_stopped) { - let request = json!({ - "command_id": format!("converge-kill-{}-{node_id}", case.seed), - "logical_node_id": node_id, - }); - replies.submit(&harness.base_url(), "/api/control/kill", &request)?; - } - wait_for_accepted_replies(&harness, case_deadline, &trace, &mut replies)?; - for command_id in replies - .accepted - .keys() - .filter(|command_id| command_id.starts_with("converge-kill-")) - { - let terminal = replies - .terminal(command_id) - .expect("accepted kill has terminal reply"); - if terminal.get("state").and_then(Value::as_str) != Some("succeeded") { - return Err(format!("convergence kill failed: {terminal}")); - } - } - - status = wait_for_state( - &harness, - case_deadline, - &trace, - &plan.survivors, - &plan.stopped, - )?; - wait_for_reachability( - &harness, - case_deadline, - &trace, - &plan.all_nodes, - &plan.survivors, - )?; - wait_for_provider_ledger( - &harness, - case_deadline, - &trace, - &plan.all_nodes, - &plan.stopped, - &plan.survivors, - )?; - let selected_offer_by_node = selected_offer_map(&status)?; - let expected_offer_by_node = plan - .selected_offers - .iter() - .enumerate() - .map(|(index, offer_id)| (index as u64 + 1, *offer_id)) - .collect::>(); - if selected_offer_by_node != expected_offer_by_node { - return Err(format!( - "public selected offers differ from searched selection: expected={expected_offer_by_node:?}, observed={selected_offer_by_node:?}" - )); - } - - if typed_rejections == 0 { - match search_offers(&harness.base_url(), 0) { - Err(error) if error.contains("409") && error.contains("error") => { - typed_rejections += 1; - trace.push(format!("typed convergence rejection: {error}")); - } - result => { - return Err(format!( - "could not establish a typed prerequisite rejection: {result:?}" - )); - } - } - } - if typed_rejections == 0 { - return Err("no typed rejection was observed".to_owned()); - } - - let convergence_restarts = match generated_restart_mode { - Some(RestartMode::Graceful) => vec![RestartMode::FlushSafeAbrupt], - Some(RestartMode::FlushSafeAbrupt) => vec![RestartMode::Graceful], - None if case.seed & 1 == 0 => { - vec![RestartMode::Graceful, RestartMode::FlushSafeAbrupt] - } - None => vec![RestartMode::FlushSafeAbrupt, RestartMode::Graceful], - }; - for mode in convergence_restarts { - flush_control(&harness.base_url())?; - replies.public_replies.push(format!( - "POST /api/control/flush reply=Flushed before convergence {mode:?}" - )); - match mode { - RestartMode::Graceful => harness.restart_gracefully(case_deadline)?, - RestartMode::FlushSafeAbrupt => harness.restart_abruptly()?, - } - let readiness_http_checkpoint = http_failure_checkpoint(); - wait_for( - case_deadline, - &trace, - "dashboard after convergence restart", - || get_status(&harness.base_url()).map(Some), - )?; - discard_http_failures_since(readiness_http_checkpoint); - trace.push(format!("convergence tail completed {mode:?} restart")); - } - status = wait_for_state( - &harness, - case_deadline, - &trace, - &plan.survivors, - &plan.stopped, - )?; - wait_for_reachability( - &harness, - case_deadline, - &trace, - &plan.all_nodes, - &plan.survivors, - )?; - wait_for_provider_ledger( - &harness, - case_deadline, - &trace, - &plan.all_nodes, - &plan.stopped, - &plan.survivors, - )?; - for node_id in &plan.survivors { - probe_node_endpoint(run_id, *node_id, &status)?; - } - - let actor_before_replays = - wait_for_stable_actor_census(&harness, case_deadline, &trace, &plan.survivors)?; - census_snapshots.push(actor_before_replays.clone()); - let replay_requests = replies - .accepted - .values() - .filter_map(|accepted| { - accepted - .submissions - .first() - .cloned() - .map(|request| (accepted.path.clone(), request)) - }) - .collect::>(); - for (path, request) in replay_requests { - replies.submit(&harness.base_url(), &path, &request)?; - } - for _ in 0..2 { - let read = get_status(&harness.base_url())?; - replies.observe_status(&read)?; - } - concurrent_status_requests(harness.dashboard_port)?; - status = wait_for_accepted_replies(&harness, case_deadline, &trace, &mut replies)?; - let actor_after_replays = - wait_for_stable_actor_census(&harness, case_deadline, &trace, &plan.survivors)?; - census_snapshots.push(actor_after_replays.clone()); - let (running_after_restart, stopped_after_restart) = node_sets(&status)?; - let reachable = reachable_nodes(run_id, &plan.all_nodes)?; - let provider_snapshot = - provider_ledger_snapshot(&harness.provider_ledger_path, harness.run_id)?; - - for node_id in &plan.survivors { - let request = json!({ - "command_id": format!("teardown-kill-{}-{node_id}", case.seed), - "logical_node_id": node_id, - }); - replies.submit(&harness.base_url(), "/api/control/kill", &request)?; - replies.submit(&harness.base_url(), "/api/control/kill", &request)?; - } - wait_for_accepted_replies(&harness, case_deadline, &trace, &mut replies)?; - wait_for_state( - &harness, - case_deadline, - &trace, - &BTreeSet::new(), - &plan.all_nodes, - )?; - wait_for_reachability( - &harness, - case_deadline, - &trace, - &plan.all_nodes, - &BTreeSet::new(), - )?; - let final_provider = wait_for_provider_ledger( - &harness, - case_deadline, - &trace, - &plan.all_nodes, - &plan.all_nodes, - &BTreeSet::new(), - )?; - let actor_after_teardown = - wait_for_baseline_actor_census(&harness, case_deadline, &trace, &actor_initial)?; - census_snapshots.push(actor_after_teardown.clone()); - let final_status = get_status(&harness.base_url())?; - replies.observe_status(&final_status)?; - if !replies.all_terminal() { - return Err(format!( - "accepted replies remained nonterminal: {:?}", - replies.accepted - )); - } - let commands = final_status - .get("commands") - .and_then(Value::as_array) - .ok_or_else(|| format!("final status has no command ledger: {final_status}"))?; - let terminal_record_counts = replies - .accepted - .keys() - .map(|command_id| { - let count = commands - .iter() - .filter(|command| { - command.get("command_id").and_then(Value::as_str) - == Some(command_id.as_str()) - && command - .get("state") - .and_then(Value::as_str) - .is_some_and(|state| matches!(state, "succeeded" | "failed")) - }) - .count(); - (command_id.clone(), count) - }) - .collect::>(); - - flush_control(&harness.base_url())?; - harness.stop_child_gracefully(case_deadline)?; - wait_for( - case_deadline, - &trace, - "all worker processes stopped", - || { - let live = plan - .all_nodes - .iter() - .map(|node_id| process_count(run_id, *node_id)) - .collect::, _>>()? - .into_iter() - .sum::(); - Ok((live == 0).then_some(())) - }, - )?; - let labeled_temp_resources = labeled_temp_resources(&harness, &plan.all_nodes); - let mut http_failures = observed_http_failures(); - http_failures.extend(replies.http_failures.clone()); - - Ok(OracleObservation { - expected_survivors: plan.survivors, - expected_stopped: plan.stopped, - running_after_restart, - stopped_after_restart, - reachable_nodes: reachable, - provider_resources: provider_snapshot.live, - provider_created: provider_snapshot.created.into_iter().collect(), - provider_destroyed: provider_snapshot.destroyed.into_iter().collect(), - selected_offers: selected_offer_by_node, - searched_offers: searched.into_iter().collect(), - actor_before_replays, - actor_after_replays, - actor_initial, - actor_after_teardown, - terminal_record_counts, - unexpected_process_exits: harness.unexpected_exits.clone(), - http_failures, - teardown_resources: final_provider.live, - labeled_temp_resources, - }) - })(); - - let observation = execution.map_err(|error| { - failure_with_diagnostics( - error, - &case, - &trace, - &replies, - &census_snapshots, - &harness, - &provider, - ) - })?; - validate_oracle(&observation).map_err(|error| { - failure_with_diagnostics( - format!("{error}; observation={observation:#?}"), - &case, - &trace, - &replies, - &census_snapshots, - &harness, - &provider, - ) - })?; - if !provider - .requests() - .iter() - .any(|request| request.method == "GET" && request.path == "/api/v0/bundles/") - { - return Err(failure_with_diagnostics( - "provider search boundary was never exercised".to_owned(), - &case, - &trace, - &replies, - &census_snapshots, - &harness, - &provider, - )); - } - - let teardown_logs = harness.logs(); - let teardown_ledger = provider_ledger_snapshot(&harness.provider_ledger_path, harness.run_id); - let provider_requests = provider.requests(); - drop(harness); - drop(provider); - temp.close().map_err(|error| { - format!( - "remove labeled E2E state directory: {error}; seed={}; actions={:#?}; \ - replies={:#?}; observation={observation:#?}; ledger={teardown_ledger:#?}; \ - provider_requests={provider_requests:#?}; logs={teardown_logs}", - case.seed, case.actions, replies.public_replies - ) - })?; - Ok(()) -} - -proptest! { - #![proptest_config(e2e_proptest_config())] - - #[test] - #[ignore = "nightly process E2E; run with --ignored"] - fn stateful_vastai_dashboard_control_survives_restarts(case in e2e_case()) { - if let Err(error) = run_stateful_case(case) { - prop_assert!(false, "{error}"); - } - } -} - -#[test] -fn e2e_oracle_rejects_controlled_lifecycle_faults() { - let baseline = OracleObservation { - expected_survivors: BTreeSet::from([2]), - expected_stopped: BTreeSet::from([1]), - running_after_restart: BTreeSet::from([2]), - stopped_after_restart: BTreeSet::from([1]), - reachable_nodes: BTreeSet::from([2]), - provider_resources: vec![2], - provider_created: BTreeSet::from([1, 2]), - provider_destroyed: BTreeSet::from([1]), - selected_offers: BTreeMap::from([(1, 101), (2, 102)]), - searched_offers: BTreeSet::from([101, 102]), - actor_before_replays: ActorCensus { - actors: 10, - orchestrator_actors: 5, - worker_actors: 5, - orchestrator_by_type: BTreeMap::from([("orchestrator".to_owned(), 5)]), - workers_by_stream: BTreeMap::from([( - "worker-2".to_owned(), - BTreeMap::from([("worker".to_owned(), 5)]), - )]), - ..ActorCensus::default() - }, - actor_after_replays: ActorCensus { - actors: 10, - orchestrator_actors: 5, - worker_actors: 5, - orchestrator_by_type: BTreeMap::from([("orchestrator".to_owned(), 5)]), - workers_by_stream: BTreeMap::from([( - "worker-2".to_owned(), - BTreeMap::from([("worker".to_owned(), 5)]), - )]), - ..ActorCensus::default() - }, - actor_initial: ActorCensus { - actors: 5, - orchestrator_actors: 5, - orchestrator_by_type: BTreeMap::from([("orchestrator".to_owned(), 5)]), - ..ActorCensus::default() - }, - actor_after_teardown: ActorCensus { - actors: 5, - orchestrator_actors: 5, - orchestrator_by_type: BTreeMap::from([("orchestrator".to_owned(), 5)]), - ..ActorCensus::default() - }, - terminal_record_counts: BTreeMap::from([("command-1".to_owned(), 1)]), - unexpected_process_exits: Vec::new(), - http_failures: Vec::new(), - teardown_resources: Vec::new(), - labeled_temp_resources: Vec::new(), - }; - validate_oracle(&baseline).expect("baseline oracle observation"); - - let mut survivor_destroy = baseline.clone(); - survivor_destroy.provider_resources.clear(); - survivor_destroy.provider_destroyed.insert(2); - assert!( - validate_oracle(&survivor_destroy) - .unwrap_err() - .contains("provider ledger mismatch") - ); - - let mut restart_loss = baseline.clone(); - restart_loss.running_after_restart.clear(); - assert!( - validate_oracle(&restart_loss) - .unwrap_err() - .contains("restart-state loss") - ); - let mut duplicate_provider = baseline.clone(); - duplicate_provider.provider_resources.push(2); - assert!( - validate_oracle(&duplicate_provider) - .unwrap_err() - .contains("duplicate provider resources") - ); - - let mut actor_poison = baseline.clone(); - actor_poison.actor_after_replays.poisoned = 1; - assert!( - validate_oracle(&actor_poison) - .unwrap_err() - .contains("actor poisoning") - ); - let mut unexpected_exit = baseline.clone(); - unexpected_exit - .unexpected_process_exits - .push("exit status 1".to_owned()); - assert!( - validate_oracle(&unexpected_exit) - .unwrap_err() - .contains("unexpected process exit") - ); - - let mut http_failure = baseline.clone(); - http_failure - .http_failures - .push("HTTP 503 or disconnect".to_owned()); - assert!( - validate_oracle(&http_failure) - .unwrap_err() - .contains("HTTP failure or disconnect") - ); - - let mut duplicate_terminal = baseline.clone(); - duplicate_terminal - .terminal_record_counts - .insert("command-1".to_owned(), 2); - assert!( - validate_oracle(&duplicate_terminal) - .unwrap_err() - .contains("exactly one terminal record") - ); - - let mut actor_leak = baseline.clone(); - actor_leak.actor_after_replays.actors = 13; - assert!( - validate_oracle(&actor_leak) - .unwrap_err() - .contains("steady-state actor census") - ); - let mut teardown_leak = baseline; - teardown_leak.teardown_resources.push(2); - teardown_leak - .labeled_temp_resources - .push(PathBuf::from("process-nodes.json")); - assert!( - validate_oracle(&teardown_leak) - .unwrap_err() - .contains("teardown leak") - ); -} diff --git a/crates/bindings/python/pyproject.toml b/crates/bindings/python/pyproject.toml index 37fe2a7..fa7ca46 100644 --- a/crates/bindings/python/pyproject.toml +++ b/crates/bindings/python/pyproject.toml @@ -8,7 +8,7 @@ version = "0.1.0" requires-python = ">=3.9" [dependency-groups] -dev = ["jupyter", "ipykernel", "pytest"] +dev = ["jupyter", "ipykernel", "maturin>=1.7,<2", "pytest", "pytest-timeout"] [tool.uv] cache-keys = [ diff --git a/crates/bindings/python/uv.lock b/crates/bindings/python/uv.lock index 2c5fab6..5f48f50 100644 --- a/crates/bindings/python/uv.lock +++ b/crates/bindings/python/uv.lock @@ -1282,6 +1282,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/af/33/ee4519fa02ed11a94aef9559552f3b17bb863f2ecfe1a35dc7f548cde231/matplotlib_inline-0.2.1-py3-none-any.whl", hash = "sha256:d56ce5156ba6085e00a9d54fead6ed29a9c47e215cd1bba2e976ef39f5710a76", size = 9516, upload-time = "2025-10-23T09:00:20.675Z" }, ] +[[package]] +name = "maturin" +version = "1.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e7/b3/addd877f871fb1860d46d3a4f206ecb10b946c85846805e6367631926fd3/maturin-1.14.1.tar.gz", hash = "sha256:9d6577a62cd08e0ceba7a0db06fb098e0c9b1b3429bad747a4f3a18215a1b3df", size = 369637, upload-time = "2026-06-19T05:19:49.774Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/f0/97c5a5bd9c71653a066c0976a484eaaae50b9369557838a4176b7b0bdaa5/maturin-1.14.1-py3-none-linux_armv6l.whl", hash = "sha256:522292398945442cdafa9daeb2271b2340fbde57027b818f923f88eab04174f8", size = 10207496, upload-time = "2026-06-19T05:19:09.321Z" }, + { url = "https://files.pythonhosted.org/packages/fe/83/294bca639b0e052f1e2f65199b3db258780c7d4e31408b934c9c974a1379/maturin-1.14.1-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:ffe5ad71f21d1e6603c4dd75f7fee34adf5ed5ebcebb692886549888ebb329ed", size = 19680113, upload-time = "2026-06-19T05:19:13.43Z" }, + { url = "https://files.pythonhosted.org/packages/43/b6/79c881410a3b1c187f7eb3d407aecae646c6a4433d630d72200359015e83/maturin-1.14.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:f3306078070c1508fd715b9116070cbcaff5959024272a9f1e6f5cb29768b86c", size = 10169205, upload-time = "2026-06-19T05:19:16.615Z" }, + { url = "https://files.pythonhosted.org/packages/93/9d/44b6f26dcb7f7a04c5501ac2dbb6ca1490150682baa525ca5860504f9eab/maturin-1.14.1-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:cd457cd88961156e26379e1155bd287cc0ec1c8b2f1582b0660fb31b87c8842d", size = 10188098, upload-time = "2026-06-19T05:19:19.736Z" }, + { url = "https://files.pythonhosted.org/packages/1a/bd/9c0d5d6983905ce2c9edaa073a7e89355a9cf7f396988e05d32f1c37785d/maturin-1.14.1-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:dfc54ae32e6fcb18302193ab9a30b0b25eefffba994ae13238974805533ef75e", size = 10627576, upload-time = "2026-06-19T05:19:22.713Z" }, + { url = "https://files.pythonhosted.org/packages/e5/33/b096412bd6a7cb399652b260666f901adf88a687181a6dbd6a3f89f0a94e/maturin-1.14.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:a131d912b5267e640bc96d70f4914e10590aed64082ec9abacba7cea52004224", size = 10085181, upload-time = "2026-06-19T05:19:25.69Z" }, + { url = "https://files.pythonhosted.org/packages/56/8d/08c3bf469c38a23c9e6c877e338193001eb604d010fedc08341974e38528/maturin-1.14.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:be18fc568fb76884c0205456336892a75105ec398e6b667cd777c6268bd06d69", size = 10026363, upload-time = "2026-06-19T05:19:28.904Z" }, + { url = "https://files.pythonhosted.org/packages/3a/a4/c4d1a92839f8745ab4aab988a7db884a79d6d710bd3b286fcf9316dece1a/maturin-1.14.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:994a0c8ba3ad8a92b3a9ee1b02645d200d610216b15cff5102b0fe65e8e08666", size = 13321347, upload-time = "2026-06-19T05:19:32.411Z" }, + { url = "https://files.pythonhosted.org/packages/b3/fa/170f04624d03fd07d2a8b1b67de83a127af93aef9eaa425839553347297b/maturin-1.14.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:be80866363e605d137991b491a741a84cde9ae350183c4c85f49690ca9aaaa65", size = 10877609, upload-time = "2026-06-19T05:19:35.448Z" }, + { url = "https://files.pythonhosted.org/packages/61/ad/1ae2e1d0ded282bf2c55ac13f0811d87deb425e200ae64a15785675dede9/maturin-1.14.1-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:5282dffd4b539d2be245f4e5b1a5ab6bc1033b58f4a4872f5833f9d43c954aa4", size = 10417316, upload-time = "2026-06-19T05:19:38.28Z" }, + { url = "https://files.pythonhosted.org/packages/fb/27/bf677183920718da49cd7982d6a3ffc440aad8919329f571d189f81b7bdf/maturin-1.14.1-py3-none-win32.whl", hash = "sha256:1a04de0a20188f95c721b5702eed18140bdcccb28c386797093eca3f62f4d4e0", size = 8931293, upload-time = "2026-06-19T05:19:41.183Z" }, + { url = "https://files.pythonhosted.org/packages/63/4b/585adeb9167b08d3cdff0032a938b0e72655c92003df4f52c3f696a1bcc2/maturin-1.14.1-py3-none-win_amd64.whl", hash = "sha256:3c9f94640ecc4895e94abaf834a0684430032c865b2748a36c12461fd9252fdd", size = 10314067, upload-time = "2026-06-19T05:19:44.389Z" }, + { url = "https://files.pythonhosted.org/packages/51/d4/dac8c0720ae246be1700afb6fbdbbea20fe35b13f6570b2f70faa005df77/maturin-1.14.1-py3-none-win_arm64.whl", hash = "sha256:15cea8fcb3ba47dd636f50092bb34baea8b04ac777392f23e6bf8a9a61efb894", size = 9718943, upload-time = "2026-06-19T05:19:47.49Z" }, +] + [[package]] name = "mistune" version = "3.2.0" @@ -1644,6 +1668,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] +[[package]] +name = "pytest-timeout" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "pytest", version = "9.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/82/4c9ecabab13363e72d880f2fb504c5f750433b2b6f16e99f4ec21ada284c/pytest_timeout-2.4.0.tar.gz", hash = "sha256:7e68e90b01f9eff71332b25001f85c75495fc4e3a836701876183c4bcfd0540a", size = 17973, upload-time = "2025-05-05T19:44:34.99Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2", size = 14382, upload-time = "2025-05-05T19:44:33.502Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -2313,8 +2350,10 @@ dev = [ { name = "ipykernel", version = "6.31.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "ipykernel", version = "7.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "jupyter" }, + { name = "maturin" }, { name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "pytest", version = "9.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "pytest-timeout" }, ] [package.metadata] @@ -2323,7 +2362,9 @@ dev = [ dev = [ { name = "ipykernel" }, { name = "jupyter" }, + { name = "maturin", specifier = ">=1.7,<2" }, { name = "pytest" }, + { name = "pytest-timeout" }, ] [[package]] diff --git a/crates/bindings/wasm-runtime/Cargo.toml b/crates/bindings/wasm-runtime/Cargo.toml index a6704bb..9202621 100644 --- a/crates/bindings/wasm-runtime/Cargo.toml +++ b/crates/bindings/wasm-runtime/Cargo.toml @@ -11,3 +11,6 @@ crate-type = ["cdylib"] swactor = { path = "../../..", default-features = false, features = ["wasm", "std"] } swactor-engine = { path = "../../engine", default-features = false } wasm-bindgen = "0.2" + +[lints] +workspace = true diff --git a/crates/bindings/wasm-runtime/src/lib.rs b/crates/bindings/wasm-runtime/src/lib.rs index 7c817ad..9d0ed74 100644 --- a/crates/bindings/wasm-runtime/src/lib.rs +++ b/crates/bindings/wasm-runtime/src/lib.rs @@ -103,6 +103,12 @@ pub struct WasmRuntime { backend: SteppingBackend, } +impl Default for WasmRuntime { + fn default() -> Self { + Self::new() + } +} + #[wasm_bindgen] impl WasmRuntime { #[wasm_bindgen(constructor)] diff --git a/crates/data-plane/tests/namespace_host_read_guarantees.rs b/crates/data-plane/tests/namespace_host_read_guarantees.rs old mode 100755 new mode 100644 diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 8b598cc..b24e796 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -15,3 +15,6 @@ tokio = { workspace = true, optional = true } [dev-dependencies] proptest = "1" + +[lints] +workspace = true diff --git a/crates/engine/src/engine.rs b/crates/engine/src/engine.rs index f26207d..c0233ba 100644 --- a/crates/engine/src/engine.rs +++ b/crates/engine/src/engine.rs @@ -18,8 +18,7 @@ use swactor::runtime::{ExternalSender, Runtime, RuntimeParts}; pub struct Engine { /// Retained so the engine owns the runtime handle it drives for its full /// lifetime. Core workers are moved into substrate tasks at construction. - #[allow(dead_code)] - runtime: Runtime, + _runtime: Runtime, backend: Arc, } @@ -41,7 +40,10 @@ impl Engine { // no Tokio feature gate — so core progression does not silently // disappear when an alternate backend is used (ENGINE_SPEC.md). crate::core_driver::install(workers, &backend); - Ok(Engine { runtime, backend }) + Ok(Engine { + _runtime: runtime, + backend, + }) } /// Return a clonable handle for scheduling engine work. diff --git a/crates/engine/tests/common/mod.rs b/crates/engine/tests/common/mod.rs index 63721c5..73c7e62 100644 --- a/crates/engine/tests/common/mod.rs +++ b/crates/engine/tests/common/mod.rs @@ -3,8 +3,6 @@ //! Imports only public `swactor` APIs and exposes no private engine state. See //! `ENGINE_SPEC.md`. -#![allow(dead_code)] - use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::time::{Duration, Instant}; @@ -23,9 +21,10 @@ pub fn default_runtime_parts() -> (RuntimeParts, Runtime) { runtime_parts(RuntimeConfig::default()) } pub fn runtime_parts_with_workers(worker_count: usize) -> (RuntimeParts, Runtime) { - let mut config = RuntimeConfig::default(); - config.worker_count = worker_count; - runtime_parts(config) + runtime_parts(RuntimeConfig { + worker_count, + ..RuntimeConfig::default() + }) } pub fn default_parts() -> RuntimeParts { diff --git a/crates/engine/tests/engine_contract.rs b/crates/engine/tests/engine_contract.rs index 861f904..e9e92af 100644 --- a/crates/engine/tests/engine_contract.rs +++ b/crates/engine/tests/engine_contract.rs @@ -10,7 +10,7 @@ //! These tests exercise the native Tokio backend specifically; the //! non-Tokio portability proof lives in `engine_unit.rs`. -mod common; +pub mod common; use common::*; use std::sync::Arc; diff --git a/crates/engine/tests/engine_unit.rs b/crates/engine/tests/engine_unit.rs index 1c0ab4b..e8d5784 100644 --- a/crates/engine/tests/engine_unit.rs +++ b/crates/engine/tests/engine_unit.rs @@ -5,7 +5,7 @@ //! These tests exercise internal logic directly and use the [`SteppingBackend`] //! to prove substrate independence without Tokio (ENGINE_SPEC.md). -mod common; +pub mod common; use common::*; use std::sync::Arc; @@ -1318,8 +1318,7 @@ fn engine_instant_is_ordered() { /// A tasks-only probe backend that shares a sentinel `Arc<()>` so the test can /// observe exactly when the engine's strong backend reference is released. struct SentinelBackend { - #[allow(dead_code)] - sentinel: Arc<()>, + _sentinel: Arc<()>, } impl ExecutionBackend for SentinelBackend { @@ -1347,7 +1346,7 @@ fn dropping_engine_releases_backend_even_with_live_handles() { let engine = Engine::new( parts, SentinelBackend { - sentinel: sentinel.clone(), + _sentinel: sentinel.clone(), }, ) .expect("tasks capability present"); diff --git a/crates/transport/Cargo.toml b/crates/transport/Cargo.toml index 6b9a404..dfb6b2b 100644 --- a/crates/transport/Cargo.toml +++ b/crates/transport/Cargo.toml @@ -14,3 +14,6 @@ ed25519-dalek = { version = "2", features = ["std", "rand_core", "serde"] } rand_core = { version = "0.6", features = ["getrandom"] } serde = { version = "1", features = ["derive"] } serde_json = "1" + +[lints] +workspace = true diff --git a/crates/transport/src/codec.rs b/crates/transport/src/codec.rs index 2a15736..21320a9 100644 --- a/crates/transport/src/codec.rs +++ b/crates/transport/src/codec.rs @@ -51,7 +51,7 @@ pub fn hex_encode(bytes: &[u8]) -> String { /// Hex-decode a string into bytes. Returns `None` on invalid input. pub fn hex_decode(hex: &str) -> Option> { - if hex.len() % 2 != 0 { + if !hex.len().is_multiple_of(2) { return None; } let mut bytes = Vec::with_capacity(hex.len() / 2); diff --git a/crates/transport/src/identity.rs b/crates/transport/src/identity.rs index 6310cf7..36c689d 100644 --- a/crates/transport/src/identity.rs +++ b/crates/transport/src/identity.rs @@ -19,7 +19,7 @@ pub fn hex_encode(bytes: &[u8]) -> String { /// Hex-decode a string into bytes. Returns `None` on invalid input. pub fn hex_decode(hex: &str) -> Option> { - if hex.len() % 2 != 0 { + if !hex.len().is_multiple_of(2) { return None; } let mut out = Vec::with_capacity(hex.len() / 2); diff --git a/crates/transport/tests/transport_api.rs b/crates/transport/tests/transport_api.rs index f80e92e..161b2e0 100644 --- a/crates/transport/tests/transport_api.rs +++ b/crates/transport/tests/transport_api.rs @@ -352,7 +352,9 @@ fn round_trip_across_two_runtimes() { // Verify all 3 replies for i in 0..3u32 { - let pong = inbox_a.try_recv().expect(&format!("missing pong #{i}")); + let pong = inbox_a + .try_recv() + .unwrap_or_else(|| panic!("missing pong #{i}")); assert_eq!(pong, Pong { value: i * 10 + 1 }); } assert!(inbox_a.try_recv().is_none(), "no extra messages"); diff --git a/src/actor.rs b/src/actor.rs index bec68de..e0fa064 100644 --- a/src/actor.rs +++ b/src/actor.rs @@ -611,7 +611,6 @@ pub(crate) struct ResumeSignal; /// Minimal core interface: send, spawn, stop, and extension access. /// Registry methods (naming, monitoring, groups) and timer scheduling /// are provided by extension traits in `swactor-std`. -#[allow(private_interfaces)] pub trait ContextInner { fn send_any(&self, addr: ActorAddress, msg: Box) -> Result<(), Error>; fn spawn_any(&self, request: SpawnRequest); diff --git a/src/admin.rs b/src/admin.rs index 3b339d3..e5f9c08 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -118,6 +118,10 @@ impl Admin { } pub(crate) type AdminBoxedReply = Box; +pub(crate) type AdminGetState = + Box AdminBoxedReply + Send>; +pub(crate) type AdminReplaceState = + Box AdminResult + Send>; pub(crate) enum AdminCommand { ListActors { @@ -130,17 +134,13 @@ pub(crate) enum AdminCommand { GetActorState { actor: ActorAddress, reply_to: ActorAddress, - get: Box< - dyn FnOnce(ActorAddress, &dyn AnyActor, ActorTypeMetadata) -> AdminBoxedReply + Send, - >, + get: AdminGetState, not_found: Box AdminBoxedReply + Send>, }, ReplaceActorState { actor: ActorAddress, reply_to: ActorAddress, - replace: Box< - dyn FnOnce(&mut dyn AnyActor, ActorTypeMetadata) -> AdminResult + Send, - >, + replace: AdminReplaceState, }, StopActor { actor: ActorAddress, diff --git a/src/worker.rs b/src/worker.rs index 8e66d5e..683b2ea 100644 --- a/src/worker.rs +++ b/src/worker.rs @@ -1,6 +1,7 @@ use crate::Instant; use std::any::Any; use std::cell::RefCell; +use std::cmp::Reverse; use std::collections::{HashMap, VecDeque}; use std::sync::Arc; use std::sync::atomic::Ordering; @@ -124,7 +125,7 @@ impl Worker { || self .worker_ext .as_ref() - .map_or(false, |e| e.has_pending_work()) + .is_some_and(|extension| extension.has_pending_work()) } /// Run one synchronous worker pass. Returns `true` if any work was done. @@ -928,7 +929,7 @@ impl ActorPool { let snap_depth = slot.mailbox.len(); let mut snap_type_counts: Vec<(&'static str, u64)> = slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect(); - snap_type_counts.sort_by(|a, b| b.1.cmp(&a.1)); + snap_type_counts.sort_by_key(|&(_, count)| Reverse(count)); let ctx = Ctx::new( wctx, @@ -1132,7 +1133,7 @@ impl ActorPool { if is_on_stop_eligible(slot.stopping, slot.poisoned) { let mut type_counts: Vec<(&'static str, u64)> = slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect(); - type_counts.sort_by(|a, b| b.1.cmp(&a.1)); + type_counts.sort_by_key(|&(_, count)| Reverse(count)); let ctx = Ctx::new( inner, addr, @@ -1159,7 +1160,7 @@ impl ActorPool { out.extend(self.actors.iter().map(|(&addr, slot)| { let mut type_counts: Vec<(&'static str, u64)> = slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect(); - type_counts.sort_by(|a, b| b.1.cmp(&a.1)); + type_counts.sort_by_key(|&(_, count)| Reverse(count)); let metadata = slot.actor.metadata(); ActorSnapshot { address: addr, diff --git a/tests/actor_lifecycle.rs b/tests/actor_lifecycle.rs index 646791a..6110be8 100644 --- a/tests/actor_lifecycle.rs +++ b/tests/actor_lifecycle.rs @@ -3,7 +3,7 @@ //! Covers: spawning, on_start, lifecycle decision paths, parent-child delegation, //! graceful stop, panic isolation, dead actor cleanup, and watching (ActorExited). -mod common; +pub mod common; use common::*; use parking_lot::Mutex; diff --git a/tests/common/mod.rs b/tests/common/mod.rs index db7b3ae..7e2e864 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,7 +1,5 @@ // Shared types and helpers for runtime test files. -#![allow(dead_code, unused_imports)] - use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; diff --git a/tests/core_extension_seams.rs b/tests/core_extension_seams.rs index 5ccfcdf..ce46cc8 100644 --- a/tests/core_extension_seams.rs +++ b/tests/core_extension_seams.rs @@ -64,10 +64,6 @@ impl RuntimeExtension for SeamExtension { &self, dead: &[(ActorAddress, StopReason, Option)], ) -> Vec<(ActorAddress, Box)> { - let _ = dead - .iter() - .map(|(_, reason, value)| (reason, value)) - .count(); let Some(report_to) = *self.state.death_report_to.lock() else { return Vec::new(); }; diff --git a/tests/lint_policy.rs b/tests/lint_policy.rs new file mode 100644 index 0000000..1b5fefc --- /dev/null +++ b/tests/lint_policy.rs @@ -0,0 +1,205 @@ +use std::collections::BTreeSet; +use std::fs; +use std::path::{Path, PathBuf}; + +use syn::visit::Visit; +use syn::{Attribute, Meta}; + +const SOURCE_ROOTS: &[&str] = &["src", "crates", "apps", "xtask", "tools", "tests"]; +const FORBIDDEN_ATTRIBUTES: &[&str] = &["allow", "expect"]; + +#[derive(Default)] +struct SuppressionVisitor { + found: bool, +} + +impl<'ast> Visit<'ast> for SuppressionVisitor { + fn visit_attribute(&mut self, attribute: &'ast Attribute) { + if FORBIDDEN_ATTRIBUTES + .iter() + .any(|name| attribute.path().is_ident(name)) + || matches!( + &attribute.meta, + Meta::List(list) + if list.path.is_ident("cfg_attr") && tokens_contain_suppression(&list.tokens) + ) + { + self.found = true; + } + syn::visit::visit_attribute(self, attribute); + } +} + +#[test] +fn workspace_sources_do_not_suppress_lints() { + let workspace = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let mut rust_sources = Vec::new(); + for root in SOURCE_ROOTS { + collect_rust_sources(&workspace.join(root), &mut rust_sources); + } + rust_sources.sort(); + + let mut violations = Vec::new(); + for source in rust_sources { + let text = fs::read_to_string(&source) + .unwrap_or_else(|error| panic!("read {}: {error}", source.display())); + let syntax = syn::parse_file(&text) + .unwrap_or_else(|error| panic!("parse {}: {error}", source.display())); + let mut visitor = SuppressionVisitor::default(); + visitor.visit_file(&syntax); + if visitor.found { + violations.push( + source + .strip_prefix(&workspace) + .unwrap_or(&source) + .display() + .to_string(), + ); + } + } + + assert!( + violations.is_empty(), + "lint suppression attributes are forbidden; fix the warning instead:\n{}", + violations.join("\n") + ); +} + +#[test] +fn suppression_visitor_detects_direct_and_conditional_attributes() { + for source in [ + "#[allow(dead_code)] fn hidden() {}", + "#![expect(unused_imports)]", + "#[cfg_attr(test, allow(clippy::too_many_arguments))] fn hidden() {}", + "#![cfg_attr(feature = \"strict\", expect(dead_code))]", + ] { + let syntax = syn::parse_file(source).expect("valid suppression probe"); + let mut visitor = SuppressionVisitor::default(); + visitor.visit_file(&syntax); + assert!(visitor.found, "suppression escaped detection: {source}"); + } + + let syntax = syn::parse_file("#[derive(Clone)] struct Clean;").unwrap(); + let mut visitor = SuppressionVisitor::default(); + visitor.visit_file(&syntax); + assert!(!visitor.found); +} + +#[test] +fn source_tree_test_modules_are_wired() { + let workspace = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let mut test_directories = Vec::new(); + for root in SOURCE_ROOTS { + collect_source_test_directories(&workspace.join(root), &mut test_directories); + } + test_directories.sort(); + + let mut unwired = Vec::new(); + for directory in test_directories { + let module_path = directory.join("mod.rs"); + let module_source = fs::read_to_string(&module_path) + .unwrap_or_else(|error| panic!("read {}: {error}", module_path.display())); + let module = syn::parse_file(&module_source) + .unwrap_or_else(|error| panic!("parse {}: {error}", module_path.display())); + let declared = module + .items + .iter() + .filter_map(|item| match item { + syn::Item::Mod(item) => Some(item.ident.to_string()), + _ => None, + }) + .collect::>(); + + for entry in fs::read_dir(&directory) + .unwrap_or_else(|error| panic!("read directory {}: {error}", directory.display())) + { + let path = entry.expect("read test-module entry").path(); + if path.extension().is_some_and(|extension| extension == "rs") + && path.file_stem().is_some_and(|stem| stem != "mod") + { + let module_name = path + .file_stem() + .expect("test module stem") + .to_string_lossy() + .to_string(); + if !declared.contains(&module_name) { + unwired.push( + path.strip_prefix(&workspace) + .unwrap_or(&path) + .display() + .to_string(), + ); + } + } + } + } + + assert!( + unwired.is_empty(), + "source-tree test modules must be declared by their adjacent mod.rs:\n{}", + unwired.join("\n") + ); +} + +fn collect_source_test_directories(directory: &Path, directories: &mut Vec) { + if !directory.exists() { + return; + } + if directory.file_name().is_some_and(|name| name == "tests") + && directory + .parent() + .and_then(Path::file_name) + .is_some_and(|name| name == "src") + { + directories.push(directory.to_path_buf()); + return; + } + let entries = fs::read_dir(directory) + .unwrap_or_else(|error| panic!("read directory {}: {error}", directory.display())); + for entry in entries { + let path = entry.expect("read source-tree entry").path(); + if path.is_dir() { + collect_source_test_directories(&path, directories); + } + } +} + +fn collect_rust_sources(directory: &Path, sources: &mut Vec) { + if !directory.exists() { + return; + } + let entries = fs::read_dir(directory) + .unwrap_or_else(|error| panic!("read directory {}: {error}", directory.display())); + for entry in entries { + let entry = entry.unwrap_or_else(|error| panic!("read directory entry: {error}")); + let path = entry.path(); + if path.is_dir() { + collect_rust_sources(&path, sources); + } else if path.extension().is_some_and(|extension| extension == "rs") { + sources.push(path); + } + } +} + +fn tokens_contain_suppression(tokens: &proc_macro2::TokenStream) -> bool { + let mut token_trees = tokens.clone().into_iter().peekable(); + while let Some(token) = token_trees.next() { + match token { + proc_macro2::TokenTree::Ident(identifier) + if FORBIDDEN_ATTRIBUTES.iter().any(|name| identifier == *name) + && matches!( + token_trees.peek(), + Some(proc_macro2::TokenTree::Group(group)) + if group.delimiter() == proc_macro2::Delimiter::Parenthesis + ) => + { + return true; + } + proc_macro2::TokenTree::Group(group) if tokens_contain_suppression(&group.stream()) => { + return true; + } + _ => {} + } + } + false +} diff --git a/tests/message_delivery.rs b/tests/message_delivery.rs index 05ce7fa..becd54a 100644 --- a/tests/message_delivery.rs +++ b/tests/message_delivery.rs @@ -3,7 +3,7 @@ //! Covers: routing correctness at scale, send-from-within-handler patterns, //! address error handling, and fairness/budgets. -mod common; +pub mod common; use common::*; use std::sync::Arc; diff --git a/tests/multicore.rs b/tests/multicore.rs index dc1d601..1f1c668 100644 --- a/tests/multicore.rs +++ b/tests/multicore.rs @@ -4,7 +4,7 @@ //! `docs/specs/drafts/MULTICORE_SPEC.md`. They observe behavior through public //! APIs only — never inspecting source layout. -mod common; +pub mod common; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -157,9 +157,10 @@ fn worker_of(stats: &swactor::stats::RuntimeStats, addr: ActorAddress) -> usize } fn config_with(workers: usize) -> RuntimeConfig { - let mut c = RuntimeConfig::default(); - c.worker_count = workers; - c + RuntimeConfig { + worker_count: workers, + ..RuntimeConfig::default() + } } // ─── Phase 1: single-thread host advances every worker once ───────────────── diff --git a/tests/proptest_runtime.rs b/tests/proptest_runtime.rs index 7fe28ec..a929f02 100644 --- a/tests/proptest_runtime.rs +++ b/tests/proptest_runtime.rs @@ -3,7 +3,7 @@ //! Uses proptest for randomized testing and proptest-state-machine for //! stateful property testing with automatic shrinking of failing sequences. -mod common; +pub mod common; use common::plain_host; use std::collections::HashMap; diff --git a/tests/runtime_admin.rs b/tests/runtime_admin.rs index c74b6d8..537ae8a 100644 --- a/tests/runtime_admin.rs +++ b/tests/runtime_admin.rs @@ -1,6 +1,6 @@ //! Runtime Admin API tests — inventory, typed actor state, lifecycle control, and scheduling. -mod common; +pub mod common; use common::*; use std::collections::HashSet; diff --git a/tests/runtime_stress.rs b/tests/runtime_stress.rs index 2e65cd7..47ef97f 100644 --- a/tests/runtime_stress.rs +++ b/tests/runtime_stress.rs @@ -3,7 +3,7 @@ //! Covers: high-volume delivery, panic isolation under load, and sustained //! throughput with no message loss. All tests are tick-driven (single worker). -mod common; +pub mod common; use common::*; use std::sync::Arc; diff --git a/tests/std_extension.rs b/tests/std_extension.rs index dd0b3ac..abcf9a7 100644 --- a/tests/std_extension.rs +++ b/tests/std_extension.rs @@ -3,7 +3,7 @@ //! Tests only the std APIs used by production crates: runtime naming, //! runtime groups, actor-side watch, actor-side group join, and extension install. -mod common; +pub mod common; use common::*; #[derive(Clone)] diff --git a/tools/actor-control-flow-lint/Cargo.toml b/tools/actor-control-flow-lint/Cargo.toml index 70380fb..09bf5cc 100644 --- a/tools/actor-control-flow-lint/Cargo.toml +++ b/tools/actor-control-flow-lint/Cargo.toml @@ -7,3 +7,6 @@ publish = false [lib] path = "src/lib.rs" + +[lints] +workspace = true diff --git a/tools/actor-control-flow-lint/tests/fixtures/fail-domain-capabilities/Cargo.lock b/tools/actor-control-flow-lint/tests/fixtures/fail-domain-capabilities/Cargo.lock index e9ec87b..fc81c68 100644 --- a/tools/actor-control-flow-lint/tests/fixtures/fail-domain-capabilities/Cargo.lock +++ b/tools/actor-control-flow-lint/tests/fixtures/fail-domain-capabilities/Cargo.lock @@ -10,6 +10,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "bitflags" version = "2.13.1" @@ -182,6 +188,7 @@ dependencies = [ name = "swactor" version = "0.1.0" dependencies = [ + "atomic-waker", "crossbeam-queue", "crossbeam-utils", "getrandom", diff --git a/tools/actor-control-flow-lint/tests/fixtures/fail-domain-capabilities/src/lib.rs b/tools/actor-control-flow-lint/tests/fixtures/fail-domain-capabilities/src/lib.rs index c313de1..8cc0e60 100755 --- a/tools/actor-control-flow-lint/tests/fixtures/fail-domain-capabilities/src/lib.rs +++ b/tools/actor-control-flow-lint/tests/fixtures/fail-domain-capabilities/src/lib.rs @@ -1,4 +1,3 @@ -#![allow(clippy::disallowed_methods, dead_code, unused_must_use)] use std::thread::sleep as renamed_sleep; use std::time::Duration; diff --git a/tools/actor-control-flow-lint/tests/fixtures/pass-actor-domain/Cargo.lock b/tools/actor-control-flow-lint/tests/fixtures/pass-actor-domain/Cargo.lock index 35f7d52..05e7ac1 100644 --- a/tools/actor-control-flow-lint/tests/fixtures/pass-actor-domain/Cargo.lock +++ b/tools/actor-control-flow-lint/tests/fixtures/pass-actor-domain/Cargo.lock @@ -10,6 +10,12 @@ dependencies = [ "swactor-engine", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "bitflags" version = "2.13.1" @@ -182,6 +188,7 @@ dependencies = [ name = "swactor" version = "0.1.0" dependencies = [ + "atomic-waker", "crossbeam-queue", "crossbeam-utils", "getrandom", diff --git a/tools/vastai/Cargo.toml b/tools/vastai/Cargo.toml index 413d382..63f953d 100644 --- a/tools/vastai/Cargo.toml +++ b/tools/vastai/Cargo.toml @@ -18,3 +18,6 @@ urlencoding = "2" [dev-dependencies] wiremock = "0.6" + +[lints] +workspace = true diff --git a/tools/vastai/src/config.rs b/tools/vastai/src/config.rs index 8a56a88..39a186e 100644 --- a/tools/vastai/src/config.rs +++ b/tools/vastai/src/config.rs @@ -48,50 +48,53 @@ fn env_optional_positive_f64(name: &str) -> Option { impl SelectionPolicy { /// Build selection policy from the historical `PP_*` environment knobs. pub fn from_env() -> Self { - let mut policy = Self::default(); - policy.min_gpu_ram_mb = env_positive_u64(ENV_GPU_MIN_RAM_MB); - if let Some(min_compute_cap) = env_positive_u64(ENV_MIN_COMPUTE_CAP) { - policy.min_compute_cap = Some(min_compute_cap); - } - policy.min_down_mbps = env_nonnegative_f64(ENV_MIN_INET_DOWN_MBPS, 100.0); - policy.min_reliability = std::env::var(ENV_MIN_RELIABILITY) - .ok() - .and_then(|s| s.trim().parse::().ok()) - .filter(|&v| (0.0..=1.0).contains(&v)) - .unwrap_or(0.95); - policy.require_verified = truthy_env(ENV_REQUIRE_VERIFIED); - policy.min_up_mbps = env_optional_positive_f64(ENV_MIN_INET_UP_MBPS); - policy.max_dph_total = env_optional_positive_f64(ENV_MAX_DPH_TOTAL); - policy.drop_cheap_frac = std::env::var(ENV_DROP_CHEAP_FRAC) - .ok() - .and_then(|s| s.trim().parse::().ok()) - .filter(|v| v.is_finite()) - .map(|v| v.clamp(0.0, 0.99)) - .unwrap_or(0.30); - policy.image_size_gb = env_optional_positive_f64(ENV_IMAGE_SIZE_GB); + let defaults = Self::default(); + let mut blacklist_hosts = defaults.blacklist_hosts; if let Ok(raw) = std::env::var(ENV_BLACKLIST_HOSTS) { - policy - .blacklist_hosts - .extend(raw.split(',').filter_map(|s| s.trim().parse::().ok())); + blacklist_hosts.extend( + raw.split(',') + .filter_map(|value| value.trim().parse::().ok()), + ); + } + Self { + gpu_name: defaults.gpu_name, + min_gpu_ram_mb: env_positive_u64(ENV_GPU_MIN_RAM_MB), + min_compute_cap: env_positive_u64(ENV_MIN_COMPUTE_CAP).or(defaults.min_compute_cap), + min_down_mbps: env_nonnegative_f64(ENV_MIN_INET_DOWN_MBPS, 100.0), + min_reliability: std::env::var(ENV_MIN_RELIABILITY) + .ok() + .and_then(|value| value.trim().parse::().ok()) + .filter(|&value| (0.0..=1.0).contains(&value)) + .unwrap_or(0.95), + require_verified: truthy_env(ENV_REQUIRE_VERIFIED), + min_up_mbps: env_optional_positive_f64(ENV_MIN_INET_UP_MBPS), + max_dph_total: env_optional_positive_f64(ENV_MAX_DPH_TOTAL), + blacklist_hosts, + drop_cheap_frac: std::env::var(ENV_DROP_CHEAP_FRAC) + .ok() + .and_then(|value| value.trim().parse::().ok()) + .filter(|value| value.is_finite()) + .map(|value| value.clamp(0.0, 0.99)) + .unwrap_or(0.30), + image_size_gb: env_optional_positive_f64(ENV_IMAGE_SIZE_GB), } - policy } } impl LifecyclePolicy { /// Build lifecycle policy from environment, using caller-provided poll cadence. pub fn from_env(poll_interval: Duration) -> Self { - let mut policy = Self::default(); - policy.lease_pace = Duration::from_millis( - std::env::var(ENV_LEASE_PACE_MS) - .ok() - .and_then(|s| s.trim().parse::().ok()) - .unwrap_or(600), - ); - policy.poll_interval = poll_interval; - if let Some(state_timeout_secs) = env_positive_u64(ENV_STATE_TIMEOUT_SECS) { - policy.state_timeout = Duration::from_secs(state_timeout_secs); + Self { + lease_pace: Duration::from_millis( + std::env::var(ENV_LEASE_PACE_MS) + .ok() + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or(600), + ), + poll_interval, + state_timeout: env_positive_u64(ENV_STATE_TIMEOUT_SECS) + .map(Duration::from_secs) + .unwrap_or_else(|| Self::default().state_timeout), } - policy } } diff --git a/tools/vastai/src/filters.rs b/tools/vastai/src/filters.rs index ea26d6f..79534aa 100644 --- a/tools/vastai/src/filters.rs +++ b/tools/vastai/src/filters.rs @@ -9,9 +9,9 @@ pub(crate) fn reachable_offers(offers: Vec, policy: &SelectionPolicy) -> .filter(|o| { o.geolocation .as_deref() - .map_or(false, |g| !g.to_uppercase().contains("CN")) + .is_some_and(|geolocation| !geolocation.to_uppercase().contains("CN")) }) - .filter(|o| o.host_id.map_or(true, |h| !blacklist.contains(&h))) + .filter(|offer| offer.host_id.is_none_or(|host| !blacklist.contains(&host))) .filter(|o| o.verification.as_deref() != Some("deverified")) .filter(|o| policy.max_dph_total.is_none_or(|max| o.dph_total <= max)) .filter(|o| { diff --git a/tools/vastai/src/lease.rs b/tools/vastai/src/lease.rs index 095515f..9603ef7 100644 --- a/tools/vastai/src/lease.rs +++ b/tools/vastai/src/lease.rs @@ -83,14 +83,14 @@ fn next_eligible_offer<'a>( failed_host_ids: &HashSet, preferred_offer_id: Option, ) -> Option<&'a Offer> { - if let Some(offer_id) = preferred_offer_id { - if let Some(offer) = pool.iter().find(|o| o.id == offer_id) { - if !tried_offer_ids.contains(&offer.id) - && offer.host_id.is_none_or(|h| !failed_host_ids.contains(&h)) - { - return Some(offer); - } - } + if let Some(offer_id) = preferred_offer_id + && let Some(offer) = pool.iter().find(|offer| offer.id == offer_id) + && !tried_offer_ids.contains(&offer.id) + && offer + .host_id + .is_none_or(|host| !failed_host_ids.contains(&host)) + { + return Some(offer); } pool.iter().find(|o| { @@ -108,18 +108,32 @@ fn env_for_index(req: &ProvisionRequest, index: u32) -> BTreeMap env } -async fn provision_one( - client: &reqwest::Client, - base_url: &str, - api_key: &str, - req: &ProvisionRequest, - pool: &[Offer], +struct ProvisionOneRequest<'a> { + client: &'a reqwest::Client, + base_url: &'a str, + api_key: &'a str, + request: &'a ProvisionRequest, + pool: &'a [Offer], index: u32, - tried_offer_ids: &mut Vec, - used_host_ids: &mut HashSet, - failed_host_ids: &mut HashSet, + tried_offer_ids: &'a mut Vec, + used_host_ids: &'a mut HashSet, + failed_host_ids: &'a mut HashSet, preferred_offer_id: Option, -) -> Result { +} + +async fn provision_one(request: ProvisionOneRequest<'_>) -> Result { + let ProvisionOneRequest { + client, + base_url, + api_key, + request: req, + pool, + index, + tried_offer_ids, + used_host_ids, + failed_host_ids, + preferred_offer_id, + } = request; let mut attempt = 1_u64; loop { let offer = match next_eligible_offer( @@ -230,20 +244,21 @@ pub async fn provision_fleet( let mut used_host_ids = HashSet::new(); let mut failed_host_ids = HashSet::new(); for index in 0..req.count { - match provision_one( + match provision_one(ProvisionOneRequest { client, base_url, api_key, - &req, - &pool, + request: &req, + pool: &pool, index, - &mut tried_offer_ids, - &mut used_host_ids, - &mut failed_host_ids, - req.preferred_offer_id + tried_offer_ids: &mut tried_offer_ids, + used_host_ids: &mut used_host_ids, + failed_host_ids: &mut failed_host_ids, + preferred_offer_id: req + .preferred_offer_id .filter(|_| req.count == 1) .or_else(|| first_wave.get(index as usize).map(|offer| offer.id)), - ) + }) .await { Ok(info) => created.push(info), @@ -279,18 +294,18 @@ pub async fn provision_fleet( ); } eprintln!("lease_chain: replacing index {index}"); - match provision_one( + match provision_one(ProvisionOneRequest { client, base_url, api_key, - &req, - &pool, + request: &req, + pool: &pool, index, - &mut tried_offer_ids, - &mut used_host_ids, - &mut failed_host_ids, - None, - ) + tried_offer_ids: &mut tried_offer_ids, + used_host_ids: &mut used_host_ids, + failed_host_ids: &mut failed_host_ids, + preferred_offer_id: None, + }) .await { Ok(info) => created[idx] = info, diff --git a/tools/vastai/src/search.rs b/tools/vastai/src/search.rs index c0962ce..809109a 100644 --- a/tools/vastai/src/search.rs +++ b/tools/vastai/src/search.rs @@ -151,10 +151,10 @@ pub fn plan_distinct_host_first_wave( if selected.len() == target { break; } - if let Some(host_id) = offer.host_id { - if !selected_hosts.insert(host_id) { - continue; - } + if let Some(host_id) = offer.host_id + && !selected_hosts.insert(host_id) + { + continue; } selected_ids.insert(offer.id); selected.push(offer.clone()); diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index ac949c5..ca7df6d 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -25,3 +25,6 @@ libc = "0.2" [dev-dependencies] proptest = "1" + +[lints] +workspace = true diff --git a/xtask/src/demo/control.rs b/xtask/src/demo/control.rs index 50c967f..882ff63 100644 --- a/xtask/src/demo/control.rs +++ b/xtask/src/demo/control.rs @@ -107,9 +107,10 @@ mod properties { codes in prop::collection::vec(any::(), 0..=32), registration in 0_usize..=32, ) { - let mut config = RuntimeConfig::default(); - config.worker_count = 1; - let parts = RuntimeParts::new(config); + let parts = RuntimeParts::new(RuntimeConfig { + worker_count: 1, + ..RuntimeConfig::default() + }); let runtime = parts.runtime().clone(); let backend = SteppingBackend::new(); let _engine = diff --git a/xtask/src/demo/edge.rs b/xtask/src/demo/edge.rs index 3bb0138..71ab866 100644 --- a/xtask/src/demo/edge.rs +++ b/xtask/src/demo/edge.rs @@ -503,7 +503,6 @@ pub struct NodeEdgeAgent { } impl NodeEdgeAgent { - #[allow(clippy::too_many_arguments)] pub fn new( attempt: u64, logical_node: String, @@ -756,7 +755,7 @@ impl swactor::actor::ActorInterface for EdgeAckRelay { if let Some(addr) = self.supervisor.get() { let _ = self .sender - .send_to(addr.clone(), crate::demo::feed::SupervisorMsg::EdgeAck(ack)); + .send_to(*addr, crate::demo::feed::SupervisorMsg::EdgeAck(ack)); } } } diff --git a/xtask/src/demo/feed.rs b/xtask/src/demo/feed.rs index 4005b96..ebfc2d2 100644 --- a/xtask/src/demo/feed.rs +++ b/xtask/src/demo/feed.rs @@ -215,25 +215,43 @@ pub struct SupervisorActor { shutdown: Option, } +pub struct SupervisorConfig { + pub driver: ClusterDriver, + pub executor: IdempotentEffectExecutor, + pub manager: NodeManager, + pub driver_handle: std::sync::Arc, + pub telemetry: SupervisorTelemetry, + pub dashboard: dashboard::DashboardHandle, + pub sender: swactor::runtime::ExternalSender, + pub registry: provisioning::BootstrapRegistry, + pub collector: std::sync::Arc, + pub engine: EngineHandle, + pub remote_sub: telemetry::TelemetrySubscription, + pub initial_slots: Vec, + pub run_id: RunId, + pub launch: crate::demo::LaunchStyle, + pub edge_actor: ActorAddress, +} + impl SupervisorActor { - #[allow(clippy::too_many_arguments)] - pub fn new( - driver: ClusterDriver, - executor: IdempotentEffectExecutor, - manager: NodeManager, - driver_handle: std::sync::Arc, - mut telemetry: SupervisorTelemetry, - dashboard: dashboard::DashboardHandle, - sender: swactor::runtime::ExternalSender, - registry: provisioning::BootstrapRegistry, - collector: std::sync::Arc, - engine: EngineHandle, - remote_sub: telemetry::TelemetrySubscription, - initial_slots: Vec, - run_id: RunId, - launch: crate::demo::LaunchStyle, - edge_actor: ActorAddress, - ) -> Self { + pub fn new(config: SupervisorConfig) -> Self { + let SupervisorConfig { + driver, + executor, + manager, + driver_handle, + mut telemetry, + dashboard, + sender, + registry, + collector, + engine, + remote_sub, + initial_slots, + run_id, + launch, + edge_actor, + } = config; let events_channel = telemetry.register("prov.reconciler.events"); let snapshot_channel = telemetry.register("prov.reconciler.snapshot"); Self { @@ -814,7 +832,7 @@ impl SupervisorActor { /// cards and the control view stay live between lifecycle transitions. fn emit_node_status(&mut self, now: SystemTime) { self.status_tick = self.status_tick.wrapping_add(1); - if self.status_tick % 4 != 0 { + if !self.status_tick.is_multiple_of(4) { return; // 250ms ticks → heartbeat every second } let attempts: Vec = self.nodes.keys().copied().collect(); @@ -968,7 +986,6 @@ impl SupervisorActor { } } -#[allow(clippy::too_many_arguments)] fn publish_frame( dashboard: &dashboard::DashboardHandle, stream: &telemetry::frame::StreamId, @@ -1489,7 +1506,7 @@ mod properties { fn control_command(kind: u8, attempt: u64, index: usize) -> dashboard::control::ControlCommand { let command_id = format!("generated-command-{index}"); - if kind % 2 == 0 { + if kind.is_multiple_of(2) { dashboard::control::ControlCommand::Kill { command_id, node: format!("node-{attempt}"), @@ -1568,9 +1585,10 @@ mod properties { fn generated_supervisor_transitions_are_once_only_nonblocking_and_clean( actions in supervisor_actions(), ) { - let mut config = RuntimeConfig::default(); - config.worker_count = 1; - let parts = RuntimeParts::new(config); + let parts = RuntimeParts::new(RuntimeConfig { + worker_count: 1, + ..RuntimeConfig::default() + }); let runtime = parts.runtime().clone(); let sender = runtime.create_sender(); let backend = SteppingBackend::new(); @@ -1644,25 +1662,25 @@ mod properties { .spawn(AnnounceActor::new(manager.clone(), sender.clone())) .expect("spawn generated announce actor"); let supervisor = runtime - .spawn(SupervisorActor::new( + .spawn(SupervisorActor::new(SupervisorConfig { driver, executor, - manager.clone(), + manager: manager.clone(), driver_handle, telemetry, dashboard, - sender.clone(), + sender: sender.clone(), registry, - Arc::new(evidence.clone()), - engine.handle(), + collector: Arc::new(evidence.clone()), + engine: engine.handle(), remote_sub, - Vec::new(), - RunId(1), - crate::demo::LaunchStyle::Process { + initial_slots: Vec::new(), + run_id: RunId(1), + launch: crate::demo::LaunchStyle::Process { exe: "generated-demo-node".into(), }, - *edge_inbox.addr(), - )) + edge_actor: *edge_inbox.addr(), + })) .expect("spawn generated supervisor actor"); drive(&backend, 8); diff --git a/xtask/src/demo/mod.rs b/xtask/src/demo/mod.rs index 49a3ed4..b021d9e 100644 --- a/xtask/src/demo/mod.rs +++ b/xtask/src/demo/mod.rs @@ -50,8 +50,8 @@ use provisioning::reconciler::ClusterDriver; use provisioning::{ClusterShape, RunId}; use feed::{ - EngineSpawner, SupervisorActor, SupervisorMsg, SupervisorTelemetry, demo_retry_policy, - initial_slots, + EngineSpawner, SupervisorActor, SupervisorConfig, SupervisorMsg, SupervisorTelemetry, + demo_retry_policy, initial_slots, }; use provider::{DemoBackend, DemoProvider, NodeManager}; @@ -68,9 +68,10 @@ pub const DEFAULT_NODES: u64 = 3; pub(super) fn shared_test_driver() -> Arc { thread_local! { static DRIVER: (Arc, Engine) = { - let mut config = RuntimeConfig::default(); - config.worker_count = 1; - let parts = RuntimeParts::new(config); + let parts = RuntimeParts::new(RuntimeConfig { + worker_count: 1, + ..RuntimeConfig::default() + }); let backend = TokioBackend::new(TokioConfig::default()) .expect("create shared demo test backend"); let engine = @@ -99,10 +100,10 @@ pub(super) fn shared_test_driver() -> Arc { /// replaced by a rebuild while this process runs, so fall back through /// argv[0] and PATH. fn resolve_exe() -> std::path::PathBuf { - if let Ok(path) = std::env::current_exe() { - if !path.to_string_lossy().ends_with(" (deleted)") { - return path; - } + if let Ok(path) = std::env::current_exe() + && !path.to_string_lossy().ends_with(" (deleted)") + { + return path; } if let Some(arg0) = std::env::args_os().next() { let candidate = std::path::PathBuf::from(&arg0); @@ -189,12 +190,14 @@ impl provisioning::NodeTelemetryCollector for DemoTelemetryCollector { }; iroh_driver::spawn_pull_collector_to_actor( &self.engine, - self.endpoint.clone(), - addr, - flow_id, - Vec::new(), - telemetry::SubscriptionRequest::all(), - Arc::clone(&self.fanout), + iroh_driver::PullCollectorConfig { + endpoint: self.endpoint.clone(), + peer: addr, + flow_id, + token: Vec::new(), + request: telemetry::SubscriptionRequest::all(), + fanout: Arc::clone(&self.fanout), + }, self.sender.clone(), header_actor, ); @@ -347,15 +350,15 @@ fn run_supervisor(args: &[String]) -> Result<(), String> { let route_view: RouteView = Arc::new(std::sync::RwLock::new(std::collections::HashMap::new())); let outbox: Outbox = Arc::new(std::sync::Mutex::new(Vec::new())); - driver.enable_actor_bridge( - runtime.clone(), - Arc::new(codec), + driver.enable_actor_bridge(iroh_driver::ActorBridgeConfig { + runtime: runtime.clone(), + codec: Arc::new(codec), routes, - announce, + swim: announce, relay_mirror, route_view, outbox, - ); + }); driver.install_actor_bridge_pump(Duration::from_millis(250)); } @@ -458,23 +461,23 @@ fn run_supervisor(args: &[String]) -> Result<(), String> { sender.clone(), supervisor_slot.clone(), )?; - let supervisor = SupervisorActor::new( - cluster_driver, + let supervisor = SupervisorActor::new(SupervisorConfig { + driver: cluster_driver, executor, - manager.clone(), + manager: manager.clone(), driver_handle, telemetry, - dashboard.clone(), - sender.clone(), - bootstrap_registry, + dashboard: dashboard.clone(), + sender: sender.clone(), + registry: bootstrap_registry, collector, - engine.handle(), + engine: engine.handle(), remote_sub, - slots, - RunId(1), - launch.clone(), + initial_slots: slots, + run_id: RunId(1), + launch: launch.clone(), edge_actor, - ); + }); // Control plane: dashboard → supervisor. control::install(&runtime, supervisor_slot.clone()); @@ -483,7 +486,7 @@ fn run_supervisor(args: &[String]) -> Result<(), String> { .spawn(supervisor) .map_err(|e| format!("spawn supervisor actor: {e}"))?; supervisor_slot - .set(supervisor_addr.clone()) + .set(supervisor_addr) .expect("supervisor address slot set once"); manager.set_spawn_actor(sender.clone(), supervisor_slot.clone()); @@ -650,9 +653,10 @@ mod properties { let pid = child.id() as i32; let stdout = child.stdout.take().expect("capture demo supervisor stdout"); - let mut config = RuntimeConfig::default(); - config.worker_count = 1; - let parts = RuntimeParts::new(config); + let parts = RuntimeParts::new(RuntimeConfig { + worker_count: 1, + ..RuntimeConfig::default() + }); let runtime = parts.runtime().clone(); let engine = Engine::new( parts, diff --git a/xtask/src/demo/node.rs b/xtask/src/demo/node.rs index 1690f29..3ac7cb8 100644 --- a/xtask/src/demo/node.rs +++ b/xtask/src/demo/node.rs @@ -179,15 +179,15 @@ pub fn run_node_role(supervisor_addr_json: &str, attempt: u64) -> Result<(), Str let route_view: RouteView = Arc::new(std::sync::RwLock::new(std::collections::HashMap::new())); let outbox: Outbox = Arc::new(std::sync::Mutex::new(Vec::new())); - driver.enable_actor_bridge( - runtime.clone(), - Arc::new(codec), + driver.enable_actor_bridge(iroh_driver::ActorBridgeConfig { + runtime: runtime.clone(), + codec: Arc::new(codec), routes, - edge_agent, + swim: edge_agent, relay_mirror, route_view, outbox, - ); + }); driver.install_actor_bridge_pump(Duration::from_millis(250)); } let driver = Arc::new(driver); @@ -199,7 +199,7 @@ pub fn run_node_role(supervisor_addr_json: &str, attempt: u64) -> Result<(), Str // Join, then announce identity + advertised address to the supervisor's // bootstrap actor over the control plane (readiness + telemetry dial). - driver.join(&[supervisor_addr.clone()]); + driver.join(std::slice::from_ref(&supervisor_addr)); let addr_json = serde_json::to_string(&driver.endpoint_addr()).map_err(|e| format!("addr: {e}"))?; @@ -546,9 +546,10 @@ mod properties { actions in node_actions(), attempt in any::(), ) { - let mut config = RuntimeConfig::default(); - config.worker_count = 1; - let parts = RuntimeParts::new(config); + let parts = RuntimeParts::new(RuntimeConfig { + worker_count: 1, + ..RuntimeConfig::default() + }); let runtime = parts.runtime().clone(); let backend = SteppingBackend::new(); let engine = diff --git a/xtask/src/demo/provider.rs b/xtask/src/demo/provider.rs index a046927..4b56698 100644 --- a/xtask/src/demo/provider.rs +++ b/xtask/src/demo/provider.rs @@ -485,10 +485,12 @@ impl EffectBackend for DemoBackend { .expect("demo provider") .create_node(spec, null_sink()) .map_err(EffectError::definite)?; - Ok(OperationOutcome::LeaseCreated(CreateLeaseResult { - lease: demo_lease(attempt), - endpoint: Some(demo_endpoint()), - })) + Ok(OperationOutcome::LeaseCreated(Box::new( + CreateLeaseResult { + lease: demo_lease(attempt), + endpoint: Some(demo_endpoint()), + }, + ))) } NodeManagerCommand::LookupEndpoint(_) => { Ok(OperationOutcome::EndpointLookup(Some(demo_endpoint()))) @@ -701,9 +703,10 @@ mod properties { actions.push(terminal); } } - let mut config = RuntimeConfig::default(); - config.worker_count = 1; - let parts = RuntimeParts::new(config); + let parts = RuntimeParts::new(RuntimeConfig { + worker_count: 1, + ..RuntimeConfig::default() + }); let runtime = parts.runtime().clone(); let backend = SteppingBackend::new(); let _engine = diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 400dd73..650cbba 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -1,4 +1,5 @@ -use std::process::{Command, ExitCode}; +use std::path::{Path, PathBuf}; +use std::process::{Command, ExitCode, Stdio}; use std::time::Instant; struct TestStep { @@ -6,46 +7,27 @@ struct TestStep { args: &'static [&'static str], } -const BASIC_TESTS: &[TestStep] = &[ +const TEST_STEPS: &[TestStep] = &[ TestStep { - label: "root crate", - args: &["test"], + label: "strict workspace lint", + args: &["lint"], }, TestStep { - label: "telemetry", - args: &["test", "-p", "telemetry"], + label: "all Rust tests (60s per-test timeout)", + args: &["nextest", "run", "--workspace", "--all-features"], }, TestStep { - label: "distribution", - args: &["test", "-p", "distribution"], - }, - TestStep { - label: "iroh-driver", - args: &["test", "-p", "iroh-driver"], - }, - TestStep { - label: "myelin", - args: &["test", "-p", "myelin"], - }, - TestStep { - label: "swactor-process", - args: &["test", "-p", "swactor-process"], - }, - TestStep { - label: "swactor-transport", - args: &["test", "-p", "swactor-transport"], - }, - TestStep { - label: "dashboard", - args: &["test", "-p", "dashboard"], - }, - TestStep { - label: "swactor-vastai", - args: &["test", "-p", "swactor-vastai"], - }, - TestStep { - label: "xtask", - args: &["test", "-p", "xtask"], + label: "all Rust doctests", + args: &[ + "test", + "--workspace", + "--all-features", + "--doc", + "--exclude", + "python", + "--exclude", + "wasm-runtime", + ], }, ]; @@ -65,16 +47,20 @@ COMMANDS: Run the visual provisioning-reconciler demo. check-telemetry-isolation Verify no frame types appear in control-plane modules. - test Run the non-binding repository test barrier." + test Run strict lint plus every Rust and Python test." ); } -fn run_step(step: &TestStep) -> bool { +fn run_step(step: &TestStep, python: &Path) -> bool { println!("\n=== {} ===", step.label); println!(" cargo {}", step.args.join(" ")); println!(); - match swactor_process::command_status(Command::new(cargo_bin()).args(step.args)) { + match swactor_process::command_status( + Command::new(cargo_bin()) + .args(step.args) + .env("PYO3_PYTHON", python), + ) { Ok(status) => status.success(), Err(error) => { eprintln!("Failed to execute cargo: {error}"); @@ -85,8 +71,17 @@ fn run_step(step: &TestStep) -> bool { fn run_tests() -> ExitCode { let start = Instant::now(); - for (index, step) in BASIC_TESTS.iter().enumerate() { - if !run_step(step) { + let Some(python) = PythonTestTools::discover() else { + return ExitCode::from(1); + }; + if !nextest_available() { + return ExitCode::from(1); + } + if !check_telemetry_isolation() { + return ExitCode::from(1); + } + for (index, step) in TEST_STEPS.iter().enumerate() { + if !run_step(step, &python.python) { eprintln!( "\n--- FAILED after {:.1}s ({index} passed, 1 failed) ---", start.elapsed().as_secs_f64() @@ -94,18 +89,99 @@ fn run_tests() -> ExitCode { return ExitCode::from(1); } } + if !python.run() { + eprintln!( + "\n--- FAILED after {:.1}s (Python tests failed) ---", + start.elapsed().as_secs_f64() + ); + return ExitCode::from(1); + } println!( "\n--- All {} step(s) passed in {:.1}s ---", - BASIC_TESTS.len(), + TEST_STEPS.len() + 1, start.elapsed().as_secs_f64() ); ExitCode::SUCCESS } +fn nextest_available() -> bool { + let available = swactor_process::command_status( + Command::new(cargo_bin()) + .args(["nextest", "--version"]) + .stdout(Stdio::null()) + .stderr(Stdio::null()), + ) + .is_ok_and(|status| status.success()); + if !available { + eprintln!("cargo-nextest is required; install it from https://nexte.st/docs/installation/"); + } + available +} + +struct PythonTestTools { + directory: PathBuf, + maturin: PathBuf, + python: PathBuf, +} + +impl PythonTestTools { + fn discover() -> Option { + let directory = workspace_root().join("crates/bindings/python"); + let tools = Self { + maturin: directory.join(".venv/bin/maturin"), + python: directory.join(".venv/bin/python"), + directory, + }; + if tools.maturin.is_file() && tools.python.is_file() { + Some(tools) + } else { + eprintln!( + "Python test environment is missing; run `uv sync --project {}` first", + tools.directory.display() + ); + None + } + } + + fn run(&self) -> bool { + println!("\n=== all Python tests ==="); + let built = swactor_process::command_status( + Command::new(&self.maturin) + .current_dir(&self.directory) + .arg("develop") + .env("PYO3_PYTHON", &self.python), + ) + .is_ok_and(|status| status.success()); + if !built { + return false; + } + swactor_process::command_status( + Command::new(&self.python) + .current_dir(&self.directory) + .args([ + "-m", + "pytest", + "-q", + "--timeout=60", + "tests/test_bootstrap.py", + "../../../tests/test_python.py", + ]), + ) + .is_ok_and(|status| status.success()) + } +} + +fn workspace_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("xtask must live directly under the workspace root") + .to_path_buf() +} + /// Verify that control-plane modules never import telemetry frame/read-side /// types. They may emit through the producer API only. -fn check_telemetry_isolation() -> ExitCode { +fn check_telemetry_isolation() -> bool { const CONTROL_DIRS: &[&str] = &[ "apps/myelin/src/orchestration", "crates/distribution/src", @@ -150,10 +226,10 @@ fn check_telemetry_isolation() -> ExitCode { "\ntelemetry-isolation: control-plane code must not import frame types \ or read-side modules. Use the telemetry producer API for emission." ); - ExitCode::from(1) + false } else { println!("telemetry-isolation: OK — no frame types in control-plane modules."); - ExitCode::SUCCESS + true } } @@ -180,7 +256,13 @@ mod demo; fn main() -> ExitCode { let mut args = std::env::args().skip(1); match args.next().as_deref() { - Some("check-telemetry-isolation") => check_telemetry_isolation(), + Some("check-telemetry-isolation") => { + if check_telemetry_isolation() { + ExitCode::SUCCESS + } else { + ExitCode::from(1) + } + } Some("test") if args.next().is_none() => run_tests(), Some("demo") => demo::run(&args.collect::>()), Some("help" | "--help" | "-h") | None => {