From 553347a8f7588d21c57f63fbbdb8d771d203e021 Mon Sep 17 00:00:00 2001 From: Zachery Aaron Shores-Chmielewski Date: Thu, 20 Aug 2026 01:38:14 +0400 Subject: [PATCH] feat(myelin): enforce actor-owned control flow Architecture enforcement: - Install a repository-owned rustc wrapper for ordinary cargo check, build, and test commands. Resolve compiler item identities so renamed imports and helper wrappers cannot hide spawning, timing, blocking, polling, thread, or runtime-driving capabilities. - Define the execution-owner crates and reject dependencies from those substrates back into Myelin policy. Add compile-pass and compile-fail contracts for actor helpers, execution owners, test waits, forbidden capabilities, suppression attempts, and owner dependency inversions. Execution ownership: - Add engine-owned actor timers with cancellation and generation identity, then migrate lifecycle deadlines and protocol ticks off application tasks. Keep networking, process output, telemetry, and blocking provider calls in their approved I/O substrates. - Move process spawn, wait, signal, Unix listener, and output-following mechanics into swactor-process. Isolate Vast.ai blocking HTTP mechanics behind its adapter while actors retain retry, recovery, and provisioning decisions. Myelin control flow: - Rework manual control, worker lifecycle, provisioning, provider recovery, job deployment, distribution, edge orchestration, and shutdown as actor state transitions and typed effects. Preserve durable provider adoption and command outcomes across graceful and abrupt restarts. - Replace controller loops and timer-forwarding tasks with actor messages; leave substrate tasks as cancellable observation streams with no durable policy state. Properties and resource ownership: - Add deterministic engine and component properties, a stateful mock-VastAI lifecycle model, persisted regression cases, controlled fault injection, and a bounded nightly workflow covering restart and teardown behavior. - Terminate reply observers, cancel telemetry collectors, bound dashboard projections, and release child observers, file descriptors, process records, and inode-verified Unix sockets on every terminal path. Verified with the compiler-policy contracts, 105 Myelin library tests, 32 swactor-process tests, telemetry cancellation contracts, randomized stateful restart cases, cargo check, and formatting checks. --- .cargo/config.toml | 3 + .github/workflows/myelin-properties.yml | 119 + Cargo.lock | 20 +- Cargo.toml | 1 + apps/myelin/Cargo.toml | 9 +- .../node/worker_node_runtime.txt | 7 + .../orchestration/app.txt | 7 + .../orchestration/manual_control.txt | 11 + .../orchestration/provisioning.txt | 8 + .../proptest-regressions/tests/e2e_vastai.txt | 5 + apps/myelin/specs/MYELIN_DAEMON.md | 81 - apps/myelin/src/job_deploy.rs | 759 +++-- apps/myelin/src/node/worker_node_runtime.rs | 2986 +++++++++++++---- .../src/observability/frame_collector.rs | 116 +- .../src/observability/orch_telemetry.rs | 2 +- .../src/observability/provisioning_logs.rs | 63 - apps/myelin/src/orchestration/app.rs | 1424 ++++++-- .../src/orchestration/cluster_reconciler.rs | 194 +- apps/myelin/src/orchestration/control.rs | 794 ++++- .../src/orchestration/distribution_stack.rs | 80 +- .../src/orchestration/job_reconciler.rs | 317 +- .../src/orchestration/manual_control.rs | 1427 +++++++- apps/myelin/src/orchestration/node_image.rs | 56 +- .../provider_adapters/vastai/mod.rs | 1974 +++++++++-- apps/myelin/src/orchestration/provisioning.rs | 2668 ++++++++++++--- apps/myelin/src/tests/engine_composition.rs | 7 +- apps/myelin/src/tests/fuzz_support.rs | 92 + .../src/tests/job_runner_integration.rs | 1 - apps/myelin/src/tests/job_runner_iroh.rs | 1 - apps/myelin/src/tests/mod.rs | 1 + apps/myelin/src/tests/node_guarantees.rs | 17 +- apps/myelin/tests/stateful_vastai.rs | 2319 +++++++++++++ clippy.toml | 55 +- crates/bindings/python/Cargo.toml | 1 + crates/bindings/python/src/lib.rs | 19 +- crates/bindings/wasm-runtime/Cargo.toml | 1 + crates/bindings/wasm-runtime/src/lib.rs | 24 +- crates/dashboard/Cargo.toml | 5 + .../proptest-regressions/control.txt | 7 + crates/dashboard/src/control.rs | 275 ++ crates/dashboard/src/lib.rs | 13 + crates/dashboard/src/server.rs | 316 ++ crates/dashboard/src/swactor/actor_view.rs | 113 +- crates/distribution/Cargo.toml | 1 + crates/distribution/tests/gossip_data.rs | 22 +- crates/distribution/tests/routing.rs | 34 +- crates/distribution/tests/swim_actor.rs | 54 +- crates/engine/Cargo.toml | 3 + crates/engine/ENGINE_SPEC.md | 4 +- crates/engine/src/core_driver.rs | 1 - crates/engine/src/engine.rs | 199 +- crates/engine/src/lib.rs | 2 +- crates/engine/src/stepping.rs | 13 +- crates/engine/src/tokio.rs | 3 - crates/engine/tests/common/mod.rs | 57 + crates/engine/tests/engine_contract.rs | 13 +- .../tests/engine_unit.proptest-regressions | 8 + crates/engine/tests/engine_unit.rs | 795 ++++- crates/iroh-driver/src/edge_transport.rs | 12 +- crates/iroh-driver/src/iroh_driver.rs | 24 +- crates/iroh-driver/src/lib.rs | 7 +- crates/iroh-driver/src/telemetry_transport.rs | 143 +- .../iroh-driver/tests/telemetry_transport.rs | 60 +- crates/job-runner/src/node.rs | 186 +- crates/job-runner/src/wire.rs | 12 + crates/process/Cargo.toml | 9 + crates/process/src/lib.rs | 19 + crates/process/src/operations.rs | 1425 ++++++++ crates/provisioning/Cargo.toml | 1 + crates/provisioning/src/bootstrap.rs | 22 +- .../provisioning/tests/process_conformance.rs | 21 +- rust-toolchain.toml | 1 + src/worker.rs | 20 +- tools/actor-control-flow-lint/Cargo.toml | 9 + tools/actor-control-flow-lint/driver.rs | 353 ++ .../actor-control-flow-lint/rustc-wrapper.py | 101 + tools/actor-control-flow-lint/src/lib.rs | 1 + .../tests/contracts.rs | 92 + .../fail-domain-capabilities/Cargo.lock | 263 ++ .../fail-domain-capabilities/Cargo.toml | 11 + .../fail-domain-capabilities/src/lib.rs | 35 + .../fixtures/fail-owner-dependency/Cargo.lock | 14 + .../fixtures/fail-owner-dependency/Cargo.toml | 10 + .../fail-owner-dependency/myelin/Cargo.toml | 8 + .../fail-owner-dependency/myelin/src/lib.rs | 1 + .../fixtures/fail-owner-dependency/src/lib.rs | 3 + .../fixtures/pass-actor-domain/Cargo.lock | 263 ++ .../fixtures/pass-actor-domain/Cargo.toml | 11 + .../fixtures/pass-actor-domain/src/lib.rs | 50 + .../fixtures/pass-execution-owner/Cargo.lock | 77 + .../fixtures/pass-execution-owner/Cargo.toml | 10 + .../fixtures/pass-execution-owner/src/lib.rs | 8 + .../tests/fixtures/pass-test-wait/Cargo.lock | 7 + .../tests/fixtures/pass-test-wait/Cargo.toml | 7 + .../tests/fixtures/pass-test-wait/src/lib.rs | 1 + .../fixtures/pass-test-wait/tests/wait.rs | 12 + tools/vastai/Cargo.toml | 4 + tools/vastai/src/blocking.rs | 91 + tools/vastai/src/client.rs | 13 +- tools/vastai/src/lib.rs | 4 + tools/vastai/src/test_http.rs | 180 + xtask/Cargo.toml | 3 + xtask/proptest-regressions/demo/feed.txt | 7 + xtask/proptest-regressions/demo/node.txt | 7 + xtask/src/demo/control.rs | 242 +- xtask/src/demo/docker.rs | 59 +- xtask/src/demo/edge.rs | 191 +- xtask/src/demo/feed.rs | 918 ++++- xtask/src/demo/mod.rs | 421 ++- xtask/src/demo/node.rs | 502 ++- xtask/src/demo/provider.rs | 371 +- xtask/src/main.rs | 2 +- 112 files changed, 20765 insertions(+), 3166 deletions(-) create mode 100644 .github/workflows/myelin-properties.yml create mode 100644 apps/myelin/proptest-regressions/node/worker_node_runtime.txt create mode 100644 apps/myelin/proptest-regressions/orchestration/app.txt create mode 100644 apps/myelin/proptest-regressions/orchestration/manual_control.txt create mode 100644 apps/myelin/proptest-regressions/orchestration/provisioning.txt create mode 100644 apps/myelin/proptest-regressions/tests/e2e_vastai.txt delete mode 100644 apps/myelin/specs/MYELIN_DAEMON.md create mode 100644 apps/myelin/src/tests/fuzz_support.rs create mode 100644 apps/myelin/tests/stateful_vastai.rs create mode 100644 crates/dashboard/proptest-regressions/control.txt create mode 100644 crates/engine/tests/engine_unit.proptest-regressions create mode 100644 crates/process/src/operations.rs create mode 100644 tools/actor-control-flow-lint/Cargo.toml create mode 100755 tools/actor-control-flow-lint/driver.rs create mode 100755 tools/actor-control-flow-lint/rustc-wrapper.py create mode 100644 tools/actor-control-flow-lint/src/lib.rs create mode 100644 tools/actor-control-flow-lint/tests/contracts.rs create mode 100644 tools/actor-control-flow-lint/tests/fixtures/fail-domain-capabilities/Cargo.lock create mode 100644 tools/actor-control-flow-lint/tests/fixtures/fail-domain-capabilities/Cargo.toml create mode 100755 tools/actor-control-flow-lint/tests/fixtures/fail-domain-capabilities/src/lib.rs create mode 100644 tools/actor-control-flow-lint/tests/fixtures/fail-owner-dependency/Cargo.lock create mode 100644 tools/actor-control-flow-lint/tests/fixtures/fail-owner-dependency/Cargo.toml create mode 100644 tools/actor-control-flow-lint/tests/fixtures/fail-owner-dependency/myelin/Cargo.toml create mode 100644 tools/actor-control-flow-lint/tests/fixtures/fail-owner-dependency/myelin/src/lib.rs create mode 100644 tools/actor-control-flow-lint/tests/fixtures/fail-owner-dependency/src/lib.rs create mode 100644 tools/actor-control-flow-lint/tests/fixtures/pass-actor-domain/Cargo.lock create mode 100644 tools/actor-control-flow-lint/tests/fixtures/pass-actor-domain/Cargo.toml create mode 100644 tools/actor-control-flow-lint/tests/fixtures/pass-actor-domain/src/lib.rs create mode 100644 tools/actor-control-flow-lint/tests/fixtures/pass-execution-owner/Cargo.lock create mode 100644 tools/actor-control-flow-lint/tests/fixtures/pass-execution-owner/Cargo.toml create mode 100644 tools/actor-control-flow-lint/tests/fixtures/pass-execution-owner/src/lib.rs create mode 100644 tools/actor-control-flow-lint/tests/fixtures/pass-test-wait/Cargo.lock create mode 100644 tools/actor-control-flow-lint/tests/fixtures/pass-test-wait/Cargo.toml create mode 100644 tools/actor-control-flow-lint/tests/fixtures/pass-test-wait/src/lib.rs create mode 100644 tools/actor-control-flow-lint/tests/fixtures/pass-test-wait/tests/wait.rs create mode 100644 tools/vastai/src/blocking.rs create mode 100644 tools/vastai/src/test_http.rs create mode 100644 xtask/proptest-regressions/demo/feed.txt create mode 100644 xtask/proptest-regressions/demo/node.txt diff --git a/.cargo/config.toml b/.cargo/config.toml index bc1639c..9ef77e8 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,3 +1,6 @@ +[build] +rustc-workspace-wrapper = "tools/actor-control-flow-lint/rustc-wrapper.py" + [alias] xtask = "run --package xtask --" myelin-chat = "run --package xtask -- myelin-chat" diff --git a/.github/workflows/myelin-properties.yml b/.github/workflows/myelin-properties.yml new file mode 100644 index 0000000..3a83e8a --- /dev/null +++ b/.github/workflows/myelin-properties.yml @@ -0,0 +1,119 @@ +name: Myelin properties + +on: + pull_request: + schedule: + - cron: "17 3 * * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + component-properties: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install pinned Rust toolchain + run: rustup toolchain install nightly-2026-02-07 --profile minimal --component rustfmt --component rustc-dev --component rust-src --component llvm-tools-preview + - name: Check formatting + run: cargo fmt --all -- --check + - name: Check default workspace members + run: cargo check + - name: Check compiler-policy contracts + run: cargo test -p actor-control-flow-lint-tests --test contracts compiler_policy_contracts -- --exact + - name: Engine scheduling properties + run: | + cargo test -p swactor-engine --test engine_unit generated_actor_timers_and_completion_are_bounded -- --exact + cargo test -p swactor-engine --test engine_unit lifecycle_invariant_detects_injected_duplicate_completion -- --exact + cargo test -p swactor-engine --test engine_unit lifecycle_invariant_detects_injected_uncancelled_periodic_timer -- --exact + - name: Process adapter properties + run: | + cargo test -p swactor-process --lib operations::properties::generated_stream_observations_close_once_and_stay_closed -- --exact + cargo test -p swactor-process --lib operations::properties::generated_lifecycle_actions_make_stop_idempotent_and_exit_terminal -- --exact + cargo test -p swactor-process --lib operations::properties::generated_stop_notifications_are_delivered_at_most_once -- --exact + cargo test -p swactor-process --lib operations::properties::generated_stdin_commands_and_eof_notify_once -- --exact + cargo test -p swactor-process --lib operations::properties::property_invariants_reject_controlled_defects -- --exact + cargo test -p swactor-process --lib operations::properties::trivial_real_child_exit_has_a_hard_timeout -- --exact + - name: Myelin component properties (exclude job and reconciler tests) + run: | + cargo test -p myelin --lib orchestration::manual_control::tests::aggressive_random_event_stream_preserves_control_invariants -- --exact + cargo test -p myelin --lib orchestration::manual_control::tests::rental_free_end_to_end_sequences_converge -- --exact + cargo test -p myelin --lib orchestration::manual_control::tests::manual_actor_generated_public_actions_and_callbacks_are_bounded -- --exact + cargo test -p myelin --lib orchestration::manual_control::tests::fixed_helper_cardinality_invariant_detects_controlled_extra_spawn -- --exact + cargo test -p myelin --lib orchestration::manual_control::tests::callback_panic_reports_typed_failure_without_poisoning_work_actor -- --exact + cargo test -p myelin --lib orchestration::manual_control::tests::callback_panic_invariant_detects_controlled_unguarded_panic -- --exact + cargo test -p myelin --lib provisioning::tests::mock_vastai_handle_state_survives_random_create_and_stop_sequences -- --exact + cargo test -p myelin --lib provisioning::tests::docker_generated_attempt_lifecycles_are_idempotent_and_bounded -- --exact + cargo test -p myelin --lib provisioning::tests::docker_duplicate_resource_detector_rejects_controlled_fault -- --exact + cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::offer_status_classes_are_offers_or_typed_rejections -- --exact + cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::malformed_offer_bodies_are_typed_rejections -- --exact + cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::wrong_or_missing_offer_fields_are_typed_rejections -- --exact + cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::duplicate_offer_records_remain_explicit_values -- --exact + cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::provider_monitor_preserves_contract_identity_and_cardinality -- --exact + cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::provider_monitor_terminal_polling_stops_after_one_typed_outcome -- --exact + cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::provider_monitor_poll_stop_orderings_cease_polling -- --exact + cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::duplicate_terminal_detector_rejects_controlled_fault -- --exact + cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_output_lines_preserve_stream_and_protocol -- --exact + cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_malformed_protocol_is_data_not_poison -- --exact + cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_eof_orderings_stop_relay_and_actor -- --exact + cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_child_failures_have_typed_attempt_outcomes -- --exact + cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_timeout_is_typed_and_stops_polling -- --exact + cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_stop_orderings_emit_one_terminal_and_stop_all_actors -- --exact + cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::runtime_actors_generated_transitions_complete_once_on_one_worker -- --exact + cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::runtime_invariant_checker_rejects_duplicate_readiness_publication -- --exact + cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::helper_wait_generated_terminal_sequences_complete_once_on_one_worker -- --exact + cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::helper_invariant_checker_rejects_expected_output_after_terminal_error -- --exact + cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::stage_fetch_generated_observations_complete_once_on_one_worker -- --exact + cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::stage_invariant_checker_rejects_wrong_terminal_classification -- --exact + cargo test -p myelin --lib orchestration::app::serve_cluster_properties::serve_cluster_production_transitions_converge_once_without_growth -- --exact + cargo test -p myelin --lib orchestration::app::serve_cluster_properties::serve_cluster_lifecycle_invariants_reject_injected_duplicate_and_growth -- --exact + cargo test -p myelin --lib orchestration::control::properties::generated_http_bridge_sequences_terminate_without_control_actor_growth -- --exact + cargo test -p myelin --lib orchestration::control::properties::generated_duplicate_control_replies_deliver_first_once_and_remove_observer -- --exact + cargo test -p myelin --lib orchestration::control::properties::reply_observer_disappearance_returns_a_bounded_terminal_http_response -- --exact + cargo test -p myelin --lib orchestration::control::properties::http_bridge_invariant_rejects_a_controlled_duplicate_forward -- --exact + - name: Controlled E2E oracle contract + run: cargo test -p myelin --features test-support --test stateful_vastai e2e_oracle_rejects_controlled_lifecycle_faults -- --exact + - name: Dashboard bridge properties + run: | + cargo test -p dashboard --features demo-control --lib control::properties::generated_concurrent_bridge_commands_forward_once_and_shutdown -- --exact + cargo test -p dashboard --features demo-control --lib control::properties::bridge_invariant_rejects_a_controlled_duplicate_delivery -- --exact + cargo test -p dashboard --features demo-control --lib server::tests::generated_control_http_sequences_are_bounded_and_typed -- --exact + cargo test -p dashboard --features demo-control --lib server::tests::control_http_invariant_rejects_a_controlled_server_error -- --exact + - name: Demo actor properties + run: | + cargo test -p xtask --bin xtask demo::control::properties::generated_control_commands_forward_only_after_supervisor_registration -- --exact + cargo test -p xtask --bin xtask demo::control::properties::control_transition_oracle_rejects_duplicate_forwarding -- --exact + cargo test -p xtask --bin xtask demo::feed::properties::generated_supervisor_transitions_are_once_only_nonblocking_and_clean -- --exact + cargo test -p xtask --bin xtask demo::feed::properties::supervisor_transition_oracle_rejects_duplicate_identity_resources -- --exact + cargo test -p xtask --bin xtask demo::node::properties::generated_node_runtime_transitions_emit_heartbeats_and_stop_once -- --exact + cargo test -p xtask --bin xtask demo::node::properties::node_transition_oracle_rejects_duplicate_resources -- --exact + cargo test -p xtask --bin xtask demo::provider::properties::generated_process_reports_complete_exit_watchers_once_and_preserve_last_state -- --exact + cargo test -p xtask --bin xtask demo::provider::properties::process_relay_oracle_rejects_lost_exit -- --exact + cargo test -p xtask --bin xtask demo::properties::direct_binary_signal_smoke_has_a_hard_timeout -- --exact + + process-e2e: + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + needs: component-properties + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - name: Install pinned Rust toolchain + run: rustup toolchain install nightly-2026-02-07 --profile minimal --component rustc-dev --component rust-src --component llvm-tools-preview + - name: Stateful VastAI process E2E + env: + PROPTEST_CASES: "4" + run: | + set -o pipefail + mkdir -p artifacts + cargo test -p myelin --features test-support --test stateful_vastai stateful_vastai_dashboard_control_survives_restarts -- --ignored --exact --nocapture 2>&1 | tee artifacts/stateful-vastai.log + - name: Preserve E2E failure artifacts + if: failure() + uses: actions/upload-artifact@v4 + with: + name: stateful-vastai-failure-${{ github.run_id }} + path: | + artifacts/stateful-vastai.log + apps/myelin/proptest-regressions/tests/e2e_vastai.txt + if-no-files-found: warn diff --git a/Cargo.lock b/Cargo.lock index 43d5fae..f364649 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,10 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "actor-control-flow-lint-tests" +version = "0.1.0" + [[package]] name = "adler2" version = "2.0.1" @@ -900,12 +904,15 @@ version = "0.1.0" dependencies = [ "axum", "parking_lot", + "proptest", "serde", "serde_json", "swactor", + "swactor-engine", "telemetry", "tokio", "tokio-stream", + "tower", ] [[package]] @@ -1131,6 +1138,7 @@ dependencies = [ "serde", "serde_json", "swactor", + "swactor-engine", "swactor-transport", "telemetry", "uuid", @@ -2491,7 +2499,6 @@ dependencies = [ "provisioning", "serde", "serde_json", - "signal-hook", "swactor", "swactor-engine", "swactor-job-runner", @@ -2503,7 +2510,6 @@ dependencies = [ "tokio", "toml 0.8.23", "ureq", - "wiremock", ] [[package]] @@ -3268,6 +3274,7 @@ dependencies = [ "serde_json", "swactor", "swactor-engine", + "swactor-process", ] [[package]] @@ -3339,6 +3346,7 @@ version = "0.1.0" dependencies = [ "pyo3", "swactor", + "swactor-engine", ] [[package]] @@ -4307,6 +4315,7 @@ name = "swactor-engine" version = "0.1.0" dependencies = [ "parking_lot", + "proptest", "swactor", "tokio", ] @@ -4333,11 +4342,16 @@ version = "0.1.0" dependencies = [ "crossbeam-queue", "libc", + "parking_lot", + "proptest", "serde", "serde_json", "serde_yaml", + "signal-hook", "swactor", + "swactor-engine", "telemetry", + "tokio", ] [[package]] @@ -5181,6 +5195,7 @@ name = "wasm-runtime" version = "0.1.0" dependencies = [ "swactor", + "swactor-engine", "wasm-bindgen", ] @@ -5775,6 +5790,7 @@ dependencies = [ "iroh-driver", "libc", "parking_lot", + "proptest", "provisioning", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 990ee8b..e4af85c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ members = [ "apps/myelin", "xtask", "tools/vastai", + "tools/actor-control-flow-lint", ] default-members = [ ".", diff --git a/apps/myelin/Cargo.toml b/apps/myelin/Cargo.toml index 632c3be..422ed8b 100644 --- a/apps/myelin/Cargo.toml +++ b/apps/myelin/Cargo.toml @@ -10,6 +10,7 @@ autobins = false [features] default = ["dashboard"] dashboard = [] +test-support = ["swactor-vastai/test-support"] [dependencies] telemetry = { path = "../../crates/telemetry" } @@ -36,12 +37,16 @@ axum = "0.8" [dev-dependencies] tempfile = "3" -wiremock = "0.6" +swactor-vastai = { path = "../../tools/vastai", features = ["test-support"] } proptest = "1" +[[test]] +name = "stateful_vastai" +path = "tests/stateful_vastai.rs" +required-features = ["test-support"] + [target.'cfg(target_os = "linux")'.dependencies] libc = "0.2" -signal-hook = "0.3" [[bin]] name = "myelin-worker" diff --git a/apps/myelin/proptest-regressions/node/worker_node_runtime.txt b/apps/myelin/proptest-regressions/node/worker_node_runtime.txt new file mode 100644 index 0000000..2da305e --- /dev/null +++ b/apps/myelin/proptest-regressions/node/worker_node_runtime.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc be434e1d0bf8073f32e4072d9093dd2df03537b9d1bf759872e760fcf74a06cb # shrinks to actions = [], extra_ticks = 1 diff --git a/apps/myelin/proptest-regressions/orchestration/app.txt b/apps/myelin/proptest-regressions/orchestration/app.txt new file mode 100644 index 0000000..04f55c6 --- /dev/null +++ b/apps/myelin/proptest-regressions/orchestration/app.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc ed6cf4326f625c499a6752aa3630891a4242ed8aa4cfe59148d901030efd4ff1 # shrinks to actions = [] diff --git a/apps/myelin/proptest-regressions/orchestration/manual_control.txt b/apps/myelin/proptest-regressions/orchestration/manual_control.txt new file mode 100644 index 0000000..32d2e9e --- /dev/null +++ b/apps/myelin/proptest-regressions/orchestration/manual_control.txt @@ -0,0 +1,11 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc a36a5c2d2904ecfb2cf31f3e446db588a7591e074994fc61142f37aad33272c8 # shrinks to actions = [Provision(163), Drive, Configure(240), StaleSearch(108), Configure(85), Search(84), Flush, Kill(76), Configure(190), StaleValidation(217), Provision(209), StaleSearch(18), StaleValidation(70), Provision(97), Drive, Flush, StaleValidation(81), Search(121), Query, Flush, StaleValidation(91), StaleValidation(110)] +cc aa67d0e8e7fd15f0b8cad5ac675b6fcee25fc5246a6841d141054b6932f0db34 # shrinks to actions = [Search(0), Search(0), Provision(86)] +cc ad4aa3d6008d92f09c315c680c258a2b3776719bca98ff49d72668542f0c4772 # shrinks to actions = [Provision(17), Flush(81), OfferSearchFinished(200), PersistenceFinished(145), Provision(202), ProviderTerminalFailure(29), Kill(222), Provision(21), Provision(117), Query(131), Provision(126), Rejoin(27), Provision(89), OfferSearchFinished(148), Query(54), Kill(77)] +cc be1220292a8604bca5d52ed98402b55bfd81ca591ee47c4139829365a5c1eddc # shrinks to actions = [ProviderValidated(17), Flush(231), OfferSearchFinished(238), Drive, Provision(32), Configure(101), Configure(194), ProviderValidated(226), Provision(152), Configure(235), Provision(208), Kill(68), ProviderValidated(50)] +cc fbc45330b5c55a483c910567ed49b8c2f40e4d9b8ad17f9a88f666fc120d4305 # shrinks to actions = [PersistenceFinished(73), Flush(190), EffectFinished(224), Query(58), Kill(182), Rejoin(82), Flush(137), OfferSearchFinished(173), Drive, Provision(204), Kill(137), EffectFinished(187), Configure(80), Configure(104), Drive, PersistenceFinished(92), Flush(180), Search(86), Configure(246), Query(33), Search(43)] diff --git a/apps/myelin/proptest-regressions/orchestration/provisioning.txt b/apps/myelin/proptest-regressions/orchestration/provisioning.txt new file mode 100644 index 0000000..4432127 --- /dev/null +++ b/apps/myelin/proptest-regressions/orchestration/provisioning.txt @@ -0,0 +1,8 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 38b940d757de89f824d145dc409f7f0d3b6d757840eff473ed9acac3eef65d21 # shrinks to actions = [Create(61), Create(253)] +cc 502380a923217c7ea1912e0a54392f1389eaafa4e35efeac26cdeb5ac2a0067e # shrinks to actions = [Create(53), Start(0), Adopt(221)] diff --git a/apps/myelin/proptest-regressions/tests/e2e_vastai.txt b/apps/myelin/proptest-regressions/tests/e2e_vastai.txt new file mode 100644 index 0000000..511ca75 --- /dev/null +++ b/apps/myelin/proptest-regressions/tests/e2e_vastai.txt @@ -0,0 +1,5 @@ +# Seeds for stateful VastAI E2E failures. Proptest replays these before generated cases. +cc 26c3cd5944cb25de70a984389373e7380a0c6e6ba1c8f3e6eff5a0f78dbef168 # current-strategy replay; E2eCase is derived from e2e_case() +cc 5efb0a6c5c348475c859a2c4267d5177aaecd5adefcc5e8b7fc2e6d728deecd1 # shrinks to case = E2eCase { seed: 7099259626237328177, node_seed: 245, kill_mask: 24, offer_offset: 3, actions: [ConcurrentQueries, Search { count: 2 }, Restart { mode: FlushSafeAbrupt }, Query, Kill { node_slot: 218, command_slot: 5 }, Query, Provision { command_slot: 5, use_searched_offers: true }, EndpointProbe { node_slot: 217 }, Restart { mode: Graceful }, Flush, Kill { node_slot: 217, command_slot: 5 }] } +cc 6d005687858520ed65af20ec5b2de056efcda15771b6420d7f67e3cb6c246d5b # shrinks to case = E2eCase { seed: 17649392557414661864, node_seed: 249, kill_mask: 159, offer_offset: 5, actions: [Kill { node_slot: 88, command_slot: 242 }, Flush, ConcurrentQueries, EndpointProbe { node_slot: 88 }, Query, Search { count: 2 }, Provision { command_slot: 242, use_searched_offers: true }, Restart { mode: FlushSafeAbrupt }, Kill { node_slot: 89, command_slot: 242 }, Query, Restart { mode: Graceful }] } +cc 2e36f51e7cb347b571ac838f81638748f8414a32ec91cb9f4b2730363d72be61 # shrinks to case = E2eCase { seed: 13301618846512983428, node_seed: 99, kill_mask: 30, offer_offset: 2, actions: [Query, Restart { mode: FlushSafeAbrupt }, Kill { node_slot: 136, command_slot: 187 }, ConcurrentQueries, Restart { mode: Graceful }, Query, EndpointProbe { node_slot: 136 }, Kill { node_slot: 137, command_slot: 187 }, Provision { command_slot: 187, use_searched_offers: false }, Flush, Search { count: 0 }] } diff --git a/apps/myelin/specs/MYELIN_DAEMON.md b/apps/myelin/specs/MYELIN_DAEMON.md deleted file mode 100644 index 7a663c2..0000000 --- a/apps/myelin/specs/MYELIN_DAEMON.md +++ /dev/null @@ -1,81 +0,0 @@ -# Myelin Fleet-Control Daemon - -Myelin is a persistent, dashboard-first control plane for manually managed compute nodes. It boots an empty fleet, accepts explicit operator commands, records intent and observations, and never performs hidden replacement or teardown. - -## Runtime shape - -```text -myelin-orchestrator - -> load stable iroh identity and cluster snapshot - -> start engine, iroh endpoint, telemetry collector, and dashboard - -> recover persisted provider resources without creating replacements - -> idle event loop - - ingest telemetry, membership, actor reports, and control messages - - dispatch explicit Provision / Kill effects - - atomically persist every state transition before dependent effects - -> on exit, detach durable-provider handles without destroying resources -``` - -There is no desired-shape reconciler or automatic node replacement in the manual control path. Provisioning, Kill, recovery, runtime readiness, and worker rejoin are typed actor protocols; provider I/O and snapshot writes run outside actor handlers on the engine. - -## Starting the daemon - -From the workspace root: - -```sh -cargo run -p myelin -``` - -The local default uses the process provider. Use Docker explicitly when required: - -```sh -cargo run -p myelin -- --provider docker -``` - -Use Vast.ai without making valid credentials a startup prerequisite: - -```sh -cargo run -p myelin -- --provider vastai -``` - -The dashboard starts in `unconfigured` or `configuration_error` state and accepts corrected credentials at runtime. - -`MYELIN_DASHBOARD_PORT` selects the dashboard port. The dashboard root remains the read-only Fleet view; `/provision` is the Myelin-owned mutation surface. Its provider-neutral control protocol includes: - -- `Provision { command_id, count, selected_offer_ids }`: create the requested nodes; Vast.ai requires and leases only the exact selected offer IDs. -- `Kill { command_id, logical_node_id }`: stop one managed process/container or destroy one Vast.ai contract while retaining its terminal snapshot record. -- `ConfigureProvider`: validate corrected in-memory Vast.ai credentials and bootstrap settings. -- `SearchOffers`: inspect filtered Vast.ai offers without leasing. -- `Query`: return provider readiness, recent commands, and managed node state. - -Every mutation carries a caller-generated command ID. The daemon persists the full command record and node intent before provider work. Reusing an ID returns the original record and never repeats create or destroy. Node IDs are monotonic and never reused. - -## Durable state - -The state directory contains: - -- `identity.key`: 32-byte iroh secret key. Preserving it keeps the daemon endpoint stable across restarts. -- `cluster.json`: schema-versioned snapshot containing the stable provider label, run id, next node id, full command records, node specs, selected offer IDs, provider references, runtime facts, phases, and errors. - -Writes use a temporary file plus rename. A corrupt identity or snapshot is a hard startup error. `--reset-state` explicitly clears both files; startup never treats corruption as an empty fleet. - -Provider state is ground truth during recovery: - -- snapshot + provider resource: adopt and observe it; -- snapshot only: mark the node stopped; never recreate it; -- provider resource without managed intent: report it as an orphan and take no action. - -The current orchestrator actor address is published under `myelin.manual-control` in the distributed name registry. A surviving worker observes a changed binding, sends `RejoinHello` with its persisted logical identity and current runtime facts, waits for the daemon to persist those facts, and only then rebinds to the returned actor address and control generation. No stable actor address is assumed. - -## Lifecycle policy - -Graceful shutdown leaves Docker containers and Vast.ai leases running so a later daemon can adopt them. Local process children are different: they cannot be adopted, so Ctrl-C stops them and clears their snapshot records. They also exit when their daemon-owned stdin supervision pipe closes, preventing an abrupt daemon crash from leaving invisible local workers. - -Destruction of durable provider resources occurs only through an explicit dashboard command. The default process path re-enters the running orchestrator executable in an internal worker mode, so `cargo run -p myelin` never depends on a separately built or stale `myelin-worker` binary. - -## Node image contract - -The standard image contains a uniform Myelin agent, SSH bootstrap, and the CUDA runtime. It does not contain tinygrad, NumPy, PyTorch, vLLM, or model weights. Frameworks and application dependencies belong to job payload images. Nodes launched by this daemon set `MYELIN_AGENT_ONLY=1`, so the agent joins membership, announces readiness, and exports telemetry without starting an inference helper. -As part of runtime-ready bootstrap, the daemon dials the node's advertised iroh endpoint on `TELEMETRY_ALPN`, requests all telemetry channels, and retains that pull stream for the node's lifetime. The node serves the pull locally; it never needs to reverse-dial the dashboard. Pulled stream descriptors and frames feed both the Fleet view and the live telemetry explorer. - -The retired chat/GGUF specification is archived at `archive/MYELIN_CHAT_SPEC.md`. diff --git a/apps/myelin/src/job_deploy.rs b/apps/myelin/src/job_deploy.rs index e3cbc03..efad062 100644 --- a/apps/myelin/src/job_deploy.rs +++ b/apps/myelin/src/job_deploy.rs @@ -13,10 +13,11 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; use serde::{Deserialize, Serialize}; -use swactor::actor::ActorAddress; -use swactor_engine::{Engine, EngineHandle, TokioBackend, TokioConfig}; +use swactor::actor::{ActorAddress, ActorInterface}; +use swactor::runtime::{Ctx, ExternalSender}; +use swactor_engine::{ActorCompletion, Engine, EngineHandle, TokioBackend, TokioConfig}; use swactor_job_runner::{ - JobDone, NodeJobActor, OUTPUTS_EDGE_ID, OrchestratorJobActor, OrchestratorJobMsg, + Job, JobDone, NodeJobActor, OUTPUTS_EDGE_ID, OrchestratorJobActor, OrchestratorJobMsg, WORKSPACE_EDGE_ID, register_job_codecs, }; use swactor_transport::hex_encode; @@ -100,35 +101,104 @@ pub(crate) fn build_composition() -> Result { Ok((engine, driver, stack)) } -fn identity_for(driver: &IrohDriver, actor: ActorAddress) -> Result { - let endpoint = advertised_endpoint_for(driver)?; - Ok(NodeIdentity { - endpoint, - actor_hex: hex_encode(&actor.0), - }) +struct ResolveIdentityActor { + driver: Option, + actor: ActorAddress, + mask: EndpointAddrMask, + started: Instant, + engine: EngineHandle, + sender: ExternalSender, + completion: ActorCompletion>, } -fn advertised_endpoint_for(driver: &IrohDriver) -> Result { - let mask = endpoint_addr_mask_from_env()?; - if !mask.requires_relay() { - return advertised_endpoint(driver.endpoint_addr(), mask); +#[derive(Clone)] +enum ResolveIdentityMsg { + Check, +} + +impl ResolveIdentityActor { + fn finish(&mut self, ctx: &swactor::runtime::Ctx, result: Result) { + let result = result.map(|identity| { + ( + self.driver + .take() + .expect("identity resolver owns driver until completion"), + identity, + ) + }); + assert!( + self.completion.complete(result).is_ok(), + "identity resolver completed twice" + ); + ctx.stop_self(); } - let started = Instant::now(); - loop { - let endpoint = driver.endpoint_addr(); - if endpoint.relay_urls().next().is_some() { - return advertised_endpoint(endpoint, mask); - } - if started.elapsed() >= RELAY_WAIT_DEADLINE { - return Err(format!( - "relay-only endpoint address mask did not observe a relay URL within {RELAY_WAIT_DEADLINE:?}; last endpoint={endpoint:?}" - )); - } - std::thread::sleep(POLL); + fn schedule_check(&self, ctx: &swactor::runtime::Ctx) { + self.engine.send_after( + POLL, + self.sender.clone(), + ctx.self_addr(), + ResolveIdentityMsg::Check, + ); } } +impl ActorInterface for ResolveIdentityActor { + type Incoming = ResolveIdentityMsg; + type Response = (); + + fn on_start(&mut self, ctx: &swactor::runtime::Ctx) { + let _ = ctx.send(ctx.self_addr(), ResolveIdentityMsg::Check); + } + + fn handle(&mut self, ctx: &swactor::runtime::Ctx, _message: Self::Incoming) { + let driver = self + .driver + .as_ref() + .expect("identity resolver handles messages only while live"); + let endpoint = driver.endpoint_addr(); + if !self.mask.requires_relay() || endpoint.relay_urls().next().is_some() { + let identity = advertised_endpoint(endpoint, self.mask) + .map(|endpoint| NodeIdentity { + endpoint, + actor_hex: hex_encode(&self.actor.0), + }) + .map_err(|error| error.to_string()); + self.finish(ctx, identity); + } else if self.started.elapsed() >= RELAY_WAIT_DEADLINE { + self.finish( + ctx, + Err(format!( + "relay-only endpoint address mask did not observe a relay URL within {RELAY_WAIT_DEADLINE:?}; last endpoint={endpoint:?}" + )), + ); + } else { + self.schedule_check(ctx); + } + } +} + +fn resolve_identity( + driver: IrohDriver, + actor: ActorAddress, + stack: &DistributionRuntimeStack, +) -> Result<(IrohDriver, NodeIdentity), String> { + let completion = ActorCompletion::new(); + stack + .runtime + .spawn(ResolveIdentityActor { + driver: Some(driver), + actor, + mask: endpoint_addr_mask_from_env()?, + started: Instant::now(), + engine: stack.engine.clone(), + sender: stack.runtime.create_sender(), + completion: completion.clone(), + }) + .map_err(|error| format!("spawn endpoint identity resolver: {error}"))?; + completion.wait() +} + fn endpoint_addr_mask_from_env() -> Result { match env_optional(MVP_IROH_ENDPOINT_ADDR_MASK_ENV) { Some(mask) => EndpointAddrMask::parse(&mask), @@ -174,6 +244,94 @@ pub(crate) fn parse_actor(hex: &str) -> Result { Ok(ActorAddress(arr)) } +#[derive(Clone)] +enum WorkerLifecycleMsg { + CheckConnection, + Stop, +} + +struct WorkerLifecycleActor { + driver: IrohDriver, + _stack: DistributionRuntimeStack, + orchestrator_endpoint: EndpointAddr, + orchestrator_node: swactor_transport::NodeId, + output_sink: Arc>>>, + started: Instant, + armed: bool, + engine: EngineHandle, + sender: ExternalSender, + completion: ActorCompletion>, +} + +impl WorkerLifecycleActor { + fn schedule_check(&self, ctx: &Ctx) { + self.engine.send_after( + POLL, + self.sender.clone(), + ctx.self_addr(), + WorkerLifecycleMsg::CheckConnection, + ); + } +} + +impl ActorInterface for WorkerLifecycleActor { + type Incoming = WorkerLifecycleMsg; + type Response = (); + + fn on_start(&mut self, ctx: &Ctx) { + let _ = ctx.send(ctx.self_addr(), WorkerLifecycleMsg::CheckConnection); + } + + fn handle(&mut self, ctx: &Ctx, message: Self::Incoming) { + match message { + WorkerLifecycleMsg::CheckConnection if !self.armed => { + if self.driver.has_active_connection(&self.orchestrator_node) + || self.started.elapsed() >= CONVERGE_DEADLINE + { + match self + .driver + .spawn_edge_send_pump(self.orchestrator_endpoint.clone(), OUTPUTS_EDGE_ID) + { + Ok(handle) => { + *self.output_sink.lock() = Some(Box::new(IrohEdgeSink(handle))); + eprintln!("job-worker: output edge sink armed"); + } + Err(error) => { + eprintln!("job-worker: failed to arm output edge sink: {error}") + } + } + self.armed = true; + } else { + self.schedule_check(ctx); + } + } + WorkerLifecycleMsg::CheckConnection => {} + WorkerLifecycleMsg::Stop => { + assert!( + self.completion.complete(Ok(())).is_ok(), + "worker lifecycle completed twice" + ); + ctx.stop_self(); + } + } + } +} + +struct WorkerStopForwarder { + sender: ExternalSender, + worker: ActorAddress, +} + +impl ActorInterface for WorkerStopForwarder { + type Incoming = (); + type Response = (); + + fn handle(&mut self, ctx: &Ctx, (): Self::Incoming) { + let _ = self.sender.send_to(self.worker, WorkerLifecycleMsg::Stop); + ctx.stop_self(); + } +} + /// Worker: connect to the orchestrator, expose a `NodeJobActor`, run jobs it /// sends over iroh. Prints this node's identity as JSON on stdout, then runs /// until killed. Bulk bytes travel over EDGE_ALPN: the orchestrator pushes the @@ -185,42 +343,38 @@ pub fn run_worker(orch_identity_json: String, workdir: PathBuf) -> Result<(), St .map_err(|e| format!("parse orch identity: {e}"))?; let orch_actor = parse_actor(&orch.actor_hex)?; let orch_endpoint = orch.endpoint.clone(); - let (_engine, driver, stack) = build_composition()?; + let (engine, driver, stack) = build_composition()?; let sender = stack.runtime.create_sender(); - // Edge workspace: the orchestrator pushes the workspace tar over EDGE_ALPN - // before submitting the job. A background thread drains those bytes, - // extracts them into `workdir`, and signals readiness; the node actor waits - // on that flag before announcing the workspace materialized. let workspace_ready = Arc::new(AtomicBool::new(false)); - let ws_events = driver.edge_events_handle(); - let ws_workdir = workdir.clone(); - let ws_ready = workspace_ready.clone(); - std::thread::Builder::new() - .name("job-worker-ws-edge".to_owned()) - .spawn(move || { - drain_workspace_edge(ws_events, ws_workdir, ws_ready); + stack + .runtime + .spawn(WorkspaceEdgeActor { + engine: stack.engine.clone(), + sender: stack.runtime.create_sender(), + events: driver.edge_events_handle(), + workdir: workdir.clone(), + ready: workspace_ready.clone(), + buf: Vec::new(), + started: Instant::now(), }) - .map_err(|e| format!("spawn workspace edge thread: {e}"))?; + .map_err(|error| format!("spawn workspace edge actor: {error}"))?; - // Edge outputs: a slot the main thread fills with an EDGE_ALPN sink to the - // orchestrator once the iroh connection is up. The node actor ships - // collected outputs through it. let output_sink_slot: Arc>>> = Arc::new(Mutex::new(None)); - let job_actor = stack .runtime .spawn( NodeJobActor::new(orch_actor, workdir, sender, 0) - .with_workspace_ready(workspace_ready.clone()) + .with_actor_timers(engine.handle()) + .with_workspace_ready(workspace_ready) .with_output_sink_slot(output_sink_slot.clone()), ) .map_err(|e| format!("spawn node job actor: {e}"))?; stack.register_local_actor(driver.register_actor(job_actor, 1)); driver.join(std::slice::from_ref(&orch.endpoint)); - let id = identity_for(&driver, job_actor)?; + let (driver, id) = resolve_identity(driver, job_actor, &stack)?; println!( "JOB_WORKER_IDENTITY {}", serde_json::to_string(&id).map_err(|e| e.to_string())? @@ -231,27 +385,33 @@ pub fn run_worker(orch_identity_json: String, workdir: PathBuf) -> Result<(), St id.actor_hex, id.endpoint ); - // Wait for the iroh connection to the orchestrator, then arm the output - // edge sink so it is ready before a CollectOutputs command can arrive. - let orch_node = swactor_transport::NodeId(*orch_endpoint.id.as_bytes()); - let conn_started = Instant::now(); - while !driver.has_active_connection(&orch_node) { - if conn_started.elapsed() >= CONVERGE_DEADLINE { - break; - } - std::thread::sleep(POLL); - } - match driver.spawn_edge_send_pump(orch_endpoint.clone(), OUTPUTS_EDGE_ID) { - Ok(handle) => { - *output_sink_slot.lock() = Some(Box::new(IrohEdgeSink(handle))); - eprintln!("job-worker: output edge sink armed"); - } - Err(e) => eprintln!("job-worker: failed to arm output edge sink: {e}"), - } - - loop { - std::thread::sleep(Duration::from_secs(3600)); - } + let runtime = stack.runtime.clone(); + let completion = ActorCompletion::new(); + let lifecycle = runtime + .spawn(WorkerLifecycleActor { + driver, + _stack: stack, + orchestrator_node: swactor_transport::NodeId(*orch_endpoint.id.as_bytes()), + orchestrator_endpoint: orch_endpoint, + output_sink: output_sink_slot, + started: Instant::now(), + armed: false, + engine: engine.handle(), + sender: runtime.create_sender(), + completion: completion.clone(), + }) + .map_err(|error| format!("spawn job worker lifecycle actor: {error}"))?; + let stop_forwarder = runtime + .spawn(WorkerStopForwarder { + sender: runtime.create_sender(), + worker: lifecycle, + }) + .map_err(|error| format!("spawn job worker stop forwarder: {error}"))?; + #[cfg(target_os = "linux")] + swactor_process::spawn_os_stop_signal_wait(runtime.create_sender(), stop_forwarder); + let result = completion.wait(); + drop(engine); + result } /// Starts the operator-side job actor and publishes enough identity for a @@ -268,7 +428,7 @@ pub(crate) fn start_orchestrator(landing: PathBuf) -> Result Result, + worker: NodeIdentity, + node_actor: ActorAddress, + phase: JobRunPhase, +} + +enum JobRunPhase { + Created, + Directory { + deadline: Instant, + }, + Connection { + started: Instant, + deadline: Instant, + }, + Running { + deadline: Instant, + output_buf: Vec, + outputs_ended: bool, + pending_done: Option, + }, + Finished, +} + +impl JobRunStateMachine { + pub(crate) fn new( + session: JobOrchestratorSession, + job: Job, + worker: NodeIdentity, + ) -> Result { + let node_actor = parse_actor(&worker.actor_hex)?; + Ok(Self { + session, + job: Some(job), + worker, + node_actor, + phase: JobRunPhase::Created, + }) + } + + pub(crate) fn start(&mut self, now: Instant) { + self.session + .driver + .join(std::slice::from_ref(&self.worker.endpoint)); + self.phase = JobRunPhase::Directory { + deadline: now + CONVERGE_DEADLINE, + }; + } + + pub(crate) fn advance(&mut self, now: Instant) -> Option> { + let phase = std::mem::replace(&mut self.phase, JobRunPhase::Finished); + match phase { + JobRunPhase::Created => { + self.phase = JobRunPhase::Created; + None + } + JobRunPhase::Directory { deadline } => { + let converged = self + .session + .stack + .route_view + .read() + .map(|view| view.contains_key(&self.node_actor)) + .unwrap_or(false); + if converged { + eprintln!( + "job-orch: directory converged; waiting for iroh connection to worker" + ); + self.phase = JobRunPhase::Connection { + started: now, + deadline: now + CONVERGE_DEADLINE, + }; + None + } else if now >= deadline { + Some(Err( + "directory did not converge: orchestrator never learned the worker actor" + .into(), + )) + } else { + self.phase = JobRunPhase::Directory { deadline }; + None + } + } + JobRunPhase::Connection { started, deadline } => { + let worker_node = swactor_transport::NodeId(*self.worker.endpoint.id.as_bytes()); + let connected = self.session.driver.has_active_connection(&worker_node); + if !connected && now < deadline { + self.phase = JobRunPhase::Connection { started, deadline }; + return None; + } + if connected { + eprintln!( + "job-orch: iroh connection to worker established after {:?}", + now.saturating_duration_since(started) + ); + eprintln!("job-orch: submitting job"); + } else { + eprintln!( + "job-orch: no iroh connection to worker after {CONVERGE_DEADLINE:?}; join_statuses={:?}; submitting best-effort", + self.session.driver.join_statuses() + ); + } + match self.submit(now) { + Ok(()) => None, + Err(error) => Some(Err(error)), + } + } + JobRunPhase::Running { + deadline, + mut output_buf, + mut outputs_ended, + mut pending_done, + } => { + let edge_events = self.session.driver.edge_events_handle(); + let drained: Vec = edge_events.lock().drain(..).collect(); + for event in drained { + match event { + WireEvent::BytesRead { edge_id, bytes, .. } + if edge_id.0 == OUTPUTS_EDGE_ID => + { + output_buf.extend_from_slice(&bytes); + } + WireEvent::StreamEnded { edge_id, .. } if edge_id.0 == OUTPUTS_EDGE_ID => { + if !output_buf.is_empty() { + if let Err(error) = swactor_job_runner::extract_tar( + &output_buf, + &self.session.landing, + ) { + eprintln!("job-orch: untar edge outputs failed: {error}"); + } + output_buf.clear(); + } + outputs_ended = true; + } + _ => {} + } + } + if pending_done.is_none() { + pending_done = self.session.done.try_recv(); + } + if let Some(done) = pending_done.as_ref() { + let need_outputs = done.exit_code.is_some() && !outputs_ended; + if !need_outputs || now >= deadline { + let done = pending_done + .take() + .expect("pending job result was observed"); + if need_outputs { + eprintln!("job-orch: output edge stream did not land before deadline"); + } + return Some(Ok(done)); + } + } + if now >= deadline { + return Some(Err("job did not complete within deadline".into())); + } + self.phase = JobRunPhase::Running { + deadline, + output_buf, + outputs_ended, + pending_done, + }; + None + } + JobRunPhase::Finished => { + Some(Err("job state machine advanced after completion".into())) + } + } + } + + fn submit(&mut self, now: Instant) -> Result<(), String> { + let job = self + .job + .take() + .ok_or_else(|| "job was already submitted".to_owned())?; + let workspace_bytes = + swactor_job_runner::pack_workspace(&job).map_err(|e| format!("pack workspace: {e}"))?; + if !workspace_bytes.is_empty() { + let pump = self + .session + .driver + .spawn_edge_send_pump(self.worker.endpoint.clone(), WORKSPACE_EDGE_ID) + .map_err(|e| format!("workspace edge pump: {e}"))?; + for record in workspace_bytes.chunks(swactor_job_runner::EDGE_RECORD_SIZE) { + pump.send(record.to_vec()) + .map_err(|e| format!("workspace edge send: {e}"))?; + } + drop(pump); + eprintln!("job-orch: workspace pushed over EDGE_ALPN"); + } + self.session + .stack + .runtime + .send_to( + self.session.orch, + OrchestratorJobMsg::Submit { + job, + node_actor: self.node_actor, + }, + ) + .map_err(|e| format!("submit: {e}"))?; + self.phase = JobRunPhase::Running { + deadline: now + JOB_DEADLINE, + output_buf: Vec::new(), + outputs_ended: false, + pending_done: None, + }; + Ok(()) + } +} + +#[derive(Clone)] +struct JobRunTick; + +struct JobRunActor { + machine: JobRunStateMachine, + engine: EngineHandle, + sender: ExternalSender, + completion: ActorCompletion>, +} + +impl JobRunActor { + fn schedule(&self, ctx: &Ctx) { + self.engine + .send_after(POLL, self.sender.clone(), ctx.self_addr(), JobRunTick); + } +} + +impl ActorInterface for JobRunActor { + type Incoming = JobRunTick; + type Response = (); + + fn on_start(&mut self, ctx: &Ctx) { + self.machine.start(Instant::now()); + let _ = ctx.send(ctx.self_addr(), JobRunTick); + } + + fn handle(&mut self, ctx: &Ctx, _message: Self::Incoming) { + if let Some(result) = self.machine.advance(Instant::now()) { + assert!( + self.completion.complete(result).is_ok(), + "job lifecycle completed twice" + ); + ctx.stop_self(); + } else { + self.schedule(ctx); + } + } +} + impl JobOrchestratorSession { pub(crate) fn identity_json(&self) -> Result { serde_json::to_string(&self.identity).map_err(|e| e.to_string()) @@ -294,151 +705,29 @@ impl JobOrchestratorSession { } pub(crate) fn run_to_completion( - &mut self, - job: swactor_job_runner::Job, + self, + job: Job, worker: NodeIdentity, ) -> Result { - let node_actor = parse_actor(&worker.actor_hex)?; - self.driver.join(std::slice::from_ref(&worker.endpoint)); - - let started = Instant::now(); - while started.elapsed() < CONVERGE_DEADLINE { - if self - .stack - .route_view - .read() - .map(|v| v.contains_key(&node_actor)) - .unwrap_or(false) - { - break; - } - std::thread::sleep(POLL); - } - if !self - .stack - .route_view - .read() - .map(|v| v.contains_key(&node_actor)) - .unwrap_or(false) - { - return Err( - "directory did not converge: orchestrator never learned the worker actor".into(), - ); - } - eprintln!("job-orch: directory converged; waiting for iroh connection to worker"); - let worker_node = swactor_transport::NodeId(*worker.endpoint.id.as_bytes()); - let conn_started = Instant::now(); - while !self.driver.has_active_connection(&worker_node) { - if conn_started.elapsed() >= CONVERGE_DEADLINE { - eprintln!( - "job-orch: no iroh connection to worker after {CONVERGE_DEADLINE:?}; join_statuses={:?}; submitting best-effort", - self.driver.join_statuses() - ); - break; - } - std::thread::sleep(POLL); - } - if self.driver.has_active_connection(&worker_node) { - eprintln!( - "job-orch: iroh connection to worker established after {:?}", - conn_started.elapsed() - ); - eprintln!("job-orch: submitting job"); - } else { - eprintln!("job-orch: submitting job (no confirmed connection)"); - } - - // EDGE: push the workspace tar over EDGE_ALPN before submitting. Small - // commands/events still travel as actor messages; only bulk bytes move - // onto the edge transport so they survive relay (NAT) traversal. - let edge_events = self.driver.edge_events_handle(); - let workspace_bytes = - swactor_job_runner::pack_workspace(&job).map_err(|e| format!("pack workspace: {e}"))?; - if !workspace_bytes.is_empty() { - let pump = self - .driver - .spawn_edge_send_pump(worker.endpoint.clone(), WORKSPACE_EDGE_ID) - .map_err(|e| format!("workspace edge pump: {e}"))?; - for record in workspace_bytes.chunks(swactor_job_runner::EDGE_RECORD_SIZE) { - pump.send(record.to_vec()) - .map_err(|e| format!("workspace edge send: {e}"))?; - } - drop(pump); // finish the edge stream → receiver observes end-of-stream - eprintln!("job-orch: workspace pushed over EDGE_ALPN"); - } - - self.stack - .runtime - .send_to(self.orch, OrchestratorJobMsg::Submit { job, node_actor }) - .map_err(|e| format!("submit: {e}"))?; - - // Drive lifecycle (actor messages) while draining the output edge stream. - let started = Instant::now(); - let mut output_buf: Vec = Vec::new(); - let mut outputs_ended = false; - // The orchestrator actor reports `JobDone` exactly once; hold it here - // while we wait for the output edge stream to land so it is not lost. - let mut pending_done: Option = None; - loop { - // Drain output edge bytes; extract the tar as soon as the stream ends - // (release the edge-event lock before the potentially slow untar). - let drained: Vec = edge_events.lock().drain(..).collect(); - for ev in drained { - match ev { - WireEvent::BytesRead { edge_id, bytes, .. } if edge_id.0 == OUTPUTS_EDGE_ID => { - output_buf.extend_from_slice(&bytes); - } - WireEvent::StreamEnded { edge_id, .. } if edge_id.0 == OUTPUTS_EDGE_ID => { - if !output_buf.is_empty() { - if let Err(e) = - swactor_job_runner::extract_tar(&output_buf, &self.landing) - { - eprintln!("job-orch: untar edge outputs failed: {e}"); - } - output_buf.clear(); - } - outputs_ended = true; - } - _ => {} - } - } - - if pending_done.is_none() { - pending_done = self.done.try_recv(); - } - - // A job that ran (exit code observed) collected outputs over edge — - // wait for that stream to land before returning so the landing dir is - // populated. A pre-run fault (no exit code) ships no outputs. - let ready = match &pending_done { - Some(done) => { - let need_outputs = done.exit_code.is_some() && !outputs_ended; - !need_outputs || started.elapsed() >= JOB_DEADLINE - } - None => false, - }; - if ready { - let done = pending_done - .take() - .expect("pending_done observed Some in ready branch"); - if done.exit_code.is_some() && !outputs_ended { - eprintln!("job-orch: output edge stream did not land before deadline"); - } - return Ok(done); - } - - if started.elapsed() >= JOB_DEADLINE { - return Err("job did not complete within deadline".into()); - } - std::thread::sleep(POLL); - } + let runtime = self.stack.runtime.clone(); + let engine = self.stack.engine.clone(); + let completion = ActorCompletion::new(); + runtime + .spawn(JobRunActor { + machine: JobRunStateMachine::new(self, job, worker)?, + engine, + sender: runtime.create_sender(), + completion: completion.clone(), + }) + .map_err(|error| format!("spawn job lifecycle actor: {error}"))?; + completion.wait() } } /// Orchestrator: expose an `OrchestratorJobActor`, print its identity, read the /// worker identity from stdin, drive the job to completion over iroh. pub fn run_serve(job: swactor_job_runner::Job, landing: PathBuf) -> Result { - let mut session = start_orchestrator(landing)?; + let session = start_orchestrator(landing)?; println!("JOB_ORCH_IDENTITY {}", session.identity_json()?); let _ = std::io::Write::flush(&mut std::io::stdout()); eprintln!("job-orch: published identity; waiting for worker identity on stdin..."); @@ -473,40 +762,64 @@ impl swactor_job_runner::JobEdgeSink for IrohEdgeSink { /// without the readiness flag being set). const WORKSPACE_EDGE_WAIT: Duration = Duration::from_secs(60 * 30); -/// Drain EDGE_ALPN workspace bytes (edge id `WORKSPACE_EDGE_ID`) the orchestrator -/// pushed, extract the tar into `workdir`, then signal readiness. Runs on a -/// background worker thread; the driver auto-accepts EDGE_ALPN connections and -/// pushes their bytes into the shared event queue drained here. -fn drain_workspace_edge( +#[derive(Clone)] +struct WorkspaceEdgePoll; + +struct WorkspaceEdgeActor { + engine: EngineHandle, + sender: ExternalSender, events: Arc>>, workdir: PathBuf, ready: Arc, -) { - let mut buf = Vec::new(); - let started = Instant::now(); - loop { - let drained: Vec = events.lock().drain(..).collect(); - for ev in drained { - match ev { + buf: Vec, + started: Instant, +} + +impl WorkspaceEdgeActor { + fn schedule(&self, ctx: &Ctx, delay: Duration) { + self.engine.send_after( + delay, + self.sender.clone(), + ctx.self_addr(), + WorkspaceEdgePoll, + ); + } +} + +impl ActorInterface for WorkspaceEdgeActor { + type Incoming = WorkspaceEdgePoll; + type Response = (); + + fn on_start(&mut self, ctx: &Ctx) { + self.schedule(ctx, Duration::ZERO); + } + + fn handle(&mut self, ctx: &Ctx, _message: Self::Incoming) { + let drained: Vec = self.events.lock().drain(..).collect(); + for event in drained { + match event { WireEvent::BytesRead { edge_id, bytes, .. } if edge_id.0 == WORKSPACE_EDGE_ID => { - buf.extend_from_slice(&bytes); + self.buf.extend_from_slice(&bytes); } WireEvent::StreamEnded { edge_id, .. } if edge_id.0 == WORKSPACE_EDGE_ID => { - if !buf.is_empty() { - if let Err(e) = swactor_job_runner::extract_tar(&buf, &workdir) { - eprintln!("job-worker: untar workspace failed: {e}"); - } + if !self.buf.is_empty() + && let Err(error) = + swactor_job_runner::extract_tar(&self.buf, &self.workdir) + { + eprintln!("job-worker: untar workspace failed: {error}"); } - ready.store(true, Ordering::Release); + self.ready.store(true, Ordering::Release); + ctx.stop_self(); return; } _ => {} } } - if started.elapsed() >= WORKSPACE_EDGE_WAIT { + if self.started.elapsed() >= WORKSPACE_EDGE_WAIT { eprintln!("job-worker: workspace edge stream did not arrive"); + ctx.stop_self(); return; } - std::thread::sleep(POLL); + self.schedule(ctx, POLL); } } diff --git a/apps/myelin/src/node/worker_node_runtime.rs b/apps/myelin/src/node/worker_node_runtime.rs index b1f5d3a..0cd32ae 100644 --- a/apps/myelin/src/node/worker_node_runtime.rs +++ b/apps/myelin/src/node/worker_node_runtime.rs @@ -11,9 +11,8 @@ use std::path::{Path, PathBuf}; use std::process::{Child, ChildStdin, Command, ExitCode, Stdio}; use std::sync::{ Arc, - mpsc::{self, Receiver, RecvTimeoutError, Sender}, + mpsc::{self, Receiver, Sender, TryRecvError}, }; -use std::thread; use std::time::{Duration, Instant}; use telemetry::frame::TelemetryEvent; @@ -51,8 +50,9 @@ use iroh_driver::{EndpointAddrMask, MVP_IROH_ENDPOINT_ADDR_MASK_ENV, advertised_ use parking_lot::Mutex; use serde_json::{Value, json}; use swactor::actor::{ActorAddress, ActorInterface}; -use swactor::runtime::{Ctx, ExternalSender, Inbox}; -use swactor_engine::{Engine, EngineHandle, TokioBackend, TokioConfig}; +use swactor::runtime::{Ctx, ExternalSender, Inbox, Runtime}; +use swactor::stats::{ActorSnapshot, StatsHook}; +use swactor_engine::{ActorCompletion, Engine, EngineHandle, TokioBackend, TokioConfig}; use tokio::io::{AsyncBufReadExt, AsyncWriteExt}; const DEFAULT_WORKER_SCRIPT: &str = "/usr/local/share/myelin/tinygrad_worker.py"; @@ -163,6 +163,61 @@ fn emit_node_event( telemetry.tick(); } +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +struct DebugActorSnapshot { + address: String, + mailbox_depth: usize, + actor_type: String, + poisoned: bool, +} + +#[derive(Clone, Debug, Default, serde::Deserialize, serde::Serialize)] +struct DebugRuntimeStats { + actors: Vec, +} + +#[derive(Clone, Default)] +struct RuntimeStatsInspector { + latest: Arc>>>, +} + +impl RuntimeStatsInspector { + fn snapshot(&self) -> DebugRuntimeStats { + let mut actors = self + .latest + .lock() + .values() + .flatten() + .cloned() + .collect::>(); + actors.sort_by(|left, right| left.address.cmp(&right.address)); + DebugRuntimeStats { actors } + } +} + +struct InspectableStatsHook { + inner: Arc, + inspector: RuntimeStatsInspector, +} + +impl StatsHook for InspectableStatsHook { + fn on_tick(&self, worker_id: usize, snapshots: &[ActorSnapshot]) { + self.inspector.latest.lock().insert( + worker_id, + snapshots + .iter() + .map(|snapshot| DebugActorSnapshot { + address: snapshot.address.to_full_hex(), + mailbox_depth: snapshot.mailbox_depth, + actor_type: snapshot.actor_type.unwrap_or("").to_owned(), + poisoned: snapshot.poisoned, + }) + .collect(), + ); + self.inner.on_tick(worker_id, snapshots); + } +} + #[derive(serde::Deserialize, serde::Serialize)] #[serde(tag = "type")] enum DebugJoinRequestWire { @@ -171,6 +226,7 @@ enum DebugJoinRequestWire { #[serde(default)] orchestrator_actor: Option, }, + RuntimeStats, } #[derive(serde::Deserialize, serde::Serialize)] @@ -185,6 +241,9 @@ enum DebugJoinResponseWire { error: String, detail: String, }, + RuntimeStats { + stats: DebugRuntimeStats, + }, } enum DebugJoinCommand { @@ -193,6 +252,9 @@ enum DebugJoinCommand { orchestrator_actor: Option, reply: tokio::sync::oneshot::Sender, }, + RuntimeStats { + reply: tokio::sync::oneshot::Sender, + }, } enum DebugJoinClientError { @@ -294,6 +356,9 @@ pub(crate) fn request_debug_join( DebugJoinResponseWire::JoinRejected { error, detail } => { Err(format!("worker join rejected: {error}: {detail}")) } + DebugJoinResponseWire::RuntimeStats { .. } => { + Err("worker join returned runtime stats unexpectedly".to_owned()) + } } } @@ -352,47 +417,15 @@ fn spawn_debug_join_listener( } } let (command_tx, command_rx) = tokio::sync::mpsc::unbounded_channel::(); - let (ready_tx, ready_rx) = std::sync::mpsc::channel::>(); - let engine_inner = engine.clone(); - engine.spawn(async move { - let listener = match tokio::net::UnixListener::bind(&path) { - Ok(l) => l, - Err(e) => { - let _ = ready_tx.send(Err(format!( - "bind debug join socket {}: {e}", - path.display() - ))); - return; - } - }; - if let Err(e) = fs::set_permissions(&path, fs::Permissions::from_mode(0o600)) { - let _ = ready_tx.send(Err(format!( - "chmod debug join socket {}: {e}", - path.display() - ))); - return; + swactor_process::spawn_unix_stream_listener(engine, &path, move |stream| { + let command_tx = command_tx.clone(); + async move { + handle_debug_join_stream(stream, command_tx).await; } - let _ = ready_tx.send(Ok(())); - loop { - match listener.accept().await { - Ok((stream, _addr)) => { - let command_tx = command_tx.clone(); - engine_inner.spawn(async move { - handle_debug_join_stream(stream, command_tx).await; - }); - } - Err(error) => { - eprintln!("myelin-worker debug join listener stopped: {error}"); - break; - } - } - } - }); - match ready_rx.recv() { - Ok(Ok(())) => {} - Ok(Err(e)) => return Err(e), - Err(_) => return Err("debug join listener task dropped".to_owned()), - } + }) + .map_err(|error| format!("bind debug join socket {}: {error}", path.display()))?; + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)) + .map_err(|error| format!("chmod debug join socket {}: {error}", path.display()))?; Ok(command_rx) } @@ -434,6 +467,25 @@ async fn handle_debug_join_stream( }) } } + Ok(DebugJoinRequestWire::RuntimeStats) => { + let (reply, response_rx) = tokio::sync::oneshot::channel(); + if command_tx + .send(DebugJoinCommand::RuntimeStats { reply }) + .is_err() + { + DebugJoinResponseWire::JoinRejected { + error: "CommandQueueClosed".to_owned(), + detail: "worker main loop is not accepting debug commands".to_owned(), + } + } else { + response_rx + .await + .unwrap_or_else(|error| DebugJoinResponseWire::JoinRejected { + error: "CommandCancelled".to_owned(), + detail: error.to_string(), + }) + } + } Err(response) => response, }, Err(error) => DebugJoinResponseWire::JoinRejected { @@ -476,6 +528,7 @@ fn drain_debug_join_commands( config: &DeploymentConfig, telemetry: &mut NodeTelemetry, pending_control_rejoin: &mut PendingControlRejoin, + runtime_stats: &RuntimeStatsInspector, ) { let Some(rx) = debug_join_rx else { return; @@ -512,6 +565,11 @@ fn drain_debug_join_commands( direct_addr_count, }); } + DebugJoinCommand::RuntimeStats { reply } => { + let _ = reply.send(DebugJoinResponseWire::RuntimeStats { + stats: runtime_stats.snapshot(), + }); + } } } } @@ -638,7 +696,76 @@ fn submit_sampler_sample_health( ); } +#[derive(Clone)] +struct SamplerTick; + +struct BlockingSamplerActor { + engine: EngineHandle, + sender: ExternalSender, + producer: TelemetryProducer, + channel: ChannelId, + health_channel: ChannelId, + health_context: SamplerHealthContext, + sampler: &'static str, + sample_channel: &'static str, + interval: Duration, + sample_fn: fn(u64) -> S, + error_of: fn(&S) -> Option<&str>, + seq: u64, +} + +impl BlockingSamplerActor { + fn schedule(&self, ctx: &Ctx) + where + S: Record + Send + 'static, + { + self.engine.send_after( + self.interval, + self.sender.clone(), + ctx.self_addr(), + SamplerTick, + ); + } +} + +impl ActorInterface for BlockingSamplerActor +where + S: Record + Send + 'static, +{ + type Incoming = SamplerTick; + type Response = (); + + fn on_start(&mut self, ctx: &Ctx) { + submit_sampler_started( + &self.producer, + self.health_channel, + self.health_context, + self.sampler, + self.sample_channel, + self.interval, + ); + self.schedule(ctx); + } + + fn handle(&mut self, ctx: &Ctx, _message: Self::Incoming) { + let sample = (self.sample_fn)(self.seq); + submit_sampler_sample_health( + &self.producer, + self.health_channel, + self.health_context, + self.sampler, + self.sample_channel, + self.seq, + (self.error_of)(&sample), + ); + self.seq = self.seq.saturating_add(1); + self.producer.submit_record(self.channel, &sample); + self.schedule(ctx); + } +} + fn spawn_blocking_sampler( + runtime: Runtime, engine: EngineHandle, producer: TelemetryProducer, channel: ChannelId, @@ -647,54 +774,29 @@ fn spawn_blocking_sampler( sampler: &'static str, sample_channel: &'static str, interval: Duration, - error_label: &'static str, sample_fn: fn(u64) -> S, - error_fn: fn(u64, String) -> S, error_of: fn(&S) -> Option<&str>, ) { - let engine_inner = engine.clone(); - engine.spawn(async move { - submit_sampler_started( - &producer, + runtime + .spawn(BlockingSamplerActor { + engine, + sender: runtime.create_sender(), + producer, + channel, health_channel, health_context, sampler, sample_channel, interval, - ); - let mut seq = 0_u64; - let mut interval = engine_inner.interval(interval); - - loop { - (&mut interval).await; - - let sample_seq = seq; - let (tx, rx) = tokio::sync::oneshot::channel(); - engine_inner.spawn_blocking(move || { - let result = sample_fn(sample_seq); - let _ = tx.send(result); - }); - let sample = match rx.await { - Ok(sample) => sample, - Err(_) => error_fn(sample_seq, format!("{error_label}: dropped")), - }; - - submit_sampler_sample_health( - &producer, - health_channel, - health_context, - sampler, - sample_channel, - sample_seq, - error_of(&sample), - ); - seq = seq.saturating_add(1); - producer.submit_record(channel, &sample); - } - }); + sample_fn, + error_of, + seq: 0, + }) + .expect("spawn telemetry sampler actor"); } fn spawn_host_gpu_sampler( + runtime: Runtime, engine: EngineHandle, producer: TelemetryProducer, channel: ChannelId, @@ -702,6 +804,7 @@ fn spawn_host_gpu_sampler( health_context: SamplerHealthContext, ) { spawn_blocking_sampler( + runtime, engine, producer, channel, @@ -710,14 +813,68 @@ fn spawn_host_gpu_sampler( "gpu", telemetry::hardware::gpu::HOST_GPU_CHANNEL, telemetry::hardware::gpu::GPU_SAMPLE_INTERVAL, - "gpu sampler task failed", telemetry::hardware::gpu::sample, - telemetry::hardware::gpu::HostGpuSample::error, - |s| s.error.as_deref(), + |sample| sample.error.as_deref(), ); } +struct HostCpuSamplerActor { + engine: EngineHandle, + sender: ExternalSender, + producer: TelemetryProducer, + channel: ChannelId, + health_channel: ChannelId, + health_context: SamplerHealthContext, + sampler: telemetry::hardware::cpu::CpuSampler, + seq: u64, +} + +impl ActorInterface for HostCpuSamplerActor { + type Incoming = SamplerTick; + type Response = (); + + fn on_start(&mut self, ctx: &Ctx) { + submit_sampler_started( + &self.producer, + self.health_channel, + self.health_context, + "cpu", + telemetry::hardware::cpu::HOST_CPU_CHANNEL, + telemetry::hardware::cpu::CPU_SAMPLE_INTERVAL, + ); + self.schedule(ctx); + } + + fn handle(&mut self, ctx: &Ctx, _message: Self::Incoming) { + let sample = self.sampler.sample(self.seq); + submit_sampler_sample_health( + &self.producer, + self.health_channel, + self.health_context, + "cpu", + telemetry::hardware::cpu::HOST_CPU_CHANNEL, + self.seq, + sample.error.as_deref(), + ); + self.seq = self.seq.saturating_add(1); + self.producer.submit_record(self.channel, &sample); + self.schedule(ctx); + } +} + +impl HostCpuSamplerActor { + fn schedule(&self, ctx: &Ctx) { + self.engine.send_after( + telemetry::hardware::cpu::CPU_SAMPLE_INTERVAL, + self.sender.clone(), + ctx.self_addr(), + SamplerTick, + ); + } +} + fn spawn_host_cpu_sampler( + runtime: Runtime, engine: EngineHandle, producer: TelemetryProducer, channel: ChannelId, @@ -725,64 +882,22 @@ fn spawn_host_cpu_sampler( health_context: SamplerHealthContext, watched_pids: Vec, ) { - let engine_inner = engine.clone(); - engine.spawn(async move { - use telemetry::hardware::cpu::{ - CPU_SAMPLE_INTERVAL, CpuSampler, HOST_CPU_CHANNEL, HostCpuSample, - }; - submit_sampler_started( - &producer, + runtime + .spawn(HostCpuSamplerActor { + engine, + sender: runtime.create_sender(), + producer, + channel, health_channel, health_context, - "cpu", - HOST_CPU_CHANNEL, - CPU_SAMPLE_INTERVAL, - ); - let mut seq = 0_u64; - let recovery_pids = watched_pids.clone(); - let mut sampler = CpuSampler::new(watched_pids); - let mut interval = engine_inner.interval(CPU_SAMPLE_INTERVAL); - - loop { - (&mut interval).await; - - // CPU sampling reads `/proc` and performs blocking filesystem - // queries, so each query runs on the engine's blocking pool rather - // than the async core-driving worker. The stateful sampler is - // carried into and back out of each blocking call so its - // previous-sample deltas persist across samples - // (ENGINE_SPEC.md). - let (tx, rx) = tokio::sync::oneshot::channel(); - engine_inner.spawn_blocking(move || { - let sample = sampler.sample(seq); - let _ = tx.send((sampler, sample)); - }); - let sample = match rx.await { - Ok((returned, sample)) => { - sampler = returned; - sample - } - Err(_) => { - sampler = CpuSampler::new(recovery_pids.clone()); - HostCpuSample::error(seq, "cpu sampler blocking task dropped".to_string()) - } - }; - - submit_sampler_sample_health( - &producer, - health_channel, - health_context, - "cpu", - HOST_CPU_CHANNEL, - seq, - sample.error.as_deref(), - ); - seq = seq.saturating_add(1); - producer.submit_record(channel, &sample); - } - }); + sampler: telemetry::hardware::cpu::CpuSampler::new(watched_pids), + seq: 0, + }) + .expect("spawn CPU sampler actor"); } + fn spawn_host_net_sampler( + runtime: Runtime, engine: EngineHandle, producer: TelemetryProducer, channel: ChannelId, @@ -790,6 +905,7 @@ fn spawn_host_net_sampler( health_context: SamplerHealthContext, ) { spawn_blocking_sampler( + runtime, engine, producer, channel, @@ -798,32 +914,64 @@ fn spawn_host_net_sampler( "net", telemetry::hardware::net::HOST_NET_CHANNEL, telemetry::hardware::net::HOST_NET_SAMPLE_INTERVAL, - "network sampler task failed", telemetry::hardware::net::sample, - telemetry::hardware::net::HostNetSample::error, - |s| s.error.as_deref(), + |sample| sample.error.as_deref(), ); } +struct ArenaSamplerActor { + engine: EngineHandle, + sender: ExternalSender, + producer: TelemetryProducer, + channel: ChannelId, + arena_manager: Arc>, + seq: u64, +} + +impl ArenaSamplerActor { + fn schedule(&self, ctx: &Ctx) { + self.engine.send_after( + arena::ARENA_SAMPLE_INTERVAL, + self.sender.clone(), + ctx.self_addr(), + SamplerTick, + ); + } +} + +impl ActorInterface for ArenaSamplerActor { + type Incoming = SamplerTick; + type Response = (); + + fn on_start(&mut self, ctx: &Ctx) { + self.schedule(ctx); + } + + fn handle(&mut self, ctx: &Ctx, _message: Self::Incoming) { + let sample: arena::ArenaSample = self.arena_manager.lock().sample(self.seq).into(); + self.seq = self.seq.saturating_add(1); + self.producer.submit_record(self.channel, &sample); + self.schedule(ctx); + } +} + fn spawn_arena_sampler( + runtime: Runtime, engine: EngineHandle, producer: TelemetryProducer, channel: ChannelId, arena_manager: Arc>, ) { - let engine_inner = engine.clone(); - engine.spawn(async move { - let mut seq = 0_u64; - let mut interval = engine_inner.interval(arena::ARENA_SAMPLE_INTERVAL); - - loop { - (&mut interval).await; - - let sample: arena::ArenaSample = arena_manager.lock().sample(seq).into(); - seq = seq.saturating_add(1); - producer.submit_record(channel, &sample); - } - }); + runtime + .spawn(ArenaSamplerActor { + engine, + sender: runtime.create_sender(), + producer, + channel, + arena_manager, + seq: 0, + }) + .expect("spawn arena sampler actor"); } struct LoadedObject { @@ -1458,8 +1606,6 @@ fn run_stage_shard_fetcher() -> Result<(), String> { }) } -// synchronous process-control sequencing: polls helper-process liveness and shutdown; the engine drives all background actor/transport/sampling work (ENGINE_SPEC.md §2) -#[allow(clippy::disallowed_methods)] fn run() -> Result<(), String> { let config = DeploymentConfig::from_env()?; let boot = |phase: &str, status: &str, detail: Value| { @@ -1497,7 +1643,16 @@ fn run() -> Result<(), String> { // integrations, then hand the workers to the engine. The engine owns both // core progression and the Tokio substrate (it schedules all background // work); components retain only cheap Runtime handles (ENGINE_SPEC.md). - let worker_stats_hook = telemetry.producer.stats_hook(); + let runtime_stats = RuntimeStatsInspector::default(); + let telemetry_stats_hook = telemetry.producer.stats_hook(); + let worker_stats_hook: Arc = if config.debug_join_socket.is_some() { + Arc::new(InspectableStatsHook { + inner: telemetry_stats_hook, + inspector: runtime_stats.clone(), + }) + } else { + telemetry_stats_hook + }; let (parts, runtime, codec, transport_router) = DistributionRuntimeStack::build_runtime( |registry| { register_myelin_actor_codecs(registry); @@ -1564,7 +1719,7 @@ fn run() -> Result<(), String> { } let stack = DistributionRuntimeStack::new_from_runtime( - runtime, + runtime.clone(), codec, transport_router, driver.node_id(), @@ -1633,14 +1788,12 @@ fn run() -> Result<(), String> { let node_runtime = |ds: &mut NodeTelemetry, phase: &str, status: &str, detail: Value| { emit_node_event(ds, &config, NODE_RUNTIME_CHANNEL, phase, status, detail) }; - let node_shutdown = |ds: &mut NodeTelemetry, phase: &str, status: &str, detail: Value| { - emit_node_event(ds, &config, NODE_SHUTDOWN_CHANNEL, phase, status, detail) - }; // Telemetry leaves this node exclusively through pull subscriptions served // by `serve_telemetry_pulls` on `TELEMETRY_ALPN`; no publisher actor. let sampler_health_channel = telemetry.channel_by_name(NODE_SAMPLER_CHANNEL); let sampler_health_context = SamplerHealthContext::from_config(&config); spawn_host_gpu_sampler( + runtime.clone(), engine.handle(), telemetry.producer.clone(), telemetry.channels.host_gpu, @@ -1648,6 +1801,7 @@ fn run() -> Result<(), String> { sampler_health_context, ); spawn_host_net_sampler( + runtime.clone(), engine.handle(), telemetry.producer.clone(), telemetry.channels.host_net, @@ -1655,6 +1809,7 @@ fn run() -> Result<(), String> { sampler_health_context, ); spawn_arena_sampler( + runtime.clone(), engine.handle(), telemetry.producer.clone(), telemetry.channels.arena, @@ -1687,7 +1842,7 @@ fn run() -> Result<(), String> { }), ); } - let mut debug_join_rx = match &config.debug_join_socket { + let debug_join_rx = match &config.debug_join_socket { Some(path) => match spawn_debug_join_listener(engine.handle(), PathBuf::from(path)) { Ok(rx) => { node_runtime( @@ -1771,7 +1926,7 @@ fn run() -> Result<(), String> { }; stack.register_local_actor(driver.register_actor(node_actor, 1)); stack.register_local_actor(driver.register_actor(*rejoin_replies.addr(), 1)); - let mut pending_control_rejoin = PendingControlRejoin::new( + let pending_control_rejoin = PendingControlRejoin::new( &config, &advertised_self_endpoint, driver.node_id(), @@ -1790,7 +1945,7 @@ fn run() -> Result<(), String> { "ready", json!({"framework":"none","workloads":"external_jobs"}), )?; - let mut pending_runtime_ready = + let pending_runtime_ready = PendingRuntimeReady::new(&config, advertised_self_endpoint.clone(), node_actor); let ready = json!({ "type":"ready", @@ -1811,7 +1966,6 @@ fn run() -> Result<(), String> { "readiness_id":pending_runtime_ready.readiness_id, }), )?; - let shutdown_rx = spawn_stdin_shutdown_listener(config.exit_on_stdin_eof); node_runtime( &mut telemetry, "main_loop", @@ -1822,81 +1976,39 @@ fn run() -> Result<(), String> { "checks":["network","telemetry","node_reports","stdin_shutdown"], }), ); - loop { - pending_control_rejoin.drive(&stack, node_actor, &rejoin_replies)?; - emit_swim_telemetry(&mut telemetry, &stack, "agent_loop"); - drain_debug_join_commands( - &mut debug_join_rx, - &mut driver, - &config, - &mut telemetry, - &mut pending_control_rejoin, - ); - telemetry.tick(); - serve_telemetry_pulls(&driver, &engine.handle(), &telemetry.endpoint); - while let Some(report) = reports.try_recv() { - if let NodeAgentReport::RuntimeReadyAck { - run_id, - node_id, - stage_index, - readiness_id, - } = report - && pending_runtime_ready.observe_ack(run_id, node_id, stage_index, readiness_id) - { - node_boot( - &mut telemetry, - "runtime_ready_ack", - "ready", - json!({ - "readiness_id":readiness_id, - "attempts":pending_runtime_ready.attempts, - "endpoint":&pending_runtime_ready.endpoint, - "node_actor":pending_runtime_ready.node_actor, - }), - ); - telemetry.submit_text(telemetry.channels.node_ready, ready.to_string()); - } - } - if !pending_runtime_ready.swim_logged && pending_runtime_ready.swim_ready(&stack) { - node_runtime( - &mut telemetry, - "coordinator_swim", - "ready", - json!({ - "coordinator":pending_runtime_ready - .coordinator - .map(|node| format!("{node:?}")) - .unwrap_or_else(|| "standalone".to_owned()), - "readiness_id":pending_runtime_ready.readiness_id, - }), - ); - pending_runtime_ready.swim_logged = true; - } - if !pending_runtime_ready.acked - && pending_runtime_ready.maybe_send(&stack, node_actor)? - { - node_runtime( - &mut telemetry, - "runtime_ready_signal", - "sent", - json!({ - "readiness_id":pending_runtime_ready.readiness_id, - "attempts":pending_runtime_ready.attempts, - "next_backoff_ms":pending_runtime_ready.backoff.as_millis(), - }), - ); - } - if shutdown_rx.try_recv().is_ok() { - node_shutdown( - &mut telemetry, - "node_exit", - "ready", - json!({"result":"ok","mode":"agent_only"}), - ); - return Ok(()); - } - thread::sleep(PUMP_INTERVAL); - } + let actor_runtime = stack.runtime.clone(); + let sender = actor_runtime.create_sender(); + let exit_on_stdin_eof = config.exit_on_stdin_eof; + let completion = ActorCompletion::new(); + let runtime_actor = actor_runtime + .spawn(AgentNodeRuntimeActor { + effects: AgentNodeRuntimeLive { + config, + telemetry, + driver, + stack, + debug_join_rx, + runtime_stats: runtime_stats.clone(), + pending_control_rejoin, + rejoin_replies, + }, + reports, + pending_runtime_ready, + ready, + node_actor, + engine: engine.handle(), + sender: sender.clone(), + completion: completion.clone(), + }) + .map_err(|error| format!("spawn agent node runtime actor: {error}"))?; + let stop_actor = actor_runtime + .spawn(StdinStopForwarder { + sender: sender.clone(), + target: runtime_actor, + }) + .map_err(|error| format!("spawn stdin stop forwarder: {error}"))?; + spawn_stdin_shutdown_listener(exit_on_stdin_eof, sender, stop_actor); + return completion.wait(); } worker_evt( @@ -1911,14 +2023,16 @@ fn run() -> Result<(), String> { "stderr":"piped", }), )?; - let mut worker = match TinygradWorker::spawn(&config, arena_fd, engine.handle()) { - Ok(worker) => worker, - Err(error) => { - worker_evt("worker_process", "failed", json!({"error":error}))?; - return Err(error); - } - }; + let mut worker = + match TinygradWorker::spawn(&config, arena_fd, runtime.clone(), engine.handle()) { + Ok(worker) => worker, + Err(error) => { + worker_evt("worker_process", "failed", json!({"error":error}))?; + return Err(error); + } + }; spawn_host_cpu_sampler( + runtime.clone(), engine.handle(), telemetry.producer.clone(), telemetry.channels.host_cpu, @@ -1942,8 +2056,8 @@ fn run() -> Result<(), String> { return Err(error); } } - let mut edge_runtime = WorkerEdgeRuntime::new(config.logical_node_id); - let mut pending_runtime_ready = + let edge_runtime = WorkerEdgeRuntime::new(config.logical_node_id); + let pending_runtime_ready = PendingRuntimeReady::new(&config, advertised_self_endpoint.clone(), node_actor); let ready = json!({ @@ -1977,7 +2091,6 @@ fn run() -> Result<(), String> { )?; } - let shutdown_rx = spawn_stdin_shutdown_listener(config.exit_on_stdin_eof); node_runtime( &mut telemetry, "stdin_shutdown_listener", @@ -1993,132 +2106,421 @@ fn run() -> Result<(), String> { "checks":["network","edge_streams","telemetry","node_reports","stdin_shutdown","worker_health"], }), ); - loop { - pending_control_rejoin.drive(&stack, node_actor, &rejoin_replies)?; - emit_swim_telemetry(&mut telemetry, &stack, "main_loop"); - drain_debug_join_commands( - &mut debug_join_rx, - &mut driver, - &config, - &mut telemetry, - &mut pending_control_rejoin, - ); - telemetry.tick(); - serve_telemetry_pulls(&driver, &engine.handle(), &telemetry.endpoint); - drain_worker_stderr(&worker.stderr_rx, &config, &mut telemetry); - edge_runtime.poll_iroh( - &mut driver, - &stack, + let actor_runtime = stack.runtime.clone(); + let sender = actor_runtime.create_sender(); + let exit_on_stdin_eof = config.exit_on_stdin_eof; + let completion = ActorCompletion::new(); + let runtime_actor = actor_runtime + .spawn(WorkerNodeRuntimeActor { + effects: WorkerNodeRuntimeLive { + config, + telemetry, + driver, + stack, + debug_join_rx, + runtime_stats: runtime_stats.clone(), + pending_control_rejoin, + rejoin_replies, + worker, + edge_runtime, + arena_manager, + }, + reports, + pending_runtime_ready, + ready, node_actor, - &mut worker, - &arena_manager, - &config, - &mut telemetry, - )?; - while let Some(report) = reports.try_recv() { - match handle_node_report( - report, - &config, - &stack, - &mut driver, - node_actor, - &mut worker, - &mut edge_runtime, - &arena_manager, - &mut telemetry, - )? { - NodeReportOutcome::None => {} - NodeReportOutcome::RuntimeReadyAck { + engine: engine.handle(), + sender: sender.clone(), + completion: completion.clone(), + }) + .map_err(|error| format!("spawn worker node runtime actor: {error}"))?; + let stop_actor = actor_runtime + .spawn(StdinStopForwarder { + sender: sender.clone(), + target: runtime_actor, + }) + .map_err(|error| format!("spawn stdin stop forwarder: {error}"))?; + spawn_stdin_shutdown_listener(exit_on_stdin_eof, sender, stop_actor); + completion.wait() +} +#[derive(Clone, Copy)] +enum NodeRuntimeMsg { + Tick, + Shutdown, +} + +struct StdinStopForwarder { + sender: ExternalSender, + target: ActorAddress, +} + +impl ActorInterface for StdinStopForwarder { + type Incoming = swactor_process::ProcessStopSignal; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, _message: Self::Incoming) { + let _ = self.sender.send_to(self.target, NodeRuntimeMsg::Shutdown); + ctx.stop_self(); + } +} + +trait AgentNodeRuntimeEffects: Send + 'static { + fn tick_before_reports(&mut self, node_actor: ActorAddress) -> Result<(), String>; + + fn publish_runtime_ready(&mut self, pending: &PendingRuntimeReady, ready: &Value); + + fn tick_after_reports( + &mut self, + pending: &mut PendingRuntimeReady, + node_actor: ActorAddress, + ) -> Result<(), String>; + + fn shutdown(&mut self); + + fn record_finish(&mut self, _result: &Result<(), String>) {} +} + +struct AgentNodeRuntimeLive { + config: DeploymentConfig, + telemetry: NodeTelemetry, + driver: IrohDriver, + stack: DistributionRuntimeStack, + debug_join_rx: Option>, + runtime_stats: RuntimeStatsInspector, + pending_control_rejoin: PendingControlRejoin, + rejoin_replies: Inbox, +} + +impl AgentNodeRuntimeEffects for AgentNodeRuntimeLive { + fn tick_before_reports(&mut self, node_actor: ActorAddress) -> Result<(), String> { + self.pending_control_rejoin + .drive(&self.stack, node_actor, &self.rejoin_replies)?; + emit_swim_telemetry(&mut self.telemetry, &self.stack, "agent_loop"); + drain_debug_join_commands( + &mut self.debug_join_rx, + &mut self.driver, + &self.config, + &mut self.telemetry, + &mut self.pending_control_rejoin, + &self.runtime_stats, + ); + self.telemetry.tick(); + serve_telemetry_pulls(&self.driver, &self.stack.engine, &self.telemetry.endpoint); + Ok(()) + } + + fn publish_runtime_ready(&mut self, pending: &PendingRuntimeReady, ready: &Value) { + emit_node_event( + &mut self.telemetry, + &self.config, + NODE_BOOTSTRAP_CHANNEL, + "runtime_ready_ack", + "ready", + json!({ + "readiness_id":pending.readiness_id, + "attempts":pending.attempts, + "endpoint":&pending.endpoint, + "node_actor":pending.node_actor, + }), + ); + self.telemetry + .submit_text(self.telemetry.channels.node_ready, ready.to_string()); + } + + fn tick_after_reports( + &mut self, + pending: &mut PendingRuntimeReady, + node_actor: ActorAddress, + ) -> Result<(), String> { + if !pending.swim_logged && pending.swim_ready(&self.stack) { + emit_node_event( + &mut self.telemetry, + &self.config, + NODE_RUNTIME_CHANNEL, + "coordinator_swim", + "ready", + json!({ + "coordinator":pending + .coordinator + .map(|node| format!("{node:?}")) + .unwrap_or_else(|| "standalone".to_owned()), + "readiness_id":pending.readiness_id, + }), + ); + pending.swim_logged = true; + } + if !pending.acked && pending.maybe_send(&self.stack, node_actor)? { + emit_node_event( + &mut self.telemetry, + &self.config, + NODE_RUNTIME_CHANNEL, + "runtime_ready_signal", + "sent", + json!({ + "readiness_id":pending.readiness_id, + "attempts":pending.attempts, + "next_backoff_ms":pending.backoff.as_millis(), + }), + ); + } + Ok(()) + } + + fn shutdown(&mut self) { + emit_node_event( + &mut self.telemetry, + &self.config, + NODE_SHUTDOWN_CHANNEL, + "node_exit", + "ready", + json!({"result":"ok","mode":"agent_only"}), + ); + } + + fn record_finish(&mut self, result: &Result<(), String>) { + if let Err(error) = result { + emit_node_event( + &mut self.telemetry, + &self.config, + NODE_SHUTDOWN_CHANNEL, + "node_runtime", + "failed", + json!({"error":error}), + ); + } + } +} + +struct AgentNodeRuntimeActor { + effects: E, + reports: Inbox, + pending_runtime_ready: PendingRuntimeReady, + ready: Value, + node_actor: ActorAddress, + engine: EngineHandle, + sender: ExternalSender, + completion: ActorCompletion>, +} + +impl AgentNodeRuntimeActor { + fn schedule_tick(&self, ctx: &Ctx) { + self.engine.send_after( + PUMP_INTERVAL, + self.sender.clone(), + ctx.self_addr(), + NodeRuntimeMsg::Tick, + ); + } + + fn tick(&mut self) -> Result<(), String> { + self.effects.tick_before_reports(self.node_actor)?; + while let Some(report) = self.reports.try_recv() { + if let NodeAgentReport::RuntimeReadyAck { + run_id, + node_id, + stage_index, + readiness_id, + } = report + && self.pending_runtime_ready.observe_ack( run_id, node_id, stage_index, readiness_id, - } => { - if pending_runtime_ready.observe_ack(run_id, node_id, stage_index, readiness_id) - { - node_boot( - &mut telemetry, - "runtime_ready_ack", - "ready", - json!({ - "readiness_id":readiness_id, - "attempts":pending_runtime_ready.attempts, - "endpoint":&pending_runtime_ready.endpoint, - "node_actor":pending_runtime_ready.node_actor, - }), - ); - telemetry.submit_text(telemetry.channels.node_ready, ready.to_string()); - node_boot( - &mut telemetry, - "telemetry_handoff", - "ready", - json!({"from":"runtime_ready_ack","to":"cluster_telemetry","channel":"myelin.node.ready"}), - ); - } - } + ) + { + self.effects + .publish_runtime_ready(&self.pending_runtime_ready, &self.ready); } } - if !pending_runtime_ready.swim_logged && pending_runtime_ready.swim_ready(&stack) { - node_runtime( - &mut telemetry, + self.effects + .tick_after_reports(&mut self.pending_runtime_ready, self.node_actor) + } + + fn finish(&mut self, ctx: &Ctx, result: Result<(), String>) { + self.effects.record_finish(&result); + assert!( + self.completion.complete(result).is_ok(), + "agent node runtime completed twice" + ); + ctx.stop_self(); + } +} + +impl ActorInterface for AgentNodeRuntimeActor { + type Incoming = NodeRuntimeMsg; + type Response = (); + + fn on_start(&mut self, ctx: &Ctx) { + let _ = ctx.send(ctx.self_addr(), NodeRuntimeMsg::Tick); + } + + fn handle(&mut self, ctx: &Ctx, message: Self::Incoming) { + match message { + NodeRuntimeMsg::Tick => match self.tick() { + Ok(()) => self.schedule_tick(ctx), + Err(error) => self.finish(ctx, Err(error)), + }, + NodeRuntimeMsg::Shutdown => { + self.effects.shutdown(); + self.finish(ctx, Ok(())); + } + } + } +} + +trait WorkerNodeRuntimeEffects: Send + 'static { + fn tick_before_reports(&mut self, node_actor: ActorAddress) -> Result<(), String>; + + fn handle_report( + &mut self, + report: NodeAgentReport, + node_actor: ActorAddress, + ) -> Result; + + fn publish_runtime_ready(&mut self, pending: &PendingRuntimeReady, ready: &Value); + + fn tick_after_reports( + &mut self, + pending: &mut PendingRuntimeReady, + node_actor: ActorAddress, + ) -> Result<(), String>; + + fn shutdown(&mut self); + + fn record_finish(&mut self, _result: &Result<(), String>) {} +} + +struct WorkerNodeRuntimeLive { + config: DeploymentConfig, + telemetry: NodeTelemetry, + driver: IrohDriver, + stack: DistributionRuntimeStack, + debug_join_rx: Option>, + runtime_stats: RuntimeStatsInspector, + pending_control_rejoin: PendingControlRejoin, + rejoin_replies: Inbox, + worker: TinygradWorker, + edge_runtime: WorkerEdgeRuntime, + arena_manager: Arc>, +} + +impl WorkerNodeRuntimeEffects for WorkerNodeRuntimeLive { + fn tick_before_reports(&mut self, node_actor: ActorAddress) -> Result<(), String> { + self.pending_control_rejoin + .drive(&self.stack, node_actor, &self.rejoin_replies)?; + emit_swim_telemetry(&mut self.telemetry, &self.stack, "main_loop"); + drain_debug_join_commands( + &mut self.debug_join_rx, + &mut self.driver, + &self.config, + &mut self.telemetry, + &mut self.pending_control_rejoin, + &self.runtime_stats, + ); + self.telemetry.tick(); + serve_telemetry_pulls(&self.driver, &self.stack.engine, &self.telemetry.endpoint); + drain_worker_stderr(&self.worker.stderr_rx, &self.config, &mut self.telemetry); + self.edge_runtime.poll_iroh( + &mut self.driver, + &self.stack, + node_actor, + &mut self.worker, + &self.arena_manager, + &self.config, + &mut self.telemetry, + ) + } + + fn handle_report( + &mut self, + report: NodeAgentReport, + node_actor: ActorAddress, + ) -> Result { + handle_node_report( + report, + &self.config, + &self.stack, + &mut self.driver, + node_actor, + &mut self.worker, + &mut self.edge_runtime, + &self.arena_manager, + &mut self.telemetry, + ) + } + + fn publish_runtime_ready(&mut self, pending: &PendingRuntimeReady, ready: &Value) { + emit_node_event( + &mut self.telemetry, + &self.config, + NODE_BOOTSTRAP_CHANNEL, + "runtime_ready_ack", + "ready", + json!({ + "readiness_id":pending.readiness_id, + "attempts":pending.attempts, + "endpoint":&pending.endpoint, + "node_actor":pending.node_actor, + }), + ); + self.telemetry + .submit_text(self.telemetry.channels.node_ready, ready.to_string()); + emit_node_event( + &mut self.telemetry, + &self.config, + NODE_BOOTSTRAP_CHANNEL, + "telemetry_handoff", + "ready", + json!({"from":"runtime_ready_ack","to":"cluster_telemetry","channel":"myelin.node.ready"}), + ); + } + + fn tick_after_reports( + &mut self, + pending: &mut PendingRuntimeReady, + node_actor: ActorAddress, + ) -> Result<(), String> { + if !pending.swim_logged && pending.swim_ready(&self.stack) { + emit_node_event( + &mut self.telemetry, + &self.config, + NODE_RUNTIME_CHANNEL, "coordinator_swim", "ready", json!({ - "coordinator":pending_runtime_ready + "coordinator":pending .coordinator .map(|node| format!("{node:?}")) .unwrap_or_else(|| "standalone".to_owned()), - "readiness_id":pending_runtime_ready.readiness_id, + "readiness_id":pending.readiness_id, }), ); - pending_runtime_ready.swim_logged = true; + pending.swim_logged = true; } - if !pending_runtime_ready.acked && pending_runtime_ready.maybe_send(&stack, node_actor)? { - node_runtime( - &mut telemetry, + if !pending.acked && pending.maybe_send(&self.stack, node_actor)? { + emit_node_event( + &mut self.telemetry, + &self.config, + NODE_RUNTIME_CHANNEL, "runtime_ready_signal", "sent", json!({ - "readiness_id":pending_runtime_ready.readiness_id, - "attempts":pending_runtime_ready.attempts, - "next_backoff_ms":pending_runtime_ready.backoff.as_millis(), + "readiness_id":pending.readiness_id, + "attempts":pending.attempts, + "next_backoff_ms":pending.backoff.as_millis(), }), ); } - if shutdown_rx.try_recv().is_ok() { - node_shutdown( - &mut telemetry, - "shutdown", - "started", - json!({"source":"stdin","command":"shutdown"}), - ); - match worker.shutdown(&config, &mut telemetry) { - Ok(()) => { - node_shutdown( - &mut telemetry, - "worker_shutdown", - "ready", - json!({"worker_event_type":"WorkerStopped"}), - ); - node_shutdown(&mut telemetry, "node_exit", "ready", json!({"result":"ok"})); - } - Err(error) => node_shutdown( - &mut telemetry, - "worker_shutdown", - "failed", - json!({"error":error}), - ), - } - return Ok(()); - } - if let Some(status) = worker.try_wait()? { - node_shutdown( - &mut telemetry, + if let Some(status) = self.worker.try_wait()? { + emit_node_event( + &mut self.telemetry, + &self.config, + NODE_SHUTDOWN_CHANNEL, "worker_process", "failed", json!({"exit_status":status.to_string()}), ); - let _ = stack.runtime.send_to( + let _ = self.stack.runtime.send_to( node_actor, NodeAgentMsg::WorkerCrashed { reason: Some(format!("tinygrad helper exited with {status}")), @@ -2126,7 +2528,126 @@ fn run() -> Result<(), String> { ); return Err(format!("tinygrad helper exited with {status}")); } - thread::sleep(PUMP_INTERVAL); + Ok(()) + } + + fn shutdown(&mut self) { + emit_node_event( + &mut self.telemetry, + &self.config, + NODE_SHUTDOWN_CHANNEL, + "shutdown", + "started", + json!({"source":"stdin","command":"shutdown"}), + ); + match self.worker.shutdown(&self.config, &mut self.telemetry) { + Ok(()) => { + emit_node_event( + &mut self.telemetry, + &self.config, + NODE_SHUTDOWN_CHANNEL, + "worker_shutdown", + "ready", + json!({"worker_event_type":"WorkerStopped"}), + ); + emit_node_event( + &mut self.telemetry, + &self.config, + NODE_SHUTDOWN_CHANNEL, + "node_exit", + "ready", + json!({"result":"ok"}), + ); + } + Err(error) => emit_node_event( + &mut self.telemetry, + &self.config, + NODE_SHUTDOWN_CHANNEL, + "worker_shutdown", + "failed", + json!({"error":error}), + ), + } + } +} + +struct WorkerNodeRuntimeActor { + effects: E, + reports: Inbox, + pending_runtime_ready: PendingRuntimeReady, + ready: Value, + node_actor: ActorAddress, + engine: EngineHandle, + sender: ExternalSender, + completion: ActorCompletion>, +} + +impl WorkerNodeRuntimeActor { + fn schedule_tick(&self, ctx: &Ctx) { + self.engine.send_after( + PUMP_INTERVAL, + self.sender.clone(), + ctx.self_addr(), + NodeRuntimeMsg::Tick, + ); + } + + fn tick(&mut self) -> Result<(), String> { + self.effects.tick_before_reports(self.node_actor)?; + while let Some(report) = self.reports.try_recv() { + match self.effects.handle_report(report, self.node_actor)? { + NodeReportOutcome::None => {} + NodeReportOutcome::RuntimeReadyAck { + run_id, + node_id, + stage_index, + readiness_id, + } => { + if self.pending_runtime_ready.observe_ack( + run_id, + node_id, + stage_index, + readiness_id, + ) { + self.effects + .publish_runtime_ready(&self.pending_runtime_ready, &self.ready); + } + } + } + } + self.effects + .tick_after_reports(&mut self.pending_runtime_ready, self.node_actor) + } + + fn finish(&mut self, ctx: &Ctx, result: Result<(), String>) { + self.effects.record_finish(&result); + assert!( + self.completion.complete(result).is_ok(), + "worker node runtime completed twice" + ); + ctx.stop_self(); + } +} + +impl ActorInterface for WorkerNodeRuntimeActor { + type Incoming = NodeRuntimeMsg; + type Response = (); + + fn on_start(&mut self, ctx: &Ctx) { + let _ = ctx.send(ctx.self_addr(), NodeRuntimeMsg::Tick); + } + + fn handle(&mut self, ctx: &Ctx, message: Self::Incoming) { + match message { + NodeRuntimeMsg::Tick => match self.tick() { + Ok(()) => self.schedule_tick(ctx), + Err(error) => self.finish(ctx, Err(error)), + }, + NodeRuntimeMsg::Shutdown => { + self.effects.shutdown(); + self.finish(ctx, Ok(())); + } + } } } @@ -2971,19 +3492,19 @@ enum StageShardFetchMsg { ReaderClosed { stream: StageShardProcessStream, }, + #[cfg(test)] + ProcessExited(std::process::ExitStatus), } -#[derive(Clone, Debug)] -enum StageShardFetchReport { - Progress(Value), - Done(PathBuf), - Failed(String), +struct StageShardFetchOutcome { + events: Vec, + result: Result, } struct StageShardFetchActor { request_json: Vec, output_path: PathBuf, - report_to: ActorAddress, + completion: ActorCompletion, sender: ExternalSender, /// The node's engine. Reader tasks and delayed messages schedule on this /// stored handle; the actor never creates another engine @@ -2994,6 +3515,7 @@ struct StageShardFetchActor { stderr_closed: bool, ready_path: Option, exit_status: Option, + events: Vec, finished: bool, } @@ -3001,14 +3523,14 @@ impl StageShardFetchActor { fn new( request_json: Vec, output_path: PathBuf, - report_to: ActorAddress, + completion: ActorCompletion, sender: ExternalSender, engine: EngineHandle, ) -> Self { Self { request_json, output_path, - report_to, + completion, sender, engine, child: None, @@ -3016,6 +3538,7 @@ impl StageShardFetchActor { stderr_closed: true, ready_path: None, exit_status: None, + events: Vec::new(), finished: false, } } @@ -3031,13 +3554,13 @@ impl StageShardFetchActor { return; } }; - let mut child = match Command::new(exe) - .arg("stage-shard-fetcher") - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - { + let mut child = match swactor_process::command_spawn( + &mut Command::new(exe) + .arg("stage-shard-fetcher") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()), + ) { Ok(child) => child, Err(error) => { self.fail(ctx, format!("spawn stage shard fetcher: {error}")); @@ -3071,7 +3594,6 @@ impl StageShardFetchActor { stdout, self.sender.clone(), ctx.self_addr(), - &self.engine, ); } else { self.stdout_closed = true; @@ -3082,7 +3604,6 @@ impl StageShardFetchActor { stderr, self.sender.clone(), ctx.self_addr(), - &self.engine, ); } else { self.stderr_closed = true; @@ -3104,12 +3625,8 @@ impl StageShardFetchActor { let Some(child) = self.child.as_mut() else { return; }; - match child.try_wait() { - Ok(Some(status)) => { - self.child = None; - self.exit_status = Some(status); - self.maybe_finish(ctx); - } + match swactor_process::child_try_wait(child) { + Ok(Some(status)) => self.handle_process_exit(ctx, status), Ok(None) => schedule_stage_shard_message( &self.engine, self.sender.clone(), @@ -3123,8 +3640,16 @@ impl StageShardFetchActor { } } } + fn handle_process_exit(&mut self, ctx: &Ctx, status: std::process::ExitStatus) { + if self.finished || self.exit_status.is_some() { + return; + } + self.child = None; + self.exit_status = Some(status); + self.maybe_finish(ctx); + } - fn handle_line(&mut self, ctx: &Ctx, stream: StageShardProcessStream, line: String) { + fn handle_line(&mut self, _ctx: &Ctx, stream: StageShardProcessStream, line: String) { if line.is_empty() || self.finished { return; } @@ -3144,7 +3669,7 @@ impl StageShardFetchActor { .map(PathBuf::from) .or_else(|| Some(self.output_path.clone())); } - let _ = ctx.send(self.report_to, StageShardFetchReport::Progress(event)); + self.events.push(event); } fn handle_reader_error(&mut self, ctx: &Ctx, stream: StageShardProcessStream, error: String) { @@ -3173,7 +3698,15 @@ impl StageShardFetchActor { .unwrap_or_else(|| self.output_path.clone()); if path.is_file() { self.finished = true; - let _ = ctx.send(self.report_to, StageShardFetchReport::Done(path)); + assert!( + self.completion + .complete(StageShardFetchOutcome { + events: std::mem::take(&mut self.events), + result: Ok(path), + }) + .is_ok(), + "stage shard fetch completed twice" + ); ctx.stop_self(); return; } @@ -3196,7 +3729,15 @@ impl StageShardFetchActor { self.finished = true; stop_stage_shard_child(&mut self.child); self.join_readers(); - let _ = ctx.send(self.report_to, StageShardFetchReport::Failed(error)); + assert!( + self.completion + .complete(StageShardFetchOutcome { + events: std::mem::take(&mut self.events), + result: Err(error), + }) + .is_ok(), + "stage shard fetch completed twice" + ); ctx.stop_self(); } @@ -3223,6 +3764,8 @@ impl ActorInterface for StageShardFetchActor { self.handle_reader_error(ctx, stream, error) } StageShardFetchMsg::ReaderClosed { stream } => self.handle_reader_closed(ctx, stream), + #[cfg(test)] + StageShardFetchMsg::ProcessExited(status) => self.handle_process_exit(ctx, status), } } @@ -3236,8 +3779,8 @@ fn stop_stage_shard_child(child: &mut Option) { let Some(mut child) = child.take() else { return; }; - let _ = child.kill(); - let _ = child.wait(); + let _ = swactor_process::child_kill(&mut child); + let _ = swactor_process::child_wait(&mut child); } fn spawn_stage_shard_reader( @@ -3245,41 +3788,15 @@ fn spawn_stage_shard_reader( reader: R, sender: ExternalSender, actor: ActorAddress, - engine: &EngineHandle, ) { - // Blocking stdout/stderr reads run on the engine's blocking pool so they - // never occupy an async core-driving worker. Each decoded line is delivered - // to the actor through the existing external sender (ENGINE_SPEC.md). - engine.spawn_blocking(move || { - let mut reader = BufReader::new(reader); - let mut line = String::new(); - loop { - line.clear(); - match reader.read_line(&mut line) { - Ok(0) => break, - Ok(_) => { - let _ = sender.send_to( - actor, - StageShardFetchMsg::ProcessLine { - stream, - line: line.trim_end_matches(['\r', '\n']).to_owned(), - }, - ); - } - Err(error) => { - let _ = sender.send_to( - actor, - StageShardFetchMsg::ReaderError { - stream, - error: error.to_string(), - }, - ); - break; - } - } - } - let _ = sender.send_to(actor, StageShardFetchMsg::ReaderClosed { stream }); - }); + swactor_process::spawn_mapped_line_reader( + reader, + sender, + actor, + move |line| StageShardFetchMsg::ProcessLine { stream, line }, + move |error| StageShardFetchMsg::ReaderError { stream, error }, + StageShardFetchMsg::ReaderClosed { stream }, + ); } fn schedule_stage_shard_message( @@ -3289,14 +3806,7 @@ fn schedule_stage_shard_message( msg: StageShardFetchMsg, delay: Duration, ) { - // Delayed actor messages use an engine task plus an engine timer, measured - // in engine time, rather than a std thread plus sleep - // (ENGINE_SPEC.md). - let timer_engine = engine.clone(); - engine.clone().spawn(async move { - timer_engine.timer(delay).await; - let _ = sender.send_to(actor, msg); - }); + engine.send_after(delay, sender, actor, msg); } fn publish_stage_shard_fetch_event( @@ -3313,8 +3823,6 @@ fn publish_stage_shard_fetch_event( Ok(()) } -// synchronous process-control sequencing: waits for an engine-driven actor; the engine drives all background work (ENGINE_SPEC.md §2) -#[allow(clippy::disallowed_methods)] fn materialize_stage_shard_with_process( plan: &StageShardPlan, config: &DeploymentConfig, @@ -3365,16 +3873,13 @@ fn materialize_stage_shard_with_process( }; let request_json = serde_json::to_vec(&request) .map_err(|e| format!("serialize stage shard fetch request: {e}"))?; - let reports = stack - .runtime - .new_inbox::() - .map_err(|e| format!("stage shard fetch report inbox: {e}"))?; + let completion = ActorCompletion::new(); let actor = stack .runtime .spawn(StageShardFetchActor::new( request_json, output_path, - *reports.addr(), + completion.clone(), stack.runtime.create_sender(), stack.engine.clone(), )) @@ -3383,23 +3888,11 @@ fn materialize_stage_shard_with_process( .runtime .send_to(actor, StageShardFetchMsg::Start) .map_err(|e| format!("start stage shard fetch actor: {e}"))?; - - loop { - while let Some(report) = reports.try_recv() { - match report { - StageShardFetchReport::Progress(event) => { - if let Err(error) = publish_stage_shard_fetch_event(telemetry, config, &event) { - let _ = stack.runtime.stop_actor(actor); - return Err(error); - } - } - StageShardFetchReport::Done(path) => return Ok(path), - StageShardFetchReport::Failed(error) => return Err(error), - } - } - telemetry.tick(); - thread::sleep(PUMP_INTERVAL); + let outcome = completion.wait(); + for event in outcome.events { + publish_stage_shard_fetch_event(telemetry, config, &event)?; } + outcome.result } fn handle_stage_command( @@ -3881,58 +4374,27 @@ impl HelperCommandWaitConfig { } } -fn spawn_helper_stdout_reader( - reader: R, - tx: Sender, - engine: &EngineHandle, -) { - // Blocking helper stdout reads run on the engine's blocking pool, never on - // an async core-driving worker. The channel, parsing, and - // actor/application-facing behavior are unchanged (ENGINE_SPEC.md). - engine.spawn_blocking(move || { - let mut reader = BufReader::new(reader); - loop { - let mut line = String::new(); - match reader.read_line(&mut line) { - Ok(0) => { - let _ = tx.send(HelperStdoutEvent::Closed); - break; - } - Ok(_) => { - if tx.send(HelperStdoutEvent::Line(line)).is_err() { - break; - } - } - Err(error) => { - let _ = tx.send(HelperStdoutEvent::ReadError(error.to_string())); - break; - } - } - } - }); +fn spawn_helper_stdout_reader(reader: R, tx: Sender) { + swactor_process::spawn_mapped_line_channel( + reader, + tx, + HelperStdoutEvent::Line, + HelperStdoutEvent::ReadError, + HelperStdoutEvent::Closed, + ); } -fn spawn_helper_stderr_reader( - reader: R, - tx: Sender, - engine: &EngineHandle, -) { - engine.spawn_blocking(move || { - for line in BufReader::new(reader).lines().map_while(Result::ok) { - if tx.send(line).is_err() { - break; - } - } - }); +fn spawn_helper_stderr_reader(reader: R, tx: Sender) { + swactor_process::spawn_line_channel(reader, tx); } fn drain_worker_stderr( - stderr_rx: &Receiver, + stderr_rx: &Arc>>, config: &DeploymentConfig, telemetry: &mut NodeTelemetry, ) { let mut emitted = false; - while let Ok(line) = stderr_rx.try_recv() { + while let Ok(line) = stderr_rx.lock().try_recv() { let payload = node_event_payload(config, "worker_stderr", "observed", json!({"line":line})); telemetry.submit_text(telemetry.channels.worker_stderr, payload.to_string()); emitted = true; @@ -3942,201 +4404,323 @@ fn drain_worker_stderr( } } +#[derive(Clone, Copy)] +struct HelperWaitTick; + +struct HelperWaitOutcome { + result: Result, + lines: Vec, + stderr_lines: Vec, + wait_samples: Vec<(u64, u64)>, +} + +struct HelperWaitActor { + stdout_rx: Arc>>, + stderr_rx: Option>>>, + expected: String, + engine: EngineHandle, + sender: ExternalSender, + completion: ActorCompletion, + wait_config: HelperCommandWaitConfig, + wait_started: Instant, + next_telemetry_at: Instant, + wait_cycles: u64, + lines: Vec, + stderr_lines: Vec, + wait_samples: Vec<(u64, u64)>, +} + +impl HelperWaitActor { + fn schedule(&self, ctx: &Ctx) { + self.engine.send_after( + self.wait_config.poll_interval, + self.sender.clone(), + ctx.self_addr(), + HelperWaitTick, + ); + } + + fn drain_stderr(&mut self) { + let Some(stderr_rx) = &self.stderr_rx else { + return; + }; + while let Ok(line) = stderr_rx.lock().try_recv() { + self.stderr_lines.push(line); + } + } + + fn finish(&mut self, ctx: &Ctx, result: Result) { + self.drain_stderr(); + assert!( + self.completion + .complete(HelperWaitOutcome { + result, + lines: std::mem::take(&mut self.lines), + stderr_lines: std::mem::take(&mut self.stderr_lines), + wait_samples: std::mem::take(&mut self.wait_samples), + }) + .is_ok(), + "helper wait completed twice" + ); + ctx.stop_self(); + } + + fn poll(&mut self, ctx: &Ctx) { + self.drain_stderr(); + loop { + let observation = self.stdout_rx.lock().try_recv(); + match observation { + Ok(HelperStdoutEvent::Line(line)) => { + let parsed = serde_json::from_str::(&line); + self.lines.push(line.clone()); + let value = match parsed { + Ok(value) => value, + Err(error) => { + self.finish(ctx, Err(format!("parse helper stdout {line:?}: {error}"))); + return; + } + }; + if value.get("type").and_then(Value::as_str) == Some("WorkerFatal") { + self.finish(ctx, Err(format!("worker fatal: {value}"))); + return; + } + if value.get("type").and_then(Value::as_str) == Some(self.expected.as_str()) { + self.finish(ctx, Ok(value)); + return; + } + } + Ok(HelperStdoutEvent::Closed) => { + self.finish( + ctx, + Err(format!( + "tinygrad helper stdout closed while waiting for {}", + self.expected + )), + ); + return; + } + Ok(HelperStdoutEvent::ReadError(error)) => { + self.finish(ctx, Err(format!("read helper stdout: {error}"))); + return; + } + Err(TryRecvError::Empty) => { + self.wait_cycles = self.wait_cycles.saturating_add(1); + let now = Instant::now(); + if now >= self.next_telemetry_at { + self.wait_samples.push(( + duration_ms_u64(now.saturating_duration_since(self.wait_started)), + self.wait_cycles, + )); + self.next_telemetry_at = now + .checked_add(self.wait_config.telemetry_interval) + .unwrap_or(now); + } + self.schedule(ctx); + return; + } + Err(TryRecvError::Disconnected) => { + self.finish( + ctx, + Err(format!( + "tinygrad helper stdout reader disconnected while waiting for {}", + self.expected + )), + ); + return; + } + } + } + } +} + +impl ActorInterface for HelperWaitActor { + type Incoming = HelperWaitTick; + type Response = (); + + fn on_start(&mut self, ctx: &Ctx) { + let _ = ctx.send(ctx.self_addr(), HelperWaitTick); + } + + fn handle(&mut self, ctx: &Ctx, _message: Self::Incoming) { + self.poll(ctx); + } +} + #[allow(clippy::too_many_arguments)] fn wait_for_helper_event( - stdout_rx: &Receiver, - stderr_rx: Option<&Receiver>, + runtime: &Runtime, + stdout_rx: Arc>>, + stderr_rx: Option>>>, expected: &str, command_type: &str, config: &DeploymentConfig, telemetry: &mut NodeTelemetry, channel: ChannelId, channel_name: &str, + engine: &EngineHandle, wait_config: HelperCommandWaitConfig, ) -> Result { - let node_worker = |ds: &mut NodeTelemetry, phase: &str, status: &str, detail: Value| { - emit_node_event(ds, config, NODE_WORKER_CHANNEL, phase, status, detail) - }; - node_worker( + emit_node_event( telemetry, + config, + NODE_WORKER_CHANNEL, "worker_stdout_read", "started", json!({"command_type":command_type,"expected_event_type":expected,"channel":channel_name}), ); - let wait_started = Instant::now(); - let mut wait_cycles = 0_u64; - let mut next_telemetry_at = wait_started; - loop { - match stdout_rx.recv_timeout(wait_config.poll_interval) { - Ok(HelperStdoutEvent::Line(line)) => { - if let Some(stderr_rx) = stderr_rx { - drain_worker_stderr(stderr_rx, config, telemetry); - } - let line_bytes = line.len(); - node_worker( - telemetry, - "worker_stdout_read", - "ready", - json!({"command_type":command_type,"expected_event_type":expected,"channel":channel_name,"line_bytes":line_bytes}), - ); - node_worker( + let completion = ActorCompletion::new(); + runtime + .spawn(HelperWaitActor { + stdout_rx, + stderr_rx, + expected: expected.to_owned(), + engine: engine.clone(), + sender: runtime.create_sender(), + completion: completion.clone(), + wait_config, + wait_started: Instant::now(), + next_telemetry_at: Instant::now(), + wait_cycles: 0, + lines: Vec::new(), + stderr_lines: Vec::new(), + wait_samples: Vec::new(), + }) + .map_err(|error| format!("spawn helper wait actor: {error}"))?; + let outcome = completion.wait(); + for line in outcome.stderr_lines { + let payload = node_event_payload(config, "worker_stderr", "observed", json!({"line":line})); + telemetry.submit_text(telemetry.channels.worker_stderr, payload.to_string()); + } + for (elapsed_ms, wait_cycles) in outcome.wait_samples { + emit_node_event( + telemetry, + config, + NODE_WORKER_CHANNEL, + "worker_command_wait", + "waiting", + json!({ + "command_type":command_type, + "expected_event_type":expected, + "channel":channel_name, + "state":"waiting_for_helper_stdout", + "elapsed_ms":elapsed_ms, + "wait_cycles":wait_cycles, + "poll_interval_ms":duration_ms_u64(wait_config.poll_interval), + }), + ); + } + for line in outcome.lines { + let line_bytes = line.len(); + emit_node_event( + telemetry, + config, + NODE_WORKER_CHANNEL, + "worker_stdout_read", + "ready", + json!({"command_type":command_type,"expected_event_type":expected,"channel":channel_name,"line_bytes":line_bytes}), + ); + let value: Value = match serde_json::from_str(&line) { + Ok(value) => value, + Err(error) => { + emit_node_event( telemetry, + config, + NODE_WORKER_CHANNEL, "worker_stdout_parse", - "started", - json!({"command_type":command_type,"expected_event_type":expected,"channel":channel_name,"line_bytes":line_bytes}), - ); - let value: Value = match serde_json::from_str(&line) { - Ok(value) => value, - Err(error) => { - node_worker( - telemetry, - "worker_stdout_parse", - "failed", - json!({"command_type":command_type,"expected_event_type":expected,"channel":channel_name,"line_bytes":line_bytes,"error":error.to_string()}), - ); - return Err(format!("parse helper stdout {line:?}: {error}")); - } - }; - let worker_event_type = value - .get("type") - .and_then(Value::as_str) - .unwrap_or("unknown"); - node_worker( - telemetry, - "worker_stdout_parse", - "ready", - json!({"command_type":command_type,"expected_event_type":expected,"channel":channel_name,"line_bytes":line_bytes,"worker_event_type":worker_event_type}), - ); - telemetry.submit_text(channel, value.to_string()); - emit_stdio_telemetry_frame(channel_name, &value) - .map_err(|e| format!("emit worker stdio telemetry frame: {e}"))?; - telemetry.tick(); - if worker_event_type == "WorkerFatal" { - return Err(format!("worker fatal: {value}")); - } - if value.get("type").and_then(Value::as_str) == Some(expected) { - return Ok(value); - } - node_worker( - telemetry, - "worker_event", - "observed", - json!({"command_type":command_type,"command_waiting_for":expected,"worker_event_type":worker_event_type,"event":value}), - ); - } - Ok(HelperStdoutEvent::Closed) => { - if let Some(stderr_rx) = stderr_rx { - drain_worker_stderr(stderr_rx, config, telemetry); - } - node_worker( - telemetry, - "worker_stdout_read", "failed", - json!({"command_type":command_type,"expected_event_type":expected,"channel":channel_name,"line_bytes":0,"error":"stdout closed"}), + json!({"command_type":command_type,"expected_event_type":expected,"channel":channel_name,"line_bytes":line_bytes,"error":error.to_string()}), ); - return Err(format!( - "tinygrad helper stdout closed while waiting for {expected}" - )); - } - Ok(HelperStdoutEvent::ReadError(error)) => { - if let Some(stderr_rx) = stderr_rx { - drain_worker_stderr(stderr_rx, config, telemetry); - } - node_worker( - telemetry, - "worker_stdout_read", - "failed", - json!({"command_type":command_type,"expected_event_type":expected,"channel":channel_name,"error":error}), - ); - return Err(format!("read helper stdout: {error}")); - } - Err(RecvTimeoutError::Timeout) => { - wait_cycles = wait_cycles.saturating_add(1); - if let Some(stderr_rx) = stderr_rx { - drain_worker_stderr(stderr_rx, config, telemetry); - } - let now = Instant::now(); - if now >= next_telemetry_at { - node_worker( - telemetry, - "worker_command_wait", - "waiting", - json!({ - "command_type":command_type, - "expected_event_type":expected, - "channel":channel_name, - "state":"busy_waiting_for_helper_stdout", - "elapsed_ms":duration_ms_u64(now.saturating_duration_since(wait_started)), - "wait_cycles":wait_cycles, - "poll_interval_ms":duration_ms_u64(wait_config.poll_interval), - }), - ); - next_telemetry_at = now - .checked_add(wait_config.telemetry_interval) - .unwrap_or(now); - } - telemetry.tick(); - } - Err(RecvTimeoutError::Disconnected) => { - node_worker( - telemetry, - "worker_stdout_read", - "failed", - json!({"command_type":command_type,"expected_event_type":expected,"channel":channel_name,"line_bytes":0,"error":"stdout reader disconnected"}), - ); - return Err(format!( - "tinygrad helper stdout reader disconnected while waiting for {expected}" - )); + continue; } + }; + let worker_event_type = value + .get("type") + .and_then(Value::as_str) + .unwrap_or("unknown"); + emit_node_event( + telemetry, + config, + NODE_WORKER_CHANNEL, + "worker_stdout_parse", + "ready", + json!({"command_type":command_type,"expected_event_type":expected,"channel":channel_name,"line_bytes":line_bytes,"worker_event_type":worker_event_type}), + ); + telemetry.submit_text(channel, value.to_string()); + emit_stdio_telemetry_frame(channel_name, &value) + .map_err(|error| format!("emit worker stdio telemetry frame: {error}"))?; + if worker_event_type != expected { + emit_node_event( + telemetry, + config, + NODE_WORKER_CHANNEL, + "worker_event", + "observed", + json!({"command_type":command_type,"command_waiting_for":expected,"worker_event_type":worker_event_type,"event":value}), + ); } } + if let Err(error) = &outcome.result { + emit_node_event( + telemetry, + config, + NODE_WORKER_CHANNEL, + "worker_stdout_read", + "failed", + json!({"command_type":command_type,"expected_event_type":expected,"channel":channel_name,"error":error}), + ); + } + outcome.result } struct TinygradWorker { child: Child, stdin: ChildStdin, - stdout_rx: Receiver, - stderr_rx: Receiver, + stdout_rx: Arc>>, + stderr_rx: Arc>>, + runtime: Runtime, + engine: EngineHandle, } impl TinygradWorker { fn spawn( config: &DeploymentConfig, arena_fd: std::os::fd::RawFd, + runtime: Runtime, engine: EngineHandle, ) -> Result { - let mut child = Command::new("python3") - .arg(&config.worker_script) - .env("DEV", &config.device) - .env("MYELIN_RUN_ID", config.run_id.to_string()) - .env("MYELIN_LOGICAL_NODE_ID", config.logical_node_id.to_string()) - .env("MYELIN_STAGE_INDEX", config.stage_index.to_string()) - .env("MYELIN_ARENA_FD", arena_fd.to_string()) - .env("MYELIN_ARENA_BYTES", config.arena_bytes.to_string()) - .env( - "MYELIN_TELEMETRY_ENDPOINT_ID", - format!( - "worker-node-{}-stage-{}-stdio-bridge", - config.logical_node_id, config.stage_index - ), - ) - .env( - "MYELIN_BENCHMARK_PRODUCER_INSTANCE", - format!( - "tinygrad-worker:{}:{}", - config.logical_node_id, config.stage_index - ), - ) - .env( - "MVP_IROH_ENDPOINT_ADDR_MASK", - config.endpoint_addr_mask.as_str(), - ) - .env("MYELIN_IROH_RELAY_MODE", format!("{:?}", config.relay_mode)) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .map_err(|e| format!("spawn tinygrad helper {}: {e}", config.worker_script))?; + let mut child = swactor_process::command_spawn( + &mut Command::new("python3") + .arg(&config.worker_script) + .env("DEV", &config.device) + .env("MYELIN_RUN_ID", config.run_id.to_string()) + .env("MYELIN_LOGICAL_NODE_ID", config.logical_node_id.to_string()) + .env("MYELIN_STAGE_INDEX", config.stage_index.to_string()) + .env("MYELIN_ARENA_FD", arena_fd.to_string()) + .env("MYELIN_ARENA_BYTES", config.arena_bytes.to_string()) + .env( + "MYELIN_TELEMETRY_ENDPOINT_ID", + format!( + "worker-node-{}-stage-{}-stdio-bridge", + config.logical_node_id, config.stage_index + ), + ) + .env( + "MYELIN_BENCHMARK_PRODUCER_INSTANCE", + format!( + "tinygrad-worker:{}:{}", + config.logical_node_id, config.stage_index + ), + ) + .env( + "MVP_IROH_ENDPOINT_ADDR_MASK", + config.endpoint_addr_mask.as_str(), + ) + .env("MYELIN_IROH_RELAY_MODE", format!("{:?}", config.relay_mode)) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()), + ) + .map_err(|e| format!("spawn tinygrad helper {}: {e}", config.worker_script))?; let stdin = child .stdin .take() @@ -4150,14 +4734,16 @@ impl TinygradWorker { .take() .ok_or_else(|| "tinygrad helper stderr missing".to_owned())?; let (stdout_tx, stdout_rx) = mpsc::channel(); - spawn_helper_stdout_reader(stdout, stdout_tx, &engine); + spawn_helper_stdout_reader(stdout, stdout_tx); let (stderr_tx, stderr_rx) = mpsc::channel(); - spawn_helper_stderr_reader(stderr, stderr_tx, &engine); + spawn_helper_stderr_reader(stderr, stderr_tx); Ok(Self { child, stdin, - stdout_rx, - stderr_rx, + stdout_rx: Arc::new(Mutex::new(stdout_rx)), + stderr_rx: Arc::new(Mutex::new(stderr_rx)), + runtime, + engine, }) } @@ -4459,8 +5045,7 @@ impl TinygradWorker { } fn try_wait(&mut self) -> Result, String> { - self.child - .try_wait() + swactor_process::child_try_wait(&mut self.child) .map_err(|e| format!("poll tinygrad helper: {e}")) } @@ -4533,14 +5118,16 @@ impl TinygradWorker { channel_name: &str, ) -> Result { wait_for_helper_event( - &self.stdout_rx, - Some(&self.stderr_rx), + &self.runtime, + Arc::clone(&self.stdout_rx), + Some(Arc::clone(&self.stderr_rx)), expected, command_type, config, telemetry, channel, channel_name, + &self.engine, HelperCommandWaitConfig::production(), ) } @@ -4548,26 +5135,1081 @@ impl TinygradWorker { impl Drop for TinygradWorker { fn drop(&mut self) { - let _ = self.child.kill(); - let _ = self.child.wait(); + let _ = swactor_process::child_kill(&mut self.child); + let _ = swactor_process::child_wait(&mut self.child); } } -// blocking user-stdin thread is process control, out of scope (ENGINE_SPEC.md §2) -#[allow(clippy::disallowed_methods)] -fn spawn_stdin_shutdown_listener(exit_on_eof: bool) -> Receiver<()> { - let (tx, rx) = mpsc::channel(); - thread::spawn(move || { - let stdin = std::io::stdin(); - for line in stdin.lock().lines().map_while(Result::ok) { - if line.trim().eq_ignore_ascii_case("shutdown") { - let _ = tx.send(()); - return; +fn spawn_stdin_shutdown_listener(exit_on_eof: bool, sender: ExternalSender, actor: ActorAddress) { + swactor_process::spawn_stdin_command_wait("shutdown", exit_on_eof, sender, actor); +} + +#[cfg(test)] +mod control_flow_properties { + use std::os::unix::process::ExitStatusExt; + use std::sync::{Arc, mpsc}; + + use iroh::{EndpointAddr, SecretKey}; + use parking_lot::Mutex; + use proptest::prelude::*; + use swactor::actor::ActorAddress; + use swactor::config::RuntimeConfig; + use swactor::runtime::{Runtime, RuntimeParts}; + use swactor_engine::{ActorCompletion, Engine, SteppingBackend}; + + use super::*; + use crate::node_actor::StageLifecycleWire; + use crate::tests::fuzz_support::{actor_census, advance_and_drive, drive_steps}; + + const DRIVE_PER_ACTION: usize = 16; + const FINAL_DRIVE_BUDGET: usize = 256; + + fn check_runtime_clean( + runtime: &Runtime, + backend: &SteppingBackend, + baseline_actors: usize, + baseline_tasks: usize, + ) -> Result<(), String> { + let stats = runtime.stats(); + let poisoned = stats + .actor_details + .iter() + .filter(|actor| actor.poisoned) + .count(); + let panics = stats + .workers + .iter() + .map(|worker| worker.panics) + .sum::(); + let mailbox_depth = stats + .workers + .iter() + .map(|worker| worker.mailbox_depth) + .sum::() + + stats + .actor_details + .iter() + .map(|actor| actor.mailbox_depth) + .sum::(); + let actors = stats.actors.len(); + let tasks = backend.pending_task_count(); + if poisoned != 0 + || panics != 0 + || mailbox_depth != 0 + || actors != baseline_actors + || tasks != baseline_tasks + { + Err(format!( + "poisoned={poisoned} panics={panics} mailbox={mailbox_depth} \ + actors={actors}/{baseline_actors} tasks={tasks}/{baseline_tasks}\n{}", + actor_census(runtime), + )) + } else { + Ok(()) + } + } + + #[derive(Clone, Debug)] + enum RuntimeAction { + Tick, + CurrentReadinessAck, + DuplicateReadinessAck, + StaleReadinessAck(u8), + Snapshot, + Lifecycle(u8), + WorkerExit, + Shutdown, + DuplicateShutdown, + } + + fn runtime_actions() -> impl Strategy> { + prop::collection::vec( + prop_oneof![ + 4 => Just(RuntimeAction::Tick), + 3 => Just(RuntimeAction::CurrentReadinessAck), + 2 => Just(RuntimeAction::DuplicateReadinessAck), + 3 => any::().prop_map(RuntimeAction::StaleReadinessAck), + 2 => Just(RuntimeAction::Snapshot), + 2 => any::().prop_map(RuntimeAction::Lifecycle), + 2 => Just(RuntimeAction::WorkerExit), + 2 => Just(RuntimeAction::Shutdown), + 2 => Just(RuntimeAction::DuplicateShutdown), + ], + 0..=32, + ) + } + + #[derive(Clone, Debug, Default, PartialEq, Eq)] + struct RuntimeEvidence { + ticks: usize, + readiness: usize, + snapshots: usize, + lifecycle: usize, + shutdowns: usize, + finishes: usize, + failed_finishes: usize, + } + + struct TestAgentRuntimeEffects(Arc>); + + impl AgentNodeRuntimeEffects for TestAgentRuntimeEffects { + fn tick_before_reports(&mut self, _node_actor: ActorAddress) -> Result<(), String> { + self.0.lock().ticks += 1; + Ok(()) + } + + fn publish_runtime_ready(&mut self, _pending: &PendingRuntimeReady, _ready: &Value) { + self.0.lock().readiness += 1; + } + + fn tick_after_reports( + &mut self, + _pending: &mut PendingRuntimeReady, + _node_actor: ActorAddress, + ) -> Result<(), String> { + Ok(()) + } + + fn shutdown(&mut self) { + self.0.lock().shutdowns += 1; + } + + fn record_finish(&mut self, result: &Result<(), String>) { + let mut evidence = self.0.lock(); + evidence.finishes += 1; + evidence.failed_finishes += result.is_err() as usize; + } + } + + struct TestWorkerRuntimeEffects { + evidence: Arc>, + fail_next_tick: Arc>, + } + + impl WorkerNodeRuntimeEffects for TestWorkerRuntimeEffects { + fn tick_before_reports(&mut self, _node_actor: ActorAddress) -> Result<(), String> { + self.evidence.lock().ticks += 1; + Ok(()) + } + + fn handle_report( + &mut self, + report: NodeAgentReport, + _node_actor: ActorAddress, + ) -> Result { + match report { + NodeAgentReport::RuntimeReadyAck { + run_id, + node_id, + stage_index, + readiness_id, + } => Ok(NodeReportOutcome::RuntimeReadyAck { + run_id, + node_id, + stage_index, + readiness_id, + }), + NodeAgentReport::Snapshot { .. } => { + self.evidence.lock().snapshots += 1; + Ok(NodeReportOutcome::None) + } + NodeAgentReport::Lifecycle(_) => { + self.evidence.lock().lifecycle += 1; + Ok(NodeReportOutcome::None) + } + _ => Ok(NodeReportOutcome::None), } } - if exit_on_eof { - let _ = tx.send(()); + + fn publish_runtime_ready(&mut self, _pending: &PendingRuntimeReady, _ready: &Value) { + self.evidence.lock().readiness += 1; } - }); - rx + + fn tick_after_reports( + &mut self, + _pending: &mut PendingRuntimeReady, + _node_actor: ActorAddress, + ) -> Result<(), String> { + if std::mem::take(&mut *self.fail_next_tick.lock()) { + Err("scripted worker process exit".to_owned()) + } else { + Ok(()) + } + } + + fn shutdown(&mut self) { + self.evidence.lock().shutdowns += 1; + } + + fn record_finish(&mut self, result: &Result<(), String>) { + let mut evidence = self.evidence.lock(); + evidence.finishes += 1; + evidence.failed_finishes += result.is_err() as usize; + } + } + + #[derive(Clone, Debug, Default)] + struct ExpectedRuntimeEvidence { + agent_readiness: usize, + worker_readiness: usize, + snapshots: usize, + lifecycle: usize, + agent_shutdowns: usize, + worker_shutdowns: usize, + worker_failed: bool, + } + + fn check_runtime_evidence( + expected: &ExpectedRuntimeEvidence, + agent: &RuntimeEvidence, + worker: &RuntimeEvidence, + ) -> Result<(), String> { + let valid = agent.readiness == expected.agent_readiness + && worker.readiness == expected.worker_readiness + && worker.snapshots == expected.snapshots + && worker.lifecycle == expected.lifecycle + && agent.shutdowns == expected.agent_shutdowns + && worker.shutdowns == expected.worker_shutdowns + && agent.finishes == 1 + && agent.failed_finishes == 0 + && worker.finishes == 1 + && worker.failed_finishes == expected.worker_failed as usize + && agent.ticks > 0 + && worker.ticks > 0; + if valid { + Ok(()) + } else { + Err(format!( + "expected={expected:?}, agent={agent:?}, worker={worker:?}" + )) + } + } + + fn pending_runtime_ready(node_actor: ActorAddress) -> PendingRuntimeReady { + PendingRuntimeReady { + run_id: 7, + node_id: 11, + stage_index: 3, + endpoint: EndpointAddr::new(SecretKey::from_bytes(&[9; 32]).public()), + node_actor, + coordinator: None, + readiness_id: 99, + attempts: 0, + next_attempt_at: Instant::now(), + backoff: RUNTIME_READY_RETRY_INITIAL, + acked: false, + swim_logged: false, + } + } + + fn readiness_report(stale: Option) -> NodeAgentReport { + let (mut run_id, mut node_id, mut stage_index, mut readiness_id) = (7, 11, 3, 99); + if let Some(kind) = stale { + match kind % 4 { + 0 => run_id += 1, + 1 => node_id += 1, + 2 => stage_index += 1, + _ => readiness_id += 1, + } + } + NodeAgentReport::RuntimeReadyAck { + run_id, + node_id, + stage_index, + readiness_id, + } + } + + fn send_report_and_tick( + runtime: &Runtime, + report_to: ActorAddress, + actor: ActorAddress, + report: NodeAgentReport, + ) { + let _ = runtime.send_to(report_to, report); + let _ = runtime.send_to(actor, NodeRuntimeMsg::Tick); + } + + proptest! { + #![proptest_config(ProptestConfig { + cases: 128, + max_shrink_iters: 2_000, + ..ProptestConfig::default() + })] + + #[test] + fn runtime_actors_generated_transitions_complete_once_on_one_worker( + actions in runtime_actions() + ) { + let mut config = RuntimeConfig::default(); + config.worker_count = 1; + let parts = RuntimeParts::new(config); + let runtime = parts.runtime().clone(); + let backend = SteppingBackend::new(); + let engine = Engine::new(parts, backend.clone()).expect("one-worker stepping engine"); + let baseline_actors = runtime.stats().actors.len(); + let baseline_tasks = backend.pending_task_count(); + let agent_reports = runtime.new_inbox::().expect("agent reports"); + let worker_reports = runtime.new_inbox::().expect("worker reports"); + let agent_reports_addr = *agent_reports.addr(); + let worker_reports_addr = *worker_reports.addr(); + let agent_node = ActorAddress([4; 32]); + let worker_node = ActorAddress([5; 32]); + let agent_completion = ActorCompletion::new(); + let worker_completion = ActorCompletion::new(); + let agent_evidence = Arc::new(Mutex::new(RuntimeEvidence::default())); + let worker_evidence = Arc::new(Mutex::new(RuntimeEvidence::default())); + let fail_next_worker_tick = Arc::new(Mutex::new(false)); + let agent_actor = runtime + .spawn(AgentNodeRuntimeActor { + effects: TestAgentRuntimeEffects(Arc::clone(&agent_evidence)), + reports: agent_reports, + pending_runtime_ready: pending_runtime_ready(agent_node), + ready: json!({"type":"ready","identity":99}), + node_actor: agent_node, + engine: engine.handle(), + sender: runtime.create_sender(), + completion: agent_completion.clone(), + }) + .expect("spawn actual agent runtime actor"); + let worker_actor = runtime + .spawn(WorkerNodeRuntimeActor { + effects: TestWorkerRuntimeEffects { + evidence: Arc::clone(&worker_evidence), + fail_next_tick: Arc::clone(&fail_next_worker_tick), + }, + reports: worker_reports, + pending_runtime_ready: pending_runtime_ready(worker_node), + ready: json!({"type":"ready","identity":99}), + node_actor: worker_node, + engine: engine.handle(), + sender: runtime.create_sender(), + completion: worker_completion.clone(), + }) + .expect("spawn actual worker runtime actor"); + let agent_stop = runtime + .spawn(StdinStopForwarder { + sender: runtime.create_sender(), + target: agent_actor, + }) + .expect("spawn agent stdin forwarder"); + let worker_stop = runtime + .spawn(StdinStopForwarder { + sender: runtime.create_sender(), + target: worker_actor, + }) + .expect("spawn worker stdin forwarder"); + drive_steps(&backend, DRIVE_PER_ACTION); + + let mut expected = ExpectedRuntimeEvidence::default(); + let (mut agent_live, mut worker_live) = (true, true); + let (mut agent_acked, mut worker_acked) = (false, false); + for action in &actions { + match action { + RuntimeAction::Tick => { + if agent_live { + let _ = runtime.send_to(agent_actor, NodeRuntimeMsg::Tick); + } + if worker_live { + let _ = runtime.send_to(worker_actor, NodeRuntimeMsg::Tick); + } + } + RuntimeAction::CurrentReadinessAck + | RuntimeAction::DuplicateReadinessAck => { + if agent_live { + send_report_and_tick( + &runtime, + agent_reports_addr, + agent_actor, + readiness_report(None), + ); + if !agent_acked { + agent_acked = true; + expected.agent_readiness = 1; + } + } + if worker_live { + send_report_and_tick( + &runtime, + worker_reports_addr, + worker_actor, + readiness_report(None), + ); + if !worker_acked { + worker_acked = true; + expected.worker_readiness = 1; + } + } + } + RuntimeAction::StaleReadinessAck(kind) => { + if agent_live { + send_report_and_tick( + &runtime, + agent_reports_addr, + agent_actor, + readiness_report(Some(*kind)), + ); + } + if worker_live { + send_report_and_tick( + &runtime, + worker_reports_addr, + worker_actor, + readiness_report(Some(*kind)), + ); + } + } + RuntimeAction::Snapshot => { + let report = NodeAgentReport::Snapshot { + commands: Vec::new(), + events: Vec::new(), + }; + if agent_live { + send_report_and_tick( + &runtime, + agent_reports_addr, + agent_actor, + report.clone(), + ); + } + if worker_live { + send_report_and_tick( + &runtime, + worker_reports_addr, + worker_actor, + report, + ); + expected.snapshots += 1; + } + } + RuntimeAction::Lifecycle(value) => { + let report = NodeAgentReport::Lifecycle( + StageLifecycleWire::StageReady { + run_id: 7, + stage_index: u32::from(*value), + }, + ); + if agent_live { + send_report_and_tick( + &runtime, + agent_reports_addr, + agent_actor, + report.clone(), + ); + } + if worker_live { + send_report_and_tick( + &runtime, + worker_reports_addr, + worker_actor, + report, + ); + expected.lifecycle += 1; + } + } + RuntimeAction::WorkerExit if worker_live => { + *fail_next_worker_tick.lock() = true; + let _ = runtime.send_to(worker_actor, NodeRuntimeMsg::Tick); + worker_live = false; + expected.worker_failed = true; + } + RuntimeAction::Shutdown => { + if agent_live { + let _ = + runtime.send_to(agent_stop, swactor_process::ProcessStopSignal); + agent_live = false; + expected.agent_shutdowns = 1; + } + if worker_live { + let _ = + runtime.send_to(worker_stop, swactor_process::ProcessStopSignal); + worker_live = false; + expected.worker_shutdowns = 1; + } + } + RuntimeAction::DuplicateShutdown => { + let _ = runtime.send_to(agent_actor, NodeRuntimeMsg::Shutdown); + let _ = runtime.send_to(agent_actor, NodeRuntimeMsg::Shutdown); + let _ = runtime.send_to(worker_actor, NodeRuntimeMsg::Shutdown); + let _ = runtime.send_to(worker_actor, NodeRuntimeMsg::Shutdown); + if agent_live { + agent_live = false; + expected.agent_shutdowns = 1; + } + if worker_live { + worker_live = false; + expected.worker_shutdowns = 1; + } + } + RuntimeAction::WorkerExit => {} + } + drive_steps(&backend, DRIVE_PER_ACTION); + } + + let _ = runtime.send_to(agent_stop, swactor_process::ProcessStopSignal); + let _ = runtime.send_to(worker_stop, swactor_process::ProcessStopSignal); + if agent_live { + expected.agent_shutdowns = 1; + } + if worker_live { + expected.worker_shutdowns = 1; + } + drive_steps(&backend, FINAL_DRIVE_BUDGET); + advance_and_drive(&backend, Duration::from_secs(1), FINAL_DRIVE_BUDGET); + + let before_wait = + check_runtime_clean(&runtime, &backend, baseline_actors, baseline_tasks); + prop_assert!( + before_wait.is_ok(), + "runtime did not converge within fixed budget: {:?}; actions={:?}; census=\n{}", + before_wait, + actions, + actor_census(&runtime), + ); + + let agent_result = agent_completion.wait(); + let worker_result = worker_completion.wait(); + let agent_observed = agent_evidence.lock().clone(); + let worker_observed = worker_evidence.lock().clone(); + let evidence = + check_runtime_evidence(&expected, &agent_observed, &worker_observed); + prop_assert!( + evidence.is_ok(), + "runtime invariant failed: {:?}; actions={:?}; expected={:?}; agent={:?}; \ + worker={:?}; replies=({:?}, {:?}); census=\n{}", + evidence, + actions, + expected, + agent_observed, + worker_observed, + agent_result, + worker_result, + actor_census(&runtime), + ); + prop_assert!(agent_result.is_ok(), "actions={:?}, agent_reply={:?}", actions, agent_result); + prop_assert_eq!( + worker_result.is_err(), + expected.worker_failed, + "actions={:?}, worker_reply={:?}, census=\n{}", + actions, + worker_result, + actor_census(&runtime), + ); + let clean = + check_runtime_clean(&runtime, &backend, baseline_actors, baseline_tasks); + prop_assert!( + clean.is_ok(), + "runtime residue: {:?}; actions={:?}; replies=({:?}, {:?}); census=\n{}", + clean, + actions, + agent_result, + worker_result, + actor_census(&runtime), + ); + } + } + + #[test] + fn runtime_invariant_checker_rejects_duplicate_readiness_publication() { + let expected = ExpectedRuntimeEvidence { + agent_readiness: 1, + agent_shutdowns: 1, + worker_shutdowns: 1, + ..ExpectedRuntimeEvidence::default() + }; + let agent = RuntimeEvidence { + ticks: 1, + readiness: 2, + shutdowns: 1, + finishes: 1, + ..RuntimeEvidence::default() + }; + let worker = RuntimeEvidence { + ticks: 1, + shutdowns: 1, + finishes: 1, + ..RuntimeEvidence::default() + }; + assert!(check_runtime_evidence(&expected, &agent, &worker).is_err()); + } + + #[derive(Clone, Debug)] + enum HelperAction { + Other, + Expected(u8), + Malformed, + Fatal, + ReadError, + Closed, + Stderr(u8), + } + + fn helper_actions() -> impl Strategy> { + prop::collection::vec( + prop_oneof![ + 3 => Just(HelperAction::Other), + 2 => any::().prop_map(HelperAction::Expected), + 2 => Just(HelperAction::Malformed), + 2 => Just(HelperAction::Fatal), + 1 => Just(HelperAction::ReadError), + 1 => Just(HelperAction::Closed), + 1 => any::().prop_map(HelperAction::Stderr), + ], + 0..=32, + ) + } + fn expected_helper_success(actions: &[HelperAction]) -> bool { + for action in actions { + match action { + HelperAction::Expected(_) => return true, + HelperAction::Malformed + | HelperAction::Fatal + | HelperAction::ReadError + | HelperAction::Closed => return false, + HelperAction::Other | HelperAction::Stderr(_) => {} + } + } + false + } + + fn check_helper_classification( + actions: &[HelperAction], + observed_success: bool, + ) -> Result<(), String> { + let expected = expected_helper_success(actions); + if observed_success == expected { + Ok(()) + } else { + Err(format!( + "helper success={observed_success}, expected={expected}, actions={actions:?}" + )) + } + } + + proptest! { + #![proptest_config(ProptestConfig { + cases: 128, + max_shrink_iters: 2_000, + ..ProptestConfig::default() + })] + + #[test] + fn helper_wait_generated_terminal_sequences_complete_once_on_one_worker( + actions in helper_actions(), + extra_ticks in 0_usize..=8, + ) { + let mut config = RuntimeConfig::default(); + config.worker_count = 1; + let parts = RuntimeParts::new(config); + let runtime = parts.runtime().clone(); + let backend = SteppingBackend::new(); + let engine = Engine::new(parts, backend.clone()).expect("one-worker stepping engine"); + let baseline_actors = runtime.stats().actors.len(); + let baseline_tasks = backend.pending_task_count(); + let (stdout_tx, stdout_rx) = mpsc::channel(); + let (stderr_tx, stderr_rx) = mpsc::channel(); + + for action in &actions { + match *action { + HelperAction::Other => { + stdout_tx + .send(HelperStdoutEvent::Line( + serde_json::json!({"type":"Progress"}).to_string(), + )) + .expect("queue helper progress"); + } + HelperAction::Expected(value) => { + stdout_tx + .send(HelperStdoutEvent::Line( + serde_json::json!({"type":"Expected","value":value}).to_string(), + )) + .expect("queue expected helper event"); + } + HelperAction::Malformed => { + stdout_tx + .send(HelperStdoutEvent::Line("{broken".to_owned())) + .expect("queue malformed helper event"); + } + HelperAction::Fatal => { + stdout_tx + .send(HelperStdoutEvent::Line( + serde_json::json!({"type":"WorkerFatal","reason":"scripted"}) + .to_string(), + )) + .expect("queue fatal helper event"); + } + HelperAction::ReadError => { + stdout_tx + .send(HelperStdoutEvent::ReadError("scripted read error".to_owned())) + .expect("queue helper read error"); + } + HelperAction::Closed => { + stdout_tx + .send(HelperStdoutEvent::Closed) + .expect("queue helper close"); + } + HelperAction::Stderr(value) => { + stderr_tx + .send(format!("stderr-{value}")) + .expect("queue helper stderr"); + } + } + } + drop(stdout_tx); + drop(stderr_tx); + + let completion = ActorCompletion::new(); + let actor = runtime + .spawn(HelperWaitActor { + stdout_rx: Arc::new(Mutex::new(stdout_rx)), + stderr_rx: Some(Arc::new(Mutex::new(stderr_rx))), + expected: "Expected".to_owned(), + engine: engine.handle(), + sender: runtime.create_sender(), + completion: completion.clone(), + wait_config: HelperCommandWaitConfig { + poll_interval: Duration::from_millis(1), + telemetry_interval: Duration::from_millis(1), + }, + wait_started: Instant::now(), + next_telemetry_at: Instant::now(), + wait_cycles: 0, + lines: Vec::new(), + stderr_lines: Vec::new(), + wait_samples: Vec::new(), + }) + .expect("spawn helper wait actor"); + for _ in 0..extra_ticks { + let _ = runtime.send_to(actor, HelperWaitTick); + } + drive_steps(&backend, FINAL_DRIVE_BUDGET); + advance_and_drive(&backend, Duration::from_secs(1), FINAL_DRIVE_BUDGET); + + let before_wait = + check_runtime_clean(&runtime, &backend, baseline_actors, baseline_tasks); + prop_assert!( + before_wait.is_ok(), + "helper did not converge within fixed budget: {:?}; actions={:?}; census=\n{}", + before_wait, + actions, + actor_census(&runtime), + ); + + let outcome = completion.wait(); + let classification = + check_helper_classification(&actions, outcome.result.is_ok()); + prop_assert!( + classification.is_ok(), + "helper invariant failed: {:?}; actions={:?}; result={:?}; lines={:?}; \ + stderr={:?}; census=\n{}", + classification, + actions, + outcome.result, + outcome.lines, + outcome.stderr_lines, + actor_census(&runtime), + ); + prop_assert!( + outcome.lines.len() <= actions.len(), + "actions={:?}, lines={:?}, census=\n{}", + actions, + outcome.lines, + actor_census(&runtime), + ); + prop_assert!( + outcome.stderr_lines.len() <= actions.len(), + "actions={:?}, stderr={:?}, census=\n{}", + actions, + outcome.stderr_lines, + actor_census(&runtime), + ); + let clean = + check_runtime_clean(&runtime, &backend, baseline_actors, baseline_tasks); + prop_assert!( + clean.is_ok(), + "helper residue: {:?}; actions={:?}; result={:?}; census=\n{}", + clean, + actions, + outcome.result, + actor_census(&runtime), + ); + } + } + #[test] + fn helper_invariant_checker_rejects_expected_output_after_terminal_error() { + for actions in [ + vec![HelperAction::Fatal, HelperAction::Expected(1)], + vec![HelperAction::Closed, HelperAction::Expected(1)], + ] { + assert!(!expected_helper_success(&actions)); + assert!(check_helper_classification(&actions, true).is_err()); + } + } + + #[derive(Clone, Debug)] + enum StageAction { + Output(u8), + MalformedOutput, + ReadyOutput, + ReadyMissing, + Stderr(u8), + ReaderError(bool), + CloseStdout, + CloseStderr, + ExitSuccess, + ExitFailure, + } + + fn stage_actions() -> impl Strategy> { + prop::collection::vec( + prop_oneof![ + 3 => any::().prop_map(StageAction::Output), + 2 => Just(StageAction::MalformedOutput), + 2 => Just(StageAction::ReadyOutput), + 2 => Just(StageAction::ReadyMissing), + 2 => any::().prop_map(StageAction::Stderr), + 1 => any::().prop_map(StageAction::ReaderError), + 2 => Just(StageAction::CloseStdout), + 2 => Just(StageAction::CloseStderr), + 2 => Just(StageAction::ExitSuccess), + 2 => Just(StageAction::ExitFailure), + ], + 0..=32, + ) + } + + #[derive(Clone, Debug)] + struct StageModel { + stdout_closed: bool, + stderr_closed: bool, + exit_success: Option, + ready_exists: Option, + events: usize, + finished: bool, + success: bool, + } + + impl StageModel { + fn new() -> Self { + Self { + stdout_closed: false, + stderr_closed: false, + exit_success: None, + ready_exists: None, + events: 0, + finished: false, + success: false, + } + } + + fn observe(&mut self, action: &StageAction) { + if self.finished { + return; + } + match action { + StageAction::Output(_) | StageAction::MalformedOutput | StageAction::Stderr(_) => { + self.events += 1; + } + StageAction::ReadyOutput => { + self.ready_exists = Some(true); + self.events += 1; + } + StageAction::ReadyMissing => { + self.ready_exists = Some(false); + self.events += 1; + } + StageAction::ReaderError(_) => self.events += 1, + StageAction::CloseStdout => self.stdout_closed = true, + StageAction::CloseStderr => self.stderr_closed = true, + StageAction::ExitSuccess if self.exit_success.is_none() => { + self.exit_success = Some(true); + } + StageAction::ExitFailure if self.exit_success.is_none() => { + self.exit_success = Some(false); + } + StageAction::ExitSuccess | StageAction::ExitFailure => {} + } + if let Some(exit_success) = self.exit_success + && self.stdout_closed + && self.stderr_closed + { + self.finished = true; + self.success = exit_success && self.ready_exists.unwrap_or(true); + } + } + } + + fn check_stage_outcome( + model: &StageModel, + observed_success: bool, + observed_events: usize, + ) -> Result<(), String> { + if observed_success == model.success && observed_events == model.events { + Ok(()) + } else { + Err(format!( + "success={observed_success}/{} events={observed_events}/{} model={model:?}", + model.success, model.events + )) + } + } + + fn stage_message( + action: &StageAction, + output_path: &Path, + missing_path: &Path, + ) -> StageShardFetchMsg { + match action { + StageAction::Output(value) => StageShardFetchMsg::ProcessLine { + stream: StageShardProcessStream::Stdout, + line: json!({"type":"StageShardProgress","value":value}).to_string(), + }, + StageAction::MalformedOutput => StageShardFetchMsg::ProcessLine { + stream: StageShardProcessStream::Stdout, + line: "{broken".to_owned(), + }, + StageAction::ReadyOutput => StageShardFetchMsg::ProcessLine { + stream: StageShardProcessStream::Stdout, + line: json!({"type":"StageShardReady","path":output_path}).to_string(), + }, + StageAction::ReadyMissing => StageShardFetchMsg::ProcessLine { + stream: StageShardProcessStream::Stdout, + line: json!({"type":"StageShardReady","path":missing_path}).to_string(), + }, + StageAction::Stderr(value) => StageShardFetchMsg::ProcessLine { + stream: StageShardProcessStream::Stderr, + line: format!("stderr-{value}"), + }, + StageAction::ReaderError(stdout) => StageShardFetchMsg::ReaderError { + stream: if *stdout { + StageShardProcessStream::Stdout + } else { + StageShardProcessStream::Stderr + }, + error: "scripted reader error".to_owned(), + }, + StageAction::CloseStdout => StageShardFetchMsg::ReaderClosed { + stream: StageShardProcessStream::Stdout, + }, + StageAction::CloseStderr => StageShardFetchMsg::ReaderClosed { + stream: StageShardProcessStream::Stderr, + }, + StageAction::ExitSuccess => { + StageShardFetchMsg::ProcessExited(std::process::ExitStatus::from_raw(0)) + } + StageAction::ExitFailure => { + StageShardFetchMsg::ProcessExited(std::process::ExitStatus::from_raw(1 << 8)) + } + } + } + + proptest! { + #![proptest_config(ProptestConfig { + cases: 128, + max_shrink_iters: 2_000, + ..ProptestConfig::default() + })] + + #[test] + fn stage_fetch_generated_observations_complete_once_on_one_worker( + actions in stage_actions() + ) { + let mut config = RuntimeConfig::default(); + config.worker_count = 1; + let parts = RuntimeParts::new(config); + let runtime = parts.runtime().clone(); + let backend = SteppingBackend::new(); + let engine = Engine::new(parts, backend.clone()).expect("one-worker stepping engine"); + let baseline_actors = runtime.stats().actors.len(); + let baseline_tasks = backend.pending_task_count(); + let output = tempfile::NamedTempFile::new().expect("stage output file"); + let output_path = output.path().to_path_buf(); + let missing_path = output_path.with_extension("missing"); + let completion = ActorCompletion::new(); + let mut stage_actor = StageShardFetchActor::new( + Vec::new(), + output_path.clone(), + completion.clone(), + runtime.create_sender(), + engine.handle(), + ); + stage_actor.stdout_closed = false; + stage_actor.stderr_closed = false; + let actor = runtime.spawn(stage_actor).expect("spawn actual stage fetch actor"); + let mut model = StageModel::new(); + + for action in &actions { + let _ = + runtime.send_to(actor, stage_message(action, &output_path, &missing_path)); + model.observe(action); + drive_steps(&backend, DRIVE_PER_ACTION); + } + for terminal in [ + StageAction::CloseStdout, + StageAction::CloseStderr, + StageAction::ExitSuccess, + ] { + if !model.finished { + let _ = runtime.send_to( + actor, + stage_message(&terminal, &output_path, &missing_path), + ); + model.observe(&terminal); + drive_steps(&backend, DRIVE_PER_ACTION); + } + } + drive_steps(&backend, FINAL_DRIVE_BUDGET); + + let before_wait = + check_runtime_clean(&runtime, &backend, baseline_actors, baseline_tasks); + prop_assert!( + before_wait.is_ok(), + "stage did not converge within fixed budget: {:?}; actions={:?}; model={:?}; census=\n{}", + before_wait, + actions, + model, + actor_census(&runtime), + ); + + let outcome = completion.wait(); + let checked = + check_stage_outcome(&model, outcome.result.is_ok(), outcome.events.len()); + prop_assert!( + checked.is_ok(), + "stage invariant failed: {:?}; actions={:?}; model={:?}; result={:?}; \ + events={:?}; census=\n{}", + checked, + actions, + model, + outcome.result, + outcome.events, + actor_census(&runtime), + ); + let clean = + check_runtime_clean(&runtime, &backend, baseline_actors, baseline_tasks); + prop_assert!( + clean.is_ok(), + "stage residue: {:?}; actions={:?}; result={:?}; events={:?}; census=\n{}", + clean, + actions, + outcome.result, + outcome.events, + actor_census(&runtime), + ); + } + } + + #[test] + fn stage_invariant_checker_rejects_wrong_terminal_classification() { + let model = StageModel { + stdout_closed: true, + stderr_closed: true, + exit_success: Some(false), + ready_exists: Some(true), + events: 1, + finished: true, + success: false, + }; + assert!(check_stage_outcome(&model, true, 1).is_err()); + } } diff --git a/apps/myelin/src/observability/frame_collector.rs b/apps/myelin/src/observability/frame_collector.rs index b4663fc..95ef4e6 100644 --- a/apps/myelin/src/observability/frame_collector.rs +++ b/apps/myelin/src/observability/frame_collector.rs @@ -7,9 +7,9 @@ //! only through these closures. use iroh::EndpointAddr; -use iroh_driver::{IrohDriver, TelemetryQuicHeader, spawn_pull_collector}; +use iroh_driver::{IrohDriver, PullCollectorHandle, TelemetryQuicHeader, spawn_pull_collector}; use parking_lot::Mutex; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::sync::{Arc, mpsc}; use swactor_engine::EngineHandle; use telemetry::frame::{ChannelRef, Frame, StreamId, TelemetryEvent}; @@ -38,6 +38,8 @@ pub(crate) struct FrameCollector { pull_header_rx: mpsc::Receiver, pull_channels: Mutex>, pull_streams: Mutex>, + pull_stream_owners: Mutex>, + pull_collectors: Mutex>, } impl FrameCollector { @@ -61,6 +63,8 @@ impl FrameCollector { pull_header_rx, pull_channels: Mutex::new(BTreeMap::new()), pull_streams: Mutex::new(BTreeMap::new()), + pull_stream_owners: Mutex::new(BTreeMap::new()), + pull_collectors: Mutex::new(BTreeMap::new()), } } @@ -76,7 +80,7 @@ impl FrameCollector { let mut flow_id = [0_u8; 16]; flow_id[..8].copy_from_slice(&run_id.to_le_bytes()); flow_id[8..].copy_from_slice(&node_id.to_le_bytes()); - spawn_pull_collector( + let collector = spawn_pull_collector( engine, endpoint, peer, @@ -86,6 +90,36 @@ impl FrameCollector { Arc::clone(&self.pull_fanout), self.pull_header_tx.clone(), ); + if let Some(previous) = self + .pull_collectors + .lock() + .insert((run_id, node_id), collector) + { + previous.cancel(); + } + } + /// Stop retaining and reconnecting a telemetry subscription for a terminal node. + pub(crate) fn unsubscribe_node(&self, run_id: u64, node_id: u64) { + if let Some(collector) = self.pull_collectors.lock().remove(&(run_id, node_id)) { + collector.cancel(); + } + let mut ended = BTreeSet::new(); + self.pull_stream_owners.lock().retain(|stream, owner| { + if *owner == (run_id, node_id) { + ended.insert(stream.clone()); + false + } else { + true + } + }); + if !ended.is_empty() { + self.pull_streams + .lock() + .retain(|stream, _| !ended.contains(stream)); + self.pull_channels + .lock() + .retain(|channel, _| !ended.contains(&channel.stream)); + } } fn pump_pulls(&self) { @@ -93,6 +127,14 @@ impl FrameCollector { self.pull_streams .lock() .insert(header.stream.stream.clone(), header.stream.clone()); + let mut run_id = [0_u8; 8]; + run_id.copy_from_slice(&header.flow_id[..8]); + let mut node_id = [0_u8; 8]; + node_id.copy_from_slice(&header.flow_id[8..]); + self.pull_stream_owners.lock().insert( + header.stream.stream.clone(), + (u64::from_le_bytes(run_id), u64::from_le_bytes(node_id)), + ); let mut channels = self.pull_channels.lock(); for descriptor in header.channels { channels.insert( @@ -145,6 +187,10 @@ impl FrameCollector { } TelemetryEvent::StreamEnded(stream) => { self.pull_streams.lock().remove(&stream); + self.pull_stream_owners.lock().remove(&stream); + self.pull_channels + .lock() + .retain(|channel, _| channel.stream != stream); } } } @@ -318,5 +364,69 @@ mod tests { assert_eq!(observed_channel, "host.net"); assert_eq!(observed_frame.position, Position(11)); assert_eq!(observed_frame.payload, br#"{"rx":1}"#); + collector + .pull_fanout + .publish(TelemetryEvent::StreamEnded(stream.clone())); + collector.pump_pulls(); + assert!(!collector.pull_streams.lock().contains_key(&stream)); + assert!( + collector + .pull_channels + .lock() + .keys() + .all(|channel| channel.stream != stream), + "ended stream retained channel descriptors" + ); + } + + #[test] + fn node_unsubscribe_releases_abrupt_stream_metadata() { + let collector = FrameCollector::new(); + let stream = StreamId::new(NodeId::new("node-9"), Lifetime(4)); + let descriptor = StreamDescriptor { + stream: stream.clone(), + label: Some("worker nine".to_owned()), + origin: StreamOrigin::RemoteNode, + }; + let channel = ChannelDescriptor { + stream: stream.clone(), + id: ChannelId(3), + name: "runtime.actors".to_owned(), + label: None, + content: ChannelContent::JsonRecord { schema: None }, + }; + let mut flow_id = [0_u8; 16]; + flow_id[..8].copy_from_slice(&5_u64.to_le_bytes()); + flow_id[8..].copy_from_slice(&9_u64.to_le_bytes()); + collector + .pull_header_tx + .send(TelemetryQuicHeader::new( + flow_id, + Vec::new(), + descriptor, + vec![channel], + )) + .unwrap(); + collector.pump_pulls(); + assert!(collector.pull_streams.lock().contains_key(&stream)); + assert!( + collector + .pull_channels + .lock() + .keys() + .any(|channel| channel.stream == stream) + ); + + collector.unsubscribe_node(5, 9); + + assert!(!collector.pull_streams.lock().contains_key(&stream)); + assert!( + collector + .pull_channels + .lock() + .keys() + .all(|channel| channel.stream != stream), + "abrupt node stop retained channel descriptors" + ); } } diff --git a/apps/myelin/src/observability/orch_telemetry.rs b/apps/myelin/src/observability/orch_telemetry.rs index f39800a..2a3cf91 100644 --- a/apps/myelin/src/observability/orch_telemetry.rs +++ b/apps/myelin/src/observability/orch_telemetry.rs @@ -285,7 +285,7 @@ impl DashboardSupport { .page_script_urls .push(crate::orchestration::control::FLEET_CONTROL_SCRIPT_URL.to_owned()); let handle = dashboard::DashboardHandle::new(config); - engine.spawn(handle.http_server_with_plugins(plugins)); + handle.spawn_with_plugins(engine, plugins); Ok(Some(Self { handle })) } diff --git a/apps/myelin/src/observability/provisioning_logs.rs b/apps/myelin/src/observability/provisioning_logs.rs index 732d576..490cd7b 100644 --- a/apps/myelin/src/observability/provisioning_logs.rs +++ b/apps/myelin/src/observability/provisioning_logs.rs @@ -1,6 +1,3 @@ -use std::io::{BufRead, BufReader, Read}; -use std::thread::{self, JoinHandle}; - use serde::{Deserialize, Serialize}; use serde_json::Value; use telemetry::{ChannelContent, Lifetime, NodeId, StreamId, TelemetryProducer}; @@ -74,66 +71,6 @@ impl BootstrapTelemetryBridge { }); } - // provider log capture is out of scope (ENGINE_SPEC.md §2) - #[allow(clippy::disallowed_methods)] - pub(crate) fn spawn_stdout_reader(&self, stdout: R) -> JoinHandle<()> - where - R: Read + Send + 'static, - { - let bridge = self.clone(); - thread::spawn(move || bridge.read_stdout(stdout)) - } - - // provider log capture is out of scope (ENGINE_SPEC.md §2) - #[allow(clippy::disallowed_methods)] - pub(crate) fn spawn_stderr_reader(&self, stderr: R) -> JoinHandle<()> - where - R: Read + Send + 'static, - { - let bridge = self.clone(); - thread::spawn(move || bridge.read_stderr(stderr)) - } - - fn read_stdout(&self, stdout: R) - where - R: Read, - { - let reader = BufReader::new(stdout); - for next in reader.lines() { - match next { - Ok(line) => self.observe_stdout_line(line), - Err(error) => { - self.sink.observe(PluginObservation::Failed { - run_id: self.spec.run_id, - node_id: self.spec.node_id, - reason: format!("read stdout: {error}"), - }); - break; - } - } - } - } - - fn read_stderr(&self, stderr: R) - where - R: Read, - { - let reader = BufReader::new(stderr); - for next in reader.lines() { - match next { - Ok(line) => self.observe_stderr_line(line), - Err(error) => { - self.sink.observe(PluginObservation::Failed { - run_id: self.spec.run_id, - node_id: self.spec.node_id, - reason: format!("read stderr: {error}"), - }); - break; - } - } - } - } - fn submit_log(&self, stream: ProvisionLogStream, line: &str) { let Some(producer) = &self.producer else { return; diff --git a/apps/myelin/src/orchestration/app.rs b/apps/myelin/src/orchestration/app.rs index 7071b07..37a1138 100644 --- a/apps/myelin/src/orchestration/app.rs +++ b/apps/myelin/src/orchestration/app.rs @@ -1,14 +1,12 @@ use std::collections::BTreeMap; use std::fs::File; -use std::io::{BufRead, BufReader, Write}; +use std::io::Write; #[cfg(target_os = "linux")] use std::os::fd::FromRawFd; use std::path::{Path, PathBuf}; use std::process::Command; -use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, mpsc}; -use std::thread; -use std::time::{Duration, Instant}; +use std::time::Duration; use crate::DEFAULT_PIPELINE_CACHED_MODEL_FILE; use crate::codecs::register_myelin_actor_codecs; @@ -22,8 +20,8 @@ use crate::orchestration::control; use crate::orchestration::daemon; use crate::orchestration::manual_control::{ CONTROL_REGISTRY_NAME, ConfigValidator, ManualActorControl, ManualControl, ManualControlMsg, - NodePhase, OfferDto, OfferSearchRequest, OfferSearcher, ProviderConfigurationRequest, - ProviderFactory, ProviderReadiness, SpecBuilder, + ManualControlReply, NodePhase, OfferDto, OfferSearchRequest, OfferSearcher, + ProviderConfigurationRequest, ProviderFactory, ProviderReadiness, SpecBuilder, }; use crate::node_provisioning::{ProviderKind, provider_kind}; @@ -52,8 +50,9 @@ use iroh_driver::{EDGE_ALPN, IrohDriver, IrohDriverConfig, TELEMETRY_ALPN}; use iroh_driver::{EndpointAddrMask, MVP_IROH_ENDPOINT_ADDR_MASK_ENV, advertised_endpoint}; use parking_lot::Mutex; use serde_json::{Value, json}; -use swactor::actor::ActorAddress; -use swactor_engine::{Engine, EngineHandle, TokioBackend, TokioConfig}; +use swactor::actor::{ActorAddress, ActorInterface}; +use swactor::runtime::{Ctx, ExternalSender, Inbox, Runtime}; +use swactor_engine::{ActorCompletion, Engine, EngineHandle, TokioBackend, TokioConfig}; const DEFAULT_IMAGE: &str = "myelin-node:latest"; const MYELIN_RUNTIME_CONFIG_ENV: &str = "MYELIN_RUNTIME_CONFIG"; const CACHED_MODEL_HOST_ENV: &str = "MYELIN_CACHED_MODEL_HOST_PATH"; @@ -66,6 +65,7 @@ const DEFAULT_MODEL_ID: &str = "llama-3.2-1b-instruct-q4"; const DEFAULT_STATE_DIR: &str = "./.config"; const DEFAULT_MAX_TOKENS: u32 = 64; const PUMP_INTERVAL: Duration = Duration::from_millis(10); +const SHUTDOWN_FLUSH_TIMEOUT: Duration = Duration::from_secs(10); const TELEMETRY_FRAME_LOG_ENV: &str = "MYELIN_TELEMETRY_FRAME_LOG"; pub(crate) fn run_with_options( @@ -103,7 +103,7 @@ where .flush() .map_err(|error| format!("flush dashboard URL to stdout: {error}"))?; } - let orch_stdio_rx = if capture_stdio { + let orch_stdio_capture = if capture_stdio { OrchStdioCapture::install()? } else { None @@ -185,13 +185,6 @@ where }), ); } - drain_orch_stdio_capture( - orch_stdio_rx.as_ref(), - &mut orch_telemetry, - None, - config.run_id, - config.node_id, - ); let actors_channel = orch_telemetry.channel_by_name("runtime.actors"); let orch_stats_hook = orch_telemetry.stats_hook_on(actors_channel); @@ -284,6 +277,9 @@ where DistributedNodeConfig::default(), engine.handle(), ); + let orch_stdio_rx = orch_stdio_capture + .map(|capture| capture.start(&stack.runtime)) + .transpose()?; bootstrap( &mut orch_telemetry, None, @@ -383,13 +379,15 @@ where })); let shared_config = Arc::new(Mutex::new(config.clone())); let provider_runtime = stack.runtime.clone(); + let provider_engine = engine.handle(); let provider_config = Arc::clone(&shared_config); let provider_registry = state_dir.process_registry_path(); let provider_factory: ProviderFactory = Arc::new(move || { - provider_config - .lock() - .clone() - .build_provisioner(provider_runtime.clone(), provider_registry.clone()) + provider_config.lock().clone().build_provisioner( + provider_runtime.clone(), + provider_engine.clone(), + provider_registry.clone(), + ) }); let spec_config = Arc::clone(&shared_config); let spec_coordinator = coordinator_endpoint.clone(); @@ -403,6 +401,7 @@ where }); let validation_config = Arc::clone(&shared_config); let validation_runtime = stack.runtime.clone(); + let validation_engine = engine.handle(); let validation_registry = state_dir.process_registry_path(); let config_validator: ConfigValidator = Arc::new(move |request| { let mut candidate = validation_config.lock().clone(); @@ -423,8 +422,11 @@ where vastai.bootstrap_command = Some(command); } if vastai.provisioning_mode == VastAiProvisioningMode::Mock { - let _ = candidate - .build_provisioner(validation_runtime.clone(), validation_registry.clone())?; + let _ = candidate.build_provisioner( + validation_runtime.clone(), + validation_engine.clone(), + validation_registry.clone(), + )?; *validation_config.lock() = candidate; return Ok(()); } @@ -432,8 +434,11 @@ where return Err("Vast.ai API key is required for live offer search".to_owned()); } candidate.prepare_vastai_ssh_key()?; - let _ = - candidate.build_provisioner(validation_runtime.clone(), validation_registry.clone())?; + let _ = candidate.build_provisioner( + validation_runtime.clone(), + validation_engine.clone(), + validation_registry.clone(), + )?; *validation_config.lock() = candidate; Ok(()) }); @@ -500,8 +505,8 @@ where }); let manual = ManualActorControl::new( ManualControl::new(snapshot, readiness), - engine.handle(), stack.runtime.clone(), + stack.engine.blocking_work_sender(), state_dir, sink, provider_factory, @@ -550,7 +555,11 @@ where dashboard = DashboardSupport::start_with_plugins( config.dashboard, &engine.handle(), - vec![control::plugin(stack.runtime.clone(), orchestrator_actor)], + vec![control::plugin( + stack.runtime.clone(), + engine.handle(), + orchestrator_actor, + )], )?; bootstrap( &mut orch_telemetry, @@ -563,8 +572,6 @@ where "transport":"direct_actor_message", }), ); - let stop_signal = spawn_stop_listener(stop_rx); - bootstrap( &mut orch_telemetry, dashboard.as_ref(), @@ -572,38 +579,34 @@ where "started", json!({"mode":"daemon","poll_interval_ms":PUMP_INTERVAL.as_millis()}), ); - let mut result = serve_cluster(ServeCluster { - driver: &mut driver, - stack: &stack, - obs_rx: &obs_rx, - collector: &collector, - orchestrator_reports: &orchestrator_reports, - stop_signal: stop_signal.as_ref(), - dashboard: dashboard.as_ref(), - orch_telemetry: &mut orch_telemetry, - orch_stdio_rx: orch_stdio_rx.as_ref(), - run_id: config.run_id, - orchestrator_node_id: config.node_id, - provider: &config.provider, - orchestrator_actor, - engine: engine.handle(), - pending_readies: BTreeMap::new(), - }); - if result.is_ok() - && let Err(error) = flush_manual_control(&stack.runtime, orchestrator_actor) - { - result = Err(error); + + let runtime = stack.runtime.clone(); + let completion = ActorCompletion::new(); + let serve_actor = runtime + .spawn(ServeClusterActor { + driver, + stack, + obs_rx, + collector, + orchestrator_reports, + dashboard, + orch_telemetry, + orch_stdio_rx, + run_id: config.run_id, + orchestrator_node_id: config.node_id, + provider: config.provider.clone(), + orchestrator_actor, + engine: engine.handle(), + sender: runtime.create_sender(), + lifecycle: ServeClusterLifecycle::new(), + flush_reply_actor: None, + completion: completion.clone(), + }) + .map_err(|error| format!("spawn daemon lifecycle actor: {error}"))?; + if let Err(error) = spawn_stop_listener(&runtime, stop_rx, serve_actor) { + let _ = runtime.send_to(serve_actor, ServeClusterMsg::Abort(error)); } - if let Err(error) = &result { - bootstrap( - &mut orch_telemetry, - dashboard.as_ref(), - "serve_cluster", - "failed", - json!({"error":error}), - ); - } - result + completion.wait() } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -1665,6 +1668,7 @@ impl Config { fn build_provisioner( &self, bootstrap_runtime: swactor::runtime::Runtime, + bootstrap_engine: EngineHandle, process_registry_path: PathBuf, ) -> Result, String> { match self.provider.as_str() { @@ -1672,10 +1676,12 @@ impl Config { self.local_worker_bin()?, process_registry_path, "process", + bootstrap_runtime.clone(), ))), "docker" => Ok(Box::new(LocalDockerPlugin::new( env_optional("MYELIN_DOCKER_CONTAINER_PREFIX") .unwrap_or_else(|| "myelin-orchestrator".to_owned()), + bootstrap_runtime.clone(), ))), "vastai" => { let vastai = self.vastai.as_ref().ok_or_else(|| { @@ -1685,6 +1691,7 @@ impl Config { return Ok(Box::new(MockVastAiPlugin::with_registry( self.local_worker_bin()?, process_registry_path, + bootstrap_runtime.clone(), ))); } if vastai.bootstrap_command.is_none() { @@ -1702,8 +1709,13 @@ impl Config { .clone() .ok_or_else(|| "VastAI SSH identity was not prepared".to_owned())?; Ok(Box::new(VastAiProvisioningPlugin::new( - ToolsVastAiLeaseClient::from_api_key(api_key)?, - SshCommandBootstrapLauncher::new(Some(ssh_identity), bootstrap_runtime), + ToolsVastAiLeaseClient::from_api_key(api_key)? + .with_actor_host(bootstrap_runtime.clone(), bootstrap_engine.clone()), + SshCommandBootstrapLauncher::new( + Some(ssh_identity), + bootstrap_runtime, + bootstrap_engine, + ), vastai.provisioning.clone(), ))) } @@ -1738,6 +1750,7 @@ impl Config { "MYELIN_LOGICAL_NODE_ID".to_owned(), logical_node_id.to_string(), ), + ("MYELIN_NODE_ATTEMPT_ID".to_owned(), "0".to_owned()), ("MYELIN_STAGE_INDEX".to_owned(), stage_index.to_string()), ( MVP_IROH_ENDPOINT_ADDR_MASK_ENV.to_owned(), @@ -1808,6 +1821,13 @@ fn runtime_ready_barrier_met(stack: &DistributionRuntimeStack, ready: &RuntimeRe && stack.route_owner(ready.node_actor) == Some(ready.swim_node_id) } +#[cfg(target_os = "linux")] +struct OrchStdioCapture { + stdout: File, + stderr: File, +} + +#[cfg(not(target_os = "linux"))] struct OrchStdioCapture; struct OrchStdioLine { @@ -1815,15 +1835,65 @@ struct OrchStdioLine { line: String, } +struct OrchStdioRelay { + tx: mpsc::Sender, + closed: u8, +} + +impl ActorInterface for OrchStdioRelay { + type Incoming = swactor_process::ProcessStreamObservation; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, observation: Self::Incoming) { + match observation { + swactor_process::ProcessStreamObservation::Line { stream, line } => { + let stream = match stream { + swactor_process::ProcessStream::Stdout => ProvisionLogStream::Stdout, + swactor_process::ProcessStream::Stderr => ProvisionLogStream::Stderr, + }; + if self.tx.send(OrchStdioLine { stream, line }).is_err() { + ctx.stop_self(); + } + } + swactor_process::ProcessStreamObservation::Error { .. } => {} + swactor_process::ProcessStreamObservation::Closed { .. } => { + self.closed = self.closed.saturating_add(1); + if self.closed == 2 { + ctx.stop_self(); + } + } + } + } +} + #[cfg(target_os = "linux")] impl OrchStdioCapture { - fn install() -> Result>, String> { - let stdout_read = Self::redirect_stream(libc::STDOUT_FILENO, "stdout")?; - let stderr_read = Self::redirect_stream(libc::STDERR_FILENO, "stderr")?; + fn install() -> Result, String> { + Ok(Some(Self { + stdout: Self::redirect_stream(libc::STDOUT_FILENO, "stdout")?, + stderr: Self::redirect_stream(libc::STDERR_FILENO, "stderr")?, + })) + } + + fn start(self, runtime: &Runtime) -> Result, String> { let (tx, rx) = mpsc::channel(); - Self::spawn_reader(stdout_read, ProvisionLogStream::Stdout, tx.clone()); - Self::spawn_reader(stderr_read, ProvisionLogStream::Stderr, tx); - Ok(Some(rx)) + let actor = runtime + .spawn(OrchStdioRelay { tx, closed: 0 }) + .map_err(|error| format!("spawn orchestrator stdio relay actor: {error}"))?; + let sender = runtime.create_sender(); + swactor_process::spawn_line_reader( + swactor_process::ProcessStream::Stdout, + self.stdout, + sender.clone(), + actor, + ); + swactor_process::spawn_line_reader( + swactor_process::ProcessStream::Stderr, + self.stderr, + sender, + actor, + ); + Ok(rx) } fn redirect_stream(fd: libc::c_int, name: &str) -> Result { @@ -1851,29 +1921,17 @@ impl OrchStdioCapture { Ok(unsafe { File::from_raw_fd(pipe_fds[0]) }) } - - // provider log capture is out of scope (ENGINE_SPEC.md §2) - #[allow(clippy::disallowed_methods)] - fn spawn_reader(file: File, stream: ProvisionLogStream, tx: mpsc::Sender) { - thread::spawn(move || { - let reader = BufReader::new(file); - for line in reader.lines() { - let Ok(line) = line else { - break; - }; - if tx.send(OrchStdioLine { stream, line }).is_err() { - break; - } - } - }); - } } #[cfg(not(target_os = "linux"))] impl OrchStdioCapture { - fn install() -> Result>, String> { + fn install() -> Result, String> { Ok(None) } + + fn start(self, _runtime: &Runtime) -> Result, String> { + unreachable!("stdio capture is unavailable on this target") + } } fn drain_orch_stdio_capture( @@ -1909,55 +1967,225 @@ impl PluginObservationSink for ChannelObservationSink { } } -fn stop_requested(stop_signal: &AtomicBool) -> bool { - stop_signal.load(Ordering::Acquire) +#[derive(Clone, Debug)] +enum ServeClusterMsg { + Tick, + Stop, + Flushed, + FlushFailed(String), + FlushTimeout, + Abort(String), } -// top-level OS signal handling is process control, out of scope (ENGINE_SPEC.md §2) -#[allow(clippy::disallowed_methods)] -fn spawn_stop_listener(external: Option>) -> Arc { - let requested = Arc::new(AtomicBool::new(false)); - let listener_requested = Arc::clone(&requested); - if let Some(external) = external { - thread::spawn(move || { - if external.recv().is_ok() { - listener_requested.store(true, Ordering::Release); +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ServeClusterState { + Running, + Flushing, + Finished, +} + +struct PendingRuntimeReady { + readiness_id: u64, + ready: T, +} + +struct ServeClusterLifecycle { + state: ServeClusterState, + pending_readies: BTreeMap>, +} + +#[derive(Debug)] +enum ServeClusterEffect { + None, + Pump, + BeginFlush, + ProviderTerminalFailure { node_id: u64, error: String }, + Complete(Result<(), String>), +} + +impl ServeClusterLifecycle { + fn new() -> Self { + Self { + state: ServeClusterState::Running, + pending_readies: BTreeMap::new(), + } + } + + fn apply_message(&mut self, message: ServeClusterMsg) -> ServeClusterEffect { + match message { + ServeClusterMsg::Tick if matches!(self.state, ServeClusterState::Running) => { + ServeClusterEffect::Pump } - }); - return requested; + ServeClusterMsg::Stop if matches!(self.state, ServeClusterState::Running) => { + self.state = ServeClusterState::Flushing; + self.pending_readies.clear(); + ServeClusterEffect::BeginFlush + } + ServeClusterMsg::Flushed if matches!(self.state, ServeClusterState::Flushing) => { + self.complete(Ok(())) + } + ServeClusterMsg::FlushFailed(error) + if matches!(self.state, ServeClusterState::Flushing) => + { + self.complete(Err(error)) + } + ServeClusterMsg::FlushTimeout if matches!(self.state, ServeClusterState::Flushing) => { + self.complete(Err( + "timed out waiting for shutdown persistence flush".to_owned() + )) + } + ServeClusterMsg::Abort(error) if !matches!(self.state, ServeClusterState::Finished) => { + self.complete(Err(error)) + } + ServeClusterMsg::Tick + | ServeClusterMsg::Stop + | ServeClusterMsg::Flushed + | ServeClusterMsg::FlushFailed(_) + | ServeClusterMsg::FlushTimeout + | ServeClusterMsg::Abort(_) => ServeClusterEffect::None, + } + } + + fn track_runtime_ready(&mut self, node_id: u64, readiness_id: u64, ready: T) -> bool { + if !matches!(self.state, ServeClusterState::Running) { + return false; + } + if let Some(current) = self.pending_readies.get(&node_id) + && current.readiness_id >= readiness_id + { + return false; + } + self.pending_readies.insert( + node_id, + PendingRuntimeReady { + readiness_id, + ready, + }, + ); + true + } + + fn acknowledge_runtime_ready(&mut self, node_id: u64, readiness_id: u64) -> bool { + let matching = self + .pending_readies + .get(&node_id) + .is_some_and(|ready| ready.readiness_id == readiness_id); + if matching { + self.pending_readies.remove(&node_id); + } + matching + } + + fn remove_runtime_ready(&mut self, node_id: u64) { + self.pending_readies.remove(&node_id); + } + + fn plugin_observation(&self, terminal: Option<(u64, String)>) -> ServeClusterEffect { + if !matches!(self.state, ServeClusterState::Running) { + return ServeClusterEffect::None; + } + match terminal { + Some((node_id, error)) => { + ServeClusterEffect::ProviderTerminalFailure { node_id, error } + } + None => ServeClusterEffect::None, + } + } + + fn complete(&mut self, result: Result<(), String>) -> ServeClusterEffect { + self.state = ServeClusterState::Finished; + self.pending_readies.clear(); + ServeClusterEffect::Complete(result) + } +} + +struct StopSignalActor { + sender: ExternalSender, + serve_actor: ActorAddress, +} + +impl ActorInterface for StopSignalActor { + type Incoming = swactor_process::ProcessStopSignal; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, _signal: Self::Incoming) { + let _ = self.sender.send_to(self.serve_actor, ServeClusterMsg::Stop); + ctx.stop_self(); + } +} + +fn spawn_stop_listener( + runtime: &Runtime, + external: Option>, + serve_actor: ActorAddress, +) -> Result<(), String> { + let actor = runtime + .spawn(StopSignalActor { + sender: runtime.create_sender(), + serve_actor, + }) + .map_err(|error| format!("spawn stop signal actor: {error}"))?; + let sender = runtime.create_sender(); + if let Some(external) = external { + swactor_process::spawn_stop_channel_wait(external, sender, actor); + return Ok(()); } #[cfg(target_os = "linux")] - thread::spawn(move || { - let Ok(mut signals) = signal_hook::iterator::Signals::new([ - signal_hook::consts::signal::SIGINT, - signal_hook::consts::signal::SIGTERM, - ]) else { - return; - }; - if signals.forever().next().is_some() { - listener_requested.store(true, Ordering::Release); - } - }); - requested + swactor_process::spawn_os_stop_signal_wait(sender, actor); + Ok(()) } -struct ServeCluster<'a> { - driver: &'a mut IrohDriver, - stack: &'a DistributionRuntimeStack, - obs_rx: &'a mpsc::Receiver, - collector: &'a FrameCollector, - orchestrator_reports: &'a swactor::runtime::Inbox, - stop_signal: &'a AtomicBool, - dashboard: Option<&'a DashboardSupport>, - orch_telemetry: &'a mut OrchTelemetry, - orch_stdio_rx: Option<&'a mpsc::Receiver>, +struct ManualFlushForwarder { + sender: ExternalSender, + serve_actor: ActorAddress, +} + +fn serve_cluster_flush_reply(reply: ManualControlReply) -> Option { + match reply { + ManualControlReply::Flushed => Some(ServeClusterMsg::Flushed), + ManualControlReply::Rejected(error) => Some(ServeClusterMsg::FlushFailed(format!( + "shutdown persistence flush rejected: {error}" + ))), + ManualControlReply::TimedOut => Some(ServeClusterMsg::FlushTimeout), + ManualControlReply::Accepted(_) + | ManualControlReply::Provider(_) + | ManualControlReply::Status(_) + | ManualControlReply::Offers(_) + | ManualControlReply::Rejoined(_) => None, + } +} + +impl ActorInterface for ManualFlushForwarder { + type Incoming = ManualControlReply; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, reply: Self::Incoming) { + if let Some(message) = serve_cluster_flush_reply(reply) { + let _ = self.sender.send_to(self.serve_actor, message); + } + ctx.stop_self(); + } +} + +struct ServeClusterActor { + driver: IrohDriver, + stack: DistributionRuntimeStack, + obs_rx: mpsc::Receiver, + collector: FrameCollector, + orchestrator_reports: Inbox, + dashboard: Option, + orch_telemetry: OrchTelemetry, + orch_stdio_rx: Option>, run_id: u64, orchestrator_node_id: u64, - provider: &'a ProviderKind, + provider: ProviderKind, orchestrator_actor: ActorAddress, engine: EngineHandle, - pending_readies: BTreeMap, + sender: ExternalSender, + lifecycle: ServeClusterLifecycle, + flush_reply_actor: Option, + completion: ActorCompletion>, } fn daemon_label(config: &Config) -> String { @@ -1973,7 +2201,7 @@ fn daemon_label(config: &Config) -> String { } } -impl ServeCluster<'_> { +impl ServeClusterActor { fn observe_report(&mut self, report: OrchestratorReport) { match report { OrchestratorReport::NodeRuntimeReady { @@ -1981,26 +2209,36 @@ impl ServeCluster<'_> { node_id, endpoint, node_actor, + readiness_id, .. } if run_id == self.run_id => { - self.pending_readies.insert( + let tracked = self.lifecycle.track_runtime_ready( node_id, + readiness_id, RuntimeReady { node_actor, swim_node_id: DistNodeId(*endpoint.id.as_bytes()), }, ); - self.driver.join(std::slice::from_ref(&endpoint)); - self.collector.subscribe_node( - &self.engine, - self.driver.endpoint(), - endpoint, - run_id, - node_id, - ); + if tracked { + self.driver.join(std::slice::from_ref(&endpoint)); + self.collector.subscribe_node( + &self.engine, + self.driver.endpoint(), + endpoint, + run_id, + node_id, + ); + } } - OrchestratorReport::NodeRuntimeReadyAck { node_id, .. } => { - self.pending_readies.remove(&node_id); + OrchestratorReport::NodeRuntimeReadyAck { + run_id, + node_id, + readiness_id, + .. + } if run_id == self.run_id => { + self.lifecycle + .acknowledge_runtime_ready(node_id, readiness_id); } _ => {} } @@ -2008,9 +2246,10 @@ impl ServeCluster<'_> { fn advance_join_barriers(&mut self) { let ready = self + .lifecycle .pending_readies .iter() - .filter(|(_, ready)| runtime_ready_barrier_met(self.stack, ready)) + .filter(|(_, pending)| runtime_ready_barrier_met(&self.stack, &pending.ready)) .map(|(node_id, _)| *node_id) .collect::>(); for node_id in ready { @@ -2023,17 +2262,36 @@ impl ServeCluster<'_> { ) .is_ok() { - self.pending_readies.remove(&node_id); + self.lifecycle.remove_runtime_ready(node_id); } } } fn drain_observations(&mut self) { - while let Ok(observation) = self.obs_rx.try_recv() { + loop { + let observation = match self.obs_rx.try_recv() { + Ok(observation) => observation, + Err(mpsc::TryRecvError::Empty) => break, + Err(mpsc::TryRecvError::Disconnected) => { + let _ = self.lifecycle.plugin_observation(None); + break; + } + }; + match &observation { + PluginObservation::Exited { + run_id, node_id, .. + } + | PluginObservation::Failed { + run_id, node_id, .. + } if *run_id == self.run_id => { + self.collector.unsubscribe_node(*run_id, *node_id); + } + _ => {} + } emit_plugin_observation( - self.orch_telemetry, - self.dashboard, - self.provider, + &mut self.orch_telemetry, + self.dashboard.as_ref(), + &self.provider, &observation, ); let terminal = match observation { @@ -2048,7 +2306,9 @@ impl ServeCluster<'_> { } => Some((node_id, reason)), _ => None, }; - if let Some((node_id, error)) = terminal { + if let ServeClusterEffect::ProviderTerminalFailure { node_id, error } = + self.lifecycle.plugin_observation(terminal) + { let _ = self.stack.runtime.send_to( self.orchestrator_actor, OrchestratorMsg::Manual(ManualControlMsg::ProviderTerminalFailure { @@ -2059,78 +2319,132 @@ impl ServeCluster<'_> { } } } -} -fn flush_manual_control( - runtime: &swactor::runtime::Runtime, - actor: ActorAddress, -) -> Result<(), String> { - let replies = runtime - .new_inbox::() - .map_err(|error| format!("create shutdown flush inbox: {error}"))?; - runtime - .send_to( - actor, - OrchestratorMsg::Manual(ManualControlMsg::Flush { - reply_to: *replies.addr(), - }), - ) - .map_err(|error| format!("request shutdown persistence flush: {error}"))?; - let deadline = Instant::now() + Duration::from_secs(10); - while Instant::now() < deadline { - if matches!( - replies.try_recv(), - Some(crate::orchestration::manual_control::ManualControlReply::Flushed) - ) { - return Ok(()); - } - thread::sleep(Duration::from_millis(10)); - } - Err("timed out waiting for shutdown persistence flush".to_owned()) -} - -#[allow(clippy::disallowed_methods)] -fn serve_cluster(mut ctx: ServeCluster<'_>) -> Result<(), String> { - loop { - ctx.collector.pump(ctx.driver); - ctx.collector.drain(|stream, descriptor, channel, frame| { - if let Some(dashboard) = ctx.dashboard { + fn pump_once(&mut self) { + self.collector.pump(&self.driver); + let dashboard = self.dashboard.as_ref(); + let telemetry = &mut self.orch_telemetry; + self.collector.drain(|stream, descriptor, channel, frame| { + if let Some(dashboard) = dashboard { dashboard.publish_frame(stream, descriptor, channel, frame); } - ctx.orch_telemetry - .archive_frame("node", stream, channel, frame); + telemetry.archive_frame("node", stream, channel, frame); }); - ctx.drain_observations(); - while let Some(report) = ctx.orchestrator_reports.try_recv() { - ctx.observe_report(report); + self.drain_observations(); + while let Some(report) = self.orchestrator_reports.try_recv() { + self.observe_report(report); } - ctx.advance_join_barriers(); + self.advance_join_barriers(); emit_swim_transitions( - ctx.orch_telemetry, - ctx.dashboard, - ctx.run_id, - ctx.orchestrator_node_id, - ctx.stack, + &mut self.orch_telemetry, + self.dashboard.as_ref(), + self.run_id, + self.orchestrator_node_id, + &self.stack, ); emit_swim_probe_events( - ctx.orch_telemetry, - ctx.dashboard, - ctx.stack, + &mut self.orch_telemetry, + self.dashboard.as_ref(), + &self.stack, "daemon_monitor", ); drain_orch_stdio_capture( - ctx.orch_stdio_rx, - ctx.orch_telemetry, - ctx.dashboard, - ctx.run_id, - ctx.orchestrator_node_id, + self.orch_stdio_rx.as_ref(), + &mut self.orch_telemetry, + self.dashboard.as_ref(), + self.run_id, + self.orchestrator_node_id, ); - if stop_requested(ctx.stop_signal) { - break; - } - thread::sleep(PUMP_INTERVAL); + self.orch_telemetry + .flush(self.dashboard.as_ref(), "orchestrator"); + } + + fn schedule(&self, ctx: &Ctx, delay: Duration, message: ServeClusterMsg) { + self.engine + .send_after(delay, self.sender.clone(), ctx.self_addr(), message); + } + + fn begin_flush(&mut self, ctx: &Ctx) -> Result<(), String> { + let reply_actor = self + .stack + .runtime + .spawn(ManualFlushForwarder { + sender: self.sender.clone(), + serve_actor: ctx.self_addr(), + }) + .map_err(|error| format!("spawn shutdown flush reply actor: {error}"))?; + if let Err(error) = self.stack.runtime.send_to( + self.orchestrator_actor, + OrchestratorMsg::Manual(ManualControlMsg::Flush { + reply_to: reply_actor, + }), + ) { + let _ = self.stack.runtime.stop_actor(reply_actor); + return Err(format!("request shutdown persistence flush: {error}")); + } + self.flush_reply_actor = Some(reply_actor); + self.schedule(ctx, SHUTDOWN_FLUSH_TIMEOUT, ServeClusterMsg::FlushTimeout); + Ok(()) + } + + fn finish(&mut self, ctx: &Ctx, result: Result<(), String>) { + if let Some(reply_actor) = self.flush_reply_actor.take() { + let _ = self.stack.runtime.stop_actor(reply_actor); + } + if let Err(error) = &result { + self.orch_telemetry.emit_bootstrap( + self.dashboard.as_ref(), + self.run_id, + self.orchestrator_node_id, + "serve_cluster", + "failed", + json!({"error":error}), + ); + } + assert!( + self.completion.complete(result).is_ok(), + "daemon lifecycle completed twice" + ); + ctx.stop_self(); + } +} + +impl ActorInterface for ServeClusterActor { + type Incoming = ServeClusterMsg; + type Response = (); + + fn on_start(&mut self, ctx: &Ctx) { + let _ = ctx.send(ctx.self_addr(), ServeClusterMsg::Tick); + } + + fn handle(&mut self, ctx: &Ctx, message: Self::Incoming) { + let effect = self.lifecycle.apply_message(message); + match effect { + ServeClusterEffect::None => {} + ServeClusterEffect::Pump => { + self.pump_once(); + self.schedule(ctx, PUMP_INTERVAL, ServeClusterMsg::Tick); + } + ServeClusterEffect::BeginFlush => { + if let Err(error) = self.begin_flush(ctx) { + let failure = self.lifecycle.apply_message(ServeClusterMsg::Abort(error)); + if let ServeClusterEffect::Complete(result) = failure { + self.finish(ctx, result); + } + } + } + ServeClusterEffect::ProviderTerminalFailure { node_id, error } => { + let _ = self.stack.runtime.send_to( + self.orchestrator_actor, + OrchestratorMsg::Manual(ManualControlMsg::ProviderTerminalFailure { + node_id, + error, + }), + ); + } + ServeClusterEffect::Complete(result) => self.finish(ctx, result), + } } - Ok(()) } fn emit_plugin_observation( @@ -2304,17 +2618,15 @@ pub(crate) fn expand_home_path(value: &str) -> Result { } pub(crate) fn derive_ssh_public_key(identity: &Path) -> Result { - let output = Command::new("ssh-keygen") - .arg("-y") - .arg("-f") - .arg(identity) - .output() - .map_err(|e| { - format!( - "derive VastAI SSH public key from {}: {e}", - identity.display() - ) - })?; + let output = swactor_process::command_output( + &mut Command::new("ssh-keygen").arg("-y").arg("-f").arg(identity), + ) + .map_err(|e| { + format!( + "derive VastAI SSH public key from {}: {e}", + identity.display() + ) + })?; let public_key = String::from_utf8_lossy(&output.stdout) .trim_end_matches(['\r', '\n']) .to_owned(); @@ -2336,12 +2648,10 @@ pub(crate) fn ssh_public_key_fingerprint(public_key: &str) -> String { if std::fs::write(&path, format!("{public_key}\n")).is_err() { return UNAVAILABLE.to_owned(); } - let output = Command::new("ssh-keygen") - .arg("-l") - .arg("-f") - .arg(&path) - .output() - .ok(); + let output = swactor_process::command_output( + &mut Command::new("ssh-keygen").arg("-l").arg("-f").arg(&path), + ) + .ok(); let _ = std::fs::remove_file(&path); let Some(output) = output.filter(|output| output.status.success()) else { return UNAVAILABLE.to_owned(); @@ -2357,10 +2667,14 @@ pub(crate) fn ssh_public_key_fingerprint(public_key: &str) -> String { } fn vastai_account_has_ssh_key(api_key: &str, public_key: &str) -> Result { - let output = Command::new("vastai") - .args(["show", "ssh-keys", "--raw", "--api-key", api_key]) - .output() - .map_err(vastai_cli_error)?; + let output = swactor_process::command_output(&mut Command::new("vastai").args([ + "show", + "ssh-keys", + "--raw", + "--api-key", + api_key, + ])) + .map_err(vastai_cli_error)?; if !output.status.success() { return Err(format!( "vastai show ssh-keys failed: {}", @@ -2378,12 +2692,13 @@ pub(crate) fn ensure_vastai_account_ssh_key(api_key: &str, public_key: &str) -> return Ok(()); } - let output = Command::new("vastai") - .args(["create", "ssh-key"]) - .arg(public_key) - .args(["-y", "--api-key", api_key]) - .output() - .map_err(vastai_cli_error)?; + let output = swactor_process::command_output( + &mut Command::new("vastai") + .args(["create", "ssh-key"]) + .arg(public_key) + .args(["-y", "--api-key", api_key]), + ) + .map_err(vastai_cli_error)?; if !output.status.success() { return Err(format!( "vastai create ssh-key failed: {}", @@ -2444,6 +2759,656 @@ where .map_err(|e| format!("invalid {name}={value:?}: {e}")) } +#[cfg(test)] +mod serve_cluster_properties { + use proptest::prelude::*; + use swactor::config::RuntimeConfig; + use swactor::runtime::RuntimeParts; + use swactor_engine::{ActorCompletion, Engine, SteppingBackend}; + + use super::*; + use crate::tests::fuzz_support::{actor_census, drive_steps}; + + const READY_NODE_DOMAIN: u8 = 8; + const STEPS_PER_ACTION: usize = 16; + const FINAL_STEPS: usize = 32; + + #[derive(Clone, Debug)] + enum LifecycleAction { + Tick, + RuntimeReady { node_id: u8, readiness_id: u8 }, + RuntimeReadyAck { node_id: u8, readiness_id: u8 }, + PluginObserved { node_id: u8 }, + PluginFailed { node_id: u8, code: u8 }, + PluginDisconnected, + Stop, + ManualFlushed, + ManualRejected(u8), + ManualTimedOut, + FlushTimeout, + Abort(u8), + } + + fn lifecycle_actions() -> impl Strategy> { + prop::collection::vec( + prop_oneof![ + 4 => Just(LifecycleAction::Tick), + 4 => (0_u8..READY_NODE_DOMAIN, any::()).prop_map( + |(node_id, readiness_id)| LifecycleAction::RuntimeReady { + node_id, + readiness_id, + } + ), + 3 => (0_u8..READY_NODE_DOMAIN, any::()).prop_map( + |(node_id, readiness_id)| LifecycleAction::RuntimeReadyAck { + node_id, + readiness_id, + } + ), + 3 => (0_u8..READY_NODE_DOMAIN) + .prop_map(|node_id| LifecycleAction::PluginObserved { node_id }), + 3 => (0_u8..READY_NODE_DOMAIN, any::()).prop_map( + |(node_id, code)| LifecycleAction::PluginFailed { node_id, code } + ), + 2 => Just(LifecycleAction::PluginDisconnected), + 3 => Just(LifecycleAction::Stop), + 2 => Just(LifecycleAction::ManualFlushed), + 2 => any::().prop_map(LifecycleAction::ManualRejected), + 2 => Just(LifecycleAction::ManualTimedOut), + 2 => Just(LifecycleAction::FlushTimeout), + 2 => any::().prop_map(LifecycleAction::Abort), + ], + 0..=32, + ) + } + + #[derive(Clone, Debug)] + enum LifecycleHarnessMsg { + Action(LifecycleAction), + ScheduledFlushTimeout, + Finalize, + } + + #[derive(Clone, Debug)] + struct LifecycleTrace { + state: ServeClusterState, + pending_readies: Vec<(u64, u64)>, + max_pending_readies: usize, + completion_results: Vec>, + completion_publication_failures: usize, + pumps: usize, + readies_tracked: usize, + readies_acknowledged: usize, + provider_failures: Vec<(u64, String)>, + disconnected_observations: usize, + manual_replies: Vec<&'static str>, + flushes_started: usize, + flush_timers_scheduled: usize, + } + + impl LifecycleTrace { + fn new() -> Self { + Self { + state: ServeClusterState::Running, + pending_readies: Vec::new(), + max_pending_readies: 0, + completion_results: Vec::new(), + completion_publication_failures: 0, + pumps: 0, + readies_tracked: 0, + readies_acknowledged: 0, + provider_failures: Vec::new(), + disconnected_observations: 0, + manual_replies: Vec::new(), + flushes_started: 0, + flush_timers_scheduled: 0, + } + } + } + + struct ServeClusterLifecycleHarness { + lifecycle: ServeClusterLifecycle<()>, + completion: ActorCompletion>, + trace: Arc>, + engine: EngineHandle, + sender: ExternalSender, + } + + impl ServeClusterLifecycleHarness { + fn record_state(&self) { + let pending_readies = self + .lifecycle + .pending_readies + .iter() + .map(|(node_id, pending)| (*node_id, pending.readiness_id)) + .collect::>(); + let mut trace = self.trace.lock(); + trace.state = self.lifecycle.state; + trace.max_pending_readies = trace.max_pending_readies.max(pending_readies.len()); + trace.pending_readies = pending_readies; + } + + fn apply_effect(&mut self, ctx: &Ctx, effect: ServeClusterEffect) { + match effect { + ServeClusterEffect::None => {} + ServeClusterEffect::Pump => { + self.trace.lock().pumps += 1; + } + ServeClusterEffect::BeginFlush => { + { + let mut trace = self.trace.lock(); + trace.flushes_started += 1; + trace.flush_timers_scheduled += 1; + } + self.engine.send_after( + SHUTDOWN_FLUSH_TIMEOUT, + self.sender.clone(), + ctx.self_addr(), + LifecycleHarnessMsg::ScheduledFlushTimeout, + ); + } + ServeClusterEffect::ProviderTerminalFailure { node_id, error } => { + self.trace.lock().provider_failures.push((node_id, error)); + } + ServeClusterEffect::Complete(result) => { + self.trace.lock().completion_results.push(result.clone()); + if self.completion.complete(result).is_err() { + self.trace.lock().completion_publication_failures += 1; + } + } + } + } + + fn apply_action(&mut self, ctx: &Ctx, action: LifecycleAction) { + let effect = match action { + LifecycleAction::Tick => self.lifecycle.apply_message(ServeClusterMsg::Tick), + LifecycleAction::RuntimeReady { + node_id, + readiness_id, + } => { + if self.lifecycle.track_runtime_ready( + u64::from(node_id), + u64::from(readiness_id), + (), + ) { + self.trace.lock().readies_tracked += 1; + } + ServeClusterEffect::None + } + LifecycleAction::RuntimeReadyAck { + node_id, + readiness_id, + } => { + if self + .lifecycle + .acknowledge_runtime_ready(u64::from(node_id), u64::from(readiness_id)) + { + self.trace.lock().readies_acknowledged += 1; + } + ServeClusterEffect::None + } + LifecycleAction::PluginObserved { node_id } => { + let _ = node_id; + self.lifecycle.plugin_observation(None) + } + LifecycleAction::PluginFailed { node_id, code } => { + self.lifecycle.plugin_observation(Some(( + u64::from(node_id), + format!("plugin-failed-{code}"), + ))) + } + LifecycleAction::PluginDisconnected => { + self.trace.lock().disconnected_observations += 1; + self.lifecycle.plugin_observation(None) + } + LifecycleAction::Stop => self.lifecycle.apply_message(ServeClusterMsg::Stop), + LifecycleAction::ManualFlushed => { + self.trace.lock().manual_replies.push("flushed"); + let message = serve_cluster_flush_reply(ManualControlReply::Flushed) + .expect("flush reply must be forwarded"); + self.lifecycle.apply_message(message) + } + LifecycleAction::ManualRejected(code) => { + self.trace.lock().manual_replies.push("rejected"); + let message = serve_cluster_flush_reply(ManualControlReply::Rejected(format!( + "manual-rejected-{code}" + ))) + .expect("rejected flush reply must be forwarded"); + self.lifecycle.apply_message(message) + } + LifecycleAction::ManualTimedOut => { + self.trace.lock().manual_replies.push("timed-out"); + let message = serve_cluster_flush_reply(ManualControlReply::TimedOut) + .expect("timed-out flush reply must be forwarded"); + self.lifecycle.apply_message(message) + } + LifecycleAction::FlushTimeout => { + self.lifecycle.apply_message(ServeClusterMsg::FlushTimeout) + } + LifecycleAction::Abort(code) => self + .lifecycle + .apply_message(ServeClusterMsg::Abort(format!("abort-{code}"))), + }; + self.apply_effect(ctx, effect); + self.record_state(); + } + } + + impl ActorInterface for ServeClusterLifecycleHarness { + type Incoming = LifecycleHarnessMsg; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, message: Self::Incoming) { + match message { + LifecycleHarnessMsg::Action(action) => self.apply_action(ctx, action), + LifecycleHarnessMsg::ScheduledFlushTimeout => { + let effect = self.lifecycle.apply_message(ServeClusterMsg::FlushTimeout); + self.apply_effect(ctx, effect); + self.record_state(); + } + LifecycleHarnessMsg::Finalize => { + if !matches!(self.lifecycle.state, ServeClusterState::Finished) { + let effect = self.lifecycle.apply_message(ServeClusterMsg::Abort( + "generated sequence exhausted".to_owned(), + )); + self.apply_effect(ctx, effect); + } + self.record_state(); + ctx.stop_self(); + } + } + } + } + + #[derive(Clone, Debug)] + struct ExpectedLifecycle { + success: bool, + readies_tracked: usize, + readies_acknowledged: usize, + provider_failures: usize, + manual_replies: usize, + flushes_started: usize, + } + + fn expected_lifecycle(actions: &[LifecycleAction]) -> ExpectedLifecycle { + let mut state = ServeClusterState::Running; + let mut pending_readies = BTreeMap::::new(); + let mut result = None; + let mut readies_tracked = 0; + let mut readies_acknowledged = 0; + let mut provider_failures = 0; + let mut manual_replies = 0; + let mut flushes_started = 0; + + for action in actions { + match action { + LifecycleAction::Tick | LifecycleAction::PluginObserved { .. } => {} + LifecycleAction::RuntimeReady { + node_id, + readiness_id, + } if matches!(state, ServeClusterState::Running) => { + let node_id = u64::from(*node_id); + let readiness_id = u64::from(*readiness_id); + let newer = pending_readies + .get(&node_id) + .is_none_or(|current| *current < readiness_id); + if newer { + pending_readies.insert(node_id, readiness_id); + readies_tracked += 1; + } + } + LifecycleAction::RuntimeReadyAck { + node_id, + readiness_id, + } => { + let node_id = u64::from(*node_id); + let readiness_id = u64::from(*readiness_id); + if pending_readies.get(&node_id) == Some(&readiness_id) { + pending_readies.remove(&node_id); + readies_acknowledged += 1; + } + } + LifecycleAction::PluginFailed { .. } + if matches!(state, ServeClusterState::Running) => + { + provider_failures += 1; + } + LifecycleAction::PluginDisconnected + | LifecycleAction::PluginFailed { .. } + | LifecycleAction::RuntimeReady { .. } => {} + LifecycleAction::Stop if matches!(state, ServeClusterState::Running) => { + state = ServeClusterState::Flushing; + pending_readies.clear(); + flushes_started += 1; + } + LifecycleAction::ManualFlushed => { + manual_replies += 1; + if matches!(state, ServeClusterState::Flushing) { + state = ServeClusterState::Finished; + pending_readies.clear(); + result = Some(true); + } + } + LifecycleAction::ManualRejected(_) | LifecycleAction::ManualTimedOut => { + manual_replies += 1; + if matches!(state, ServeClusterState::Flushing) { + state = ServeClusterState::Finished; + pending_readies.clear(); + result = Some(false); + } + } + LifecycleAction::FlushTimeout if matches!(state, ServeClusterState::Flushing) => { + state = ServeClusterState::Finished; + pending_readies.clear(); + result = Some(false); + } + LifecycleAction::Abort(_) if !matches!(state, ServeClusterState::Finished) => { + state = ServeClusterState::Finished; + pending_readies.clear(); + result = Some(false); + } + LifecycleAction::Stop + | LifecycleAction::FlushTimeout + | LifecycleAction::Abort(_) => {} + } + } + + if matches!(state, ServeClusterState::Flushing) { + result = Some(false); + } + ExpectedLifecycle { + success: result.unwrap_or(false), + readies_tracked, + readies_acknowledged, + provider_failures, + manual_replies, + flushes_started, + } + } + + #[derive(Clone, Debug)] + struct LifecycleResources { + baseline_actors: usize, + peak_actors: usize, + final_actors: usize, + baseline_tasks: usize, + peak_tasks: usize, + final_tasks: usize, + poisoned_actors: usize, + worker_panics: u64, + mailbox_depth: usize, + } + + impl LifecycleResources { + fn new(baseline_actors: usize, baseline_tasks: usize) -> Self { + Self { + baseline_actors, + peak_actors: baseline_actors, + final_actors: baseline_actors, + baseline_tasks, + peak_tasks: baseline_tasks, + final_tasks: baseline_tasks, + poisoned_actors: 0, + worker_panics: 0, + mailbox_depth: 0, + } + } + + fn observe(&mut self, runtime: &Runtime, backend: &SteppingBackend) { + let stats = runtime.stats(); + self.peak_actors = self.peak_actors.max(stats.actors.len()); + self.final_actors = stats.actors.len(); + self.peak_tasks = self.peak_tasks.max(backend.pending_task_count()); + self.final_tasks = backend.pending_task_count(); + self.poisoned_actors = stats + .actor_details + .iter() + .filter(|actor| actor.poisoned) + .count(); + self.worker_panics = stats.workers.iter().map(|worker| worker.panics).sum(); + self.mailbox_depth = stats + .workers + .iter() + .map(|worker| worker.mailbox_depth) + .sum::() + + stats + .actor_details + .iter() + .map(|actor| actor.mailbox_depth) + .sum::(); + } + } + + fn lifecycle_invariant_errors( + trace: &LifecycleTrace, + resources: &LifecycleResources, + expected: &ExpectedLifecycle, + published_result: &Option>, + ) -> Vec { + let mut errors = Vec::new(); + if trace.state != ServeClusterState::Finished { + errors.push(format!( + "lifecycle did not converge: state={:?}", + trace.state + )); + } + if !trace.pending_readies.is_empty() { + errors.push(format!( + "pending readiness did not drain: {:?}", + trace.pending_readies + )); + } + if trace.max_pending_readies > usize::from(READY_NODE_DOMAIN) { + errors.push(format!( + "pending readiness exceeded node domain: max={} domain={}", + trace.max_pending_readies, READY_NODE_DOMAIN + )); + } + if trace.completion_results.len() != 1 { + errors.push(format!( + "expected one completion, observed {:?}", + trace.completion_results + )); + } + if trace.completion_publication_failures != 0 { + errors.push(format!( + "completion publication failed {} time(s)", + trace.completion_publication_failures + )); + } + if trace.completion_results.first().map(Result::is_ok) != Some(expected.success) { + errors.push(format!( + "completion/model mismatch: completion={:?} expected_success={}", + trace.completion_results, expected.success + )); + } + if published_result.as_ref() != trace.completion_results.first() { + errors.push(format!( + "published completion mismatch: published={published_result:?} trace={:?}", + trace.completion_results + )); + } + if trace.readies_tracked != expected.readies_tracked + || trace.readies_acknowledged != expected.readies_acknowledged + { + errors.push(format!( + "readiness model mismatch: tracked={}/{} acknowledged={}/{}", + trace.readies_tracked, + expected.readies_tracked, + trace.readies_acknowledged, + expected.readies_acknowledged + )); + } + if trace.provider_failures.len() != expected.provider_failures { + errors.push(format!( + "manual provider-failure forwarding mismatch: observed={:?} expected_count={}", + trace.provider_failures, expected.provider_failures + )); + } + if trace.manual_replies.len() != expected.manual_replies { + errors.push(format!( + "manual reply evidence mismatch: observed={:?} expected_count={}", + trace.manual_replies, expected.manual_replies + )); + } + if trace.flushes_started != expected.flushes_started + || trace.flush_timers_scheduled != trace.flushes_started + || trace.flushes_started > 1 + { + errors.push(format!( + "flush state was not bounded: starts={} timers={} expected_starts={}", + trace.flushes_started, trace.flush_timers_scheduled, expected.flushes_started + )); + } + if resources.peak_actors > resources.baseline_actors.saturating_add(1) { + errors.push(format!( + "per-event actor growth: baseline={} peak={}", + resources.baseline_actors, resources.peak_actors + )); + } + if resources.final_actors != resources.baseline_actors { + errors.push(format!( + "actor census did not return to baseline: baseline={} final={}", + resources.baseline_actors, resources.final_actors + )); + } + if resources.peak_tasks > resources.baseline_tasks.saturating_add(1) { + errors.push(format!( + "pending tasks were unbounded: baseline={} peak={}", + resources.baseline_tasks, resources.peak_tasks + )); + } + if resources.final_tasks != resources.baseline_tasks { + errors.push(format!( + "pending tasks did not drain: baseline={} final={}", + resources.baseline_tasks, resources.final_tasks + )); + } + if resources.poisoned_actors != 0 || resources.worker_panics != 0 { + errors.push(format!( + "actor poison detected: poisoned={} worker_panics={}", + resources.poisoned_actors, resources.worker_panics + )); + } + if resources.mailbox_depth != 0 { + errors.push(format!( + "mailboxes did not drain: depth={}", + resources.mailbox_depth + )); + } + errors + } + + proptest! { + #![proptest_config(ProptestConfig { + cases: 128, + max_shrink_iters: 2_000, + ..ProptestConfig::default() + })] + + #[test] + fn serve_cluster_production_transitions_converge_once_without_growth( + actions in lifecycle_actions() + ) { + let mut config = RuntimeConfig::default(); + config.worker_count = 1; + let parts = RuntimeParts::new(config); + let runtime = parts.runtime().clone(); + let backend = SteppingBackend::new(); + let engine = Engine::new(parts, backend.clone()).expect("one-worker stepping engine"); + let baseline_actors = runtime.stats().actors.len(); + let baseline_tasks = backend.pending_task_count(); + let completion = ActorCompletion::new(); + let trace = Arc::new(Mutex::new(LifecycleTrace::new())); + let actor = runtime + .spawn(ServeClusterLifecycleHarness { + lifecycle: ServeClusterLifecycle::new(), + completion: completion.clone(), + trace: Arc::clone(&trace), + engine: engine.handle(), + sender: runtime.create_sender(), + }) + .expect("spawn production lifecycle transition harness"); + let mut resources = LifecycleResources::new(baseline_actors, baseline_tasks); + resources.observe(&runtime, &backend); + + for action in &actions { + runtime + .send_to(actor, LifecycleHarnessMsg::Action(action.clone())) + .expect("queue generated serve-cluster action"); + drive_steps(&backend, STEPS_PER_ACTION); + resources.observe(&runtime, &backend); + } + + backend.advance_time(SHUTDOWN_FLUSH_TIMEOUT); + drive_steps(&backend, FINAL_STEPS); + resources.observe(&runtime, &backend); + runtime + .send_to(actor, LifecycleHarnessMsg::Finalize) + .expect("queue serve-cluster lifecycle finalizer"); + drive_steps(&backend, FINAL_STEPS); + resources.observe(&runtime, &backend); + + let trace = trace.lock().clone(); + let published_result = if trace.completion_results.len() == 1 + && trace.completion_publication_failures == 0 + { + Some(completion.wait()) + } else { + None + }; + let expected = expected_lifecycle(&actions); + let errors = + lifecycle_invariant_errors(&trace, &resources, &expected, &published_result); + prop_assert!( + errors.is_empty(), + "serve-cluster lifecycle invariant failure\nerrors={errors:#?}\nactions={actions:#?}\nstate/replies={trace:#?}\nresources={resources:#?}\ncensus=\n{}", + actor_census(&runtime), + ); + } + } + + #[test] + fn serve_cluster_lifecycle_invariants_reject_injected_duplicate_and_growth() { + let mut trace = LifecycleTrace::new(); + trace.state = ServeClusterState::Finished; + trace.completion_results = vec![ + Err("injected-first".to_owned()), + Err("injected-duplicate".to_owned()), + ]; + trace.completion_publication_failures = 1; + let expected = ExpectedLifecycle { + success: false, + readies_tracked: 0, + readies_acknowledged: 0, + provider_failures: 0, + manual_replies: 0, + flushes_started: 0, + }; + let resources = LifecycleResources { + baseline_actors: 3, + peak_actors: 5, + final_actors: 3, + baseline_tasks: 0, + peak_tasks: 0, + final_tasks: 0, + poisoned_actors: 0, + worker_panics: 0, + mailbox_depth: 0, + }; + let published_result = Some(Err("injected-first".to_owned())); + + let errors = lifecycle_invariant_errors(&trace, &resources, &expected, &published_result); + assert!( + errors.iter().any(|error| error.contains("one completion")) + && errors + .iter() + .any(|error| error.contains("per-event actor growth")), + "controlled defects were not rejected by property invariants: {errors:#?}" + ); + } +} + #[cfg(test)] mod lifecycle_policy_tests { use super::{ActorAddress, ConfigBuilder, EndpointAddr, VastAiProvisioningMode}; @@ -2469,6 +3434,13 @@ mod lifecycle_policy_tests { .node_spec_for_stage(coordinator, ActorAddress::default(), 1, 0) .unwrap(); assert_eq!(spec.args, [crate::ORCHESTRATOR_WORKER_MODE_ARG]); + let attempt_id = spec.attempt_id.to_string(); + assert_eq!( + spec.env.iter().find_map( + |(key, value)| (key == "MYELIN_NODE_ATTEMPT_ID").then_some(value.as_str()) + ), + Some(attempt_id.as_str()), + ); assert!(vastai.bootstrap_command.is_none()); } diff --git a/apps/myelin/src/orchestration/cluster_reconciler.rs b/apps/myelin/src/orchestration/cluster_reconciler.rs index f7e9664..dcddb85 100644 --- a/apps/myelin/src/orchestration/cluster_reconciler.rs +++ b/apps/myelin/src/orchestration/cluster_reconciler.rs @@ -1,7 +1,9 @@ //! Myelin integration for the provider-neutral cluster reconciler. use std::collections::{BTreeMap, BTreeSet, VecDeque}; -use std::sync::mpsc::{self, Receiver, Sender, TryRecvError}; +#[cfg(test)] +use std::sync::mpsc::TryRecvError; +use std::sync::mpsc::{self, Receiver, Sender}; use std::sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard}; use std::time::{Duration, SystemTime}; @@ -12,7 +14,9 @@ use provisioning::{ NodeManagerCommand, NodeObservation, NodeStage, OperationOutcome, PlannedEffect, ProviderLeaseId, RetryPolicy, SshEndpoint, SwactorId, }; -use swactor_engine::EngineHandle; +use swactor::actor::{ActorAddress, ActorInterface}; +use swactor::runtime::{Ctx, ExternalSender, Runtime}; +use swactor_engine::{BlockingWorkSender, EngineHandle}; use crate::provisioning::{ NodeProvisionSpec, PluginNodeHandle, PluginObservation, PluginObservationSink, PluginSink, @@ -447,12 +451,14 @@ impl EffectBackend for MyelinEffectBackend { #[derive(Clone)] pub(crate) struct EngineEffectSpawner { - engine: EngineHandle, + blocking: BlockingWorkSender, } impl EngineEffectSpawner { - fn new(engine: EngineHandle) -> Self { - Self { engine } + fn new(engine: &EngineHandle) -> Self { + Self { + blocking: engine.blocking_work_sender(), + } } } @@ -463,11 +469,9 @@ impl BlockingEffectSpawner for EngineEffectSpawner { &self, work: provisioning::BlockingEffectWork, ) -> Result<(), Self::SpawnError> { - if !self.engine.capabilities().blocking { - return Err("engine blocking work capability is unavailable".to_owned()); - } - self.engine.spawn_blocking(work); - Ok(()) + self.blocking + .submit(work) + .map_err(|_| "engine stopped before provider effect submission".to_owned()) } } @@ -476,14 +480,79 @@ enum ControllerWake { Periodic, } +#[derive(Clone)] +enum ControllerTimerMsg { + ScheduleDeadline(Option), + DeadlineElapsed(SystemTime), + PeriodicElapsed, +} + +struct ControllerTimerActor { + engine: EngineHandle, + sender: ExternalSender, + wake: Sender, + scheduled_deadline: Option, +} + +impl ControllerTimerActor { + fn schedule_periodic(&self, ctx: &Ctx) { + self.engine.send_after( + PERIODIC_RECONCILE, + self.sender.clone(), + ctx.self_addr(), + ControllerTimerMsg::PeriodicElapsed, + ); + } +} + +impl ActorInterface for ControllerTimerActor { + type Incoming = ControllerTimerMsg; + type Response = (); + + fn on_start(&mut self, ctx: &Ctx) { + self.schedule_periodic(ctx); + } + + fn handle(&mut self, ctx: &Ctx, message: Self::Incoming) { + match message { + ControllerTimerMsg::ScheduleDeadline(deadline) => { + self.scheduled_deadline = deadline; + if let Some(deadline) = deadline { + self.engine.send_after( + deadline + .duration_since(SystemTime::now()) + .unwrap_or(Duration::ZERO), + self.sender.clone(), + ctx.self_addr(), + ControllerTimerMsg::DeadlineElapsed(deadline), + ); + } + } + ControllerTimerMsg::DeadlineElapsed(deadline) => { + if self.scheduled_deadline == Some(deadline) { + self.scheduled_deadline = None; + let _ = self.wake.send(ControllerWake::Deadline(deadline)); + } + } + ControllerTimerMsg::PeriodicElapsed => { + if self.wake.send(ControllerWake::Periodic).is_ok() { + self.schedule_periodic(ctx); + } else { + ctx.stop_self(); + } + } + } + } +} + pub(crate) struct ProvisionedClusterGuard { driver: ClusterDriver, executor: IdempotentEffectExecutor, external_nodes: BTreeMap, failure_rx: Receiver, deferred_failures: VecDeque, - engine: EngineHandle, - wake_tx: Sender, + runtime: Runtime, + timer_actor: ActorAddress, wake_rx: Receiver, scheduled_deadline: Option, stopped: bool, @@ -495,6 +564,7 @@ impl ProvisionedClusterGuard { bindings: Vec, retry: RetryPolicy, engine: EngineHandle, + runtime: Runtime, sink: PluginSink, ) -> Result { let expanded = desired.expand().map_err(|error| error.to_string())?; @@ -520,19 +590,25 @@ impl ProvisionedClusterGuard { } let (failure_tx, failure_rx) = mpsc::channel(); let (backend, external_nodes) = MyelinEffectBackend::new(bindings, sink, failure_tx)?; - let executor = - IdempotentEffectExecutor::new(backend, EngineEffectSpawner::new(engine.clone())); + let executor = IdempotentEffectExecutor::new(backend, EngineEffectSpawner::new(&engine)); let driver = ClusterDriver::new(desired, retry).map_err(|error| error.to_string())?; let (wake_tx, wake_rx) = mpsc::channel(); - spawn_periodic_wake(&engine, wake_tx.clone()); + let timer_actor = runtime + .spawn(ControllerTimerActor { + engine: engine.clone(), + sender: runtime.create_sender(), + wake: wake_tx, + scheduled_deadline: None, + }) + .map_err(|error| format!("spawn cluster timer actor: {error}"))?; Ok(Self { driver, executor, external_nodes, failure_rx, deferred_failures: VecDeque::new(), - engine, - wake_tx, + runtime, + timer_actor, wake_rx, scheduled_deadline: None, stopped: false, @@ -657,7 +733,7 @@ impl ProvisionedClusterGuard { Ok(submitted) } - fn begin_shutdown(&mut self) -> Result<(), String> { + pub(crate) fn begin_shutdown(&mut self) -> Result<(), String> { if self.stopped { return Ok(()); } @@ -673,31 +749,46 @@ impl ProvisionedClusterGuard { self.driver.state().nodes.is_empty() } - // Synchronous orchestration waits while all provider work remains engine-hosted. - #[allow(clippy::disallowed_methods)] + pub(crate) fn finish_shutdown(&mut self) -> Result<(), String> { + if !self.stopped { + self.executor.backend().stop_all()?; + self.stopped = true; + } + Ok(()) + } + + #[cfg(test)] pub(crate) fn stop(&mut self) -> Result<(), String> { self.begin_shutdown()?; while !self.is_stopped() { self.poll(SystemTime::now()) .map_err(|error| error.to_string())?; - std::thread::sleep(Duration::from_millis(10)); + self.wait_for_work(Duration::from_millis(10)); + } + self.finish_shutdown() + } + + #[cfg(test)] + pub(crate) fn wait_for_work(&mut self, timeout: Duration) { + if let Ok(wake) = self.wake_rx.recv_timeout(timeout) { + self.apply_wake(wake, SystemTime::now()); } - self.executor.backend().stop_all()?; - self.stopped = true; - Ok(()) } fn drain_wakes(&mut self, now: SystemTime) { - loop { - match self.wake_rx.try_recv() { - Ok(ControllerWake::Periodic) => self.driver.trigger(), - Ok(ControllerWake::Deadline(deadline)) => { - if self.scheduled_deadline == Some(deadline) { - self.scheduled_deadline = None; - } - self.driver.trigger_if_due(now); + while let Ok(wake) = self.wake_rx.try_recv() { + self.apply_wake(wake, now); + } + } + + fn apply_wake(&mut self, wake: ControllerWake, now: SystemTime) { + match wake { + ControllerWake::Periodic => self.driver.trigger(), + ControllerWake::Deadline(deadline) => { + if self.scheduled_deadline == Some(deadline) { + self.scheduled_deadline = None; } - Err(TryRecvError::Empty | TryRecvError::Disconnected) => return, + self.driver.trigger_if_due(now); } } } @@ -798,20 +889,19 @@ impl ProvisionedClusterGuard { }) { self.scheduled_deadline = None; + let _ = self + .runtime + .send_to(self.timer_actor, ControllerTimerMsg::ScheduleDeadline(None)); return; } - if deadline.is_none() || deadline == self.scheduled_deadline { + if deadline == self.scheduled_deadline { return; } - let deadline = deadline.expect("checked deadline"); - self.scheduled_deadline = Some(deadline); - let delay = deadline.duration_since(now).unwrap_or(Duration::ZERO); - let timer = self.engine.timer(delay); - let wake = self.wake_tx.clone(); - self.engine.spawn(async move { - timer.await; - let _ = wake.send(ControllerWake::Deadline(deadline)); - }); + self.scheduled_deadline = deadline; + let _ = self.runtime.send_to( + self.timer_actor, + ControllerTimerMsg::ScheduleDeadline(deadline), + ); } } @@ -823,18 +913,6 @@ impl Drop for ProvisionedClusterGuard { } } -fn spawn_periodic_wake(engine: &EngineHandle, wake: Sender) { - let mut interval = engine.interval(PERIODIC_RECONCILE); - engine.spawn(async move { - loop { - (&mut interval).await; - if wake.send(ControllerWake::Periodic).is_err() { - return; - } - } - }); -} - fn lock_node(node: &Mutex) -> MutexGuard<'_, NodeEffects> { node.lock() .unwrap_or_else(std::sync::PoisonError::into_inner) @@ -1353,7 +1431,6 @@ mod tests { } #[test] - #[allow(clippy::disallowed_methods)] fn engine_hosted_controller_converges_and_cleans_up_end_to_end() { let stats = Arc::new(PluginStats::default()); let desired_node = desired(); @@ -1391,6 +1468,7 @@ mod tests { }), }; let parts = swactor::runtime::RuntimeParts::new(swactor::config::RuntimeConfig::default()); + let runtime = parts.runtime().clone(); let backend = swactor_engine::TokioBackend::new(swactor_engine::TokioConfig::default()).unwrap(); let engine = swactor_engine::Engine::new(parts, backend).unwrap(); @@ -1400,6 +1478,7 @@ mod tests { vec![binding], RetryPolicy::default(), engine.handle(), + runtime, sink, ) .unwrap(); @@ -1503,7 +1582,6 @@ mod tests { assert_eq!(scaled_stats.stops.load(Ordering::SeqCst), 1); } #[test] - #[allow(clippy::disallowed_methods)] fn ambiguous_create_timeout_retries_by_adoption_and_discards_late_success() { let stats = Arc::new(PluginStats::default()); let (entered_tx, entered_rx) = mpsc::channel(); @@ -1545,6 +1623,7 @@ mod tests { }), }; let parts = swactor::runtime::RuntimeParts::new(swactor::config::RuntimeConfig::default()); + let runtime = parts.runtime().clone(); let backend = swactor_engine::TokioBackend::new(swactor_engine::TokioConfig::default()).unwrap(); let engine = swactor_engine::Engine::new(parts, backend).unwrap(); @@ -1556,6 +1635,7 @@ mod tests { ..RetryPolicy::default() }, engine.handle(), + runtime, PluginSink::new(Arc::new(NullSink)), ) .unwrap(); diff --git a/apps/myelin/src/orchestration/control.rs b/apps/myelin/src/orchestration/control.rs index 66dcaf5..abaa42c 100644 --- a/apps/myelin/src/orchestration/control.rs +++ b/apps/myelin/src/orchestration/control.rs @@ -1,3 +1,4 @@ +use std::sync::{Arc, Mutex}; use std::time::Duration; use axum::extract::{Path, State}; @@ -6,8 +7,9 @@ use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use axum::{Json, Router}; use serde::Serialize; -use swactor::actor::ActorAddress; -use swactor::runtime::Runtime; +use swactor::actor::{ActorAddress, ActorInterface}; +use swactor::runtime::{Ctx, ExternalSender, Runtime}; +use swactor_engine::EngineHandle; use swactor_vastai::VastClient; use crate::orchestration::actor::OrchestratorMsg; @@ -20,7 +22,38 @@ const CONTROL_REPLY_TIMEOUT: Duration = Duration::from_secs(2); const OFFER_SEARCH_REPLY_MARGIN: Duration = Duration::from_secs(5); const OFFER_SEARCH_REPLY_TIMEOUT: Duration = VastClient::REQUEST_TIMEOUT.saturating_add(OFFER_SEARCH_REPLY_MARGIN); -const CONTROL_REPLY_POLL: Duration = Duration::from_millis(5); +struct ControlReplyObserver { + reply: Arc>>>, + engine: EngineHandle, + sender: ExternalSender, + timeout: Duration, +} + +impl ActorInterface for ControlReplyObserver { + type Incoming = ManualControlReply; + type Response = (); + + fn on_start(&mut self, ctx: &Ctx) { + self.engine.send_after( + self.timeout, + self.sender.clone(), + ctx.self_addr(), + ManualControlReply::TimedOut, + ); + } + + fn handle(&mut self, ctx: &Ctx, reply: Self::Incoming) { + if let Some(response) = self + .reply + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + { + let _ = response.send(reply); + } + ctx.stop_self(); + } +} const PROVISION_PAGE: &str = include_str!("provision_page.html"); const FLEET_CONTROL_SCRIPT: &str = include_str!("fleet_control.js"); pub(crate) const FLEET_CONTROL_SCRIPT_URL: &str = "/assets/myelin-fleet-control.js"; @@ -28,21 +61,29 @@ pub(crate) const FLEET_CONTROL_SCRIPT_URL: &str = "/assets/myelin-fleet-control. #[derive(Clone)] struct ControlHttpState { runtime: Runtime, + engine: EngineHandle, orchestrator: ActorAddress, } -pub(crate) fn plugin(runtime: Runtime, orchestrator: ActorAddress) -> dashboard::DashboardPlugin { +pub(crate) fn plugin( + runtime: Runtime, + engine: EngineHandle, + orchestrator: ActorAddress, +) -> dashboard::DashboardPlugin { let state = ControlHttpState { runtime, + engine, orchestrator, }; let routes = Router::new() .route(FLEET_CONTROL_SCRIPT_URL, get(fleet_control_script)) .route("/api/control/status", get(status)) + .route("/api/control/actors", get(actor_stats)) .route("/api/control/provision", post(provision)) .route("/api/control/kill", post(kill)) .route("/api/control/provider", post(configure_provider)) .route("/api/control/offers", post(search_offers)) + .route("/api/control/flush", post(flush)) .route("/api/control/nodes/{logical_node_id}/kill", post(kill_path)) .with_state(state); dashboard::DashboardPlugin::new(routes).with_page(dashboard::PluginPage::new( @@ -125,6 +166,10 @@ async fn status(State(state): State) -> Response { .await } +async fn actor_stats(State(state): State) -> impl IntoResponse { + Json(state.runtime.stats()) +} + async fn search_offers( State(state): State, Json(request): Json, @@ -135,6 +180,13 @@ async fn search_offers( .await } +async fn flush(State(state): State) -> Response { + request_reply(&state, CONTROL_REPLY_TIMEOUT, |reply_to| { + ManualControlMsg::Flush { reply_to } + }) + .await +} + fn route_mutation(state: &ControlHttpState, msg: ManualControlMsg) -> Response { match state .runtime @@ -156,55 +208,717 @@ async fn request_reply( timeout: Duration, build: impl FnOnce(ActorAddress) -> ManualControlMsg, ) -> Response { - let inbox = match state.runtime.new_inbox::() { - Ok(inbox) => inbox, - Err(error) => { - return ( + let response_rx = match begin_request_reply(state, timeout, build) { + Ok(response_rx) => response_rx, + Err(response) => return response, + }; + + match response_rx.await { + Ok(ManualControlReply::Rejected(error)) => { + (StatusCode::CONFLICT, Json(ErrorResponse { error })).into_response() + } + Ok(ManualControlReply::TimedOut) => ( + StatusCode::GATEWAY_TIMEOUT, + Json(ErrorResponse { + error: "orchestrator control reply timed out".to_owned(), + }), + ) + .into_response(), + Ok(reply) => Json(reply).into_response(), + Err(error) => ( + StatusCode::SERVICE_UNAVAILABLE, + Json(ErrorResponse { + error: format!("control reply observer stopped: {error}"), + }), + ) + .into_response(), + } +} + +fn begin_request_reply( + state: &ControlHttpState, + timeout: Duration, + build: impl FnOnce(ActorAddress) -> ManualControlMsg, +) -> Result, Response> { + let (response_tx, response_rx) = tokio::sync::oneshot::channel(); + let response_tx = Arc::new(Mutex::new(Some(response_tx))); + let reply_to = state + .runtime + .spawn(ControlReplyObserver { + reply: response_tx, + engine: state.engine.clone(), + sender: state.runtime.create_sender(), + timeout, + }) + .map_err(|error| { + ( StatusCode::SERVICE_UNAVAILABLE, Json(ErrorResponse { - error: format!("create control reply inbox: {error}"), + error: format!("create control reply observer: {error}"), }), ) - .into_response(); - } - }; - if let Err(error) = state.runtime.send_to( - state.orchestrator, - OrchestratorMsg::Manual(build(*inbox.addr())), - ) { - return ( + .into_response() + })?; + if let Err(error) = state + .runtime + .send_to(state.orchestrator, OrchestratorMsg::Manual(build(reply_to))) + { + let _ = state.runtime.stop_actor(reply_to); + return Err(( StatusCode::SERVICE_UNAVAILABLE, Json(ErrorResponse { error: format!("orchestrator control actor unavailable: {error}"), }), ) - .into_response(); - } - - let deadline = tokio::time::Instant::now() + timeout; - loop { - if let Some(reply) = inbox.try_recv() { - return match reply { - ManualControlReply::Rejected(error) => { - (StatusCode::CONFLICT, Json(ErrorResponse { error })).into_response() - } - reply => Json(reply).into_response(), - }; - } - if tokio::time::Instant::now() >= deadline { - return ( - StatusCode::GATEWAY_TIMEOUT, - Json(ErrorResponse { - error: "orchestrator control reply timed out".to_owned(), - }), - ) - .into_response(); - } - tokio::time::sleep(CONTROL_REPLY_POLL).await; + .into_response()); } + Ok(response_rx) } #[derive(Serialize)] struct ErrorResponse { error: String, } + +#[cfg(test)] +mod properties { + use proptest::prelude::*; + use swactor::config::RuntimeConfig; + use swactor::runtime::RuntimeParts; + use swactor_engine::{Engine, SteppingBackend}; + + use super::*; + use crate::tests::fuzz_support::{ + actor_census, assert_actor_delta_at_most, assert_mailboxes_drained, assert_no_poison, + drive_steps, + }; + + const STEP_BUDGET: usize = 64; + + fn reply(code: u8) -> ManualControlReply { + match code % 3 { + 0 => ManualControlReply::Flushed, + 1 => ManualControlReply::Rejected(format!("rejected-{code}")), + _ => ManualControlReply::TimedOut, + } + } + + #[derive(Clone, Debug)] + enum HttpAction { + Provision { command_slot: u8, count: u8 }, + Kill { command_slot: u8, node: u8 }, + KillPath { command_slot: u8, node: u8 }, + Configure { value: u8 }, + Status, + ActorStats, + Flush, + SearchOffers, + } + + impl HttpAction { + fn from_raw(kind: u8, command_slot: u8, value: u8) -> Self { + match kind % 8 { + 0 => Self::Provision { + command_slot, + count: value, + }, + 1 => Self::Kill { + command_slot, + node: value, + }, + 2 => Self::KillPath { + command_slot, + node: value, + }, + 3 => Self::Configure { value }, + 4 => Self::Status, + 5 => Self::ActorStats, + 6 => Self::Flush, + _ => Self::SearchOffers, + } + } + + fn command_id(command_slot: u8) -> String { + format!("repeated-{}", command_slot % 4) + } + + fn expected_bridge_observation(&self) -> Option { + match self { + Self::Provision { command_slot, .. } => { + Some(format!("Provision:{}", Self::command_id(*command_slot))) + } + Self::Kill { command_slot, .. } | Self::KillPath { command_slot, .. } => { + Some(format!("Kill:{}", Self::command_id(*command_slot))) + } + Self::Configure { .. } => Some("Configure".to_owned()), + Self::Status => Some("Status".to_owned()), + Self::ActorStats => None, + Self::Flush => Some("Flush".to_owned()), + Self::SearchOffers => Some("SearchOffers".to_owned()), + } + } + + fn is_mutation(&self) -> bool { + matches!( + self, + Self::Provision { .. } + | Self::Kill { .. } + | Self::KillPath { .. } + | Self::Configure { .. } + ) + } + + fn uses_reply_observer(&self) -> bool { + matches!(self, Self::Status | Self::Flush | Self::SearchOffers) + } + } + + #[derive(Clone, Debug)] + struct HttpObservation { + index: usize, + action: HttpAction, + status: StatusCode, + } + + struct HttpBridgeProbe { + observations: Arc>>, + disappear_reply_observers: bool, + } + + impl HttpBridgeProbe { + fn finish_reply(&self, ctx: &Ctx, reply_to: ActorAddress, reply: ManualControlReply) { + if self.disappear_reply_observers { + let _ = ctx.stop_actor(reply_to); + } else { + let _ = ctx.send(reply_to, reply); + } + } + } + + impl ActorInterface for HttpBridgeProbe { + type Incoming = OrchestratorMsg; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, message: Self::Incoming) { + let OrchestratorMsg::Manual(message) = message else { + return; + }; + match message { + ManualControlMsg::Provision { request, .. } => self + .observations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(format!("Provision:{}", request.command_id)), + ManualControlMsg::Kill { request, .. } => self + .observations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(format!("Kill:{}", request.command_id)), + ManualControlMsg::Configure { .. } => self + .observations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push("Configure".to_owned()), + ManualControlMsg::Query { reply_to } => { + self.observations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push("Status".to_owned()); + self.finish_reply(ctx, reply_to, ManualControlReply::Flushed); + } + ManualControlMsg::Flush { reply_to } => { + self.observations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push("Flush".to_owned()); + self.finish_reply(ctx, reply_to, ManualControlReply::Flushed); + } + ManualControlMsg::SearchOffers { reply_to, .. } => { + self.observations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push("SearchOffers".to_owned()); + self.finish_reply(ctx, reply_to, ManualControlReply::Offers(Vec::new())); + } + _ => {} + } + } + } + + enum PendingHttpObservation { + Ready(HttpObservation), + Reply { + index: usize, + action: HttpAction, + receiver: tokio::sync::oneshot::Receiver, + }, + } + + fn begin_http_action( + state: &ControlHttpState, + index: usize, + action: HttpAction, + ) -> PendingHttpObservation { + let immediate = |status| { + PendingHttpObservation::Ready(HttpObservation { + index, + action: action.clone(), + status, + }) + }; + let reply = |result: Result<_, Response>| match result { + Ok(receiver) => PendingHttpObservation::Reply { + index, + action: action.clone(), + receiver, + }, + Err(response) => immediate(response.status()), + }; + + match &action { + HttpAction::Provision { + command_slot, + count, + } => immediate( + route_mutation( + state, + ManualControlMsg::Provision { + request: ProvisionRequest { + command_id: HttpAction::command_id(*command_slot), + count: u32::from(*count), + selected_offer_ids: Vec::new(), + }, + reply_to: None, + }, + ) + .status(), + ), + HttpAction::Kill { command_slot, node } + | HttpAction::KillPath { command_slot, node } => immediate( + route_mutation( + state, + ManualControlMsg::Kill { + request: KillRequest { + command_id: HttpAction::command_id(*command_slot), + logical_node_id: u64::from(*node), + }, + reply_to: None, + }, + ) + .status(), + ), + HttpAction::Configure { value } => immediate( + route_mutation( + state, + ManualControlMsg::Configure { + request: ProviderConfigurationRequest { + api_key: Some(format!("generated-key-{value}")), + ssh_identity: None, + bootstrap_command: None, + }, + reply_to: None, + }, + ) + .status(), + ), + HttpAction::Status => reply(begin_request_reply( + state, + CONTROL_REPLY_TIMEOUT, + |reply_to| ManualControlMsg::Query { reply_to }, + )), + HttpAction::ActorStats => immediate(StatusCode::OK), + HttpAction::Flush => reply(begin_request_reply( + state, + CONTROL_REPLY_TIMEOUT, + |reply_to| ManualControlMsg::Flush { reply_to }, + )), + HttpAction::SearchOffers => reply(begin_request_reply( + state, + OFFER_SEARCH_REPLY_TIMEOUT, + |reply_to| ManualControlMsg::SearchOffers { + request: OfferSearchRequest::default(), + reply_to, + }, + )), + } + } + + fn finish_http_action(observation: PendingHttpObservation) -> Result { + let PendingHttpObservation::Reply { + index, + action, + mut receiver, + } = observation + else { + let PendingHttpObservation::Ready(observation) = observation else { + unreachable!("pending HTTP observation variant changed") + }; + return Ok(observation); + }; + let status = match receiver.try_recv() { + Ok(ManualControlReply::Rejected(_)) => StatusCode::CONFLICT, + Ok(ManualControlReply::TimedOut) => StatusCode::GATEWAY_TIMEOUT, + Ok(_) => StatusCode::OK, + Err(tokio::sync::oneshot::error::TryRecvError::Closed) => { + StatusCode::SERVICE_UNAVAILABLE + } + Err(tokio::sync::oneshot::error::TryRecvError::Empty) => { + return Err(format!( + "control reply remained pending after fixed step budget; \ + index={index}, action={action:?}" + )); + } + }; + Ok(HttpObservation { + index, + action, + status, + }) + } + + fn http_bridge_invariant_failure( + actions: &[HttpAction], + responses: &[HttpObservation], + observed: &[String], + state: &ControlHttpState, + baseline_actors: usize, + disappear_reply_observers: bool, + ) -> Option { + let mut expected_bridge = actions + .iter() + .filter_map(HttpAction::expected_bridge_observation) + .collect::>(); + expected_bridge.sort(); + let mut actual_bridge = observed.to_vec(); + actual_bridge.sort(); + let statuses_valid = responses.iter().all(|response| { + let expected = if response.action.is_mutation() { + StatusCode::ACCEPTED + } else if disappear_reply_observers && response.action.uses_reply_observer() { + StatusCode::SERVICE_UNAVAILABLE + } else { + StatusCode::OK + }; + response.status == expected + }); + let stats = state.runtime.stats(); + let panics = stats + .workers + .iter() + .map(|worker| worker.panics) + .sum::(); + let mailbox_depth = stats + .workers + .iter() + .map(|worker| worker.mailbox_depth) + .sum::() + + stats + .actor_details + .iter() + .map(|actor| actor.mailbox_depth) + .sum::(); + if responses.len() == actions.len() + && statuses_valid + && actual_bridge == expected_bridge + && stats.actors.len() == baseline_actors + && stats.actor_details.iter().all(|actor| !actor.poisoned) + && panics == 0 + && mailbox_depth == 0 + { + None + } else { + Some(format!( + "responses={responses:?}, expected_bridge={expected_bridge:?}, \ + observed_bridge={actual_bridge:?}, expected_actor_count={baseline_actors}, \ + mailbox_depth={mailbox_depth}, actor_census=\n{}", + actor_census(&state.runtime), + )) + } + } + + proptest! { + #![proptest_config(ProptestConfig { + cases: 128, + max_shrink_iters: 2_000, + ..ProptestConfig::default() + })] + + #[test] + fn generated_http_bridge_sequences_terminate_without_control_actor_growth( + raw_actions in prop::collection::vec( + (any::(), any::(), any::()), + 0..=32, + ), + concurrent in any::(), + disappear_reply_observers in any::(), + ) { + let actions = raw_actions + .into_iter() + .map(|(kind, command_slot, value)| { + HttpAction::from_raw(kind, command_slot, value) + }) + .collect::>(); + let mut config = RuntimeConfig::default(); + config.worker_count = 1; + let parts = RuntimeParts::new(config); + let runtime = parts.runtime().clone(); + let backend = SteppingBackend::new(); + let engine = + Engine::new(parts, backend.clone()).expect("control HTTP stepping engine"); + let observations = Arc::new(Mutex::new(Vec::new())); + let orchestrator = runtime + .spawn(HttpBridgeProbe { + observations: Arc::clone(&observations), + disappear_reply_observers, + }) + .expect("spawn control HTTP bridge probe"); + drive_steps(&backend, STEP_BUDGET); + let baseline_actors = runtime.stats().actors.len(); + let baseline_tasks = backend.pending_task_count(); + let state = ControlHttpState { + runtime: runtime.clone(), + engine: engine.handle(), + orchestrator, + }; + let outcome = (|| { + let mut responses = Vec::with_capacity(actions.len()); + if concurrent { + let pending = actions + .iter() + .cloned() + .enumerate() + .map(|(index, action)| begin_http_action(&state, index, action)) + .collect::>(); + drive_steps(&backend, STEP_BUDGET); + drive_steps(&backend, STEP_BUDGET); + for observation in pending { + responses.push(finish_http_action(observation)?); + } + } else { + for (index, action) in actions.iter().cloned().enumerate() { + let pending = begin_http_action(&state, index, action); + drive_steps(&backend, STEP_BUDGET); + drive_steps(&backend, STEP_BUDGET); + responses.push(finish_http_action(pending)?); + } + } + responses.sort_by_key(|response| response.index); + Ok::<_, String>(responses) + })(); + prop_assert!( + outcome.is_ok(), + "control HTTP request did not terminate; actions={:?}; error={:?}; \ + responses=[]; actor_census=\n{}", + actions, + outcome.as_ref().err(), + actor_census(&runtime), + ); + let responses = outcome.expect("outcome checked above"); + drive_steps(&backend, STEP_BUDGET); + let observed = observations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + let failure = http_bridge_invariant_failure( + &actions, + &responses, + &observed, + &state, + baseline_actors, + disappear_reply_observers, + ); + prop_assert!( + failure.is_none(), + "control HTTP/bridge invariant failed; actions={:?}; responses={:?}; failure={}", + actions, + responses, + failure.unwrap_or_default(), + ); + + backend.advance_time(OFFER_SEARCH_REPLY_TIMEOUT); + drive_steps(&backend, STEP_BUDGET); + prop_assert_eq!( + backend.pending_task_count(), + baseline_tasks, + "control reply timers survived their fixed drain budget; actions={:?}; \ + responses={:?}; actor_census=\n{}", + actions, + responses, + actor_census(&runtime), + ); + runtime + .stop_actor(orchestrator) + .expect("stop control HTTP bridge probe"); + drive_steps(&backend, STEP_BUDGET); + assert_no_poison(&runtime); + assert_actor_delta_at_most(&runtime, 0, 0); + assert_mailboxes_drained(&runtime); + } + + #[test] + fn generated_duplicate_control_replies_deliver_first_once_and_remove_observer( + replies in prop::collection::vec(any::(), 0..=16) + ) { + let mut config = RuntimeConfig::default(); + config.worker_count = 1; + let parts = RuntimeParts::new(config); + let runtime = parts.runtime().clone(); + let backend = SteppingBackend::new(); + let engine = + Engine::new(parts, backend.clone()).expect("control reply stepping engine"); + let baseline_actors = runtime.stats().actors.len(); + let baseline_tasks = backend.pending_task_count(); + let (response_tx, mut response_rx) = tokio::sync::oneshot::channel(); + let observer = runtime + .spawn(ControlReplyObserver { + reply: Arc::new(Mutex::new(Some(response_tx))), + engine: engine.handle(), + sender: runtime.create_sender(), + timeout: Duration::from_millis(1), + }) + .expect("spawn control reply observer"); + for code in &replies { + runtime + .send_to(observer, reply(*code)) + .expect("send generated control reply"); + } + drive_steps(&backend, 32); + if replies.is_empty() { + backend.advance_time(Duration::from_millis(1)); + drive_steps(&backend, 32); + } + + let observed = response_rx + .try_recv() + .expect("control reply observer produced a terminal reply"); + let expected = replies + .first() + .map(|code| reply(*code)) + .unwrap_or(ManualControlReply::TimedOut); + prop_assert_eq!( + observed, + expected, + "control reply observer accepted a stale duplicate; replies={:?}; \ + actor_census=\n{}", + replies, + actor_census(&runtime), + ); + + backend.advance_time(Duration::from_millis(1)); + drive_steps(&backend, 32); + prop_assert_eq!(backend.pending_task_count(), baseline_tasks); + assert_no_poison(&runtime); + assert_actor_delta_at_most(&runtime, baseline_actors, 0); + assert_mailboxes_drained(&runtime); + } + } + + #[test] + fn disappeared_orchestrator_removes_control_reply_actor() { + let parts = RuntimeParts::new(RuntimeConfig::default()); + let runtime = parts.runtime().clone(); + let backend = SteppingBackend::new(); + let engine = Engine::new(parts, backend.clone()).expect("control failure stepping engine"); + let baseline_tasks = backend.pending_task_count(); + let state = ControlHttpState { + runtime: runtime.clone(), + engine: engine.handle(), + orchestrator: ActorAddress::default(), + }; + let response = match begin_request_reply(&state, Duration::from_millis(1), |reply_to| { + ManualControlMsg::Query { reply_to } + }) { + Ok(_) => panic!("missing orchestrator unexpectedly accepted a control query"), + Err(response) => response, + }; + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + drive_steps(&backend, STEP_BUDGET); + backend.advance_time(Duration::from_millis(1)); + drive_steps(&backend, STEP_BUDGET); + assert_eq!(backend.pending_task_count(), baseline_tasks); + assert_no_poison(&runtime); + assert_actor_delta_at_most(&runtime, 0, 0); + assert_mailboxes_drained(&runtime); + } + + #[test] + fn reply_observer_disappearance_returns_a_bounded_terminal_http_response() { + let mut config = RuntimeConfig::default(); + config.worker_count = 1; + let parts = RuntimeParts::new(config); + let runtime = parts.runtime().clone(); + let backend = SteppingBackend::new(); + let engine = + Engine::new(parts, backend.clone()).expect("reply disappearance stepping engine"); + let observations = Arc::new(Mutex::new(Vec::new())); + let orchestrator = runtime + .spawn(HttpBridgeProbe { + observations, + disappear_reply_observers: true, + }) + .expect("spawn disappearing-reply bridge"); + drive_steps(&backend, STEP_BUDGET); + let baseline_actors = runtime.stats().actors.len(); + let baseline_tasks = backend.pending_task_count(); + let state = ControlHttpState { + runtime: runtime.clone(), + engine: engine.handle(), + orchestrator, + }; + let pending = begin_http_action(&state, 0, HttpAction::Status); + drive_steps(&backend, STEP_BUDGET); + drive_steps(&backend, STEP_BUDGET); + let response = finish_http_action(pending).unwrap_or_else(|error| { + panic!( + "{error}; responses=[]; actor_census=\n{}", + actor_census(&runtime), + ) + }); + assert_eq!( + response.status, + StatusCode::SERVICE_UNAVAILABLE, + "reply observer disappearance returned {}; actor_census=\n{}", + response.status, + actor_census(&runtime), + ); + drive_steps(&backend, STEP_BUDGET); + assert_actor_delta_at_most(&runtime, baseline_actors, 0); + backend.advance_time(CONTROL_REPLY_TIMEOUT); + drive_steps(&backend, STEP_BUDGET); + assert_eq!(backend.pending_task_count(), baseline_tasks); + runtime + .stop_actor(orchestrator) + .expect("stop disappearing-reply bridge"); + drive_steps(&backend, STEP_BUDGET); + assert_no_poison(&runtime); + assert_actor_delta_at_most(&runtime, 0, 0); + assert_mailboxes_drained(&runtime); + } + + #[test] + fn http_bridge_invariant_rejects_a_controlled_duplicate_forward() { + let parts = RuntimeParts::new(RuntimeConfig::default()); + let runtime = parts.runtime().clone(); + let backend = SteppingBackend::new(); + let engine = + Engine::new(parts, backend.clone()).expect("control invariant stepping engine"); + let state = ControlHttpState { + runtime, + engine: engine.handle(), + orchestrator: ActorAddress::default(), + }; + let actions = vec![HttpAction::Status]; + let responses = vec![HttpObservation { + index: 0, + action: HttpAction::Status, + status: StatusCode::OK, + }]; + let duplicated = vec!["Status".to_owned(), "Status".to_owned()]; + + assert!( + http_bridge_invariant_failure(&actions, &responses, &duplicated, &state, 0, false,) + .is_some(), + "control HTTP/bridge invariant accepted a controlled duplicate forward" + ); + } +} diff --git a/apps/myelin/src/orchestration/distribution_stack.rs b/apps/myelin/src/orchestration/distribution_stack.rs index 0020115..284e073 100644 --- a/apps/myelin/src/orchestration/distribution_stack.rs +++ b/apps/myelin/src/orchestration/distribution_stack.rs @@ -13,7 +13,7 @@ use std::time::{Duration, Instant}; use swactor::actor::{ActorAddress, ActorInterface}; use swactor::config::RuntimeConfig; -use swactor::runtime::{Ctx, Runtime, RuntimeParts}; +use swactor::runtime::{Ctx, ExternalSender, Runtime, RuntimeParts}; use swactor::stats::StatsHook; use swactor::std::StdExtension; use swactor_engine::EngineHandle; @@ -37,6 +37,49 @@ use distribution::transport_bridge::{ }; use distribution::types::{DirectoryEntry, MemberState, NodeId}; +#[derive(Clone)] +struct ProtocolTick; + +struct ProtocolTicker { + runtime: Runtime, + engine: EngineHandle, + sender: ExternalSender, + period: Duration, + swim: ActorAddress, + registry: ActorAddress, + metadata: ActorAddress, + directory: ActorAddress, +} + +impl ProtocolTicker { + fn schedule(&self, ctx: &Ctx) { + self.engine.send_after( + self.period, + self.sender.clone(), + ctx.self_addr(), + ProtocolTick, + ); + } +} + +impl ActorInterface for ProtocolTicker { + type Incoming = ProtocolTick; + type Response = (); + + fn on_start(&mut self, ctx: &Ctx) { + self.schedule(ctx); + } + + fn handle(&mut self, ctx: &Ctx, _message: Self::Incoming) { + let now = self.engine.now().to_instant(); + let _ = self.runtime.send_to(self.swim, SwimIn::Tick { now }); + let _ = self.runtime.send_to(self.registry, RegistryIn::Tick); + let _ = self.runtime.send_to(self.metadata, MetadataIn::Tick); + let _ = self.runtime.send_to(self.directory, DirectoryIn::Tick); + self.schedule(ctx); + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) struct DistributionActorAddrs { pub swim: ActorAddress, @@ -223,29 +266,20 @@ impl DistributionRuntimeStack { routes } - /// Spawn an engine-hosted interval task that injects protocol Tick messages - /// (SWIM, registry, metadata, directory), replacing the manual tick - /// injection previously done by the application pump loop - /// (ENGINE_SPEC.md). The engine owns protocol progression; the - /// application loop no longer calls tick or core-driving methods. + /// Spawn the actor that owns periodic distribution protocol ticks. pub(crate) fn spawn_protocol_ticker(&self, period: Duration) { - let runtime = self.runtime.clone(); - let swim = self.actors.swim; - let registry = self.actors.registry; - let metadata = self.actors.metadata; - let directory = self.actors.directory; - let engine = self.engine.clone(); - engine.clone().spawn(async move { - let mut interval = engine.interval(period); - loop { - (&mut interval).await; - let now = engine.now().to_instant(); - let _ = runtime.send_to(swim, SwimIn::Tick { now }); - let _ = runtime.send_to(registry, RegistryIn::Tick); - let _ = runtime.send_to(metadata, MetadataIn::Tick); - let _ = runtime.send_to(directory, DirectoryIn::Tick); - } - }); + self.runtime + .spawn(ProtocolTicker { + runtime: self.runtime.clone(), + engine: self.engine.clone(), + sender: self.runtime.create_sender(), + period, + swim: self.actors.swim, + registry: self.actors.registry, + metadata: self.actors.metadata, + directory: self.actors.directory, + }) + .expect("spawn distribution protocol ticker actor"); } pub(crate) fn register_local_actor(&self, entry: DirectoryEntry) { diff --git a/apps/myelin/src/orchestration/job_reconciler.rs b/apps/myelin/src/orchestration/job_reconciler.rs index 2b8d86b..399d309 100644 --- a/apps/myelin/src/orchestration/job_reconciler.rs +++ b/apps/myelin/src/orchestration/job_reconciler.rs @@ -14,10 +14,13 @@ use provisioning::{ BootSpec, ClusterShape, DesiredNodeShape, LogicalNodeId, NodeAttemptId, NodeGroupId, ProviderKind, RetryPolicy, RoleId, RunId, RunNodeGroupSpec, SwactorId, SwarmJoinTemplate, }; +use swactor::actor::ActorInterface; +use swactor::runtime::{Ctx, ExternalSender}; +use swactor_engine::{ActorCompletion, EngineHandle}; use swactor_job_runner::{Job, JobDone}; use swactor_vastai::SelectionPolicy; -use crate::job_deploy::{self, NodeIdentity}; +use crate::job_deploy::{self, JobRunStateMachine, NodeIdentity}; use crate::orchestration::app::{ derive_ssh_public_key, ensure_vastai_account_ssh_key, resolve_vastai_ssh_identity, ssh_public_key_fingerprint, @@ -124,7 +127,7 @@ pub(crate) fn run_vastai_job( validate_non_empty("worker workdir", &options.worker_workdir)?; validate_non_empty("endpoint address mask", &options.endpoint_addr_mask)?; - let mut session = job_deploy::start_orchestrator(landing)?; + let session = job_deploy::start_orchestrator(landing)?; let orch_json = session.identity_json()?; println!("JOB_ORCH_IDENTITY {orch_json}"); let _ = std::io::Write::flush(&mut std::io::stdout()); @@ -134,102 +137,235 @@ pub(crate) fn run_vastai_job( ); let (sink, observations) = observation_channel(); - let provisioner = build_vastai_provisioner(&api_key, &options, session.runtime())?; + let runtime = session.runtime(); + let engine = session.engine_handle(); + let provisioner = + build_vastai_provisioner(&api_key, &options, runtime.clone(), engine.clone())?; let spec = job_node_spec(&options, image, &orch_json)?; - let mut cluster = build_cluster(&options, spec, provisioner, session.engine_handle(), sink)?; + let cluster = build_cluster( + &options, + spec, + provisioner, + engine.clone(), + runtime.clone(), + sink, + )?; + let completion = ActorCompletion::new(); + runtime + .spawn(ReconciledJobActor { + cluster, + observations, + session: Some(session), + job: Some(job), + phase: ReconciledJobPhase::Provisioning { + deadline: Instant::now() + options.provision_timeout, + }, + node_id: options.node_id, + engine, + sender: runtime.create_sender(), + completion: completion.clone(), + }) + .map_err(|error| format!("spawn reconciled job actor: {error}"))?; + completion.wait() +} - let result = run_with_cluster(job, &mut session, &mut cluster, &observations, &options); - let stop_result = cluster.stop(); - match (result, stop_result) { - (Ok(done), Ok(())) => Ok(done), - (Ok(_), Err(cleanup)) => Err(format!( +#[derive(Clone)] +struct ReconciledJobTick; + +enum ReconciledJobPhase { + Provisioning { + deadline: Instant, + }, + Converging { + deadline: Instant, + worker: NodeIdentity, + }, + Running(JobRunStateMachine), + Stopping { + result: Result, + }, + Finished, +} + +struct ReconciledJobActor { + cluster: ProvisionedClusterGuard, + observations: mpsc::Receiver, + session: Option, + job: Option, + phase: ReconciledJobPhase, + node_id: u64, + engine: EngineHandle, + sender: ExternalSender, + completion: ActorCompletion>, +} + +impl ReconciledJobActor { + fn schedule(&self, ctx: &Ctx) { + self.engine.send_after( + POLL, + self.sender.clone(), + ctx.self_addr(), + ReconciledJobTick, + ); + } + + fn begin_stop(&mut self, result: Result) { + let result = match self.cluster.begin_shutdown() { + Ok(()) => result, + Err(cleanup) => merge_cleanup(result, cleanup), + }; + self.phase = ReconciledJobPhase::Stopping { result }; + } + + fn complete( + &mut self, + ctx: &Ctx, + result: Result, + cleanup: Result<(), String>, + ) { + let result = match cleanup { + Ok(()) => result, + Err(cleanup) => merge_cleanup(result, cleanup), + }; + assert!( + self.completion.complete(result).is_ok(), + "reconciled job completed twice" + ); + self.phase = ReconciledJobPhase::Finished; + ctx.stop_self(); + } +} + +impl ActorInterface for ReconciledJobActor { + type Incoming = ReconciledJobTick; + type Response = (); + + fn on_start(&mut self, ctx: &Ctx) { + let _ = ctx.send(ctx.self_addr(), ReconciledJobTick); + } + + fn handle(&mut self, ctx: &Ctx, _message: Self::Incoming) { + if let Err(error) = self.cluster.poll(SystemTime::now()) { + let cleanup = self.cluster.finish_shutdown(); + self.complete(ctx, Err(format!("job reconciler poll: {error}")), cleanup); + return; + } + + let phase = std::mem::replace(&mut self.phase, ReconciledJobPhase::Finished); + match phase { + ReconciledJobPhase::Provisioning { deadline } => { + let mut worker = None; + if let Err(error) = drain_observations(&self.observations, &mut worker) { + self.begin_stop(Err(error)); + } else if let Some(worker) = worker { + eprintln!("job-reconcile: worker identity observed through reconciler stdout"); + let result = job_deploy::parse_actor(&worker.actor_hex).and_then(|actor| { + let attempt = self.cluster.current_attempt(self.node_id).ok_or_else(|| { + format!( + "reconciler has no active attempt for node {}", + self.node_id + ) + })?; + self.cluster + .observe_runtime_ready( + self.node_id, + NodeAttemptId(attempt.0), + SwactorId(format!("{actor:?}")), + SystemTime::now(), + ) + .then_some(()) + .ok_or_else(|| { + format!( + "reconciler rejected runtime-ready observation for node {} attempt {}", + self.node_id, attempt.0 + ) + }) + }); + match result { + Ok(()) => { + self.phase = ReconciledJobPhase::Converging { + deadline: Instant::now() + RUNTIME_CONVERGENCE_TIMEOUT, + worker, + }; + } + Err(error) => self.begin_stop(Err(error)), + } + } else if Instant::now() >= deadline { + self.begin_stop(Err(format!( + "timed out waiting for reconciled job worker identity" + ))); + } else { + self.phase = ReconciledJobPhase::Provisioning { deadline }; + } + } + ReconciledJobPhase::Converging { deadline, worker } => { + let mut ignored = None; + if let Err(error) = drain_observations(&self.observations, &mut ignored) { + self.begin_stop(Err(error)); + } else if self.cluster.is_converged() { + eprintln!("job-reconcile: reconciler accepted runtime-ready worker"); + let machine = self + .session + .take() + .zip(self.job.take()) + .ok_or_else(|| "reconciled job lost session state".to_owned()) + .and_then(|(session, job)| JobRunStateMachine::new(session, job, worker)); + match machine { + Ok(mut machine) => { + machine.start(Instant::now()); + self.phase = ReconciledJobPhase::Running(machine); + } + Err(error) => self.begin_stop(Err(error)), + } + } else if Instant::now() >= deadline { + self.begin_stop(Err(format!( + "timed out after {RUNTIME_CONVERGENCE_TIMEOUT:?} waiting for reconciler convergence" + ))); + } else { + self.phase = ReconciledJobPhase::Converging { deadline, worker }; + } + } + ReconciledJobPhase::Running(mut machine) => { + let mut ignored = None; + if let Err(error) = drain_observations(&self.observations, &mut ignored) { + self.begin_stop(Err(error)); + } else if let Some(result) = machine.advance(Instant::now()) { + self.begin_stop(result); + } else { + self.phase = ReconciledJobPhase::Running(machine); + } + } + ReconciledJobPhase::Stopping { result } => { + if self.cluster.is_stopped() { + let cleanup = self.cluster.finish_shutdown(); + self.complete(ctx, result, cleanup); + return; + } + self.phase = ReconciledJobPhase::Stopping { result }; + } + ReconciledJobPhase::Finished => { + ctx.stop_self(); + return; + } + } + self.schedule(ctx); + } +} + +fn merge_cleanup(result: Result, cleanup: String) -> Result { + match result { + Ok(_) => Err(format!( "job completed but reconciler cleanup failed: {cleanup}" )), - (Err(error), Ok(())) => Err(error), - (Err(error), Err(cleanup)) => Err(format!("{error}; reconciler cleanup failed: {cleanup}")), + Err(error) => Err(format!("{error}; reconciler cleanup failed: {cleanup}")), } } -fn run_with_cluster( - job: Job, - session: &mut job_deploy::JobOrchestratorSession, - cluster: &mut ProvisionedClusterGuard, - observations: &mpsc::Receiver, - options: &VastAiJobOptions, -) -> Result { - let worker = wait_for_worker_identity(cluster, observations, options)?; - let actor = job_deploy::parse_actor(&worker.actor_hex)?; - let attempt = cluster.current_attempt(options.node_id).ok_or_else(|| { - format!( - "reconciler has no active attempt for node {}", - options.node_id - ) - })?; - if !cluster.observe_runtime_ready( - options.node_id, - NodeAttemptId(attempt.0), - SwactorId(format!("{actor:?}")), - SystemTime::now(), - ) { - return Err(format!( - "reconciler rejected runtime-ready observation for node {} attempt {}", - options.node_id, attempt.0 - )); - } - wait_for_cluster_convergence(cluster, observations)?; - session.run_to_completion(job, worker) -} - -fn wait_for_worker_identity( - cluster: &mut ProvisionedClusterGuard, - observations: &mpsc::Receiver, - options: &VastAiJobOptions, -) -> Result { - let started = Instant::now(); - let mut worker = None; - while started.elapsed() < options.provision_timeout { - cluster - .poll(SystemTime::now()) - .map_err(|error| format!("job reconciler poll: {error}"))?; - drain_observations(observations, &mut worker)?; - if let Some(worker) = worker.take() { - eprintln!("job-reconcile: worker identity observed through reconciler stdout"); - return Ok(worker); - } - std::thread::sleep(POLL); - } - Err(format!( - "timed out after {:?} waiting for reconciled job worker identity", - options.provision_timeout - )) -} - -fn wait_for_cluster_convergence( - cluster: &mut ProvisionedClusterGuard, - observations: &mpsc::Receiver, -) -> Result<(), String> { - let started = Instant::now(); - let mut ignored = None; - while started.elapsed() < RUNTIME_CONVERGENCE_TIMEOUT { - cluster - .poll(SystemTime::now()) - .map_err(|error| format!("job reconciler convergence poll: {error}"))?; - drain_observations(observations, &mut ignored)?; - if cluster.is_converged() { - eprintln!("job-reconcile: reconciler accepted runtime-ready worker"); - return Ok(()); - } - std::thread::sleep(POLL); - } - Err(format!( - "timed out after {RUNTIME_CONVERGENCE_TIMEOUT:?} waiting for reconciler convergence" - )) -} - fn build_vastai_provisioner( api_key: &str, options: &VastAiJobOptions, runtime: swactor::runtime::Runtime, + engine: swactor_engine::EngineHandle, ) -> Result, String> { let identity = resolve_vastai_ssh_identity(options.ssh_identity.clone())?; if !identity.is_file() { @@ -259,8 +395,9 @@ fn build_vastai_provisioner( } Ok(Box::new(VastAiProvisioningPlugin::new( - ToolsVastAiLeaseClient::from_api_key(api_key.to_owned())?, - SshCommandBootstrapLauncher::new(Some(identity), runtime), + ToolsVastAiLeaseClient::from_api_key(api_key.to_owned())? + .with_actor_host(runtime.clone(), engine.clone()), + SshCommandBootstrapLauncher::new(Some(identity), runtime, engine), config, ))) } @@ -320,6 +457,7 @@ fn build_cluster( spec: NodeProvisionSpec, provisioner: Box, engine: swactor_engine::EngineHandle, + runtime: swactor::runtime::Runtime, sink: PluginSink, ) -> Result { let group_id = NodeGroupId(format!("job-node-{}", spec.node_id)); @@ -375,6 +513,7 @@ fn build_cluster( }], retry, engine, + runtime, sink, ) } diff --git a/apps/myelin/src/orchestration/manual_control.rs b/apps/myelin/src/orchestration/manual_control.rs index cefbdbd..f9d73f6 100644 --- a/apps/myelin/src/orchestration/manual_control.rs +++ b/apps/myelin/src/orchestration/manual_control.rs @@ -8,9 +8,9 @@ use std::sync::Arc; use parking_lot::Mutex; use serde::{Deserialize, Serialize}; -use swactor::actor::ActorAddress; +use swactor::actor::{ActorAddress, ActorInterface}; use swactor::runtime::{Ctx, ExternalSender, Runtime}; -use swactor_engine::EngineHandle; +use swactor_engine::BlockingWorkSender; use swactor_transport::{CodecRegistry, JsonCodec, NetworkMessage}; use swactor_vastai::OfferBrowseCriteria; @@ -610,11 +610,12 @@ impl ManualControl { kind: EffectKind, result: Result, ) -> Result<(), String> { - if self.in_flight.remove(&node_id) != Some(kind) { + if self.in_flight.get(&node_id).copied() != Some(kind) { return Err(format!( "node {node_id} completed {kind:?} without matching in-flight effect" )); } + self.in_flight.remove(&node_id); match kind { EffectKind::Create => self.created(node_id, result), EffectKind::StartBootstrap => self.bootstrap_started(node_id, result), @@ -755,6 +756,7 @@ impl ManualControl { node.phase = NodePhase::Running; node.last_error = None; node.last_seen_unix_ms = unix_ms_now(); + self.succeed_commands(hello.logical_node_id, CommandKind::Provision); let binding = RejoinBinding { orchestrator_actor, control_generation, @@ -1234,7 +1236,6 @@ impl ManualControl { node.last_error = Some(error.clone()); } } - self.in_flight.clear(); } } @@ -1267,15 +1268,15 @@ pub(crate) enum ManualControlMsg { reply_to: Option, }, ProviderValidated { + work_id: u64, error: Option, - reply_to: Option, }, SearchOffers { request: OfferSearchRequest, reply_to: ActorAddress, }, OfferSearchFinished { - reply_to: ActorAddress, + work_id: u64, result: Result, String>, }, Query { @@ -1291,6 +1292,7 @@ pub(crate) enum ManualControlMsg { EffectFinished { node_id: u64, kind: EffectKind, + effect_id: u64, outcome: Option, error: Option, }, @@ -1316,6 +1318,8 @@ pub(crate) enum ManualControlReply { Rejoined(RejoinBinding), Flushed, Rejected(String), + #[serde(skip)] + TimedOut, } pub(crate) type ProviderFactory = @@ -1366,12 +1370,57 @@ struct NodeLane { type SharedLane = Arc>>; -/// Engine-backed adapter owned by the orchestrator actor. All methods called -/// from actor handlers are transition-only; filesystem and provider work is -/// scheduled on the engine and reports back as `ManualControlMsg`. +type ManualActorWork = Box; +type ManualWorkFailure = Box; + +struct ManualWork { + work: Arc>>, + failure: Arc>>, +} + +impl Clone for ManualWork { + fn clone(&self) -> Self { + Self { + work: Arc::clone(&self.work), + failure: Arc::clone(&self.failure), + } + } +} + +struct ManualWorkActor { + blocking_work: BlockingWorkSender, +} + +impl ActorInterface for ManualWorkActor { + type Incoming = ManualWork; + type Response = (); + + fn handle(&mut self, _ctx: &Ctx, message: Self::Incoming) { + let Some(work) = message.work.lock().take() else { + return; + }; + let failure = Arc::clone(&message.failure); + let failure_after_panic = Arc::clone(&failure); + let guarded_work = Box::new(move || { + if std::panic::catch_unwind(std::panic::AssertUnwindSafe(work)).is_err() + && let Some(report) = failure_after_panic.lock().take() + { + report("manual control work panicked".to_owned()); + } + }); + if self.blocking_work.submit(guarded_work).is_err() + && let Some(report) = failure.lock().take() + { + report("manual control work backend is unavailable".to_owned()); + } + } +} + +/// Actor-backed adapter owned by the orchestrator actor. Filesystem and provider +/// work is delivered to a dedicated actor and reports back as `ManualControlMsg`. pub(crate) struct ManualActorControl { core: ManualControl, - engine: EngineHandle, + work_actor: ActorAddress, runtime: Runtime, sender: ExternalSender, state_dir: StateDir, @@ -1381,10 +1430,16 @@ pub(crate) struct ManualActorControl { config_validator: Option, offer_searcher: Option, lanes: BTreeMap, + active_effect_ids: BTreeMap, persistence_queue: VecDeque<(u64, ClusterSnapshot)>, - persistence_in_flight: bool, - pending_command_replies: BTreeMap, + persistence_in_flight: Option, + flush_failure: Option, flush_waiters: Vec, + pending_command_replies: BTreeMap, + pending_validations: BTreeMap>, + latest_validation_id: Option, + pending_offer_searches: BTreeMap, + next_work_id: u64, control_generation: u64, } @@ -1392,8 +1447,8 @@ impl ManualActorControl { #[allow(clippy::too_many_arguments)] pub(crate) fn new( core: ManualControl, - engine: EngineHandle, runtime: Runtime, + blocking_work: BlockingWorkSender, state_dir: StateDir, sink: PluginSink, provider_factory: ProviderFactory, @@ -1403,9 +1458,12 @@ impl ManualActorControl { control_generation: u64, ) -> Self { let sender = runtime.create_sender(); + let work_actor = runtime + .spawn(ManualWorkActor { blocking_work }) + .expect("spawn manual control work actor"); Self { core, - engine, + work_actor, runtime, sender, state_dir, @@ -1415,10 +1473,16 @@ impl ManualActorControl { config_validator, offer_searcher, lanes: BTreeMap::new(), + active_effect_ids: BTreeMap::new(), persistence_queue: VecDeque::new(), - persistence_in_flight: false, - pending_command_replies: BTreeMap::new(), + persistence_in_flight: None, + flush_failure: None, flush_waiters: Vec::new(), + pending_command_replies: BTreeMap::new(), + pending_validations: BTreeMap::new(), + latest_validation_id: None, + pending_offer_searches: BTreeMap::new(), + next_work_id: 0, control_generation, } } @@ -1432,6 +1496,15 @@ impl ManualActorControl { self.dispatch_actions(actor); } + fn allocate_work_id(&mut self) -> u64 { + let work_id = self.next_work_id; + self.next_work_id = self + .next_work_id + .checked_add(1) + .expect("manual control work id exhausted"); + work_id + } + pub(crate) fn handle(&mut self, ctx: &Ctx, msg: ManualControlMsg) { match msg { ManualControlMsg::Provision { request, reply_to } => { @@ -1482,22 +1555,54 @@ impl ManualActorControl { } return; }; + let work_id = self.allocate_work_id(); + self.latest_validation_id = Some(work_id); + self.pending_validations.insert(work_id, reply_to); let sender = self.sender.clone(); + let failed_sender = self.sender.clone(); let actor = ctx.self_addr(); - self.engine.spawn_blocking(move || { - let error = validator(request).err(); - let _ = sender.send_to( - actor, - OrchestratorMsg::Manual(ManualControlMsg::ProviderValidated { - error, - reply_to, - }), - ); - }); + self.spawn_work( + move || { + let error = validator(request).err(); + let _ = sender.send_to( + actor, + OrchestratorMsg::Manual(ManualControlMsg::ProviderValidated { + work_id, + error, + }), + ); + }, + move |error| { + let _ = failed_sender.send_to( + actor, + OrchestratorMsg::Manual(ManualControlMsg::ProviderValidated { + work_id, + error: Some(error), + }), + ); + }, + ); } - ManualControlMsg::ProviderValidated { error, reply_to } => { + ManualControlMsg::ProviderValidated { work_id, error } => { + let Some(reply_to) = self.pending_validations.remove(&work_id) else { + return; + }; + if self.latest_validation_id != Some(work_id) { + if let Some(reply_to) = reply_to { + let _ = ctx.send( + reply_to, + ManualControlReply::Rejected( + "provider validation was superseded".to_owned(), + ), + ); + } + return; + } + self.latest_validation_id = None; self.core.set_provider_validation(error.map_or(Ok(()), Err)); - if self.core.provider().kind == ProviderReadinessKind::Ready { + if self.core.provider().kind == ProviderReadinessKind::Ready + && self.lanes.is_empty() + { self.core.begin_recovery(); } if let Some(reply_to) = reply_to { @@ -1522,18 +1627,32 @@ impl ManualActorControl { ), ); } else if let Some(searcher) = self.offer_searcher.clone() { + let work_id = self.allocate_work_id(); + self.pending_offer_searches.insert(work_id, reply_to); let sender = self.sender.clone(); + let failed_sender = self.sender.clone(); let actor = ctx.self_addr(); - self.engine.spawn_blocking(move || { - let result = searcher(request); - let _ = sender.send_to( - actor, - OrchestratorMsg::Manual(ManualControlMsg::OfferSearchFinished { - reply_to, - result, - }), - ); - }); + self.spawn_work( + move || { + let result = searcher(request); + let _ = sender.send_to( + actor, + OrchestratorMsg::Manual(ManualControlMsg::OfferSearchFinished { + work_id, + result, + }), + ); + }, + move |error| { + let _ = failed_sender.send_to( + actor, + OrchestratorMsg::Manual(ManualControlMsg::OfferSearchFinished { + work_id, + result: Err(error), + }), + ); + }, + ); } else { let _ = ctx.send( reply_to, @@ -1543,7 +1662,10 @@ impl ManualActorControl { ); } } - ManualControlMsg::OfferSearchFinished { reply_to, result } => { + ManualControlMsg::OfferSearchFinished { work_id, result } => { + let Some(reply_to) = self.pending_offer_searches.remove(&work_id) else { + return; + }; let reply = match result { Ok(offers) => ManualControlReply::Offers(offers), Err(error) => ManualControlReply::Rejected(error), @@ -1554,43 +1676,77 @@ impl ManualActorControl { let _ = ctx.send(reply_to, ManualControlReply::Status(self.core.read_model())); } ManualControlMsg::PersistenceFinished { generation, error } => { - self.persistence_in_flight = false; - let persisted = self - .core - .persisted(generation, error.clone().map_or(Ok(()), Err)); + if self.persistence_in_flight != Some(generation) { + return; + } + self.persistence_in_flight = None; + let persistence_error = error + .as_ref() + .map(|error| format!("persist control state: {error}")); + let persisted = self.core.persisted(generation, error.map_or(Ok(()), Err)); + if let Some(error) = persistence_error.as_ref() { + self.flush_failure = Some(error.clone()); + self.persistence_queue.clear(); + } if let Some((reply_to, command_id)) = self.pending_command_replies.remove(&generation) { - let reply = match persisted { - Ok(()) => self - .core - .snapshot() - .commands - .get(&command_id) - .cloned() - .map(ManualControlReply::Accepted) - .unwrap_or_else(|| { - ManualControlReply::Rejected(format!( - "persisted command {command_id} is absent" - )) - }), - Err(error) => ManualControlReply::Rejected(error), + let reply = if let Some(error) = persistence_error.as_ref() { + ManualControlReply::Rejected(error.clone()) + } else { + match persisted { + Ok(()) => self + .core + .snapshot() + .commands + .get(&command_id) + .cloned() + .map(ManualControlReply::Accepted) + .unwrap_or_else(|| { + ManualControlReply::Rejected(format!( + "persisted command {command_id} is absent" + )) + }), + Err(error) => ManualControlReply::Rejected(error), + } }; let _ = ctx.send(reply_to, reply); } + if let Some(error) = persistence_error { + for (_, (reply_to, _)) in std::mem::take(&mut self.pending_command_replies) { + let _ = ctx.send(reply_to, ManualControlReply::Rejected(error.clone())); + } + } } ManualControlMsg::EffectFinished { node_id, kind, + effect_id, outcome, error, } => { + if self.active_effect_ids.get(&node_id).copied() != Some(effect_id) { + return; + } + self.active_effect_ids.remove(&node_id); let result = match (outcome, error) { (Some(outcome), None) => Ok(outcome), (_, Some(error)) => Err(error), (None, None) => Err("provider effect returned no outcome".to_owned()), }; + let releases_lane = (kind == EffectKind::Stop && result.is_ok()) + || (kind == EffectKind::Create && result.is_err()) + || (kind == EffectKind::Recover + && !matches!( + &result, + Ok(EffectOutcome::Recovered { + provider_ref: Some(_) + }) + )); let _ = self.core.effect_finished(node_id, kind, result); + if releases_lane { + self.lanes.remove(&node_id); + } } ManualControlMsg::JoinBarrierSatisfied { node_id } => { let _ = self.core.join_barrier_satisfied(node_id); @@ -1766,58 +1922,120 @@ impl ManualActorControl { } fn start_next_persistence(&mut self, actor: ActorAddress) { - if self.persistence_in_flight { + if self.persistence_in_flight.is_some() { return; } let Some((generation, snapshot)) = self.persistence_queue.pop_front() else { return; }; - self.persistence_in_flight = true; + self.persistence_in_flight = Some(generation); let state_dir = self.state_dir.clone(); let sender = self.sender.clone(); - self.engine.spawn_blocking(move || { - let error = state_dir.save_snapshot(&snapshot).err(); - let _ = sender.send_to( - actor, - OrchestratorMsg::Manual(ManualControlMsg::PersistenceFinished { - generation, - error, - }), - ); - }); + let failed_sender = self.sender.clone(); + self.spawn_work( + move || { + let error = state_dir.save_snapshot(&snapshot).err(); + let _ = sender.send_to( + actor, + OrchestratorMsg::Manual(ManualControlMsg::PersistenceFinished { + generation, + error, + }), + ); + }, + move |error| { + let _ = failed_sender.send_to( + actor, + OrchestratorMsg::Manual(ManualControlMsg::PersistenceFinished { + generation, + error: Some(error), + }), + ); + }, + ); } fn finish_flush_waiters(&mut self, ctx: &Ctx) { - if self.persistence_in_flight + if self.persistence_in_flight.is_some() || !self.persistence_queue.is_empty() || !self.core.persistence_idle() { return; } + if self.flush_waiters.is_empty() { + return; + } + let failure = self.flush_failure.take(); for reply_to in self.flush_waiters.drain(..) { - let _ = ctx.send(reply_to, ManualControlReply::Flushed); + let reply = failure + .as_ref() + .map_or(ManualControlReply::Flushed, |error| { + ManualControlReply::Rejected(error.clone()) + }); + let _ = ctx.send(reply_to, reply); } } - fn spawn_effect(&self, actor: ActorAddress, node_id: u64, kind: EffectKind, work: F) + fn spawn_effect(&mut self, actor: ActorAddress, node_id: u64, kind: EffectKind, work: F) where F: FnOnce() -> Result + Send + 'static, { + let effect_id = self.allocate_work_id(); + assert!( + self.active_effect_ids.insert(node_id, effect_id).is_none(), + "manual control dispatched overlapping effects for node {node_id}" + ); let sender = self.sender.clone(); - self.engine.spawn_blocking(move || { - let (outcome, error) = match work() { - Ok(outcome) => (Some(outcome), None), - Err(error) => (None, Some(error)), - }; - let _ = sender.send_to( - actor, - OrchestratorMsg::Manual(ManualControlMsg::EffectFinished { - node_id, - kind, - outcome, - error, - }), - ); - }); + let failed_sender = self.sender.clone(); + self.spawn_work( + move || { + let (outcome, error) = match work() { + Ok(outcome) => (Some(outcome), None), + Err(error) => (None, Some(error)), + }; + let _ = sender.send_to( + actor, + OrchestratorMsg::Manual(ManualControlMsg::EffectFinished { + node_id, + kind, + effect_id, + outcome, + error, + }), + ); + }, + move |error| { + let _ = failed_sender.send_to( + actor, + OrchestratorMsg::Manual(ManualControlMsg::EffectFinished { + node_id, + kind, + effect_id, + outcome: None, + error: Some(error), + }), + ); + }, + ); + } + + fn spawn_work( + &self, + work: impl FnOnce() + Send + 'static, + failure: impl FnOnce(String) + Send + 'static, + ) { + let _ = self.runtime.send_to( + self.work_actor, + ManualWork { + work: Arc::new(Mutex::new(Some(Box::new(work)))), + failure: Arc::new(Mutex::new(Some(Box::new(failure)))), + }, + ); + } +} + +impl Drop for ManualActorControl { + fn drop(&mut self) { + let _ = self.runtime.stop_actor(self.work_actor); } } @@ -1838,9 +2056,19 @@ fn send_command_reply( #[cfg(test)] mod tests { use std::collections::{BTreeMap, BTreeSet, VecDeque}; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; use distribution::types::NodeId as DistNodeId; use proptest::prelude::*; + use swactor::runtime::{RuntimeConfig, RuntimeParts}; + use swactor_engine::{Engine, SteppingBackend}; + + use crate::provisioning::{PluginObservation, PluginObservationSink}; + use crate::tests::fuzz_support::{ + actor_census, assert_actor_delta_at_most, assert_mailboxes_drained, assert_no_poison, + assert_no_poison_with_context, drive_steps, + }; use super::*; @@ -2237,6 +2465,16 @@ mod tests { last_error: None, last_seen_unix_ms: 0, }); + snapshot.commands.insert( + "provision".to_owned(), + CommandRecord { + command_id: "provision".to_owned(), + kind: CommandKind::Provision, + state: CommandState::Running, + node_ids: vec![1], + error: None, + }, + ); let mut core = ManualControl::new(snapshot, ProviderReadiness::ready()); let current = ActorAddress([9; 32]); let binding = core @@ -2276,6 +2514,10 @@ mod tests { node.runtime.as_ref().unwrap().node_actor, ActorAddress([8; 32]) ); + assert_eq!( + core.snapshot().commands["provision"].state, + CommandState::Succeeded + ); assert!( core.rejoin( &RejoinHello { @@ -2391,14 +2633,14 @@ mod tests { proptest! { #![proptest_config(ProptestConfig { - cases: 2048, - max_shrink_iters: 20_000, + cases: 128, + max_shrink_iters: 2_000, ..ProptestConfig::default() })] #[test] fn aggressive_random_event_stream_preserves_control_invariants( - operations in prop::collection::vec(any::(), 1..768) + operations in prop::collection::vec(any::(), 0..=32) ) { let mut core = ready_core(); let mut pending = VecDeque::new(); @@ -2621,14 +2863,14 @@ mod tests { proptest! { #![proptest_config(ProptestConfig { - cases: 256, - max_shrink_iters: 10_000, + cases: 128, + max_shrink_iters: 2_000, ..ProptestConfig::default() })] #[test] fn rental_free_end_to_end_sequences_converge( - operations in prop::collection::vec((any::(), any::()), 1..256) + operations in prop::collection::vec((any::(), any::()), 0..=32) ) { let mut core = ready_core(); let mut resources = BTreeSet::new(); @@ -2731,4 +2973,1003 @@ mod tests { prop_assert!(core.snapshot().commands.values().all(|command| command.state.is_terminal())); } } + + struct DiscardObservations; + + impl PluginObservationSink for DiscardObservations { + fn observe(&self, _observation: PluginObservation) {} + } + + struct ScriptedPlugin { + resources: Arc>>, + } + + impl ProvisionPlugin for ScriptedPlugin { + fn create_node( + &mut self, + spec: NodeProvisionSpec, + _sink: PluginSink, + ) -> Result { + match spec.node_id % 7 { + 0 => return Err("scripted provider create failure".to_owned()), + 1 => panic!("scripted provider callback panic"), + _ => {} + } + if !self.resources.lock().insert(spec.node_id) { + return Err(format!("duplicate resource for node {}", spec.node_id)); + } + Ok(PluginNodeHandle { + id: spec.node_id, + provider_process_id: None, + }) + } + + fn create_node_selected( + &mut self, + spec: NodeProvisionSpec, + sink: PluginSink, + _selected_offer_id: Option, + ) -> Result { + self.create_node(spec, sink) + } + + fn start_bootstrap(&mut self, handle: &PluginNodeHandle) -> Result<(), String> { + if handle.id % 11 == 0 { + return Err("scripted bootstrap failure".to_owned()); + } + Ok(()) + } + + fn complete_bootstrap(&mut self, _handle: &PluginNodeHandle) -> Result<(), String> { + Ok(()) + } + + fn stop_node(&mut self, handle: &PluginNodeHandle) -> Result<(), String> { + self.resources.lock().remove(&handle.id); + Ok(()) + } + + fn stop_by_spec( + &mut self, + spec: &NodeProvisionSpec, + _sink: PluginSink, + ) -> Result { + Ok(self.resources.lock().remove(&spec.node_id)) + } + + fn provider_ref_for(&self, spec: &NodeProvisionSpec) -> String { + if spec.node_id % 5 == 0 { + String::new() + } else { + format!("scripted-resource-{}", spec.node_id) + } + } + } + + #[derive(Clone, Debug, Default)] + struct ManualActorEvidence { + lanes: usize, + lane_ids: Vec, + persistence_queue: Vec, + persistence_in_flight: Option, + pending_command_replies: Vec, + flush_waiters: usize, + pending_validations: Vec, + pending_offer_searches: Vec, + active_effects: Vec<(u64, EffectKind, u64)>, + node_ids: Vec, + node_phases: Vec<(u64, NodePhase)>, + } + + impl ManualActorEvidence { + fn capture(control: &ManualActorControl) -> Self { + Self { + lanes: control.lanes.len(), + lane_ids: control.lanes.keys().copied().collect(), + persistence_queue: control + .persistence_queue + .iter() + .map(|(generation, _)| *generation) + .collect(), + persistence_in_flight: control.persistence_in_flight, + pending_command_replies: control.pending_command_replies.keys().copied().collect(), + flush_waiters: control.flush_waiters.len(), + pending_validations: control.pending_validations.keys().copied().collect(), + pending_offer_searches: control.pending_offer_searches.keys().copied().collect(), + active_effects: control + .active_effect_ids + .iter() + .map(|(node_id, effect_id)| { + let kind = *control + .core + .in_flight + .get(node_id) + .expect("actor effect identity matches core effect"); + (*node_id, kind, *effect_id) + }) + .collect(), + node_ids: control + .core + .snapshot() + .nodes + .iter() + .filter(|node| node.logical_node_id != 0) + .map(|node| node.logical_node_id) + .collect(), + node_phases: control + .core + .snapshot() + .nodes + .iter() + .filter(|node| node.logical_node_id != 0) + .map(|node| (node.logical_node_id, node.phase)) + .collect(), + } + } + + fn pending_count(&self) -> usize { + self.persistence_queue.len() + + usize::from(self.persistence_in_flight.is_some()) + + self.pending_command_replies.len() + + self.flush_waiters + + self.pending_validations.len() + + self.pending_offer_searches.len() + + self.active_effects.len() + } + + fn persistence_drained(&self) -> bool { + self.persistence_in_flight.is_none() && self.persistence_queue.is_empty() + } + } + + struct ManualHarnessActor { + control: ManualActorControl, + evidence: Arc>, + } + + impl ManualHarnessActor { + fn record_evidence(&self) { + *self.evidence.lock() = ManualActorEvidence::capture(&self.control); + } + } + + impl ActorInterface for ManualHarnessActor { + type Incoming = OrchestratorMsg; + type Response = (); + + fn on_start(&mut self, ctx: &Ctx) { + self.control.start(ctx.self_addr()); + self.record_evidence(); + } + + fn handle(&mut self, ctx: &Ctx, message: Self::Incoming) { + if let OrchestratorMsg::Manual(message) = message { + self.control.handle(ctx, message); + self.record_evidence(); + } + } + } + + struct ReplySlot { + inbox: Option>, + expect_reply: bool, + flush_barrier: Option, + } + + #[derive(Default)] + struct TerminalReplyCollector { + slots: BTreeMap, + replies: BTreeMap>, + } + + impl TerminalReplyCollector { + fn reply_to( + &mut self, + runtime: &Runtime, + request_id: String, + selector: u8, + allow_missing: bool, + ) -> Option { + let outcome = selector % 3; + if outcome == 2 && allow_missing { + self.slots.insert( + request_id, + ReplySlot { + inbox: None, + expect_reply: false, + flush_barrier: None, + }, + ); + return None; + } + let inbox = runtime + .new_inbox::() + .expect("manual terminal reply inbox"); + let address = *inbox.addr(); + let closed = outcome == 1 || (outcome == 2 && !allow_missing); + self.slots.insert( + request_id, + ReplySlot { + inbox: (!closed).then_some(inbox), + expect_reply: !closed, + flush_barrier: None, + }, + ); + Some(address) + } + fn mark_flush_barrier(&mut self, request_id: &str, evidence: &ManualActorEvidence) { + let barrier = evidence + .persistence_queue + .iter() + .copied() + .chain(evidence.persistence_in_flight) + .max() + .unwrap_or(0); + self.slots + .get_mut(request_id) + .expect("flush reply slot exists") + .flush_barrier = Some(barrier); + } + + fn drain(&mut self, evidence: &ManualActorEvidence, actions: &[ActorControlAction]) { + for (request_id, slot) in &self.slots { + let Some(inbox) = slot.inbox.as_ref() else { + continue; + }; + while let Some(reply) = inbox.try_recv() { + if let (ManualControlReply::Flushed, Some(barrier)) = + (&reply, slot.flush_barrier) + { + let older_persistence_pending = evidence + .persistence_queue + .iter() + .copied() + .chain(evidence.persistence_in_flight) + .any(|generation| generation <= barrier); + assert!( + !older_persistence_pending, + "flush replied before its persistence barrier drained; \ + request={request_id}, barrier={barrier}, actions={actions:?}, \ + evidence={evidence:?}" + ); + } + self.replies + .entry(request_id.clone()) + .or_default() + .push(reply); + } + assert!( + self.replies.get(request_id).map_or(0, Vec::len) <= 1, + "request produced duplicate terminal replies; request={request_id}, actions={actions:?}, replies={}, evidence={evidence:?}", + self.describe(), + ); + } + } + + fn assert_complete( + &self, + actions: &[ActorControlAction], + evidence: &ManualActorEvidence, + runtime: &Runtime, + ) { + for (request_id, slot) in &self.slots { + let count = self.replies.get(request_id).map_or(0, Vec::len); + assert!( + count <= 1, + "request produced more than one terminal reply; request={request_id}, actions={actions:?}, replies={}, evidence={evidence:?}\n{}", + self.describe(), + actor_census(runtime), + ); + assert_eq!( + count, + usize::from(slot.expect_reply), + "open request did not produce exactly one terminal reply, or closed/missing request was observed; request={request_id}, actions={actions:?}, replies={}, evidence={evidence:?}\n{}", + self.describe(), + actor_census(runtime), + ); + } + } + + fn describe(&self) -> String { + format!("{:?}", self.replies) + } + } + + #[derive(Clone, Debug)] + enum ActorControlAction { + Configure(u8), + Search(u8), + Provision(u8), + Kill(u8), + Query(u8), + Flush(u8), + Rejoin(u8), + ProviderTerminalFailure(u8), + ProviderValidated(u8), + OfferSearchFinished(u8), + PersistenceFinished(u8), + EffectFinished(u8), + Drive, + } + + fn actor_control_actions() -> impl Strategy> { + prop::collection::vec( + prop_oneof![ + 2 => any::().prop_map(ActorControlAction::Configure), + 2 => any::().prop_map(ActorControlAction::Search), + 3 => any::().prop_map(ActorControlAction::Provision), + 2 => any::().prop_map(ActorControlAction::Kill), + 2 => any::().prop_map(ActorControlAction::Query), + 2 => any::().prop_map(ActorControlAction::Flush), + 1 => any::().prop_map(ActorControlAction::Rejoin), + 1 => any::().prop_map(ActorControlAction::ProviderTerminalFailure), + 1 => any::().prop_map(ActorControlAction::ProviderValidated), + 1 => any::().prop_map(ActorControlAction::OfferSearchFinished), + 1 => any::().prop_map(ActorControlAction::PersistenceFinished), + 1 => any::().prop_map(ActorControlAction::EffectFinished), + 2 => Just(ActorControlAction::Drive), + ], + 0..=32, + ) + } + + fn settle_manual_work(backend: &SteppingBackend) { + for _ in 0..96 { + drive_steps(backend, 8); + backend + .join_blocking() + .expect("manual control blocking work must not panic"); + drive_steps(backend, 8); + } + } + + fn send_manual(runtime: &Runtime, actor: ActorAddress, message: ManualControlMsg) { + runtime + .send_to(actor, OrchestratorMsg::Manual(message)) + .expect("send manual control message"); + } + + fn send_duplicate_manual(runtime: &Runtime, actor: ActorAddress, message: ManualControlMsg) { + send_manual(runtime, actor, message.clone()); + send_manual(runtime, actor, message); + } + + fn scripted_offer(malformed: bool) -> OfferDto { + OfferDto { + offer_id: 44, + host_id: Some(55), + gpu_model: "scripted".to_owned(), + gpu_ram_mb: Some(24_000.0), + compute_cap: 89, + verification: Some("verified".to_owned()), + reliability: Some(if malformed { f64::NAN } else { 0.99 }), + download_mbps: Some(1_000.0), + upload_mbps: Some(1_000.0), + location: Some("test".to_owned()), + hourly_price: if malformed { f64::INFINITY } else { 0.5 }, + download_cost_per_tb: 0.0, + upload_cost_per_tb: 0.0, + } + } + + fn effect_outcome(node_id: u64, kind: EffectKind) -> EffectOutcome { + match kind { + EffectKind::Create => EffectOutcome::Created { + provider_ref: format!("injected-resource-{node_id}"), + }, + EffectKind::StartBootstrap => EffectOutcome::BootstrapStarted, + EffectKind::CompleteBootstrap => EffectOutcome::BootstrapCompleted, + EffectKind::Stop => EffectOutcome::Stopped, + EffectKind::Recover => EffectOutcome::Recovered { + provider_ref: Some(format!("injected-resource-{node_id}")), + }, + } + } + + fn assert_pending_bounded( + evidence: &ManualActorEvidence, + action_count: usize, + actions: &[ActorControlAction], + replies: &TerminalReplyCollector, + runtime: &Runtime, + ) { + let bound = action_count.saturating_mul(3).saturating_add(4); + assert!( + evidence.pending_count() <= bound, + "manual pending state exceeded live generated work; bound={bound}, actions={actions:?}, replies={}, evidence={evidence:?}\n{}", + replies.describe(), + actor_census(runtime), + ); + assert!( + evidence.lanes <= action_count, + "provider lanes exceeded generated actions; actions={actions:?}, replies={}, evidence={evidence:?}\n{}", + replies.describe(), + actor_census(runtime), + ); + } + + fn assert_manual_no_poison( + runtime: &Runtime, + actions: &[ActorControlAction], + replies: &TerminalReplyCollector, + evidence: &ManualActorEvidence, + resources: &Arc>>, + ) { + let context = format!( + "actions={actions:?}\nreplies={}\npending/resource state: evidence={evidence:?}, resources={:?}\nactor census follows", + replies.describe(), + *resources.lock(), + ); + assert_no_poison_with_context(runtime, &context); + } + + proptest! { + #![proptest_config(ProptestConfig { + cases: 128, + max_shrink_iters: 2_000, + ..ProptestConfig::default() + })] + + #[test] + fn manual_actor_generated_public_actions_and_callbacks_are_bounded( + actions in actor_control_actions() + ) { + let parts = RuntimeParts::new(RuntimeConfig::default()); + let runtime = parts.runtime().clone(); + let backend = SteppingBackend::new(); + let engine = Engine::new(parts, backend.clone()).expect("stepping engine"); + let baseline = runtime.stats().actors.len(); + let resources = Arc::new(Mutex::new(BTreeSet::new())); + let provider_resources = Arc::clone(&resources); + let factory_calls = Arc::new(AtomicUsize::new(0)); + let provider_factory_calls = Arc::clone(&factory_calls); + let provider_factory: ProviderFactory = Arc::new(move || { + match provider_factory_calls.fetch_add(1, Ordering::Relaxed) % 5 { + 1 => Err("scripted provider factory failure".to_owned()), + 2 => panic!("scripted provider factory panic"), + _ => Ok(Box::new(ScriptedPlugin { + resources: Arc::clone(&provider_resources), + })), + } + }); + let spec_builder: SpecBuilder = Arc::new(|node_id, _actor| { + if node_id % 9 == 0 { + return Err("malformed scripted provision specification".to_owned()); + } + Ok(spec(node_id)) + }); + let validator: ConfigValidator = Arc::new(|request| { + match request.api_key.as_deref() { + Some("panic") => panic!("scripted validation callback panic"), + Some("bad") => Err("scripted validation failure".to_owned()), + Some("") => Err("malformed provider configuration".to_owned()), + _ => Ok(()), + } + }); + let searcher: OfferSearcher = Arc::new(|request| { + match request.gpu_model.as_deref() { + Some("panic") => panic!("scripted search callback panic"), + Some("bad") => Err("scripted search failure".to_owned()), + Some("malformed") => Ok(vec![scripted_offer(true)]), + _ => Ok(vec![scripted_offer(false)]), + } + }); + let temp = tempfile::tempdir().expect("manual state tempdir"); + let evidence = Arc::new(Mutex::new(ManualActorEvidence::default())); + let control = ManualActorControl::new( + ready_core(), + runtime.clone(), + engine.handle().blocking_work_sender(), + StateDir::new(temp.path()), + PluginSink::new(Arc::new(DiscardObservations)), + provider_factory, + spec_builder, + Some(validator), + Some(searcher), + 1, + ); + prop_assert_eq!( + runtime.stats().actors.len(), + baseline + 1, + "manual control construction did not add exactly one work actor\n{}", + actor_census(&runtime), + ); + let actor = runtime + .spawn(ManualHarnessActor { + control, + evidence: Arc::clone(&evidence), + }) + .expect("spawn manual harness"); + drive_steps(&backend, 8); + prop_assert_eq!( + runtime.stats().actors.len(), + baseline + 2, + "manual control harness did not have fixed helper cardinality\n{}", + actor_census(&runtime), + ); + + let mut replies = TerminalReplyCollector::default(); + let mut request_serial = 0_u64; + for action in &actions { + let request_id = format!("request-{request_serial}"); + request_serial += 1; + match *action { + ActorControlAction::Configure(selector) => { + let reply_to = replies.reply_to( + &runtime, + format!("{request_id}:configure"), + selector >> 4, + true, + ); + let api_key = match selector % 4 { + 0 => Some("good".to_owned()), + 1 => Some("bad".to_owned()), + 2 => Some("panic".to_owned()), + _ => Some(String::new()), + }; + send_manual( + &runtime, + actor, + ManualControlMsg::Configure { + request: ProviderConfigurationRequest { + api_key, + ssh_identity: None, + bootstrap_command: None, + }, + reply_to, + }, + ); + } + ActorControlAction::Search(selector) => { + let reply_to = replies + .reply_to( + &runtime, + format!("{request_id}:search"), + selector >> 4, + false, + ) + .expect("required search reply address"); + let request = match selector % 5 { + 0 => OfferSearchRequest { + gpu_model: Some("good".to_owned()), + count: Some(1), + ..OfferSearchRequest::default() + }, + 1 => OfferSearchRequest { + gpu_model: Some("bad".to_owned()), + count: Some(1), + ..OfferSearchRequest::default() + }, + 2 => OfferSearchRequest { + gpu_model: Some("panic".to_owned()), + count: Some(1), + ..OfferSearchRequest::default() + }, + 3 => OfferSearchRequest { + gpu_model: Some("malformed".to_owned()), + count: Some(1), + ..OfferSearchRequest::default() + }, + _ => OfferSearchRequest { + count: Some(0), + ..OfferSearchRequest::default() + }, + }; + send_manual( + &runtime, + actor, + ManualControlMsg::SearchOffers { request, reply_to }, + ); + } + ActorControlAction::Provision(selector) => { + let key = format!("{request_id}:provision"); + let reply_to = replies.reply_to( + &runtime, + key.clone(), + selector >> 4, + true, + ); + let count = u32::from(selector % 5 != 0); + send_manual( + &runtime, + actor, + ManualControlMsg::Provision { + request: ProvisionRequest { + command_id: key, + count, + selected_offer_ids: (count == 1) + .then_some(vec![10_000 + request_serial]) + .unwrap_or_default(), + }, + reply_to, + }, + ); + } + ActorControlAction::Kill(selector) => { + let key = format!("{request_id}:kill"); + let reply_to = replies.reply_to( + &runtime, + key.clone(), + selector >> 4, + true, + ); + let state = evidence.lock().clone(); + let node_id = state + .node_ids + .get(usize::from(selector) % state.node_ids.len().max(1)) + .copied() + .unwrap_or(u64::from(selector) + 1); + send_manual( + &runtime, + actor, + ManualControlMsg::Kill { + request: KillRequest { + command_id: key, + logical_node_id: node_id, + }, + reply_to, + }, + ); + } + ActorControlAction::Query(selector) => { + let reply_to = replies + .reply_to( + &runtime, + format!("{request_id}:query"), + selector, + false, + ) + .expect("required query reply address"); + send_manual(&runtime, actor, ManualControlMsg::Query { reply_to }); + } + ActorControlAction::Flush(selector) => { + let key = format!("{request_id}:flush"); + let reply_to = replies + .reply_to(&runtime, key.clone(), selector, false) + .expect("required flush reply address"); + replies.mark_flush_barrier(&key, &evidence.lock()); + send_manual(&runtime, actor, ManualControlMsg::Flush { reply_to }); + } + ActorControlAction::Rejoin(selector) => { + let reply_to = replies + .reply_to( + &runtime, + format!("{request_id}:rejoin"), + selector, + false, + ) + .expect("required rejoin reply address"); + send_manual( + &runtime, + actor, + ManualControlMsg::Rejoin { + hello: RejoinHello { + run_id: 7, + logical_node_id: u64::from(selector) + 10_000, + attempt_id: 0, + selected_offer_id: None, + endpoint: "generated-rejoin".to_owned(), + swim_node_id: DistNodeId([selector; 32]), + stage_index: 0, + node_actor: ActorAddress([selector; 32]), + }, + reply_to, + }, + ); + } + ActorControlAction::ProviderTerminalFailure(selector) => { + let state = evidence.lock().clone(); + let node_id = state + .node_ids + .get(usize::from(selector) % state.node_ids.len().max(1)) + .copied() + .unwrap_or(u64::from(selector) + 1); + send_manual( + &runtime, + actor, + ManualControlMsg::ProviderTerminalFailure { + node_id, + error: format!("generated terminal provider failure {selector}"), + }, + ); + } + ActorControlAction::ProviderValidated(selector) => { + let state = evidence.lock().clone(); + let work_id = state + .pending_validations + .get(usize::from(selector) % state.pending_validations.len().max(1)) + .copied() + .unwrap_or(u64::MAX - u64::from(selector)); + send_duplicate_manual( + &runtime, + actor, + ManualControlMsg::ProviderValidated { + work_id, + error: (selector & 1 != 0) + .then(|| "injected validation failure".to_owned()), + }, + ); + } + ActorControlAction::OfferSearchFinished(selector) => { + let state = evidence.lock().clone(); + let work_id = state + .pending_offer_searches + .get(usize::from(selector) % state.pending_offer_searches.len().max(1)) + .copied() + .unwrap_or(u64::MAX - u64::from(selector)); + let result = match selector % 3 { + 0 => Ok(vec![scripted_offer(false)]), + 1 => Err("injected search failure".to_owned()), + _ => Ok(vec![scripted_offer(true)]), + }; + send_duplicate_manual( + &runtime, + actor, + ManualControlMsg::OfferSearchFinished { work_id, result }, + ); + } + ActorControlAction::PersistenceFinished(selector) => { + let state = evidence.lock().clone(); + let generation = match selector % 3 { + 0 => state.persistence_in_flight, + 1 => state.persistence_queue.last().copied(), + _ => None, + } + .unwrap_or(u64::MAX - u64::from(selector)); + send_duplicate_manual( + &runtime, + actor, + ManualControlMsg::PersistenceFinished { + generation, + error: (selector & 4 != 0) + .then(|| "injected persistence failure".to_owned()), + }, + ); + } + ActorControlAction::EffectFinished(selector) => { + let state = evidence.lock().clone(); + let (node_id, kind, active_effect_id) = state + .active_effects + .get(usize::from(selector) % state.active_effects.len().max(1)) + .copied() + .unwrap_or(( + u64::MAX - u64::from(selector), + EffectKind::Create, + u64::MAX, + )); + let effect_id = active_effect_id.wrapping_add(1); + let (outcome, error) = match selector % 3 { + 0 => (Some(effect_outcome(node_id, kind)), None), + 1 => (None, Some("injected provider effect failure".to_owned())), + _ => (None, None), + }; + send_duplicate_manual( + &runtime, + actor, + ManualControlMsg::EffectFinished { + node_id, + kind, + effect_id, + outcome, + error, + }, + ); + } + ActorControlAction::Drive => settle_manual_work(&backend), + } + + drive_steps(&backend, 8); + let state = evidence.lock().clone(); + replies.drain(&state, &actions); + assert_pending_bounded( + &state, + actions.len().max(1), + &actions, + &replies, + &runtime, + ); + prop_assert_eq!( + runtime.stats().actors.len(), + baseline + 2, + "manual commands changed fixed helper cardinality; actions={:?}, replies={}, evidence={:?}\n{}", + actions, + replies.describe(), + state, + actor_census(&runtime), + ); + assert_manual_no_poison(&runtime, &actions, &replies, &state, &resources); + } + + settle_manual_work(&backend); + let settled = evidence.lock().clone(); + replies.drain(&settled, &actions); + prop_assert!( + settled.persistence_drained() + && settled.pending_command_replies.is_empty() + && settled.flush_waiters == 0 + && settled.pending_validations.is_empty() + && settled.pending_offer_searches.is_empty() + && settled.active_effects.is_empty(), + "manual pending state did not drain; actions={:?}, replies={}, evidence={:?}\n{}", + actions, + replies.describe(), + settled, + actor_census(&runtime), + ); + replies.assert_complete(&actions, &settled, &runtime); + assert_manual_no_poison(&runtime, &actions, &replies, &settled, &resources); + assert_actor_delta_at_most(&runtime, baseline, 2); + + for node_id in settled.node_ids.iter().copied() { + send_manual( + &runtime, + actor, + ManualControlMsg::Kill { + request: KillRequest { + command_id: format!("cleanup-{node_id}"), + logical_node_id: node_id, + }, + reply_to: None, + }, + ); + } + settle_manual_work(&backend); + let cleanup = evidence.lock().clone(); + prop_assert!( + resources.lock().is_empty(), + "manual control leaked scripted resources; actions={:?}, replies={}, resources={:?}, evidence={:?}\n{}", + actions, + replies.describe(), + *resources.lock(), + cleanup, + actor_census(&runtime), + ); + prop_assert!( + cleanup.lanes == 0 && cleanup.lane_ids.is_empty(), + "provider lanes did not drain after terminal cleanup; actions={:?}, replies={}, \ + lane_ids={:?}, node_phases={:?}, evidence={:?}\n{}", + actions, + replies.describe(), + cleanup.lane_ids, + cleanup.node_phases, + cleanup, + actor_census(&runtime), + ); + + let _ = runtime.stop_actor(actor); + let _ = runtime.stop_actor(actor); + drive_steps(&backend, 32); + backend.join_blocking().expect("join final manual work"); + drive_steps(&backend, 32); + assert_manual_no_poison(&runtime, &actions, &replies, &cleanup, &resources); + prop_assert_eq!( + runtime.stats().actors.len(), + baseline, + "manual control or work actor survived owner stop; actions={:?}, replies={}, evidence={:?}\n{}", + actions, + replies.describe(), + cleanup, + actor_census(&runtime), + ); + assert_mailboxes_drained(&runtime); + } + } + + struct IdleHelperActor; + + impl ActorInterface for IdleHelperActor { + type Incoming = (); + type Response = (); + + fn handle(&mut self, _ctx: &Ctx, (): ()) {} + } + + #[test] + fn fixed_helper_cardinality_invariant_detects_controlled_extra_spawn() { + let parts = RuntimeParts::new(RuntimeConfig::default()); + let runtime = parts.runtime().clone(); + let backend = SteppingBackend::new(); + let _engine = Engine::new(parts, backend.clone()).expect("stepping engine"); + let baseline = runtime.stats().actors.len(); + let expected = runtime.spawn(IdleHelperActor).expect("expected helper"); + let injected = runtime + .spawn(IdleHelperActor) + .expect("controlled extra helper"); + let detected = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + assert_actor_delta_at_most(&runtime, baseline, 1); + })); + assert!( + detected.is_err(), + "actor-cardinality invariant accepted a controlled extra helper\n{}", + actor_census(&runtime), + ); + let _ = runtime.stop_actor(expected); + let _ = runtime.stop_actor(injected); + drive_steps(&backend, 16); + assert_no_poison(&runtime); + assert_eq!(runtime.stats().actors.len(), baseline); + } + + struct PanickingCallbackActor; + + impl ActorInterface for PanickingCallbackActor { + type Incoming = (); + type Response = (); + + fn handle(&mut self, _ctx: &Ctx, (): ()) { + panic!("controlled unguarded callback panic"); + } + } + + #[test] + fn callback_panic_invariant_detects_controlled_unguarded_panic() { + let parts = RuntimeParts::new(RuntimeConfig::default()); + let runtime = parts.runtime().clone(); + let backend = SteppingBackend::new(); + let _engine = Engine::new(parts, backend.clone()).expect("stepping engine"); + let actor = runtime + .spawn(PanickingCallbackActor) + .expect("controlled panicking callback actor"); + runtime + .send_to(actor, ()) + .expect("send controlled unguarded callback"); + drive_steps(&backend, 8); + let detected = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + assert_no_poison_with_context( + &runtime, + "controlled unguarded callback must be rejected by the property invariant", + ); + })); + assert!( + detected.is_err(), + "poison invariant accepted a controlled unguarded callback panic\n{}", + actor_census(&runtime), + ); + } + + #[test] + fn callback_panic_reports_typed_failure_without_poisoning_work_actor() { + let parts = RuntimeParts::new(RuntimeConfig::default()); + let runtime = parts.runtime().clone(); + let backend = SteppingBackend::new(); + let engine = Engine::new(parts, backend.clone()).expect("stepping engine"); + let baseline = runtime.stats().actors.len(); + let failures = Arc::new(Mutex::new(Vec::new())); + let observed_failures = Arc::clone(&failures); + let actor = runtime + .spawn(ManualWorkActor { + blocking_work: engine.handle().blocking_work_sender(), + }) + .expect("manual work actor"); + runtime + .send_to( + actor, + ManualWork { + work: Arc::new(Mutex::new(Some(Box::new(|| { + panic!("controlled callback panic"); + })))), + failure: Arc::new(Mutex::new(Some(Box::new(move |error| { + observed_failures.lock().push(error); + })))), + }, + ) + .expect("send controlled callback"); + drive_steps(&backend, 8); + backend + .join_blocking() + .expect("guarded callback must not escape the work boundary"); + drive_steps(&backend, 8); + assert_eq!( + failures.lock().as_slice(), + ["manual control work panicked"], + "callback panic was not converted into its typed terminal failure", + ); + assert_no_poison(&runtime); + assert_eq!(runtime.stats().actors.len(), baseline + 1); + let _ = runtime.stop_actor(actor); + drive_steps(&backend, 16); + assert_no_poison(&runtime); + assert_eq!(runtime.stats().actors.len(), baseline); + assert_mailboxes_drained(&runtime); + } } diff --git a/apps/myelin/src/orchestration/node_image.rs b/apps/myelin/src/orchestration/node_image.rs index 5bada4b..cd2f7b6 100644 --- a/apps/myelin/src/orchestration/node_image.rs +++ b/apps/myelin/src/orchestration/node_image.rs @@ -220,11 +220,10 @@ fn prepare_node_image_inner( } fn workspace_root() -> Result { - let output = Command::new("git") + let output = swactor_process::command_output(&mut Command::new("git") .args(["rev-parse", "--show-toplevel"]) .current_dir(env!("CARGO_MANIFEST_DIR")) - .stdin(Stdio::null()) - .output() + .stdin(Stdio::null())) .map_err(|e| format!("locate repository root with git: {e}"))?; if !output.status.success() { return Err(format!( @@ -251,11 +250,10 @@ fn image_version_tag(root: &Path, image_content_hash: &str) -> Result Result { - let output = Command::new("git") + let output = swactor_process::command_output(&mut Command::new("git") .current_dir(root) .args(args) - .stdin(Stdio::null()) - .output() + .stdin(Stdio::null())) .map_err(|e| format!("run git {}: {e}", args.join(" ")))?; if output.status.success() { Ok(String::from_utf8_lossy(&output.stdout).to_string()) @@ -613,8 +611,6 @@ enum CommandOutputLine { Stderr(String), } -// container image build is provisioning infrastructure, out of scope (ENGINE_SPEC.md §2) -#[allow(clippy::disallowed_methods)] fn spawn_line_reader( reader: R, to_line: fn(String) -> CommandOutputLine, @@ -654,8 +650,6 @@ fn drain_command_lines( } } -// container image build is provisioning infrastructure, out of scope (ENGINE_SPEC.md §2) -#[allow(clippy::disallowed_methods)] fn run_status_command( root: &Path, program: &str, @@ -667,13 +661,12 @@ fn run_status_command( let args: Vec = args.iter().map(|arg| (*arg).to_owned()).collect(); eprintln!("myelin-node-image: {label}"); if progress.is_none() { - let status = Command::new(program) + let status = swactor_process::command_status(&mut Command::new(program) .current_dir(root) .args(&args) .stdin(Stdio::null()) .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .status() + .stderr(Stdio::inherit())) .map_err(|e| format!("run {label}: {e}"))?; return if status.success() { Ok(()) @@ -693,13 +686,12 @@ fn run_status_command( args: args.to_vec(), }, ); - let mut child = match Command::new(program) + let mut child = match swactor_process::command_spawn(&mut Command::new(program) .current_dir(root) .args(&args) .stdin(Stdio::null()) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() + .stderr(Stdio::piped())) { Ok(child) => child, Err(error) => { @@ -737,7 +729,7 @@ fn run_status_command( drop(tx); let status = loop { - match child.try_wait() { + match swactor_process::child_try_wait(&mut child) { Ok(Some(status)) => break status, Ok(None) => { drain_command_lines(&rx, progress, label, image_ref, started); @@ -785,13 +777,12 @@ fn run_status_command( } fn docker_image_exists(root: &Path, image_ref: &str) -> bool { - Command::new("docker") + swactor_process::command_status(&mut Command::new("docker") .current_dir(root) .args(["image", "inspect", image_ref]) .stdin(Stdio::null()) .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() + .stderr(Stdio::null())) .map(|status| status.success()) .unwrap_or(false) } @@ -800,7 +791,7 @@ fn docker_image_labels( root: &Path, image_ref: &str, ) -> Result>, String> { - let output = Command::new("docker") + let output = swactor_process::command_output(&mut Command::new("docker") .current_dir(root) .args([ "image", @@ -809,8 +800,7 @@ fn docker_image_labels( "{{ json .Config.Labels }}", image_ref, ]) - .stdin(Stdio::null()) - .output() + .stdin(Stdio::null())) .map_err(|e| format!("inspect docker image {image_ref}: {e}"))?; if !output.status.success() { return Ok(None); @@ -822,19 +812,18 @@ fn docker_image_labels( } fn docker_manifest_exists(root: &Path, image_ref: &str) -> bool { - Command::new("docker") + swactor_process::command_status(&mut Command::new("docker") .current_dir(root) .args(["manifest", "inspect", image_ref]) .stdin(Stdio::null()) .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() + .stderr(Stdio::null())) .map(|status| status.success()) .unwrap_or(false) } fn docker_image_has_container(root: &Path, image_ref: &str) -> bool { - Command::new("docker") + swactor_process::command_output(&mut Command::new("docker") .current_dir(root) .args([ "ps", @@ -844,14 +833,13 @@ fn docker_image_has_container(root: &Path, image_ref: &str) -> bool { "--format", "{{.ID}}", ]) - .stdin(Stdio::null()) - .output() + .stdin(Stdio::null())) .map(|output| output.status.success() && !output.stdout.is_empty()) .unwrap_or(true) } fn docker_image_tags(root: &Path, repository: &str) -> Result, String> { - let output = Command::new("docker") + let output = swactor_process::command_output(&mut Command::new("docker") .current_dir(root) .args([ "image", @@ -860,8 +848,7 @@ fn docker_image_tags(root: &Path, repository: &str) -> Result Result Result<(), String> { - let status = Command::new("docker") + let status = swactor_process::command_status(&mut Command::new("docker") .current_dir(root) .args(["image", "rm", image_ref]) .stdin(Stdio::null()) .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() + .stderr(Stdio::null())) .map_err(|error| format!("docker image rm failed: {error}"))?; if status.success() { Ok(()) diff --git a/apps/myelin/src/orchestration/provider_adapters/vastai/mod.rs b/apps/myelin/src/orchestration/provider_adapters/vastai/mod.rs index aeb8901..c0f9df8 100644 --- a/apps/myelin/src/orchestration/provider_adapters/vastai/mod.rs +++ b/apps/myelin/src/orchestration/provider_adapters/vastai/mod.rs @@ -1,24 +1,16 @@ -// VastAI provider adapter: owns private blocking facades and a legacy provider -// monitor thread outside the orchestration engine. The monitor's swactor core -// is driven by an explicit SingleThreadRuntime owned by that thread; the main -// orchestration engine owns all bootstrap actors spawned on its runtime handle. -#![allow(clippy::disallowed_methods)] +// VastAI provider adapter. Actors own provider lifecycle policy; the +// swactor-vastai and swactor-process crates own API and process mechanics. use std::collections::{BTreeMap, BTreeSet, HashSet}; -use std::io::{BufRead, BufReader, Read}; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::thread::{self, JoinHandle}; use std::time::{Duration, Instant}; use swactor::actor::{ActorAddress, ActorInterface}; -use swactor::runtime::{ - Ctx, ExternalSender, Runtime, RuntimeConfig, RuntimeParts, SingleThreadRuntime, -}; +use swactor::runtime::{Ctx, ExternalSender, Runtime}; +use swactor_engine::EngineHandle; use swactor_vastai::{ - CreateInstanceRequest, LifecyclePolicy, Offer, OfferBrowseCriteria, ProvisionRequest, - ProvisionedInstance, SelectionPolicy, classify_vastai_error, + BlockingVastClient, CreateInstanceRequest, LifecyclePolicy, Offer, OfferBrowseCriteria, + ProvisionRequest, ProvisionedInstance, SelectionPolicy, classify_vastai_error, }; use telemetry::TelemetryProducer; @@ -27,6 +19,7 @@ use crate::provisioning::{ AdoptedNode, NodeProvisionSpec, PluginNodeHandle, PluginObservation, PluginSink, ProvisionPlugin, }; +use swactor_process::{LineReaderHandle, ProcessStream, ProcessStreamObservation}; #[derive(Clone, Debug)] pub(crate) struct VastAiProvisioningConfig { @@ -65,42 +58,17 @@ pub(crate) struct VastAiSshEndpoint { pub(crate) struct VastAiProviderMonitor { runtime: Runtime, actor: ActorAddress, - tick_thread: Option>, - stop_flag: Arc, } impl VastAiProviderMonitor { - fn new(runtime: Runtime, mut host: SingleThreadRuntime, actor: ActorAddress) -> Self { - let stop_flag = Arc::new(AtomicBool::new(false)); - - let flag = Arc::clone(&stop_flag); - let tick_thread = thread::spawn(move || { - while !flag.load(Ordering::Relaxed) || host.has_work() { - if host.has_work() { - host.tick(); - } else { - thread::sleep(Duration::from_millis(10)); - } - } - }); - - Self { - runtime, - actor, - tick_thread: Some(tick_thread), - stop_flag, - } + fn new(runtime: Runtime, actor: ActorAddress) -> Self { + Self { runtime, actor } } fn stop(&mut self) { - let Some(tick_thread) = self.tick_thread.take() else { - return; - }; let _ = self .runtime .send_to(self.actor, VastAiProviderMonitorMsg::Stop); - self.stop_flag.store(true, Ordering::Relaxed); - let _ = tick_thread.join(); } } @@ -126,6 +94,21 @@ pub(crate) trait VastAiLeaseClient: Send { /// Resolves the live contract id carrying `label`, if any. fn contract_by_label(&mut self, label: &str) -> Result, String>; + fn contract_by_label_with_retry( + &mut self, + label: &str, + attempts: usize, + _pace: Duration, + ) -> Result, String> { + for _ in 0..attempts.max(1) { + let contract = self.contract_by_label(label)?; + if contract.is_some() { + return Ok(contract); + } + } + Ok(None) + } + fn ssh_endpoint( &mut self, contract_id: u64, @@ -148,17 +131,21 @@ pub(crate) trait VastAiLeaseClient: Send { } pub(crate) struct ToolsVastAiLeaseClient { - client: swactor_vastai::VastClient, - runtime: tokio::runtime::Runtime, + client: BlockingVastClient, + actor_host: Option<(Runtime, EngineHandle)>, } impl ToolsVastAiLeaseClient { pub(crate) fn new(client: swactor_vastai::VastClient) -> Result { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|e| format!("vastai tokio runtime: {e}"))?; - Ok(Self { client, runtime }) + Ok(Self { + client: BlockingVastClient::new(client)?, + actor_host: None, + }) + } + + pub(crate) fn with_actor_host(mut self, runtime: Runtime, engine: EngineHandle) -> Self { + self.actor_host = Some((runtime, engine)); + self } pub(crate) fn from_api_key(api_key: impl Into) -> Result { @@ -169,7 +156,7 @@ impl ToolsVastAiLeaseClient { &mut self, criteria: &OfferBrowseCriteria, ) -> Result, String> { - self.runtime.block_on(self.client.browse_offers(criteria)) + self.client.browse_offers(criteria) } fn create_request_for_offer( @@ -195,8 +182,7 @@ impl ToolsVastAiLeaseClient { } fn candidate_pool(&mut self, request: &ProvisionRequest) -> Result, String> { - self.runtime - .block_on(self.client.search_offers(&request.selection, 1)) + self.client.search_offers(&request.selection, 1) } fn create_from_offer( @@ -205,9 +191,7 @@ impl ToolsVastAiLeaseClient { offer: &Offer, ) -> Result { let create = Self::create_request_for_offer(request, offer.id); - let info = self - .runtime - .block_on(self.client.create_instance(&create))?; + let info = self.client.create_instance(&create)?; Ok(ProvisionedInstance { index: 0, contract_id: info.contract_id, @@ -234,6 +218,7 @@ struct VastAiProviderMonitorActor { spec: NodeProvisionSpec, sink: PluginSink, sender: ExternalSender, + engine: EngineHandle, last_state: Option, state_since: Instant, poll: u64, @@ -249,6 +234,7 @@ impl VastAiProviderMonitorActor { spec: NodeProvisionSpec, sink: PluginSink, sender: ExternalSender, + engine: EngineHandle, ) -> Self { Self { client, @@ -258,6 +244,7 @@ impl VastAiProviderMonitorActor { spec, sink, sender, + engine, last_state: None, state_since: Instant::now(), poll: 0, @@ -282,10 +269,11 @@ impl VastAiProviderMonitorActor { } fn schedule_next_poll(&self, ctx: &Ctx) { - schedule_provider_monitor_poll( + self.engine.send_after( + self.lifecycle.poll_interval, self.sender.clone(), ctx.self_addr(), - self.lifecycle.poll_interval, + VastAiProviderMonitorMsg::Poll, ); } @@ -295,11 +283,7 @@ impl VastAiProviderMonitorActor { } self.poll = self.poll.saturating_add(1); let poll = self.poll; - let status = match self - .client - .runtime - .block_on(self.client.client.instance_status(self.contract_id)) - { + let status = match self.client.client.instance_status(self.contract_id) { Ok(status) => status, Err(error) if error.contains("not found while fetching provider status") @@ -415,21 +399,11 @@ impl ActorInterface for VastAiProviderMonitorActor { } } -fn schedule_provider_monitor_poll(sender: ExternalSender, actor: ActorAddress, delay: Duration) { - thread::spawn(move || { - thread::sleep(delay); - let _ = sender.send_to(actor, VastAiProviderMonitorMsg::Poll); - }); -} - impl Clone for ToolsVastAiLeaseClient { fn clone(&self) -> Self { Self { client: self.client.clone(), - runtime: tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .expect("clone VastAI lease client runtime"), + actor_host: self.actor_host.clone(), } } } @@ -448,7 +422,30 @@ fn adopted_instance(contract_id: u64) -> ProvisionedInstance { impl VastAiLeaseClient for ToolsVastAiLeaseClient { fn contract_by_label(&mut self, label: &str) -> Result, String> { - let instances = self.runtime.block_on(self.client.list_by_label(label))?; + let instances = self.client.list_by_label(label)?; + match instances.as_slice() { + [] => Ok(None), + [instance] => Ok(Some(instance.contract_id)), + _ => Err(format!( + "multiple VastAI contracts share stable label {label}: {}", + instances + .iter() + .map(|instance| instance.contract_id.to_string()) + .collect::>() + .join(",") + )), + } + } + + fn contract_by_label_with_retry( + &mut self, + label: &str, + attempts: usize, + pace: Duration, + ) -> Result, String> { + let instances = self + .client + .list_by_label_with_retry(label, attempts, pace)?; match instances.as_slice() { [] => Ok(None), [instance] => Ok(Some(instance.contract_id)), @@ -565,11 +562,9 @@ impl VastAiLeaseClient for ToolsVastAiLeaseClient { lifecycle: &LifecyclePolicy, ssh_user: &str, ) -> Result { - let endpoint = self.runtime.block_on(self.client.wait_for_ssh_endpoint( - contract_id, - label, - lifecycle, - ))?; + let endpoint = self + .client + .wait_for_ssh_endpoint(contract_id, label, lifecycle)?; let host = endpoint.ip; let port = endpoint.port; if host.is_empty() || host == "unknown" { @@ -593,8 +588,7 @@ impl VastAiLeaseClient for ToolsVastAiLeaseClient { spec: NodeProvisionSpec, sink: PluginSink, ) -> Option { - let parts = RuntimeParts::new(RuntimeConfig::default()); - let runtime = parts.runtime().clone(); + let (runtime, engine) = self.actor_host.as_ref()?.clone(); let sender = runtime.create_sender(); let actor = runtime .spawn(VastAiProviderMonitorActor::new( @@ -605,18 +599,14 @@ impl VastAiLeaseClient for ToolsVastAiLeaseClient { spec, sink, sender, + engine, )) .ok()?; - Some(VastAiProviderMonitor::new( - runtime, - SingleThreadRuntime::new(parts), - actor, - )) + Some(VastAiProviderMonitor::new(runtime, actor)) } fn destroy_contract(&mut self, contract_id: u64) -> Result<(), String> { - self.runtime - .block_on(self.client.destroy_instance_with_retry(contract_id)) + self.client.destroy_instance_with_retry(contract_id) } } @@ -692,17 +682,55 @@ enum SshBootstrapMsg { ReaderClosed { stream: SshBootstrapStream, }, + #[cfg(test)] + ScriptedAttemptFinished { + result: Result, + }, Stop, } +struct SshOutputRelay { + target: ActorAddress, +} + +impl ActorInterface for SshOutputRelay { + type Incoming = ProcessStreamObservation; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, observation: Self::Incoming) { + let message = match observation { + ProcessStreamObservation::Line { stream, line } => SshBootstrapMsg::OutputLine { + stream: map_process_stream(stream), + line, + }, + ProcessStreamObservation::Error { stream, error } => SshBootstrapMsg::ReaderError { + stream: map_process_stream(stream), + error, + }, + ProcessStreamObservation::Closed { stream } => SshBootstrapMsg::ReaderClosed { + stream: map_process_stream(stream), + }, + }; + let _ = ctx.send(self.target, message); + } +} + +fn map_process_stream(stream: ProcessStream) -> SshBootstrapStream { + match stream { + ProcessStream::Stdout => SshBootstrapStream::Stdout, + ProcessStream::Stderr => SshBootstrapStream::Stderr, + } +} struct SshBootstrapActor { bridge: BootstrapTelemetryBridge, endpoint: VastAiSshEndpoint, ssh_identity: Option, sender: ExternalSender, + engine: EngineHandle, child: Option, - stdout_reader: Option>, - stderr_reader: Option>, + reader_relay: Option, + stdout_reader: Option, + stderr_reader: Option, stdout_closed: bool, stderr_closed: bool, pending_status: Option, @@ -711,6 +739,10 @@ struct SshBootstrapActor { backoff: Duration, observation_class: Option<&'static str>, stopped: bool, + #[cfg(test)] + disable_attempt_spawn: bool, + #[cfg(test)] + spawn_pending_test_child: bool, } impl SshBootstrapActor { @@ -719,16 +751,19 @@ impl SshBootstrapActor { endpoint: VastAiSshEndpoint, ssh_identity: Option, sender: ExternalSender, + engine: EngineHandle, ) -> Self { Self { bridge, endpoint, ssh_identity, sender, + engine, child: None, stdout_reader: None, stderr_reader: None, stdout_closed: true, + reader_relay: None, stderr_closed: true, pending_status: None, pending_wait_error: None, @@ -736,6 +771,10 @@ impl SshBootstrapActor { backoff: Duration::from_secs(1), observation_class: None, stopped: false, + #[cfg(test)] + disable_attempt_spawn: false, + #[cfg(test)] + spawn_pending_test_child: false, } } @@ -747,6 +786,32 @@ impl SshBootstrapActor { self.bridge.spec().node_id } + #[cfg(test)] + fn with_attempt_spawn_disabled(mut self) -> Self { + self.disable_attempt_spawn = true; + self + } + + #[cfg(test)] + fn with_pending_test_child(mut self) -> Self { + self.disable_attempt_spawn = true; + self.spawn_pending_test_child = true; + self + } + + #[cfg(test)] + fn with_open_test_streams(mut self) -> Self { + self.disable_attempt_spawn = true; + self.stdout_closed = false; + self.stderr_closed = false; + self + } + + fn schedule(&self, ctx: &Ctx, message: SshBootstrapMsg, delay: Duration) { + self.engine + .send_after(delay, self.sender.clone(), ctx.self_addr(), message); + } + fn start_attempt(&mut self, ctx: &Ctx) { if self.stopped { return; @@ -756,11 +821,28 @@ impl SshBootstrapActor { self.attempt, self.endpoint.user, self.endpoint.host, self.endpoint.port )); - match spawn_ssh_bootstrap_attempt( + #[cfg(test)] + let attempt = if self.disable_attempt_spawn { + if self.spawn_pending_test_child { + spawn_pending_test_child() + } else { + return; + } + } else { + spawn_ssh_bootstrap_attempt( + self.bridge.spec(), + &self.endpoint, + self.ssh_identity.as_deref(), + ) + }; + #[cfg(not(test))] + let attempt = spawn_ssh_bootstrap_attempt( self.bridge.spec(), &self.endpoint, self.ssh_identity.as_deref(), - ) { + ); + + match attempt { Ok((child, stdout, stderr)) => { self.child = Some(child); self.stdout_closed = false; @@ -768,24 +850,22 @@ impl SshBootstrapActor { self.observation_class = None; self.pending_status = None; self.pending_wait_error = None; - self.stdout_reader = Some(spawn_ssh_output_reader( - SshBootstrapStream::Stdout, + let reader_relay = self + .reader_relay + .expect("SSH output relay is installed before attempts start"); + self.stdout_reader = Some(swactor_process::spawn_line_reader( + ProcessStream::Stdout, stdout, self.sender.clone(), - ctx.self_addr(), + reader_relay, )); - self.stderr_reader = Some(spawn_ssh_output_reader( - SshBootstrapStream::Stderr, + self.stderr_reader = Some(swactor_process::spawn_line_reader( + ProcessStream::Stderr, stderr, self.sender.clone(), - ctx.self_addr(), + reader_relay, )); - schedule_ssh_message( - self.sender.clone(), - ctx.self_addr(), - SshBootstrapMsg::PollChild, - Duration::from_millis(100), - ); + self.schedule(ctx, SshBootstrapMsg::PollChild, Duration::from_millis(100)); } Err(error) => { self.bridge.observe_provider_line(format!( @@ -804,18 +884,15 @@ impl SshBootstrapActor { let Some(child) = self.child.as_mut() else { return; }; - match child.try_wait() { + match swactor_process::child_try_wait(child) { Ok(Some(status)) => { self.child = None; self.pending_status = Some(status); self.maybe_finish_attempt(ctx); } - Ok(None) => schedule_ssh_message( - self.sender.clone(), - ctx.self_addr(), - SshBootstrapMsg::PollChild, - Duration::from_millis(100), - ), + Ok(None) => { + self.schedule(ctx, SshBootstrapMsg::PollChild, Duration::from_millis(100)); + } Err(error) => { self.child = None; self.pending_wait_error = Some(error.to_string()); @@ -862,6 +939,29 @@ impl SshBootstrapActor { self.maybe_finish_attempt(ctx); } + fn finish_completed_attempt(&mut self, ctx: &Ctx, status: String, success: bool) { + self.join_readers(); + let readiness = if success { + "exited before runtime ready" + } else { + "not ready before runtime ready" + }; + let observation_class = self.observation_class.unwrap_or("process_exit"); + self.bridge.observe_provider_line( + serde_json::json!({ + "type": "VastAiBootstrapAttemptCompleted", + "run_id": self.run_id(), + "node_id": self.node_id(), + "attempt": self.attempt, + "status": status, + "class": observation_class, + "classification": readiness, + }) + .to_string(), + ); + self.schedule_retry(ctx); + } + fn maybe_finish_attempt(&mut self, ctx: &Ctx) { if self.stopped || !self.stdout_closed || !self.stderr_closed { return; @@ -878,26 +978,8 @@ impl SshBootstrapActor { let Some(status) = self.pending_status.take() else { return; }; - self.join_readers(); - let readiness = if status.success() { - "exited before runtime ready" - } else { - "not ready before runtime ready" - }; - let observation_class = self.observation_class.unwrap_or("process_exit"); - self.bridge.observe_provider_line( - serde_json::json!({ - "type": "VastAiBootstrapAttemptCompleted", - "run_id": self.run_id(), - "node_id": self.node_id(), - "attempt": self.attempt, - "status": status.to_string(), - "class": observation_class, - "classification": readiness, - }) - .to_string(), - ); - self.schedule_retry(ctx); + let success = status.success(); + self.finish_completed_attempt(ctx, status.to_string(), success); } fn schedule_retry(&mut self, ctx: &Ctx) { @@ -912,20 +994,15 @@ impl SshBootstrapActor { )); self.backoff = std::cmp::min(self.backoff.saturating_mul(2), Duration::from_secs(30)); self.attempt = self.attempt.saturating_add(1); - schedule_ssh_message( - self.sender.clone(), - ctx.self_addr(), - SshBootstrapMsg::StartAttempt, - delay, - ); + self.schedule(ctx, SshBootstrapMsg::StartAttempt, delay); } fn join_readers(&mut self) { if let Some(reader) = self.stdout_reader.take() { - let _ = reader.join(); + reader.join(); } if let Some(reader) = self.stderr_reader.take() { - let _ = reader.join(); + reader.join(); } self.stdout_closed = true; self.stderr_closed = true; @@ -935,6 +1012,30 @@ impl SshBootstrapActor { stop_ssh_child(&mut self.child); self.join_readers(); } + + fn stop_relay(&mut self, ctx: &Ctx) { + if let Some(reader_relay) = self.reader_relay.take() { + let _ = ctx.stop_actor(reader_relay); + } + } + + fn mark_stopped(&mut self) { + if self.stopped { + return; + } + self.stopped = true; + self.bridge.observe_provider_line( + serde_json::json!({ + "type": "VastAiBootstrapStopped", + "run_id": self.run_id(), + "node_id": self.node_id(), + "attempt": self.attempt, + "child_active": self.child.is_some(), + "classification": "bootstrap_stopped", + }) + .to_string(), + ); + } } impl ActorInterface for SshBootstrapActor { @@ -942,7 +1043,19 @@ impl ActorInterface for SshBootstrapActor { type Response = (); fn on_start(&mut self, ctx: &Ctx) { - let _ = ctx.send(ctx.self_addr(), SshBootstrapMsg::StartAttempt); + match ctx.spawn(SshOutputRelay { + target: ctx.self_addr(), + }) { + Ok(reader_relay) => { + self.reader_relay = Some(reader_relay); + let _ = ctx.send(ctx.self_addr(), SshBootstrapMsg::StartAttempt); + } + Err(error) => { + self.bridge + .observe_provider_line(format!("spawn SSH output relay actor: {error}")); + ctx.stop_self(); + } + } } fn handle(&mut self, ctx: &Ctx, msg: Self::Incoming) { @@ -954,17 +1067,36 @@ impl ActorInterface for SshBootstrapActor { self.handle_reader_error(stream, error) } SshBootstrapMsg::ReaderClosed { stream } => self.handle_reader_closed(ctx, stream), + #[cfg(test)] + SshBootstrapMsg::ScriptedAttemptFinished { result } => { + self.stdout_closed = true; + self.stderr_closed = true; + match result { + Ok(status) => self.finish_completed_attempt( + ctx, + format!("exit status: {status}"), + status == 0, + ), + Err(error) => { + self.stop_child(); + self.pending_wait_error = Some(error); + self.maybe_finish_attempt(ctx); + } + } + } SshBootstrapMsg::Stop => { - self.stopped = true; + self.mark_stopped(); self.stop_child(); + self.stop_relay(ctx); ctx.stop_self(); } } } - fn on_stop(&mut self, _ctx: &Ctx) { - self.stopped = true; + fn on_stop(&mut self, ctx: &Ctx) { + self.mark_stopped(); self.stop_child(); + self.stop_relay(ctx); } } @@ -972,14 +1104,37 @@ fn stop_ssh_child(child: &mut Option) { let Some(mut child) = child.take() else { return; }; - let _ = child.kill(); - let _ = child.wait(); + let _ = swactor_process::child_kill(&mut child); + let _ = swactor_process::child_wait(&mut child); +} + +#[cfg(test)] +fn spawn_pending_test_child() +-> Result<(Child, std::process::ChildStdout, std::process::ChildStderr), String> { + let mut command = Command::new("sh"); + command + .args(["-c", "read _"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let mut child = swactor_process::command_spawn(&mut command) + .map_err(|error| format!("spawn pending SSH test child: {error}"))?; + let stdout = child + .stdout + .take() + .ok_or_else(|| "pending SSH test child missing stdout".to_owned())?; + let stderr = child + .stderr + .take() + .ok_or_else(|| "pending SSH test child missing stderr".to_owned())?; + Ok((child, stdout, stderr)) } #[derive(Clone)] pub(crate) struct SshCommandBootstrapLauncher { ssh_identity: Option, runtime: Runtime, + engine: EngineHandle, } pub(crate) struct SshCommandBootstrapHandle { actor: ActorAddress, @@ -987,10 +1142,15 @@ pub(crate) struct SshCommandBootstrapHandle { } impl SshCommandBootstrapLauncher { - pub(crate) fn new(ssh_identity: Option, runtime: Runtime) -> Self { + pub(crate) fn new( + ssh_identity: Option, + runtime: Runtime, + engine: EngineHandle, + ) -> Self { Self { ssh_identity, runtime, + engine, } } } @@ -1022,6 +1182,7 @@ impl VastAiBootstrapLauncher for SshCommandBootstrapLauncher { endpoint, self.ssh_identity.clone(), sender, + self.engine.clone(), )) .map_err(|e| format!("spawn VastAI SSH bootstrap actor: {e}"))?; @@ -1059,50 +1220,6 @@ fn classify_ssh_observation(line: &str) -> Option<&'static str> { None } -fn spawn_ssh_output_reader( - stream: SshBootstrapStream, - reader: R, - sender: ExternalSender, - actor: ActorAddress, -) -> JoinHandle<()> -where - R: Read + Send + 'static, -{ - thread::spawn(move || { - let reader = BufReader::new(reader); - for next in reader.lines() { - match next { - Ok(line) => { - let _ = sender.send_to(actor, SshBootstrapMsg::OutputLine { stream, line }); - } - Err(error) => { - let _ = sender.send_to( - actor, - SshBootstrapMsg::ReaderError { - stream, - error: error.to_string(), - }, - ); - break; - } - } - } - let _ = sender.send_to(actor, SshBootstrapMsg::ReaderClosed { stream }); - }) -} - -fn schedule_ssh_message( - sender: ExternalSender, - actor: ActorAddress, - msg: SshBootstrapMsg, - delay: Duration, -) { - thread::spawn(move || { - thread::sleep(delay); - let _ = sender.send_to(actor, msg); - }); -} - fn spawn_ssh_bootstrap_attempt( spec: &NodeProvisionSpec, endpoint: &VastAiSshEndpoint, @@ -1115,8 +1232,7 @@ fn spawn_ssh_bootstrap_attempt( .stdout(Stdio::piped()) .stderr(Stdio::piped()); - let mut child = command - .spawn() + let mut child = swactor_process::command_spawn(&mut command) .map_err(|e| format!("spawn VastAI SSH bootstrap {}: {e}", spec.node_id))?; let stdout = child .stdout @@ -1571,16 +1687,11 @@ where sink: PluginSink, ) -> Result, String> { let label = self.label_for(spec); - let mut contract = None; - for attempt in 0..10 { - contract = self.client.contract_by_label(&label)?; - if contract.is_some() { - break; - } - if attempt < 9 { - thread::sleep(self.config.lifecycle.lease_pace); - } - } + let contract = self.client.contract_by_label_with_retry( + &label, + 10, + self.config.lifecycle.lease_pace, + )?; if contract.is_none() { return Ok(None); } @@ -1624,9 +1735,16 @@ where #[cfg(test)] mod tests { + use parking_lot::Mutex; + use proptest::prelude::*; use serde_json::json; - use wiremock::matchers::{method, path}; - use wiremock::{Mock, MockServer, ResponseTemplate}; + use std::sync::Arc; + use std::sync::atomic::Ordering; + use swactor::runtime::{RuntimeConfig, RuntimeParts}; + use swactor_engine::{Engine, SteppingBackend}; + use swactor_vastai::test_http::{TestHttpRoute, TestHttpServer}; + + use crate::tests::fuzz_support::{actor_census, advance_and_drive, drive_steps}; use super::*; @@ -1653,27 +1771,22 @@ mod tests { #[test] fn provision_one_adopts_stable_label_before_offer_search() { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - let server = runtime.block_on(MockServer::start()); - runtime.block_on(async { - Mock::given(method("GET")) - .and(path("/api/v0/instances/")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "instances": [{ - "id": 73, - "label": "run-5-node-7-attempt-9", - "actual_status": "loading", - "ssh_host": "", - "ssh_port": 0, - "public_ipaddr": "" - }] - }))) - .mount(&server) - .await; - }); + let server = TestHttpServer::start(vec![TestHttpRoute::json( + "GET", + "/api/v0/instances/", + 200, + json!({ + "instances": [{ + "id": 73, + "label": "run-5-node-7-attempt-9", + "actual_status": "loading", + "ssh_host": "", + "ssh_port": 0, + "public_ipaddr": "" + }] + }), + )]) + .unwrap(); let client = swactor_vastai::VastClient::with_base_url(server.uri(), "secret"); let mut client = ToolsVastAiLeaseClient::new(client).unwrap(); let adopted = client @@ -1694,9 +1807,9 @@ mod tests { assert_eq!(adopted.contract_id, 73); assert_eq!(adopted.offer_id, 0); - let requests = runtime.block_on(server.received_requests()).unwrap(); + let requests = server.requests(); assert_eq!(requests.len(), 1); - assert_eq!(requests[0].url.path(), "/api/v0/instances/"); + assert_eq!(requests[0].path, "/api/v0/instances/"); } use std::sync::atomic::AtomicUsize; @@ -1895,30 +2008,25 @@ mod tests { #[test] fn offer_search_is_read_only() { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - let server = runtime.block_on(MockServer::start()); - runtime.block_on(async { - Mock::given(method("GET")) - .and(path("/api/v0/bundles/")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "offers": [{ - "id": 41, - "gpu_name": "A", - "dph_total": 0.2, - "host_id": 1, - "compute_cap": 800, - "reliability2": 0.99, - "inet_down": 500.0, - "inet_up": 500.0, - "geolocation": "US" - }] - }))) - .mount(&server) - .await; - }); + let server = TestHttpServer::start(vec![TestHttpRoute::json( + "GET", + "/api/v0/bundles/", + 200, + json!({ + "offers": [{ + "id": 41, + "gpu_name": "A", + "dph_total": 0.2, + "host_id": 1, + "compute_cap": 800, + "reliability2": 0.99, + "inet_down": 500.0, + "inet_up": 500.0, + "geolocation": "US" + }] + }), + )]) + .unwrap(); let mut client = ToolsVastAiLeaseClient::new(swactor_vastai::VastClient::with_base_url( server.uri(), "secret", @@ -1933,28 +2041,21 @@ mod tests { offers.iter().map(|offer| offer.id).collect::>(), [41] ); - let requests = runtime.block_on(server.received_requests()).unwrap(); + let requests = server.requests(); assert_eq!(requests.len(), 1); assert_eq!(requests[0].method.as_str(), "GET"); - assert_eq!(requests[0].url.path(), "/api/v0/bundles/"); + assert_eq!(requests[0].path, "/api/v0/bundles/"); } #[test] fn exact_offer_creation_never_requests_an_alternative() { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - let server = runtime.block_on(MockServer::start()); - runtime.block_on(async { - Mock::given(method("GET")) - .and(path("/api/v0/instances/")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({"instances": []}))) - .mount(&server) - .await; - Mock::given(method("GET")) - .and(path("/api/v0/bundles/")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + let server = TestHttpServer::start(vec![ + TestHttpRoute::json("GET", "/api/v0/instances/", 200, json!({"instances": []})), + TestHttpRoute::json( + "GET", + "/api/v0/bundles/", + 200, + json!({ "offers": [ {"id": 41, "gpu_name": "A", "dph_total": 0.2, "host_id": 1, "compute_cap": 800, "reliability2": 0.99, "inet_down": 500.0, @@ -1963,17 +2064,11 @@ mod tests { "compute_cap": 800, "reliability2": 0.99, "inet_down": 500.0, "geolocation": "US"} ] - }))) - .mount(&server) - .await; - Mock::given(method("PUT")) - .and(path("/api/v0/asks/42/")) - .respond_with( - ResponseTemplate::new(200).set_body_json(json!({"new_contract": 700})), - ) - .mount(&server) - .await; - }); + }), + ), + TestHttpRoute::json("PUT", "/api/v0/asks/42/", 200, json!({"new_contract": 700})), + ]) + .unwrap(); let mut client = ToolsVastAiLeaseClient::new(swactor_vastai::VastClient::with_base_url( server.uri(), "secret", @@ -1982,12 +2077,12 @@ mod tests { let instance = client.provision_exact(exact_request(), 42).unwrap(); assert_eq!(instance.offer_id, 42); assert_eq!(instance.contract_id, 700); - let requests = runtime.block_on(server.received_requests()).unwrap(); + let requests = server.requests(); assert_eq!( requests .iter() .filter(|request| request.method.as_str() == "PUT") - .map(|request| request.url.path().to_owned()) + .map(|request| request.path.clone()) .collect::>(), vec!["/api/v0/asks/42/"] ); @@ -1995,29 +2090,22 @@ mod tests { #[test] fn unavailable_exact_offer_fails_without_any_create_request() { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - let server = runtime.block_on(MockServer::start()); - runtime.block_on(async { - Mock::given(method("GET")) - .and(path("/api/v0/instances/")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({"instances": []}))) - .mount(&server) - .await; - Mock::given(method("GET")) - .and(path("/api/v0/bundles/")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + let server = TestHttpServer::start(vec![ + TestHttpRoute::json("GET", "/api/v0/instances/", 200, json!({"instances": []})), + TestHttpRoute::json( + "GET", + "/api/v0/bundles/", + 200, + json!({ "offers": [{ "id": 41, "gpu_name": "A", "dph_total": 0.2, "host_id": 1, "compute_cap": 800, "reliability2": 0.99, "inet_down": 500.0, "geolocation": "US" }] - }))) - .mount(&server) - .await; - }); + }), + ), + ]) + .unwrap(); let mut client = ToolsVastAiLeaseClient::new(swactor_vastai::VastClient::with_base_url( server.uri(), "secret", @@ -2029,11 +2117,1301 @@ mod tests { .unwrap_err() .contains("selected offer 42") ); - let requests = runtime.block_on(server.received_requests()).unwrap(); + let requests = server.requests(); assert!( requests .iter() .all(|request| request.method.as_str() != "PUT") ); } + + #[derive(Default)] + struct RecordingSink { + observations: Mutex>, + } + + impl PluginObservationSink for RecordingSink { + fn observe(&self, observation: PluginObservation) { + self.observations.lock().push(observation); + } + } + + fn offer_fixture(id: u64) -> serde_json::Value { + json!({ + "id": id, + "gpu_name": "RTX 4090", + "dph_total": 0.2, + "gpu_ram": 24_000.0, + "host_id": id.saturating_add(100), + "compute_cap": 890, + "reliability2": 0.99, + "inet_down": 500.0, + "inet_up": 250.0, + "geolocation": "US" + }) + } + + #[derive(Debug, PartialEq)] + enum OfferBrowseOutcome { + Offers(Vec), + Rejected(String), + } + + fn browse_offer_fixture(route: TestHttpRoute) -> (OfferBrowseOutcome, usize) { + let server = TestHttpServer::start(vec![route]).expect("start offer HTTP fixture"); + let mut client = ToolsVastAiLeaseClient::new(swactor_vastai::VastClient::with_base_url( + server.uri(), + "secret", + )) + .expect("blocking VastAI client"); + let outcome = match client.browse_offers(&OfferBrowseCriteria::default()) { + Ok(offers) => { + OfferBrowseOutcome::Offers(offers.into_iter().map(|offer| offer.id).collect()) + } + Err(error) => OfferBrowseOutcome::Rejected(error), + }; + (outcome, server.requests().len()) + } + + #[derive(Clone, Debug, PartialEq, Eq)] + enum TerminalOutcome { + MonitorRejected { + run_id: u64, + node_id: u64, + reason: String, + }, + BootstrapStopped { + run_id: u64, + node_id: u64, + attempt: u64, + }, + } + + #[derive(Debug, PartialEq, Eq)] + enum TerminalInvariantError { + MalformedTerminal(String), + DuplicateTerminal { + first: TerminalOutcome, + duplicate: TerminalOutcome, + }, + } + + fn single_terminal_outcome( + observations: &[PluginObservation], + ) -> Result, TerminalInvariantError> { + let mut terminal = None; + for observation in observations { + let candidate = match observation { + PluginObservation::Failed { + run_id, + node_id, + reason, + } => Some(TerminalOutcome::MonitorRejected { + run_id: *run_id, + node_id: *node_id, + reason: reason.clone(), + }), + PluginObservation::ProviderLine { + run_id: observed_run_id, + node_id: observed_node_id, + line, + } => { + let Ok(value) = serde_json::from_str::(line) else { + continue; + }; + if value.get("type").and_then(serde_json::Value::as_str) + != Some("VastAiBootstrapStopped") + { + continue; + } + let run_id = value + .get("run_id") + .and_then(serde_json::Value::as_u64) + .ok_or_else(|| TerminalInvariantError::MalformedTerminal(line.clone()))?; + let node_id = value + .get("node_id") + .and_then(serde_json::Value::as_u64) + .ok_or_else(|| TerminalInvariantError::MalformedTerminal(line.clone()))?; + let attempt = value + .get("attempt") + .and_then(serde_json::Value::as_u64) + .ok_or_else(|| TerminalInvariantError::MalformedTerminal(line.clone()))?; + if run_id != *observed_run_id || node_id != *observed_node_id { + return Err(TerminalInvariantError::MalformedTerminal(line.clone())); + } + Some(TerminalOutcome::BootstrapStopped { + run_id, + node_id, + attempt, + }) + } + _ => None, + }; + let Some(candidate) = candidate else { + continue; + }; + if let Some(first) = terminal { + return Err(TerminalInvariantError::DuplicateTerminal { + first, + duplicate: candidate, + }); + } + terminal = Some(candidate); + } + Ok(terminal) + } + + fn observations(recording: &RecordingSink) -> Vec { + recording.observations.lock().clone() + } + + fn runtime_is_clean(runtime: &Runtime, baseline: usize, actor_limit: usize) -> bool { + let stats = runtime.stats(); + stats.actors.len() <= baseline.saturating_add(actor_limit) + && stats.actor_details.iter().all(|actor| !actor.poisoned) + && stats + .workers + .iter() + .map(|worker| worker.panics) + .sum::() + == 0 + } + + fn monitor_spec(run_id: u64, node_id: u64) -> NodeProvisionSpec { + NodeProvisionSpec { + run_id, + node_id, + attempt_id: 9, + stage_index: Some(0), + image: "node:v1".to_owned(), + env: Vec::new(), + args: Vec::new(), + mounts: Vec::new(), + } + } + + fn monitor_route(contract_id: u64, kind: u8) -> TestHttpRoute { + let path = format!("/api/v0/instances/{contract_id}/"); + match kind { + 0 => TestHttpRoute::json( + "GET", + &path, + 200, + json!({"instances": { + "actual_status": "running", + "intended_status": "running", + "public_ipaddr": "127.0.0.1", + "ssh_port": 22 + }}), + ), + 1 => TestHttpRoute::json( + "GET", + &path, + 200, + json!({"instances": { + "actual_status": "error", + "intended_status": "running", + "status_msg": "container failed" + }}), + ), + 2 => TestHttpRoute::raw("GET", &path, 404, b"missing".to_vec()), + 3 => TestHttpRoute::raw("GET", &path, 200, b"{broken".to_vec()), + 4 => TestHttpRoute::raw("GET", &path, 500, b"retry".to_vec()), + _ => TestHttpRoute::json( + "GET", + &path, + 200, + json!({"instances": [ + {"actual_status": "running", "intended_status": "running"}, + {"actual_status": "error", "intended_status": "running"} + ]}), + ), + } + } + + struct MonitorHarness { + server: TestHttpServer, + runtime: Runtime, + backend: SteppingBackend, + _engine: Engine, + actor: ActorAddress, + baseline: usize, + recording: Arc, + } + + fn monitor_harness(contract_id: u64, spec: NodeProvisionSpec, kind: u8) -> MonitorHarness { + let server = TestHttpServer::start(vec![monitor_route(contract_id, kind)]) + .expect("start monitor HTTP fixture"); + let client = ToolsVastAiLeaseClient::new(swactor_vastai::VastClient::with_base_url( + server.uri(), + "secret", + )) + .expect("blocking VastAI client"); + let parts = RuntimeParts::new(RuntimeConfig::default()); + let runtime = parts.runtime().clone(); + let backend = SteppingBackend::new(); + let engine = Engine::new(parts, backend.clone()).expect("stepping engine"); + let baseline = runtime.stats().actors.len(); + let recording = Arc::new(RecordingSink::default()); + let actor = runtime + .spawn(VastAiProviderMonitorActor::new( + client, + contract_id, + format!( + "run-{}-node-{}-attempt-{}", + spec.run_id, spec.node_id, spec.attempt_id + ), + LifecyclePolicy { + lease_pace: Duration::ZERO, + poll_interval: Duration::from_millis(1), + state_timeout: Duration::from_millis(10), + }, + spec, + PluginSink::new(recording.clone()), + runtime.create_sender(), + engine.handle(), + )) + .expect("spawn VastAI monitor"); + MonitorHarness { + server, + runtime, + backend, + _engine: engine, + actor, + baseline, + recording, + } + } + + struct SshHarness { + runtime: Runtime, + backend: SteppingBackend, + _engine: Engine, + actor: ActorAddress, + baseline: usize, + recording: Arc, + } + + #[derive(Clone, Copy)] + enum SshHarnessMode { + Idle, + OpenStreams, + PendingChild, + } + + fn ssh_harness_with_mode(mode: SshHarnessMode) -> SshHarness { + let parts = RuntimeParts::new(RuntimeConfig::default()); + let runtime = parts.runtime().clone(); + let backend = SteppingBackend::new(); + let engine = Engine::new(parts, backend.clone()).expect("stepping engine"); + let baseline = runtime.stats().actors.len(); + let recording = Arc::new(RecordingSink::default()); + let bridge = BootstrapTelemetryBridge::new( + monitor_spec(5, 7), + PluginSink::new(recording.clone()), + None, + ); + let actor = SshBootstrapActor::new( + bridge, + VastAiSshEndpoint { + host: "scripted.invalid".to_owned(), + port: 22, + user: "root".to_owned(), + }, + None, + runtime.create_sender(), + engine.handle(), + ); + let actor = match mode { + SshHarnessMode::Idle => actor.with_attempt_spawn_disabled(), + SshHarnessMode::OpenStreams => actor.with_open_test_streams(), + SshHarnessMode::PendingChild => actor.with_pending_test_child(), + }; + let actor = runtime.spawn(actor).expect("spawn SSH bootstrap actor"); + drive_steps(&backend, 8); + SshHarness { + runtime, + backend, + _engine: engine, + actor, + baseline, + recording, + } + } + + fn ssh_harness() -> SshHarness { + ssh_harness_with_mode(SshHarnessMode::Idle) + } + + fn malformed_protocol_line(kind: u8, nonce: u16) -> String { + match kind % 8 { + 0 => "{broken".to_owned(), + 1 => json!({"myelin_stdio_event": 1}).to_string(), + 2 => json!({ + "myelin_stdio_event": 2, + "kind": "telemetry_frame", + "channel": "test", + "payload": {"nonce": nonce} + }) + .to_string(), + 3 => json!({ + "myelin_stdio_event": 1, + "kind": "wrong", + "channel": "test", + "payload": {"nonce": nonce} + }) + .to_string(), + 4 => json!({ + "myelin_stdio_event": 1, + "kind": "telemetry_frame", + "channel": nonce + }) + .to_string(), + 5 => json!([1, 2, nonce]).to_string(), + 6 => "null".to_owned(), + _ => format!("malformed-protocol-{nonce}"), + } + } + + #[derive(Clone, Debug)] + enum SshOutputAction { + Stdout(String), + Stderr(String), + Protocol(u16), + } + + fn ssh_output_actions() -> impl Strategy> { + prop::collection::vec( + prop_oneof![ + "[ -~]{0,32}".prop_map(SshOutputAction::Stdout), + "[ -~]{0,32}".prop_map(SshOutputAction::Stderr), + any::().prop_map(SshOutputAction::Protocol), + ], + 0..=32, + ) + } + + #[derive(Clone, Debug)] + enum SshOrderingAction { + Poll, + Stdout(String), + ReaderError(bool), + Advance(u16), + Stop, + } + + fn ssh_ordering_actions() -> impl Strategy> { + prop::collection::vec( + prop_oneof![ + Just(SshOrderingAction::Poll), + "[ -~]{0,16}".prop_map(SshOrderingAction::Stdout), + any::().prop_map(SshOrderingAction::ReaderError), + (0_u16..=1_000).prop_map(SshOrderingAction::Advance), + Just(SshOrderingAction::Stop), + ], + 0..=32, + ) + } + + proptest! { + #![proptest_config(ProptestConfig { + cases: 128, + max_shrink_iters: 2_000, + ..ProptestConfig::default() + })] + + #[test] + fn offer_status_classes_are_offers_or_typed_rejections( + status_index in 0_usize..11, + empty in any::(), + ) { + let statuses = [200, 201, 206, 204, 300, 302, 400, 404, 429, 500, 503]; + let status = statuses[status_index]; + let expected_ids = if empty { Vec::new() } else { vec![41] }; + let offers = if empty { + Vec::new() + } else { + vec![offer_fixture(41)] + }; + let (outcome, request_count) = browse_offer_fixture(TestHttpRoute::json( + "GET", + "/api/v0/bundles/", + status, + json!({"offers": offers}), + )); + let should_parse = matches!(status, 200 | 201 | 206); + prop_assert_eq!( + matches!(&outcome, OfferBrowseOutcome::Offers(_)), + should_parse, + "status={}, outcome={:?}", + status, + outcome, + ); + if should_parse { + prop_assert_eq!( + &outcome, + &OfferBrowseOutcome::Offers(expected_ids), + "status={}, empty={}, outcome={:?}", + status, + empty, + outcome, + ); + } + if let OfferBrowseOutcome::Rejected(error) = &outcome { + prop_assert!( + !error.trim().is_empty(), + "status={}, typed rejection was empty: {:?}", + status, + outcome, + ); + } + prop_assert_eq!( + request_count, + 1, + "status={}, outcome={:?}", + status, + outcome, + ); + } + + #[test] + fn malformed_offer_bodies_are_typed_rejections( + kind in 0_u8..5, + fuzz in prop::collection::vec(any::(), 0..=32), + ) { + let body = match kind { + 0 => Vec::new(), + 1 => { + let mut body = b"{broken".to_vec(); + body.extend_from_slice(&fuzz); + body.push(0xff); + body + } + 2 => { + let mut body = b"{\"offers\":[".to_vec(); + body.extend_from_slice(&fuzz); + body.push(0xff); + body + } + 3 => { + let mut body = fuzz.clone(); + body.push(0xff); + body + } + _ => { + let mut body = b"[".to_vec(); + body.extend_from_slice(&fuzz); + body.push(0xff); + body + } + }; + let (outcome, request_count) = browse_offer_fixture(TestHttpRoute::raw( + "GET", + "/api/v0/bundles/", + 200, + body, + )); + prop_assert!( + matches!(&outcome, OfferBrowseOutcome::Rejected(error) if !error.trim().is_empty()), + "kind={}, fuzz={:?}, outcome={:?}", + kind, + fuzz, + outcome, + ); + prop_assert_eq!( + request_count, + 1, + "kind={}, fuzz={:?}, outcome={:?}", + kind, + fuzz, + outcome, + ); + } + + #[test] + fn wrong_or_missing_offer_fields_are_typed_rejections(kind in 0_u8..8) { + let body = match kind { + 0 => json!({}), + 1 => json!({"offers": "wrong"}), + 2 => json!({"offers": {}}), + 3 => json!({"offers": [{"gpu_name": "A", "dph_total": 0.2}]}), + 4 => json!({"offers": [{"id": "41", "gpu_name": "A", "dph_total": 0.2}]}), + 5 => json!({"offers": [{"id": 41, "gpu_name": 9, "dph_total": 0.2}]}), + 6 => json!({"offers": [{"id": 41, "gpu_name": "A", "dph_total": "cheap"}]}), + _ => json!({"offers": null}), + }; + let (outcome, request_count) = browse_offer_fixture(TestHttpRoute::json( + "GET", + "/api/v0/bundles/", + 200, + body, + )); + prop_assert!( + matches!(&outcome, OfferBrowseOutcome::Rejected(error) if !error.trim().is_empty()), + "wrong-field kind={}, outcome={:?}", + kind, + outcome, + ); + prop_assert_eq!( + request_count, + 1, + "wrong-field kind={}, outcome={:?}", + kind, + outcome, + ); + } + + #[test] + fn duplicate_offer_records_remain_explicit_values( + id in 0_u64..=1_000_000, + repetitions in 2_usize..=32, + ) { + let repeated = (0..repetitions).map(|_| offer_fixture(id)).collect::>(); + let (outcome, request_count) = browse_offer_fixture(TestHttpRoute::json( + "GET", + "/api/v0/bundles/", + 200, + json!({"offers": repeated}), + )); + prop_assert_eq!( + outcome, + OfferBrowseOutcome::Offers(vec![id; repetitions]), + "id={}, repetitions={}", + id, + repetitions, + ); + prop_assert_eq!( + request_count, + 1, + "id={}, repetitions={}", + id, + repetitions, + ); + } + + #[test] + fn provider_monitor_preserves_contract_identity_and_cardinality( + run_id in 1_u64..=10_000, + node_id in 1_u64..=10_000, + contract_id in 1_u64..=10_000, + ) { + let actions = [ + format!("spawn({contract_id})"), + "initial_poll".to_owned(), + "stop".to_owned(), + ]; + let harness = monitor_harness(contract_id, monitor_spec(run_id, node_id), 0); + drive_steps(&harness.backend, 8); + let before_stop = observations(&harness.recording); + let census = actor_census(&harness.runtime); + prop_assert!( + harness.runtime.stats().actors.len() == harness.baseline + 1, + "actions={:?}, outcomes={:?}, census={}", + actions, + before_stop, + census, + ); + prop_assert!( + harness.server.requests().iter().all(|request| { + request.path == format!("/api/v0/instances/{contract_id}/") + }), + "actions={:?}, outcomes={:?}, requests={:?}, census={}", + actions, + before_stop, + harness.server.requests(), + census, + ); + let identities = before_stop + .iter() + .filter_map(|observation| match observation { + PluginObservation::ProviderLine { + run_id: observed_run, + node_id: observed_node, + line, + } => serde_json::from_str::(line) + .ok() + .filter(|value| { + value.get("type").and_then(serde_json::Value::as_str) + == Some("VastAiProviderStatusObserved") + }) + .map(|value| (*observed_run, *observed_node, value)), + _ => None, + }) + .collect::>(); + prop_assert!( + !identities.is_empty() + && identities.iter().all(|(observed_run, observed_node, value)| { + *observed_run == run_id + && *observed_node == node_id + && value.get("run_id").and_then(serde_json::Value::as_u64) + == Some(run_id) + && value.get("node_id").and_then(serde_json::Value::as_u64) + == Some(node_id) + && value.get("contract_id").and_then(serde_json::Value::as_u64) + == Some(contract_id) + }), + "actions={:?}, identities={:?}, outcomes={:?}, census={}", + actions, + identities, + before_stop, + census, + ); + let _ = harness + .runtime + .send_to(harness.actor, VastAiProviderMonitorMsg::Stop); + advance_and_drive(&harness.backend, Duration::from_secs(1), 64); + let final_outcomes = observations(&harness.recording); + prop_assert!( + runtime_is_clean(&harness.runtime, harness.baseline, 0), + "actions={:?}, outcomes={:?}, census={}", + actions, + final_outcomes, + actor_census(&harness.runtime), + ); + } + + #[test] + fn provider_monitor_terminal_polling_stops_after_one_typed_outcome( + terminal_kind in 0_usize..4, + extra_polls in 0_usize..=32, + ) { + let kinds = [1_u8, 2, 3, 5]; + let kind = kinds[terminal_kind]; + let actions = vec!["poll"; extra_polls]; + let harness = monitor_harness(73, monitor_spec(5, 7), kind); + for _ in 0..extra_polls { + let _ = harness + .runtime + .send_to(harness.actor, VastAiProviderMonitorMsg::Poll); + } + drive_steps(&harness.backend, 64); + let first_request_count = harness.server.requests().len(); + advance_and_drive(&harness.backend, Duration::from_secs(1), 64); + let second_request_count = harness.server.requests().len(); + let outcomes = observations(&harness.recording); + let terminal = single_terminal_outcome(&outcomes); + let census = actor_census(&harness.runtime); + prop_assert_eq!( + first_request_count, + second_request_count, + "kind={}, actions={:?}, outcomes={:?}, census={}", + kind, + actions, + outcomes, + census, + ); + prop_assert_eq!( + first_request_count, + 1, + "kind={}, actions={:?}, outcomes={:?}, census={}", + kind, + actions, + outcomes, + census, + ); + prop_assert!( + matches!( + &terminal, + Ok(Some(TerminalOutcome::MonitorRejected { reason, .. })) + if reason.contains("[class=") + ), + "kind={}, actions={:?}, terminal={:?}, outcomes={:?}, census={}", + kind, + actions, + terminal, + outcomes, + census, + ); + prop_assert!( + runtime_is_clean(&harness.runtime, harness.baseline, 0), + "kind={}, actions={:?}, terminal={:?}, outcomes={:?}, census={}", + kind, + actions, + terminal, + outcomes, + census, + ); + } + + #[test] + fn provider_monitor_poll_stop_orderings_cease_polling( + retrying in any::(), + actions in prop::collection::vec(0_u8..3, 0..=32), + ) { + let kind = if retrying { 4 } else { 0 }; + let harness = monitor_harness(73, monitor_spec(5, 7), kind); + drive_steps(&harness.backend, 8); + let mut stopped = false; + let mut stopped_request_count = None; + for action in &actions { + match action { + 0 => { + let _ = harness + .runtime + .send_to(harness.actor, VastAiProviderMonitorMsg::Poll); + drive_steps(&harness.backend, 8); + } + 1 => advance_and_drive( + &harness.backend, + Duration::from_millis(1), + 8, + ), + _ => { + let _ = harness + .runtime + .send_to(harness.actor, VastAiProviderMonitorMsg::Stop); + drive_steps(&harness.backend, 8); + stopped = true; + stopped_request_count + .get_or_insert_with(|| harness.server.requests().len()); + } + } + let outcomes = observations(&harness.recording); + prop_assert!( + runtime_is_clean(&harness.runtime, harness.baseline, 1), + "actions={:?}, stopped={}, outcomes={:?}, census={}", + actions, + stopped, + outcomes, + actor_census(&harness.runtime), + ); + if let Some(count) = stopped_request_count { + prop_assert_eq!( + harness.server.requests().len(), + count, + "polling resumed after stop; actions={:?}, outcomes={:?}, census={}", + actions, + outcomes, + actor_census(&harness.runtime), + ); + } + } + let _ = harness + .runtime + .send_to(harness.actor, VastAiProviderMonitorMsg::Stop); + advance_and_drive(&harness.backend, Duration::from_secs(1), 64); + let settled_requests = harness.server.requests().len(); + advance_and_drive(&harness.backend, Duration::from_secs(1), 64); + let outcomes = observations(&harness.recording); + prop_assert_eq!( + harness.server.requests().len(), + settled_requests, + "polling did not cease; actions={:?}, stopped={}, outcomes={:?}, census={}", + actions, + stopped, + outcomes, + actor_census(&harness.runtime), + ); + prop_assert!( + runtime_is_clean(&harness.runtime, harness.baseline, 0), + "actions={:?}, stopped={}, outcomes={:?}, census={}", + actions, + stopped, + outcomes, + actor_census(&harness.runtime), + ); + } + + #[test] + fn duplicate_terminal_detector_rejects_controlled_fault( + run_id in any::(), + node_id in any::(), + ) { + let injected = vec![ + PluginObservation::ProviderLine { + run_id, + node_id, + line: json!({ + "type": "VastAiBootstrapStopped", + "run_id": run_id, + "node_id": node_id, + "attempt": 1, + }) + .to_string(), + }, + PluginObservation::ProviderLine { + run_id, + node_id, + line: json!({ + "type": "VastAiBootstrapStopped", + "run_id": run_id, + "node_id": node_id, + "attempt": 2, + }) + .to_string(), + }, + ]; + let detected = single_terminal_outcome(&injected); + prop_assert!( + matches!( + &detected, + Err(TerminalInvariantError::DuplicateTerminal { .. }) + ), + "controlled duplicate terminal escaped detector; actions=[inject_first, inject_duplicate], outcomes={:?}, detected={:?}", + injected, + detected, + ); + } + + #[test] + fn ssh_bootstrap_output_lines_preserve_stream_and_protocol(actions in ssh_output_actions()) { + let harness = ssh_harness(); + let initial_census = actor_census(&harness.runtime); + prop_assert!( + harness.runtime.stats().actors.len() == harness.baseline + 2, + "actions={:?}, outcomes={:?}, census={}", + actions, + observations(&harness.recording), + initial_census, + ); + for action in &actions { + let message = match action { + SshOutputAction::Stdout(line) => SshBootstrapMsg::OutputLine { + stream: SshBootstrapStream::Stdout, + line: line.clone(), + }, + SshOutputAction::Stderr(line) => SshBootstrapMsg::OutputLine { + stream: SshBootstrapStream::Stderr, + line: line.clone(), + }, + SshOutputAction::Protocol(nonce) => SshBootstrapMsg::OutputLine { + stream: SshBootstrapStream::Stdout, + line: json!({ + "myelin_stdio_event": 1, + "kind": "telemetry_frame", + "channel": "generated", + "payload": {"nonce": nonce} + }) + .to_string(), + }, + }; + harness + .runtime + .send_to(harness.actor, message) + .unwrap_or_else(|error| { + panic!( + "queue SSH output failed: error={error}, action={action:?}, actions={actions:?}, outcomes={:?}, census={}", + observations(&harness.recording), + actor_census(&harness.runtime), + ) + }); + drive_steps(&harness.backend, 4); + } + let _ = harness.runtime.send_to(harness.actor, SshBootstrapMsg::Stop); + advance_and_drive(&harness.backend, Duration::from_secs(31), 128); + let outcomes = observations(&harness.recording); + let expected_stdout = actions + .iter() + .filter(|action| matches!(action, SshOutputAction::Stdout(_))) + .count(); + let expected_stderr = actions + .iter() + .filter(|action| matches!(action, SshOutputAction::Stderr(_))) + .count(); + let expected_protocol = actions + .iter() + .filter(|action| matches!(action, SshOutputAction::Protocol(_))) + .count(); + prop_assert_eq!( + outcomes + .iter() + .filter(|outcome| matches!(outcome, PluginObservation::StdoutLine { .. })) + .count(), + expected_stdout, + "actions={:?}, outcomes={:?}, census={}", + actions, + outcomes, + actor_census(&harness.runtime), + ); + prop_assert_eq!( + outcomes + .iter() + .filter(|outcome| matches!(outcome, PluginObservation::StderrLine { .. })) + .count(), + expected_stderr, + "actions={:?}, outcomes={:?}, census={}", + actions, + outcomes, + actor_census(&harness.runtime), + ); + prop_assert_eq!( + outcomes + .iter() + .filter(|outcome| matches!(outcome, PluginObservation::TelemetryFrame { .. })) + .count(), + expected_protocol, + "actions={:?}, outcomes={:?}, census={}", + actions, + outcomes, + actor_census(&harness.runtime), + ); + let terminal = single_terminal_outcome(&outcomes); + prop_assert!( + matches!(&terminal, Ok(Some(TerminalOutcome::BootstrapStopped { .. }))), + "actions={:?}, terminal={:?}, outcomes={:?}, census={}", + actions, + terminal, + outcomes, + actor_census(&harness.runtime), + ); + prop_assert!( + runtime_is_clean(&harness.runtime, harness.baseline, 0), + "actions={:?}, terminal={:?}, outcomes={:?}, census={}", + actions, + terminal, + outcomes, + actor_census(&harness.runtime), + ); + } + + #[test] + fn ssh_bootstrap_malformed_protocol_is_data_not_poison( + actions in prop::collection::vec((0_u8..8, any::()), 0..=32), + ) { + let harness = ssh_harness(); + for (kind, nonce) in &actions { + harness + .runtime + .send_to( + harness.actor, + SshBootstrapMsg::OutputLine { + stream: SshBootstrapStream::Stdout, + line: malformed_protocol_line(*kind, *nonce), + }, + ) + .unwrap_or_else(|error| { + panic!( + "queue malformed SSH protocol failed: error={error}, action=({kind}, {nonce}), actions={actions:?}, outcomes={:?}, census={}", + observations(&harness.recording), + actor_census(&harness.runtime), + ) + }); + drive_steps(&harness.backend, 4); + } + let _ = harness.runtime.send_to(harness.actor, SshBootstrapMsg::Stop); + advance_and_drive(&harness.backend, Duration::from_secs(31), 128); + let outcomes = observations(&harness.recording); + prop_assert_eq!( + outcomes + .iter() + .filter(|outcome| matches!(outcome, PluginObservation::StdoutLine { .. })) + .count(), + actions.len(), + "actions={:?}, outcomes={:?}, census={}", + actions, + outcomes, + actor_census(&harness.runtime), + ); + prop_assert_eq!( + outcomes + .iter() + .filter(|outcome| matches!(outcome, PluginObservation::TelemetryFrame { .. })) + .count(), + 0, + "actions={:?}, outcomes={:?}, census={}", + actions, + outcomes, + actor_census(&harness.runtime), + ); + let terminal = single_terminal_outcome(&outcomes); + prop_assert!( + matches!(&terminal, Ok(Some(TerminalOutcome::BootstrapStopped { .. }))) + && runtime_is_clean(&harness.runtime, harness.baseline, 0), + "actions={:?}, terminal={:?}, outcomes={:?}, census={}", + actions, + terminal, + outcomes, + actor_census(&harness.runtime), + ); + } + + #[test] + fn ssh_bootstrap_eof_orderings_stop_relay_and_actor( + generated_actions in prop::collection::vec(any::(), 0..=30), + ) { + let mut actions = generated_actions; + actions.push(false); + actions.push(true); + let harness = ssh_harness_with_mode(SshHarnessMode::OpenStreams); + for stderr in &actions { + harness + .runtime + .send_to( + harness.actor, + SshBootstrapMsg::ReaderClosed { + stream: if *stderr { + SshBootstrapStream::Stderr + } else { + SshBootstrapStream::Stdout + }, + }, + ) + .unwrap_or_else(|error| { + panic!( + "queue SSH EOF failed: error={error}, action={stderr}, actions={actions:?}, outcomes={:?}, census={}", + observations(&harness.recording), + actor_census(&harness.runtime), + ) + }); + drive_steps(&harness.backend, 4); + } + let _ = harness.runtime.send_to(harness.actor, SshBootstrapMsg::Stop); + advance_and_drive(&harness.backend, Duration::from_secs(31), 128); + let outcomes = observations(&harness.recording); + let terminal = single_terminal_outcome(&outcomes); + prop_assert!( + matches!(&terminal, Ok(Some(TerminalOutcome::BootstrapStopped { .. }))) + && runtime_is_clean(&harness.runtime, harness.baseline, 0), + "actions={:?}, terminal={:?}, outcomes={:?}, census={}", + actions, + terminal, + outcomes, + actor_census(&harness.runtime), + ); + } + + #[test] + fn ssh_bootstrap_child_failures_have_typed_attempt_outcomes( + actions in prop::collection::vec(1_i32..=255, 1..=32), + ) { + let harness = ssh_harness(); + for status in &actions { + harness + .runtime + .send_to( + harness.actor, + SshBootstrapMsg::ScriptedAttemptFinished { + result: Ok(*status), + }, + ) + .unwrap_or_else(|error| { + panic!( + "queue SSH child failure failed: error={error}, action={status}, actions={actions:?}, outcomes={:?}, census={}", + observations(&harness.recording), + actor_census(&harness.runtime), + ) + }); + drive_steps(&harness.backend, 4); + } + let _ = harness.runtime.send_to(harness.actor, SshBootstrapMsg::Stop); + advance_and_drive(&harness.backend, Duration::from_secs(31), 128); + let outcomes = observations(&harness.recording); + let completions = outcomes + .iter() + .filter_map(|outcome| match outcome { + PluginObservation::ProviderLine { line, .. } => { + serde_json::from_str::(line).ok() + } + _ => None, + }) + .filter(|value| { + value.get("type").and_then(serde_json::Value::as_str) + == Some("VastAiBootstrapAttemptCompleted") + && value + .get("classification") + .and_then(serde_json::Value::as_str) + == Some("not ready before runtime ready") + }) + .count(); + prop_assert_eq!( + completions, + actions.len(), + "actions={:?}, outcomes={:?}, census={}", + actions, + outcomes, + actor_census(&harness.runtime), + ); + let terminal = single_terminal_outcome(&outcomes); + prop_assert!( + matches!(&terminal, Ok(Some(TerminalOutcome::BootstrapStopped { .. }))) + && runtime_is_clean(&harness.runtime, harness.baseline, 0), + "actions={:?}, terminal={:?}, outcomes={:?}, census={}", + actions, + terminal, + outcomes, + actor_census(&harness.runtime), + ); + } + + #[test] + fn ssh_bootstrap_timeout_is_typed_and_stops_polling( + advances in prop::collection::vec(0_u16..=1_000, 0..=31), + ) { + let harness = ssh_harness_with_mode(SshHarnessMode::PendingChild); + advance_and_drive(&harness.backend, Duration::from_millis(100), 8); + let initial_outcomes = observations(&harness.recording); + prop_assert!( + harness.runtime.stats().actors.len() == harness.baseline + 2 + && !initial_outcomes.iter().any(|outcome| matches!( + outcome, + PluginObservation::ProviderLine { line, .. } + if line.contains("spawn VastAI SSH bootstrap attempt") + && line.contains("failed") + )), + "actions=[start_pending_child, poll, timeout, {:?}, stop], outcomes={:?}, census={}", + advances, + initial_outcomes, + actor_census(&harness.runtime), + ); + harness + .runtime + .send_to( + harness.actor, + SshBootstrapMsg::ScriptedAttemptFinished { + result: Err("bootstrap timeout".to_owned()), + }, + ) + .unwrap_or_else(|error| { + panic!( + "queue SSH timeout failed: error={error}, actions=[timeout, {advances:?}], outcomes={:?}, census={}", + observations(&harness.recording), + actor_census(&harness.runtime), + ) + }); + drive_steps(&harness.backend, 4); + for millis in &advances { + advance_and_drive( + &harness.backend, + Duration::from_millis(u64::from(*millis)), + 4, + ); + } + let _ = harness.runtime.send_to(harness.actor, SshBootstrapMsg::Stop); + advance_and_drive(&harness.backend, Duration::from_secs(31), 128); + let outcomes = observations(&harness.recording); + prop_assert!( + outcomes.iter().any(|outcome| matches!( + outcome, + PluginObservation::ProviderLine { line, .. } + if line.contains("bootstrap timeout") && line.contains("retrying") + )), + "actions=[timeout, {:?}, stop], outcomes={:?}, census={}", + advances, + outcomes, + actor_census(&harness.runtime), + ); + let settled_attempts = outcomes + .iter() + .filter(|outcome| matches!( + outcome, + PluginObservation::ProviderLine { line, .. } + if line.contains("VastAI SSH bootstrap attempt") + )) + .count(); + advance_and_drive(&harness.backend, Duration::from_secs(31), 128); + let final_outcomes = observations(&harness.recording); + let final_attempts = final_outcomes + .iter() + .filter(|outcome| matches!( + outcome, + PluginObservation::ProviderLine { line, .. } + if line.contains("VastAI SSH bootstrap attempt") + )) + .count(); + let terminal = single_terminal_outcome(&final_outcomes); + prop_assert_eq!( + final_attempts, + settled_attempts, + "polling resumed after timeout stop; actions=[timeout, {:?}, stop], terminal={:?}, outcomes={:?}, census={}", + advances, + terminal, + final_outcomes, + actor_census(&harness.runtime), + ); + prop_assert!( + matches!(&terminal, Ok(Some(TerminalOutcome::BootstrapStopped { .. }))) + && runtime_is_clean(&harness.runtime, harness.baseline, 0), + "actions=[timeout, {:?}, stop], terminal={:?}, outcomes={:?}, census={}", + advances, + terminal, + final_outcomes, + actor_census(&harness.runtime), + ); + } + + #[test] + fn ssh_bootstrap_stop_orderings_emit_one_terminal_and_stop_all_actors( + actions in ssh_ordering_actions(), + ) { + let harness = ssh_harness(); + let mut stopped = false; + for action in &actions { + match action { + SshOrderingAction::Poll => { + let _ = harness + .runtime + .send_to(harness.actor, SshBootstrapMsg::PollChild); + drive_steps(&harness.backend, 4); + } + SshOrderingAction::Stdout(line) => { + let _ = harness.runtime.send_to( + harness.actor, + SshBootstrapMsg::OutputLine { + stream: SshBootstrapStream::Stdout, + line: line.clone(), + }, + ); + drive_steps(&harness.backend, 4); + } + SshOrderingAction::ReaderError(stderr) => { + let _ = harness.runtime.send_to( + harness.actor, + SshBootstrapMsg::ReaderError { + stream: if *stderr { + SshBootstrapStream::Stderr + } else { + SshBootstrapStream::Stdout + }, + error: "scripted reader failure".to_owned(), + }, + ); + drive_steps(&harness.backend, 4); + } + SshOrderingAction::Advance(millis) => advance_and_drive( + &harness.backend, + Duration::from_millis(u64::from(*millis)), + 4, + ), + SshOrderingAction::Stop => { + let _ = harness + .runtime + .send_to(harness.actor, SshBootstrapMsg::Stop); + drive_steps(&harness.backend, 4); + stopped = true; + } + } + let outcomes = observations(&harness.recording); + prop_assert!( + runtime_is_clean(&harness.runtime, harness.baseline, 2), + "actions={:?}, stopped={}, outcomes={:?}, census={}", + actions, + stopped, + outcomes, + actor_census(&harness.runtime), + ); + } + let _ = harness.runtime.send_to(harness.actor, SshBootstrapMsg::Stop); + advance_and_drive(&harness.backend, Duration::from_secs(31), 128); + let outcomes = observations(&harness.recording); + let terminal = single_terminal_outcome(&outcomes); + prop_assert!( + matches!(&terminal, Ok(Some(TerminalOutcome::BootstrapStopped { .. }))), + "actions={:?}, stopped={}, terminal={:?}, outcomes={:?}, census={}", + actions, + stopped, + terminal, + outcomes, + actor_census(&harness.runtime), + ); + prop_assert!( + runtime_is_clean(&harness.runtime, harness.baseline, 0), + "actions={:?}, stopped={}, terminal={:?}, outcomes={:?}, census={}", + actions, + stopped, + terminal, + outcomes, + actor_census(&harness.runtime), + ); + } + } } diff --git a/apps/myelin/src/orchestration/provisioning.rs b/apps/myelin/src/orchestration/provisioning.rs index 5a55bea..41734ea 100644 --- a/apps/myelin/src/orchestration/provisioning.rs +++ b/apps/myelin/src/orchestration/provisioning.rs @@ -6,12 +6,11 @@ use std::collections::BTreeMap; use std::fs::{self, File, OpenOptions}; -use std::io::{Read, Write}; +use std::io::Write; use std::path::{Path, PathBuf}; use std::process::{Child, ChildStdin, Command, Stdio}; use std::sync::{Arc, Mutex}; -use std::thread; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; #[cfg(target_os = "linux")] use std::os::unix::process::CommandExt; @@ -22,15 +21,19 @@ pub use ::provisioning::plugin::{ ProvisionLogStream, ProvisionPlugin, }; use iroh::EndpointAddr; -use swactor::actor::ActorAddress; +use swactor::actor::{ActorAddress, ActorInterface}; +use swactor::runtime::{Ctx, Runtime}; use crate::node::worker_node_runtime::request_debug_join; use crate::observability::provisioning_logs::BootstrapTelemetryBridge; +use crate::orchestration::manual_control::SELECTED_OFFER_ID_ENV; pub(crate) struct LocalDockerPlugin { container_name_prefix: String, next_handle_id: u64, nodes: BTreeMap, + runtime: Runtime, + backend: Arc, } struct LocalDockerNode { @@ -39,12 +42,28 @@ struct LocalDockerNode { container_name: String, } +trait DockerLifecycleBackend: Send + Sync { + fn container_state(&self, name: &str) -> Result, String>; + fn start(&self, prefix: &str, runtime: &Runtime, node: &LocalDockerNode) -> Result<(), String>; + fn remove(&self, name: &str) -> Result<(), String>; + fn find( + &self, + prefix: &str, + spec: &NodeProvisionSpec, + ) -> Result, String>; + fn observe(&self, runtime: &Runtime, node: &LocalDockerNode, tail: &str) -> Result<(), String>; + fn list_managed(&self, prefix: &str) -> Result, String>; +} + +struct SystemDockerLifecycle; + pub(crate) struct LocalProcessPlugin { program: PathBuf, registry_path: Option, provider_prefix: String, next_handle_id: u64, nodes: BTreeMap, + runtime: Runtime, } /// Safe Vast.ai provisioning simulator. Marketplace selection remains real; @@ -53,6 +72,8 @@ pub(crate) struct LocalProcessPlugin { pub(crate) struct MockVastAiPlugin { inner: LocalProcessPlugin, selected_offers: BTreeMap, + #[cfg(feature = "test-support")] + lifecycle_observer_path: Option, } struct LocalProcessNode { @@ -65,6 +86,7 @@ struct LocalProcessNode { struct LocalProcessRuntime { stdin: ChildStdin, child: Arc>>, + exit_actor: Option, } #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] @@ -77,17 +99,152 @@ struct LocalProcessRecord { attempt_id: u64, } +impl LocalProcessRecord { + fn identity(&self) -> swactor_process::ProcessIdentity { + swactor_process::ProcessIdentity { + pid: self.pid, + environment: vec![ + ("MYELIN_RUN_ID".to_owned(), self.run_id.to_string()), + ( + "MYELIN_LOGICAL_NODE_ID".to_owned(), + self.node_id.to_string(), + ), + ], + process_group_leader: true, + } + } +} + +struct BootstrapOutputActor { + bridge: BootstrapTelemetryBridge, + closed: u8, +} + +impl ActorInterface for BootstrapOutputActor { + type Incoming = swactor_process::ProcessStreamObservation; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, observation: Self::Incoming) { + match observation { + swactor_process::ProcessStreamObservation::Line { stream, line } => match stream { + swactor_process::ProcessStream::Stdout => self.bridge.observe_stdout_line(line), + swactor_process::ProcessStream::Stderr => self.bridge.observe_stderr_line(line), + }, + swactor_process::ProcessStreamObservation::Error { stream, error } => { + self.bridge.observe_provider_line(format!( + "read {}: {error}", + match stream { + swactor_process::ProcessStream::Stdout => "stdout", + swactor_process::ProcessStream::Stderr => "stderr", + } + )); + } + swactor_process::ProcessStreamObservation::Closed { .. } => { + self.closed = self.closed.saturating_add(1); + if self.closed == 2 { + ctx.stop_self(); + } + } + } + } +} + +struct ProcessExitActor { + spec: NodeProvisionSpec, + sink: PluginSink, + registry: Option<(PathBuf, String)>, +} + +impl ActorInterface for ProcessExitActor { + type Incoming = swactor_process::ProcessExitObservation; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, observation: Self::Incoming) { + if let Some((path, provider_ref)) = &self.registry { + let _ = update_process_registry(path, |registry| { + registry.remove(provider_ref); + }); + } + let cleanup_error = remove_debug_join_socket(&self.spec).err(); + let event = match (observation.error, cleanup_error) { + (Some(reason), Some(cleanup_error)) => PluginObservation::Failed { + run_id: self.spec.run_id, + node_id: self.spec.node_id, + reason: format!("wait local process node: {reason}; {cleanup_error}"), + }, + (Some(reason), None) => PluginObservation::Failed { + run_id: self.spec.run_id, + node_id: self.spec.node_id, + reason: format!("wait local process node: {reason}"), + }, + (None, Some(reason)) => PluginObservation::Failed { + run_id: self.spec.run_id, + node_id: self.spec.node_id, + reason, + }, + (None, None) => PluginObservation::Exited { + run_id: self.spec.run_id, + node_id: self.spec.node_id, + status: observation.status, + }, + }; + self.sink.observe(event); + ctx.stop_self(); + } +} + +struct DockerWaitActor { + spec: NodeProvisionSpec, + sink: PluginSink, +} + +impl ActorInterface for DockerWaitActor { + type Incoming = swactor_process::CommandOutputObservation; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, observation: Self::Incoming) { + let status = observation + .error + .is_none() + .then(|| { + String::from_utf8_lossy(&observation.stdout) + .trim() + .parse::() + .ok() + }) + .flatten(); + self.sink.observe(PluginObservation::Exited { + run_id: self.spec.run_id, + node_id: self.spec.node_id, + status, + }); + ctx.stop_self(); + } +} + +struct DiscardProcessExit; + +impl ActorInterface for DiscardProcessExit { + type Incoming = swactor_process::ProcessExitObservation; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, _observation: Self::Incoming) { + ctx.stop_self(); + } +} + static PROCESS_REGISTRY_LOCK: Mutex<()> = Mutex::new(()); impl LocalProcessPlugin { #[cfg(test)] - pub(crate) fn new(program: impl Into) -> Self { + pub(crate) fn new(program: impl Into, runtime: Runtime) -> Self { Self { program: program.into(), registry_path: None, provider_prefix: "process".to_owned(), next_handle_id: 1, nodes: BTreeMap::new(), + runtime, } } @@ -95,6 +252,7 @@ impl LocalProcessPlugin { program: impl Into, registry_path: impl Into, provider_prefix: impl Into, + runtime: Runtime, ) -> Self { Self { program: program.into(), @@ -102,31 +260,177 @@ impl LocalProcessPlugin { provider_prefix: provider_prefix.into(), next_handle_id: 1, nodes: BTreeMap::new(), + runtime, } } } impl MockVastAiPlugin { #[cfg(test)] - pub(crate) fn new(program: impl Into) -> Self { - let mut inner = LocalProcessPlugin::new(program); + pub(crate) fn new(program: impl Into, runtime: Runtime) -> Self { + let mut inner = LocalProcessPlugin::new(program, runtime); inner.provider_prefix = "mock-vastai".to_owned(); Self { inner, selected_offers: BTreeMap::new(), + #[cfg(feature = "test-support")] + lifecycle_observer_path: None, } } pub(crate) fn with_registry( program: impl Into, registry_path: impl Into, + runtime: Runtime, ) -> Self { Self { - inner: LocalProcessPlugin::with_registry(program, registry_path, "mock-vastai"), + inner: LocalProcessPlugin::with_registry( + program, + registry_path, + "mock-vastai", + runtime, + ), selected_offers: BTreeMap::new(), + #[cfg(feature = "test-support")] + lifecycle_observer_path: std::env::var_os("MYELIN_MOCK_VASTAI_LEDGER_PATH") + .map(PathBuf::from), } } } +fn selected_offer_from_spec(spec: &NodeProvisionSpec) -> Option { + spec.env + .iter() + .find(|(key, _)| key == SELECTED_OFFER_ID_ENV) + .and_then(|(_, value)| value.parse().ok()) +} + +fn observe_mock_vastai_contract( + sink: &PluginSink, + event_type: &str, + spec: &NodeProvisionSpec, + provider_ref: &str, + selected_offer_id: Option, +) { + sink.observe(PluginObservation::ProviderLine { + run_id: spec.run_id, + node_id: spec.node_id, + line: serde_json::json!({ + "type": event_type, + "simulated": true, + "provider_ref": provider_ref, + "selected_offer_id": selected_offer_id, + }) + .to_string(), + }); +} +#[cfg(feature = "test-support")] +fn observe_mock_vastai_lifecycle( + path: Option<&Path>, + event: &str, + spec: &NodeProvisionSpec, + provider_ref: &str, + selected_offer_id: Option, +) -> Result<(), String> { + let Some(path) = path else { + return Ok(()); + }; + let mut file = OpenOptions::new() + .create(true) + .append(true) + .open(path) + .map_err(|error| { + format!( + "open mock Vast.ai lifecycle observer {}: {error}", + path.display() + ) + })?; + let mut observation = serde_json::to_string(&serde_json::json!({ + "event": event, + "run_id": spec.run_id, + "node_id": spec.node_id, + "attempt_id": spec.attempt_id, + "provider_ref": provider_ref, + "selected_offer_id": selected_offer_id, + })) + .map_err(|error| { + format!( + "encode mock Vast.ai lifecycle observation {}: {error}", + path.display() + ) + })?; + observation.push('\n'); + file.write_all(observation.as_bytes()).map_err(|error| { + format!( + "append mock Vast.ai lifecycle observation {}: {error}", + path.display() + ) + }) +} + +#[cfg(feature = "test-support")] +fn mock_vastai_lifecycle_path(plugin: &MockVastAiPlugin) -> Option<&Path> { + plugin.lifecycle_observer_path.as_deref() +} + +#[cfg(feature = "test-support")] +fn mock_vastai_resource_is_live( + path: Option<&Path>, + spec: &NodeProvisionSpec, + provider_ref: &str, +) -> Result { + let Some(path) = path else { + return Ok(false); + }; + let contents = match fs::read_to_string(path) { + Ok(contents) => contents, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(error) => { + return Err(format!( + "read mock Vast.ai provider state {}: {error}", + path.display() + )); + } + }; + let mut live = false; + for (index, line) in contents.lines().enumerate() { + let event: serde_json::Value = serde_json::from_str(line).map_err(|error| { + format!( + "parse mock Vast.ai provider state {} line {}: {error}", + path.display(), + index + 1 + ) + })?; + if event + .get("provider_ref") + .and_then(serde_json::Value::as_str) + != Some(provider_ref) + { + continue; + } + let event_run_id = event.get("run_id").and_then(serde_json::Value::as_u64); + let event_node_id = event.get("node_id").and_then(serde_json::Value::as_u64); + let event_attempt_id = event.get("attempt_id").and_then(serde_json::Value::as_u64); + if (event_run_id, event_node_id, event_attempt_id) + != (Some(spec.run_id), Some(spec.node_id), Some(spec.attempt_id)) + { + return Err(format!( + "mock Vast.ai provider reference {provider_ref} belongs to another node: {event}" + )); + } + match event.get("event").and_then(serde_json::Value::as_str) { + Some("created") => live = true, + Some("destroyed") => live = false, + Some("adopted" | "recreated") => {} + other => { + return Err(format!( + "unknown mock Vast.ai provider lifecycle event {other:?}: {event}" + )); + } + } + } + Ok(live) +} + fn local_process_provider_ref(prefix: &str, spec: &NodeProvisionSpec) -> String { format!( "{prefix}-{}-{}-attempt-{}", @@ -198,102 +502,70 @@ fn update_process_registry( } fn process_record_matches(record: &LocalProcessRecord) -> bool { - #[cfg(target_os = "linux")] - { - let Ok(stat) = fs::read_to_string(format!("/proc/{}/stat", record.pid)) else { - return false; - }; - let Some((_, tail)) = stat.rsplit_once(')') else { - return false; - }; - let mut fields = tail.split_whitespace(); - let state = fields.next(); - let _parent_pid = fields.next(); - let process_group = fields.next().and_then(|value| value.parse::().ok()); - if state == Some("Z") || process_group != Some(record.pid) { - return false; - } - let Ok(environ) = fs::read(format!("/proc/{}/environ", record.pid)) else { - return false; - }; - let expected = [ - ("MYELIN_RUN_ID", record.run_id.to_string()), - ("MYELIN_LOGICAL_NODE_ID", record.node_id.to_string()), - ]; - return expected.iter().all(|(key, value)| { - environ.split(|byte| *byte == 0).any(|entry| { - entry - .strip_prefix(format!("{key}=").as_bytes()) - .is_some_and(|actual| actual == value.as_bytes()) - }) - }); - } - #[cfg(not(target_os = "linux"))] - { - let _ = record; - false - } -} - -struct FollowProcessFile { - file: File, - record: LocalProcessRecord, -} - -impl Read for FollowProcessFile { - fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { - loop { - let count = self.file.read(buffer)?; - if count > 0 || !process_record_matches(&self.record) { - return Ok(count); - } - thread::sleep(Duration::from_millis(50)); - } - } + record.identity().matches() } fn discover_process_record( spec: &NodeProvisionSpec, stdout_path: PathBuf, stderr_path: PathBuf, ) -> Result, String> { - #[cfg(target_os = "linux")] - { - let entries = - fs::read_dir("/proc").map_err(|error| format!("scan /proc for worker: {error}"))?; - let mut matches = Vec::new(); - for entry in entries.flatten() { - let Some(pid) = entry - .file_name() - .to_str() - .and_then(|name| name.parse::().ok()) - else { - continue; - }; - let record = LocalProcessRecord { - pid, - stdout_path: stdout_path.clone(), - stderr_path: stderr_path.clone(), + let environment = vec![ + ("MYELIN_RUN_ID".to_owned(), spec.run_id.to_string()), + ( + "MYELIN_LOGICAL_NODE_ID".to_owned(), + spec.node_id.to_string(), + ), + ]; + let mut matches = swactor_process::find_process_identities_with_retry( + &environment, + true, + 40, + Duration::from_millis(50), + ) + .map_err(|error| format!("scan processes for worker: {error}"))?; + match matches.len() { + 0 => Ok(None), + 1 => { + let identity = matches.pop().expect("one process identity"); + Ok(Some(LocalProcessRecord { + pid: identity.pid, + stdout_path, + stderr_path, run_id: spec.run_id, node_id: spec.node_id, attempt_id: spec.attempt_id, - }; - if process_record_matches(&record) { - matches.push(record); - } + })) } - return match matches.len() { - 0 => Ok(None), - 1 => Ok(matches.pop()), - count => Err(format!( - "{count} local worker processes match run {} node {}", - spec.run_id, spec.node_id - )), - }; + count => Err(format!( + "{count} local worker processes match run {} node {}", + spec.run_id, spec.node_id + )), } - #[cfg(not(target_os = "linux"))] - { - let _ = (spec, stdout_path, stderr_path); - Ok(None) +} + +fn debug_join_socket_path(spec: &NodeProvisionSpec) -> PathBuf { + spec.env + .iter() + .find_map(|(key, value)| { + (key == "MYELIN_DEBUG_JOIN_SOCKET").then_some(PathBuf::from(value)) + }) + .unwrap_or_else(|| { + PathBuf::from(format!( + "/tmp/myelin-node-debug-join-{}-{}.sock", + spec.run_id, spec.node_id + )) + }) +} + +fn remove_debug_join_socket(spec: &NodeProvisionSpec) -> Result<(), String> { + let socket = debug_join_socket_path(spec); + match fs::remove_file(&socket) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(format!( + "remove debug join socket {}: {error}", + socket.display() + )), } } @@ -314,34 +586,19 @@ fn request_process_rejoin(spec: &NodeProvisionSpec) -> Result<(), String> { .ok_or_else(|| "recovery spec has no orchestrator actor".to_owned())?; let orchestrator_actor = serde_json::from_str::(actor_json) .map_err(|error| format!("parse recovery orchestrator actor: {error}"))?; - let socket = spec - .env - .iter() - .find_map(|(key, value)| { - (key == "MYELIN_DEBUG_JOIN_SOCKET").then_some(PathBuf::from(value)) - }) - .unwrap_or_else(|| { - PathBuf::from(format!( - "/tmp/myelin-node-debug-join-{}-{}.sock", - spec.run_id, spec.node_id - )) - }); - let mut last_error = None; - for _ in 0..40 { - match request_debug_join(&socket, endpoint.clone(), orchestrator_actor) { - Ok(()) => return Ok(()), - Err(error) => last_error = Some(error), - } - thread::sleep(Duration::from_millis(50)); + let socket = debug_join_socket_path(spec); + if !swactor_process::wait_for_path(&socket, 40, Duration::from_millis(50)) { + return Err(format!( + "rejoin socket {} did not become ready", + socket.display() + )); } - Err(format!( - "rejoin local worker through {}: {}", - socket.display(), - last_error.unwrap_or_else(|| "unknown error".to_owned()) - )) + request_debug_join(&socket, endpoint, orchestrator_actor) + .map_err(|error| format!("rejoin local worker through {}: {error}", socket.display())) } fn observe_process_files( + runtime: &Runtime, spec: NodeProvisionSpec, sink: PluginSink, record: LocalProcessRecord, @@ -358,31 +615,72 @@ fn observe_process_files( record.stderr_path.display() ) })?; - spawn_stdout_reader( - spec.clone(), - sink.clone(), - FollowProcessFile { - file: stdout, - record: record.clone(), - }, - ); - spawn_stderr_reader( + observe_output_streams( + runtime, spec, sink, - FollowProcessFile { - file: stderr, - record, - }, + swactor_process::FollowProcessFile::new( + stdout, + record.identity(), + Duration::from_millis(50), + ), + swactor_process::FollowProcessFile::new( + stderr, + record.identity(), + Duration::from_millis(50), + ), + ) +} + +fn observe_output_streams( + runtime: &Runtime, + spec: NodeProvisionSpec, + sink: PluginSink, + stdout: impl std::io::Read + Send + 'static, + stderr: impl std::io::Read + Send + 'static, +) -> Result<(), String> { + let actor = runtime + .spawn(BootstrapOutputActor { + bridge: BootstrapTelemetryBridge::new(spec, sink, None), + closed: 0, + }) + .map_err(|error| format!("spawn bootstrap output actor: {error}"))?; + let sender = runtime.create_sender(); + swactor_process::spawn_line_reader( + swactor_process::ProcessStream::Stdout, + stdout, + sender.clone(), + actor, + ); + swactor_process::spawn_line_reader( + swactor_process::ProcessStream::Stderr, + stderr, + sender, + actor, ); Ok(()) } impl LocalDockerPlugin { - pub(crate) fn new(container_name_prefix: impl Into) -> Self { + pub(crate) fn new(container_name_prefix: impl Into, runtime: Runtime) -> Self { + Self::with_backend( + container_name_prefix, + runtime, + Arc::new(SystemDockerLifecycle), + ) + } + + fn with_backend( + container_name_prefix: impl Into, + runtime: Runtime, + backend: Arc, + ) -> Self { Self { container_name_prefix: container_name_prefix.into(), next_handle_id: 1, nodes: BTreeMap::new(), + runtime, + backend, } } } @@ -411,13 +709,10 @@ fn docker_inspect_error_is_absent(stderr: &str) -> bool { stderr.contains("no such object") || stderr.contains("no such container") } -#[allow(clippy::disallowed_methods)] fn docker_container_is_absent(name: &str) -> Result { - let output = Command::new("docker") - .arg("inspect") - .arg(name) - .output() - .map_err(|error| format!("inspect Docker container {name}: {error}"))?; + let output = + swactor_process::command_output(&mut Command::new("docker").arg("inspect").arg(name)) + .map_err(|error| format!("inspect Docker container {name}: {error}"))?; if output.status.success() { return Ok(false); } @@ -433,13 +728,13 @@ fn docker_container_is_absent(name: &str) -> Result { } } -#[allow(clippy::disallowed_methods)] fn docker_container_is_running(name: &str) -> Result { - let output = Command::new("docker") - .args(["inspect", "-f", "{{.State.Running}}"]) - .arg(name) - .output() - .map_err(|error| format!("inspect Docker container {name} state: {error}"))?; + let output = swactor_process::command_output( + &mut Command::new("docker") + .args(["inspect", "-f", "{{.State.Running}}"]) + .arg(name), + ) + .map_err(|error| format!("inspect Docker container {name} state: {error}"))?; if !output.status.success() { return Err(format!( "inspect Docker container {name} state exited with {}: {}", @@ -451,19 +746,16 @@ fn docker_container_is_running(name: &str) -> Result { } /// Lists container names carrying this daemon's label, running or not. -#[allow(clippy::disallowed_methods)] fn docker_labeled_containers(prefix: &str) -> Result, String> { - let output = Command::new("docker") - .args([ - "ps", - "-a", - "--filter", - &format!("label=myelin.daemon={prefix}"), - "--format", - "{{.Names}}", - ]) - .output() - .map_err(|error| format!("list labeled Docker containers: {error}"))?; + let output = swactor_process::command_output(&mut Command::new("docker").args([ + "ps", + "-a", + "--filter", + &format!("label=myelin.daemon={prefix}"), + "--format", + "{{.Names}}", + ])) + .map_err(|error| format!("list labeled Docker containers: {error}"))?; if !output.status.success() { return Err(format!( "list labeled Docker containers exited with {}: {}", @@ -483,17 +775,18 @@ fn docker_containers_for_spec( prefix: &str, spec: &NodeProvisionSpec, ) -> Result, String> { - let output = Command::new("docker") - .args(["ps", "-a"]) - .arg("--filter") - .arg(format!("label=myelin.daemon={prefix}")) - .arg("--filter") - .arg(format!("label=myelin.run={}", spec.run_id)) - .arg("--filter") - .arg(format!("label=myelin.node={}", spec.node_id)) - .args(["--format", "{{.Names}}"]) - .output() - .map_err(|error| format!("list Docker containers for node {}: {error}", spec.node_id))?; + let output = swactor_process::command_output( + &mut Command::new("docker") + .args(["ps", "-a"]) + .arg("--filter") + .arg(format!("label=myelin.daemon={prefix}")) + .arg("--filter") + .arg(format!("label=myelin.run={}", spec.run_id)) + .arg("--filter") + .arg(format!("label=myelin.node={}", spec.node_id)) + .args(["--format", "{{.Names}}"]), + ) + .map_err(|error| format!("list Docker containers for node {}: {error}", spec.node_id))?; if !output.status.success() { return Err(format!( "list Docker containers for node {} exited with {}: {}", @@ -582,11 +875,12 @@ fn prepare_docker_file_volume( "create cached model docker volume", )?; let loader_name = format!("myelin-cache-load-{}-{volume}", std::process::id()); - let _ = Command::new("docker") - .args(["rm", "-f", &loader_name]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); + let _ = swactor_process::command_status( + &mut Command::new("docker") + .args(["rm", "-f", &loader_name]) + .stdout(Stdio::null()) + .stderr(Stdio::null()), + ); docker_status_vec( vec![ "create".to_owned(), @@ -610,11 +904,12 @@ fn prepare_docker_file_volume( ], "copy cached model into docker volume", ); - let _ = Command::new("docker") - .args(["rm", &loader_name]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); + let _ = swactor_process::command_status( + &mut Command::new("docker") + .args(["rm", &loader_name]) + .stdout(Stdio::null()) + .stderr(Stdio::null()), + ); copy_result?; // tinygrad opens GGUF files read-write even when it does not intend to mutate @@ -660,13 +955,14 @@ fn safe_docker_volume_component(value: &str) -> String { } fn docker_status(args: &[&str], label: &str) -> Result<(), String> { - let status = Command::new("docker") - .args(args) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::inherit()) - .status() - .map_err(|e| format!("{label}: {e}"))?; + let status = swactor_process::command_status( + &mut Command::new("docker") + .args(args) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()), + ) + .map_err(|e| format!("{label}: {e}"))?; if status.success() { Ok(()) } else { @@ -675,13 +971,14 @@ fn docker_status(args: &[&str], label: &str) -> Result<(), String> { } fn docker_status_vec(args: Vec, label: &str) -> Result<(), String> { - let status = Command::new("docker") - .args(&args) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::inherit()) - .status() - .map_err(|e| format!("{label}: {e}"))?; + let status = swactor_process::command_status( + &mut Command::new("docker") + .args(&args) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()), + ) + .map_err(|e| format!("{label}: {e}"))?; if status.success() { Ok(()) } else { @@ -689,112 +986,54 @@ fn docker_status_vec(args: Vec, label: &str) -> Result<(), String> { } } -fn lock_process_child( - child: &Arc>>, -) -> std::sync::MutexGuard<'_, Option> { - child - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) -} fn stop_owned_process(runtime: &mut LocalProcessRuntime) -> Result, String> { let _ = runtime.stdin.write_all(b"shutdown\n"); let _ = runtime.stdin.flush(); - let deadline = Instant::now() + Duration::from_secs(2); - while Instant::now() < deadline { - let mut slot = lock_process_child(&runtime.child); - let Some(child) = slot.as_mut() else { - return Ok(None); - }; - match child.try_wait() { - Ok(Some(status)) => { - *slot = None; - return Ok(status.code()); - } - Ok(None) => { - drop(slot); - thread::sleep(Duration::from_millis(50)); - } - Err(error) => return Err(format!("wait local process node: {error}")), - } - } - - let mut slot = lock_process_child(&runtime.child); - let Some(child) = slot.as_mut() else { - return Ok(None); - }; - #[cfg(target_os = "linux")] - unsafe { - let _ = libc::kill(-(child.id() as i32), libc::SIGKILL); - } - let kill_error = child.kill().err(); - match child.wait() { - Ok(status) => { - *slot = None; - Ok(status.code()) - } - Err(error) => Err(match kill_error { - Some(kill_error) => { - format!("kill local process node: {kill_error}; wait failed: {error}") - } - None => format!("wait for killed local process node: {error}"), - }), - } + swactor_process::wait_shared_child_or_kill( + &runtime.child, + Duration::from_secs(2), + true, + Duration::from_millis(50), + ) + .map(|status| status.and_then(|status| status.code())) + .map_err(|error| format!("stop local process node: {error}")) } fn stop_adopted_process(record: &LocalProcessRecord) -> Result<(), String> { - if !process_record_matches(record) { - return Ok(()); - } #[cfg(target_os = "linux")] - unsafe { - if libc::kill(-(record.pid as i32), libc::SIGTERM) != 0 { - return Err(format!( - "terminate adopted local process {}: {}", - record.pid, - std::io::Error::last_os_error() - )); - } - } - let deadline = Instant::now() + Duration::from_secs(2); - while Instant::now() < deadline { - if !process_record_matches(record) { - return Ok(()); - } - thread::sleep(Duration::from_millis(50)); - } - #[cfg(target_os = "linux")] - unsafe { - if libc::kill(-(record.pid as i32), libc::SIGKILL) != 0 && process_record_matches(record) { - return Err(format!( - "kill adopted local process {}: {}", - record.pid, - std::io::Error::last_os_error() - )); - } + { + return swactor_process::terminate_process_group( + &record.identity(), + Duration::from_secs(2), + Duration::from_millis(50), + ); } + #[cfg(not(target_os = "linux"))] Ok(()) } fn observe_adopted_process( + runtime: &Runtime, spec: NodeProvisionSpec, sink: PluginSink, registry_path: PathBuf, provider_ref: String, record: LocalProcessRecord, -) { - thread::spawn(move || { - while process_record_matches(&record) { - thread::sleep(Duration::from_millis(100)); - } - let _ = update_process_registry(®istry_path, |registry| { - registry.remove(&provider_ref); - }); - sink.observe(PluginObservation::Exited { - run_id: spec.run_id, - node_id: spec.node_id, - status: None, - }); - }); +) -> Result<(), String> { + let actor = runtime + .spawn(ProcessExitActor { + spec, + sink, + registry: Some((registry_path, provider_ref)), + }) + .map_err(|error| format!("spawn adopted process observer actor: {error}"))?; + swactor_process::spawn_identity_exit_wait( + record.identity(), + Duration::from_millis(100), + runtime.create_sender(), + actor, + ); + Ok(()) } impl ProvisionPlugin for LocalProcessPlugin { @@ -820,8 +1059,6 @@ impl ProvisionPlugin for LocalProcessPlugin { Ok(handle) } - // provider process supervision/lifecycle is out of scope (ENGINE_SPEC.md §2) - #[allow(clippy::disallowed_methods)] fn start_bootstrap(&mut self, handle: &PluginNodeHandle) -> Result<(), String> { let registry_path = self.registry_path.clone(); let provider_prefix = self.provider_prefix.clone(); @@ -889,7 +1126,7 @@ impl ProvisionPlugin for LocalProcessPlugin { }); } - let mut child = command.spawn().map_err(|error| { + let mut child = swactor_process::command_spawn(&mut command).map_err(|error| { format!( "spawn local process node {} with {}: {error}", spec.node_id, @@ -898,8 +1135,8 @@ impl ProvisionPlugin for LocalProcessPlugin { })?; let pid = child.id(); let Some(stdin) = child.stdin.take() else { - let _ = child.kill(); - let _ = child.wait(); + let _ = swactor_process::child_kill(&mut child); + let _ = swactor_process::child_wait(&mut child); return Err(format!( "local process node {} did not expose piped stdin", spec.node_id @@ -922,8 +1159,8 @@ impl ProvisionPlugin for LocalProcessPlugin { unsafe { let _ = libc::kill(-(pid as i32), libc::SIGKILL); } - let _ = child.kill(); - let _ = child.wait(); + let _ = swactor_process::child_kill(&mut child); + let _ = swactor_process::child_wait(&mut child); return Err(error); } @@ -932,44 +1169,27 @@ impl ProvisionPlugin for LocalProcessPlugin { node.runtime = Some(LocalProcessRuntime { stdin, child: Arc::clone(&child), + exit_actor: None, }); - observe_process_files(spec.clone(), sink.clone(), record)?; - thread::spawn(move || { - loop { - let observation = { - let mut slot = lock_process_child(&child); - let Some(child) = slot.as_mut() else { - return; - }; - match child.try_wait() { - Ok(Some(status)) => { - *slot = None; - Some(PluginObservation::Exited { - run_id: spec.run_id, - node_id: spec.node_id, - status: status.code(), - }) - } - Ok(None) => None, - Err(error) => Some(PluginObservation::Failed { - run_id: spec.run_id, - node_id: spec.node_id, - reason: format!("wait local process node: {error}"), - }), - } - }; - if let Some(observation) = observation { - if let Some(path) = registry_path.as_deref() { - let _ = update_process_registry(path, |registry| { - registry.remove(&provider_ref); - }); - } - sink.observe(observation); - return; - } - thread::sleep(Duration::from_millis(100)); - } - }); + observe_process_files(&self.runtime, spec.clone(), sink.clone(), record)?; + let exit_actor = self + .runtime + .spawn(ProcessExitActor { + spec, + sink, + registry: registry_path.map(|path| (path, provider_ref)), + }) + .map_err(|error| format!("spawn local process observer actor: {error}"))?; + node.runtime + .as_mut() + .expect("local process runtime was installed before its exit observer") + .exit_actor = Some(exit_actor); + swactor_process::spawn_shared_child_wait( + child, + Duration::from_millis(100), + self.runtime.create_sender(), + exit_actor, + ); Ok(()) } @@ -977,8 +1197,6 @@ impl ProvisionPlugin for LocalProcessPlugin { Ok(()) } - // provider process supervision/lifecycle is out of scope (ENGINE_SPEC.md §2) - #[allow(clippy::disallowed_methods)] fn stop_node(&mut self, handle: &PluginNodeHandle) -> Result<(), String> { let Some(mut node) = self.nodes.remove(&handle.id) else { return Ok(()); @@ -991,12 +1209,24 @@ impl ProvisionPlugin for LocalProcessPlugin { .transpose()? .and_then(|registry| registry.get(&provider_ref).cloned()); let result = match node.runtime.as_mut() { - Some(runtime) => stop_owned_process(runtime), + Some(runtime) => { + let result = stop_owned_process(runtime); + if result.is_ok() + && let Some(exit_actor) = runtime.exit_actor.take() + { + let _ = self.runtime.stop_actor(exit_actor); + } + result + } None => match record.as_ref() { Some(record) => stop_adopted_process(record).map(|()| None), None => Ok(None), }, }; + let result = result.and_then(|status| { + remove_debug_join_socket(&node.spec)?; + Ok(status) + }); match result { Ok(status) => { if let Some(path) = self.registry_path.as_deref() { @@ -1059,13 +1289,7 @@ impl ProvisionPlugin for LocalProcessPlugin { if record.is_none() { let (stdout_path, stderr_path) = process_output_paths(Some(®istry_path), &provider_ref); - for _ in 0..40 { - record = discover_process_record(spec, stdout_path.clone(), stderr_path.clone())?; - if record.is_some() { - break; - } - thread::sleep(Duration::from_millis(50)); - } + record = discover_process_record(spec, stdout_path, stderr_path)?; if let Some(discovered) = record.as_ref() { update_process_registry(®istry_path, |registry| { registry.insert(provider_ref.clone(), discovered.clone()); @@ -1076,14 +1300,15 @@ impl ProvisionPlugin for LocalProcessPlugin { return Ok(None); }; request_process_rejoin(spec)?; - observe_process_files(spec.clone(), sink.clone(), record.clone())?; + observe_process_files(&self.runtime, spec.clone(), sink.clone(), record.clone())?; observe_adopted_process( + &self.runtime, spec.clone(), sink.clone(), registry_path, provider_ref.clone(), record.clone(), - ); + )?; let handle = PluginNodeHandle { id: self.next_handle_id, provider_process_id: Some(record.pid), @@ -1160,18 +1385,25 @@ impl ProvisionPlugin for MockVastAiPlugin { .ok_or_else(|| { "mock Vast.ai provisioning requires an exact selected offer".to_owned() })?; - sink.observe(PluginObservation::ProviderLine { - run_id: spec.run_id, - node_id: spec.node_id, - line: serde_json::json!({ - "type": "MockVastAiContractCreated", - "simulated": true, - "selected_offer_id": offer_id, - }) - .to_string(), - }); - let handle = self.inner.create_node(spec, sink)?; + let provider_ref = self.inner.provider_ref_for(&spec); + let event_spec = spec.clone(); + let handle = self.inner.create_node(spec, sink.clone())?; self.selected_offers.insert(handle.id, offer_id); + observe_mock_vastai_contract( + &sink, + "MockVastAiContractCreated", + &event_spec, + &provider_ref, + Some(offer_id), + ); + #[cfg(feature = "test-support")] + observe_mock_vastai_lifecycle( + mock_vastai_lifecycle_path(self), + "created", + &event_spec, + &provider_ref, + Some(offer_id), + )?; Ok(handle) } @@ -1188,8 +1420,38 @@ impl ProvisionPlugin for MockVastAiPlugin { } fn stop_node(&mut self, handle: &PluginNodeHandle) -> Result<(), String> { + let event = self.inner.nodes.get(&handle.id).map(|node| { + let offer_id = self + .selected_offers + .get(&handle.id) + .copied() + .or_else(|| selected_offer_from_spec(&node.spec)); + ( + node.spec.clone(), + node.sink.clone(), + self.inner.provider_ref_for(&node.spec), + offer_id, + ) + }); self.inner.stop_node(handle)?; self.selected_offers.remove(&handle.id); + if let Some((spec, sink, provider_ref, offer_id)) = event { + observe_mock_vastai_contract( + &sink, + "MockVastAiContractDestroyed", + &spec, + &provider_ref, + offer_id, + ); + #[cfg(feature = "test-support")] + observe_mock_vastai_lifecycle( + mock_vastai_lifecycle_path(self), + "destroyed", + &spec, + &provider_ref, + offer_id, + )?; + } Ok(()) } @@ -1198,7 +1460,44 @@ impl ProvisionPlugin for MockVastAiPlugin { spec: &NodeProvisionSpec, sink: PluginSink, ) -> Result, String> { - self.inner.adopt_by_spec(spec, sink) + let adopted = self.inner.adopt_by_spec(spec, sink.clone())?; + #[cfg(feature = "test-support")] + let adopted = { + let provider_ref = self.inner.provider_ref_for(spec); + if adopted.is_none() + && mock_vastai_resource_is_live( + mock_vastai_lifecycle_path(self), + spec, + &provider_ref, + )? + { + self.inner.prepare_missing_bootstrap(spec, sink.clone())? + } else { + adopted + } + }; + if let Some(adopted) = &adopted { + let offer_id = selected_offer_from_spec(spec); + if let Some(offer_id) = offer_id { + self.selected_offers.insert(adopted.handle.id, offer_id); + } + observe_mock_vastai_contract( + &sink, + "MockVastAiContractAdopted", + spec, + &adopted.provider_ref, + offer_id, + ); + #[cfg(feature = "test-support")] + observe_mock_vastai_lifecycle( + mock_vastai_lifecycle_path(self), + "adopted", + spec, + &adopted.provider_ref, + offer_id, + )?; + } + Ok(adopted) } fn prepare_missing_bootstrap( @@ -1206,7 +1505,29 @@ impl ProvisionPlugin for MockVastAiPlugin { spec: &NodeProvisionSpec, sink: PluginSink, ) -> Result, String> { - self.inner.prepare_missing_bootstrap(spec, sink) + let prepared = self.inner.prepare_missing_bootstrap(spec, sink.clone())?; + if let Some(prepared) = &prepared { + let offer_id = selected_offer_from_spec(spec); + if let Some(offer_id) = offer_id { + self.selected_offers.insert(prepared.handle.id, offer_id); + } + observe_mock_vastai_contract( + &sink, + "MockVastAiContractRecreated", + spec, + &prepared.provider_ref, + offer_id, + ); + #[cfg(feature = "test-support")] + observe_mock_vastai_lifecycle( + mock_vastai_lifecycle_path(self), + "recreated", + spec, + &prepared.provider_ref, + offer_id, + )?; + } + Ok(prepared) } fn provider_ref_for(&self, spec: &NodeProvisionSpec) -> String { @@ -1218,16 +1539,156 @@ impl ProvisionPlugin for MockVastAiPlugin { } fn stop_by_spec(&mut self, spec: &NodeProvisionSpec, sink: PluginSink) -> Result { - self.inner.stop_by_spec(spec, sink) + let provider_ref = self.inner.provider_ref_for(spec); + let stopped = self.inner.stop_by_spec(spec, sink.clone())?; + if stopped { + observe_mock_vastai_contract( + &sink, + "MockVastAiContractDestroyed", + spec, + &provider_ref, + selected_offer_from_spec(spec), + ); + #[cfg(feature = "test-support")] + observe_mock_vastai_lifecycle( + mock_vastai_lifecycle_path(self), + "destroyed", + spec, + &provider_ref, + selected_offer_from_spec(spec), + )?; + } + Ok(stopped) } } +impl DockerLifecycleBackend for SystemDockerLifecycle { + fn container_state(&self, name: &str) -> Result, String> { + if docker_container_is_absent(name)? { + Ok(None) + } else { + docker_container_is_running(name).map(Some) + } + } + + fn start(&self, prefix: &str, runtime: &Runtime, node: &LocalDockerNode) -> Result<(), String> { + start_system_docker_container(prefix, runtime, node) + } + + fn remove(&self, name: &str) -> Result<(), String> { + remove_docker_container(name) + } + + fn find( + &self, + prefix: &str, + spec: &NodeProvisionSpec, + ) -> Result, String> { + let Some(name) = docker_container_for_spec(prefix, spec)? else { + return Ok(None); + }; + let running = docker_container_is_running(&name)?; + Ok(Some((name, running))) + } + + fn observe(&self, runtime: &Runtime, node: &LocalDockerNode, tail: &str) -> Result<(), String> { + observe_docker_container( + runtime, + node.spec.clone(), + node.sink.clone(), + node.container_name.clone(), + tail, + ) + } + + fn list_managed(&self, prefix: &str) -> Result, String> { + docker_labeled_containers(prefix) + } +} + +fn start_system_docker_container( + prefix: &str, + runtime: &Runtime, + node: &LocalDockerNode, +) -> Result<(), String> { + let spec = node.spec.clone(); + let sink = node.sink.clone(); + let container_name = node.container_name.clone(); + let mut command = Command::new("docker"); + command + .arg("run") + .arg("-d") + .arg("--add-host") + .arg("host.docker.internal:host-gateway") + .arg("--name") + .arg(&container_name); + for label in docker_container_labels(prefix, &spec) { + command.arg("--label").arg(label); + } + let docker_gpus = spec + .env + .iter() + .find(|(key, _)| key == "MYELIN_DOCKER_GPUS") + .map(|(_, value)| value.clone()) + .or_else(|| std::env::var("MYELIN_DOCKER_GPUS").ok()) + .filter(|value| !value.trim().is_empty()); + sink.observe(PluginObservation::TelemetryFrame { + run_id: spec.run_id, + node_id: spec.node_id, + channel: "myelin.provisioning.events".to_owned(), + payload: serde_json::json!({ + "type":"DockerGpuConfigResolved", + "provider":"Docker", + "gpus_arg":docker_gpus.as_deref(), + "has_gpus_arg":docker_gpus.is_some(), + }) + .to_string(), + }); + if let Some(gpus) = docker_gpus.as_deref() { + command.arg("--gpus").arg(gpus); + } + for mount in &spec.mounts { + command + .arg("--mount") + .arg(docker_runtime_mount_arg(&spec.image, mount)?); + } + for (key, value) in &spec.env { + command.arg("-e").arg(format!("{key}={value}")); + } + command.arg(&spec.image); + for arg in &spec.args { + command.arg(arg); + } + let output = swactor_process::command_output(&mut command) + .map_err(|error| format!("run Docker node {}: {error}", spec.node_id))?; + if !output.status.success() { + return Err(format!( + "run Docker node {} exited with {}: {}", + spec.node_id, + output.status, + String::from_utf8_lossy(&output.stderr).trim() + )); + } + observe_docker_container(runtime, spec, sink, container_name, "all") +} + impl ProvisionPlugin for LocalDockerPlugin { fn create_node( &mut self, spec: NodeProvisionSpec, sink: PluginSink, ) -> Result { + let container_name = docker_container_name(&self.container_name_prefix, &spec); + if self + .nodes + .values() + .any(|node| node.container_name == container_name) + { + return Err(format!( + "Docker attempt {} is already registered", + container_name + )); + } let handle = PluginNodeHandle { id: self.next_handle_id, provider_process_id: None, @@ -1236,7 +1697,7 @@ impl ProvisionPlugin for LocalDockerPlugin { self.nodes.insert( handle.id, LocalDockerNode { - container_name: docker_container_name(&self.container_name_prefix, &spec), + container_name, spec, sink, }, @@ -1244,83 +1705,21 @@ impl ProvisionPlugin for LocalDockerPlugin { Ok(handle) } - // provider process supervision/lifecycle is out of scope (ENGINE_SPEC.md §2) - #[allow(clippy::disallowed_methods)] fn start_bootstrap(&mut self, handle: &PluginNodeHandle) -> Result<(), String> { let node = self .nodes .get(&handle.id) .ok_or_else(|| format!("Docker node handle {} is absent", handle.id))?; - if !docker_container_is_absent(&node.container_name)? { - return if docker_container_is_running(&node.container_name)? { - Ok(()) - } else { - Err(format!( - "Docker container {} exists but is not running", - node.container_name - )) - }; + match self.backend.container_state(&node.container_name)? { + Some(true) => Ok(()), + Some(false) => Err(format!( + "Docker container {} exists but is not running", + node.container_name + )), + None => self + .backend + .start(&self.container_name_prefix, &self.runtime, node), } - let spec = node.spec.clone(); - let sink = node.sink.clone(); - let container_name = node.container_name.clone(); - let mut command = Command::new("docker"); - command - .arg("run") - .arg("-d") - .arg("--add-host") - .arg("host.docker.internal:host-gateway") - .arg("--name") - .arg(&container_name); - for label in docker_container_labels(&self.container_name_prefix, &spec) { - command.arg("--label").arg(label); - } - let docker_gpus = spec - .env - .iter() - .find(|(key, _)| key == "MYELIN_DOCKER_GPUS") - .map(|(_, value)| value.clone()) - .or_else(|| std::env::var("MYELIN_DOCKER_GPUS").ok()) - .filter(|value| !value.trim().is_empty()); - sink.observe(PluginObservation::TelemetryFrame { - run_id: spec.run_id, - node_id: spec.node_id, - channel: "myelin.provisioning.events".to_owned(), - payload: serde_json::json!({ - "type":"DockerGpuConfigResolved", - "provider":"Docker", - "gpus_arg":docker_gpus.as_deref(), - "has_gpus_arg":docker_gpus.is_some(), - }) - .to_string(), - }); - if let Some(gpus) = docker_gpus.as_deref() { - command.arg("--gpus").arg(gpus); - } - for mount in &spec.mounts { - command - .arg("--mount") - .arg(docker_runtime_mount_arg(&spec.image, mount)?); - } - for (key, value) in &spec.env { - command.arg("-e").arg(format!("{key}={value}")); - } - command.arg(&spec.image); - for arg in &spec.args { - command.arg(arg); - } - let output = command - .output() - .map_err(|error| format!("run Docker node {}: {error}", spec.node_id))?; - if !output.status.success() { - return Err(format!( - "run Docker node {} exited with {}: {}", - spec.node_id, - output.status, - String::from_utf8_lossy(&output.stderr).trim() - )); - } - observe_docker_container(spec, sink, container_name, "all") } fn complete_bootstrap(&mut self, _handle: &PluginNodeHandle) -> Result<(), String> { @@ -1331,7 +1730,7 @@ impl ProvisionPlugin for LocalDockerPlugin { let Some(node) = self.nodes.remove(&handle.id) else { return Ok(()); }; - let result = remove_docker_container(&node.container_name); + let result = self.backend.remove(&node.container_name); if result.is_err() { self.nodes.insert(handle.id, node); } @@ -1343,31 +1742,41 @@ impl ProvisionPlugin for LocalDockerPlugin { spec: &NodeProvisionSpec, sink: PluginSink, ) -> Result, String> { - let Some(container_name) = docker_container_for_spec(&self.container_name_prefix, spec)? + if let Some((&id, node)) = self.nodes.iter_mut().find(|(_, node)| { + node.container_name == docker_container_name(&self.container_name_prefix, spec) + }) { + node.sink = sink; + return Ok(Some(AdoptedNode { + handle: PluginNodeHandle { + id, + provider_process_id: None, + }, + provider_ref: node.container_name.clone(), + })); + } + let Some((container_name, running)) = + self.backend.find(&self.container_name_prefix, spec)? else { return Ok(None); }; - let running = docker_container_is_running(&container_name)?; let handle = PluginNodeHandle { id: self.next_handle_id, provider_process_id: None, }; self.next_handle_id = self.next_handle_id.wrapping_add(1).max(1); + let node = LocalDockerNode { + spec: spec.clone(), + sink: sink.clone(), + container_name: container_name.clone(), + }; if running { // Replay this container's bootstrap log into the fresh daemon, // then follow new output. The replay supplies runtime facts when // the prior daemon died before persisting readiness. - observe_docker_container(spec.clone(), sink.clone(), container_name.clone(), "all")?; + self.backend.observe(&self.runtime, &node, "all")?; } - let adopted_name = container_name.clone(); - self.nodes.insert( - handle.id, - LocalDockerNode { - spec: spec.clone(), - sink: sink.clone(), - container_name, - }, - ); + let adopted_name = container_name; + self.nodes.insert(handle.id, node); sink.observe(PluginObservation::TelemetryFrame { run_id: spec.run_id, node_id: spec.node_id, @@ -1402,15 +1811,16 @@ impl ProvisionPlugin for LocalDockerPlugin { } fn list_managed_refs(&self) -> Result, String> { - docker_labeled_containers(&self.container_name_prefix) + self.backend.list_managed(&self.container_name_prefix) } fn stop_by_spec(&mut self, spec: &NodeProvisionSpec, sink: PluginSink) -> Result { - let Some(container_name) = docker_container_for_spec(&self.container_name_prefix, spec)? + let Some((container_name, _running)) = + self.backend.find(&self.container_name_prefix, spec)? else { return Ok(false); }; - remove_docker_container(&container_name)?; + self.backend.remove(&container_name)?; sink.observe(PluginObservation::TelemetryFrame { run_id: spec.run_id, node_id: spec.node_id, @@ -1429,14 +1839,15 @@ impl ProvisionPlugin for LocalDockerPlugin { } fn remove_docker_container(container_name: &str) -> Result<(), String> { - let status = Command::new("docker") - .arg("rm") - .arg("-f") - .arg(container_name) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .map_err(|error| format!("docker rm {container_name}: {error}"))?; + let status = swactor_process::command_status( + &mut Command::new("docker") + .arg("rm") + .arg("-f") + .arg(container_name) + .stdout(Stdio::null()) + .stderr(Stdio::null()), + ) + .map_err(|error| format!("docker rm {container_name}: {error}"))?; if status.success() || matches!(docker_container_is_absent(container_name), Ok(true)) { Ok(()) } else { @@ -1445,22 +1856,23 @@ fn remove_docker_container(container_name: &str) -> Result<(), String> { } fn observe_docker_container( + runtime: &Runtime, spec: NodeProvisionSpec, sink: PluginSink, container_name: String, tail: &str, ) -> Result<(), String> { let mut logs = Command::new("docker"); - let mut logs = logs - .arg("logs") - .arg("--follow") - .arg("--tail") - .arg(tail) - .arg(&container_name) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .map_err(|error| format!("follow Docker logs for {container_name}: {error}"))?; + let mut logs = swactor_process::command_spawn( + logs.arg("logs") + .arg("--follow") + .arg("--tail") + .arg(tail) + .arg(&container_name) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()), + ) + .map_err(|error| format!("follow Docker logs for {container_name}: {error}"))?; let stdout = logs .stdout .take() @@ -1469,50 +1881,31 @@ fn observe_docker_container( .stderr .take() .ok_or_else(|| format!("Docker logs for {container_name} has no stderr"))?; - spawn_stdout_reader(spec.clone(), sink.clone(), stdout); - spawn_stderr_reader(spec.clone(), sink.clone(), stderr); - thread::spawn(move || { - let _ = logs.wait(); - }); + observe_output_streams(runtime, spec.clone(), sink.clone(), stdout, stderr)?; - let wait_name = container_name; - thread::spawn(move || { - let status = Command::new("docker").arg("wait").arg(&wait_name).output(); - let code = status.ok().and_then(|output| { - String::from_utf8_lossy(&output.stdout) - .trim() - .parse::() - .ok() - }); - sink.observe(PluginObservation::Exited { - run_id: spec.run_id, - node_id: spec.node_id, - status: code, - }); - }); + let logs_waiter = runtime + .spawn(DiscardProcessExit) + .map_err(|error| format!("spawn Docker log waiter actor: {error}"))?; + swactor_process::spawn_child_wait(logs, runtime.create_sender(), logs_waiter); + + let wait_actor = runtime + .spawn(DockerWaitActor { spec, sink }) + .map_err(|error| format!("spawn Docker waiter actor: {error}"))?; + let mut wait = Command::new("docker"); + wait.arg("wait").arg(container_name); + swactor_process::spawn_command_output(wait, runtime.create_sender(), wait_actor); Ok(()) } -fn spawn_stdout_reader( - spec: NodeProvisionSpec, - sink: PluginSink, - stdout: impl std::io::Read + Send + 'static, -) { - BootstrapTelemetryBridge::new(spec, sink, None).spawn_stdout_reader(stdout); -} - -fn spawn_stderr_reader( - spec: NodeProvisionSpec, - sink: PluginSink, - stderr: impl std::io::Read + Send + 'static, -) { - BootstrapTelemetryBridge::new(spec, sink, None).spawn_stderr_reader(stderr); -} - #[cfg(test)] mod tests { use super::*; + use parking_lot::Mutex as ParkingMutex; + use proptest::prelude::*; + use std::collections::BTreeSet; use std::sync::mpsc; + use swactor::runtime::RuntimeParts; + use swactor_engine::{Engine, SteppingBackend, TokioBackend, TokioConfig}; #[test] fn docker_absence_detection_is_case_insensitive() { @@ -1546,11 +1939,32 @@ mod tests { } } + fn test_runtime() -> (Engine, Runtime) { + let parts = RuntimeParts::new(swactor::config::RuntimeConfig::default()); + let runtime = parts.runtime().clone(); + let backend = TokioBackend::new(TokioConfig::default()).expect("test Tokio backend"); + let engine = Engine::new(parts, backend).expect("test actor engine"); + (engine, runtime) + } + + fn recv_provider_event(rx: &mpsc::Receiver) -> serde_json::Value { + for _ in 0..32 { + let observation = rx + .recv_timeout(Duration::from_secs(1)) + .expect("provider event"); + if let PluginObservation::ProviderLine { line, .. } = observation { + return serde_json::from_str(&line).expect("provider event JSON"); + } + } + panic!("provider event was not observed"); + } + #[test] fn mock_vastai_preserves_exact_offer_identity_without_starting_a_lease() { + let (_engine, runtime) = test_runtime(); let (tx, rx) = mpsc::channel(); let sink = PluginSink::new(Arc::new(ChannelSink(tx))); - let mut plugin = MockVastAiPlugin::new("/not/executed"); + let mut plugin = MockVastAiPlugin::new("/not/executed", runtime); let spec = test_spec(); let handle = plugin @@ -1560,24 +1974,27 @@ mod tests { assert_eq!(plugin.provider_ref_for(&spec), "mock-vastai-5-7-attempt-11"); assert_eq!(plugin.selected_offers.get(&handle.id), Some(&8_675_309)); assert_eq!(plugin.inner.nodes.len(), 1); - let PluginObservation::ProviderLine { line, .. } = rx.try_recv().unwrap() else { - panic!("mock lease must emit a provider event"); - }; - let event: serde_json::Value = serde_json::from_str(&line).unwrap(); + let event = recv_provider_event(&rx); assert_eq!(event["type"], "MockVastAiContractCreated"); assert_eq!(event["simulated"], true); assert_eq!(event["selected_offer_id"], 8_675_309); + assert_eq!(event["provider_ref"], "mock-vastai-5-7-attempt-11"); plugin.stop_node(&handle).unwrap(); assert!(plugin.selected_offers.is_empty()); assert!(plugin.inner.nodes.is_empty()); + let event = recv_provider_event(&rx); + assert_eq!(event["type"], "MockVastAiContractDestroyed"); + assert_eq!(event["provider_ref"], "mock-vastai-5-7-attempt-11"); + assert_eq!(event["selected_offer_id"], 8_675_309); } #[test] fn mock_vastai_rejects_create_without_an_exact_offer() { + let (_engine, runtime) = test_runtime(); let (tx, _) = mpsc::channel(); let sink = PluginSink::new(Arc::new(ChannelSink(tx))); - let mut plugin = MockVastAiPlugin::new("/not/executed"); + let mut plugin = MockVastAiPlugin::new("/not/executed", runtime); let error = plugin.create_node(test_spec(), sink).unwrap_err(); @@ -1587,6 +2004,11 @@ mod tests { } proptest::proptest! { + #![proptest_config(ProptestConfig { + cases: 128, + max_shrink_iters: 2_000, + ..ProptestConfig::default() + })] #[test] fn mock_vastai_handle_state_survives_random_create_and_stop_sequences( operations in proptest::collection::vec( @@ -1594,12 +2016,13 @@ mod tests { proptest::prelude::any::(), proptest::prelude::any::(), ), - 1..128, + 0..=32, ) ) { + let (_engine, runtime) = test_runtime(); let (tx, _) = mpsc::channel(); let sink = PluginSink::new(Arc::new(ChannelSink(tx))); - let mut plugin = MockVastAiPlugin::new("/not/executed"); + let mut plugin = MockVastAiPlugin::new("/not/executed", runtime); let mut handles = Vec::new(); for (offer_id, should_stop) in operations { @@ -1630,35 +2053,74 @@ mod tests { #[cfg(target_os = "linux")] #[test] - fn local_process_creation_does_not_start_bootstrap() { + fn local_process_stop_releases_all_observer_actors() { + let (_engine, runtime) = test_runtime(); + let baseline_actors = runtime.stats().actors.len(); + let baseline_fds = fs::read_dir("/proc/self/fd").unwrap().count(); let (tx, rx) = mpsc::channel(); let sink = PluginSink::new(Arc::new(ChannelSink(tx))); let mut spec = test_spec(); + let socket_path = std::env::temp_dir().join(format!( + "myelin-local-process-socket-test-{}-{}.sock", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::write(&socket_path, []).unwrap(); + spec.env.push(( + "MYELIN_DEBUG_JOIN_SOCKET".to_owned(), + socket_path.display().to_string(), + )); spec.args = vec![ "-c".to_owned(), - "printf 'started\\n'; IFS= read -r line".to_owned(), + "printf 'started\n'; IFS= read -r line".to_owned(), ]; - let mut plugin = LocalProcessPlugin::new("/bin/sh"); + let mut plugin = LocalProcessPlugin::new("/bin/sh", runtime.clone()); let handle = plugin.create_node(spec, sink).unwrap(); assert!(matches!(rx.try_recv(), Err(mpsc::TryRecvError::Empty))); plugin.start_bootstrap(&handle).unwrap(); - let deadline = Instant::now() + Duration::from_secs(2); - let line = loop { - let remaining = deadline.saturating_duration_since(Instant::now()); - let observation = rx.recv_timeout(remaining).unwrap(); - if let PluginObservation::StdoutLine { line, .. } = observation { - break line; - } - }; - assert_eq!(line, "started"); + let node = plugin + .nodes + .get(&handle.id) + .expect("registered local process"); + assert!(node.pid.is_some(), "bootstrap did not start its process"); + assert!( + node.runtime.is_some(), + "bootstrap did not retain its process runtime" + ); plugin.stop_node(&handle).unwrap(); + assert!( + !socket_path.exists(), + "local process debug socket survived explicit stop" + ); + + let deadline = std::time::Instant::now() + Duration::from_secs(2); + while (runtime.stats().actors.len() != baseline_actors + || fs::read_dir("/proc/self/fd").unwrap().count() > baseline_fds) + && std::time::Instant::now() < deadline + { + std::thread::sleep(Duration::from_millis(10)); + } + assert_eq!( + runtime.stats().actors.len(), + baseline_actors, + "local process observers survived explicit process stop" + ); + let final_fds = fs::read_dir("/proc/self/fd").unwrap().count(); + assert!( + final_fds <= baseline_fds, + "local process stop leaked file descriptors: baseline={baseline_fds}, final={final_fds}" + ); } #[cfg(target_os = "linux")] #[test] fn persistent_process_is_adopted_after_plugin_drop() { + let (_engine, runtime) = test_runtime(); let registry_path = std::env::temp_dir().join(format!( "myelin-process-registry-test-{}-{}.json", std::process::id(), @@ -1683,8 +2145,12 @@ mod tests { let provider_ref = local_process_provider_ref("process", &spec); let record = { - let mut plugin = - LocalProcessPlugin::with_registry("/bin/sh", ®istry_path, "process"); + let mut plugin = LocalProcessPlugin::with_registry( + "/bin/sh", + ®istry_path, + "process", + runtime.clone(), + ); let handle = plugin.create_node(spec.clone(), sink.clone()).unwrap(); plugin.start_bootstrap(&handle).unwrap(); read_process_registry(®istry_path) @@ -1697,7 +2163,8 @@ mod tests { // Model a crash after spawn but before the parent commits its registry update. fs::remove_file(®istry_path).unwrap(); - let mut restarted = LocalProcessPlugin::with_registry("/bin/sh", ®istry_path, "process"); + let mut restarted = + LocalProcessPlugin::with_registry("/bin/sh", ®istry_path, "process", runtime); let adopted = restarted .adopt_by_spec(&spec, sink) .unwrap() @@ -1712,6 +2179,7 @@ mod tests { #[cfg(target_os = "linux")] #[test] fn mock_vastai_process_is_adopted_after_plugin_drop() { + let (_engine, runtime) = test_runtime(); let registry_path = std::env::temp_dir().join(format!( "myelin-mock-registry-test-{}-{}.json", std::process::id(), @@ -1723,6 +2191,8 @@ mod tests { let (tx, _) = mpsc::channel(); let sink = PluginSink::new(Arc::new(ChannelSink(tx))); let mut spec = test_spec(); + spec.run_id = 6; + spec.node_id = 8; spec.env .push(("MYELIN_RUN_ID".to_owned(), spec.run_id.to_string())); spec.env.push(( @@ -1735,7 +2205,8 @@ mod tests { ]; let record = { - let mut plugin = MockVastAiPlugin::with_registry("/bin/sh", ®istry_path); + let mut plugin = + MockVastAiPlugin::with_registry("/bin/sh", ®istry_path, runtime.clone()); let handle = plugin .create_node_selected(spec.clone(), sink.clone(), Some(42)) .unwrap(); @@ -1747,7 +2218,7 @@ mod tests { .cloned() .unwrap() }; - let mut restarted = MockVastAiPlugin::with_registry("/bin/sh", ®istry_path); + let mut restarted = MockVastAiPlugin::with_registry("/bin/sh", ®istry_path, runtime); let adopted = restarted .adopt_by_spec(&spec, sink) .unwrap() @@ -1760,6 +2231,7 @@ mod tests { #[cfg(target_os = "linux")] #[test] fn mixed_mock_processes_survive_kill_provision_and_restart() { + let (_engine, runtime) = test_runtime(); let registry_path = std::env::temp_dir().join(format!( "myelin-mixed-mock-test-{}-{}.json", std::process::id(), @@ -1787,7 +2259,8 @@ mod tests { }; { - let mut plugin = MockVastAiPlugin::with_registry("/bin/sh", ®istry_path); + let mut plugin = + MockVastAiPlugin::with_registry("/bin/sh", ®istry_path, runtime.clone()); for node_id in 1..=3 { let handle = plugin .create_node_selected(make_spec(node_id), sink.clone(), Some(node_id)) @@ -1796,7 +2269,8 @@ mod tests { } } { - let mut restarted = MockVastAiPlugin::with_registry("/bin/sh", ®istry_path); + let mut restarted = + MockVastAiPlugin::with_registry("/bin/sh", ®istry_path, runtime.clone()); let killed = restarted .adopt_by_spec(&make_spec(2), sink.clone()) .unwrap() @@ -1808,7 +2282,7 @@ mod tests { restarted.start_bootstrap(&new_node).unwrap(); } - let mut final_restart = MockVastAiPlugin::with_registry("/bin/sh", ®istry_path); + let mut final_restart = MockVastAiPlugin::with_registry("/bin/sh", ®istry_path, runtime); assert_eq!( final_restart.list_managed_refs().unwrap(), [ @@ -1829,10 +2303,11 @@ mod tests { #[test] fn missing_local_bootstrap_recreates_only_the_provider_handle() { + let (_engine, runtime) = test_runtime(); let (tx, _) = mpsc::channel(); let sink = PluginSink::new(Arc::new(ChannelSink(tx))); let spec = test_spec(); - let mut process = LocalProcessPlugin::new("/not/executed"); + let mut process = LocalProcessPlugin::new("/not/executed", runtime.clone()); let process_node = process .prepare_missing_bootstrap(&spec, sink.clone()) .unwrap() @@ -1843,7 +2318,7 @@ mod tests { None ); - let mut docker = LocalDockerPlugin::new("myelin"); + let mut docker = LocalDockerPlugin::new("myelin", runtime); let docker_node = docker .prepare_missing_bootstrap(&spec, sink) .unwrap() @@ -1866,4 +2341,1155 @@ mod tests { "myelin-5-7-attempt-12" ); } + + #[derive(Clone, Debug)] + struct ScriptedDockerResource { + id: u64, + attempt: String, + running: bool, + } + + #[derive(Default, Debug)] + struct ScriptedDockerState { + resources: Vec, + waiters: BTreeMap, + next_resource_id: u64, + fail_next_start: bool, + fail_next_remove: bool, + } + + #[derive(Default)] + struct ScriptedDockerBackend { + state: ParkingMutex, + } + + #[derive(Clone, Copy, Debug)] + enum ScriptedDockerTerminal { + Exit(i32), + CommandFailure, + MalformedOutput, + } + + impl ScriptedDockerTerminal { + fn observation(self) -> swactor_process::CommandOutputObservation { + match self { + Self::Exit(status) => swactor_process::CommandOutputObservation { + status: Some(0), + stdout: format!("{status}\n").into_bytes(), + stderr: Vec::new(), + error: None, + }, + Self::CommandFailure => swactor_process::CommandOutputObservation { + status: None, + stdout: Vec::new(), + stderr: Vec::new(), + error: Some("scripted docker wait command failure".to_owned()), + }, + Self::MalformedOutput => swactor_process::CommandOutputObservation { + status: Some(0), + stdout: b"not-an-exit-status\n".to_vec(), + stderr: Vec::new(), + error: None, + }, + } + } + } + + impl ScriptedDockerBackend { + fn fail_start(&self) { + self.state.lock().fail_next_start = true; + } + + fn fail_remove(&self) { + self.state.lock().fail_next_remove = true; + } + + fn clear_failures(&self) { + let mut state = self.state.lock(); + state.fail_next_start = false; + state.fail_next_remove = false; + } + + fn seed_orphan(&self, name: &str, running: bool) -> Result<(), String> { + let mut state = self.state.lock(); + if state + .resources + .iter() + .any(|resource| resource.attempt == name) + { + return Err(format!("scripted Docker resource {name} already exists")); + } + state.next_resource_id = state.next_resource_id.wrapping_add(1).max(1); + let id = state.next_resource_id; + state.resources.push(ScriptedDockerResource { + id, + attempt: name.to_owned(), + running, + }); + Ok(()) + } + + fn inject_duplicate_resource(&self, name: &str) -> Result<(), String> { + let mut state = self.state.lock(); + let running = state + .resources + .iter() + .find(|resource| resource.attempt == name) + .ok_or_else(|| format!("cannot duplicate absent Docker resource {name}"))? + .running; + state.next_resource_id = state.next_resource_id.wrapping_add(1).max(1); + let id = state.next_resource_id; + state.resources.push(ScriptedDockerResource { + id, + attempt: name.to_owned(), + running, + }); + Ok(()) + } + + fn has_waiter(&self, name: &str) -> bool { + self.state.lock().waiters.contains_key(name) + } + + fn finish( + &self, + runtime: &Runtime, + name: &str, + terminal: ScriptedDockerTerminal, + ) -> Result { + let waiter = { + let mut state = self.state.lock(); + for resource in &mut state.resources { + if resource.attempt == name { + resource.running = false; + } + } + state.waiters.remove(name) + }; + let Some(waiter) = waiter else { + return Ok(false); + }; + runtime + .send_to(waiter, terminal.observation()) + .map_err(|error| { + format!("deliver scripted Docker terminal observation: {error}") + })?; + Ok(true) + } + + fn finish_all(&self, runtime: &Runtime) -> Result<(), String> { + let waiters = { + let mut state = self.state.lock(); + std::mem::take(&mut state.waiters) + }; + for (_, waiter) in waiters { + runtime + .send_to(waiter, ScriptedDockerTerminal::CommandFailure.observation()) + .map_err(|error| { + format!("deliver scripted Docker cleanup observation: {error}") + })?; + } + Ok(()) + } + + fn observe_node(&self, runtime: &Runtime, node: &LocalDockerNode) -> Result<(), String> { + { + let state = self.state.lock(); + if state.waiters.contains_key(&node.container_name) { + return Ok(()); + } + } + let waiter = runtime + .spawn(DockerWaitActor { + spec: node.spec.clone(), + sink: node.sink.clone(), + }) + .map_err(|error| format!("spawn scripted Docker waiter: {error}"))?; + self.state + .lock() + .waiters + .insert(node.container_name.clone(), waiter); + Ok(()) + } + } + + impl DockerLifecycleBackend for ScriptedDockerBackend { + fn container_state(&self, name: &str) -> Result, String> { + let state = self.state.lock(); + let matches = state + .resources + .iter() + .filter(|resource| resource.attempt == name) + .collect::>(); + match matches.as_slice() { + [] => Ok(None), + [resource] => Ok(Some(resource.running)), + _ => Err(format!( + "multiple scripted Docker resources match {name}: {matches:?}" + )), + } + } + + fn start( + &self, + _prefix: &str, + runtime: &Runtime, + node: &LocalDockerNode, + ) -> Result<(), String> { + { + let mut state = self.state.lock(); + if std::mem::take(&mut state.fail_next_start) { + return Err("scripted Docker start command failure".to_owned()); + } + if state + .resources + .iter() + .any(|resource| resource.attempt == node.container_name) + { + return Err(format!( + "duplicate scripted Docker container {}", + node.container_name + )); + } + state.next_resource_id = state.next_resource_id.wrapping_add(1).max(1); + let id = state.next_resource_id; + state.resources.push(ScriptedDockerResource { + id, + attempt: node.container_name.clone(), + running: true, + }); + } + self.observe_node(runtime, node) + } + + fn remove(&self, name: &str) -> Result<(), String> { + let mut state = self.state.lock(); + if std::mem::take(&mut state.fail_next_remove) { + return Err("scripted Docker remove command failure".to_owned()); + } + state.resources.retain(|resource| resource.attempt != name); + Ok(()) + } + + fn find( + &self, + prefix: &str, + spec: &NodeProvisionSpec, + ) -> Result, String> { + let name = docker_container_name(prefix, spec); + Ok(self.container_state(&name)?.map(|running| (name, running))) + } + + fn observe( + &self, + runtime: &Runtime, + node: &LocalDockerNode, + _tail: &str, + ) -> Result<(), String> { + self.observe_node(runtime, node) + } + + fn list_managed(&self, _prefix: &str) -> Result, String> { + Ok(self + .state + .lock() + .resources + .iter() + .map(|resource| resource.attempt.clone()) + .collect()) + } + } + + #[derive(Default)] + struct RecordingDockerSink { + observations: ParkingMutex>, + } + + impl RecordingDockerSink { + fn snapshot(&self) -> Vec { + self.observations.lock().clone() + } + } + + impl PluginObservationSink for RecordingDockerSink { + fn observe(&self, observation: PluginObservation) { + self.observations.lock().push(observation); + } + } + + #[derive(Clone, Debug)] + enum DockerAction { + Create(u8), + StartBootstrap(u8), + Inspect(u8), + Stop(u8), + ExitBeforeReadiness(u8), + CommandFailure(u8), + MalformedCommandOutput(u8), + StopWhilePending(u8), + FailStart(u8), + FailStop(u8), + AdoptOrReject(u8), + StopBySpec(u8), + } + + fn docker_actions() -> impl Strategy> { + prop::collection::vec( + prop_oneof![ + 3 => any::().prop_map(DockerAction::Create), + 3 => any::().prop_map(DockerAction::StartBootstrap), + 2 => any::().prop_map(DockerAction::Inspect), + 2 => any::().prop_map(DockerAction::Stop), + 1 => any::().prop_map(DockerAction::ExitBeforeReadiness), + 1 => any::().prop_map(DockerAction::CommandFailure), + 1 => any::().prop_map(DockerAction::MalformedCommandOutput), + 1 => any::().prop_map(DockerAction::StopWhilePending), + 1 => any::().prop_map(DockerAction::FailStart), + 1 => any::().prop_map(DockerAction::FailStop), + 2 => any::().prop_map(DockerAction::AdoptOrReject), + 2 => any::().prop_map(DockerAction::StopBySpec), + ], + 0..=32, + ) + } + + fn generated_docker_spec(selector: u8) -> NodeProvisionSpec { + let mut spec = test_spec(); + spec.node_id = u64::from(selector % 4) + 1; + spec.attempt_id = u64::from((selector / 4) % 3) + 1; + spec + } + + fn registered_docker_handle( + plugin: &LocalDockerPlugin, + spec: &NodeProvisionSpec, + ) -> Option { + let name = docker_container_name(&plugin.container_name_prefix, spec); + plugin.nodes.iter().find_map(|(&id, node)| { + (node.container_name == name).then_some(PluginNodeHandle { + id, + provider_process_id: None, + }) + }) + } + + fn ensure_registered_docker_attempt( + plugin: &mut LocalDockerPlugin, + spec: &NodeProvisionSpec, + sink: &PluginSink, + ) -> Result { + if let Some(handle) = registered_docker_handle(plugin, spec) { + return Ok(handle); + } + plugin.create_node(spec.clone(), sink.clone()) + } + + const DOCKER_STEP_BUDGET: usize = 32; + const DOCKER_TIME_BUDGET: Duration = Duration::from_millis(1); + + fn settle_docker_actors(stepping: &SteppingBackend) { + crate::tests::fuzz_support::advance_and_drive( + stepping, + DOCKER_TIME_BUDGET, + DOCKER_STEP_BUDGET, + ); + } + + fn reset_scripted_attempt( + plugin: &mut LocalDockerPlugin, + backend: &ScriptedDockerBackend, + runtime: &Runtime, + stepping: &SteppingBackend, + spec: &NodeProvisionSpec, + ) -> Result<(), String> { + backend.clear_failures(); + if let Some(handle) = registered_docker_handle(plugin, spec) { + plugin.stop_node(&handle)?; + plugin.stop_node(&handle)?; + } + let name = docker_container_name(&plugin.container_name_prefix, spec); + backend.remove(&name)?; + let _ = backend.finish(runtime, &name, ScriptedDockerTerminal::CommandFailure)?; + settle_docker_actors(stepping); + Ok(()) + } + + fn prepare_pending_docker_attempt( + plugin: &mut LocalDockerPlugin, + backend: &ScriptedDockerBackend, + runtime: &Runtime, + stepping: &SteppingBackend, + spec: &NodeProvisionSpec, + sink: &PluginSink, + ) -> Result { + reset_scripted_attempt(plugin, backend, runtime, stepping, spec)?; + let handle = plugin.create_node(spec.clone(), sink.clone())?; + plugin.start_bootstrap(&handle)?; + let name = docker_container_name(&plugin.container_name_prefix, spec); + if !backend.has_waiter(&name) { + return Err(format!( + "scripted Docker attempt {name} started without a terminal observer" + )); + } + Ok(handle) + } + + fn docker_lifecycle_invariant( + plugin: &LocalDockerPlugin, + backend: &ScriptedDockerBackend, + runtime: &Runtime, + baseline_actors: usize, + actions: &[DockerAction], + replies: &[PluginObservation], + ) -> Result<(), String> { + let state = backend.state.lock(); + let resources = format!("{:?}", state.resources); + let waiters = state.waiters.keys().cloned().collect::>(); + let evidence = || { + format!( + "actions={actions:?}; resources={resources}; waiters={waiters:?}; replies={replies:?}; census=\n{}", + crate::tests::fuzz_support::actor_census(runtime), + ) + }; + + let registered_attempts = plugin + .nodes + .values() + .map(|node| node.container_name.as_str()) + .collect::>(); + if registered_attempts.len() != plugin.nodes.len() { + return Err(format!( + "duplicate Docker attempt handles were registered; {}", + evidence() + )); + } + + let mut resource_counts = BTreeMap::<&str, usize>::new(); + let mut resource_ids = BTreeSet::new(); + for resource in &state.resources { + *resource_counts + .entry(resource.attempt.as_str()) + .or_default() += 1; + if !resource_ids.insert(resource.id) { + return Err(format!( + "scripted Docker resource id {} was reused; {}", + resource.id, + evidence() + )); + } + } + if let Some((attempt, count)) = resource_counts.iter().find(|(_, count)| **count > 1) { + return Err(format!( + "Docker attempt {attempt} owns {count} external resources; {}", + evidence() + )); + } + + let stats = runtime.stats(); + let poisoned = stats + .actor_details + .iter() + .filter(|actor| actor.poisoned) + .collect::>(); + let worker_panics = stats + .workers + .iter() + .map(|worker| worker.panics) + .sum::(); + if !poisoned.is_empty() || worker_panics != 0 { + return Err(format!( + "Docker lifecycle poisoned actors: poisoned={poisoned:?}, worker_panics={worker_panics}; {}", + evidence() + )); + } + let actor_ceiling = baseline_actors.saturating_add(state.waiters.len()); + if stats.actors.len() > actor_ceiling { + return Err(format!( + "Docker lifecycle actor count {} exceeds baseline {baseline_actors} plus {} pending terminal observers; {}", + stats.actors.len(), + state.waiters.len(), + evidence() + )); + } + Ok(()) + } + + proptest! { + #![proptest_config(ProptestConfig { + cases: 128, + max_shrink_iters: 2_000, + ..ProptestConfig::default() + })] + + #[test] + fn docker_generated_attempt_lifecycles_are_idempotent_and_bounded( + actions in docker_actions() + ) { + let parts = RuntimeParts::new(swactor::config::RuntimeConfig::default()); + let runtime = parts.runtime().clone(); + let stepping = SteppingBackend::new(); + let _engine = Engine::new(parts, stepping.clone()).expect("stepping engine"); + let baseline_actors = runtime.stats().actors.len(); + let recording = Arc::new(RecordingDockerSink::default()); + let sink = PluginSink::new(recording.clone()); + let backend = Arc::new(ScriptedDockerBackend::default()); + let mut plugin = LocalDockerPlugin::with_backend( + "myelin", + runtime.clone(), + backend.clone(), + ); + + for action in &actions { + let selector = match *action { + DockerAction::Create(selector) + | DockerAction::StartBootstrap(selector) + | DockerAction::Inspect(selector) + | DockerAction::Stop(selector) + | DockerAction::ExitBeforeReadiness(selector) + | DockerAction::CommandFailure(selector) + | DockerAction::MalformedCommandOutput(selector) + | DockerAction::StopWhilePending(selector) + | DockerAction::FailStart(selector) + | DockerAction::FailStop(selector) + | DockerAction::AdoptOrReject(selector) + | DockerAction::StopBySpec(selector) => selector, + }; + let spec = generated_docker_spec(selector); + let name = docker_container_name("myelin", &spec); + + match *action { + DockerAction::Create(_) => { + let existing = registered_docker_handle(&plugin, &spec); + let first = plugin.create_node(spec.clone(), sink.clone()); + if existing.is_some() { + prop_assert!( + first.is_err(), + "repeated create unexpectedly replaced {}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + } else { + prop_assert!( + first.is_ok(), + "first create rejected {}: {:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + first, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + } + let registered = registered_docker_handle(&plugin, &spec); + let repeated = plugin.create_node(spec.clone(), sink.clone()); + prop_assert!( + repeated.is_err() && registered_docker_handle(&plugin, &spec) == registered, + "duplicate create was not deterministically rejected for {}: repeated={:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + repeated, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + } + DockerAction::StartBootstrap(_) => { + let handle = ensure_registered_docker_attempt(&mut plugin, &spec, &sink); + prop_assert!( + handle.is_ok(), + "register before start failed for {}: {:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + handle, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + let handle = handle.unwrap(); + let first = plugin.start_bootstrap(&handle); + let repeated = plugin.start_bootstrap(&handle); + prop_assert_eq!( + &repeated, + &first, + "repeated Docker start was non-deterministic for {}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + if first.is_ok() { + let complete = plugin.complete_bootstrap(&handle); + let repeated_complete = plugin.complete_bootstrap(&handle); + prop_assert!( + complete.is_ok() && repeated_complete.is_ok(), + "repeated Docker bootstrap completion failed for {}: first={:?}, repeated={:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + complete, + repeated_complete, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + } + } + DockerAction::Inspect(_) => { + let first = backend.container_state(&name); + let repeated = backend.container_state(&name); + prop_assert_eq!( + &repeated, + &first, + "repeated Docker inspect was non-deterministic for {}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + } + DockerAction::Stop(_) => { + let handle = ensure_registered_docker_attempt(&mut plugin, &spec, &sink); + prop_assert!( + handle.is_ok(), + "register before stop failed for {}: {:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + handle, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + let handle = handle.unwrap(); + let first = plugin.stop_node(&handle); + let repeated = plugin.stop_node(&handle); + prop_assert!( + first.is_ok() && repeated.is_ok(), + "Docker stop was not idempotent for {}: first={:?}, repeated={:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + first, + repeated, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + let terminal = backend.finish( + &runtime, + &name, + ScriptedDockerTerminal::Exit(137), + ); + prop_assert!( + terminal.is_ok(), + "stopped Docker terminal delivery failed for {}: {:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + terminal, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + settle_docker_actors(&stepping); + } + DockerAction::ExitBeforeReadiness(_) + | DockerAction::CommandFailure(_) + | DockerAction::MalformedCommandOutput(_) + | DockerAction::StopWhilePending(_) => { + let handle = prepare_pending_docker_attempt( + &mut plugin, + &backend, + &runtime, + &stepping, + &spec, + &sink, + ); + prop_assert!( + handle.is_ok(), + "prepare pending Docker operation failed for {}: {:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + handle, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + let handle = handle.unwrap(); + let before_replies = recording.snapshot().len(); + let terminal = match *action { + DockerAction::ExitBeforeReadiness(_) => { + ScriptedDockerTerminal::Exit(23) + } + DockerAction::CommandFailure(_) => { + ScriptedDockerTerminal::CommandFailure + } + DockerAction::MalformedCommandOutput(_) => { + ScriptedDockerTerminal::MalformedOutput + } + DockerAction::StopWhilePending(_) => { + let first = plugin.stop_node(&handle); + let repeated = plugin.stop_node(&handle); + prop_assert!( + first.is_ok() && repeated.is_ok(), + "stop while Docker observation was pending was not idempotent for {}: first={:?}, repeated={:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + first, + repeated, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + ScriptedDockerTerminal::Exit(137) + } + _ => unreachable!(), + }; + let delivered = backend.finish(&runtime, &name, terminal); + prop_assert!( + matches!(&delivered, Ok(true)), + "Docker terminal observation was not delivered for {}: {:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + delivered, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + settle_docker_actors(&stepping); + let observations = recording.snapshot(); + let expected_status = match terminal { + ScriptedDockerTerminal::Exit(status) => Some(status), + ScriptedDockerTerminal::CommandFailure + | ScriptedDockerTerminal::MalformedOutput => None, + }; + prop_assert!( + observations[before_replies..].iter().any(|observation| { + matches!( + observation, + PluginObservation::Exited { status, .. } + if *status == expected_status + ) + }), + "Docker terminal reply was missing for {}: expected_status={:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + expected_status, + actions, + backend.state.lock().resources, + observations, + crate::tests::fuzz_support::actor_census(&runtime), + ); + } + DockerAction::FailStart(_) => { + let reset = reset_scripted_attempt( + &mut plugin, + &backend, + &runtime, + &stepping, + &spec, + ); + prop_assert!( + reset.is_ok(), + "reset before Docker start failure failed for {}: {:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + reset, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + let handle = plugin.create_node(spec.clone(), sink.clone()); + prop_assert!( + handle.is_ok(), + "create before Docker start failure failed for {}: {:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + handle, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + let handle = handle.unwrap(); + backend.fail_start(); + let failed = plugin.start_bootstrap(&handle); + prop_assert!( + matches!(&failed, Err(error) if error.contains("start command failure")), + "scripted Docker start command failure was not propagated for {}: {:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + failed, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + let retry = plugin.start_bootstrap(&handle); + prop_assert!( + retry.is_ok(), + "Docker start did not recover deterministically after one command failure for {}: {:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + retry, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + } + DockerAction::FailStop(_) => { + let handle = prepare_pending_docker_attempt( + &mut plugin, + &backend, + &runtime, + &stepping, + &spec, + &sink, + ); + prop_assert!( + handle.is_ok(), + "prepare before Docker stop failure failed for {}: {:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + handle, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + let handle = handle.unwrap(); + backend.fail_remove(); + let failed = plugin.stop_node(&handle); + prop_assert!( + matches!(&failed, Err(error) if error.contains("remove command failure")) + && registered_docker_handle(&plugin, &spec) == Some(handle.clone()), + "Docker stop failure did not retain its registered attempt for retry {}: {:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + failed, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + let retry = plugin.stop_node(&handle); + let repeated = plugin.stop_node(&handle); + prop_assert!( + retry.is_ok() && repeated.is_ok(), + "Docker stop retry was not idempotent for {}: retry={:?}, repeated={:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + retry, + repeated, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + let terminal = backend.finish( + &runtime, + &name, + ScriptedDockerTerminal::CommandFailure, + ); + prop_assert!( + matches!(&terminal, Ok(true)), + "Docker stop failure cleanup did not close its observer for {}: {:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + terminal, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + settle_docker_actors(&stepping); + } + DockerAction::AdoptOrReject(_) => { + let reset = reset_scripted_attempt( + &mut plugin, + &backend, + &runtime, + &stepping, + &spec, + ); + prop_assert!( + reset.is_ok(), + "reset before Docker adoption failed for {}: {:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + reset, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + let should_exist = selector % 2 == 0; + if should_exist { + let seeded = backend.seed_orphan(&name, selector & 2 != 0); + prop_assert!( + seeded.is_ok(), + "seed Docker orphan failed for {}: {:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + seeded, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + } + let before_resources = backend.state.lock().resources.len(); + let first = plugin.adopt_by_spec(&spec, sink.clone()); + let repeated = plugin.adopt_by_spec(&spec, sink.clone()); + prop_assert!( + first.is_ok() && repeated.is_ok(), + "Docker adoption errored for {}: first={:?}, repeated={:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + first, + repeated, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + let first = first.unwrap(); + let repeated = repeated.unwrap(); + prop_assert_eq!( + first.as_ref().map(|node| &node.handle), + repeated.as_ref().map(|node| &node.handle), + "Docker adoption was non-deterministic for {}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + prop_assert_eq!( + first.is_some(), + should_exist, + "Docker adoption did not deterministically adopt/reject {}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + let after_state = backend.state.lock(); + let after_resource_count = after_state.resources.len(); + let after_resources = after_state.resources.clone(); + drop(after_state); + prop_assert_eq!( + after_resource_count, + before_resources, + "Docker adoption multiplied external resources for {}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + actions, + after_resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + } + DockerAction::StopBySpec(_) => { + let first = plugin.stop_by_spec(&spec, sink.clone()); + let repeated = plugin.stop_by_spec(&spec, sink.clone()); + prop_assert!( + first.is_ok() + && repeated.is_ok() + && matches!(&repeated, Ok(false)), + "Docker spec-addressed stop was not idempotent for {}: first={:?}, repeated={:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + first, + repeated, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + let terminal = backend.finish( + &runtime, + &name, + ScriptedDockerTerminal::Exit(137), + ); + prop_assert!( + terminal.is_ok(), + "Docker spec-addressed stop failed to close its observer for {}: {:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + terminal, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + settle_docker_actors(&stepping); + } + } + settle_docker_actors(&stepping); + + + let observations = recording.snapshot(); + let invariant = docker_lifecycle_invariant( + &plugin, + &backend, + &runtime, + baseline_actors, + &actions, + &observations, + ); + prop_assert!(invariant.is_ok(), "{}", invariant.unwrap_err()); + } + + backend.clear_failures(); + let handles = plugin + .nodes + .keys() + .copied() + .map(|id| PluginNodeHandle { + id, + provider_process_id: None, + }) + .collect::>(); + for handle in handles { + let first = plugin.stop_node(&handle); + let repeated = plugin.stop_node(&handle); + prop_assert!( + first.is_ok() && repeated.is_ok(), + "final Docker stop was not idempotent: first={:?}, repeated={:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + first, + repeated, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + } + let finished = backend.finish_all(&runtime); + prop_assert!( + finished.is_ok(), + "final Docker terminal-observer cleanup failed: {:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + finished, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + settle_docker_actors(&stepping); + + let remaining = backend + .state + .lock() + .resources + .iter() + .map(|resource| resource.attempt.clone()) + .collect::>(); + for name in remaining { + let removed = backend.remove(&name); + prop_assert!( + removed.is_ok(), + "final scripted Docker resource cleanup failed for {}: {:?}; actions={:?}; resources={:?}; replies={:?}; census=\n{}", + name, + removed, + actions, + backend.state.lock().resources, + recording.snapshot(), + crate::tests::fuzz_support::actor_census(&runtime), + ); + } + let observations = recording.snapshot(); + let invariant = docker_lifecycle_invariant( + &plugin, + &backend, + &runtime, + baseline_actors, + &actions, + &observations, + ); + prop_assert!(invariant.is_ok(), "{}", invariant.unwrap_err()); + let final_state = backend.state.lock(); + let resources_empty = final_state.resources.is_empty(); + let waiters_empty = final_state.waiters.is_empty(); + let final_resources = final_state.resources.clone(); + let final_waiters = final_state.waiters.keys().cloned().collect::>(); + drop(final_state); + prop_assert!( + resources_empty + && waiters_empty + && plugin.nodes.is_empty() + && runtime.stats().actors.len() == baseline_actors, + "Docker lifecycle did not return to actor/resource baseline; actions={:?}; resources={:?}; waiters={:?}; replies={:?}; census=\n{}", + actions, + final_resources, + final_waiters, + observations, + crate::tests::fuzz_support::actor_census(&runtime), + ); + } + } + + #[test] + fn docker_duplicate_resource_detector_rejects_controlled_fault() { + let parts = RuntimeParts::new(swactor::config::RuntimeConfig::default()); + let runtime = parts.runtime().clone(); + let stepping = SteppingBackend::new(); + let _engine = Engine::new(parts, stepping.clone()).expect("stepping engine"); + let baseline_actors = runtime.stats().actors.len(); + let recording = Arc::new(RecordingDockerSink::default()); + let sink = PluginSink::new(recording.clone()); + let backend = Arc::new(ScriptedDockerBackend::default()); + let mut plugin = + LocalDockerPlugin::with_backend("myelin", runtime.clone(), backend.clone()); + let spec = generated_docker_spec(0); + let name = docker_container_name("myelin", &spec); + let handle = plugin + .create_node(spec, sink) + .expect("register Docker attempt"); + plugin + .start_bootstrap(&handle) + .expect("start one Docker resource"); + let actions = vec![DockerAction::Create(0), DockerAction::StartBootstrap(0)]; + let observations = recording.snapshot(); + docker_lifecycle_invariant( + &plugin, + &backend, + &runtime, + baseline_actors, + &actions, + &observations, + ) + .expect("valid single-resource lifecycle"); + + backend + .inject_duplicate_resource(&name) + .expect("inject controlled duplicate resource"); + let error = docker_lifecycle_invariant( + &plugin, + &backend, + &runtime, + baseline_actors, + &actions, + &recording.snapshot(), + ) + .expect_err("duplicate-resource invariant must reject controlled fault"); + assert!( + error.contains("owns 2 external resources"), + "wrong duplicate-resource failure: {error}" + ); + + backend.clear_failures(); + plugin + .stop_node(&handle) + .expect("remove controlled duplicate resources"); + backend + .finish_all(&runtime) + .expect("finish controlled fault observer"); + settle_docker_actors(&stepping); + let observations = recording.snapshot(); + docker_lifecycle_invariant( + &plugin, + &backend, + &runtime, + baseline_actors, + &actions, + &observations, + ) + .expect("controlled fault cleanup"); + assert_eq!( + runtime.stats().actors.len(), + baseline_actors, + "controlled fault leaked actors; actions={actions:?}; resources={:?}; replies={observations:?}; census=\n{}", + backend.state.lock().resources, + crate::tests::fuzz_support::actor_census(&runtime), + ); + } } diff --git a/apps/myelin/src/tests/engine_composition.rs b/apps/myelin/src/tests/engine_composition.rs index 9fb83fd..6073402 100644 --- a/apps/myelin/src/tests/engine_composition.rs +++ b/apps/myelin/src/tests/engine_composition.rs @@ -101,9 +101,7 @@ fn build_composition() -> (Engine, IrohDriver, DistributionRuntimeStack) { (engine, driver, stack) } -/// Poll an inbox until a value arrives or the deadline elapses. The only -/// `thread::sleep` in this module: test observation, not engine work. -#[allow(clippy::disallowed_methods)] +/// Bounded inbox polling for test observation; not runtime work. fn recv_within(inbox: &Inbox, deadline: Duration) -> Option { let started = Instant::now(); loop { @@ -161,7 +159,7 @@ fn dashboard_server_is_scheduled_through_the_engine() { let mut config = dashboard::DashboardConfig::default(); config.port = free_port; let handle = dashboard::DashboardHandle::new(config); - engine.handle().spawn(handle.http_server()); + handle.spawn(&engine.handle()); // Behavioral proof the server future is actually running on the engine: // the bound port accepts a TCP connection. No second runtime is involved. @@ -171,7 +169,6 @@ fn dashboard_server_is_scheduled_through_the_engine() { } #[cfg(feature = "dashboard")] -#[allow(clippy::disallowed_methods)] fn poll_connect(addr: (&str, u16), deadline: Duration) -> bool { use std::net::TcpStream; let started = Instant::now(); diff --git a/apps/myelin/src/tests/fuzz_support.rs b/apps/myelin/src/tests/fuzz_support.rs new file mode 100644 index 0000000..5f580d6 --- /dev/null +++ b/apps/myelin/src/tests/fuzz_support.rs @@ -0,0 +1,92 @@ +use std::time::Duration; + +use swactor::runtime::Runtime; +use swactor_engine::SteppingBackend; + +pub(crate) fn drive_steps(backend: &SteppingBackend, count: usize) { + for _ in 0..count { + backend.step(); + } +} + +pub(crate) fn advance_and_drive(backend: &SteppingBackend, duration: Duration, count: usize) { + backend.advance_time(duration); + drive_steps(backend, count); +} + +pub(crate) fn actor_census(runtime: &Runtime) -> String { + let stats = runtime.stats(); + if stats.actor_details.is_empty() { + return format!("actors={:?}, workers={:?}", stats.actors, stats.workers); + } + + stats + .actor_details + .iter() + .map(|actor| { + format!( + "address={} name={:?} worker={} mailbox={} last={:?} processed={} poisoned={}", + actor.address, + actor.name, + actor.worker_id, + actor.mailbox_depth, + actor.last_msg_type, + actor.messages_processed, + actor.poisoned, + ) + }) + .collect::>() + .join("\n") +} + +pub(crate) fn assert_no_poison(runtime: &Runtime) { + assert_no_poison_with_context(runtime, ""); +} + +pub(crate) fn assert_no_poison_with_context(runtime: &Runtime, context: &str) { + let stats = runtime.stats(); + let poisoned = stats + .actor_details + .iter() + .filter(|actor| actor.poisoned) + .collect::>(); + let panics = stats + .workers + .iter() + .map(|worker| worker.panics) + .sum::(); + assert!( + poisoned.is_empty() && panics == 0, + "actor poison/panic detected: poisoned={poisoned:?}, worker_panics={panics}\n{context}\n{}", + actor_census(runtime), + ); +} + +pub(crate) fn assert_actor_delta_at_most(runtime: &Runtime, baseline: usize, limit: usize) { + let current = runtime.stats().actors.len(); + assert!( + current <= baseline.saturating_add(limit), + "actor count grew from {baseline} to {current}, limit={limit}\n{}", + actor_census(runtime), + ); +} + +pub(crate) fn assert_mailboxes_drained(runtime: &Runtime) { + let stats = runtime.stats(); + let worker_depth = stats + .workers + .iter() + .map(|worker| worker.mailbox_depth) + .sum::(); + let actor_depth = stats + .actor_details + .iter() + .map(|actor| actor.mailbox_depth) + .sum::(); + assert_eq!( + worker_depth + actor_depth, + 0, + "mailboxes did not drain\n{}", + actor_census(runtime), + ); +} diff --git a/apps/myelin/src/tests/job_runner_integration.rs b/apps/myelin/src/tests/job_runner_integration.rs index f041fad..6617427 100644 --- a/apps/myelin/src/tests/job_runner_integration.rs +++ b/apps/myelin/src/tests/job_runner_integration.rs @@ -26,7 +26,6 @@ use crate::orchestration::distribution_stack::DistributionRuntimeStack; const POLL: Duration = Duration::from_millis(15); const DEADLINE: Duration = Duration::from_secs(20); -#[allow(clippy::disallowed_methods)] fn recv_within(inbox: &Inbox, deadline: Duration) -> Option { let started = Instant::now(); loop { diff --git a/apps/myelin/src/tests/job_runner_iroh.rs b/apps/myelin/src/tests/job_runner_iroh.rs index 0c90d9a..4d2c4bb 100644 --- a/apps/myelin/src/tests/job_runner_iroh.rs +++ b/apps/myelin/src/tests/job_runner_iroh.rs @@ -179,7 +179,6 @@ fn driver_a_join( driver_a.join(std::slice::from_ref(&driver_b.endpoint_addr())); } -#[allow(clippy::disallowed_methods)] fn wait_until(deadline: Duration, mut check: impl FnMut() -> bool) -> bool { let started = Instant::now(); loop { diff --git a/apps/myelin/src/tests/mod.rs b/apps/myelin/src/tests/mod.rs index 54facf0..55a4651 100644 --- a/apps/myelin/src/tests/mod.rs +++ b/apps/myelin/src/tests/mod.rs @@ -1,4 +1,5 @@ mod engine_composition; +pub(crate) mod fuzz_support; mod harness; mod job_runner_iroh; mod node_guarantees; diff --git a/apps/myelin/src/tests/node_guarantees.rs b/apps/myelin/src/tests/node_guarantees.rs index db5350a..90b2380 100644 --- a/apps/myelin/src/tests/node_guarantees.rs +++ b/apps/myelin/src/tests/node_guarantees.rs @@ -1,7 +1,7 @@ //! Behavior guarantees for the `node` module. //! -//! These unit tests drive a manual `SingleThreadRuntime` host in isolation to verify actor -//! message routing — they are not engine integration tests. +//! These tests use the engine's deterministic stepping backend to verify actor +//! message routing without constructing or driving a runtime directly. use crate::node_actor::{NodeAgentActor, NodeAgentMsg, NodeAgentReport}; use crate::orchestration::actor::OrchestratorMsg; @@ -9,13 +9,15 @@ use iroh::{EndpointAddr, SecretKey}; use myelin::staging as stage; use swactor::actor::ActorAddress; use swactor::config::RuntimeConfig; -use swactor::runtime::{RuntimeParts, SingleThreadRuntime}; +use swactor::runtime::RuntimeParts; +use swactor_engine::{Engine, SteppingBackend}; #[test] fn node_agent_runtime_loaded_reports_orchestrator() { let parts = RuntimeParts::new(RuntimeConfig::default()); let runtime = parts.runtime().clone(); - let mut host = SingleThreadRuntime::new(parts); + let backend = SteppingBackend::new(); + let _engine = Engine::new(parts, backend.clone()).expect("stepping engine"); let orchestrator_inbox = runtime .new_inbox::() .expect("orchestrator inbox"); @@ -39,7 +41,7 @@ fn node_agent_runtime_loaded_reports_orchestrator() { }, ) .expect("send runtime loaded"); - host.tick(); + backend.step(); assert_eq!( orchestrator_inbox.try_recv(), @@ -58,7 +60,8 @@ fn node_agent_runtime_loaded_reports_orchestrator() { fn node_agent_runtime_ready_ack_reports_worker_loop() { let parts = RuntimeParts::new(RuntimeConfig::default()); let runtime = parts.runtime().clone(); - let mut host = SingleThreadRuntime::new(parts); + let backend = SteppingBackend::new(); + let _engine = Engine::new(parts, backend.clone()).expect("stepping engine"); let orchestrator_inbox = runtime .new_inbox::() .expect("orchestrator inbox"); @@ -84,7 +87,7 @@ fn node_agent_runtime_ready_ack_reports_worker_loop() { }, ) .expect("send runtime ready ack"); - host.tick(); + backend.step(); assert_eq!( reports.try_recv(), diff --git a/apps/myelin/tests/stateful_vastai.rs b/apps/myelin/tests/stateful_vastai.rs new file mode 100644 index 0000000..d365e30 --- /dev/null +++ b/apps/myelin/tests/stateful_vastai.rs @@ -0,0 +1,2319 @@ +#![cfg(target_os = "linux")] + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::OpenOptions; +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::os::unix::net::UnixStream; +use std::path::PathBuf; +use std::process::{Child, Command, Stdio}; +use std::sync::OnceLock; +use std::time::{Duration, Instant}; + +use proptest::prelude::*; +use proptest::test_runner::FileFailurePersistence; +use serde_json::{Value, json}; +use swactor_process::{ + child_kill, child_try_wait, child_wait, command_spawn, find_process_identities_by_environment, + request_child_termination, terminate_process_group, +}; +use swactor_vastai::test_http::{TestHttpRoute, TestHttpServer}; + +const CASE_DEADLINE: Duration = Duration::from_secs(10); +const SUITE_DEADLINE: Duration = Duration::from_secs(30); +const POLL: Duration = Duration::from_millis(25); +const CENSUS_STABILITY: Duration = Duration::from_millis(25); +const ACTORS_PER_NODE_LIMIT: u64 = 32; +const OFFER_COUNT: usize = 8; + +static SUITE_STARTED: OnceLock = OnceLock::new(); +std::thread_local! { + static OBSERVED_HTTP_FAILURES: std::cell::RefCell> = + const { std::cell::RefCell::new(Vec::new()) }; +} + +fn http_failure_checkpoint() -> usize { + OBSERVED_HTTP_FAILURES.with(|failures| failures.borrow().len()) +} + +fn discard_http_failures_since(checkpoint: usize) { + OBSERVED_HTTP_FAILURES.with(|failures| failures.borrow_mut().truncate(checkpoint)); +} + +fn record_http_failure(failure: String) { + OBSERVED_HTTP_FAILURES.with(|failures| failures.borrow_mut().push(failure)); +} + +fn observed_http_failures() -> Vec { + OBSERVED_HTTP_FAILURES.with(|failures| failures.borrow().clone()) +} + +#[derive(Clone, Copy, Debug)] +enum RestartMode { + Graceful, + FlushSafeAbrupt, +} + +#[derive(Clone, Debug)] +enum ExternalAction { + Search { + count: usize, + }, + Provision { + command_slot: u8, + use_searched_offers: bool, + }, + Query, + Kill { + node_slot: u8, + command_slot: u8, + }, + Flush, + Restart { + mode: RestartMode, + }, + EndpointProbe { + node_slot: u8, + }, + ConcurrentQueries, +} + +#[derive(Clone, Debug)] +struct E2eCase { + seed: u64, + node_seed: u8, + kill_mask: u8, + offer_offset: usize, + actions: Vec, +} + +fn e2e_case() -> impl Strategy { + ( + any::(), + any::(), + any::(), + 0_usize..OFFER_COUNT, + any::(), + any::(), + any::(), + any::(), + proptest::collection::vec(any::(), 11), + ) + .prop_map( + |( + seed, + node_seed, + kill_mask, + offer_offset, + invalid_search, + use_searched_offers, + node_slot, + command_slot, + ordering, + )| { + let mut keyed_actions = vec![ + ExternalAction::Search { + count: if invalid_search { 0 } else { 2 }, + }, + ExternalAction::Provision { + command_slot, + use_searched_offers, + }, + ExternalAction::Query, + ExternalAction::Kill { + node_slot, + command_slot, + }, + ExternalAction::Flush, + ExternalAction::Restart { + mode: RestartMode::Graceful, + }, + ExternalAction::EndpointProbe { node_slot }, + ExternalAction::ConcurrentQueries, + ExternalAction::Query, + ExternalAction::Kill { + node_slot: node_slot.wrapping_add(1), + command_slot, + }, + ExternalAction::Restart { + mode: RestartMode::FlushSafeAbrupt, + }, + ] + .into_iter() + .enumerate() + .map(|(index, action)| ((ordering[index], index), action)) + .collect::>(); + keyed_actions.sort_by_key(|(key, _)| *key); + E2eCase { + seed, + node_seed, + kill_mask, + offer_offset, + actions: keyed_actions + .into_iter() + .map(|(_, action)| action) + .collect(), + } + }, + ) +} + +fn e2e_proptest_config() -> ProptestConfig { + let has_case_override = std::env::var_os("PROPTEST_CASES").is_some(); + let mut config = ProptestConfig::default(); + if !has_case_override { + config.cases = 4; + } + config.failure_persistence = Some(Box::new(FileFailurePersistence::Direct(concat!( + env!("CARGO_MANIFEST_DIR"), + "/proptest-regressions/tests/e2e_vastai.txt" + )))); + config.max_shrink_iters = 0; + config +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct ActorCensus { + actors: u64, + orchestrator_actors: u64, + worker_actors: u64, + poisoned: u64, + type_mismatches: u64, + mailbox_depth: u64, + orchestrator_by_type: BTreeMap, + workers_by_stream: BTreeMap>, +} + +#[derive(Clone, Debug)] +struct OracleObservation { + expected_survivors: BTreeSet, + expected_stopped: BTreeSet, + running_after_restart: BTreeSet, + stopped_after_restart: BTreeSet, + reachable_nodes: BTreeSet, + provider_resources: Vec, + provider_created: BTreeSet, + provider_destroyed: BTreeSet, + selected_offers: BTreeMap, + searched_offers: BTreeSet, + actor_before_replays: ActorCensus, + actor_after_replays: ActorCensus, + actor_initial: ActorCensus, + actor_after_teardown: ActorCensus, + terminal_record_counts: BTreeMap, + unexpected_process_exits: Vec, + http_failures: Vec, + teardown_resources: Vec, + labeled_temp_resources: Vec, +} + +fn validate_oracle(observation: &OracleObservation) -> Result<(), String> { + if observation + .terminal_record_counts + .values() + .any(|count| *count != 1) + { + return Err(format!( + "accepted reply did not have exactly one terminal record: {:?}", + observation.terminal_record_counts + )); + } + if !observation.unexpected_process_exits.is_empty() { + return Err(format!( + "unexpected process exit: {:?}", + observation.unexpected_process_exits + )); + } + if !observation.http_failures.is_empty() { + return Err(format!( + "HTTP failure or disconnect: {:?}", + observation.http_failures + )); + } + if observation.running_after_restart != observation.expected_survivors { + return Err(format!( + "restart-state loss: expected running {:?}, observed {:?}", + observation.expected_survivors, observation.running_after_restart + )); + } + if observation.stopped_after_restart != observation.expected_stopped { + return Err(format!( + "destroy set mismatch: expected stopped {:?}, observed {:?}", + observation.expected_stopped, observation.stopped_after_restart + )); + } + if observation.reachable_nodes != observation.expected_survivors { + return Err(format!( + "survivor reachability mismatch: expected {:?}, observed {:?}", + observation.expected_survivors, observation.reachable_nodes + )); + } + let provider_set = observation + .provider_resources + .iter() + .copied() + .collect::>(); + if provider_set.len() != observation.provider_resources.len() { + return Err(format!( + "duplicate provider resources: {:?}", + observation.provider_resources + )); + } + if provider_set != observation.expected_survivors { + return Err(format!( + "provider ledger mismatch: expected {:?}, observed {:?}", + observation.expected_survivors, provider_set + )); + } + let expected_all = observation + .expected_survivors + .union(&observation.expected_stopped) + .copied() + .collect::>(); + if observation.provider_created != expected_all { + return Err(format!( + "provider create ledger mismatch: expected {:?}, observed {:?}", + expected_all, observation.provider_created + )); + } + if observation.provider_destroyed != observation.expected_stopped { + return Err(format!( + "provider destroy ledger mismatch: expected {:?}, observed {:?}", + observation.expected_stopped, observation.provider_destroyed + )); + } + if observation.selected_offers.len() + != observation + .expected_survivors + .len() + .saturating_add(observation.expected_stopped.len()) + || observation + .selected_offers + .values() + .any(|offer| !observation.searched_offers.contains(offer)) + { + return Err(format!( + "selected offer did not come from search response: selected={:?}, searched={:?}", + observation.selected_offers, observation.searched_offers + )); + } + if observation.actor_initial.poisoned != 0 + || observation.actor_before_replays.poisoned != 0 + || observation.actor_after_replays.poisoned != 0 + || observation.actor_after_teardown.poisoned != 0 + { + return Err(format!( + "actor poisoning observed: initial={:?}, before={:?}, after={:?}, teardown={:?}", + observation.actor_initial, + observation.actor_before_replays, + observation.actor_after_replays, + observation.actor_after_teardown, + )); + } + if observation.actor_initial.type_mismatches != 0 + || observation.actor_before_replays.type_mismatches != 0 + || observation.actor_after_replays.type_mismatches != 0 + || observation.actor_after_teardown.type_mismatches != 0 + { + return Err(format!( + "actor message type mismatch observed: initial={:?}, before={:?}, after={:?}, teardown={:?}", + observation.actor_initial, + observation.actor_before_replays, + observation.actor_after_replays, + observation.actor_after_teardown, + )); + } + for (phase, census) in [ + ("before replay", &observation.actor_before_replays), + ("after replay", &observation.actor_after_replays), + ] { + let typed_worker_count = census + .workers_by_stream + .values() + .flat_map(|types| types.iter()) + .map(|(actor_type, count)| { + if actor_type == "" { + Err(format!( + "{phase} worker census contains an unknown actor type" + )) + } else { + Ok(*count) + } + }) + .collect::, _>>()? + .into_iter() + .sum::(); + if census.workers_by_stream.len() != observation.expected_survivors.len() + || typed_worker_count != census.worker_actors + { + return Err(format!( + "{phase} worker census is incomplete: expected_nodes={}, census={census:?}", + observation.expected_survivors.len() + )); + } + } + let mut before_types = observation + .actor_before_replays + .orchestrator_by_type + .clone(); + let mut after_types = observation.actor_after_replays.orchestrator_by_type.clone(); + before_types.remove("myelin::orchestration::control::ControlReplyObserver"); + after_types.remove("myelin::orchestration::control::ControlReplyObserver"); + if observation.actor_before_replays.actors != observation.actor_after_replays.actors + || observation.actor_before_replays.orchestrator_actors + != observation.actor_after_replays.orchestrator_actors + || observation.actor_before_replays.worker_actors + != observation.actor_after_replays.worker_actors + || observation.actor_before_replays.mailbox_depth + != observation.actor_after_replays.mailbox_depth + || observation.actor_before_replays.workers_by_stream + != observation.actor_after_replays.workers_by_stream + || before_types != after_types + { + return Err(format!( + "steady-state actor census changed under replay: before={:?}, after={:?}", + observation.actor_before_replays, observation.actor_after_replays + )); + } + let live_node_count = observation.expected_survivors.len() as u64; + if observation.actor_before_replays.worker_actors + > live_node_count.saturating_mul(ACTORS_PER_NODE_LIMIT) + { + return Err(format!( + "per-node actor bound exceeded: live_nodes={live_node_count}, census={:?}", + observation.actor_before_replays + )); + } + if observation.actor_after_teardown.orchestrator_actors + != observation.actor_initial.orchestrator_actors + || observation.actor_after_teardown.mailbox_depth != 0 + { + return Err(format!( + "actor census did not return to baseline: initial={:?}, teardown={:?}", + observation.actor_initial, observation.actor_after_teardown + )); + } + if !observation.teardown_resources.is_empty() || !observation.labeled_temp_resources.is_empty() + { + return Err(format!( + "teardown leak: provider={:?}, labeled_temp={:?}", + observation.teardown_resources, observation.labeled_temp_resources + )); + } + Ok(()) +} + +struct ScenarioHarness { + state_dir: PathBuf, + provider_url: String, + dashboard_port: u16, + run_id: u64, + provider_ledger_path: PathBuf, + restart: usize, + child: Option, + log_paths: Vec, + unexpected_exits: Vec, +} + +impl ScenarioHarness { + fn new(state_dir: PathBuf, provider_url: String, dashboard_port: u16, run_id: u64) -> Self { + let provider_ledger_path = state_dir.join("mock-vastai-lifecycle.jsonl"); + Self { + state_dir, + provider_url, + dashboard_port, + run_id, + provider_ledger_path, + restart: 0, + child: None, + log_paths: Vec::new(), + unexpected_exits: Vec::new(), + } + } + + fn start(&mut self) -> Result<(), String> { + if self.child.is_some() { + return Err("orchestrator is already running".to_owned()); + } + let log_path = self + .state_dir + .join(format!("orchestrator-{}.log", self.restart)); + let stdout = OpenOptions::new() + .create(true) + .append(true) + .open(&log_path) + .map_err(|error| format!("open {}: {error}", log_path.display()))?; + let stderr = stdout + .try_clone() + .map_err(|error| format!("clone {}: {error}", log_path.display()))?; + let mut command = Command::new(env!("CARGO_BIN_EXE_myelin-orchestrator")); + command + .current_dir(&self.state_dir) + .args([ + "--provider", + "vastai", + "--vastai-provisioning", + "mock", + "--dashboard", + "--state-dir", + ]) + .arg(&self.state_dir) + .args(["--run-id", &self.run_id.to_string()]) + .env("VASTAI_BASE_URL", &self.provider_url) + .env("VAST_API_KEY", "stateful-e2e-secret") + .env("MYELIN_DASHBOARD_PORT", self.dashboard_port.to_string()) + .env("MYELIN_MOCK_VASTAI_LEDGER_PATH", &self.provider_ledger_path) + .env("MYELIN_VASTAI_POLL_INTERVAL_SECS", "1") + .env("MYELIN_IROH_RELAY_MODE", "disabled") + .env("RUST_BACKTRACE", "1") + .stdin(Stdio::null()) + .stdout(Stdio::from(stdout)) + .stderr(Stdio::from(stderr)); + let child = command_spawn(&mut command) + .map_err(|error| format!("spawn Cargo-built Myelin orchestrator: {error}"))?; + self.child = Some(child); + self.log_paths.push(log_path); + self.restart += 1; + Ok(()) + } + fn ensure_running(&mut self) -> Result<(), String> { + let child = self + .child + .as_mut() + .ok_or_else(|| "orchestrator process is absent".to_owned())?; + if let Some(status) = child_try_wait(child) + .map_err(|error| format!("inspect live orchestrator process: {error}"))? + { + let failure = format!("orchestrator exited unexpectedly with {status}"); + self.unexpected_exits.push(failure.clone()); + return Err(failure); + } + Ok(()) + } + + fn restart_gracefully(&mut self, deadline: Instant) -> Result<(), String> { + self.stop_child_gracefully(deadline)?; + self.start() + } + + fn restart_abruptly(&mut self) -> Result<(), String> { + self.stop_child_abruptly()?; + self.start() + } + + fn stop_child_gracefully(&mut self, deadline: Instant) -> Result<(), String> { + let Some(mut child) = self.child.take() else { + return Ok(()); + }; + if child_try_wait(&mut child) + .map_err(|error| format!("inspect orchestrator process: {error}"))? + .is_none() + { + request_child_termination(&child) + .map_err(|error| format!("request graceful orchestrator stop: {error}"))?; + } + loop { + if let Some(status) = child_try_wait(&mut child) + .map_err(|error| format!("inspect graceful orchestrator stop: {error}"))? + { + if status.success() { + return Ok(()); + } + return Err(format!( + "graceful orchestrator stop exited with {status}; logs={}", + self.logs() + )); + } + if Instant::now() >= deadline { + let _ = child_kill(&mut child); + let _ = child_wait(&mut child); + return Err(format!( + "graceful orchestrator stop exceeded case deadline; logs={}", + self.logs() + )); + } + std::thread::sleep(POLL); + } + } + + fn stop_child_abruptly(&mut self) -> Result<(), String> { + let Some(mut child) = self.child.take() else { + return Ok(()); + }; + if child_try_wait(&mut child) + .map_err(|error| format!("inspect orchestrator process: {error}"))? + .is_none() + { + child_kill(&mut child) + .map_err(|error| format!("kill orchestrator process: {error}"))?; + } + child_wait(&mut child) + .map_err(|error| format!("wait for orchestrator process: {error}"))?; + Ok(()) + } + + fn base_url(&self) -> String { + format!("http://127.0.0.1:{}", self.dashboard_port) + } + + fn logs(&self) -> String { + let mut paths = self.log_paths.clone(); + if let Ok(entries) = std::fs::read_dir(self.state_dir.join("process-output")) { + paths.extend(entries.filter_map(|entry| entry.ok().map(|entry| entry.path()))); + } + paths.sort(); + paths + .iter() + .map(|path| { + std::fs::read_to_string(path) + .map(|text| format!("\n--- {} ---\n{text}", path.display())) + .unwrap_or_else(|error| { + format!("\n--- {} unavailable: {error} ---", path.display()) + }) + }) + .collect() + } + + fn cleanup_workers(&self) { + let environment = [("MYELIN_RUN_ID".to_owned(), self.run_id.to_string())]; + let Ok(processes) = find_process_identities_by_environment(&environment, true) else { + return; + }; + for process in processes { + let _ = terminate_process_group( + &process, + Duration::from_secs(2), + Duration::from_millis(20), + ); + } + } + fn remove_worker_sockets(&self, nodes: &BTreeSet) -> Result<(), String> { + for node_id in nodes { + let path = node_socket(self.run_id, *node_id); + match std::fs::remove_file(&path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(format!( + "remove labeled worker socket {}: {error}", + path.display() + )); + } + } + } + Ok(()) + } +} + +impl Drop for ScenarioHarness { + fn drop(&mut self) { + let _ = self.stop_child_abruptly(); + self.cleanup_workers(); + let possible_nodes = BTreeSet::from([1, 2]); + let _ = self.remove_worker_sockets(&possible_nodes); + } +} + +fn production_offers() -> Value { + let offers = (0..OFFER_COUNT) + .map(|index| { + json!({ + "id": 8_675_300_u64 + index as u64, + "gpu_name": "RTX 4090", + "num_gpus": 1, + "gpu_ram": 24_576.0, + "dph_total": 0.20 + index as f64 / 100.0, + "host_id": 90_000_u64 + index as u64, + "compute_cap": 890, + "verification": "verified", + "reliability2": 0.995, + "inet_down": 1_000.0, + "inet_up": 800.0, + "internet_down_cost_per_tb": 1.5, + "internet_up_cost_per_tb": 2.5, + "geolocation": "US", + "disk_bw": 2_000.0, + "duration": 86_400.0, + "rentable": true + }) + }) + .collect::>(); + json!({"offers": offers}) +} + +fn reserve_port() -> Result { + let listener = TcpListener::bind(("127.0.0.1", 0)) + .map_err(|error| format!("reserve dashboard port: {error}"))?; + listener + .local_addr() + .map(|address| address.port()) + .map_err(|error| format!("read reserved dashboard address: {error}")) +} + +fn check_deadlines(case_deadline: Instant, trace: &[String]) -> Result<(), String> { + let now = Instant::now(); + let suite_started = *SUITE_STARTED.get_or_init(Instant::now); + if now >= case_deadline { + return Err(format!( + "{}-second E2E case deadline exceeded; trace={trace:#?}", + CASE_DEADLINE.as_secs() + )); + } + if now.duration_since(suite_started) >= SUITE_DEADLINE { + return Err(format!( + "{}-second E2E suite deadline exceeded; trace={trace:#?}", + SUITE_DEADLINE.as_secs() + )); + } + Ok(()) +} + +fn wait_for( + case_deadline: Instant, + trace: &[String], + description: &str, + mut probe: impl FnMut() -> Result, String>, +) -> Result { + let mut last_error = None; + loop { + check_deadlines(case_deadline, trace)?; + match probe() { + Ok(Some(value)) => return Ok(value), + Ok(None) => {} + Err(error) => last_error = Some(error), + } + std::thread::sleep(POLL); + if Instant::now() >= case_deadline { + return Err(format!( + "timed out waiting for {description}; last_error={last_error:?}; trace={trace:#?}" + )); + } + } +} + +fn request_json( + method: &str, + url: &str, + body: Option<&Value>, +) -> Result<(u16, Option), String> { + let agent = ureq::AgentBuilder::new() + .timeout_connect(Duration::from_millis(250)) + .timeout_read(Duration::from_secs(3)) + .timeout_write(Duration::from_secs(1)) + .build(); + let request = agent.request(method, url).set("Accept", "application/json"); + let result = match body { + Some(body) => request + .set("Content-Type", "application/json") + .send_string(&body.to_string()), + None => request.call(), + }; + let response = match result { + Ok(response) => response, + Err(ureq::Error::Status(_, response)) => response, + Err(error) => { + let failure = format!("{method} {url} disconnected: {error}"); + record_http_failure(failure.clone()); + return Err(failure); + } + }; + let status = response.status(); + if status >= 500 { + record_http_failure(format!("{method} {url} returned HTTP {status}")); + } + let raw = response.into_string().map_err(|error| { + let failure = format!("read {method} {url} response: {error}"); + record_http_failure(failure.clone()); + failure + })?; + let json = if raw.trim().is_empty() { + None + } else { + Some(serde_json::from_str(&raw).map_err(|error| { + let failure = format!("parse {method} {url} response {raw:?}: {error}"); + record_http_failure(failure.clone()); + failure + })?) + }; + Ok((status, json)) +} + +fn get_status(base_url: &str) -> Result { + let url = format!("{base_url}/api/control/status"); + let (status, body) = request_json("GET", &url, None)?; + if status != 200 { + return Err(format!("status endpoint returned {status}: {body:?}")); + } + body.and_then(|body| body.get("Status").cloned()) + .ok_or_else(|| "status response has no Status model".to_owned()) +} + +fn search_offers(base_url: &str, count: usize) -> Result, String> { + let url = format!("{base_url}/api/control/offers"); + let request = json!({ + "gpu_model": "RTX 4090", + "min_gpu_ram_mb": 20_000, + "min_compute_cap": 800, + "min_reliability": 0.99, + "require_verified": true, + "min_download_mbps": 500.0, + "min_upload_mbps": 400.0, + "max_hourly_price": 1.0, + "blacklist_hosts": [], + "count": count + }); + let (status, body) = request_json("POST", &url, Some(&request))?; + if status != 200 { + return Err(format!("offer search returned {status}: {body:?}")); + } + body.and_then(|body| body.get("Offers").cloned()) + .and_then(|offers| offers.as_array().cloned()) + .ok_or_else(|| "offer response has no Offers array".to_owned())? + .into_iter() + .map(|offer| { + offer + .get("offer_id") + .and_then(Value::as_u64) + .ok_or_else(|| format!("offer has no offer_id: {offer}")) + }) + .collect() +} +fn assert_typed_prerequisite_rejection(base_url: &str) -> Result { + match search_offers(base_url, 0) { + Err(error) if error.contains("409") && error.contains("error") => Ok(error), + result => Err(format!( + "prerequisite-invalid action did not produce a typed rejection: {result:?}" + )), + } +} + +#[derive(Clone, Debug)] +struct AcceptedCommand { + path: String, + submissions: Vec, + terminal: Option, +} + +#[derive(Clone, Debug, Default)] +struct ExternalReplyCollector { + accepted: BTreeMap, + public_replies: Vec, + http_failures: Vec, +} + +impl ExternalReplyCollector { + fn submit(&mut self, base_url: &str, path: &str, body: &Value) -> Result<(), String> { + let command_id = body + .get("command_id") + .and_then(Value::as_str) + .ok_or_else(|| format!("external mutation has no command_id: {body}"))? + .to_owned(); + let url = format!("{base_url}{path}"); + let response = request_json("POST", &url, Some(body)); + let (status, response_body) = match response { + Ok(response) => response, + Err(error) => { + self.http_failures + .push(format!("POST {path} disconnected: {error}")); + return Err(error); + } + }; + if status >= 500 { + self.http_failures + .push(format!("POST {path} returned {status}: {response_body:?}")); + } + if status != 202 { + return Err(format!("POST {path} returned {status}: {response_body:?}")); + } + self.public_replies + .push(format!("POST {path} command_id={command_id} status=202")); + self.accepted + .entry(command_id) + .or_insert_with(|| AcceptedCommand { + path: path.to_owned(), + submissions: Vec::new(), + terminal: None, + }) + .submissions + .push(body.clone()); + Ok(()) + } + + fn observe_status(&mut self, status: &Value) -> Result<(), String> { + let commands = status + .get("commands") + .and_then(Value::as_array) + .ok_or_else(|| format!("status has no command ledger: {status}"))?; + for (command_id, accepted) in &mut self.accepted { + let matches = commands + .iter() + .filter(|command| { + command.get("command_id").and_then(Value::as_str) == Some(command_id.as_str()) + }) + .collect::>(); + if matches.len() > 1 { + return Err(format!( + "accepted command {command_id} has {} public records: {matches:?}", + matches.len() + )); + } + let Some(record) = matches.first() else { + continue; + }; + if record + .get("state") + .and_then(Value::as_str) + .is_some_and(|state| matches!(state, "succeeded" | "failed")) + { + if accepted + .terminal + .as_ref() + .is_some_and(|terminal| terminal != *record) + { + return Err(format!( + "accepted command {command_id} changed terminal reply: before={:?}, after={record}", + accepted.terminal + )); + } + accepted.terminal = Some((*record).clone()); + } + } + Ok(()) + } + + fn all_terminal(&self) -> bool { + self.accepted + .values() + .all(|accepted| accepted.terminal.is_some()) + } + + fn terminal(&self, command_id: &str) -> Option<&Value> { + self.accepted + .get(command_id) + .and_then(|accepted| accepted.terminal.as_ref()) + } +} + +fn flush_control(base_url: &str) -> Result<(), String> { + let url = format!("{base_url}/api/control/flush"); + let (status, response) = request_json("POST", &url, None)?; + if status != 200 || response != Some(Value::String("Flushed".to_owned())) { + return Err(format!("control flush returned {status}: {response:?}")); + } + Ok(()) +} + +fn wait_for_accepted_replies( + harness: &ScenarioHarness, + case_deadline: Instant, + trace: &[String], + replies: &mut ExternalReplyCollector, +) -> Result { + wait_for(case_deadline, trace, "all accepted command replies", || { + let status = get_status(&harness.base_url())?; + replies.observe_status(&status)?; + Ok(replies.all_terminal().then_some(status)) + }) +} + +fn concurrent_status_requests(port: u16) -> Result<(), String> { + let request = + b"GET /api/control/status HTTP/1.1\r\nHost: 127.0.0.1\r\nAccept: application/json\r\nConnection: close\r\n\r\n"; + let mut streams = Vec::new(); + for _ in 0..2 { + let mut stream = TcpStream::connect(("127.0.0.1", port)) + .map_err(|error| format!("connect concurrent dashboard request: {error}"))?; + stream + .set_read_timeout(Some(Duration::from_secs(1))) + .map_err(|error| format!("set concurrent dashboard read timeout: {error}"))?; + stream + .write_all(request) + .map_err(|error| format!("write concurrent dashboard request: {error}"))?; + stream + .flush() + .map_err(|error| format!("flush concurrent dashboard request: {error}"))?; + streams.push(stream); + } + for mut stream in streams { + let mut response = String::new(); + stream + .read_to_string(&mut response) + .map_err(|error| format!("read concurrent dashboard response: {error}"))?; + let status = response + .lines() + .next() + .and_then(|line| line.split_whitespace().nth(1)) + .and_then(|status| status.parse::().ok()) + .ok_or_else(|| format!("malformed concurrent dashboard response: {response:?}"))?; + if status != 200 { + return Err(format!( + "concurrent dashboard request returned {status}: {response:?}" + )); + } + } + Ok(()) +} + +fn node_sets(status: &Value) -> Result<(BTreeSet, BTreeSet), String> { + let nodes = status + .get("nodes") + .and_then(Value::as_array) + .ok_or_else(|| format!("status has no nodes: {status}"))?; + let mut running = BTreeSet::new(); + let mut stopped = BTreeSet::new(); + for node in nodes { + let id = node + .get("logical_node_id") + .and_then(Value::as_u64) + .ok_or_else(|| format!("node has no logical_node_id: {node}"))?; + match node.get("phase").and_then(Value::as_str) { + Some("running") => { + if node.get("runtime").is_none_or(Value::is_null) { + return Err(format!("running node {id} has no runtime readiness facts")); + } + running.insert(id); + } + Some("stopped") => { + stopped.insert(id); + } + _ => {} + } + } + Ok((running, stopped)) +} + +fn selected_offer_map(status: &Value) -> Result, String> { + status + .get("nodes") + .and_then(Value::as_array) + .ok_or_else(|| format!("status has no nodes: {status}"))? + .iter() + .map(|node| { + let node_id = node + .get("logical_node_id") + .and_then(Value::as_u64) + .ok_or_else(|| format!("node has no logical_node_id: {node}"))?; + let offer_id = node + .get("selected_offer_id") + .and_then(Value::as_u64) + .ok_or_else(|| format!("node {node_id} has no selected_offer_id"))?; + Ok((node_id, offer_id)) + }) + .collect() +} + +#[derive(Clone, Debug, Default)] +struct ProviderSnapshot { + live: Vec, + created: Vec, + destroyed: Vec, + events: Vec, +} + +fn provider_ledger_snapshot(path: &PathBuf, run_id: u64) -> Result { + let contents = match std::fs::read_to_string(path) { + Ok(contents) => contents, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(error) => { + return Err(format!( + "read independent provider ledger {}: {error}", + path.display() + )); + } + }; + let mut resources = BTreeMap::::new(); + let mut snapshot = ProviderSnapshot::default(); + for (index, line) in contents.lines().enumerate() { + let event: Value = serde_json::from_str(line).map_err(|error| { + format!( + "parse independent provider ledger {} line {}: {error}; line={line:?}", + path.display(), + index + 1 + ) + })?; + if event.get("run_id").and_then(Value::as_u64) != Some(run_id) { + return Err(format!( + "independent provider ledger contains a foreign run: {event}" + )); + } + let event_type = event + .get("event") + .and_then(Value::as_str) + .ok_or_else(|| format!("provider lifecycle observation has no event: {event}"))?; + let node_id = event + .get("node_id") + .and_then(Value::as_u64) + .ok_or_else(|| format!("provider lifecycle observation has no node_id: {event}"))?; + let provider_ref = event + .get("provider_ref") + .and_then(Value::as_str) + .ok_or_else(|| { + format!("provider lifecycle observation has no provider_ref: {event}") + })?; + match event_type { + "created" => { + if event + .get("selected_offer_id") + .and_then(Value::as_u64) + .is_none_or(|offer_id| offer_id == 0) + { + return Err(format!( + "provider create did not preserve a selected offer: {event}" + )); + } + snapshot.created.push(node_id); + if let Some(previous_node) = resources.insert(provider_ref.to_owned(), node_id) { + return Err(format!( + "duplicate provider resource {provider_ref}: previous node {previous_node}, event={event}" + )); + } + } + "adopted" | "recreated" => match resources.get(provider_ref) { + Some(resource_node) if *resource_node == node_id => {} + _ => { + return Err(format!( + "provider recovered a resource absent from the independent ledger: {event}; live={resources:?}" + )); + } + }, + "destroyed" => { + snapshot.destroyed.push(node_id); + if resources.remove(provider_ref) != Some(node_id) { + return Err(format!( + "provider destroyed a resource absent from the independent ledger: {event}; live={resources:?}" + )); + } + } + other => { + return Err(format!( + "unknown independent provider lifecycle event {other:?}: {event}" + )); + } + } + snapshot.events.push(event); + } + snapshot.live = resources.into_values().collect(); + Ok(snapshot) +} + +fn node_socket(run_id: u64, node_id: u64) -> PathBuf { + std::env::temp_dir().join(format!("myelin-node-debug-join-{run_id}-{node_id}.sock")) +} + +fn process_count(run_id: u64, node_id: u64) -> Result { + find_process_identities_by_environment( + &[ + ("MYELIN_RUN_ID".to_owned(), run_id.to_string()), + ("MYELIN_LOGICAL_NODE_ID".to_owned(), node_id.to_string()), + ], + true, + ) + .map(|matches| matches.len()) + .map_err(|error| format!("find worker process for node {node_id}: {error}")) +} + +fn reachable_nodes(run_id: u64, all_nodes: &BTreeSet) -> Result, String> { + let mut reachable = BTreeSet::new(); + for &node_id in all_nodes { + let count = process_count(run_id, node_id)?; + if count > 1 { + return Err(format!("node {node_id} owns {count} worker processes")); + } + if count == 1 && UnixStream::connect(node_socket(run_id, node_id)).is_ok() { + reachable.insert(node_id); + } + } + Ok(reachable) +} + +fn node_endpoint_request(run_id: u64, node_id: u64, request: &str) -> Result { + let mut stream = UnixStream::connect(node_socket(run_id, node_id)) + .map_err(|error| format!("connect node {node_id} debug endpoint: {error}"))?; + stream + .set_read_timeout(Some(Duration::from_secs(1))) + .map_err(|error| format!("set node {node_id} endpoint read timeout: {error}"))?; + stream + .set_write_timeout(Some(Duration::from_secs(1))) + .map_err(|error| format!("set node {node_id} endpoint write timeout: {error}"))?; + stream + .write_all(request.as_bytes()) + .map_err(|error| format!("write node {node_id} endpoint request: {error}"))?; + stream + .flush() + .map_err(|error| format!("flush node {node_id} endpoint request: {error}"))?; + let mut response = String::new(); + BufReader::new(stream) + .read_line(&mut response) + .map_err(|error| format!("read node {node_id} endpoint response: {error}"))?; + serde_json::from_str(&response) + .map_err(|error| format!("parse node {node_id} endpoint response: {error}")) +} + +fn probe_node_endpoint(run_id: u64, node_id: u64, status: &Value) -> Result<(), String> { + let node = status + .get("nodes") + .and_then(Value::as_array) + .and_then(|nodes| { + nodes + .iter() + .find(|node| node.get("logical_node_id").and_then(Value::as_u64) == Some(node_id)) + }) + .ok_or_else(|| format!("status has no node {node_id}: {status}"))?; + let endpoint_json = node + .pointer("/runtime/endpoint") + .and_then(Value::as_str) + .ok_or_else(|| format!("running node {node_id} has no endpoint: {node}"))?; + let _: Value = serde_json::from_str(endpoint_json) + .map_err(|error| format!("parse node {node_id} endpoint: {error}"))?; + let response = node_endpoint_request(run_id, node_id, "{}\n")?; + if response.get("type").and_then(Value::as_str) != Some("JoinRejected") + || response.get("error").and_then(Value::as_str) != Some("MalformedCommand") + { + return Err(format!( + "node {node_id} endpoint did not complete a typed readiness round trip: {response}" + )); + } + Ok(()) +} + +fn actor_census( + harness: &ScenarioHarness, + worker_nodes: &BTreeSet, +) -> Result { + let base_url = harness.base_url(); + let actor_url = format!("{base_url}/api/control/actors"); + let (actor_status, actor_body) = request_json("GET", &actor_url, None)?; + if actor_status != 200 { + return Err(format!( + "live orchestrator actor census returned {actor_status}: {actor_body:?}" + )); + } + let actor_body = + actor_body.ok_or_else(|| "live orchestrator actor census returned no body".to_owned())?; + let mut census = ActorCensus::default(); + census.orchestrator_actors = actor_body + .get("actors") + .and_then(Value::as_array) + .map_or(0, |actors| actors.len() as u64); + census.actors = census.orchestrator_actors; + if let Some(workers) = actor_body.get("workers").and_then(Value::as_array) { + for worker in workers { + census.mailbox_depth = census.mailbox_depth.saturating_add( + worker + .get("mailbox_depth") + .and_then(Value::as_u64) + .unwrap_or(0), + ); + census.poisoned = census + .poisoned + .saturating_add(worker.get("panics").and_then(Value::as_u64).unwrap_or(0)); + census.type_mismatches = census.type_mismatches.saturating_add( + worker + .get("type_mismatches") + .and_then(Value::as_u64) + .unwrap_or(0), + ); + } + } + + let url = format!("{base_url}/api/view/fleet"); + let (status, body) = request_json("GET", &url, None)?; + if status != 200 { + return Err(format!("fleet actor census returned {status}: {body:?}")); + } + let body = body.ok_or_else(|| "fleet actor census returned no body".to_owned())?; + let live = body + .get("live") + .and_then(Value::as_array) + .ok_or_else(|| format!("fleet actor census has no live array: {body}"))?; + if live.is_empty() { + return Err("fleet actor census has no live streams".to_owned()); + } + for node in live { + let Some(summary) = node.get("actor_summary") else { + continue; + }; + let _actors = summary.get("actors").and_then(Value::as_u64).unwrap_or(0); + let orchestrator = node + .pointer("/stream/origin") + .and_then(Value::as_str) + .is_some_and(|origin| origin == "orchestrator"); + let mut by_type = BTreeMap::new(); + if let Some(roster) = node.get("roster").and_then(Value::as_array) { + for actor in roster { + let actor_type = actor + .get("actor_type") + .and_then(Value::as_str) + .unwrap_or("") + .to_owned(); + *by_type.entry(actor_type).or_insert(0) += 1; + } + } + if orchestrator { + for (actor_type, count) in by_type { + *census.orchestrator_by_type.entry(actor_type).or_insert(0) += count; + } + } + } + for &node_id in worker_nodes { + let response = + node_endpoint_request(harness.run_id, node_id, "{\"type\":\"RuntimeStats\"}\n")?; + if response.get("type").and_then(Value::as_str) != Some("RuntimeStats") { + return Err(format!( + "node {node_id} rejected runtime census request: {response}" + )); + } + let stats = response + .get("stats") + .ok_or_else(|| format!("node {node_id} runtime census has no stats: {response}"))?; + let details = stats + .get("actors") + .and_then(Value::as_array) + .ok_or_else(|| format!("node {node_id} runtime census has no actors: {stats}"))?; + if details.is_empty() { + return Err(format!( + "node {node_id} runtime census is not populated yet" + )); + } + let actors = details.len() as u64; + census.worker_actors = census.worker_actors.saturating_add(actors); + census.actors = census.actors.saturating_add(actors); + census.poisoned = census.poisoned.saturating_add( + details + .iter() + .filter(|actor| actor.get("poisoned").and_then(Value::as_bool) == Some(true)) + .count() as u64, + ); + census.type_mismatches = census.type_mismatches.saturating_add( + stats + .get("workers") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|worker| worker.get("type_mismatches").and_then(Value::as_u64)) + .sum::(), + ); + census.mailbox_depth = census.mailbox_depth.saturating_add( + details + .iter() + .filter_map(|actor| actor.get("mailbox_depth").and_then(Value::as_u64)) + .sum(), + ); + let mut by_type = BTreeMap::new(); + for actor in details { + let actor_type = actor + .get("actor_type") + .and_then(Value::as_str) + .unwrap_or("") + .to_owned(); + *by_type.entry(actor_type).or_insert(0) += 1; + } + census + .workers_by_stream + .insert(node_id.to_string(), by_type); + } + let typed_orchestrator_actors = census.orchestrator_by_type.values().sum::(); + if typed_orchestrator_actors == 0 && !worker_nodes.is_empty() { + return Err(format!( + "live orchestrator roster is not populated yet: fleet={body}" + )); + } + if typed_orchestrator_actors != 0 && typed_orchestrator_actors != census.orchestrator_actors { + return Err(format!( + "orchestrator actor roster disagrees with the live runtime: \ + roster={typed_orchestrator_actors}, runtime={}; fleet={body}; runtime_stats={actor_body}", + census.orchestrator_actors + )); + } + if census.orchestrator_actors == 0 { + return Err(format!( + "live orchestrator actor census has no actors: {actor_body}" + )); + } + Ok(census) +} + +fn wait_for_stable_actor_census( + harness: &ScenarioHarness, + case_deadline: Instant, + trace: &[String], + worker_nodes: &BTreeSet, +) -> Result { + let mut candidate: Option<(ActorCensus, Instant)> = None; + wait_for(case_deadline, trace, "stable actor census", || { + let census = actor_census(harness, worker_nodes)?; + if census.mailbox_depth != 0 { + candidate = None; + return Ok(None); + } + match &candidate { + Some((previous, since)) if previous == &census => { + Ok((since.elapsed() >= CENSUS_STABILITY).then_some(census)) + } + _ => { + candidate = Some((census, Instant::now())); + Ok(None) + } + } + }) +} + +fn wait_for_baseline_actor_census( + harness: &ScenarioHarness, + case_deadline: Instant, + trace: &[String], + baseline: &ActorCensus, +) -> Result { + wait_for(case_deadline, trace, "baseline actor census", || { + let census = actor_census(harness, &BTreeSet::new())?; + Ok((census.actors == baseline.actors + && census.orchestrator_actors == baseline.orchestrator_actors + && census.worker_actors == baseline.worker_actors + && census.poisoned == baseline.poisoned + && census.type_mismatches == baseline.type_mismatches + && census.mailbox_depth == baseline.mailbox_depth) + .then_some(census)) + }) +} + +fn wait_for_state( + harness: &ScenarioHarness, + case_deadline: Instant, + trace: &[String], + expected_running: &BTreeSet, + expected_stopped: &BTreeSet, +) -> Result { + wait_for(case_deadline, trace, "durable node state", || { + let status = get_status(&harness.base_url())?; + let (running, stopped) = node_sets(&status)?; + Ok((running == *expected_running && stopped == *expected_stopped).then_some(status)) + }) +} + +fn wait_for_reachability( + harness: &ScenarioHarness, + case_deadline: Instant, + trace: &[String], + all_nodes: &BTreeSet, + expected: &BTreeSet, +) -> Result, String> { + wait_for(case_deadline, trace, "worker reachability", || { + let reachable = reachable_nodes(harness.run_id, all_nodes)?; + Ok((reachable == *expected).then_some(reachable)) + }) +} + +fn wait_for_provider_ledger( + harness: &ScenarioHarness, + case_deadline: Instant, + trace: &[String], + expected_created: &BTreeSet, + expected_destroyed: &BTreeSet, + expected_live: &BTreeSet, +) -> Result { + wait_for( + case_deadline, + trace, + "independent mock-provider lifecycle ledger", + || { + let snapshot = provider_ledger_snapshot(&harness.provider_ledger_path, harness.run_id)?; + let live_set = snapshot.live.iter().copied().collect::>(); + let created_set = snapshot.created.iter().copied().collect::>(); + let destroyed_set = snapshot.destroyed.iter().copied().collect::>(); + Ok((created_set == *expected_created + && snapshot.created.len() == expected_created.len() + && destroyed_set == *expected_destroyed + && snapshot.destroyed.len() == expected_destroyed.len() + && live_set == *expected_live + && snapshot.live.len() == expected_live.len()) + .then_some(snapshot)) + }, + ) +} +#[derive(Clone, Debug)] +struct LifecyclePlan { + selected_offers: Vec, + all_nodes: BTreeSet, + stopped: BTreeSet, + survivors: BTreeSet, +} + +fn lifecycle_plan(case: &E2eCase, searched: &[u64]) -> Result { + let usable = searched.len().min(2); + if usable == 0 { + return Err("cannot derive lifecycle plan without a usable search result".to_owned()); + } + let node_count = 1 + usize::from(case.node_seed) % usable; + let selected_offers = (0..node_count) + .map(|offset| searched[(case.offer_offset + offset) % searched.len()]) + .collect::>(); + let all_nodes = (1..=node_count as u64).collect::>(); + let protected_survivor = 1 + case.seed % node_count as u64; + let mut stopped = BTreeSet::new(); + for node_id in &all_nodes { + if *node_id != protected_survivor + && case.kill_mask & (1_u8 << ((*node_id as usize - 1) % 8)) != 0 + { + stopped.insert(*node_id); + } + } + let survivors = all_nodes + .difference(&stopped) + .copied() + .collect::>(); + Ok(LifecyclePlan { + selected_offers, + all_nodes, + stopped, + survivors, + }) +} + +fn labeled_temp_resources(harness: &ScenarioHarness, all_nodes: &BTreeSet) -> Vec { + let mut leaked = all_nodes + .iter() + .map(|node_id| node_socket(harness.run_id, *node_id)) + .filter(|path| path.exists()) + .collect::>(); + let registry_path = harness.state_dir.join("process-nodes.json"); + let registry_leaked = match std::fs::read_to_string(®istry_path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => false, + Ok(contents) => !matches!( + serde_json::from_str::(&contents), + Ok(Value::Object(registry)) if registry.is_empty() + ), + Err(_) => true, + }; + if registry_leaked { + leaked.push(registry_path); + } + leaked +} + +fn setup_failure_with_diagnostics(error: String, case: &E2eCase, trace: &[String]) -> String { + format!( + "{error}; seed={}; generated_actions={:#?}; trace={trace:#?}; \ + public_replies=[]; http_failures=[]; process_logs=; \ + census=; reachability=; provider_ledger=", + case.seed, case.actions + ) +} + +fn failure_with_diagnostics( + error: String, + case: &E2eCase, + trace: &[String], + replies: &ExternalReplyCollector, + census_snapshots: &[ActorCensus], + harness: &ScenarioHarness, + provider: &TestHttpServer, +) -> String { + let status = get_status(&harness.base_url()); + let node_sets = status + .as_ref() + .ok() + .and_then(|status| node_sets(status).ok()); + let observed_nodes = node_sets + .as_ref() + .map(|(running, stopped)| running.union(stopped).copied().collect()) + .unwrap_or_default(); + let reachability = reachable_nodes(harness.run_id, &observed_nodes); + let census = node_sets + .as_ref() + .map(|(running, _)| actor_census(harness, running)) + .transpose(); + let provider_ledger = provider_ledger_snapshot(&harness.provider_ledger_path, harness.run_id); + let worker_processes = find_process_identities_by_environment( + &[("MYELIN_RUN_ID".to_owned(), harness.run_id.to_string())], + true, + ); + let worker_sockets = observed_nodes + .iter() + .map(|node_id| (*node_id, node_socket(harness.run_id, *node_id).exists())) + .collect::>(); + format!( + "{error}; seed={}; generated_actions={:#?}; trace={trace:#?}; \ + public_replies={:#?}; http_failures={:#?}; observed_http_failures={:#?}; \ + unexpected_process_exits={:#?}; \ + status={status:#?}; census_snapshots={census_snapshots:#?}; current_census={census:#?}; \ + reachability={reachability:#?}; worker_processes={worker_processes:#?}; \ + worker_sockets={worker_sockets:#?}; provider_ledger={provider_ledger:#?}; \ + provider_http_requests={:#?}; process_logs={}", + case.seed, + case.actions, + replies.public_replies, + replies.http_failures, + observed_http_failures(), + harness.unexpected_exits, + provider.requests(), + harness.logs(), + ) +} + +fn run_stateful_case(case: E2eCase) -> Result<(), String> { + let case_started = Instant::now(); + let case_deadline = case_started + CASE_DEADLINE; + let mut trace = vec![format!( + "seed={} generated_actions={:#?}", + case.seed, case.actions + )]; + discard_http_failures_since(0); + check_deadlines(case_deadline, &trace) + .map_err(|error| setup_failure_with_diagnostics(error, &case, &trace))?; + + let provider = TestHttpServer::start(vec![TestHttpRoute::json( + "GET", + "/api/v0/bundles/", + 200, + production_offers(), + )]) + .map_err(|error| setup_failure_with_diagnostics(error, &case, &trace))?; + let temp = tempfile::tempdir().map_err(|error| { + setup_failure_with_diagnostics(format!("create E2E state dir: {error}"), &case, &trace) + })?; + let run_id = 7_000_000_u64 + case.seed % 1_000_000; + let dashboard_port = + reserve_port().map_err(|error| setup_failure_with_diagnostics(error, &case, &trace))?; + let mut harness = ScenarioHarness::new( + temp.path().to_path_buf(), + provider.uri(), + dashboard_port, + run_id, + ); + let mut replies = ExternalReplyCollector::default(); + let mut census_snapshots = Vec::new(); + + let execution = (|| -> Result { + harness.start()?; + trace.push("launch direct Cargo-built Myelin orchestrator".to_owned()); + let readiness_http_checkpoint = http_failure_checkpoint(); + wait_for(case_deadline, &trace, "dashboard readiness", || { + get_status(&harness.base_url()).map(Some) + })?; + discard_http_failures_since(readiness_http_checkpoint); + harness.ensure_running()?; + let mut generated_restart_mode = None; + concurrent_status_requests(harness.dashboard_port)?; + trace.push("two concurrent independent status requests completed".to_owned()); + let actor_initial = + wait_for_stable_actor_census(&harness, case_deadline, &trace, &BTreeSet::new())?; + census_snapshots.push(actor_initial.clone()); + + let mut searched = Vec::new(); + let mut generated_restarts = 0_usize; + let mut typed_rejections = 0_usize; + for (action_index, action) in case.actions.clone().into_iter().enumerate() { + check_deadlines(case_deadline, &trace)?; + harness.ensure_running()?; + trace.push(format!("generated action {action_index}: {action:?}")); + match action { + ExternalAction::Search { count } => match search_offers(&harness.base_url(), count) + { + Ok(offers) if count > 0 => { + searched = offers; + trace.push(format!("search returned usable offers {searched:?}")); + } + Ok(offers) => { + return Err(format!( + "prerequisite-invalid search unexpectedly succeeded: {offers:?}" + )); + } + Err(error) + if count == 0 && error.contains("409") && error.contains("error") => + { + typed_rejections += 1; + trace.push(format!("typed invalid-search rejection: {error}")); + } + Err(error) => { + replies + .http_failures + .push(format!("offer search failed: {error}")); + return Err(error); + } + }, + ExternalAction::Provision { + command_slot, + use_searched_offers, + } => { + let Some(plan) = use_searched_offers + .then(|| lifecycle_plan(&case, &searched).ok()) + .flatten() + else { + let rejection = assert_typed_prerequisite_rejection(&harness.base_url())?; + typed_rejections += 1; + trace.push(format!( + "provision rejected before usable search selection: {rejection}" + )); + continue; + }; + let request = json!({ + "command_id": format!("generated-provision-{command_slot}"), + "count": plan.selected_offers.len(), + "selected_offer_ids": plan.selected_offers, + }); + replies.submit(&harness.base_url(), "/api/control/provision", &request)?; + } + ExternalAction::Query => { + let status = get_status(&harness.base_url()).map_err(|error| { + replies + .http_failures + .push(format!("status query failed: {error}")); + error + })?; + replies.observe_status(&status)?; + } + ExternalAction::Kill { + node_slot, + command_slot, + } => { + let Some(plan) = lifecycle_plan(&case, &searched).ok() else { + let rejection = assert_typed_prerequisite_rejection(&harness.base_url())?; + typed_rejections += 1; + trace.push(format!( + "kill rejected before usable search selection: {rejection}" + )); + continue; + }; + let Some(target) = (!plan.stopped.is_empty()).then(|| { + let index = usize::from(node_slot) % plan.stopped.len(); + *plan.stopped.iter().nth(index).expect("bounded kill index") + }) else { + let rejection = assert_typed_prerequisite_rejection(&harness.base_url())?; + typed_rejections += 1; + trace.push(format!( + "kill rejected because its generated subset is empty: {rejection}" + )); + continue; + }; + let status = get_status(&harness.base_url())?; + replies.observe_status(&status)?; + let (running, stopped) = node_sets(&status)?; + if !running.contains(&target) && !stopped.contains(&target) { + let rejection = assert_typed_prerequisite_rejection(&harness.base_url())?; + typed_rejections += 1; + trace.push(format!( + "kill of node {target} rejected before provision: {rejection}" + )); + continue; + } + let request = json!({ + "command_id": format!("generated-kill-{command_slot}"), + "logical_node_id": target, + }); + replies.submit(&harness.base_url(), "/api/control/kill", &request)?; + } + ExternalAction::Flush => { + flush_control(&harness.base_url()).map_err(|error| { + replies + .http_failures + .push(format!("control flush failed: {error}")); + error + })?; + replies + .public_replies + .push("POST /api/control/flush reply=Flushed".to_owned()); + } + ExternalAction::Restart { mode } => { + if replies.accepted.is_empty() { + trace.push(format!( + "restart {mode:?} rejected by external model before any acknowledged mutation" + )); + continue; + } + if generated_restarts == 1 { + trace.push(format!( + "restart {mode:?} rejected by the bounded external model after one generated restart" + )); + continue; + } + flush_control(&harness.base_url())?; + replies.public_replies.push(format!( + "POST /api/control/flush reply=Flushed before {mode:?}" + )); + match mode { + RestartMode::Graceful => { + harness.restart_gracefully(case_deadline)?; + } + RestartMode::FlushSafeAbrupt => { + harness.restart_abruptly()?; + } + } + generated_restarts += 1; + generated_restart_mode = Some(mode); + let readiness_http_checkpoint = http_failure_checkpoint(); + wait_for( + case_deadline, + &trace, + "dashboard after generated restart", + || get_status(&harness.base_url()).map(Some), + )?; + discard_http_failures_since(readiness_http_checkpoint); + trace.push(format!( + "completed generated {mode:?} restart between acknowledged operations" + )); + } + ExternalAction::EndpointProbe { node_slot } => { + let status = get_status(&harness.base_url())?; + replies.observe_status(&status)?; + let (running, _) = node_sets(&status)?; + if !running.is_empty() { + let index = usize::from(node_slot) % running.len(); + let node_id = *running.iter().nth(index).expect("bounded probe index"); + probe_node_endpoint(run_id, node_id, &status)?; + typed_rejections += 1; + trace.push(format!( + "node {node_id} returned typed MalformedCommand rejection" + )); + } else { + trace.push( + "endpoint probe rejected by external model before provision".to_owned(), + ); + } + } + ExternalAction::ConcurrentQueries => { + concurrent_status_requests(harness.dashboard_port).map_err(|error| { + replies + .http_failures + .push(format!("concurrent status requests failed: {error}")); + error + })?; + } + } + } + + if searched.is_empty() { + searched = search_offers(&harness.base_url(), 2)?; + trace.push(format!( + "convergence search returned usable offers {searched:?}" + )); + } + let plan = lifecycle_plan(&case, &searched)?; + if plan.selected_offers.len() > 2 || plan.survivors.is_empty() { + return Err(format!("invalid bounded lifecycle plan: {plan:?}")); + } + + let mut status = if replies.accepted.is_empty() { + get_status(&harness.base_url())? + } else { + wait_for_accepted_replies(&harness, case_deadline, &trace, &mut replies)? + }; + for (command_id, accepted) in &replies.accepted { + if let Some(terminal) = &accepted.terminal + && terminal.get("state").and_then(Value::as_str) == Some("failed") + { + if terminal.get("error").and_then(Value::as_str).is_none() { + return Err(format!( + "failed command {command_id} has no typed rejection: {terminal}" + )); + } + typed_rejections += 1; + } + } + + let (running, stopped) = node_sets(&status)?; + let observed = running.union(&stopped).copied().collect::>(); + if observed.is_empty() { + let request = json!({ + "command_id": format!("converge-provision-{}", case.seed), + "count": plan.selected_offers.len(), + "selected_offer_ids": plan.selected_offers, + }); + replies.submit(&harness.base_url(), "/api/control/provision", &request)?; + wait_for_accepted_replies(&harness, case_deadline, &trace, &mut replies)?; + let terminal = replies + .terminal(request["command_id"].as_str().expect("command id")) + .expect("accepted command has terminal reply"); + if terminal.get("state").and_then(Value::as_str) != Some("succeeded") { + return Err(format!("convergence provision failed: {terminal}")); + } + } else if observed != plan.all_nodes { + return Err(format!( + "accepted replies produced an unexpected node set: expected {:?}, observed {observed:?}, status={status}", + plan.all_nodes + )); + } + + status = wait_for( + case_deadline, + &trace, + "provisioned state from accepted replies", + || { + let status = get_status(&harness.base_url())?; + let (running, stopped) = node_sets(&status)?; + let observed = running.union(&stopped).copied().collect::>(); + Ok((observed == plan.all_nodes).then_some(status)) + }, + )?; + let (_, already_stopped) = node_sets(&status)?; + if !already_stopped.is_subset(&plan.stopped) { + return Err(format!( + "generated successful kills exceeded the survivor-preserving subset: desired={:?}, observed={already_stopped:?}", + plan.stopped + )); + } + for node_id in plan.stopped.difference(&already_stopped) { + let request = json!({ + "command_id": format!("converge-kill-{}-{node_id}", case.seed), + "logical_node_id": node_id, + }); + replies.submit(&harness.base_url(), "/api/control/kill", &request)?; + } + wait_for_accepted_replies(&harness, case_deadline, &trace, &mut replies)?; + for command_id in replies + .accepted + .keys() + .filter(|command_id| command_id.starts_with("converge-kill-")) + { + let terminal = replies + .terminal(command_id) + .expect("accepted kill has terminal reply"); + if terminal.get("state").and_then(Value::as_str) != Some("succeeded") { + return Err(format!("convergence kill failed: {terminal}")); + } + } + + status = wait_for_state( + &harness, + case_deadline, + &trace, + &plan.survivors, + &plan.stopped, + )?; + wait_for_reachability( + &harness, + case_deadline, + &trace, + &plan.all_nodes, + &plan.survivors, + )?; + wait_for_provider_ledger( + &harness, + case_deadline, + &trace, + &plan.all_nodes, + &plan.stopped, + &plan.survivors, + )?; + let selected_offer_by_node = selected_offer_map(&status)?; + let expected_offer_by_node = plan + .selected_offers + .iter() + .enumerate() + .map(|(index, offer_id)| (index as u64 + 1, *offer_id)) + .collect::>(); + if selected_offer_by_node != expected_offer_by_node { + return Err(format!( + "public selected offers differ from searched selection: expected={expected_offer_by_node:?}, observed={selected_offer_by_node:?}" + )); + } + + if typed_rejections == 0 { + match search_offers(&harness.base_url(), 0) { + Err(error) if error.contains("409") && error.contains("error") => { + typed_rejections += 1; + trace.push(format!("typed convergence rejection: {error}")); + } + result => { + return Err(format!( + "could not establish a typed prerequisite rejection: {result:?}" + )); + } + } + } + if typed_rejections == 0 { + return Err("no typed rejection was observed".to_owned()); + } + + let convergence_restarts = match generated_restart_mode { + Some(RestartMode::Graceful) => vec![RestartMode::FlushSafeAbrupt], + Some(RestartMode::FlushSafeAbrupt) => vec![RestartMode::Graceful], + None if case.seed & 1 == 0 => { + vec![RestartMode::Graceful, RestartMode::FlushSafeAbrupt] + } + None => vec![RestartMode::FlushSafeAbrupt, RestartMode::Graceful], + }; + for mode in convergence_restarts { + flush_control(&harness.base_url())?; + replies.public_replies.push(format!( + "POST /api/control/flush reply=Flushed before convergence {mode:?}" + )); + match mode { + RestartMode::Graceful => harness.restart_gracefully(case_deadline)?, + RestartMode::FlushSafeAbrupt => harness.restart_abruptly()?, + } + let readiness_http_checkpoint = http_failure_checkpoint(); + wait_for( + case_deadline, + &trace, + "dashboard after convergence restart", + || get_status(&harness.base_url()).map(Some), + )?; + discard_http_failures_since(readiness_http_checkpoint); + trace.push(format!("convergence tail completed {mode:?} restart")); + } + status = wait_for_state( + &harness, + case_deadline, + &trace, + &plan.survivors, + &plan.stopped, + )?; + wait_for_reachability( + &harness, + case_deadline, + &trace, + &plan.all_nodes, + &plan.survivors, + )?; + wait_for_provider_ledger( + &harness, + case_deadline, + &trace, + &plan.all_nodes, + &plan.stopped, + &plan.survivors, + )?; + for node_id in &plan.survivors { + probe_node_endpoint(run_id, *node_id, &status)?; + } + + let actor_before_replays = + wait_for_stable_actor_census(&harness, case_deadline, &trace, &plan.survivors)?; + census_snapshots.push(actor_before_replays.clone()); + let replay_requests = replies + .accepted + .values() + .filter_map(|accepted| { + accepted + .submissions + .first() + .cloned() + .map(|request| (accepted.path.clone(), request)) + }) + .collect::>(); + for (path, request) in replay_requests { + replies.submit(&harness.base_url(), &path, &request)?; + } + for _ in 0..2 { + let read = get_status(&harness.base_url())?; + replies.observe_status(&read)?; + } + concurrent_status_requests(harness.dashboard_port)?; + status = wait_for_accepted_replies(&harness, case_deadline, &trace, &mut replies)?; + let actor_after_replays = + wait_for_stable_actor_census(&harness, case_deadline, &trace, &plan.survivors)?; + census_snapshots.push(actor_after_replays.clone()); + let (running_after_restart, stopped_after_restart) = node_sets(&status)?; + let reachable = reachable_nodes(run_id, &plan.all_nodes)?; + let provider_snapshot = + provider_ledger_snapshot(&harness.provider_ledger_path, harness.run_id)?; + + for node_id in &plan.survivors { + let request = json!({ + "command_id": format!("teardown-kill-{}-{node_id}", case.seed), + "logical_node_id": node_id, + }); + replies.submit(&harness.base_url(), "/api/control/kill", &request)?; + replies.submit(&harness.base_url(), "/api/control/kill", &request)?; + } + wait_for_accepted_replies(&harness, case_deadline, &trace, &mut replies)?; + wait_for_state( + &harness, + case_deadline, + &trace, + &BTreeSet::new(), + &plan.all_nodes, + )?; + wait_for_reachability( + &harness, + case_deadline, + &trace, + &plan.all_nodes, + &BTreeSet::new(), + )?; + let final_provider = wait_for_provider_ledger( + &harness, + case_deadline, + &trace, + &plan.all_nodes, + &plan.all_nodes, + &BTreeSet::new(), + )?; + let actor_after_teardown = + wait_for_baseline_actor_census(&harness, case_deadline, &trace, &actor_initial)?; + census_snapshots.push(actor_after_teardown.clone()); + let final_status = get_status(&harness.base_url())?; + replies.observe_status(&final_status)?; + if !replies.all_terminal() { + return Err(format!( + "accepted replies remained nonterminal: {:?}", + replies.accepted + )); + } + let commands = final_status + .get("commands") + .and_then(Value::as_array) + .ok_or_else(|| format!("final status has no command ledger: {final_status}"))?; + let terminal_record_counts = replies + .accepted + .keys() + .map(|command_id| { + let count = commands + .iter() + .filter(|command| { + command.get("command_id").and_then(Value::as_str) + == Some(command_id.as_str()) + && command + .get("state") + .and_then(Value::as_str) + .is_some_and(|state| matches!(state, "succeeded" | "failed")) + }) + .count(); + (command_id.clone(), count) + }) + .collect::>(); + + flush_control(&harness.base_url())?; + harness.stop_child_gracefully(case_deadline)?; + wait_for( + case_deadline, + &trace, + "all worker processes stopped", + || { + let live = plan + .all_nodes + .iter() + .map(|node_id| process_count(run_id, *node_id)) + .collect::, _>>()? + .into_iter() + .sum::(); + Ok((live == 0).then_some(())) + }, + )?; + let labeled_temp_resources = labeled_temp_resources(&harness, &plan.all_nodes); + let mut http_failures = observed_http_failures(); + http_failures.extend(replies.http_failures.clone()); + + Ok(OracleObservation { + expected_survivors: plan.survivors, + expected_stopped: plan.stopped, + running_after_restart, + stopped_after_restart, + reachable_nodes: reachable, + provider_resources: provider_snapshot.live, + provider_created: provider_snapshot.created.into_iter().collect(), + provider_destroyed: provider_snapshot.destroyed.into_iter().collect(), + selected_offers: selected_offer_by_node, + searched_offers: searched.into_iter().collect(), + actor_before_replays, + actor_after_replays, + actor_initial, + actor_after_teardown, + terminal_record_counts, + unexpected_process_exits: harness.unexpected_exits.clone(), + http_failures, + teardown_resources: final_provider.live, + labeled_temp_resources, + }) + })(); + + let observation = execution.map_err(|error| { + failure_with_diagnostics( + error, + &case, + &trace, + &replies, + &census_snapshots, + &harness, + &provider, + ) + })?; + validate_oracle(&observation).map_err(|error| { + failure_with_diagnostics( + format!("{error}; observation={observation:#?}"), + &case, + &trace, + &replies, + &census_snapshots, + &harness, + &provider, + ) + })?; + if !provider + .requests() + .iter() + .any(|request| request.method == "GET" && request.path == "/api/v0/bundles/") + { + return Err(failure_with_diagnostics( + "provider search boundary was never exercised".to_owned(), + &case, + &trace, + &replies, + &census_snapshots, + &harness, + &provider, + )); + } + + let teardown_logs = harness.logs(); + let teardown_ledger = provider_ledger_snapshot(&harness.provider_ledger_path, harness.run_id); + let provider_requests = provider.requests(); + drop(harness); + drop(provider); + temp.close().map_err(|error| { + format!( + "remove labeled E2E state directory: {error}; seed={}; actions={:#?}; \ + replies={:#?}; observation={observation:#?}; ledger={teardown_ledger:#?}; \ + provider_requests={provider_requests:#?}; logs={teardown_logs}", + case.seed, case.actions, replies.public_replies + ) + })?; + Ok(()) +} + +proptest! { + #![proptest_config(e2e_proptest_config())] + + #[test] + #[ignore = "nightly process E2E; run with --ignored"] + fn stateful_vastai_dashboard_control_survives_restarts(case in e2e_case()) { + if let Err(error) = run_stateful_case(case) { + prop_assert!(false, "{error}"); + } + } +} + +#[test] +fn e2e_oracle_rejects_controlled_lifecycle_faults() { + let baseline = OracleObservation { + expected_survivors: BTreeSet::from([2]), + expected_stopped: BTreeSet::from([1]), + running_after_restart: BTreeSet::from([2]), + stopped_after_restart: BTreeSet::from([1]), + reachable_nodes: BTreeSet::from([2]), + provider_resources: vec![2], + provider_created: BTreeSet::from([1, 2]), + provider_destroyed: BTreeSet::from([1]), + selected_offers: BTreeMap::from([(1, 101), (2, 102)]), + searched_offers: BTreeSet::from([101, 102]), + actor_before_replays: ActorCensus { + actors: 10, + orchestrator_actors: 5, + worker_actors: 5, + orchestrator_by_type: BTreeMap::from([("orchestrator".to_owned(), 5)]), + workers_by_stream: BTreeMap::from([( + "worker-2".to_owned(), + BTreeMap::from([("worker".to_owned(), 5)]), + )]), + ..ActorCensus::default() + }, + actor_after_replays: ActorCensus { + actors: 10, + orchestrator_actors: 5, + worker_actors: 5, + orchestrator_by_type: BTreeMap::from([("orchestrator".to_owned(), 5)]), + workers_by_stream: BTreeMap::from([( + "worker-2".to_owned(), + BTreeMap::from([("worker".to_owned(), 5)]), + )]), + ..ActorCensus::default() + }, + actor_initial: ActorCensus { + actors: 5, + orchestrator_actors: 5, + orchestrator_by_type: BTreeMap::from([("orchestrator".to_owned(), 5)]), + ..ActorCensus::default() + }, + actor_after_teardown: ActorCensus { + actors: 5, + orchestrator_actors: 5, + orchestrator_by_type: BTreeMap::from([("orchestrator".to_owned(), 5)]), + ..ActorCensus::default() + }, + terminal_record_counts: BTreeMap::from([("command-1".to_owned(), 1)]), + unexpected_process_exits: Vec::new(), + http_failures: Vec::new(), + teardown_resources: Vec::new(), + labeled_temp_resources: Vec::new(), + }; + validate_oracle(&baseline).expect("baseline oracle observation"); + + let mut survivor_destroy = baseline.clone(); + survivor_destroy.provider_resources.clear(); + survivor_destroy.provider_destroyed.insert(2); + assert!( + validate_oracle(&survivor_destroy) + .unwrap_err() + .contains("provider ledger mismatch") + ); + + let mut restart_loss = baseline.clone(); + restart_loss.running_after_restart.clear(); + assert!( + validate_oracle(&restart_loss) + .unwrap_err() + .contains("restart-state loss") + ); + let mut duplicate_provider = baseline.clone(); + duplicate_provider.provider_resources.push(2); + assert!( + validate_oracle(&duplicate_provider) + .unwrap_err() + .contains("duplicate provider resources") + ); + + let mut actor_poison = baseline.clone(); + actor_poison.actor_after_replays.poisoned = 1; + assert!( + validate_oracle(&actor_poison) + .unwrap_err() + .contains("actor poisoning") + ); + let mut unexpected_exit = baseline.clone(); + unexpected_exit + .unexpected_process_exits + .push("exit status 1".to_owned()); + assert!( + validate_oracle(&unexpected_exit) + .unwrap_err() + .contains("unexpected process exit") + ); + + let mut http_failure = baseline.clone(); + http_failure + .http_failures + .push("HTTP 503 or disconnect".to_owned()); + assert!( + validate_oracle(&http_failure) + .unwrap_err() + .contains("HTTP failure or disconnect") + ); + + let mut duplicate_terminal = baseline.clone(); + duplicate_terminal + .terminal_record_counts + .insert("command-1".to_owned(), 2); + assert!( + validate_oracle(&duplicate_terminal) + .unwrap_err() + .contains("exactly one terminal record") + ); + + let mut actor_leak = baseline.clone(); + actor_leak.actor_after_replays.actors = 13; + assert!( + validate_oracle(&actor_leak) + .unwrap_err() + .contains("steady-state actor census") + ); + let mut teardown_leak = baseline; + teardown_leak.teardown_resources.push(2); + teardown_leak + .labeled_temp_resources + .push(PathBuf::from("process-nodes.json")); + assert!( + validate_oracle(&teardown_leak) + .unwrap_err() + .contains("teardown leak") + ); +} diff --git a/clippy.toml b/clippy.toml index 9f6e760..51ad1ae 100644 --- a/clippy.toml +++ b/clippy.toml @@ -1,52 +1,3 @@ -# Clippy enforcement policy for the swactor engine boundary -# (ENGINE_SPEC.md §2 / §3.1). -# -# These direct runtime / scheduling / time / core-driving operations are -# disallowed outside the engine's own substrate implementation. Integrations -# (iroh-driver, myelin, ...) must go through `EngineHandle`. The -# `swactor-engine` Tokio backend and the core driver carry narrow -# `#[allow(clippy::disallowed_methods)]` exemptions because they ARE the -# substrate implementor; the VastAI provider module carries a temporary -# module-level exemption pending its separate redesign (out of scope per §2). -# -# In-scope work that backs actors, transport, RPC, sampling, or node/orchestrator -# progression must schedule through `EngineHandle`. The only retained direct -# uses are narrow exclusions (§2): provider adapters/lifecycle (VastAI), -# provider-specific process supervision and log capture, and top-level OS-signal -# / blocking user-stdin / synchronous process-control sequencing. Each retained -# use carries a local `#[allow]` with its exclusion reason. -# -# Workspace-wide enforcement: `swactor-engine`, `iroh-driver`, and in-scope -# `myelin` carry `#![deny(clippy::disallowed_methods)]` and pass clean. - -disallowed-methods = [ - { path = "tokio::runtime::Runtime::new", reason = "runtime ownership belongs to the engine; construct an engine-owned substrate instead" }, - { path = "tokio::runtime::Builder::new_current_thread", reason = "runtime ownership belongs to the engine; use EngineHandle" }, - { path = "tokio::runtime::Builder::new_multi_thread", reason = "runtime ownership belongs to the engine; use EngineHandle" }, - { path = "tokio::runtime::Handle::current", reason = "ambient runtime detection is forbidden; construct an engine-owned substrate instead" }, - { path = "tokio::runtime::Handle::try_current", reason = "ambient runtime detection is forbidden; construct an engine-owned substrate instead" }, - { path = "tokio::runtime::Runtime::block_on", reason = "blocking on a runtime is forbidden; schedule through EngineHandle" }, - { path = "tokio::runtime::Handle::block_on", reason = "blocking on a runtime is forbidden; schedule through EngineHandle" }, - - { path = "tokio::spawn", reason = "direct scheduling is forbidden; use EngineHandle::spawn" }, - { path = "tokio::task::spawn", reason = "direct scheduling is forbidden; use EngineHandle::spawn" }, - { path = "tokio::task::spawn_blocking", reason = "use EngineHandle::spawn_blocking" }, - { path = "tokio::runtime::Runtime::spawn", reason = "direct scheduling is forbidden; use EngineHandle::spawn" }, - { path = "tokio::runtime::Handle::spawn", reason = "direct scheduling is forbidden; use EngineHandle::spawn" }, - { path = "tokio::runtime::Runtime::spawn_blocking", reason = "use EngineHandle::spawn_blocking" }, - { path = "tokio::runtime::Handle::spawn_blocking", reason = "use EngineHandle::spawn_blocking" }, - - { path = "tokio::time::sleep", reason = "use EngineHandle::timer" }, - { path = "tokio::time::sleep_until", reason = "use EngineHandle::timer" }, - { path = "tokio::time::interval", reason = "use EngineHandle::interval" }, - { path = "tokio::time::interval_at", reason = "use EngineHandle::interval" }, - { path = "tokio::time::timeout", reason = "use an engine-derived timeout" }, - { path = "tokio::time::timeout_at", reason = "use an engine-derived timeout" }, - - { path = "std::thread::spawn", reason = "direct thread scheduling is forbidden; schedule through EngineHandle" }, - { path = "std::thread::sleep", reason = "use EngineHandle::timer; retained only for narrow process-control exclusions (ENGINE_SPEC.md §2)" }, - - { path = "swactor::runtime::Runtime::tick", reason = "manual core driving is forbidden; the engine owns core progression" }, - { path = "swactor::runtime::Runtime::try_tick", reason = "manual core driving is forbidden; the engine owns core progression" }, - { path = "swactor::runtime::Runtime::has_work", reason = "manual core driving is forbidden; the engine owns core progression" }, -] +# Actor control-flow policy is enforced by the repository rustc workspace +# wrapper configured in `.cargo/config.toml`. Clippy is intentionally not a +# second architecture-policy mechanism. diff --git a/crates/bindings/python/Cargo.toml b/crates/bindings/python/Cargo.toml index 5e3afdd..bde3cc8 100644 --- a/crates/bindings/python/Cargo.toml +++ b/crates/bindings/python/Cargo.toml @@ -10,4 +10,5 @@ crate-type = ["cdylib"] [dependencies] swactor = { path = "../../.." } +swactor-engine = { path = "../../engine", default-features = false } pyo3 = { version = "0.23", features = ["extension-module"] } diff --git a/crates/bindings/python/src/lib.rs b/crates/bindings/python/src/lib.rs index 7ba5003..311cdf2 100644 --- a/crates/bindings/python/src/lib.rs +++ b/crates/bindings/python/src/lib.rs @@ -8,9 +8,8 @@ use ::swactor::actor::{ Actor, ActorAddress, ActorInterface, AnyActor, Ctx, Environment, SpawnRequest, }; use ::swactor::config::RuntimeConfig; -use ::swactor::runtime::{ - Inbox, Runtime, RuntimeParts, SingleThreadRuntime as SingleThreadRuntimeHost, -}; +use ::swactor::runtime::{Inbox, Runtime, RuntimeParts}; +use swactor_engine::{Engine, SteppingBackend}; // ─── PyMsg newtype ─────────────────────────────────────────────────────────── @@ -279,7 +278,8 @@ impl From for RuntimeConfig { #[pyclass(name = "Runtime", unsendable)] pub struct PyRuntime { runtime: Runtime, - host: SingleThreadRuntimeHost, + _engine: Engine, + backend: SteppingBackend, } #[pymethods] @@ -293,8 +293,13 @@ impl PyRuntime { }; let parts = RuntimeParts::new(config); let runtime = parts.runtime().clone(); - let host = SingleThreadRuntimeHost::new(parts); - Self { runtime, host } + let backend = SteppingBackend::new(); + let engine = Engine::new(parts, backend.clone()).expect("create Python actor engine"); + Self { + runtime, + _engine: engine, + backend, + } } fn spawn(&self, handler: PyObject) -> PyResult { @@ -316,7 +321,7 @@ impl PyRuntime { } fn tick(&mut self) -> PyResult<()> { - self.host.tick(); + self.backend.step(); Ok(()) } diff --git a/crates/bindings/wasm-runtime/Cargo.toml b/crates/bindings/wasm-runtime/Cargo.toml index c47943c..a6704bb 100644 --- a/crates/bindings/wasm-runtime/Cargo.toml +++ b/crates/bindings/wasm-runtime/Cargo.toml @@ -9,4 +9,5 @@ crate-type = ["cdylib"] [dependencies] swactor = { path = "../../..", default-features = false, features = ["wasm", "std"] } +swactor-engine = { path = "../../engine", default-features = false } wasm-bindgen = "0.2" diff --git a/crates/bindings/wasm-runtime/src/lib.rs b/crates/bindings/wasm-runtime/src/lib.rs index 7dbb6df..7c817ad 100644 --- a/crates/bindings/wasm-runtime/src/lib.rs +++ b/crates/bindings/wasm-runtime/src/lib.rs @@ -3,11 +3,9 @@ use std::sync::Arc; use wasm_bindgen::prelude::*; use swactor::actor::{ActorAddress, ActorExited, ActorInterface}; -use swactor::runtime::{ - Ctx, Inbox, Runtime, RuntimeConfig, RuntimeParts, - SingleThreadRuntime as SingleThreadRuntimeHost, -}; +use swactor::runtime::{Ctx, Inbox, Runtime, RuntimeConfig, RuntimeParts}; use swactor::std::{CtxGroups, CtxWatching, RuntimeGroups, RuntimeNaming, StdExtension}; +use swactor_engine::{Engine, SteppingBackend}; // ─── Core JS-facing types ─────────────────────────────────────────────────── @@ -94,14 +92,15 @@ impl WasmInboxString { /// The browser-facing swactor runtime. /// -/// Owns a cloneable `swactor::Runtime` handle plus the single-threaded host that -/// drives its workers, with StdExtension installed (naming, monitoring, groups). +/// Owns a cloneable `swactor::Runtime` handle plus an engine-backed stepping +/// substrate, with StdExtension installed (naming, monitoring, groups). /// Actors are spawned via dedicated spawn functions (one per actor type). The /// runtime is driven by calling `tick()`. #[wasm_bindgen] pub struct WasmRuntime { rt: Runtime, - host: SingleThreadRuntimeHost, + _engine: Engine, + backend: SteppingBackend, } #[wasm_bindgen] @@ -111,13 +110,18 @@ impl WasmRuntime { let parts = RuntimeParts::new(RuntimeConfig::default()) .with_extension(Arc::new(StdExtension::new())); let rt = parts.runtime().clone(); - let host = SingleThreadRuntimeHost::new(parts); - Self { rt, host } + let backend = SteppingBackend::new(); + let engine = Engine::new(parts, backend.clone()).expect("create wasm actor engine"); + Self { + rt, + _engine: engine, + backend, + } } /// Drive one tick of the runtime. pub fn tick(&mut self) { - self.host.tick(); + self.backend.step(); } /// Number of actors currently alive. diff --git a/crates/dashboard/Cargo.toml b/crates/dashboard/Cargo.toml index 3a3ba2a..3ae7410 100644 --- a/crates/dashboard/Cargo.toml +++ b/crates/dashboard/Cargo.toml @@ -8,6 +8,7 @@ license = "AGPL-3.0-only" axum = "0.8" telemetry = { path = "../telemetry" } swactor = { path = "../..", features = ["serde"] } +swactor-engine = { path = "../engine" } parking_lot = "0.12" serde = { version = "1", features = ["derive"] } serde_json = "1" @@ -18,3 +19,7 @@ tokio-stream = "0.1" # Live control actions (kill processes, provision nodes) for the # provisioning-reconciler demo. Never enabled in shipping builds. demo-control = [] + +[dev-dependencies] +proptest = "1" +tower = { version = "0.5", features = ["util"] } diff --git a/crates/dashboard/proptest-regressions/control.txt b/crates/dashboard/proptest-regressions/control.txt new file mode 100644 index 0000000..b94e141 --- /dev/null +++ b/crates/dashboard/proptest-regressions/control.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 6dbd7f7ab6013cac23e3e815f076193eac00fc906957d486983845194cf20103 # shrinks to inputs = [(0, 0)], split = 0, concurrent = false diff --git a/crates/dashboard/src/control.rs b/crates/dashboard/src/control.rs index 4592133..03c8a5d 100644 --- a/crates/dashboard/src/control.rs +++ b/crates/dashboard/src/control.rs @@ -50,8 +50,283 @@ pub fn set_control_sender(sender: Sender) { let _ = CONTROL_SENDER.set(sender); } +/// Install an actor destination for dashboard-issued control commands. +/// +/// The dashboard owns the blocking channel reader; each command becomes one +/// typed actor observation. +pub fn install_actor_sink( + sender: swactor::runtime::ExternalSender, + actor: swactor::actor::ActorAddress, +) { + let (control_sender, receiver) = std::sync::mpsc::channel(); + set_control_sender(control_sender); + drop(spawn_actor_sink_forwarder(receiver, sender, actor)); +} + +fn spawn_actor_sink_forwarder( + receiver: std::sync::mpsc::Receiver, + sender: swactor::runtime::ExternalSender, + actor: swactor::actor::ActorAddress, +) -> std::thread::JoinHandle<()> { + std::thread::spawn(move || { + while let Ok(command) = receiver.recv() { + if sender.send_to(actor, command).is_err() { + return; + } + } + }) +} + pub(crate) fn dispatch(command: ControlCommand) -> bool { CONTROL_SENDER .get() .is_some_and(|sender| sender.send(command).is_ok()) } + +#[cfg(test)] +mod properties { + use std::sync::Arc; + use std::time::Duration; + + use parking_lot::Mutex; + use proptest::prelude::*; + use swactor::actor::{ActorInterface, Ctx}; + use swactor::config::RuntimeConfig; + use swactor::runtime::{Runtime, RuntimeParts}; + use swactor_engine::{Engine, SteppingBackend}; + + use super::*; + + const STEP_BUDGET: usize = 64; + const FORWARDER_BUDGET: Duration = Duration::from_secs(1); + + struct CommandProbe { + observed: Arc>>, + } + + impl ActorInterface for CommandProbe { + type Incoming = ControlCommand; + type Response = (); + + fn handle(&mut self, _ctx: &Ctx, command: Self::Incoming) { + self.observed.lock().push(format!("{command:?}")); + } + } + + fn command(kind: u8, value: u8) -> ControlCommand { + let command_id = format!("repeated-{}", value % 4); + match kind % 4 { + 0 => ControlCommand::Kill { + command_id, + node: format!("node-{value}"), + }, + 1 => ControlCommand::Provision { + command_id, + count: u32::from(value), + }, + 2 => ControlCommand::Remove { + command_id, + count: u32::from(value), + }, + _ => ControlCommand::EstablishEdge { + command_id, + node: format!("node-{value}"), + }, + } + } + + fn bridge_invariant_failure( + expected: &[String], + observed: &[String], + runtime: &Runtime, + expected_actor_count: usize, + ) -> Option { + let stats = runtime.stats(); + let panics = stats + .workers + .iter() + .map(|worker| worker.panics) + .sum::(); + let mailbox_depth = stats + .workers + .iter() + .map(|worker| worker.mailbox_depth) + .sum::() + + stats + .actor_details + .iter() + .map(|actor| actor.mailbox_depth) + .sum::(); + if observed != expected + || stats.actors.len() != expected_actor_count + || stats.actor_details.iter().any(|actor| actor.poisoned) + || panics != 0 + || mailbox_depth != 0 + { + Some(format!( + "expected={expected:?}, observed={observed:?}, \ + expected_actor_count={expected_actor_count}, mailbox_depth={mailbox_depth}, \ + actor_census={stats:?}" + )) + } else { + None + } + } + + proptest! { + #![proptest_config(ProptestConfig { + cases: 128, + max_shrink_iters: 2_000, + ..ProptestConfig::default() + })] + + #[test] + fn generated_concurrent_bridge_commands_forward_once_and_shutdown( + inputs in prop::collection::vec((any::(), any::()), 0..=32), + split in 0_usize..=32, + concurrent in any::(), + destination_disappears in any::(), + ) { + let mut config = RuntimeConfig::default(); + config.worker_count = 1; + let parts = RuntimeParts::new(config); + let runtime = parts.runtime().clone(); + let backend = SteppingBackend::new(); + let _engine = + Engine::new(parts, backend.clone()).expect("dashboard bridge stepping engine"); + let observed = Arc::new(Mutex::new(Vec::new())); + let probe = runtime + .spawn(CommandProbe { + observed: Arc::clone(&observed), + }) + .expect("spawn dashboard control probe"); + if destination_disappears { + runtime + .stop_actor(probe) + .expect("stop dashboard destination before forwarding"); + for _ in 0..STEP_BUDGET { + backend.step(); + } + } + + let (tx, rx) = std::sync::mpsc::channel(); + let forwarder = + spawn_actor_sink_forwarder(rx, runtime.create_sender(), probe); + let commands = inputs + .iter() + .map(|(kind, value)| command(*kind, *value)) + .collect::>(); + let command_log = commands + .iter() + .map(|command| format!("{command:?}")) + .collect::>(); + + if concurrent { + let split = split.min(commands.len()); + let left = commands[..split].to_vec(); + let right = commands[split..].to_vec(); + let left_tx = tx.clone(); + let left_sender = std::thread::spawn(move || { + left.into_iter() + .map(|command| left_tx.send(command).is_ok()) + .collect::>() + }); + let right_tx = tx.clone(); + let right_sender = std::thread::spawn(move || { + right + .into_iter() + .map(|command| right_tx.send(command).is_ok()) + .collect::>() + }); + let _ = left_sender.join().expect("join left dashboard sender"); + let _ = right_sender.join().expect("join right dashboard sender"); + } else { + for command in commands { + if !destination_disappears { + tx.send(command).expect("send dashboard command"); + } else { + let _ = tx.send(command); + } + } + } + drop(tx); + + let (done_tx, done_rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = done_tx.send(forwarder.join()); + }); + let forwarder_result = done_rx.recv_timeout(FORWARDER_BUDGET).unwrap_or_else(|error| { + panic!( + "dashboard bridge did not terminate after disconnect: {error}; \ + actions={command_log:?}; actor_census={:?}", + runtime.stats(), + ) + }); + forwarder_result.expect("dashboard bridge forwarder panicked"); + for _ in 0..STEP_BUDGET { + backend.step(); + } + + let mut actual = observed.lock().clone(); + actual.sort(); + let mut expected = if destination_disappears { + Vec::new() + } else { + command_log.clone() + }; + expected.sort(); + let expected_actor_count = usize::from(!destination_disappears); + prop_assert!( + bridge_invariant_failure( + &expected, + &actual, + &runtime, + expected_actor_count, + ) + .is_none(), + "dashboard bridge invariant failed; actions={:?}; disconnect={}; failure={}", + command_log, + destination_disappears, + bridge_invariant_failure( + &expected, + &actual, + &runtime, + expected_actor_count, + ) + .unwrap_or_default(), + ); + + if !destination_disappears { + runtime + .stop_actor(probe) + .expect("stop dashboard control probe"); + for _ in 0..STEP_BUDGET { + backend.step(); + } + } + prop_assert!( + bridge_invariant_failure(&expected, &actual, &runtime, 0).is_none(), + "dashboard bridge teardown leaked forwarding actors; actions={:?}; failure={}", + command_log, + bridge_invariant_failure(&expected, &actual, &runtime, 0) + .unwrap_or_default(), + ); + } + } + + #[test] + fn bridge_invariant_rejects_a_controlled_duplicate_delivery() { + let parts = RuntimeParts::new(RuntimeConfig::default()); + let runtime = parts.runtime().clone(); + let expected = vec!["Provision repeated-0".to_owned()]; + let duplicated = vec![ + "Provision repeated-0".to_owned(), + "Provision repeated-0".to_owned(), + ]; + + assert!( + bridge_invariant_failure(&expected, &duplicated, &runtime, 0).is_some(), + "bridge invariant accepted a controlled duplicate delivery" + ); + } +} diff --git a/crates/dashboard/src/lib.rs b/crates/dashboard/src/lib.rs index 4d2e592..8184c14 100644 --- a/crates/dashboard/src/lib.rs +++ b/crates/dashboard/src/lib.rs @@ -270,6 +270,19 @@ impl DashboardHandle { server::run_server_with_routes(state, port, routes).await; } } + /// Schedule the dashboard HTTP server on its approved execution owner. + pub fn spawn(&self, engine: &swactor_engine::EngineHandle) { + engine.spawn(self.http_server()); + } + + /// Schedule the dashboard HTTP server on its approved execution owner. + pub fn spawn_with_plugins( + &self, + engine: &swactor_engine::EngineHandle, + plugins: Vec, + ) { + engine.spawn(self.http_server_with_plugins(plugins)); + } } /// Create the telemetry dashboard state. diff --git a/crates/dashboard/src/server.rs b/crates/dashboard/src/server.rs index 71db8cd..a65eba2 100644 --- a/crates/dashboard/src/server.rs +++ b/crates/dashboard/src/server.rs @@ -313,7 +313,20 @@ async fn frame_stream( #[cfg(test)] mod tests { + #[cfg(feature = "demo-control")] + use std::sync::{LazyLock, Mutex}; + #[cfg(feature = "demo-control")] + use std::time::Duration; + use super::*; + #[cfg(feature = "demo-control")] + use axum::body::Body; + #[cfg(feature = "demo-control")] + use axum::http::Request; + #[cfg(feature = "demo-control")] + use proptest::prelude::*; + #[cfg(feature = "demo-control")] + use tower::util::ServiceExt; fn state_with_plugin(page: PluginPage) -> AppState { let views = Arc::new(ViewRegistry::new()); @@ -341,4 +354,307 @@ mod tests { assert!(rendered.contains(r#"Provision"#)); } + + #[cfg(feature = "demo-control")] + const HTTP_RESPONSE_BUDGET: Duration = Duration::from_secs(1); + + #[cfg(feature = "demo-control")] + static TEST_CONTROL_RECEIVER: LazyLock< + Mutex>, + > = LazyLock::new(|| { + let (sender, receiver) = std::sync::mpsc::channel(); + crate::control::set_control_sender(sender); + Mutex::new(receiver) + }); + + #[cfg(feature = "demo-control")] + fn test_control_receiver() + -> &'static Mutex> { + &TEST_CONTROL_RECEIVER + } + + #[cfg(feature = "demo-control")] + #[derive(Clone, Debug)] + struct HttpAction { + route: u8, + payload: u8, + value: u8, + } + + #[cfg(feature = "demo-control")] + impl HttpAction { + fn uri(&self) -> &'static str { + match self.route % 4 { + 0 => "/control/kill", + 1 => "/control/provision", + 2 => "/control/remove", + _ => "/control/edge", + } + } + + fn command_json(&self, route: u8) -> String { + let command_id = format!("repeated-{}", self.value % 4); + match route % 4 { + 0 => serde_json::json!({ + "Kill": { + "command_id": command_id, + "node": format!("node-{}", self.value), + } + }) + .to_string(), + 1 => serde_json::json!({ + "Provision": { + "command_id": command_id, + "count": self.value, + } + }) + .to_string(), + 2 => serde_json::json!({ + "Remove": { + "command_id": command_id, + "count": self.value, + } + }) + .to_string(), + _ => serde_json::json!({ + "EstablishEdge": { + "command_id": command_id, + "node": format!("node-{}", self.value), + } + }) + .to_string(), + } + } + + fn body(&self) -> String { + match self.payload % 5 { + 0 => self.command_json(self.route), + 1 => match self.value % 6 { + 0 => String::new(), + 1 => "{".to_owned(), + 2 => "[".to_owned(), + 3 => "{\"".to_owned(), + 4 => "{\"command_id\":".to_owned(), + _ => "not-json".to_owned(), + }, + 2 => { + let command_id = format!("repeated-{}", self.value % 4); + match self.route % 4 { + 0 => serde_json::json!({ + "Kill": {"command_id": command_id} + }) + .to_string(), + 1 => serde_json::json!({ + "Provision": {"count": self.value} + }) + .to_string(), + 2 => serde_json::json!({ + "Remove": {"command_id": command_id} + }) + .to_string(), + _ => serde_json::json!({ + "EstablishEdge": {"node": format!("node-{}", self.value)} + }) + .to_string(), + } + } + 3 => self.command_json(self.route.wrapping_add(1)), + _ => match self.route % 4 { + 0 => r#"{"Kill":{"command_id":7,"node":[]}}"#.to_owned(), + 1 => r#"{"Provision":{"command_id":7,"count":"one"}}"#.to_owned(), + 2 => r#"{"Remove":{"command_id":[],"count":-1}}"#.to_owned(), + _ => r#"{"EstablishEdge":{"command_id":false,"node":7}}"#.to_owned(), + }, + } + } + + fn is_valid_for_route(&self) -> bool { + self.payload % 5 == 0 + } + } + + #[cfg(feature = "demo-control")] + #[derive(Clone, Debug)] + struct HttpObservation { + index: usize, + uri: &'static str, + body: String, + expected_valid: bool, + status: StatusCode, + } + + #[cfg(feature = "demo-control")] + async fn send_control_request( + app: Router, + index: usize, + action: HttpAction, + ) -> Result { + let uri = action.uri(); + let body = action.body(); + let expected_valid = action.is_valid_for_route(); + let request = Request::post(uri) + .header("content-type", "application/json") + .body(Body::from(body.clone())) + .map_err(|error| format!("build dashboard request: {error}"))?; + let response = tokio::time::timeout(HTTP_RESPONSE_BUDGET, app.oneshot(request)) + .await + .map_err(|_| format!("request {index} {uri} exceeded {HTTP_RESPONSE_BUDGET:?}"))? + .map_err(|error| format!("route dashboard request: {error}"))?; + Ok(HttpObservation { + index, + uri, + body, + expected_valid, + status: response.status(), + }) + } + + #[cfg(feature = "demo-control")] + fn http_invariant_failure( + actions: &[HttpAction], + responses: &[HttpObservation], + forwarded: usize, + ) -> Option { + let expected_forwarded = actions + .iter() + .filter(|action| action.is_valid_for_route()) + .count(); + let terminal = responses.len() == actions.len(); + let response_log = responses + .iter() + .map(|response| { + format!( + "#{} {} body={:?} -> {}", + response.index, response.uri, response.body, response.status, + ) + }) + .collect::>(); + let statuses_valid = responses.iter().all(|response| { + !response.status.is_server_error() + && if response.expected_valid { + response.status == StatusCode::ACCEPTED + } else { + response.status.is_client_error() + } + }); + if terminal && statuses_valid && forwarded == expected_forwarded { + None + } else { + Some(format!( + "terminal={terminal}, expected_forwarded={expected_forwarded}, \ + forwarded={forwarded}, responses={response_log:?}, \ + actor_census=dashboard HTTP routes own no actors" + )) + } + } + + #[cfg(feature = "demo-control")] + proptest! { + #![proptest_config(ProptestConfig { + cases: 128, + max_shrink_iters: 2_000, + ..ProptestConfig::default() + })] + + #[test] + fn generated_control_http_sequences_are_bounded_and_typed( + raw_actions in prop::collection::vec( + (any::(), any::(), any::()), + 0..=32, + ), + concurrent in any::(), + ) { + let actions = raw_actions + .into_iter() + .map(|(route, payload, value)| HttpAction { + route, + payload, + value, + }) + .collect::>(); + let receiver = test_control_receiver(); + while receiver + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .try_recv() + .is_ok() + {} + let state = state_with_plugin(PluginPage::new( + "control-test", + "Control test", + "/control-test", + "", + )); + let current_thread = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("build current-thread dashboard HTTP runtime"); + let outcome = current_thread.block_on(async { + let mut responses = Vec::with_capacity(actions.len()); + if concurrent { + let mut requests = tokio::task::JoinSet::new(); + for (index, action) in actions.iter().cloned().enumerate() { + requests.spawn(send_control_request(router(state.clone()), index, action)); + } + while let Some(result) = requests.join_next().await { + responses.push( + result + .map_err(|error| format!("dashboard request task failed: {error}"))??, + ); + } + } else { + for (index, action) in actions.iter().cloned().enumerate() { + responses.push( + send_control_request(router(state.clone()), index, action).await?, + ); + } + } + responses.sort_by_key(|response| response.index); + Ok::<_, String>(responses) + }); + prop_assert!( + outcome.is_ok(), + "dashboard HTTP request did not terminate; actions={:?}; error={:?}; \ + responses=[]; actor_census=dashboard HTTP routes own no actors", + actions, + outcome.as_ref().err(), + ); + let responses = outcome.expect("outcome checked above"); + let forwarded = { + let receiver = receiver + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + receiver.try_iter().count() + }; + let failure = http_invariant_failure(&actions, &responses, forwarded); + prop_assert!( + failure.is_none(), + "dashboard HTTP invariant failed; actions={:?}; responses={:?}; failure={}", + actions, + responses, + failure.unwrap_or_default(), + ); + } + } + + #[cfg(feature = "demo-control")] + #[test] + fn control_http_invariant_rejects_a_controlled_server_error() { + let actions = vec![HttpAction { + route: 0, + payload: 1, + value: 0, + }]; + let responses = vec![HttpObservation { + index: 0, + uri: actions[0].uri(), + body: actions[0].body(), + expected_valid: false, + status: StatusCode::INTERNAL_SERVER_ERROR, + }]; + assert!( + http_invariant_failure(&actions, &responses, 0).is_some(), + "HTTP invariant accepted a controlled 5xx response for invalid JSON" + ); + } } diff --git a/crates/dashboard/src/swactor/actor_view.rs b/crates/dashboard/src/swactor/actor_view.rs index 45be0ba..2dabfbf 100644 --- a/crates/dashboard/src/swactor/actor_view.rs +++ b/crates/dashboard/src/swactor/actor_view.rs @@ -52,6 +52,7 @@ pub(crate) struct RuntimeState { pub(crate) uptime_ms: Option, pub(crate) actors: BTreeMap, pub(crate) history: VecDeque, + actor_snapshot_generation: u64, } impl RuntimeState { @@ -62,6 +63,7 @@ impl RuntimeState { uptime_ms: None, actors: BTreeMap::new(), history: VecDeque::with_capacity(HISTORY_CAP), + actor_snapshot_generation: 0, } } @@ -79,28 +81,45 @@ impl RuntimeState { } fn apply_actors(&mut self, value: &Value, now: Instant) { - // Per-worker `worker_id` wrapper (TelemetryStatsHook shape) is the - // default placement for actors that do not carry one inline. + // A wrapped actor list is a complete snapshot for one worker. A list + // without a worker is a complete merged-runtime snapshot. Bare actor + // frames remain incremental. let wrapper_worker = u32_field(value, &["worker_id", "worker"]); - if let Some(actors) = value.get("actors").and_then(Value::as_array) { - for actor in actors { - self.apply_actor(actor, now, wrapper_worker); - } - return; - } - // A bare actor object per frame (no envelope). - self.apply_actor(value, now, wrapper_worker); - } - - fn apply_actor(&mut self, value: &Value, now: Instant, default_worker: Option) { - let Some(address) = string_field(value, &["address", "addr", "actor_addr"]) else { + let Some(actors) = value.get("actors").and_then(Value::as_array) else { + let _ = self.apply_actor(value, now, wrapper_worker); return; }; + + self.actor_snapshot_generation = self.actor_snapshot_generation.wrapping_add(1); + let generation = self.actor_snapshot_generation; + for actor in actors { + if let Some(actor) = self.apply_actor(actor, now, wrapper_worker) { + actor.snapshot_generation = generation; + } + } + match wrapper_worker { + Some(worker_id) => self.actors.retain(|_, actor| { + actor.worker_id != Some(worker_id) || actor.snapshot_generation == generation + }), + None => self + .actors + .retain(|_, actor| actor.snapshot_generation == generation), + } + } + + fn apply_actor( + &mut self, + value: &Value, + now: Instant, + default_worker: Option, + ) -> Option<&mut ActorState> { + let address = string_field(value, &["address", "addr", "actor_addr"])?; let actor = self .actors .entry(address.clone()) .or_insert_with(|| ActorState::new(address)); actor.apply_json(value, now, default_worker); + Some(actor) } fn apply_stats(&mut self, value: &Value, now: Instant) { @@ -128,7 +147,7 @@ impl RuntimeState { // Some publishers carry full per-actor detail under `actor_details`. if let Some(details) = value.get("actor_details").and_then(Value::as_array) { for actor in details { - self.apply_actor(actor, now, None); + let _ = self.apply_actor(actor, now, None); } } } @@ -185,6 +204,7 @@ pub(crate) struct ActorState { /// Messages folded away by the receipt sampling interval. pub(crate) sampled_out: u64, pub(crate) last_update: Option, + snapshot_generation: u64, } impl ActorState { @@ -206,6 +226,7 @@ impl ActorState { receipts: VecDeque::with_capacity(RECEIPT_CAP), sampled_out: 0, last_update: None, + snapshot_generation: 0, } } @@ -433,3 +454,65 @@ fn parse_message_type_counts(value: Option<&Value>) -> Option } None } + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn per_worker_snapshots_remove_stopped_actors_without_touching_other_workers() { + let now = Instant::now(); + let mut runtime = RuntimeState::new(now); + + runtime.apply_actors( + &json!({ + "worker_id": 0, + "actors": [ + {"address": "stable", "actor_type": "StableActor"}, + {"address": "observer", "actor_type": "ControlReplyObserver"}, + ], + }), + now, + ); + runtime.apply_actors( + &json!({ + "worker_id": 1, + "actors": [ + {"address": "other-worker", "actor_type": "OtherActor"}, + ], + }), + now, + ); + assert_eq!(runtime.actors.len(), 3); + + runtime.apply_actors( + &json!({ + "worker_id": 0, + "actors": [ + {"address": "stable", "actor_type": "StableActor"}, + ], + }), + now, + ); + assert_eq!( + runtime + .actors + .keys() + .map(String::as_str) + .collect::>(), + vec!["other-worker", "stable"] + ); + + runtime.apply_actors(&json!({"worker_id": 0, "actors": []}), now); + assert_eq!( + runtime + .actors + .keys() + .map(String::as_str) + .collect::>(), + vec!["other-worker"] + ); + } +} diff --git a/crates/distribution/Cargo.toml b/crates/distribution/Cargo.toml index 9366164..794d30f 100644 --- a/crates/distribution/Cargo.toml +++ b/crates/distribution/Cargo.toml @@ -21,3 +21,4 @@ libc = "0.2" [dev-dependencies] serde_json = "1" proptest = "1" +swactor-engine = { path = "../engine", default-features = false } diff --git a/crates/distribution/tests/gossip_data.rs b/crates/distribution/tests/gossip_data.rs index a57c7bd..58a6f67 100644 --- a/crates/distribution/tests/gossip_data.rs +++ b/crates/distribution/tests/gossip_data.rs @@ -228,14 +228,14 @@ mod standalone_gossip_transport { //! Actorized registry/metadata/directory gossip over one codec and transport, proving //! standalone frames converge without piggybacking on SWIM. - use std::cell::RefCell; use std::collections::HashMap; use std::sync::{Arc, RwLock}; use swactor::Error; use swactor::actor::ActorAddress; - use swactor::runtime::{Inbox, Runtime, RuntimeConfig, RuntimeParts, SingleThreadRuntime}; + use swactor::runtime::{Inbox, Runtime, RuntimeConfig, RuntimeParts}; use swactor::std::StdExtension; + use swactor_engine::{Engine, SteppingBackend}; use swactor_transport::{CodecRegistry, Transport, TransportRouter, WireEnvelope}; use distribution::crypto::{Keypair, KeypairExt}; @@ -272,7 +272,8 @@ mod standalone_gossip_transport { /// plus the shared state needed to wire it into a mesh. struct Node { rt: Runtime, - host: RefCell, + _engine: Engine, + backend: SteppingBackend, registry: ActorAddress, metadata: ActorAddress, directory: ActorAddress, @@ -305,7 +306,9 @@ mod standalone_gossip_transport { codec.clone(), router.clone(), ))); - let host = RefCell::new(SingleThreadRuntime::new(parts)); + let backend = SteppingBackend::new(); + let engine = + Engine::new(parts, backend.clone()).expect("create stepping actor engine"); let dir = SharedPeerDirectory::new(); let relay_mirror: RelayMirror = Arc::new(RwLock::new(HashMap::new())); @@ -336,7 +339,8 @@ mod standalone_gossip_transport { nodes.push(Node { rt, - host, + _engine: engine, + backend, registry, metadata, directory, @@ -407,7 +411,7 @@ mod standalone_gossip_transport { fn pump(&self, k: usize) { for _ in 0..k { for node in &self.nodes { - node.host.borrow_mut().tick(); + node.backend.step(); } } } @@ -448,7 +452,7 @@ mod standalone_gossip_transport { }, ) .unwrap(); - self.nodes[observer].host.borrow_mut().tick(); + self.nodes[observer].backend.step(); inbox.try_recv().and_then(|r| r.binding) } @@ -465,7 +469,7 @@ mod standalone_gossip_transport { }, ) .unwrap(); - self.nodes[observer].host.borrow_mut().tick(); + self.nodes[observer].backend.step(); inbox.try_recv().and_then(|r| r.relay_url) } @@ -482,7 +486,7 @@ mod standalone_gossip_transport { }, ) .unwrap(); - self.nodes[observer].host.borrow_mut().tick(); + self.nodes[observer].backend.step(); inbox.try_recv().and_then(|located| located.host) } } diff --git a/crates/distribution/tests/routing.rs b/crates/distribution/tests/routing.rs index f9633e4..f3488b5 100644 --- a/crates/distribution/tests/routing.rs +++ b/crates/distribution/tests/routing.rs @@ -16,15 +16,15 @@ mod directory_actor { //! DirectoryActor convergence and safety: signed claims, supersede, deterministic conflict //! resolution, dead-host hiding, catch-up, quieting, and retained recovery claims. - use std::cell::RefCell; use std::collections::HashMap; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, RwLock}; use swactor::Error; use swactor::actor::ActorAddress; - use swactor::runtime::{Inbox, Runtime, RuntimeConfig, RuntimeParts, SingleThreadRuntime}; + use swactor::runtime::{Inbox, Runtime, RuntimeConfig, RuntimeParts}; use swactor::std::StdExtension; + use swactor_engine::{Engine, SteppingBackend}; use swactor_transport::{CodecRegistry, Transport, TransportRouter, WireEnvelope}; use distribution::crypto::{Keypair, KeypairExt}; @@ -63,7 +63,8 @@ mod directory_actor { /// wire it into a mesh and observe it. struct Node { rt: Runtime, - host: RefCell, + _engine: Engine, + backend: SteppingBackend, directory: ActorAddress, dir: SharedPeerDirectory, router: Arc, @@ -94,7 +95,9 @@ mod directory_actor { codec.clone(), router.clone(), ))); - let host = RefCell::new(SingleThreadRuntime::new(parts)); + let backend = SteppingBackend::new(); + let engine = + Engine::new(parts, backend.clone()).expect("create stepping actor engine"); let dir = SharedPeerDirectory::new(); let route_view: RouteView = Arc::new(RwLock::new(HashMap::new())); @@ -109,7 +112,8 @@ mod directory_actor { nodes.push(Node { rt, - host, + _engine: engine, + backend, directory, dir, router, @@ -181,7 +185,7 @@ mod directory_actor { fn pump(&self, k: usize) { for _ in 0..k { for node in &self.nodes { - node.host.borrow_mut().tick(); + node.backend.step(); } } } @@ -246,7 +250,7 @@ mod directory_actor { }, ) .unwrap(); - self.nodes[observer].host.borrow_mut().tick(); + self.nodes[observer].backend.step(); inbox.try_recv().and_then(|located| located.host) } @@ -585,15 +589,15 @@ mod directory_route_path { //! Application delivery through the directory route view: address-only sends, supersede, and //! best-effort drops. - use std::cell::RefCell; use std::collections::HashMap; use std::sync::{Arc, Mutex, RwLock}; use serde::{Deserialize, Serialize}; use swactor::Error; use swactor::actor::{ActorAddress, ActorInterface}; - use swactor::runtime::{Ctx, Runtime, RuntimeConfig, RuntimeParts, SingleThreadRuntime}; + use swactor::runtime::{Ctx, Runtime, RuntimeConfig, RuntimeParts}; use swactor::std::StdExtension; + use swactor_engine::{Engine, SteppingBackend}; use swactor_transport::{CodecRegistry, NetworkMessage, TransportRouter}; use distribution::crypto::{Keypair, KeypairExt}; @@ -659,7 +663,8 @@ mod directory_route_path { struct RouteNode { rt: Runtime, - host: RefCell, + _engine: Engine, + backend: SteppingBackend, outbox: Outbox, route_view: RouteView, directory: ActorAddress, @@ -702,7 +707,9 @@ mod directory_route_path { codec.clone(), router.clone(), ))); - let host = RefCell::new(SingleThreadRuntime::new(parts)); + let backend = SteppingBackend::new(); + let engine = + Engine::new(parts, backend.clone()).expect("create stepping actor engine"); let outbox: Outbox = Arc::new(Mutex::new(Vec::new())); let route_view: RouteView = Arc::new(RwLock::new(HashMap::new())); @@ -731,7 +738,8 @@ mod directory_route_path { nodes.push(RouteNode { rt, - host, + _engine: engine, + backend, outbox, route_view, directory, @@ -801,7 +809,7 @@ mod directory_route_path { fn settle(&self, k: usize) { for _ in 0..k { for node in &self.nodes { - node.host.borrow_mut().tick(); + node.backend.step(); } self.deliver_wire(); } diff --git a/crates/distribution/tests/swim_actor.rs b/crates/distribution/tests/swim_actor.rs index e4f7d2b..a3d0e02 100644 --- a/crates/distribution/tests/swim_actor.rs +++ b/crates/distribution/tests/swim_actor.rs @@ -16,13 +16,13 @@ mod single_runtime_actor { //! SwimActor behavior in one runtime: subscription stream convergence and genuine unreachable- //! peer death detection. - use std::cell::RefCell; use std::collections::BTreeMap; use std::sync::Arc; use std::time::{Duration, Instant}; - use swactor::runtime::{Runtime, RuntimeConfig, RuntimeParts, SingleThreadRuntime}; + use swactor::runtime::{Runtime, RuntimeConfig, RuntimeParts}; use swactor::std::StdExtension; + use swactor_engine::{Engine, SteppingBackend}; use distribution::swim::actor::{ MembershipChanged, PeerDirectory, SharedPeerDirectory, SwimActor, SwimIn, @@ -55,7 +55,8 @@ mod single_runtime_actor { /// `MembershipChanged` subscriber inbox, and a shared Binding. struct ActorCluster { rt: Runtime, - host: RefCell, + _engine: Engine, + backend: SteppingBackend, ids: Vec, addrs: Vec, inboxes: Vec>, @@ -72,7 +73,8 @@ mod single_runtime_actor { let parts = RuntimeParts::new(RuntimeConfig::default()) .with_extension(Arc::new(StdExtension::new())); let rt = parts.runtime().clone(); - let host = RefCell::new(SingleThreadRuntime::new(parts)); + let backend = SteppingBackend::new(); + let engine = Engine::new(parts, backend.clone()).expect("create stepping actor engine"); let dir = SharedPeerDirectory::new(); let now = Instant::now(); let ids: Vec = (0..n).map(|i| id(i as u8)).collect(); @@ -103,7 +105,8 @@ mod single_runtime_actor { } let mut c = ActorCluster { rt, - host, + _engine: engine, + backend, ids, addrs, inboxes, @@ -128,7 +131,7 @@ mod single_runtime_actor { fn pump(&self, n: usize) { for _ in 0..n { - self.host.borrow_mut().tick(); + self.backend.step(); } } @@ -242,15 +245,15 @@ mod transport_runtime_actor { //! SwimActor behavior across separate runtimes through codec, TransportRouter, deliver_raw, and //! transport send failure. - use std::cell::RefCell; use std::collections::{BTreeMap, HashSet}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use swactor::Error; use swactor::actor::ActorAddress; - use swactor::runtime::{Inbox, Runtime, RuntimeConfig, RuntimeParts, SingleThreadRuntime}; + use swactor::runtime::{Inbox, Runtime, RuntimeConfig, RuntimeParts}; use swactor::std::StdExtension; + use swactor_engine::{Engine, SteppingBackend}; use swactor_transport::{CodecRegistry, Transport, TransportRouter, WireEnvelope}; use distribution::messages::actor_codec_registry; @@ -321,7 +324,8 @@ mod transport_runtime_actor { /// `Link` transports — the multi-runtime analog of `swim_actor.rs::ActorCluster`. struct TransportCluster { rts: Vec, - hosts: Vec>, + _engines: Vec, + backends: Vec, swims: Vec, inboxes: Vec>, streams: Vec>, @@ -338,7 +342,8 @@ mod transport_runtime_actor { let partition = Arc::new(Mutex::new(HashSet::new())); let mut rts = Vec::new(); - let mut hosts = Vec::new(); + let mut engines = Vec::new(); + let mut backends = Vec::new(); let mut swims = Vec::new(); let mut dirs = Vec::new(); let mut routers = Vec::new(); @@ -355,7 +360,9 @@ mod transport_runtime_actor { codec.clone(), router.clone(), ))); - let host = RefCell::new(SingleThreadRuntime::new(parts)); + let backend = SteppingBackend::new(); + let engine = + Engine::new(parts, backend.clone()).expect("create stepping actor engine"); let dir = SharedPeerDirectory::new(); let swim = rt @@ -378,7 +385,8 @@ mod transport_runtime_actor { .unwrap(); rts.push(rt); - hosts.push(host); + engines.push(engine); + backends.push(backend); swims.push(swim); dirs.push(dir); routers.push(router); @@ -412,7 +420,8 @@ mod transport_runtime_actor { let mut c = TransportCluster { rts, - hosts, + _engines: engines, + backends, swims, inboxes, streams: vec![Vec::new(); n], @@ -443,8 +452,8 @@ mod transport_runtime_actor { /// iterations; `k` is sized so a probe + its notification settle per round. fn pump(&self, k: usize) { for _ in 0..k { - for host in &self.hosts { - host.borrow_mut().tick(); + for backend in &self.backends { + backend.step(); } } } @@ -547,13 +556,13 @@ mod actor_membership_safety_edges { //! Actor-observable safety edges: resurrection after silence, multi-hop death dissemination, //! and bounded stale-refute behavior. - use std::cell::RefCell; use std::collections::BTreeMap; use std::sync::Arc; use std::time::{Duration, Instant}; - use swactor::runtime::{Runtime, RuntimeConfig, RuntimeParts, SingleThreadRuntime}; + use swactor::runtime::{Runtime, RuntimeConfig, RuntimeParts}; use swactor::std::StdExtension; + use swactor_engine::{Engine, SteppingBackend}; use distribution::swim::actor::{ MembershipChanged, PeerDirectory, SharedPeerDirectory, SwimActor, SwimIn, @@ -589,7 +598,8 @@ mod actor_membership_safety_edges { /// but lets the test choose the config and rebind a dropped node. struct Cluster { rt: Runtime, - host: RefCell, + _engine: Engine, + backend: SteppingBackend, ids: Vec, addrs: Vec, inboxes: Vec>, @@ -603,7 +613,8 @@ mod actor_membership_safety_edges { let parts = RuntimeParts::new(RuntimeConfig::default()) .with_extension(Arc::new(StdExtension::new())); let rt = parts.runtime().clone(); - let host = RefCell::new(SingleThreadRuntime::new(parts)); + let backend = SteppingBackend::new(); + let engine = Engine::new(parts, backend.clone()).expect("create stepping actor engine"); let dir = SharedPeerDirectory::new(); let now = Instant::now(); let ids: Vec = (0..n).map(|i| id(i as u8)).collect(); @@ -633,7 +644,8 @@ mod actor_membership_safety_edges { } let mut c = Cluster { rt, - host, + _engine: engine, + backend, ids, addrs, inboxes, @@ -655,7 +667,7 @@ mod actor_membership_safety_edges { fn pump(&self, n: usize) { for _ in 0..n { - self.host.borrow_mut().tick(); + self.backend.step(); } } diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 822d1ee..8b598cc 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -12,3 +12,6 @@ tokio = ["dep:tokio"] swactor = { path = "../.." } parking_lot = "0.12" tokio = { workspace = true, optional = true } + +[dev-dependencies] +proptest = "1" diff --git a/crates/engine/ENGINE_SPEC.md b/crates/engine/ENGINE_SPEC.md index d1c2bfe..7719490 100644 --- a/crates/engine/ENGINE_SPEC.md +++ b/crates/engine/ENGINE_SPEC.md @@ -66,7 +66,7 @@ Constructing a swactor engine consumes configured `RuntimeParts` and the selecte | operation | meaning | |---|---| | `spawn(task)` | Schedule an async unit of work on the substrate. | -| `spawn_blocking(work)` | Schedule blocking CPU / syscall work off the async path. | +| `blocking_work_sender().submit(work)` | Route blocking I/O work to the substrate's dedicated blocking pool. | | `timer(delay)` / `interval(period)` | Schedule future or recurring work. | | `now()` | The engine's monotonic clock. | @@ -88,7 +88,7 @@ The primitives an engine may provide. Capabilities are **per-implementation and - **Tasks** — `spawn` of an async unit of work; the substrate's unit of concurrency. - **Timers** — one-shot delay and recurring interval. - **I/O** — streams, sockets, files, and protocol endpoints used by engine-hosted work. An engine may implement I/O through asynchronous operations, blocking operations on managed threads, callbacks, or host-native facilities. Integrations declare the I/O capabilities they require, and binding fails at construction when the selected engine cannot provide them. -- **Blocking** — `spawn_blocking` for CPU-bound or syscall work that must not stall the executor. +- **Blocking** — a `BlockingWorkSender` routes syscall work off actor and async workers. - **Time** — `now()`. In a test engine this is virtual, advanced by the test; this is what makes deterministic testing possible. An engine that provides only tasks + time is still valid. Blocking and I/O are additional capabilities declared by integrations that require them. diff --git a/crates/engine/src/core_driver.rs b/crates/engine/src/core_driver.rs index 027d648..318ca30 100644 --- a/crates/engine/src/core_driver.rs +++ b/crates/engine/src/core_driver.rs @@ -57,7 +57,6 @@ struct CoreDriver { // Core drivers are the engine's sole core-progression path; this is the one // place permitted to call `Worker::try_tick` (ENGINE_SPEC.md §2). -#[allow(clippy::disallowed_methods)] impl Future for CoreDriver { type Output = (); diff --git a/crates/engine/src/engine.rs b/crates/engine/src/engine.rs index 2ee5552..f26207d 100644 --- a/crates/engine/src/engine.rs +++ b/crates/engine/src/engine.rs @@ -1,11 +1,14 @@ //! Composite engine and cloneable scheduler handle. +use parking_lot::{Condvar, Mutex}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Weak}; use std::time::Duration; use crate::backend::{Capabilities, EngineError, ExecutionBackend}; use crate::time::{EngineInstant, Interval, Timeout, Timer}; -use swactor::runtime::{Runtime, RuntimeParts}; +use swactor::actor::{ActorAddress, Message}; +use swactor::runtime::{ExternalSender, Runtime, RuntimeParts}; /// The composite engine: retains a configured core runtime handle and its /// execution backend, and owns one core-driving loop per worker. @@ -67,6 +70,116 @@ impl Engine { pub struct EngineHandle { backend: Weak, } +/// A clonable substrate route for one-shot blocking I/O work. +/// +/// Domain actors decide which effect to execute; this handle only moves its +/// mechanics onto the engine backend's dedicated blocking pool. +#[derive(Clone)] +pub struct BlockingWorkSender { + backend: Weak, +} + +impl BlockingWorkSender { + /// Submit one blocking I/O operation without occupying an actor worker. + /// + /// Returns the operation unchanged if the owning engine has stopped. + pub fn submit(&self, work: crate::BoxWork) -> Result<(), crate::BoxWork> { + let Some(backend) = self.backend.upgrade() else { + return Err(work); + }; + backend.spawn_blocking(work); + Ok(()) + } +} + +/// Cancellation handle for an engine-owned actor message timer. +/// +/// Cancellation is idempotent. A timer may already have fired when cancellation +/// races its deadline, so actor messages should still carry an operation or +/// generation identity that lets the receiver reject stale work. +#[derive(Clone, Debug)] +pub struct ActorTimer { + cancelled: Arc, +} + +impl ActorTimer { + /// Prevent this timer from delivering future messages. + pub fn cancel(&self) { + self.cancelled.store(true, Ordering::Release); + } + + /// Report whether cancellation has been requested. + pub fn is_cancelled(&self) -> bool { + self.cancelled.load(Ordering::Acquire) + } +} + +enum CompletionState { + Pending, + Ready(T), + Consumed, +} + +/// One actor-owned terminal observation for a synchronous process entrypoint. +/// +/// The waiting thread cannot poll, set a deadline, or advance domain state. +/// An actor decides when the operation is complete and publishes the value. +pub struct ActorCompletion { + inner: Arc<(Mutex>, Condvar)>, +} + +impl Clone for ActorCompletion { + fn clone(&self) -> Self { + Self { + inner: Arc::clone(&self.inner), + } + } +} + +impl Default for ActorCompletion { + fn default() -> Self { + Self::new() + } +} + +impl ActorCompletion { + pub fn new() -> Self { + Self { + inner: Arc::new((Mutex::new(CompletionState::Pending), Condvar::new())), + } + } + + /// Publish the terminal observation once. + pub fn complete(&self, value: T) -> Result<(), T> { + let (state, ready) = &*self.inner; + let mut state = state.lock(); + if !matches!(*state, CompletionState::Pending) { + return Err(value); + } + *state = CompletionState::Ready(value); + ready.notify_all(); + Ok(()) + } + + /// Block the process entrypoint until the owning actor completes. + pub fn wait(&self) -> T { + let (state, ready) = &*self.inner; + let mut state = state.lock(); + loop { + if matches!(*state, CompletionState::Pending) { + ready.wait(&mut state); + continue; + } + match std::mem::replace(&mut *state, CompletionState::Consumed) { + CompletionState::Ready(value) => return value, + CompletionState::Consumed => { + panic!("ActorCompletion::wait called after value was consumed") + } + CompletionState::Pending => unreachable!("pending state handled above"), + } + } + } +} impl EngineHandle { /// Upgrade to the live backend, or `None` if the owning engine is gone. @@ -86,16 +199,10 @@ impl EngineHandle { backend.spawn(Box::pin(task)); } } - - /// Schedule `work` on a dedicated blocking thread. - /// - /// A no-op once the owning engine has been dropped. - pub fn spawn_blocking(&self, work: F) - where - F: FnOnce() + Send + 'static, - { - if let Some(backend) = self.backend() { - backend.spawn_blocking(Box::new(work)); + /// Create a route to the backend's dedicated blocking-I/O pool. + pub fn blocking_work_sender(&self) -> BlockingWorkSender { + BlockingWorkSender { + backend: self.backend.clone(), } } @@ -120,6 +227,76 @@ impl EngineHandle { current: None, } } + + /// Schedule one typed message for delivery after `delay`. + /// + /// The engine owns the timer task; domain code receives no future or + /// scheduling callback. The actor receiving `message` owns the deadline + /// decision and should reject stale operation identities. + pub fn send_after( + &self, + delay: Duration, + sender: ExternalSender, + actor: ActorAddress, + message: M, + ) -> ActorTimer + where + M: Message, + { + let actor_timer = ActorTimer { + cancelled: Arc::new(AtomicBool::new(false)), + }; + let cancelled = Arc::clone(&actor_timer.cancelled); + let timer = self.timer(delay); + self.spawn(async move { + timer.await; + if !cancelled.load(Ordering::Acquire) { + let _ = sender.send_to(actor, message); + } + }); + actor_timer + } + + /// Schedule a cloned typed message after every `period`. + /// + /// Delivery stops after cancellation or when the actor address no longer + /// accepts messages. + pub fn send_every( + &self, + period: Duration, + sender: ExternalSender, + actor: ActorAddress, + message: M, + ) -> ActorTimer + where + M: Message, + { + let actor_timer = ActorTimer { + cancelled: Arc::new(AtomicBool::new(false)), + }; + let cancelled = Arc::clone(&actor_timer.cancelled); + let handle = self.clone(); + let mut interval = Box::pin(handle.interval(period)); + self.spawn(std::future::poll_fn(move |cx| { + if cancelled.load(Ordering::Acquire) { + return std::task::Poll::Ready(()); + } + match std::future::Future::poll(interval.as_mut(), cx) { + std::task::Poll::Ready(()) => { + if cancelled.load(Ordering::Acquire) + || sender.send_to(actor, message.clone()).is_err() + { + std::task::Poll::Ready(()) + } else { + cx.waker().wake_by_ref(); + std::task::Poll::Pending + } + } + std::task::Poll::Pending => std::task::Poll::Pending, + } + })); + actor_timer + } /// Race `future` against an engine timer. /// /// Resolves to `Ok` with the future's output if it completes within diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index aeee854..97333dc 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -15,7 +15,7 @@ mod time; mod tokio; pub use backend::{BoxTask, BoxTimer, BoxWork, Capabilities, EngineError, ExecutionBackend}; -pub use engine::{Engine, EngineHandle}; +pub use engine::{ActorCompletion, ActorTimer, BlockingWorkSender, Engine, EngineHandle}; pub use stepping::SteppingBackend; pub use time::{Elapsed, EngineInstant, Interval, Timeout, Timer}; diff --git a/crates/engine/src/stepping.rs b/crates/engine/src/stepping.rs index 110aaa7..0809dc0 100644 --- a/crates/engine/src/stepping.rs +++ b/crates/engine/src/stepping.rs @@ -181,11 +181,22 @@ impl SteppingBackend { pub fn pending_task_count(&self) -> usize { self.inner.tasks.lock().len() } + + /// Join every blocking operation submitted so far. + /// + /// Deterministic tests use this as a barrier before stepping actor replies. + /// Returns the first worker panic instead of silently discarding it. + pub fn join_blocking(&self) -> std::thread::Result<()> { + let handles: Vec<_> = self.inner.blocking.lock().drain(..).collect(); + for handle in handles { + handle.join()?; + } + Ok(()) + } } /// This impl is the substrate implementor for the deterministic stepping /// backend; blocking work runs on a std thread (ENGINE_SPEC.md §2). -#[allow(clippy::disallowed_methods)] impl ExecutionBackend for SteppingBackend { fn spawn(&self, task: BoxTask) { self.inner.tasks.lock().push(task); diff --git a/crates/engine/src/tokio.rs b/crates/engine/src/tokio.rs index 4dc2f27..b3363c3 100644 --- a/crates/engine/src/tokio.rs +++ b/crates/engine/src/tokio.rs @@ -55,7 +55,6 @@ impl TokioBackend { /// deterministic. // The engine's Tokio backend is the substrate owner: it is the one place // permitted to construct a Tokio runtime (ENGINE_SPEC.md §2). - #[allow(clippy::disallowed_methods)] pub fn new(config: TokioConfig) -> Result { let runtime = tokio::runtime::Builder::new_multi_thread() .worker_threads(config.worker_threads) @@ -80,7 +79,6 @@ impl TokioBackend { /// This impl is the Tokio substrate implementor: it is the one place permitted /// to schedule directly on the owned runtime (ENGINE_SPEC.md §2). -#[allow(clippy::disallowed_methods)] impl ExecutionBackend for TokioBackend { fn spawn(&self, task: BoxTask) { // The handle is used ephemerally and never stored or returned. @@ -152,7 +150,6 @@ impl LazySleep { // `LazySleep` arms a `tokio::time::sleep` inside an engine task where the time // driver is available; this is the substrate's own time primitive. -#[allow(clippy::disallowed_methods)] impl Future for LazySleep { type Output = (); diff --git a/crates/engine/tests/common/mod.rs b/crates/engine/tests/common/mod.rs index ddf7b42..63721c5 100644 --- a/crates/engine/tests/common/mod.rs +++ b/crates/engine/tests/common/mod.rs @@ -11,6 +11,7 @@ use std::time::{Duration, Instant}; use swactor::actor::{ActorInterface, Ctx}; use swactor::runtime::{Runtime, RuntimeConfig, RuntimeParts}; +use swactor_engine::SteppingBackend; pub fn runtime_parts(config: RuntimeConfig) -> (RuntimeParts, Runtime) { let parts = RuntimeParts::new(config); @@ -111,3 +112,59 @@ pub async fn yield_once() { }) .await; } + +pub fn drive_steps(backend: &SteppingBackend, count: usize) { + for _ in 0..count { + backend.step(); + } +} + +pub fn advance_and_drive(backend: &SteppingBackend, duration: Duration, count: usize) { + backend.advance_time(duration); + drive_steps(backend, count); +} + +pub fn assert_no_poison(runtime: &Runtime) { + let stats = runtime.stats(); + let panics = stats + .workers + .iter() + .map(|worker| worker.panics) + .sum::(); + let poisoned = stats + .actor_details + .iter() + .filter(|actor| actor.poisoned) + .collect::>(); + assert!( + panics == 0 && poisoned.is_empty(), + "runtime contains poisoned actors: worker_panics={panics}, poisoned={poisoned:?}, \ + actors={:?}, details={:?}", + stats.actors, + stats.actor_details, + ); +} + +pub fn assert_actor_delta_at_most(runtime: &Runtime, baseline: usize, limit: usize) { + let stats = runtime.stats(); + assert!( + stats.actors.len() <= baseline.saturating_add(limit), + "actor count grew from {baseline} to {}, limit={limit}: {:?}", + stats.actors.len(), + stats.actors, + ); +} + +pub fn assert_mailboxes_drained(runtime: &Runtime) { + let stats = runtime.stats(); + let mailbox_depth = stats + .workers + .iter() + .map(|worker| worker.mailbox_depth) + .sum::(); + assert_eq!( + mailbox_depth, 0, + "mailboxes did not drain: workers={:?}, details={:?}", + stats.workers, stats.actor_details, + ); +} diff --git a/crates/engine/tests/engine_contract.rs b/crates/engine/tests/engine_contract.rs index 82adb15..861f904 100644 --- a/crates/engine/tests/engine_contract.rs +++ b/crates/engine/tests/engine_contract.rs @@ -215,9 +215,15 @@ fn blocking_work_does_not_stop_actor_ticks() { // task so the owned runtime shuts down deterministically. let _release = BarrierRelease(barrier.clone()); let barrier_for_work = barrier.clone(); - handle.spawn_blocking(move || { - barrier_for_work.wait(); - }); + assert!( + handle + .blocking_work_sender() + .submit(Box::new(move || { + barrier_for_work.wait(); + })) + .is_ok(), + "submit blocking work" + ); // Deliver an actor message while the blocking work remains blocked. runtime.send_to(addr, Probe).expect("deliver probe message"); @@ -322,7 +328,6 @@ fn engine_timer_can_be_created_off_runtime() { // Tokio runtime to hand the engine — the one test-only use of the substrate // constructor (ENGINE_SPEC.md §2). #[test] -#[allow(clippy::disallowed_methods)] fn engine_adopts_caller_tuned_tokio_runtime() { // ENGINE_SPEC.md §9: the native engine supports // consuming an explicitly tuned Tokio runtime rather than always building diff --git a/crates/engine/tests/engine_unit.proptest-regressions b/crates/engine/tests/engine_unit.proptest-regressions new file mode 100644 index 0000000..6f15ba2 --- /dev/null +++ b/crates/engine/tests/engine_unit.proptest-regressions @@ -0,0 +1,8 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 540459f422fe16f29c58b74f5e357a9caa7deb8f56bb46a52a182f44ff91b470 # shrinks to actions = [Complete(0)] +cc 41f8ea6a4dfca245bc21a381049ec2e855e33fa34f5a1adcbdf064b7bb38b182 # shrinks to actions = [ScheduleOnce(1), ScheduleOnce(1), ScheduleEvery(1), DropEngine, Advance(1)] diff --git a/crates/engine/tests/engine_unit.rs b/crates/engine/tests/engine_unit.rs index f1f58de..1c0ab4b 100644 --- a/crates/engine/tests/engine_unit.rs +++ b/crates/engine/tests/engine_unit.rs @@ -13,7 +13,13 @@ use std::sync::atomic::Ordering::SeqCst; use std::sync::atomic::{AtomicBool, AtomicUsize}; use std::time::Duration; -use swactor_engine::{Capabilities, Engine, EngineError, ExecutionBackend, SteppingBackend}; +use parking_lot::Mutex; +use proptest::prelude::*; +use swactor::actor::{ActorInterface, Ctx}; +use swactor_engine::{ + ActorCompletion, ActorTimer, Capabilities, Engine, EngineError, ExecutionBackend, + SteppingBackend, +}; #[cfg(feature = "tokio")] use swactor_engine::{TokioBackend, TokioConfig}; @@ -473,6 +479,60 @@ fn stepping_timer_fires_after_virtual_time_advance() { ); } +#[test] +fn actor_message_timer_uses_engine_clock() { + let (parts, runtime) = default_runtime_parts(); + let received = Arc::new(AtomicUsize::new(0)); + let actor = runtime + .spawn(RecordingProbe { + received: Arc::clone(&received), + }) + .expect("spawn recording actor"); + let sender = runtime.create_sender(); + let backend = SteppingBackend::new(); + let engine = Engine::new(parts, backend.clone()).expect("construct engine"); + + engine + .handle() + .send_after(Duration::from_secs(1), sender, actor, Probe); + for _ in 0..4 { + backend.step(); + } + assert_eq!(received.load(SeqCst), 0); + + backend.advance_time(Duration::from_secs(1)); + for _ in 0..4 { + backend.step(); + } + assert_eq!(received.load(SeqCst), 1); +} + +#[test] +fn cancelling_actor_message_timer_prevents_delivery() { + let (parts, runtime) = default_runtime_parts(); + let received = Arc::new(AtomicUsize::new(0)); + let actor = runtime + .spawn(RecordingProbe { + received: Arc::clone(&received), + }) + .expect("spawn recording actor"); + let sender = runtime.create_sender(); + let backend = SteppingBackend::new(); + let engine = Engine::new(parts, backend.clone()).expect("construct engine"); + + let timer = engine + .handle() + .send_after(Duration::from_secs(1), sender, actor, Probe); + timer.cancel(); + backend.advance_time(Duration::from_secs(1)); + for _ in 0..4 { + backend.step(); + } + + assert!(timer.is_cancelled()); + assert_eq!(received.load(SeqCst), 0); +} + #[test] fn stepping_blocking_work_runs_isolated() { let parts = default_parts(); @@ -482,10 +542,16 @@ fn stepping_blocking_work_runs_isolated() { let done = Arc::new(AtomicBool::new(false)); let done_clone = done.clone(); - handle.spawn_blocking(move || { - std::thread::sleep(Duration::from_millis(10)); - done_clone.store(true, SeqCst); - }); + assert!( + handle + .blocking_work_sender() + .submit(Box::new(move || { + std::thread::sleep(Duration::from_millis(10)); + done_clone.store(true, SeqCst); + })) + .is_ok(), + "submit stepping work" + ); let deadline = std::time::Instant::now() + Duration::from_secs(2); loop { @@ -499,6 +565,714 @@ fn stepping_blocking_work_runs_isolated() { } } +// ═══════════════════════════════════════════════════════════════════════════════ +// §6 Generated control-flow contracts +// ═══════════════════════════════════════════════════════════════════════════════ + +const LIFECYCLE_DRAIN_STEPS: usize = 4; +const MAX_GENERATED_TIMER_DELAY: Duration = Duration::from_millis(4); + +#[derive(Clone, Debug)] +struct TimerFuzzMessage { + token: usize, + generation: usize, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct TimerObservation { + token: usize, + message_generation: usize, + actor_generation: usize, +} + +struct TimerFuzzProbe { + generation: Arc, + observations: Arc>>, + deliveries: Arc>>, +} + +impl ActorInterface for TimerFuzzProbe { + type Incoming = TimerFuzzMessage; + type Response = (); + + fn handle(&mut self, _ctx: &Ctx, message: TimerFuzzMessage) { + let observation = TimerObservation { + token: message.token, + message_generation: message.generation, + actor_generation: self.generation.load(SeqCst), + }; + self.observations.lock().push(observation.clone()); + if observation.message_generation == observation.actor_generation { + self.deliveries.lock().push(observation); + } + } +} + +#[derive(Clone, Copy, Debug)] +enum TimerKind { + Once, + Periodic, +} + +#[derive(Debug)] +struct TimerRecord { + timer: ActorTimer, + token: usize, + kind: TimerKind, + scheduled_while_live: bool, + cancelled_at: Option, +} + +type TerminalOutcome = Result; + +#[derive(Debug, Default)] +struct CompletionCensus { + attempts: Vec, + accepted: Vec, + rejected: Vec, +} + +#[derive(Clone, Copy, Debug)] +struct LifecycleCensus { + pending_tasks: usize, + task_limit: usize, + timer_handles: usize, + timer_limit: usize, + uncancelled_periodic: usize, + quiesced: bool, +} + +fn record_completion_attempt( + completion: &ActorCompletion, + census: &mut CompletionCensus, + outcome: TerminalOutcome, +) { + census.attempts.push(outcome); + match completion.complete(outcome) { + Ok(()) => census.accepted.push(outcome), + Err(rejected) => census.rejected.push(rejected), + } +} + +fn check_lifecycle_invariants( + completions: &CompletionCensus, + census: LifecycleCensus, +) -> Result<(), String> { + let expected_accepted = usize::from(!completions.attempts.is_empty()); + if completions.accepted.len() != expected_accepted { + return Err(format!( + "completion accepted {} terminal observations, expected {}; \ + completion_census={completions:?}, lifecycle_census={census:?}", + completions.accepted.len(), + expected_accepted, + )); + } + if completions.accepted.first() != completions.attempts.first() { + return Err(format!( + "completion did not preserve its first terminal observation; \ + completion_census={completions:?}, lifecycle_census={census:?}", + )); + } + if completions.rejected.as_slice() != &completions.attempts[expected_accepted..] { + return Err(format!( + "completion did not reject every duplicate terminal observation; \ + completion_census={completions:?}, lifecycle_census={census:?}", + )); + } + if census.quiesced && census.uncancelled_periodic != 0 { + return Err(format!( + "{} uncancelled periodic timer(s) survived quiescence; \ + completion_census={completions:?}, lifecycle_census={census:?}", + census.uncancelled_periodic, + )); + } + if census.pending_tasks > census.task_limit { + return Err(format!( + "task cardinality {} exceeded generated-operation bound {}; \ + completion_census={completions:?}, lifecycle_census={census:?}", + census.pending_tasks, census.task_limit, + )); + } + if census.timer_handles > census.timer_limit { + return Err(format!( + "timer cardinality {} exceeded generated-operation bound {}; \ + completion_census={completions:?}, lifecycle_census={census:?}", + census.timer_handles, census.timer_limit, + )); + } + Ok(()) +} + +fn observations_for_token(observations: &[TimerObservation], token: usize) -> usize { + observations + .iter() + .filter(|observation| observation.token == token) + .count() +} + +#[derive(Clone, Debug)] +enum TimerAction { + ScheduleOnce(u8), + ScheduleEvery(u8), + Cancel(u8), + Advance(u8), + BumpGeneration, + StopActor, + DropEngine, + CompleteSuccess(u8), + CompleteError(u8), +} + +fn timer_actions() -> impl Strategy> { + proptest::collection::vec( + prop_oneof![ + 3 => (1_u8..=4).prop_map(TimerAction::ScheduleOnce), + 2 => (1_u8..=4).prop_map(TimerAction::ScheduleEvery), + 2 => any::().prop_map(TimerAction::Cancel), + 3 => (0_u8..=4).prop_map(TimerAction::Advance), + 1 => Just(TimerAction::BumpGeneration), + 1 => Just(TimerAction::StopActor), + 1 => Just(TimerAction::DropEngine), + 1 => any::().prop_map(TimerAction::CompleteSuccess), + 1 => any::().prop_map(TimerAction::CompleteError), + ], + 0..=32, + ) +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: 128, + max_shrink_iters: 2_000, + ..ProptestConfig::default() + })] + + #[test] + fn generated_actor_timers_and_completion_are_bounded(actions in timer_actions()) { + let (parts, runtime) = default_runtime_parts(); + let baseline = runtime.stats().actors.len(); + let generation = Arc::new(AtomicUsize::new(0)); + let observations = Arc::new(Mutex::new(Vec::new())); + let deliveries = Arc::new(Mutex::new(Vec::new())); + let actor = runtime + .spawn(TimerFuzzProbe { + generation: Arc::clone(&generation), + observations: Arc::clone(&observations), + deliveries: Arc::clone(&deliveries), + }) + .expect("spawn timer fuzz probe"); + let sender = runtime.create_sender(); + let backend = SteppingBackend::new(); + let mut engine = Some(Engine::new(parts, backend.clone()).expect("construct engine")); + let handle = engine.as_ref().unwrap().handle(); + let driver_task_count = backend.pending_task_count(); + let completion = ActorCompletion::new(); + let mut completion_census = CompletionCensus::default(); + let mut timers = Vec::::new(); + let mut scheduled_while_live = 0; + let mut generated_timer_actions = 0; + let mut stopped_at = None; + + let assert_post_drop_scheduling_is_inert = || -> Result<(), String> { + let pending_before = backend.pending_task_count(); + let generation = generation.load(SeqCst); + let one_shot = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + handle.send_after( + Duration::from_millis(1), + sender.clone(), + actor, + TimerFuzzMessage { + token: usize::MAX, + generation, + }, + ) + })); + let periodic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + handle.send_every( + Duration::from_millis(1), + sender.clone(), + actor, + TimerFuzzMessage { + token: usize::MAX, + generation, + }, + ) + })); + if one_shot.is_err() || periodic.is_err() { + return Err("post-drop one-shot or periodic scheduling panicked".to_owned()); + } + let pending_after = backend.pending_task_count(); + if pending_after != pending_before { + return Err(format!( + "post-drop scheduling changed pending tasks from {pending_before} to \ + {pending_after}", + )); + } + if handle.capabilities() != Capabilities::NONE + || handle.require(Capabilities::TASKS_ONLY).is_ok() + { + return Err("dropped engine still advertised scheduling capabilities".to_owned()); + } + Ok(()) + }; + + drive_steps(&backend, LIFECYCLE_DRAIN_STEPS); + for action in &actions { + match *action { + TimerAction::ScheduleOnce(delay) => { + let token = timers.len(); + let engine_was_live = engine.is_some(); + let tasks_before = backend.pending_task_count(); + let scheduled = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + handle.send_after( + Duration::from_millis(u64::from(delay)), + sender.clone(), + actor, + TimerFuzzMessage { + token, + generation: generation.load(SeqCst), + }, + ) + })); + prop_assert!( + scheduled.is_ok(), + "one-shot scheduling panicked; actions={:?}, action={:?}, stats={:?}", + actions, + action, + runtime.stats(), + ); + let timer = scheduled.unwrap(); + generated_timer_actions += 1; + if engine_was_live { + scheduled_while_live += 1; + prop_assert_eq!( + backend.pending_task_count(), + tasks_before + 1, + "live one-shot did not create exactly one task; actions={:?}, \ + action={:?}, stats={:?}", + actions, + action, + runtime.stats(), + ); + } else { + prop_assert_eq!( + backend.pending_task_count(), + tasks_before, + "post-drop one-shot was not inert; actions={:?}, action={:?}, \ + stats={:?}", + actions, + action, + runtime.stats(), + ); + } + timers.push(TimerRecord { + timer, + token, + kind: TimerKind::Once, + scheduled_while_live: engine_was_live, + cancelled_at: None, + }); + } + TimerAction::ScheduleEvery(period) => { + let token = timers.len(); + let engine_was_live = engine.is_some(); + let tasks_before = backend.pending_task_count(); + let scheduled = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + handle.send_every( + Duration::from_millis(u64::from(period)), + sender.clone(), + actor, + TimerFuzzMessage { + token, + generation: generation.load(SeqCst), + }, + ) + })); + prop_assert!( + scheduled.is_ok(), + "periodic scheduling panicked; actions={:?}, action={:?}, stats={:?}", + actions, + action, + runtime.stats(), + ); + let timer = scheduled.unwrap(); + generated_timer_actions += 1; + if engine_was_live { + scheduled_while_live += 1; + prop_assert_eq!( + backend.pending_task_count(), + tasks_before + 1, + "live periodic send did not create exactly one task; actions={:?}, \ + action={:?}, stats={:?}", + actions, + action, + runtime.stats(), + ); + } else { + prop_assert_eq!( + backend.pending_task_count(), + tasks_before, + "post-drop periodic send was not inert; actions={:?}, action={:?}, \ + stats={:?}", + actions, + action, + runtime.stats(), + ); + } + timers.push(TimerRecord { + timer, + token, + kind: TimerKind::Periodic, + scheduled_while_live: engine_was_live, + cancelled_at: None, + }); + } + TimerAction::Cancel(index) => { + if !timers.is_empty() { + drive_steps(&backend, LIFECYCLE_DRAIN_STEPS); + let index = usize::from(index) % timers.len(); + let delivered = + observations_for_token(&observations.lock(), timers[index].token); + timers[index].timer.cancel(); + timers[index].timer.cancel(); + timers[index].cancelled_at.get_or_insert(delivered); + prop_assert!( + timers[index].timer.is_cancelled(), + "repeated cancellation was not idempotent; actions={:?}, \ + action={:?}, timer_census={:?}", + actions, + action, + timers, + ); + } + } + TimerAction::Advance(milliseconds) => { + advance_and_drive( + &backend, + Duration::from_millis(u64::from(milliseconds)), + LIFECYCLE_DRAIN_STEPS, + ); + } + TimerAction::BumpGeneration => { + generation.fetch_add(1, SeqCst); + } + TimerAction::StopActor => { + drive_steps(&backend, LIFECYCLE_DRAIN_STEPS); + let observed = observations.lock().len(); + let _ = runtime.stop_actor(actor); + stopped_at.get_or_insert(observed); + } + TimerAction::DropEngine => { + drop(engine.take()); + let post_drop = assert_post_drop_scheduling_is_inert(); + prop_assert!( + post_drop.is_ok(), + "post-drop invariant failed: {}; actions={:?}, action={:?}, \ + timer_census={:?}, completion_census={:?}, stats={:?}", + post_drop.as_ref().unwrap_err(), + actions, + action, + timers, + completion_census, + runtime.stats(), + ); + } + TimerAction::CompleteSuccess(value) => { + record_completion_attempt(&completion, &mut completion_census, Ok(value)); + } + TimerAction::CompleteError(value) => { + record_completion_attempt(&completion, &mut completion_census, Err(value)); + } + } + + drive_steps(&backend, LIFECYCLE_DRAIN_STEPS); + let observed = observations.lock().clone(); + let delivered = deliveries.lock().clone(); + let expected_deliveries = observed + .iter() + .filter(|observation| { + observation.message_generation == observation.actor_generation + }) + .cloned() + .collect::>(); + prop_assert_eq!( + delivered, + expected_deliveries, + "stale generation was delivered or a current generation was lost; actions={:?}, \ + action={:?}, observations={:?}, timer_census={:?}, \ + completion_census={:?}, stats={:?}", + actions, + action, + observed, + timers, + completion_census, + runtime.stats(), + ); + for timer in &timers { + if let Some(cancelled_at) = timer.cancelled_at { + let current = observations_for_token(&observed, timer.token); + prop_assert_eq!( + current, + cancelled_at, + "timer {} delivered after cancellation; actions={:?}, action={:?}, \ + observations={:?}, timer_census={:?}, completion_census={:?}, \ + stats={:?}", + timer.token, + actions, + action, + observed, + timers, + completion_census, + runtime.stats(), + ); + } + } + if let Some(stopped_at) = stopped_at { + prop_assert_eq!( + observed.len(), + stopped_at, + "message delivered after actor stop; actions={:?}, action={:?}, \ + observations={:?}, timer_census={:?}, completion_census={:?}, stats={:?}", + actions, + action, + observed, + timers, + completion_census, + runtime.stats(), + ); + } + let lifecycle_census = LifecycleCensus { + pending_tasks: backend.pending_task_count(), + task_limit: driver_task_count + scheduled_while_live, + timer_handles: timers.len(), + timer_limit: generated_timer_actions, + uncancelled_periodic: timers + .iter() + .filter(|timer| { + timer.scheduled_while_live + && matches!(timer.kind, TimerKind::Periodic) + && !timer.timer.is_cancelled() + }) + .count(), + quiesced: false, + }; + let invariant = + check_lifecycle_invariants(&completion_census, lifecycle_census); + prop_assert!( + invariant.is_ok(), + "lifecycle invariant failed: {}; actions={:?}, action={:?}, \ + observations={:?}, timer_census={:?}, completion_census={:?}, stats={:?}", + invariant.as_ref().unwrap_err(), + actions, + action, + observed, + timers, + completion_census, + runtime.stats(), + ); + prop_assert!( + runtime.stats().actors.len() <= baseline + 1, + "actor cardinality exceeded fixed bound; actions={:?}, action={:?}, \ + timer_census={:?}, completion_census={:?}, stats={:?}", + actions, + action, + timers, + completion_census, + runtime.stats(), + ); + } + + if let Some(expected) = completion_census.accepted.first().copied() { + let observed = completion.wait(); + prop_assert_eq!( + observed, + expected, + "completion wait returned a different terminal observation; actions={:?}, \ + timer_census={:?}, completion_census={:?}, stats={:?}", + actions, + timers, + completion_census, + runtime.stats(), + ); + record_completion_attempt(&completion, &mut completion_census, expected); + } + + drive_steps(&backend, LIFECYCLE_DRAIN_STEPS); + let observed = observations.lock().clone(); + for timer in &mut timers { + let delivered = observations_for_token(&observed, timer.token); + timer.timer.cancel(); + timer.timer.cancel(); + timer.cancelled_at.get_or_insert(delivered); + } + advance_and_drive( + &backend, + MAX_GENERATED_TIMER_DELAY, + LIFECYCLE_DRAIN_STEPS, + ); + drop(engine.take()); + let post_drop = assert_post_drop_scheduling_is_inert(); + prop_assert!( + post_drop.is_ok(), + "final post-drop invariant failed: {}; actions={:?}, timer_census={:?}, \ + completion_census={:?}, stats={:?}", + post_drop.as_ref().unwrap_err(), + actions, + timers, + completion_census, + runtime.stats(), + ); + drive_steps(&backend, LIFECYCLE_DRAIN_STEPS); + let final_observations = observations.lock().clone(); + let final_deliveries = deliveries.lock().clone(); + let expected_final_deliveries = final_observations + .iter() + .filter(|observation| { + observation.message_generation == observation.actor_generation + }) + .cloned() + .collect::>(); + prop_assert_eq!( + final_deliveries, + expected_final_deliveries, + "final drain delivered a stale generation or lost a current generation; \ + actions={:?}, observations={:?}, timer_census={:?}, completion_census={:?}, \ + stats={:?}", + actions, + final_observations, + timers, + completion_census, + runtime.stats(), + ); + for timer in &timers { + let delivered = observations_for_token(&final_observations, timer.token); + prop_assert_eq!( + delivered, + timer.cancelled_at.expect("all timers cancelled during final drain"), + "timer {} delivered during the final post-cancellation drain; actions={:?}, \ + observations={:?}, timer_census={:?}, completion_census={:?}, stats={:?}", + timer.token, + actions, + final_observations, + timers, + completion_census, + runtime.stats(), + ); + } + + let lifecycle_census = LifecycleCensus { + pending_tasks: backend.pending_task_count(), + task_limit: driver_task_count, + timer_handles: timers.len(), + timer_limit: generated_timer_actions, + uncancelled_periodic: timers + .iter() + .filter(|timer| { + timer.scheduled_while_live + && matches!(timer.kind, TimerKind::Periodic) + && !timer.timer.is_cancelled() + }) + .count(), + quiesced: true, + }; + let invariant = check_lifecycle_invariants(&completion_census, lifecycle_census); + prop_assert!( + invariant.is_ok(), + "final lifecycle invariant failed: {}; actions={:?}, observations={:?}, \ + timer_census={:?}, completion_census={:?}, lifecycle_census={:?}, stats={:?}", + invariant.as_ref().unwrap_err(), + actions, + observations.lock(), + timers, + completion_census, + lifecycle_census, + runtime.stats(), + ); + + let final_stats = runtime.stats(); + let worker_panics = final_stats + .workers + .iter() + .map(|worker| worker.panics) + .sum::(); + let poisoned = final_stats + .actor_details + .iter() + .filter(|actor| actor.poisoned) + .collect::>(); + prop_assert!( + worker_panics == 0 && poisoned.is_empty(), + "runtime poisoned during generated lifecycle; actions={:?}, observations={:?}, \ + timer_census={:?}, completion_census={:?}, lifecycle_census={:?}, stats={:?}", + actions, + final_observations, + timers, + completion_census, + lifecycle_census, + final_stats, + ); + prop_assert!( + final_stats.actors.len() <= baseline + 1, + "final actor cardinality exceeded fixed bound; actions={:?}, observations={:?}, \ + timer_census={:?}, completion_census={:?}, lifecycle_census={:?}, stats={:?}", + actions, + final_observations, + timers, + completion_census, + lifecycle_census, + final_stats, + ); + } +} + +#[test] +fn lifecycle_invariant_detects_injected_duplicate_completion() { + let completions = CompletionCensus { + attempts: vec![Ok(7), Err(9)], + accepted: vec![Ok(7), Err(9)], + rejected: Vec::new(), + }; + let census = LifecycleCensus { + pending_tasks: 1, + task_limit: 1, + timer_handles: 0, + timer_limit: 0, + uncancelled_periodic: 0, + quiesced: false, + }; + + let violation = check_lifecycle_invariants(&completions, census) + .expect_err("the exactly-once invariant must reject an injected duplicate completion"); + assert!( + violation.contains("completion accepted 2 terminal observations"), + "wrong detector failure for injected actions=[CompleteSuccess(7), CompleteError(9)]: \ + violation={violation}, completion_census={completions:?}, lifecycle_census={census:?}", + ); +} + +#[test] +fn lifecycle_invariant_detects_injected_uncancelled_periodic_timer() { + let completions = CompletionCensus::default(); + let census = LifecycleCensus { + pending_tasks: 2, + task_limit: 1, + timer_handles: 1, + timer_limit: 1, + uncancelled_periodic: 1, + quiesced: true, + }; + + let violation = check_lifecycle_invariants(&completions, census) + .expect_err("the leak invariant must reject an injected uncancelled periodic timer"); + assert!( + violation.contains("uncancelled periodic timer(s) survived quiescence"), + "wrong detector failure for injected actions=[ScheduleEvery(1), DropEngine]: \ + violation={violation}, completion_census={completions:?}, lifecycle_census={census:?}", + ); +} + #[test] fn stepping_spawned_task_completing_is_removed_from_queue() { let parts = default_parts(); @@ -620,10 +1394,15 @@ fn handle_used_after_engine_drop_degrades_gracefully() { // Time falls back to the wall clock without panicking. let _ = handle.now(); - // Scheduling work and creating primitives are no-ops / never fire, never - // panic, and never retain the backend. + // Scheduling work and creating primitives reject / no-op / never fire, + // never panic, and never retain the backend. handle.spawn(async {}); - handle.spawn_blocking(|| {}); + assert!( + handle + .blocking_work_sender() + .submit(Box::new(|| {})) + .is_err() + ); let _never_fires = handle.timer(Duration::from_secs(1)); let _never_ticks = handle.interval(Duration::from_secs(1)); } diff --git a/crates/iroh-driver/src/edge_transport.rs b/crates/iroh-driver/src/edge_transport.rs index 691e6d3..128b51c 100644 --- a/crates/iroh-driver/src/edge_transport.rs +++ b/crates/iroh-driver/src/edge_transport.rs @@ -45,6 +45,7 @@ pub(crate) fn spawn_edge_send_pump( endpoint: Endpoint, peer: EndpointAddr, edge_id: u64, + ready_timeout: Option, ) -> Result { let (tx, mut rx) = tokio_mpsc::unbounded_channel::>(); let (ready_tx, ready_rx) = std::sync::mpsc::channel::>(); @@ -108,9 +109,14 @@ pub(crate) fn spawn_edge_send_pump( let _ = ready_tx.send(Err(error)); } }); - ready_rx - .recv() - .map_err(|e| format!("edge {edge_id} sender startup channel closed: {e}"))??; + match ready_timeout { + Some(timeout) => ready_rx + .recv_timeout(timeout) + .map_err(|error| format!("edge {edge_id} sender startup: {error}"))??, + None => ready_rx + .recv() + .map_err(|error| format!("edge {edge_id} sender startup channel closed: {error}"))??, + } Ok(EdgeSendHandle { tx }) } diff --git a/crates/iroh-driver/src/iroh_driver.rs b/crates/iroh-driver/src/iroh_driver.rs index 01a98a0..905a4d9 100644 --- a/crates/iroh-driver/src/iroh_driver.rs +++ b/crates/iroh-driver/src/iroh_driver.rs @@ -598,7 +598,29 @@ impl IrohDriver { peer: EndpointAddr, edge_id: u64, ) -> Result { - spawn_edge_sender_task(self.engine.clone(), self.endpoint.clone(), peer, edge_id) + spawn_edge_sender_task( + self.engine.clone(), + self.endpoint.clone(), + peer, + edge_id, + None, + ) + } + + /// Start an EDGE_ALPN send pump with a bounded connection handshake. + pub fn spawn_edge_send_pump_timeout( + &self, + peer: EndpointAddr, + edge_id: u64, + timeout: std::time::Duration, + ) -> Result { + spawn_edge_sender_task( + self.engine.clone(), + self.endpoint.clone(), + peer, + edge_id, + Some(timeout), + ) } /// The node's identity. diff --git a/crates/iroh-driver/src/lib.rs b/crates/iroh-driver/src/lib.rs index a91e948..1239ce0 100644 --- a/crates/iroh-driver/src/lib.rs +++ b/crates/iroh-driver/src/lib.rs @@ -25,9 +25,10 @@ pub use iroh_driver::{ pub use edge_transport::{EDGE_ALPN, EdgeSendHandle}; pub use telemetry_transport::{ - TELEMETRY_ALPN, TelemetryQuicHeader, TelemetryQuicRead, TelemetryQuicWriteStats, - read_events_from_stream, read_next_event, read_next_uni_from_connection, read_pull_request, - read_stream_header, read_stream_into_fanout, spawn_connection_reader, spawn_pull_collector, + PullCollectorHandle, TELEMETRY_ALPN, TelemetryQuicHeader, TelemetryQuicRead, + TelemetryQuicWriteStats, read_events_from_stream, read_next_event, + read_next_uni_from_connection, read_pull_request, read_stream_header, read_stream_into_fanout, + spawn_connection_reader, spawn_pull_collector, spawn_pull_collector_to_actor, spawn_pull_server, spawn_subscription_writer, write_available_subscription, write_event, write_pull_request, write_subscription_until_closed, }; diff --git a/crates/iroh-driver/src/telemetry_transport.rs b/crates/iroh-driver/src/telemetry_transport.rs index 21a8832..722ac72 100644 --- a/crates/iroh-driver/src/telemetry_transport.rs +++ b/crates/iroh-driver/src/telemetry_transport.rs @@ -7,6 +7,8 @@ use std::time::Duration; use crossbeam_channel::TryRecvError; use iroh::endpoint::{Connection, RecvStream, SendStream}; use iroh::{Endpoint, EndpointAddr}; +use swactor::actor::ActorAddress; +use swactor::runtime::ExternalSender; use swactor_engine::EngineHandle; use telemetry::frame::{ ChannelDescriptor, ChannelId, ChannelRef, FrameDelivery, Position, StreamDescriptor, @@ -104,6 +106,44 @@ pub fn spawn_pull_server( }); } +/// Cancellation handle for one collector-initiated telemetry subscription. +/// +/// Cancellation is idempotent and immediately interrupts network I/O, +/// reconnect backoff, and all subsequent reconnect attempts. +#[derive(Debug)] +pub struct PullCollectorHandle { + cancellation: tokio::sync::watch::Sender, + completion: tokio::sync::watch::Receiver, +} + +impl PullCollectorHandle { + pub fn cancel(&self) { + self.cancellation.send_replace(true); + } + + pub fn is_cancelled(&self) -> bool { + *self.cancellation.borrow() + } + + pub fn is_finished(&self) -> bool { + *self.completion.borrow() + } +} + +impl Drop for PullCollectorHandle { + fn drop(&mut self) { + self.cancel(); + } +} + +struct PullCollectorCompletion(tokio::sync::watch::Sender); + +impl Drop for PullCollectorCompletion { + fn drop(&mut self) { + self.0.send_replace(true); + } +} + /// Supervisor side: retain a pull subscription to a node on `TELEMETRY_ALPN`. /// /// A transport interruption reconnects with bounded backoff. Returning after @@ -118,17 +158,91 @@ pub fn spawn_pull_collector( request: telemetry::SubscriptionRequest, fanout: std::sync::Arc, on_header: std::sync::mpsc::Sender, -) { +) -> PullCollectorHandle { + spawn_pull_collector_with_sink( + engine, + endpoint, + peer, + flow_id, + token, + request, + fanout, + PullHeaderSink::Channel(on_header), + ) +} + +/// Supervisor side variant that delivers each connection header directly to +/// an actor. Transport owns the subscription task; the actor owns how the +/// stream identity changes domain state. +pub fn spawn_pull_collector_to_actor( + engine: &EngineHandle, + endpoint: Endpoint, + peer: EndpointAddr, + flow_id: [u8; 16], + token: Vec, + request: telemetry::SubscriptionRequest, + fanout: std::sync::Arc, + sender: ExternalSender, + actor: ActorAddress, +) -> PullCollectorHandle { + spawn_pull_collector_with_sink( + engine, + endpoint, + peer, + flow_id, + token, + request, + fanout, + PullHeaderSink::Actor { sender, actor }, + ) +} + +enum PullHeaderSink { + Channel(std::sync::mpsc::Sender), + Actor { + sender: ExternalSender, + actor: ActorAddress, + }, +} + +impl PullHeaderSink { + fn deliver(&self, header: TelemetryQuicHeader) -> bool { + match self { + Self::Channel(sender) => sender.send(header).is_ok(), + Self::Actor { sender, actor } => sender.send_to(*actor, header).is_ok(), + } + } +} + +#[allow(clippy::too_many_arguments)] +fn spawn_pull_collector_with_sink( + engine: &EngineHandle, + endpoint: Endpoint, + peer: EndpointAddr, + flow_id: [u8; 16], + token: Vec, + request: telemetry::SubscriptionRequest, + fanout: std::sync::Arc, + on_header: PullHeaderSink, +) -> PullCollectorHandle { + let (cancellation, mut cancellation_rx) = tokio::sync::watch::channel(false); + let (completion, completion_rx) = tokio::sync::watch::channel(false); let engine_handle = engine.clone(); engine.spawn(async move { + let _completion = PullCollectorCompletion(completion); let peer_id = peer.id.to_string(); let mut retry_delay = Duration::from_millis(250); loop { - match collect_pull_once( - &endpoint, &peer, flow_id, &token, &request, &fanout, &on_header, - ) - .await - { + if *cancellation_rx.borrow() { + return; + } + let result = tokio::select! { + _ = cancellation_rx.changed() => return, + result = collect_pull_once( + &endpoint, &peer, flow_id, &token, &request, &fanout, &on_header, + ) => result, + }; + match result { Ok(()) => return, Err(error) => { eprintln!( @@ -137,13 +251,20 @@ pub fn spawn_pull_collector( ); } } - engine_handle.timer(retry_delay).await; + tokio::select! { + _ = cancellation_rx.changed() => return, + _ = engine_handle.timer(retry_delay) => {} + } retry_delay = retry_delay .checked_mul(2) .unwrap_or(Duration::from_secs(5)) .min(Duration::from_secs(5)); } }); + PullCollectorHandle { + cancellation, + completion: completion_rx, + } } async fn collect_pull_once( @@ -153,7 +274,7 @@ async fn collect_pull_once( token: &[u8], request: &telemetry::SubscriptionRequest, fanout: &telemetry::DeliveryFanout, - on_header: &std::sync::mpsc::Sender, + on_header: &PullHeaderSink, ) -> Result<(), String> { let conn = endpoint .connect(peer.clone(), TELEMETRY_ALPN) @@ -173,14 +294,18 @@ async fn collect_pull_once( let header = read_header(&mut recv) .await .map_err(|error| format!("answer header unreadable: {error}"))?; - if on_header.send(header.clone()).is_err() { + if !on_header.deliver(header.clone()) { return Ok(()); } let stream = header.stream; loop { match read_next_event(&mut recv, &stream).await { Ok(Some(event)) => { + let ended = matches!(event, TelemetryEvent::StreamEnded(_)); fanout.publish(event); + if ended { + return Ok(()); + } } Ok(None) => return Err("answer stream closed".to_owned()), Err(error) => return Err(format!("read answer stream failed: {error}")), diff --git a/crates/iroh-driver/tests/telemetry_transport.rs b/crates/iroh-driver/tests/telemetry_transport.rs index bfe05d7..65564f1 100644 --- a/crates/iroh-driver/tests/telemetry_transport.rs +++ b/crates/iroh-driver/tests/telemetry_transport.rs @@ -1,15 +1,20 @@ use std::net::{IpAddr, SocketAddr}; +use std::sync::Arc; +use std::time::{Duration, Instant}; use iroh::{Endpoint, EndpointAddr, RelayMode}; use iroh_driver::{ - TELEMETRY_ALPN, TelemetryQuicHeader, read_next_uni_from_connection, + TELEMETRY_ALPN, TelemetryQuicHeader, read_next_uni_from_connection, spawn_pull_collector, write_available_subscription, }; use swactor::config::RuntimeConfig; use swactor::runtime::RuntimeParts; use swactor_engine::{Engine, TokioBackend, TokioConfig}; use telemetry::frame::TelemetryEvent; -use telemetry::{ChannelContent, Lifetime, NodeId, Position, StreamId, TelemetryEndpoint}; +use telemetry::{ + ChannelContent, DeliveryFanout, Lifetime, NodeId, Position, StreamId, SubscriptionRequest, + TelemetryEndpoint, +}; /// Telemetry transport test scheduled through `EngineHandle`, not an ambient /// `#[tokio::test]` runtime (ENGINE_SPEC.md). @@ -112,6 +117,57 @@ fn iroh_telemetry_alpn_carries_catalog_and_numeric_frames() { } } +#[test] +fn pull_collector_cancellation_interrupts_inflight_io() { + let parts = RuntimeParts::new(RuntimeConfig::default()); + let engine = Engine::new( + parts, + TokioBackend::new(TokioConfig::default()).expect("test backend"), + ) + .expect("test engine"); + let handle = engine.handle(); + let (resource_tx, resource_rx) = std::sync::mpsc::channel(); + let setup_handle = handle.clone(); + handle.spawn(async move { + let collector_endpoint = test_endpoint().await; + let silent_peer = test_endpoint().await; + let (header_tx, header_rx) = std::sync::mpsc::channel(); + let collector = spawn_pull_collector( + &setup_handle, + collector_endpoint.clone(), + endpoint_addr(&silent_peer), + [3; 16], + Vec::new(), + SubscriptionRequest::all(), + Arc::new(DeliveryFanout::new(8)), + header_tx, + ); + resource_tx + .send((collector, collector_endpoint, silent_peer, header_rx)) + .expect("return collector resources"); + }); + let (collector, collector_endpoint, silent_peer, _header_rx) = resource_rx + .recv_timeout(Duration::from_secs(5)) + .expect("collector setup"); + + std::thread::sleep(Duration::from_millis(100)); + assert!( + !collector.is_finished(), + "collector was not retained in silent-peer network I/O" + ); + + collector.cancel(); + let stopped_deadline = Instant::now() + Duration::from_secs(2); + while !collector.is_finished() && Instant::now() < stopped_deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + collector.is_finished(), + "cancelled collector remained blocked in network I/O" + ); + drop((collector_endpoint, silent_peer)); +} + async fn test_endpoint() -> Endpoint { Endpoint::builder(iroh::endpoint::presets::Minimal) .relay_mode(RelayMode::Disabled) diff --git a/crates/job-runner/src/node.rs b/crates/job-runner/src/node.rs index 91946d4..d7dd2cb 100644 --- a/crates/job-runner/src/node.rs +++ b/crates/job-runner/src/node.rs @@ -20,6 +20,7 @@ use crate::wire::{ }; use swactor::actor::{ActorAddress, ActorInterface, Ctx}; use swactor::runtime::ExternalSender; +use swactor_engine::EngineHandle; use swactor_process::{ ExitStatus, ProcessOutput, ProcessOutputConfig, ProcessSpec, spawn_local_process, }; @@ -31,6 +32,12 @@ pub enum JobPhase { Run, } +struct PendingOutputs { + job_id: u64, + outputs: Vec, + deadline: Instant, +} + /// The node-side executor. `Incoming` is the orchestrator↔node wire command. pub struct NodeJobActor { orchestrator: ActorAddress, @@ -38,6 +45,9 @@ pub struct NodeJobActor { sender: ExternalSender, job_id: u64, workspace_buf: Vec, + actor_timers: Option, + workspace_wait: Option<(u64, Instant)>, + pending_outputs: Option, /// Edge-mode workspace-ready flag. When set, the orchestrator pushed the /// workspace tar over EDGE_ALPN (drained + extracted by the integration /// layer); `MaterializeWorkspace` waits for it before emitting @@ -70,11 +80,20 @@ impl NodeJobActor { sender, job_id, workspace_buf: Vec::new(), + actor_timers: None, + workspace_wait: None, + pending_outputs: None, workspace_ready: None, output_sink: None, } } + /// Give edge-mode waits access only to engine-owned typed actor timers. + pub fn with_actor_timers(mut self, engine: EngineHandle) -> Self { + self.actor_timers = Some(engine); + self + } + /// Edge mode: workspace bytes arrive over EDGE_ALPN and are extracted by the /// integration layer, which sets `flag` once the workspace is on disk. pub fn with_workspace_ready(mut self, flag: Arc) -> Self { @@ -148,12 +167,8 @@ impl NodeJobActor { ctx: &Ctx, job_id: u64, outputs: &[String], - slot: Arc>>>, + sink: Box, ) { - let sink = match self.take_output_sink(ctx, job_id, &slot) { - Some(sink) => sink, - None => return, // already faulted while waiting for the sink - }; let bytes = match pack_outputs_tar(&self.workdir, outputs) { Ok(b) => b, Err(e) => { @@ -191,33 +206,106 @@ impl NodeJobActor { self.emit(ctx, NodeJobEvent::OutputsCollected { job_id }); } - /// Take the edge output sink from the shared slot, waiting briefly for the - /// integration layer to arm it. Emits a fault and returns `None` on timeout. - fn take_output_sink( - &self, - ctx: &Ctx, - job_id: u64, - slot: &Arc>>>, - ) -> Option> { - let deadline = Instant::now() + OUTPUT_EDGE_ARM_WAIT; - loop { - { - let mut guard = slot.lock(); - if guard.is_some() { - return guard.take(); - } - } - if Instant::now() >= deadline { - self.emit( - ctx, - NodeJobEvent::NodeFault { - job_id, - reason: "output edge sink was never armed".to_owned(), - }, - ); - return None; - } - std::thread::sleep(EDGE_SPIN); + fn schedule_edge_check(&self, ctx: &Ctx, job_id: u64, message: NodeJobCommand) -> bool { + let Some(engine) = &self.actor_timers else { + self.emit( + ctx, + NodeJobEvent::NodeFault { + job_id, + reason: "edge mode requires engine-owned actor timers".to_owned(), + }, + ); + return false; + }; + engine.send_after(EDGE_SPIN, self.sender.clone(), ctx.self_addr(), message); + true + } + + fn begin_workspace_wait(&mut self, ctx: &Ctx, job_id: u64) { + if self + .workspace_ready + .as_ref() + .is_some_and(|ready| ready.load(Ordering::Acquire)) + { + self.emit(ctx, NodeJobEvent::WorkspaceMaterialized { job_id }); + return; + } + self.workspace_wait = Some((job_id, Instant::now() + WORKSPACE_EDGE_WAIT)); + if !self.schedule_edge_check(ctx, job_id, NodeJobCommand::CheckWorkspaceReady { job_id }) { + self.workspace_wait = None; + } + } + + fn check_workspace_ready(&mut self, ctx: &Ctx, job_id: u64) { + let Some((pending_job, deadline)) = self.workspace_wait else { + return; + }; + if pending_job != job_id { + return; + } + if self + .workspace_ready + .as_ref() + .is_some_and(|ready| ready.load(Ordering::Acquire)) + { + self.workspace_wait = None; + self.emit(ctx, NodeJobEvent::WorkspaceMaterialized { job_id }); + } else if Instant::now() >= deadline { + self.workspace_wait = None; + self.emit( + ctx, + NodeJobEvent::NodeFault { + job_id, + reason: "workspace edge transfer did not land".to_owned(), + }, + ); + } else { + self.schedule_edge_check(ctx, job_id, NodeJobCommand::CheckWorkspaceReady { job_id }); + } + } + + fn take_ready_output_sink(&self) -> Option> { + self.output_sink + .as_ref() + .and_then(|slot| slot.lock().take()) + } + + fn begin_output_wait(&mut self, ctx: &Ctx, job_id: u64, outputs: Vec) { + if let Some(sink) = self.take_ready_output_sink() { + self.collect_outputs_edge(ctx, job_id, &outputs, sink); + return; + } + self.pending_outputs = Some(PendingOutputs { + job_id, + outputs, + deadline: Instant::now() + OUTPUT_EDGE_ARM_WAIT, + }); + if !self.schedule_edge_check(ctx, job_id, NodeJobCommand::CheckOutputSink { job_id }) { + self.pending_outputs = None; + } + } + + fn check_output_sink(&mut self, ctx: &Ctx, job_id: u64) { + let Some(pending) = self.pending_outputs.as_ref() else { + return; + }; + if pending.job_id != job_id { + return; + } + if let Some(sink) = self.take_ready_output_sink() { + let pending = self.pending_outputs.take().expect("pending output state"); + self.collect_outputs_edge(ctx, job_id, &pending.outputs, sink); + } else if Instant::now() >= pending.deadline { + self.pending_outputs = None; + self.emit( + ctx, + NodeJobEvent::NodeFault { + job_id, + reason: "output edge sink was never armed".to_owned(), + }, + ); + } else { + self.schedule_edge_check(ctx, job_id, NodeJobCommand::CheckOutputSink { job_id }); } } } @@ -229,25 +317,8 @@ impl ActorInterface for NodeJobActor { fn handle(&mut self, ctx: &Ctx, cmd: NodeJobCommand) { match cmd { NodeJobCommand::MaterializeWorkspace { job_id } => { - if let Some(flag) = self.workspace_ready.as_ref() { - // Edge mode: the workspace tar traveled over EDGE_ALPN and - // was extracted into `workdir` by the integration layer. - // Wait for its readiness signal before announcing ready. - let deadline = Instant::now() + WORKSPACE_EDGE_WAIT; - while !flag.load(Ordering::Acquire) { - if Instant::now() >= deadline { - self.emit( - ctx, - NodeJobEvent::NodeFault { - job_id, - reason: "workspace edge transfer did not land".to_owned(), - }, - ); - return; - } - std::thread::sleep(EDGE_SPIN); - } - self.emit(ctx, NodeJobEvent::WorkspaceMaterialized { job_id }); + if self.workspace_ready.is_some() { + self.begin_workspace_wait(ctx, job_id); } else { // Chunk mode: clear the buffer; bytes arrive as WorkspaceChunk. self.workspace_buf.clear(); @@ -278,11 +349,8 @@ impl ActorInterface for NodeJobActor { self.spawn_supervised(ctx, JobPhase::Run, command, &env); } NodeJobCommand::CollectOutputs { job_id, outputs } => { - if let Some(slot) = self.output_sink.clone() { - // Edge mode: pack all outputs into one tar and ship over - // EDGE_ALPN, then announce collection. Dropping the sink - // finishes the stream so the orchestrator sees end-of-stream. - self.collect_outputs_edge(ctx, job_id, &outputs, slot); + if self.output_sink.is_some() { + self.begin_output_wait(ctx, job_id, outputs); } else { for name in &outputs { let path = self.workdir.join(name); @@ -304,6 +372,12 @@ impl ActorInterface for NodeJobActor { self.emit(ctx, NodeJobEvent::OutputsCollected { job_id }); } } + NodeJobCommand::CheckWorkspaceReady { job_id } => { + self.check_workspace_ready(ctx, job_id); + } + NodeJobCommand::CheckOutputSink { job_id } => { + self.check_output_sink(ctx, job_id); + } } } } diff --git a/crates/job-runner/src/wire.rs b/crates/job-runner/src/wire.rs index b06b245..37bdcd5 100644 --- a/crates/job-runner/src/wire.rs +++ b/crates/job-runner/src/wire.rs @@ -66,6 +66,18 @@ pub enum NodeJobCommand { job_id: u64, outputs: Vec, }, + /// Engine-owned timer observation used while an edge workspace transfer is + /// pending. `job_id` rejects stale timer delivery. + #[doc(hidden)] + CheckWorkspaceReady { + job_id: u64, + }, + /// Engine-owned timer observation used while an edge output sink is + /// pending. `job_id` rejects stale timer delivery. + #[doc(hidden)] + CheckOutputSink { + job_id: u64, + }, } impl NetworkMessage for NodeJobCommand { diff --git a/crates/process/Cargo.toml b/crates/process/Cargo.toml index ddeb475..256c8bb 100644 --- a/crates/process/Cargo.toml +++ b/crates/process/Cargo.toml @@ -6,9 +6,18 @@ license = "AGPL-3.0-only" [dependencies] swactor = { path = "../..", default-features = false, features = ["no_random"] } +swactor-engine = { path = "../engine" } +tokio.workspace = true telemetry = { path = "../telemetry" } crossbeam-queue = "0.3.12" libc = "0.2" serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml = "0.9" + +[target.'cfg(target_os = "linux")'.dependencies] +signal-hook = "0.3" + +[dev-dependencies] +parking_lot = "0.12" +proptest = "1" diff --git a/crates/process/src/lib.rs b/crates/process/src/lib.rs index 171b148..04614c0 100644 --- a/crates/process/src/lib.rs +++ b/crates/process/src/lib.rs @@ -1,6 +1,7 @@ mod actor; mod lifecycle; mod message; +mod operations; mod spawn; mod supervisor; mod types; @@ -10,6 +11,24 @@ pub mod yaml; pub use lifecycle::{ProcessLifecycleObservability, ProcessOutputConfig}; pub use message::{ProcessCommand, ProcessOutput}; +#[cfg(unix)] +pub use operations::request_child_termination; +#[cfg(target_os = "linux")] +pub use operations::spawn_os_stop_signal_wait; +#[cfg(unix)] +pub use operations::spawn_unix_stream_listener; +#[cfg(target_os = "linux")] +pub use operations::terminate_process_group; +pub use operations::{ + CommandOutputObservation, FollowProcessFile, LineReaderHandle, ProcessExitObservation, + ProcessIdentity, ProcessStopSignal, ProcessStream, ProcessStreamObservation, child_kill, + child_try_wait, child_wait, child_wait_with_output, command_output, command_spawn, + command_status, find_process_identities_by_environment, find_process_identities_with_retry, + spawn_child_wait, spawn_command_output, spawn_detached_command_status, + spawn_identity_exit_wait, spawn_line_channel, spawn_line_reader, spawn_mapped_line_channel, + spawn_mapped_line_reader, spawn_shared_child_wait, spawn_stdin_command_wait, + spawn_stop_channel_wait, wait_for_path, wait_shared_child_or_kill, +}; pub use pipeline::{ JobComplete, JobDefinition, JobFailure, JobId, JobProgress, JobStatus, JobSuccess, LocalPipelineConfig, LocalStartJob, PipelineId, PipelineStatus, diff --git a/crates/process/src/operations.rs b/crates/process/src/operations.rs new file mode 100644 index 0000000..2ab0b2e --- /dev/null +++ b/crates/process/src/operations.rs @@ -0,0 +1,1425 @@ +//! Concrete operating-system process mechanics owned by `swactor-process`. +//! +//! Domain crates may choose a command or decide to stop a resource, but the +//! actual process creation, waiting, and signaling stays behind this substrate +//! boundary. + +use std::fs::File; +use std::io::{self, BufRead, BufReader, Read}; +use std::path::Path; +use std::process::{Child, Command, ExitStatus, Output}; +use std::sync::mpsc::Receiver; +use std::sync::{Arc, Mutex}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; + +use swactor::actor::ActorAddress; +use swactor::runtime::ExternalSender; + +pub fn command_output(command: &mut Command) -> io::Result { + command.output() +} + +pub fn command_status(command: &mut Command) -> io::Result { + command.status() +} + +pub fn command_spawn(command: &mut Command) -> io::Result { + command.spawn() +} + +pub fn child_kill(child: &mut Child) -> io::Result<()> { + child.kill() +} + +/// Ask a child process to shut down through its ordinary SIGTERM path. +/// +/// Unlike [`child_kill`], this gives the child an opportunity to flush durable +/// state and release owned resources before exiting. +#[cfg(unix)] +pub fn request_child_termination(child: &Child) -> io::Result<()> { + if unsafe { libc::kill(child.id() as i32, libc::SIGTERM) } == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } +} + +pub fn child_wait(child: &mut Child) -> io::Result { + child.wait() +} + +pub fn child_try_wait(child: &mut Child) -> io::Result> { + child.try_wait() +} + +pub fn child_wait_with_output(child: Child) -> io::Result { + child.wait_with_output() +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ProcessStream { + Stdout, + Stderr, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ProcessStreamObservation { + Line { + stream: ProcessStream, + line: String, + }, + Error { + stream: ProcessStream, + error: String, + }, + Closed { + stream: ProcessStream, + }, +} + +pub struct LineReaderHandle { + join: JoinHandle<()>, +} + +impl LineReaderHandle { + pub fn join(self) { + let _ = self.join.join(); + } +} + +/// Read one child stream and deliver typed observations to an actor relay. +pub fn spawn_line_reader( + stream: ProcessStream, + reader: R, + sender: ExternalSender, + actor: ActorAddress, +) -> LineReaderHandle +where + R: Read + Send + 'static, +{ + let join = thread::spawn(move || { + for next in BufReader::new(reader).lines() { + let observation = match next { + Ok(line) => ProcessStreamObservation::Line { stream, line }, + Err(error) => { + let _ = sender.send_to( + actor, + ProcessStreamObservation::Error { + stream, + error: error.to_string(), + }, + ); + break; + } + }; + if sender.send_to(actor, observation).is_err() { + return; + } + } + let _ = sender.send_to(actor, ProcessStreamObservation::Closed { stream }); + }); + LineReaderHandle { join } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ProcessStopSignal; + +/// Wait for an embedding process-control channel and notify an actor once. +pub fn spawn_stop_channel_wait( + receiver: Receiver<()>, + sender: ExternalSender, + actor: ActorAddress, +) { + drop(spawn_stop_channel_wait_thread(receiver, sender, actor)); +} + +fn spawn_stop_channel_wait_thread( + receiver: Receiver<()>, + sender: ExternalSender, + actor: ActorAddress, +) -> JoinHandle<()> { + thread::spawn(move || { + if receiver.recv().is_ok() { + let _ = sender.send_to(actor, ProcessStopSignal); + } + }) +} + +/// Wait for SIGINT/SIGTERM and notify an actor once. +#[cfg(target_os = "linux")] +pub fn spawn_os_stop_signal_wait(sender: ExternalSender, actor: ActorAddress) { + thread::spawn(move || { + let Ok(mut signals) = signal_hook::iterator::Signals::new([ + signal_hook::consts::signal::SIGINT, + signal_hook::consts::signal::SIGTERM, + ]) else { + return; + }; + if signals.forever().next().is_some() { + let _ = sender.send_to(actor, ProcessStopSignal); + } + }); +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProcessIdentity { + pub pid: u32, + pub environment: Vec<(String, String)>, + pub process_group_leader: bool, +} + +impl ProcessIdentity { + pub fn matches(&self) -> bool { + #[cfg(target_os = "linux")] + { + let Ok(stat) = std::fs::read_to_string(format!("/proc/{}/stat", self.pid)) else { + return false; + }; + let Some((_, tail)) = stat.rsplit_once(')') else { + return false; + }; + let mut fields = tail.split_whitespace(); + let state = fields.next(); + let _parent_pid = fields.next(); + let process_group = fields.next().and_then(|value| value.parse::().ok()); + if state == Some("Z") || self.process_group_leader && process_group != Some(self.pid) { + return false; + } + let Ok(environ) = std::fs::read(format!("/proc/{}/environ", self.pid)) else { + return false; + }; + return self.environment.iter().all(|(key, value)| { + environ.split(|byte| *byte == 0).any(|entry| { + entry + .strip_prefix(format!("{key}=").as_bytes()) + .is_some_and(|actual| actual == value.as_bytes()) + }) + }); + } + #[cfg(not(target_os = "linux"))] + false + } +} + +pub struct FollowProcessFile { + file: File, + identity: ProcessIdentity, + poll_interval: Duration, +} + +impl FollowProcessFile { + pub fn new(file: File, identity: ProcessIdentity, poll_interval: Duration) -> Self { + Self { + file, + identity, + poll_interval, + } + } +} + +impl Read for FollowProcessFile { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + loop { + let count = self.file.read(buffer)?; + if count > 0 || !self.identity.matches() { + return Ok(count); + } + thread::sleep(self.poll_interval); + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProcessExitObservation { + pub status: Option, + pub error: Option, +} + +pub fn spawn_shared_child_wait( + child: Arc>>, + poll_interval: Duration, + sender: ExternalSender, + actor: ActorAddress, +) { + thread::spawn(move || { + loop { + let observation = { + let mut slot = child + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let Some(child) = slot.as_mut() else { + return; + }; + match child.try_wait() { + Ok(Some(status)) => { + *slot = None; + Some(ProcessExitObservation { + status: status.code(), + error: None, + }) + } + Ok(None) => None, + Err(error) => Some(ProcessExitObservation { + status: None, + error: Some(error.to_string()), + }), + } + }; + if let Some(observation) = observation { + let _ = sender.send_to(actor, observation); + return; + } + thread::sleep(poll_interval); + } + }); +} + +pub fn wait_shared_child_or_kill( + child: &Arc>>, + timeout: Duration, + process_group: bool, + poll_interval: Duration, +) -> io::Result> { + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + let mut slot = child + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let Some(child) = slot.as_mut() else { + return Ok(None); + }; + match child.try_wait()? { + Some(status) => { + *slot = None; + return Ok(Some(status)); + } + None => drop(slot), + } + thread::sleep(poll_interval); + } + + let mut slot = child + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let Some(child) = slot.as_mut() else { + return Ok(None); + }; + #[cfg(target_os = "linux")] + if process_group { + let _ = unsafe { libc::kill(-(child.id() as i32), libc::SIGKILL) }; + } + child.kill()?; + let status = child.wait()?; + *slot = None; + Ok(Some(status)) +} + +#[cfg(target_os = "linux")] +pub fn terminate_process_group( + identity: &ProcessIdentity, + timeout: Duration, + poll_interval: Duration, +) -> Result<(), String> { + if !identity.matches() { + return Ok(()); + } + if unsafe { libc::kill(-(identity.pid as i32), libc::SIGTERM) } != 0 { + return Err(format!( + "terminate process group {}: {}", + identity.pid, + io::Error::last_os_error() + )); + } + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if !identity.matches() { + return Ok(()); + } + thread::sleep(poll_interval); + } + if unsafe { libc::kill(-(identity.pid as i32), libc::SIGKILL) } != 0 && identity.matches() { + return Err(format!( + "kill process group {}: {}", + identity.pid, + io::Error::last_os_error() + )); + } + Ok(()) +} + +pub fn spawn_identity_exit_wait( + identity: ProcessIdentity, + poll_interval: Duration, + sender: ExternalSender, + actor: ActorAddress, +) { + thread::spawn(move || { + while identity.matches() { + thread::sleep(poll_interval); + } + let _ = sender.send_to( + actor, + ProcessExitObservation { + status: None, + error: None, + }, + ); + }); +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CommandOutputObservation { + pub status: Option, + pub stdout: Vec, + pub stderr: Vec, + pub error: Option, +} + +pub fn spawn_command_output(mut command: Command, sender: ExternalSender, actor: ActorAddress) { + thread::spawn(move || { + let observation = match command.output() { + Ok(output) => CommandOutputObservation { + status: output.status.code(), + stdout: output.stdout, + stderr: output.stderr, + error: None, + }, + Err(error) => CommandOutputObservation { + status: None, + stdout: Vec::new(), + stderr: Vec::new(), + error: Some(error.to_string()), + }, + }; + let _ = sender.send_to(actor, observation); + }); +} + +pub fn spawn_detached_command_status(mut command: Command, label: impl Into) { + let label = label.into(); + thread::spawn(move || { + if let Err(error) = command.status() { + eprintln!("{label}: {error}"); + } + }); +} + +pub fn spawn_child_wait(mut child: Child, sender: ExternalSender, actor: ActorAddress) { + thread::spawn(move || { + let observation = match child.wait() { + Ok(status) => ProcessExitObservation { + status: status.code(), + error: None, + }, + Err(error) => ProcessExitObservation { + status: None, + error: Some(error.to_string()), + }, + }; + let _ = sender.send_to(actor, observation); + }); +} + +pub fn spawn_mapped_line_reader( + reader: R, + sender: ExternalSender, + actor: ActorAddress, + line_message: L, + error_message: E, + closed_message: M, +) where + R: Read + Send + 'static, + M: swactor::actor::Message, + L: Fn(String) -> M + Send + 'static, + E: Fn(String) -> M + Send + 'static, +{ + thread::spawn(move || { + for line in BufReader::new(reader).lines() { + match line { + Ok(line) => { + let _ = sender.send_to(actor, line_message(line)); + } + Err(error) => { + let _ = sender.send_to(actor, error_message(error.to_string())); + break; + } + } + } + let _ = sender.send_to(actor, closed_message); + }); +} + +pub fn spawn_mapped_line_channel( + reader: R, + sender: std::sync::mpsc::Sender, + line_message: L, + error_message: E, + closed_message: M, +) where + R: Read + Send + 'static, + M: Send + 'static, + L: Fn(String) -> M + Send + 'static, + E: Fn(String) -> M + Send + 'static, +{ + thread::spawn(move || { + for line in BufReader::new(reader).lines() { + let message = match line { + Ok(line) => line_message(line), + Err(error) => { + let _ = sender.send(error_message(error.to_string())); + break; + } + }; + if sender.send(message).is_err() { + return; + } + } + let _ = sender.send(closed_message); + }); +} + +pub fn spawn_line_channel(reader: R, sender: std::sync::mpsc::Sender) +where + R: Read + Send + 'static, +{ + thread::spawn(move || { + for line in BufReader::new(reader).lines().map_while(Result::ok) { + if sender.send(line).is_err() { + return; + } + } + }); +} + +fn forward_stdin_command( + reader: R, + command: &str, + trigger_on_eof: bool, + sender: ExternalSender, + actor: ActorAddress, +) where + R: BufRead, +{ + for line in reader.lines().map_while(Result::ok) { + if line.trim().eq_ignore_ascii_case(command) { + let _ = sender.send_to(actor, ProcessStopSignal); + return; + } + } + if trigger_on_eof { + let _ = sender.send_to(actor, ProcessStopSignal); + } +} + +pub fn spawn_stdin_command_wait( + command: &'static str, + trigger_on_eof: bool, + sender: ExternalSender, + actor: ActorAddress, +) { + thread::spawn(move || { + forward_stdin_command( + std::io::stdin().lock(), + command, + trigger_on_eof, + sender, + actor, + ); + }); +} + +pub fn find_process_identities_by_environment( + environment: &[(String, String)], + process_group_leader: bool, +) -> io::Result> { + #[cfg(target_os = "linux")] + { + let mut matches = Vec::new(); + for entry in std::fs::read_dir("/proc")?.flatten() { + let Some(pid) = entry + .file_name() + .to_str() + .and_then(|name| name.parse::().ok()) + else { + continue; + }; + let identity = ProcessIdentity { + pid, + environment: environment.to_vec(), + process_group_leader, + }; + if identity.matches() { + matches.push(identity); + } + } + return Ok(matches); + } + #[cfg(not(target_os = "linux"))] + Ok(Vec::new()) +} + +pub fn find_process_identities_with_retry( + environment: &[(String, String)], + process_group_leader: bool, + attempts: usize, + poll_interval: Duration, +) -> io::Result> { + let attempts = attempts.max(1); + for attempt in 0..attempts { + let matches = find_process_identities_by_environment(environment, process_group_leader)?; + if !matches.is_empty() || attempt + 1 == attempts { + return Ok(matches); + } + thread::sleep(poll_interval); + } + unreachable!("at least one process discovery attempt runs") +} + +pub fn wait_for_path(path: &Path, attempts: usize, poll_interval: Duration) -> bool { + let attempts = attempts.max(1); + for attempt in 0..attempts { + if path.exists() { + return true; + } + if attempt + 1 < attempts { + thread::sleep(poll_interval); + } + } + false +} + +#[cfg(unix)] +struct BoundUnixSocketPath { + path: std::path::PathBuf, + device: u64, + inode: u64, +} + +#[cfg(unix)] +impl BoundUnixSocketPath { + fn new(path: std::path::PathBuf) -> io::Result { + use std::os::unix::fs::MetadataExt; + + let metadata = std::fs::symlink_metadata(&path)?; + Ok(Self { + path, + device: metadata.dev(), + inode: metadata.ino(), + }) + } +} + +#[cfg(unix)] +impl Drop for BoundUnixSocketPath { + fn drop(&mut self) { + use std::os::unix::fs::MetadataExt; + + let owns_path = std::fs::symlink_metadata(&self.path) + .is_ok_and(|metadata| metadata.dev() == self.device && metadata.ino() == self.inode); + if owns_path { + let _ = std::fs::remove_file(&self.path); + } + } +} + +#[cfg(unix)] +pub fn spawn_unix_stream_listener( + engine: swactor_engine::EngineHandle, + path: impl AsRef, + handler: H, +) -> io::Result<()> +where + H: Fn(tokio::net::UnixStream) -> F + Clone + Send + Sync + 'static, + F: Future + Send + 'static, +{ + use std::os::unix::net::UnixListener; + + let path = path.as_ref().to_path_buf(); + let listener = UnixListener::bind(&path)?; + let bound_path = BoundUnixSocketPath::new(path)?; + if let Err(error) = listener.set_nonblocking(true) { + drop(bound_path); + return Err(error); + } + let connection_engine = engine.clone(); + engine.spawn(async move { + let _bound_path = bound_path; + let listener = match tokio::net::UnixListener::from_std(listener) { + Ok(listener) => listener, + Err(error) => { + eprintln!("Unix stream listener stopped during startup: {error}"); + return; + } + }; + loop { + match listener.accept().await { + Ok((stream, _address)) => { + connection_engine.spawn(handler.clone()(stream)); + } + Err(error) => { + eprintln!("Unix stream listener stopped: {error}"); + return; + } + } + } + }); + Ok(()) +} + +#[cfg(test)] +mod properties { + use std::collections::VecDeque; + use std::fmt::Debug; + use std::io::Cursor; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::mpsc; + + use parking_lot::Mutex as ParkingMutex; + use proptest::prelude::*; + use swactor::actor::{ActorInterface, Ctx}; + use swactor::config::RuntimeConfig; + use swactor::runtime::{Runtime, RuntimeParts, SingleThreadRuntime}; + + use super::*; + + const DRIVER_BUDGET: usize = 64; + #[cfg(unix)] + #[test] + fn unix_listener_unlinks_its_bound_path_when_engine_stops() { + let path = std::env::temp_dir().join(format!( + "swactor-process-listener-cleanup-{}-{}.sock", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system clock after epoch") + .as_nanos() + )); + let parts = RuntimeParts::new(RuntimeConfig::default()); + let backend = swactor_engine::SteppingBackend::new(); + let engine = swactor_engine::Engine::new(parts, backend).expect("stepping engine"); + + spawn_unix_stream_listener(engine.handle(), &path, |_stream| async {}) + .expect("bind test Unix listener"); + assert!(path.exists(), "listener path was never created"); + + drop(engine); + assert!(!path.exists(), "listener path survived its owning engine"); + } + + struct StreamProbe { + observations: Arc>>, + closes_left: usize, + } + + impl ActorInterface for StreamProbe { + type Incoming = ProcessStreamObservation; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, observation: Self::Incoming) { + if matches!(observation, ProcessStreamObservation::Closed { .. }) + && self.closes_left > 0 + { + self.closes_left -= 1; + } + self.observations.lock().push(observation); + if self.closes_left == 0 { + ctx.stop_self(); + } + } + } + + struct SignalProbe { + count: Arc, + } + + impl ActorInterface for SignalProbe { + type Incoming = ProcessStopSignal; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, _signal: Self::Incoming) { + self.count.fetch_add(1, Ordering::SeqCst); + ctx.stop_self(); + } + } + + struct ExitProbe { + observations: Arc>>, + } + + impl ActorInterface for ExitProbe { + type Incoming = ProcessExitObservation; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, observation: Self::Incoming) { + self.observations.lock().push(observation); + ctx.stop_self(); + } + } + + #[derive(Clone, Debug)] + enum ReadAction { + Line(String), + Error, + Eof, + } + + #[derive(Clone, Debug)] + struct StreamAction { + stream: ProcessStream, + action: ReadAction, + } + + struct ScriptedReader { + actions: VecDeque, + terminal: bool, + } + + impl ScriptedReader { + fn new(actions: impl IntoIterator) -> Self { + Self { + actions: actions.into_iter().collect(), + terminal: false, + } + } + } + + impl Read for ScriptedReader { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + if self.terminal { + return Ok(0); + } + match self.actions.pop_front() { + Some(ReadAction::Line(line)) => { + let bytes = format!("{line}\n").into_bytes(); + assert!( + bytes.len() <= buffer.len(), + "scripted line exceeds reader buffer" + ); + buffer[..bytes.len()].copy_from_slice(&bytes); + Ok(bytes.len()) + } + Some(ReadAction::Error) => { + self.terminal = true; + Err(io::Error::other("scripted read failure")) + } + Some(ReadAction::Eof) | None => { + self.terminal = true; + Ok(0) + } + } + } + } + + #[derive(Debug, PartialEq, Eq)] + struct ExpectedStream { + lines: Vec, + error: bool, + } + + #[derive(Clone, Debug)] + enum LifecycleAction { + Stop, + Exit(ProcessExitObservation), + } + + struct LifecycleProbe { + observations: Arc>>, + stop_effects: Arc, + stop_requested: bool, + } + + impl ActorInterface for LifecycleProbe { + type Incoming = LifecycleAction; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, action: Self::Incoming) { + match action { + LifecycleAction::Stop if !self.stop_requested => { + self.stop_requested = true; + self.stop_effects.fetch_add(1, Ordering::SeqCst); + } + LifecycleAction::Stop => {} + LifecycleAction::Exit(observation) => { + self.observations.lock().push(observation); + ctx.stop_self(); + } + } + } + } + + #[derive(Clone, Debug)] + enum StopAction { + Notify, + Disconnect, + } + + #[derive(Clone, Debug)] + enum StdinAction { + Command, + MixedCaseCommand, + Other(String), + } + + fn runtime_host() -> (Runtime, SingleThreadRuntime) { + let mut config = RuntimeConfig::default(); + config.worker_count = 1; + let parts = RuntimeParts::new(config); + let runtime = parts.runtime().clone(); + (runtime, SingleThreadRuntime::new(parts)) + } + + fn drive(host: &mut SingleThreadRuntime) { + for _ in 0..DRIVER_BUDGET { + host.try_tick(); + } + } + + fn stream_action_strategy() -> impl Strategy { + let line = proptest::string::string_regex("[a-zA-Z0-9 ]{0,16}") + .expect("valid generated line expression") + .prop_map(ReadAction::Line); + ( + any::(), + prop_oneof![8 => line, 1 => Just(ReadAction::Error), 1 => Just(ReadAction::Eof)], + ) + .prop_map(|(stderr, action)| StreamAction { + stream: if stderr { + ProcessStream::Stderr + } else { + ProcessStream::Stdout + }, + action, + }) + } + + fn lifecycle_action_strategy() -> impl Strategy { + prop_oneof![ + 4 => Just(LifecycleAction::Stop), + 5 => (-2_i32..=2).prop_map(|status| { + LifecycleAction::Exit(ProcessExitObservation { + status: Some(status), + error: None, + }) + }), + 1 => Just(LifecycleAction::Exit(ProcessExitObservation { + status: None, + error: Some("scripted wait failure".to_owned()), + })), + ] + } + + fn stop_action_strategy() -> impl Strategy { + prop_oneof![4 => Just(StopAction::Notify), 1 => Just(StopAction::Disconnect)] + } + + fn stdin_action_strategy() -> impl Strategy { + prop_oneof![ + 2 => Just(StdinAction::Command), + 1 => Just(StdinAction::MixedCaseCommand), + 5 => proptest::string::string_regex("[a-zA-Z0-9 ]{0,16}") + .expect("valid generated stdin expression") + .prop_filter("other input must not be the command", |line| { + !line.trim().eq_ignore_ascii_case("stop") + }) + .prop_map(StdinAction::Other), + ] + } + + fn expected_stream(actions: &[StreamAction], stream: ProcessStream) -> ExpectedStream { + let mut expected = ExpectedStream { + lines: Vec::new(), + error: false, + }; + for action in actions + .iter() + .filter(|action| action.stream == stream) + .map(|action| &action.action) + { + match action { + ReadAction::Line(line) => expected.lines.push(line.clone()), + ReadAction::Error => { + expected.error = true; + break; + } + ReadAction::Eof => break, + } + } + expected + } + + fn check_stream_invariant( + observations: &[ProcessStreamObservation], + stream: ProcessStream, + expected: &ExpectedStream, + ) -> Result<(), String> { + let mut lines = Vec::new(); + let mut errors = 0; + let mut closes = 0; + let mut terminal_seen = false; + for observation in observations { + let observed_stream = match observation { + ProcessStreamObservation::Line { stream, .. } + | ProcessStreamObservation::Error { stream, .. } + | ProcessStreamObservation::Closed { stream } => *stream, + }; + if observed_stream != stream { + continue; + } + match observation { + ProcessStreamObservation::Line { line, .. } => { + if terminal_seen { + return Err(format!("line after terminal observation: {line:?}")); + } + lines.push(line.clone()); + } + ProcessStreamObservation::Error { error, .. } => { + if terminal_seen { + return Err(format!("duplicate terminal error: {error}")); + } + if !error.contains("scripted read failure") { + return Err(format!("unexpected read error: {error}")); + } + errors += 1; + terminal_seen = true; + } + ProcessStreamObservation::Closed { .. } => { + if closes > 0 { + return Err("stream closed more than once".to_owned()); + } + closes += 1; + terminal_seen = true; + } + } + } + if lines != expected.lines { + return Err(format!( + "line order/content changed: expected={:?}, actual={lines:?}", + expected.lines + )); + } + if errors != usize::from(expected.error) { + return Err(format!( + "read error count changed: expected={}, actual={errors}", + usize::from(expected.error) + )); + } + if closes != 1 { + return Err(format!("expected one stream close, actual={closes}")); + } + Ok(()) + } + + fn check_lifecycle_invariant( + observations: &[ProcessExitObservation], + expected: &ProcessExitObservation, + ) -> Result<(), String> { + match observations { + [actual] if actual == expected => Ok(()), + [actual] => Err(format!( + "terminal process observation changed: expected={expected:?}, actual={actual:?}" + )), + _ => Err(format!( + "expected one terminal process observation, actual={observations:?}" + )), + } + } + + fn check_signal_count(actual: usize, expected: usize) -> Result<(), String> { + if actual == expected { + Ok(()) + } else { + Err(format!( + "stop notification count changed: expected={expected}, actual={actual}" + )) + } + } + + fn quiescence_violation(runtime: &Runtime) -> Option { + let stats = runtime.stats(); + let mailbox_depth = stats + .workers + .iter() + .map(|worker| worker.mailbox_depth) + .sum::(); + let panics = stats + .workers + .iter() + .map(|worker| worker.panics) + .sum::(); + if stats.actors.is_empty() && mailbox_depth == 0 && panics == 0 { + None + } else { + Some(format!( + "actors={:?}, mailbox_depth={mailbox_depth}, panics={panics}", + stats.actors + )) + } + } + + fn evidence( + actions: &A, + observations: &O, + runtime: &Runtime, + outstanding: &S, + ) -> String { + let stats = runtime.stats(); + format!( + "actions={actions:?}; observations={observations:?}; actor_census={:?}; \ + workers={:?}; outstanding={outstanding:?}", + stats.actors, stats.workers + ) + } + + #[cfg(unix)] + #[test] + fn trivial_real_child_exit_has_a_hard_timeout() { + let (runtime, mut host) = runtime_host(); + let observations = Arc::new(ParkingMutex::new(Vec::new())); + let actor = runtime + .spawn(ExitProbe { + observations: Arc::clone(&observations), + }) + .expect("spawn process exit probe"); + let mut command = Command::new("/bin/sh"); + command.args(["-c", "exit 0"]); + let child = command_spawn(&mut command).expect("spawn trivial child"); + spawn_child_wait(child, runtime.create_sender(), actor); + + let deadline = Instant::now() + Duration::from_secs(2); + while observations.lock().is_empty() && Instant::now() < deadline { + host.try_tick(); + thread::yield_now(); + } + drive(&mut host); + + let observations = observations.lock().clone(); + assert_eq!( + observations, + vec![ProcessExitObservation { + status: Some(0), + error: None, + }], + "trivial child did not exit before hard timeout; observations={observations:?}; \ + actor_census={:?}", + runtime.stats().actors + ); + assert_eq!(quiescence_violation(&runtime), None); + } + + #[test] + fn property_invariants_reject_controlled_defects() { + let late_output = vec![ + ProcessStreamObservation::Closed { + stream: ProcessStream::Stdout, + }, + ProcessStreamObservation::Line { + stream: ProcessStream::Stdout, + line: "late".to_owned(), + }, + ]; + assert!( + check_stream_invariant( + &late_output, + ProcessStream::Stdout, + &ExpectedStream { + lines: Vec::new(), + error: false, + }, + ) + .is_err(), + "stream invariant accepted output after close" + ); + let duplicate_exit = ProcessExitObservation { + status: Some(0), + error: None, + }; + assert!( + check_lifecycle_invariant( + &[duplicate_exit.clone(), duplicate_exit.clone()], + &duplicate_exit, + ) + .is_err(), + "lifecycle invariant accepted duplicate terminal output" + ); + assert!( + check_signal_count(2, 1).is_err(), + "stop invariant accepted a duplicate notification" + ); + } + + proptest! { + #![proptest_config(ProptestConfig { + cases: 128, + max_shrink_iters: 2_000, + ..ProptestConfig::default() + })] + + #[test] + fn generated_stream_observations_close_once_and_stay_closed( + actions in prop::collection::vec(stream_action_strategy(), 0..=32), + ) { + let (runtime, mut host) = runtime_host(); + let observations = Arc::new(ParkingMutex::new(Vec::new())); + let actor = runtime + .spawn(StreamProbe { + observations: Arc::clone(&observations), + closes_left: 2, + }) + .expect("spawn process stream probe"); + let sender = runtime.create_sender(); + let stdout = ScriptedReader::new( + actions + .iter() + .filter(|action| action.stream == ProcessStream::Stdout) + .map(|action| action.action.clone()), + ); + let stderr = ScriptedReader::new( + actions + .iter() + .filter(|action| action.stream == ProcessStream::Stderr) + .map(|action| action.action.clone()), + ); + let stdout_handle = + spawn_line_reader(ProcessStream::Stdout, stdout, sender.clone(), actor); + let stderr_handle = + spawn_line_reader(ProcessStream::Stderr, stderr, sender, actor); + stdout_handle.join(); + stderr_handle.join(); + drive(&mut host); + + let observations = observations.lock().clone(); + let stdout_result = check_stream_invariant( + &observations, + ProcessStream::Stdout, + &expected_stream(&actions, ProcessStream::Stdout), + ); + let stderr_result = check_stream_invariant( + &observations, + ProcessStream::Stderr, + &expected_stream(&actions, ProcessStream::Stderr), + ); + let quiescence = quiescence_violation(&runtime); + let diagnostic = evidence( + &actions, + &observations, + &runtime, + &("reader_threads=0", &quiescence), + ); + prop_assert!( + stdout_result.is_ok(), + "{diagnostic}; stdout_violation={stdout_result:?}" + ); + prop_assert!( + stderr_result.is_ok(), + "{diagnostic}; stderr_violation={stderr_result:?}" + ); + prop_assert!( + quiescence.is_none(), + "{diagnostic}; quiescence_violation={quiescence:?}" + ); + } + + #[test] + fn generated_lifecycle_actions_make_stop_idempotent_and_exit_terminal( + generated in prop::collection::vec(lifecycle_action_strategy(), 0..=31), + ) { + let mut actions = generated; + if !actions + .iter() + .any(|action| matches!(action, LifecycleAction::Exit(_))) + { + actions.push(LifecycleAction::Exit(ProcessExitObservation { + status: Some(0), + error: None, + })); + } + let first_exit = actions + .iter() + .position(|action| matches!(action, LifecycleAction::Exit(_))) + .expect("normalization adds an exit"); + let expected_exit = match &actions[first_exit] { + LifecycleAction::Exit(observation) => observation.clone(), + LifecycleAction::Stop => unreachable!("first_exit points at exit"), + }; + let expected_stop_effects = usize::from( + actions[..first_exit] + .iter() + .any(|action| matches!(action, LifecycleAction::Stop)), + ); + + let (runtime, mut host) = runtime_host(); + let observations = Arc::new(ParkingMutex::new(Vec::new())); + let stop_effects = Arc::new(AtomicUsize::new(0)); + let actor = runtime + .spawn(LifecycleProbe { + observations: Arc::clone(&observations), + stop_effects: Arc::clone(&stop_effects), + stop_requested: false, + }) + .expect("spawn lifecycle probe"); + let mut rejected = Vec::new(); + for (index, action) in actions.iter().cloned().enumerate() { + if runtime.send_to(actor, action).is_err() { + rejected.push(index); + } + drive(&mut host); + } + + let observations = observations.lock().clone(); + let lifecycle_result = + check_lifecycle_invariant(&observations, &expected_exit); + let actual_stop_effects = stop_effects.load(Ordering::SeqCst); + let stop_result = + check_signal_count(actual_stop_effects, expected_stop_effects); + let quiescence = quiescence_violation(&runtime); + let diagnostic = evidence( + &actions, + &observations, + &runtime, + &( + format!("rejected_action_indices={rejected:?}"), + &quiescence, + ), + ); + prop_assert!( + lifecycle_result.is_ok(), + "{diagnostic}; lifecycle_violation={lifecycle_result:?}" + ); + prop_assert!( + stop_result.is_ok(), + "{diagnostic}; stop_violation={stop_result:?}" + ); + prop_assert!( + quiescence.is_none(), + "{diagnostic}; quiescence_violation={quiescence:?}" + ); + } + + #[test] + fn generated_stop_notifications_are_delivered_at_most_once( + actions in prop::collection::vec(stop_action_strategy(), 0..=32), + ) { + let (runtime, mut host) = runtime_host(); + let count = Arc::new(AtomicUsize::new(0)); + let actor = runtime + .spawn(SignalProbe { + count: Arc::clone(&count), + }) + .expect("spawn process signal probe"); + let (sender, receiver) = mpsc::channel(); + let mut sender = Some(sender); + let waiter = + spawn_stop_channel_wait_thread(receiver, runtime.create_sender(), actor); + let mut expected = 0; + let mut rejected = Vec::new(); + for (index, action) in actions.iter().enumerate() { + match action { + StopAction::Notify => match sender.as_ref() { + Some(sender) => { + if expected == 0 { + expected = 1; + } + if sender.send(()).is_err() { + rejected.push(index); + } + } + None => rejected.push(index), + }, + StopAction::Disconnect => drop(sender.take()), + } + } + drop(sender); + waiter.join().expect("join process stop waiter"); + drive(&mut host); + if expected == 0 { + runtime.stop_actor(actor).expect("stop unused signal probe"); + drive(&mut host); + } + + let actual = count.load(Ordering::SeqCst); + let signal_result = check_signal_count(actual, expected); + let quiescence = quiescence_violation(&runtime); + let diagnostic = evidence( + &actions, + &format!("stop_notifications={actual}"), + &runtime, + &( + format!("rejected_action_indices={rejected:?}"), + &quiescence, + ), + ); + prop_assert!( + signal_result.is_ok(), + "{diagnostic}; signal_violation={signal_result:?}" + ); + prop_assert!( + quiescence.is_none(), + "{diagnostic}; quiescence_violation={quiescence:?}" + ); + } + + #[test] + fn generated_stdin_commands_and_eof_notify_once( + actions in prop::collection::vec(stdin_action_strategy(), 0..=32), + trigger_on_eof in any::(), + ) { + let mut bytes = Vec::new(); + let mut has_command = false; + for action in &actions { + let line = match action { + StdinAction::Command => { + has_command = true; + "stop" + } + StdinAction::MixedCaseCommand => { + has_command = true; + " StOp " + } + StdinAction::Other(line) => line, + }; + bytes.extend_from_slice(line.as_bytes()); + bytes.push(b'\n'); + } + let expected = usize::from(has_command || trigger_on_eof); + let (runtime, mut host) = runtime_host(); + let count = Arc::new(AtomicUsize::new(0)); + let actor = runtime + .spawn(SignalProbe { + count: Arc::clone(&count), + }) + .expect("spawn stdin signal probe"); + forward_stdin_command( + Cursor::new(bytes), + "stop", + trigger_on_eof, + runtime.create_sender(), + actor, + ); + drive(&mut host); + if expected == 0 { + runtime.stop_actor(actor).expect("stop unused stdin probe"); + drive(&mut host); + } + + let actual = count.load(Ordering::SeqCst); + let signal_result = check_signal_count(actual, expected); + let quiescence = quiescence_violation(&runtime); + let diagnostic = evidence( + &(actions, trigger_on_eof), + &format!("stop_notifications={actual}"), + &runtime, + &("stdin_reader=closed", &quiescence), + ); + prop_assert!( + signal_result.is_ok(), + "{diagnostic}; signal_violation={signal_result:?}" + ); + prop_assert!( + quiescence.is_none(), + "{diagnostic}; quiescence_violation={quiescence:?}" + ); + } + } +} diff --git a/crates/provisioning/Cargo.toml b/crates/provisioning/Cargo.toml index 4a894d4..31ba2ff 100644 --- a/crates/provisioning/Cargo.toml +++ b/crates/provisioning/Cargo.toml @@ -13,3 +13,4 @@ swactor-engine = { path = "../engine" } [dev-dependencies] parking_lot = "0.12" serde_json = "1" +swactor-process = { path = "../process" } diff --git a/crates/provisioning/src/bootstrap.rs b/crates/provisioning/src/bootstrap.rs index 3023d3a..3d46e12 100644 --- a/crates/provisioning/src/bootstrap.rs +++ b/crates/provisioning/src/bootstrap.rs @@ -193,6 +193,8 @@ pub struct BootstrapActor { sender: ExternalSender, collector: Option>, phase: Phase, + /// Launch was requested; duplicate `Start` messages are inert. + started: bool, /// `Bootstrapped` reported (and collector fired) exactly once. announced: bool, /// Terminal report (Failed/Exited) emitted exactly once. @@ -208,6 +210,7 @@ impl BootstrapActor { sender: config.sender, collector: config.collector, phase: Phase::Bootstrapping, + started: false, announced: false, closed: false, } @@ -299,9 +302,10 @@ impl ActorInterface for BootstrapActor { fn handle(&mut self, ctx: &Ctx, msg: BootstrapMsg) { match msg { BootstrapMsg::Start => { - if self.phase != Phase::Bootstrapping { - return; // Restart of a started/stopped attempt: ignore. + if self.phase != Phase::Bootstrapping || self.started { + return; } + self.started = true; if let Err(reason) = self.logic.start(ctx, ctx.self_addr(), &self.sender) { self.fail(format!("launch failed: {reason}")); } @@ -334,22 +338,12 @@ pub fn spawn_bootstrap_actor( config: BootstrapConfig, ) -> Result { let sender = config.sender.clone(); - let start_sender = sender.clone(); let period = config.probe_period; let actor = ctx .spawn(BootstrapActor::new(logic, config)) .map_err(|error| format!("spawn bootstrap actor: {error}"))?; - let probe_engine = engine.clone(); - engine.spawn(async move { - let mut interval = probe_engine.interval(period); - loop { - (&mut interval).await; - if sender.send_to(actor, BootstrapMsg::Probe).is_err() { - return; - } - } - }); - let _ = start_sender.send_to(actor, BootstrapMsg::Start); + engine.send_every(period, sender.clone(), actor, BootstrapMsg::Probe); + let _ = sender.send_to(actor, BootstrapMsg::Start); Ok(actor) } diff --git a/crates/provisioning/tests/process_conformance.rs b/crates/provisioning/tests/process_conformance.rs index f146532..2e7f205 100644 --- a/crates/provisioning/tests/process_conformance.rs +++ b/crates/provisioning/tests/process_conformance.rs @@ -12,6 +12,7 @@ use std::sync::Arc; use parking_lot::Mutex; use provisioning::plugin::{NodeProvisionSpec, PluginNodeHandle, PluginSink, ProvisionPlugin}; +use swactor_process::{child_kill, child_wait, command_spawn}; use common::{ AMBIGUOUS_FAULT_MARKER, Fault, PluginBackendAdapter, TestablePlugin, assert_plugin_contracts, @@ -49,8 +50,8 @@ impl Drop for ProcessPlugin { let mut state = self.state.lock(); let children: Vec = std::mem::take(&mut state.children).into_values().collect(); for mut child in children { - let _ = child.kill(); - let _ = child.wait(); + let _ = child_kill(&mut child); + let _ = child_wait(&mut child); } } } @@ -108,10 +109,10 @@ impl ProvisionPlugin for ProcessPlugin { if matches!(fault, Some(Fault::Definite)) { return Err("scripted definite failure".to_owned()); } - let child = Command::new("sleep") - .arg("infinity") - .spawn() - .map_err(|error| format!("spawn failed: {error}"))?; + let mut command = Command::new("sleep"); + command.arg("infinity"); + let child = + command_spawn(&mut command).map_err(|error| format!("spawn failed: {error}"))?; let pid = child.id(); state.children.insert(attempt, child); state.created += 1; @@ -141,12 +142,8 @@ impl ProvisionPlugin for ProcessPlugin { fn stop_node(&mut self, handle: &PluginNodeHandle) -> Result<(), String> { let mut state = self.state.lock(); if let Some(mut child) = state.children.remove(&handle.id) { - child - .kill() - .map_err(|error| format!("kill failed: {error}"))?; - child - .wait() - .map_err(|error| format!("reap failed: {error}"))?; + child_kill(&mut child).map_err(|error| format!("kill failed: {error}"))?; + child_wait(&mut child).map_err(|error| format!("reap failed: {error}"))?; } Ok(()) } diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 5f3bc6d..f3d03eb 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,3 @@ [toolchain] channel = "nightly-2026-02-07" +components = ["rustc-dev"] diff --git a/src/worker.rs b/src/worker.rs index c001584..8e66d5e 100644 --- a/src/worker.rs +++ b/src/worker.rs @@ -322,13 +322,29 @@ impl Worker { ); } - // 9. Clean up poisoned and stopping actors - did_work |= Self::cleanup_dead_actors( + // 9. Clean up poisoned and stopping actors. Cleanup changes the pool after + // phase 8 published its snapshot, so republish cardinality and mailbox + // state when actors were removed rather than leaving observability stale + // until unrelated work reaches this worker. + let cleaned_dead = Self::cleanup_dead_actors( &mut self.pool, &mut self.worker_ext, &mut self.deferred_transfers, &tc, ); + if cleaned_dead { + did_work = true; + self.stats + .num_actors + .store(self.pool.len(), Ordering::Relaxed); + self.stats + .total_mailbox_depth + .store(self.pool.total_mailbox_depth(), Ordering::Relaxed); + if let Some(hook) = tc.stats_hook { + self.pool.mailbox_depths_into(&mut self.snapshot_buf); + hook.on_tick(wid.index(), &self.snapshot_buf); + } + } self.has_backlog = did_work; did_work diff --git a/tools/actor-control-flow-lint/Cargo.toml b/tools/actor-control-flow-lint/Cargo.toml new file mode 100644 index 0000000..70380fb --- /dev/null +++ b/tools/actor-control-flow-lint/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "actor-control-flow-lint-tests" +version = "0.1.0" +edition = "2024" +license = "AGPL-3.0-only" +publish = false + +[lib] +path = "src/lib.rs" diff --git a/tools/actor-control-flow-lint/driver.rs b/tools/actor-control-flow-lint/driver.rs new file mode 100755 index 0000000..5b73224 --- /dev/null +++ b/tools/actor-control-flow-lint/driver.rs @@ -0,0 +1,353 @@ +#![feature(rustc_private)] + +extern crate rustc_driver; +extern crate rustc_hir; +extern crate rustc_interface; +extern crate rustc_middle; +extern crate rustc_span; + +use std::env; +use std::process::ExitCode; + +use rustc_driver::{Callbacks, Compilation}; +use rustc_hir as hir; +use rustc_hir::def::Res; +use rustc_hir::intravisit::{self, Visitor}; +use rustc_interface::interface::Compiler; +use rustc_middle::ty::{TyCtxt, TypeckResults}; +use rustc_span::{Span, symbol::Symbol}; + +/// Crates whose purpose is to run the actor engine or turn external I/O, +/// process, telemetry, and provider API streams into actor observations. +/// Adding an entry changes the architecture. +const EXECUTION_OWNERS: &[&str] = &[ + "dashboard", + "iroh-driver", + "swactor", + "swactor-engine", + "swactor-process", + "swactor-transport", + "swactor-vastai", + "telemetry", +]; + +/// This package owns only the compile-contract subprocess harness. It cannot be +/// used as a workspace dependency. +const TEST_SUPPORT_OWNERS: &[&str] = &["actor-control-flow-lint-tests"]; + +/// Policy-bearing crates that must never enter an execution owner's dependency +/// closure. +const DOMAIN_CONTROL_CRATES: &[&str] = &[ + "myelin", + "provisioning", + "swactor-job-runner", + "xtask", +]; + +#[derive(Clone, Copy)] +struct Capability { + label: &'static str, + resolution: &'static str, + test_wait: bool, + paths: &'static [&'static str], +} + +const MOVE_TO_OWNER: &str = + "move stream mechanics into an approved execution owner or move the decision into an actor"; +const USE_ACTOR_TIMER: &str = + "schedule a typed actor message through the engine; the receiving actor owns the deadline decision"; + +/// Stable resolved item paths. These are deliberately compiler identities, not +/// spellings found in source, so re-exports, renamed imports, and local wrappers +/// cannot evade the boundary. +const CAPABILITIES: &[Capability] = &[ + Capability { + label: "asynchronous task spawning", + resolution: MOVE_TO_OWNER, + test_wait: false, + paths: &[ + "tokio::runtime::Handle::spawn", + "tokio::runtime::Runtime::spawn", + "tokio::spawn", + "tokio::task::spawn", + "tokio::task::spawn_local", + ], + }, + Capability { + label: "blocking task spawning", + resolution: MOVE_TO_OWNER, + test_wait: false, + paths: &[ + "tokio::runtime::Handle::spawn_blocking", + "tokio::runtime::Runtime::spawn_blocking", + "tokio::task::spawn_blocking", + ], + }, + Capability { + label: "engine task scheduling", + resolution: MOVE_TO_OWNER, + test_wait: false, + paths: &["swactor_engine::EngineHandle::spawn"], + }, + Capability { + label: "OS thread creation", + resolution: MOVE_TO_OWNER, + test_wait: false, + paths: &[ + "std::thread::Builder::spawn", + "std::thread::Builder::spawn_unchecked", + "std::thread::spawn", + ], + }, + Capability { + label: "thread sleeping", + resolution: USE_ACTOR_TIMER, + test_wait: true, + paths: &[ + "std::thread::park", + "std::thread::park_timeout", + "std::thread::sleep", + ], + }, + Capability { + label: "direct timer driving", + resolution: USE_ACTOR_TIMER, + test_wait: false, + paths: &[ + "swactor_engine::EngineHandle::interval", + "swactor_engine::EngineHandle::timer", + "swactor_engine::EngineHandle::timeout", + "tokio::time::interval", + "tokio::time::interval_at", + "tokio::time::sleep", + "tokio::time::sleep_until", + "tokio::time::timeout", + "tokio::time::timeout_at", + ], + }, + Capability { + label: "runtime construction or driving", + resolution: "the actor engine owns runtime construction and progression", + test_wait: false, + paths: &[ + "swactor::runtime::SingleThreadRuntime::tick", + "swactor::runtime::SingleThreadRuntime::try_tick", + "swactor::runtime::SingleThreadRuntime::has_work", + "tokio::runtime::Builder::new_current_thread", + "tokio::runtime::Builder::new_multi_thread", + "tokio::runtime::Handle::block_on", + "tokio::runtime::Runtime::block_on", + "tokio::runtime::Runtime::new", + ], + }, + Capability { + label: "blocking receive used as a controller", + resolution: "receive observations in an actor; tests may use a bounded observation wait", + test_wait: true, + paths: &[ + "crossbeam_channel::channel::Receiver::recv", + "crossbeam_channel::channel::Receiver::recv_deadline", + "crossbeam_channel::channel::Receiver::recv_timeout", + "std::sync::mpsc::Receiver::recv", + "std::sync::mpsc::Receiver::recv_deadline", + "std::sync::mpsc::Receiver::recv_timeout", + "tokio::sync::mpsc::bounded::Receiver::blocking_recv", + "tokio::sync::oneshot::Receiver::blocking_recv", + ], + }, + Capability { + label: "process creation", + resolution: "send a command to the process I/O owner and return exit/output observations to an actor", + test_wait: false, + paths: &[ + "std::process::Child::kill", + "std::process::Child::try_wait", + "std::process::Child::wait", + "std::process::Child::wait_with_output", + "std::process::Command::output", + "std::process::Command::spawn", + "std::process::Command::status", + "tokio::process::Child::kill", + "tokio::process::Child::start_kill", + "tokio::process::Child::try_wait", + "tokio::process::Child::wait", + "tokio::process::Child::wait_with_output", + "tokio::process::Command::output", + "tokio::process::Command::spawn", + "tokio::process::Command::status", + ], + }, +]; + +struct ActorControlFlowCallbacks { + package: String, + test_build: bool, + trace: bool, +} + +impl Callbacks for ActorControlFlowCallbacks { + fn after_analysis<'tcx>( + &mut self, + _compiler: &Compiler, + tcx: TyCtxt<'tcx>, + ) -> Compilation { + if TEST_SUPPORT_OWNERS.contains(&self.package.as_str()) { + return Compilation::Continue; + } + if EXECUTION_OWNERS.contains(&self.package.as_str()) { + check_owner_dependencies(tcx, &self.package); + return Compilation::Continue; + } + + for owner in tcx.hir_body_owners() { + let typeck = tcx.typeck(owner); + let body = tcx.hir_body_owned_by(owner); + let mut visitor = CapabilityVisitor { + tcx, + typeck, + package: &self.package, + test_build: self.test_build, + trace: self.trace, + }; + visitor.visit_body(body); + } + + Compilation::Continue + } +} + +fn check_owner_dependencies(tcx: TyCtxt<'_>, package: &str) { + for &crate_num in tcx.crates(()) { + let dependency_symbol = tcx.crate_name(crate_num); + let dependency = dependency_symbol.as_str(); + if DOMAIN_CONTROL_CRATES.contains(&dependency) { + tcx.dcx().err(format!( + "actor control-flow policy: execution owner `{package}` depends on domain-control crate `{dependency}`; execution owners must remain below domain policy in the dependency graph" + )); + } + } +} + +struct CapabilityVisitor<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + typeck: &'tcx TypeckResults<'tcx>, + package: &'a str, + test_build: bool, + trace: bool, +} + +impl<'tcx> Visitor<'tcx> for CapabilityVisitor<'_, 'tcx> { + fn visit_expr(&mut self, expr: &'tcx hir::Expr<'tcx>) { + match &expr.kind { + hir::ExprKind::MethodCall(..) => { + if let Some(def_id) = self.typeck.type_dependent_def_id(expr.hir_id) { + self.check(def_id, expr.span); + } + } + hir::ExprKind::Path(qpath) => { + if let Res::Def(_, def_id) = self.typeck.qpath_res(qpath, expr.hir_id) { + self.check(def_id, expr.span); + } + } + _ => {} + } + intravisit::walk_expr(self, expr); + } +} + +impl CapabilityVisitor<'_, '_> { + fn check(&self, def_id: rustc_hir::def_id::DefId, span: Span) { + let path = self.tcx.def_path_str(def_id); + let normalized_path = normalize_def_path(&path); + if self.trace && is_candidate_name(self.tcx.item_name(def_id)) { + eprintln!("actor-control-flow trace: {path}"); + } + + let Some(capability) = CAPABILITIES + .iter() + .find(|capability| capability.paths.contains(&normalized_path.as_str())) + else { + return; + }; + + + if self.test_build && capability.test_wait { + return; + } + + self.tcx.dcx().span_err( + span, + format!( + "actor control-flow violation: `{}` is forbidden in workspace crate `{}`; {}", + capability.label, self.package, capability.resolution + ), + ); + } +} + +fn normalize_def_path(path: &str) -> String { + let mut normalized = String::with_capacity(path.len()); + let mut cursor = 0; + while let Some(relative_start) = path[cursor..].find("::<") { + let start = cursor + relative_start; + normalized.push_str(&path[cursor..start]); + let generic_start = start + 3; + let mut depth = 1_usize; + let mut end = path.len(); + for (offset, character) in path[generic_start..].char_indices() { + match character { + '<' => depth += 1, + '>' => { + depth -= 1; + if depth == 0 { + end = generic_start + offset + character.len_utf8(); + break; + } + } + _ => {} + } + } + cursor = end; + } + normalized.push_str(&path[cursor..]); + normalized +} + +fn is_candidate_name(name: Symbol) -> bool { + matches!( + name.as_str(), + "block_on" + | "blocking_recv" + | "has_work" + | "interval" + | "interval_at" + | "new_current_thread" + | "new_multi_thread" + | "recv" + | "recv_deadline" + | "recv_timeout" + | "sleep" + | "sleep_until" + | "spawn" + | "spawn_blocking" + | "spawn_local" + | "tick" + | "timeout" + | "timeout_at" + | "timer" + | "try_tick" + ) +} + +fn main() -> ExitCode { + let args: Vec = env::args().collect(); + let package = env::var("MYELIN_ACTOR_LINT_PACKAGE").unwrap_or_else(|_| "unknown".to_owned()); + let test_build = env::var_os("MYELIN_ACTOR_LINT_TEST_BUILD").is_some(); + let trace = env::var_os("MYELIN_ACTOR_LINT_TRACE").is_some(); + let mut callbacks = ActorControlFlowCallbacks { + package, + test_build, + trace, + }; + rustc_driver::catch_with_exit_code(|| rustc_driver::run_compiler(&args, &mut callbacks)) +} diff --git a/tools/actor-control-flow-lint/rustc-wrapper.py b/tools/actor-control-flow-lint/rustc-wrapper.py new file mode 100755 index 0000000..e7e679e --- /dev/null +++ b/tools/actor-control-flow-lint/rustc-wrapper.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Build, cache, and execute the repository-owned rustc policy driver.""" + +from __future__ import annotations + +import fcntl +import hashlib +import os +from pathlib import Path +import subprocess +import sys + + +def fail(message: str) -> "None": + print(f"actor-control-flow wrapper: {message}", file=sys.stderr) + raise SystemExit(1) + + +def main() -> None: + if len(sys.argv) < 2: + fail("Cargo did not supply the real rustc path") + + real_rustc = sys.argv[1] + rustc_args = sys.argv[2:] + root = Path(__file__).resolve().parents[2] + source = Path(__file__).with_name("driver.rs") + target_root = Path(os.environ.get("CARGO_TARGET_DIR", root / "target")) + if not target_root.is_absolute(): + target_root = root / target_root + cache = target_root / "actor-control-flow-lint" + cache.mkdir(parents=True, exist_ok=True) + + bootstrap_env = os.environ.copy() + bootstrap_env.pop("CARGO_MAKEFLAGS", None) + bootstrap_env.pop("MAKEFLAGS", None) + + try: + version = subprocess.check_output( + [real_rustc, "--version", "--verbose"], + text=True, + stderr=subprocess.STDOUT, + env=bootstrap_env, + ) + sysroot = subprocess.check_output( + [real_rustc, "--print", "sysroot"], + text=True, + stderr=subprocess.STDOUT, + env=bootstrap_env, + ).strip() + except (OSError, subprocess.CalledProcessError) as error: + fail(f"cannot inspect pinned rustc: {error}") + + digest = hashlib.sha256(source.read_bytes() + version.encode()).hexdigest()[:20] + driver = cache / f"driver-{digest}" + lock_path = cache / "build.lock" + + with lock_path.open("a+b") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + if not driver.exists(): + temporary = cache / f".{driver.name}.{os.getpid()}.tmp" + command = [ + real_rustc, + str(source), + "--crate-name", + "myelin_actor_control_flow_lint", + "--edition=2024", + "-Cprefer-dynamic", + "-L", + str(Path(sysroot) / "lib"), + "-o", + str(temporary), + ] + result = subprocess.run(command, env=bootstrap_env) + if result.returncode != 0: + temporary.unlink(missing_ok=True) + fail( + "failed to build compiler driver; the pinned toolchain must include " + "the rustc-dev component" + ) + os.replace(temporary, driver) + + package = os.environ.get("CARGO_PKG_NAME", "unknown") + test_build = "--test" in rustc_args + child_env = os.environ.copy() + child_env["MYELIN_ACTOR_LINT_PACKAGE"] = package + if test_build: + child_env["MYELIN_ACTOR_LINT_TEST_BUILD"] = "1" + else: + child_env.pop("MYELIN_ACTOR_LINT_TEST_BUILD", None) + + rustc_lib = str(Path(sysroot) / "lib") + current_library_path = child_env.get("LD_LIBRARY_PATH") + child_env["LD_LIBRARY_PATH"] = ( + f"{rustc_lib}:{current_library_path}" if current_library_path else rustc_lib + ) + + os.execvpe(str(driver), [real_rustc, *rustc_args], child_env) + + +if __name__ == "__main__": + main() diff --git a/tools/actor-control-flow-lint/src/lib.rs b/tools/actor-control-flow-lint/src/lib.rs new file mode 100644 index 0000000..c1464f2 --- /dev/null +++ b/tools/actor-control-flow-lint/src/lib.rs @@ -0,0 +1 @@ +//! Compile-contract harness for the repository-owned actor control-flow driver. diff --git a/tools/actor-control-flow-lint/tests/contracts.rs b/tools/actor-control-flow-lint/tests/contracts.rs new file mode 100644 index 0000000..cb9ae19 --- /dev/null +++ b/tools/actor-control-flow-lint/tests/contracts.rs @@ -0,0 +1,92 @@ +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +fn repository_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .expect("lint package lives under tools/") + .to_path_buf() +} + +fn cargo_check(fixture: &str, extra_args: &[&str]) -> Output { + let root = repository_root(); + let fixture_dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures") + .join(fixture); + let target_dir = root + .join("target/actor-control-flow-contracts") + .join(fixture); + let wrapper = root.join("tools/actor-control-flow-lint/rustc-wrapper.py"); + + let mut command = Command::new(env!("CARGO")); + command + .arg("check") + .arg("--quiet") + .args(extra_args) + .current_dir(fixture_dir) + .env("CARGO_TARGET_DIR", target_dir) + .env("CARGO_TERM_COLOR", "never") + .env("RUSTC_WORKSPACE_WRAPPER", wrapper) + .env_remove("CARGO_MAKEFLAGS") + .env_remove("MAKEFLAGS"); + command.output().expect("run fixture cargo check") +} + +fn stderr(output: &Output) -> String { + String::from_utf8_lossy(&output.stderr).into_owned() +} + +#[test] +fn compiler_policy_contracts() { + let direct = cargo_check("fail-domain-capabilities", &[]); + assert!( + !direct.status.success(), + "forbidden domain fixture compiled" + ); + let direct_stderr = stderr(&direct); + for expected in [ + "asynchronous task spawning", + "blocking task spawning", + "engine task scheduling", + "OS thread creation", + "thread sleeping", + "direct timer driving", + "runtime construction or driving", + "blocking receive used as a controller", + "process creation", + ] { + assert!( + direct_stderr.contains(expected), + "missing `{expected}` diagnostic:\n{direct_stderr}" + ); + } + + let dependency = cargo_check("fail-owner-dependency", &[]); + assert!( + !dependency.status.success(), + "execution owner depending on domain control compiled" + ); + let dependency_stderr = stderr(&dependency); + assert!( + dependency_stderr + .contains("execution owner `swactor-engine` depends on domain-control crate `myelin`"), + "missing owner dependency diagnostic:\n{dependency_stderr}" + ); + + for fixture in ["pass-actor-domain", "pass-execution-owner"] { + let output = cargo_check(fixture, &[]); + assert!( + output.status.success(), + "compile-pass fixture `{fixture}` failed:\n{}", + stderr(&output) + ); + } + + let test_wait = cargo_check("pass-test-wait", &["--tests"]); + assert!( + test_wait.status.success(), + "narrow test wait fixture failed:\n{}", + stderr(&test_wait) + ); +} diff --git a/tools/actor-control-flow-lint/tests/fixtures/fail-domain-capabilities/Cargo.lock b/tools/actor-control-flow-lint/tests/fixtures/fail-domain-capabilities/Cargo.lock new file mode 100644 index 0000000..e9ec87b --- /dev/null +++ b/tools/actor-control-flow-lint/tests/fixtures/fail-domain-capabilities/Cargo.lock @@ -0,0 +1,263 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "actor-lint-domain-fail" +version = "0.0.0" +dependencies = [ + "swactor-engine", + "tokio", +] + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "crossbeam-queue" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "swactor" +version = "0.1.0" +dependencies = [ + "crossbeam-queue", + "crossbeam-utils", + "getrandom", + "parking_lot", +] + +[[package]] +name = "swactor-engine" +version = "0.1.0" +dependencies = [ + "parking_lot", + "swactor", + "tokio", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] diff --git a/tools/actor-control-flow-lint/tests/fixtures/fail-domain-capabilities/Cargo.toml b/tools/actor-control-flow-lint/tests/fixtures/fail-domain-capabilities/Cargo.toml new file mode 100644 index 0000000..3402f9a --- /dev/null +++ b/tools/actor-control-flow-lint/tests/fixtures/fail-domain-capabilities/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "actor-lint-domain-fail" +version = "0.0.0" +edition = "2024" +publish = false + +[dependencies] +swactor-engine = { path = "../../../../../crates/engine" } +tokio = { version = "1", features = ["rt", "time"] } + +[workspace] diff --git a/tools/actor-control-flow-lint/tests/fixtures/fail-domain-capabilities/src/lib.rs b/tools/actor-control-flow-lint/tests/fixtures/fail-domain-capabilities/src/lib.rs new file mode 100755 index 0000000..c313de1 --- /dev/null +++ b/tools/actor-control-flow-lint/tests/fixtures/fail-domain-capabilities/src/lib.rs @@ -0,0 +1,35 @@ +#![allow(clippy::disallowed_methods, dead_code, unused_must_use)] + +use std::thread::sleep as renamed_sleep; +use std::time::Duration; +use swactor_engine::EngineHandle; + +fn local_sleep_wrapper() { + renamed_sleep(Duration::from_millis(1)); +} + +fn forbidden_engine_controls(handle: &EngineHandle) { + handle.spawn(async {}); + handle.timer(Duration::from_millis(1)); +} + +fn forbidden_thread_and_receive() { + std::thread::spawn(|| {}); + local_sleep_wrapper(); + let (_sender, receiver) = std::sync::mpsc::channel::<()>(); + let _ = receiver.recv(); + let _ = std::process::Command::new("true").output(); +} + +fn forbidden_runtime_driver() { + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .expect("runtime"); + runtime.block_on(async {}); +} + +async fn forbidden_tokio_controls() { + tokio::spawn(async {}); + tokio::task::spawn_blocking(|| {}); + tokio::time::sleep(Duration::from_millis(1)).await; +} diff --git a/tools/actor-control-flow-lint/tests/fixtures/fail-owner-dependency/Cargo.lock b/tools/actor-control-flow-lint/tests/fixtures/fail-owner-dependency/Cargo.lock new file mode 100644 index 0000000..711ffe9 --- /dev/null +++ b/tools/actor-control-flow-lint/tests/fixtures/fail-owner-dependency/Cargo.lock @@ -0,0 +1,14 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "myelin" +version = "0.0.0" + +[[package]] +name = "swactor-engine" +version = "0.0.0" +dependencies = [ + "myelin", +] diff --git a/tools/actor-control-flow-lint/tests/fixtures/fail-owner-dependency/Cargo.toml b/tools/actor-control-flow-lint/tests/fixtures/fail-owner-dependency/Cargo.toml new file mode 100644 index 0000000..f55e2ee --- /dev/null +++ b/tools/actor-control-flow-lint/tests/fixtures/fail-owner-dependency/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "swactor-engine" +version = "0.0.0" +edition = "2024" +publish = false + +[dependencies] +myelin = { path = "myelin" } + +[workspace] diff --git a/tools/actor-control-flow-lint/tests/fixtures/fail-owner-dependency/myelin/Cargo.toml b/tools/actor-control-flow-lint/tests/fixtures/fail-owner-dependency/myelin/Cargo.toml new file mode 100644 index 0000000..d2df276 --- /dev/null +++ b/tools/actor-control-flow-lint/tests/fixtures/fail-owner-dependency/myelin/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "myelin" +version = "0.0.0" +edition = "2024" +publish = false + +[lib] +path = "src/lib.rs" diff --git a/tools/actor-control-flow-lint/tests/fixtures/fail-owner-dependency/myelin/src/lib.rs b/tools/actor-control-flow-lint/tests/fixtures/fail-owner-dependency/myelin/src/lib.rs new file mode 100644 index 0000000..6331207 --- /dev/null +++ b/tools/actor-control-flow-lint/tests/fixtures/fail-owner-dependency/myelin/src/lib.rs @@ -0,0 +1 @@ +pub fn domain_policy() {} diff --git a/tools/actor-control-flow-lint/tests/fixtures/fail-owner-dependency/src/lib.rs b/tools/actor-control-flow-lint/tests/fixtures/fail-owner-dependency/src/lib.rs new file mode 100644 index 0000000..d7e6e66 --- /dev/null +++ b/tools/actor-control-flow-lint/tests/fixtures/fail-owner-dependency/src/lib.rs @@ -0,0 +1,3 @@ +pub fn forbidden_dependency() { + myelin::domain_policy(); +} diff --git a/tools/actor-control-flow-lint/tests/fixtures/pass-actor-domain/Cargo.lock b/tools/actor-control-flow-lint/tests/fixtures/pass-actor-domain/Cargo.lock new file mode 100644 index 0000000..35f7d52 --- /dev/null +++ b/tools/actor-control-flow-lint/tests/fixtures/pass-actor-domain/Cargo.lock @@ -0,0 +1,263 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "actor-lint-domain-pass" +version = "0.0.0" +dependencies = [ + "swactor", + "swactor-engine", +] + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "crossbeam-queue" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "swactor" +version = "0.1.0" +dependencies = [ + "crossbeam-queue", + "crossbeam-utils", + "getrandom", + "parking_lot", +] + +[[package]] +name = "swactor-engine" +version = "0.1.0" +dependencies = [ + "parking_lot", + "swactor", + "tokio", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] diff --git a/tools/actor-control-flow-lint/tests/fixtures/pass-actor-domain/Cargo.toml b/tools/actor-control-flow-lint/tests/fixtures/pass-actor-domain/Cargo.toml new file mode 100644 index 0000000..0c74364 --- /dev/null +++ b/tools/actor-control-flow-lint/tests/fixtures/pass-actor-domain/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "actor-lint-domain-pass" +version = "0.0.0" +edition = "2024" +publish = false + +[dependencies] +swactor = { path = "../../../../.." } +swactor-engine = { path = "../../../../../crates/engine" } + +[workspace] diff --git a/tools/actor-control-flow-lint/tests/fixtures/pass-actor-domain/src/lib.rs b/tools/actor-control-flow-lint/tests/fixtures/pass-actor-domain/src/lib.rs new file mode 100644 index 0000000..57aeef8 --- /dev/null +++ b/tools/actor-control-flow-lint/tests/fixtures/pass-actor-domain/src/lib.rs @@ -0,0 +1,50 @@ +use std::time::Duration; + +use swactor::actor::{ActorAddress, ActorInterface, Ctx}; +use swactor::runtime::ExternalSender; +use swactor_engine::EngineHandle; + +#[derive(Clone)] +pub struct Tick { + pub generation: u64, +} + +struct Child; + +impl ActorInterface for Child { + type Incoming = Tick; + type Response = (); + + fn handle(&mut self, _ctx: &Ctx, _message: Tick) {} +} + +pub struct Parent; + +impl Parent { + fn spawn_helper(&self, ctx: &Ctx) { + let _ = ctx.spawn(Child); + } +} + +impl ActorInterface for Parent { + type Incoming = Tick; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, _message: Tick) { + self.spawn_helper(ctx); + } +} + +pub fn schedule_actor_tick( + engine: &EngineHandle, + sender: ExternalSender, + actor: ActorAddress, + generation: u64, +) { + engine.send_after( + Duration::from_millis(10), + sender, + actor, + Tick { generation }, + ); +} diff --git a/tools/actor-control-flow-lint/tests/fixtures/pass-execution-owner/Cargo.lock b/tools/actor-control-flow-lint/tests/fixtures/pass-execution-owner/Cargo.lock new file mode 100644 index 0000000..70f7809 --- /dev/null +++ b/tools/actor-control-flow-lint/tests/fixtures/pass-execution-owner/Cargo.lock @@ -0,0 +1,77 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "iroh-driver" +version = "0.0.0" +dependencies = [ + "tokio", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] diff --git a/tools/actor-control-flow-lint/tests/fixtures/pass-execution-owner/Cargo.toml b/tools/actor-control-flow-lint/tests/fixtures/pass-execution-owner/Cargo.toml new file mode 100644 index 0000000..48f519b --- /dev/null +++ b/tools/actor-control-flow-lint/tests/fixtures/pass-execution-owner/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "iroh-driver" +version = "0.0.0" +edition = "2024" +publish = false + +[dependencies] +tokio = { version = "1", features = ["net", "rt", "time"] } + +[workspace] diff --git a/tools/actor-control-flow-lint/tests/fixtures/pass-execution-owner/src/lib.rs b/tools/actor-control-flow-lint/tests/fixtures/pass-execution-owner/src/lib.rs new file mode 100644 index 0000000..7503fe8 --- /dev/null +++ b/tools/actor-control-flow-lint/tests/fixtures/pass-execution-owner/src/lib.rs @@ -0,0 +1,8 @@ +use std::time::Duration; + +pub fn start_transport_pump() { + tokio::spawn(async { + let _io_type: Option = None; + tokio::time::sleep(Duration::from_millis(1)).await; + }); +} diff --git a/tools/actor-control-flow-lint/tests/fixtures/pass-test-wait/Cargo.lock b/tools/actor-control-flow-lint/tests/fixtures/pass-test-wait/Cargo.lock new file mode 100644 index 0000000..2a52c3b --- /dev/null +++ b/tools/actor-control-flow-lint/tests/fixtures/pass-test-wait/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "actor-lint-test-wait-pass" +version = "0.0.0" diff --git a/tools/actor-control-flow-lint/tests/fixtures/pass-test-wait/Cargo.toml b/tools/actor-control-flow-lint/tests/fixtures/pass-test-wait/Cargo.toml new file mode 100644 index 0000000..8973e00 --- /dev/null +++ b/tools/actor-control-flow-lint/tests/fixtures/pass-test-wait/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "actor-lint-test-wait-pass" +version = "0.0.0" +edition = "2024" +publish = false + +[workspace] diff --git a/tools/actor-control-flow-lint/tests/fixtures/pass-test-wait/src/lib.rs b/tools/actor-control-flow-lint/tests/fixtures/pass-test-wait/src/lib.rs new file mode 100644 index 0000000..0082cb1 --- /dev/null +++ b/tools/actor-control-flow-lint/tests/fixtures/pass-test-wait/src/lib.rs @@ -0,0 +1 @@ +pub fn marker() {} diff --git a/tools/actor-control-flow-lint/tests/fixtures/pass-test-wait/tests/wait.rs b/tools/actor-control-flow-lint/tests/fixtures/pass-test-wait/tests/wait.rs new file mode 100644 index 0000000..6b42541 --- /dev/null +++ b/tools/actor-control-flow-lint/tests/fixtures/pass-test-wait/tests/wait.rs @@ -0,0 +1,12 @@ +use std::time::Duration; + +#[test] +fn bounded_observation_wait_is_allowed() { + let (sender, receiver) = std::sync::mpsc::channel(); + sender.send(7_u8).expect("send observation"); + assert_eq!( + receiver.recv_timeout(Duration::from_millis(10)), + Ok(7) + ); + std::thread::sleep(Duration::from_millis(1)); +} diff --git a/tools/vastai/Cargo.toml b/tools/vastai/Cargo.toml index b360cce..413d382 100644 --- a/tools/vastai/Cargo.toml +++ b/tools/vastai/Cargo.toml @@ -5,6 +5,10 @@ edition = "2024" license = "AGPL-3.0-only" publish = false + +[features] +default = [] +test-support = [] [dependencies] reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } serde = { version = "1", features = ["derive"] } diff --git a/tools/vastai/src/blocking.rs b/tools/vastai/src/blocking.rs new file mode 100644 index 0000000..14b2ae4 --- /dev/null +++ b/tools/vastai/src/blocking.rs @@ -0,0 +1,91 @@ +//! Synchronous facade for concrete Vast.ai API operations. +//! +//! The provider-I/O crate owns the Tokio runtime used to drive HTTP requests; +//! domain actors receive only operation results and never drive a runtime. + +use crate::{ + CreateInstanceRequest, InstanceInfo, LabeledInstance, LifecyclePolicy, Offer, + OfferBrowseCriteria, ProviderInstanceStatus, RunningInstance, SelectionPolicy, VastClient, +}; +use std::time::Duration; + +pub struct BlockingVastClient { + client: VastClient, + runtime: tokio::runtime::Runtime, +} + +impl BlockingVastClient { + pub fn new(client: VastClient) -> Result { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| format!("build Vast.ai I/O runtime: {error}"))?; + Ok(Self { client, runtime }) + } + + pub fn browse_offers(&self, criteria: &OfferBrowseCriteria) -> Result, String> { + self.runtime.block_on(self.client.browse_offers(criteria)) + } + + pub fn search_offers( + &self, + policy: &SelectionPolicy, + target_count: u32, + ) -> Result, String> { + self.runtime + .block_on(self.client.search_offers(policy, target_count)) + } + + pub fn create_instance(&self, request: &CreateInstanceRequest) -> Result { + self.runtime.block_on(self.client.create_instance(request)) + } + + pub fn instance_status(&self, contract_id: u64) -> Result { + self.runtime + .block_on(self.client.instance_status(contract_id)) + } + + pub fn list_by_label(&self, label: &str) -> Result, String> { + self.runtime.block_on(self.client.list_by_label(label)) + } + + pub fn list_by_label_with_retry( + &self, + label: &str, + attempts: usize, + pace: Duration, + ) -> Result, String> { + let attempts = attempts.max(1); + for attempt in 0..attempts { + let instances = self.list_by_label(label)?; + if !instances.is_empty() || attempt + 1 == attempts { + return Ok(instances); + } + std::thread::sleep(pace); + } + unreachable!("at least one Vast.ai label lookup attempt runs") + } + + pub fn wait_for_ssh_endpoint( + &self, + contract_id: u64, + label: &str, + policy: &LifecyclePolicy, + ) -> Result { + self.runtime.block_on( + self.client + .wait_for_ssh_endpoint(contract_id, label, policy), + ) + } + + pub fn destroy_instance_with_retry(&self, contract_id: u64) -> Result<(), String> { + self.runtime + .block_on(self.client.destroy_instance_with_retry(contract_id)) + } +} + +impl Clone for BlockingVastClient { + fn clone(&self) -> Self { + Self::new(self.client.clone()).expect("clone Vast.ai I/O runtime") + } +} diff --git a/tools/vastai/src/client.rs b/tools/vastai/src/client.rs index c50dfb0..8ead45f 100644 --- a/tools/vastai/src/client.rs +++ b/tools/vastai/src/client.rs @@ -1,5 +1,8 @@ use std::time::Duration; +pub const VASTAI_BASE_URL_ENV: &str = "VASTAI_BASE_URL"; +const DEFAULT_VASTAI_BASE_URL: &str = "https://cloud.vast.ai"; + use crate::search::OfferBrowseCriteria; use crate::types::{ @@ -18,8 +21,16 @@ pub struct VastClient { impl VastClient { pub const REQUEST_TIMEOUT: Duration = Duration::from_secs(45); + /// Build a client for the configured Vast.ai API endpoint. + /// + /// `VASTAI_BASE_URL` supports operator proxies and production-shaped local + /// fixtures. Empty values retain the public Vast.ai endpoint. pub fn new(api_key: impl Into) -> Self { - Self::with_base_url("https://cloud.vast.ai", api_key) + let base_url = std::env::var(VASTAI_BASE_URL_ENV) + .ok() + .filter(|url| !url.trim().is_empty()) + .unwrap_or_else(|| DEFAULT_VASTAI_BASE_URL.to_owned()); + Self::with_base_url(base_url, api_key) } pub fn with_base_url(base_url: impl Into, api_key: impl Into) -> Self { diff --git a/tools/vastai/src/lib.rs b/tools/vastai/src/lib.rs index 55d3b4a..8155367 100644 --- a/tools/vastai/src/lib.rs +++ b/tools/vastai/src/lib.rs @@ -4,6 +4,7 @@ //! general-purpose actor runtime, while this utility rents, monitors, and tears //! down vast.ai machines for apps that choose to use it. +mod blocking; pub mod client; pub mod config; pub mod filters; @@ -15,8 +16,11 @@ pub mod provision; pub mod search; pub mod state; pub mod teardown; +#[cfg(feature = "test-support")] +pub mod test_http; pub mod types; +pub use blocking::BlockingVastClient; pub use client::VastClient; pub use lease::{confirm_lease, provision_fleet}; pub use logs::{fetch_logs, request_logs}; diff --git a/tools/vastai/src/test_http.rs b/tools/vastai/src/test_http.rs new file mode 100644 index 0000000..58f9f72 --- /dev/null +++ b/tools/vastai/src/test_http.rs @@ -0,0 +1,180 @@ +//! Synchronous HTTP fixture owned by the provider-I/O substrate. +//! +//! The fixture accepts only declarative method/path/JSON routes. Domain tests +//! cannot supply futures, callbacks, or other independently scheduled work. + +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; +use std::time::Duration; + +#[derive(Clone, Debug)] +pub struct TestHttpRoute { + method: String, + path: String, + status: u16, + body: Vec, +} + +impl TestHttpRoute { + pub fn json(method: &str, path: &str, status: u16, body: serde_json::Value) -> Self { + Self { + method: method.to_owned(), + path: path.to_owned(), + status, + body: serde_json::to_vec(&body).expect("test HTTP JSON serializes"), + } + } + + pub fn raw(method: &str, path: &str, status: u16, body: impl Into>) -> Self { + Self { + method: method.to_owned(), + path: path.to_owned(), + status, + body: body.into(), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TestHttpRequest { + pub method: String, + pub path: String, +} + +pub struct TestHttpServer { + address: std::net::SocketAddr, + requests: Arc>>, + stop: Arc, + join: Option>, +} + +impl TestHttpServer { + pub fn start(routes: Vec) -> Result { + let listener = TcpListener::bind(("127.0.0.1", 0)) + .map_err(|error| format!("bind test HTTP server: {error}"))?; + listener + .set_nonblocking(true) + .map_err(|error| format!("configure test HTTP server: {error}"))?; + let address = listener + .local_addr() + .map_err(|error| format!("read test HTTP address: {error}"))?; + let requests = Arc::new(Mutex::new(Vec::new())); + let stop = Arc::new(AtomicBool::new(false)); + let thread_requests = Arc::clone(&requests); + let thread_stop = Arc::clone(&stop); + let join = std::thread::spawn(move || { + while !thread_stop.load(Ordering::Acquire) { + match listener.accept() { + Ok((stream, _)) => serve(stream, &routes, &thread_requests), + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(5)); + } + Err(_) => break, + } + } + }); + Ok(Self { + address, + requests, + stop, + join: Some(join), + }) + } + + pub fn uri(&self) -> String { + format!("http://{}", self.address) + } + + pub fn requests(&self) -> Vec { + self.requests.lock().expect("test HTTP requests").clone() + } +} + +impl Drop for TestHttpServer { + fn drop(&mut self) { + self.stop.store(true, Ordering::Release); + if let Some(join) = self.join.take() { + let _ = join.join(); + } + } +} + +fn serve(mut stream: TcpStream, routes: &[TestHttpRoute], requests: &Mutex>) { + let _ = stream.set_read_timeout(Some(Duration::from_secs(2))); + let Ok(request) = read_request(&mut stream) else { + return; + }; + requests + .lock() + .expect("test HTTP requests") + .push(request.clone()); + let route = routes.iter().find(|route| { + route.method == request.method + && request + .path + .split_once('?') + .map_or(request.path.as_str(), |(path, _)| path) + == route.path + }); + let (status, body) = route + .map(|route| (route.status, route.body.as_slice())) + .unwrap_or((404, b"{}")); + let reason = match status { + 200 => "OK", + 201 => "Created", + 204 => "No Content", + 400 => "Bad Request", + 404 => "Not Found", + _ => "Response", + }; + let header = format!( + "HTTP/1.1 {status} {reason}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + let _ = stream.write_all(header.as_bytes()); + let _ = stream.write_all(body); + let _ = stream.flush(); +} + +fn read_request(stream: &mut TcpStream) -> Result { + let mut bytes = Vec::new(); + let mut buffer = [0_u8; 4096]; + loop { + let read = stream + .read(&mut buffer) + .map_err(|error| format!("read test HTTP request: {error}"))?; + if read == 0 { + break; + } + bytes.extend_from_slice(&buffer[..read]); + if bytes.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + if bytes.len() > 64 * 1024 { + return Err("test HTTP request headers too large".to_owned()); + } + } + let line_end = bytes + .windows(2) + .position(|window| window == b"\r\n") + .ok_or_else(|| "test HTTP request has no request line".to_owned())?; + let line = std::str::from_utf8(&bytes[..line_end]) + .map_err(|error| format!("test HTTP request line is not UTF-8: {error}"))?; + let mut fields = line.split_whitespace(); + let method = fields + .next() + .ok_or_else(|| "test HTTP request has no method".to_owned())?; + let path = fields + .next() + .ok_or_else(|| "test HTTP request has no path".to_owned())?; + Ok(TestHttpRequest { + method: method.to_owned(), + path: path + .split_once('?') + .map_or(path, |(path, _)| path) + .to_owned(), + }) +} diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index fcf78e5..ac949c5 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -22,3 +22,6 @@ parking_lot = "0.12" [target.'cfg(target_os = "linux")'.dependencies] libc = "0.2" + +[dev-dependencies] +proptest = "1" diff --git a/xtask/proptest-regressions/demo/feed.txt b/xtask/proptest-regressions/demo/feed.txt new file mode 100644 index 0000000..dcc40e3 --- /dev/null +++ b/xtask/proptest-regressions/demo/feed.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 734889bb022ba3f70176b64e70fa84e73e172f999533fb15f3c72115080cf4e3 # shrinks to actions = [Spawn { attempt: 60000, succeeds: true }, Spawn { attempt: 60001, succeeds: false }, Started { attempt: 60000, pid: 41 }, Announce { attempt: 60000 }, Heartbeat { attempt: 60000 }, ProviderBlock, Control { kind: 0, attempt: 60000 }, ChildExit { attempt: 60000, code: 0 }, Shutdown, Shutdown] diff --git a/xtask/proptest-regressions/demo/node.txt b/xtask/proptest-regressions/demo/node.txt new file mode 100644 index 0000000..9833cf1 --- /dev/null +++ b/xtask/proptest-regressions/demo/node.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 42f7016ceaa65ddef7717389b08c508eb30e22e8ade6e9cf0749088a258d7e8b # shrinks to actions = [Announce, Heartbeat, Heartbeat, Heartbeat, Heartbeat, Heartbeat, Heartbeat, Heartbeat, Heartbeat, Heartbeat, Stop, Stop], attempt = 0 diff --git a/xtask/src/demo/control.rs b/xtask/src/demo/control.rs index c68a5a5..50c967f 100644 --- a/xtask/src/demo/control.rs +++ b/xtask/src/demo/control.rs @@ -1,39 +1,217 @@ //! Control plumbing: dashboard control commands → supervisor actor. -//! -//! The dashboard (under `demo-control`) dispatches into a std mpsc channel; -//! a blocking-pool task forwards each command as a `SupervisorMsg::Control` -//! message to the supervisor actor. -use std::sync::mpsc as std_mpsc; +use std::sync::{Arc, OnceLock}; -use swactor::runtime::ExternalSender; -use swactor_engine::EngineHandle; +use swactor::actor::{ActorAddress, ActorInterface}; +use swactor::runtime::{Ctx, Runtime}; use crate::demo::feed::SupervisorMsg; -/// Wire the dashboard control channel to the supervisor actor. -pub fn install( - engine: &EngineHandle, - sender: ExternalSender, - supervisor: std::sync::Arc>, -) { - let (control_tx, control_rx) = std_mpsc::channel::(); - dashboard::control::set_control_sender(control_tx); - - let engine = engine.clone(); - // A std-mpsc recv blocks its thread, so this forwarder must live on the - // engine's blocking pool — as an async task it would park one of the - // (two) Tokio workers indefinitely and starve the reconciler ticks. - engine.spawn_blocking(move || { - loop { - match control_rx.recv() { - Ok(command) => { - if let Some(addr) = supervisor.get() { - let _ = sender.send_to(addr.clone(), SupervisorMsg::Control(command)); - } - } - Err(_) => return, - } - } - }); +struct ControlForwarder { + supervisor: Arc>, +} + +impl ActorInterface for ControlForwarder { + type Incoming = dashboard::control::ControlCommand; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, command: Self::Incoming) { + if let Some(supervisor) = self.supervisor.get() { + let _ = ctx.send(*supervisor, SupervisorMsg::Control(command)); + } + } +} + +/// Wire the dashboard control channel to a typed actor relay. +pub fn install(runtime: &Runtime, supervisor: Arc>) { + let actor = runtime + .spawn(ControlForwarder { supervisor }) + .expect("spawn dashboard control forwarder actor"); + dashboard::control::install_actor_sink(runtime.create_sender(), actor); +} + +#[cfg(test)] +mod properties { + use proptest::prelude::*; + use swactor::config::RuntimeConfig; + use swactor::runtime::RuntimeParts; + use swactor_engine::{Engine, SteppingBackend}; + + use super::*; + + fn command(code: u8, index: usize) -> dashboard::control::ControlCommand { + let command_id = format!("command-{index}"); + match code % 4 { + 0 => dashboard::control::ControlCommand::Kill { + command_id, + node: format!("node-{code}"), + }, + 1 => dashboard::control::ControlCommand::Provision { + command_id, + count: u32::from(code), + }, + 2 => dashboard::control::ControlCommand::Remove { + command_id, + count: u32::from(code), + }, + _ => dashboard::control::ControlCommand::EstablishEdge { + command_id, + node: format!("node-{code}"), + }, + } + } + + fn drive(backend: &SteppingBackend) { + for _ in 0..16 { + backend.step(); + } + } + + fn check_control_invariants( + received: &[String], + expected: &[String], + active_actors: usize, + final_actors: usize, + worker_panics: u64, + ) -> Result<(), String> { + if received != expected { + return Err(format!( + "forwarded command mismatch: received={received:?} expected={expected:?}" + )); + } + if active_actors != 1 { + return Err(format!( + "one control route must own one actor: observed={active_actors}" + )); + } + if final_actors != 0 { + return Err(format!( + "control actors did not return to baseline: observed={final_actors}" + )); + } + if worker_panics != 0 { + return Err(format!("control worker panicked {worker_panics} time(s)")); + } + Ok(()) + } + + proptest! { + #![proptest_config(ProptestConfig { + cases: 128, + max_shrink_iters: 2_000, + ..ProptestConfig::default() + })] + + #[test] + fn generated_control_commands_forward_only_after_supervisor_registration( + codes in prop::collection::vec(any::(), 0..=32), + registration in 0_usize..=32, + ) { + let mut config = RuntimeConfig::default(); + config.worker_count = 1; + let parts = RuntimeParts::new(config); + let runtime = parts.runtime().clone(); + let backend = SteppingBackend::new(); + let _engine = + Engine::new(parts, backend.clone()).expect("demo control stepping engine"); + let supervisor = runtime + .new_inbox::() + .expect("create demo supervisor inbox"); + let supervisor_slot = Arc::new(OnceLock::new()); + let forwarder = runtime + .spawn(ControlForwarder { + supervisor: Arc::clone(&supervisor_slot), + }) + .expect("spawn demo control forwarder"); + let registration = registration.min(codes.len()); + let active_actors = runtime.stats().actors.len(); + prop_assert_eq!( + active_actors, + 1, + "control identity created the wrong actor set; codes={:?} \ + registration={} census={:?}", + codes, + registration, + runtime.stats() + ); + + for (index, code) in codes.iter().take(registration).enumerate() { + runtime + .send_to(forwarder, command(*code, index)) + .expect("send pre-registration control command"); + } + drive(&backend); + prop_assert!( + supervisor.try_recv().is_none(), + "control command crossed an uninitialized supervisor route; \ + codes={:?} registration={} census={:?}", + codes, + registration, + runtime.stats() + ); + + supervisor_slot + .set(*supervisor.addr()) + .expect("register supervisor address"); + for (index, code) in codes.iter().enumerate().skip(registration) { + runtime + .send_to(forwarder, command(*code, index)) + .expect("send registered control command"); + } + drive(&backend); + + let mut received_ids = Vec::new(); + while let Some(message) = supervisor.try_recv() { + if let SupervisorMsg::Control(command) = message { + received_ids.push(command.command_id().to_owned()); + } + } + let expected_ids = (registration..codes.len()) + .map(|index| format!("command-{index}")) + .collect::>(); + + runtime + .stop_actor(forwarder) + .expect("stop demo control forwarder"); + drive(&backend); + let stats = runtime.stats(); + let worker_panics = stats + .workers + .iter() + .map(|worker| worker.panics) + .sum::(); + prop_assert!( + check_control_invariants( + &received_ids, + &expected_ids, + active_actors, + stats.actors.len(), + worker_panics, + ) + .is_ok(), + "demo control invariant failed; codes={:?} registration={} \ + received={:?} expected={:?} census={:?}", + codes, + registration, + received_ids, + expected_ids, + stats + ); + } + } + + #[test] + fn control_transition_oracle_rejects_duplicate_forwarding() { + let rejected = check_control_invariants( + &["command-0".to_owned(), "command-0".to_owned()], + &["command-0".to_owned()], + 1, + 0, + 0, + ); + assert!( + rejected.is_err(), + "control property oracle accepted a controlled duplicate forwarding defect" + ); + } } diff --git a/xtask/src/demo/docker.rs b/xtask/src/demo/docker.rs index 258692c..7cd268c 100644 --- a/xtask/src/demo/docker.rs +++ b/xtask/src/demo/docker.rs @@ -91,17 +91,16 @@ impl DockerProcessLogic { } self.removed = true; let name = container_name(self.spec.attempt); - std::thread::spawn(move || { - let status = std::process::Command::new("docker") - .args(["rm", "-f", "--", &name]) - .stdin(std::process::Stdio::null()) - .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()) - .status(); - if let Err(error) = status { - eprintln!("demo: docker rm -f {name} failed: {error}"); - } - }); + let mut command = std::process::Command::new("docker"); + command + .args(["rm", "-f", "--", &name]) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()); + swactor_process::spawn_detached_command_status( + command, + format!("demo: docker rm -f {name} failed"), + ); } } @@ -361,19 +360,20 @@ fn build_image(root: &Path, image: &str) -> Result<(), String> { } fn run_cargo_build(root: &Path) -> Result<(), String> { - let output = std::process::Command::new("cargo") - .args([ - "build", - "--release", - "--target", - IMAGE_TARGET, - "--package", - "xtask", - ]) - .current_dir(root) - .stdin(std::process::Stdio::null()) - .output() - .map_err(|e| format!("run cargo: {e}"))?; + let output = swactor_process::command_output( + std::process::Command::new("cargo") + .args([ + "build", + "--release", + "--target", + IMAGE_TARGET, + "--package", + "xtask", + ]) + .current_dir(root) + .stdin(std::process::Stdio::null()), + ) + .map_err(|e| format!("run cargo: {e}"))?; if !output.status.success() { let mut stderr = String::from_utf8_lossy(&output.stderr).into_owned(); if stderr.len() > 4000 { @@ -414,11 +414,12 @@ fn docker_output(args: &[&str], label: &str) -> Result { } fn docker_raw(args: &[&str]) -> Result { - std::process::Command::new("docker") - .args(args) - .stdin(std::process::Stdio::null()) - .output() - .map_err(|e| format!("run docker {args:?}: {e}")) + swactor_process::command_output( + std::process::Command::new("docker") + .args(args) + .stdin(std::process::Stdio::null()), + ) + .map_err(|e| format!("run docker {args:?}: {e}")) } fn unix_ms() -> u64 { diff --git a/xtask/src/demo/edge.rs b/xtask/src/demo/edge.rs index a69071f..3bb0138 100644 --- a/xtask/src/demo/edge.rs +++ b/xtask/src/demo/edge.rs @@ -27,6 +27,9 @@ use data_plane::edge_wire::EdgeTransport; use data_plane::ids::{EdgeId, RunId}; use data_plane::object_record::{self, ObjectRecord}; use iroh::EndpointAddr; +use swactor::actor::{ActorAddress, ActorInterface}; +use swactor::runtime::{Ctx, ExternalSender, Runtime}; +use swactor_engine::EngineHandle; use telemetry::TelemetryProducer; /// Wire tag of the edge-provision gossip frame (supervisor → node). @@ -121,20 +124,8 @@ impl EdgeTransport for DriverTransport { edge_id: EdgeId, peer: &EndpointAddr, ) -> Result { - // The driver's send pump blocks its calling thread until the - // connect handshake completes — indefinitely for a dead peer. - // Bound it: run the spawn on a helper thread and give up after - // EDGE_CONNECT_TIMEOUT. (This runs on the edge pump thread, but a - // EDGE_CONNECT_TIMEOUT. (This runs on the edge pump thread, but a - // dead node must not wedge edge polling for the whole cluster.) - let driver = Arc::clone(&self.0); - let peer = peer.clone(); - let (tx, rx) = std::sync::mpsc::channel(); - std::thread::spawn(move || { - let _ = tx.send(driver.spawn_edge_send_pump(peer, edge_id.0)); - }); - rx.recv_timeout(EDGE_CONNECT_TIMEOUT) - .map_err(|error| format!("edge {} connect handshake: {error}", edge_id.0))? + self.0 + .spawn_edge_send_pump_timeout(peer.clone(), edge_id.0, EDGE_CONNECT_TIMEOUT) } fn drain_events(&mut self) -> Vec { @@ -283,21 +274,21 @@ impl EdgeSession { } } -/// Command from the supervisor actor into the edge pump thread. The pump -/// thread solely owns the sessions: every mutation travels through this -/// channel, so the actor never touches pump-owned state (and the pump's -/// blocking connect handshakes can never stall the actor). pub enum EdgePumpCmd { - /// Adopt a freshly created session (replaces any session for the same - /// attempt). Establish(Box), - /// A node acked its inbound edge. Ack(EdgeAck), - /// The current set of live node attempts; sessions for other attempts - /// are torn down. LiveAttempts(Vec), - /// Tear everything down (supervisor shutdown). DropAll, + Tick, +} + +#[derive(Clone)] +pub struct EdgePumpMessage(Arc>>); + +impl EdgePumpMessage { + pub fn new(command: EdgePumpCmd) -> Self { + Self(Arc::new(std::sync::Mutex::new(Some(command)))) + } } /// One pump→actor update: feed lines plus the full edge-state mirror for @@ -308,73 +299,103 @@ pub struct EdgePumpUpdate { pub states: Vec, } -/// Start the edge pump on the engine's blocking pool. Every tick: apply -/// pending commands, retry unacked provisions, poll each session's -/// runtime (the blocking connect handshakes belong on this thread — see -/// `control.rs` for the same constraint), and report an update to the -/// supervisor actor. -pub fn start_edge_pump( - engine: &swactor_engine::EngineHandle, +struct EdgePumpActor { + engine: EngineHandle, + sender: ExternalSender, driver: Arc, - cmds: std::sync::mpsc::Receiver, - sender: swactor::runtime::ExternalSender, - supervisor: Arc>, -) { - let engine = engine.clone(); - engine.spawn_blocking(move || { - let mut sessions: Vec = Vec::new(); - let mut next_poll = std::time::Instant::now(); - loop { - // Drain commands (blocking with the remaining tick budget). - loop { - let timeout = next_poll.saturating_duration_since(std::time::Instant::now()); - match cmds.recv_timeout(timeout) { - Ok(EdgePumpCmd::Establish(session)) => { - let attempt = session.attempt; - sessions.retain(|existing| existing.attempt != attempt); - sessions.push(*session); - } - Ok(EdgePumpCmd::Ack(ack)) => { - if let Some(session) = sessions - .iter_mut() - .find(|session| session.edge_id.0 == ack.edge_id) - { - session.acked = Some(ack.outcome.clone()); - } - } - Ok(EdgePumpCmd::LiveAttempts(live)) => { - let mut i = 0; - while i < sessions.len() { - if live.contains(&sessions[i].attempt) { - i += 1; - } else { - let dead = sessions.remove(i); - // Dropping the session drops its writer, - // finishing the edge stream. - eprintln!("demo: edge {} torn down (node gone)", dead.edge_id.0); - } - } - } - Ok(EdgePumpCmd::DropAll) => sessions.clear(), - Err(std::sync::mpsc::RecvTimeoutError::Timeout) => break, - Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => return, + supervisor: Arc>, + sessions: Vec, +} + +impl EdgePumpActor { + fn schedule(&self, ctx: &Ctx) { + self.engine.send_after( + Duration::from_millis(250), + self.sender.clone(), + ctx.self_addr(), + EdgePumpMessage::new(EdgePumpCmd::Tick), + ); + } +} + +impl ActorInterface for EdgePumpActor { + type Incoming = EdgePumpMessage; + type Response = (); + + fn on_start(&mut self, ctx: &Ctx) { + self.schedule(ctx); + } + + fn handle(&mut self, ctx: &Ctx, message: Self::Incoming) { + let Some(command) = message + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + else { + return; + }; + match command { + EdgePumpCmd::Establish(session) => { + let attempt = session.attempt; + self.sessions.retain(|existing| existing.attempt != attempt); + self.sessions.push(*session); + } + EdgePumpCmd::Ack(ack) => { + if let Some(session) = self + .sessions + .iter_mut() + .find(|session| session.edge_id.0 == ack.edge_id) + { + session.acked = Some(ack.outcome); } } - next_poll = std::time::Instant::now() + Duration::from_millis(250); - - let mut update = EdgePumpUpdate::default(); - for session in sessions.iter_mut() { - pump_one(session, &driver, &mut update); + EdgePumpCmd::LiveAttempts(live) => { + let mut index = 0; + while index < self.sessions.len() { + if live.contains(&self.sessions[index].attempt) { + index += 1; + } else { + let dead = self.sessions.remove(index); + eprintln!("demo: edge {} torn down (node gone)", dead.edge_id.0); + } + } } - update.states = sessions.iter().map(session_state_json).collect(); - if let Some(addr) = supervisor.get() { - let _ = sender.send_to( - addr.clone(), - crate::demo::feed::SupervisorMsg::EdgeUpdate(update), - ); + EdgePumpCmd::DropAll => self.sessions.clear(), + EdgePumpCmd::Tick => { + let mut update = EdgePumpUpdate::default(); + for session in &mut self.sessions { + pump_one(session, &self.driver, &mut update); + } + update.states = self.sessions.iter().map(session_state_json).collect(); + if let Some(supervisor) = self.supervisor.get() { + let _ = self.sender.send_to( + *supervisor, + crate::demo::feed::SupervisorMsg::EdgeUpdate(update), + ); + } + self.schedule(ctx); } } - }); + } +} + +pub fn start_edge_pump( + runtime: &Runtime, + engine: EngineHandle, + driver: Arc, + sender: ExternalSender, + supervisor: Arc>, +) -> Result { + runtime + .spawn(EdgePumpActor { + engine, + sender, + driver, + supervisor, + sessions: Vec::new(), + }) + .map_err(|error| format!("spawn edge pump actor: {error}")) } fn session_state_json(session: &EdgeSession) -> serde_json::Value { diff --git a/xtask/src/demo/feed.rs b/xtask/src/demo/feed.rs index 3faf843..4005b96 100644 --- a/xtask/src/demo/feed.rs +++ b/xtask/src/demo/feed.rs @@ -22,12 +22,12 @@ use provisioning::reconciler::{ ClusterShape, NodeObservation, OperationOutcome, PlannedEffect, RetryPolicy, }; use serde_json::json; -use swactor::actor::{ActorInterface, Ctx}; -use swactor_engine::EngineHandle; +use swactor::actor::{ActorAddress, ActorInterface, Ctx}; +use swactor_engine::{ActorCompletion, BlockingWorkSender, EngineHandle}; use telemetry::{ChannelContent, StreamDescriptor, TelemetryEndpoint, TelemetryProducer}; use crate::demo::edge; -use crate::demo::edge::{EdgeAck, EdgePumpCmd, EdgeSession}; +use crate::demo::edge::{EdgeAck, EdgePumpCmd, EdgePumpMessage, EdgeSession}; use crate::demo::provider::{ DemoBackend, NodeManager, NodeTelemetry, register_node_channels, unix_ms, }; @@ -79,24 +79,26 @@ impl SupervisorTelemetry { } } -/// Engine-backed spawner for executor blocking work. #[derive(Clone)] pub struct EngineSpawner { - engine: EngineHandle, + blocking: BlockingWorkSender, } impl EngineSpawner { - pub fn new(engine: EngineHandle) -> Self { - Self { engine } + pub fn new(engine: &EngineHandle) -> Self { + Self { + blocking: engine.blocking_work_sender(), + } } } impl BlockingEffectSpawner for EngineSpawner { - type SpawnError = std::convert::Infallible; + type SpawnError = String; fn spawn_blocking(&self, work: BlockingEffectWork) -> Result<(), Self::SpawnError> { - self.engine.spawn_blocking(move || work()); - Ok(()) + self.blocking + .submit(work) + .map_err(|_| "engine stopped before demo effect submission".to_owned()) } } @@ -147,7 +149,7 @@ pub enum SupervisorMsg { /// State + feed update from the edge pump thread (sole session owner). EdgeUpdate(crate::demo::edge::EdgePumpUpdate), Shutdown { - drained: std::sync::Arc, + completion: ActorCompletion<()>, }, } @@ -167,6 +169,12 @@ struct NodeStreams { status_channel: telemetry::ChannelId, } +struct ShutdownState { + completion: ActorCompletion<()>, + ticks: u32, + settled_ticks: u32, +} + /// The provisioning supervisor actor. pub struct SupervisorActor { pub driver: ClusterDriver, @@ -199,11 +207,12 @@ pub struct SupervisorActor { pub dashboard: dashboard::DashboardHandle, status_tick: u64, launch: crate::demo::LaunchStyle, - /// Command channel into the edge pump thread (sole session owner). - edge_cmd: std::sync::mpsc::Sender, + /// Actor that owns and advances all edge sessions. + edge_actor: ActorAddress, /// Last reported edge-state mirror from the pump (snapshot data). edge_states: Vec, next_edge_id: u64, + shutdown: Option, } impl SupervisorActor { @@ -223,7 +232,7 @@ impl SupervisorActor { initial_slots: Vec, run_id: RunId, launch: crate::demo::LaunchStyle, - edge_cmd: std::sync::mpsc::Sender, + edge_actor: ActorAddress, ) -> Self { let events_channel = telemetry.register("prov.reconciler.events"); let snapshot_channel = telemetry.register("prov.reconciler.snapshot"); @@ -251,12 +260,48 @@ impl SupervisorActor { dashboard, status_tick: 0, launch, - edge_cmd, + edge_actor, edge_states: Vec::new(), next_edge_id: 0, + shutdown: None, } } + fn schedule_tick(&self, ctx: &Ctx) { + self.engine.send_after( + crate::demo::TICK, + self.sender.clone(), + ctx.self_addr(), + SupervisorMsg::Tick, + ); + } + + fn advance_shutdown(&mut self, ctx: &Ctx) -> bool { + let converged = self.driver.is_converged(); + let Some(shutdown) = self.shutdown.as_mut() else { + return false; + }; + shutdown.ticks = shutdown.ticks.saturating_add(1); + if converged { + shutdown.settled_ticks = shutdown.settled_ticks.saturating_add(1); + } else { + shutdown.settled_ticks = 0; + } + if shutdown.settled_ticks < 8 && shutdown.ticks < 80 { + return false; + } + let shutdown = self + .shutdown + .take() + .expect("shutdown state was present while completing"); + assert!( + shutdown.completion.complete(()).is_ok(), + "demo supervisor completed shutdown twice" + ); + ctx.stop_self(); + true + } + fn desired_shape(&self, generation: u64) -> ClusterShape { ClusterShape { run_id: self.run_id.clone(), @@ -356,7 +401,13 @@ impl SupervisorActor { let logic = match self.registry.create(&spec) { Ok(logic) => logic, Err(error) => { - let _ = request.reply.send(Err(format!("bootstrap logic: {error}"))); + assert!( + request + .reply + .complete(Err(format!("bootstrap logic: {error}"))) + .is_ok(), + "spawn request completed twice" + ); return; } }; @@ -379,12 +430,19 @@ impl SupervisorActor { last_announce_ms: None, endpoint_addr: None, }; - let _ = request.reply.send(Ok(runtime)); + assert!( + request.reply.complete(Ok(runtime)).is_ok(), + "spawn request completed twice" + ); } Err(error) => { - let _ = request - .reply - .send(Err(format!("spawn bootstrap actor: {error}"))); + assert!( + request + .reply + .complete(Err(format!("spawn bootstrap actor: {error}"))) + .is_ok(), + "spawn request completed twice" + ); } } } @@ -936,6 +994,10 @@ impl ActorInterface for SupervisorActor { type Incoming = SupervisorMsg; type Response = (); + fn on_start(&mut self, ctx: &Ctx) { + self.schedule_tick(ctx); + } + fn handle(&mut self, ctx: &Ctx, msg: SupervisorMsg) { match msg { SupervisorMsg::Tick => { @@ -950,6 +1012,9 @@ impl ActorInterface for SupervisorActor { self.emit_feed(now); self.flush_telemetry(); self.flush_remote_streams(); + if !self.advance_shutdown(ctx) { + self.schedule_tick(ctx); + } } SupervisorMsg::Control(command) => self.handle_control(command), SupervisorMsg::Spawn(request) => self.spawn_node(ctx, request), @@ -964,7 +1029,9 @@ impl ActorInterface for SupervisorActor { attempt, } => self.register_remote_stream(header, logical_node, attempt), SupervisorMsg::EdgeAck(ack) => { - let _ = self.edge_cmd.send(EdgePumpCmd::Ack(ack)); + let _ = self + .sender + .send_to(self.edge_actor, EdgePumpMessage::new(EdgePumpCmd::Ack(ack))); } SupervisorMsg::EdgeUpdate(update) => { self.edge_states = update.states; @@ -972,15 +1039,23 @@ impl ActorInterface for SupervisorActor { self.emit_event("edge", &node, detail); } } - SupervisorMsg::Shutdown { drained } => { - self.slots.clear(); - let generation = self.driver.desired().generation.saturating_add(1); - if let Err(error) = self.driver.update_desired(self.desired_shape(generation)) { - eprintln!("demo: shutdown update_desired failed: {error}"); + SupervisorMsg::Shutdown { completion } => { + if self.shutdown.is_none() { + self.slots.clear(); + let generation = self.driver.desired().generation.saturating_add(1); + if let Err(error) = self.driver.update_desired(self.desired_shape(generation)) { + eprintln!("demo: shutdown update_desired failed: {error}"); + } + let _ = self + .sender + .send_to(self.edge_actor, EdgePumpMessage::new(EdgePumpCmd::DropAll)); + self.emit_event("control", "", "shutdown: desired → empty".to_owned()); + self.shutdown = Some(ShutdownState { + completion, + ticks: 0, + settled_ticks: 0, + }); } - let _ = self.edge_cmd.send(EdgePumpCmd::DropAll); - self.emit_event("control", "", "shutdown: desired → empty".to_owned()); - drained.store(true, std::sync::atomic::Ordering::SeqCst); } } } @@ -1100,8 +1175,11 @@ impl SupervisorActor { ); } if self - .edge_cmd - .send(EdgePumpCmd::Establish(Box::new(session))) + .sender + .send_to( + self.edge_actor, + EdgePumpMessage::new(EdgePumpCmd::Establish(Box::new(session))), + ) .is_err() { self.emit_event("edge", node, "edge: pump gone".to_owned()); @@ -1134,7 +1212,10 @@ impl SupervisorActor { .filter(|runtime| runtime.exited.is_none()) .map(|runtime| runtime.attempt) .collect(); - let _ = self.edge_cmd.send(EdgePumpCmd::LiveAttempts(live)); + let _ = self.sender.send_to( + self.edge_actor, + EdgePumpMessage::new(EdgePumpCmd::LiveAttempts(live)), + ); } } @@ -1197,3 +1278,776 @@ pub fn demo_retry_policy() -> RetryPolicy { endpoint_probe_interval: Duration::from_secs(1), } } + +#[cfg(test)] +mod properties { + use std::collections::{BTreeMap, BTreeSet}; + use std::sync::{Arc, Mutex, mpsc}; + use std::time::{Duration, SystemTime}; + + use proptest::prelude::*; + use provisioning::bootstrap::{ + BootstrapLogic, LogicProbe, NodeIdentity, NodeLaunchSpec, NodeTelemetryCollector, + }; + use provisioning::plugin::{ + NodeProvisionSpec, PluginObservation, PluginObservationSink, PluginSink, ProvisionPlugin, + }; + use swactor::config::RuntimeConfig; + use swactor::runtime::RuntimeParts; + use swactor_engine::{ActorCompletion, Engine, SteppingBackend}; + use swactor_process::{ExitStatus, ProcessOutput}; + + use super::*; + use crate::demo::provider::{ + AnnounceActor, DemoProvider, NodeRelayActor, NodeRuntime, SpawnNodeRequest, + }; + + #[derive(Clone, Copy, Debug)] + enum SupervisorAction { + Spawn { attempt: u16, succeeds: bool }, + Started { attempt: u16, pid: u16 }, + Announce { attempt: u16 }, + Heartbeat { attempt: u16 }, + ChildExit { attempt: u16, code: u8 }, + Control { kind: u8, attempt: u16 }, + ProviderBlock, + Shutdown, + } + + fn supervisor_actions() -> impl Strategy> { + let generated = prop::collection::vec( + prop_oneof![ + 4 => (0_u16..=3, any::()).prop_map(|(attempt, succeeds)| { + SupervisorAction::Spawn { attempt, succeeds } + }), + 2 => (0_u16..=3, any::()) + .prop_map(|(attempt, pid)| SupervisorAction::Started { attempt, pid }), + 3 => (0_u16..=3) + .prop_map(|attempt| SupervisorAction::Announce { attempt }), + 3 => (0_u16..=3) + .prop_map(|attempt| SupervisorAction::Heartbeat { attempt }), + 2 => (0_u16..=3, any::()) + .prop_map(|(attempt, code)| SupervisorAction::ChildExit { attempt, code }), + 3 => (any::(), 0_u16..=3) + .prop_map(|(kind, attempt)| SupervisorAction::Control { kind, attempt }), + ], + 0..=22, + ); + generated.prop_map(|mut actions| { + actions.extend([ + SupervisorAction::Spawn { + attempt: 60_000, + succeeds: true, + }, + SupervisorAction::Spawn { + attempt: 60_001, + succeeds: false, + }, + SupervisorAction::Started { + attempt: 60_000, + pid: 41, + }, + SupervisorAction::Announce { attempt: 60_000 }, + SupervisorAction::Heartbeat { attempt: 60_000 }, + SupervisorAction::ProviderBlock, + SupervisorAction::Control { + kind: 0, + attempt: 60_000, + }, + SupervisorAction::ChildExit { + attempt: 60_000, + code: 0, + }, + SupervisorAction::Shutdown, + SupervisorAction::Shutdown, + ]); + actions + }) + } + + #[derive(Clone, Default)] + struct Evidence { + starts: Arc>>, + terminations: Arc>>, + relays: Arc>>, + collections: Arc>>, + } + + struct GeneratedLogic { + spec: NodeLaunchSpec, + manager: NodeManager, + evidence: Evidence, + } + + impl BootstrapLogic for GeneratedLogic { + fn start( + &mut self, + ctx: &Ctx, + owner: ActorAddress, + _sender: &swactor::runtime::ExternalSender, + ) -> Result<(), String> { + let relay = ctx + .spawn(NodeRelayActor::new(self.manager.clone(), self.spec.attempt)) + .map_err(|error| format!("spawn generated process relay: {error}"))?; + let replaced = self + .evidence + .relays + .lock() + .expect("generated relay evidence") + .insert(self.spec.attempt, relay); + if replaced.is_some() { + return Err(format!( + "duplicate actor set for attempt {}", + self.spec.attempt + )); + } + *self + .evidence + .starts + .lock() + .expect("generated start evidence") + .entry(self.spec.attempt) + .or_default() += 1; + self.manager.register(NodeRuntime { + attempt: self.spec.attempt, + logical_node: self.spec.logical_node.clone(), + bootstrap: owner, + pid: None, + exited: None, + spawn_failed: None, + last_announce_ms: None, + endpoint_addr: None, + }); + Ok(()) + } + + fn probe(&mut self, _now: SystemTime) -> LogicProbe { + let Some(runtime) = self.manager.get(self.spec.attempt) else { + return LogicProbe::Exited("generated runtime removed".to_owned()); + }; + if let Some(reason) = runtime.spawn_failed { + return LogicProbe::Failed(reason); + } + if let Some(status) = runtime.exited { + return LogicProbe::Exited(format!("{status:?}")); + } + LogicProbe::Pending + } + + fn terminate( + &mut self, + _sender: &swactor::runtime::ExternalSender, + _kill_after: Option, + ) { + *self + .evidence + .terminations + .lock() + .expect("generated termination evidence") + .entry(self.spec.attempt) + .or_default() += 1; + } + } + + impl NodeTelemetryCollector for Evidence { + fn collect(&self, identity: &NodeIdentity) { + *self + .collections + .lock() + .expect("generated collection evidence") + .entry(identity.attempt) + .or_default() += 1; + } + } + + fn drive(backend: &SteppingBackend, steps: usize) { + for _ in 0..steps { + backend.step(); + } + } + + fn null_sink() -> PluginSink { + struct NullSink; + impl PluginObservationSink for NullSink { + fn observe(&self, _observation: PluginObservation) {} + } + PluginSink::new(Arc::new(NullSink)) + } + + fn provision_spec(attempt: u64) -> NodeProvisionSpec { + NodeProvisionSpec { + run_id: 1, + node_id: attempt, + attempt_id: attempt, + stage_index: None, + image: "generated-demo-node".to_owned(), + env: vec![("DEMO_LOGICAL_NODE".to_owned(), format!("node-{attempt}"))], + args: Vec::new(), + mounts: Vec::new(), + } + } + + fn control_command(kind: u8, attempt: u64, index: usize) -> dashboard::control::ControlCommand { + let command_id = format!("generated-command-{index}"); + if kind % 2 == 0 { + dashboard::control::ControlCommand::Kill { + command_id, + node: format!("node-{attempt}"), + } + } else { + dashboard::control::ControlCommand::EstablishEdge { + command_id, + node: format!("node-{attempt}"), + } + } + } + + fn check_supervisor_invariants( + identities: &[u64], + starts: &BTreeMap, + relays: &BTreeMap, + collections: &BTreeMap, + replies: &[(u64, bool)], + final_actors: usize, + worker_panics: u64, + ) -> Result<(), String> { + let unique = identities.iter().copied().collect::>(); + if unique.len() != identities.len() { + return Err(format!("duplicate manager identity: {identities:?}")); + } + for attempt in &unique { + if starts.get(attempt) != Some(&1) { + return Err(format!( + "attempt {attempt} started {:?} times", + starts.get(attempt) + )); + } + if !relays.contains_key(attempt) { + return Err(format!("attempt {attempt} has no process relay actor")); + } + if collections.get(attempt).copied().unwrap_or(0) > 1 { + return Err(format!( + "attempt {attempt} collected telemetry more than once" + )); + } + } + if collections.get(&60_000) != Some(&1) { + return Err(format!( + "announce/heartbeat did not collect canonical identity exactly once: {:?}", + collections.get(&60_000) + )); + } + if !replies.iter().any(|(_, success)| *success) + || !replies.iter().any(|(_, success)| !*success) + { + return Err(format!( + "spawn replies did not cover success and failure: {replies:?}" + )); + } + if final_actors != 0 { + return Err(format!( + "supervisor resources did not return to baseline: {final_actors}" + )); + } + if worker_panics != 0 { + return Err(format!( + "supervisor worker panicked {worker_panics} time(s)" + )); + } + Ok(()) + } + + proptest! { + #![proptest_config(ProptestConfig { + cases: 128, + max_shrink_iters: 2_000, + ..ProptestConfig::default() + })] + + #[test] + fn generated_supervisor_transitions_are_once_only_nonblocking_and_clean( + actions in supervisor_actions(), + ) { + let mut config = RuntimeConfig::default(); + config.worker_count = 1; + let parts = RuntimeParts::new(config); + let runtime = parts.runtime().clone(); + let sender = runtime.create_sender(); + let backend = SteppingBackend::new(); + let engine = + Engine::new(parts, backend.clone()).expect("one-worker supervisor stepping engine"); + let manager = NodeManager::new(); + let evidence = Evidence::default(); + let outcomes = Arc::new(Mutex::new(BTreeMap::::new())); + + let mut registry = provisioning::BootstrapRegistry::new(); + registry.register("process", { + let manager = manager.clone(); + let evidence = evidence.clone(); + let outcomes = Arc::clone(&outcomes); + Arc::new(move |spec| { + if !outcomes + .lock() + .expect("generated provider outcomes") + .get(&spec.attempt) + .copied() + .unwrap_or(true) + { + return Err(format!("generated provider failure {}", spec.attempt)); + } + Ok(Box::new(GeneratedLogic { + spec: spec.clone(), + manager: manager.clone(), + evidence: evidence.clone(), + }) as Box) + }) + }); + let telemetry = SupervisorTelemetry::new("generated-supervisor"); + let fanout = Arc::new(telemetry::DeliveryFanout::new(32)); + let remote_sub = fanout.subscribe_all( + "generated-dashboard", + telemetry::TelemetrySnapshot { + streams: Vec::new(), + channels: Vec::new(), + }, + ); + let io_driver = crate::demo::shared_test_driver(); + let driver_handle = Arc::new(crate::demo::DemoDriverHandle { + supervisor_addr_json: serde_json::to_string(&io_driver.endpoint_addr()) + .expect("serialize generated supervisor endpoint"), + driver: io_driver, + }); + let shape = ClusterShape { + run_id: RunId(1), + generation: 1, + groups: Vec::new(), + }; + let driver = + ClusterDriver::new(shape, demo_retry_policy()).expect("generated cluster driver"); + let plugin = Arc::new(Mutex::new(DemoProvider::new(manager.clone()))); + let executor = IdempotentEffectExecutor::new( + DemoBackend { + plugin: Arc::clone(&plugin), + manager: manager.clone(), + sender: sender.clone(), + }, + EngineSpawner::new(&engine.handle()), + ); + let dashboard = dashboard::DashboardHandle::new(dashboard::DashboardConfig { + port: 0, + ..dashboard::DashboardConfig::default() + }); + let edge_inbox = runtime + .new_inbox::() + .expect("create generated edge inbox"); + let announce = runtime + .spawn(AnnounceActor::new(manager.clone(), sender.clone())) + .expect("spawn generated announce actor"); + let supervisor = runtime + .spawn(SupervisorActor::new( + driver, + executor, + manager.clone(), + driver_handle, + telemetry, + dashboard, + sender.clone(), + registry, + Arc::new(evidence.clone()), + engine.handle(), + remote_sub, + Vec::new(), + RunId(1), + crate::demo::LaunchStyle::Process { + exe: "generated-demo-node".into(), + }, + *edge_inbox.addr(), + )) + .expect("spawn generated supervisor actor"); + drive(&backend, 8); + + let mut replies = Vec::new(); + let mut shutdown = None; + let mut provider_gate: Option> = None; + for (index, action) in actions.iter().enumerate() { + match *action { + SupervisorAction::Spawn { attempt, succeeds } => { + let attempt = u64::from(attempt); + if manager.get(attempt).is_some() { + let actors_before = runtime.stats().actors.len(); + let handle = plugin + .lock() + .expect("generated demo provider") + .create_node(provision_spec(attempt), null_sink()) + .expect("adopt generated demo node"); + prop_assert_eq!( + handle.id, + attempt, + "adopted wrong identity; actions={:?}", + actions + ); + prop_assert_eq!( + runtime.stats().actors.len(), + actors_before, + "provider adoption duplicated actor resources; actions={:?} \ + attempt={} census={:?}", + actions, + attempt, + runtime.stats() + ); + continue; + } + outcomes + .lock() + .expect("generated provider outcomes") + .insert(attempt, succeeds); + let actors_before = runtime.stats().actors.len(); + let reply = ActorCompletion::new(); + runtime + .send_to( + supervisor, + SupervisorMsg::Spawn(SpawnNodeRequest { + attempt, + logical_node: format!("node-{attempt}"), + reply: reply.clone(), + }), + ) + .expect("send generated spawn request"); + drive(&backend, 12); + prop_assert!( + reply.complete(Err("spawn completion probe".to_owned())).is_err(), + "spawn request did not complete within fixed budget; \ + actions={:?} attempt={} census={:?}", + actions, + attempt, + runtime.stats() + ); + let result = reply.wait(); + replies.push((attempt, result.is_ok())); + match result { + Ok(runtime_node) => { + prop_assert!( + succeeds, + "provider unexpectedly succeeded; actions={:?}", + actions + ); + runtime + .send_to(runtime_node.bootstrap, provisioning::BootstrapMsg::Start) + .expect("start generated bootstrap actor"); + drive(&backend, 12); + prop_assert!( + manager.get(attempt).is_some(), + "successful provider did not register identity; actions={:?} \ + attempt={} replies={:?}", + actions, + attempt, + replies + ); + prop_assert_eq!( + runtime.stats().actors.len(), + actors_before + 2, + "identity did not own exactly bootstrap+relay resources; \ + actions={:?} attempt={} census={:?}", + actions, + attempt, + runtime.stats() + ); + } + Err(_) => { + prop_assert!( + !succeeds, + "provider unexpectedly failed; actions={:?}", + actions + ); + prop_assert_eq!( + runtime.stats().actors.len(), + actors_before, + "failed provider leaked actor resources; actions={:?} \ + attempt={} census={:?}", + actions, + attempt, + runtime.stats() + ); + } + } + } + SupervisorAction::Started { attempt, pid } => { + if let Some(relay) = evidence + .relays + .lock() + .expect("generated relay evidence") + .get(&u64::from(attempt)) + .copied() + { + let _ = runtime.send_to( + relay, + ProcessOutput::Started { + pid: u32::from(pid) + 1, + }, + ); + drive(&backend, 4); + } + } + SupervisorAction::Announce { attempt } + | SupervisorAction::Heartbeat { attempt } => { + let attempt = u64::from(attempt); + let _ = runtime.send_to( + announce, + crate::demo::node::NodeAnnounce { + attempt, + logical_node: format!("node-{attempt}"), + key_hex: format!("{attempt:016x}"), + endpoint_addr_json: serde_json::to_string( + &crate::demo::shared_test_driver().endpoint_addr(), + ) + .expect("serialize generated announce endpoint"), + at_ms: index as u64 + 1, + }, + ); + drive(&backend, 8); + } + SupervisorAction::ChildExit { attempt, code } => { + let attempt = u64::from(attempt); + if let Some(relay) = evidence + .relays + .lock() + .expect("generated relay evidence") + .get(&attempt) + .copied() + { + let _ = runtime.send_to( + relay, + ProcessOutput::Exited { + status: ExitStatus::Code(i32::from(code)), + }, + ); + drive(&backend, 4); + } + if let Some(runtime_node) = manager.get(attempt) { + let _ = runtime.send_to( + runtime_node.bootstrap, + provisioning::BootstrapMsg::Probe, + ); + drive(&backend, 8); + } + } + SupervisorAction::Control { kind, attempt } => { + let attempt = u64::from(attempt); + let terminations_before = evidence + .terminations + .lock() + .expect("generated termination evidence") + .get(&attempt) + .copied() + .unwrap_or(0); + runtime + .send_to( + supervisor, + SupervisorMsg::Control(control_command(kind, attempt, index)), + ) + .expect("send generated dashboard control"); + drive(&backend, 8); + if let Some(release) = provider_gate.take() { + let terminations_after = evidence + .terminations + .lock() + .expect("generated termination evidence") + .get(&attempt) + .copied() + .unwrap_or(0); + prop_assert!( + terminations_after > terminations_before, + "blocking provider work prevented supervisor control progress; \ + actions={:?} attempt={} terminations={:?} census={:?}", + actions, + attempt, + evidence.terminations, + runtime.stats() + ); + release + .send(()) + .expect("release generated blocking provider work"); + } + } + SupervisorAction::ProviderBlock => { + let (entered_tx, entered_rx) = mpsc::sync_channel(1); + let (release_tx, release_rx) = mpsc::sync_channel(1); + EngineSpawner::new(&engine.handle()) + .spawn_blocking(Box::new(move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + })) + .expect("spawn generated blocking provider work"); + entered_rx + .recv_timeout(Duration::from_secs(2)) + .expect("blocking provider work did not enter within hard budget"); + provider_gate = Some(release_tx); + } + SupervisorAction::Shutdown => { + let completion = shutdown.get_or_insert_with(ActorCompletion::new); + runtime + .send_to( + supervisor, + SupervisorMsg::Shutdown { + completion: completion.clone(), + }, + ) + .expect("send generated supervisor shutdown"); + drive(&backend, 4); + } + } + } + prop_assert!( + provider_gate.is_none(), + "provider gate was not released; actions={:?}", + actions + ); + + let canonical = manager + .get(60_000) + .expect("canonical generated identity remains registered through shutdown"); + prop_assert_eq!( + canonical.pid, + Some(42), + "provider Started report was lost; actions={:?} replies={:?} \ + census={:?}", + actions, + replies, + runtime.stats() + ); + prop_assert_eq!( + canonical.exited.as_ref(), + Some(&ExitStatus::Code(0)), + "child exit report was lost; actions={:?} replies={:?} census={:?}", + actions, + replies, + runtime.stats() + ); + prop_assert!( + canonical.last_announce_ms.is_some() && canonical.endpoint_addr.is_some(), + "announce/heartbeat facts were lost; actions={:?} replies={:?} \ + last_announce={:?} endpoint={:?} census={:?}", + actions, + replies, + canonical.last_announce_ms, + canonical.endpoint_addr, + runtime.stats() + ); + + let identities = manager + .nodes() + .into_iter() + .map(|node| node.attempt) + .collect::>(); + let starts = evidence + .starts + .lock() + .expect("generated start evidence") + .clone(); + let relays = evidence + .relays + .lock() + .expect("generated relay evidence") + .clone(); + let collections = evidence + .collections + .lock() + .expect("generated collection evidence") + .clone(); + + for runtime_node in manager.nodes() { + manager.set_exited(runtime_node.attempt, ExitStatus::Code(0)); + let _ = runtime.send_to( + runtime_node.bootstrap, + provisioning::BootstrapMsg::Stop { + kill_after: Some(Duration::ZERO), + }, + ); + let _ = runtime.send_to( + runtime_node.bootstrap, + provisioning::BootstrapMsg::Probe, + ); + manager.remove(runtime_node.attempt); + } + for relay in relays.values().copied() { + runtime + .stop_actor(relay) + .expect("stop generated process relay"); + } + runtime + .stop_actor(announce) + .expect("stop generated announce actor"); + drive(&backend, 32); + + for _ in 0..12 { + backend.advance_time(crate::demo::TICK); + drive(&backend, 16); + } + let shutdown = shutdown.expect("bounded action suffix installs shutdown"); + prop_assert!( + shutdown.complete(()).is_err(), + "supervisor shutdown did not complete within fixed budget; \ + actions={:?} identities={:?} replies={:?} census={:?}", + actions, + identities, + replies, + runtime.stats() + ); + shutdown.wait(); + prop_assert!( + shutdown.complete(()).is_err(), + "supervisor shutdown completion accepted a second terminal value; actions={:?}", + actions + ); + + let final_stats = runtime.stats(); + let worker_panics = final_stats + .workers + .iter() + .map(|worker| worker.panics) + .sum::(); + prop_assert!( + check_supervisor_invariants( + &identities, + &starts, + &relays, + &collections, + &replies, + final_stats.actors.len(), + worker_panics, + ) + .is_ok(), + "supervisor invariant failed; actions={:?} identities={:?} \ + starts={:?} relays={:?} collections={:?} \ + replies={:?} census={:?}", + actions, + identities, + starts, + relays, + collections, + replies, + final_stats + ); + } + } + + #[test] + fn supervisor_transition_oracle_rejects_duplicate_identity_resources() { + let starts = BTreeMap::from([(7, 2)]); + let relays = BTreeMap::from([(7, ActorAddress::default())]); + let collections = BTreeMap::from([(60_000, 1)]); + let rejected = check_supervisor_invariants( + &[7], + &starts, + &relays, + &collections, + &[(7, true), (8, false)], + 0, + 0, + ); + assert!( + rejected.is_err(), + "supervisor property oracle accepted a controlled duplicate resource set" + ); + } +} diff --git a/xtask/src/demo/mod.rs b/xtask/src/demo/mod.rs index e379397..49a3ed4 100644 --- a/xtask/src/demo/mod.rs +++ b/xtask/src/demo/mod.rs @@ -38,9 +38,10 @@ use std::sync::Arc; use std::time::Duration; use iroh::RelayMode; +use swactor::actor::{ActorAddress, ActorInterface, Ctx}; use swactor::config::RuntimeConfig; -use swactor::runtime::RuntimeParts; -use swactor_engine::{Engine, TokioBackend, TokioConfig}; +use swactor::runtime::{ExternalSender, Runtime, RuntimeParts}; +use swactor_engine::{ActorCompletion, Engine, TokioBackend, TokioConfig}; use distribution::node::DistributedNodeConfig; use iroh_driver::{IrohDriver, IrohDriverConfig}; @@ -63,6 +64,36 @@ pub const DEFAULT_PORT: u16 = 9871; /// Default initial cluster size. pub const DEFAULT_NODES: u64 = 3; +#[cfg(test)] +pub(super) fn shared_test_driver() -> Arc { + thread_local! { + static DRIVER: (Arc, Engine) = { + let mut config = RuntimeConfig::default(); + config.worker_count = 1; + let parts = RuntimeParts::new(config); + let backend = TokioBackend::new(TokioConfig::default()) + .expect("create shared demo test backend"); + let engine = + Engine::new(parts, backend).expect("create shared demo test I/O engine"); + let driver = Arc::new( + IrohDriver::with_engine( + engine.handle(), + IrohDriverConfig { + secret_key: None, + relay_mode: RelayMode::Disabled, + node: DistributedNodeConfig::default(), + peer_auth: None, + additional_alpns: vec![], + }, + ) + .expect("create shared demo test driver"), + ); + (driver, engine) + }; + } + DRIVER.with(|(driver, _)| Arc::clone(driver)) +} + /// Resolve the executable path for node-role re-execs. `current_exe()` /// returns a "(deleted)" path or fails once the binary file has been /// replaced by a rebuild while this process runs, so fall back through @@ -116,10 +147,11 @@ impl DemoDriverHandle { /// the supervisor actor so frames can be classified before publishing. struct DemoTelemetryCollector { engine: swactor_engine::EngineHandle, + runtime: Runtime, endpoint: iroh::Endpoint, fanout: Arc, sender: swactor::runtime::ExternalSender, - supervisor_slot: Arc>, + supervisor_slot: Arc>, } impl provisioning::NodeTelemetryCollector for DemoTelemetryCollector { @@ -138,8 +170,24 @@ impl provisioning::NodeTelemetryCollector for DemoTelemetryCollector { ); let mut flow_id = [0u8; 16]; flow_id[..8].copy_from_slice(&identity.attempt.to_le_bytes()); - let (header_tx, header_rx) = std::sync::mpsc::channel(); - iroh_driver::spawn_pull_collector( + let Some(supervisor) = self.supervisor_slot.get().copied() else { + return; + }; + let logical_node = identity.logical_node.clone(); + let attempt = identity.attempt; + let header_actor = match self.runtime.spawn(PullHeaderActor { + sender: self.sender.clone(), + supervisor, + logical_node, + attempt, + }) { + Ok(actor) => actor, + Err(error) => { + eprintln!("demo: cannot spawn telemetry header actor: {error}"); + return; + } + }; + iroh_driver::spawn_pull_collector_to_actor( &self.engine, self.endpoint.clone(), addr, @@ -147,26 +195,32 @@ impl provisioning::NodeTelemetryCollector for DemoTelemetryCollector { Vec::new(), telemetry::SubscriptionRequest::all(), Arc::clone(&self.fanout), - header_tx, + self.sender.clone(), + header_actor, + ); + } +} + +struct PullHeaderActor { + sender: swactor::runtime::ExternalSender, + supervisor: ActorAddress, + logical_node: String, + attempt: u64, +} + +impl ActorInterface for PullHeaderActor { + type Incoming = iroh_driver::TelemetryQuicHeader; + type Response = (); + + fn handle(&mut self, _ctx: &Ctx, header: Self::Incoming) { + let _ = self.sender.send_to( + self.supervisor, + feed::SupervisorMsg::NodeStream { + header, + logical_node: self.logical_node.clone(), + attempt: self.attempt, + }, ); - let sender = self.sender.clone(); - if let Some(supervisor) = self.supervisor_slot.get() { - let supervisor = supervisor.clone(); - let logical_node = identity.logical_node.clone(); - let attempt = identity.attempt; - std::thread::spawn(move || { - if let Ok(header) = header_rx.recv() { - let _ = sender.send_to( - supervisor, - feed::SupervisorMsg::NodeStream { - header, - logical_node, - attempt, - }, - ); - } - }); - } } } @@ -212,11 +266,9 @@ fn run_supervisor(args: &[String]) -> Result<(), String> { .unwrap_or(DEFAULT_NODES); let docker_mode = args.iter().any(|arg| arg == "--docker"); - // Node registry + spawn channel: needed before the actor bridge so the - // announce relay can route wire announces into bootstrap actors. + // Node registry: wire announces and provisioning effects converge through + // the supervisor actor once it is installed below. let manager = NodeManager::new(); - let (spawn_tx, spawn_rx) = std::sync::mpsc::channel::(); - manager.set_spawn_channel(spawn_tx); // Telemetry endpoint with the runtime stats hook attached before the // engine takes the parts. @@ -318,7 +370,7 @@ fn run_supervisor(args: &[String]) -> Result<(), String> { ..dashboard::DashboardConfig::default() }); dashboard.register_view(Arc::new(view::ReconcilerDashboardView::default())); - engine.handle().spawn(dashboard.http_server()); + dashboard.spawn(&engine.handle()); // Provisioning: driver + plugin + executor. let launch = if docker_mode { @@ -349,7 +401,7 @@ fn run_supervisor(args: &[String]) -> Result<(), String> { ClusterDriver::new(shape, demo_retry_policy()).map_err(|e| format!("driver: {e}"))?; let plugin = DemoProvider::new(manager.clone()); - let spawner = EngineSpawner::new(engine.handle()); + let spawner = EngineSpawner::new(&engine.handle()); let executor = IdempotentEffectExecutor::new( DemoBackend { plugin: Arc::new(std::sync::Mutex::new(plugin)), @@ -392,17 +444,24 @@ fn run_supervisor(args: &[String]) -> Result<(), String> { let collector: Arc = Arc::new(DemoTelemetryCollector { engine: engine.handle(), + runtime: runtime.clone(), endpoint: driver_handle.endpoint(), fanout: Arc::clone(&fanout), sender: sender.clone(), supervisor_slot: supervisor_slot.clone(), }); - let (edge_cmd_tx, edge_cmd_rx) = std::sync::mpsc::channel::(); + let edge_actor = edge::start_edge_pump( + &runtime, + engine.handle(), + edge_driver, + sender.clone(), + supervisor_slot.clone(), + )?; let supervisor = SupervisorActor::new( cluster_driver, executor, - manager, + manager.clone(), driver_handle, telemetry, dashboard.clone(), @@ -414,42 +473,11 @@ fn run_supervisor(args: &[String]) -> Result<(), String> { slots, RunId(1), launch.clone(), - edge_cmd_tx, + edge_actor, ); // Control plane: dashboard → supervisor. - control::install(&engine.handle(), sender.clone(), supervisor_slot.clone()); - - // Spawn-request pump: provider blocking threads → supervisor actor. - let pump_sender = sender.clone(); - let pump_slot = supervisor_slot.clone(); - engine.handle().spawn_blocking(move || { - loop { - match spawn_rx.recv() { - Ok(request) => { - if let Some(addr) = pump_slot.get() { - let _ = pump_sender.send_to(addr.clone(), SupervisorMsg::Spawn(request)); - } - } - Err(_) => return, - } - } - }); - - // Tick: engine interval → supervisor actor. - let tick_sender = sender.clone(); - let tick_engine = engine.handle(); - let interval_engine = tick_engine.clone(); - let tick_slot = supervisor_slot.clone(); - tick_engine.spawn(async move { - let mut interval = interval_engine.interval(TICK); - loop { - (&mut interval).await; - if let Some(addr) = tick_slot.get() { - let _ = tick_sender.send_to(addr.clone(), SupervisorMsg::Tick); - } - } - }); + control::install(&runtime, supervisor_slot.clone()); let supervisor_addr = runtime .spawn(supervisor) @@ -457,85 +485,220 @@ fn run_supervisor(args: &[String]) -> Result<(), String> { supervisor_slot .set(supervisor_addr.clone()) .expect("supervisor address slot set once"); + manager.set_spawn_actor(sender.clone(), supervisor_slot.clone()); - // Edge pump on the blocking pool: it solely owns the edge sessions — - // edge runtime polls block their thread (connect handshakes), which - // must never run on a Tokio worker or hold a lock the actor needs. - edge::start_edge_pump( - &engine.handle(), - edge_driver, - edge_cmd_rx, - sender.clone(), - supervisor_slot.clone(), - ); - + let completion = ActorCompletion::new(); + let stop_actor = runtime + .spawn(SupervisorStopActor { + sender: sender.clone(), + supervisor: supervisor_addr, + completion: completion.clone(), + }) + .map_err(|error| format!("spawn supervisor stop actor: {error}"))?; + #[cfg(target_os = "linux")] + swactor_process::spawn_os_stop_signal_wait(runtime.create_sender(), stop_actor); println!("demo: dashboard on http://localhost:{port}"); println!(" /view/fleet — per-node cards (pid, lifecycle)"); println!(" /view/demo-control — Fleet Control: stages, feeds, kill / provision / edge"); println!(" Ctrl-C to tear down."); + completion.wait(); - // Block until Ctrl-C (synchronous signal flag — the wait must not depend - // on engine task progression). - install_sigint_flag(); - while !sigint_requested() { - std::thread::sleep(Duration::from_millis(100)); - } - - // Teardown: drain the cluster through the real destroy path (desired → - // empty → BeginDelete → DestroyLease → process actors stop children), - // pumping ticks until the driver quiesces. - let drained = Arc::new(std::sync::atomic::AtomicBool::new(false)); - let _ = sender.send_to( - supervisor_addr.clone(), - SupervisorMsg::Shutdown { - drained: drained.clone(), - }, - ); - let mut settle_ticks = 0_u32; - let deadline = std::time::Instant::now() + Duration::from_secs(20); - while std::time::Instant::now() < deadline { - if drained.load(std::sync::atomic::Ordering::SeqCst) { - // Give DestroyLease results a few extra ticks to land. - settle_ticks += 1; - if settle_ticks >= 8 { - break; - } - } - std::thread::sleep(TICK); - } - // Best-effort drain window closed: children still alive (if any) are - // killed by the kernel parent-death signal armed in the node role - // (process kind) or swept by label below (docker kind). if let LaunchStyle::Docker(docker) = &launch { docker::sweep_run(docker); } - std::process::exit(0); + Ok(()) } -static SIGINT_REQUESTED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); - -fn sigint_requested() -> bool { - SIGINT_REQUESTED.load(std::sync::atomic::Ordering::SeqCst) +struct SupervisorStopActor { + sender: ExternalSender, + supervisor: ActorAddress, + completion: ActorCompletion<()>, } -#[cfg(target_os = "linux")] -fn install_sigint_flag() { - unsafe { - let handler: extern "C" fn(libc::c_int) = sigint_handler; - libc::signal(libc::SIGINT, handler as usize); - // Supervisors under process managers (systemd, container runtimes, - // harnesses) stop children with SIGTERM; treat it exactly like - // Ctrl-C so the drain + sweep path runs instead of a default kill. - libc::signal(libc::SIGTERM, handler as usize); +impl ActorInterface for SupervisorStopActor { + type Incoming = swactor_process::ProcessStopSignal; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, _message: Self::Incoming) { + let _ = self.sender.send_to( + self.supervisor, + SupervisorMsg::Shutdown { + completion: self.completion.clone(), + }, + ); + ctx.stop_self(); } } -#[cfg(target_os = "linux")] -extern "C" fn sigint_handler(_signal: libc::c_int) { - SIGINT_REQUESTED.store(true, std::sync::atomic::Ordering::SeqCst); -} +#[cfg(all(test, target_os = "linux"))] +mod properties { + use std::process::{Command, Stdio}; + use std::time::Duration; -#[cfg(not(target_os = "linux"))] -fn install_sigint_flag() { - // Non-Linux builds wait for SIGTERM's default disposition instead. + use swactor_process::ProcessExitObservation; + + use super::*; + + #[derive(Clone, Debug)] + enum ReadinessObservation { + Line(String), + Error(String), + Closed, + Timeout, + } + + struct ReadinessObserver { + pid: i32, + completion: ActorCompletion>, + } + + impl ReadinessObserver { + fn fail(&self, ctx: &Ctx, error: String) { + let _ = unsafe { libc::kill(self.pid, libc::SIGKILL) }; + let _ = self.completion.complete(Err(error)); + ctx.stop_self(); + } + } + + impl ActorInterface for ReadinessObserver { + type Incoming = ReadinessObservation; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, observation: Self::Incoming) { + match observation { + ReadinessObservation::Line(line) if line.contains("demo: dashboard on") => { + if unsafe { libc::kill(self.pid, libc::SIGTERM) } != 0 { + self.fail( + ctx, + format!( + "send SIGTERM to direct demo binary: {}", + std::io::Error::last_os_error() + ), + ); + } else { + ctx.stop_self(); + } + } + ReadinessObservation::Line(_) => {} + ReadinessObservation::Error(error) => { + self.fail(ctx, format!("read direct demo stdout: {error}")); + } + ReadinessObservation::Closed => { + self.fail(ctx, "direct demo stdout closed before readiness".to_owned()); + } + ReadinessObservation::Timeout => { + self.fail( + ctx, + "direct demo binary did not become ready within ten seconds".to_owned(), + ); + } + } + } + } + + struct ExitObserver { + pid: i32, + completion: ActorCompletion>, + } + + impl ActorInterface for ExitObserver { + type Incoming = ProcessExitObservation; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, observation: Self::Incoming) { + let result = match (observation.status, observation.error) { + (Some(0), None) => Ok(()), + (status, Some(error)) => { + let _ = unsafe { libc::kill(self.pid, libc::SIGKILL) }; + Err(format!( + "direct demo process observation failed: status={status:?}, error={error}" + )) + } + (status, None) => Err(format!( + "direct demo binary did not exit cleanly after SIGTERM: status={status:?}" + )), + }; + let _ = self.completion.complete(result); + ctx.stop_self(); + } + } + + #[test] + #[ignore = "subprocess entrypoint for direct_binary_signal_smoke_has_a_hard_timeout"] + fn direct_binary_signal_smoke_child() { + run_supervisor(&[ + "--nodes".to_owned(), + "0".to_owned(), + "--port".to_owned(), + "0".to_owned(), + ]) + .expect("empty demo supervisor exits after its OS stop signal"); + } + + #[test] + fn direct_binary_signal_smoke_has_a_hard_timeout() { + const CHILD_TEST: &str = "demo::properties::direct_binary_signal_smoke_child"; + const HARD_TIMEOUT: Duration = Duration::from_secs(10); + + let mut command = Command::new(std::env::current_exe().expect("current xtask test binary")); + command + .args(["--ignored", "--exact", CHILD_TEST, "--nocapture"]) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()); + let mut child = + swactor_process::command_spawn(&mut command).expect("spawn direct xtask test binary"); + let pid = child.id() as i32; + let stdout = child.stdout.take().expect("capture demo supervisor stdout"); + + let mut config = RuntimeConfig::default(); + config.worker_count = 1; + let parts = RuntimeParts::new(config); + let runtime = parts.runtime().clone(); + let engine = Engine::new( + parts, + TokioBackend::new(TokioConfig::default()).expect("create signal smoke backend"), + ) + .expect("create signal smoke engine"); + let sender = runtime.create_sender(); + let completion = ActorCompletion::new(); + let readiness = runtime + .spawn(ReadinessObserver { + pid, + completion: completion.clone(), + }) + .expect("spawn readiness observer"); + let exit = runtime + .spawn(ExitObserver { + pid, + completion: completion.clone(), + }) + .expect("spawn exit observer"); + + swactor_process::spawn_mapped_line_reader( + stdout, + sender.clone(), + readiness, + ReadinessObservation::Line, + ReadinessObservation::Error, + ReadinessObservation::Closed, + ); + swactor_process::spawn_child_wait(child, sender.clone(), exit); + let _readiness_timeout = engine.handle().send_after( + HARD_TIMEOUT, + sender.clone(), + readiness, + ReadinessObservation::Timeout, + ); + let _exit_timeout = engine.handle().send_after( + HARD_TIMEOUT, + sender, + exit, + ProcessExitObservation { + status: None, + error: Some("demo subprocess exceeded ten-second hard timeout".to_owned()), + }, + ); + + completion.wait().expect("direct demo signal smoke"); + } } diff --git a/xtask/src/demo/node.rs b/xtask/src/demo/node.rs index 4cbed78..1690f29 100644 --- a/xtask/src/demo/node.rs +++ b/xtask/src/demo/node.rs @@ -24,10 +24,10 @@ use std::time::Duration; use iroh::RelayMode; use serde_json::json; -use swactor::actor::{ActorInterface, Ctx}; +use swactor::actor::{ActorAddress, ActorInterface, Ctx}; use swactor::config::RuntimeConfig; -use swactor::runtime::RuntimeParts; -use swactor_engine::{Engine, TokioBackend, TokioConfig}; +use swactor::runtime::{ExternalSender, RuntimeParts}; +use swactor_engine::{ActorCompletion, Engine, EngineHandle, TokioBackend, TokioConfig}; use distribution::node::DistributedNodeConfig; use iroh_driver::{IrohDriver, IrohDriverConfig, TELEMETRY_ALPN, spawn_pull_server}; @@ -196,54 +196,15 @@ pub fn run_node_role(supervisor_addr_json: &str, attempt: u64) -> Result<(), Str // TELEMETRY_ALPN connections out of the driver-owned ingress so the // serve loop below can drain them. driver.retain_telemetry_connections(); - { - // Edge agent tick: poll the edge runtime on the supervisor's - // session cadence. - let sender = runtime.create_sender(); - let edge_engine = engine.handle(); - edge_engine.clone().spawn(async move { - let mut interval = edge_engine.interval(Duration::from_millis(250)); - loop { - (&mut interval).await; - let _ = sender.send_to(edge_agent, edge::NodeEdgeMsg::Tick); - } - }); - } // Join, then announce identity + advertised address to the supervisor's // bootstrap actor over the control plane (readiness + telemetry dial). driver.join(&[supervisor_addr.clone()]); let addr_json = serde_json::to_string(&driver.endpoint_addr()).map_err(|e| format!("addr: {e}"))?; - let announce_driver = Arc::clone(&driver); - let announce_logical = logical_node.clone(); - let announce_key = node_hex.clone(); - let announce_engine = engine.handle(); - announce_engine.clone().spawn(async move { - let mut interval = announce_engine.interval(HEARTBEAT_PERIOD); - loop { - (&mut interval).await; - let announce = NodeAnnounce { - attempt, - logical_node: announce_logical.clone(), - key_hex: announce_key.clone(), - endpoint_addr_json: addr_json.clone(), - at_ms: unix_ms(), - }; - let Ok(bytes) = serde_json::to_vec(&announce) else { - continue; - }; - announce_driver.send_tagged_gossip( - supervisor_addr.clone(), - ANNOUNCE_TAG.as_bytes(), - bytes, - ); - } - }); // Beat actors: real actors with real message flow, so the node's // runtime.actors roster (pulled by the supervisor) is visibly alive. - let sender = runtime.create_sender(); let mut beat_addrs = Vec::new(); for index in 0..BEAT_ACTORS { let name = format!("beat-{index}"); @@ -255,80 +216,183 @@ pub fn run_node_role(supervisor_addr_json: &str, attempt: u64) -> Result<(), Str channel: beats_channel, }) .map_err(|error| format!("spawn beat actor: {error}"))?; - beat_addrs.push((addr, name)); - } - - // Heartbeats: a node.status telemetry record (the wire announce above is - // the supervisor-facing liveness channel). - let heartbeat_producer = producer.clone(); - let heartbeat_logical = logical_node.clone(); - let heartbeat_key = node_hex.clone(); - let interval_engine = engine.handle(); - interval_engine.clone().spawn(async move { - let mut interval = interval_engine.interval(HEARTBEAT_PERIOD); - let mut seq: u64 = 0; - loop { - (&mut interval).await; - seq += 1; - let payload = json!({ - "at_ms": unix_ms(), - "node": heartbeat_logical, - "key": heartbeat_key, - "seq": seq, - "alive": true, - "pid": std::process::id(), - "beats": BEAT_ACTORS, - }); - let bytes = serde_json::to_vec(&payload).expect("status serializes"); - heartbeat_producer.submit_bytes(status_channel, bytes); - } - }); - - // Beat driver: engine intervals poking the beat actors. - for (index, (addr, _name)) in beat_addrs.iter().enumerate() { - let addr = *addr; - let beat_sender = sender.clone(); - let beat_engine = engine.handle(); - // Stagger the beat periods so roster entries tick at different rates. - let period = Duration::from_millis(1000 + 500 * index as u64); - beat_engine.clone().spawn(async move { - let mut interval = beat_engine.interval(period); - loop { - (&mut interval).await; - let _ = beat_sender.send_to(addr, BeatMsg::Wake); - } - }); + beat_addrs.push(addr); } // Serve telemetry pulls: accepted TELEMETRY_ALPN connections answer with // this endpoint's subscription stream. let telemetry_endpoint = Arc::new(endpoint); - let serve_engine = engine.handle(); - let serve_driver = Arc::clone(&driver); - serve_engine.clone().spawn(async move { - let mut interval = serve_engine.interval(PULL_POLL_PERIOD); - loop { - (&mut interval).await; - // Drain the mux into the endpoint fanout so producer frames - // reach the pull subscription; without this tick the mux fills - // and nothing is ever streamed. - let _ = telemetry_endpoint.tick(); - for (_node, conn) in serve_driver.drain_accepted_for_alpn(TELEMETRY_ALPN) { - spawn_pull_server( - &serve_engine, - conn, - Arc::clone(&telemetry_endpoint), - WRITER_IDLE, + let completion = ActorCompletion::new(); + let runtime_actor = NodeRuntimeActor { + engine: engine.handle(), + sender: runtime.create_sender(), + edge_agent, + driver: Arc::clone(&driver), + attempt, + supervisor_addr, + logical_node, + node_hex, + endpoint_addr_json: addr_json, + endpoint: Arc::clone(&telemetry_endpoint), + producer, + status_channel, + beats: beat_addrs + .into_iter() + .enumerate() + .map(|(index, actor)| (actor, Duration::from_millis(1000 + 500 * index as u64))) + .collect(), + heartbeat_seq: 0, + completion: completion.clone(), + }; + let runtime_actor = runtime + .spawn(runtime_actor) + .map_err(|error| format!("spawn node runtime actor: {error}"))?; + let stop_actor = runtime + .spawn(NodeStopForwarder { + sender: runtime.create_sender(), + runtime_actor, + }) + .map_err(|error| format!("spawn node stop actor: {error}"))?; + #[cfg(target_os = "linux")] + swactor_process::spawn_os_stop_signal_wait(runtime.create_sender(), stop_actor); + + // The actor owns lifecycle; the entrypoint waits only for its terminal signal. + completion.wait(); + Ok(()) +} + +#[derive(Clone, Debug)] +enum NodeRuntimeMsg { + EdgeTick, + Announce, + Heartbeat, + Beat(usize), + PullTelemetry, + Stop, +} + +struct NodeRuntimeActor { + engine: EngineHandle, + sender: ExternalSender, + attempt: u64, + edge_agent: ActorAddress, + driver: Arc, + supervisor_addr: iroh::EndpointAddr, + logical_node: String, + node_hex: String, + endpoint_addr_json: String, + endpoint: Arc, + producer: TelemetryProducer, + status_channel: telemetry::ChannelId, + beats: Vec<(ActorAddress, Duration)>, + heartbeat_seq: u64, + completion: ActorCompletion<()>, +} + +impl NodeRuntimeActor { + fn schedule(&self, ctx: &Ctx, delay: Duration, message: NodeRuntimeMsg) { + self.engine + .send_after(delay, self.sender.clone(), ctx.self_addr(), message); + } +} + +impl ActorInterface for NodeRuntimeActor { + type Incoming = NodeRuntimeMsg; + type Response = (); + + fn on_start(&mut self, ctx: &Ctx) { + self.schedule(ctx, Duration::from_millis(250), NodeRuntimeMsg::EdgeTick); + self.schedule(ctx, Duration::ZERO, NodeRuntimeMsg::Announce); + self.schedule(ctx, HEARTBEAT_PERIOD, NodeRuntimeMsg::Heartbeat); + self.schedule(ctx, PULL_POLL_PERIOD, NodeRuntimeMsg::PullTelemetry); + for (index, (_, period)) in self.beats.iter().enumerate() { + self.schedule(ctx, *period, NodeRuntimeMsg::Beat(index)); + } + } + + fn handle(&mut self, ctx: &Ctx, message: Self::Incoming) { + match message { + NodeRuntimeMsg::EdgeTick => { + let _ = self + .sender + .send_to(self.edge_agent, edge::NodeEdgeMsg::Tick); + self.schedule(ctx, Duration::from_millis(250), NodeRuntimeMsg::EdgeTick); + } + NodeRuntimeMsg::Announce => { + let announce = NodeAnnounce { + attempt: self.attempt, + logical_node: self.logical_node.clone(), + key_hex: self.node_hex.clone(), + endpoint_addr_json: self.endpoint_addr_json.clone(), + at_ms: unix_ms(), + }; + if let Ok(bytes) = serde_json::to_vec(&announce) { + self.driver.send_tagged_gossip( + self.supervisor_addr.clone(), + ANNOUNCE_TAG.as_bytes(), + bytes, + ); + } + self.schedule(ctx, HEARTBEAT_PERIOD, NodeRuntimeMsg::Announce); + } + NodeRuntimeMsg::Heartbeat => { + self.heartbeat_seq = self.heartbeat_seq.saturating_add(1); + let payload = json!({ + "at_ms": unix_ms(), + "node": self.logical_node, + "key": self.node_hex, + "seq": self.heartbeat_seq, + "alive": true, + "pid": std::process::id(), + "beats": BEAT_ACTORS, + }); + let bytes = serde_json::to_vec(&payload).expect("status serializes"); + self.producer.submit_bytes(self.status_channel, bytes); + self.schedule(ctx, HEARTBEAT_PERIOD, NodeRuntimeMsg::Heartbeat); + } + NodeRuntimeMsg::Beat(index) => { + if let Some((actor, period)) = self.beats.get(index).copied() { + let _ = self.sender.send_to(actor, BeatMsg::Wake); + self.schedule(ctx, period, NodeRuntimeMsg::Beat(index)); + } + } + NodeRuntimeMsg::PullTelemetry => { + let _ = self.endpoint.tick(); + for (_node, connection) in self.driver.drain_accepted_for_alpn(TELEMETRY_ALPN) { + spawn_pull_server( + &self.engine, + connection, + Arc::clone(&self.endpoint), + WRITER_IDLE, + ); + } + self.schedule(ctx, PULL_POLL_PERIOD, NodeRuntimeMsg::PullTelemetry); + } + NodeRuntimeMsg::Stop => { + assert!( + self.completion.complete(()).is_ok(), + "demo node completed twice" ); + ctx.stop_self(); } } - }); + } +} - // The engine owns progression; park this thread until killed. `driver` - // stays alive until process exit. - let _keep_driver = driver; - loop { - std::thread::park(); +struct NodeStopForwarder { + sender: ExternalSender, + runtime_actor: ActorAddress, +} + +impl ActorInterface for NodeStopForwarder { + type Incoming = swactor_process::ProcessStopSignal; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, _message: Self::Incoming) { + let _ = self + .sender + .send_to(self.runtime_actor, NodeRuntimeMsg::Stop); + ctx.stop_self(); } } @@ -397,3 +461,227 @@ fn install_parent_death_signal() -> Result<(), String> { // Non-Linux builds rely on the supervisor's graceful teardown path. Ok(()) } + +#[cfg(test)] +mod properties { + use proptest::prelude::*; + use swactor::config::RuntimeConfig; + use swactor::runtime::RuntimeParts; + use swactor_engine::{ActorCompletion, Engine, SteppingBackend}; + + use super::*; + + #[derive(Clone, Debug)] + enum NodeAction { + Announce, + Heartbeat, + EdgeTick, + PullTelemetry, + Stop, + } + + fn node_actions() -> impl Strategy> { + prop::collection::vec( + prop_oneof![ + 3 => Just(NodeAction::Announce), + 4 => Just(NodeAction::Heartbeat), + 2 => Just(NodeAction::EdgeTick), + 1 => Just(NodeAction::PullTelemetry), + ], + 0..=28, + ) + .prop_map(|mut generated| { + let mut actions = vec![NodeAction::Announce, NodeAction::Heartbeat]; + actions.append(&mut generated); + actions.extend([NodeAction::Stop, NodeAction::Stop]); + actions + }) + } + + fn drive(backend: &SteppingBackend, steps: usize) { + for _ in 0..steps { + backend.step(); + } + } + + fn check_node_invariants( + expected_heartbeats: usize, + observed_heartbeats: usize, + active_actors: usize, + final_actors: usize, + worker_panics: u64, + ) -> Result<(), String> { + if observed_heartbeats != expected_heartbeats { + return Err(format!( + "heartbeat mismatch: expected={expected_heartbeats} observed={observed_heartbeats}" + )); + } + if active_actors != 2 { + return Err(format!( + "one node identity must own runtime+stop actors: observed={active_actors}" + )); + } + if final_actors != 0 { + return Err(format!( + "node actors did not return to baseline: observed={final_actors}" + )); + } + if worker_panics != 0 { + return Err(format!( + "node actor worker panicked {worker_panics} time(s)" + )); + } + Ok(()) + } + + proptest! { + #![proptest_config(ProptestConfig { + cases: 128, + max_shrink_iters: 2_000, + ..ProptestConfig::default() + })] + + #[test] + fn generated_node_runtime_transitions_emit_heartbeats_and_stop_once( + actions in node_actions(), + attempt in any::(), + ) { + let mut config = RuntimeConfig::default(); + config.worker_count = 1; + let parts = RuntimeParts::new(config); + let runtime = parts.runtime().clone(); + let backend = SteppingBackend::new(); + let engine = + Engine::new(parts, backend.clone()).expect("one-worker node stepping engine"); + let driver = crate::demo::shared_test_driver(); + let supervisor_addr = driver.endpoint_addr(); + let endpoint = Arc::new(TelemetryEndpoint::with_descriptor( + telemetry::frame::StreamDescriptor { + stream: telemetry::frame::StreamId::new( + telemetry::frame::NodeId::new(format!("generated-node-{attempt}")), + telemetry::frame::Lifetime(1), + ), + label: Some("generated demo node".to_owned()), + origin: telemetry::frame::StreamOrigin::RemoteNode, + }, + 128, + 64, + )); + let producer = endpoint.producer(); + let status_channel = endpoint.register_channel( + "node.status", + ChannelContent::JsonRecord { + schema: Some("demo.node.status.v1".to_owned()), + }, + ); + let status_frames = endpoint.subscribe_all("generated-node-property"); + let edge_inbox = runtime + .new_inbox::() + .expect("create generated node edge inbox"); + let completion = ActorCompletion::new(); + let runtime_actor = runtime + .spawn(NodeRuntimeActor { + engine: engine.handle(), + sender: runtime.create_sender(), + attempt, + edge_agent: *edge_inbox.addr(), + driver: Arc::clone(&driver), + supervisor_addr, + logical_node: format!("node-{attempt}"), + node_hex: format!("{attempt:016x}"), + endpoint_addr_json: serde_json::to_string(&driver.endpoint_addr()) + .expect("serialize generated node endpoint"), + endpoint: Arc::clone(&endpoint), + producer, + status_channel, + beats: Vec::new(), + heartbeat_seq: 0, + completion: completion.clone(), + }) + .expect("spawn generated node runtime actor"); + let stop_actor = runtime + .spawn(NodeStopForwarder { + sender: runtime.create_sender(), + runtime_actor, + }) + .expect("spawn generated node stop actor"); + drive(&backend, 8); + let active_stats = runtime.stats(); + + let mut expected_heartbeats = 0; + for action in actions + .iter() + .take(actions.len().saturating_sub(2)) + { + let message = match action { + NodeAction::Announce => NodeRuntimeMsg::Announce, + NodeAction::Heartbeat => { + expected_heartbeats += 1; + NodeRuntimeMsg::Heartbeat + } + NodeAction::EdgeTick => NodeRuntimeMsg::EdgeTick, + NodeAction::PullTelemetry => NodeRuntimeMsg::PullTelemetry, + NodeAction::Stop => unreachable!("stop actions are the bounded suffix"), + }; + runtime + .send_to(runtime_actor, message) + .expect("send generated node action"); + drive(&backend, 4); + } + + runtime + .send_to(stop_actor, swactor_process::ProcessStopSignal) + .expect("send first generated OS stop signal"); + runtime + .send_to(stop_actor, swactor_process::ProcessStopSignal) + .expect("send repeated generated OS stop signal"); + drive(&backend, 32); + prop_assert!( + completion.complete(()).is_err(), + "node completion remained pending; actions={actions:?} active={active_stats:?}" + ); + completion.wait(); + + endpoint.tick(); + let observed_heartbeats = status_frames + .drain_available() + .into_iter() + .filter(|event| { + matches!( + event, + telemetry::frame::TelemetryEvent::Frame(delivery) + if delivery.channel.channel == status_channel + ) + }) + .count(); + let final_stats = runtime.stats(); + let worker_panics = final_stats + .workers + .iter() + .map(|worker| worker.panics) + .sum::(); + prop_assert!( + check_node_invariants( + expected_heartbeats, + observed_heartbeats, + active_stats.actors.len(), + final_stats.actors.len(), + worker_panics, + ) + .is_ok(), + "node transition invariant failed; actions={actions:?} attempt={attempt} \ + expected_heartbeats={expected_heartbeats} observed_heartbeats={observed_heartbeats} \ + active={active_stats:?} final={final_stats:?}" + ); + } + } + + #[test] + fn node_transition_oracle_rejects_duplicate_resources() { + let rejected = check_node_invariants(3, 3, 3, 0, 0); + assert!( + rejected.is_err(), + "node property oracle accepted a controlled duplicate actor resource" + ); + } +} diff --git a/xtask/src/demo/provider.rs b/xtask/src/demo/provider.rs index 012e5ac..a046927 100644 --- a/xtask/src/demo/provider.rs +++ b/xtask/src/demo/provider.rs @@ -23,6 +23,7 @@ use std::time::{Duration, SystemTime}; use swactor::actor::{ActorAddress, ActorInterface, Ctx}; use swactor::runtime::ExternalSender; +use swactor_engine::ActorCompletion; use swactor_process::{ExitStatus, ProcessOutput}; use provisioning::executor::{EffectBackend, EffectError}; @@ -34,9 +35,6 @@ use provisioning::plugin::{NodeProvisionSpec, PluginNodeHandle, PluginSink, Prov use provisioning::reconciler::{OperationId, OperationOutcome, PlannedEffect}; use telemetry::{ChannelContent, StreamDescriptor, TelemetryEndpoint, TelemetryProducer}; -/// How long a blocking plugin call waits for the supervisor actor. -pub const BACKEND_WAIT: Duration = Duration::from_secs(20); - /// One provisioned node's runtime facts. #[derive(Clone)] pub struct NodeRuntime { @@ -61,20 +59,26 @@ pub struct NodeRuntime { pub struct SpawnNodeRequest { pub attempt: u64, pub logical_node: String, - pub reply: std::sync::mpsc::Sender>, + pub reply: ActorCompletion>, } -/// Shared node registry + spawn queue: hub between executor blocking threads, -/// relay actors, and the supervisor actor. +/// Shared node registry and actor route for provisioning requests. #[derive(Clone, Default)] pub struct NodeManager { inner: Arc>, } +#[derive(Clone)] +struct SpawnRoute { + sender: ExternalSender, + supervisor: Arc>, +} + #[derive(Default)] struct NodeManagerInner { nodes: BTreeMap, - spawn_tx: Option>, + spawn_route: Option, + exit_waiters: BTreeMap>>, } impl NodeManager { @@ -82,30 +86,41 @@ impl NodeManager { Self::default() } - pub fn set_spawn_channel(&self, sender: std::sync::mpsc::Sender) { - self.inner.lock().expect("node manager").spawn_tx = Some(sender); + pub fn set_spawn_actor( + &self, + sender: ExternalSender, + supervisor: Arc>, + ) { + self.inner.lock().expect("node manager").spawn_route = + Some(SpawnRoute { sender, supervisor }); } pub fn request_spawn(&self, attempt: u64, logical_node: String) -> Result { - let (reply_tx, reply_rx) = std::sync::mpsc::channel(); - let request = SpawnNodeRequest { - attempt, - logical_node, - reply: reply_tx, - }; - { - let inner = self.inner.lock().expect("node manager"); - let sender = inner - .spawn_tx - .as_ref() - .ok_or_else(|| "supervisor spawn channel not installed".to_owned())?; - sender - .send(request) - .map_err(|_| "supervisor actor gone".to_owned())?; - } - reply_rx - .recv_timeout(BACKEND_WAIT) - .map_err(|_| "timed out waiting for node spawn".to_owned())? + let reply = ActorCompletion::new(); + let route = self + .inner + .lock() + .expect("node manager") + .spawn_route + .clone() + .ok_or_else(|| "supervisor actor route not installed".to_owned())?; + let supervisor = route + .supervisor + .get() + .copied() + .ok_or_else(|| "supervisor actor not ready".to_owned())?; + route + .sender + .send_to( + supervisor, + crate::demo::feed::SupervisorMsg::Spawn(SpawnNodeRequest { + attempt, + logical_node, + reply: reply.clone(), + }), + ) + .map_err(|_| "supervisor actor gone".to_owned())?; + reply.wait() } pub fn register(&self, runtime: NodeRuntime) { @@ -163,7 +178,41 @@ impl NodeManager { } pub fn set_exited(&self, attempt: u64, status: ExitStatus) { - self.update(attempt, |runtime| runtime.exited = Some(status)); + let waiters = { + let mut inner = self.inner.lock().expect("node manager"); + if let Some(runtime) = inner.nodes.get_mut(&attempt) { + runtime.exited = Some(status); + } + inner.exit_waiters.remove(&attempt).unwrap_or_default() + }; + for waiter in waiters { + let _ = waiter.complete(()); + } + } + + pub fn watch_exit(&self, attempt: u64) -> ActorCompletion<()> { + let completion = ActorCompletion::new(); + let already_exited = { + let mut inner = self.inner.lock().expect("node manager"); + if inner + .nodes + .get(&attempt) + .is_some_and(|runtime| runtime.exited.is_some()) + { + true + } else { + inner + .exit_waiters + .entry(attempt) + .or_default() + .push(completion.clone()); + false + } + }; + if already_exited { + let _ = completion.complete(()); + } + completion } pub fn set_spawn_failed(&self, attempt: u64, reason: String) { @@ -394,12 +443,6 @@ pub fn session_id_for(operation: OperationId) -> BootstrapSessionId { BootstrapSessionId(operation.attempt.0) } -fn runtime_exited(manager: &NodeManager, attempt: u64) -> bool { - manager - .get(attempt) - .is_some_and(|runtime| runtime.exited.is_some()) -} - /// The demo `EffectBackend`: routes effects through the plugin. pub struct DemoBackend { /// The plugin lives behind a mutex: `EffectBackend::execute` is `&self` @@ -496,22 +539,20 @@ fn stop_node_with_sender( handle: &PluginNodeHandle, sender: &ExternalSender, ) -> Result<(), String> { - if let Some(runtime) = manager.get(handle.id) { - if runtime.pid.is_some() && runtime.exited.is_none() { - let _ = sender.send_to( + if let Some(runtime) = manager.get(handle.id) + && runtime.pid.is_some() + && runtime.exited.is_none() + { + let exited = manager.watch_exit(handle.id); + sender + .send_to( runtime.bootstrap, provisioning::BootstrapMsg::Stop { kill_after: Some(Duration::from_secs(1)), }, - ); - } - let deadline = std::time::Instant::now() + BACKEND_WAIT; - while !runtime_exited(manager, handle.id) { - if std::time::Instant::now() > deadline { - break; - } - std::thread::sleep(Duration::from_millis(50)); - } + ) + .map_err(|_| "bootstrap actor stopped before process shutdown".to_owned())?; + exited.wait(); } plugin.lock().expect("demo provider").stop_node(handle) } @@ -540,3 +581,239 @@ pub fn unix_ms(now: SystemTime) -> u64 { .map(|duration| duration.as_millis() as u64) .unwrap_or(0) } + +#[cfg(test)] +mod properties { + use proptest::prelude::*; + use swactor::config::RuntimeConfig; + use swactor::runtime::RuntimeParts; + use swactor_engine::{Engine, SteppingBackend}; + + use super::*; + + #[derive(Clone, Debug)] + enum RelayAction { + Started(u16), + ExitedCode(u8), + ExitedSignal(u8), + SpawnFailed(u8), + Error, + } + + fn relay_actions() -> impl Strategy> { + prop::collection::vec( + prop_oneof![ + 3 => any::().prop_map(RelayAction::Started), + 2 => any::().prop_map(RelayAction::ExitedCode), + 2 => any::().prop_map(RelayAction::ExitedSignal), + 1 => any::().prop_map(RelayAction::SpawnFailed), + 1 => Just(RelayAction::Error), + ], + 0..=32, + ) + } + + fn output(action: &RelayAction) -> ProcessOutput { + match *action { + RelayAction::Started(pid) => ProcessOutput::Started { + pid: u32::from(pid) + 1, + }, + RelayAction::ExitedCode(code) => ProcessOutput::Exited { + status: ExitStatus::Code(i32::from(code)), + }, + RelayAction::ExitedSignal(signal) => ProcessOutput::Exited { + status: ExitStatus::Signal(i32::from(signal) + 1), + }, + RelayAction::SpawnFailed(code) => ProcessOutput::SpawnFailed { + error: format!("spawn-{code}"), + }, + RelayAction::Error => ProcessOutput::Error { + error: "scripted process error".to_owned(), + }, + } + } + + fn drive(backend: &SteppingBackend) { + for _ in 0..64 { + backend.step(); + } + } + + fn check_relay_invariants( + observed: &NodeRuntime, + expected_pid: Option, + expected_exit: &Option, + expected_spawn_failure: &Option, + final_actors: usize, + worker_panics: u64, + ) -> Result<(), String> { + if observed.pid != expected_pid { + return Err(format!( + "pid mismatch: observed={:?} expected={expected_pid:?}", + observed.pid + )); + } + if observed.exited.as_ref() != expected_exit.as_ref() { + return Err(format!( + "exit mismatch: observed={:?} expected={expected_exit:?}", + observed.exited + )); + } + if observed.spawn_failed.as_ref() != expected_spawn_failure.as_ref() { + return Err(format!( + "spawn failure mismatch: observed={:?} expected={expected_spawn_failure:?}", + observed.spawn_failed + )); + } + if final_actors != 0 { + return Err(format!( + "process relay did not return to baseline: observed={final_actors}" + )); + } + if worker_panics != 0 { + return Err(format!( + "process relay worker panicked {worker_panics} time(s)" + )); + } + Ok(()) + } + + proptest! { + #![proptest_config(ProptestConfig { + cases: 128, + max_shrink_iters: 2_000, + ..ProptestConfig::default() + })] + + #[test] + fn generated_process_reports_complete_exit_watchers_once_and_preserve_last_state( + mut actions in relay_actions(), + watcher_count in 0_usize..=8, + fallback_exit in any::(), + ) { + if !actions.iter().any(|action| { + matches!(action, RelayAction::ExitedCode(_) | RelayAction::ExitedSignal(_)) + }) { + let terminal = RelayAction::ExitedCode(fallback_exit); + if let Some(last) = actions.get_mut(31) { + *last = terminal; + } else { + actions.push(terminal); + } + } + let mut config = RuntimeConfig::default(); + config.worker_count = 1; + let parts = RuntimeParts::new(config); + let runtime = parts.runtime().clone(); + let backend = SteppingBackend::new(); + let _engine = + Engine::new(parts, backend.clone()).expect("demo process stepping engine"); + let manager = NodeManager::new(); + let attempt = 41; + manager.register(NodeRuntime { + attempt, + logical_node: "generated-node".to_owned(), + bootstrap: ActorAddress::default(), + pid: None, + exited: None, + spawn_failed: None, + last_announce_ms: None, + endpoint_addr: None, + }); + let relay = runtime + .spawn(NodeRelayActor::new(manager.clone(), attempt)) + .expect("spawn demo process relay"); + let watchers = (0..watcher_count) + .map(|_| manager.watch_exit(attempt)) + .collect::>(); + + let mut expected_pid = None; + let mut expected_exit = None; + let mut expected_spawn_failure = None; + for action in &actions { + match action { + RelayAction::Started(pid) => expected_pid = Some(u32::from(*pid) + 1), + RelayAction::ExitedCode(code) => { + expected_exit = Some(ExitStatus::Code(i32::from(*code))); + } + RelayAction::ExitedSignal(signal) => { + expected_exit = Some(ExitStatus::Signal(i32::from(*signal) + 1)); + } + RelayAction::SpawnFailed(code) => { + expected_spawn_failure = Some(format!("spawn-{code}")); + } + RelayAction::Error => {} + } + runtime + .send_to(relay, output(action)) + .expect("send demo process report"); + } + drive(&backend); + + for watcher in watchers { + prop_assert!( + watcher.complete(()).is_err(), + "exit watcher remained pending after fixed drive budget; actions={actions:?} \ + census={:?}", + runtime.stats() + ); + watcher.wait(); + } + let late_watcher = manager.watch_exit(attempt); + prop_assert!( + late_watcher.complete(()).is_err(), + "late exit watcher did not complete immediately; actions={actions:?} census={:?}", + runtime.stats() + ); + late_watcher.wait(); + let observed = manager.get(attempt).expect("registered demo node"); + + runtime.stop_actor(relay).expect("stop demo process relay"); + drive(&backend); + let stats = runtime.stats(); + let worker_panics = stats + .workers + .iter() + .map(|worker| worker.panics) + .sum::(); + prop_assert!( + check_relay_invariants( + &observed, + expected_pid, + &expected_exit, + &expected_spawn_failure, + stats.actors.len(), + worker_panics, + ) + .is_ok(), + "demo process relay invariant failed; actions={actions:?} \ + expected_pid={expected_pid:?} expected_exit={expected_exit:?} \ + expected_spawn_failure={expected_spawn_failure:?} observed_pid={:?} \ + observed_exit={:?} observed_spawn_failure={:?} census={stats:?}", + observed.pid, + observed.exited, + observed.spawn_failed, + ); + } + } + + #[test] + fn process_relay_oracle_rejects_lost_exit() { + let observed = NodeRuntime { + attempt: 7, + logical_node: "fault-sensitive-node".to_owned(), + bootstrap: ActorAddress::default(), + pid: Some(9), + exited: None, + spawn_failed: None, + last_announce_ms: None, + endpoint_addr: None, + }; + let rejected = + check_relay_invariants(&observed, Some(9), &Some(ExitStatus::Code(0)), &None, 0, 0); + assert!( + rejected.is_err(), + "provider property oracle accepted a controlled lost-exit defect" + ); + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 8932b89..400dd73 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -74,7 +74,7 @@ fn run_step(step: &TestStep) -> bool { println!(" cargo {}", step.args.join(" ")); println!(); - match Command::new(cargo_bin()).args(step.args).status() { + match swactor_process::command_status(Command::new(cargo_bin()).args(step.args)) { Ok(status) => status.success(), Err(error) => { eprintln!("Failed to execute cargo: {error}");