zacheryasc/swactor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Enforce actor-owned Myelin control flow

Browse source 
Add a repository-owned rustc wrapper that enforces execution ownership and dependency boundaries during ordinary Cargo commands, with compile-pass and compile-fail policy contracts.

Move scheduling, timers, provider polling, provisioning, recovery, supervision, and shutdown decisions behind engine and actor APIs. Add deterministic component properties, stateful Myelin lifecycle coverage, persisted regression cases, and the bounded CI workflow.

Tighten resource ownership by cancelling telemetry collectors, terminating reply observers, bounding dashboard projections, and releasing process file descriptors, child observers, and inode-verified Unix socket paths on every exit path.
This commit is contained in:
Zachery Aaron Shores-Chmielewski 2026-08-20 01:38:14 +04:00
parent 3389abfe33
commit 23c829a9ba
112 changed files with 20765 additions and 3166 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.cargo/config.toml
View file

@ -1,3 +1,6 @@
[build]
rustc-workspace-wrapper = "tools/actor-control-flow-lint/rustc-wrapper.py"
[alias]
xtask = "run --package xtask --"
myelin-chat = "run --package xtask -- myelin-chat"

119
.github/workflows/myelin-properties.yml vendored Normal file
View file

@ -0,0 +1,119 @@
name: Myelin properties
on:
pull_request:
schedule:
- cron: "17 3 * * *"
workflow_dispatch:
permissions:
contents: read
jobs:
component-properties:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install pinned Rust toolchain
run: rustup toolchain install nightly-2026-02-07 --profile minimal --component rustfmt --component rustc-dev --component rust-src --component llvm-tools-preview
- name: Check formatting
run: cargo fmt --all -- --check
- name: Check default workspace members
run: cargo check
- name: Check compiler-policy contracts
run: cargo test -p actor-control-flow-lint-tests --test contracts compiler_policy_contracts -- --exact
- name: Engine scheduling properties
run: |
cargo test -p swactor-engine --test engine_unit generated_actor_timers_and_completion_are_bounded -- --exact
cargo test -p swactor-engine --test engine_unit lifecycle_invariant_detects_injected_duplicate_completion -- --exact
cargo test -p swactor-engine --test engine_unit lifecycle_invariant_detects_injected_uncancelled_periodic_timer -- --exact
- name: Process adapter properties
run: |
cargo test -p swactor-process --lib operations::properties::generated_stream_observations_close_once_and_stay_closed -- --exact
cargo test -p swactor-process --lib operations::properties::generated_lifecycle_actions_make_stop_idempotent_and_exit_terminal -- --exact
cargo test -p swactor-process --lib operations::properties::generated_stop_notifications_are_delivered_at_most_once -- --exact
cargo test -p swactor-process --lib operations::properties::generated_stdin_commands_and_eof_notify_once -- --exact
cargo test -p swactor-process --lib operations::properties::property_invariants_reject_controlled_defects -- --exact
cargo test -p swactor-process --lib operations::properties::trivial_real_child_exit_has_a_hard_timeout -- --exact
- name: Myelin component properties (exclude job and reconciler tests)
run: |
cargo test -p myelin --lib orchestration::manual_control::tests::aggressive_random_event_stream_preserves_control_invariants -- --exact
cargo test -p myelin --lib orchestration::manual_control::tests::rental_free_end_to_end_sequences_converge -- --exact
cargo test -p myelin --lib orchestration::manual_control::tests::manual_actor_generated_public_actions_and_callbacks_are_bounded -- --exact
cargo test -p myelin --lib orchestration::manual_control::tests::fixed_helper_cardinality_invariant_detects_controlled_extra_spawn -- --exact
cargo test -p myelin --lib orchestration::manual_control::tests::callback_panic_reports_typed_failure_without_poisoning_work_actor -- --exact
cargo test -p myelin --lib orchestration::manual_control::tests::callback_panic_invariant_detects_controlled_unguarded_panic -- --exact
cargo test -p myelin --lib provisioning::tests::mock_vastai_handle_state_survives_random_create_and_stop_sequences -- --exact
cargo test -p myelin --lib provisioning::tests::docker_generated_attempt_lifecycles_are_idempotent_and_bounded -- --exact
cargo test -p myelin --lib provisioning::tests::docker_duplicate_resource_detector_rejects_controlled_fault -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::offer_status_classes_are_offers_or_typed_rejections -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::malformed_offer_bodies_are_typed_rejections -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::wrong_or_missing_offer_fields_are_typed_rejections -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::duplicate_offer_records_remain_explicit_values -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::provider_monitor_preserves_contract_identity_and_cardinality -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::provider_monitor_terminal_polling_stops_after_one_typed_outcome -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::provider_monitor_poll_stop_orderings_cease_polling -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::duplicate_terminal_detector_rejects_controlled_fault -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_output_lines_preserve_stream_and_protocol -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_malformed_protocol_is_data_not_poison -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_eof_orderings_stop_relay_and_actor -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_child_failures_have_typed_attempt_outcomes -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_timeout_is_typed_and_stops_polling -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_stop_orderings_emit_one_terminal_and_stop_all_actors -- --exact
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::runtime_actors_generated_transitions_complete_once_on_one_worker -- --exact
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::runtime_invariant_checker_rejects_duplicate_readiness_publication -- --exact
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::helper_wait_generated_terminal_sequences_complete_once_on_one_worker -- --exact
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::helper_invariant_checker_rejects_expected_output_after_terminal_error -- --exact
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::stage_fetch_generated_observations_complete_once_on_one_worker -- --exact
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::stage_invariant_checker_rejects_wrong_terminal_classification -- --exact
cargo test -p myelin --lib orchestration::app::serve_cluster_properties::serve_cluster_production_transitions_converge_once_without_growth -- --exact
cargo test -p myelin --lib orchestration::app::serve_cluster_properties::serve_cluster_lifecycle_invariants_reject_injected_duplicate_and_growth -- --exact
cargo test -p myelin --lib orchestration::control::properties::generated_http_bridge_sequences_terminate_without_control_actor_growth -- --exact
cargo test -p myelin --lib orchestration::control::properties::generated_duplicate_control_replies_deliver_first_once_and_remove_observer -- --exact
cargo test -p myelin --lib orchestration::control::properties::reply_observer_disappearance_returns_a_bounded_terminal_http_response -- --exact
cargo test -p myelin --lib orchestration::control::properties::http_bridge_invariant_rejects_a_controlled_duplicate_forward -- --exact
- name: Controlled E2E oracle contract
run: cargo test -p myelin --features test-support --test stateful_vastai e2e_oracle_rejects_controlled_lifecycle_faults -- --exact
- name: Dashboard bridge properties
run: |
cargo test -p dashboard --features demo-control --lib control::properties::generated_concurrent_bridge_commands_forward_once_and_shutdown -- --exact
cargo test -p dashboard --features demo-control --lib control::properties::bridge_invariant_rejects_a_controlled_duplicate_delivery -- --exact
cargo test -p dashboard --features demo-control --lib server::tests::generated_control_http_sequences_are_bounded_and_typed -- --exact
cargo test -p dashboard --features demo-control --lib server::tests::control_http_invariant_rejects_a_controlled_server_error -- --exact
- name: Demo actor properties
run: |
cargo test -p xtask --bin xtask demo::control::properties::generated_control_commands_forward_only_after_supervisor_registration -- --exact
cargo test -p xtask --bin xtask demo::control::properties::control_transition_oracle_rejects_duplicate_forwarding -- --exact
cargo test -p xtask --bin xtask demo::feed::properties::generated_supervisor_transitions_are_once_only_nonblocking_and_clean -- --exact
cargo test -p xtask --bin xtask demo::feed::properties::supervisor_transition_oracle_rejects_duplicate_identity_resources -- --exact
cargo test -p xtask --bin xtask demo::node::properties::generated_node_runtime_transitions_emit_heartbeats_and_stop_once -- --exact
cargo test -p xtask --bin xtask demo::node::properties::node_transition_oracle_rejects_duplicate_resources -- --exact
cargo test -p xtask --bin xtask demo::provider::properties::generated_process_reports_complete_exit_watchers_once_and_preserve_last_state -- --exact
cargo test -p xtask --bin xtask demo::provider::properties::process_relay_oracle_rejects_lost_exit -- --exact
cargo test -p xtask --bin xtask demo::properties::direct_binary_signal_smoke_has_a_hard_timeout -- --exact
process-e2e:
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
needs: component-properties
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
- name: Install pinned Rust toolchain
run: rustup toolchain install nightly-2026-02-07 --profile minimal --component rustc-dev --component rust-src --component llvm-tools-preview
- name: Stateful VastAI process E2E
env:
PROPTEST_CASES: "4"
run: |
set -o pipefail
mkdir -p artifacts
cargo test -p myelin --features test-support --test stateful_vastai stateful_vastai_dashboard_control_survives_restarts -- --ignored --exact --nocapture 2>&1 | tee artifacts/stateful-vastai.log
- name: Preserve E2E failure artifacts
if: failure()
uses: actions/upload-artifact@v4
with:
name: stateful-vastai-failure-${{ github.run_id }}
path: |
artifacts/stateful-vastai.log
apps/myelin/proptest-regressions/tests/e2e_vastai.txt
if-no-files-found: warn

20
Cargo.lock generated
View file

@ -2,6 +2,10 @@
# It is not intended for manual editing.
version = 4
[[package]]
name = "actor-control-flow-lint-tests"
version = "0.1.0"
[[package]]
name = "adler2"
version = "2.0.1"
@ -900,12 +904,15 @@ version = "0.1.0"
dependencies = [
"axum",
"parking_lot",
"proptest",
"serde",
"serde_json",
"swactor",
"swactor-engine",
"telemetry",
"tokio",
"tokio-stream",
"tower",
]
[[package]]
@ -1131,6 +1138,7 @@ dependencies = [
"serde",
"serde_json",
"swactor",
"swactor-engine",
"swactor-transport",
"telemetry",
"uuid",
@ -2491,7 +2499,6 @@ dependencies = [
"provisioning",
"serde",
"serde_json",
"signal-hook",
"swactor",
"swactor-engine",
"swactor-job-runner",
@ -2503,7 +2510,6 @@ dependencies = [
"tokio",
"toml 0.8.23",
"ureq",
"wiremock",
]
[[package]]
@ -3268,6 +3274,7 @@ dependencies = [
"serde_json",
"swactor",
"swactor-engine",
"swactor-process",
]
[[package]]
@ -3339,6 +3346,7 @@ version = "0.1.0"
dependencies = [
"pyo3",
"swactor",
"swactor-engine",
]
[[package]]
@ -4307,6 +4315,7 @@ name = "swactor-engine"
version = "0.1.0"
dependencies = [
"parking_lot",
"proptest",
"swactor",
"tokio",
]
@ -4333,11 +4342,16 @@ version = "0.1.0"
dependencies = [
"crossbeam-queue",
"libc",
"parking_lot",
"proptest",
"serde",
"serde_json",
"serde_yaml",
"signal-hook",
"swactor",
"swactor-engine",
"telemetry",
"tokio",
]
[[package]]
@ -5181,6 +5195,7 @@ name = "wasm-runtime"
version = "0.1.0"
dependencies = [
"swactor",
"swactor-engine",
"wasm-bindgen",
]
@ -5775,6 +5790,7 @@ dependencies = [
"iroh-driver",
"libc",
"parking_lot",
"proptest",
"provisioning",
"serde",
"serde_json",

1
Cargo.toml
View file

@ -13,6 +13,7 @@ members = [
"apps/myelin",
"xtask",
"tools/vastai",
"tools/actor-control-flow-lint",
]
default-members = [
".",

9
apps/myelin/Cargo.toml
View file

@ -10,6 +10,7 @@ autobins = false
[features]
default = ["dashboard"]
dashboard = []
test-support = ["swactor-vastai/test-support"]
[dependencies]
telemetry = { path = "../../crates/telemetry" }
@ -36,12 +37,16 @@ axum = "0.8"
[dev-dependencies]
tempfile = "3"
wiremock = "0.6"
swactor-vastai = { path = "../../tools/vastai", features = ["test-support"] }
proptest = "1"
[[test]]
name = "stateful_vastai"
path = "tests/stateful_vastai.rs"
required-features = ["test-support"]
[target.'cfg(target_os = "linux")'.dependencies]
libc = "0.2"
signal-hook = "0.3"
[[bin]]
name = "myelin-worker"

7
apps/myelin/proptest-regressions/node/worker_node_runtime.txt Normal file
View file

@ -0,0 +1,7 @@
# Seeds for failure cases proptest has generated in the past. It is
# automatically read and these particular cases re-run before any
# novel cases are generated.
#
# It is recommended to check this file in to source control so that
# everyone who runs the test benefits from these saved cases.
cc be434e1d0bf8073f32e4072d9093dd2df03537b9d1bf759872e760fcf74a06cb # shrinks to actions = [], extra_ticks = 1

7
apps/myelin/proptest-regressions/orchestration/app.txt Normal file
View file

@ -0,0 +1,7 @@
# Seeds for failure cases proptest has generated in the past. It is
# automatically read and these particular cases re-run before any
# novel cases are generated.
#
# It is recommended to check this file in to source control so that
# everyone who runs the test benefits from these saved cases.
cc ed6cf4326f625c499a6752aa3630891a4242ed8aa4cfe59148d901030efd4ff1 # shrinks to actions = []

11
apps/myelin/proptest-regressions/orchestration/manual_control.txt Normal file
View file

@ -0,0 +1,11 @@
# Seeds for failure cases proptest has generated in the past. It is
# automatically read and these particular cases re-run before any
# novel cases are generated.
#
# It is recommended to check this file in to source control so that
# everyone who runs the test benefits from these saved cases.
cc a36a5c2d2904ecfb2cf31f3e446db588a7591e074994fc61142f37aad33272c8 # shrinks to actions = [Provision(163), Drive, Configure(240), StaleSearch(108), Configure(85), Search(84), Flush, Kill(76), Configure(190), StaleValidation(217), Provision(209), StaleSearch(18), StaleValidation(70), Provision(97), Drive, Flush, StaleValidation(81), Search(121), Query, Flush, StaleValidation(91), StaleValidation(110)]
cc aa67d0e8e7fd15f0b8cad5ac675b6fcee25fc5246a6841d141054b6932f0db34 # shrinks to actions = [Search(0), Search(0), Provision(86)]
cc ad4aa3d6008d92f09c315c680c258a2b3776719bca98ff49d72668542f0c4772 # shrinks to actions = [Provision(17), Flush(81), OfferSearchFinished(200), PersistenceFinished(145), Provision(202), ProviderTerminalFailure(29), Kill(222), Provision(21), Provision(117), Query(131), Provision(126), Rejoin(27), Provision(89), OfferSearchFinished(148), Query(54), Kill(77)]
cc be1220292a8604bca5d52ed98402b55bfd81ca591ee47c4139829365a5c1eddc # shrinks to actions = [ProviderValidated(17), Flush(231), OfferSearchFinished(238), Drive, Provision(32), Configure(101), Configure(194), ProviderValidated(226), Provision(152), Configure(235), Provision(208), Kill(68), ProviderValidated(50)]
cc fbc45330b5c55a483c910567ed49b8c2f40e4d9b8ad17f9a88f666fc120d4305 # shrinks to actions = [PersistenceFinished(73), Flush(190), EffectFinished(224), Query(58), Kill(182), Rejoin(82), Flush(137), OfferSearchFinished(173), Drive, Provision(204), Kill(137), EffectFinished(187), Configure(80), Configure(104), Drive, PersistenceFinished(92), Flush(180), Search(86), Configure(246), Query(33), Search(43)]

8
apps/myelin/proptest-regressions/orchestration/provisioning.txt Normal file
View file

@ -0,0 +1,8 @@
# Seeds for failure cases proptest has generated in the past. It is
# automatically read and these particular cases re-run before any
# novel cases are generated.
#
# It is recommended to check this file in to source control so that
# everyone who runs the test benefits from these saved cases.
cc 38b940d757de89f824d145dc409f7f0d3b6d757840eff473ed9acac3eef65d21 # shrinks to actions = [Create(61), Create(253)]
cc 502380a923217c7ea1912e0a54392f1389eaafa4e35efeac26cdeb5ac2a0067e # shrinks to actions = [Create(53), Start(0), Adopt(221)]

5
apps/myelin/proptest-regressions/tests/e2e_vastai.txt Normal file
View file

@ -0,0 +1,5 @@
# Seeds for stateful VastAI E2E failures. Proptest replays these before generated cases.
cc 26c3cd5944cb25de70a984389373e7380a0c6e6ba1c8f3e6eff5a0f78dbef168 # current-strategy replay; E2eCase is derived from e2e_case()
cc 5efb0a6c5c348475c859a2c4267d5177aaecd5adefcc5e8b7fc2e6d728deecd1 # shrinks to case = E2eCase { seed: 7099259626237328177, node_seed: 245, kill_mask: 24, offer_offset: 3, actions: [ConcurrentQueries, Search { count: 2 }, Restart { mode: FlushSafeAbrupt }, Query, Kill { node_slot: 218, command_slot: 5 }, Query, Provision { command_slot: 5, use_searched_offers: true }, EndpointProbe { node_slot: 217 }, Restart { mode: Graceful }, Flush, Kill { node_slot: 217, command_slot: 5 }] }
cc 6d005687858520ed65af20ec5b2de056efcda15771b6420d7f67e3cb6c246d5b # shrinks to case = E2eCase { seed: 17649392557414661864, node_seed: 249, kill_mask: 159, offer_offset: 5, actions: [Kill { node_slot: 88, command_slot: 242 }, Flush, ConcurrentQueries, EndpointProbe { node_slot: 88 }, Query, Search { count: 2 }, Provision { command_slot: 242, use_searched_offers: true }, Restart { mode: FlushSafeAbrupt }, Kill { node_slot: 89, command_slot: 242 }, Query, Restart { mode: Graceful }] }
cc 2e36f51e7cb347b571ac838f81638748f8414a32ec91cb9f4b2730363d72be61 # shrinks to case = E2eCase { seed: 13301618846512983428, node_seed: 99, kill_mask: 30, offer_offset: 2, actions: [Query, Restart { mode: FlushSafeAbrupt }, Kill { node_slot: 136, command_slot: 187 }, ConcurrentQueries, Restart { mode: Graceful }, Query, EndpointProbe { node_slot: 136 }, Kill { node_slot: 137, command_slot: 187 }, Provision { command_slot: 187, use_searched_offers: false }, Flush, Search { count: 0 }] }

81
apps/myelin/specs/MYELIN_DAEMON.md
View file

@ -1,81 +0,0 @@
# Myelin Fleet-Control Daemon
Myelin is a persistent, dashboard-first control plane for manually managed compute nodes. It boots an empty fleet, accepts explicit operator commands, records intent and observations, and never performs hidden replacement or teardown.
## Runtime shape
```text
myelin-orchestrator
-> load stable iroh identity and cluster snapshot
-> start engine, iroh endpoint, telemetry collector, and dashboard
-> recover persisted provider resources without creating replacements
-> idle event loop
- ingest telemetry, membership, actor reports, and control messages
- dispatch explicit Provision / Kill effects
- atomically persist every state transition before dependent effects
-> on exit, detach durable-provider handles without destroying resources
```
There is no desired-shape reconciler or automatic node replacement in the manual control path. Provisioning, Kill, recovery, runtime readiness, and worker rejoin are typed actor protocols; provider I/O and snapshot writes run outside actor handlers on the engine.
## Starting the daemon
From the workspace root:
```sh
cargo run -p myelin
```
The local default uses the process provider. Use Docker explicitly when required:
```sh
cargo run -p myelin -- --provider docker
```
Use Vast.ai without making valid credentials a startup prerequisite:
```sh
cargo run -p myelin -- --provider vastai
```
The dashboard starts in `unconfigured` or `configuration_error` state and accepts corrected credentials at runtime.
`MYELIN_DASHBOARD_PORT` selects the dashboard port. The dashboard root remains the read-only Fleet view; `/provision` is the Myelin-owned mutation surface. Its provider-neutral control protocol includes:
- `Provision { command_id, count, selected_offer_ids }`: create the requested nodes; Vast.ai requires and leases only the exact selected offer IDs.
- `Kill { command_id, logical_node_id }`: stop one managed process/container or destroy one Vast.ai contract while retaining its terminal snapshot record.
- `ConfigureProvider`: validate corrected in-memory Vast.ai credentials and bootstrap settings.
- `SearchOffers`: inspect filtered Vast.ai offers without leasing.
- `Query`: return provider readiness, recent commands, and managed node state.
Every mutation carries a caller-generated command ID. The daemon persists the full command record and node intent before provider work. Reusing an ID returns the original record and never repeats create or destroy. Node IDs are monotonic and never reused.
## Durable state
The state directory contains:
- `identity.key`: 32-byte iroh secret key. Preserving it keeps the daemon endpoint stable across restarts.
- `cluster.json`: schema-versioned snapshot containing the stable provider label, run id, next node id, full command records, node specs, selected offer IDs, provider references, runtime facts, phases, and errors.
Writes use a temporary file plus rename. A corrupt identity or snapshot is a hard startup error. `--reset-state` explicitly clears both files; startup never treats corruption as an empty fleet.
Provider state is ground truth during recovery:
- snapshot + provider resource: adopt and observe it;
- snapshot only: mark the node stopped; never recreate it;
- provider resource without managed intent: report it as an orphan and take no action.
The current orchestrator actor address is published under `myelin.manual-control` in the distributed name registry. A surviving worker observes a changed binding, sends `RejoinHello` with its persisted logical identity and current runtime facts, waits for the daemon to persist those facts, and only then rebinds to the returned actor address and control generation. No stable actor address is assumed.
## Lifecycle policy
Graceful shutdown leaves Docker containers and Vast.ai leases running so a later daemon can adopt them. Local process children are different: they cannot be adopted, so Ctrl-C stops them and clears their snapshot records. They also exit when their daemon-owned stdin supervision pipe closes, preventing an abrupt daemon crash from leaving invisible local workers.
Destruction of durable provider resources occurs only through an explicit dashboard command. The default process path re-enters the running orchestrator executable in an internal worker mode, so `cargo run -p myelin` never depends on a separately built or stale `myelin-worker` binary.
## Node image contract
The standard image contains a uniform Myelin agent, SSH bootstrap, and the CUDA runtime. It does not contain tinygrad, NumPy, PyTorch, vLLM, or model weights. Frameworks and application dependencies belong to job payload images. Nodes launched by this daemon set `MYELIN_AGENT_ONLY=1`, so the agent joins membership, announces readiness, and exports telemetry without starting an inference helper.
As part of runtime-ready bootstrap, the daemon dials the node's advertised iroh endpoint on `TELEMETRY_ALPN`, requests all telemetry channels, and retains that pull stream for the node's lifetime. The node serves the pull locally; it never needs to reverse-dial the dashboard. Pulled stream descriptors and frames feed both the Fleet view and the live telemetry explorer.
The retired chat/GGUF specification is archived at `archive/MYELIN_CHAT_SPEC.md`.

759
apps/myelin/src/job_deploy.rs
View file

@ -13,10 +13,11 @@ use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, Instant};
use serde::{Deserialize, Serialize};
use swactor::actor::ActorAddress;
use swactor_engine::{Engine, EngineHandle, TokioBackend, TokioConfig};
use swactor::actor::{ActorAddress, ActorInterface};
use swactor::runtime::{Ctx, ExternalSender};
use swactor_engine::{ActorCompletion, Engine, EngineHandle, TokioBackend, TokioConfig};
use swactor_job_runner::{
JobDone, NodeJobActor, OUTPUTS_EDGE_ID, OrchestratorJobActor, OrchestratorJobMsg,
Job, JobDone, NodeJobActor, OUTPUTS_EDGE_ID, OrchestratorJobActor, OrchestratorJobMsg,
WORKSPACE_EDGE_ID, register_job_codecs,
};
use swactor_transport::hex_encode;
@ -100,35 +101,104 @@ pub(crate) fn build_composition() -> Result<JobComposition, String> {
Ok((engine, driver, stack))
}
fn identity_for(driver: &IrohDriver, actor: ActorAddress) -> Result<NodeIdentity, String> {
let endpoint = advertised_endpoint_for(driver)?;
Ok(NodeIdentity {
endpoint,
actor_hex: hex_encode(&actor.0),
})
struct ResolveIdentityActor {
driver: Option<IrohDriver>,
actor: ActorAddress,
mask: EndpointAddrMask,
started: Instant,
engine: EngineHandle,
sender: ExternalSender,
completion: ActorCompletion<Result<(IrohDriver, NodeIdentity), String>>,
}
fn advertised_endpoint_for(driver: &IrohDriver) -> Result<EndpointAddr, String> {
let mask = endpoint_addr_mask_from_env()?;
if !mask.requires_relay() {
return advertised_endpoint(driver.endpoint_addr(), mask);
#[derive(Clone)]
enum ResolveIdentityMsg {
Check,
}
impl ResolveIdentityActor {
fn finish(&mut self, ctx: &swactor::runtime::Ctx, result: Result<NodeIdentity, String>) {
let result = result.map(|identity| {
(
self.driver
.take()
.expect("identity resolver owns driver until completion"),
identity,
)
});
assert!(
self.completion.complete(result).is_ok(),
"identity resolver completed twice"
);
ctx.stop_self();
}
let started = Instant::now();
loop {
let endpoint = driver.endpoint_addr();
if endpoint.relay_urls().next().is_some() {
return advertised_endpoint(endpoint, mask);
}
if started.elapsed() >= RELAY_WAIT_DEADLINE {
return Err(format!(
"relay-only endpoint address mask did not observe a relay URL within {RELAY_WAIT_DEADLINE:?}; last endpoint={endpoint:?}"
));
}
std::thread::sleep(POLL);
fn schedule_check(&self, ctx: &swactor::runtime::Ctx) {
self.engine.send_after(
POLL,
self.sender.clone(),
ctx.self_addr(),
ResolveIdentityMsg::Check,
);
}
}
impl ActorInterface for ResolveIdentityActor {
type Incoming = ResolveIdentityMsg;
type Response = ();
fn on_start(&mut self, ctx: &swactor::runtime::Ctx) {
let _ = ctx.send(ctx.self_addr(), ResolveIdentityMsg::Check);
}
fn handle(&mut self, ctx: &swactor::runtime::Ctx, _message: Self::Incoming) {
let driver = self
.driver
.as_ref()
.expect("identity resolver handles messages only while live");
let endpoint = driver.endpoint_addr();
if !self.mask.requires_relay() || endpoint.relay_urls().next().is_some() {
let identity = advertised_endpoint(endpoint, self.mask)
.map(|endpoint| NodeIdentity {
endpoint,
actor_hex: hex_encode(&self.actor.0),
})
.map_err(|error| error.to_string());
self.finish(ctx, identity);
} else if self.started.elapsed() >= RELAY_WAIT_DEADLINE {
self.finish(
ctx,
Err(format!(
"relay-only endpoint address mask did not observe a relay URL within {RELAY_WAIT_DEADLINE:?}; last endpoint={endpoint:?}"
)),
);
} else {
self.schedule_check(ctx);
}
}
}
fn resolve_identity(
driver: IrohDriver,
actor: ActorAddress,
stack: &DistributionRuntimeStack,
) -> Result<(IrohDriver, NodeIdentity), String> {
let completion = ActorCompletion::new();
stack
.runtime
.spawn(ResolveIdentityActor {
driver: Some(driver),
actor,
mask: endpoint_addr_mask_from_env()?,
started: Instant::now(),
engine: stack.engine.clone(),
sender: stack.runtime.create_sender(),
completion: completion.clone(),
})
.map_err(|error| format!("spawn endpoint identity resolver: {error}"))?;
completion.wait()
}
fn endpoint_addr_mask_from_env() -> Result<EndpointAddrMask, String> {
match env_optional(MVP_IROH_ENDPOINT_ADDR_MASK_ENV) {
Some(mask) => EndpointAddrMask::parse(&mask),
@ -174,6 +244,94 @@ pub(crate) fn parse_actor(hex: &str) -> Result<ActorAddress, String> {
Ok(ActorAddress(arr))
}
#[derive(Clone)]
enum WorkerLifecycleMsg {
CheckConnection,
Stop,
}
struct WorkerLifecycleActor {
driver: IrohDriver,
_stack: DistributionRuntimeStack,
orchestrator_endpoint: EndpointAddr,
orchestrator_node: swactor_transport::NodeId,
output_sink: Arc<Mutex<Option<Box<dyn swactor_job_runner::JobEdgeSink>>>>,
started: Instant,
armed: bool,
engine: EngineHandle,
sender: ExternalSender,
completion: ActorCompletion<Result<(), String>>,
}
impl WorkerLifecycleActor {
fn schedule_check(&self, ctx: &Ctx) {
self.engine.send_after(
POLL,
self.sender.clone(),
ctx.self_addr(),
WorkerLifecycleMsg::CheckConnection,
);
}
}
impl ActorInterface for WorkerLifecycleActor {
type Incoming = WorkerLifecycleMsg;
type Response = ();
fn on_start(&mut self, ctx: &Ctx) {
let _ = ctx.send(ctx.self_addr(), WorkerLifecycleMsg::CheckConnection);
}
fn handle(&mut self, ctx: &Ctx, message: Self::Incoming) {
match message {
WorkerLifecycleMsg::CheckConnection if !self.armed => {
if self.driver.has_active_connection(&self.orchestrator_node)
|| self.started.elapsed() >= CONVERGE_DEADLINE
{
match self
.driver
.spawn_edge_send_pump(self.orchestrator_endpoint.clone(), OUTPUTS_EDGE_ID)
{
Ok(handle) => {
*self.output_sink.lock() = Some(Box::new(IrohEdgeSink(handle)));
eprintln!("job-worker: output edge sink armed");
}
Err(error) => {
eprintln!("job-worker: failed to arm output edge sink: {error}")
}
}
self.armed = true;
} else {
self.schedule_check(ctx);
}
}
WorkerLifecycleMsg::CheckConnection => {}
WorkerLifecycleMsg::Stop => {
assert!(
self.completion.complete(Ok(())).is_ok(),
"worker lifecycle completed twice"
);
ctx.stop_self();
}
}
}
}
struct WorkerStopForwarder {
sender: ExternalSender,
worker: ActorAddress,
}
impl ActorInterface for WorkerStopForwarder {
type Incoming = ();
type Response = ();
fn handle(&mut self, ctx: &Ctx, (): Self::Incoming) {
let _ = self.sender.send_to(self.worker, WorkerLifecycleMsg::Stop);
ctx.stop_self();
}
}
/// Worker: connect to the orchestrator, expose a `NodeJobActor`, run jobs it
/// sends over iroh. Prints this node's identity as JSON on stdout, then runs
/// until killed. Bulk bytes travel over EDGE_ALPN: the orchestrator pushes the
@ -185,42 +343,38 @@ pub fn run_worker(orch_identity_json: String, workdir: PathBuf) -> Result<(), St
.map_err(|e| format!("parse orch identity: {e}"))?;
let orch_actor = parse_actor(&orch.actor_hex)?;
let orch_endpoint = orch.endpoint.clone();
let (_engine, driver, stack) = build_composition()?;
let (engine, driver, stack) = build_composition()?;
let sender = stack.runtime.create_sender();
// Edge workspace: the orchestrator pushes the workspace tar over EDGE_ALPN
// before submitting the job. A background thread drains those bytes,
// extracts them into `workdir`, and signals readiness; the node actor waits
// on that flag before announcing the workspace materialized.
let workspace_ready = Arc::new(AtomicBool::new(false));
let ws_events = driver.edge_events_handle();
let ws_workdir = workdir.clone();
let ws_ready = workspace_ready.clone();
std::thread::Builder::new()
.name("job-worker-ws-edge".to_owned())
.spawn(move || {
drain_workspace_edge(ws_events, ws_workdir, ws_ready);
stack
.runtime
.spawn(WorkspaceEdgeActor {
engine: stack.engine.clone(),
sender: stack.runtime.create_sender(),
events: driver.edge_events_handle(),
workdir: workdir.clone(),
ready: workspace_ready.clone(),
buf: Vec::new(),
started: Instant::now(),
})
.map_err(|e| format!("spawn workspace edge thread: {e}"))?;
.map_err(|error| format!("spawn workspace edge actor: {error}"))?;
// Edge outputs: a slot the main thread fills with an EDGE_ALPN sink to the
// orchestrator once the iroh connection is up. The node actor ships
// collected outputs through it.
let output_sink_slot: Arc<Mutex<Option<Box<dyn swactor_job_runner::JobEdgeSink>>>> =
Arc::new(Mutex::new(None));
let job_actor = stack
.runtime
.spawn(
NodeJobActor::new(orch_actor, workdir, sender, 0)
.with_workspace_ready(workspace_ready.clone())
.with_actor_timers(engine.handle())
.with_workspace_ready(workspace_ready)
.with_output_sink_slot(output_sink_slot.clone()),
)
.map_err(|e| format!("spawn node job actor: {e}"))?;
stack.register_local_actor(driver.register_actor(job_actor, 1));
driver.join(std::slice::from_ref(&orch.endpoint));
let id = identity_for(&driver, job_actor)?;
let (driver, id) = resolve_identity(driver, job_actor, &stack)?;
println!(
"JOB_WORKER_IDENTITY {}",
serde_json::to_string(&id).map_err(|e| e.to_string())?
@ -231,27 +385,33 @@ pub fn run_worker(orch_identity_json: String, workdir: PathBuf) -> Result<(), St
id.actor_hex, id.endpoint
);
// Wait for the iroh connection to the orchestrator, then arm the output
// edge sink so it is ready before a CollectOutputs command can arrive.
let orch_node = swactor_transport::NodeId(*orch_endpoint.id.as_bytes());
let conn_started = Instant::now();
while !driver.has_active_connection(&orch_node) {
if conn_started.elapsed() >= CONVERGE_DEADLINE {
break;
}
std::thread::sleep(POLL);
}
match driver.spawn_edge_send_pump(orch_endpoint.clone(), OUTPUTS_EDGE_ID) {
Ok(handle) => {
*output_sink_slot.lock() = Some(Box::new(IrohEdgeSink(handle)));
eprintln!("job-worker: output edge sink armed");
}
Err(e) => eprintln!("job-worker: failed to arm output edge sink: {e}"),
}
loop {
std::thread::sleep(Duration::from_secs(3600));
}
let runtime = stack.runtime.clone();
let completion = ActorCompletion::new();
let lifecycle = runtime
.spawn(WorkerLifecycleActor {
driver,
_stack: stack,
orchestrator_node: swactor_transport::NodeId(*orch_endpoint.id.as_bytes()),
orchestrator_endpoint: orch_endpoint,
output_sink: output_sink_slot,
started: Instant::now(),
armed: false,
engine: engine.handle(),
sender: runtime.create_sender(),
completion: completion.clone(),
})
.map_err(|error| format!("spawn job worker lifecycle actor: {error}"))?;
let stop_forwarder = runtime
.spawn(WorkerStopForwarder {
sender: runtime.create_sender(),
worker: lifecycle,
})
.map_err(|error| format!("spawn job worker stop forwarder: {error}"))?;
#[cfg(target_os = "linux")]
swactor_process::spawn_os_stop_signal_wait(runtime.create_sender(), stop_forwarder);
let result = completion.wait();
drop(engine);
result
}
/// Starts the operator-side job actor and publishes enough identity for a
@ -268,7 +428,7 @@ pub(crate) fn start_orchestrator(landing: PathBuf) -> Result<JobOrchestratorSess
.map_err(|e| format!("spawn orchestrator: {e}"))?;
stack.register_local_actor(driver.register_actor(orch, 1));
let identity = identity_for(&driver, orch)?;
let (driver, identity) = resolve_identity(driver, orch, &stack)?;
Ok(JobOrchestratorSession {
_engine: engine,
driver,
@ -280,6 +440,257 @@ pub(crate) fn start_orchestrator(landing: PathBuf) -> Result<JobOrchestratorSess
})
}
pub(crate) struct JobRunStateMachine {
session: JobOrchestratorSession,
job: Option<Job>,
worker: NodeIdentity,
node_actor: ActorAddress,
phase: JobRunPhase,
}
enum JobRunPhase {
Created,
Directory {
deadline: Instant,
},
Connection {
started: Instant,
deadline: Instant,
},
Running {
deadline: Instant,
output_buf: Vec<u8>,
outputs_ended: bool,
pending_done: Option<JobDone>,
},
Finished,
}
impl JobRunStateMachine {
pub(crate) fn new(
session: JobOrchestratorSession,
job: Job,
worker: NodeIdentity,
) -> Result<Self, String> {
let node_actor = parse_actor(&worker.actor_hex)?;
Ok(Self {
session,
job: Some(job),
worker,
node_actor,
phase: JobRunPhase::Created,
})
}
pub(crate) fn start(&mut self, now: Instant) {
self.session
.driver
.join(std::slice::from_ref(&self.worker.endpoint));
self.phase = JobRunPhase::Directory {
deadline: now + CONVERGE_DEADLINE,
};
}
pub(crate) fn advance(&mut self, now: Instant) -> Option<Result<JobDone, String>> {
let phase = std::mem::replace(&mut self.phase, JobRunPhase::Finished);
match phase {
JobRunPhase::Created => {
self.phase = JobRunPhase::Created;
None
}
JobRunPhase::Directory { deadline } => {
let converged = self
.session
.stack
.route_view
.read()
.map(|view| view.contains_key(&self.node_actor))
.unwrap_or(false);
if converged {
eprintln!(
"job-orch: directory converged; waiting for iroh connection to worker"
);
self.phase = JobRunPhase::Connection {
started: now,
deadline: now + CONVERGE_DEADLINE,
};
None
} else if now >= deadline {
Some(Err(
"directory did not converge: orchestrator never learned the worker actor"
.into(),
))
} else {
self.phase = JobRunPhase::Directory { deadline };
None
}
}
JobRunPhase::Connection { started, deadline } => {
let worker_node = swactor_transport::NodeId(*self.worker.endpoint.id.as_bytes());
let connected = self.session.driver.has_active_connection(&worker_node);
if !connected && now < deadline {
self.phase = JobRunPhase::Connection { started, deadline };
return None;
}
if connected {
eprintln!(
"job-orch: iroh connection to worker established after {:?}",
now.saturating_duration_since(started)
);
eprintln!("job-orch: submitting job");
} else {
eprintln!(
"job-orch: no iroh connection to worker after {CONVERGE_DEADLINE:?}; join_statuses={:?}; submitting best-effort",
self.session.driver.join_statuses()
);
}
match self.submit(now) {
Ok(()) => None,
Err(error) => Some(Err(error)),
}
}
JobRunPhase::Running {
deadline,
mut output_buf,
mut outputs_ended,
mut pending_done,
} => {
let edge_events = self.session.driver.edge_events_handle();
let drained: Vec<WireEvent> = edge_events.lock().drain(..).collect();
for event in drained {
match event {
WireEvent::BytesRead { edge_id, bytes, .. }
if edge_id.0 == OUTPUTS_EDGE_ID =>
{
output_buf.extend_from_slice(&bytes);
}
WireEvent::StreamEnded { edge_id, .. } if edge_id.0 == OUTPUTS_EDGE_ID => {
if !output_buf.is_empty() {
if let Err(error) = swactor_job_runner::extract_tar(
&output_buf,
&self.session.landing,
) {
eprintln!("job-orch: untar edge outputs failed: {error}");
}
output_buf.clear();
}
outputs_ended = true;
}
_ => {}
}
}
if pending_done.is_none() {
pending_done = self.session.done.try_recv();
}
if let Some(done) = pending_done.as_ref() {
let need_outputs = done.exit_code.is_some() && !outputs_ended;
if !need_outputs || now >= deadline {
let done = pending_done
.take()
.expect("pending job result was observed");
if need_outputs {
eprintln!("job-orch: output edge stream did not land before deadline");
}
return Some(Ok(done));
}
}
if now >= deadline {
return Some(Err("job did not complete within deadline".into()));
}
self.phase = JobRunPhase::Running {
deadline,
output_buf,
outputs_ended,
pending_done,
};
None
}
JobRunPhase::Finished => {
Some(Err("job state machine advanced after completion".into()))
}
}
}
fn submit(&mut self, now: Instant) -> Result<(), String> {
let job = self
.job
.take()
.ok_or_else(|| "job was already submitted".to_owned())?;
let workspace_bytes =
swactor_job_runner::pack_workspace(&job).map_err(|e| format!("pack workspace: {e}"))?;
if !workspace_bytes.is_empty() {
let pump = self
.session
.driver
.spawn_edge_send_pump(self.worker.endpoint.clone(), WORKSPACE_EDGE_ID)
.map_err(|e| format!("workspace edge pump: {e}"))?;
for record in workspace_bytes.chunks(swactor_job_runner::EDGE_RECORD_SIZE) {
pump.send(record.to_vec())
.map_err(|e| format!("workspace edge send: {e}"))?;
}
drop(pump);
eprintln!("job-orch: workspace pushed over EDGE_ALPN");
}
self.session
.stack
.runtime
.send_to(
self.session.orch,
OrchestratorJobMsg::Submit {
job,
node_actor: self.node_actor,
},
)
.map_err(|e| format!("submit: {e}"))?;
self.phase = JobRunPhase::Running {
deadline: now + JOB_DEADLINE,
output_buf: Vec::new(),
outputs_ended: false,
pending_done: None,
};
Ok(())
}
}
#[derive(Clone)]
struct JobRunTick;
struct JobRunActor {
machine: JobRunStateMachine,
engine: EngineHandle,
sender: ExternalSender,
completion: ActorCompletion<Result<JobDone, String>>,
}
impl JobRunActor {
fn schedule(&self, ctx: &Ctx) {
self.engine
.send_after(POLL, self.sender.clone(), ctx.self_addr(), JobRunTick);
}
}
impl ActorInterface for JobRunActor {
type Incoming = JobRunTick;
type Response = ();
fn on_start(&mut self, ctx: &Ctx) {
self.machine.start(Instant::now());
let _ = ctx.send(ctx.self_addr(), JobRunTick);
}
fn handle(&mut self, ctx: &Ctx, _message: Self::Incoming) {
if let Some(result) = self.machine.advance(Instant::now()) {
assert!(
self.completion.complete(result).is_ok(),
"job lifecycle completed twice"
);
ctx.stop_self();
} else {
self.schedule(ctx);
}
}
}
impl JobOrchestratorSession {
pub(crate) fn identity_json(&self) -> Result<String, String> {
serde_json::to_string(&self.identity).map_err(|e| e.to_string())
@ -294,151 +705,29 @@ impl JobOrchestratorSession {
}
pub(crate) fn run_to_completion(
&mut self,
job: swactor_job_runner::Job,
self,
job: Job,
worker: NodeIdentity,
) -> Result<JobDone, String> {
let node_actor = parse_actor(&worker.actor_hex)?;
self.driver.join(std::slice::from_ref(&worker.endpoint));
let started = Instant::now();
while started.elapsed() < CONVERGE_DEADLINE {
if self
.stack
.route_view
.read()
.map(|v| v.contains_key(&node_actor))
.unwrap_or(false)
{
break;
}
std::thread::sleep(POLL);
}
if !self
.stack
.route_view
.read()
.map(|v| v.contains_key(&node_actor))
.unwrap_or(false)
{
return Err(
"directory did not converge: orchestrator never learned the worker actor".into(),
);
}
eprintln!("job-orch: directory converged; waiting for iroh connection to worker");
let worker_node = swactor_transport::NodeId(*worker.endpoint.id.as_bytes());
let conn_started = Instant::now();
while !self.driver.has_active_connection(&worker_node) {
if conn_started.elapsed() >= CONVERGE_DEADLINE {
eprintln!(
"job-orch: no iroh connection to worker after {CONVERGE_DEADLINE:?}; join_statuses={:?}; submitting best-effort",
self.driver.join_statuses()
);
break;
}
std::thread::sleep(POLL);
}
if self.driver.has_active_connection(&worker_node) {
eprintln!(
"job-orch: iroh connection to worker established after {:?}",
conn_started.elapsed()
);
eprintln!("job-orch: submitting job");
} else {
eprintln!("job-orch: submitting job (no confirmed connection)");
}
// EDGE: push the workspace tar over EDGE_ALPN before submitting. Small
// commands/events still travel as actor messages; only bulk bytes move
// onto the edge transport so they survive relay (NAT) traversal.
let edge_events = self.driver.edge_events_handle();
let workspace_bytes =
swactor_job_runner::pack_workspace(&job).map_err(|e| format!("pack workspace: {e}"))?;
if !workspace_bytes.is_empty() {
let pump = self
.driver
.spawn_edge_send_pump(worker.endpoint.clone(), WORKSPACE_EDGE_ID)
.map_err(|e| format!("workspace edge pump: {e}"))?;
for record in workspace_bytes.chunks(swactor_job_runner::EDGE_RECORD_SIZE) {
pump.send(record.to_vec())
.map_err(|e| format!("workspace edge send: {e}"))?;
}
drop(pump); // finish the edge stream → receiver observes end-of-stream
eprintln!("job-orch: workspace pushed over EDGE_ALPN");
}
self.stack
.runtime
.send_to(self.orch, OrchestratorJobMsg::Submit { job, node_actor })
.map_err(|e| format!("submit: {e}"))?;
// Drive lifecycle (actor messages) while draining the output edge stream.
let started = Instant::now();
let mut output_buf: Vec<u8> = Vec::new();
let mut outputs_ended = false;
// The orchestrator actor reports `JobDone` exactly once; hold it here
// while we wait for the output edge stream to land so it is not lost.
let mut pending_done: Option<JobDone> = None;
loop {
// Drain output edge bytes; extract the tar as soon as the stream ends
// (release the edge-event lock before the potentially slow untar).
let drained: Vec<WireEvent> = edge_events.lock().drain(..).collect();
for ev in drained {
match ev {
WireEvent::BytesRead { edge_id, bytes, .. } if edge_id.0 == OUTPUTS_EDGE_ID => {
output_buf.extend_from_slice(&bytes);
}
WireEvent::StreamEnded { edge_id, .. } if edge_id.0 == OUTPUTS_EDGE_ID => {
if !output_buf.is_empty() {
if let Err(e) =
swactor_job_runner::extract_tar(&output_buf, &self.landing)
{
eprintln!("job-orch: untar edge outputs failed: {e}");
}
output_buf.clear();
}
outputs_ended = true;
}
_ => {}
}
}
if pending_done.is_none() {
pending_done = self.done.try_recv();
}
// A job that ran (exit code observed) collected outputs over edge —
// wait for that stream to land before returning so the landing dir is
// populated. A pre-run fault (no exit code) ships no outputs.
let ready = match &pending_done {
Some(done) => {
let need_outputs = done.exit_code.is_some() && !outputs_ended;
!need_outputs || started.elapsed() >= JOB_DEADLINE
}
None => false,
};
if ready {
let done = pending_done
.take()
.expect("pending_done observed Some in ready branch");
if done.exit_code.is_some() && !outputs_ended {
eprintln!("job-orch: output edge stream did not land before deadline");
}
return Ok(done);
}
if started.elapsed() >= JOB_DEADLINE {
return Err("job did not complete within deadline".into());
}
std::thread::sleep(POLL);
}
let runtime = self.stack.runtime.clone();
let engine = self.stack.engine.clone();
let completion = ActorCompletion::new();
runtime
.spawn(JobRunActor {
machine: JobRunStateMachine::new(self, job, worker)?,
engine,
sender: runtime.create_sender(),
completion: completion.clone(),
})
.map_err(|error| format!("spawn job lifecycle actor: {error}"))?;
completion.wait()
}
}
/// Orchestrator: expose an `OrchestratorJobActor`, print its identity, read the
/// worker identity from stdin, drive the job to completion over iroh.
pub fn run_serve(job: swactor_job_runner::Job, landing: PathBuf) -> Result<JobDone, String> {
let mut session = start_orchestrator(landing)?;
let session = start_orchestrator(landing)?;
println!("JOB_ORCH_IDENTITY {}", session.identity_json()?);
let _ = std::io::Write::flush(&mut std::io::stdout());
eprintln!("job-orch: published identity; waiting for worker identity on stdin...");
@ -473,40 +762,64 @@ impl swactor_job_runner::JobEdgeSink for IrohEdgeSink {
/// without the readiness flag being set).
const WORKSPACE_EDGE_WAIT: Duration = Duration::from_secs(60 * 30);
/// Drain EDGE_ALPN workspace bytes (edge id `WORKSPACE_EDGE_ID`) the orchestrator
/// pushed, extract the tar into `workdir`, then signal readiness. Runs on a
/// background worker thread; the driver auto-accepts EDGE_ALPN connections and
/// pushes their bytes into the shared event queue drained here.
fn drain_workspace_edge(
#[derive(Clone)]
struct WorkspaceEdgePoll;
struct WorkspaceEdgeActor {
engine: EngineHandle,
sender: ExternalSender,
events: Arc<Mutex<Vec<WireEvent>>>,
workdir: PathBuf,
ready: Arc<AtomicBool>,
) {
let mut buf = Vec::new();
let started = Instant::now();
loop {
let drained: Vec<WireEvent> = events.lock().drain(..).collect();
for ev in drained {
match ev {
buf: Vec<u8>,
started: Instant,
}
impl WorkspaceEdgeActor {
fn schedule(&self, ctx: &Ctx, delay: Duration) {
self.engine.send_after(
delay,
self.sender.clone(),
ctx.self_addr(),
WorkspaceEdgePoll,
);
}
}
impl ActorInterface for WorkspaceEdgeActor {
type Incoming = WorkspaceEdgePoll;
type Response = ();
fn on_start(&mut self, ctx: &Ctx) {
self.schedule(ctx, Duration::ZERO);
}
fn handle(&mut self, ctx: &Ctx, _message: Self::Incoming) {
let drained: Vec<WireEvent> = self.events.lock().drain(..).collect();
for event in drained {
match event {
WireEvent::BytesRead { edge_id, bytes, .. } if edge_id.0 == WORKSPACE_EDGE_ID => {
buf.extend_from_slice(&bytes);
self.buf.extend_from_slice(&bytes);
}
WireEvent::StreamEnded { edge_id, .. } if edge_id.0 == WORKSPACE_EDGE_ID => {
if !buf.is_empty() {
if let Err(e) = swactor_job_runner::extract_tar(&buf, &workdir) {
eprintln!("job-worker: untar workspace failed: {e}");
}
if !self.buf.is_empty()
&& let Err(error) =
swactor_job_runner::extract_tar(&self.buf, &self.workdir)
{
eprintln!("job-worker: untar workspace failed: {error}");
}
ready.store(true, Ordering::Release);
self.ready.store(true, Ordering::Release);
ctx.stop_self();
return;
}
_ => {}
}
}
if started.elapsed() >= WORKSPACE_EDGE_WAIT {
if self.started.elapsed() >= WORKSPACE_EDGE_WAIT {
eprintln!("job-worker: workspace edge stream did not arrive");
ctx.stop_self();
return;
}
std::thread::sleep(POLL);
self.schedule(ctx, POLL);
}
}

2986
apps/myelin/src/node/worker_node_runtime.rs
View file

File diff suppressed because it is too large Load diff

116
apps/myelin/src/observability/frame_collector.rs
View file

@ -7,9 +7,9 @@
//! only through these closures.
use iroh::EndpointAddr;
use iroh_driver::{IrohDriver, TelemetryQuicHeader, spawn_pull_collector};
use iroh_driver::{IrohDriver, PullCollectorHandle, TelemetryQuicHeader, spawn_pull_collector};
use parking_lot::Mutex;
use std::collections::BTreeMap;
use std::collections::{BTreeMap, BTreeSet};
use std::sync::{Arc, mpsc};
use swactor_engine::EngineHandle;
use telemetry::frame::{ChannelRef, Frame, StreamId, TelemetryEvent};
@ -38,6 +38,8 @@ pub(crate) struct FrameCollector {
pull_header_rx: mpsc::Receiver<TelemetryQuicHeader>,
pull_channels: Mutex<BTreeMap<ChannelRef, String>>,
pull_streams: Mutex<BTreeMap<StreamId, StreamDescriptor>>,
pull_stream_owners: Mutex<BTreeMap<StreamId, (u64, u64)>>,
pull_collectors: Mutex<BTreeMap<(u64, u64), PullCollectorHandle>>,
}
impl FrameCollector {
@ -61,6 +63,8 @@ impl FrameCollector {
pull_header_rx,
pull_channels: Mutex::new(BTreeMap::new()),
pull_streams: Mutex::new(BTreeMap::new()),
pull_stream_owners: Mutex::new(BTreeMap::new()),
pull_collectors: Mutex::new(BTreeMap::new()),
}
}
@ -76,7 +80,7 @@ impl FrameCollector {
let mut flow_id = [0_u8; 16];
flow_id[..8].copy_from_slice(&run_id.to_le_bytes());
flow_id[8..].copy_from_slice(&node_id.to_le_bytes());
spawn_pull_collector(
let collector = spawn_pull_collector(
engine,
endpoint,
peer,
@ -86,6 +90,36 @@ impl FrameCollector {
Arc::clone(&self.pull_fanout),
self.pull_header_tx.clone(),
);
if let Some(previous) = self
.pull_collectors
.lock()
.insert((run_id, node_id), collector)
{
previous.cancel();
}
}
/// Stop retaining and reconnecting a telemetry subscription for a terminal node.
pub(crate) fn unsubscribe_node(&self, run_id: u64, node_id: u64) {
if let Some(collector) = self.pull_collectors.lock().remove(&(run_id, node_id)) {
collector.cancel();
}
let mut ended = BTreeSet::new();
self.pull_stream_owners.lock().retain(|stream, owner| {
if *owner == (run_id, node_id) {
ended.insert(stream.clone());
false
} else {
true
}
});
if !ended.is_empty() {
self.pull_streams
.lock()
.retain(|stream, _| !ended.contains(stream));
self.pull_channels
.lock()
.retain(|channel, _| !ended.contains(&channel.stream));
}
}
fn pump_pulls(&self) {
@ -93,6 +127,14 @@ impl FrameCollector {
self.pull_streams
.lock()
.insert(header.stream.stream.clone(), header.stream.clone());
let mut run_id = [0_u8; 8];
run_id.copy_from_slice(&header.flow_id[..8]);
let mut node_id = [0_u8; 8];
node_id.copy_from_slice(&header.flow_id[8..]);
self.pull_stream_owners.lock().insert(
header.stream.stream.clone(),
(u64::from_le_bytes(run_id), u64::from_le_bytes(node_id)),
);
let mut channels = self.pull_channels.lock();
for descriptor in header.channels {
channels.insert(
@ -145,6 +187,10 @@ impl FrameCollector {
}
TelemetryEvent::StreamEnded(stream) => {
self.pull_streams.lock().remove(&stream);
self.pull_stream_owners.lock().remove(&stream);
self.pull_channels
.lock()
.retain(|channel, _| channel.stream != stream);
}
}
}
@ -318,5 +364,69 @@ mod tests {
assert_eq!(observed_channel, "host.net");
assert_eq!(observed_frame.position, Position(11));
assert_eq!(observed_frame.payload, br#"{"rx":1}"#);
collector
.pull_fanout
.publish(TelemetryEvent::StreamEnded(stream.clone()));
collector.pump_pulls();
assert!(!collector.pull_streams.lock().contains_key(&stream));
assert!(
collector
.pull_channels
.lock()
.keys()
.all(|channel| channel.stream != stream),
"ended stream retained channel descriptors"
);
}
#[test]
fn node_unsubscribe_releases_abrupt_stream_metadata() {
let collector = FrameCollector::new();
let stream = StreamId::new(NodeId::new("node-9"), Lifetime(4));
let descriptor = StreamDescriptor {
stream: stream.clone(),
label: Some("worker nine".to_owned()),
origin: StreamOrigin::RemoteNode,
};
let channel = ChannelDescriptor {
stream: stream.clone(),
id: ChannelId(3),
name: "runtime.actors".to_owned(),
label: None,
content: ChannelContent::JsonRecord { schema: None },
};
let mut flow_id = [0_u8; 16];
flow_id[..8].copy_from_slice(&5_u64.to_le_bytes());
flow_id[8..].copy_from_slice(&9_u64.to_le_bytes());
collector
.pull_header_tx
.send(TelemetryQuicHeader::new(
flow_id,
Vec::new(),
descriptor,
vec![channel],
))
.unwrap();
collector.pump_pulls();
assert!(collector.pull_streams.lock().contains_key(&stream));
assert!(
collector
.pull_channels
.lock()
.keys()
.any(|channel| channel.stream == stream)
);
collector.unsubscribe_node(5, 9);
assert!(!collector.pull_streams.lock().contains_key(&stream));
assert!(
collector
.pull_channels
.lock()
.keys()
.all(|channel| channel.stream != stream),
"abrupt node stop retained channel descriptors"
);
}
}

2
apps/myelin/src/observability/orch_telemetry.rs
View file

@ -285,7 +285,7 @@ impl DashboardSupport {
.page_script_urls
.push(crate::orchestration::control::FLEET_CONTROL_SCRIPT_URL.to_owned());
let handle = dashboard::DashboardHandle::new(config);
engine.spawn(handle.http_server_with_plugins(plugins));
handle.spawn_with_plugins(engine, plugins);
Ok(Some(Self { handle }))
}

63
apps/myelin/src/observability/provisioning_logs.rs
View file

@ -1,6 +1,3 @@
use std::io::{BufRead, BufReader, Read};
use std::thread::{self, JoinHandle};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use telemetry::{ChannelContent, Lifetime, NodeId, StreamId, TelemetryProducer};
@ -74,66 +71,6 @@ impl BootstrapTelemetryBridge {
});
}
// provider log capture is out of scope (ENGINE_SPEC.md §2)
#[allow(clippy::disallowed_methods)]
pub(crate) fn spawn_stdout_reader<R>(&self, stdout: R) -> JoinHandle<()>
where
R: Read + Send + 'static,
{
let bridge = self.clone();
thread::spawn(move || bridge.read_stdout(stdout))
}
// provider log capture is out of scope (ENGINE_SPEC.md §2)
#[allow(clippy::disallowed_methods)]
pub(crate) fn spawn_stderr_reader<R>(&self, stderr: R) -> JoinHandle<()>
where
R: Read + Send + 'static,
{
let bridge = self.clone();
thread::spawn(move || bridge.read_stderr(stderr))
}
fn read_stdout<R>(&self, stdout: R)
where
R: Read,
{
let reader = BufReader::new(stdout);
for next in reader.lines() {
match next {
Ok(line) => self.observe_stdout_line(line),
Err(error) => {
self.sink.observe(PluginObservation::Failed {
run_id: self.spec.run_id,
node_id: self.spec.node_id,
reason: format!("read stdout: {error}"),
});
break;
}
}
}
}
fn read_stderr<R>(&self, stderr: R)
where
R: Read,
{
let reader = BufReader::new(stderr);
for next in reader.lines() {
match next {
Ok(line) => self.observe_stderr_line(line),
Err(error) => {
self.sink.observe(PluginObservation::Failed {
run_id: self.spec.run_id,
node_id: self.spec.node_id,
reason: format!("read stderr: {error}"),
});
break;
}
}
}
}
fn submit_log(&self, stream: ProvisionLogStream, line: &str) {
let Some(producer) = &self.producer else {
return;

1424
apps/myelin/src/orchestration/app.rs
View file

File diff suppressed because it is too large Load diff

194
apps/myelin/src/orchestration/cluster_reconciler.rs
View file

@ -1,7 +1,9 @@
//! Myelin integration for the provider-neutral cluster reconciler.
use std::collections::{BTreeMap, BTreeSet, VecDeque};
use std::sync::mpsc::{self, Receiver, Sender, TryRecvError};
#[cfg(test)]
use std::sync::mpsc::TryRecvError;
use std::sync::mpsc::{self, Receiver, Sender};
use std::sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard};
use std::time::{Duration, SystemTime};
@ -12,7 +14,9 @@ use provisioning::{
NodeManagerCommand, NodeObservation, NodeStage, OperationOutcome, PlannedEffect,
ProviderLeaseId, RetryPolicy, SshEndpoint, SwactorId,
};
use swactor_engine::EngineHandle;
use swactor::actor::{ActorAddress, ActorInterface};
use swactor::runtime::{Ctx, ExternalSender, Runtime};
use swactor_engine::{BlockingWorkSender, EngineHandle};
use crate::provisioning::{
NodeProvisionSpec, PluginNodeHandle, PluginObservation, PluginObservationSink, PluginSink,
@ -447,12 +451,14 @@ impl EffectBackend for MyelinEffectBackend {
#[derive(Clone)]
pub(crate) struct EngineEffectSpawner {
engine: EngineHandle,
blocking: BlockingWorkSender,
}
impl EngineEffectSpawner {
fn new(engine: EngineHandle) -> Self {
Self { engine }
fn new(engine: &EngineHandle) -> Self {
Self {
blocking: engine.blocking_work_sender(),
}
}
}
@ -463,11 +469,9 @@ impl BlockingEffectSpawner for EngineEffectSpawner {
&self,
work: provisioning::BlockingEffectWork,
) -> Result<(), Self::SpawnError> {
if !self.engine.capabilities().blocking {
return Err("engine blocking work capability is unavailable".to_owned());
}
self.engine.spawn_blocking(work);
Ok(())
self.blocking
.submit(work)
.map_err(|_| "engine stopped before provider effect submission".to_owned())
}
}
@ -476,14 +480,79 @@ enum ControllerWake {
Periodic,
}
#[derive(Clone)]
enum ControllerTimerMsg {
ScheduleDeadline(Option<SystemTime>),
DeadlineElapsed(SystemTime),
PeriodicElapsed,
}
struct ControllerTimerActor {
engine: EngineHandle,
sender: ExternalSender,
wake: Sender<ControllerWake>,
scheduled_deadline: Option<SystemTime>,
}
impl ControllerTimerActor {
fn schedule_periodic(&self, ctx: &Ctx) {
self.engine.send_after(
PERIODIC_RECONCILE,
self.sender.clone(),
ctx.self_addr(),
ControllerTimerMsg::PeriodicElapsed,
);
}
}
impl ActorInterface for ControllerTimerActor {
type Incoming = ControllerTimerMsg;
type Response = ();
fn on_start(&mut self, ctx: &Ctx) {
self.schedule_periodic(ctx);
}
fn handle(&mut self, ctx: &Ctx, message: Self::Incoming) {
match message {
ControllerTimerMsg::ScheduleDeadline(deadline) => {
self.scheduled_deadline = deadline;
if let Some(deadline) = deadline {
self.engine.send_after(
deadline
.duration_since(SystemTime::now())
.unwrap_or(Duration::ZERO),
self.sender.clone(),
ctx.self_addr(),
ControllerTimerMsg::DeadlineElapsed(deadline),
);
}
}
ControllerTimerMsg::DeadlineElapsed(deadline) => {
if self.scheduled_deadline == Some(deadline) {
self.scheduled_deadline = None;
let _ = self.wake.send(ControllerWake::Deadline(deadline));
}
}
ControllerTimerMsg::PeriodicElapsed => {
if self.wake.send(ControllerWake::Periodic).is_ok() {
self.schedule_periodic(ctx);
} else {
ctx.stop_self();
}
}
}
}
}
pub(crate) struct ProvisionedClusterGuard {
driver: ClusterDriver,
executor: IdempotentEffectExecutor<MyelinEffectBackend, EngineEffectSpawner>,
external_nodes: BTreeMap<u64, LogicalNodeId>,
failure_rx: Receiver<TaggedFailure>,
deferred_failures: VecDeque<TaggedFailure>,
engine: EngineHandle,
wake_tx: Sender<ControllerWake>,
runtime: Runtime,
timer_actor: ActorAddress,
wake_rx: Receiver<ControllerWake>,
scheduled_deadline: Option<SystemTime>,
stopped: bool,
@ -495,6 +564,7 @@ impl ProvisionedClusterGuard {
bindings: Vec<ReconcilerNodeBinding>,
retry: RetryPolicy,
engine: EngineHandle,
runtime: Runtime,
sink: PluginSink,
) -> Result<Self, String> {
let expanded = desired.expand().map_err(|error| error.to_string())?;
@ -520,19 +590,25 @@ impl ProvisionedClusterGuard {
}
let (failure_tx, failure_rx) = mpsc::channel();
let (backend, external_nodes) = MyelinEffectBackend::new(bindings, sink, failure_tx)?;
let executor =
IdempotentEffectExecutor::new(backend, EngineEffectSpawner::new(engine.clone()));
let executor = IdempotentEffectExecutor::new(backend, EngineEffectSpawner::new(&engine));
let driver = ClusterDriver::new(desired, retry).map_err(|error| error.to_string())?;
let (wake_tx, wake_rx) = mpsc::channel();
spawn_periodic_wake(&engine, wake_tx.clone());
let timer_actor = runtime
.spawn(ControllerTimerActor {
engine: engine.clone(),
sender: runtime.create_sender(),
wake: wake_tx,
scheduled_deadline: None,
})
.map_err(|error| format!("spawn cluster timer actor: {error}"))?;
Ok(Self {
driver,
executor,
external_nodes,
failure_rx,
deferred_failures: VecDeque::new(),
engine,
wake_tx,
runtime,
timer_actor,
wake_rx,
scheduled_deadline: None,
stopped: false,
@ -657,7 +733,7 @@ impl ProvisionedClusterGuard {
Ok(submitted)
}
fn begin_shutdown(&mut self) -> Result<(), String> {
pub(crate) fn begin_shutdown(&mut self) -> Result<(), String> {
if self.stopped {
return Ok(());
}
@ -673,31 +749,46 @@ impl ProvisionedClusterGuard {
self.driver.state().nodes.is_empty()
}
// Synchronous orchestration waits while all provider work remains engine-hosted.
#[allow(clippy::disallowed_methods)]
pub(crate) fn finish_shutdown(&mut self) -> Result<(), String> {
if !self.stopped {
self.executor.backend().stop_all()?;
self.stopped = true;
}
Ok(())
}
#[cfg(test)]
pub(crate) fn stop(&mut self) -> Result<(), String> {
self.begin_shutdown()?;
while !self.is_stopped() {
self.poll(SystemTime::now())
.map_err(|error| error.to_string())?;
std::thread::sleep(Duration::from_millis(10));
self.wait_for_work(Duration::from_millis(10));
}
self.finish_shutdown()
}
#[cfg(test)]
pub(crate) fn wait_for_work(&mut self, timeout: Duration) {
if let Ok(wake) = self.wake_rx.recv_timeout(timeout) {
self.apply_wake(wake, SystemTime::now());
}
self.executor.backend().stop_all()?;
self.stopped = true;
Ok(())
}
fn drain_wakes(&mut self, now: SystemTime) {
loop {
match self.wake_rx.try_recv() {
Ok(ControllerWake::Periodic) => self.driver.trigger(),
Ok(ControllerWake::Deadline(deadline)) => {
if self.scheduled_deadline == Some(deadline) {
self.scheduled_deadline = None;
}
self.driver.trigger_if_due(now);
while let Ok(wake) = self.wake_rx.try_recv() {
self.apply_wake(wake, now);
}
}
fn apply_wake(&mut self, wake: ControllerWake, now: SystemTime) {
match wake {
ControllerWake::Periodic => self.driver.trigger(),
ControllerWake::Deadline(deadline) => {
if self.scheduled_deadline == Some(deadline) {
self.scheduled_deadline = None;
}
Err(TryRecvError::Empty | TryRecvError::Disconnected) => return,
self.driver.trigger_if_due(now);
}
}
}
@ -798,20 +889,19 @@ impl ProvisionedClusterGuard {
})
{
self.scheduled_deadline = None;
let _ = self
.runtime
.send_to(self.timer_actor, ControllerTimerMsg::ScheduleDeadline(None));
return;
}
if deadline.is_none() || deadline == self.scheduled_deadline {
if deadline == self.scheduled_deadline {
return;
}
let deadline = deadline.expect("checked deadline");
self.scheduled_deadline = Some(deadline);
let delay = deadline.duration_since(now).unwrap_or(Duration::ZERO);
let timer = self.engine.timer(delay);
let wake = self.wake_tx.clone();
self.engine.spawn(async move {
timer.await;
let _ = wake.send(ControllerWake::Deadline(deadline));
});
self.scheduled_deadline = deadline;
let _ = self.runtime.send_to(
self.timer_actor,
ControllerTimerMsg::ScheduleDeadline(deadline),
);
}
}
@ -823,18 +913,6 @@ impl Drop for ProvisionedClusterGuard {
}
}
fn spawn_periodic_wake(engine: &EngineHandle, wake: Sender<ControllerWake>) {
let mut interval = engine.interval(PERIODIC_RECONCILE);
engine.spawn(async move {
loop {
(&mut interval).await;
if wake.send(ControllerWake::Periodic).is_err() {
return;
}
}
});
}
fn lock_node(node: &Mutex<NodeEffects>) -> MutexGuard<'_, NodeEffects> {
node.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
@ -1353,7 +1431,6 @@ mod tests {
}
#[test]
#[allow(clippy::disallowed_methods)]
fn engine_hosted_controller_converges_and_cleans_up_end_to_end() {
let stats = Arc::new(PluginStats::default());
let desired_node = desired();
@ -1391,6 +1468,7 @@ mod tests {
}),
};
let parts = swactor::runtime::RuntimeParts::new(swactor::config::RuntimeConfig::default());
let runtime = parts.runtime().clone();
let backend =
swactor_engine::TokioBackend::new(swactor_engine::TokioConfig::default()).unwrap();
let engine = swactor_engine::Engine::new(parts, backend).unwrap();
@ -1400,6 +1478,7 @@ mod tests {
vec![binding],
RetryPolicy::default(),
engine.handle(),
runtime,
sink,
)
.unwrap();
@ -1503,7 +1582,6 @@ mod tests {
assert_eq!(scaled_stats.stops.load(Ordering::SeqCst), 1);
}
#[test]
#[allow(clippy::disallowed_methods)]
fn ambiguous_create_timeout_retries_by_adoption_and_discards_late_success() {
let stats = Arc::new(PluginStats::default());
let (entered_tx, entered_rx) = mpsc::channel();
@ -1545,6 +1623,7 @@ mod tests {
}),
};
let parts = swactor::runtime::RuntimeParts::new(swactor::config::RuntimeConfig::default());
let runtime = parts.runtime().clone();
let backend =
swactor_engine::TokioBackend::new(swactor_engine::TokioConfig::default()).unwrap();
let engine = swactor_engine::Engine::new(parts, backend).unwrap();
@ -1556,6 +1635,7 @@ mod tests {
..RetryPolicy::default()
},
engine.handle(),
runtime,
PluginSink::new(Arc::new(NullSink)),
)
.unwrap();

794
apps/myelin/src/orchestration/control.rs
View file

@ -1,3 +1,4 @@
use std::sync::{Arc, Mutex};
use std::time::Duration;
use axum::extract::{Path, State};
@ -6,8 +7,9 @@ use axum::response::{IntoResponse, Response};
use axum::routing::{get, post};
use axum::{Json, Router};
use serde::Serialize;
use swactor::actor::ActorAddress;
use swactor::runtime::Runtime;
use swactor::actor::{ActorAddress, ActorInterface};
use swactor::runtime::{Ctx, ExternalSender, Runtime};
use swactor_engine::EngineHandle;
use swactor_vastai::VastClient;
use crate::orchestration::actor::OrchestratorMsg;
@ -20,7 +22,38 @@ const CONTROL_REPLY_TIMEOUT: Duration = Duration::from_secs(2);
const OFFER_SEARCH_REPLY_MARGIN: Duration = Duration::from_secs(5);
const OFFER_SEARCH_REPLY_TIMEOUT: Duration =
VastClient::REQUEST_TIMEOUT.saturating_add(OFFER_SEARCH_REPLY_MARGIN);
const CONTROL_REPLY_POLL: Duration = Duration::from_millis(5);
struct ControlReplyObserver {
reply: Arc<Mutex<Option<tokio::sync::oneshot::Sender<ManualControlReply>>>>,
engine: EngineHandle,
sender: ExternalSender,
timeout: Duration,
}
impl ActorInterface for ControlReplyObserver {
type Incoming = ManualControlReply;
type Response = ();
fn on_start(&mut self, ctx: &Ctx) {
self.engine.send_after(
self.timeout,
self.sender.clone(),
ctx.self_addr(),
ManualControlReply::TimedOut,
);
}
fn handle(&mut self, ctx: &Ctx, reply: Self::Incoming) {
if let Some(response) = self
.reply
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take()
{
let _ = response.send(reply);
}
ctx.stop_self();
}
}
const PROVISION_PAGE: &str = include_str!("provision_page.html");
const FLEET_CONTROL_SCRIPT: &str = include_str!("fleet_control.js");
pub(crate) const FLEET_CONTROL_SCRIPT_URL: &str = "/assets/myelin-fleet-control.js";
@ -28,21 +61,29 @@ pub(crate) const FLEET_CONTROL_SCRIPT_URL: &str = "/assets/myelin-fleet-control.
#[derive(Clone)]
struct ControlHttpState {
runtime: Runtime,
engine: EngineHandle,
orchestrator: ActorAddress,
}
pub(crate) fn plugin(runtime: Runtime, orchestrator: ActorAddress) -> dashboard::DashboardPlugin {
pub(crate) fn plugin(
runtime: Runtime,
engine: EngineHandle,
orchestrator: ActorAddress,
) -> dashboard::DashboardPlugin {
let state = ControlHttpState {
runtime,
engine,
orchestrator,
};
let routes = Router::new()
.route(FLEET_CONTROL_SCRIPT_URL, get(fleet_control_script))
.route("/api/control/status", get(status))
.route("/api/control/actors", get(actor_stats))
.route("/api/control/provision", post(provision))
.route("/api/control/kill", post(kill))
.route("/api/control/provider", post(configure_provider))
.route("/api/control/offers", post(search_offers))
.route("/api/control/flush", post(flush))
.route("/api/control/nodes/{logical_node_id}/kill", post(kill_path))
.with_state(state);
dashboard::DashboardPlugin::new(routes).with_page(dashboard::PluginPage::new(
@ -125,6 +166,10 @@ async fn status(State(state): State<ControlHttpState>) -> Response {
.await
}
async fn actor_stats(State(state): State<ControlHttpState>) -> impl IntoResponse {
Json(state.runtime.stats())
}
async fn search_offers(
State(state): State<ControlHttpState>,
Json(request): Json<OfferSearchRequest>,
@ -135,6 +180,13 @@ async fn search_offers(
.await
}
async fn flush(State(state): State<ControlHttpState>) -> Response {
request_reply(&state, CONTROL_REPLY_TIMEOUT, |reply_to| {
ManualControlMsg::Flush { reply_to }
})
.await
}
fn route_mutation(state: &ControlHttpState, msg: ManualControlMsg) -> Response {
match state
.runtime
@ -156,55 +208,717 @@ async fn request_reply(
timeout: Duration,
build: impl FnOnce(ActorAddress) -> ManualControlMsg,
) -> Response {
let inbox = match state.runtime.new_inbox::<ManualControlReply>() {
Ok(inbox) => inbox,
Err(error) => {
return (
let response_rx = match begin_request_reply(state, timeout, build) {
Ok(response_rx) => response_rx,
Err(response) => return response,
};
match response_rx.await {
Ok(ManualControlReply::Rejected(error)) => {
(StatusCode::CONFLICT, Json(ErrorResponse { error })).into_response()
}
Ok(ManualControlReply::TimedOut) => (
StatusCode::GATEWAY_TIMEOUT,
Json(ErrorResponse {
error: "orchestrator control reply timed out".to_owned(),
}),
)
.into_response(),
Ok(reply) => Json(reply).into_response(),
Err(error) => (
StatusCode::SERVICE_UNAVAILABLE,
Json(ErrorResponse {
error: format!("control reply observer stopped: {error}"),
}),
)
.into_response(),
}
}
fn begin_request_reply(
state: &ControlHttpState,
timeout: Duration,
build: impl FnOnce(ActorAddress) -> ManualControlMsg,
) -> Result<tokio::sync::oneshot::Receiver<ManualControlReply>, Response> {
let (response_tx, response_rx) = tokio::sync::oneshot::channel();
let response_tx = Arc::new(Mutex::new(Some(response_tx)));
let reply_to = state
.runtime
.spawn(ControlReplyObserver {
reply: response_tx,
engine: state.engine.clone(),
sender: state.runtime.create_sender(),
timeout,
})
.map_err(|error| {
(
StatusCode::SERVICE_UNAVAILABLE,
Json(ErrorResponse {
error: format!("create control reply inbox: {error}"),
error: format!("create control reply observer: {error}"),
}),
)
.into_response();
}
};
if let Err(error) = state.runtime.send_to(
state.orchestrator,
OrchestratorMsg::Manual(build(*inbox.addr())),
) {
return (
.into_response()
})?;
if let Err(error) = state
.runtime
.send_to(state.orchestrator, OrchestratorMsg::Manual(build(reply_to)))
{
let _ = state.runtime.stop_actor(reply_to);
return Err((
StatusCode::SERVICE_UNAVAILABLE,
Json(ErrorResponse {
error: format!("orchestrator control actor unavailable: {error}"),
}),
)
.into_response();
}
let deadline = tokio::time::Instant::now() + timeout;
loop {
if let Some(reply) = inbox.try_recv() {
return match reply {
ManualControlReply::Rejected(error) => {
(StatusCode::CONFLICT, Json(ErrorResponse { error })).into_response()
}
reply => Json(reply).into_response(),
};
}
if tokio::time::Instant::now() >= deadline {
return (
StatusCode::GATEWAY_TIMEOUT,
Json(ErrorResponse {
error: "orchestrator control reply timed out".to_owned(),
}),
)
.into_response();
}
tokio::time::sleep(CONTROL_REPLY_POLL).await;
.into_response());
}
Ok(response_rx)
}
#[derive(Serialize)]
struct ErrorResponse {
error: String,
}
#[cfg(test)]
mod properties {
use proptest::prelude::*;
use swactor::config::RuntimeConfig;
use swactor::runtime::RuntimeParts;
use swactor_engine::{Engine, SteppingBackend};
use super::*;
use crate::tests::fuzz_support::{
actor_census, assert_actor_delta_at_most, assert_mailboxes_drained, assert_no_poison,
drive_steps,
};
const STEP_BUDGET: usize = 64;
fn reply(code: u8) -> ManualControlReply {
match code % 3 {
0 => ManualControlReply::Flushed,
1 => ManualControlReply::Rejected(format!("rejected-{code}")),
_ => ManualControlReply::TimedOut,
}
}
#[derive(Clone, Debug)]
enum HttpAction {
Provision { command_slot: u8, count: u8 },
Kill { command_slot: u8, node: u8 },
KillPath { command_slot: u8, node: u8 },
Configure { value: u8 },
Status,
ActorStats,
Flush,
SearchOffers,
}
impl HttpAction {
fn from_raw(kind: u8, command_slot: u8, value: u8) -> Self {
match kind % 8 {
0 => Self::Provision {
command_slot,
count: value,
},
1 => Self::Kill {
command_slot,
node: value,
},
2 => Self::KillPath {
command_slot,
node: value,
},
3 => Self::Configure { value },
4 => Self::Status,
5 => Self::ActorStats,
6 => Self::Flush,
_ => Self::SearchOffers,
}
}
fn command_id(command_slot: u8) -> String {
format!("repeated-{}", command_slot % 4)
}
fn expected_bridge_observation(&self) -> Option<String> {
match self {
Self::Provision { command_slot, .. } => {
Some(format!("Provision:{}", Self::command_id(*command_slot)))
}
Self::Kill { command_slot, .. } | Self::KillPath { command_slot, .. } => {
Some(format!("Kill:{}", Self::command_id(*command_slot)))
}
Self::Configure { .. } => Some("Configure".to_owned()),
Self::Status => Some("Status".to_owned()),
Self::ActorStats => None,
Self::Flush => Some("Flush".to_owned()),
Self::SearchOffers => Some("SearchOffers".to_owned()),
}
}
fn is_mutation(&self) -> bool {
matches!(
self,
Self::Provision { .. }
| Self::Kill { .. }
| Self::KillPath { .. }
| Self::Configure { .. }
)
}
fn uses_reply_observer(&self) -> bool {
matches!(self, Self::Status | Self::Flush | Self::SearchOffers)
}
}
#[derive(Clone, Debug)]
struct HttpObservation {
index: usize,
action: HttpAction,
status: StatusCode,
}
struct HttpBridgeProbe {
observations: Arc<Mutex<Vec<String>>>,
disappear_reply_observers: bool,
}
impl HttpBridgeProbe {
fn finish_reply(&self, ctx: &Ctx, reply_to: ActorAddress, reply: ManualControlReply) {
if self.disappear_reply_observers {
let _ = ctx.stop_actor(reply_to);
} else {
let _ = ctx.send(reply_to, reply);
}
}
}
impl ActorInterface for HttpBridgeProbe {
type Incoming = OrchestratorMsg;
type Response = ();
fn handle(&mut self, ctx: &Ctx, message: Self::Incoming) {
let OrchestratorMsg::Manual(message) = message else {
return;
};
match message {
ManualControlMsg::Provision { request, .. } => self
.observations
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(format!("Provision:{}", request.command_id)),
ManualControlMsg::Kill { request, .. } => self
.observations
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(format!("Kill:{}", request.command_id)),
ManualControlMsg::Configure { .. } => self
.observations
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push("Configure".to_owned()),
ManualControlMsg::Query { reply_to } => {
self.observations
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push("Status".to_owned());
self.finish_reply(ctx, reply_to, ManualControlReply::Flushed);
}
ManualControlMsg::Flush { reply_to } => {
self.observations
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push("Flush".to_owned());
self.finish_reply(ctx, reply_to, ManualControlReply::Flushed);
}
ManualControlMsg::SearchOffers { reply_to, .. } => {
self.observations
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push("SearchOffers".to_owned());
self.finish_reply(ctx, reply_to, ManualControlReply::Offers(Vec::new()));
}
_ => {}
}
}
}
enum PendingHttpObservation {
Ready(HttpObservation),
Reply {
index: usize,
action: HttpAction,
receiver: tokio::sync::oneshot::Receiver<ManualControlReply>,
},
}
fn begin_http_action(
state: &ControlHttpState,
index: usize,
action: HttpAction,
) -> PendingHttpObservation {
let immediate = |status| {
PendingHttpObservation::Ready(HttpObservation {
index,
action: action.clone(),
status,
})
};
let reply = |result: Result<_, Response>| match result {
Ok(receiver) => PendingHttpObservation::Reply {
index,
action: action.clone(),
receiver,
},
Err(response) => immediate(response.status()),
};
match &action {
HttpAction::Provision {
command_slot,
count,
} => immediate(
route_mutation(
state,
ManualControlMsg::Provision {
request: ProvisionRequest {
command_id: HttpAction::command_id(*command_slot),
count: u32::from(*count),
selected_offer_ids: Vec::new(),
},
reply_to: None,
},
)
.status(),
),
HttpAction::Kill { command_slot, node }
| HttpAction::KillPath { command_slot, node } => immediate(
route_mutation(
state,
ManualControlMsg::Kill {
request: KillRequest {
command_id: HttpAction::command_id(*command_slot),
logical_node_id: u64::from(*node),
},
reply_to: None,
},
)
.status(),
),
HttpAction::Configure { value } => immediate(
route_mutation(
state,
ManualControlMsg::Configure {
request: ProviderConfigurationRequest {
api_key: Some(format!("generated-key-{value}")),
ssh_identity: None,
bootstrap_command: None,
},
reply_to: None,
},
)
.status(),
),
HttpAction::Status => reply(begin_request_reply(
state,
CONTROL_REPLY_TIMEOUT,
|reply_to| ManualControlMsg::Query { reply_to },
)),
HttpAction::ActorStats => immediate(StatusCode::OK),
HttpAction::Flush => reply(begin_request_reply(
state,
CONTROL_REPLY_TIMEOUT,
|reply_to| ManualControlMsg::Flush { reply_to },
)),
HttpAction::SearchOffers => reply(begin_request_reply(
state,
OFFER_SEARCH_REPLY_TIMEOUT,
|reply_to| ManualControlMsg::SearchOffers {
request: OfferSearchRequest::default(),
reply_to,
},
)),
}
}
fn finish_http_action(observation: PendingHttpObservation) -> Result<HttpObservation, String> {
let PendingHttpObservation::Reply {
index,
action,
mut receiver,
} = observation
else {
let PendingHttpObservation::Ready(observation) = observation else {
unreachable!("pending HTTP observation variant changed")
};
return Ok(observation);
};
let status = match receiver.try_recv() {
Ok(ManualControlReply::Rejected(_)) => StatusCode::CONFLICT,
Ok(ManualControlReply::TimedOut) => StatusCode::GATEWAY_TIMEOUT,
Ok(_) => StatusCode::OK,
Err(tokio::sync::oneshot::error::TryRecvError::Closed) => {
StatusCode::SERVICE_UNAVAILABLE
}
Err(tokio::sync::oneshot::error::TryRecvError::Empty) => {
return Err(format!(
"control reply remained pending after fixed step budget; \
index={index}, action={action:?}"
));
}
};
Ok(HttpObservation {
index,
action,
status,
})
}
fn http_bridge_invariant_failure(
actions: &[HttpAction],
responses: &[HttpObservation],
observed: &[String],
state: &ControlHttpState,
baseline_actors: usize,
disappear_reply_observers: bool,
) -> Option<String> {
let mut expected_bridge = actions
.iter()
.filter_map(HttpAction::expected_bridge_observation)
.collect::<Vec<_>>();
expected_bridge.sort();
let mut actual_bridge = observed.to_vec();
actual_bridge.sort();
let statuses_valid = responses.iter().all(|response| {
let expected = if response.action.is_mutation() {
StatusCode::ACCEPTED
} else if disappear_reply_observers && response.action.uses_reply_observer() {
StatusCode::SERVICE_UNAVAILABLE
} else {
StatusCode::OK
};
response.status == expected
});
let stats = state.runtime.stats();
let panics = stats
.workers
.iter()
.map(|worker| worker.panics)
.sum::<u64>();
let mailbox_depth = stats
.workers
.iter()
.map(|worker| worker.mailbox_depth)
.sum::<usize>()
+ stats
.actor_details
.iter()
.map(|actor| actor.mailbox_depth)
.sum::<usize>();
if responses.len() == actions.len()
&& statuses_valid
&& actual_bridge == expected_bridge
&& stats.actors.len() == baseline_actors
&& stats.actor_details.iter().all(|actor| !actor.poisoned)
&& panics == 0
&& mailbox_depth == 0
{
None
} else {
Some(format!(
"responses={responses:?}, expected_bridge={expected_bridge:?}, \
observed_bridge={actual_bridge:?}, expected_actor_count={baseline_actors}, \
mailbox_depth={mailbox_depth}, actor_census=\n{}",
actor_census(&state.runtime),
))
}
}
proptest! {
#![proptest_config(ProptestConfig {
cases: 128,
max_shrink_iters: 2_000,
..ProptestConfig::default()
})]
#[test]
fn generated_http_bridge_sequences_terminate_without_control_actor_growth(
raw_actions in prop::collection::vec(
(any::<u8>(), any::<u8>(), any::<u8>()),
0..=32,
),
concurrent in any::<bool>(),
disappear_reply_observers in any::<bool>(),
) {
let actions = raw_actions
.into_iter()
.map(|(kind, command_slot, value)| {
HttpAction::from_raw(kind, command_slot, value)
})
.collect::<Vec<_>>();
let mut config = RuntimeConfig::default();
config.worker_count = 1;
let parts = RuntimeParts::new(config);
let runtime = parts.runtime().clone();
let backend = SteppingBackend::new();
let engine =
Engine::new(parts, backend.clone()).expect("control HTTP stepping engine");
let observations = Arc::new(Mutex::new(Vec::new()));
let orchestrator = runtime
.spawn(HttpBridgeProbe {
observations: Arc::clone(&observations),
disappear_reply_observers,
})
.expect("spawn control HTTP bridge probe");
drive_steps(&backend, STEP_BUDGET);
let baseline_actors = runtime.stats().actors.len();
let baseline_tasks = backend.pending_task_count();
let state = ControlHttpState {
runtime: runtime.clone(),
engine: engine.handle(),
orchestrator,
};
let outcome = (|| {
let mut responses = Vec::with_capacity(actions.len());
if concurrent {
let pending = actions
.iter()
.cloned()
.enumerate()
.map(|(index, action)| begin_http_action(&state, index, action))
.collect::<Vec<_>>();
drive_steps(&backend, STEP_BUDGET);
drive_steps(&backend, STEP_BUDGET);
for observation in pending {
responses.push(finish_http_action(observation)?);
}
} else {
for (index, action) in actions.iter().cloned().enumerate() {
let pending = begin_http_action(&state, index, action);
drive_steps(&backend, STEP_BUDGET);
drive_steps(&backend, STEP_BUDGET);
responses.push(finish_http_action(pending)?);
}
}
responses.sort_by_key(|response| response.index);
Ok::<_, String>(responses)
})();
prop_assert!(
outcome.is_ok(),
"control HTTP request did not terminate; actions={:?}; error={:?}; \
responses=[]; actor_census=\n{}",
actions,
outcome.as_ref().err(),
actor_census(&runtime),
);
let responses = outcome.expect("outcome checked above");
drive_steps(&backend, STEP_BUDGET);
let observed = observations
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
let failure = http_bridge_invariant_failure(
&actions,
&responses,
&observed,
&state,
baseline_actors,
disappear_reply_observers,
);
prop_assert!(
failure.is_none(),
"control HTTP/bridge invariant failed; actions={:?}; responses={:?}; failure={}",
actions,
responses,
failure.unwrap_or_default(),
);
backend.advance_time(OFFER_SEARCH_REPLY_TIMEOUT);
drive_steps(&backend, STEP_BUDGET);
prop_assert_eq!(
backend.pending_task_count(),
baseline_tasks,
"control reply timers survived their fixed drain budget; actions={:?}; \
responses={:?}; actor_census=\n{}",
actions,
responses,
actor_census(&runtime),
);
runtime
.stop_actor(orchestrator)
.expect("stop control HTTP bridge probe");
drive_steps(&backend, STEP_BUDGET);
assert_no_poison(&runtime);
assert_actor_delta_at_most(&runtime, 0, 0);
assert_mailboxes_drained(&runtime);
}
#[test]
fn generated_duplicate_control_replies_deliver_first_once_and_remove_observer(
replies in prop::collection::vec(any::<u8>(), 0..=16)
) {
let mut config = RuntimeConfig::default();
config.worker_count = 1;
let parts = RuntimeParts::new(config);
let runtime = parts.runtime().clone();
let backend = SteppingBackend::new();
let engine =
Engine::new(parts, backend.clone()).expect("control reply stepping engine");
let baseline_actors = runtime.stats().actors.len();
let baseline_tasks = backend.pending_task_count();
let (response_tx, mut response_rx) = tokio::sync::oneshot::channel();
let observer = runtime
.spawn(ControlReplyObserver {
reply: Arc::new(Mutex::new(Some(response_tx))),
engine: engine.handle(),
sender: runtime.create_sender(),
timeout: Duration::from_millis(1),
})
.expect("spawn control reply observer");
for code in &replies {
runtime
.send_to(observer, reply(*code))
.expect("send generated control reply");
}
drive_steps(&backend, 32);
if replies.is_empty() {
backend.advance_time(Duration::from_millis(1));
drive_steps(&backend, 32);
}
let observed = response_rx
.try_recv()
.expect("control reply observer produced a terminal reply");
let expected = replies
.first()
.map(|code| reply(*code))
.unwrap_or(ManualControlReply::TimedOut);
prop_assert_eq!(
observed,
expected,
"control reply observer accepted a stale duplicate; replies={:?}; \
actor_census=\n{}",
replies,
actor_census(&runtime),
);
backend.advance_time(Duration::from_millis(1));
drive_steps(&backend, 32);
prop_assert_eq!(backend.pending_task_count(), baseline_tasks);
assert_no_poison(&runtime);
assert_actor_delta_at_most(&runtime, baseline_actors, 0);
assert_mailboxes_drained(&runtime);
}
}
#[test]
fn disappeared_orchestrator_removes_control_reply_actor() {
let parts = RuntimeParts::new(RuntimeConfig::default());
let runtime = parts.runtime().clone();
let backend = SteppingBackend::new();
let engine = Engine::new(parts, backend.clone()).expect("control failure stepping engine");
let baseline_tasks = backend.pending_task_count();
let state = ControlHttpState {
runtime: runtime.clone(),
engine: engine.handle(),
orchestrator: ActorAddress::default(),
};
let response = match begin_request_reply(&state, Duration::from_millis(1), |reply_to| {
ManualControlMsg::Query { reply_to }
}) {
Ok(_) => panic!("missing orchestrator unexpectedly accepted a control query"),
Err(response) => response,
};
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
drive_steps(&backend, STEP_BUDGET);
backend.advance_time(Duration::from_millis(1));
drive_steps(&backend, STEP_BUDGET);
assert_eq!(backend.pending_task_count(), baseline_tasks);
assert_no_poison(&runtime);
assert_actor_delta_at_most(&runtime, 0, 0);
assert_mailboxes_drained(&runtime);
}
#[test]
fn reply_observer_disappearance_returns_a_bounded_terminal_http_response() {
let mut config = RuntimeConfig::default();
config.worker_count = 1;
let parts = RuntimeParts::new(config);
let runtime = parts.runtime().clone();
let backend = SteppingBackend::new();
let engine =
Engine::new(parts, backend.clone()).expect("reply disappearance stepping engine");
let observations = Arc::new(Mutex::new(Vec::new()));
let orchestrator = runtime
.spawn(HttpBridgeProbe {
observations,
disappear_reply_observers: true,
})
.expect("spawn disappearing-reply bridge");
drive_steps(&backend, STEP_BUDGET);
let baseline_actors = runtime.stats().actors.len();
let baseline_tasks = backend.pending_task_count();
let state = ControlHttpState {
runtime: runtime.clone(),
engine: engine.handle(),
orchestrator,
};
let pending = begin_http_action(&state, 0, HttpAction::Status);
drive_steps(&backend, STEP_BUDGET);
drive_steps(&backend, STEP_BUDGET);
let response = finish_http_action(pending).unwrap_or_else(|error| {
panic!(
"{error}; responses=[]; actor_census=\n{}",
actor_census(&runtime),
)
});
assert_eq!(
response.status,
StatusCode::SERVICE_UNAVAILABLE,
"reply observer disappearance returned {}; actor_census=\n{}",
response.status,
actor_census(&runtime),
);
drive_steps(&backend, STEP_BUDGET);
assert_actor_delta_at_most(&runtime, baseline_actors, 0);
backend.advance_time(CONTROL_REPLY_TIMEOUT);
drive_steps(&backend, STEP_BUDGET);
assert_eq!(backend.pending_task_count(), baseline_tasks);
runtime
.stop_actor(orchestrator)
.expect("stop disappearing-reply bridge");
drive_steps(&backend, STEP_BUDGET);
assert_no_poison(&runtime);
assert_actor_delta_at_most(&runtime, 0, 0);
assert_mailboxes_drained(&runtime);
}
#[test]
fn http_bridge_invariant_rejects_a_controlled_duplicate_forward() {
let parts = RuntimeParts::new(RuntimeConfig::default());
let runtime = parts.runtime().clone();
let backend = SteppingBackend::new();
let engine =
Engine::new(parts, backend.clone()).expect("control invariant stepping engine");
let state = ControlHttpState {
runtime,
engine: engine.handle(),
orchestrator: ActorAddress::default(),
};
let actions = vec![HttpAction::Status];
let responses = vec![HttpObservation {
index: 0,
action: HttpAction::Status,
status: StatusCode::OK,
}];
let duplicated = vec!["Status".to_owned(), "Status".to_owned()];
assert!(
http_bridge_invariant_failure(&actions, &responses, &duplicated, &state, 0, false,)
.is_some(),
"control HTTP/bridge invariant accepted a controlled duplicate forward"
);
}
}

80
apps/myelin/src/orchestration/distribution_stack.rs
View file

@ -13,7 +13,7 @@ use std::time::{Duration, Instant};
use swactor::actor::{ActorAddress, ActorInterface};
use swactor::config::RuntimeConfig;
use swactor::runtime::{Ctx, Runtime, RuntimeParts};
use swactor::runtime::{Ctx, ExternalSender, Runtime, RuntimeParts};
use swactor::stats::StatsHook;
use swactor::std::StdExtension;
use swactor_engine::EngineHandle;
@ -37,6 +37,49 @@ use distribution::transport_bridge::{
};
use distribution::types::{DirectoryEntry, MemberState, NodeId};
#[derive(Clone)]
struct ProtocolTick;
struct ProtocolTicker {
runtime: Runtime,
engine: EngineHandle,
sender: ExternalSender,
period: Duration,
swim: ActorAddress,
registry: ActorAddress,
metadata: ActorAddress,
directory: ActorAddress,
}
impl ProtocolTicker {
fn schedule(&self, ctx: &Ctx) {
self.engine.send_after(
self.period,
self.sender.clone(),
ctx.self_addr(),
ProtocolTick,
);
}
}
impl ActorInterface for ProtocolTicker {
type Incoming = ProtocolTick;
type Response = ();
fn on_start(&mut self, ctx: &Ctx) {
self.schedule(ctx);
}
fn handle(&mut self, ctx: &Ctx, _message: Self::Incoming) {
let now = self.engine.now().to_instant();
let _ = self.runtime.send_to(self.swim, SwimIn::Tick { now });
let _ = self.runtime.send_to(self.registry, RegistryIn::Tick);
let _ = self.runtime.send_to(self.metadata, MetadataIn::Tick);
let _ = self.runtime.send_to(self.directory, DirectoryIn::Tick);
self.schedule(ctx);
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) struct DistributionActorAddrs {
pub swim: ActorAddress,
@ -223,29 +266,20 @@ impl DistributionRuntimeStack {
routes
}
/// Spawn an engine-hosted interval task that injects protocol Tick messages
/// (SWIM, registry, metadata, directory), replacing the manual tick
/// injection previously done by the application pump loop
/// (ENGINE_SPEC.md). The engine owns protocol progression; the
/// application loop no longer calls tick or core-driving methods.
/// Spawn the actor that owns periodic distribution protocol ticks.
pub(crate) fn spawn_protocol_ticker(&self, period: Duration) {
let runtime = self.runtime.clone();
let swim = self.actors.swim;
let registry = self.actors.registry;
let metadata = self.actors.metadata;
let directory = self.actors.directory;
let engine = self.engine.clone();
engine.clone().spawn(async move {
let mut interval = engine.interval(period);
loop {
(&mut interval).await;
let now = engine.now().to_instant();
let _ = runtime.send_to(swim, SwimIn::Tick { now });
let _ = runtime.send_to(registry, RegistryIn::Tick);
let _ = runtime.send_to(metadata, MetadataIn::Tick);
let _ = runtime.send_to(directory, DirectoryIn::Tick);
}
});
self.runtime
.spawn(ProtocolTicker {
runtime: self.runtime.clone(),
engine: self.engine.clone(),
sender: self.runtime.create_sender(),
period,
swim: self.actors.swim,
registry: self.actors.registry,
metadata: self.actors.metadata,
directory: self.actors.directory,
})
.expect("spawn distribution protocol ticker actor");
}
pub(crate) fn register_local_actor(&self, entry: DirectoryEntry) {

317
apps/myelin/src/orchestration/job_reconciler.rs
View file

@ -14,10 +14,13 @@ use provisioning::{
BootSpec, ClusterShape, DesiredNodeShape, LogicalNodeId, NodeAttemptId, NodeGroupId,
ProviderKind, RetryPolicy, RoleId, RunId, RunNodeGroupSpec, SwactorId, SwarmJoinTemplate,
};
use swactor::actor::ActorInterface;
use swactor::runtime::{Ctx, ExternalSender};
use swactor_engine::{ActorCompletion, EngineHandle};
use swactor_job_runner::{Job, JobDone};
use swactor_vastai::SelectionPolicy;
use crate::job_deploy::{self, NodeIdentity};
use crate::job_deploy::{self, JobRunStateMachine, NodeIdentity};
use crate::orchestration::app::{
derive_ssh_public_key, ensure_vastai_account_ssh_key, resolve_vastai_ssh_identity,
ssh_public_key_fingerprint,
@ -124,7 +127,7 @@ pub(crate) fn run_vastai_job(
validate_non_empty("worker workdir", &options.worker_workdir)?;
validate_non_empty("endpoint address mask", &options.endpoint_addr_mask)?;
let mut session = job_deploy::start_orchestrator(landing)?;
let session = job_deploy::start_orchestrator(landing)?;
let orch_json = session.identity_json()?;
println!("JOB_ORCH_IDENTITY {orch_json}");
let _ = std::io::Write::flush(&mut std::io::stdout());
@ -134,102 +137,235 @@ pub(crate) fn run_vastai_job(
);
let (sink, observations) = observation_channel();
let provisioner = build_vastai_provisioner(&api_key, &options, session.runtime())?;
let runtime = session.runtime();
let engine = session.engine_handle();
let provisioner =
build_vastai_provisioner(&api_key, &options, runtime.clone(), engine.clone())?;
let spec = job_node_spec(&options, image, &orch_json)?;
let mut cluster = build_cluster(&options, spec, provisioner, session.engine_handle(), sink)?;
let cluster = build_cluster(
&options,
spec,
provisioner,
engine.clone(),
runtime.clone(),
sink,
)?;
let completion = ActorCompletion::new();
runtime
.spawn(ReconciledJobActor {
cluster,
observations,
session: Some(session),
job: Some(job),
phase: ReconciledJobPhase::Provisioning {
deadline: Instant::now() + options.provision_timeout,
},
node_id: options.node_id,
engine,
sender: runtime.create_sender(),
completion: completion.clone(),
})
.map_err(|error| format!("spawn reconciled job actor: {error}"))?;
completion.wait()
}
let result = run_with_cluster(job, &mut session, &mut cluster, &observations, &options);
let stop_result = cluster.stop();
match (result, stop_result) {
(Ok(done), Ok(())) => Ok(done),
(Ok(_), Err(cleanup)) => Err(format!(
#[derive(Clone)]
struct ReconciledJobTick;
enum ReconciledJobPhase {
Provisioning {
deadline: Instant,
},
Converging {
deadline: Instant,
worker: NodeIdentity,
},
Running(JobRunStateMachine),
Stopping {
result: Result<JobDone, String>,
},
Finished,
}
struct ReconciledJobActor {
cluster: ProvisionedClusterGuard,
observations: mpsc::Receiver<PluginObservation>,
session: Option<job_deploy::JobOrchestratorSession>,
job: Option<Job>,
phase: ReconciledJobPhase,
node_id: u64,
engine: EngineHandle,
sender: ExternalSender,
completion: ActorCompletion<Result<JobDone, String>>,
}
impl ReconciledJobActor {
fn schedule(&self, ctx: &Ctx) {
self.engine.send_after(
POLL,
self.sender.clone(),
ctx.self_addr(),
ReconciledJobTick,
);
}
fn begin_stop(&mut self, result: Result<JobDone, String>) {
let result = match self.cluster.begin_shutdown() {
Ok(()) => result,
Err(cleanup) => merge_cleanup(result, cleanup),
};
self.phase = ReconciledJobPhase::Stopping { result };
}
fn complete(
&mut self,
ctx: &Ctx,
result: Result<JobDone, String>,
cleanup: Result<(), String>,
) {
let result = match cleanup {
Ok(()) => result,
Err(cleanup) => merge_cleanup(result, cleanup),
};
assert!(
self.completion.complete(result).is_ok(),
"reconciled job completed twice"
);
self.phase = ReconciledJobPhase::Finished;
ctx.stop_self();
}
}
impl ActorInterface for ReconciledJobActor {
type Incoming = ReconciledJobTick;
type Response = ();
fn on_start(&mut self, ctx: &Ctx) {
let _ = ctx.send(ctx.self_addr(), ReconciledJobTick);
}
fn handle(&mut self, ctx: &Ctx, _message: Self::Incoming) {
if let Err(error) = self.cluster.poll(SystemTime::now()) {
let cleanup = self.cluster.finish_shutdown();
self.complete(ctx, Err(format!("job reconciler poll: {error}")), cleanup);
return;
}
let phase = std::mem::replace(&mut self.phase, ReconciledJobPhase::Finished);
match phase {
ReconciledJobPhase::Provisioning { deadline } => {
let mut worker = None;
if let Err(error) = drain_observations(&self.observations, &mut worker) {
self.begin_stop(Err(error));
} else if let Some(worker) = worker {
eprintln!("job-reconcile: worker identity observed through reconciler stdout");
let result = job_deploy::parse_actor(&worker.actor_hex).and_then(|actor| {
let attempt = self.cluster.current_attempt(self.node_id).ok_or_else(|| {
format!(
"reconciler has no active attempt for node {}",
self.node_id
)
})?;
self.cluster
.observe_runtime_ready(
self.node_id,
NodeAttemptId(attempt.0),
SwactorId(format!("{actor:?}")),
SystemTime::now(),
)
.then_some(())
.ok_or_else(|| {
format!(
"reconciler rejected runtime-ready observation for node {} attempt {}",
self.node_id, attempt.0
)
})
});
match result {
Ok(()) => {
self.phase = ReconciledJobPhase::Converging {
deadline: Instant::now() + RUNTIME_CONVERGENCE_TIMEOUT,
worker,
};
}
Err(error) => self.begin_stop(Err(error)),
}
} else if Instant::now() >= deadline {
self.begin_stop(Err(format!(
"timed out waiting for reconciled job worker identity"
)));
} else {
self.phase = ReconciledJobPhase::Provisioning { deadline };
}
}
ReconciledJobPhase::Converging { deadline, worker } => {
let mut ignored = None;
if let Err(error) = drain_observations(&self.observations, &mut ignored) {
self.begin_stop(Err(error));
} else if self.cluster.is_converged() {
eprintln!("job-reconcile: reconciler accepted runtime-ready worker");
let machine = self
.session
.take()
.zip(self.job.take())
.ok_or_else(|| "reconciled job lost session state".to_owned())
.and_then(|(session, job)| JobRunStateMachine::new(session, job, worker));
match machine {
Ok(mut machine) => {
machine.start(Instant::now());
self.phase = ReconciledJobPhase::Running(machine);
}
Err(error) => self.begin_stop(Err(error)),
}
} else if Instant::now() >= deadline {
self.begin_stop(Err(format!(
"timed out after {RUNTIME_CONVERGENCE_TIMEOUT:?} waiting for reconciler convergence"
)));
} else {
self.phase = ReconciledJobPhase::Converging { deadline, worker };
}
}
ReconciledJobPhase::Running(mut machine) => {
let mut ignored = None;
if let Err(error) = drain_observations(&self.observations, &mut ignored) {
self.begin_stop(Err(error));
} else if let Some(result) = machine.advance(Instant::now()) {
self.begin_stop(result);
} else {
self.phase = ReconciledJobPhase::Running(machine);
}
}
ReconciledJobPhase::Stopping { result } => {
if self.cluster.is_stopped() {
let cleanup = self.cluster.finish_shutdown();
self.complete(ctx, result, cleanup);
return;
}
self.phase = ReconciledJobPhase::Stopping { result };
}
ReconciledJobPhase::Finished => {
ctx.stop_self();
return;
}
}
self.schedule(ctx);
}
}
fn merge_cleanup(result: Result<JobDone, String>, cleanup: String) -> Result<JobDone, String> {
match result {
Ok(_) => Err(format!(
"job completed but reconciler cleanup failed: {cleanup}"
)),
(Err(error), Ok(())) => Err(error),
(Err(error), Err(cleanup)) => Err(format!("{error}; reconciler cleanup failed: {cleanup}")),
Err(error) => Err(format!("{error}; reconciler cleanup failed: {cleanup}")),
}
}
fn run_with_cluster(
job: Job,
session: &mut job_deploy::JobOrchestratorSession,
cluster: &mut ProvisionedClusterGuard,
observations: &mpsc::Receiver<PluginObservation>,
options: &VastAiJobOptions,
) -> Result<JobDone, String> {
let worker = wait_for_worker_identity(cluster, observations, options)?;
let actor = job_deploy::parse_actor(&worker.actor_hex)?;
let attempt = cluster.current_attempt(options.node_id).ok_or_else(|| {
format!(
"reconciler has no active attempt for node {}",
options.node_id
)
})?;
if !cluster.observe_runtime_ready(
options.node_id,
NodeAttemptId(attempt.0),
SwactorId(format!("{actor:?}")),
SystemTime::now(),
) {
return Err(format!(
"reconciler rejected runtime-ready observation for node {} attempt {}",
options.node_id, attempt.0
));
}
wait_for_cluster_convergence(cluster, observations)?;
session.run_to_completion(job, worker)
}
fn wait_for_worker_identity(
cluster: &mut ProvisionedClusterGuard,
observations: &mpsc::Receiver<PluginObservation>,
options: &VastAiJobOptions,
) -> Result<NodeIdentity, String> {
let started = Instant::now();
let mut worker = None;
while started.elapsed() < options.provision_timeout {
cluster
.poll(SystemTime::now())
.map_err(|error| format!("job reconciler poll: {error}"))?;
drain_observations(observations, &mut worker)?;
if let Some(worker) = worker.take() {
eprintln!("job-reconcile: worker identity observed through reconciler stdout");
return Ok(worker);
}
std::thread::sleep(POLL);
}
Err(format!(
"timed out after {:?} waiting for reconciled job worker identity",
options.provision_timeout
))
}
fn wait_for_cluster_convergence(
cluster: &mut ProvisionedClusterGuard,
observations: &mpsc::Receiver<PluginObservation>,
) -> Result<(), String> {
let started = Instant::now();
let mut ignored = None;
while started.elapsed() < RUNTIME_CONVERGENCE_TIMEOUT {
cluster
.poll(SystemTime::now())
.map_err(|error| format!("job reconciler convergence poll: {error}"))?;
drain_observations(observations, &mut ignored)?;
if cluster.is_converged() {
eprintln!("job-reconcile: reconciler accepted runtime-ready worker");
return Ok(());
}
std::thread::sleep(POLL);
}
Err(format!(
"timed out after {RUNTIME_CONVERGENCE_TIMEOUT:?} waiting for reconciler convergence"
))
}
fn build_vastai_provisioner(
api_key: &str,
options: &VastAiJobOptions,
runtime: swactor::runtime::Runtime,
engine: swactor_engine::EngineHandle,
) -> Result<Box<dyn ProvisionPlugin>, String> {
let identity = resolve_vastai_ssh_identity(options.ssh_identity.clone())?;
if !identity.is_file() {
@ -259,8 +395,9 @@ fn build_vastai_provisioner(
}
Ok(Box::new(VastAiProvisioningPlugin::new(
ToolsVastAiLeaseClient::from_api_key(api_key.to_owned())?,
SshCommandBootstrapLauncher::new(Some(identity), runtime),
ToolsVastAiLeaseClient::from_api_key(api_key.to_owned())?
.with_actor_host(runtime.clone(), engine.clone()),
SshCommandBootstrapLauncher::new(Some(identity), runtime, engine),
config,
)))
}
@ -320,6 +457,7 @@ fn build_cluster(
spec: NodeProvisionSpec,
provisioner: Box<dyn ProvisionPlugin>,
engine: swactor_engine::EngineHandle,
runtime: swactor::runtime::Runtime,
sink: PluginSink,
) -> Result<ProvisionedClusterGuard, String> {
let group_id = NodeGroupId(format!("job-node-{}", spec.node_id));
@ -375,6 +513,7 @@ fn build_cluster(
}],
retry,
engine,
runtime,
sink,
)
}

1427
apps/myelin/src/orchestration/manual_control.rs
View file

File diff suppressed because it is too large Load diff

56
apps/myelin/src/orchestration/node_image.rs
View file

@ -220,11 +220,10 @@ fn prepare_node_image_inner(
}
fn workspace_root() -> Result<PathBuf, String> {
let output = Command::new("git")
let output = swactor_process::command_output(&mut Command::new("git")
.args(["rev-parse", "--show-toplevel"])
.current_dir(env!("CARGO_MANIFEST_DIR"))
.stdin(Stdio::null())
.output()
.stdin(Stdio::null()))
.map_err(|e| format!("locate repository root with git: {e}"))?;
if !output.status.success() {
return Err(format!(
@ -251,11 +250,10 @@ fn image_version_tag(root: &Path, image_content_hash: &str) -> Result<String, St
}
fn git_capture(root: &Path, args: &[&str]) -> Result<String, String> {
let output = Command::new("git")
let output = swactor_process::command_output(&mut Command::new("git")
.current_dir(root)
.args(args)
.stdin(Stdio::null())
.output()
.stdin(Stdio::null()))
.map_err(|e| format!("run git {}: {e}", args.join(" ")))?;
if output.status.success() {
Ok(String::from_utf8_lossy(&output.stdout).to_string())
@ -613,8 +611,6 @@ enum CommandOutputLine {
Stderr(String),
}
// container image build is provisioning infrastructure, out of scope (ENGINE_SPEC.md §2)
#[allow(clippy::disallowed_methods)]
fn spawn_line_reader<R>(
reader: R,
to_line: fn(String) -> CommandOutputLine,
@ -654,8 +650,6 @@ fn drain_command_lines(
}
}
// container image build is provisioning infrastructure, out of scope (ENGINE_SPEC.md §2)
#[allow(clippy::disallowed_methods)]
fn run_status_command(
root: &Path,
program: &str,
@ -667,13 +661,12 @@ fn run_status_command(
let args: Vec<String> = args.iter().map(|arg| (*arg).to_owned()).collect();
eprintln!("myelin-node-image: {label}");
if progress.is_none() {
let status = Command::new(program)
let status = swactor_process::command_status(&mut Command::new(program)
.current_dir(root)
.args(&args)
.stdin(Stdio::null())
.stdout(Stdio::inherit())
.stderr(Stdio::inherit())
.status()
.stderr(Stdio::inherit()))
.map_err(|e| format!("run {label}: {e}"))?;
return if status.success() {
Ok(())
@ -693,13 +686,12 @@ fn run_status_command(
args: args.to_vec(),
},
);
let mut child = match Command::new(program)
let mut child = match swactor_process::command_spawn(&mut Command::new(program)
.current_dir(root)
.args(&args)
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.stderr(Stdio::piped()))
{
Ok(child) => child,
Err(error) => {
@ -737,7 +729,7 @@ fn run_status_command(
drop(tx);
let status = loop {
match child.try_wait() {
match swactor_process::child_try_wait(&mut child) {
Ok(Some(status)) => break status,
Ok(None) => {
drain_command_lines(&rx, progress, label, image_ref, started);
@ -785,13 +777,12 @@ fn run_status_command(
}
fn docker_image_exists(root: &Path, image_ref: &str) -> bool {
Command::new("docker")
swactor_process::command_status(&mut Command::new("docker")
.current_dir(root)
.args(["image", "inspect", image_ref])
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.stderr(Stdio::null()))
.map(|status| status.success())
.unwrap_or(false)
}
@ -800,7 +791,7 @@ fn docker_image_labels(
root: &Path,
image_ref: &str,
) -> Result<Option<BTreeMap<String, String>>, String> {
let output = Command::new("docker")
let output = swactor_process::command_output(&mut Command::new("docker")
.current_dir(root)
.args([
"image",
@ -809,8 +800,7 @@ fn docker_image_labels(
"{{ json .Config.Labels }}",
image_ref,
])
.stdin(Stdio::null())
.output()
.stdin(Stdio::null()))
.map_err(|e| format!("inspect docker image {image_ref}: {e}"))?;
if !output.status.success() {
return Ok(None);
@ -822,19 +812,18 @@ fn docker_image_labels(
}
fn docker_manifest_exists(root: &Path, image_ref: &str) -> bool {
Command::new("docker")
swactor_process::command_status(&mut Command::new("docker")
.current_dir(root)
.args(["manifest", "inspect", image_ref])
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.stderr(Stdio::null()))
.map(|status| status.success())
.unwrap_or(false)
}
fn docker_image_has_container(root: &Path, image_ref: &str) -> bool {
Command::new("docker")
swactor_process::command_output(&mut Command::new("docker")
.current_dir(root)
.args([
"ps",
@ -844,14 +833,13 @@ fn docker_image_has_container(root: &Path, image_ref: &str) -> bool {
"--format",
"{{.ID}}",
])
.stdin(Stdio::null())
.output()
.stdin(Stdio::null()))
.map(|output| output.status.success() && !output.stdout.is_empty())
.unwrap_or(true)
}
fn docker_image_tags(root: &Path, repository: &str) -> Result<Vec<(String, String)>, String> {
let output = Command::new("docker")
let output = swactor_process::command_output(&mut Command::new("docker")
.current_dir(root)
.args([
"image",
@ -860,8 +848,7 @@ fn docker_image_tags(root: &Path, repository: &str) -> Result<Vec<(String, Strin
"{{.Repository}}\t{{.Tag}}",
repository,
])
.stdin(Stdio::null())
.output()
.stdin(Stdio::null()))
.map_err(|error| format!("docker image ls failed: {error}"))?;
if !output.status.success() {
return Err(format!("docker image ls failed with {}", output.status));
@ -877,13 +864,12 @@ fn docker_image_tags(root: &Path, repository: &str) -> Result<Vec<(String, Strin
}
fn docker_image_remove(root: &Path, image_ref: &str) -> Result<(), String> {
let status = Command::new("docker")
let status = swactor_process::command_status(&mut Command::new("docker")
.current_dir(root)
.args(["image", "rm", image_ref])
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.stderr(Stdio::null()))
.map_err(|error| format!("docker image rm failed: {error}"))?;
if status.success() {
Ok(())

1974
apps/myelin/src/orchestration/provider_adapters/vastai/mod.rs
View file

File diff suppressed because it is too large Load diff

2668
apps/myelin/src/orchestration/provisioning.rs
View file

File diff suppressed because it is too large Load diff

7
apps/myelin/src/tests/engine_composition.rs
View file

@ -101,9 +101,7 @@ fn build_composition() -> (Engine, IrohDriver, DistributionRuntimeStack) {
(engine, driver, stack)
}
/// Poll an inbox until a value arrives or the deadline elapses. The only
/// `thread::sleep` in this module: test observation, not engine work.
#[allow(clippy::disallowed_methods)]
/// Bounded inbox polling for test observation; not runtime work.
fn recv_within<T: Message>(inbox: &Inbox<T>, deadline: Duration) -> Option<T> {
let started = Instant::now();
loop {
@ -161,7 +159,7 @@ fn dashboard_server_is_scheduled_through_the_engine() {
let mut config = dashboard::DashboardConfig::default();
config.port = free_port;
let handle = dashboard::DashboardHandle::new(config);
engine.handle().spawn(handle.http_server());
handle.spawn(&engine.handle());
// Behavioral proof the server future is actually running on the engine:
// the bound port accepts a TCP connection. No second runtime is involved.
@ -171,7 +169,6 @@ fn dashboard_server_is_scheduled_through_the_engine() {
}
#[cfg(feature = "dashboard")]
#[allow(clippy::disallowed_methods)]
fn poll_connect(addr: (&str, u16), deadline: Duration) -> bool {
use std::net::TcpStream;
let started = Instant::now();

92
apps/myelin/src/tests/fuzz_support.rs Normal file
View file

@ -0,0 +1,92 @@
use std::time::Duration;
use swactor::runtime::Runtime;
use swactor_engine::SteppingBackend;
pub(crate) fn drive_steps(backend: &SteppingBackend, count: usize) {
for _ in 0..count {
backend.step();
}
}
pub(crate) fn advance_and_drive(backend: &SteppingBackend, duration: Duration, count: usize) {
backend.advance_time(duration);
drive_steps(backend, count);
}
pub(crate) fn actor_census(runtime: &Runtime) -> String {
let stats = runtime.stats();
if stats.actor_details.is_empty() {
return format!("actors={:?}, workers={:?}", stats.actors, stats.workers);
}
stats
.actor_details
.iter()
.map(|actor| {
format!(
"address={} name={:?} worker={} mailbox={} last={:?} processed={} poisoned={}",
actor.address,
actor.name,
actor.worker_id,
actor.mailbox_depth,
actor.last_msg_type,
actor.messages_processed,
actor.poisoned,
)
})
.collect::<Vec<_>>()
.join("\n")
}
pub(crate) fn assert_no_poison(runtime: &Runtime) {
assert_no_poison_with_context(runtime, "");
}
pub(crate) fn assert_no_poison_with_context(runtime: &Runtime, context: &str) {
let stats = runtime.stats();
let poisoned = stats
.actor_details
.iter()
.filter(|actor| actor.poisoned)
.collect::<Vec<_>>();
let panics = stats
.workers
.iter()
.map(|worker| worker.panics)
.sum::<u64>();
assert!(
poisoned.is_empty() && panics == 0,
"actor poison/panic detected: poisoned={poisoned:?}, worker_panics={panics}\n{context}\n{}",
actor_census(runtime),
);
}
pub(crate) fn assert_actor_delta_at_most(runtime: &Runtime, baseline: usize, limit: usize) {
let current = runtime.stats().actors.len();
assert!(
current <= baseline.saturating_add(limit),
"actor count grew from {baseline} to {current}, limit={limit}\n{}",
actor_census(runtime),
);
}
pub(crate) fn assert_mailboxes_drained(runtime: &Runtime) {
let stats = runtime.stats();
let worker_depth = stats
.workers
.iter()
.map(|worker| worker.mailbox_depth)
.sum::<usize>();
let actor_depth = stats
.actor_details
.iter()
.map(|actor| actor.mailbox_depth)
.sum::<usize>();
assert_eq!(
worker_depth + actor_depth,
0,
"mailboxes did not drain\n{}",
actor_census(runtime),
);
}

1
apps/myelin/src/tests/job_runner_integration.rs
View file

@ -26,7 +26,6 @@ use crate::orchestration::distribution_stack::DistributionRuntimeStack;
const POLL: Duration = Duration::from_millis(15);
const DEADLINE: Duration = Duration::from_secs(20);
#[allow(clippy::disallowed_methods)]
fn recv_within<T: Message>(inbox: &Inbox<T>, deadline: Duration) -> Option<T> {
let started = Instant::now();
loop {

1
apps/myelin/src/tests/job_runner_iroh.rs
View file

@ -179,7 +179,6 @@ fn driver_a_join(
driver_a.join(std::slice::from_ref(&driver_b.endpoint_addr()));
}
#[allow(clippy::disallowed_methods)]
fn wait_until(deadline: Duration, mut check: impl FnMut() -> bool) -> bool {
let started = Instant::now();
loop {

1
apps/myelin/src/tests/mod.rs
View file

@ -1,4 +1,5 @@
mod engine_composition;
pub(crate) mod fuzz_support;
mod harness;
mod job_runner_iroh;
mod node_guarantees;

17
apps/myelin/src/tests/node_guarantees.rs
View file

@ -1,7 +1,7 @@
//! Behavior guarantees for the `node` module.
//!
//! These unit tests drive a manual `SingleThreadRuntime` host in isolation to verify actor
//! message routing — they are not engine integration tests.
//! These tests use the engine's deterministic stepping backend to verify actor
//! message routing without constructing or driving a runtime directly.
use crate::node_actor::{NodeAgentActor, NodeAgentMsg, NodeAgentReport};
use crate::orchestration::actor::OrchestratorMsg;
@ -9,13 +9,15 @@ use iroh::{EndpointAddr, SecretKey};
use myelin::staging as stage;
use swactor::actor::ActorAddress;
use swactor::config::RuntimeConfig;
use swactor::runtime::{RuntimeParts, SingleThreadRuntime};
use swactor::runtime::RuntimeParts;
use swactor_engine::{Engine, SteppingBackend};
#[test]
fn node_agent_runtime_loaded_reports_orchestrator() {
let parts = RuntimeParts::new(RuntimeConfig::default());
let runtime = parts.runtime().clone();
let mut host = SingleThreadRuntime::new(parts);
let backend = SteppingBackend::new();
let _engine = Engine::new(parts, backend.clone()).expect("stepping engine");
let orchestrator_inbox = runtime
.new_inbox::<OrchestratorMsg>()
.expect("orchestrator inbox");
@ -39,7 +41,7 @@ fn node_agent_runtime_loaded_reports_orchestrator() {
},
)
.expect("send runtime loaded");
host.tick();
backend.step();
assert_eq!(
orchestrator_inbox.try_recv(),
@ -58,7 +60,8 @@ fn node_agent_runtime_loaded_reports_orchestrator() {
fn node_agent_runtime_ready_ack_reports_worker_loop() {
let parts = RuntimeParts::new(RuntimeConfig::default());
let runtime = parts.runtime().clone();
let mut host = SingleThreadRuntime::new(parts);
let backend = SteppingBackend::new();
let _engine = Engine::new(parts, backend.clone()).expect("stepping engine");
let orchestrator_inbox = runtime
.new_inbox::<OrchestratorMsg>()
.expect("orchestrator inbox");
@ -84,7 +87,7 @@ fn node_agent_runtime_ready_ack_reports_worker_loop() {
},
)
.expect("send runtime ready ack");
host.tick();
backend.step();
assert_eq!(
reports.try_recv(),

2319
apps/myelin/tests/stateful_vastai.rs Normal file
View file

File diff suppressed because it is too large Load diff

55
clippy.toml
View file

@ -1,52 +1,3 @@
# Clippy enforcement policy for the swactor engine boundary
# (ENGINE_SPEC.md §2 / §3.1).
#
# These direct runtime / scheduling / time / core-driving operations are
# disallowed outside the engine's own substrate implementation. Integrations
# (iroh-driver, myelin, ...) must go through `EngineHandle`. The
# `swactor-engine` Tokio backend and the core driver carry narrow
# `#[allow(clippy::disallowed_methods)]` exemptions because they ARE the
# substrate implementor; the VastAI provider module carries a temporary
# module-level exemption pending its separate redesign (out of scope per §2).
#
# In-scope work that backs actors, transport, RPC, sampling, or node/orchestrator
# progression must schedule through `EngineHandle`. The only retained direct
# uses are narrow exclusions (§2): provider adapters/lifecycle (VastAI),
# provider-specific process supervision and log capture, and top-level OS-signal
# / blocking user-stdin / synchronous process-control sequencing. Each retained
# use carries a local `#[allow]` with its exclusion reason.
#
# Workspace-wide enforcement: `swactor-engine`, `iroh-driver`, and in-scope
# `myelin` carry `#![deny(clippy::disallowed_methods)]` and pass clean.
disallowed-methods = [
{ path = "tokio::runtime::Runtime::new", reason = "runtime ownership belongs to the engine; construct an engine-owned substrate instead" },
{ path = "tokio::runtime::Builder::new_current_thread", reason = "runtime ownership belongs to the engine; use EngineHandle" },
{ path = "tokio::runtime::Builder::new_multi_thread", reason = "runtime ownership belongs to the engine; use EngineHandle" },
{ path = "tokio::runtime::Handle::current", reason = "ambient runtime detection is forbidden; construct an engine-owned substrate instead" },
{ path = "tokio::runtime::Handle::try_current", reason = "ambient runtime detection is forbidden; construct an engine-owned substrate instead" },
{ path = "tokio::runtime::Runtime::block_on", reason = "blocking on a runtime is forbidden; schedule through EngineHandle" },
{ path = "tokio::runtime::Handle::block_on", reason = "blocking on a runtime is forbidden; schedule through EngineHandle" },
{ path = "tokio::spawn", reason = "direct scheduling is forbidden; use EngineHandle::spawn" },
{ path = "tokio::task::spawn", reason = "direct scheduling is forbidden; use EngineHandle::spawn" },
{ path = "tokio::task::spawn_blocking", reason = "use EngineHandle::spawn_blocking" },
{ path = "tokio::runtime::Runtime::spawn", reason = "direct scheduling is forbidden; use EngineHandle::spawn" },
{ path = "tokio::runtime::Handle::spawn", reason = "direct scheduling is forbidden; use EngineHandle::spawn" },
{ path = "tokio::runtime::Runtime::spawn_blocking", reason = "use EngineHandle::spawn_blocking" },
{ path = "tokio::runtime::Handle::spawn_blocking", reason = "use EngineHandle::spawn_blocking" },
{ path = "tokio::time::sleep", reason = "use EngineHandle::timer" },
{ path = "tokio::time::sleep_until", reason = "use EngineHandle::timer" },
{ path = "tokio::time::interval", reason = "use EngineHandle::interval" },
{ path = "tokio::time::interval_at", reason = "use EngineHandle::interval" },
{ path = "tokio::time::timeout", reason = "use an engine-derived timeout" },
{ path = "tokio::time::timeout_at", reason = "use an engine-derived timeout" },
{ path = "std::thread::spawn", reason = "direct thread scheduling is forbidden; schedule through EngineHandle" },
{ path = "std::thread::sleep", reason = "use EngineHandle::timer; retained only for narrow process-control exclusions (ENGINE_SPEC.md §2)" },
{ path = "swactor::runtime::Runtime::tick", reason = "manual core driving is forbidden; the engine owns core progression" },
{ path = "swactor::runtime::Runtime::try_tick", reason = "manual core driving is forbidden; the engine owns core progression" },
{ path = "swactor::runtime::Runtime::has_work", reason = "manual core driving is forbidden; the engine owns core progression" },
]
# Actor control-flow policy is enforced by the repository rustc workspace
# wrapper configured in `.cargo/config.toml`. Clippy is intentionally not a
# second architecture-policy mechanism.

1
crates/bindings/python/Cargo.toml
View file

@ -10,4 +10,5 @@ crate-type = ["cdylib"]
[dependencies]
swactor = { path = "../../.." }
swactor-engine = { path = "../../engine", default-features = false }
pyo3 = { version = "0.23", features = ["extension-module"] }

19
crates/bindings/python/src/lib.rs
View file

@ -8,9 +8,8 @@ use ::swactor::actor::{
Actor, ActorAddress, ActorInterface, AnyActor, Ctx, Environment, SpawnRequest,
};
use ::swactor::config::RuntimeConfig;
use ::swactor::runtime::{
Inbox, Runtime, RuntimeParts, SingleThreadRuntime as SingleThreadRuntimeHost,
};
use ::swactor::runtime::{Inbox, Runtime, RuntimeParts};
use swactor_engine::{Engine, SteppingBackend};
// ─── PyMsg newtype ───────────────────────────────────────────────────────────
@ -279,7 +278,8 @@ impl From<PyRuntimeConfig> for RuntimeConfig {
#[pyclass(name = "Runtime", unsendable)]
pub struct PyRuntime {
runtime: Runtime,
host: SingleThreadRuntimeHost,
_engine: Engine,
backend: SteppingBackend,
}
#[pymethods]
@ -293,8 +293,13 @@ impl PyRuntime {
};
let parts = RuntimeParts::new(config);
let runtime = parts.runtime().clone();
let host = SingleThreadRuntimeHost::new(parts);
Self { runtime, host }
let backend = SteppingBackend::new();
let engine = Engine::new(parts, backend.clone()).expect("create Python actor engine");
Self {
runtime,
_engine: engine,
backend,
}
}
fn spawn(&self, handler: PyObject) -> PyResult<PyActorAddress> {
@ -316,7 +321,7 @@ impl PyRuntime {
}
fn tick(&mut self) -> PyResult<()> {
self.host.tick();
self.backend.step();
Ok(())
}

1
crates/bindings/wasm-runtime/Cargo.toml
View file

@ -9,4 +9,5 @@ crate-type = ["cdylib"]
[dependencies]
swactor = { path = "../../..", default-features = false, features = ["wasm", "std"] }
swactor-engine = { path = "../../engine", default-features = false }
wasm-bindgen = "0.2"

24
crates/bindings/wasm-runtime/src/lib.rs
View file

@ -3,11 +3,9 @@ use std::sync::Arc;
use wasm_bindgen::prelude::*;
use swactor::actor::{ActorAddress, ActorExited, ActorInterface};
use swactor::runtime::{
Ctx, Inbox, Runtime, RuntimeConfig, RuntimeParts,
SingleThreadRuntime as SingleThreadRuntimeHost,
};
use swactor::runtime::{Ctx, Inbox, Runtime, RuntimeConfig, RuntimeParts};
use swactor::std::{CtxGroups, CtxWatching, RuntimeGroups, RuntimeNaming, StdExtension};
use swactor_engine::{Engine, SteppingBackend};
// ─── Core JS-facing types ───────────────────────────────────────────────────
@ -94,14 +92,15 @@ impl WasmInboxString {
/// The browser-facing swactor runtime.
///
/// Owns a cloneable `swactor::Runtime` handle plus the single-threaded host that
/// drives its workers, with StdExtension installed (naming, monitoring, groups).
/// Owns a cloneable `swactor::Runtime` handle plus an engine-backed stepping
/// substrate, with StdExtension installed (naming, monitoring, groups).
/// Actors are spawned via dedicated spawn functions (one per actor type). The
/// runtime is driven by calling `tick()`.
#[wasm_bindgen]
pub struct WasmRuntime {
rt: Runtime,
host: SingleThreadRuntimeHost,
_engine: Engine,
backend: SteppingBackend,
}
#[wasm_bindgen]
@ -111,13 +110,18 @@ impl WasmRuntime {
let parts = RuntimeParts::new(RuntimeConfig::default())
.with_extension(Arc::new(StdExtension::new()));
let rt = parts.runtime().clone();
let host = SingleThreadRuntimeHost::new(parts);
Self { rt, host }
let backend = SteppingBackend::new();
let engine = Engine::new(parts, backend.clone()).expect("create wasm actor engine");
Self {
rt,
_engine: engine,
backend,
}
}
/// Drive one tick of the runtime.
pub fn tick(&mut self) {
self.host.tick();
self.backend.step();
}
/// Number of actors currently alive.

5
crates/dashboard/Cargo.toml
View file

@ -8,6 +8,7 @@ license = "AGPL-3.0-only"
axum = "0.8"
telemetry = { path = "../telemetry" }
swactor = { path = "../..", features = ["serde"] }
swactor-engine = { path = "../engine" }
parking_lot = "0.12"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
@ -18,3 +19,7 @@ tokio-stream = "0.1"
# Live control actions (kill processes, provision nodes) for the
# provisioning-reconciler demo. Never enabled in shipping builds.
demo-control = []
[dev-dependencies]
proptest = "1"
tower = { version = "0.5", features = ["util"] }

7
crates/dashboard/proptest-regressions/control.txt Normal file
View file

@ -0,0 +1,7 @@
# Seeds for failure cases proptest has generated in the past. It is
# automatically read and these particular cases re-run before any
# novel cases are generated.
#
# It is recommended to check this file in to source control so that
# everyone who runs the test benefits from these saved cases.
cc 6dbd7f7ab6013cac23e3e815f076193eac00fc906957d486983845194cf20103 # shrinks to inputs = [(0, 0)], split = 0, concurrent = false

275
crates/dashboard/src/control.rs
View file

@ -50,8 +50,283 @@ pub fn set_control_sender(sender: Sender<ControlCommand>) {
let _ = CONTROL_SENDER.set(sender);
}
/// Install an actor destination for dashboard-issued control commands.
///
/// The dashboard owns the blocking channel reader; each command becomes one
/// typed actor observation.
pub fn install_actor_sink(
sender: swactor::runtime::ExternalSender,
actor: swactor::actor::ActorAddress,
) {
let (control_sender, receiver) = std::sync::mpsc::channel();
set_control_sender(control_sender);
drop(spawn_actor_sink_forwarder(receiver, sender, actor));
}
fn spawn_actor_sink_forwarder(
receiver: std::sync::mpsc::Receiver<ControlCommand>,
sender: swactor::runtime::ExternalSender,
actor: swactor::actor::ActorAddress,
) -> std::thread::JoinHandle<()> {
std::thread::spawn(move || {
while let Ok(command) = receiver.recv() {
if sender.send_to(actor, command).is_err() {
return;
}
}
})
}
pub(crate) fn dispatch(command: ControlCommand) -> bool {
CONTROL_SENDER
.get()
.is_some_and(|sender| sender.send(command).is_ok())
}
#[cfg(test)]
mod properties {
use std::sync::Arc;
use std::time::Duration;
use parking_lot::Mutex;
use proptest::prelude::*;
use swactor::actor::{ActorInterface, Ctx};
use swactor::config::RuntimeConfig;
use swactor::runtime::{Runtime, RuntimeParts};
use swactor_engine::{Engine, SteppingBackend};
use super::*;
const STEP_BUDGET: usize = 64;
const FORWARDER_BUDGET: Duration = Duration::from_secs(1);
struct CommandProbe {
observed: Arc<Mutex<Vec<String>>>,
}
impl ActorInterface for CommandProbe {
type Incoming = ControlCommand;
type Response = ();
fn handle(&mut self, _ctx: &Ctx, command: Self::Incoming) {
self.observed.lock().push(format!("{command:?}"));
}
}
fn command(kind: u8, value: u8) -> ControlCommand {
let command_id = format!("repeated-{}", value % 4);
match kind % 4 {
0 => ControlCommand::Kill {
command_id,
node: format!("node-{value}"),
},
1 => ControlCommand::Provision {
command_id,
count: u32::from(value),
},
2 => ControlCommand::Remove {
command_id,
count: u32::from(value),
},
_ => ControlCommand::EstablishEdge {
command_id,
node: format!("node-{value}"),
},
}
}
fn bridge_invariant_failure(
expected: &[String],
observed: &[String],
runtime: &Runtime,
expected_actor_count: usize,
) -> Option<String> {
let stats = runtime.stats();
let panics = stats
.workers
.iter()
.map(|worker| worker.panics)
.sum::<u64>();
let mailbox_depth = stats
.workers
.iter()
.map(|worker| worker.mailbox_depth)
.sum::<usize>()
+ stats
.actor_details
.iter()
.map(|actor| actor.mailbox_depth)
.sum::<usize>();
if observed != expected
|| stats.actors.len() != expected_actor_count
|| stats.actor_details.iter().any(|actor| actor.poisoned)
|| panics != 0
|| mailbox_depth != 0
{
Some(format!(
"expected={expected:?}, observed={observed:?}, \
expected_actor_count={expected_actor_count}, mailbox_depth={mailbox_depth}, \
actor_census={stats:?}"
))
} else {
None
}
}
proptest! {
#![proptest_config(ProptestConfig {
cases: 128,
max_shrink_iters: 2_000,
..ProptestConfig::default()
})]
#[test]
fn generated_concurrent_bridge_commands_forward_once_and_shutdown(
inputs in prop::collection::vec((any::<u8>(), any::<u8>()), 0..=32),
split in 0_usize..=32,
concurrent in any::<bool>(),
destination_disappears in any::<bool>(),
) {
let mut config = RuntimeConfig::default();
config.worker_count = 1;
let parts = RuntimeParts::new(config);
let runtime = parts.runtime().clone();
let backend = SteppingBackend::new();
let _engine =
Engine::new(parts, backend.clone()).expect("dashboard bridge stepping engine");
let observed = Arc::new(Mutex::new(Vec::new()));
let probe = runtime
.spawn(CommandProbe {
observed: Arc::clone(&observed),
})
.expect("spawn dashboard control probe");
if destination_disappears {
runtime
.stop_actor(probe)
.expect("stop dashboard destination before forwarding");
for _ in 0..STEP_BUDGET {
backend.step();
}
}
let (tx, rx) = std::sync::mpsc::channel();
let forwarder =
spawn_actor_sink_forwarder(rx, runtime.create_sender(), probe);
let commands = inputs
.iter()
.map(|(kind, value)| command(*kind, *value))
.collect::<Vec<_>>();
let command_log = commands
.iter()
.map(|command| format!("{command:?}"))
.collect::<Vec<_>>();
if concurrent {
let split = split.min(commands.len());
let left = commands[..split].to_vec();
let right = commands[split..].to_vec();
let left_tx = tx.clone();
let left_sender = std::thread::spawn(move || {
left.into_iter()
.map(|command| left_tx.send(command).is_ok())
.collect::<Vec<_>>()
});
let right_tx = tx.clone();
let right_sender = std::thread::spawn(move || {
right
.into_iter()
.map(|command| right_tx.send(command).is_ok())
.collect::<Vec<_>>()
});
let _ = left_sender.join().expect("join left dashboard sender");
let _ = right_sender.join().expect("join right dashboard sender");
} else {
for command in commands {
if !destination_disappears {
tx.send(command).expect("send dashboard command");
} else {
let _ = tx.send(command);
}
}
}
drop(tx);
let (done_tx, done_rx) = std::sync::mpsc::channel();
std::thread::spawn(move || {
let _ = done_tx.send(forwarder.join());
});
let forwarder_result = done_rx.recv_timeout(FORWARDER_BUDGET).unwrap_or_else(|error| {
panic!(
"dashboard bridge did not terminate after disconnect: {error}; \
actions={command_log:?}; actor_census={:?}",
runtime.stats(),
)
});
forwarder_result.expect("dashboard bridge forwarder panicked");
for _ in 0..STEP_BUDGET {
backend.step();
}
let mut actual = observed.lock().clone();
actual.sort();
let mut expected = if destination_disappears {
Vec::new()
} else {
command_log.clone()
};
expected.sort();
let expected_actor_count = usize::from(!destination_disappears);
prop_assert!(
bridge_invariant_failure(
&expected,
&actual,
&runtime,
expected_actor_count,
)
.is_none(),
"dashboard bridge invariant failed; actions={:?}; disconnect={}; failure={}",
command_log,
destination_disappears,
bridge_invariant_failure(
&expected,
&actual,
&runtime,
expected_actor_count,
)
.unwrap_or_default(),
);
if !destination_disappears {
runtime
.stop_actor(probe)
.expect("stop dashboard control probe");
for _ in 0..STEP_BUDGET {
backend.step();
}
}
prop_assert!(
bridge_invariant_failure(&expected, &actual, &runtime, 0).is_none(),
"dashboard bridge teardown leaked forwarding actors; actions={:?}; failure={}",
command_log,
bridge_invariant_failure(&expected, &actual, &runtime, 0)
.unwrap_or_default(),
);
}
}
#[test]
fn bridge_invariant_rejects_a_controlled_duplicate_delivery() {
let parts = RuntimeParts::new(RuntimeConfig::default());
let runtime = parts.runtime().clone();
let expected = vec!["Provision repeated-0".to_owned()];
let duplicated = vec![
"Provision repeated-0".to_owned(),
"Provision repeated-0".to_owned(),
];
assert!(
bridge_invariant_failure(&expected, &duplicated, &runtime, 0).is_some(),
"bridge invariant accepted a controlled duplicate delivery"
);
}
}

13
crates/dashboard/src/lib.rs
View file

@ -270,6 +270,19 @@ impl DashboardHandle {
server::run_server_with_routes(state, port, routes).await;
}
}
/// Schedule the dashboard HTTP server on its approved execution owner.
pub fn spawn(&self, engine: &swactor_engine::EngineHandle) {
engine.spawn(self.http_server());
}
/// Schedule the dashboard HTTP server on its approved execution owner.
pub fn spawn_with_plugins(
&self,
engine: &swactor_engine::EngineHandle,
plugins: Vec<DashboardPlugin>,
) {
engine.spawn(self.http_server_with_plugins(plugins));
}
}
/// Create the telemetry dashboard state.

316
crates/dashboard/src/server.rs
View file

@ -313,7 +313,20 @@ async fn frame_stream(
#[cfg(test)]
mod tests {
#[cfg(feature = "demo-control")]
use std::sync::{LazyLock, Mutex};
#[cfg(feature = "demo-control")]
use std::time::Duration;
use super::*;
#[cfg(feature = "demo-control")]
use axum::body::Body;
#[cfg(feature = "demo-control")]
use axum::http::Request;
#[cfg(feature = "demo-control")]
use proptest::prelude::*;
#[cfg(feature = "demo-control")]
use tower::util::ServiceExt;
fn state_with_plugin(page: PluginPage) -> AppState {
let views = Arc::new(ViewRegistry::new());
@ -341,4 +354,307 @@ mod tests {
assert!(rendered.contains(r#"<a href="/provision""#));
assert!(rendered.contains(r#"aria-current="page" data-active="true">Provision</a>"#));
}
#[cfg(feature = "demo-control")]
const HTTP_RESPONSE_BUDGET: Duration = Duration::from_secs(1);
#[cfg(feature = "demo-control")]
static TEST_CONTROL_RECEIVER: LazyLock<
Mutex<std::sync::mpsc::Receiver<crate::control::ControlCommand>>,
> = LazyLock::new(|| {
let (sender, receiver) = std::sync::mpsc::channel();
crate::control::set_control_sender(sender);
Mutex::new(receiver)
});
#[cfg(feature = "demo-control")]
fn test_control_receiver()
-> &'static Mutex<std::sync::mpsc::Receiver<crate::control::ControlCommand>> {
&TEST_CONTROL_RECEIVER
}
#[cfg(feature = "demo-control")]
#[derive(Clone, Debug)]
struct HttpAction {
route: u8,
payload: u8,
value: u8,
}
#[cfg(feature = "demo-control")]
impl HttpAction {
fn uri(&self) -> &'static str {
match self.route % 4 {
0 => "/control/kill",
1 => "/control/provision",
2 => "/control/remove",
_ => "/control/edge",
}
}
fn command_json(&self, route: u8) -> String {
let command_id = format!("repeated-{}", self.value % 4);
match route % 4 {
0 => serde_json::json!({
"Kill": {
"command_id": command_id,
"node": format!("node-{}", self.value),
}
})
.to_string(),
1 => serde_json::json!({
"Provision": {
"command_id": command_id,
"count": self.value,
}
})
.to_string(),
2 => serde_json::json!({
"Remove": {
"command_id": command_id,
"count": self.value,
}
})
.to_string(),
_ => serde_json::json!({
"EstablishEdge": {
"command_id": command_id,
"node": format!("node-{}", self.value),
}
})
.to_string(),
}
}
fn body(&self) -> String {
match self.payload % 5 {
0 => self.command_json(self.route),
1 => match self.value % 6 {
0 => String::new(),
1 => "{".to_owned(),
2 => "[".to_owned(),
3 => "{\"".to_owned(),
4 => "{\"command_id\":".to_owned(),
_ => "not-json".to_owned(),
},
2 => {
let command_id = format!("repeated-{}", self.value % 4);
match self.route % 4 {
0 => serde_json::json!({
"Kill": {"command_id": command_id}
})
.to_string(),
1 => serde_json::json!({
"Provision": {"count": self.value}
})
.to_string(),
2 => serde_json::json!({
"Remove": {"command_id": command_id}
})
.to_string(),
_ => serde_json::json!({
"EstablishEdge": {"node": format!("node-{}", self.value)}
})
.to_string(),
}
}
3 => self.command_json(self.route.wrapping_add(1)),
_ => match self.route % 4 {
0 => r#"{"Kill":{"command_id":7,"node":[]}}"#.to_owned(),
1 => r#"{"Provision":{"command_id":7,"count":"one"}}"#.to_owned(),
2 => r#"{"Remove":{"command_id":[],"count":-1}}"#.to_owned(),
_ => r#"{"EstablishEdge":{"command_id":false,"node":7}}"#.to_owned(),
},
}
}
fn is_valid_for_route(&self) -> bool {
self.payload % 5 == 0
}
}
#[cfg(feature = "demo-control")]
#[derive(Clone, Debug)]
struct HttpObservation {
index: usize,
uri: &'static str,
body: String,
expected_valid: bool,
status: StatusCode,
}
#[cfg(feature = "demo-control")]
async fn send_control_request(
app: Router,
index: usize,
action: HttpAction,
) -> Result<HttpObservation, String> {
let uri = action.uri();
let body = action.body();
let expected_valid = action.is_valid_for_route();
let request = Request::post(uri)
.header("content-type", "application/json")
.body(Body::from(body.clone()))
.map_err(|error| format!("build dashboard request: {error}"))?;
let response = tokio::time::timeout(HTTP_RESPONSE_BUDGET, app.oneshot(request))
.await
.map_err(|_| format!("request {index} {uri} exceeded {HTTP_RESPONSE_BUDGET:?}"))?
.map_err(|error| format!("route dashboard request: {error}"))?;
Ok(HttpObservation {
index,
uri,
body,
expected_valid,
status: response.status(),
})
}
#[cfg(feature = "demo-control")]
fn http_invariant_failure(
actions: &[HttpAction],
responses: &[HttpObservation],
forwarded: usize,
) -> Option<String> {
let expected_forwarded = actions
.iter()
.filter(|action| action.is_valid_for_route())
.count();
let terminal = responses.len() == actions.len();
let response_log = responses
.iter()
.map(|response| {
format!(
"#{} {} body={:?} -> {}",
response.index, response.uri, response.body, response.status,
)
})
.collect::<Vec<_>>();
let statuses_valid = responses.iter().all(|response| {
!response.status.is_server_error()
&& if response.expected_valid {
response.status == StatusCode::ACCEPTED
} else {
response.status.is_client_error()
}
});
if terminal && statuses_valid && forwarded == expected_forwarded {
None
} else {
Some(format!(
"terminal={terminal}, expected_forwarded={expected_forwarded}, \
forwarded={forwarded}, responses={response_log:?}, \
actor_census=dashboard HTTP routes own no actors"
))
}
}
#[cfg(feature = "demo-control")]
proptest! {
#![proptest_config(ProptestConfig {
cases: 128,
max_shrink_iters: 2_000,
..ProptestConfig::default()
})]
#[test]
fn generated_control_http_sequences_are_bounded_and_typed(
raw_actions in prop::collection::vec(
(any::<u8>(), any::<u8>(), any::<u8>()),
0..=32,
),
concurrent in any::<bool>(),
) {
let actions = raw_actions
.into_iter()
.map(|(route, payload, value)| HttpAction {
route,
payload,
value,
})
.collect::<Vec<_>>();
let receiver = test_control_receiver();
while receiver
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.try_recv()
.is_ok()
{}
let state = state_with_plugin(PluginPage::new(
"control-test",
"Control test",
"/control-test",
"",
));
let current_thread = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("build current-thread dashboard HTTP runtime");
let outcome = current_thread.block_on(async {
let mut responses = Vec::with_capacity(actions.len());
if concurrent {
let mut requests = tokio::task::JoinSet::new();
for (index, action) in actions.iter().cloned().enumerate() {
requests.spawn(send_control_request(router(state.clone()), index, action));
}
while let Some(result) = requests.join_next().await {
responses.push(
result
.map_err(|error| format!("dashboard request task failed: {error}"))??,
);
}
} else {
for (index, action) in actions.iter().cloned().enumerate() {
responses.push(
send_control_request(router(state.clone()), index, action).await?,
);
}
}
responses.sort_by_key(|response| response.index);
Ok::<_, String>(responses)
});
prop_assert!(
outcome.is_ok(),
"dashboard HTTP request did not terminate; actions={:?}; error={:?}; \
responses=[]; actor_census=dashboard HTTP routes own no actors",
actions,
outcome.as_ref().err(),
);
let responses = outcome.expect("outcome checked above");
let forwarded = {
let receiver = receiver
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
receiver.try_iter().count()
};
let failure = http_invariant_failure(&actions, &responses, forwarded);
prop_assert!(
failure.is_none(),
"dashboard HTTP invariant failed; actions={:?}; responses={:?}; failure={}",
actions,
responses,
failure.unwrap_or_default(),
);
}
}
#[cfg(feature = "demo-control")]
#[test]
fn control_http_invariant_rejects_a_controlled_server_error() {
let actions = vec![HttpAction {
route: 0,
payload: 1,
value: 0,
}];
let responses = vec![HttpObservation {
index: 0,
uri: actions[0].uri(),
body: actions[0].body(),
expected_valid: false,
status: StatusCode::INTERNAL_SERVER_ERROR,
}];
assert!(
http_invariant_failure(&actions, &responses, 0).is_some(),
"HTTP invariant accepted a controlled 5xx response for invalid JSON"
);
}
}

113
crates/dashboard/src/swactor/actor_view.rs
View file

@ -52,6 +52,7 @@ pub(crate) struct RuntimeState {
pub(crate) uptime_ms: Option<u64>,
pub(crate) actors: BTreeMap<String, ActorState>,
pub(crate) history: VecDeque<HistorySample>,
actor_snapshot_generation: u64,
}
impl RuntimeState {
@ -62,6 +63,7 @@ impl RuntimeState {
uptime_ms: None,
actors: BTreeMap::new(),
history: VecDeque::with_capacity(HISTORY_CAP),
actor_snapshot_generation: 0,
}
}
@ -79,28 +81,45 @@ impl RuntimeState {
}
fn apply_actors(&mut self, value: &Value, now: Instant) {
// Per-worker `worker_id` wrapper (TelemetryStatsHook shape) is the
// default placement for actors that do not carry one inline.
// A wrapped actor list is a complete snapshot for one worker. A list
// without a worker is a complete merged-runtime snapshot. Bare actor
// frames remain incremental.
let wrapper_worker = u32_field(value, &["worker_id", "worker"]);
if let Some(actors) = value.get("actors").and_then(Value::as_array) {
for actor in actors {
self.apply_actor(actor, now, wrapper_worker);
}
return;
}
// A bare actor object per frame (no envelope).
self.apply_actor(value, now, wrapper_worker);
}
fn apply_actor(&mut self, value: &Value, now: Instant, default_worker: Option<u32>) {
let Some(address) = string_field(value, &["address", "addr", "actor_addr"]) else {
let Some(actors) = value.get("actors").and_then(Value::as_array) else {
let _ = self.apply_actor(value, now, wrapper_worker);
return;
};
self.actor_snapshot_generation = self.actor_snapshot_generation.wrapping_add(1);
let generation = self.actor_snapshot_generation;
for actor in actors {
if let Some(actor) = self.apply_actor(actor, now, wrapper_worker) {
actor.snapshot_generation = generation;
}
}
match wrapper_worker {
Some(worker_id) => self.actors.retain(|_, actor| {
actor.worker_id != Some(worker_id) || actor.snapshot_generation == generation
}),
None => self
.actors
.retain(|_, actor| actor.snapshot_generation == generation),
}
}
fn apply_actor(
&mut self,
value: &Value,
now: Instant,
default_worker: Option<u32>,
) -> Option<&mut ActorState> {
let address = string_field(value, &["address", "addr", "actor_addr"])?;
let actor = self
.actors
.entry(address.clone())
.or_insert_with(|| ActorState::new(address));
actor.apply_json(value, now, default_worker);
Some(actor)
}
fn apply_stats(&mut self, value: &Value, now: Instant) {
@ -128,7 +147,7 @@ impl RuntimeState {
// Some publishers carry full per-actor detail under `actor_details`.
if let Some(details) = value.get("actor_details").and_then(Value::as_array) {
for actor in details {
self.apply_actor(actor, now, None);
let _ = self.apply_actor(actor, now, None);
}
}
}
@ -185,6 +204,7 @@ pub(crate) struct ActorState {
/// Messages folded away by the receipt sampling interval.
pub(crate) sampled_out: u64,
pub(crate) last_update: Option<Instant>,
snapshot_generation: u64,
}
impl ActorState {
@ -206,6 +226,7 @@ impl ActorState {
receipts: VecDeque::with_capacity(RECEIPT_CAP),
sampled_out: 0,
last_update: None,
snapshot_generation: 0,
}
}
@ -433,3 +454,65 @@ fn parse_message_type_counts(value: Option<&Value>) -> Option<Vec<(String, u64)>
}
None
}
#[cfg(test)]
mod tests {
use serde_json::json;
use super::*;
#[test]
fn per_worker_snapshots_remove_stopped_actors_without_touching_other_workers() {
let now = Instant::now();
let mut runtime = RuntimeState::new(now);
runtime.apply_actors(
&json!({
"worker_id": 0,
"actors": [
{"address": "stable", "actor_type": "StableActor"},
{"address": "observer", "actor_type": "ControlReplyObserver"},
],
}),
now,
);
runtime.apply_actors(
&json!({
"worker_id": 1,
"actors": [
{"address": "other-worker", "actor_type": "OtherActor"},
],
}),
now,
);
assert_eq!(runtime.actors.len(), 3);
runtime.apply_actors(
&json!({
"worker_id": 0,
"actors": [
{"address": "stable", "actor_type": "StableActor"},
],
}),
now,
);
assert_eq!(
runtime
.actors
.keys()
.map(String::as_str)
.collect::<Vec<_>>(),
vec!["other-worker", "stable"]
);
runtime.apply_actors(&json!({"worker_id": 0, "actors": []}), now);
assert_eq!(
runtime
.actors
.keys()
.map(String::as_str)
.collect::<Vec<_>>(),
vec!["other-worker"]
);
}
}

1
crates/distribution/Cargo.toml
View file

@ -21,3 +21,4 @@ libc = "0.2"
[dev-dependencies]
serde_json = "1"
proptest = "1"
swactor-engine = { path = "../engine", default-features = false }

22
crates/distribution/tests/gossip_data.rs
View file

@ -228,14 +228,14 @@ mod standalone_gossip_transport {
//! Actorized registry/metadata/directory gossip over one codec and transport, proving
//! standalone frames converge without piggybacking on SWIM.
use std::cell::RefCell;
use std::collections::HashMap;
use std::sync::{Arc, RwLock};
use swactor::Error;
use swactor::actor::ActorAddress;
use swactor::runtime::{Inbox, Runtime, RuntimeConfig, RuntimeParts, SingleThreadRuntime};
use swactor::runtime::{Inbox, Runtime, RuntimeConfig, RuntimeParts};
use swactor::std::StdExtension;
use swactor_engine::{Engine, SteppingBackend};
use swactor_transport::{CodecRegistry, Transport, TransportRouter, WireEnvelope};
use distribution::crypto::{Keypair, KeypairExt};
@ -272,7 +272,8 @@ mod standalone_gossip_transport {
/// plus the shared state needed to wire it into a mesh.
struct Node {
rt: Runtime,
host: RefCell<SingleThreadRuntime>,
_engine: Engine,
backend: SteppingBackend,
registry: ActorAddress,
metadata: ActorAddress,
directory: ActorAddress,
@ -305,7 +306,9 @@ mod standalone_gossip_transport {
codec.clone(),
router.clone(),
)));
let host = RefCell::new(SingleThreadRuntime::new(parts));
let backend = SteppingBackend::new();
let engine =
Engine::new(parts, backend.clone()).expect("create stepping actor engine");
let dir = SharedPeerDirectory::new();
let relay_mirror: RelayMirror = Arc::new(RwLock::new(HashMap::new()));
@ -336,7 +339,8 @@ mod standalone_gossip_transport {
nodes.push(Node {
rt,
host,
_engine: engine,
backend,
registry,
metadata,
directory,
@ -407,7 +411,7 @@ mod standalone_gossip_transport {
fn pump(&self, k: usize) {
for _ in 0..k {
for node in &self.nodes {
node.host.borrow_mut().tick();
node.backend.step();
}
}
}
@ -448,7 +452,7 @@ mod standalone_gossip_transport {
},
)
.unwrap();
self.nodes[observer].host.borrow_mut().tick();
self.nodes[observer].backend.step();
inbox.try_recv().and_then(|r| r.binding)
}
@ -465,7 +469,7 @@ mod standalone_gossip_transport {
},
)
.unwrap();
self.nodes[observer].host.borrow_mut().tick();
self.nodes[observer].backend.step();
inbox.try_recv().and_then(|r| r.relay_url)
}
@ -482,7 +486,7 @@ mod standalone_gossip_transport {
},
)
.unwrap();
self.nodes[observer].host.borrow_mut().tick();
self.nodes[observer].backend.step();
inbox.try_recv().and_then(|located| located.host)
}
}

34
crates/distribution/tests/routing.rs
View file

@ -16,15 +16,15 @@ mod directory_actor {
//! DirectoryActor convergence and safety: signed claims, supersede, deterministic conflict
//! resolution, dead-host hiding, catch-up, quieting, and retained recovery claims.
use std::cell::RefCell;
use std::collections::HashMap;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, RwLock};
use swactor::Error;
use swactor::actor::ActorAddress;
use swactor::runtime::{Inbox, Runtime, RuntimeConfig, RuntimeParts, SingleThreadRuntime};
use swactor::runtime::{Inbox, Runtime, RuntimeConfig, RuntimeParts};
use swactor::std::StdExtension;
use swactor_engine::{Engine, SteppingBackend};
use swactor_transport::{CodecRegistry, Transport, TransportRouter, WireEnvelope};
use distribution::crypto::{Keypair, KeypairExt};
@ -63,7 +63,8 @@ mod directory_actor {
/// wire it into a mesh and observe it.
struct Node {
rt: Runtime,
host: RefCell<SingleThreadRuntime>,
_engine: Engine,
backend: SteppingBackend,
directory: ActorAddress,
dir: SharedPeerDirectory,
router: Arc<TransportRouter>,
@ -94,7 +95,9 @@ mod directory_actor {
codec.clone(),
router.clone(),
)));
let host = RefCell::new(SingleThreadRuntime::new(parts));
let backend = SteppingBackend::new();
let engine =
Engine::new(parts, backend.clone()).expect("create stepping actor engine");
let dir = SharedPeerDirectory::new();
let route_view: RouteView = Arc::new(RwLock::new(HashMap::new()));
@ -109,7 +112,8 @@ mod directory_actor {
nodes.push(Node {
rt,
host,
_engine: engine,
backend,
directory,
dir,
router,
@ -181,7 +185,7 @@ mod directory_actor {
fn pump(&self, k: usize) {
for _ in 0..k {
for node in &self.nodes {
node.host.borrow_mut().tick();
node.backend.step();
}
}
}
@ -246,7 +250,7 @@ mod directory_actor {
},
)
.unwrap();
self.nodes[observer].host.borrow_mut().tick();
self.nodes[observer].backend.step();
inbox.try_recv().and_then(|located| located.host)
}
@ -585,15 +589,15 @@ mod directory_route_path {
//! Application delivery through the directory route view: address-only sends, supersede, and
//! best-effort drops.
use std::cell::RefCell;
use std::collections::HashMap;
use std::sync::{Arc, Mutex, RwLock};
use serde::{Deserialize, Serialize};
use swactor::Error;
use swactor::actor::{ActorAddress, ActorInterface};
use swactor::runtime::{Ctx, Runtime, RuntimeConfig, RuntimeParts, SingleThreadRuntime};
use swactor::runtime::{Ctx, Runtime, RuntimeConfig, RuntimeParts};
use swactor::std::StdExtension;
use swactor_engine::{Engine, SteppingBackend};
use swactor_transport::{CodecRegistry, NetworkMessage, TransportRouter};
use distribution::crypto::{Keypair, KeypairExt};
@ -659,7 +663,8 @@ mod directory_route_path {
struct RouteNode {
rt: Runtime,
host: RefCell<SingleThreadRuntime>,
_engine: Engine,
backend: SteppingBackend,
outbox: Outbox,
route_view: RouteView,
directory: ActorAddress,
@ -702,7 +707,9 @@ mod directory_route_path {
codec.clone(),
router.clone(),
)));
let host = RefCell::new(SingleThreadRuntime::new(parts));
let backend = SteppingBackend::new();
let engine =
Engine::new(parts, backend.clone()).expect("create stepping actor engine");
let outbox: Outbox = Arc::new(Mutex::new(Vec::new()));
let route_view: RouteView = Arc::new(RwLock::new(HashMap::new()));
@ -731,7 +738,8 @@ mod directory_route_path {
nodes.push(RouteNode {
rt,
host,
_engine: engine,
backend,
outbox,
route_view,
directory,
@ -801,7 +809,7 @@ mod directory_route_path {
fn settle(&self, k: usize) {
for _ in 0..k {
for node in &self.nodes {
node.host.borrow_mut().tick();
node.backend.step();
}
self.deliver_wire();
}

54
crates/distribution/tests/swim_actor.rs
View file

@ -16,13 +16,13 @@ mod single_runtime_actor {
//! SwimActor behavior in one runtime: subscription stream convergence and genuine unreachable-
//! peer death detection.
use std::cell::RefCell;
use std::collections::BTreeMap;
use std::sync::Arc;
use std::time::{Duration, Instant};
use swactor::runtime::{Runtime, RuntimeConfig, RuntimeParts, SingleThreadRuntime};
use swactor::runtime::{Runtime, RuntimeConfig, RuntimeParts};
use swactor::std::StdExtension;
use swactor_engine::{Engine, SteppingBackend};
use distribution::swim::actor::{
MembershipChanged, PeerDirectory, SharedPeerDirectory, SwimActor, SwimIn,
@ -55,7 +55,8 @@ mod single_runtime_actor {
/// `MembershipChanged` subscriber inbox, and a shared Binding.
struct ActorCluster {
rt: Runtime,
host: RefCell<SingleThreadRuntime>,
_engine: Engine,
backend: SteppingBackend,
ids: Vec<NodeId>,
addrs: Vec<swactor::actor::ActorAddress>,
inboxes: Vec<swactor::runtime::Inbox<MembershipChanged>>,
@ -72,7 +73,8 @@ mod single_runtime_actor {
let parts = RuntimeParts::new(RuntimeConfig::default())
.with_extension(Arc::new(StdExtension::new()));
let rt = parts.runtime().clone();
let host = RefCell::new(SingleThreadRuntime::new(parts));
let backend = SteppingBackend::new();
let engine = Engine::new(parts, backend.clone()).expect("create stepping actor engine");
let dir = SharedPeerDirectory::new();
let now = Instant::now();
let ids: Vec<NodeId> = (0..n).map(|i| id(i as u8)).collect();
@ -103,7 +105,8 @@ mod single_runtime_actor {
}
let mut c = ActorCluster {
rt,
host,
_engine: engine,
backend,
ids,
addrs,
inboxes,
@ -128,7 +131,7 @@ mod single_runtime_actor {
fn pump(&self, n: usize) {
for _ in 0..n {
self.host.borrow_mut().tick();
self.backend.step();
}
}
@ -242,15 +245,15 @@ mod transport_runtime_actor {
//! SwimActor behavior across separate runtimes through codec, TransportRouter, deliver_raw, and
//! transport send failure.
use std::cell::RefCell;
use std::collections::{BTreeMap, HashSet};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use swactor::Error;
use swactor::actor::ActorAddress;
use swactor::runtime::{Inbox, Runtime, RuntimeConfig, RuntimeParts, SingleThreadRuntime};
use swactor::runtime::{Inbox, Runtime, RuntimeConfig, RuntimeParts};
use swactor::std::StdExtension;
use swactor_engine::{Engine, SteppingBackend};
use swactor_transport::{CodecRegistry, Transport, TransportRouter, WireEnvelope};
use distribution::messages::actor_codec_registry;
@ -321,7 +324,8 @@ mod transport_runtime_actor {
/// `Link` transports — the multi-runtime analog of `swim_actor.rs::ActorCluster`.
struct TransportCluster {
rts: Vec<Runtime>,
hosts: Vec<RefCell<SingleThreadRuntime>>,
_engines: Vec<Engine>,
backends: Vec<SteppingBackend>,
swims: Vec<ActorAddress>,
inboxes: Vec<Inbox<MembershipChanged>>,
streams: Vec<Vec<MembershipChanged>>,
@ -338,7 +342,8 @@ mod transport_runtime_actor {
let partition = Arc::new(Mutex::new(HashSet::new()));
let mut rts = Vec::new();
let mut hosts = Vec::new();
let mut engines = Vec::new();
let mut backends = Vec::new();
let mut swims = Vec::new();
let mut dirs = Vec::new();
let mut routers = Vec::new();
@ -355,7 +360,9 @@ mod transport_runtime_actor {
codec.clone(),
router.clone(),
)));
let host = RefCell::new(SingleThreadRuntime::new(parts));
let backend = SteppingBackend::new();
let engine =
Engine::new(parts, backend.clone()).expect("create stepping actor engine");
let dir = SharedPeerDirectory::new();
let swim = rt
@ -378,7 +385,8 @@ mod transport_runtime_actor {
.unwrap();
rts.push(rt);
hosts.push(host);
engines.push(engine);
backends.push(backend);
swims.push(swim);
dirs.push(dir);
routers.push(router);
@ -412,7 +420,8 @@ mod transport_runtime_actor {
let mut c = TransportCluster {
rts,
hosts,
_engines: engines,
backends,
swims,
inboxes,
streams: vec![Vec::new(); n],
@ -443,8 +452,8 @@ mod transport_runtime_actor {
/// iterations; `k` is sized so a probe + its notification settle per round.
fn pump(&self, k: usize) {
for _ in 0..k {
for host in &self.hosts {
host.borrow_mut().tick();
for backend in &self.backends {
backend.step();
}
}
}
@ -547,13 +556,13 @@ mod actor_membership_safety_edges {
//! Actor-observable safety edges: resurrection after silence, multi-hop death dissemination,
//! and bounded stale-refute behavior.
use std::cell::RefCell;
use std::collections::BTreeMap;
use std::sync::Arc;
use std::time::{Duration, Instant};
use swactor::runtime::{Runtime, RuntimeConfig, RuntimeParts, SingleThreadRuntime};
use swactor::runtime::{Runtime, RuntimeConfig, RuntimeParts};
use swactor::std::StdExtension;
use swactor_engine::{Engine, SteppingBackend};
use distribution::swim::actor::{
MembershipChanged, PeerDirectory, SharedPeerDirectory, SwimActor, SwimIn,
@ -589,7 +598,8 @@ mod actor_membership_safety_edges {
/// but lets the test choose the config and rebind a dropped node.
struct Cluster {
rt: Runtime,
host: RefCell<SingleThreadRuntime>,
_engine: Engine,
backend: SteppingBackend,
ids: Vec<NodeId>,
addrs: Vec<swactor::actor::ActorAddress>,
inboxes: Vec<swactor::runtime::Inbox<MembershipChanged>>,
@ -603,7 +613,8 @@ mod actor_membership_safety_edges {
let parts = RuntimeParts::new(RuntimeConfig::default())
.with_extension(Arc::new(StdExtension::new()));
let rt = parts.runtime().clone();
let host = RefCell::new(SingleThreadRuntime::new(parts));
let backend = SteppingBackend::new();
let engine = Engine::new(parts, backend.clone()).expect("create stepping actor engine");
let dir = SharedPeerDirectory::new();
let now = Instant::now();
let ids: Vec<NodeId> = (0..n).map(|i| id(i as u8)).collect();
@ -633,7 +644,8 @@ mod actor_membership_safety_edges {
}
let mut c = Cluster {
rt,
host,
_engine: engine,
backend,
ids,
addrs,
inboxes,
@ -655,7 +667,7 @@ mod actor_membership_safety_edges {
fn pump(&self, n: usize) {
for _ in 0..n {
self.host.borrow_mut().tick();
self.backend.step();
}
}

3
crates/engine/Cargo.toml
View file

@ -12,3 +12,6 @@ tokio = ["dep:tokio"]
swactor = { path = "../.." }
parking_lot = "0.12"
tokio = { workspace = true, optional = true }
[dev-dependencies]
proptest = "1"

4
crates/engine/ENGINE_SPEC.md
View file

@ -66,7 +66,7 @@ Constructing a swactor engine consumes configured `RuntimeParts` and the selecte
| operation | meaning |
|---|---|
| `spawn(task)` | Schedule an async unit of work on the substrate. |
| `spawn_blocking(work)` | Schedule blocking CPU / syscall work off the async path. |
| `blocking_work_sender().submit(work)` | Route blocking I/O work to the substrate's dedicated blocking pool. |
| `timer(delay)` / `interval(period)` | Schedule future or recurring work. |
| `now()` | The engine's monotonic clock. |
@ -88,7 +88,7 @@ The primitives an engine may provide. Capabilities are **per-implementation and
- **Tasks** — `spawn` of an async unit of work; the substrate's unit of concurrency.
- **Timers** — one-shot delay and recurring interval.
- **I/O** — streams, sockets, files, and protocol endpoints used by engine-hosted work. An engine may implement I/O through asynchronous operations, blocking operations on managed threads, callbacks, or host-native facilities. Integrations declare the I/O capabilities they require, and binding fails at construction when the selected engine cannot provide them.
- **Blocking** — `spawn_blocking` for CPU-bound or syscall work that must not stall the executor.
- **Blocking** — a `BlockingWorkSender` routes syscall work off actor and async workers.
- **Time** — `now()`. In a test engine this is virtual, advanced by the test; this is what makes deterministic testing possible.
An engine that provides only tasks + time is still valid. Blocking and I/O are additional capabilities declared by integrations that require them.

1
crates/engine/src/core_driver.rs
View file

@ -57,7 +57,6 @@ struct CoreDriver {
// Core drivers are the engine's sole core-progression path; this is the one
// place permitted to call `Worker::try_tick` (ENGINE_SPEC.md §2).
#[allow(clippy::disallowed_methods)]
impl Future for CoreDriver {
type Output = ();

199
crates/engine/src/engine.rs
View file

@ -1,11 +1,14 @@
//! Composite engine and cloneable scheduler handle.
use parking_lot::{Condvar, Mutex};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Weak};
use std::time::Duration;
use crate::backend::{Capabilities, EngineError, ExecutionBackend};
use crate::time::{EngineInstant, Interval, Timeout, Timer};
use swactor::runtime::{Runtime, RuntimeParts};
use swactor::actor::{ActorAddress, Message};
use swactor::runtime::{ExternalSender, Runtime, RuntimeParts};
/// The composite engine: retains a configured core runtime handle and its
/// execution backend, and owns one core-driving loop per worker.
@ -67,6 +70,116 @@ impl Engine {
pub struct EngineHandle {
backend: Weak<dyn ExecutionBackend>,
}
/// A clonable substrate route for one-shot blocking I/O work.
///
/// Domain actors decide which effect to execute; this handle only moves its
/// mechanics onto the engine backend's dedicated blocking pool.
#[derive(Clone)]
pub struct BlockingWorkSender {
backend: Weak<dyn ExecutionBackend>,
}
impl BlockingWorkSender {
/// Submit one blocking I/O operation without occupying an actor worker.
///
/// Returns the operation unchanged if the owning engine has stopped.
pub fn submit(&self, work: crate::BoxWork) -> Result<(), crate::BoxWork> {
let Some(backend) = self.backend.upgrade() else {
return Err(work);
};
backend.spawn_blocking(work);
Ok(())
}
}
/// Cancellation handle for an engine-owned actor message timer.
///
/// Cancellation is idempotent. A timer may already have fired when cancellation
/// races its deadline, so actor messages should still carry an operation or
/// generation identity that lets the receiver reject stale work.
#[derive(Clone, Debug)]
pub struct ActorTimer {
cancelled: Arc<AtomicBool>,
}
impl ActorTimer {
/// Prevent this timer from delivering future messages.
pub fn cancel(&self) {
self.cancelled.store(true, Ordering::Release);
}
/// Report whether cancellation has been requested.
pub fn is_cancelled(&self) -> bool {
self.cancelled.load(Ordering::Acquire)
}
}
enum CompletionState<T> {
Pending,
Ready(T),
Consumed,
}
/// One actor-owned terminal observation for a synchronous process entrypoint.
///
/// The waiting thread cannot poll, set a deadline, or advance domain state.
/// An actor decides when the operation is complete and publishes the value.
pub struct ActorCompletion<T> {
inner: Arc<(Mutex<CompletionState<T>>, Condvar)>,
}
impl<T> Clone for ActorCompletion<T> {
fn clone(&self) -> Self {
Self {
inner: Arc::clone(&self.inner),
}
}
}
impl<T> Default for ActorCompletion<T> {
fn default() -> Self {
Self::new()
}
}
impl<T> ActorCompletion<T> {
pub fn new() -> Self {
Self {
inner: Arc::new((Mutex::new(CompletionState::Pending), Condvar::new())),
}
}
/// Publish the terminal observation once.
pub fn complete(&self, value: T) -> Result<(), T> {
let (state, ready) = &*self.inner;
let mut state = state.lock();
if !matches!(*state, CompletionState::Pending) {
return Err(value);
}
*state = CompletionState::Ready(value);
ready.notify_all();
Ok(())
}
/// Block the process entrypoint until the owning actor completes.
pub fn wait(&self) -> T {
let (state, ready) = &*self.inner;
let mut state = state.lock();
loop {
if matches!(*state, CompletionState::Pending) {
ready.wait(&mut state);
continue;
}
match std::mem::replace(&mut *state, CompletionState::Consumed) {
CompletionState::Ready(value) => return value,
CompletionState::Consumed => {
panic!("ActorCompletion::wait called after value was consumed")
}
CompletionState::Pending => unreachable!("pending state handled above"),
}
}
}
}
impl EngineHandle {
/// Upgrade to the live backend, or `None` if the owning engine is gone.
@ -86,16 +199,10 @@ impl EngineHandle {
backend.spawn(Box::pin(task));
}
}
/// Schedule `work` on a dedicated blocking thread.
///
/// A no-op once the owning engine has been dropped.
pub fn spawn_blocking<F>(&self, work: F)
where
F: FnOnce() + Send + 'static,
{
if let Some(backend) = self.backend() {
backend.spawn_blocking(Box::new(work));
/// Create a route to the backend's dedicated blocking-I/O pool.
pub fn blocking_work_sender(&self) -> BlockingWorkSender {
BlockingWorkSender {
backend: self.backend.clone(),
}
}
@ -120,6 +227,76 @@ impl EngineHandle {
current: None,
}
}
/// Schedule one typed message for delivery after `delay`.
///
/// The engine owns the timer task; domain code receives no future or
/// scheduling callback. The actor receiving `message` owns the deadline
/// decision and should reject stale operation identities.
pub fn send_after<M>(
&self,
delay: Duration,
sender: ExternalSender,
actor: ActorAddress,
message: M,
) -> ActorTimer
where
M: Message,
{
let actor_timer = ActorTimer {
cancelled: Arc::new(AtomicBool::new(false)),
};
let cancelled = Arc::clone(&actor_timer.cancelled);
let timer = self.timer(delay);
self.spawn(async move {
timer.await;
if !cancelled.load(Ordering::Acquire) {
let _ = sender.send_to(actor, message);
}
});
actor_timer
}
/// Schedule a cloned typed message after every `period`.
///
/// Delivery stops after cancellation or when the actor address no longer
/// accepts messages.
pub fn send_every<M>(
&self,
period: Duration,
sender: ExternalSender,
actor: ActorAddress,
message: M,
) -> ActorTimer
where
M: Message,
{
let actor_timer = ActorTimer {
cancelled: Arc::new(AtomicBool::new(false)),
};
let cancelled = Arc::clone(&actor_timer.cancelled);
let handle = self.clone();
let mut interval = Box::pin(handle.interval(period));
self.spawn(std::future::poll_fn(move |cx| {
if cancelled.load(Ordering::Acquire) {
return std::task::Poll::Ready(());
}
match std::future::Future::poll(interval.as_mut(), cx) {
std::task::Poll::Ready(()) => {
if cancelled.load(Ordering::Acquire)
|| sender.send_to(actor, message.clone()).is_err()
{
std::task::Poll::Ready(())
} else {
cx.waker().wake_by_ref();
std::task::Poll::Pending
}
}
std::task::Poll::Pending => std::task::Poll::Pending,
}
}));
actor_timer
}
/// Race `future` against an engine timer.
///
/// Resolves to `Ok` with the future's output if it completes within

2
crates/engine/src/lib.rs
View file

@ -15,7 +15,7 @@ mod time;
mod tokio;
pub use backend::{BoxTask, BoxTimer, BoxWork, Capabilities, EngineError, ExecutionBackend};
pub use engine::{Engine, EngineHandle};
pub use engine::{ActorCompletion, ActorTimer, BlockingWorkSender, Engine, EngineHandle};
pub use stepping::SteppingBackend;
pub use time::{Elapsed, EngineInstant, Interval, Timeout, Timer};

13
crates/engine/src/stepping.rs
View file

@ -181,11 +181,22 @@ impl SteppingBackend {
pub fn pending_task_count(&self) -> usize {
self.inner.tasks.lock().len()
}
/// Join every blocking operation submitted so far.
///
/// Deterministic tests use this as a barrier before stepping actor replies.
/// Returns the first worker panic instead of silently discarding it.
pub fn join_blocking(&self) -> std::thread::Result<()> {
let handles: Vec<_> = self.inner.blocking.lock().drain(..).collect();
for handle in handles {
handle.join()?;
}
Ok(())
}
}
/// This impl is the substrate implementor for the deterministic stepping
/// backend; blocking work runs on a std thread (ENGINE_SPEC.md §2).
#[allow(clippy::disallowed_methods)]
impl ExecutionBackend for SteppingBackend {
fn spawn(&self, task: BoxTask) {
self.inner.tasks.lock().push(task);

3
crates/engine/src/tokio.rs
View file

@ -55,7 +55,6 @@ impl TokioBackend {
/// deterministic.
// The engine's Tokio backend is the substrate owner: it is the one place
// permitted to construct a Tokio runtime (ENGINE_SPEC.md §2).
#[allow(clippy::disallowed_methods)]
pub fn new(config: TokioConfig) -> Result<Self, EngineError> {
let runtime = tokio::runtime::Builder::new_multi_thread()
.worker_threads(config.worker_threads)
@ -80,7 +79,6 @@ impl TokioBackend {
/// This impl is the Tokio substrate implementor: it is the one place permitted
/// to schedule directly on the owned runtime (ENGINE_SPEC.md §2).
#[allow(clippy::disallowed_methods)]
impl ExecutionBackend for TokioBackend {
fn spawn(&self, task: BoxTask) {
// The handle is used ephemerally and never stored or returned.
@ -152,7 +150,6 @@ impl LazySleep {
// `LazySleep` arms a `tokio::time::sleep` inside an engine task where the time
// driver is available; this is the substrate's own time primitive.
#[allow(clippy::disallowed_methods)]
impl Future for LazySleep {
type Output = ();

57
crates/engine/tests/common/mod.rs
View file

@ -11,6 +11,7 @@ use std::time::{Duration, Instant};
use swactor::actor::{ActorInterface, Ctx};
use swactor::runtime::{Runtime, RuntimeConfig, RuntimeParts};
use swactor_engine::SteppingBackend;
pub fn runtime_parts(config: RuntimeConfig) -> (RuntimeParts, Runtime) {
let parts = RuntimeParts::new(config);
@ -111,3 +112,59 @@ pub async fn yield_once() {
})
.await;
}
pub fn drive_steps(backend: &SteppingBackend, count: usize) {
for _ in 0..count {
backend.step();
}
}
pub fn advance_and_drive(backend: &SteppingBackend, duration: Duration, count: usize) {
backend.advance_time(duration);
drive_steps(backend, count);
}
pub fn assert_no_poison(runtime: &Runtime) {
let stats = runtime.stats();
let panics = stats
.workers
.iter()
.map(|worker| worker.panics)
.sum::<u64>();
let poisoned = stats
.actor_details
.iter()
.filter(|actor| actor.poisoned)
.collect::<Vec<_>>();
assert!(
panics == 0 && poisoned.is_empty(),
"runtime contains poisoned actors: worker_panics={panics}, poisoned={poisoned:?}, \
actors={:?}, details={:?}",
stats.actors,
stats.actor_details,
);
}
pub fn assert_actor_delta_at_most(runtime: &Runtime, baseline: usize, limit: usize) {
let stats = runtime.stats();
assert!(
stats.actors.len() <= baseline.saturating_add(limit),
"actor count grew from {baseline} to {}, limit={limit}: {:?}",
stats.actors.len(),
stats.actors,
);
}
pub fn assert_mailboxes_drained(runtime: &Runtime) {
let stats = runtime.stats();
let mailbox_depth = stats
.workers
.iter()
.map(|worker| worker.mailbox_depth)
.sum::<usize>();
assert_eq!(
mailbox_depth, 0,
"mailboxes did not drain: workers={:?}, details={:?}",
stats.workers, stats.actor_details,
);
}

13
crates/engine/tests/engine_contract.rs
View file

@ -215,9 +215,15 @@ fn blocking_work_does_not_stop_actor_ticks() {
// task so the owned runtime shuts down deterministically.
let _release = BarrierRelease(barrier.clone());
let barrier_for_work = barrier.clone();
handle.spawn_blocking(move || {
barrier_for_work.wait();
});
assert!(
handle
.blocking_work_sender()
.submit(Box::new(move || {
barrier_for_work.wait();
}))
.is_ok(),
"submit blocking work"
);
// Deliver an actor message while the blocking work remains blocked.
runtime.send_to(addr, Probe).expect("deliver probe message");
@ -322,7 +328,6 @@ fn engine_timer_can_be_created_off_runtime() {
// Tokio runtime to hand the engine — the one test-only use of the substrate
// constructor (ENGINE_SPEC.md §2).
#[test]
#[allow(clippy::disallowed_methods)]
fn engine_adopts_caller_tuned_tokio_runtime() {
// ENGINE_SPEC.md §9: the native engine supports
// consuming an explicitly tuned Tokio runtime rather than always building

8
crates/engine/tests/engine_unit.proptest-regressions Normal file
View file

@ -0,0 +1,8 @@
# Seeds for failure cases proptest has generated in the past. It is
# automatically read and these particular cases re-run before any
# novel cases are generated.
#
# It is recommended to check this file in to source control so that
# everyone who runs the test benefits from these saved cases.
cc 540459f422fe16f29c58b74f5e357a9caa7deb8f56bb46a52a182f44ff91b470 # shrinks to actions = [Complete(0)]
cc 41f8ea6a4dfca245bc21a381049ec2e855e33fa34f5a1adcbdf064b7bb38b182 # shrinks to actions = [ScheduleOnce(1), ScheduleOnce(1), ScheduleEvery(1), DropEngine, Advance(1)]

795
crates/engine/tests/engine_unit.rs
View file

@ -13,7 +13,13 @@ use std::sync::atomic::Ordering::SeqCst;
use std::sync::atomic::{AtomicBool, AtomicUsize};
use std::time::Duration;
use swactor_engine::{Capabilities, Engine, EngineError, ExecutionBackend, SteppingBackend};
use parking_lot::Mutex;
use proptest::prelude::*;
use swactor::actor::{ActorInterface, Ctx};
use swactor_engine::{
ActorCompletion, ActorTimer, Capabilities, Engine, EngineError, ExecutionBackend,
SteppingBackend,
};
#[cfg(feature = "tokio")]
use swactor_engine::{TokioBackend, TokioConfig};
@ -473,6 +479,60 @@ fn stepping_timer_fires_after_virtual_time_advance() {
);
}
#[test]
fn actor_message_timer_uses_engine_clock() {
let (parts, runtime) = default_runtime_parts();
let received = Arc::new(AtomicUsize::new(0));
let actor = runtime
.spawn(RecordingProbe {
received: Arc::clone(&received),
})
.expect("spawn recording actor");
let sender = runtime.create_sender();
let backend = SteppingBackend::new();
let engine = Engine::new(parts, backend.clone()).expect("construct engine");
engine
.handle()
.send_after(Duration::from_secs(1), sender, actor, Probe);
for _ in 0..4 {
backend.step();
}
assert_eq!(received.load(SeqCst), 0);
backend.advance_time(Duration::from_secs(1));
for _ in 0..4 {
backend.step();
}
assert_eq!(received.load(SeqCst), 1);
}
#[test]
fn cancelling_actor_message_timer_prevents_delivery() {
let (parts, runtime) = default_runtime_parts();
let received = Arc::new(AtomicUsize::new(0));
let actor = runtime
.spawn(RecordingProbe {
received: Arc::clone(&received),
})
.expect("spawn recording actor");
let sender = runtime.create_sender();
let backend = SteppingBackend::new();
let engine = Engine::new(parts, backend.clone()).expect("construct engine");
let timer = engine
.handle()
.send_after(Duration::from_secs(1), sender, actor, Probe);
timer.cancel();
backend.advance_time(Duration::from_secs(1));
for _ in 0..4 {
backend.step();
}
assert!(timer.is_cancelled());
assert_eq!(received.load(SeqCst), 0);
}
#[test]
fn stepping_blocking_work_runs_isolated() {
let parts = default_parts();
@ -482,10 +542,16 @@ fn stepping_blocking_work_runs_isolated() {
let done = Arc::new(AtomicBool::new(false));
let done_clone = done.clone();
handle.spawn_blocking(move || {
std::thread::sleep(Duration::from_millis(10));
done_clone.store(true, SeqCst);
});
assert!(
handle
.blocking_work_sender()
.submit(Box::new(move || {
std::thread::sleep(Duration::from_millis(10));
done_clone.store(true, SeqCst);
}))
.is_ok(),
"submit stepping work"
);
let deadline = std::time::Instant::now() + Duration::from_secs(2);
loop {
@ -499,6 +565,714 @@ fn stepping_blocking_work_runs_isolated() {
}
}
// ═══════════════════════════════════════════════════════════════════════════════
// §6 Generated control-flow contracts
// ═══════════════════════════════════════════════════════════════════════════════
const LIFECYCLE_DRAIN_STEPS: usize = 4;
const MAX_GENERATED_TIMER_DELAY: Duration = Duration::from_millis(4);
#[derive(Clone, Debug)]
struct TimerFuzzMessage {
token: usize,
generation: usize,
}
#[derive(Clone, Debug, PartialEq, Eq)]
struct TimerObservation {
token: usize,
message_generation: usize,
actor_generation: usize,
}
struct TimerFuzzProbe {
generation: Arc<AtomicUsize>,
observations: Arc<Mutex<Vec<TimerObservation>>>,
deliveries: Arc<Mutex<Vec<TimerObservation>>>,
}
impl ActorInterface for TimerFuzzProbe {
type Incoming = TimerFuzzMessage;
type Response = ();
fn handle(&mut self, _ctx: &Ctx, message: TimerFuzzMessage) {
let observation = TimerObservation {
token: message.token,
message_generation: message.generation,
actor_generation: self.generation.load(SeqCst),
};
self.observations.lock().push(observation.clone());
if observation.message_generation == observation.actor_generation {
self.deliveries.lock().push(observation);
}
}
}
#[derive(Clone, Copy, Debug)]
enum TimerKind {
Once,
Periodic,
}
#[derive(Debug)]
struct TimerRecord {
timer: ActorTimer,
token: usize,
kind: TimerKind,
scheduled_while_live: bool,
cancelled_at: Option<usize>,
}
type TerminalOutcome = Result<u8, u8>;
#[derive(Debug, Default)]
struct CompletionCensus {
attempts: Vec<TerminalOutcome>,
accepted: Vec<TerminalOutcome>,
rejected: Vec<TerminalOutcome>,
}
#[derive(Clone, Copy, Debug)]
struct LifecycleCensus {
pending_tasks: usize,
task_limit: usize,
timer_handles: usize,
timer_limit: usize,
uncancelled_periodic: usize,
quiesced: bool,
}
fn record_completion_attempt(
completion: &ActorCompletion<TerminalOutcome>,
census: &mut CompletionCensus,
outcome: TerminalOutcome,
) {
census.attempts.push(outcome);
match completion.complete(outcome) {
Ok(()) => census.accepted.push(outcome),
Err(rejected) => census.rejected.push(rejected),
}
}
fn check_lifecycle_invariants(
completions: &CompletionCensus,
census: LifecycleCensus,
) -> Result<(), String> {
let expected_accepted = usize::from(!completions.attempts.is_empty());
if completions.accepted.len() != expected_accepted {
return Err(format!(
"completion accepted {} terminal observations, expected {}; \
completion_census={completions:?}, lifecycle_census={census:?}",
completions.accepted.len(),
expected_accepted,
));
}
if completions.accepted.first() != completions.attempts.first() {
return Err(format!(
"completion did not preserve its first terminal observation; \
completion_census={completions:?}, lifecycle_census={census:?}",
));
}
if completions.rejected.as_slice() != &completions.attempts[expected_accepted..] {
return Err(format!(
"completion did not reject every duplicate terminal observation; \
completion_census={completions:?}, lifecycle_census={census:?}",
));
}
if census.quiesced && census.uncancelled_periodic != 0 {
return Err(format!(
"{} uncancelled periodic timer(s) survived quiescence; \
completion_census={completions:?}, lifecycle_census={census:?}",
census.uncancelled_periodic,
));
}
if census.pending_tasks > census.task_limit {
return Err(format!(
"task cardinality {} exceeded generated-operation bound {}; \
completion_census={completions:?}, lifecycle_census={census:?}",
census.pending_tasks, census.task_limit,
));
}
if census.timer_handles > census.timer_limit {
return Err(format!(
"timer cardinality {} exceeded generated-operation bound {}; \
completion_census={completions:?}, lifecycle_census={census:?}",
census.timer_handles, census.timer_limit,
));
}
Ok(())
}
fn observations_for_token(observations: &[TimerObservation], token: usize) -> usize {
observations
.iter()
.filter(|observation| observation.token == token)
.count()
}
#[derive(Clone, Debug)]
enum TimerAction {
ScheduleOnce(u8),
ScheduleEvery(u8),
Cancel(u8),
Advance(u8),
BumpGeneration,
StopActor,
DropEngine,
CompleteSuccess(u8),
CompleteError(u8),
}
fn timer_actions() -> impl Strategy<Value = Vec<TimerAction>> {
proptest::collection::vec(
prop_oneof![
3 => (1_u8..=4).prop_map(TimerAction::ScheduleOnce),
2 => (1_u8..=4).prop_map(TimerAction::ScheduleEvery),
2 => any::<u8>().prop_map(TimerAction::Cancel),
3 => (0_u8..=4).prop_map(TimerAction::Advance),
1 => Just(TimerAction::BumpGeneration),
1 => Just(TimerAction::StopActor),
1 => Just(TimerAction::DropEngine),
1 => any::<u8>().prop_map(TimerAction::CompleteSuccess),
1 => any::<u8>().prop_map(TimerAction::CompleteError),
],
0..=32,
)
}
proptest! {
#![proptest_config(ProptestConfig {
cases: 128,
max_shrink_iters: 2_000,
..ProptestConfig::default()
})]
#[test]
fn generated_actor_timers_and_completion_are_bounded(actions in timer_actions()) {
let (parts, runtime) = default_runtime_parts();
let baseline = runtime.stats().actors.len();
let generation = Arc::new(AtomicUsize::new(0));
let observations = Arc::new(Mutex::new(Vec::new()));
let deliveries = Arc::new(Mutex::new(Vec::new()));
let actor = runtime
.spawn(TimerFuzzProbe {
generation: Arc::clone(&generation),
observations: Arc::clone(&observations),
deliveries: Arc::clone(&deliveries),
})
.expect("spawn timer fuzz probe");
let sender = runtime.create_sender();
let backend = SteppingBackend::new();
let mut engine = Some(Engine::new(parts, backend.clone()).expect("construct engine"));
let handle = engine.as_ref().unwrap().handle();
let driver_task_count = backend.pending_task_count();
let completion = ActorCompletion::new();
let mut completion_census = CompletionCensus::default();
let mut timers = Vec::<TimerRecord>::new();
let mut scheduled_while_live = 0;
let mut generated_timer_actions = 0;
let mut stopped_at = None;
let assert_post_drop_scheduling_is_inert = || -> Result<(), String> {
let pending_before = backend.pending_task_count();
let generation = generation.load(SeqCst);
let one_shot = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
handle.send_after(
Duration::from_millis(1),
sender.clone(),
actor,
TimerFuzzMessage {
token: usize::MAX,
generation,
},
)
}));
let periodic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
handle.send_every(
Duration::from_millis(1),
sender.clone(),
actor,
TimerFuzzMessage {
token: usize::MAX,
generation,
},
)
}));
if one_shot.is_err() || periodic.is_err() {
return Err("post-drop one-shot or periodic scheduling panicked".to_owned());
}
let pending_after = backend.pending_task_count();
if pending_after != pending_before {
return Err(format!(
"post-drop scheduling changed pending tasks from {pending_before} to \
{pending_after}",
));
}
if handle.capabilities() != Capabilities::NONE
|| handle.require(Capabilities::TASKS_ONLY).is_ok()
{
return Err("dropped engine still advertised scheduling capabilities".to_owned());
}
Ok(())
};
drive_steps(&backend, LIFECYCLE_DRAIN_STEPS);
for action in &actions {
match *action {
TimerAction::ScheduleOnce(delay) => {
let token = timers.len();
let engine_was_live = engine.is_some();
let tasks_before = backend.pending_task_count();
let scheduled =
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
handle.send_after(
Duration::from_millis(u64::from(delay)),
sender.clone(),
actor,
TimerFuzzMessage {
token,
generation: generation.load(SeqCst),
},
)
}));
prop_assert!(
scheduled.is_ok(),
"one-shot scheduling panicked; actions={:?}, action={:?}, stats={:?}",
actions,
action,
runtime.stats(),
);
let timer = scheduled.unwrap();
generated_timer_actions += 1;
if engine_was_live {
scheduled_while_live += 1;
prop_assert_eq!(
backend.pending_task_count(),
tasks_before + 1,
"live one-shot did not create exactly one task; actions={:?}, \
action={:?}, stats={:?}",
actions,
action,
runtime.stats(),
);
} else {
prop_assert_eq!(
backend.pending_task_count(),
tasks_before,
"post-drop one-shot was not inert; actions={:?}, action={:?}, \
stats={:?}",
actions,
action,
runtime.stats(),
);
}
timers.push(TimerRecord {
timer,
token,
kind: TimerKind::Once,
scheduled_while_live: engine_was_live,
cancelled_at: None,
});
}
TimerAction::ScheduleEvery(period) => {
let token = timers.len();
let engine_was_live = engine.is_some();
let tasks_before = backend.pending_task_count();
let scheduled =
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
handle.send_every(
Duration::from_millis(u64::from(period)),
sender.clone(),
actor,
TimerFuzzMessage {
token,
generation: generation.load(SeqCst),
},
)
}));
prop_assert!(
scheduled.is_ok(),
"periodic scheduling panicked; actions={:?}, action={:?}, stats={:?}",
actions,
action,
runtime.stats(),
);
let timer = scheduled.unwrap();
generated_timer_actions += 1;
if engine_was_live {
scheduled_while_live += 1;
prop_assert_eq!(
backend.pending_task_count(),
tasks_before + 1,
"live periodic send did not create exactly one task; actions={:?}, \
action={:?}, stats={:?}",
actions,
action,
runtime.stats(),
);
} else {
prop_assert_eq!(
backend.pending_task_count(),
tasks_before,
"post-drop periodic send was not inert; actions={:?}, action={:?}, \
stats={:?}",
actions,
action,
runtime.stats(),
);
}
timers.push(TimerRecord {
timer,
token,
kind: TimerKind::Periodic,
scheduled_while_live: engine_was_live,
cancelled_at: None,
});
}
TimerAction::Cancel(index) => {
if !timers.is_empty() {
drive_steps(&backend, LIFECYCLE_DRAIN_STEPS);
let index = usize::from(index) % timers.len();
let delivered =
observations_for_token(&observations.lock(), timers[index].token);
timers[index].timer.cancel();
timers[index].timer.cancel();
timers[index].cancelled_at.get_or_insert(delivered);
prop_assert!(
timers[index].timer.is_cancelled(),
"repeated cancellation was not idempotent; actions={:?}, \
action={:?}, timer_census={:?}",
actions,
action,
timers,
);
}
}
TimerAction::Advance(milliseconds) => {
advance_and_drive(
&backend,
Duration::from_millis(u64::from(milliseconds)),
LIFECYCLE_DRAIN_STEPS,
);
}
TimerAction::BumpGeneration => {
generation.fetch_add(1, SeqCst);
}
TimerAction::StopActor => {
drive_steps(&backend, LIFECYCLE_DRAIN_STEPS);
let observed = observations.lock().len();
let _ = runtime.stop_actor(actor);
stopped_at.get_or_insert(observed);
}
TimerAction::DropEngine => {
drop(engine.take());
let post_drop = assert_post_drop_scheduling_is_inert();
prop_assert!(
post_drop.is_ok(),
"post-drop invariant failed: {}; actions={:?}, action={:?}, \
timer_census={:?}, completion_census={:?}, stats={:?}",
post_drop.as_ref().unwrap_err(),
actions,
action,
timers,
completion_census,
runtime.stats(),
);
}
TimerAction::CompleteSuccess(value) => {
record_completion_attempt(&completion, &mut completion_census, Ok(value));
}
TimerAction::CompleteError(value) => {
record_completion_attempt(&completion, &mut completion_census, Err(value));
}
}
drive_steps(&backend, LIFECYCLE_DRAIN_STEPS);
let observed = observations.lock().clone();
let delivered = deliveries.lock().clone();
let expected_deliveries = observed
.iter()
.filter(|observation| {
observation.message_generation == observation.actor_generation
})
.cloned()
.collect::<Vec<_>>();
prop_assert_eq!(
delivered,
expected_deliveries,
"stale generation was delivered or a current generation was lost; actions={:?}, \
action={:?}, observations={:?}, timer_census={:?}, \
completion_census={:?}, stats={:?}",
actions,
action,
observed,
timers,
completion_census,
runtime.stats(),
);
for timer in &timers {
if let Some(cancelled_at) = timer.cancelled_at {
let current = observations_for_token(&observed, timer.token);
prop_assert_eq!(
current,
cancelled_at,
"timer {} delivered after cancellation; actions={:?}, action={:?}, \
observations={:?}, timer_census={:?}, completion_census={:?}, \
stats={:?}",
timer.token,
actions,
action,
observed,
timers,
completion_census,
runtime.stats(),
);
}
}
if let Some(stopped_at) = stopped_at {
prop_assert_eq!(
observed.len(),
stopped_at,
"message delivered after actor stop; actions={:?}, action={:?}, \
observations={:?}, timer_census={:?}, completion_census={:?}, stats={:?}",
actions,
action,
observed,
timers,
completion_census,
runtime.stats(),
);
}
let lifecycle_census = LifecycleCensus {
pending_tasks: backend.pending_task_count(),
task_limit: driver_task_count + scheduled_while_live,
timer_handles: timers.len(),
timer_limit: generated_timer_actions,
uncancelled_periodic: timers
.iter()
.filter(|timer| {
timer.scheduled_while_live
&& matches!(timer.kind, TimerKind::Periodic)
&& !timer.timer.is_cancelled()
})
.count(),
quiesced: false,
};
let invariant =
check_lifecycle_invariants(&completion_census, lifecycle_census);
prop_assert!(
invariant.is_ok(),
"lifecycle invariant failed: {}; actions={:?}, action={:?}, \
observations={:?}, timer_census={:?}, completion_census={:?}, stats={:?}",
invariant.as_ref().unwrap_err(),
actions,
action,
observed,
timers,
completion_census,
runtime.stats(),
);
prop_assert!(
runtime.stats().actors.len() <= baseline + 1,
"actor cardinality exceeded fixed bound; actions={:?}, action={:?}, \
timer_census={:?}, completion_census={:?}, stats={:?}",
actions,
action,
timers,
completion_census,
runtime.stats(),
);
}
if let Some(expected) = completion_census.accepted.first().copied() {
let observed = completion.wait();
prop_assert_eq!(
observed,
expected,
"completion wait returned a different terminal observation; actions={:?}, \
timer_census={:?}, completion_census={:?}, stats={:?}",
actions,
timers,
completion_census,
runtime.stats(),
);
record_completion_attempt(&completion, &mut completion_census, expected);
}
drive_steps(&backend, LIFECYCLE_DRAIN_STEPS);
let observed = observations.lock().clone();
for timer in &mut timers {
let delivered = observations_for_token(&observed, timer.token);
timer.timer.cancel();
timer.timer.cancel();
timer.cancelled_at.get_or_insert(delivered);
}
advance_and_drive(
&backend,
MAX_GENERATED_TIMER_DELAY,
LIFECYCLE_DRAIN_STEPS,
);
drop(engine.take());
let post_drop = assert_post_drop_scheduling_is_inert();
prop_assert!(
post_drop.is_ok(),
"final post-drop invariant failed: {}; actions={:?}, timer_census={:?}, \
completion_census={:?}, stats={:?}",
post_drop.as_ref().unwrap_err(),
actions,
timers,
completion_census,
runtime.stats(),
);
drive_steps(&backend, LIFECYCLE_DRAIN_STEPS);
let final_observations = observations.lock().clone();
let final_deliveries = deliveries.lock().clone();
let expected_final_deliveries = final_observations
.iter()
.filter(|observation| {
observation.message_generation == observation.actor_generation
})
.cloned()
.collect::<Vec<_>>();
prop_assert_eq!(
final_deliveries,
expected_final_deliveries,
"final drain delivered a stale generation or lost a current generation; \
actions={:?}, observations={:?}, timer_census={:?}, completion_census={:?}, \
stats={:?}",
actions,
final_observations,
timers,
completion_census,
runtime.stats(),
);
for timer in &timers {
let delivered = observations_for_token(&final_observations, timer.token);
prop_assert_eq!(
delivered,
timer.cancelled_at.expect("all timers cancelled during final drain"),
"timer {} delivered during the final post-cancellation drain; actions={:?}, \
observations={:?}, timer_census={:?}, completion_census={:?}, stats={:?}",
timer.token,
actions,
final_observations,
timers,
completion_census,
runtime.stats(),
);
}
let lifecycle_census = LifecycleCensus {
pending_tasks: backend.pending_task_count(),
task_limit: driver_task_count,
timer_handles: timers.len(),
timer_limit: generated_timer_actions,
uncancelled_periodic: timers
.iter()
.filter(|timer| {
timer.scheduled_while_live
&& matches!(timer.kind, TimerKind::Periodic)
&& !timer.timer.is_cancelled()
})
.count(),
quiesced: true,
};
let invariant = check_lifecycle_invariants(&completion_census, lifecycle_census);
prop_assert!(
invariant.is_ok(),
"final lifecycle invariant failed: {}; actions={:?}, observations={:?}, \
timer_census={:?}, completion_census={:?}, lifecycle_census={:?}, stats={:?}",
invariant.as_ref().unwrap_err(),
actions,
observations.lock(),
timers,
completion_census,
lifecycle_census,
runtime.stats(),
);
let final_stats = runtime.stats();
let worker_panics = final_stats
.workers
.iter()
.map(|worker| worker.panics)
.sum::<u64>();
let poisoned = final_stats
.actor_details
.iter()
.filter(|actor| actor.poisoned)
.collect::<Vec<_>>();
prop_assert!(
worker_panics == 0 && poisoned.is_empty(),
"runtime poisoned during generated lifecycle; actions={:?}, observations={:?}, \
timer_census={:?}, completion_census={:?}, lifecycle_census={:?}, stats={:?}",
actions,
final_observations,
timers,
completion_census,
lifecycle_census,
final_stats,
);
prop_assert!(
final_stats.actors.len() <= baseline + 1,
"final actor cardinality exceeded fixed bound; actions={:?}, observations={:?}, \
timer_census={:?}, completion_census={:?}, lifecycle_census={:?}, stats={:?}",
actions,
final_observations,
timers,
completion_census,
lifecycle_census,
final_stats,
);
}
}
#[test]
fn lifecycle_invariant_detects_injected_duplicate_completion() {
let completions = CompletionCensus {
attempts: vec![Ok(7), Err(9)],
accepted: vec![Ok(7), Err(9)],
rejected: Vec::new(),
};
let census = LifecycleCensus {
pending_tasks: 1,
task_limit: 1,
timer_handles: 0,
timer_limit: 0,
uncancelled_periodic: 0,
quiesced: false,
};
let violation = check_lifecycle_invariants(&completions, census)
.expect_err("the exactly-once invariant must reject an injected duplicate completion");
assert!(
violation.contains("completion accepted 2 terminal observations"),
"wrong detector failure for injected actions=[CompleteSuccess(7), CompleteError(9)]: \
violation={violation}, completion_census={completions:?}, lifecycle_census={census:?}",
);
}
#[test]
fn lifecycle_invariant_detects_injected_uncancelled_periodic_timer() {
let completions = CompletionCensus::default();
let census = LifecycleCensus {
pending_tasks: 2,
task_limit: 1,
timer_handles: 1,
timer_limit: 1,
uncancelled_periodic: 1,
quiesced: true,
};
let violation = check_lifecycle_invariants(&completions, census)
.expect_err("the leak invariant must reject an injected uncancelled periodic timer");
assert!(
violation.contains("uncancelled periodic timer(s) survived quiescence"),
"wrong detector failure for injected actions=[ScheduleEvery(1), DropEngine]: \
violation={violation}, completion_census={completions:?}, lifecycle_census={census:?}",
);
}
#[test]
fn stepping_spawned_task_completing_is_removed_from_queue() {
let parts = default_parts();
@ -620,10 +1394,15 @@ fn handle_used_after_engine_drop_degrades_gracefully() {
// Time falls back to the wall clock without panicking.
let _ = handle.now();
// Scheduling work and creating primitives are no-ops / never fire, never
// panic, and never retain the backend.
// Scheduling work and creating primitives reject / no-op / never fire,
// never panic, and never retain the backend.
handle.spawn(async {});
handle.spawn_blocking(|| {});
assert!(
handle
.blocking_work_sender()
.submit(Box::new(|| {}))
.is_err()
);
let _never_fires = handle.timer(Duration::from_secs(1));
let _never_ticks = handle.interval(Duration::from_secs(1));
}

12
crates/iroh-driver/src/edge_transport.rs
View file

@ -45,6 +45,7 @@ pub(crate) fn spawn_edge_send_pump(
endpoint: Endpoint,
peer: EndpointAddr,
edge_id: u64,
ready_timeout: Option<std::time::Duration>,
) -> Result<EdgeSendHandle, String> {
let (tx, mut rx) = tokio_mpsc::unbounded_channel::<Vec<u8>>();
let (ready_tx, ready_rx) = std::sync::mpsc::channel::<Result<(), String>>();
@ -108,9 +109,14 @@ pub(crate) fn spawn_edge_send_pump(
let _ = ready_tx.send(Err(error));
}
});
ready_rx
.recv()
.map_err(|e| format!("edge {edge_id} sender startup channel closed: {e}"))??;
match ready_timeout {
Some(timeout) => ready_rx
.recv_timeout(timeout)
.map_err(|error| format!("edge {edge_id} sender startup: {error}"))??,
None => ready_rx
.recv()
.map_err(|error| format!("edge {edge_id} sender startup channel closed: {error}"))??,
}
Ok(EdgeSendHandle { tx })
}

24
crates/iroh-driver/src/iroh_driver.rs
View file

@ -598,7 +598,29 @@ impl IrohDriver {
peer: EndpointAddr,
edge_id: u64,
) -> Result<EdgeSendHandle, String> {
spawn_edge_sender_task(self.engine.clone(), self.endpoint.clone(), peer, edge_id)
spawn_edge_sender_task(
self.engine.clone(),
self.endpoint.clone(),
peer,
edge_id,
None,
)
}
/// Start an EDGE_ALPN send pump with a bounded connection handshake.
pub fn spawn_edge_send_pump_timeout(
&self,
peer: EndpointAddr,
edge_id: u64,
timeout: std::time::Duration,
) -> Result<EdgeSendHandle, String> {
spawn_edge_sender_task(
self.engine.clone(),
self.endpoint.clone(),
peer,
edge_id,
Some(timeout),
)
}
/// The node's identity.

7
crates/iroh-driver/src/lib.rs
View file

@ -25,9 +25,10 @@ pub use iroh_driver::{
pub use edge_transport::{EDGE_ALPN, EdgeSendHandle};
pub use telemetry_transport::{
TELEMETRY_ALPN, TelemetryQuicHeader, TelemetryQuicRead, TelemetryQuicWriteStats,
read_events_from_stream, read_next_event, read_next_uni_from_connection, read_pull_request,
read_stream_header, read_stream_into_fanout, spawn_connection_reader, spawn_pull_collector,
PullCollectorHandle, TELEMETRY_ALPN, TelemetryQuicHeader, TelemetryQuicRead,
TelemetryQuicWriteStats, read_events_from_stream, read_next_event,
read_next_uni_from_connection, read_pull_request, read_stream_header, read_stream_into_fanout,
spawn_connection_reader, spawn_pull_collector, spawn_pull_collector_to_actor,
spawn_pull_server, spawn_subscription_writer, write_available_subscription, write_event,
write_pull_request, write_subscription_until_closed,
};

143
crates/iroh-driver/src/telemetry_transport.rs
View file

@ -7,6 +7,8 @@ use std::time::Duration;
use crossbeam_channel::TryRecvError;
use iroh::endpoint::{Connection, RecvStream, SendStream};
use iroh::{Endpoint, EndpointAddr};
use swactor::actor::ActorAddress;
use swactor::runtime::ExternalSender;
use swactor_engine::EngineHandle;
use telemetry::frame::{
ChannelDescriptor, ChannelId, ChannelRef, FrameDelivery, Position, StreamDescriptor,
@ -104,6 +106,44 @@ pub fn spawn_pull_server(
});
}
/// Cancellation handle for one collector-initiated telemetry subscription.
///
/// Cancellation is idempotent and immediately interrupts network I/O,
/// reconnect backoff, and all subsequent reconnect attempts.
#[derive(Debug)]
pub struct PullCollectorHandle {
cancellation: tokio::sync::watch::Sender<bool>,
completion: tokio::sync::watch::Receiver<bool>,
}
impl PullCollectorHandle {
pub fn cancel(&self) {
self.cancellation.send_replace(true);
}
pub fn is_cancelled(&self) -> bool {
*self.cancellation.borrow()
}
pub fn is_finished(&self) -> bool {
*self.completion.borrow()
}
}
impl Drop for PullCollectorHandle {
fn drop(&mut self) {
self.cancel();
}
}
struct PullCollectorCompletion(tokio::sync::watch::Sender<bool>);
impl Drop for PullCollectorCompletion {
fn drop(&mut self) {
self.0.send_replace(true);
}
}
/// Supervisor side: retain a pull subscription to a node on `TELEMETRY_ALPN`.
///
/// A transport interruption reconnects with bounded backoff. Returning after
@ -118,17 +158,91 @@ pub fn spawn_pull_collector(
request: telemetry::SubscriptionRequest,
fanout: std::sync::Arc<telemetry::DeliveryFanout>,
on_header: std::sync::mpsc::Sender<TelemetryQuicHeader>,
) {
) -> PullCollectorHandle {
spawn_pull_collector_with_sink(
engine,
endpoint,
peer,
flow_id,
token,
request,
fanout,
PullHeaderSink::Channel(on_header),
)
}
/// Supervisor side variant that delivers each connection header directly to
/// an actor. Transport owns the subscription task; the actor owns how the
/// stream identity changes domain state.
pub fn spawn_pull_collector_to_actor(
engine: &EngineHandle,
endpoint: Endpoint,
peer: EndpointAddr,
flow_id: [u8; 16],
token: Vec<u8>,
request: telemetry::SubscriptionRequest,
fanout: std::sync::Arc<telemetry::DeliveryFanout>,
sender: ExternalSender,
actor: ActorAddress,
) -> PullCollectorHandle {
spawn_pull_collector_with_sink(
engine,
endpoint,
peer,
flow_id,
token,
request,
fanout,
PullHeaderSink::Actor { sender, actor },
)
}
enum PullHeaderSink {
Channel(std::sync::mpsc::Sender<TelemetryQuicHeader>),
Actor {
sender: ExternalSender,
actor: ActorAddress,
},
}
impl PullHeaderSink {
fn deliver(&self, header: TelemetryQuicHeader) -> bool {
match self {
Self::Channel(sender) => sender.send(header).is_ok(),
Self::Actor { sender, actor } => sender.send_to(*actor, header).is_ok(),
}
}
}
#[allow(clippy::too_many_arguments)]
fn spawn_pull_collector_with_sink(
engine: &EngineHandle,
endpoint: Endpoint,
peer: EndpointAddr,
flow_id: [u8; 16],
token: Vec<u8>,
request: telemetry::SubscriptionRequest,
fanout: std::sync::Arc<telemetry::DeliveryFanout>,
on_header: PullHeaderSink,
) -> PullCollectorHandle {
let (cancellation, mut cancellation_rx) = tokio::sync::watch::channel(false);
let (completion, completion_rx) = tokio::sync::watch::channel(false);
let engine_handle = engine.clone();
engine.spawn(async move {
let _completion = PullCollectorCompletion(completion);
let peer_id = peer.id.to_string();
let mut retry_delay = Duration::from_millis(250);
loop {
match collect_pull_once(
&endpoint, &peer, flow_id, &token, &request, &fanout, &on_header,
)
.await
{
if *cancellation_rx.borrow() {
return;
}
let result = tokio::select! {
_ = cancellation_rx.changed() => return,
result = collect_pull_once(
&endpoint, &peer, flow_id, &token, &request, &fanout, &on_header,
) => result,
};
match result {
Ok(()) => return,
Err(error) => {
eprintln!(
@ -137,13 +251,20 @@ pub fn spawn_pull_collector(
);
}
}
engine_handle.timer(retry_delay).await;
tokio::select! {
_ = cancellation_rx.changed() => return,
_ = engine_handle.timer(retry_delay) => {}
}
retry_delay = retry_delay
.checked_mul(2)
.unwrap_or(Duration::from_secs(5))
.min(Duration::from_secs(5));
}
});
PullCollectorHandle {
cancellation,
completion: completion_rx,
}
}
async fn collect_pull_once(
@ -153,7 +274,7 @@ async fn collect_pull_once(
token: &[u8],
request: &telemetry::SubscriptionRequest,
fanout: &telemetry::DeliveryFanout,
on_header: &std::sync::mpsc::Sender<TelemetryQuicHeader>,
on_header: &PullHeaderSink,
) -> Result<(), String> {
let conn = endpoint
.connect(peer.clone(), TELEMETRY_ALPN)
@ -173,14 +294,18 @@ async fn collect_pull_once(
let header = read_header(&mut recv)
.await
.map_err(|error| format!("answer header unreadable: {error}"))?;
if on_header.send(header.clone()).is_err() {
if !on_header.deliver(header.clone()) {
return Ok(());
}
let stream = header.stream;
loop {
match read_next_event(&mut recv, &stream).await {
Ok(Some(event)) => {
let ended = matches!(event, TelemetryEvent::StreamEnded(_));
fanout.publish(event);
if ended {
return Ok(());
}
}
Ok(None) => return Err("answer stream closed".to_owned()),
Err(error) => return Err(format!("read answer stream failed: {error}")),

60
crates/iroh-driver/tests/telemetry_transport.rs
View file

@ -1,15 +1,20 @@
use std::net::{IpAddr, SocketAddr};
use std::sync::Arc;
use std::time::{Duration, Instant};
use iroh::{Endpoint, EndpointAddr, RelayMode};
use iroh_driver::{
TELEMETRY_ALPN, TelemetryQuicHeader, read_next_uni_from_connection,
TELEMETRY_ALPN, TelemetryQuicHeader, read_next_uni_from_connection, spawn_pull_collector,
write_available_subscription,
};
use swactor::config::RuntimeConfig;
use swactor::runtime::RuntimeParts;
use swactor_engine::{Engine, TokioBackend, TokioConfig};
use telemetry::frame::TelemetryEvent;
use telemetry::{ChannelContent, Lifetime, NodeId, Position, StreamId, TelemetryEndpoint};
use telemetry::{
ChannelContent, DeliveryFanout, Lifetime, NodeId, Position, StreamId, SubscriptionRequest,
TelemetryEndpoint,
};
/// Telemetry transport test scheduled through `EngineHandle`, not an ambient
/// `#[tokio::test]` runtime (ENGINE_SPEC.md).
@ -112,6 +117,57 @@ fn iroh_telemetry_alpn_carries_catalog_and_numeric_frames() {
}
}
#[test]
fn pull_collector_cancellation_interrupts_inflight_io() {
let parts = RuntimeParts::new(RuntimeConfig::default());
let engine = Engine::new(
parts,
TokioBackend::new(TokioConfig::default()).expect("test backend"),
)
.expect("test engine");
let handle = engine.handle();
let (resource_tx, resource_rx) = std::sync::mpsc::channel();
let setup_handle = handle.clone();
handle.spawn(async move {
let collector_endpoint = test_endpoint().await;
let silent_peer = test_endpoint().await;
let (header_tx, header_rx) = std::sync::mpsc::channel();
let collector = spawn_pull_collector(
&setup_handle,
collector_endpoint.clone(),
endpoint_addr(&silent_peer),
[3; 16],
Vec::new(),
SubscriptionRequest::all(),
Arc::new(DeliveryFanout::new(8)),
header_tx,
);
resource_tx
.send((collector, collector_endpoint, silent_peer, header_rx))
.expect("return collector resources");
});
let (collector, collector_endpoint, silent_peer, _header_rx) = resource_rx
.recv_timeout(Duration::from_secs(5))
.expect("collector setup");
std::thread::sleep(Duration::from_millis(100));
assert!(
!collector.is_finished(),
"collector was not retained in silent-peer network I/O"
);
collector.cancel();
let stopped_deadline = Instant::now() + Duration::from_secs(2);
while !collector.is_finished() && Instant::now() < stopped_deadline {
std::thread::sleep(Duration::from_millis(10));
}
assert!(
collector.is_finished(),
"cancelled collector remained blocked in network I/O"
);
drop((collector_endpoint, silent_peer));
}
async fn test_endpoint() -> Endpoint {
Endpoint::builder(iroh::endpoint::presets::Minimal)
.relay_mode(RelayMode::Disabled)

186
crates/job-runner/src/node.rs
View file

@ -20,6 +20,7 @@ use crate::wire::{
};
use swactor::actor::{ActorAddress, ActorInterface, Ctx};
use swactor::runtime::ExternalSender;
use swactor_engine::EngineHandle;
use swactor_process::{
ExitStatus, ProcessOutput, ProcessOutputConfig, ProcessSpec, spawn_local_process,
};
@ -31,6 +32,12 @@ pub enum JobPhase {
Run,
}
struct PendingOutputs {
job_id: u64,
outputs: Vec<String>,
deadline: Instant,
}
/// The node-side executor. `Incoming` is the orchestrator↔node wire command.
pub struct NodeJobActor {
orchestrator: ActorAddress,
@ -38,6 +45,9 @@ pub struct NodeJobActor {
sender: ExternalSender,
job_id: u64,
workspace_buf: Vec<u8>,
actor_timers: Option<EngineHandle>,
workspace_wait: Option<(u64, Instant)>,
pending_outputs: Option<PendingOutputs>,
/// Edge-mode workspace-ready flag. When set, the orchestrator pushed the
/// workspace tar over EDGE_ALPN (drained + extracted by the integration
/// layer); `MaterializeWorkspace` waits for it before emitting
@ -70,11 +80,20 @@ impl NodeJobActor {
sender,
job_id,
workspace_buf: Vec::new(),
actor_timers: None,
workspace_wait: None,
pending_outputs: None,
workspace_ready: None,
output_sink: None,
}
}
/// Give edge-mode waits access only to engine-owned typed actor timers.
pub fn with_actor_timers(mut self, engine: EngineHandle) -> Self {
self.actor_timers = Some(engine);
self
}
/// Edge mode: workspace bytes arrive over EDGE_ALPN and are extracted by the
/// integration layer, which sets `flag` once the workspace is on disk.
pub fn with_workspace_ready(mut self, flag: Arc<AtomicBool>) -> Self {
@ -148,12 +167,8 @@ impl NodeJobActor {
ctx: &Ctx,
job_id: u64,
outputs: &[String],
slot: Arc<Mutex<Option<Box<dyn JobEdgeSink>>>>,
sink: Box<dyn JobEdgeSink>,
) {
let sink = match self.take_output_sink(ctx, job_id, &slot) {
Some(sink) => sink,
None => return, // already faulted while waiting for the sink
};
let bytes = match pack_outputs_tar(&self.workdir, outputs) {
Ok(b) => b,
Err(e) => {
@ -191,33 +206,106 @@ impl NodeJobActor {
self.emit(ctx, NodeJobEvent::OutputsCollected { job_id });
}
/// Take the edge output sink from the shared slot, waiting briefly for the
/// integration layer to arm it. Emits a fault and returns `None` on timeout.
fn take_output_sink(
&self,
ctx: &Ctx,
job_id: u64,
slot: &Arc<Mutex<Option<Box<dyn JobEdgeSink>>>>,
) -> Option<Box<dyn JobEdgeSink>> {
let deadline = Instant::now() + OUTPUT_EDGE_ARM_WAIT;
loop {
{
let mut guard = slot.lock();
if guard.is_some() {
return guard.take();
}
}
if Instant::now() >= deadline {
self.emit(
ctx,
NodeJobEvent::NodeFault {
job_id,
reason: "output edge sink was never armed".to_owned(),
},
);
return None;
}
std::thread::sleep(EDGE_SPIN);
fn schedule_edge_check(&self, ctx: &Ctx, job_id: u64, message: NodeJobCommand) -> bool {
let Some(engine) = &self.actor_timers else {
self.emit(
ctx,
NodeJobEvent::NodeFault {
job_id,
reason: "edge mode requires engine-owned actor timers".to_owned(),
},
);
return false;
};
engine.send_after(EDGE_SPIN, self.sender.clone(), ctx.self_addr(), message);
true
}
fn begin_workspace_wait(&mut self, ctx: &Ctx, job_id: u64) {
if self
.workspace_ready
.as_ref()
.is_some_and(|ready| ready.load(Ordering::Acquire))
{
self.emit(ctx, NodeJobEvent::WorkspaceMaterialized { job_id });
return;
}
self.workspace_wait = Some((job_id, Instant::now() + WORKSPACE_EDGE_WAIT));
if !self.schedule_edge_check(ctx, job_id, NodeJobCommand::CheckWorkspaceReady { job_id }) {
self.workspace_wait = None;
}
}
fn check_workspace_ready(&mut self, ctx: &Ctx, job_id: u64) {
let Some((pending_job, deadline)) = self.workspace_wait else {
return;
};
if pending_job != job_id {
return;
}
if self
.workspace_ready
.as_ref()
.is_some_and(|ready| ready.load(Ordering::Acquire))
{
self.workspace_wait = None;
self.emit(ctx, NodeJobEvent::WorkspaceMaterialized { job_id });
} else if Instant::now() >= deadline {
self.workspace_wait = None;
self.emit(
ctx,
NodeJobEvent::NodeFault {
job_id,
reason: "workspace edge transfer did not land".to_owned(),
},
);
} else {
self.schedule_edge_check(ctx, job_id, NodeJobCommand::CheckWorkspaceReady { job_id });
}
}
fn take_ready_output_sink(&self) -> Option<Box<dyn JobEdgeSink>> {
self.output_sink
.as_ref()
.and_then(|slot| slot.lock().take())
}
fn begin_output_wait(&mut self, ctx: &Ctx, job_id: u64, outputs: Vec<String>) {
if let Some(sink) = self.take_ready_output_sink() {
self.collect_outputs_edge(ctx, job_id, &outputs, sink);
return;
}
self.pending_outputs = Some(PendingOutputs {
job_id,
outputs,
deadline: Instant::now() + OUTPUT_EDGE_ARM_WAIT,
});
if !self.schedule_edge_check(ctx, job_id, NodeJobCommand::CheckOutputSink { job_id }) {
self.pending_outputs = None;
}
}
fn check_output_sink(&mut self, ctx: &Ctx, job_id: u64) {
let Some(pending) = self.pending_outputs.as_ref() else {
return;
};
if pending.job_id != job_id {
return;
}
if let Some(sink) = self.take_ready_output_sink() {
let pending = self.pending_outputs.take().expect("pending output state");
self.collect_outputs_edge(ctx, job_id, &pending.outputs, sink);
} else if Instant::now() >= pending.deadline {
self.pending_outputs = None;
self.emit(
ctx,
NodeJobEvent::NodeFault {
job_id,
reason: "output edge sink was never armed".to_owned(),
},
);
} else {
self.schedule_edge_check(ctx, job_id, NodeJobCommand::CheckOutputSink { job_id });
}
}
}
@ -229,25 +317,8 @@ impl ActorInterface for NodeJobActor {
fn handle(&mut self, ctx: &Ctx, cmd: NodeJobCommand) {
match cmd {
NodeJobCommand::MaterializeWorkspace { job_id } => {
if let Some(flag) = self.workspace_ready.as_ref() {
// Edge mode: the workspace tar traveled over EDGE_ALPN and
// was extracted into `workdir` by the integration layer.
// Wait for its readiness signal before announcing ready.
let deadline = Instant::now() + WORKSPACE_EDGE_WAIT;
while !flag.load(Ordering::Acquire) {
if Instant::now() >= deadline {
self.emit(
ctx,
NodeJobEvent::NodeFault {
job_id,
reason: "workspace edge transfer did not land".to_owned(),
},
);
return;
}
std::thread::sleep(EDGE_SPIN);
}
self.emit(ctx, NodeJobEvent::WorkspaceMaterialized { job_id });
if self.workspace_ready.is_some() {
self.begin_workspace_wait(ctx, job_id);
} else {
// Chunk mode: clear the buffer; bytes arrive as WorkspaceChunk.
self.workspace_buf.clear();
@ -278,11 +349,8 @@ impl ActorInterface for NodeJobActor {
self.spawn_supervised(ctx, JobPhase::Run, command, &env);
}
NodeJobCommand::CollectOutputs { job_id, outputs } => {
if let Some(slot) = self.output_sink.clone() {
// Edge mode: pack all outputs into one tar and ship over
// EDGE_ALPN, then announce collection. Dropping the sink
// finishes the stream so the orchestrator sees end-of-stream.
self.collect_outputs_edge(ctx, job_id, &outputs, slot);
if self.output_sink.is_some() {
self.begin_output_wait(ctx, job_id, outputs);
} else {
for name in &outputs {
let path = self.workdir.join(name);
@ -304,6 +372,12 @@ impl ActorInterface for NodeJobActor {
self.emit(ctx, NodeJobEvent::OutputsCollected { job_id });
}
}
NodeJobCommand::CheckWorkspaceReady { job_id } => {
self.check_workspace_ready(ctx, job_id);
}
NodeJobCommand::CheckOutputSink { job_id } => {
self.check_output_sink(ctx, job_id);
}
}
}
}

12
crates/job-runner/src/wire.rs
View file

@ -66,6 +66,18 @@ pub enum NodeJobCommand {
job_id: u64,
outputs: Vec<String>,
},
/// Engine-owned timer observation used while an edge workspace transfer is
/// pending. `job_id` rejects stale timer delivery.
#[doc(hidden)]
CheckWorkspaceReady {
job_id: u64,
},
/// Engine-owned timer observation used while an edge output sink is
/// pending. `job_id` rejects stale timer delivery.
#[doc(hidden)]
CheckOutputSink {
job_id: u64,
},
}
impl NetworkMessage for NodeJobCommand {

9
crates/process/Cargo.toml
View file

@ -6,9 +6,18 @@ license = "AGPL-3.0-only"
[dependencies]
swactor = { path = "../..", default-features = false, features = ["no_random"] }
swactor-engine = { path = "../engine" }
tokio.workspace = true
telemetry = { path = "../telemetry" }
crossbeam-queue = "0.3.12"
libc = "0.2"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
serde_yaml = "0.9"
[target.'cfg(target_os = "linux")'.dependencies]
signal-hook = "0.3"
[dev-dependencies]
parking_lot = "0.12"
proptest = "1"

19
crates/process/src/lib.rs
View file

@ -1,6 +1,7 @@
mod actor;
mod lifecycle;
mod message;
mod operations;
mod spawn;
mod supervisor;
mod types;
@ -10,6 +11,24 @@ pub mod yaml;
pub use lifecycle::{ProcessLifecycleObservability, ProcessOutputConfig};
pub use message::{ProcessCommand, ProcessOutput};
#[cfg(unix)]
pub use operations::request_child_termination;
#[cfg(target_os = "linux")]
pub use operations::spawn_os_stop_signal_wait;
#[cfg(unix)]
pub use operations::spawn_unix_stream_listener;
#[cfg(target_os = "linux")]
pub use operations::terminate_process_group;
pub use operations::{
CommandOutputObservation, FollowProcessFile, LineReaderHandle, ProcessExitObservation,
ProcessIdentity, ProcessStopSignal, ProcessStream, ProcessStreamObservation, child_kill,
child_try_wait, child_wait, child_wait_with_output, command_output, command_spawn,
command_status, find_process_identities_by_environment, find_process_identities_with_retry,
spawn_child_wait, spawn_command_output, spawn_detached_command_status,
spawn_identity_exit_wait, spawn_line_channel, spawn_line_reader, spawn_mapped_line_channel,
spawn_mapped_line_reader, spawn_shared_child_wait, spawn_stdin_command_wait,
spawn_stop_channel_wait, wait_for_path, wait_shared_child_or_kill,
};
pub use pipeline::{
JobComplete, JobDefinition, JobFailure, JobId, JobProgress, JobStatus, JobSuccess,
LocalPipelineConfig, LocalStartJob, PipelineId, PipelineStatus,

1425
crates/process/src/operations.rs Normal file
View file

File diff suppressed because it is too large Load diff

1
crates/provisioning/Cargo.toml
View file

@ -13,3 +13,4 @@ swactor-engine = { path = "../engine" }
[dev-dependencies]
parking_lot = "0.12"
serde_json = "1"
swactor-process = { path = "../process" }

22
crates/provisioning/src/bootstrap.rs
View file

@ -193,6 +193,8 @@ pub struct BootstrapActor {
sender: ExternalSender,
collector: Option<Arc<dyn NodeTelemetryCollector>>,
phase: Phase,
/// Launch was requested; duplicate `Start` messages are inert.
started: bool,
/// `Bootstrapped` reported (and collector fired) exactly once.
announced: bool,
/// Terminal report (Failed/Exited) emitted exactly once.
@ -208,6 +210,7 @@ impl BootstrapActor {
sender: config.sender,
collector: config.collector,
phase: Phase::Bootstrapping,
started: false,
announced: false,
closed: false,
}
@ -299,9 +302,10 @@ impl ActorInterface for BootstrapActor {
fn handle(&mut self, ctx: &Ctx, msg: BootstrapMsg) {
match msg {
BootstrapMsg::Start => {
if self.phase != Phase::Bootstrapping {
return; // Restart of a started/stopped attempt: ignore.
if self.phase != Phase::Bootstrapping || self.started {
return;
}
self.started = true;
if let Err(reason) = self.logic.start(ctx, ctx.self_addr(), &self.sender) {
self.fail(format!("launch failed: {reason}"));
}
@ -334,22 +338,12 @@ pub fn spawn_bootstrap_actor(
config: BootstrapConfig,
) -> Result<ActorAddress, String> {
let sender = config.sender.clone();
let start_sender = sender.clone();
let period = config.probe_period;
let actor = ctx
.spawn(BootstrapActor::new(logic, config))
.map_err(|error| format!("spawn bootstrap actor: {error}"))?;
let probe_engine = engine.clone();
engine.spawn(async move {
let mut interval = probe_engine.interval(period);
loop {
(&mut interval).await;
if sender.send_to(actor, BootstrapMsg::Probe).is_err() {
return;
}
}
});
let _ = start_sender.send_to(actor, BootstrapMsg::Start);
engine.send_every(period, sender.clone(), actor, BootstrapMsg::Probe);
let _ = sender.send_to(actor, BootstrapMsg::Start);
Ok(actor)
}

21
crates/provisioning/tests/process_conformance.rs
View file

@ -12,6 +12,7 @@ use std::sync::Arc;
use parking_lot::Mutex;
use provisioning::plugin::{NodeProvisionSpec, PluginNodeHandle, PluginSink, ProvisionPlugin};
use swactor_process::{child_kill, child_wait, command_spawn};
use common::{
AMBIGUOUS_FAULT_MARKER, Fault, PluginBackendAdapter, TestablePlugin, assert_plugin_contracts,
@ -49,8 +50,8 @@ impl Drop for ProcessPlugin {
let mut state = self.state.lock();
let children: Vec<Child> = std::mem::take(&mut state.children).into_values().collect();
for mut child in children {
let _ = child.kill();
let _ = child.wait();
let _ = child_kill(&mut child);
let _ = child_wait(&mut child);
}
}
}
@ -108,10 +109,10 @@ impl ProvisionPlugin for ProcessPlugin {
if matches!(fault, Some(Fault::Definite)) {
return Err("scripted definite failure".to_owned());
}
let child = Command::new("sleep")
.arg("infinity")
.spawn()
.map_err(|error| format!("spawn failed: {error}"))?;
let mut command = Command::new("sleep");
command.arg("infinity");
let child =
command_spawn(&mut command).map_err(|error| format!("spawn failed: {error}"))?;
let pid = child.id();
state.children.insert(attempt, child);
state.created += 1;
@ -141,12 +142,8 @@ impl ProvisionPlugin for ProcessPlugin {
fn stop_node(&mut self, handle: &PluginNodeHandle) -> Result<(), String> {
let mut state = self.state.lock();
if let Some(mut child) = state.children.remove(&handle.id) {
child
.kill()
.map_err(|error| format!("kill failed: {error}"))?;
child
.wait()
.map_err(|error| format!("reap failed: {error}"))?;
child_kill(&mut child).map_err(|error| format!("kill failed: {error}"))?;
child_wait(&mut child).map_err(|error| format!("reap failed: {error}"))?;
}
Ok(())
}

1
rust-toolchain.toml
View file

@ -1,2 +1,3 @@
[toolchain]
channel = "nightly-2026-02-07"
components = ["rustc-dev"]

20
src/worker.rs
View file

@ -322,13 +322,29 @@ impl Worker {
);
}
// 9. Clean up poisoned and stopping actors
did_work |= Self::cleanup_dead_actors(
// 9. Clean up poisoned and stopping actors. Cleanup changes the pool after
// phase 8 published its snapshot, so republish cardinality and mailbox
// state when actors were removed rather than leaving observability stale
// until unrelated work reaches this worker.
let cleaned_dead = Self::cleanup_dead_actors(
&mut self.pool,
&mut self.worker_ext,
&mut self.deferred_transfers,
&tc,
);
if cleaned_dead {
did_work = true;
self.stats
.num_actors
.store(self.pool.len(), Ordering::Relaxed);
self.stats
.total_mailbox_depth
.store(self.pool.total_mailbox_depth(), Ordering::Relaxed);
if let Some(hook) = tc.stats_hook {
self.pool.mailbox_depths_into(&mut self.snapshot_buf);
hook.on_tick(wid.index(), &self.snapshot_buf);
}
}
self.has_backlog = did_work;
did_work

9
tools/actor-control-flow-lint/Cargo.toml Normal file
View file

@ -0,0 +1,9 @@
[package]
name = "actor-control-flow-lint-tests"
version = "0.1.0"
edition = "2024"
license = "AGPL-3.0-only"
publish = false
[lib]
path = "src/lib.rs"

353
tools/actor-control-flow-lint/driver.rs Executable file
View file

@ -0,0 +1,353 @@
#![feature(rustc_private)]
extern crate rustc_driver;
extern crate rustc_hir;
extern crate rustc_interface;
extern crate rustc_middle;
extern crate rustc_span;
use std::env;
use std::process::ExitCode;
use rustc_driver::{Callbacks, Compilation};
use rustc_hir as hir;
use rustc_hir::def::Res;
use rustc_hir::intravisit::{self, Visitor};
use rustc_interface::interface::Compiler;
use rustc_middle::ty::{TyCtxt, TypeckResults};
use rustc_span::{Span, symbol::Symbol};
/// Crates whose purpose is to run the actor engine or turn external I/O,
/// process, telemetry, and provider API streams into actor observations.
/// Adding an entry changes the architecture.
const EXECUTION_OWNERS: &[&str] = &[
"dashboard",
"iroh-driver",
"swactor",
"swactor-engine",
"swactor-process",
"swactor-transport",
"swactor-vastai",
"telemetry",
];
/// This package owns only the compile-contract subprocess harness. It cannot be
/// used as a workspace dependency.
const TEST_SUPPORT_OWNERS: &[&str] = &["actor-control-flow-lint-tests"];
/// Policy-bearing crates that must never enter an execution owner's dependency
/// closure.
const DOMAIN_CONTROL_CRATES: &[&str] = &[
"myelin",
"provisioning",
"swactor-job-runner",
"xtask",
];
#[derive(Clone, Copy)]
struct Capability {
label: &'static str,
resolution: &'static str,
test_wait: bool,
paths: &'static [&'static str],
}
const MOVE_TO_OWNER: &str =
"move stream mechanics into an approved execution owner or move the decision into an actor";
const USE_ACTOR_TIMER: &str =
"schedule a typed actor message through the engine; the receiving actor owns the deadline decision";
/// Stable resolved item paths. These are deliberately compiler identities, not
/// spellings found in source, so re-exports, renamed imports, and local wrappers
/// cannot evade the boundary.
const CAPABILITIES: &[Capability] = &[
Capability {
label: "asynchronous task spawning",
resolution: MOVE_TO_OWNER,
test_wait: false,
paths: &[
"tokio::runtime::Handle::spawn",
"tokio::runtime::Runtime::spawn",
"tokio::spawn",
"tokio::task::spawn",
"tokio::task::spawn_local",
],
},
Capability {
label: "blocking task spawning",
resolution: MOVE_TO_OWNER,
test_wait: false,
paths: &[
"tokio::runtime::Handle::spawn_blocking",
"tokio::runtime::Runtime::spawn_blocking",
"tokio::task::spawn_blocking",
],
},
Capability {
label: "engine task scheduling",
resolution: MOVE_TO_OWNER,
test_wait: false,
paths: &["swactor_engine::EngineHandle::spawn"],
},
Capability {
label: "OS thread creation",
resolution: MOVE_TO_OWNER,
test_wait: false,
paths: &[
"std::thread::Builder::spawn",
"std::thread::Builder::spawn_unchecked",
"std::thread::spawn",
],
},
Capability {
label: "thread sleeping",
resolution: USE_ACTOR_TIMER,
test_wait: true,
paths: &[
"std::thread::park",
"std::thread::park_timeout",
"std::thread::sleep",
],
},
Capability {
label: "direct timer driving",
resolution: USE_ACTOR_TIMER,
test_wait: false,
paths: &[
"swactor_engine::EngineHandle::interval",
"swactor_engine::EngineHandle::timer",
"swactor_engine::EngineHandle::timeout",
"tokio::time::interval",
"tokio::time::interval_at",
"tokio::time::sleep",
"tokio::time::sleep_until",
"tokio::time::timeout",
"tokio::time::timeout_at",
],
},
Capability {
label: "runtime construction or driving",
resolution: "the actor engine owns runtime construction and progression",
test_wait: false,
paths: &[
"swactor::runtime::SingleThreadRuntime::tick",
"swactor::runtime::SingleThreadRuntime::try_tick",
"swactor::runtime::SingleThreadRuntime::has_work",
"tokio::runtime::Builder::new_current_thread",
"tokio::runtime::Builder::new_multi_thread",
"tokio::runtime::Handle::block_on",
"tokio::runtime::Runtime::block_on",
"tokio::runtime::Runtime::new",
],
},
Capability {
label: "blocking receive used as a controller",
resolution: "receive observations in an actor; tests may use a bounded observation wait",
test_wait: true,
paths: &[
"crossbeam_channel::channel::Receiver::recv",
"crossbeam_channel::channel::Receiver::recv_deadline",
"crossbeam_channel::channel::Receiver::recv_timeout",
"std::sync::mpsc::Receiver::recv",
"std::sync::mpsc::Receiver::recv_deadline",
"std::sync::mpsc::Receiver::recv_timeout",
"tokio::sync::mpsc::bounded::Receiver::blocking_recv",
"tokio::sync::oneshot::Receiver::blocking_recv",
],
},
Capability {
label: "process creation",
resolution: "send a command to the process I/O owner and return exit/output observations to an actor",
test_wait: false,
paths: &[
"std::process::Child::kill",
"std::process::Child::try_wait",
"std::process::Child::wait",
"std::process::Child::wait_with_output",
"std::process::Command::output",
"std::process::Command::spawn",
"std::process::Command::status",
"tokio::process::Child::kill",
"tokio::process::Child::start_kill",
"tokio::process::Child::try_wait",
"tokio::process::Child::wait",
"tokio::process::Child::wait_with_output",
"tokio::process::Command::output",
"tokio::process::Command::spawn",
"tokio::process::Command::status",
],
},
];
struct ActorControlFlowCallbacks {
package: String,
test_build: bool,
trace: bool,
}
impl Callbacks for ActorControlFlowCallbacks {
fn after_analysis<'tcx>(
&mut self,
_compiler: &Compiler,
tcx: TyCtxt<'tcx>,
) -> Compilation {
if TEST_SUPPORT_OWNERS.contains(&self.package.as_str()) {
return Compilation::Continue;
}
if EXECUTION_OWNERS.contains(&self.package.as_str()) {
check_owner_dependencies(tcx, &self.package);
return Compilation::Continue;
}
for owner in tcx.hir_body_owners() {
let typeck = tcx.typeck(owner);
let body = tcx.hir_body_owned_by(owner);
let mut visitor = CapabilityVisitor {
tcx,
typeck,
package: &self.package,
test_build: self.test_build,
trace: self.trace,
};
visitor.visit_body(body);
}
Compilation::Continue
}
}
fn check_owner_dependencies(tcx: TyCtxt<'_>, package: &str) {
for &crate_num in tcx.crates(()) {
let dependency_symbol = tcx.crate_name(crate_num);
let dependency = dependency_symbol.as_str();
if DOMAIN_CONTROL_CRATES.contains(&dependency) {
tcx.dcx().err(format!(
"actor control-flow policy: execution owner `{package}` depends on domain-control crate `{dependency}`; execution owners must remain below domain policy in the dependency graph"
));
}
}
}
struct CapabilityVisitor<'a, 'tcx> {
tcx: TyCtxt<'tcx>,
typeck: &'tcx TypeckResults<'tcx>,
package: &'a str,
test_build: bool,
trace: bool,
}
impl<'tcx> Visitor<'tcx> for CapabilityVisitor<'_, 'tcx> {
fn visit_expr(&mut self, expr: &'tcx hir::Expr<'tcx>) {
match &expr.kind {
hir::ExprKind::MethodCall(..) => {
if let Some(def_id) = self.typeck.type_dependent_def_id(expr.hir_id) {
self.check(def_id, expr.span);
}
}
hir::ExprKind::Path(qpath) => {
if let Res::Def(_, def_id) = self.typeck.qpath_res(qpath, expr.hir_id) {
self.check(def_id, expr.span);
}
}
_ => {}
}
intravisit::walk_expr(self, expr);
}
}
impl CapabilityVisitor<'_, '_> {
fn check(&self, def_id: rustc_hir::def_id::DefId, span: Span) {
let path = self.tcx.def_path_str(def_id);
let normalized_path = normalize_def_path(&path);
if self.trace && is_candidate_name(self.tcx.item_name(def_id)) {
eprintln!("actor-control-flow trace: {path}");
}
let Some(capability) = CAPABILITIES
.iter()
.find(|capability| capability.paths.contains(&normalized_path.as_str()))
else {
return;
};
if self.test_build && capability.test_wait {
return;
}
self.tcx.dcx().span_err(
span,
format!(
"actor control-flow violation: `{}` is forbidden in workspace crate `{}`; {}",
capability.label, self.package, capability.resolution
),
);
}
}
fn normalize_def_path(path: &str) -> String {
let mut normalized = String::with_capacity(path.len());
let mut cursor = 0;
while let Some(relative_start) = path[cursor..].find("::<") {
let start = cursor + relative_start;
normalized.push_str(&path[cursor..start]);
let generic_start = start + 3;
let mut depth = 1_usize;
let mut end = path.len();
for (offset, character) in path[generic_start..].char_indices() {
match character {
'<' => depth += 1,
'>' => {
depth -= 1;
if depth == 0 {
end = generic_start + offset + character.len_utf8();
break;
}
}
_ => {}
}
}
cursor = end;
}
normalized.push_str(&path[cursor..]);
normalized
}
fn is_candidate_name(name: Symbol) -> bool {
matches!(
name.as_str(),
"block_on"
| "blocking_recv"
| "has_work"
| "interval"
| "interval_at"
| "new_current_thread"
| "new_multi_thread"
| "recv"
| "recv_deadline"
| "recv_timeout"
| "sleep"
| "sleep_until"
| "spawn"
| "spawn_blocking"
| "spawn_local"
| "tick"
| "timeout"
| "timeout_at"
| "timer"
| "try_tick"
)
}
fn main() -> ExitCode {
let args: Vec<String> = env::args().collect();
let package = env::var("MYELIN_ACTOR_LINT_PACKAGE").unwrap_or_else(|_| "unknown".to_owned());
let test_build = env::var_os("MYELIN_ACTOR_LINT_TEST_BUILD").is_some();
let trace = env::var_os("MYELIN_ACTOR_LINT_TRACE").is_some();
let mut callbacks = ActorControlFlowCallbacks {
package,
test_build,
trace,
};
rustc_driver::catch_with_exit_code(|| rustc_driver::run_compiler(&args, &mut callbacks))
}

101
tools/actor-control-flow-lint/rustc-wrapper.py Executable file
View file

@ -0,0 +1,101 @@
#!/usr/bin/env python3
"""Build, cache, and execute the repository-owned rustc policy driver."""
from __future__ import annotations
import fcntl
import hashlib
import os
from pathlib import Path
import subprocess
import sys
def fail(message: str) -> "None":
print(f"actor-control-flow wrapper: {message}", file=sys.stderr)
raise SystemExit(1)
def main() -> None:
if len(sys.argv) < 2:
fail("Cargo did not supply the real rustc path")
real_rustc = sys.argv[1]
rustc_args = sys.argv[2:]
root = Path(__file__).resolve().parents[2]
source = Path(__file__).with_name("driver.rs")
target_root = Path(os.environ.get("CARGO_TARGET_DIR", root / "target"))
if not target_root.is_absolute():
target_root = root / target_root
cache = target_root / "actor-control-flow-lint"
cache.mkdir(parents=True, exist_ok=True)
bootstrap_env = os.environ.copy()
bootstrap_env.pop("CARGO_MAKEFLAGS", None)
bootstrap_env.pop("MAKEFLAGS", None)
try:
version = subprocess.check_output(
[real_rustc, "--version", "--verbose"],
text=True,
stderr=subprocess.STDOUT,
env=bootstrap_env,
)
sysroot = subprocess.check_output(
[real_rustc, "--print", "sysroot"],
text=True,
stderr=subprocess.STDOUT,
env=bootstrap_env,
).strip()
except (OSError, subprocess.CalledProcessError) as error:
fail(f"cannot inspect pinned rustc: {error}")
digest = hashlib.sha256(source.read_bytes() + version.encode()).hexdigest()[:20]
driver = cache / f"driver-{digest}"
lock_path = cache / "build.lock"
with lock_path.open("a+b") as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
if not driver.exists():
temporary = cache / f".{driver.name}.{os.getpid()}.tmp"
command = [
real_rustc,
str(source),
"--crate-name",
"myelin_actor_control_flow_lint",
"--edition=2024",
"-Cprefer-dynamic",
"-L",
str(Path(sysroot) / "lib"),
"-o",
str(temporary),
]
result = subprocess.run(command, env=bootstrap_env)
if result.returncode != 0:
temporary.unlink(missing_ok=True)
fail(
"failed to build compiler driver; the pinned toolchain must include "
"the rustc-dev component"
)
os.replace(temporary, driver)
package = os.environ.get("CARGO_PKG_NAME", "unknown")
test_build = "--test" in rustc_args
child_env = os.environ.copy()
child_env["MYELIN_ACTOR_LINT_PACKAGE"] = package
if test_build:
child_env["MYELIN_ACTOR_LINT_TEST_BUILD"] = "1"
else:
child_env.pop("MYELIN_ACTOR_LINT_TEST_BUILD", None)
rustc_lib = str(Path(sysroot) / "lib")
current_library_path = child_env.get("LD_LIBRARY_PATH")
child_env["LD_LIBRARY_PATH"] = (
f"{rustc_lib}:{current_library_path}" if current_library_path else rustc_lib
)
os.execvpe(str(driver), [real_rustc, *rustc_args], child_env)
if __name__ == "__main__":
main()

1
tools/actor-control-flow-lint/src/lib.rs Normal file
View file

@ -0,0 +1 @@
//! Compile-contract harness for the repository-owned actor control-flow driver.

92
tools/actor-control-flow-lint/tests/contracts.rs Normal file
View file

@ -0,0 +1,92 @@
use std::path::{Path, PathBuf};
use std::process::{Command, Output};
fn repository_root() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.and_then(Path::parent)
.expect("lint package lives under tools/")
.to_path_buf()
}
fn cargo_check(fixture: &str, extra_args: &[&str]) -> Output {
let root = repository_root();
let fixture_dir = Path::new(env!("CARGO_MANIFEST_DIR"))
.join("tests/fixtures")
.join(fixture);
let target_dir = root
.join("target/actor-control-flow-contracts")
.join(fixture);
let wrapper = root.join("tools/actor-control-flow-lint/rustc-wrapper.py");
let mut command = Command::new(env!("CARGO"));
command
.arg("check")
.arg("--quiet")
.args(extra_args)
.current_dir(fixture_dir)
.env("CARGO_TARGET_DIR", target_dir)
.env("CARGO_TERM_COLOR", "never")
.env("RUSTC_WORKSPACE_WRAPPER", wrapper)
.env_remove("CARGO_MAKEFLAGS")
.env_remove("MAKEFLAGS");
command.output().expect("run fixture cargo check")
}
fn stderr(output: &Output) -> String {
String::from_utf8_lossy(&output.stderr).into_owned()
}
#[test]
fn compiler_policy_contracts() {
let direct = cargo_check("fail-domain-capabilities", &[]);
assert!(
!direct.status.success(),
"forbidden domain fixture compiled"
);
let direct_stderr = stderr(&direct);
for expected in [
"asynchronous task spawning",
"blocking task spawning",
"engine task scheduling",
"OS thread creation",
"thread sleeping",
"direct timer driving",
"runtime construction or driving",
"blocking receive used as a controller",
"process creation",
] {
assert!(
direct_stderr.contains(expected),
"missing `{expected}` diagnostic:\n{direct_stderr}"
);
}
let dependency = cargo_check("fail-owner-dependency", &[]);
assert!(
!dependency.status.success(),
"execution owner depending on domain control compiled"
);
let dependency_stderr = stderr(&dependency);
assert!(
dependency_stderr
.contains("execution owner `swactor-engine` depends on domain-control crate `myelin`"),
"missing owner dependency diagnostic:\n{dependency_stderr}"
);
for fixture in ["pass-actor-domain", "pass-execution-owner"] {
let output = cargo_check(fixture, &[]);
assert!(
output.status.success(),
"compile-pass fixture `{fixture}` failed:\n{}",
stderr(&output)
);
}
let test_wait = cargo_check("pass-test-wait", &["--tests"]);
assert!(
test_wait.status.success(),
"narrow test wait fixture failed:\n{}",
stderr(&test_wait)
);
}

263
tools/actor-control-flow-lint/tests/fixtures/fail-domain-capabilities/Cargo.lock generated vendored Normal file
View file

@ -0,0 +1,263 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "actor-lint-domain-fail"
version = "0.0.0"
dependencies = [
"swactor-engine",
"tokio",
]
[[package]]
name = "bitflags"
version = "2.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
[[package]]
name = "bytes"
version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
[[package]]
name = "cfg-if"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
[[package]]
name = "crossbeam-queue"
version = "0.3.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
[[package]]
name = "errno"
version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys",
]
[[package]]
name = "getrandom"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
dependencies = [
"cfg-if",
"libc",
"wasi",
]
[[package]]
name = "libc"
version = "0.2.189"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
[[package]]
name = "lock_api"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
dependencies = [
"scopeguard",
]
[[package]]
name = "mio"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427"
dependencies = [
"libc",
"wasi",
"windows-sys",
]
[[package]]
name = "parking_lot"
version = "0.12.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
dependencies = [
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.9.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"smallvec",
"windows-link",
]
[[package]]
name = "pin-project-lite"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
[[package]]
name = "proc-macro2"
version = "1.0.107"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
dependencies = [
"proc-macro2",
]
[[package]]
name = "redox_syscall"
version = "0.5.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
dependencies = [
"bitflags",
]
[[package]]
name = "scopeguard"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "signal-hook-registry"
version = "1.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b"
dependencies = [
"errno",
"libc",
]
[[package]]
name = "smallvec"
version = "1.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
[[package]]
name = "socket2"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"
dependencies = [
"libc",
"windows-sys",
]
[[package]]
name = "swactor"
version = "0.1.0"
dependencies = [
"crossbeam-queue",
"crossbeam-utils",
"getrandom",
"parking_lot",
]
[[package]]
name = "swactor-engine"
version = "0.1.0"
dependencies = [
"parking_lot",
"swactor",
"tokio",
]
[[package]]
name = "syn"
version = "3.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "tokio"
version = "1.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"
dependencies = [
"bytes",
"libc",
"mio",
"pin-project-lite",
"signal-hook-registry",
"socket2",
"tokio-macros",
"windows-sys",
]
[[package]]
name = "tokio-macros"
version = "2.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "unicode-ident"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
[[package]]
name = "wasi"
version = "0.11.1+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
[[package]]
name = "windows-link"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
[[package]]
name = "windows-sys"
version = "0.61.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
dependencies = [
"windows-link",
]

11
tools/actor-control-flow-lint/tests/fixtures/fail-domain-capabilities/Cargo.toml vendored Normal file
View file

@ -0,0 +1,11 @@
[package]
name = "actor-lint-domain-fail"
version = "0.0.0"
edition = "2024"
publish = false
[dependencies]
swactor-engine = { path = "../../../../../crates/engine" }
tokio = { version = "1", features = ["rt", "time"] }
[workspace]

35
tools/actor-control-flow-lint/tests/fixtures/fail-domain-capabilities/src/lib.rs vendored Executable file
View file

@ -0,0 +1,35 @@
#![allow(clippy::disallowed_methods, dead_code, unused_must_use)]
use std::thread::sleep as renamed_sleep;
use std::time::Duration;
use swactor_engine::EngineHandle;
fn local_sleep_wrapper() {
renamed_sleep(Duration::from_millis(1));
}
fn forbidden_engine_controls(handle: &EngineHandle) {
handle.spawn(async {});
handle.timer(Duration::from_millis(1));
}
fn forbidden_thread_and_receive() {
std::thread::spawn(|| {});
local_sleep_wrapper();
let (_sender, receiver) = std::sync::mpsc::channel::<()>();
let _ = receiver.recv();
let _ = std::process::Command::new("true").output();
}
fn forbidden_runtime_driver() {
let runtime = tokio::runtime::Builder::new_current_thread()
.build()
.expect("runtime");
runtime.block_on(async {});
}
async fn forbidden_tokio_controls() {
tokio::spawn(async {});
tokio::task::spawn_blocking(|| {});
tokio::time::sleep(Duration::from_millis(1)).await;
}

14
tools/actor-control-flow-lint/tests/fixtures/fail-owner-dependency/Cargo.lock generated vendored Normal file
View file

@ -0,0 +1,14 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "myelin"
version = "0.0.0"
[[package]]
name = "swactor-engine"
version = "0.0.0"
dependencies = [
"myelin",
]

10
tools/actor-control-flow-lint/tests/fixtures/fail-owner-dependency/Cargo.toml vendored Normal file
View file

@ -0,0 +1,10 @@
[package]
name = "swactor-engine"
version = "0.0.0"
edition = "2024"
publish = false
[dependencies]
myelin = { path = "myelin" }
[workspace]

8
tools/actor-control-flow-lint/tests/fixtures/fail-owner-dependency/myelin/Cargo.toml vendored Normal file
View file

@ -0,0 +1,8 @@
[package]
name = "myelin"
version = "0.0.0"
edition = "2024"
publish = false
[lib]
path = "src/lib.rs"

1
tools/actor-control-flow-lint/tests/fixtures/fail-owner-dependency/myelin/src/lib.rs vendored Normal file
View file

@ -0,0 +1 @@
pub fn domain_policy() {}

3
tools/actor-control-flow-lint/tests/fixtures/fail-owner-dependency/src/lib.rs vendored Normal file
View file

@ -0,0 +1,3 @@
pub fn forbidden_dependency() {
myelin::domain_policy();
}

263
tools/actor-control-flow-lint/tests/fixtures/pass-actor-domain/Cargo.lock generated vendored Normal file
View file

@ -0,0 +1,263 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "actor-lint-domain-pass"
version = "0.0.0"
dependencies = [
"swactor",
"swactor-engine",
]
[[package]]
name = "bitflags"
version = "2.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
[[package]]
name = "bytes"
version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
[[package]]
name = "cfg-if"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
[[package]]
name = "crossbeam-queue"
version = "0.3.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
[[package]]
name = "errno"
version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys",
]
[[package]]
name = "getrandom"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
dependencies = [
"cfg-if",
"libc",
"wasi",
]
[[package]]
name = "libc"
version = "0.2.189"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
[[package]]
name = "lock_api"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
dependencies = [
"scopeguard",
]
[[package]]
name = "mio"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427"
dependencies = [
"libc",
"wasi",
"windows-sys",
]
[[package]]
name = "parking_lot"
version = "0.12.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
dependencies = [
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.9.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"smallvec",
"windows-link",
]
[[package]]
name = "pin-project-lite"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
[[package]]
name = "proc-macro2"
version = "1.0.107"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
dependencies = [
"proc-macro2",
]
[[package]]
name = "redox_syscall"
version = "0.5.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
dependencies = [
"bitflags",
]
[[package]]
name = "scopeguard"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "signal-hook-registry"
version = "1.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b"
dependencies = [
"errno",
"libc",
]
[[package]]
name = "smallvec"
version = "1.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
[[package]]
name = "socket2"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"
dependencies = [
"libc",
"windows-sys",
]
[[package]]
name = "swactor"
version = "0.1.0"
dependencies = [
"crossbeam-queue",
"crossbeam-utils",
"getrandom",
"parking_lot",
]
[[package]]
name = "swactor-engine"
version = "0.1.0"
dependencies = [
"parking_lot",
"swactor",
"tokio",
]
[[package]]
name = "syn"
version = "3.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "tokio"
version = "1.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"
dependencies = [
"bytes",
"libc",
"mio",
"pin-project-lite",
"signal-hook-registry",
"socket2",
"tokio-macros",
"windows-sys",
]
[[package]]
name = "tokio-macros"
version = "2.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "unicode-ident"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
[[package]]
name = "wasi"
version = "0.11.1+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
[[package]]
name = "windows-link"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
[[package]]
name = "windows-sys"
version = "0.61.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
dependencies = [
"windows-link",
]

11
tools/actor-control-flow-lint/tests/fixtures/pass-actor-domain/Cargo.toml vendored Normal file
View file

@ -0,0 +1,11 @@
[package]
name = "actor-lint-domain-pass"
version = "0.0.0"
edition = "2024"
publish = false
[dependencies]
swactor = { path = "../../../../.." }
swactor-engine = { path = "../../../../../crates/engine" }
[workspace]

50
tools/actor-control-flow-lint/tests/fixtures/pass-actor-domain/src/lib.rs vendored Normal file
View file

@ -0,0 +1,50 @@
use std::time::Duration;
use swactor::actor::{ActorAddress, ActorInterface, Ctx};
use swactor::runtime::ExternalSender;
use swactor_engine::EngineHandle;
#[derive(Clone)]
pub struct Tick {
pub generation: u64,
}
struct Child;
impl ActorInterface for Child {
type Incoming = Tick;
type Response = ();
fn handle(&mut self, _ctx: &Ctx, _message: Tick) {}
}
pub struct Parent;
impl Parent {
fn spawn_helper(&self, ctx: &Ctx) {
let _ = ctx.spawn(Child);
}
}
impl ActorInterface for Parent {
type Incoming = Tick;
type Response = ();
fn handle(&mut self, ctx: &Ctx, _message: Tick) {
self.spawn_helper(ctx);
}
}
pub fn schedule_actor_tick(
engine: &EngineHandle,
sender: ExternalSender,
actor: ActorAddress,
generation: u64,
) {
engine.send_after(
Duration::from_millis(10),
sender,
actor,
Tick { generation },
);
}

77
tools/actor-control-flow-lint/tests/fixtures/pass-execution-owner/Cargo.lock generated vendored Normal file
View file

@ -0,0 +1,77 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "iroh-driver"
version = "0.0.0"
dependencies = [
"tokio",
]
[[package]]
name = "libc"
version = "0.2.189"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
[[package]]
name = "mio"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427"
dependencies = [
"libc",
"wasi",
"windows-sys",
]
[[package]]
name = "pin-project-lite"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
[[package]]
name = "socket2"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"
dependencies = [
"libc",
"windows-sys",
]
[[package]]
name = "tokio"
version = "1.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"
dependencies = [
"libc",
"mio",
"pin-project-lite",
"socket2",
"windows-sys",
]
[[package]]
name = "wasi"
version = "0.11.1+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
[[package]]
name = "windows-link"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
[[package]]
name = "windows-sys"
version = "0.61.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
dependencies = [
"windows-link",
]

10
tools/actor-control-flow-lint/tests/fixtures/pass-execution-owner/Cargo.toml vendored Normal file
View file

@ -0,0 +1,10 @@
[package]
name = "iroh-driver"
version = "0.0.0"
edition = "2024"
publish = false
[dependencies]
tokio = { version = "1", features = ["net", "rt", "time"] }
[workspace]

8
tools/actor-control-flow-lint/tests/fixtures/pass-execution-owner/src/lib.rs vendored Normal file
View file

@ -0,0 +1,8 @@
use std::time::Duration;
pub fn start_transport_pump() {
tokio::spawn(async {
let _io_type: Option<tokio::net::TcpStream> = None;
tokio::time::sleep(Duration::from_millis(1)).await;
});
}

7
tools/actor-control-flow-lint/tests/fixtures/pass-test-wait/Cargo.lock generated vendored Normal file
View file

@ -0,0 +1,7 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "actor-lint-test-wait-pass"
version = "0.0.0"

7
tools/actor-control-flow-lint/tests/fixtures/pass-test-wait/Cargo.toml vendored Normal file
View file

@ -0,0 +1,7 @@
[package]
name = "actor-lint-test-wait-pass"
version = "0.0.0"
edition = "2024"
publish = false
[workspace]

1
tools/actor-control-flow-lint/tests/fixtures/pass-test-wait/src/lib.rs vendored Normal file
View file

@ -0,0 +1 @@
pub fn marker() {}

12
tools/actor-control-flow-lint/tests/fixtures/pass-test-wait/tests/wait.rs vendored Normal file
View file

@ -0,0 +1,12 @@
use std::time::Duration;
#[test]
fn bounded_observation_wait_is_allowed() {
let (sender, receiver) = std::sync::mpsc::channel();
sender.send(7_u8).expect("send observation");
assert_eq!(
receiver.recv_timeout(Duration::from_millis(10)),
Ok(7)
);
std::thread::sleep(Duration::from_millis(1));
}

4
tools/vastai/Cargo.toml
View file

@ -5,6 +5,10 @@ edition = "2024"
license = "AGPL-3.0-only"
publish = false
[features]
default = []
test-support = []
[dependencies]
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
serde = { version = "1", features = ["derive"] }

91
tools/vastai/src/blocking.rs Normal file
View file

@ -0,0 +1,91 @@
//! Synchronous facade for concrete Vast.ai API operations.
//!
//! The provider-I/O crate owns the Tokio runtime used to drive HTTP requests;
//! domain actors receive only operation results and never drive a runtime.
use crate::{
CreateInstanceRequest, InstanceInfo, LabeledInstance, LifecyclePolicy, Offer,
OfferBrowseCriteria, ProviderInstanceStatus, RunningInstance, SelectionPolicy, VastClient,
};
use std::time::Duration;
pub struct BlockingVastClient {
client: VastClient,
runtime: tokio::runtime::Runtime,
}
impl BlockingVastClient {
pub fn new(client: VastClient) -> Result<Self, String> {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.map_err(|error| format!("build Vast.ai I/O runtime: {error}"))?;
Ok(Self { client, runtime })
}
pub fn browse_offers(&self, criteria: &OfferBrowseCriteria) -> Result<Vec<Offer>, String> {
self.runtime.block_on(self.client.browse_offers(criteria))
}
pub fn search_offers(
&self,
policy: &SelectionPolicy,
target_count: u32,
) -> Result<Vec<Offer>, String> {
self.runtime
.block_on(self.client.search_offers(policy, target_count))
}
pub fn create_instance(&self, request: &CreateInstanceRequest) -> Result<InstanceInfo, String> {
self.runtime.block_on(self.client.create_instance(request))
}
pub fn instance_status(&self, contract_id: u64) -> Result<ProviderInstanceStatus, String> {
self.runtime
.block_on(self.client.instance_status(contract_id))
}
pub fn list_by_label(&self, label: &str) -> Result<Vec<LabeledInstance>, String> {
self.runtime.block_on(self.client.list_by_label(label))
}
pub fn list_by_label_with_retry(
&self,
label: &str,
attempts: usize,
pace: Duration,
) -> Result<Vec<LabeledInstance>, String> {
let attempts = attempts.max(1);
for attempt in 0..attempts {
let instances = self.list_by_label(label)?;
if !instances.is_empty() || attempt + 1 == attempts {
return Ok(instances);
}
std::thread::sleep(pace);
}
unreachable!("at least one Vast.ai label lookup attempt runs")
}
pub fn wait_for_ssh_endpoint(
&self,
contract_id: u64,
label: &str,
policy: &LifecyclePolicy,
) -> Result<RunningInstance, String> {
self.runtime.block_on(
self.client
.wait_for_ssh_endpoint(contract_id, label, policy),
)
}
pub fn destroy_instance_with_retry(&self, contract_id: u64) -> Result<(), String> {
self.runtime
.block_on(self.client.destroy_instance_with_retry(contract_id))
}
}
impl Clone for BlockingVastClient {
fn clone(&self) -> Self {
Self::new(self.client.clone()).expect("clone Vast.ai I/O runtime")
}
}

13
tools/vastai/src/client.rs
View file

@ -1,5 +1,8 @@
use std::time::Duration;
pub const VASTAI_BASE_URL_ENV: &str = "VASTAI_BASE_URL";
const DEFAULT_VASTAI_BASE_URL: &str = "https://cloud.vast.ai";
use crate::search::OfferBrowseCriteria;
use crate::types::{
@ -18,8 +21,16 @@ pub struct VastClient {
impl VastClient {
pub const REQUEST_TIMEOUT: Duration = Duration::from_secs(45);
/// Build a client for the configured Vast.ai API endpoint.
///
/// `VASTAI_BASE_URL` supports operator proxies and production-shaped local
/// fixtures. Empty values retain the public Vast.ai endpoint.
pub fn new(api_key: impl Into<String>) -> Self {
Self::with_base_url("https://cloud.vast.ai", api_key)
let base_url = std::env::var(VASTAI_BASE_URL_ENV)
.ok()
.filter(|url| !url.trim().is_empty())
.unwrap_or_else(|| DEFAULT_VASTAI_BASE_URL.to_owned());
Self::with_base_url(base_url, api_key)
}
pub fn with_base_url(base_url: impl Into<String>, api_key: impl Into<String>) -> Self {

4
tools/vastai/src/lib.rs
View file

@ -4,6 +4,7 @@
//! general-purpose actor runtime, while this utility rents, monitors, and tears
//! down vast.ai machines for apps that choose to use it.
mod blocking;
pub mod client;
pub mod config;
pub mod filters;
@ -15,8 +16,11 @@ pub mod provision;
pub mod search;
pub mod state;
pub mod teardown;
#[cfg(feature = "test-support")]
pub mod test_http;
pub mod types;
pub use blocking::BlockingVastClient;
pub use client::VastClient;
pub use lease::{confirm_lease, provision_fleet};
pub use logs::{fetch_logs, request_logs};

Some files were not shown because too many files have changed in this diff Show more