zacheryasc/swactor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Compare commits

base: zacheryasc:0e2fc2b164c3046a039331e3ce06ba4c95a86416
Branches Tags
zacheryasc:master
zacheryasc:development
zacheryasc:backup/pre-rewrite-msg
...
compare: zacheryasc:52bbc661a2f78fe690dce5c7bbb4dc47415968fb
Branches Tags
zacheryasc:development
zacheryasc:master
zacheryasc:backup/pre-rewrite-msg

3 commits
0e2fc2b164 ... 52bbc661a2

Author SHA1 Message Date
Zachery Aaron Shores-Chmielewski
52bbc661a2 chore(test): enforce complete local test barrier 
Deny workspace warnings and lint suppressions, consolidate Rust and Python coverage under cargo xtask test with 60-second per-test limits, and remove stale flaky, stateful, Docker, and orphaned test artifacts.
 2026-08-22 21:18:25 +04:00
Zachery Aaron Shores-Chmielewski
71471de0a6 fix(dashboard): improve control and hardware views 
Refine fleet and provisioning interactions, expand control-plane projections, and add CPU, memory, pressure, and storage telemetry for the local dashboard surfaces.
 2026-08-22 21:17:42 +04:00
Zachery Aaron Shores-Chmielewski
cd870d3141 feat(data-plane): complete virtual blob namespace 
Add durable namespace control, Iroh-backed source transfer, session-independent publication lifetime, and wire managed Myelin jobs and bindings through the reusable data-plane API.
 2026-08-22 21:16:56 +04:00
168 changed files with 10083 additions and 5983 deletions
Show stats Expand all files Collapse all files

2
.cargo/config.toml
View file

@ -3,4 +3,4 @@ rustc-workspace-wrapper = "tools/actor-control-flow-lint/rustc-wrapper.py"
[alias]
xtask = "run --package xtask --"
myelin-chat = "run --package xtask -- myelin-chat"
lint = "clippy --workspace --all-targets --all-features"

2
.config/nextest.toml Normal file
View file

@ -0,0 +1,2 @@
[profile.default]
slow-timeout = { period = "60s", terminate-after = 1 }

119
.github/workflows/myelin-properties.yml vendored
View file

@ -1,119 +0,0 @@
name: Myelin properties
on:
pull_request:
schedule:
- cron: "17 3 * * *"
workflow_dispatch:
permissions:
contents: read
jobs:
component-properties:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install pinned Rust toolchain
run: rustup toolchain install nightly-2026-02-07 --profile minimal --component rustfmt --component rustc-dev --component rust-src --component llvm-tools-preview
- name: Check formatting
run: cargo fmt --all -- --check
- name: Check default workspace members
run: cargo check
- name: Check compiler-policy contracts
run: cargo test -p actor-control-flow-lint-tests --test contracts compiler_policy_contracts -- --exact
- name: Engine scheduling properties
run: |
cargo test -p swactor-engine --test engine_unit generated_actor_timers_and_completion_are_bounded -- --exact
cargo test -p swactor-engine --test engine_unit lifecycle_invariant_detects_injected_duplicate_completion -- --exact
cargo test -p swactor-engine --test engine_unit lifecycle_invariant_detects_injected_uncancelled_periodic_timer -- --exact
- name: Process adapter properties
run: |
cargo test -p swactor-process --lib operations::properties::generated_stream_observations_close_once_and_stay_closed -- --exact
cargo test -p swactor-process --lib operations::properties::generated_lifecycle_actions_make_stop_idempotent_and_exit_terminal -- --exact
cargo test -p swactor-process --lib operations::properties::generated_stop_notifications_are_delivered_at_most_once -- --exact
cargo test -p swactor-process --lib operations::properties::generated_stdin_commands_and_eof_notify_once -- --exact
cargo test -p swactor-process --lib operations::properties::property_invariants_reject_controlled_defects -- --exact
cargo test -p swactor-process --lib operations::properties::trivial_real_child_exit_has_a_hard_timeout -- --exact
- name: Myelin component properties (exclude job and reconciler tests)
run: |
cargo test -p myelin --lib orchestration::manual_control::tests::aggressive_random_event_stream_preserves_control_invariants -- --exact
cargo test -p myelin --lib orchestration::manual_control::tests::rental_free_end_to_end_sequences_converge -- --exact
cargo test -p myelin --lib orchestration::manual_control::tests::manual_actor_generated_public_actions_and_callbacks_are_bounded -- --exact
cargo test -p myelin --lib orchestration::manual_control::tests::fixed_helper_cardinality_invariant_detects_controlled_extra_spawn -- --exact
cargo test -p myelin --lib orchestration::manual_control::tests::callback_panic_reports_typed_failure_without_poisoning_work_actor -- --exact
cargo test -p myelin --lib orchestration::manual_control::tests::callback_panic_invariant_detects_controlled_unguarded_panic -- --exact
cargo test -p myelin --lib provisioning::tests::mock_vastai_handle_state_survives_random_create_and_stop_sequences -- --exact
cargo test -p myelin --lib provisioning::tests::docker_generated_attempt_lifecycles_are_idempotent_and_bounded -- --exact
cargo test -p myelin --lib provisioning::tests::docker_duplicate_resource_detector_rejects_controlled_fault -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::offer_status_classes_are_offers_or_typed_rejections -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::malformed_offer_bodies_are_typed_rejections -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::wrong_or_missing_offer_fields_are_typed_rejections -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::duplicate_offer_records_remain_explicit_values -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::provider_monitor_preserves_contract_identity_and_cardinality -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::provider_monitor_terminal_polling_stops_after_one_typed_outcome -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::provider_monitor_poll_stop_orderings_cease_polling -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::duplicate_terminal_detector_rejects_controlled_fault -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_output_lines_preserve_stream_and_protocol -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_malformed_protocol_is_data_not_poison -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_eof_orderings_stop_relay_and_actor -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_child_failures_have_typed_attempt_outcomes -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_timeout_is_typed_and_stops_polling -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_stop_orderings_emit_one_terminal_and_stop_all_actors -- --exact
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::runtime_actors_generated_transitions_complete_once_on_one_worker -- --exact
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::runtime_invariant_checker_rejects_duplicate_readiness_publication -- --exact
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::helper_wait_generated_terminal_sequences_complete_once_on_one_worker -- --exact
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::helper_invariant_checker_rejects_expected_output_after_terminal_error -- --exact
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::stage_fetch_generated_observations_complete_once_on_one_worker -- --exact
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::stage_invariant_checker_rejects_wrong_terminal_classification -- --exact
cargo test -p myelin --lib orchestration::app::serve_cluster_properties::serve_cluster_production_transitions_converge_once_without_growth -- --exact
cargo test -p myelin --lib orchestration::app::serve_cluster_properties::serve_cluster_lifecycle_invariants_reject_injected_duplicate_and_growth -- --exact
cargo test -p myelin --lib orchestration::control::properties::generated_http_bridge_sequences_terminate_without_control_actor_growth -- --exact
cargo test -p myelin --lib orchestration::control::properties::generated_duplicate_control_replies_deliver_first_once_and_remove_observer -- --exact
cargo test -p myelin --lib orchestration::control::properties::reply_observer_disappearance_returns_a_bounded_terminal_http_response -- --exact
cargo test -p myelin --lib orchestration::control::properties::http_bridge_invariant_rejects_a_controlled_duplicate_forward -- --exact
- name: Controlled E2E oracle contract
run: cargo test -p myelin --features test-support --test stateful_vastai e2e_oracle_rejects_controlled_lifecycle_faults -- --exact
- name: Dashboard bridge properties
run: |
cargo test -p dashboard --features demo-control --lib control::properties::generated_concurrent_bridge_commands_forward_once_and_shutdown -- --exact
cargo test -p dashboard --features demo-control --lib control::properties::bridge_invariant_rejects_a_controlled_duplicate_delivery -- --exact
cargo test -p dashboard --features demo-control --lib server::tests::generated_control_http_sequences_are_bounded_and_typed -- --exact
cargo test -p dashboard --features demo-control --lib server::tests::control_http_invariant_rejects_a_controlled_server_error -- --exact
- name: Demo actor properties
run: |
cargo test -p xtask --bin xtask demo::control::properties::generated_control_commands_forward_only_after_supervisor_registration -- --exact
cargo test -p xtask --bin xtask demo::control::properties::control_transition_oracle_rejects_duplicate_forwarding -- --exact
cargo test -p xtask --bin xtask demo::feed::properties::generated_supervisor_transitions_are_once_only_nonblocking_and_clean -- --exact
cargo test -p xtask --bin xtask demo::feed::properties::supervisor_transition_oracle_rejects_duplicate_identity_resources -- --exact
cargo test -p xtask --bin xtask demo::node::properties::generated_node_runtime_transitions_emit_heartbeats_and_stop_once -- --exact
cargo test -p xtask --bin xtask demo::node::properties::node_transition_oracle_rejects_duplicate_resources -- --exact
cargo test -p xtask --bin xtask demo::provider::properties::generated_process_reports_complete_exit_watchers_once_and_preserve_last_state -- --exact
cargo test -p xtask --bin xtask demo::provider::properties::process_relay_oracle_rejects_lost_exit -- --exact
cargo test -p xtask --bin xtask demo::properties::direct_binary_signal_smoke_has_a_hard_timeout -- --exact
process-e2e:
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
needs: component-properties
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
- name: Install pinned Rust toolchain
run: rustup toolchain install nightly-2026-02-07 --profile minimal --component rustc-dev --component rust-src --component llvm-tools-preview
- name: Stateful VastAI process E2E
env:
PROPTEST_CASES: "4"
run: |
set -o pipefail
mkdir -p artifacts
cargo test -p myelin --features test-support --test stateful_vastai stateful_vastai_dashboard_control_survives_restarts -- --ignored --exact --nocapture 2>&1 | tee artifacts/stateful-vastai.log
- name: Preserve E2E failure artifacts
if: failure()
uses: actions/upload-artifact@v4
with:
name: stateful-vastai-failure-${{ github.run_id }}
path: |
artifacts/stateful-vastai.log
apps/myelin/proptest-regressions/tests/e2e_vastai.txt
if-no-files-found: warn

6
Cargo.lock generated
View file

@ -962,7 +962,9 @@ dependencies = [
"futures-lite",
"libc",
"parking_lot",
"proptest",
"serde",
"serde_json",
"sha2 0.10.9",
"swactor",
"swactor-engine",
@ -4335,9 +4337,11 @@ dependencies = [
"getrandom 0.2.17",
"loom",
"parking_lot",
"proc-macro2",
"proptest",
"proptest-state-machine",
"serde",
"syn",
"tracing",
"web-time 0.2.4",
]
@ -4489,11 +4493,13 @@ name = "telemetry"
version = "0.1.0"
dependencies = [
"crossbeam-channel",
"futures-channel",
"iroh",
"libc",
"serde",
"serde_json",
"swactor",
"swactor-engine",
"swactor-transport",
]

11
Cargo.toml
View file

@ -75,4 +75,15 @@ criterion = { version = "0.5", features = ["html_reports"] }
proptest = "1"
proptest-state-machine = "0.3"
loom = "0.7"
proc-macro2 = "1"
syn = { version = "2", features = ["full", "visit"] }
[workspace.lints.rust]
warnings = "deny"
[workspace.lints.clippy]
allow_attributes = "forbid"
[lints]
workspace = true

403
DATA_PLANE_SPSC_STREAM_MVP.md Normal file
View file

@ -0,0 +1,403 @@
# Data-plane SPSC stream MVP
## Status
Pending implementation.
This document is the authoritative plan for the remaining data-plane stream slice. Blob namespace and resolution work is defined separately in `DATA_PLANE_VIRTUAL_BLOB_NAMESPACE_MVP.md`.
## Goal
Complete the existing public stream API for one producer and one consumer:
```python
reader = await ctx.data.read_stream(path)
async with ctx.data.write_stream(path) as writer:
await writer.write(data)
```
Swactor must resolve the logical path, match exactly one source with exactly one sink, provision the local arena rings and remote transport, preserve byte order, apply lossless backpressure, propagate terminal state, and clean up every binding.
The final Myelin acceptance path replaces the temporary Unix-socket result bridge with this implementation.
## Scope
The MVP supports only:
- one producer;
- one consumer;
- ordered opaque bytes;
- lossless delivery while both endpoints remain live;
- explicit EOF and fault termination;
- bounded buffering and backpressure;
- actor-owned protocol state;
- arena-backed SPSC byte rings;
- local or Iroh-backed transfer chosen internally.
The MVP does not support:
- fanout or SPMC streams;
- multiple writers;
- topics or best-effort delivery;
- replay or resume;
- reconnecting an existing stream incarnation;
- application records, tensor schemas, JSON, dtype, or shape semantics;
- application-visible actors, nodes, edges, rings, sockets, or arena offsets.
## Public semantics
A stream path names one SPSC rendezvous.
```text
DataPath
-> waiting source or sink
-> matched source/sink incarnation
-> active transfer
-> EOF or fault
```
The stream is opaque ordered bytes. Application framing remains application code.
Each match creates one incarnation. A failed or closed incarnation is never silently resumed or joined to a replacement endpoint.
### Writer
```python
async with ctx.data.write_stream(path) as writer:
await writer.write(data)
```
- Entering the context resolves and authorizes the path, matches the sink, provisions the transfer, and waits for readiness.
- `write()` completes only after Swactor has accepted the complete input into its bounded ordered pipeline.
- Backpressure suspends `write()` rather than dropping bytes.
- A clean context exit publishes EOF exactly once.
- Exceptional exit publishes a fault or abort terminal state.
- Writes after terminal state fail typed.
### Reader
```python
reader = await ctx.data.read_stream(path)
chunk = await reader.read()
```
- Opening resolves and authorizes the path, matches the source, provisions the transfer, and waits for readiness.
- Reads observe source byte order.
- EOF is distinguishable from a zero-length temporary read.
- Source, transport, ring, and session faults become typed stream failures.
- Cancellation releases any held ring view and permits cleanup.
The precise Python read method shape may follow existing binding conventions, but it must not return arena, ring, edge, actor, or transport identities.
## Namespace rendezvous
The centralized namespace service from `DATA_PLANE_VIRTUAL_BLOB_NAMESPACE_MVP.md` gains stream entries without changing blob semantics.
Conceptually:
```rust
enum NamespaceEntry {
Blob(BlobBinding),
Stream(StreamBinding),
}
enum StreamBinding {
WaitingSink {
sink: ActorAddress,
},
WaitingSource {
source: ActorAddress,
},
Matched {
incarnation: u64,
source: ActorAddress,
sink: ActorAddress,
coordinator: ActorAddress,
},
}
```
The directory actor serializes registration and matching. A path cannot simultaneously name a blob and a stream.
For the first Myelin flow, orchestration may register the result sink before the job opens its writer. The actor protocol should not require application-managed edge IDs or socket paths.
## Existing foundation to retain
`crates/data-plane/src/byte_ring.rs` already provides the correct stream storage foundation:
- arena-resident SPSC header and data region;
- release/acquire `commit` and `consume` cursors;
- generation and producer/consumer role enforcement;
- `Data`, `Eof`, and `Fault` record framing;
- lossless bounded backpressure;
- ordering and wraparound handling;
- corruption containment.
Retain its valid guarantee tests.
Also retain:
- one inherited arena descriptor;
- automatically driven child runtime;
- persistent host and child data-plane session actors;
- one operation actor per public open;
- one host binding/coordinator actor per match;
- existing actor routing and Iroh transport;
- the rule that protocol state lives in actor FSM fields rather than mutex-protected pending maps.
## Required byte-ring correction
The current consuming path returns owned bytes through `recv_record() -> Vec<u8>`. That is not zero-copy.
Add a pinned borrowed record view:
```rust
struct PinnedRecordView {
endpoint_guard: Arc<...>,
generation: u64,
kind: RecordKind,
first_span: ...,
second_span: Option<...>,
}
```
A record may wrap around the ring and therefore expose two ordered spans. The view pins the committed range. The consumer cursor advances only after the view is released or explicitly consumed.
Required behavior:
```text
producer commits record
-> consumer receives pinned view
-> transport/Python consumes one or two spans
-> view release publishes consume cursor
-> producer capacity becomes available
```
Holding a view intentionally applies backpressure.
No hot-path stream operation should assemble a record into `Vec<u8>` merely to cross the host/child boundary or feed the transport.
## Actor notification model
Actor messages replace eventfd wakeups and polling.
Messages indicate that protocol state may have advanced; ring cursors remain authoritative.
Conceptually:
```text
producer commits data
-> ProgressAvailable notification
-> consumer checks committed cursor
consumer advances consume cursor
-> CapacityAvailable notification
-> producer retries pending reservation
```
Notifications may be coalesced or redundant. Correctness must not depend on wake counts.
Each operation actor owns its pending action in FSM fields. Do not add a shared request-ID map or protocol mutex.
## Standard happy path
```text
Sink registration
-> namespace records waiting sink
Python write_stream(path)
-> PyDataPlane delegates to general DataPlane
-> ChildDataPlaneSessionActor spawns WriteStreamOperationActor
-> host session authorizes canonical path
-> namespace matches source operation with waiting sink
-> StreamCoordinatorActor is created
Coordinator
-> resolves persistent endpoint actors to nodes
-> allocates and installs arena SPSC rings
-> establishes local or Iroh transfer
-> waits for source and sink readiness
-> sends Opened to both operation paths
Python writer.write(bytes)
-> reserves ring capacity
-> writes bytes into producer spans
-> release-commits Data record
-> actor notification drives downstream progress
-> transport consumes pinned source record
-> destination producer commits to consumer ring
-> reader consumes bytes in order
-> consume progress releases backpressure
Clean writer exit
-> commits EOF once
-> reader observes all preceding bytes, then EOF
-> coordinator quiesces pumps
-> rings are uninstalled and leases released
-> namespace incarnation terminates
```
## Backpressure
Backpressure is end-to-end and lossless:
```text
slow reader
-> destination ring fills
-> network receive stops advancing
-> source transport stops draining
-> source ring fills
-> writer.write awaits capacity
```
No layer may drop bytes, overwrite unread bytes, busy-spin, or create an unbounded overflow queue.
A held reader or transport view is part of the backpressure mechanism and must keep the relevant range pinned.
## Ordering
For one stream incarnation:
- producer writes have one total order;
- the consumer observes exactly that order;
- EOF follows every successfully accepted data byte;
- a fault terminates the incarnation and no later data is delivered;
- transport chunk boundaries are not application-visible stream boundaries.
The existing Iroh QUIC byte stream already provides remote ordered delivery. Swactor must not implement another TCP-like acknowledgement, retransmission, congestion-control, or reorder protocol.
## Terminal behavior
Each incarnation has exactly one terminal result:
```text
EOF
Fault
Cancelled
SessionClosed
```
Terminal state is sticky. Duplicate or late terminal messages are ignored after the first accepted terminal transition.
A clean writer close produces EOF. Writer failure, transport failure, ring corruption, endpoint death, or exceptional writer exit produces a fault. Cancellation and session shutdown reclaim resources without presenting successful EOF.
## Python binding boundary
General functionality belongs in `crates/data-plane`:
- stream reader and writer types;
- operation actors;
- pinned ring record views;
- ring lifetime guards;
- backpressure and terminal FSMs;
- namespace and transfer protocols.
Python owns only:
- `PyStreamReader` and `PyStreamWriter` wrappers;
- Rust-future to awaitable conversion;
- buffer exposure where used;
- async context-manager behavior;
- typed exception conversion.
Python must not manually tick the runtime or poll shared cursors.
## Myelin migration
The current result path uses a temporary Unix-domain socket exposed through `legacy_output` and `SWACTOR_DATA_PLANE_OUTPUT`.
After actor-driven streams satisfy the acceptance path:
1. Register the Myelin result sink at `/runs/<run-id>/results/inference`.
2. Let `/runs/self/results/inference` resolve through `JobContext`.
3. Route `ctx.data.write_stream(...)` through the SPSC stream implementation.
4. Remove the Unix listener, socket path, legacy environment variable, connector, and compatibility code.
5. Preserve application-owned JSON/result framing.
6. Rerun the complete Tinygrad CUDA scenario.
Expected result:
```text
CUDA -> [2.75, -8.75]
```
## Failure behavior
| Failure | Required behavior |
|---|---|
| Path absent or unmatched under chosen open policy | Typed path/rendezvous failure or pending operation, never fallback socket |
| Unauthorized path | Fail before ring or edge provisioning |
| Duplicate source or sink | Typed namespace conflict |
| Arena exhaustion | Fail both sides and release partial provisioning |
| Ring generation mismatch or corruption | Fault incarnation and stop using the ring |
| Iroh stream fault | Fault both endpoint operations |
| Writer exception | Publish fault/abort, never clean EOF |
| Reader or writer cancellation | Release held views and terminate binding |
| Child or host session death | Stop bindings, quiesce pumps, reclaim leases |
| EOF | Deliver all prior bytes, then complete reader exactly once |
## Behavioral invariants
1. One stream incarnation has exactly one producer and one consumer.
2. A live path cannot simultaneously name a blob and a stream.
3. Neither endpoint receives `Opened` before required rings and transport are ready.
4. Successfully accepted bytes are delivered exactly once and in order while both endpoints remain live.
5. The producer never overwrites unread bytes.
6. The consumer never observes uncommitted bytes.
7. A pinned record prevents its range from being consumed or reused.
8. Backpressure is bounded and lossless.
9. EOF follows all accepted data and appears exactly once.
10. A fault prevents subsequent data or EOF success.
11. Every incarnation reaches exactly one terminal outcome.
12. Every arena lease is released only after pumps and views are quiescent.
13. Wake notifications are hints; ring state is authoritative.
14. Python never observes actor, node, edge, ring, socket, or arena identities.
15. Swactor never interprets application byte schemas.
## Implementation sequence
1. Extend the centralized namespace with SPSC stream source/sink registration and atomic matching.
2. Define shared stream session, operation, coordinator, progress, and terminal messages.
3. Implement pinned borrowed byte-ring record views with wraparound spans.
4. Add actor-driven capacity and data-progress notifications.
5. Implement general `StreamReader` and `StreamWriter` operation actors in `crates/data-plane`.
6. Provision local arena rings and Iroh edges through one coordinator FSM.
7. Add thin Python reader/writer wrappers and async context behavior.
8. Add focused tests for ordering, backpressure, wraparound, held views, EOF, faults, cancellation, and lease cleanup.
9. Substitute the SPSC path for Myelin's temporary Unix result bridge.
10. Remove all legacy stream socket and environment plumbing.
11. Run affected Rust and Python suites.
12. Exercise the full Tinygrad CUDA scenario and verify the deterministic result.
## Verification
Focused behavioral proof must cover:
- source-first or sink-first rendezvous behavior selected for the MVP;
- concurrent independent SPSC paths;
- exact byte ordering through wraparound;
- writer suspension and resumption under backpressure;
- no consume advancement while a view is pinned;
- EOF after all prior data;
- fault dominance over later progress;
- cancellation and session-failure reclamation;
- no stream payload assembly into `Vec<u8>` on zero-copy host/child paths;
- absence of the legacy Unix stream bridge after migration;
- the complete Myelin CUDA result path.
Affected suites include:
```bash
cargo test -p data-plane
cargo test -p myelin --lib
PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 cargo check -p python
```
Python binding tests run after rebuilding the extension from `crates/bindings/python`.
## Final boundary
The remaining stream MVP is intentionally narrow:
> One logical path rendezvous-matches one writer with one reader. Swactor provisions ordered bounded SPSC byte movement, uses actor messages for progress, applies lossless backpressure, propagates EOF or faults, and removes the temporary Unix bridge without exposing transport mechanics to applications.

748
DATA_PLANE_VIRTUAL_BLOB_NAMESPACE_MVP.md Normal file
View file

@ -0,0 +1,748 @@
# Data-plane virtual blob namespace MVP
## Purpose
This document defines the MVP for registering, resolving, transferring, and exposing fixed-size blobs through Swactor.
The immediate acceptance path is Myelin loading `apps/myelin/jobs/tiny_linear.weights` from disk, registering it at `/models/tiny-linear/weights`, transferring it into a job's arena, and exposing it to Python as a read-only blob.
The design deliberately targets the centralized happy path:
- one authoritative namespace actor;
- one current blob target per path;
- one disk-backed source implementation;
- complete eager transfer into the current arena-backed Python view;
- existing actor routing and Iroh byte transport.
Content identities, verification, replication, leaderless consistency, lazy paging, and additional source types are deferred.
## Core abstraction
A blob is a fixed-length virtual byte space.
A `DataPath` behaves like a named pointer:
```text
DataPath -> current blob target
```
The path is the stable identity. It is not tied to a content hash or immutable object ID.
```text
/models/tiny-linear/weights
-> current FileBlobSourceActor
```
A later registration may atomically replace the target:
```text
/models/tiny-linear/weights
-> replacement source
```
### Dereference semantics
`read_blob(path)` dereferences the path once.
```text
read begins while path -> target A
path is later changed -> target B
existing read continues against A
new reads resolve B
```
An in-flight operation is never redirected midway through transfer.
This is pointer replacement, not live shared mutation of bytes underneath an existing view. Supporting visible in-place mutation requires a separate coherence model and is outside the MVP.
## Developer experience
A developer registers a disk-backed blob with one control-plane operation:
```rust
data.register(
"/models/tiny-linear/weights",
blob::file(weights_path),
)
.await?;
```
A job reads it by path:
```python
weights = await ctx.data.read_blob(
"/models/tiny-linear/weights",
)
with weights.map() as mapped:
values = struct.unpack_from("<6f", mapped)
```
Swactor owns everything between these calls:
- namespace consistency;
- file opening and source lifetime;
- actor and node resolution;
- transfer negotiation;
- destination arena allocation;
- completion and failure;
- lease and source cleanup.
The developer does not supply:
- byte length;
- digest;
- provider actor;
- directory actor;
- node identity;
- edge identity;
- source chunks;
- provenance metadata;
- cleanup behavior.
## Central namespace
One authoritative `DataDirectoryActor` owns the cluster namespace:
```rust
struct DataDirectoryActor {
blobs: BTreeMap<DataPath, BlobBinding>,
}
struct BlobBinding {
source: ActorAddress,
length: u64,
revision: u64,
}
```
The actor serializes all registration, replacement, lookup, and removal operations.
Conceptual protocol:
```rust
enum DataDirectoryIn {
Register {
path: DataPath,
source: ActorAddress,
operation_id: OperationId,
length: u64,
reply_to: ActorAddress,
},
Resolve {
path: DataPath,
reply_to: ActorAddress,
},
Unregister {
path: DataPath,
operation_id: OperationId,
reply_to: ActorAddress,
},
}
```
`Register` is an atomic durable store:
- an absent path receives its first binding;
- an existing path has its binding replaced;
- the namespace revision advances once;
- success is returned only after the new binding and operation result are durable and authoritative.
`Unregister` follows the same durable commit rule. A client-generated `OperationId` makes mutation retry idempotent across a crash after commit but before reply:
```text
same operation ID + same request -> return the committed result
same operation ID + different request -> reject
```
The revision is actor-message fencing and recovery state. It is not a content identity and is not exposed to applications.
### Availability and restart model
The orchestrator hosts the one authoritative directory for the MVP:
- no consensus;
- no replication;
- no leader election;
- exactly one orchestrator instance may own the namespace;
- namespace mutations and current bindings are persisted locally.
If the orchestrator or directory is unavailable, operations that still require namespace authority wait for service rediscovery instead of failing merely because the authority restarted. Callers may cancel or impose their own deadline. Reads that already selected a source no longer depend on the directory and may complete while it is unavailable.
On restart, the orchestrator restores the durable namespace, reconstructs recoverable sources, starts a new `DataDirectoryActor`, increments its authority epoch, and publishes the actor through stable service discovery. Waiting clients discover the new actor and retry unresolved or idempotent operations.
Already-returned `Blob`s, mapped views, job arenas, and independently hosted source actors do not depend on the orchestrator process. A transfer whose source actor died with the orchestrator may fail; it is never silently redirected to a replacement target.
Concurrent orchestrators and recovery from network partitions are outside the MVP. Deployment must provide singleton process ownership; split-brain safety requires external fencing or a future consensus design.
## Registration lifecycle
The control-plane registration call performs:
```text
open file
-> determine fixed length
-> create disk source behavior
-> make its actor routable
-> durably commit its recovery descriptor and namespace binding
-> atomically install it in DataDirectoryActor
```
Registration persists until:
- explicit `unregister(path)`;
- replacement by a later registration.
It survives orchestrator restart and does not depend on retaining an application-visible registration guard. A registration acknowledged before a crash is present after recovery.
### Replacement
A later registration atomically replaces the target:
```rust
data.register(
"/models/tiny-linear/weights",
blob::file(new_weights_path),
)
.await?;
```
```text
before swap:
new reads -> old source
after swap:
new reads -> new source
```
Existing reads retain the old source binding until completion or failure.
The retired source remains alive while active operations use it. Once it is no longer the namespace target and has no active transfers, Swactor closes it.
`unregister(path)` follows the same lifetime discipline:
```text
remove namespace binding
-> reject new reads with PathNotFound
-> permit already-pinned reads to finish
-> retire source when unreferenced
```
## Disk source behavior
The MVP implements one concrete source:
```text
FileBlobSourceActor
```
It owns:
- an opened read-only file;
- the fixed length observed at registration;
- private source information needed to read it;
- active source transfer bindings.
The public namespace record contains no file path. The durable namespace store retains a private recovery descriptor sufficient to reopen the source after orchestrator restart; that descriptor is never sent to Python or returned by path resolution.
The trusted-cluster MVP contract is sufficient:
- the registrant supplies a stable file at a restart-stable private location;
- Swactor retains the opened file while the source actor is live;
- recovery reopens the file and checks the fixed registration length before republishing the source;
- source length is fixed;
- digest and mutation verification are deferred.
If recovery cannot reopen or stat the file at the recorded length, the binding remains known but unavailable and reads fail with a typed source failure until control replaces or unregisters it. Recovery must not silently bind different bytes.
HTTP, object storage, caching, and remote-origin plugins are not implemented now. The source boundary and durable descriptor remain opaque so `DataDirectoryActor` does not depend on disk-specific fields.
## Swactor service installation
The namespace and source plumbing are Swactor services, not application setup.
Internally:
```text
Swactor/Myelin orchestrator runtime
-> opens durable namespace store
-> restores current bindings and idempotency records
-> reconstructs recoverable source actors
-> starts DataDirectoryActor with a new authority epoch
-> publishes the namespace service through stable discovery
```
Host job sessions discover the namespace service internally. They retain the logical service locator rather than treating one runtime actor address as permanent.
Application code does not:
- spawn the directory;
- publish or retain its actor address;
- wire routes;
- pass it through job configuration;
- implement reconnect or mutation retry.
The existing distribution `DirectoryActor` remains responsible for locating persistent source actors on nodes. The data directory resolves logical paths; the distribution directory resolves actor locations; stable service discovery locates the current data-directory actor after restart.
## Python API
### `read_blob`
```python
blob = await ctx.data.read_blob(path)
```
The public API guarantees:
- the path was canonicalized and authorized;
- one current source target was selected;
- the returned `Blob` represents one fixed byte space;
- namespace-authority restart before resolution stalls the awaitable rather than changing its meaning;
- source, placement, and recovery details remain opaque.
The MVP implementation additionally:
- allocates the complete destination lease;
- transfers all bytes;
- checks exact length;
- seals the lease;
- returns only after the complete blob is ready.
That eager behavior is not a permanent public placement promise. A future implementation may provide lazy virtual mappings without changing the `Blob` abstraction.
### `Blob`
```python
blob.length
blob.digest # normally None in this MVP
blob.map()
```
### `BlobView`
```python
with blob.map() as mapped:
consume(mapped)
```
`BlobView` is the public name. `ArenaView` exposes the current backing strategy.
The MVP `BlobView`:
- wraps the existing arena-backed view;
- exports Python's read-only buffer protocol;
- holds the blob lease alive;
- prevents close while buffer exports remain active;
- never constructs Python `bytes` or `bytearray`.
## Standard read path
```text
Python:
ctx.data.read_blob(path)
PyDataPlane:
converts Rust future to Python awaitable
DataPlane:
asks ChildDataPlaneSessionActor
ChildDataPlaneSessionActor:
spawns ReadBlobOperationActor
ReadBlobOperationActor:
sends OpenReadBlob to HostDataPlaneSessionActor
HostDataPlaneSessionActor:
expands /runs/self
checks JobContext read prefixes
spawns HostReadBlobBindingActor
HostReadBlobBindingActor:
discovers the current DataDirectoryActor
waits and rediscovers while namespace authority is unavailable
asks DataDirectoryActor to resolve path
DataDirectoryActor:
returns one current source actor, fixed length, and binding revision
HostReadBlobBindingActor:
pins that source for this operation
allocates FillingRead lease in child arena
asks source actor to begin transfer
FileBlobSourceActor:
spawns one source transfer binding
reads the opened file
sends bytes through existing Iroh transfer
Destination binding:
writes incoming chunks into final arena payload
tracks exact received length
seals the lease after source completion
HostReadBlobBindingActor:
sends BlobOpened through child session
ReadBlobOperationActor:
validates sealed lease
constructs Blob and lease guard
completes original local ask
PyDataPlane:
returns PyBlob
```
If authority disappears before resolution completes, the binding returns to discovery and keeps the original awaitable pending. It does not allocate a destination lease while waiting. Cancellation stops discovery. After a source binding has been returned, that logical read is pinned and is never re-resolved to a newer path target.
The current Iroh API produces `WireEvent::BytesRead(Vec<u8>)`, so the MVP performs one copy from each network buffer into the final arena payload. It must not add another blob-assembly buffer or pass blob bytes through the reusable byte ring.
## Python `write_blob`
`write_blob` updates the same namespace:
```python
async with ctx.data.write_blob(path, length=n) as blob:
with blob.map() as mapped:
mapped[:] = result
```
Flow:
```text
allocate writable candidate
-> Python fills candidate
-> clean context exit seals candidate
-> host binding becomes a readable source
-> discover or wait for namespace authority
-> durably and atomically replace path binding
```
Exceptional exit:
```text
abort candidate
-> publish nothing
-> retain previous path target
```
No separate host-session `published` namespace remains. Disk registrations and Python-produced blobs are both current targets in `DataDirectoryActor`.
A sealed candidate waiting for directory recovery retains its arena lease. Cancellation or child-session close aborts it and reclaims the lease. A committed publication is retried by operation ID if the directory crashes before its reply.
## Reuse from the current implementation
Retain:
- `DataPath` parsing and segment-safe prefix checks;
- `/runs/self` expansion in `JobContext`;
- host-side read/write authorization;
- child session and per-operation actors;
- host binding actors;
- arena allocator actor;
- `AllocationKind::FillingRead`;
- incoming chunk writes into the final lease;
- exact-length sealing;
- `Blob`, lease guard, and Python buffer lifetime handling;
- Iroh data transfer;
- one inherited arena descriptor.
Refactor:
- `IncomingBlobSourceActor` becomes destination transfer machinery rather than a session-local namespace owner;
- current `BeginBlobSource`, `BlobSourceChunk`, `FinishBlobSource`, and `FailBlobSource` calls become internal transfer events;
- `HostDataPlaneSessionActor::blobs` and `published` no longer define application paths.
Remove from application setup:
- `InputBlobAssignment` edge/path registration;
- manual source begin/push/finish calls;
- application-managed model edge IDs;
- `include_bytes!` as the production model source.
## Myelin acceptance path
Control side:
```rust
data.register(
"/models/tiny-linear/weights",
blob::file("apps/myelin/jobs/tiny_linear.weights"),
)
.await?;
```
Job side:
```python
weights = await ctx.data.read_blob(
"/models/tiny-linear/weights",
)
with weights.map() as mapped:
values = struct.unpack_from("<6f", mapped)
```
Swactor resolves the source actor, negotiates the transfer, fills the job arena, and returns the blob.
The final scenario remains:
```text
CUDA -> [2.75, -8.75]
```
The temporary result-stream bridge remains until the later SPSC stream slice replaces it.
## State machines
### Namespace binding
```text
Absent
-> Bound(revision)
-> Replaced(new revision)
-> Unbound(new revision)
```
Every mutation is durably committed before acknowledgement. Replacement installs the new source atomically. The previous source transitions independently toward retirement.
### Destination read binding
```text
DiscoveringDirectory
-> Resolving
-> Allocating
-> Negotiating
-> Filling
-> Sealing
-> Granted
-> Released
DiscoveringDirectory or Resolving
-> WaitingForDirectory
-> DiscoveringDirectory
any pre-grant state
-> Faulted
-> Releasing
```
### File source
```text
Opening
-> Bound
-> Serving
-> Retiring
-> Closed
```
Each read uses an operation-specific source binding. The persistent source actor does not require a request-ID hashmap.
## Failure behavior
| Failure | Behavior |
|---|---|
| Path absent | `PathNotFound` |
| Unauthorized read | `Unauthorized` before discovery or transfer |
| File open/stat failure during registration | Registration fails; existing binding remains |
| File recovery open/stat/length failure | Binding remains known but unavailable; reads return a typed source failure |
| Source route unavailable after resolution | Read fails with source or session failure; it is not redirected |
| File read failure | Partial lease aborts; no `Blob` is returned |
| Arena exhausted | `ArenaExhausted` |
| Short or oversized transfer | Length failure; no `Blob` is returned |
| Iroh failure | Partial lease aborts |
| Python await cancellation | Waiting discovery, operation, transfer, and partial lease cancel |
| Child session close | Active bindings terminate and leases reclaim |
| Directory or orchestrator unavailable before resolution | Namespace-dependent operation waits for rediscovery; active independent transfers may finish |
| Directory crash after committed mutation but before reply | Client retries the same operation ID and receives the committed result |
## Correctness guarantees
### Safety and consistency properties
1. Namespace operations are linearizable through the singleton authority: each mutation takes effect at one revision in one total order, and each successful resolve observes one revision.
2. Each path has at most one current binding at every revision.
3. A read observes one binding snapshot. Once resolved, rebinding and authority restart cannot redirect it.
4. Blob visibility is all-or-nothing: no partial, failed, short, oversized, or unsealed transfer produces a `Blob`.
5. A source exposes one fixed extent for the lifetime of a selected binding.
6. Authorization precedes namespace discovery, arena allocation, and transfer.
7. Disk and Python-produced blobs have the same publication and replacement semantics.
### Durability and recovery guarantees
1. Acknowledged namespace mutations survive orchestrator restart.
2. A mutation retried with the same operation ID has at-most-once logical effect and returns its recorded result.
3. Recovery never substitutes a different source for an already-selected read.
4. A file binding is republished only after its private recovery descriptor reopens at the registered length.
### Conditional liveness and availability guarantees
1. An unresolved namespace operation remains pending while the authority is unavailable and resumes once the singleton authority is discoverable again.
2. Callers may cancel or impose a deadline while waiting.
3. Already-selected transfers may complete without the directory when their source and destination processes remain live.
4. No progress guarantee is made while the singleton authority is unavailable, or when a selected source process has failed.
### Resource and lifetime invariants
1. A live `Blob` or `BlobView` prevents destination lease reclamation.
2. Cancellation and failure reclaim candidate and partial leases.
3. A retired source remains alive until its pinned reads finish while its source process remains live.
4. The directory contains no payload bytes or transfer buffers.
5. Python observes paths, fixed bytes, waiting, and typed failures—not actor, node, file, recovery, or transfer identities.
Safety assumes one orchestrator namespace authority. Concurrent authorities, partition recovery, and automatic failover are outside the MVP.
## Verification methodology
Verification combines deterministic conformance tests, stateful property testing, deliberate fault injection, and real-system integration.
1. **Deterministic guarantee tests (conformance scenarios)** encode each safety, durability, liveness, and lifetime guarantee as a small reproducible scenario over controlled actor runtimes.
2. **Stateful model-based property tests (model-based fuzzing)** maintain a simple reference namespace model, generate bounded sequences of legal actions, apply each action to the model and implementation, and check the guarantees after every transition. Actions include register, replace, unregister, begin/resolve/finish read, publish/abort write, cancel, retire, lose authority, and recover authority. Seeds are replayable and failing sequences are shrunk.
3. **Failpoint and fault-injection tests** deliberately fail storage before commit, after durable commit, and before reply; drop directory routes; fail file reads; break Iroh streams; cancel operations in each pre-grant state; and close child sessions. Each test checks both the reported result and the absence of leaked leases or unintended namespace changes.
4. **Crash-recovery tests** reopen real temporary durable stores and run an actual orchestrator stop/restart to verify acknowledged state recovery, idempotent retry, and pending-operation resumption.
5. **Real-adapter integration tests** use Iroh loopback to prove network buffers fill the final `FillingRead` lease directly and use the compiled Python extension to verify `BlobView` buffer and lifetime semantics.
6. **End-to-end acceptance** registers the real weights file, runs the Myelin/Tinygrad job, and verifies `CUDA -> [2.75, -8.75]`.
The reference model covers externally observable namespace and operation state, not implementation fields. Stateful generation produces only legal commands for the current model state; targeted invalid-input tests remain separate.
The build loop for each vertical slice is:
```text
state the guarantee
-> add deterministic conformance case and model transition
-> implement the narrow production path
-> run bounded generated action sequences
-> run relevant failpoints
-> run directly affected existing tests
```
Mocks are limited to deterministic scheduling and failpoint control at storage and transport ports. Final transport, Python, process-restart, and CUDA verification use real implementations.
## Code architecture
### New data-plane modules
| Location | Ownership |
|---|---|
| `crates/data-plane/src/control.rs` | Reusable `DataNamespaceService` recovery and public `DataPlaneControl` file registration, ensure, and unregister API. |
| `crates/data-plane/src/namespace.rs` | `DataDirectoryActor`, runtime `BlobBinding`, revisions, operation IDs, register/resolve/unregister network protocol, and typed replies. |
| `crates/data-plane/src/namespace_store.rs` | Versioned durable schema, opaque source recovery records, committed operation results, load/recovery, and crash-safe temporary-write/sync/rename persistence. |
| `crates/data-plane/src/source.rs` | `FileBlobSourceActor`, recovery-time reopening and length validation, retirement, and one operation-specific source binding per read. |
| `crates/data-plane/src/blob_transfer.rs` | Transport-neutral transfer offers, source/destination ports, completion/failure events, and transfer identifiers. It contains no Iroh types. |
`crates/data-plane/src/lib.rs` exports those modules. `crates/data-plane/src/protocol.rs` registers their wire codecs and adds only shared typed failures. `crates/data-plane/Cargo.toml` gains only dependencies required by the durable schema; distribution and Iroh remain outside this crate.
### Existing data-plane modules
| Location | Change |
|---|---|
| `crates/data-plane/src/host.rs` | Remove session-local application namespace ownership. Refactor `IncomingBlobSourceActor` into destination transfer state and route host reads/writes through the namespace client. Preserve the arena allocator as sole lease allocator/releaser. |
| `crates/data-plane/src/data_plane.rs` | Preserve child session and operation APIs; extend cancellation and pending-publication handling for namespace rediscovery. |
| `crates/data-plane/src/blob.rs` | Rename the public read view to `BlobView` without changing sealing, mapping, or lease lifetime. |
### Transport and Myelin composition
| Location | Change |
|---|---|
| `crates/iroh-driver/src/blob_transfer.rs` | Implement the data-plane transfer ports using Iroh streams and forward each received network buffer directly to the destination binding. |
| `apps/myelin/src/data_namespace.rs` | New composition layer: open the namespace store, reconstruct sources, spawn the orchestrator directory, publish `swactor.data-directory`, and run the node-local namespace client/proxy that discovers and retries against the current authority epoch. |
| `apps/myelin/src/orchestration/distribution_stack.rs` | Expose the existing `RegistryActor`/`RegistryView` to the namespace composition. It remains generic distribution infrastructure and gains no blob semantics. |
| `apps/myelin/src/codecs.rs` | Register namespace, source, and transfer actor codecs. |
| `apps/myelin/src/job_data_plane.rs` | Inject the namespace client and Iroh transfer ports into host sessions; remove direct blob maps and manual source chunk methods. |
| `apps/myelin/src/job_deploy.rs` | Remove `InputBlobAssignment`, model edge setup, and incoming manual chunk dispatch. Keep the temporary result-stream bridge. |
| `apps/myelin/src/node/worker_node_runtime.rs` | Install the node-local namespace client and transfer receiver when constructing job services. |
| `apps/myelin/src/orchestration/app.rs` and deployable orchestrator startup | Install the authoritative namespace service and its configured durable state path. |
| `crates/bindings/python/src/job.rs` | Expose `BlobView`, retain typed error mapping, and replace the embedded manual test source with the real registration path. |
### Architectural invariants
1. `DataDirectoryActor` is the only in-memory writer of current logical bindings and revisions.
2. `NamespaceStore` is the only durable namespace writer. One actor serializes calls into it.
3. A mutation constructs the next complete state, durably commits it, swaps the runtime state, then replies—in that order.
4. A repeated operation ID returns its recorded result without creating another source, revision, or retirement transition.
5. The node-local namespace client owns discovery and retry. Host sessions never retain a directory actor address as durable configuration.
6. A host read accepts one successful resolve reply. After that transition, directory changes cannot alter its source actor, length, or revision.
7. `DataDirectoryActor` and `NamespaceStore` never receive payload bytes or transfer buffers.
8. The data-plane crate defines transport ports; Iroh-driver implements them; Myelin wires them. Dependencies never point from data-plane into Iroh-driver, distribution, or Myelin.
9. `FileBlobSourceActor` exclusively owns its opened file. Each transfer child owns one read attempt and reports completion before retirement can close the source.
10. `ArenaAllocatorActor` remains the sole allocator and releaser. Destination bindings write only within their granted `FillingRead` lease and seal only at the exact advertised length.
11. Cancellation is an explicit state transition that stops discovery or transfer and reclaims any candidate or partial lease.
12. Python receives only `Blob`, `BlobView`, lengths, optional digests, and typed failures. Actor, node, file, recovery, and transfer identities remain internal.
## Implementation sequence
1. Add deterministic guarantee tests and a reference state model for namespace ordering, durable acknowledgement, idempotent retry, resolve-once reads, waiting, cancellation, and lease cleanup; then add bounded legal-action sequence generation over that model.
2. Implement `namespace_store.rs` and make its persistence and crash-injection contracts pass.
3. Implement `namespace.rs`, wire codecs, and directory actor tests over the durable store.
4. Implement `source.rs`, disk registration, source recovery, retirement, and source lifecycle tests.
5. Define `blob_transfer.rs`, implement the Iroh adapter, and pass real loopback exact-length and failure tests.
6. Add Myelin namespace bootstrap, stable `RegistryActor` discovery, authority epochs, and the node-local retrying namespace client.
7. Refactor `host.rs` so read authorization precedes discovery, resolution precedes allocation, and transfer fills and seals the final arena lease.
8. Move `write_blob` publication into the directory with durable operation IDs and lease retention/cancellation while authority is unavailable.
9. Remove session-local `blobs` and `published` ownership and delete the manual begin/chunk/finish protocol and its callers.
10. Rename Rust and Python `ArenaView` to `BlobView`, migrate every caller, and run the Python buffer-lifetime contracts.
11. Replace Myelin model edge assignment and `include_bytes!` with one disk registration.
12. Run the process-level orchestrator stop/restart contracts, including waiting read, waiting publication, committed-before-reply retry, and no post-resolution redirection.
13. Run all affected suites, then exercise Tinygrad CUDA and verify `CUDA -> [2.75, -8.75]`.
## Deferred
- content IDs and digest verification;
- replicated or leaderless namespace authority;
- concurrent-orchestrator fencing, partition recovery, and automatic failover;
- multiple providers or replicas;
- payload durability across source-node loss;
- caching and spilling;
- lazy page-fault-backed mappings;
- partial range reads;
- HTTP and object-store sources;
- SPSC stream namespace matching;
- topics, fanout, and SPMC streams;
- GPU-native blob realizations.
## Final MVP boundary
The MVP is intentionally narrow:
> A restartable singleton orchestrator durably owns a centralized actor registry that maps pointer-like paths to current fixed-byte sources. Namespace-dependent operations wait while that authority restarts; already-selected independent transfers and returned blobs do not depend on it. Disk registration is one operation. Python dereferences by path. Swactor resolves one source, performs the complete transfer into the existing sealed arena backing, and returns a read-only blob without exposing placement or recovery mechanics.
## Completion status
The remaining MVP work is complete.
### Delivered behavior
- Public `DataPlaneControl` registers disk sources through `blob::file(...)`; Myelin supplies only cluster publication, discovery, and Iroh transport composition.
- A file registered on the authority node is readable by a session on a remote node.
- A blob committed by one session is readable by an independent remote session.
- Committed publication ownership detaches from the producer session. Producer close does not unregister it.
- Replacement or unregister retires the source after active transfers, then reclaims the source actor and arena lease.
- Tinygrad CUDA output and the temporary result socket behavior remain unchanged.
### Verification coverage
- `crates/data-plane/tests/namespace_host_read_guarantees.rs` deterministically covers public file registration, committed publication lifetime across producer close, and source reclamation after unregister.
- `apps/myelin/src/tests/data_namespace_guarantees.rs` composes two real Iroh-backed nodes and covers remote control registration reads, remote session publication reads, producer close, and unregister reclamation.
- The data-plane and Iroh blob-transfer suites cover namespace, source, transfer, exact-length, cancellation, and lease behavior.
- The compiled Python extension suite covers `BlobView` buffer lifetime, write publication, the temporary output socket, real process attachment, and the Tinygrad CUDA result `[2.75, -8.75]`.
### Final code architecture
```text
crates/data-plane
├── control
│ Reusable namespace service and public control API
├── namespace + namespace_store
│ Namespace semantics and durability
├── source + blob_transfer
│ Source lifetime and transport-neutral movement
└── host + data_plane + blob
Reusable arena, transfer, publication, and mapping machinery
apps/myelin
├── data_namespace
│ Cluster-level discovery, publication, and Iroh composition
└── job_data_plane
Creates data-plane sessions for managed jobs
```

8
apps/myelin/Cargo.toml
View file

@ -10,7 +10,6 @@ autobins = false
[features]
default = ["dashboard"]
dashboard = []
test-support = ["swactor-vastai/test-support"]
[dependencies]
telemetry = { path = "../../crates/telemetry" }
@ -41,10 +40,6 @@ tempfile = "3"
swactor-vastai = { path = "../../tools/vastai", features = ["test-support"] }
proptest = "1"
[[test]]
name = "stateful_vastai"
path = "tests/stateful_vastai.rs"
required-features = ["test-support"]
[target.'cfg(target_os = "linux")'.dependencies]
libc = "0.2"
@ -65,3 +60,6 @@ path = "src/bin/job_worker.rs"
name = "myelin-job"
path = "src/bin/job.rs"
[lints]
workspace = true

54
apps/myelin/node-image/local_docker_e2e.sh
View file

@ -1,54 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
BASE_IMAGE=${BASE_IMAGE:-myelin-node-base:cuda12.6}
IMAGE=${IMAGE:-myelin-node:latest}
CONTAINER=${CONTAINER:-myelin-node-e2e-$$}
GPUS=${MYELIN_CUDA_GPUS:-all}
PROMPT=${MYELIN_NODE_SELF_TEST_PROMPT:-ping}
TIMEOUT_SECS=${MYELIN_NODE_E2E_TIMEOUT_SECS:-1800}
FRAME_LOG=${MYELIN_TELEMETRY_FRAME_LOG:-/var/log/myelin-telemetry.ndjson}
cleanup() {
docker rm -f "$CONTAINER" >/dev/null 2>&1 || true
}
trap cleanup EXIT
cargo build --release -p myelin --bin myelin-worker
docker build -f apps/myelin/node-image/Dockerfile.base -t "$BASE_IMAGE" .
docker build -f apps/myelin/node-image/Dockerfile --build-arg BASE_IMAGE="$BASE_IMAGE" -t "$IMAGE" .
docker run -d \
--name "$CONTAINER" \
--gpus "$GPUS" \
-e MYELIN_NODE_SELF_TEST_PROMPT="$PROMPT" \
-e MYELIN_NODE_MAX_RUNTIME_SECS=1 \
-e MYELIN_SELF_TEST_MAX_TOKENS="${MYELIN_SELF_TEST_MAX_TOKENS:-1}" \
-e MYELIN_MODEL_CACHE_DIR=/var/cache/myelin-models \
-e MYELIN_TELEMETRY_FRAME_LOG="$FRAME_LOG" \
${HF_TOKEN:+-e HF_TOKEN="$HF_TOKEN"} \
"$IMAGE" >/dev/null
deadline=$((SECONDS + TIMEOUT_SECS))
while (( SECONDS < deadline )); do
logs=$(docker logs "$CONTAINER" 2>&1 || true)
if grep -q '"type":"ready"' <<<"$logs" && grep -q '"type":"self_test_completed"' <<<"$logs"; then
frames=$(docker exec "$CONTAINER" cat "$FRAME_LOG" 2>/dev/null || true)
if grep -q '"channel":"myelin.node.ready"' <<<"$frames" &&
grep -q '"channel":"myelin.worker.weights"' <<<"$frames" &&
grep -q '"channel":"myelin.worker.prompt"' <<<"$frames"; then
printf '%s\n' "$logs"
printf '%s\n' "$frames"
exit 0
fi
fi
if grep -q 'WorkerFatal\|myelin-node: .*failed\|ModelLoadFailed\|GgufDownloadFailed' <<<"$logs"; then
printf '%s\n' "$logs" >&2
exit 1
fi
sleep 5
done
docker logs "$CONTAINER" 2>&1 || true
echo "myelin-node Docker E2E timed out after ${TIMEOUT_SECS}s" >&2
exit 1

5
apps/myelin/proptest-regressions/tests/e2e_vastai.txt
View file

@ -1,5 +0,0 @@
# Seeds for stateful VastAI E2E failures. Proptest replays these before generated cases.
cc 26c3cd5944cb25de70a984389373e7380a0c6e6ba1c8f3e6eff5a0f78dbef168 # current-strategy replay; E2eCase is derived from e2e_case()
cc 5efb0a6c5c348475c859a2c4267d5177aaecd5adefcc5e8b7fc2e6d728deecd1 # shrinks to case = E2eCase { seed: 7099259626237328177, node_seed: 245, kill_mask: 24, offer_offset: 3, actions: [ConcurrentQueries, Search { count: 2 }, Restart { mode: FlushSafeAbrupt }, Query, Kill { node_slot: 218, command_slot: 5 }, Query, Provision { command_slot: 5, use_searched_offers: true }, EndpointProbe { node_slot: 217 }, Restart { mode: Graceful }, Flush, Kill { node_slot: 217, command_slot: 5 }] }
cc 6d005687858520ed65af20ec5b2de056efcda15771b6420d7f67e3cb6c246d5b # shrinks to case = E2eCase { seed: 17649392557414661864, node_seed: 249, kill_mask: 159, offer_offset: 5, actions: [Kill { node_slot: 88, command_slot: 242 }, Flush, ConcurrentQueries, EndpointProbe { node_slot: 88 }, Query, Search { count: 2 }, Provision { command_slot: 242, use_searched_offers: true }, Restart { mode: FlushSafeAbrupt }, Kill { node_slot: 89, command_slot: 242 }, Query, Restart { mode: Graceful }] }
cc 2e36f51e7cb347b571ac838f81638748f8414a32ec91cb9f4b2730363d72be61 # shrinks to case = E2eCase { seed: 13301618846512983428, node_seed: 99, kill_mask: 30, offer_offset: 2, actions: [Query, Restart { mode: FlushSafeAbrupt }, Kill { node_slot: 136, command_slot: 187 }, ConcurrentQueries, Restart { mode: Graceful }, Query, EndpointProbe { node_slot: 136 }, Kill { node_slot: 137, command_slot: 187 }, Provision { command_slot: 187, use_searched_offers: false }, Flush, Search { count: 0 }] }

197
apps/myelin/src/data_namespace.rs Normal file
View file

@ -0,0 +1,197 @@
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
use data_plane::blob_transfer::BlobTransferSender;
use data_plane::control::{DataNamespaceService, DataPlaneControl};
use data_plane::namespace::{NamespaceClient, NamespaceClientActor, NamespaceDiscovery};
use data_plane::source::BlobSourcePublisher;
use distribution::directory_actor::DirectoryIn;
use distribution::registry_actor::{RegistryIn, RegistryView};
use iroh_driver::{ActorRegistrar, IrohBlobTransferSender, IrohDriver};
use swactor::actor::{ActorAddress, ActorInterface, Ctx};
use swactor::runtime::Runtime;
use crate::orchestration::distribution_stack::DistributionRuntimeStack;
pub(crate) const DATA_DIRECTORY_SERVICE: &str = "swactor.data-directory";
const NAMESPACE_RETRY: Duration = Duration::from_millis(100);
#[derive(Clone)]
struct LocalActorPublisher {
runtime: Runtime,
distribution_directory: ActorAddress,
registrar: ActorRegistrar,
}
impl LocalActorPublisher {
fn publish(&self, actor: ActorAddress) -> Result<(), String> {
let claim = self.registrar.register_actor(actor, 1);
self.runtime
.send_to(self.distribution_directory, DirectoryIn::Register(claim))
.map_err(|error| error.to_string())
}
}
impl BlobSourcePublisher for LocalActorPublisher {
fn publish_source(&self, source: ActorAddress) -> Result<(), String> {
self.publish(source)
}
}
struct RegistryNamespaceDiscovery {
view: RegistryView,
}
impl NamespaceDiscovery for RegistryNamespaceDiscovery {
fn current_directory(&self) -> Option<ActorAddress> {
self.view
.read()
.expect("registry view poisoned")
.entries
.iter()
.find(|entry| entry.name == DATA_DIRECTORY_SERVICE && !entry.tombstone)
.map(|entry| entry.actor_addr)
}
}
#[derive(Clone)]
enum NamespaceServicePublisherIn {
Tick,
}
struct NamespaceServicePublisher {
registry: ActorAddress,
view: RegistryView,
directory: ActorAddress,
}
impl ActorInterface for NamespaceServicePublisher {
type Incoming = NamespaceServicePublisherIn;
type Response = ();
fn handle(&mut self, ctx: &Ctx<'_>, message: NamespaceServicePublisherIn) {
match message {
NamespaceServicePublisherIn::Tick => {
let published = self
.view
.read()
.expect("registry view poisoned")
.entries
.iter()
.any(|entry| {
entry.name == DATA_DIRECTORY_SERVICE
&& !entry.tombstone
&& entry.actor_addr == self.directory
});
if !published {
let _ = ctx.send(
self.registry,
RegistryIn::RegisterName {
name: DATA_DIRECTORY_SERVICE.to_owned(),
actor_addr: self.directory,
},
);
}
}
}
}
}
pub(crate) struct DataNamespaceAuthority {
service: DataNamespaceService,
}
impl DataNamespaceAuthority {
pub(crate) fn start(
stack: &DistributionRuntimeStack,
driver: &IrohDriver,
state_path: impl AsRef<Path>,
) -> Result<Self, String> {
let runtime = stack.runtime.clone();
let source_sender: Arc<dyn BlobTransferSender> = Arc::new(IrohBlobTransferSender::new(
driver.edge_connector(),
&stack.engine,
));
let publisher = Arc::new(LocalActorPublisher {
runtime: runtime.clone(),
distribution_directory: stack.actors.directory,
registrar: driver.actor_registrar(),
});
let source_publisher: Arc<dyn BlobSourcePublisher> = publisher.clone();
let service = DataNamespaceService::recover(
runtime.clone(),
state_path,
source_sender,
source_publisher,
)
.map_err(|error| format!("recover data namespace: {error}"))?;
let directory = service.directory();
publisher.publish(directory)?;
runtime
.send_to(
stack.actors.registry,
RegistryIn::RegisterName {
name: DATA_DIRECTORY_SERVICE.to_owned(),
actor_addr: directory,
},
)
.map_err(|error| format!("publish data directory service: {error}"))?;
let service_publisher = runtime
.spawn(NamespaceServicePublisher {
registry: stack.actors.registry,
view: Arc::clone(&stack.registry_view),
directory,
})
.map_err(|error| format!("spawn namespace service publisher: {error}"))?;
stack.engine.send_every(
Duration::from_secs(1),
runtime.create_sender(),
service_publisher,
NamespaceServicePublisherIn::Tick,
);
Ok(Self { service })
}
#[cfg(test)]
pub(crate) fn directory(&self) -> ActorAddress {
self.service.directory()
}
pub(crate) fn control(&self) -> DataPlaneControl {
self.service.control()
}
}
pub(crate) struct InstalledNamespaceClient {
pub(crate) client: NamespaceClient,
pub(crate) source_publisher: Arc<dyn BlobSourcePublisher>,
}
pub(crate) fn install_namespace_client(
stack: &DistributionRuntimeStack,
driver: &IrohDriver,
) -> Result<InstalledNamespaceClient, String> {
let discovery: Arc<dyn NamespaceDiscovery> = Arc::new(RegistryNamespaceDiscovery {
view: Arc::clone(&stack.registry_view),
});
let proxy = stack
.runtime
.spawn(NamespaceClientActor::new(
stack.engine.clone(),
stack.runtime.create_sender(),
discovery,
NAMESPACE_RETRY,
))
.map_err(|error| format!("spawn namespace client: {error}"))?;
stack.register_local_actor(driver.register_actor(proxy, 1));
let source_publisher: Arc<dyn BlobSourcePublisher> = Arc::new(LocalActorPublisher {
runtime: stack.runtime.clone(),
distribution_directory: stack.actors.directory,
registrar: driver.actor_registrar(),
});
Ok(InstalledNamespaceClient {
client: NamespaceClient::new(stack.runtime.clone(), proxy),
source_publisher,
})
}

105
apps/myelin/src/job_data_plane.rs
View file

@ -2,14 +2,15 @@ use std::collections::BTreeMap;
use std::sync::Arc;
use data_plane::arena::{ArenaConfig, ArenaManager, NodeId as ArenaNodeId};
use data_plane::blob::{BlobMetadata, ContentDigest};
use data_plane::blob_transfer::{BlobTransferReceiver, BlobTransferSender};
use data_plane::bootstrap::{self, BootstrapSpec, ENV_DATA_PLANE_ENDPOINT, JobHandoff};
use data_plane::host::{
BlobSource, HostDataPlaneConfig, HostDataPlaneSessionActor, HostRouteRegistrar,
install_session_env,
HostDataPlaneConfig, HostDataPlaneSessionActor, HostRouteRegistrar, install_session_env,
};
use data_plane::path::{DataPath, JobContext};
use data_plane::namespace::NamespaceClient;
use data_plane::path::JobContext;
use data_plane::protocol::JobCapability;
use data_plane::source::BlobSourcePublisher;
use distribution::transport_bridge::{OutboxRouteBinder, RouteBinder, RouteView};
use distribution::types::NodeId;
use swactor::actor::ActorAddress;
@ -78,17 +79,33 @@ pub(crate) struct ActorJobDataPlane {
runtime: Runtime,
}
pub(crate) struct ActorJobDataPlaneConfig {
pub(crate) arena_bytes: u64,
pub(crate) arena_generation: u64,
pub(crate) session_generation: u64,
pub(crate) capability: JobCapability,
pub(crate) job_context: JobContext,
pub(crate) namespace: Option<NamespaceClient>,
pub(crate) transfer_receiver: Option<Arc<dyn BlobTransferReceiver>>,
pub(crate) source_sender: Option<Arc<dyn BlobTransferSender>>,
pub(crate) source_publisher: Option<Arc<dyn BlobSourcePublisher>>,
pub(crate) route_registrar: Option<Arc<dyn HostRouteRegistrar>>,
}
impl ActorJobDataPlane {
pub(crate) fn new(
runtime: &Runtime,
arena_bytes: u64,
arena_generation: u64,
session_generation: u64,
capability: JobCapability,
job_context: JobContext,
blobs: BTreeMap<DataPath, BlobSource>,
route_registrar: Option<Arc<dyn HostRouteRegistrar>>,
) -> Result<Self, String> {
pub(crate) fn new(runtime: &Runtime, config: ActorJobDataPlaneConfig) -> Result<Self, String> {
let ActorJobDataPlaneConfig {
arena_bytes,
arena_generation,
session_generation,
capability,
job_context,
namespace,
transfer_receiver,
source_sender,
source_publisher,
route_registrar,
} = config;
let mut arena = ArenaManager::boot(ArenaConfig {
node_id: ArenaNodeId(1),
reservation_ceiling: arena_bytes,
@ -106,12 +123,16 @@ impl ActorJobDataPlane {
let host_session = runtime
.spawn(
HostDataPlaneSessionActor::new(HostDataPlaneConfig {
runtime: runtime.clone(),
arena,
arena_generation,
session_generation,
capability,
job_context,
blobs,
namespace,
transfer_receiver,
source_sender,
source_publisher,
route_registrar,
})
.map_err(|error| format!("configure host data-plane session: {error}"))?,
@ -141,55 +162,6 @@ impl ActorJobDataPlane {
})
}
pub(crate) fn begin_blob_source(
&self,
path: DataPath,
length: u64,
digest: Option<ContentDigest>,
) -> Result<(), String> {
futures_lite::future::block_on(async {
self.runtime
.ask::<data_plane::protocol::HostSessionIn, Result<(), data_plane::protocol::DataPlaneError>>(
self.host_session,
|reply_to| data_plane::protocol::HostSessionIn::BeginBlobSource {
path,
metadata: BlobMetadata { length, digest },
reply_to,
},
)
.map_err(|error| format!("begin data-plane blob source: {error}"))?
.await
.map_err(|error| format!("begin data-plane blob source: {error}"))
})
}
pub(crate) fn push_blob_chunk(&self, path: DataPath, bytes: Vec<u8>) -> Result<(), String> {
self.runtime
.send_to(
self.host_session,
data_plane::protocol::HostSessionIn::BlobSourceChunk { path, bytes },
)
.map_err(|error| format!("publish data-plane blob chunk: {error}"))
}
pub(crate) fn finish_blob_source(&self, path: DataPath) -> Result<(), String> {
self.runtime
.send_to(
self.host_session,
data_plane::protocol::HostSessionIn::FinishBlobSource { path },
)
.map_err(|error| format!("finish data-plane blob source: {error}"))
}
pub(crate) fn fail_blob_source(&self, path: DataPath, reason: String) -> Result<(), String> {
self.runtime
.send_to(
self.host_session,
data_plane::protocol::HostSessionIn::FailBlobSource { path, reason },
)
.map_err(|error| format!("fault data-plane blob source: {error}"))
}
pub(crate) fn close(&self) {
let _ = self.runtime.send_to(
self.host_session,
@ -206,11 +178,6 @@ impl ActorJobDataPlane {
env
}
#[cfg(test)]
pub(crate) fn host_session(&self) -> ActorAddress {
self.host_session
}
#[cfg(test)]
pub(crate) fn arena_fd(&self) -> std::os::fd::RawFd {
use std::os::fd::AsRawFd;

142
apps/myelin/src/job_deploy.rs
View file

@ -4,6 +4,9 @@
//! in the directory, and exchanges its `EndpointAddr` + actor address
//! out-of-band so each side can route to the other over the iroh actor plane.
use crate::job_data_plane::{
ActorJobDataPlane, ActorJobDataPlaneConfig, MyelinChildRouteRegistrar,
};
use parking_lot::Mutex;
use std::collections::BTreeMap;
use std::env;
@ -23,11 +26,12 @@ use swactor_job_runner::{
};
use swactor_transport::hex_encode;
use data_plane::blob::BlobMetadata;
use data_plane::blob_transfer::{BlobTransferReceiver, BlobTransferSender};
use data_plane::edge_wire::WireEvent;
use data_plane::host::BlobSource;
use data_plane::namespace::NamespaceClient;
use data_plane::path::{DataPath, JobContext};
use data_plane::protocol::{JobCapability, register_data_plane_codecs};
use data_plane::source::BlobSourcePublisher;
use distribution::node::DistributedNodeConfig;
use iroh::{EndpointAddr, RelayMode};
use iroh_driver::{
@ -37,7 +41,6 @@ use iroh_driver::{
use tokio::io::AsyncReadExt;
use tokio::sync::Notify;
use crate::job_data_plane::{ActorJobDataPlane, MyelinChildRouteRegistrar};
use crate::orchestration::distribution_stack::DistributionRuntimeStack;
const POLL: Duration = Duration::from_millis(25);
@ -51,24 +54,15 @@ const JOB_OUTPUT_SOCKET: &str = "inference-results.sock";
const DATA_PLANE_CONNECT_DEADLINE: Duration = Duration::from_secs(30);
const JOB_ARENA_BYTES: u64 = 1 << 20;
#[derive(Clone, Debug, Serialize, Deserialize)]
struct InputBlobAssignment {
edge_id: u64,
path: DataPath,
metadata: BlobMetadata,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
struct EmbeddedDataPlaneAssignment {
result_endpoint: EndpointAddr,
input_blobs: Vec<InputBlobAssignment>,
}
/// Actor-driven finite-blob ingress plus the retained temporary Unix output
/// stream bridge. Remote bytes remain on `EDGE_ALPN`.
#[derive(Clone)]
pub(crate) struct EmbeddedJobDataPlane {
input_paths: Arc<Mutex<BTreeMap<u64, DataPath>>>,
result_sink: Arc<Mutex<Option<EdgeSendHandle>>>,
result_ready: Arc<Notify>,
connector: EdgeConnector,
@ -77,14 +71,32 @@ pub(crate) struct EmbeddedJobDataPlane {
output_path: PathBuf,
}
pub(crate) struct EmbeddedJobDataPlaneConfig<'a> {
pub(crate) engine: EngineHandle,
pub(crate) connector: EdgeConnector,
pub(crate) root: &'a Path,
pub(crate) host_endpoint: EndpointAddr,
pub(crate) namespace: NamespaceClient,
pub(crate) transfer_receiver: Arc<dyn BlobTransferReceiver>,
pub(crate) source_sender: Arc<dyn BlobTransferSender>,
pub(crate) source_publisher: Arc<dyn BlobSourcePublisher>,
}
impl EmbeddedJobDataPlane {
pub(crate) fn start(
engine: EngineHandle,
connector: EdgeConnector,
root: &Path,
stack: &DistributionRuntimeStack,
host_endpoint: EndpointAddr,
config: EmbeddedJobDataPlaneConfig<'_>,
) -> Result<Self, String> {
let EmbeddedJobDataPlaneConfig {
engine,
connector,
root,
host_endpoint,
namespace,
transfer_receiver,
source_sender,
source_publisher,
} = config;
std::fs::create_dir_all(root)
.map_err(|error| format!("create job data-plane root {}: {error}", root.display()))?;
let output_path = root.join(JOB_OUTPUT_SOCKET);
@ -97,22 +109,26 @@ impl EmbeddedJobDataPlane {
));
let actor_plane = Arc::new(ActorJobDataPlane::new(
&stack.runtime,
JOB_ARENA_BYTES,
1,
1,
capability,
JobContext {
run_id: "unconfigured".to_owned(),
read_prefixes: vec![DataPath::parse("/models").expect("static model prefix")],
write_prefixes: vec![DataPath::parse("/runs").expect("static run prefix")],
ActorJobDataPlaneConfig {
arena_bytes: JOB_ARENA_BYTES,
arena_generation: 1,
session_generation: 1,
capability,
job_context: JobContext {
run_id: "unconfigured".to_owned(),
read_prefixes: vec![DataPath::parse("/models").expect("static model prefix")],
write_prefixes: vec![DataPath::parse("/runs").expect("static run prefix")],
},
namespace: Some(namespace),
transfer_receiver: Some(transfer_receiver),
source_sender: Some(source_sender),
source_publisher: Some(source_publisher),
route_registrar: Some(route_registrar),
},
BTreeMap::<DataPath, BlobSource>::new(),
Some(route_registrar),
)?);
let host_endpoint_json = serde_json::to_string(&host_endpoint)
.map_err(|error| format!("serialize host data-plane endpoint: {error}"))?;
let input_paths = Arc::new(Mutex::new(BTreeMap::new()));
let result_sink: Arc<Mutex<Option<EdgeSendHandle>>> = Arc::new(Mutex::new(None));
let result_ready = Arc::new(Notify::new());
@ -147,7 +163,6 @@ impl EmbeddedJobDataPlane {
.map_err(|error| format!("bind job data-plane output: {error}"))?;
Ok(Self {
input_paths,
result_sink,
result_ready,
connector,
@ -156,40 +171,6 @@ impl EmbeddedJobDataPlane {
host_endpoint_json,
})
}
pub(crate) fn drain_input_events(&self, events: &Arc<Mutex<Vec<WireEvent>>>) {
let input_paths = self.input_paths.lock().clone();
let mut events = events.lock();
let mut remaining = Vec::with_capacity(events.len());
for event in events.drain(..) {
match event {
WireEvent::BytesRead { edge_id, bytes, .. }
if input_paths.contains_key(&edge_id.0) =>
{
let path = input_paths[&edge_id.0].clone();
let _ = self.actor_plane.push_blob_chunk(path, bytes);
}
WireEvent::StreamEnded { edge_id, .. } if input_paths.contains_key(&edge_id.0) => {
let path = input_paths[&edge_id.0].clone();
let _ = self.actor_plane.finish_blob_source(path);
}
WireEvent::StreamFault {
edge_id: Some(edge_id),
reason,
..
} if input_paths.contains_key(&edge_id.0) => {
let path = input_paths[&edge_id.0].clone();
let _ = self
.actor_plane
.fail_blob_source(path, format!("{reason:?}"));
}
WireEvent::StreamArrived { edge_id, .. }
if input_paths.contains_key(&edge_id.0) => {}
event => remaining.push(event),
}
}
events.extend(remaining);
}
}
impl JobDataPlanePort for EmbeddedJobDataPlane {
@ -208,24 +189,6 @@ impl JobDataPlanePort for EmbeddedJobDataPlane {
*self.result_sink.lock() = Some(sink);
self.result_ready.notify_one();
self.actor_plane.configure_run(job_id.to_string())?;
let mut input_paths = BTreeMap::new();
for input in assignment.input_blobs {
if input_paths
.insert(input.edge_id, input.path.clone())
.is_some()
{
return Err(format!(
"duplicate input blob edge id {} in job assignment",
input.edge_id
));
}
self.actor_plane.begin_blob_source(
input.path,
input.metadata.length,
input.metadata.digest,
)?;
}
*self.input_paths.lock() = input_paths;
let mut env = self.actor_plane.handoff_env(&self.host_endpoint_json);
env.insert(
"SWACTOR_DATA_PLANE_OUTPUT".to_owned(),
@ -235,7 +198,6 @@ impl JobDataPlanePort for EmbeddedJobDataPlane {
}
fn session_ended(&self, _job_id: u64) {
self.input_paths.lock().clear();
self.result_sink.lock().take();
self.actor_plane.close();
}
@ -307,15 +269,15 @@ fn build_composition_with_relay(relay_mode: RelayMode) -> Result<JobComposition,
DistributedNodeConfig::default(),
engine.handle(),
);
driver.enable_actor_bridge(
stack.runtime.clone(),
stack.codec.clone(),
stack.actor_bridge_routes(),
stack.actors.swim,
stack.relay_mirror.clone(),
stack.route_view.clone(),
stack.outbox.clone(),
);
driver.enable_actor_bridge(iroh_driver::ActorBridgeConfig {
runtime: stack.runtime.clone(),
codec: stack.codec.clone(),
routes: stack.actor_bridge_routes(),
swim: stack.actors.swim,
relay_mirror: stack.relay_mirror.clone(),
route_view: stack.route_view.clone(),
outbox: stack.outbox.clone(),
});
stack.spawn_protocol_ticker(POLL);
driver.install_actor_bridge_pump(POLL);
Ok((engine, driver, stack))

43
apps/myelin/src/lib.rs
View file

@ -1,8 +1,5 @@
// Engine boundary enforcement: disallowed scheduling/time/core-driving methods
// are hard errors in this crate (ENGINE_SPEC.md §2). The VastAI
// provider adapter carries a module-level `#![allow]` pending its separate
// redesign; unit tests that drive a raw Runtime in isolation are exempted
// locally.
// Engine boundary enforcement: disallowed scheduling, time, and core-driving
// methods are hard errors in this crate (ENGINE_SPEC.md §2).
#![deny(clippy::disallowed_methods)]
#![recursion_limit = "256"]
@ -23,8 +20,9 @@ pub fn run_worker_node_from_env() -> std::process::ExitCode {
node::worker_node_runtime::run_from_env()
}
mod job_deploy;
mod data_namespace;
mod job_data_plane;
mod job_deploy;
/// `myelin-job-worker` — GPU-node side of the iroh job runner.
pub fn run_job_worker_from_args<I>(args: I) -> std::process::ExitCode
@ -84,16 +82,17 @@ where
"--provider" => provider = it.next(),
"--image" | "--node-image" => vastai.image = it.next(),
"--vastai-api-key" => vastai.api_key = it.next(),
"--vastai-ssh-identity" => match it.next() {
Some(path) => match orchestration::app::expand_home_path(&path) {
Ok(path) => vastai.ssh_identity = Some(path),
Err(error) => {
eprintln!("myelin-job: {error}");
return std::process::ExitCode::from(2);
"--vastai-ssh-identity" => {
if let Some(path) = it.next() {
match orchestration::app::expand_home_path(&path) {
Ok(path) => vastai.ssh_identity = Some(path),
Err(error) => {
eprintln!("myelin-job: {error}");
return std::process::ExitCode::from(2);
}
}
},
None => {}
},
}
}
"--remote-worker-bin" => {
vastai.remote_worker_bin = it.next().unwrap_or_default();
}
@ -227,12 +226,14 @@ where
fn vastai_job_options_from_env() -> Result<orchestration::job_reconciler::VastAiJobOptions, String>
{
let mut options = orchestration::job_reconciler::VastAiJobOptions::default();
options.api_key = first_env(["VAST_API_KEY", "MYELIN_VASTAI_API_KEY", "VASTAI_API_KEY"]);
options.image = env_optional("MYELIN_NODE_IMAGE");
options.ssh_identity = env_optional("MYELIN_VASTAI_SSH_IDENTITY")
.map(|path| orchestration::app::expand_home_path(&path))
.transpose()?;
let mut options = orchestration::job_reconciler::VastAiJobOptions {
api_key: first_env(["VAST_API_KEY", "MYELIN_VASTAI_API_KEY", "VASTAI_API_KEY"]),
image: env_optional("MYELIN_NODE_IMAGE"),
ssh_identity: env_optional("MYELIN_VASTAI_SSH_IDENTITY")
.map(|path| orchestration::app::expand_home_path(&path))
.transpose()?,
..orchestration::job_reconciler::VastAiJobOptions::default()
};
apply_env("MYELIN_JOB_REMOTE_WORKER_BIN", |value| {
options.remote_worker_bin = value;
Ok(())

13
apps/myelin/src/node/actor.rs
View file

@ -129,7 +129,7 @@ impl StageProvisionWire {
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) enum NodeAgentMsg {
ProvisionStage(StageProvisionWire),
ProvisionStage(Box<StageProvisionWire>),
MarkWorkerReady,
RuntimeLoaded {
run_id: u64,
@ -296,7 +296,7 @@ pub(crate) enum StageLifecycleWire {
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) enum NodeAgentReport {
Command(StageCommandWire),
Command(Box<StageCommandWire>),
Lifecycle(StageLifecycleWire),
PromptRequested {
request_id: u64,
@ -455,7 +455,7 @@ impl NodeAgentActor {
NodeAgentMsg::ProvisionStage(provision) => {
self.core.observe(stage::StageEvent::ProvisionStage {
from: stage::NodeId(provision.authorized_orchestrator),
provision: provision.to_core(),
provision: Box::new(provision.to_core()),
});
self.inbound_edge = provision.inbound_edge;
self.outbound_edge = provision.outbound_edge;
@ -615,7 +615,10 @@ impl NodeAgentActor {
fn drain_outputs(&mut self, ctx: &Ctx) {
for command in &self.core.commands()[self.command_cursor..] {
self.report(ctx, NodeAgentReport::Command(self.command_wire(command)));
self.report(
ctx,
NodeAgentReport::Command(Box::new(self.command_wire(command))),
);
}
self.command_cursor = self.core.commands().len();
@ -741,7 +744,7 @@ impl From<&stage::StageCommand> for StageCommandWire {
tokenizer: source.tokenizer.clone(),
layer_start: range.start,
layer_end_exclusive: range.end_exclusive,
stage_shard_plan: shard_plan.clone(),
stage_shard_plan: shard_plan.as_ref().clone(),
},
stage::StageCommand::ExecuteStep(step) => Self::ExecuteStep {
step_id: step.step_id.0,

1109
apps/myelin/src/node/worker_node_runtime.rs
View file

File diff suppressed because it is too large Load diff

15
apps/myelin/src/observability/frame_collector.rs
View file

@ -7,6 +7,7 @@
//! only through these closures.
use iroh::EndpointAddr;
use iroh_driver::telemetry_transport::PullCollectorConfig;
use iroh_driver::{IrohDriver, PullCollectorHandle, TelemetryQuicHeader, spawn_pull_collector};
use parking_lot::Mutex;
use std::collections::{BTreeMap, BTreeSet};
@ -82,12 +83,14 @@ impl FrameCollector {
flow_id[8..].copy_from_slice(&node_id.to_le_bytes());
let collector = spawn_pull_collector(
engine,
endpoint,
peer,
flow_id,
Vec::new(),
SubscriptionRequest::all(),
Arc::clone(&self.pull_fanout),
PullCollectorConfig {
endpoint,
peer,
flow_id,
token: Vec::new(),
request: SubscriptionRequest::all(),
fanout: Arc::clone(&self.pull_fanout),
},
self.pull_header_tx.clone(),
);
if let Some(previous) = self

39
apps/myelin/src/observability/orch_telemetry.rs
View file

@ -43,6 +43,15 @@ pub(crate) struct OrchTelemetry {
descriptor: StreamDescriptor,
}
pub(crate) struct BootstrapEmission<'a> {
pub(crate) dashboard: Option<&'a DashboardSupport>,
pub(crate) channel: &'a str,
pub(crate) run_id: u64,
pub(crate) node_id: u64,
pub(crate) phase: &'a str,
pub(crate) status: &'a str,
pub(crate) detail: Value,
}
impl OrchTelemetry {
pub(crate) fn new(run_id: u64, frame_log: Option<&Path>) -> Result<Self, String> {
let stream = StreamId::new(NodeId::new("myelin-orchestrator"), Lifetime(run_id));
@ -110,6 +119,10 @@ impl OrchTelemetry {
id
}
pub(crate) fn producer(&self) -> TelemetryProducer {
self.producer.clone()
}
pub(crate) fn emit_event(
&mut self,
dashboard: Option<&DashboardSupport>,
@ -140,27 +153,27 @@ impl OrchTelemetry {
status: &str,
detail: Value,
) {
self.emit_bootstrap_to_channel(
self.emit_bootstrap_to_channel(BootstrapEmission {
dashboard,
MYELIN_ORCH_BOOTSTRAP,
channel: MYELIN_ORCH_BOOTSTRAP,
run_id,
node_id,
phase,
status,
detail,
);
});
}
pub(crate) fn emit_bootstrap_to_channel(
&mut self,
dashboard: Option<&DashboardSupport>,
channel: &str,
run_id: u64,
node_id: u64,
phase: &str,
status: &str,
detail: Value,
) {
pub(crate) fn emit_bootstrap_to_channel(&mut self, emission: BootstrapEmission<'_>) {
let BootstrapEmission {
dashboard,
channel,
run_id,
node_id,
phase,
status,
detail,
} = emission;
let benchmark = benchmark::stamp("myelin-orchestrator");
let payload = serde_json::to_vec(&json!({
"schema_version": benchmark["schema_version"].clone(),

2
apps/myelin/src/observability/provisioning_logs.rs
View file

@ -10,7 +10,7 @@ use crate::provisioning::{
};
pub(crate) fn node_stream_id(run_id: u64, node_id: u64) -> StreamId {
StreamId::new(NodeId::new(&node_id.to_string()), Lifetime(run_id))
StreamId::new(NodeId::new(node_id.to_string()), Lifetime(run_id))
}
#[derive(Clone)]

22
apps/myelin/src/orchestration/actor.rs
View file

@ -139,10 +139,10 @@ pub(crate) enum RunCommandWire {
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) enum LifecycleEventWire {
RunFaulted { run_id: u64 },
RunCompleted { run_id: u64 },
RunOperatorStopped { run_id: u64 },
RunTornDown { run_id: u64 },
Faulted { run_id: u64 },
Completed { run_id: u64 },
OperatorStopped { run_id: u64 },
TornDown { run_id: u64 },
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@ -512,16 +512,12 @@ impl From<&core::RunCommand> for RunCommandWire {
impl From<&core::LifecycleEvent> for LifecycleEventWire {
fn from(event: &core::LifecycleEvent) -> Self {
match event {
core::LifecycleEvent::RunFaulted { run_id, .. } => {
Self::RunFaulted { run_id: run_id.0 }
core::LifecycleEvent::Faulted { run_id, .. } => Self::Faulted { run_id: run_id.0 },
core::LifecycleEvent::Completed { run_id } => Self::Completed { run_id: run_id.0 },
core::LifecycleEvent::OperatorStopped { run_id } => {
Self::OperatorStopped { run_id: run_id.0 }
}
core::LifecycleEvent::RunCompleted { run_id } => {
Self::RunCompleted { run_id: run_id.0 }
}
core::LifecycleEvent::RunOperatorStopped { run_id } => {
Self::RunOperatorStopped { run_id: run_id.0 }
}
core::LifecycleEvent::RunTornDown { run_id } => Self::RunTornDown { run_id: run_id.0 },
core::LifecycleEvent::TornDown { run_id } => Self::TornDown { run_id: run_id.0 },
}
}
}

248
apps/myelin/src/orchestration/app.rs
View file

@ -12,16 +12,16 @@ use crate::DEFAULT_PIPELINE_CACHED_MODEL_FILE;
use crate::codecs::register_myelin_actor_codecs;
use crate::observability::frame_collector::FrameCollector;
use crate::observability::orch_telemetry::{
DashboardSupport, MYELIN_SWIM_MEMBERSHIP, OrchTelemetry,
BootstrapEmission, DashboardSupport, MYELIN_SWIM_MEMBERSHIP, OrchTelemetry,
};
use crate::orchestration::actor::{OrchestratorActor, OrchestratorMsg, OrchestratorReport};
use crate::orchestration::config::{DEFAULT_CONFIG_PATH, TomlConfigOverlay};
use crate::orchestration::control;
use crate::orchestration::daemon;
use crate::orchestration::manual_control::{
CONTROL_REGISTRY_NAME, ConfigValidator, ManualActorControl, ManualControl, ManualControlMsg,
ManualControlReply, NodePhase, OfferDto, OfferSearchRequest, OfferSearcher,
ProviderConfigurationRequest, ProviderFactory, ProviderReadiness, SpecBuilder,
CONTROL_REGISTRY_NAME, ConfigValidator, ManualActorControl, ManualActorControlConfig,
ManualControl, ManualControlMsg, ManualControlReply, NodePhase, OfferDto, OfferSearchRequest,
OfferSearcher, ProviderConfigurationRequest, ProviderFactory, ProviderReadiness, SpecBuilder,
};
use swactor_job_runner::{JobDone, OrchestratorJobActor};
@ -69,6 +69,154 @@ const PUMP_INTERVAL: Duration = Duration::from_millis(10);
const SHUTDOWN_FLUSH_TIMEOUT: Duration = Duration::from_secs(10);
const TELEMETRY_FRAME_LOG_ENV: &str = "MYELIN_TELEMETRY_FRAME_LOG";
fn sample_orchestrator_cpu(
mut sampler: telemetry::hardware::cpu::CpuSampler,
seq: u64,
) -> (
telemetry::hardware::cpu::CpuSampler,
telemetry::hardware::cpu::HostCpuSample,
) {
let sample = sampler.sample(seq);
(sampler, sample)
}
fn sample_orchestrator_gpu((): (), seq: u64) -> ((), telemetry::hardware::gpu::HostGpuSample) {
((), telemetry::hardware::gpu::sample(seq))
}
fn sample_orchestrator_memory(
(): (),
seq: u64,
) -> ((), telemetry::hardware::memory::HostMemorySample) {
((), telemetry::hardware::memory::sample(seq))
}
fn sample_orchestrator_net((): (), seq: u64) -> ((), telemetry::hardware::net::HostNetSample) {
((), telemetry::hardware::net::sample(seq))
}
fn sample_orchestrator_storage(
(): (),
seq: u64,
) -> ((), telemetry::hardware::storage::HostStorageSample) {
((), telemetry::hardware::storage::sample(seq))
}
fn spawn_orchestrator_hardware_samplers(engine: &EngineHandle, telemetry: &mut OrchTelemetry) {
let cpu_channel = telemetry.record_channel::<telemetry::hardware::cpu::HostCpuSample>();
let cpu_producer = telemetry.producer();
telemetry::hardware::spawn_blocking_sampler(
engine.clone(),
telemetry::hardware::cpu::CPU_SAMPLE_INTERVAL,
telemetry::hardware::cpu::CpuSampler::new([std::process::id()]),
sample_orchestrator_cpu,
|| {},
move |_, sample| {
cpu_producer.submit_record(cpu_channel, &sample);
},
);
let gpu_channel = telemetry.record_channel::<telemetry::hardware::gpu::HostGpuSample>();
let gpu_producer = telemetry.producer();
telemetry::hardware::spawn_blocking_sampler(
engine.clone(),
telemetry::hardware::gpu::GPU_SAMPLE_INTERVAL,
(),
sample_orchestrator_gpu,
|| {},
move |_, sample| {
gpu_producer.submit_record(gpu_channel, &sample);
},
);
let memory_channel =
telemetry.record_channel::<telemetry::hardware::memory::HostMemorySample>();
let memory_producer = telemetry.producer();
telemetry::hardware::spawn_blocking_sampler(
engine.clone(),
telemetry::hardware::memory::MEMORY_SAMPLE_INTERVAL,
(),
sample_orchestrator_memory,
|| {},
move |_, sample| {
memory_producer.submit_record(memory_channel, &sample);
},
);
let net_channel = telemetry.record_channel::<telemetry::hardware::net::HostNetSample>();
let net_producer = telemetry.producer();
telemetry::hardware::spawn_blocking_sampler(
engine.clone(),
telemetry::hardware::net::HOST_NET_SAMPLE_INTERVAL,
(),
sample_orchestrator_net,
|| {},
move |_, sample| {
net_producer.submit_record(net_channel, &sample);
},
);
let storage_channel =
telemetry.record_channel::<telemetry::hardware::storage::HostStorageSample>();
let storage_producer = telemetry.producer();
telemetry::hardware::spawn_blocking_sampler(
engine.clone(),
telemetry::hardware::storage::STORAGE_SAMPLE_INTERVAL,
(),
sample_orchestrator_storage,
|| {},
move |_, sample| {
storage_producer.submit_record(storage_channel, &sample);
},
);
}
#[cfg(test)]
mod hardware_telemetry_tests {
use std::time::{Duration, Instant};
use swactor::config::RuntimeConfig;
use swactor::runtime::RuntimeParts;
use super::*;
#[test]
fn orchestrator_emits_all_host_hardware_channels() {
let frame_log = tempfile::NamedTempFile::new().expect("frame log");
let mut telemetry =
OrchTelemetry::new(1, Some(frame_log.path())).expect("orchestrator telemetry");
let engine = Engine::new(
RuntimeParts::new(RuntimeConfig::default()),
TokioBackend::new(TokioConfig::default()).expect("Tokio backend"),
)
.expect("engine");
spawn_orchestrator_hardware_samplers(&engine.handle(), &mut telemetry);
let deadline = Instant::now() + Duration::from_secs(5);
loop {
telemetry.flush(None, "test");
let frames = std::fs::read_to_string(frame_log.path()).expect("read frame log");
if [
"host.cpu",
"host.gpu",
"host.memory",
"host.net",
"host.storage",
]
.iter()
.all(|channel| frames.contains(&format!(r#""channel":"{channel}""#)))
{
return;
}
assert!(
Instant::now() < deadline,
"orchestrator hardware channels were incomplete: {frames}"
);
std::thread::sleep(Duration::from_millis(10));
}
}
}
pub(crate) fn run_with_options<I>(
args: I,
capture_stdio: bool,
@ -86,6 +234,7 @@ where
.overlay_cli(args)?
.finalize()?;
let state_dir = daemon::StateDir::new(config.state_dir.clone());
let data_namespace_path = config.state_dir.join("data-namespace.json");
if config.reset_state {
state_dir.reset()?;
}
@ -224,6 +373,7 @@ where
return Err(format!("create engine: {error}"));
}
};
spawn_orchestrator_hardware_samplers(&engine.handle(), &mut orch_telemetry);
let mut driver = match IrohDriver::with_engine(
engine.handle(),
IrohDriverConfig {
@ -295,15 +445,28 @@ where
"ready",
json!({"registered":["node_agent","orchestrator","provisioner","prompt_rpc","telemetry"]}),
);
driver.enable_actor_bridge(
stack.runtime.clone(),
stack.codec.clone(),
stack.actor_bridge_routes(),
stack.actors.swim,
stack.relay_mirror.clone(),
stack.route_view.clone(),
stack.outbox.clone(),
);
driver.enable_actor_bridge(iroh_driver::ActorBridgeConfig {
runtime: stack.runtime.clone(),
codec: stack.codec.clone(),
routes: stack.actor_bridge_routes(),
swim: stack.actors.swim,
relay_mirror: stack.relay_mirror.clone(),
route_view: stack.route_view.clone(),
outbox: stack.outbox.clone(),
});
let data_namespace =
crate::data_namespace::DataNamespaceAuthority::start(&stack, &driver, data_namespace_path)?;
let tiny_linear_weights = std::env::var_os("MYELIN_TINY_LINEAR_WEIGHTS")
.map(std::path::PathBuf::from)
.unwrap_or_else(|| std::path::PathBuf::from("apps/myelin/jobs/tiny_linear.weights"));
futures_lite::future::block_on(
data_namespace.control().ensure(
data_plane::path::DataPath::parse("/models/tiny-linear/weights")
.expect("static tiny-linear data path"),
data_plane::blob::file(tiny_linear_weights),
),
)
.map_err(|error| format!("register tiny-linear weights: {error}"))?;
let recovery_nodes = snapshot
.nodes
.iter()
@ -508,13 +671,15 @@ where
ManualControl::new(snapshot, readiness),
stack.runtime.clone(),
stack.engine.blocking_work_sender(),
state_dir,
sink,
provider_factory,
spec_builder,
Some(config_validator),
offer_searcher,
daemon::unix_ms_now(),
ManualActorControlConfig {
state_dir,
sink,
provider_factory,
spec_builder,
config_validator: Some(config_validator),
offer_searcher,
control_generation: daemon::unix_ms_now(),
},
);
let orchestrator_actor = match stack.runtime.spawn(
OrchestratorActor::new(
@ -1476,10 +1641,11 @@ impl ConfigBuilder {
});
let provider_name = provider.as_str();
let mut image = self.image.clone();
if provider_name == "vastai" && !self.image_overridden_after_toml {
if let Some(vastai_image) = &self.toml_vastai_image {
image = vastai_image.clone();
}
if provider_name == "vastai"
&& !self.image_overridden_after_toml
&& let Some(vastai_image) = &self.toml_vastai_image
{
image = vastai_image.clone();
}
if self.pipeline_stages == 0 {
return Err("--pipeline-stages must be greater than 0".to_owned());
@ -2367,6 +2533,8 @@ impl ServeClusterActor {
}
telemetry.archive_frame("node", stream, channel, frame);
});
self.orch_telemetry
.flush(self.dashboard.as_ref(), "orchestrator");
self.drain_observations();
while let Some(report) = self.orchestrator_reports.try_recv() {
self.observe_report(report);
@ -2587,14 +2755,14 @@ fn emit_swim_transitions(
let last_ack_age_ms = transition.last_ack_age.map(duration_ms_u64);
let consecutive_timeouts = transition.consecutive_timeouts;
let recent_probe_targets = stack.swim_recent_probe_targets();
orch_telemetry.emit_bootstrap_to_channel(
orch_telemetry.emit_bootstrap_to_channel(BootstrapEmission {
dashboard,
MYELIN_SWIM_MEMBERSHIP,
channel: MYELIN_SWIM_MEMBERSHIP,
run_id,
node_id,
"membership_transition",
"observed",
json!({
phase: "membership_transition",
status: "observed",
detail: json!({
"peer":peer.clone(),
"from":from.clone(),
"to":to.clone(),
@ -2604,7 +2772,7 @@ fn emit_swim_transitions(
"recent_probe_targets":recent_probe_targets.clone(),
"member_state":member_state.clone(),
}),
);
});
orch_telemetry.emit_record(dashboard, &stack.membership_transition(transition));
}
transitions
@ -2656,7 +2824,7 @@ pub(crate) fn expand_home_path(value: &str) -> Result<PathBuf, String> {
pub(crate) fn derive_ssh_public_key(identity: &Path) -> Result<String, String> {
let output = swactor_process::command_output(
&mut Command::new("ssh-keygen").arg("-y").arg("-f").arg(identity),
Command::new("ssh-keygen").arg("-y").arg("-f").arg(identity),
)
.map_err(|e| {
format!(
@ -2685,10 +2853,9 @@ pub(crate) fn ssh_public_key_fingerprint(public_key: &str) -> String {
if std::fs::write(&path, format!("{public_key}\n")).is_err() {
return UNAVAILABLE.to_owned();
}
let output = swactor_process::command_output(
&mut Command::new("ssh-keygen").arg("-l").arg("-f").arg(&path),
)
.ok();
let output =
swactor_process::command_output(Command::new("ssh-keygen").arg("-l").arg("-f").arg(&path))
.ok();
let _ = std::fs::remove_file(&path);
let Some(output) = output.filter(|output| output.status.success()) else {
return UNAVAILABLE.to_owned();
@ -2704,7 +2871,7 @@ pub(crate) fn ssh_public_key_fingerprint(public_key: &str) -> String {
}
fn vastai_account_has_ssh_key(api_key: &str, public_key: &str) -> Result<bool, String> {
let output = swactor_process::command_output(&mut Command::new("vastai").args([
let output = swactor_process::command_output(Command::new("vastai").args([
"show",
"ssh-keys",
"--raw",
@ -2730,7 +2897,7 @@ pub(crate) fn ensure_vastai_account_ssh_key(api_key: &str, public_key: &str) ->
}
let output = swactor_process::command_output(
&mut Command::new("vastai")
Command::new("vastai")
.args(["create", "ssh-key"])
.arg(public_key)
.args(["-y", "--api-key", api_key]),
@ -3347,9 +3514,10 @@ mod serve_cluster_properties {
fn serve_cluster_production_transitions_converge_once_without_growth(
actions in lifecycle_actions()
) {
let mut config = RuntimeConfig::default();
config.worker_count = 1;
let parts = RuntimeParts::new(config);
let parts = RuntimeParts::new(RuntimeConfig {
worker_count: 1,
..RuntimeConfig::default()
});
let runtime = parts.runtime().clone();
let backend = SteppingBackend::new();
let engine = Engine::new(parts, backend.clone()).expect("one-worker stepping engine");

20
apps/myelin/src/orchestration/cluster_reconciler.rs
View file

@ -257,10 +257,12 @@ impl MyelinEffectBackend {
let mut effects = lock_node(&effects);
if let Some(live) = &effects.live {
if live.attempt == effect.operation.attempt {
return Ok(OperationOutcome::LeaseCreated(CreateLeaseResult {
lease: live.lease.clone(),
endpoint: Some(live.endpoint.clone()),
}));
return Ok(OperationOutcome::LeaseCreated(Box::new(
CreateLeaseResult {
lease: live.lease.clone(),
endpoint: Some(live.endpoint.clone()),
},
)));
}
return Err(EffectError::ambiguous(format!(
"node {} still owns attempt {} while creating attempt {}",
@ -335,10 +337,12 @@ impl MyelinEffectBackend {
bootstrap_started: false,
});
failure_sink.arm();
Ok(OperationOutcome::LeaseCreated(CreateLeaseResult {
lease,
endpoint: Some(endpoint),
}))
Ok(OperationOutcome::LeaseCreated(Box::new(
CreateLeaseResult {
lease,
endpoint: Some(endpoint),
},
)))
}
}

59
apps/myelin/src/orchestration/control.rs
View file

@ -538,7 +538,7 @@ async fn submit_node_job(
ManualControlMsg::Query { reply_to }
}) {
Ok(response_rx) => response_rx,
Err(response) => return response,
Err(response) => return *response,
};
let model = match response_rx.await {
Ok(ManualControlReply::Status(model)) => model,
@ -689,7 +689,7 @@ async fn request_reply(
) -> Response {
let response_rx = match begin_request_reply(state, timeout, build) {
Ok(response_rx) => response_rx,
Err(response) => return response,
Err(response) => return *response,
};
match response_rx.await {
@ -718,7 +718,7 @@ fn begin_request_reply(
state: &ControlHttpState,
timeout: Duration,
build: impl FnOnce(ActorAddress) -> ManualControlMsg,
) -> Result<tokio::sync::oneshot::Receiver<ManualControlReply>, Response> {
) -> Result<tokio::sync::oneshot::Receiver<ManualControlReply>, Box<Response>> {
let (response_tx, response_rx) = tokio::sync::oneshot::channel();
let response_tx = Arc::new(Mutex::new(Some(response_tx)));
let reply_to = state
@ -730,26 +730,30 @@ fn begin_request_reply(
timeout,
})
.map_err(|error| {
(
StatusCode::SERVICE_UNAVAILABLE,
Json(ErrorResponse {
error: format!("create control reply observer: {error}"),
}),
Box::new(
(
StatusCode::SERVICE_UNAVAILABLE,
Json(ErrorResponse {
error: format!("create control reply observer: {error}"),
}),
)
.into_response(),
)
.into_response()
})?;
if let Err(error) = state
.runtime
.send_to(state.orchestrator, OrchestratorMsg::Manual(build(reply_to)))
{
let _ = state.runtime.stop_actor(reply_to);
return Err((
StatusCode::SERVICE_UNAVAILABLE,
Json(ErrorResponse {
error: format!("orchestrator control actor unavailable: {error}"),
}),
)
.into_response());
return Err(Box::new(
(
StatusCode::SERVICE_UNAVAILABLE,
Json(ErrorResponse {
error: format!("orchestrator control actor unavailable: {error}"),
}),
)
.into_response(),
));
}
Ok(response_rx)
}
@ -998,7 +1002,7 @@ mod properties {
status,
})
};
let reply = |result: Result<_, Response>| match result {
let reply = |result: Result<_, Box<Response>>| match result {
Ok(receiver) => PendingHttpObservation::Reply {
index,
action: action.clone(),
@ -1191,9 +1195,10 @@ mod properties {
HttpAction::from_raw(kind, command_slot, value)
})
.collect::<Vec<_>>();
let mut config = RuntimeConfig::default();
config.worker_count = 1;
let parts = RuntimeParts::new(config);
let parts = RuntimeParts::new(RuntimeConfig {
worker_count: 1,
..RuntimeConfig::default()
});
let runtime = parts.runtime().clone();
let backend = SteppingBackend::new();
let engine =
@ -1294,9 +1299,10 @@ mod properties {
fn generated_duplicate_control_replies_deliver_first_once_and_remove_observer(
replies in prop::collection::vec(any::<u8>(), 0..=16)
) {
let mut config = RuntimeConfig::default();
config.worker_count = 1;
let parts = RuntimeParts::new(config);
let parts = RuntimeParts::new(RuntimeConfig {
worker_count: 1,
..RuntimeConfig::default()
});
let runtime = parts.runtime().clone();
let backend = SteppingBackend::new();
let engine =
@ -1380,9 +1386,10 @@ mod properties {
#[test]
fn reply_observer_disappearance_returns_a_bounded_terminal_http_response() {
let mut config = RuntimeConfig::default();
config.worker_count = 1;
let parts = RuntimeParts::new(config);
let parts = RuntimeParts::new(RuntimeConfig {
worker_count: 1,
..RuntimeConfig::default()
});
let runtime = parts.runtime().clone();
let backend = SteppingBackend::new();
let engine =

61
apps/myelin/src/orchestration/fleet_control.js
View file

@ -1,14 +1,50 @@
(() => {
const CONTROL_ID = 'myelin-fleet-control';
const CONFIRM_ID = 'myelin-confirm-dialog';
const CONFIRM_STYLE_ID = 'myelin-confirm-dialog-style';
const selectedJobs = new Map();
async function syncControl() {
const nodeView = document.querySelector('.node-view[data-node]');
if (!nodeView) return;
const rawNodeId = nodeView.getAttribute('data-node') || '';
if (!/^\d+$/.test(rawNodeId)) return;
const logicalNodeId = Number(rawNodeId);
function confirmKill(logicalNodeId) {
let dialog = document.getElementById(CONFIRM_ID);
if (!dialog) {
const style = document.createElement('style');
style.id = CONFIRM_STYLE_ID;
style.textContent = `
.myelin-confirm { width:min(440px,calc(100vw - 32px));padding:0;color:var(--text);background:var(--panel);border:1px solid var(--bad);border-radius:var(--r);box-shadow:0 18px 60px rgba(0,0,0,.55) }
.myelin-confirm::backdrop { background:rgba(0,6,12,.78) }
.myelin-confirm form { display:grid;gap:14px;padding:18px }
.myelin-confirm h2,.myelin-confirm p { margin:0 }
.myelin-confirm h2 { color:var(--bad) }
.myelin-confirm-actions { display:flex;justify-content:flex-end;gap:8px }
.myelin-confirm button { padding:6px 12px;background:transparent;color:var(--text);border:1px solid var(--border);border-radius:var(--r);cursor:pointer;font:600 13px var(--mono) }
.myelin-confirm button[value="confirm"] { color:var(--danger-ink);background:var(--danger-fill);border-color:var(--danger-border) }
`;
document.head.append(style);
dialog = document.createElement('dialog');
dialog.id = CONFIRM_ID;
dialog.className = 'myelin-confirm';
dialog.setAttribute('aria-labelledby', 'myelin-confirm-title');
dialog.setAttribute('aria-describedby', 'myelin-confirm-message');
dialog.innerHTML = `<form method="dialog">
<h2 id="myelin-confirm-title">Terminate managed node?</h2>
<p id="myelin-confirm-message"></p>
<div class="myelin-confirm-actions">
<button value="cancel" autofocus>Cancel</button>
<button value="confirm">Terminate node</button>
</div>
</form>`;
document.body.append(dialog);
}
dialog.querySelector('#myelin-confirm-message').textContent =
`Terminate managed node ${logicalNodeId}? Vast.ai contracts are destroyed and billing stops.`;
dialog.returnValue = 'cancel';
return new Promise(resolve => {
dialog.addEventListener('close', () => resolve(dialog.returnValue === 'confirm'), { once: true });
dialog.showModal();
});
}
async function syncControl() {
let model;
try {
const response = await fetch('/api/control/status', { cache: 'no-store' });
@ -18,6 +54,17 @@
} catch (_) {
return;
}
window.dispatchEvent(new CustomEvent('dashboard-hardware-source', {
detail: {
source: model?.provider?.provisioning_mode === 'mock' ? 'orchestrator' : 'node',
},
}));
const nodeView = document.querySelector('.node-view[data-node]');
if (!nodeView) return;
const rawNodeId = nodeView.getAttribute('data-node') || '';
if (!/^\d+$/.test(rawNodeId)) return;
const logicalNodeId = Number(rawNodeId);
const node = model?.nodes?.find(candidate => candidate.logical_node_id === logicalNodeId);
if (!node) return;
@ -130,7 +177,7 @@
};
killButton.onclick = async () => {
if (!window.confirm(`Kill managed node ${logicalNodeId}? Vast.ai contracts are destroyed and billing stops.`)) return;
if (!await confirmKill(logicalNodeId)) return;
killButton.disabled = true;
message.textContent = 'Submitting kill…';
const commandId = `fleet-kill-${globalThis.crypto?.randomUUID?.() || Date.now()}`;

34
apps/myelin/src/orchestration/job_reconciler.rs
View file

@ -180,7 +180,7 @@ enum ReconciledJobPhase {
deadline: Instant,
worker: NodeIdentity,
},
Running(JobRunStateMachine),
Running(Box<JobRunStateMachine>),
Stopping {
result: Result<JobDone, String>,
},
@ -291,9 +291,9 @@ impl ActorInterface for ReconciledJobActor {
Err(error) => self.begin_stop(Err(error)),
}
} else if Instant::now() >= deadline {
self.begin_stop(Err(format!(
"timed out waiting for reconciled job worker identity"
)));
self.begin_stop(Err(
"timed out waiting for reconciled job worker identity".to_string()
));
} else {
self.phase = ReconciledJobPhase::Provisioning { deadline };
}
@ -313,7 +313,7 @@ impl ActorInterface for ReconciledJobActor {
match machine {
Ok(mut machine) => {
machine.start(Instant::now());
self.phase = ReconciledJobPhase::Running(machine);
self.phase = ReconciledJobPhase::Running(Box::new(machine));
}
Err(error) => self.begin_stop(Err(error)),
}
@ -382,14 +382,16 @@ fn build_vastai_provisioner(
ssh_public_key_fingerprint(&public_key)
);
let mut config = VastAiProvisioningConfig::default();
config.label_prefix = options.label_prefix.clone();
config.disk_gb = options.disk_gb;
config.ssh_user = options.ssh_user.clone();
config.confirm_lease = options.confirm_lease;
config.onstart = options.onstart.clone();
config.ssh_public_key = Some(public_key);
config.selection = selection_policy(options);
let mut config = VastAiProvisioningConfig {
label_prefix: options.label_prefix.clone(),
disk_gb: options.disk_gb,
ssh_user: options.ssh_user.clone(),
confirm_lease: options.confirm_lease,
onstart: options.onstart.clone(),
ssh_public_key: Some(public_key),
selection: selection_policy(options),
..VastAiProvisioningConfig::default()
};
if let Some(poll_interval) = options.poll_interval {
config.lifecycle.poll_interval = poll_interval;
}
@ -502,8 +504,10 @@ fn build_cluster(
},
}],
};
let mut retry = RetryPolicy::default();
retry.operation_timeout = options.provision_timeout;
let retry = RetryPolicy {
operation_timeout: options.provision_timeout,
..RetryPolicy::default()
};
ProvisionedClusterGuard::new(
desired,
vec![ReconcilerNodeBinding {

48
apps/myelin/src/orchestration/manual_control.rs
View file

@ -1458,20 +1458,32 @@ pub(crate) struct ManualActorControl {
control_generation: u64,
}
pub(crate) struct ManualActorControlConfig {
pub(crate) state_dir: StateDir,
pub(crate) sink: PluginSink,
pub(crate) provider_factory: ProviderFactory,
pub(crate) spec_builder: SpecBuilder,
pub(crate) config_validator: Option<ConfigValidator>,
pub(crate) offer_searcher: Option<OfferSearcher>,
pub(crate) control_generation: u64,
}
impl ManualActorControl {
#[allow(clippy::too_many_arguments)]
pub(crate) fn new(
core: ManualControl,
runtime: Runtime,
blocking_work: BlockingWorkSender,
state_dir: StateDir,
sink: PluginSink,
provider_factory: ProviderFactory,
spec_builder: SpecBuilder,
config_validator: Option<ConfigValidator>,
offer_searcher: Option<OfferSearcher>,
control_generation: u64,
config: ManualActorControlConfig,
) -> Self {
let ManualActorControlConfig {
state_dir,
sink,
provider_factory,
spec_builder,
config_validator,
offer_searcher,
control_generation,
} = config;
let sender = runtime.create_sender();
let work_actor = runtime
.spawn(ManualWorkActor { blocking_work })
@ -3038,7 +3050,7 @@ mod tests {
}
fn start_bootstrap(&mut self, handle: &PluginNodeHandle) -> Result<(), String> {
if handle.id % 11 == 0 {
if handle.id.is_multiple_of(11) {
return Err("scripted bootstrap failure".to_owned());
}
Ok(())
@ -3062,7 +3074,7 @@ mod tests {
}
fn provider_ref_for(&self, spec: &NodeProvisionSpec) -> String {
if spec.node_id % 5 == 0 {
if spec.node_id.is_multiple_of(5) {
String::new()
} else {
format!("scripted-resource-{}", spec.node_id)
@ -3484,13 +3496,15 @@ mod tests {
ready_core(),
runtime.clone(),
engine.handle().blocking_work_sender(),
StateDir::new(temp.path()),
PluginSink::new(Arc::new(DiscardObservations)),
provider_factory,
spec_builder,
Some(validator),
Some(searcher),
1,
ManualActorControlConfig {
state_dir: StateDir::new(temp.path()),
sink: PluginSink::new(Arc::new(DiscardObservations)),
provider_factory,
spec_builder,
config_validator: Some(validator),
offer_searcher: Some(searcher),
control_generation: 1,
},
);
prop_assert_eq!(
runtime.stats().actors.len(),

80
apps/myelin/src/orchestration/provider_adapters/vastai/mod.rs
View file

@ -225,17 +225,29 @@ struct VastAiProviderMonitorActor {
stopped: bool,
}
struct VastAiProviderMonitorConfig {
client: ToolsVastAiLeaseClient,
contract_id: u64,
label: String,
lifecycle: LifecyclePolicy,
spec: NodeProvisionSpec,
sink: PluginSink,
sender: ExternalSender,
engine: EngineHandle,
}
impl VastAiProviderMonitorActor {
fn new(
client: ToolsVastAiLeaseClient,
contract_id: u64,
label: String,
lifecycle: LifecyclePolicy,
spec: NodeProvisionSpec,
sink: PluginSink,
sender: ExternalSender,
engine: EngineHandle,
) -> Self {
fn new(config: VastAiProviderMonitorConfig) -> Self {
let VastAiProviderMonitorConfig {
client,
contract_id,
label,
lifecycle,
spec,
sink,
sender,
engine,
} = config;
Self {
client,
contract_id,
@ -592,14 +604,16 @@ impl VastAiLeaseClient for ToolsVastAiLeaseClient {
let sender = runtime.create_sender();
let actor = runtime
.spawn(VastAiProviderMonitorActor::new(
self.clone(),
contract_id,
label,
lifecycle,
spec,
sink,
sender,
engine,
VastAiProviderMonitorConfig {
client: self.clone(),
contract_id,
label,
lifecycle,
spec,
sink,
sender,
engine,
},
))
.ok()?;
Some(VastAiProviderMonitor::new(runtime, actor))
@ -2355,21 +2369,23 @@ mod tests {
let recording = Arc::new(RecordingSink::default());
let actor = runtime
.spawn(VastAiProviderMonitorActor::new(
client,
contract_id,
format!(
"run-{}-node-{}-attempt-{}",
spec.run_id, spec.node_id, spec.attempt_id
),
LifecyclePolicy {
lease_pace: Duration::ZERO,
poll_interval: Duration::from_millis(1),
state_timeout: Duration::from_millis(10),
VastAiProviderMonitorConfig {
client,
contract_id,
label: format!(
"run-{}-node-{}-attempt-{}",
spec.run_id, spec.node_id, spec.attempt_id
),
lifecycle: LifecyclePolicy {
lease_pace: Duration::ZERO,
poll_interval: Duration::from_millis(1),
state_timeout: Duration::from_millis(10),
},
spec,
sink: PluginSink::new(recording.clone()),
sender: runtime.create_sender(),
engine: engine.handle(),
},
spec,
PluginSink::new(recording.clone()),
runtime.create_sender(),
engine.handle(),
))
.expect("spawn VastAI monitor");
MonitorHarness {

39
apps/myelin/src/orchestration/provision_page.html
View file

@ -92,6 +92,19 @@
.sort-button[data-direction="asc"]::before { top: 3px; border-bottom: 5px solid currentColor; }
.sort-button[data-direction="desc"]::before { bottom: 3px; border-top: 5px solid currentColor; }
.sort-button:hover:not(:disabled), .sort-button[aria-pressed="true"] { color: var(--cyan); border-color: var(--cyan); background: var(--panel-hover); }
.offers-heading { display: flex; align-items: center; justify-content: space-between; gap: 12px; flex-wrap: wrap; margin-bottom: 12px; }
.offers-heading h2 { margin: 0; }
.offer-actions { display: flex; align-items: center; gap: 10px; flex-wrap: wrap; }
.offer-selection { margin: 0; font: 12px var(--mono); }
dialog.confirm-dialog {
width: min(440px, calc(100vw - 32px)); padding: 0; color: var(--text);
background: var(--panel); border: 1px solid var(--amber); border-radius: var(--r);
box-shadow: 0 18px 60px rgba(0, 0, 0, .55);
}
dialog.confirm-dialog::backdrop { background: rgba(0, 6, 12, .78); }
.confirm-form { display: grid; gap: 14px; padding: 18px; }
.confirm-form h2, .confirm-form p { margin: 0; }
.confirm-actions { display: flex; justify-content: flex-end; gap: 8px; }
@media (max-width: 680px) { #search-form { grid-template-columns: 1fr; } }
@media (prefers-reduced-motion: reduce) { * { transition-duration: .01ms !important; } }
</style>
@ -144,12 +157,27 @@
</section>
</div>
<section id="vast-offers-card" class="card" style="margin-top:1rem">
<h2>Offers</h2><p id="offer-status" class="muted"></p><p id="offer-error" class="error"></p>
<div class="offers-heading">
<h2>Offers</h2>
<div class="offer-actions">
<p class="offer-selection">Selected: <code id="selection">none (0/8)</code></p>
<button id="provision-button" class="primary" disabled>Provision selected offers</button>
</div>
</div>
<p id="offer-status" class="muted"></p><p id="offer-error" class="error"></p>
<div class="offers-scroll"><table id="offers-table"><thead><tr><th>Select</th><th>Offer<span class="sort-controls"><button type="button" class="sort-button" data-sort="offer_id" data-direction="asc" title="Sort offer ascending" aria-label="Sort offer ascending">↑</button><button type="button" class="sort-button" data-sort="offer_id" data-direction="desc" title="Sort offer descending" aria-label="Sort offer descending">↓</button></span></th><th>Host<span class="sort-controls"><button type="button" class="sort-button" data-sort="host_id" data-direction="asc" title="Sort host ascending" aria-label="Sort host ascending">↑</button><button type="button" class="sort-button" data-sort="host_id" data-direction="desc" title="Sort host descending" aria-label="Sort host descending">↓</button></span></th><th>GPU<span class="sort-controls"><button type="button" class="sort-button" data-sort="gpu_model" data-direction="asc" title="Sort GPU ascending" aria-label="Sort GPU ascending">↑</button><button type="button" class="sort-button" data-sort="gpu_model" data-direction="desc" title="Sort GPU descending" aria-label="Sort GPU descending">↓</button></span></th><th>RAM<span class="sort-controls"><button type="button" class="sort-button" data-sort="gpu_ram_mb" data-direction="asc" title="Sort RAM ascending" aria-label="Sort RAM ascending">↑</button><button type="button" class="sort-button" data-sort="gpu_ram_mb" data-direction="desc" title="Sort RAM descending" aria-label="Sort RAM descending">↓</button></span></th><th>CC<span class="sort-controls"><button type="button" class="sort-button" data-sort="compute_cap" data-direction="asc" title="Sort compute capability ascending" aria-label="Sort compute capability ascending">↑</button><button type="button" class="sort-button" data-sort="compute_cap" data-direction="desc" title="Sort compute capability descending" aria-label="Sort compute capability descending">↓</button></span></th><th>Verified<span class="sort-controls"><button type="button" class="sort-button" data-sort="verification" data-direction="asc" title="Sort verification ascending" aria-label="Sort verification ascending">↑</button><button type="button" class="sort-button" data-sort="verification" data-direction="desc" title="Sort verification descending" aria-label="Sort verification descending">↓</button></span></th><th>Reliability<span class="sort-controls"><button type="button" class="sort-button" data-sort="reliability" data-direction="asc" title="Sort reliability ascending" aria-label="Sort reliability ascending">↑</button><button type="button" class="sort-button" data-sort="reliability" data-direction="desc" title="Sort reliability descending" aria-label="Sort reliability descending">↓</button></span></th><th>Down<span class="sort-controls"><button type="button" class="sort-button" data-sort="download_mbps" data-direction="asc" title="Sort download speed ascending" aria-label="Sort download speed ascending">↑</button><button type="button" class="sort-button" data-sort="download_mbps" data-direction="desc" title="Sort download speed descending" aria-label="Sort download speed descending">↓</button></span></th><th>Up<span class="sort-controls"><button type="button" class="sort-button" data-sort="upload_mbps" data-direction="asc" title="Sort upload speed ascending" aria-label="Sort upload speed ascending">↑</button><button type="button" class="sort-button" data-sort="upload_mbps" data-direction="desc" title="Sort upload speed descending" aria-label="Sort upload speed descending">↓</button></span></th><th>Location<span class="sort-controls"><button type="button" class="sort-button" data-sort="location" data-direction="asc" title="Sort location ascending" aria-label="Sort location ascending">↑</button><button type="button" class="sort-button" data-sort="location" data-direction="desc" title="Sort location descending" aria-label="Sort location descending">↓</button></span></th><th>$/hr<span class="sort-controls"><button type="button" class="sort-button" data-sort="hourly_price" data-direction="asc" title="Sort price ascending" aria-label="Sort price ascending">↑</button><button type="button" class="sort-button" data-sort="hourly_price" data-direction="desc" title="Sort price descending" aria-label="Sort price descending">↓</button></span></th></tr></thead><tbody id="offers"></tbody></table></div>
<p>Selected exact offer IDs: <code id="selection">none (0/8)</code></p>
<button id="provision-button" class="primary" disabled>Provision selected offers</button>
</section>
</main>
<dialog id="confirm-dialog" class="confirm-dialog" aria-labelledby="confirm-title" aria-describedby="confirm-message">
<form method="dialog" class="confirm-form">
<h2 id="confirm-title">Confirm action</h2>
<p id="confirm-message"></p>
<div class="confirm-actions">
<button value="cancel" autofocus>Cancel</button>
<button id="confirm-submit" class="primary" value="confirm">Confirm</button>
</div>
</form>
</dialog>
<script>
'use strict';
const $=id=>document.getElementById(id); const MAX_SELECTED_OFFERS=8; const PAGE_STATE_KEY='myelin.provision.page.v1'; const SEARCH_FIELDS=['gpu_model','min_gpu_ram_mb','min_compute_cap','min_reliability','min_download_mbps','min_upload_mbps','max_hourly_price','blacklist_hosts','require_verified']; let selected=new Set(); let provisioningMode='real'; let currentOffers=[]; let offerSorts=[]; let sortDoubleClickStartedActive=null;
@ -158,6 +186,7 @@ const number=(form,name)=>{const raw=value(form,name);return raw===null?null:Num
const escapeHtml=raw=>String(raw??'').replace(/[&<>"']/g,char=>({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[char]));
async function jsonFetch(url,options={}){const response=await fetch(url,{...options,headers:{'content-type':'application/json',...(options.headers||{})}});const text=await response.text();let body=null;try{body=text?JSON.parse(text):null}catch{body={error:text}}if(!response.ok)throw new Error(body?.error||`${response.status} ${response.statusText}`);return body}
function commandId(prefix){return `${prefix}-${crypto.randomUUID()}`}
function confirmAction(title,message,confirmLabel){const dialog=$('confirm-dialog');$('confirm-title').textContent=title;$('confirm-message').textContent=message;$('confirm-submit').textContent=confirmLabel;dialog.returnValue='cancel';return new Promise(resolve=>{dialog.addEventListener('close',()=>resolve(dialog.returnValue==='confirm'),{once:true});dialog.showModal()})}
function formSnapshot(form,names){return Object.fromEntries(names.map(name=>{const field=form.elements[name];return [name,field.type==='checkbox'?field.checked:field.value]}))}
function restoreForm(form,state){if(!state)return;Object.entries(state).forEach(([name,saved])=>{const field=form.elements[name];if(!field)return;if(field.type==='checkbox')field.checked=Boolean(saved);else field.value=String(saved??'')})}
function persistPageState(){try{sessionStorage.setItem(PAGE_STATE_KEY,JSON.stringify({config:formSnapshot($('config-form'),['ssh_identity','bootstrap_command']),search:formSnapshot($('search-form'),SEARCH_FIELDS),local:formSnapshot($('local-provision-form'),['count']),offers:currentOffers,selected:[...selected],sorts:offerSorts}))}catch{}}
@ -165,13 +194,13 @@ function restorePageState(){try{const saved=JSON.parse(sessionStorage.getItem(PA
function updateSelectionUi(){const count=selected.size;$('selection').textContent=count?`${[...selected].join(', ')} (${count}/${MAX_SELECTED_OFFERS})`:`none (0/${MAX_SELECTED_OFFERS})`;$('provision-button').disabled=count===0;$('provision-button').textContent=count===0?(provisioningMode==='mock'?'Simulate selected offers':'Provision selected offers'):`${provisioningMode==='mock'?'Simulate':'Provision'} ${count} selected offer${count===1?'':'s'}`}
async function refresh(){try{const reply=await jsonFetch('/api/control/status');const model=reply.Status;const provider=model.provider.name;provisioningMode=model.provider.provisioning_mode||'real';const vast=provider==='vastai';const simulated=vast&&provisioningMode==='mock';const local=provider==='process'||provider==='docker';$('provider-config-card').hidden=!vast;$('vast-search-card').hidden=!vast;$('vast-offers-card').hidden=!vast;$('local-provision-card').hidden=!local;$('simulation-banner').hidden=!simulated;document.querySelectorAll('[data-real-vastai]').forEach(element=>element.hidden=simulated);$('local-provider-name').textContent=provider;$('readiness').textContent=`${provider} · ${provisioningMode} · ${model.provider.kind}`;$('readiness').className=model.provider.kind==='ready'?'status ok':'status';$('readiness-error').textContent=model.provider.error||'';$('search-button').disabled=!vast||model.provider.kind!=='ready';$('local-provision-button').disabled=!local||model.provider.kind!=='ready';updateSelectionUi()}catch(error){$('readiness-error').textContent=error.message}}
$('config-form').addEventListener('submit',async event=>{event.preventDefault();const form=event.currentTarget;try{await jsonFetch('/api/control/provider',{method:'POST',body:JSON.stringify({api_key:value(form,'api_key'),ssh_identity:value(form,'ssh_identity'),bootstrap_command:value(form,'bootstrap_command')})});form.elements.api_key.value='';persistPageState();await refresh()}catch(error){$('readiness-error').textContent=error.message}});
$('local-provision-form').addEventListener('submit',async event=>{event.preventDefault();const form=event.currentTarget;const count=number(form,'count')||1;const provider=$('local-provider-name').textContent;if(!confirm(`Provision ${count} ${provider} node${count===1?'':'s'}?`))return;try{await jsonFetch('/api/control/provision',{method:'POST',body:JSON.stringify({command_id:commandId('provision'),count,selected_offer_ids:[],image:$('runtime-image').value.trim()})});$('local-provision-error').textContent='';await refresh()}catch(error){$('local-provision-error').textContent=error.message}});
$('local-provision-form').addEventListener('submit',async event=>{event.preventDefault();const form=event.currentTarget;const count=number(form,'count')||1;const provider=$('local-provider-name').textContent;if(!await confirmAction('Provision nodes?',`Provision ${count} ${provider} node${count===1?'':'s'}?`,'Provision'))return;try{await jsonFetch('/api/control/provision',{method:'POST',body:JSON.stringify({command_id:commandId('provision'),count,selected_offer_ids:[],image:$('runtime-image').value.trim()})});$('local-provision-error').textContent='';await refresh()}catch(error){$('local-provision-error').textContent=error.message}});
$('search-form').addEventListener('submit',async event=>{event.preventDefault();const form=event.currentTarget;const blacklist=(value(form,'blacklist_hosts')||'').split(',').map(v=>Number(v.trim())).filter(Number.isFinite);$('offer-status').textContent='Searching…';$('offer-error').textContent='';try{const reply=await jsonFetch('/api/control/offers',{method:'POST',body:JSON.stringify({gpu_model:value(form,'gpu_model'),min_gpu_ram_mb:number(form,'min_gpu_ram_mb'),min_compute_cap:number(form,'min_compute_cap'),min_reliability:number(form,'min_reliability'),require_verified:form.elements.require_verified.checked,min_download_mbps:number(form,'min_download_mbps'),min_upload_mbps:number(form,'min_upload_mbps'),max_hourly_price:number(form,'max_hourly_price'),blacklist_hosts:blacklist,count:null})});currentOffers=reply.Offers;selected.clear();renderOffers();updateSelectionUi();$('offer-status').textContent=currentOffers.length===0?'No offers match this search.':`${currentOffers.length} matching offer${currentOffers.length===1?'':'s'}`;persistPageState()}catch(error){$('offer-status').textContent='';$('offer-error').textContent=error.message}});
function compareOfferValues(left,right){const leftMissing=left===null||left===undefined||left==='';const rightMissing=right===null||right===undefined||right==='';if(leftMissing||rightMissing)return leftMissing===rightMissing?0:leftMissing?1:-1;if(typeof left==='number'&&typeof right==='number')return left-right;return String(left).localeCompare(String(right),undefined,{numeric:true,sensitivity:'base'})}
function renderOffers(){const offers=[...currentOffers];if(offerSorts.length)offers.sort((left,right)=>{for(const sort of offerSorts){const compared=compareOfferValues(left[sort.key],right[sort.key]);if(compared!==0)return compared*(sort.direction==='asc'?1:-1)}return Number(left.offer_id)-Number(right.offer_id)});$('offers').replaceChildren(...offers.map(offer=>{const tr=document.createElement('tr');tr.innerHTML=`<td><input type="checkbox" data-offer="${Number(offer.offer_id)}" style="width:auto"></td><td>${escapeHtml(offer.offer_id)}</td><td>${escapeHtml(offer.host_id)}</td><td>${escapeHtml(offer.gpu_model)}</td><td>${escapeHtml(offer.gpu_ram_mb)}</td><td>${escapeHtml(offer.compute_cap)}</td><td>${escapeHtml(offer.verification)}</td><td>${escapeHtml(offer.reliability)}</td><td>${escapeHtml(offer.download_mbps)}</td><td>${escapeHtml(offer.upload_mbps)}</td><td>${escapeHtml(offer.location)}</td><td>${escapeHtml(Number(offer.hourly_price).toFixed(3))}</td>`;tr.querySelector('[data-offer]').checked=selected.has(Number(offer.offer_id));return tr}));document.querySelectorAll('.sort-button').forEach(button=>button.setAttribute('aria-pressed',String(offerSorts.some(sort=>sort.key===button.dataset.sort&&sort.direction===button.dataset.direction))))}
$('offers-table').addEventListener('click',event=>{const button=event.target.closest('.sort-button');if(!button)return;const id=`${button.dataset.sort}:${button.dataset.direction}`;const activeIndex=offerSorts.findIndex(sort=>sort.key===button.dataset.sort&&sort.direction===button.dataset.direction);if(event.detail===1)sortDoubleClickStartedActive=activeIndex>=0?id:null;if(event.detail===2&&sortDoubleClickStartedActive===id){offerSorts.splice(activeIndex,1);sortDoubleClickStartedActive=null}else if(activeIndex<0){const columnIndex=offerSorts.findIndex(sort=>sort.key===button.dataset.sort);const next={key:button.dataset.sort,direction:button.dataset.direction};if(columnIndex>=0)offerSorts.splice(columnIndex,1,next);else offerSorts.push(next)}renderOffers();persistPageState()});
$('offers').addEventListener('change',event=>{const id=Number(event.target.dataset.offer);if(!Number.isFinite(id))return;if(event.target.checked){if(selected.size>=MAX_SELECTED_OFFERS){event.target.checked=false;$('offer-error').textContent=`Select at most ${MAX_SELECTED_OFFERS} offers.`;return}selected.add(id)}else selected.delete(id);$('offer-error').textContent='';updateSelectionUi();persistPageState()});
$('provision-button').addEventListener('click',async()=>{try{const ids=[...selected];if(ids.length===0)return;const question=provisioningMode==='mock'?`Simulate provisioning exact Vast.ai offer IDs ${ids.join(', ')}? No rental will be created.`:`Provision exact Vast.ai offer IDs ${ids.join(', ')}? This can incur charges.`;if(!confirm(question))return;await jsonFetch('/api/control/provision',{method:'POST',body:JSON.stringify({command_id:commandId('provision'),count:ids.length,selected_offer_ids:ids,image:$('runtime-image').value.trim()})});const provisioned=new Set(ids);currentOffers=currentOffers.filter(offer=>!provisioned.has(Number(offer.offer_id)));selected.clear();renderOffers();updateSelectionUi();persistPageState();await refresh()}catch(error){$('offer-error').textContent=error.message}});
$('provision-button').addEventListener('click',async()=>{try{const ids=[...selected];if(ids.length===0)return;const simulated=provisioningMode==='mock';const question=simulated?`Simulate provisioning exact Vast.ai offer IDs ${ids.join(', ')}? No rental will be created.`:`Provision exact Vast.ai offer IDs ${ids.join(', ')}? This can incur charges.`;if(!await confirmAction(simulated?'Simulate provisioning?':'Provision selected offers?',question,simulated?'Simulate':'Provision'))return;await jsonFetch('/api/control/provision',{method:'POST',body:JSON.stringify({command_id:commandId('provision'),count:ids.length,selected_offer_ids:ids,image:$('runtime-image').value.trim()})});const provisioned=new Set(ids);currentOffers=currentOffers.filter(offer=>!provisioned.has(Number(offer.offer_id)));selected.clear();renderOffers();updateSelectionUi();persistPageState();await refresh()}catch(error){$('offer-error').textContent=error.message}});
$('config-form').addEventListener('input',persistPageState);$('search-form').addEventListener('input',persistPageState);$('local-provision-form').addEventListener('input',persistPageState);restorePageState();refresh();setInterval(refresh,1000);
</script>
</body></html>

264
apps/myelin/src/orchestration/provisioning.rs
View file

@ -73,8 +73,6 @@ pub(crate) struct MockVastAiPlugin {
inner: Box<dyn ProvisionPlugin>,
contracts: BTreeMap<u64, MockVastAiContract>,
selected_offers: BTreeMap<u64, u64>,
#[cfg(feature = "test-support")]
lifecycle_observer_path: Option<PathBuf>,
}
#[derive(Clone)]
@ -280,9 +278,6 @@ impl MockVastAiPlugin {
inner,
contracts: BTreeMap::new(),
selected_offers: BTreeMap::new(),
#[cfg(feature = "test-support")]
lifecycle_observer_path: std::env::var_os("MYELIN_MOCK_VASTAI_LEDGER_PATH")
.map(PathBuf::from),
}
}
@ -333,113 +328,6 @@ fn observe_mock_vastai_contract(
.to_string(),
});
}
#[cfg(feature = "test-support")]
fn observe_mock_vastai_lifecycle(
path: Option<&Path>,
event: &str,
spec: &NodeProvisionSpec,
provider_ref: &str,
selected_offer_id: Option<u64>,
) -> Result<(), String> {
let Some(path) = path else {
return Ok(());
};
let mut file = OpenOptions::new()
.create(true)
.append(true)
.open(path)
.map_err(|error| {
format!(
"open mock Vast.ai lifecycle observer {}: {error}",
path.display()
)
})?;
let mut observation = serde_json::to_string(&serde_json::json!({
"event": event,
"run_id": spec.run_id,
"node_id": spec.node_id,
"attempt_id": spec.attempt_id,
"provider_ref": provider_ref,
"selected_offer_id": selected_offer_id,
}))
.map_err(|error| {
format!(
"encode mock Vast.ai lifecycle observation {}: {error}",
path.display()
)
})?;
observation.push('\n');
file.write_all(observation.as_bytes()).map_err(|error| {
format!(
"append mock Vast.ai lifecycle observation {}: {error}",
path.display()
)
})
}
#[cfg(feature = "test-support")]
fn mock_vastai_lifecycle_path(plugin: &MockVastAiPlugin) -> Option<&Path> {
plugin.lifecycle_observer_path.as_deref()
}
#[cfg(feature = "test-support")]
fn mock_vastai_resource_is_live(
path: Option<&Path>,
spec: &NodeProvisionSpec,
provider_ref: &str,
) -> Result<bool, String> {
let Some(path) = path else {
return Ok(false);
};
let contents = match fs::read_to_string(path) {
Ok(contents) => contents,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false),
Err(error) => {
return Err(format!(
"read mock Vast.ai provider state {}: {error}",
path.display()
));
}
};
let mut live = false;
for (index, line) in contents.lines().enumerate() {
let event: serde_json::Value = serde_json::from_str(line).map_err(|error| {
format!(
"parse mock Vast.ai provider state {} line {}: {error}",
path.display(),
index + 1
)
})?;
if event
.get("provider_ref")
.and_then(serde_json::Value::as_str)
!= Some(provider_ref)
{
continue;
}
let event_run_id = event.get("run_id").and_then(serde_json::Value::as_u64);
let event_node_id = event.get("node_id").and_then(serde_json::Value::as_u64);
let event_attempt_id = event.get("attempt_id").and_then(serde_json::Value::as_u64);
if (event_run_id, event_node_id, event_attempt_id)
!= (Some(spec.run_id), Some(spec.node_id), Some(spec.attempt_id))
{
return Err(format!(
"mock Vast.ai provider reference {provider_ref} belongs to another node: {event}"
));
}
match event.get("event").and_then(serde_json::Value::as_str) {
Some("created") => live = true,
Some("destroyed") => live = false,
Some("adopted" | "recreated") => {}
other => {
return Err(format!(
"unknown mock Vast.ai provider lifecycle event {other:?}: {event}"
));
}
}
}
Ok(live)
}
fn local_process_provider_ref(prefix: &str, spec: &NodeProvisionSpec) -> String {
format!(
@ -720,9 +608,8 @@ fn docker_inspect_error_is_absent(stderr: &str) -> bool {
}
fn docker_container_is_absent(name: &str) -> Result<bool, String> {
let output =
swactor_process::command_output(&mut Command::new("docker").arg("inspect").arg(name))
.map_err(|error| format!("inspect Docker container {name}: {error}"))?;
let output = swactor_process::command_output(Command::new("docker").arg("inspect").arg(name))
.map_err(|error| format!("inspect Docker container {name}: {error}"))?;
if output.status.success() {
return Ok(false);
}
@ -740,7 +627,7 @@ fn docker_container_is_absent(name: &str) -> Result<bool, String> {
fn docker_container_is_running(name: &str) -> Result<bool, String> {
let output = swactor_process::command_output(
&mut Command::new("docker")
Command::new("docker")
.args(["inspect", "-f", "{{.State.Running}}"])
.arg(name),
)
@ -757,7 +644,7 @@ fn docker_container_is_running(name: &str) -> Result<bool, String> {
/// Lists container names carrying this daemon's label, running or not.
fn docker_labeled_containers(prefix: &str) -> Result<Vec<String>, String> {
let output = swactor_process::command_output(&mut Command::new("docker").args([
let output = swactor_process::command_output(Command::new("docker").args([
"ps",
"-a",
"--filter",
@ -786,7 +673,7 @@ fn docker_containers_for_spec(
spec: &NodeProvisionSpec,
) -> Result<Vec<String>, String> {
let output = swactor_process::command_output(
&mut Command::new("docker")
Command::new("docker")
.args(["ps", "-a"])
.arg("--filter")
.arg(format!("label=myelin.daemon={prefix}"))
@ -886,7 +773,7 @@ fn prepare_docker_file_volume(
)?;
let loader_name = format!("myelin-cache-load-{}-{volume}", std::process::id());
let _ = swactor_process::command_status(
&mut Command::new("docker")
Command::new("docker")
.args(["rm", "-f", &loader_name])
.stdout(Stdio::null())
.stderr(Stdio::null()),
@ -915,7 +802,7 @@ fn prepare_docker_file_volume(
"copy cached model into docker volume",
);
let _ = swactor_process::command_status(
&mut Command::new("docker")
Command::new("docker")
.args(["rm", &loader_name])
.stdout(Stdio::null())
.stderr(Stdio::null()),
@ -966,7 +853,7 @@ fn safe_docker_volume_component(value: &str) -> String {
fn docker_status(args: &[&str], label: &str) -> Result<(), String> {
let status = swactor_process::command_status(
&mut Command::new("docker")
Command::new("docker")
.args(args)
.stdin(Stdio::null())
.stdout(Stdio::null())
@ -982,7 +869,7 @@ fn docker_status(args: &[&str], label: &str) -> Result<(), String> {
fn docker_status_vec(args: Vec<String>, label: &str) -> Result<(), String> {
let status = swactor_process::command_status(
&mut Command::new("docker")
Command::new("docker")
.args(&args)
.stdin(Stdio::null())
.stdout(Stdio::null())
@ -1012,11 +899,11 @@ fn stop_owned_process(runtime: &mut LocalProcessRuntime) -> Result<Option<i32>,
fn stop_adopted_process(record: &LocalProcessRecord) -> Result<(), String> {
#[cfg(target_os = "linux")]
{
return swactor_process::terminate_process_group(
swactor_process::terminate_process_group(
&record.identity(),
Duration::from_secs(2),
Duration::from_millis(50),
);
)
}
#[cfg(not(target_os = "linux"))]
Ok(())
@ -1414,14 +1301,6 @@ impl ProvisionPlugin for MockVastAiPlugin {
&provider_ref,
Some(offer_id),
);
#[cfg(feature = "test-support")]
observe_mock_vastai_lifecycle(
mock_vastai_lifecycle_path(self),
"created",
&spec,
&provider_ref,
Some(offer_id),
)?;
Ok(handle)
}
@ -1450,14 +1329,6 @@ impl ProvisionPlugin for MockVastAiPlugin {
&contract.provider_ref,
contract.selected_offer_id,
);
#[cfg(feature = "test-support")]
observe_mock_vastai_lifecycle(
mock_vastai_lifecycle_path(self),
"destroyed",
&contract.spec,
&contract.provider_ref,
contract.selected_offer_id,
)?;
}
Ok(())
}
@ -1468,21 +1339,6 @@ impl ProvisionPlugin for MockVastAiPlugin {
sink: PluginSink,
) -> Result<Option<AdoptedNode>, String> {
let adopted = self.inner.adopt_by_spec(spec, sink.clone())?;
#[cfg(feature = "test-support")]
let adopted = {
let provider_ref = self.inner.provider_ref_for(spec);
if adopted.is_none()
&& mock_vastai_resource_is_live(
mock_vastai_lifecycle_path(self),
spec,
&provider_ref,
)?
{
self.inner.prepare_missing_bootstrap(spec, sink.clone())?
} else {
adopted
}
};
if let Some(adopted) = &adopted {
let offer_id = selected_offer_from_spec(spec);
if let Some(offer_id) = offer_id {
@ -1504,14 +1360,6 @@ impl ProvisionPlugin for MockVastAiPlugin {
&adopted.provider_ref,
offer_id,
);
#[cfg(feature = "test-support")]
observe_mock_vastai_lifecycle(
mock_vastai_lifecycle_path(self),
"adopted",
spec,
&adopted.provider_ref,
offer_id,
)?;
}
Ok(adopted)
}
@ -1543,14 +1391,6 @@ impl ProvisionPlugin for MockVastAiPlugin {
&prepared.provider_ref,
offer_id,
);
#[cfg(feature = "test-support")]
observe_mock_vastai_lifecycle(
mock_vastai_lifecycle_path(self),
"recreated",
spec,
&prepared.provider_ref,
offer_id,
)?;
}
Ok(prepared)
}
@ -1588,14 +1428,6 @@ impl ProvisionPlugin for MockVastAiPlugin {
&provider_ref,
selected_offer_from_spec(spec),
);
#[cfg(feature = "test-support")]
observe_mock_vastai_lifecycle(
mock_vastai_lifecycle_path(self),
"destroyed",
spec,
&provider_ref,
selected_offer_from_spec(spec),
)?;
}
Ok(stopped)
}
@ -1879,7 +1711,7 @@ impl ProvisionPlugin for LocalDockerPlugin {
fn remove_docker_container(container_name: &str) -> Result<(), String> {
let status = swactor_process::command_status(
&mut Command::new("docker")
Command::new("docker")
.arg("rm")
.arg("-f")
.arg(container_name)
@ -2274,78 +2106,6 @@ mod tests {
assert!(!process_record_matches(&record));
let _ = fs::remove_file(registry_path);
}
#[cfg(target_os = "linux")]
#[test]
fn mixed_mock_processes_survive_kill_provision_and_restart() {
let (_engine, runtime) = test_runtime();
let registry_path = std::env::temp_dir().join(format!(
"myelin-mixed-mock-test-{}-{}.json",
std::process::id(),
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let (tx, _) = mpsc::channel();
let sink = PluginSink::new(Arc::new(ChannelSink(tx)));
let make_spec = |node_id: u64| {
let mut spec = test_spec();
spec.node_id = node_id;
spec.env
.push(("MYELIN_RUN_ID".to_owned(), spec.run_id.to_string()));
spec.env.push((
"MYELIN_LOGICAL_NODE_ID".to_owned(),
spec.node_id.to_string(),
));
spec.args = vec![
"-c".to_owned(),
"trap 'exit 0' TERM; while :; do sleep 1; done".to_owned(),
];
spec
};
{
let mut plugin =
MockVastAiPlugin::with_registry("/bin/sh", &registry_path, runtime.clone());
for node_id in 1..=3 {
let handle = plugin
.create_node_selected(make_spec(node_id), sink.clone(), Some(node_id))
.unwrap();
plugin.start_bootstrap(&handle).unwrap();
}
}
{
let mut restarted =
MockVastAiPlugin::with_registry("/bin/sh", &registry_path, runtime.clone());
let killed = restarted
.adopt_by_spec(&make_spec(2), sink.clone())
.unwrap()
.expect("node 2");
restarted.stop_node(&killed.handle).unwrap();
let new_node = restarted
.create_node_selected(make_spec(4), sink.clone(), Some(4))
.unwrap();
restarted.start_bootstrap(&new_node).unwrap();
}
let mut final_restart = MockVastAiPlugin::with_registry("/bin/sh", &registry_path, runtime);
assert_eq!(
final_restart.list_managed_refs().unwrap(),
[
"mock-vastai-5-1-attempt-11",
"mock-vastai-5-3-attempt-11",
"mock-vastai-5-4-attempt-11",
]
);
for node_id in [1, 3, 4] {
let adopted = final_restart
.adopt_by_spec(&make_spec(node_id), sink.clone())
.unwrap()
.expect("surviving mock process");
final_restart.stop_node(&adopted.handle).unwrap();
}
let _ = fs::remove_file(registry_path);
}
#[test]
fn missing_local_bootstrap_recreates_only_the_provider_handle() {

18
apps/myelin/src/orchestration/run_fsm.rs
View file

@ -114,17 +114,17 @@ pub(crate) enum RunFaultReason {
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) enum LifecycleEvent {
RunFaulted {
Faulted {
run_id: RunId,
reason: RunFaultReason,
},
RunCompleted {
Completed {
run_id: RunId,
},
RunOperatorStopped {
OperatorStopped {
run_id: RunId,
},
RunTornDown {
TornDown {
run_id: RunId,
},
}
@ -373,7 +373,7 @@ impl OrchestratorRun {
return;
}
self.terminal = true;
self.events.push(LifecycleEvent::RunCompleted {
self.events.push(LifecycleEvent::Completed {
run_id: self.config.run_id,
});
self.start_teardown();
@ -383,7 +383,7 @@ impl OrchestratorRun {
return;
}
self.terminal = true;
self.events.push(LifecycleEvent::RunOperatorStopped {
self.events.push(LifecycleEvent::OperatorStopped {
run_id: self.config.run_id,
});
self.start_teardown();
@ -394,7 +394,7 @@ impl OrchestratorRun {
return;
}
self.terminal = true;
self.events.push(LifecycleEvent::RunFaulted {
self.events.push(LifecycleEvent::Faulted {
run_id: self.config.run_id,
reason,
});
@ -437,9 +437,9 @@ impl OrchestratorRun {
if !self
.events
.iter()
.any(|event| matches!(event, LifecycleEvent::RunTornDown { .. }))
.any(|event| matches!(event, LifecycleEvent::TornDown { .. }))
{
self.events.push(LifecycleEvent::RunTornDown {
self.events.push(LifecycleEvent::TornDown {
run_id: self.config.run_id,
});
}

8
apps/myelin/src/staging/control.rs
View file

@ -91,7 +91,7 @@ pub(crate) struct ProvisionStage {
pub(crate) enum StageEvent {
ProvisionStage {
from: NodeId,
provision: ProvisionStage,
provision: Box<ProvisionStage>,
},
WorkerReady,
WeightsReady,
@ -211,7 +211,7 @@ pub(crate) enum StageCommand {
LoadWeights {
source: WeightSource,
range: LayerRange,
shard_plan: Option<StageShardPlan>,
shard_plan: Box<Option<StageShardPlan>>,
},
ExecuteStep(ExecuteStep),
ReleaseInputHandle {
@ -277,7 +277,7 @@ impl StageController {
pub(crate) fn observe(&mut self, event: StageEvent) {
match event {
StageEvent::ProvisionStage { from, provision } => self.provision(from, provision),
StageEvent::ProvisionStage { from, provision } => self.provision(from, *provision),
StageEvent::WorkerReady => self.worker_ready = true,
StageEvent::WeightsReady => self.weights_ready = true,
StageEvent::InboundEdgeReady { edge_id } => {
@ -353,7 +353,7 @@ impl StageController {
self.commands.push(StageCommand::LoadWeights {
source: provision.weight_source.clone(),
range: provision.layer_range,
shard_plan: provision.shard_plan.clone(),
shard_plan: Box::new(provision.shard_plan.clone()),
});
self.provision = Some(provision);
}

4
apps/myelin/src/staging/gguf_shard.rs
View file

@ -299,7 +299,7 @@ where
"bytes_total":bytes_total,
}));
}
let metadata_prefix = fetch_http_range(&url, 0, plan.metadata_end)?;
let metadata_prefix = fetch_http_range(url, 0, plan.metadata_end)?;
bytes_done = bytes_done.saturating_add(plan.metadata_end);
if plan.metadata_end > 0 {
emit(serde_json::json!({
@ -348,7 +348,7 @@ where
"bytes_done":bytes_done,
"bytes_total":bytes_total,
}));
let range_bytes = fetch_http_range(&url, range.start, range.len)?;
let range_bytes = fetch_http_range(url, range.start, range.len)?;
bytes_done = bytes_done.saturating_add(range.len);
emit(serde_json::json!({
"type":"StageShardRangeFetchReady",

246
apps/myelin/src/tests/data_namespace_guarantees.rs Normal file
View file

@ -0,0 +1,246 @@
#![cfg(target_os = "linux")]
use std::sync::Arc;
use std::time::{Duration, Instant};
use data_plane::arena::{ArenaConfig, ArenaManager, NodeId as ArenaNodeId};
use data_plane::blob;
use data_plane::blob_transfer::{BlobTransferReceiver, BlobTransferSender};
use data_plane::bootstrap::{self, BootstrapSpec};
use data_plane::data_plane::DataPlaneBootstrap;
use data_plane::host::{HostDataPlaneConfig, HostDataPlaneSessionActor};
use data_plane::namespace::{DirectoryClient, NamespaceError};
use data_plane::path::{DataPath, JobContext};
use data_plane::protocol::JobCapability;
use iroh_driver::{IrohBlobTransferReceiver, IrohBlobTransferSender, IrohDriver};
use swactor_engine::Engine;
use crate::data_namespace::{
DATA_DIRECTORY_SERVICE, DataNamespaceAuthority, InstalledNamespaceClient,
install_namespace_client,
};
use crate::orchestration::distribution_stack::DistributionRuntimeStack;
use crate::tests::harness::build_iroh_composition;
const POLL: Duration = Duration::from_millis(25);
const DEADLINE: Duration = Duration::from_secs(30);
const CAPABILITY: JobCapability = JobCapability::new([0x77; 32]);
struct DataNode {
namespace: InstalledNamespaceClient,
receiver: Arc<dyn BlobTransferReceiver>,
sender: Arc<dyn BlobTransferSender>,
stack: DistributionRuntimeStack,
driver: IrohDriver,
_engine: Engine,
}
impl DataNode {
fn start() -> Self {
let (engine, driver, stack) = build_iroh_composition(POLL);
let namespace = install_namespace_client(&stack, &driver).expect("namespace client");
let receiver = Arc::new(IrohBlobTransferReceiver::new(
driver.endpoint_addr(),
driver.edge_events_handle(),
));
receiver.install_pump(&engine.handle(), stack.runtime.clone(), POLL);
let receiver: Arc<dyn BlobTransferReceiver> = receiver;
let sender: Arc<dyn BlobTransferSender> = Arc::new(IrohBlobTransferSender::new(
driver.edge_connector(),
&engine.handle(),
));
Self {
namespace,
receiver,
sender,
stack,
driver,
_engine: engine,
}
}
fn session(
&self,
generation: u64,
read_prefixes: Vec<DataPath>,
write_prefixes: Vec<DataPath>,
) -> DataPlaneBootstrap {
let mut arena = ArenaManager::boot(ArenaConfig {
node_id: ArenaNodeId(generation),
reservation_ceiling: 64 * 1024,
base_alignment: 64,
})
.expect("session arena");
let handoff = bootstrap::write_bootstrap(
&mut arena,
BootstrapSpec {
arena_generation: generation,
alignment: 64,
},
)
.expect("session bootstrap");
let host = self
.stack
.runtime
.spawn(
HostDataPlaneSessionActor::new(HostDataPlaneConfig {
runtime: self.stack.runtime.clone(),
arena,
arena_generation: generation,
session_generation: generation,
capability: CAPABILITY,
job_context: JobContext {
run_id: format!("run-{generation}"),
read_prefixes,
write_prefixes,
},
namespace: Some(self.namespace.client.clone()),
transfer_receiver: Some(Arc::clone(&self.receiver)),
source_sender: Some(Arc::clone(&self.sender)),
source_publisher: Some(Arc::clone(&self.namespace.source_publisher)),
route_registrar: None,
})
.expect("host session"),
)
.expect("spawn host session");
futures_lite::future::block_on(DataPlaneBootstrap::attach(
handoff.arena_fd,
self.stack.runtime.clone(),
host,
CAPABILITY,
))
.expect("attach child session")
}
}
#[test]
fn control_and_session_publications_cross_real_iroh_and_outlive_the_producer() {
let state = tempfile::tempdir().expect("namespace state");
let file = state.path().join("registered.bin");
std::fs::write(&file, b"control-file-over-iroh").expect("registered file");
let node_a = DataNode::start();
let node_b = DataNode::start();
let authority = DataNamespaceAuthority::start(
&node_a.stack,
&node_a.driver,
state.path().join("namespace.json"),
)
.expect("namespace authority");
node_a
.driver
.join(std::slice::from_ref(&node_b.driver.endpoint_addr()));
assert!(wait_until(DEADLINE, || {
namespace_visible(&node_a.stack, authority.directory())
&& namespace_visible(&node_b.stack, authority.directory())
}));
let file_path = DataPath::parse("/shared/registered").unwrap();
futures_lite::future::block_on(
authority
.control()
.register(file_path.clone(), blob::file(&file)),
)
.expect("public file registration");
let directory = DirectoryClient::new(node_a.stack.runtime.clone(), authority.directory());
let file_source = futures_lite::future::block_on(directory.resolve(file_path.clone()))
.expect("registered binding")
.source;
assert!(wait_until(DEADLINE, || {
node_b.stack.route_owner(file_source).is_some()
}));
let reader_b = node_b.session(1, vec![DataPath::parse("/shared").unwrap()], vec![]);
let registered = futures_lite::future::block_on(reader_b.data_plane.read_blob(&file_path))
.expect("remote registered-file read");
assert_eq!(
registered.map().unwrap().as_ref(),
b"control-file-over-iroh"
);
drop(registered);
let published_path = DataPath::parse("/shared/published").unwrap();
let producer = node_b.session(
2,
vec![DataPath::parse("/shared").unwrap()],
vec![DataPath::parse("/shared").unwrap()],
);
let mut writer = futures_lite::future::block_on(
producer
.data_plane
.write_blob(&published_path, b"session-over-iroh".len() as u64),
)
.expect("open publication");
writer
.map()
.unwrap()
.as_mut()
.copy_from_slice(b"session-over-iroh");
futures_lite::future::block_on(writer.seal()).expect("commit publication");
let published_source =
futures_lite::future::block_on(directory.resolve(published_path.clone()))
.expect("published binding")
.source;
assert!(wait_until(DEADLINE, || {
node_a.stack.route_owner(published_source).is_some()
}));
producer.data_plane.close().expect("close producer");
let reader_a = node_a.session(3, vec![DataPath::parse("/shared").unwrap()], vec![]);
let published = futures_lite::future::block_on(reader_a.data_plane.read_blob(&published_path))
.expect("read after producer close");
assert_eq!(published.map().unwrap().as_ref(), b"session-over-iroh");
drop(published);
futures_lite::future::block_on(authority.control().unregister(published_path.clone()))
.expect("unregister publication");
assert!(matches!(
futures_lite::future::block_on(directory.resolve(published_path)),
Err(NamespaceError::PathNotFound(_))
));
assert!(wait_until(DEADLINE, || {
node_b
.stack
.runtime
.stats()
.actors
.iter()
.all(|(actor, _)| *actor != published_source)
}));
reader_a.data_plane.close().expect("close reader A");
reader_b.data_plane.close().expect("close reader B");
}
fn namespace_visible(
stack: &DistributionRuntimeStack,
directory: swactor::actor::ActorAddress,
) -> bool {
stack.route_owner(directory).is_some()
&& stack
.registry_view
.read()
.expect("registry view")
.entries
.iter()
.any(|entry| {
entry.name == DATA_DIRECTORY_SERVICE
&& entry.actor_addr == directory
&& !entry.tombstone
})
}
fn wait_until(deadline: Duration, mut check: impl FnMut() -> bool) -> bool {
let started = Instant::now();
loop {
if check() {
return true;
}
if started.elapsed() >= deadline {
return false;
}
std::thread::sleep(POLL);
}
}

24
apps/myelin/src/tests/engine_composition.rs
View file

@ -84,15 +84,15 @@ fn build_composition() -> (Engine, IrohDriver, DistributionRuntimeStack) {
engine.handle(),
);
driver.enable_actor_bridge(
stack.runtime.clone(),
stack.codec.clone(),
stack.actor_bridge_routes(),
stack.actors.swim,
stack.relay_mirror.clone(),
stack.route_view.clone(),
stack.outbox.clone(),
);
driver.enable_actor_bridge(iroh_driver::ActorBridgeConfig {
runtime: stack.runtime.clone(),
codec: stack.codec.clone(),
routes: stack.actor_bridge_routes(),
swim: stack.actors.swim,
relay_mirror: stack.relay_mirror.clone(),
route_view: stack.route_view.clone(),
outbox: stack.outbox.clone(),
});
// Engine-hosted protocol tick injection + adapter progression — no manual
// pump is wired anywhere.
stack.spawn_protocol_ticker(PROBE_TICK);
@ -156,8 +156,10 @@ fn dashboard_server_is_scheduled_through_the_engine() {
.port();
let (engine, _driver, _stack) = build_composition();
let mut config = dashboard::DashboardConfig::default();
config.port = free_port;
let config = dashboard::DashboardConfig {
port: free_port,
..dashboard::DashboardConfig::default()
};
let handle = dashboard::DashboardHandle::new(config);
handle.spawn(&engine.handle());

51
apps/myelin/src/tests/harness.rs
View file

@ -8,6 +8,14 @@
//! such as `fsm::RunPlan::test_linear(..)` and
//! `stage::WeightSource::embedded_gguf(..)` work unchanged.
use std::time::Duration;
use distribution::node::DistributedNodeConfig;
use iroh::RelayMode;
use iroh_driver::{EDGE_ALPN, IrohDriver, IrohDriverConfig};
use swactor_engine::{Engine, TokioBackend, TokioConfig};
use crate::orchestration::distribution_stack::DistributionRuntimeStack;
use crate::run_fsm::{NodeId as FsmNodeId, OrchestratorRun, RunId as FsmRunId, RunPlan, StageRef};
use crate::run_plan::{GgufSource, TokenizerSource};
use crate::staging::{DeviceHandle, StageController, WeightSource};
@ -46,5 +54,48 @@ impl WeightSource {
/// Test alias for the orchestrator run core, retained for readable test prose.
pub(crate) type OrchestratorHarness = OrchestratorRun;
pub(crate) fn build_iroh_composition(
poll: Duration,
) -> (Engine, IrohDriver, DistributionRuntimeStack) {
let (parts, runtime, codec, transport_router) =
DistributionRuntimeStack::build_runtime(crate::codecs::register_myelin_actor_codecs, None);
let engine = Engine::new(
parts,
TokioBackend::new(TokioConfig::default()).expect("tokio backend"),
)
.expect("engine");
let mut driver = IrohDriver::with_engine(
engine.handle(),
IrohDriverConfig {
secret_key: None,
relay_mode: RelayMode::Disabled,
node: DistributedNodeConfig::default(),
peer_auth: None,
additional_alpns: vec![EDGE_ALPN.to_vec()],
},
)
.expect("iroh driver");
let stack = DistributionRuntimeStack::new_from_runtime(
runtime,
codec,
transport_router,
driver.node_id(),
DistributedNodeConfig::default(),
engine.handle(),
);
driver.enable_actor_bridge(iroh_driver::ActorBridgeConfig {
runtime: stack.runtime.clone(),
codec: stack.codec.clone(),
routes: stack.actor_bridge_routes(),
swim: stack.actors.swim,
relay_mirror: stack.relay_mirror.clone(),
route_view: stack.route_view.clone(),
outbox: stack.outbox.clone(),
});
stack.spawn_protocol_ticker(poll);
driver.install_actor_bridge_pump(poll);
(engine, driver, stack)
}
/// Test alias for the stage controller core, retained for readable test prose.
pub(crate) type StageControllerHarness = StageController;

50
apps/myelin/src/tests/job_data_plane_guarantees.rs
View file

@ -1,12 +1,10 @@
#![cfg(target_os = "linux")]
use std::collections::{BTreeMap, BTreeSet};
use std::collections::BTreeSet;
use data_plane::blob::ContentDigest;
use data_plane::bootstrap::{
ENV_ARENA_FD, ENV_DATA_PLANE_ACTOR, ENV_DATA_PLANE_ENDPOINT, ENV_JOB_CAPABILITY,
};
use data_plane::host::BlobSource;
use data_plane::path::{DataPath, JobContext};
use data_plane::protocol::JobCapability;
use swactor::config::RuntimeConfig;
@ -24,24 +22,24 @@ fn path(value: &str) -> DataPath {
fn plane() -> ActorJobDataPlane {
let parts = RuntimeParts::new(RuntimeConfig::default());
let runtime = parts.runtime().clone();
let mut blobs = BTreeMap::new();
blobs.insert(
path("/models/fixture"),
BlobSource::with_sha256(b"opaque fixture".as_slice()),
);
ActorJobDataPlane::new(
&runtime,
ARENA_BYTES,
11,
13,
CAPABILITY,
JobContext {
run_id: "run-1".to_owned(),
read_prefixes: vec![path("/models")],
write_prefixes: vec![path("/runs/run-1/results")],
crate::job_data_plane::ActorJobDataPlaneConfig {
arena_bytes: ARENA_BYTES,
arena_generation: 11,
session_generation: 13,
capability: CAPABILITY,
job_context: JobContext {
run_id: "run-1".to_owned(),
read_prefixes: vec![path("/models")],
write_prefixes: vec![path("/runs/run-1/results")],
},
namespace: None,
transfer_receiver: None,
source_sender: None,
source_publisher: None,
route_registrar: None,
},
blobs,
None,
)
.expect("actor data-plane")
}
@ -71,19 +69,3 @@ fn handoff_contains_one_descriptor_and_private_actor_metadata() {
assert!(flags >= 0);
assert_eq!(flags & libc::FD_CLOEXEC, 0);
}
#[test]
fn fixture_source_metadata_is_application_opaque() {
let source = BlobSource::with_sha256(b"opaque fixture".as_slice());
assert_eq!(source.length(), 14);
assert_eq!(
source.digest(),
Some(ContentDigest::sha256(b"opaque fixture"))
);
let plane = plane();
assert_ne!(
plane.host_session(),
swactor::actor::ActorAddress::default()
);
}

76
apps/myelin/src/tests/job_runner_integration.rs
View file

@ -13,8 +13,8 @@ use swactor::actor::Message;
use swactor::runtime::Inbox;
use swactor_engine::{Engine, TokioBackend, TokioConfig};
use swactor_job_runner::{
register_job_codecs, Job, JobDone, JobState, NodeJobActor, OrchestratorJobActor,
OrchestratorJobMsg, Workspace,
Job, JobDone, JobState, NodeJobActor, OrchestratorJobActor, OrchestratorJobMsg, Workspace,
register_job_codecs,
};
use distribution::node::DistributedNodeConfig;
@ -41,9 +41,12 @@ fn recv_within<T: Message>(inbox: &Inbox<T>, deadline: Duration) -> Option<T> {
fn build_composition() -> (Engine, IrohDriver, DistributionRuntimeStack) {
let (parts, runtime, codec, transport_router) =
DistributionRuntimeStack::build_runtime(|c| register_job_codecs(c), None);
let engine = Engine::new(parts, TokioBackend::new(TokioConfig::default()).expect("tokio backend"))
.expect("engine");
DistributionRuntimeStack::build_runtime(register_job_codecs, None);
let engine = Engine::new(
parts,
TokioBackend::new(TokioConfig::default()).expect("tokio backend"),
)
.expect("engine");
let mut driver = IrohDriver::with_engine(
engine.handle(),
IrohDriverConfig {
@ -63,15 +66,15 @@ fn build_composition() -> (Engine, IrohDriver, DistributionRuntimeStack) {
DistributedNodeConfig::default(),
engine.handle(),
);
driver.enable_actor_bridge(
stack.runtime.clone(),
stack.codec.clone(),
stack.actor_bridge_routes(),
stack.actors.swim,
stack.relay_mirror.clone(),
stack.route_view.clone(),
stack.outbox.clone(),
);
driver.enable_actor_bridge(iroh_driver::ActorBridgeConfig {
runtime: stack.runtime.clone(),
codec: stack.codec.clone(),
routes: stack.actor_bridge_routes(),
swim: stack.actors.swim,
relay_mirror: stack.relay_mirror.clone(),
route_view: stack.route_view.clone(),
outbox: stack.outbox.clone(),
});
stack.spawn_protocol_ticker(POLL);
driver.install_actor_bridge_pump(POLL);
(engine, driver, stack)
@ -90,17 +93,28 @@ fn job_runs_through_swactor_inside_myelin_composition() {
let done = runtime.new_inbox::<JobDone>().expect("done inbox");
let orch = runtime
.spawn(OrchestratorJobActor::new(*done.addr(), landing.path().to_path_buf()))
.spawn(OrchestratorJobActor::new(
*done.addr(),
landing.path().to_path_buf(),
))
.expect("spawn orchestrator");
let node = runtime
.spawn(NodeJobActor::new(orch, node_workdir.path().to_path_buf(), sender, 0))
.spawn(NodeJobActor::new(
orch,
node_workdir.path().to_path_buf(),
sender,
0,
))
.expect("spawn node");
let job = Job {
name: "myelin-probe".to_owned(),
setup: Some("echo setup-ok > setup_done.txt".to_owned()),
run: "echo hello-from-myelin-swactor > greeting.txt".to_owned(),
workspace: Some(Workspace { workdir: ws.path().to_path_buf(), exclude: vec![] }),
workspace: Some(Workspace {
workdir: ws.path().to_path_buf(),
exclude: vec![],
}),
outputs: vec![
"greeting.txt".to_owned(),
"setup_done.txt".to_owned(),
@ -109,18 +123,36 @@ fn job_runs_through_swactor_inside_myelin_composition() {
env: BTreeMap::new(),
};
runtime
.send_to(orch, OrchestratorJobMsg::Submit { job, node_actor: node })
.send_to(
orch,
OrchestratorJobMsg::Submit {
job,
node_actor: node,
},
)
.expect("submit");
let result = recv_within(&done, DEADLINE);
drop(engine);
let done = result.expect("job did not reach a terminal state");
assert_eq!(done.state, JobState::Completed, "expected COMPLETED, got {:?}", done);
assert_eq!(
done.state,
JobState::Completed,
"expected COMPLETED, got {:?}",
done
);
assert_eq!(done.exit_code, Some(0));
let greeting = std::fs::read_to_string(landing.path().join("greeting.txt"))
.expect("collected greeting.txt");
assert!(greeting.contains("hello-from-myelin-swactor"), "greeting: {greeting}");
let seed = std::fs::read_to_string(landing.path().join("seed.txt")).expect("collected seed.txt");
assert_eq!(seed, "seed-value", "workspace materialized + collected through swactor");
assert!(
greeting.contains("hello-from-myelin-swactor"),
"greeting: {greeting}"
);
let seed =
std::fs::read_to_string(landing.path().join("seed.txt")).expect("collected seed.txt");
assert_eq!(
seed, "seed-value",
"workspace materialized + collected through swactor"
);
}

53
apps/myelin/src/tests/job_runner_iroh.rs
View file

@ -9,63 +9,20 @@
use std::collections::BTreeMap;
use std::time::{Duration, Instant};
use swactor_engine::{Engine, TokioBackend, TokioConfig};
use swactor_job_runner::{
Job, JobDone, JobState, NodeJobActor, OrchestratorJobActor, OrchestratorJobMsg, Workspace,
register_job_codecs,
};
use distribution::node::DistributedNodeConfig;
use iroh::RelayMode;
use iroh_driver::{IrohDriver, IrohDriverConfig};
use iroh_driver::IrohDriver;
use crate::orchestration::distribution_stack::DistributionRuntimeStack;
use crate::tests::harness::build_iroh_composition;
const POLL: Duration = Duration::from_millis(25);
const CONVERGE_DEADLINE: Duration = Duration::from_secs(30);
const JOB_DEADLINE: Duration = Duration::from_secs(30);
fn build_composition() -> (Engine, IrohDriver, DistributionRuntimeStack) {
let (parts, runtime, codec, transport_router) =
DistributionRuntimeStack::build_runtime(|c| register_job_codecs(c), None);
let engine = Engine::new(
parts,
TokioBackend::new(TokioConfig::default()).expect("tokio backend"),
)
.expect("engine");
let mut driver = IrohDriver::with_engine(
engine.handle(),
IrohDriverConfig {
secret_key: None,
relay_mode: RelayMode::Disabled,
node: DistributedNodeConfig::default(),
peer_auth: None,
additional_alpns: vec![],
},
)
.expect("iroh driver");
let stack = DistributionRuntimeStack::new_from_runtime(
runtime.clone(),
codec,
transport_router,
driver.node_id(),
DistributedNodeConfig::default(),
engine.handle(),
);
driver.enable_actor_bridge(
stack.runtime.clone(),
stack.codec.clone(),
stack.actor_bridge_routes(),
stack.actors.swim,
stack.relay_mirror.clone(),
stack.route_view.clone(),
stack.outbox.clone(),
);
stack.spawn_protocol_ticker(POLL);
driver.install_actor_bridge_pump(POLL);
(engine, driver, stack)
}
#[test]
fn job_runs_across_two_nodes_over_real_iroh() {
let node_workdir = tempfile::tempdir().expect("node workdir");
@ -75,13 +32,13 @@ fn job_runs_across_two_nodes_over_real_iroh() {
// Worker composition (B): spawn + register the NodeJobActor so peers can
// route to it through the converged directory.
let (engine_b, driver_b, stack_b) = build_composition();
let (engine_b, driver_b, stack_b) = build_iroh_composition(POLL);
let sender_b = stack_b.runtime.create_sender();
// Placeholder orchestrator address: the real one is on A; the node only
// needs it once the orchestrator submits. We point the node at A's
// orchestrator after it exists (address is fixed below), but the node actor
// captures the address at construction — so spawn it after A's orchestrator.
let (_engine_a, _driver_a, stack_a) = build_composition();
let (_engine_a, _driver_a, stack_a) = build_iroh_composition(POLL);
let done = stack_a
.runtime

2
apps/myelin/src/tests/mod.rs
View file

@ -1,7 +1,9 @@
mod data_namespace_guarantees;
mod engine_composition;
pub(crate) mod fuzz_support;
mod harness;
mod job_data_plane_guarantees;
mod job_runner_integration;
mod job_runner_iroh;
mod node_guarantees;
mod observability_guarantees;

26
apps/myelin/src/tests/orchestration_guarantees.rs
View file

@ -193,7 +193,7 @@ mod run_fsm {
invalid
.events()
.iter()
.any(|event| { matches!(event, fsm::LifecycleEvent::RunFaulted { .. }) })
.any(|event| { matches!(event, fsm::LifecycleEvent::Faulted { .. }) })
);
}
@ -249,7 +249,7 @@ mod run_fsm {
fsm::RunCommand::InjectTokenObject { object, .. } => Some(object),
_ => None,
})
.last()
.next_back()
.expect("decode injection must be recorded");
assert_eq!(
*decode_object,
@ -315,7 +315,7 @@ mod run_fsm {
.events()
.iter()
.filter_map(|event| match event {
fsm::LifecycleEvent::RunFaulted { reason, .. } => Some(reason),
fsm::LifecycleEvent::Faulted { reason, .. } => Some(reason),
_ => None,
})
.collect::<Vec<_>>();
@ -344,7 +344,7 @@ mod run_fsm {
assert!(membership_lost.events().iter().any(|event| {
matches!(
event,
fsm::LifecycleEvent::RunFaulted {
fsm::LifecycleEvent::Faulted {
reason: fsm::RunFaultReason::MembershipLost {
node_id: fsm::NodeId(12)
},
@ -380,12 +380,12 @@ mod run_fsm {
let completed = harness
.events()
.iter()
.filter(|event| matches!(event, fsm::LifecycleEvent::RunCompleted { .. }))
.filter(|event| matches!(event, fsm::LifecycleEvent::Completed { .. }))
.count();
let faulted = harness
.events()
.iter()
.filter(|event| matches!(event, fsm::LifecycleEvent::RunFaulted { .. }))
.filter(|event| matches!(event, fsm::LifecycleEvent::Faulted { .. }))
.count();
assert_eq!(completed, 1);
assert_eq!(faulted, 0);
@ -432,12 +432,12 @@ mod run_fsm {
let stopped_count = stopped
.events()
.iter()
.filter(|event| matches!(event, fsm::LifecycleEvent::RunOperatorStopped { .. }))
.filter(|event| matches!(event, fsm::LifecycleEvent::OperatorStopped { .. }))
.count();
let stopped_faults = stopped
.events()
.iter()
.filter(|event| matches!(event, fsm::LifecycleEvent::RunFaulted { .. }))
.filter(|event| matches!(event, fsm::LifecycleEvent::Faulted { .. }))
.count();
assert_eq!(stopped_count, 1);
assert_eq!(stopped_faults, 0);
@ -475,7 +475,7 @@ mod run_fsm {
!harness
.events()
.iter()
.any(|event| { matches!(event, fsm::LifecycleEvent::RunTornDown { .. }) })
.any(|event| { matches!(event, fsm::LifecycleEvent::TornDown { .. }) })
);
// StageStopped for every stage still is not enough until local endpoints stop.
@ -491,7 +491,7 @@ mod run_fsm {
!harness
.events()
.iter()
.any(|event| { matches!(event, fsm::LifecycleEvent::RunTornDown { .. }) })
.any(|event| { matches!(event, fsm::LifecycleEvent::TornDown { .. }) })
);
harness.observe(fsm::RunEvent::TokenEndpointsStopped);
@ -499,14 +499,14 @@ mod run_fsm {
let torn_down_count = harness
.events()
.iter()
.filter(|event| matches!(event, fsm::LifecycleEvent::RunTornDown { .. }))
.filter(|event| matches!(event, fsm::LifecycleEvent::TornDown { .. }))
.count();
assert_eq!(torn_down_count, 1);
// Ordering is proven over the lifecycle transcript.
let fault_pos = position_of(
harness.events(),
&fsm::LifecycleEvent::RunFaulted {
&fsm::LifecycleEvent::Faulted {
run_id: fsm::RunId(7),
reason: fsm::RunFaultReason::StageFault {
stage_index: 0,
@ -516,7 +516,7 @@ mod run_fsm {
);
let torn_down_pos = position_of(
harness.events(),
&fsm::LifecycleEvent::RunTornDown {
&fsm::LifecycleEvent::TornDown {
run_id: fsm::RunId(7),
},
);

8
apps/myelin/src/tests/staging_guarantees.rs
View file

@ -65,7 +65,7 @@ mod stage_controller {
let mut harness = new_controller();
harness.observe(stage::StageEvent::ProvisionStage {
from: stage::NodeId(99),
provision: valid_provision(),
provision: Box::new(valid_provision()),
});
for event in preparation_ready_events() {
harness.observe(event);
@ -81,7 +81,7 @@ mod stage_controller {
let mut harness = new_controller();
harness.observe(stage::StageEvent::ProvisionStage {
from: stage::NodeId(99),
provision: valid_provision(),
provision: Box::new(valid_provision()),
});
// Setup commands should be derived from the provided assignment.
@ -108,7 +108,7 @@ mod stage_controller {
let mut unauthorized = new_controller();
unauthorized.observe(stage::StageEvent::ProvisionStage {
from: stage::NodeId(123),
provision: valid_provision(),
provision: Box::new(valid_provision()),
});
assert!(unauthorized.events().iter().any(|event| {
matches!(
@ -134,7 +134,7 @@ mod stage_controller {
let mut harness = new_controller();
harness.observe(stage::StageEvent::ProvisionStage {
from: stage::NodeId(99),
provision: valid_provision(),
provision: Box::new(valid_provision()),
});
// Feed every readiness event except the final one and prove no prefix is

305
apps/myelin/src/tests/tinygrad_device_bridge_backend.py
View file

@ -1,305 +0,0 @@
#!/usr/bin/env python3
"""Tinygrad-backed device bridge probe for Rust Myelin bridge tests.
Line-delimited JSON control only. Payload bytes live in the arena file whose
path Rust passes during initialize.
"""
from __future__ import annotations
import json
import os
import struct
import sys
from dataclasses import dataclass
from typing import Any
try:
from tinygrad import Tensor # type: ignore
except Exception as exc: # pragma: no cover - exercised from Rust process tests
print(
json.dumps(
{
"type": "worker_fatal",
"reason": "tinygrad_unavailable",
"message": str(exc),
}
),
flush=True,
)
raise SystemExit(2)
@dataclass
class DeviceObject:
dtype: str
shape: str
extent: int
values: list[int]
tensor: Any
@dataclass
class PendingCopy:
kind: str
copy_id: int
handle_id: int
host_offset: int
device_offset: int
length: int
arena_path: str | None = None
arena_bytes = 0
generation = 0
objects: dict[int, DeviceObject] = {}
pending_copies: dict[int, PendingCopy] = {}
fail_next: str | None = None
def emit(obj: dict[str, Any]) -> None:
print(json.dumps(obj, separators=(",", ":")), flush=True)
def backend_error(reason: str) -> None:
emit({"type": "backend_error", "reason": reason})
def fatal(reason: str, message: str) -> None:
emit({"type": "worker_fatal", "reason": reason, "message": message})
def require_arena() -> str:
if arena_path is None:
raise RuntimeError("arena is not initialized")
return arena_path
def consume_failure(reason: str) -> bool:
global fail_next
if fail_next == reason:
fail_next = None
backend_error(reason)
return True
return False
def check_u32_range(offset: int, length: int) -> None:
if offset < 0 or length < 0 or offset % 4 != 0 or length % 4 != 0:
raise ValueError("u32 ranges must be non-negative and 4-byte aligned")
def tensor_from_values(values: list[int]) -> Any:
try:
return Tensor(values, dtype="uint32").realize()
except Exception:
return Tensor(values, dtype="int32").realize()
def allocate_tensor(dtype: str, extent: int) -> tuple[list[int], Any]:
if dtype == "u32":
if extent % 4 != 0:
raise ValueError("u32 extent must be 4-byte aligned")
values = [0] * (extent // 4)
return values, tensor_from_values(values)
if dtype == "f16":
values = [0] * (extent // 2)
return values, Tensor(values, dtype="float16").realize()
raise ValueError(f"unsupported dtype {dtype!r}")
def update_tensor(obj: DeviceObject) -> None:
if obj.dtype == "u32":
obj.tensor = tensor_from_values(obj.values)
elif obj.dtype == "f16":
obj.tensor = Tensor(obj.values, dtype="float16").realize()
else:
raise ValueError(f"unsupported dtype {obj.dtype!r}")
def materialized_values(obj: DeviceObject) -> list[int]:
return [int(v) for v in obj.tensor.tolist()]
def read_arena(offset: int, length: int) -> bytes:
path = require_arena()
with open(path, "rb", buffering=0) as f:
f.seek(offset)
data = f.read(length)
if len(data) != length:
raise EOFError("short arena read")
return data
def write_arena(offset: int, data: bytes) -> None:
path = require_arena()
with open(path, "r+b", buffering=0) as f:
f.seek(offset)
f.write(data)
f.flush()
def perform_host_to_device(copy: PendingCopy) -> None:
obj = objects[copy.handle_id]
if obj.dtype != "u32":
raise ValueError("payload copy is implemented for u32 test objects only")
check_u32_range(copy.device_offset, copy.length)
payload = read_arena(copy.host_offset, copy.length)
words = list(struct.unpack("<" + "I" * (copy.length // 4), payload))
start = copy.device_offset // 4
end = start + len(words)
if end > len(obj.values):
raise ValueError("device range out of bounds")
obj.values[start:end] = words
update_tensor(obj)
def perform_device_to_host(copy: PendingCopy) -> None:
obj = objects[copy.handle_id]
if obj.dtype != "u32":
raise ValueError("payload copy is implemented for u32 test objects only")
check_u32_range(copy.device_offset, copy.length)
values = materialized_values(obj)
start = copy.device_offset // 4
end = start + (copy.length // 4)
if end > len(values):
raise ValueError("device range out of bounds")
payload = struct.pack("<" + "I" * (end - start), *values[start:end])
write_arena(copy.host_offset, payload)
def perform_copy(copy: PendingCopy) -> None:
if copy.kind == "host_to_device":
perform_host_to_device(copy)
elif copy.kind == "device_to_host":
perform_device_to_host(copy)
else:
raise ValueError(f"unknown copy kind {copy.kind!r}")
def handle(req: dict[str, Any]) -> bool:
global arena_path, arena_bytes, generation, fail_next
typ = req.get("type")
if typ == "initialize":
arena_path = str(req["arena_path"])
arena_bytes = int(req["arena_bytes"])
generation = int(req["generation"])
with open(arena_path, "r+b", buffering=0) as f:
f.truncate(arena_bytes)
emit({"type": "worker_ready", "generation": generation})
return True
if typ == "alloc":
if consume_failure("allocation_failed"):
return True
handle_id = int(req["handle_id"])
dtype = str(req["dtype"])
shape = str(req["shape"])
extent = int(req["extent"])
values, tensor = allocate_tensor(dtype, extent)
objects[handle_id] = DeviceObject(dtype=dtype, shape=shape, extent=extent, values=values, tensor=tensor)
emit({"type": "allocated", "handle_id": handle_id})
return True
if typ in ("host_to_device", "device_to_host"):
if consume_failure("copy_failed"):
return True
copy = PendingCopy(
kind=typ,
copy_id=int(req["copy_id"]),
handle_id=int(req["handle_id"]),
host_offset=int(req["host_offset"]),
device_offset=int(req["device_offset"]),
length=int(req["len"]),
)
if copy.handle_id not in objects:
backend_error("copy_failed")
return True
if bool(req.get("defer", False)):
pending_copies[copy.copy_id] = copy
emit({"type": "copy_started", "copy_id": copy.copy_id})
else:
perform_copy(copy)
emit({"type": "copy_completed", "copy_id": copy.copy_id})
return True
if typ == "complete_copy":
copy_id = int(req["copy_id"])
copy = pending_copies.pop(copy_id, None)
if copy is None:
backend_error("copy_failed")
return True
perform_copy(copy)
emit({"type": "copy_completed", "copy_id": copy_id})
return True
if typ == "wrap_for_tinygrad":
if consume_failure("invalid_view"):
return True
handle_id = int(req["handle_id"])
obj = objects.get(handle_id)
if obj is None:
backend_error("invalid_view")
return True
dtype = str(req["dtype"])
shape = str(req["shape"])
if obj.dtype != dtype or obj.shape != shape:
backend_error("invalid_view")
return True
# Force materialization at view time so success depends on live tensor state.
_ = obj.tensor.tolist()
emit({"type": "view", "handle_id": handle_id, "dtype": dtype, "shape": shape})
return True
if typ == "free":
handle_id = int(req["handle_id"])
if handle_id not in objects:
backend_error("invalid_view")
return True
del objects[handle_id]
emit({"type": "freed", "handle_id": handle_id})
return True
if typ == "restart":
generation = int(req["generation"])
objects.clear()
pending_copies.clear()
fail_next = None
emit({"type": "worker_ready", "generation": generation})
return True
if typ == "fail_next":
fail_next = str(req["failure"])
emit({"type": "ok"})
return True
if typ == "shutdown":
emit({"type": "worker_stopped"})
return False
fatal("protocol_error", f"unknown command type {typ!r}")
return False
def main() -> int:
for line in sys.stdin:
line = line.strip()
if not line:
continue
try:
req = json.loads(line)
if not isinstance(req, dict):
raise ValueError("request must be an object")
if not handle(req):
return 0
except SystemExit:
raise
except Exception as exc:
fatal("backend_exception", str(exc))
return 2
return 0
if __name__ == "__main__":
raise SystemExit(main())

2319
apps/myelin/tests/stateful_vastai.rs
View file

File diff suppressed because it is too large Load diff

3
crates/bindings/python/Cargo.toml
View file

@ -23,3 +23,6 @@ serde_json = "1"
pyo3 = { version = "0.23", features = ["extension-module"] }
pyo3-async-runtimes = { version = "0.23", features = ["tokio-runtime"] }
tokio.workspace = true
[lints]
workspace = true

2
crates/bindings/python/pyproject.toml
View file

@ -8,7 +8,7 @@ version = "0.1.0"
requires-python = ">=3.9"
[dependency-groups]
dev = ["jupyter", "ipykernel", "pytest"]
dev = ["jupyter", "ipykernel", "maturin>=1.7,<2", "pytest", "pytest-timeout"]
[tool.uv]
cache-keys = [

276
crates/bindings/python/src/job.rs
View file

@ -5,7 +5,7 @@ use std::ptr;
use std::sync::{Arc, Mutex, RwLock};
use std::time::Duration;
use data_plane::blob::{ArenaView, Blob, ContentDigest, WritableArenaView};
use data_plane::blob::{Blob, BlobView, ContentDigest, WritableArenaView};
use data_plane::bootstrap as dp_bootstrap;
use data_plane::data_plane::{BlobWriter, DataPlane, DataPlaneBootstrap, parse_actor_address};
use data_plane::path::DataPath;
@ -55,9 +55,6 @@ fn data_plane_error(error: DataPlaneError) -> PyErr {
DataPlaneError::PathNotFound(path) => {
PyErr::new::<DataPathError, _>(format!("data path not found: {path}"))
}
DataPlaneError::PathAlreadyExists(path) => {
PyErr::new::<DataPathError, _>(format!("data path already exists: {path}"))
}
DataPlaneError::Blob(reason) => PyErr::new::<BlobError, _>(format!("{reason:?}")),
DataPlaneError::Attachment(reason) => {
PyErr::new::<SessionError, _>(format!("attachment failed: {reason:?}"))
@ -190,15 +187,15 @@ fn build_child_routing(
let binder = OutboxRouteBinder::new(router, route_transport);
binder.ensure_routable(host_session);
driver.enable_actor_bridge(
runtime.clone(),
codecs,
HashMap::new(),
ActorAddress::default(),
driver.enable_actor_bridge(iroh_driver::ActorBridgeConfig {
runtime: runtime.clone(),
codec: codecs,
routes: HashMap::new(),
swim: ActorAddress::default(),
relay_mirror,
route_view,
outbox,
);
});
driver.install_actor_bridge_pump(ROUTE_POLL);
driver.join(std::slice::from_ref(&host_endpoint));
@ -295,14 +292,14 @@ impl PyBlob {
self.inner.digest().map(digest_hex)
}
fn map(&self, py: Python<'_>) -> PyResult<Py<PyArenaView>> {
fn map(&self, py: Python<'_>) -> PyResult<Py<PyBlobView>> {
let inner = self
.inner
.map()
.map_err(|error| data_plane_error(DataPlaneError::Blob(BlobFailure::from(error))))?;
Py::new(
py,
PyArenaView {
PyBlobView {
inner: Some(inner),
exports: 0,
},
@ -319,17 +316,17 @@ fn digest_hex(digest: &ContentDigest) -> String {
encoded
}
#[pyclass(name = "ArenaView")]
pub struct PyArenaView {
inner: Option<ArenaView>,
#[pyclass(name = "BlobView")]
pub struct PyBlobView {
inner: Option<BlobView>,
exports: usize,
}
#[pymethods]
impl PyArenaView {
impl PyBlobView {
fn __enter__(slf: PyRef<'_, Self>) -> PyResult<PyRef<'_, Self>> {
if slf.inner.is_none() {
return Err(PyBufferError::new_err("arena view is closed"));
return Err(PyBufferError::new_err("blob view is closed"));
}
Ok(slf)
}
@ -347,7 +344,7 @@ impl PyArenaView {
fn close(&mut self) -> PyResult<()> {
if self.exports != 0 {
return Err(PyBufferError::new_err(
"cannot close an arena view with active buffer exports",
"cannot close a blob view with active buffer exports",
));
}
self.inner.take();
@ -364,10 +361,10 @@ impl PyArenaView {
let inner = borrowed
.inner
.as_ref()
.ok_or_else(|| PyBufferError::new_err("arena view is closed"))?;
.ok_or_else(|| PyBufferError::new_err("blob view is closed"))?;
(inner.as_ptr().cast_mut(), inner.len())
};
// SAFETY: the retained `ArenaView` owns the stable mapping and the
// SAFETY: the retained `BlobView` owns the stable mapping and the
// Python buffer owns the cloned `slf` reference until release.
unsafe { fill_buffer(view, flags, pointer, length, true, slf.clone().into_any()) }?;
slf.borrow_mut().exports += 1;
@ -727,8 +724,10 @@ fn run(py: Python<'_>, main: Bound<'_, PyAny>) -> PyResult<()> {
host_session,
capability,
Some(child_node),
attachment_engine,
ROUTE_DEADLINE,
data_plane::data_plane::AttachDeadline {
engine: attachment_engine,
timeout: ROUTE_DEADLINE,
},
))
.map_err(data_plane_error)?;
@ -769,14 +768,101 @@ impl data_plane::host::HostRouteRegistrar for DebugHostRouteRegistrar {
}
}
#[cfg(all(debug_assertions, target_os = "linux"))]
struct DebugBlobSender {
runtime: Runtime,
}
#[cfg(all(debug_assertions, target_os = "linux"))]
impl data_plane::blob_transfer::BlobTransferSender for DebugBlobSender {
fn start_file(
&self,
request: data_plane::blob_transfer::FileTransferRequest,
) -> Result<(), String> {
use std::os::unix::fs::FileExt;
let mut bytes = vec![0_u8; request.length as usize];
request
.file
.read_exact_at(&mut bytes, request.offset)
.map_err(|error| error.to_string())?;
self.runtime
.send_to(
request.offer.destination,
data_plane::blob_transfer::BlobTransferEvent::Chunk {
transfer_id: request.offer.transfer_id,
bytes,
},
)
.map_err(|error| error.to_string())?;
self.runtime
.send_to(
request.offer.destination,
data_plane::blob_transfer::BlobTransferEvent::Finished {
transfer_id: request.offer.transfer_id,
},
)
.map_err(|error| error.to_string())?;
request.completion.complete(Ok(()));
Ok(())
}
}
#[cfg(all(debug_assertions, target_os = "linux"))]
struct DebugBlobReceiver;
#[cfg(all(debug_assertions, target_os = "linux"))]
impl data_plane::blob_transfer::BlobTransferReceiver for DebugBlobReceiver {
fn open(
&self,
destination: ActorAddress,
transfer_id: data_plane::blob_transfer::BlobTransferId,
) -> Result<data_plane::blob_transfer::BlobTransferOffer, String> {
Ok(data_plane::blob_transfer::BlobTransferOffer {
transfer_id,
destination,
failure_proxy: None,
transport: Vec::new(),
})
}
fn cancel(&self, _offer: &data_plane::blob_transfer::BlobTransferOffer) {}
}
#[cfg(all(debug_assertions, target_os = "linux"))]
struct DebugNamespaceDiscovery(ActorAddress);
#[cfg(all(debug_assertions, target_os = "linux"))]
impl data_plane::namespace::NamespaceDiscovery for DebugNamespaceDiscovery {
fn current_directory(&self) -> Option<ActorAddress> {
Some(self.0)
}
}
#[cfg(all(debug_assertions, target_os = "linux"))]
struct DebugSourceRegistrar;
#[cfg(all(debug_assertions, target_os = "linux"))]
impl data_plane::source::BlobSourcePublisher for DebugSourceRegistrar {
fn publish_source(&self, _source: ActorAddress) -> Result<(), String> {
Ok(())
}
}
#[cfg(all(debug_assertions, target_os = "linux"))]
#[pyclass(name = "_TestDataPlaneHost")]
struct PyTestDataPlaneHost {
_driver: Arc<IrohDriver>,
_engine: Engine,
runtime: Runtime,
host_session: ActorAddress,
handoff: data_plane::bootstrap::JobHandoff,
namespace_root: std::path::PathBuf,
}
#[cfg(all(debug_assertions, target_os = "linux"))]
impl Drop for PyTestDataPlaneHost {
fn drop(&mut self) {
self._driver.shutdown();
let _ = std::fs::remove_dir_all(&self.namespace_root);
}
}
#[cfg(all(debug_assertions, target_os = "linux"))]
@ -793,51 +879,6 @@ impl PyTestDataPlaneHost {
fn arena_fd(&self) -> RawFd {
std::os::fd::AsRawFd::as_raw_fd(&self.handoff.arena_fd)
}
fn published<'py>(
&self,
py: Python<'py>,
path: String,
) -> PyResult<Option<Bound<'py, pyo3::types::PyBytes>>> {
let path =
DataPath::parse(path).map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
let published = future::block_on(async {
let ask = self
.runtime
.ask::<
data_plane::protocol::HostSessionIn,
Option<data_plane::protocol::PublishedBlobInfo>,
>(
self.host_session,
|reply_to| data_plane::protocol::HostSessionIn::InspectPublished {
path,
reply_to,
},
)
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
Ok::<_, PyErr>(ask.await)
})?;
let Some(published) = published else {
return Ok(None);
};
// SAFETY: dup returns a fresh descriptor or -1.
let fd = unsafe { libc::dup(std::os::fd::AsRawFd::as_raw_fd(&self.handoff.arena_fd)) };
if fd < 0 {
return Err(PyRuntimeError::new_err(
std::io::Error::last_os_error().to_string(),
));
}
// SAFETY: `fd` is a fresh uniquely-owned duplicate.
let file = std::fs::File::from(unsafe { OwnedFd::from_raw_fd(fd) });
let mut bytes = vec![0_u8; published.metadata.length as usize];
std::os::unix::fs::FileExt::read_exact_at(
&file,
&mut bytes,
published.lease.offset + data_plane::blob::BLOB_HEADER_LEN,
)
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
Ok(Some(pyo3::types::PyBytes::new(py, &bytes)))
}
}
#[cfg(all(debug_assertions, target_os = "linux"))]
@ -896,15 +937,50 @@ fn _test_data_plane_host() -> PyResult<PyTestDataPlaneHost> {
)
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
let capability = JobCapability::new([0x5a; 32]);
let weights: Arc<[u8]> =
Arc::from(include_bytes!("../../../../apps/myelin/jobs/tiny_linear.weights").as_slice());
let namespace_root = std::env::temp_dir().join(format!(
"swactor-python-namespace-{}",
ActorAddress::new_random().to_full_hex()
));
std::fs::create_dir_all(&namespace_root)
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
let source_sender: Arc<dyn data_plane::blob_transfer::BlobTransferSender> =
Arc::new(DebugBlobSender {
runtime: runtime.clone(),
});
let source_publisher: Arc<dyn data_plane::source::BlobSourcePublisher> =
Arc::new(DebugSourceRegistrar);
let namespace_service = data_plane::control::DataNamespaceService::recover(
runtime.clone(),
namespace_root.join("namespace.json"),
Arc::clone(&source_sender),
Arc::clone(&source_publisher),
)
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
let directory = namespace_service.directory();
let weights_path = DataPath::parse("/models/tiny-linear/weights")
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
let blobs = std::collections::BTreeMap::new();
let weights_file = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../../apps/myelin/jobs/tiny_linear.weights");
future::block_on(
namespace_service
.control()
.register(weights_path, data_plane::blob::file(weights_file)),
)
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
let namespace_proxy = runtime
.spawn(data_plane::namespace::NamespaceClientActor::new(
engine.handle(),
runtime.create_sender(),
Arc::new(DebugNamespaceDiscovery(directory)),
ROUTE_POLL,
))
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
let namespace = data_plane::namespace::NamespaceClient::new(runtime.clone(), namespace_proxy);
let host_session = runtime
.spawn(
data_plane::host::HostDataPlaneSessionActor::new(
data_plane::host::HostDataPlaneConfig {
runtime: runtime.clone(),
arena,
arena_generation: 1,
session_generation: 1,
@ -914,13 +990,18 @@ fn _test_data_plane_host() -> PyResult<PyTestDataPlaneHost> {
read_prefixes: vec![
DataPath::parse("/models")
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?,
DataPath::parse("/runs/test-run/results")
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?,
],
write_prefixes: vec![
DataPath::parse("/runs/test-run/results")
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?,
],
},
blobs,
namespace: Some(namespace),
transfer_receiver: Some(Arc::new(DebugBlobReceiver)),
source_sender: Some(source_sender),
source_publisher: Some(source_publisher),
route_registrar: Some(registrar),
},
)
@ -928,48 +1009,16 @@ fn _test_data_plane_host() -> PyResult<PyTestDataPlaneHost> {
)
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
data_plane::host::install_session_env(&mut handoff, host_session, capability);
future::block_on(async {
runtime
.ask::<data_plane::protocol::HostSessionIn, Result<(), DataPlaneError>>(
host_session,
|reply_to| data_plane::protocol::HostSessionIn::BeginBlobSource {
path: weights_path.clone(),
metadata: data_plane::blob::BlobMetadata {
length: weights.len() as u64,
digest: Some(ContentDigest::sha256(&weights)),
},
reply_to,
},
)
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?
.await
.map_err(data_plane_error)
})?;
runtime
.send_to(
host_session,
data_plane::protocol::HostSessionIn::BlobSourceChunk {
path: weights_path.clone(),
bytes: weights.to_vec(),
},
)
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
runtime
.send_to(
host_session,
data_plane::protocol::HostSessionIn::FinishBlobSource { path: weights_path },
)
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
driver.enable_actor_bridge(
runtime.clone(),
codecs,
HashMap::new(),
ActorAddress::default(),
driver.enable_actor_bridge(iroh_driver::ActorBridgeConfig {
runtime: runtime.clone(),
codec: codecs,
routes: HashMap::new(),
swim: ActorAddress::default(),
relay_mirror,
route_view,
outbox,
);
});
driver.install_actor_bridge_pump(ROUTE_POLL);
handoff.env.insert(
dp_bootstrap::ENV_DATA_PLANE_ENDPOINT.to_owned(),
@ -980,9 +1029,8 @@ fn _test_data_plane_host() -> PyResult<PyTestDataPlaneHost> {
Ok(PyTestDataPlaneHost {
_driver: Arc::new(driver),
_engine: engine,
runtime,
host_session,
handoff,
namespace_root,
})
}
@ -994,7 +1042,7 @@ pub fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
module.add("SessionError", module.py().get_type::<SessionError>())?;
module.add_class::<PyDataPlane>()?;
module.add_class::<PyBlob>()?;
module.add_class::<PyArenaView>()?;
module.add_class::<PyBlobView>()?;
module.add_class::<PyStreamWriter>()?;
module.add_class::<PyContext>()?;
module.add_function(wrap_pyfunction!(run, module)?)?;

14
crates/bindings/python/tests/test_bootstrap.py
View file

@ -126,9 +126,9 @@ def test_run_attaches_before_main_and_maps_blob_buffer_directly(monkeypatch, hos
blob = await ctx.data.read_blob("/models/tiny-linear/weights")
assert isinstance(blob, swactor.Blob)
assert blob.length == 24
assert len(blob.digest) == 64
assert blob.digest is None
with blob.map() as mapped:
assert isinstance(mapped, swactor.ArenaView)
assert isinstance(mapped, swactor.BlobView)
assert struct.unpack_from("<6f", mapped) == pytest.approx(
(1.5, -2.0, 0.5, 4.0, 0.25, -0.75)
)
@ -194,9 +194,13 @@ def test_write_blob_seals_cleanly_and_exception_aborts(monkeypatch, host):
except AbortWrite:
pass
complete = await ctx.data.read_blob("/runs/self/results/complete")
with complete.map() as mapped:
assert bytes(mapped) == b"result"
with pytest.raises(swactor.DataPathError, match="not found"):
await ctx.data.read_blob("/runs/self/results/aborted")
swactor.run(main)
assert bytes(host.published("/runs/self/results/complete")) == b"result"
assert host.published("/runs/self/results/aborted") is None
def test_missing_path_and_authorization_are_typed(monkeypatch, host):
@ -206,7 +210,7 @@ def test_missing_path_and_authorization_are_typed(monkeypatch, host):
with pytest.raises(swactor.DataPathError, match="not found"):
await ctx.data.read_blob("/models/missing")
with pytest.raises(PermissionError):
await ctx.data.read_blob("/runs/self/results/private")
await ctx.data.read_blob("/runs/self/private")
swactor.run(main)

41
crates/bindings/python/uv.lock
View file

@ -1282,6 +1282,30 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/af/33/ee4519fa02ed11a94aef9559552f3b17bb863f2ecfe1a35dc7f548cde231/matplotlib_inline-0.2.1-py3-none-any.whl", hash = "sha256:d56ce5156ba6085e00a9d54fead6ed29a9c47e215cd1bba2e976ef39f5710a76", size = 9516, upload-time = "2025-10-23T09:00:20.675Z" },
]
[[package]]
name = "maturin"
version = "1.14.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "tomli", marker = "python_full_version < '3.11'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/e7/b3/addd877f871fb1860d46d3a4f206ecb10b946c85846805e6367631926fd3/maturin-1.14.1.tar.gz", hash = "sha256:9d6577a62cd08e0ceba7a0db06fb098e0c9b1b3429bad747a4f3a18215a1b3df", size = 369637, upload-time = "2026-06-19T05:19:49.774Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/f4/f0/97c5a5bd9c71653a066c0976a484eaaae50b9369557838a4176b7b0bdaa5/maturin-1.14.1-py3-none-linux_armv6l.whl", hash = "sha256:522292398945442cdafa9daeb2271b2340fbde57027b818f923f88eab04174f8", size = 10207496, upload-time = "2026-06-19T05:19:09.321Z" },
{ url = "https://files.pythonhosted.org/packages/fe/83/294bca639b0e052f1e2f65199b3db258780c7d4e31408b934c9c974a1379/maturin-1.14.1-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:ffe5ad71f21d1e6603c4dd75f7fee34adf5ed5ebcebb692886549888ebb329ed", size = 19680113, upload-time = "2026-06-19T05:19:13.43Z" },
{ url = "https://files.pythonhosted.org/packages/43/b6/79c881410a3b1c187f7eb3d407aecae646c6a4433d630d72200359015e83/maturin-1.14.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:f3306078070c1508fd715b9116070cbcaff5959024272a9f1e6f5cb29768b86c", size = 10169205, upload-time = "2026-06-19T05:19:16.615Z" },
{ url = "https://files.pythonhosted.org/packages/93/9d/44b6f26dcb7f7a04c5501ac2dbb6ca1490150682baa525ca5860504f9eab/maturin-1.14.1-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:cd457cd88961156e26379e1155bd287cc0ec1c8b2f1582b0660fb31b87c8842d", size = 10188098, upload-time = "2026-06-19T05:19:19.736Z" },
{ url = "https://files.pythonhosted.org/packages/1a/bd/9c0d5d6983905ce2c9edaa073a7e89355a9cf7f396988e05d32f1c37785d/maturin-1.14.1-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:dfc54ae32e6fcb18302193ab9a30b0b25eefffba994ae13238974805533ef75e", size = 10627576, upload-time = "2026-06-19T05:19:22.713Z" },
{ url = "https://files.pythonhosted.org/packages/e5/33/b096412bd6a7cb399652b260666f901adf88a687181a6dbd6a3f89f0a94e/maturin-1.14.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:a131d912b5267e640bc96d70f4914e10590aed64082ec9abacba7cea52004224", size = 10085181, upload-time = "2026-06-19T05:19:25.69Z" },
{ url = "https://files.pythonhosted.org/packages/56/8d/08c3bf469c38a23c9e6c877e338193001eb604d010fedc08341974e38528/maturin-1.14.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:be18fc568fb76884c0205456336892a75105ec398e6b667cd777c6268bd06d69", size = 10026363, upload-time = "2026-06-19T05:19:28.904Z" },
{ url = "https://files.pythonhosted.org/packages/3a/a4/c4d1a92839f8745ab4aab988a7db884a79d6d710bd3b286fcf9316dece1a/maturin-1.14.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:994a0c8ba3ad8a92b3a9ee1b02645d200d610216b15cff5102b0fe65e8e08666", size = 13321347, upload-time = "2026-06-19T05:19:32.411Z" },
{ url = "https://files.pythonhosted.org/packages/b3/fa/170f04624d03fd07d2a8b1b67de83a127af93aef9eaa425839553347297b/maturin-1.14.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:be80866363e605d137991b491a741a84cde9ae350183c4c85f49690ca9aaaa65", size = 10877609, upload-time = "2026-06-19T05:19:35.448Z" },
{ url = "https://files.pythonhosted.org/packages/61/ad/1ae2e1d0ded282bf2c55ac13f0811d87deb425e200ae64a15785675dede9/maturin-1.14.1-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:5282dffd4b539d2be245f4e5b1a5ab6bc1033b58f4a4872f5833f9d43c954aa4", size = 10417316, upload-time = "2026-06-19T05:19:38.28Z" },
{ url = "https://files.pythonhosted.org/packages/fb/27/bf677183920718da49cd7982d6a3ffc440aad8919329f571d189f81b7bdf/maturin-1.14.1-py3-none-win32.whl", hash = "sha256:1a04de0a20188f95c721b5702eed18140bdcccb28c386797093eca3f62f4d4e0", size = 8931293, upload-time = "2026-06-19T05:19:41.183Z" },
{ url = "https://files.pythonhosted.org/packages/63/4b/585adeb9167b08d3cdff0032a938b0e72655c92003df4f52c3f696a1bcc2/maturin-1.14.1-py3-none-win_amd64.whl", hash = "sha256:3c9f94640ecc4895e94abaf834a0684430032c865b2748a36c12461fd9252fdd", size = 10314067, upload-time = "2026-06-19T05:19:44.389Z" },
{ url = "https://files.pythonhosted.org/packages/51/d4/dac8c0720ae246be1700afb6fbdbbea20fe35b13f6570b2f70faa005df77/maturin-1.14.1-py3-none-win_arm64.whl", hash = "sha256:15cea8fcb3ba47dd636f50092bb34baea8b04ac777392f23e6bf8a9a61efb894", size = 9718943, upload-time = "2026-06-19T05:19:47.49Z" },
]
[[package]]
name = "mistune"
version = "3.2.0"
@ -1644,6 +1668,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" },
]
[[package]]
name = "pytest-timeout"
version = "2.4.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
{ name = "pytest", version = "9.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/ac/82/4c9ecabab13363e72d880f2fb504c5f750433b2b6f16e99f4ec21ada284c/pytest_timeout-2.4.0.tar.gz", hash = "sha256:7e68e90b01f9eff71332b25001f85c75495fc4e3a836701876183c4bcfd0540a", size = 17973, upload-time = "2025-05-05T19:44:34.99Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2", size = 14382, upload-time = "2025-05-05T19:44:33.502Z" },
]
[[package]]
name = "python-dateutil"
version = "2.9.0.post0"
@ -2313,8 +2350,10 @@ dev = [
{ name = "ipykernel", version = "6.31.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
{ name = "ipykernel", version = "7.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
{ name = "jupyter" },
{ name = "maturin" },
{ name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
{ name = "pytest", version = "9.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
{ name = "pytest-timeout" },
]
[package.metadata]
@ -2323,7 +2362,9 @@ dev = [
dev = [
{ name = "ipykernel" },
{ name = "jupyter" },
{ name = "maturin", specifier = ">=1.7,<2" },
{ name = "pytest" },
{ name = "pytest-timeout" },
]
[[package]]

3
crates/bindings/wasm-runtime/Cargo.toml
View file

@ -11,3 +11,6 @@ crate-type = ["cdylib"]
swactor = { path = "../../..", default-features = false, features = ["wasm", "std"] }
swactor-engine = { path = "../../engine", default-features = false }
wasm-bindgen = "0.2"
[lints]
workspace = true

6
crates/bindings/wasm-runtime/src/lib.rs
View file

@ -103,6 +103,12 @@ pub struct WasmRuntime {
backend: SteppingBackend,
}
impl Default for WasmRuntime {
fn default() -> Self {
Self::new()
}
}
#[wasm_bindgen]
impl WasmRuntime {
#[wasm_bindgen(constructor)]

3
crates/dashboard/Cargo.toml
View file

@ -23,3 +23,6 @@ demo-control = []
[dev-dependencies]
proptest = "1"
tower = { version = "0.5", features = ["util"] }
[lints]
workspace = true

7
crates/dashboard/src/control.rs
View file

@ -187,9 +187,10 @@ mod properties {
concurrent in any::<bool>(),
destination_disappears in any::<bool>(),
) {
let mut config = RuntimeConfig::default();
config.worker_count = 1;
let parts = RuntimeParts::new(config);
let parts = RuntimeParts::new(RuntimeConfig {
worker_count: 1,
..RuntimeConfig::default()
});
let runtime = parts.runtime().clone();
let backend = SteppingBackend::new();
let _engine =

208
crates/dashboard/src/control_plane.rs
View file

@ -236,7 +236,9 @@ struct NodeCard {
errors: Vec<String>,
cpu: Option<CpuSnapshot>,
gpu: Option<GpuSnapshot>,
memory: Option<telemetry::hardware::memory::HostMemorySample>,
net: Option<crate::hardware_view::NetSnapshot>,
storage: Option<telemetry::hardware::storage::HostStorageSample>,
process: Option<crate::hardware_view::ProcessSnapshot>,
history: Vec<HardwareHistorySnapshot>,
actor_summary: ActorSummarySnapshot,
@ -329,15 +331,9 @@ fn stream_key(stream: &StreamEvent) -> String {
fn node_card(node: &FusedNode, now: Instant) -> NodeCard {
let summary = node.hardware.summary();
let totals = node.actors.totals();
let mut roster: Vec<RosterRow> = node.actors.actors.values().map(roster_row).collect();
// Busiest actors first; ties fall back to address for stable rendering.
roster.sort_by(|left, right| {
right
.msg_per_sec
.partial_cmp(&left.msg_per_sec)
.map_or(std::cmp::Ordering::Equal, |order| order)
.then_with(|| left.address.cmp(&right.address))
});
// Address-keyed map order is the stable default. Volatile telemetry must
// not move a row out from under the pointer; the page offers explicit sorts.
let roster: Vec<RosterRow> = node.actors.actors.values().map(roster_row).collect();
NodeCard {
stream: StreamKeySnapshot {
key: stream_key(&node.stream),
@ -352,7 +348,9 @@ fn node_card(node: &FusedNode, now: Instant) -> NodeCard {
errors: node.hardware.errors(),
cpu: node.hardware.cpu.as_ref().map(CpuSnapshot::from),
gpu: node.hardware.gpu.as_ref().map(GpuSnapshot::from),
memory: node.hardware.memory.clone(),
net: node.hardware.net.clone(),
storage: node.hardware.storage.clone(),
process: node.hardware.process.clone(),
history: node
.hardware
@ -362,11 +360,16 @@ fn node_card(node: &FusedNode, now: Instant) -> NodeCard {
ms_ago: duration_ms(now.duration_since(sample.at)),
sample_unix_ms: sample.sample_unix_ms,
cpu_total_percent: sample.cpu_total_percent,
cpu_cores_percent: sample.cpu_cores_percent.clone(),
gpu_max_percent: sample.gpu_max_percent,
gpu_memory_used_mib: sample.gpu_memory_used_mib,
gpu_memory_total_mib: sample.gpu_memory_total_mib,
net_rx_bps: sample.net_rx_bps,
net_tx_bps: sample.net_tx_bps,
memory_used_percent: sample.memory_used_percent,
memory_pressure_some_avg10: sample.memory_pressure_some_avg10,
storage_used_percent: sample.storage_used_percent,
io_pressure_some_avg10: sample.io_pressure_some_avg10,
})
.collect(),
actor_summary: ActorSummarySnapshot {
@ -667,6 +670,191 @@ mod tests {
assert_eq!(still_live.first(), Some(&"orch"));
}
#[test]
fn hardware_channels_fold_into_fleet_snapshot() {
let view = ControlPlaneView::default();
let stream = StreamId::new(NodeId::new("worker"), Lifetime(1));
let cpu = json!({
"schema":"host.cpu.v1",
"seq":1,
"sample_unix_ms":1_000,
"query_elapsed_ms":1,
"host":{
"logical_cpus":8,
"total_percent":42.5,
"idle_percent":57.5,
"iowait_percent":0.0,
"steal_percent":0.0,
"load1":1.0,
"load5":0.5,
"load15":0.25
},
"cores":[
{"index":0,"total_percent":25.0,"idle_percent":75.0,"iowait_percent":0.0,"steal_percent":0.0},
{"index":1,"total_percent":60.0,"idle_percent":40.0,"iowait_percent":0.0,"steal_percent":0.0}
],
"processes":[],
"error":null
});
let gpu = json!({
"schema":"host.gpu.v1",
"seq":1,
"sample_unix_ms":1_000,
"query_elapsed_ms":2,
"gpus":[{
"index":0,
"uuid":"gpu-0",
"name":"test gpu",
"memory_used_mib":512,
"memory_total_mib":4096,
"utilization_gpu_percent":71,
"utilization_memory_percent":12,
"temperature_c":55,
"power_draw_w":25.0
}],
"processes":[],
"error":null
});
let memory = json!({
"schema":"host.memory.v1",
"seq":1,
"sample_unix_ms":2_000,
"query_elapsed_ms":1,
"total_bytes":16_000,
"available_bytes":4_000,
"used_bytes":12_000,
"cached_bytes":2_000,
"swap_total_bytes":8_000,
"swap_used_bytes":1_000,
"pressure":{
"some_avg10":1.25,
"some_avg60":0.75,
"some_avg300":0.5,
"some_total_us":100,
"full_avg10":0.1,
"full_avg60":0.05,
"full_avg300":0.01,
"full_total_us":10
},
"error":null
});
let net_sample = |seq, sample_unix_ms, rx_bytes, tx_bytes| {
json!({
"schema":"host.net.v1",
"seq":seq,
"sample_unix_ms":sample_unix_ms,
"interfaces":[{
"name":"eth0",
"rx_bytes":rx_bytes,
"tx_bytes":tx_bytes,
"rx_packets":10,
"tx_packets":10,
"rx_errors":0,
"tx_errors":0,
"rx_dropped":0,
"tx_dropped":0
}],
"error":null
})
};
let storage = json!({
"schema":"host.storage.v1",
"seq":1,
"sample_unix_ms":2_000,
"query_elapsed_ms":1,
"filesystems":[{
"mount":"/",
"total_bytes":100_000,
"used_bytes":80_000,
"available_bytes":20_000,
"used_percent":80.0
}],
"pressure":{
"some_avg10":2.5,
"some_avg60":1.5,
"some_avg300":0.5,
"some_total_us":200,
"full_avg10":0.2,
"full_avg60":0.1,
"full_avg300":0.05,
"full_total_us":20
},
"error":null
});
for (position, channel, payload) in [
(0, "host.cpu", cpu),
(1, "host.gpu", gpu),
(2, "host.memory", memory),
(3, "host.net", net_sample(0, 1_000, 1_000, 2_000)),
(4, "host.net", net_sample(1, 2_000, 2_000, 3_500)),
(5, "host.storage", storage),
] {
ingest_json(
&view,
&stream,
position,
channel,
serde_json::to_vec(&payload).expect("hardware payload"),
);
}
let snapshot = view.snapshot_json();
let node = &snapshot["live"][0];
assert_eq!(node["cpu"]["host"]["total_percent"], json!(42.5));
assert_eq!(node["gpu"]["gpus"][0]["utilization_gpu_percent"], json!(71));
assert_eq!(node["cpu"]["cores"][1]["total_percent"], json!(60.0));
assert_eq!(node["memory"]["used_bytes"], json!(12_000));
assert_eq!(node["net"]["interfaces"][0]["rx_bps"], json!(1_000.0));
assert_eq!(node["net"]["interfaces"][0]["tx_bps"], json!(1_500.0));
assert_eq!(
node["storage"]["filesystems"][0]["used_percent"],
json!(80.0)
);
assert_eq!(node["history"][0]["cpu_cores_percent"], json!([25.0, 60.0]));
assert_eq!(
node["history"][0]["memory_pressure_some_avg10"],
json!(1.25)
);
assert_eq!(node["history"][0]["io_pressure_some_avg10"], json!(2.5));
assert_eq!(node["last_sample_unix_ms"], json!(2_000));
assert!(node["errors"].as_array().expect("errors").is_empty());
}
#[test]
fn roster_default_order_does_not_follow_volatile_throughput() {
let view = ControlPlaneView::default();
let stream = StreamId::new(NodeId::new("node"), Lifetime(1));
ingest_json(
&view,
&stream,
0,
"runtime.actors",
actors_payload(
0,
json!([
{ "address": "zz", "messages_processed": 100 },
{ "address": "aa", "messages_processed": 1 }
]),
),
);
{
let mut state = view.state.write();
let actors = &mut state.streams.get_mut("node#1").expect("node").actors.actors;
actors.get_mut("zz").expect("zz actor").msg_per_sec = 10_000.0;
actors.get_mut("aa").expect("aa actor").msg_per_sec = 1.0;
}
let snapshot = view.snapshot_json();
let addresses: Vec<&str> = snapshot["live"][0]["roster"]
.as_array()
.expect("roster")
.iter()
.map(|actor| actor["address"].as_str().expect("address"))
.collect();
assert_eq!(addresses, vec!["aa", "zz"]);
}
#[test]
fn newer_life_generation_evicts_superseded_stream() {
let view = ControlPlaneView::default();
@ -701,7 +889,7 @@ mod tests {
fn stale_pool_is_hard_capped() {
let view = ControlPlaneView::default();
for index in 0..(STALE_POOL_CAP as u64 + 5) {
let stream = StreamId::new(NodeId::new(&format!("old-{index}")), Lifetime(1));
let stream = StreamId::new(NodeId::new(format!("old-{index}")), Lifetime(1));
ingest_json(
&view,
&stream,

286
crates/dashboard/src/control_plane_page.html
View file

@ -87,6 +87,12 @@
table { width: 100%; border-collapse: collapse; font-family: var(--mono); font-size: 12px; }
th, td { text-align: left; padding: 7px 8px; border-bottom: 1px solid var(--divider); font-variant-numeric: tabular-nums; }
th { color: var(--amber); font-weight: 600; font-size: 10px; text-transform: uppercase; letter-spacing: .08em; }
.roster-sort-controls { display: inline-flex; gap: 2px; margin-left: 4px; vertical-align: middle; }
.roster-sort-button { position: relative; width: 14px; height: 14px; padding: 0; color: var(--muted); background: transparent; border: 1px solid transparent; border-radius: var(--r); font-size: 0; line-height: 0; cursor: pointer; }
.roster-sort-button::before { content: ""; position: absolute; left: 3px; width: 0; height: 0; border-left: 3px solid transparent; border-right: 3px solid transparent; }
.roster-sort-button[data-direction="asc"]::before { top: 3px; border-bottom: 5px solid currentColor; }
.roster-sort-button[data-direction="desc"]::before { bottom: 3px; border-top: 5px solid currentColor; }
.roster-sort-button:hover, .roster-sort-button[aria-pressed="true"] { color: var(--cyan); border-color: var(--cyan); background: var(--panel-hover); }
tbody tr[data-addr] { cursor: pointer; transition: background var(--t); }
tbody tr[data-addr]:hover { background: var(--row-hover); }
tbody tr[data-selected="true"] { background: var(--selected); box-shadow: inset 2px 0 0 var(--selected-edge); }
@ -102,6 +108,33 @@
.empty { padding: 24px; border: 1px dashed var(--border); border-radius: var(--r); color: var(--muted); font-family: var(--mono); font-size: 12px; }
.notice { font-size: 12px; color: var(--muted); margin: 6px 0; }
canvas { width: 100%; height: 64px; background: var(--inset); border: 1px solid var(--border); border-radius: var(--r); }
.hardware-source { margin-bottom: 8px; }
.hardware-grid { display: grid; grid-template-columns: repeat(12, minmax(0, 1fr)); gap: 8px; }
.hw-card { min-width: 0; padding: 10px; background: var(--inset); border: 1px solid var(--divider); border-radius: var(--r); }
.hw-card.cpu { grid-column: span 7; }
.hw-card.memory { grid-column: span 5; }
.hw-card.storage, .hw-card.gpu, .hw-card.network { grid-column: span 4; }
.hw-card-head { display: flex; align-items: baseline; justify-content: space-between; gap: 8px; margin-bottom: 7px; }
.hw-card-title { color: var(--muted); font: 700 10px/1.2 var(--mono); text-transform: uppercase; letter-spacing: .09em; }
.hw-card-value { color: var(--text); font: 700 16px/1 var(--mono); font-variant-numeric: tabular-nums; }
.hw-card-subtitle { margin-top: 5px; color: var(--muted); font: 11px/1.35 var(--mono); }
.metric-pair { display: flex; justify-content: space-between; gap: 8px; margin-top: 6px; color: var(--muted); font: 11px/1.3 var(--mono); }
.metric-pair strong { color: var(--text); font-weight: 600; }
.pressure-value.ok { color: var(--ok); } .pressure-value.warn { color: var(--amber); } .pressure-value.bad { color: var(--bad); }
canvas.thread-graph { height: 72px; background: var(--bg); border-color: var(--divider); image-rendering: pixelated; }
.thread-strip { display: grid; grid-template-columns: repeat(auto-fit, minmax(26px, 1fr)); gap: 3px; margin-top: 5px; }
.thread-cell { min-width: 0; height: 20px; display: grid; place-items: center; border: 1px solid var(--divider); border-radius: var(--r); color: var(--text); font: 9px/1 var(--mono); font-variant-numeric: tabular-nums; }
.network-list { display: grid; gap: 6px; }
.network-row { display: grid; grid-template-columns: minmax(0, 1fr) auto; gap: 8px; font: 11px/1.25 var(--mono); }
.network-row .interface { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; color: var(--muted); }
.hw-empty { color: var(--muted); font: 11px/1.35 var(--mono); }
@media (max-width: 900px) {
.hw-card.cpu { grid-column: span 12; }
.hw-card.memory, .hw-card.storage, .hw-card.gpu, .hw-card.network { grid-column: span 6; }
}
@media (max-width: 600px) {
.hw-card.cpu, .hw-card.memory, .hw-card.storage, .hw-card.gpu, .hw-card.network { grid-column: span 12; }
}
.err { color: var(--bad); }
@media (prefers-reduced-motion: reduce) { * { transition-duration: 0.01ms !important; animation: none !important; } }
</style>
@ -134,6 +167,8 @@ const params = new URLSearchParams(window.location.search);
let selectedStream = params.get('stream') || null;
let selectedActor = params.get('actor') || null;
let rosterFilter = '';
let rosterSort = { key: 'address', direction: 'asc' };
let hardwareSource = 'node';
let lastSnapshot = null;
let detailTimer = null;
// A1: last-rendered HTML per region. A poll that yields identical markup
@ -144,6 +179,14 @@ let lastRosterHtml = null;
let lastMachineHtml = null;
let lastDossierHtml = null;
window.addEventListener('dashboard-hardware-source', event => {
const next = event.detail?.source === 'orchestrator' ? 'orchestrator' : 'node';
if (next === hardwareSource) return;
hardwareSource = next;
lastMachineHtml = null;
if (lastSnapshot) render();
});
// Receipt ages (dossier) render as empty spans carrying an absolute epoch —
// stable across polls — and a 1 Hz pass rewrites their textContent. All other
// ticking text (card seen/stats) is written per-poll by updateCardTexts.
@ -173,6 +216,17 @@ function fmtRate(value) {
if (n >= 1e3) return (n / 1e3).toFixed(1) + ' k';
return n.toFixed(n < 10 ? 1 : 0);
}
function fmtBytes(value) {
const n = Number(value);
if (!Number.isFinite(n)) return '—';
const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
let scaled = Math.max(0, n), unit = 0;
while (scaled >= 1024 && unit < units.length - 1) {
scaled /= 1024;
unit += 1;
}
return scaled.toFixed(scaled < 10 && unit > 0 ? 1 : 0) + ' ' + units[unit];
}
function typeShort(name) {
if (!name) return '—';
const parts = String(name).split('::');
@ -291,6 +345,14 @@ function updateCardTexts(nodes) {
}
}
function hardwareNodeFor(node, live, stale) {
if (hardwareSource !== 'orchestrator' || node.stream.origin === 'orchestrator') {
return { node, mirrored: false };
}
const orchestrator = live.concat(stale).find(candidate => candidate.stream.origin === 'orchestrator');
return orchestrator ? { node: orchestrator, mirrored: true } : { node, mirrored: false };
}
function renderNode(page, node, live, stale) {
const summary = node.actor_summary || {};
const active = document.activeElement;
@ -333,7 +395,8 @@ function renderNode(page, node, live, stale) {
if (actorCount) actorCount.textContent = fmt(summary.actors);
const seen = page.querySelector('h2 [data-seen]');
if (seen) seen.textContent = node.live ? 'live' : 'stale ' + ago(node.last_seen_ms_ago) + ' ago';
const machine = machineDetail(node);
const machineSource = hardwareNodeFor(node, live, stale);
const machine = machineDetail(machineSource.node, machineSource.mirrored);
const machineSlot = document.getElementById('machine-detail');
if (machineSlot && machine !== lastMachineHtml) {
machineSlot.innerHTML = machine;
@ -347,67 +410,192 @@ function renderNode(page, node, live, stale) {
}
}
function machineDetail(node) {
const cpu = node.cpu, gpu = node.gpu, net = node.net;
const rows = [];
const cpuTotal = cpu && cpu.host ? cpu.host.total_percent : null;
rows.push(`<div class="bar-row"><span class="name">CPU</span>${bar(cpuTotal)}<span class="num">${cpuTotal == null ? '—' : fmt(cpuTotal, 1) + '%'}</span></div>`);
if (gpu && gpu.gpus) {
gpu.gpus.forEach((device, index) => {
const util = device.utilization_gpu_percent;
const used = device.memory_used_mib || 0, total = device.memory_total_mib || 0;
rows.push(`<div class="bar-row"><span class="name">GPU ${fmt(index)}</span>${bar(util)}<span class="num">${util == null ? '—' : fmt(util) + '%'}</span></div>`);
if (total) rows.push(`<div class="bar-row"><span class="name">&nbsp;</span>${bar(used, total)}<span class="num">${fmt(used)}/${fmt(total)} MiB</span></div>`);
});
}
if (net && net.interfaces) {
net.interfaces.slice(0, 4).forEach(nic => {
rows.push(`<div class="bar-row"><span class="name" title="${esc(nic.name)}">${esc(nic.name.slice(0, 12))}</span><span class="num" style="width:auto;flex:1">${fmtRate(nic.rx_bps)}↓ ${fmtRate(nic.tx_bps)}↑</span></div>`);
});
}
if (node.history && node.history.length) drawMachineHistory(node.history);
const errors = (node.errors || []).map(e => `<div class="notice err">${esc(e)}</div>`).join('');
return `<div class="bars">${rows.join('')}</div><canvas id="machine-spark" width="600" height="64"></canvas>${errors}`;
function percentOf(used, total) {
const numerator = Number(used), denominator = Number(total);
return Number.isFinite(numerator) && Number.isFinite(denominator) && denominator > 0
? numerator * 100 / denominator
: null;
}
function drawMachineHistory(history) {
function pressureTone(value, warn = 1, bad = 5) {
const pressure = Number(value);
if (!Number.isFinite(pressure)) return '';
return pressure >= bad ? 'bad' : pressure >= warn ? 'warn' : 'ok';
}
function machineDetail(node, mirrored = false) {
const cpu = node.cpu, gpu = node.gpu, memory = node.memory, net = node.net, storage = node.storage;
const cpuHost = cpu?.host;
const cpuTotal = cpuHost?.total_percent;
const cores = [...(cpu?.cores || [])].sort((left, right) => left.index - right.index);
const threadHeight = Math.min(92, Math.max(42, cores.length * 4));
drawThreadGraph(node.history || [], cores.map(core => core.total_percent));
const threadCells = cores.map(core => {
const value = core.total_percent;
const load = value == null ? 0 : Math.max(0, Math.min(100, Number(value)));
const label = value == null ? '—' : fmt(value);
return `<span class="thread-cell" title="Thread ${fmt(core.index)} · ${value == null ? 'no sample' : fmt(value, 1) + '%'}" style="background:linear-gradient(to top,var(--ok) 0%,var(--ok) ${load}%,var(--bg) ${load}%,var(--bg) 100%)">${label}</span>`;
}).join('');
const load = [cpuHost?.load1, cpuHost?.load5, cpuHost?.load15].map(value => fmt(value, 2)).join(' / ');
const cpuLabel = cpuTotal == null ? '—' : fmt(cpuTotal, 1) + '%';
const memoryPercent = percentOf(memory?.used_bytes, memory?.total_bytes);
const memoryPressure = memory?.pressure?.some_avg10;
const swapPercent = percentOf(memory?.swap_used_bytes, memory?.swap_total_bytes);
const memoryLabel = memoryPercent == null ? '—' : fmt(memoryPercent, 1) + '%';
const filesystem = storage?.filesystems?.[0];
const storagePercent = filesystem?.used_percent;
const ioPressure = storage?.pressure?.some_avg10;
const storageLabel = storagePercent == null ? '—' : fmt(storagePercent, 1) + '%';
const gpuDevices = gpu?.gpus || [];
const gpuMax = gpuDevices.reduce((maximum, device) => {
const value = Number(device.utilization_gpu_percent);
return Number.isFinite(value) ? Math.max(maximum, value) : maximum;
}, 0);
const gpuRows = gpuDevices.map((device, index) => {
const utilization = device.utilization_gpu_percent;
const memoryPercent = percentOf(device.memory_used_mib, device.memory_total_mib);
return `<div class="metric-pair"><span>GPU ${fmt(index)}</span><strong>${utilization == null ? '—' : fmt(utilization) + '%'}</strong></div>
${bar(utilization)}
<div class="metric-pair"><span>Memory</span><strong>${device.memory_total_mib ? fmt(device.memory_used_mib) + ' / ' + fmt(device.memory_total_mib) + ' MiB' : '—'}</strong></div>
${device.memory_total_mib ? bar(memoryPercent) : ''}`;
}).join('');
const networkRows = (net?.interfaces || []).slice(0, 4).map(nic => `
<div class="network-row">
<span class="interface" title="${esc(nic.name)}">${esc(nic.name)}</span>
<span>${fmtRate(nic.rx_bps)}↓&nbsp; ${fmtRate(nic.tx_bps)}↑</span>
</div>`).join('');
const source = mirrored ? '<div class="hardware-source muted">Mock node · local orchestrator hardware</div>' : '';
const errors = (node.errors || []).map(error => `<div class="notice err">${esc(error)}</div>`).join('');
return `${source}<div class="hardware-grid">
<section class="hw-card cpu">
<div class="hw-card-head"><span class="hw-card-title">CPU · ${fmt(cores.length)} threads</span><span class="hw-card-value">${cpuLabel}</span></div>
<canvas id="thread-graph" class="thread-graph" width="560" height="${threadHeight}" style="height:${threadHeight}px" role="img" aria-label="Per-thread CPU utilization history"></canvas>
<div class="thread-strip">${threadCells || '<span class="hw-empty">Waiting for per-thread samples</span>'}</div>
<div class="hw-card-subtitle">Load 1 / 5 / 15 min&nbsp; ${load}</div>
</section>
<section class="hw-card memory">
<div class="hw-card-head"><span class="hw-card-title">Memory</span><span class="hw-card-value">${memoryLabel}</span></div>
${bar(memoryPercent)}
<div class="metric-pair"><span>Used</span><strong>${fmtBytes(memory?.used_bytes)} / ${fmtBytes(memory?.total_bytes)}</strong></div>
<div class="metric-pair"><span>Available</span><strong>${fmtBytes(memory?.available_bytes)}</strong></div>
<div class="metric-pair"><span>Swap</span><strong>${swapPercent == null ? '—' : fmt(swapPercent, 1) + '%'}</strong></div>
<div class="metric-pair"><span>PSI some · 10s</span><strong class="pressure-value ${pressureTone(memoryPressure)}">${memoryPressure == null ? '—' : fmt(memoryPressure, 2) + '%'}</strong></div>
</section>
<section class="hw-card storage">
<div class="hw-card-head"><span class="hw-card-title">Storage · ${esc(filesystem?.mount || '/')}</span><span class="hw-card-value">${storageLabel}</span></div>
${bar(storagePercent)}
<div class="metric-pair"><span>Used</span><strong>${fmtBytes(filesystem?.used_bytes)} / ${fmtBytes(filesystem?.total_bytes)}</strong></div>
<div class="metric-pair"><span>Available</span><strong>${fmtBytes(filesystem?.available_bytes)}</strong></div>
<div class="metric-pair"><span>I/O PSI · 10s</span><strong class="pressure-value ${pressureTone(ioPressure)}">${ioPressure == null ? '—' : fmt(ioPressure, 2) + '%'}</strong></div>
</section>
<section class="hw-card gpu">
<div class="hw-card-head"><span class="hw-card-title">GPU · ${fmt(gpuDevices.length)} devices</span><span class="hw-card-value">${gpuDevices.length ? fmt(gpuMax) + '%' : '—'}</span></div>
${gpuRows || '<div class="hw-empty">No GPU telemetry</div>'}
</section>
<section class="hw-card network">
<div class="hw-card-head"><span class="hw-card-title">Network</span><span class="hw-card-value">${fmt(net?.interfaces?.length || 0)}</span></div>
<div class="network-list">${networkRows || '<div class="hw-empty">No interface telemetry</div>'}</div>
</section>
</div>${errors}`;
}
function drawThreadGraph(history, currentCores) {
requestAnimationFrame(() => {
const canvas = document.getElementById('machine-spark');
const canvas = document.getElementById('thread-graph');
if (!canvas) return;
const ratio = Math.max(1, window.devicePixelRatio || 1);
const rect = canvas.getBoundingClientRect();
const width = Math.max(1, Math.round(rect.width));
const height = Math.max(1, Math.round(rect.height));
const pixelWidth = Math.round(width * ratio), pixelHeight = Math.round(height * ratio);
if (canvas.width !== pixelWidth || canvas.height !== pixelHeight) {
canvas.width = pixelWidth;
canvas.height = pixelHeight;
}
const ctx = canvas.getContext('2d');
ctx.clearRect(0, 0, canvas.width, canvas.height);
const points = history.map(h => ({ cpu: h.cpu_total_percent, gpu: h.gpu_max_percent }));
drawLine(ctx, points.map(p => p.gpu), '#fbbf24');
drawLine(ctx, points.map(p => p.cpu), '#34d399');
ctx.setTransform(ratio, 0, 0, ratio, 0, 0);
const styles = getComputedStyle(document.documentElement);
const background = styles.getPropertyValue('--bg').trim() || '#00060c';
const divider = styles.getPropertyValue('--divider').trim() || '#0d2c4a';
const foreground = styles.getPropertyValue('--ok').trim() || '#00d400';
ctx.globalAlpha = 1;
ctx.fillStyle = background;
ctx.fillRect(0, 0, width, height);
const threadCount = Math.max(currentCores.length, ...history.map(point => point.cpu_cores_percent?.length || 0));
if (!threadCount) return;
const rowHeight = height / threadCount;
ctx.fillStyle = divider;
for (let row = 1; row < threadCount; row += 1) {
ctx.fillRect(0, Math.floor(row * rowHeight), width, 1);
}
const slot = 2;
const points = history
.map(point => point.cpu_cores_percent || [])
.slice(-Math.floor(width / slot));
if (!points.length) points.push(currentCores);
ctx.fillStyle = foreground;
points.forEach((values, pointIndex) => {
const x = width - (points.length - pointIndex) * slot;
values.forEach((value, threadIndex) => {
if (value == null || !Number.isFinite(Number(value))) return;
const utilization = Math.max(0, Math.min(100, Number(value)));
ctx.globalAlpha = 0.10 + utilization * 0.009;
const y = Math.ceil(threadIndex * rowHeight);
ctx.fillRect(x, y, slot, Math.max(1, Math.floor(rowHeight) - 1));
});
});
ctx.globalAlpha = 1;
});
}
function drawLine(ctx, values, color) {
const valid = values.filter(v => v != null);
if (valid.length < 2) return;
const max = Math.max(100, ...valid);
const step = ctx.canvas.width / (values.length - 1 || 1);
ctx.strokeStyle = color;
ctx.lineWidth = 1.5;
ctx.beginPath();
let started = false;
values.forEach((value, index) => {
if (value == null) return;
const x = index * step;
const y = ctx.canvas.height - (value / max) * (ctx.canvas.height - 6) - 3;
if (!started) { ctx.moveTo(x, y); started = true; } else ctx.lineTo(x, y);
});
ctx.stroke();
function rosterSortControls(key, label) {
return `<span class="roster-sort-controls">
<button type="button" class="roster-sort-button" data-sort="${key}" data-direction="asc" aria-label="Sort ${label} ascending" title="Sort ${label} ascending" aria-pressed="${rosterSort.key === key && rosterSort.direction === 'asc'}">↑</button>
<button type="button" class="roster-sort-button" data-sort="${key}" data-direction="desc" aria-label="Sort ${label} descending" title="Sort ${label} descending" aria-pressed="${rosterSort.key === key && rosterSort.direction === 'desc'}">↓</button>
</span>`;
}
function compareRosterValues(left, right) {
const leftMissing = left == null || left === '';
const rightMissing = right == null || right === '';
if (leftMissing || rightMissing) return leftMissing === rightMissing ? 0 : leftMissing ? 1 : -1;
if (typeof left === 'number' && typeof right === 'number') return left - right;
if (typeof left === 'boolean' && typeof right === 'boolean') return Number(left) - Number(right);
return String(left).localeCompare(String(right), undefined, { numeric: true, sensitivity: 'base' });
}
function isLegacyHardwareSampler(actor) {
const actorType = String(actor.actor_type || '');
return (actorType.includes('BlockingSamplerActor') && actorType.includes('telemetry::hardware'))
|| /Host(?:Cpu|Gpu|Net)SamplerActor/.test(actorType);
}
function renderRoster(node) {
const wrap = document.getElementById('roster-wrap');
if (!wrap) return;
let rows = node.roster || [];
let rows = [...(node.roster || [])];
if (hardwareSource === 'orchestrator') {
// Mock containers may come from an older local image. Current samplers are
// engine tasks; do not leak legacy sampler actors into the simulated roster.
rows = rows.filter(actor => !isLegacyHardwareSampler(actor));
const actorCount = document.querySelector('[data-actor-count]');
if (actorCount) actorCount.textContent = fmt(rows.length);
}
if (rosterFilter) {
const q = rosterFilter.toLowerCase();
rows = rows.filter(a => `${a.name || ''} ${a.actor_type || ''} ${a.address} ${a.worker_id ?? ''}`.toLowerCase().includes(q));
}
rows.sort((left, right) => {
const compared = compareRosterValues(left[rosterSort.key], right[rosterSort.key]);
return (rosterSort.direction === 'asc' ? compared : -compared)
|| String(left.address).localeCompare(String(right.address));
});
const capped = rows.slice(0, ROSTER_RENDER_CAP);
const notice = rows.length > capped.length
? `<div class="notice">showing ${fmt(capped.length)} of ${fmt(rows.length)} — refine the filter to see more</div>`
@ -417,7 +605,7 @@ function renderRoster(node) {
html = '<div class="empty">No actors on this stream (or none match the filter).</div>';
} else {
html = notice + `<table>
<thead><tr><th>Actor</th><th>Type</th><th>State</th><th>Mailbox</th><th>Msg/s</th><th>Processed</th><th>Worker</th><th>Last message</th></tr></thead>
<thead><tr><th>Actor${rosterSortControls('address', 'actor')}</th><th>Type${rosterSortControls('actor_type', 'type')}</th><th>State${rosterSortControls('poisoned', 'state')}</th><th>Mailbox${rosterSortControls('mailbox_depth', 'mailbox')}</th><th>Msg/s${rosterSortControls('msg_per_sec', 'throughput')}</th><th>Processed${rosterSortControls('messages_processed', 'processed count')}</th><th>Worker${rosterSortControls('worker_id', 'worker')}</th><th>Last message${rosterSortControls('last_msg_type', 'last message')}</th></tr></thead>
<tbody>${capped.map(a => `<tr data-addr="${esc(a.address)}" tabindex="0" ${a.address === selectedActor ? 'data-selected="true"' : ''}>
<td class="mono">${esc(shortAddr(a.address))}${a.name ? `<br><span class="muted">${esc(a.name)}</span>` : ''}</td>
<td><span class="muted" title="${esc(a.actor_type || '')}">${typeShort(a.actor_type)}</span></td>
@ -552,6 +740,12 @@ pageEl.addEventListener('click', e => {
render();
return;
}
const sortButton = e.target.closest('.roster-sort-button');
if (sortButton) {
rosterSort = { key: sortButton.dataset.sort, direction: sortButton.dataset.direction };
render();
return;
}
const rosterRow = e.target.closest('tr[data-addr]');
if (rosterRow) {
selectedActor = rosterRow.getAttribute('data-addr');

99
crates/dashboard/src/hardware_view.rs
View file

@ -8,7 +8,9 @@ use telemetry::hardware::cpu::{
use telemetry::hardware::gpu::{
GpuDeviceSample, GpuProcessSample, HOST_GPU_CHANNEL, HostGpuSample,
};
use telemetry::hardware::memory::{HOST_MEMORY_CHANNEL, HostMemorySample};
use telemetry::hardware::net::{HOST_NET_CHANNEL, HostNetSample, NetInterfaceSample};
use telemetry::hardware::storage::{HOST_STORAGE_CHANNEL, HostStorageSample};
use serde::Serialize;
@ -27,7 +29,9 @@ pub(crate) struct NodeHardwareState {
decode_errors: BTreeMap<&'static str, String>,
pub(crate) cpu: Option<HostCpuSample>,
pub(crate) gpu: Option<HostGpuSample>,
pub(crate) memory: Option<HostMemorySample>,
pub(crate) net: Option<NetSnapshot>,
pub(crate) storage: Option<HostStorageSample>,
pub(crate) process: Option<ProcessSnapshot>,
pub(crate) history: VecDeque<HardwareHistoryState>,
}
@ -39,7 +43,9 @@ impl NodeHardwareState {
decode_errors: BTreeMap::new(),
cpu: None,
gpu: None,
memory: None,
net: None,
storage: None,
process: None,
history: VecDeque::with_capacity(HISTORY_CAP),
}
@ -64,6 +70,14 @@ impl NodeHardwareState {
}
Err(error) => self.store_decode_error(HOST_GPU_CHANNEL, error),
},
HOST_MEMORY_CHANNEL => match HostMemorySample::decode(payload) {
Ok(sample) => {
self.memory = Some(sample);
self.decode_errors.remove(HOST_MEMORY_CHANNEL);
self.update_history(now);
}
Err(error) => self.store_decode_error(HOST_MEMORY_CHANNEL, error),
},
HOST_NET_CHANNEL => match HostNetSample::decode(payload) {
Ok(sample) => {
self.net = Some(NetSnapshot::from_sample(sample, self.net.as_ref()));
@ -72,6 +86,14 @@ impl NodeHardwareState {
}
Err(error) => self.store_decode_error(HOST_NET_CHANNEL, error),
},
HOST_STORAGE_CHANNEL => match HostStorageSample::decode(payload) {
Ok(sample) => {
self.storage = Some(sample);
self.decode_errors.remove(HOST_STORAGE_CHANNEL);
self.update_history(now);
}
Err(error) => self.store_decode_error(HOST_STORAGE_CHANNEL, error),
},
_ => {
if channel.starts_with("proc.") && channel.ends_with(".lifecycle") {
self.process = decode_process_snapshot(payload);
@ -99,6 +121,35 @@ impl NodeHardwareState {
.as_ref()
.and_then(|sample| sample.host.as_ref())
.and_then(|host| host.total_percent);
let cpu_cores_percent = self
.cpu
.as_ref()
.map(|sample| {
sample
.cores
.iter()
.map(|core| core.total_percent)
.collect::<Vec<_>>()
})
.unwrap_or_default();
let memory_used_percent = self.memory.as_ref().and_then(|sample| {
Some(sample.used_bytes? as f64 * 100.0 / sample.total_bytes?.max(1) as f64)
});
let memory_pressure_some_avg10 = self
.memory
.as_ref()
.and_then(|sample| sample.pressure.as_ref())
.map(|pressure| pressure.some_avg10);
let storage_used_percent = self
.storage
.as_ref()
.and_then(|sample| sample.filesystems.first())
.and_then(|filesystem| filesystem.used_percent);
let io_pressure_some_avg10 = self
.storage
.as_ref()
.and_then(|sample| sample.pressure.as_ref())
.map(|pressure| pressure.some_avg10);
let mut gpu_max_percent = None;
let mut gpu_memory_used_mib = 0_u64;
@ -128,17 +179,24 @@ impl NodeHardwareState {
sample_unix_ms: [
self.cpu.as_ref().map(|sample| sample.sample_unix_ms),
self.gpu.as_ref().map(|sample| sample.sample_unix_ms),
self.memory.as_ref().map(|sample| sample.sample_unix_ms),
self.net.as_ref().map(|sample| sample.sample_unix_ms),
self.storage.as_ref().map(|sample| sample.sample_unix_ms),
]
.into_iter()
.flatten()
.max(),
cpu_total_percent,
cpu_cores_percent,
gpu_max_percent,
gpu_memory_used_mib,
gpu_memory_total_mib,
net_rx_bps,
net_tx_bps,
memory_used_percent,
memory_pressure_some_avg10,
storage_used_percent,
io_pressure_some_avg10,
}
}
@ -149,11 +207,16 @@ impl NodeHardwareState {
{
last.sample_unix_ms = summary.sample_unix_ms;
last.cpu_total_percent = summary.cpu_total_percent;
last.cpu_cores_percent = summary.cpu_cores_percent.clone();
last.gpu_max_percent = summary.gpu_max_percent;
last.gpu_memory_used_mib = summary.gpu_memory_used_mib;
last.gpu_memory_total_mib = summary.gpu_memory_total_mib;
last.net_rx_bps = summary.net_rx_bps;
last.net_tx_bps = summary.net_tx_bps;
last.memory_used_percent = summary.memory_used_percent;
last.memory_pressure_some_avg10 = summary.memory_pressure_some_avg10;
last.storage_used_percent = summary.storage_used_percent;
last.io_pressure_some_avg10 = summary.io_pressure_some_avg10;
return;
}
@ -164,11 +227,16 @@ impl NodeHardwareState {
at: now,
sample_unix_ms: summary.sample_unix_ms,
cpu_total_percent: summary.cpu_total_percent,
cpu_cores_percent: summary.cpu_cores_percent,
gpu_max_percent: summary.gpu_max_percent,
gpu_memory_used_mib: summary.gpu_memory_used_mib,
gpu_memory_total_mib: summary.gpu_memory_total_mib,
net_rx_bps: summary.net_rx_bps,
net_tx_bps: summary.net_tx_bps,
memory_used_percent: summary.memory_used_percent,
memory_pressure_some_avg10: summary.memory_pressure_some_avg10,
storage_used_percent: summary.storage_used_percent,
io_pressure_some_avg10: summary.io_pressure_some_avg10,
});
}
@ -180,9 +248,23 @@ impl NodeHardwareState {
if let Some(error) = self.gpu.as_ref().and_then(|sample| sample.error.as_ref()) {
errors.push(format!("{HOST_GPU_CHANNEL}: {error}"));
}
if let Some(error) = self
.memory
.as_ref()
.and_then(|sample| sample.error.as_ref())
{
errors.push(format!("{HOST_MEMORY_CHANNEL}: {error}"));
}
if let Some(error) = self.net.as_ref().and_then(|sample| sample.error.as_ref()) {
errors.push(format!("{HOST_NET_CHANNEL}: {error}"));
}
if let Some(error) = self
.storage
.as_ref()
.and_then(|sample| sample.error.as_ref())
{
errors.push(format!("{HOST_STORAGE_CHANNEL}: {error}"));
}
errors
}
}
@ -308,26 +390,36 @@ impl NetInterfaceSnapshot {
}
}
#[derive(Clone, Copy)]
#[derive(Clone)]
pub(crate) struct HardwareSummary {
pub(crate) sample_unix_ms: Option<u64>,
pub(crate) cpu_total_percent: Option<f64>,
pub(crate) cpu_cores_percent: Vec<Option<f64>>,
pub(crate) gpu_max_percent: Option<u64>,
pub(crate) gpu_memory_used_mib: u64,
pub(crate) gpu_memory_total_mib: u64,
pub(crate) net_rx_bps: f64,
pub(crate) net_tx_bps: f64,
pub(crate) memory_used_percent: Option<f64>,
pub(crate) memory_pressure_some_avg10: Option<f64>,
pub(crate) storage_used_percent: Option<f64>,
pub(crate) io_pressure_some_avg10: Option<f64>,
}
pub(crate) struct HardwareHistoryState {
pub(crate) at: Instant,
pub(crate) sample_unix_ms: Option<u64>,
pub(crate) cpu_total_percent: Option<f64>,
pub(crate) cpu_cores_percent: Vec<Option<f64>>,
pub(crate) gpu_max_percent: Option<u64>,
pub(crate) gpu_memory_used_mib: u64,
pub(crate) gpu_memory_total_mib: u64,
pub(crate) net_rx_bps: f64,
pub(crate) net_tx_bps: f64,
pub(crate) memory_used_percent: Option<f64>,
pub(crate) memory_pressure_some_avg10: Option<f64>,
pub(crate) storage_used_percent: Option<f64>,
pub(crate) io_pressure_some_avg10: Option<f64>,
}
#[derive(Serialize)]
@ -383,11 +475,16 @@ pub(crate) struct HardwareHistorySnapshot {
pub(crate) ms_ago: u64,
pub(crate) sample_unix_ms: Option<u64>,
pub(crate) cpu_total_percent: Option<f64>,
pub(crate) cpu_cores_percent: Vec<Option<f64>>,
pub(crate) gpu_max_percent: Option<u64>,
pub(crate) gpu_memory_used_mib: u64,
pub(crate) gpu_memory_total_mib: u64,
pub(crate) net_rx_bps: f64,
pub(crate) net_tx_bps: f64,
pub(crate) memory_used_percent: Option<f64>,
pub(crate) memory_pressure_some_avg10: Option<f64>,
pub(crate) storage_used_percent: Option<f64>,
pub(crate) io_pressure_some_avg10: Option<f64>,
}
pub(crate) fn duration_ms(duration: Duration) -> u64 {

2
crates/dashboard/src/server.rs
View file

@ -469,7 +469,7 @@ mod tests {
}
fn is_valid_for_route(&self) -> bool {
self.payload % 5 == 0
self.payload.is_multiple_of(5)
}
}

9
crates/dashboard/src/swactor/actor_view.rs
View file

@ -20,6 +20,7 @@
//! out. Receipts are bounded and interval-spaced so noisy actors cannot flood
//! the page.
use std::cmp::Reverse;
use std::collections::{BTreeMap, VecDeque};
use std::time::{Duration, Instant};
@ -172,8 +173,10 @@ impl RuntimeState {
}
pub(crate) fn totals(&self) -> Totals {
let mut totals = Totals::default();
totals.actors = self.actors.len().min(u32::MAX as usize) as u32;
let mut totals = Totals {
actors: self.actors.len().min(u32::MAX as usize) as u32,
..Totals::default()
};
for actor in self.actors.values() {
totals.mailbox_depth = totals.mailbox_depth.saturating_add(actor.mailbox_depth);
totals.msg_per_sec += actor.msg_per_sec;
@ -449,7 +452,7 @@ fn parse_message_type_counts(value: Option<&Value>) -> Option<Vec<(String, u64)>
.iter()
.filter_map(|(name, count)| value_to_u64(count).map(|count| (name.clone(), count)))
.collect();
out.sort_by(|a, b| b.1.cmp(&a.1));
out.sort_by_key(|&(_, count)| Reverse(count));
return Some(out);
}
None

5
crates/data-plane/Cargo.toml
View file

@ -8,6 +8,7 @@ publish = false
[dependencies]
telemetry = { path = "../telemetry" }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
sha2 = "0.10"
swactor = { path = "../..", features = ["serde", "transport"] }
swactor-transport = { path = "../transport" }
@ -17,6 +18,10 @@ swactor-engine = { path = "../engine" }
[dev-dependencies]
parking_lot = "0.12"
futures-lite = "2"
proptest = "1"
tokio.workspace = true
[target.'cfg(target_os = "linux")'.dependencies]
libc = "0.2"
[lints]
workspace = true

18
crates/data-plane/src/arena.rs
View file

@ -345,10 +345,10 @@ impl ArenaManager {
}];
}
if self.pending.is_empty() {
if let Some(lease) = self.try_allocate(&request) {
return vec![ArenaEvent::RingLeased { lease }];
}
if self.pending.is_empty()
&& let Some(lease) = self.try_allocate(&request)
{
return vec![ArenaEvent::RingLeased { lease }];
}
let request_id = request.request_id;
@ -517,11 +517,11 @@ impl ArenaManager {
if range.start == range.end {
continue;
}
if let Some(last) = coalesced.last_mut() {
if range.start <= last.end {
last.end = last.end.max(range.end);
continue;
}
if let Some(last) = coalesced.last_mut()
&& range.start <= last.end
{
last.end = last.end.max(range.end);
continue;
}
coalesced.push(range);
}

85
crates/data-plane/src/blob.rs
View file

@ -2,6 +2,7 @@
use std::fmt;
use std::ops::{Deref, DerefMut};
use std::path::{Path, PathBuf};
use std::ptr::NonNull;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
@ -25,6 +26,21 @@ const DIGEST_OFFSET: usize = 32;
const DIGEST_LEN: usize = 32;
const RESERVED_OFFSET: usize = 64;
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct FileRegistration {
path: PathBuf,
}
impl FileRegistration {
pub(crate) fn path(&self) -> &Path {
&self.path
}
}
pub fn file(path: impl Into<PathBuf>) -> FileRegistration {
FileRegistration { path: path.into() }
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[repr(u8)]
pub enum BlobAccess {
@ -89,20 +105,6 @@ pub enum BlobSharedState {
Released = 5,
}
impl BlobSharedState {
fn from_u64(value: u64) -> Option<Self> {
match value {
0 => Some(Self::Vacant),
1 => Some(Self::Filling),
2 => Some(Self::Writable),
3 => Some(Self::Sealed),
4 => Some(Self::Aborted),
5 => Some(Self::Released),
_ => None,
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum BlobError {
RangeOutOfBounds {
@ -249,7 +251,7 @@ impl Blob {
self.guard.lease
}
pub fn map(&self) -> Result<ArenaView, BlobError> {
pub fn map(&self) -> Result<BlobView, BlobError> {
validate_mapped_header(
&self.guard.arena,
self.guard.lease,
@ -264,7 +266,7 @@ impl Blob {
},
)?;
let range = mapped_range(&self.guard.arena, payload_offset, self.guard.lease.length)?;
Ok(ArenaView {
Ok(BlobView {
guard: self.guard.clone(),
payload_offset: range.start,
length: range.len(),
@ -272,13 +274,13 @@ impl Blob {
}
}
pub struct ArenaView {
pub struct BlobView {
guard: Arc<BlobLeaseGuard>,
payload_offset: usize,
length: usize,
}
impl ArenaView {
impl BlobView {
pub fn len(&self) -> usize {
self.length
}
@ -296,7 +298,7 @@ impl ArenaView {
}
}
impl AsRef<[u8]> for ArenaView {
impl AsRef<[u8]> for BlobView {
fn as_ref(&self) -> &[u8] {
// SAFETY: construction bounds-checks the range, the arena mapping is
// stable, and the shared guard prevents lease reuse while this view is
@ -305,7 +307,7 @@ impl AsRef<[u8]> for ArenaView {
}
}
impl Deref for ArenaView {
impl Deref for BlobView {
type Target = [u8];
fn deref(&self) -> &Self::Target {
@ -484,38 +486,6 @@ impl Drop for WritableArenaView {
}
}
pub(crate) fn install_read_blob(
arena: &ArenaManager,
allocation: &RingLease,
generation: u64,
bytes: &[u8],
digest: Option<ContentDigest>,
) -> Result<(BlobLease, BlobMetadata), BlobError> {
let length = u64::try_from(bytes.len()).map_err(|_| BlobError::InvalidHostLease)?;
let lease = descriptor_from_allocation(allocation, generation, length, BlobAccess::ReadOnly)?;
let metadata = BlobMetadata { length, digest };
let header = host_header_ptr(arena, lease)?;
write_header(header, lease, &metadata, BlobSharedState::Filling);
if let Some(expected) = &metadata.digest {
if !expected.matches(bytes) {
host_state(arena, lease)?.store(BlobSharedState::Aborted as u64, Ordering::Release);
return Err(BlobError::DigestMismatch);
}
}
let payload = arena
.region_ptr(lease.offset + BLOB_HEADER_LEN, length)
.ok_or(BlobError::InvalidHostLease)?;
// SAFETY: the allocation owns exactly `length` payload bytes and `bytes`
// has that same length. This is the only host-to-child process-boundary copy.
unsafe {
std::ptr::copy_nonoverlapping(bytes.as_ptr(), payload.as_ptr(), bytes.len());
}
host_state(arena, lease)?.store(BlobSharedState::Sealed as u64, Ordering::Release);
Ok((lease, metadata))
}
pub(crate) fn install_filling_read_blob(
arena: &ArenaManager,
allocation: &RingLease,
@ -727,7 +697,7 @@ fn validate_header(
length: lease.length,
arena_size,
})?;
if state_address as usize % std::mem::align_of::<AtomicU64>() != 0 {
if !(state_address as usize).is_multiple_of(std::mem::align_of::<AtomicU64>()) {
return Err(BlobError::UnalignedHeader {
offset: lease.offset,
});
@ -808,7 +778,7 @@ fn validate_header(
fn mapped_state(arena: &MappedArena, lease: BlobLease) -> Result<&AtomicU64, BlobError> {
let range = mapped_range(arena, lease.offset, BLOB_HEADER_LEN + lease.length)?;
let state_offset = range.start + STATE_OFFSET;
if state_offset % std::mem::align_of::<AtomicU64>() != 0 {
if !state_offset.is_multiple_of(std::mem::align_of::<AtomicU64>()) {
return Err(BlobError::UnalignedHeader {
offset: lease.offset,
});
@ -826,7 +796,7 @@ fn host_header_ptr(arena: &ArenaManager, lease: BlobLease) -> Result<NonNull<u8>
fn host_state(arena: &ArenaManager, lease: BlobLease) -> Result<&AtomicU64, BlobError> {
let header = host_header_ptr(arena, lease)?;
let address = lease.offset + STATE_OFFSET as u64;
if address as usize % std::mem::align_of::<AtomicU64>() != 0 {
if !(address as usize).is_multiple_of(std::mem::align_of::<AtomicU64>()) {
return Err(BlobError::UnalignedHeader {
offset: lease.offset,
});
@ -860,8 +830,3 @@ fn checked_raw_range(arena_size: u64, offset: u64, length: u64) -> Result<(), Bl
arena_size,
})
}
#[allow(dead_code)]
fn state_name(value: u64) -> Option<BlobSharedState> {
BlobSharedState::from_u64(value)
}

103
crates/data-plane/src/blob_transfer.rs Normal file
View file

@ -0,0 +1,103 @@
//! Transport-neutral contracts for one-shot fixed-length blob transfers.
use std::fmt;
use std::fs::File;
use crate::blob::{BlobLease, BlobMetadata};
use crate::protocol::DataPlaneError;
use serde::{Deserialize, Serialize};
use swactor::actor::ActorAddress;
use swactor_transport::{CodecRegistry, JsonCodec, NetworkMessage};
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct BlobTransferId(pub u64);
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BlobTransferOffer {
pub transfer_id: BlobTransferId,
pub destination: ActorAddress,
pub failure_proxy: Option<ActorAddress>,
pub transport: Vec<u8>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum BlobTransferEvent {
Chunk {
transfer_id: BlobTransferId,
bytes: Vec<u8>,
},
Finished {
transfer_id: BlobTransferId,
},
Failed {
transfer_id: BlobTransferId,
reason: String,
},
Allocated(Result<(BlobLease, BlobMetadata), DataPlaneError>),
AllocatorFailed(DataPlaneError),
Sealed(Result<(), DataPlaneError>),
Released(Result<(), DataPlaneError>),
Cancel,
}
impl NetworkMessage for BlobTransferEvent {
fn type_tag() -> &'static str {
"data-plane.blob-transfer.event.v1"
}
}
pub trait BlobTransferCompletion: Send + 'static {
fn complete(self: Box<Self>, result: Result<(), String>);
}
pub struct FileTransferRequest {
pub offer: BlobTransferOffer,
pub file: File,
pub offset: u64,
pub length: u64,
pub completion: Box<dyn BlobTransferCompletion>,
}
pub trait BlobTransferSender: Send + Sync + 'static {
fn start_file(&self, request: FileTransferRequest) -> Result<(), String>;
}
pub trait BlobTransferReceiver: Send + Sync + 'static {
fn open(
&self,
destination: ActorAddress,
transfer_id: BlobTransferId,
) -> Result<BlobTransferOffer, String>;
fn cancel(&self, offer: &BlobTransferOffer);
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum BlobTransferFailure {
Start(String),
Source(String),
Transport(String),
Length { expected: u64, found: u64 },
}
impl fmt::Display for BlobTransferFailure {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Start(reason) => write!(f, "blob transfer did not start: {reason}"),
Self::Source(reason) => write!(f, "blob source failed: {reason}"),
Self::Transport(reason) => write!(f, "blob transport failed: {reason}"),
Self::Length { expected, found } => {
write!(
f,
"blob transfer length mismatch: expected {expected}, found {found}"
)
}
}
}
}
impl std::error::Error for BlobTransferFailure {}
pub fn register_blob_transfer_codecs(registry: &mut CodecRegistry) {
registry.register::<BlobTransferEvent, _>(JsonCodec::default());
}

133
crates/data-plane/src/byte_ring.rs
View file

@ -27,29 +27,32 @@
//! Properties a correct implementation must uphold (each defended in
//! `tests/byte_ring_guarantees.rs`):
//!
//! - P1 exactly-once, in-order byte delivery across interleavings,
//! wake timings, and wraparound;
//! - P2 unread bytes are never overwritten by the producer;
//! - P3 single-writer cursor fields (SPSC), enforced by endpoint roles;
//! - P4 data publication is release-ordered before `commit` advance,
//! and `consume` advance after reads (acquire);
//! - P5 untrusted-header containment: arbitrary header bytes yield typed
//! errors or bounded access, never out-of-region access;
//! - P6 (binding slice) Python performs ring operations via helpers only;
//! - P7 backpressure: reserving beyond free space fails cleanly with the
//! exact free amount, and succeeds again once the peer consumes;
//! - P8 (binding slice) no lost wakeups: signals follow publication;
//! - P9 completion is explicit: `Data`/`Eof`/`Fault` records, `Eof` only
//! after all bytes, torn (uncommitted) records stay invisible;
//! - P1 exactly-once, in-order byte delivery across interleavings,
//! wake timings, and wraparound;
//! - P2 unread bytes are never overwritten by the producer;
//! - P3 single-writer cursor fields (SPSC), enforced by endpoint roles;
//! - P4 data publication is release-ordered before `commit` advance,
//! and `consume` advance after reads (acquire);
//! - P5 untrusted-header containment: arbitrary header bytes yield typed
//! errors or bounded access, never out-of-region access;
//! - P6 (binding slice) Python performs ring operations via helpers only;
//! - P7 backpressure: reserving beyond free space fails cleanly with the
//! exact free amount, and succeeds again once the peer consumes;
//! - P8 (binding slice) no lost wakeups: signals follow publication;
//! - P9 completion is explicit: `Data`/`Eof`/`Fault` records, `Eof` only
//! after all bytes, torn (uncommitted) records stay invisible;
//! - P10 generation fence: stale reservations and stale generations are
//! rejected, never applied;
//! rejected, never applied;
//! - P11 (binding slice) fast path performs no syscalls;
//! - P12 one copy per side per byte.
use std::ptr::NonNull;
use std::sync::atomic::{AtomicU64, Ordering};
use crate::arena::{ArenaEvent, ArenaManager, ArenaRequest, LeaseRing, LeaseRequestId, RingLease, RingLeaseRejection, RingSpec};
use crate::arena::{
ArenaEvent, ArenaManager, ArenaRequest, LeaseRequestId, LeaseRing, RingLease,
RingLeaseRejection, RingSpec,
};
/// `"SWRG"` read little-endian.
pub const RING_MAGIC: u32 = u32::from_le_bytes(*b"SWRG");
@ -98,7 +101,6 @@ pub enum Role {
Consumer,
}
/// A reserved, not-yet-committed span of the data region.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Reservation {
@ -149,19 +151,42 @@ pub enum InstallError {
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum HeaderError {
BadMagic { found: u32 },
UnsupportedVersion { found: u16, supported: u16 },
ReservedBytesNotZero { at: u64 },
CapacityMismatch { header: u64, handle: u64 },
GenerationMismatch { header: u64, handle: u64 },
CommitBelowConsume { commit: u64, consume: u64 },
ReadableExceedsCapacity { commit: u64, consume: u64, capacity: u64 },
BadMagic {
found: u32,
},
UnsupportedVersion {
found: u16,
supported: u16,
},
ReservedBytesNotZero {
at: u64,
},
CapacityMismatch {
header: u64,
handle: u64,
},
GenerationMismatch {
header: u64,
handle: u64,
},
CommitBelowConsume {
commit: u64,
consume: u64,
},
ReadableExceedsCapacity {
commit: u64,
consume: u64,
capacity: u64,
},
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum AttachError {
/// The handle itself points outside the arena.
OutOfBounds { end: u64, arena_len: u64 },
OutOfBounds {
end: u64,
arena_len: u64,
},
Header(HeaderError),
}
@ -173,12 +198,24 @@ pub enum RecordError {
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum FlowError {
InsufficientSpace { requested: u64, free: u64 },
BeyondCommitted { requested: u64, readable: u64 },
InsufficientSpace {
requested: u64,
free: u64,
},
BeyondCommitted {
requested: u64,
readable: u64,
},
/// The reservation's generation no longer matches the ring header.
StaleReservation { reservation: u64, ring: u64 },
StaleReservation {
reservation: u64,
ring: u64,
},
/// Operation reserved for the other role (property P3).
RoleViolation { operation: &'static str, role: Role },
RoleViolation {
operation: &'static str,
role: Role,
},
/// The header stopped validating mid-protocol (property P5).
Corrupt(HeaderError),
BadRecord(RecordError),
@ -289,16 +326,16 @@ pub fn attach(
arena_len: arena.arena_len(),
})?;
let endpoint = Endpoint { header, info: handle, role };
endpoint
.validate_fixed()
.map_err(AttachError::Header)?;
let endpoint = Endpoint {
header,
info: handle,
role,
};
endpoint.validate_fixed().map_err(AttachError::Header)?;
endpoint
.validate_generation(handle.generation)
.map_err(AttachError::Header)?;
endpoint
.validate_cursors()
.map_err(AttachError::Header)?;
endpoint.validate_cursors().map_err(AttachError::Header)?;
Ok(endpoint)
}
@ -462,7 +499,11 @@ impl Endpoint {
// SAFETY: as `copy_into`.
unsafe {
std::ptr::copy_nonoverlapping(self.data_ptr().add(start), bytes.as_mut_ptr(), first);
std::ptr::copy_nonoverlapping(self.data_ptr(), bytes.as_mut_ptr().add(first), len - first);
std::ptr::copy_nonoverlapping(
self.data_ptr(),
bytes.as_mut_ptr().add(first),
len - first,
);
}
bytes
}
@ -478,7 +519,10 @@ impl Endpoint {
let used = commit - consume;
let free = self.info.capacity - used;
if len > free {
return Err(FlowError::InsufficientSpace { requested: len, free });
return Err(FlowError::InsufficientSpace {
requested: len,
free,
});
}
Ok(Reservation {
start: commit,
@ -492,7 +536,8 @@ impl Endpoint {
self.check("write", Role::Producer)?;
self.stale_check(reservation)?;
assert_eq!(
reservation.len as usize, bytes.len(),
reservation.len as usize,
bytes.len(),
"reservation length must match the payload"
);
self.copy_into(reservation.start, bytes);
@ -539,7 +584,10 @@ impl Endpoint {
let (commit, consume) = (self.commit_cursor(), self.consume_cursor());
let readable = commit - consume;
if len > readable {
return Err(FlowError::BeyondCommitted { requested: len, readable });
return Err(FlowError::BeyondCommitted {
requested: len,
readable,
});
}
Ok(self.copy_out(consume, len))
}
@ -550,7 +598,10 @@ impl Endpoint {
let (commit, consume) = (self.commit_cursor(), self.consume_cursor());
let readable = commit - consume;
if len > readable {
return Err(FlowError::BeyondCommitted { requested: len, readable });
return Err(FlowError::BeyondCommitted {
requested: len,
readable,
});
}
self.atomic(OFF_CONSUME)
.store(consume + len, Ordering::Release);

158
crates/data-plane/src/control.rs Normal file
View file

@ -0,0 +1,158 @@
//! Reusable namespace service lifecycle and public file-registration control.
use std::path::Path;
use std::sync::Arc;
use swactor::actor::ActorAddress;
use swactor::runtime::Runtime;
use crate::blob::FileRegistration;
use crate::blob_transfer::BlobTransferSender;
use crate::namespace::{DataDirectoryActor, DirectoryClient, NamespaceError, OperationId};
use crate::namespace_store::SourceRecovery;
use crate::path::DataPath;
use crate::source::{BlobSourceIn, BlobSourcePublisher, FileBlobSourceActor};
pub struct DataNamespaceService {
directory: ActorAddress,
control: DataPlaneControl,
}
impl DataNamespaceService {
pub fn recover(
runtime: Runtime,
store_path: impl AsRef<Path>,
source_sender: Arc<dyn BlobTransferSender>,
source_publisher: Arc<dyn BlobSourcePublisher>,
) -> Result<Self, NamespaceError> {
let recovery_runtime = runtime.clone();
let recovery_sender = Arc::clone(&source_sender);
let recovery_publisher = Arc::clone(&source_publisher);
let directory = DataDirectoryActor::recover(
store_path,
move |recovery, expected_length| match recovery {
SourceRecovery::File { path } => {
let source = FileBlobSourceActor::recover(
recovery_runtime.clone(),
Arc::clone(&recovery_sender),
path,
expected_length,
)?;
let source = recovery_runtime.spawn(source).map_err(|error| {
NamespaceError::SourceRecovery(format!(
"spawn recovered file source {}: {error}",
path.display()
))
})?;
if let Err(error) = recovery_publisher.publish_source(source) {
let _ = recovery_runtime.send_to(source, BlobSourceIn::Retire);
return Err(NamespaceError::SourceRecovery(error));
}
Ok(source)
}
SourceRecovery::Actor { actor } => Ok(*actor),
},
)?;
let directory = runtime.spawn(directory).map_err(|error| {
NamespaceError::SourceRecovery(format!("spawn data directory: {error}"))
})?;
let control = DataPlaneControl {
runtime: runtime.clone(),
directory: DirectoryClient::new(runtime, directory),
source_sender,
source_publisher,
};
Ok(Self { directory, control })
}
pub fn directory(&self) -> ActorAddress {
self.directory
}
pub fn control(&self) -> DataPlaneControl {
self.control.clone()
}
}
#[derive(Clone)]
pub struct DataPlaneControl {
runtime: Runtime,
directory: DirectoryClient,
source_sender: Arc<dyn BlobTransferSender>,
source_publisher: Arc<dyn BlobSourcePublisher>,
}
impl DataPlaneControl {
pub async fn ensure(
&self,
path: DataPath,
registration: FileRegistration,
) -> Result<(), NamespaceError> {
match self.directory.resolve(path.clone()).await {
Ok(_) => Ok(()),
Err(NamespaceError::PathNotFound(_)) | Err(NamespaceError::SourceRecovery(_)) => {
self.register(path, registration).await
}
Err(error) => Err(error),
}
}
pub async fn register(
&self,
path: DataPath,
registration: FileRegistration,
) -> Result<(), NamespaceError> {
let source = FileBlobSourceActor::open(
self.runtime.clone(),
Arc::clone(&self.source_sender),
registration.path(),
)?;
let length = source.length();
let recovery = source.recovery();
let source = self
.runtime
.spawn(source)
.map_err(|error| NamespaceError::SourceRecovery(error.to_string()))?;
let mut cleanup = PendingSourceRegistration {
runtime: self.runtime.clone(),
source,
armed: true,
};
self.source_publisher
.publish_source(source)
.map_err(NamespaceError::SourceRecovery)?;
self.directory
.register(path, source, length, recovery, random_operation_id())
.await?;
cleanup.armed = false;
Ok(())
}
pub async fn unregister(&self, path: DataPath) -> Result<(), NamespaceError> {
self.directory
.unregister(path, random_operation_id())
.await?;
Ok(())
}
}
struct PendingSourceRegistration {
runtime: Runtime,
source: ActorAddress,
armed: bool,
}
impl Drop for PendingSourceRegistration {
fn drop(&mut self) {
if self.armed {
let _ = self.runtime.send_to(self.source, BlobSourceIn::Retire);
}
}
}
fn random_operation_id() -> OperationId {
let actor = ActorAddress::new_random();
let mut bytes = [0_u8; 16];
bytes.copy_from_slice(&actor.0[..16]);
OperationId::from_u128(u128::from_be_bytes(bytes))
}

90
crates/data-plane/src/data_plane.rs
View file

@ -1,6 +1,6 @@
//! Child-side data-plane session, per-operation actors, and native API.
use std::collections::HashSet;
use std::collections::{HashMap, HashSet};
use std::os::fd::OwnedFd;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
@ -30,6 +30,11 @@ pub struct DataPlaneBootstrap {
pub data_plane: DataPlane,
}
pub struct AttachDeadline {
pub engine: EngineHandle,
pub timeout: Duration,
}
impl DataPlaneBootstrap {
pub async fn attach(
arena_fd: OwnedFd,
@ -94,8 +99,7 @@ impl DataPlaneBootstrap {
host_session: ActorAddress,
job_capability: JobCapability,
child_node: Option<[u8; 32]>,
engine: EngineHandle,
timeout: Duration,
deadline: AttachDeadline,
) -> Result<Self, DataPlaneError> {
let sender = runtime.create_sender();
Self::attach_mapped_inner(
@ -105,7 +109,7 @@ impl DataPlaneBootstrap {
host_session,
job_capability,
child_node,
Some((engine, sender, timeout)),
Some((deadline.engine, sender, deadline.timeout)),
)
.await
}
@ -134,6 +138,7 @@ impl DataPlaneBootstrap {
session_generation: None,
attach_reply: Some(attach_reply),
operations: HashSet::new(),
read_operations: HashMap::new(),
state: ChildSessionState::Attaching,
})
.map_err(|error| DataPlaneError::SessionFailed(error.to_string()))?;
@ -153,6 +158,26 @@ impl DataPlaneBootstrap {
}
}
struct ReadCancellation {
runtime: Runtime,
child_session: ActorAddress,
reply_to: ActorAddress,
armed: bool,
}
impl Drop for ReadCancellation {
fn drop(&mut self) {
if self.armed {
let _ = self.runtime.send_to(
self.child_session,
ChildSessionIn::CancelRead {
reply_to: self.reply_to,
},
);
}
}
}
#[derive(Clone)]
pub struct DataPlane {
runtime: Runtime,
@ -170,16 +195,29 @@ impl DataPlane {
}
pub async fn read_blob(&self, path: &DataPath) -> Result<Blob, DataPlaneError> {
let ask = self
let inbox = self
.runtime
.ask::<ChildSessionIn, Result<Blob, DataPlaneError>>(self.child_session, |reply_to| {
.new_inbox::<Result<Blob, DataPlaneError>>()
.map_err(|error| DataPlaneError::SessionFailed(error.to_string()))?;
let reply_to = *inbox.addr();
self.runtime
.send_to(
self.child_session,
ChildSessionIn::ReadBlob {
path: path.clone(),
reply_to,
}
})
},
)
.map_err(|error| DataPlaneError::SessionFailed(error.to_string()))?;
ask.await
let mut cancellation = ReadCancellation {
runtime: self.runtime.clone(),
child_session: self.child_session,
reply_to,
armed: true,
};
let result = inbox.recv().await;
cancellation.armed = false;
result
}
pub async fn read_blob_path(&self, path: &str) -> Result<Blob, DataPlaneError> {
@ -259,7 +297,6 @@ impl BlobWriter {
pub async fn seal(&mut self) -> Result<(), DataPlaneError> {
let metadata = self.writable.seal()?;
self.finalized = true;
let lease = self.writable.lease();
let ask = self
.runtime
@ -271,7 +308,11 @@ impl BlobWriter {
}
})
.map_err(|error| DataPlaneError::SessionFailed(error.to_string()))?;
ask.await
let result = ask.await;
if result.is_ok() {
self.finalized = true;
}
result
}
pub async fn abort(&mut self) -> Result<(), DataPlaneError> {
@ -293,10 +334,11 @@ impl BlobWriter {
impl Drop for BlobWriter {
fn drop(&mut self) {
if self.finalized || self.writable.is_finished() {
if self.finalized {
return;
}
if self.writable.abort().is_ok() {
let should_abort = self.writable.is_finished() || self.writable.abort().is_ok();
if should_abort {
let _ = self.runtime.send_to(
self.operation,
ChildOperationIn::AbortRequested {
@ -318,6 +360,7 @@ pub struct ChildDataPlaneSessionActor {
session_generation: Option<u64>,
attach_reply: Option<ActorAddress>,
operations: HashSet<ActorAddress>,
read_operations: HashMap<ActorAddress, ActorAddress>,
state: ChildSessionState,
}
@ -408,6 +451,7 @@ impl ActorInterface for ChildDataPlaneSessionActor {
match ctx.spawn(operation) {
Ok(operation) => {
self.operations.insert(operation);
self.read_operations.insert(reply_to, operation);
}
Err(error) => self.fail_local_open(
ctx,
@ -416,6 +460,12 @@ impl ActorInterface for ChildDataPlaneSessionActor {
),
}
}
ChildSessionIn::CancelRead { reply_to } => {
if let Some(operation) = self.read_operations.remove(&reply_to) {
self.operations.remove(&operation);
let _ = ctx.stop_actor(operation);
}
}
ChildSessionIn::OpenWriteBlob {
path,
length,
@ -504,6 +554,8 @@ impl ActorInterface for ChildDataPlaneSessionActor {
}
ChildSessionIn::OperationDone { operation } => {
self.operations.remove(&operation);
self.read_operations
.retain(|_, read_operation| *read_operation != operation);
}
ChildSessionIn::Close => {
if matches!(
@ -516,6 +568,7 @@ impl ActorInterface for ChildDataPlaneSessionActor {
for operation in self.operations.iter().copied() {
let _ = ctx.stop_actor(operation);
}
self.read_operations.clear();
let _ = ctx.send(self.host_session, HostSessionIn::Close);
self.state = ChildSessionState::Closed;
}
@ -641,6 +694,12 @@ impl ActorInterface for ReadBlobOperationActor {
fn on_stop(&mut self, ctx: &Ctx<'_>) {
if !self.replied {
let _ = ctx.send(
self.host_session,
HostSessionIn::CancelReadBlob {
operation: ctx.self_addr(),
},
);
let _ = ctx.send(
self.reply_to,
Err::<Blob, _>(DataPlaneError::OperationCancelled),
@ -850,7 +909,10 @@ impl ActorInterface for WriteBlobOperationActor {
}
}
ChildOperationIn::AbortRequested { reply_to, lease }
if self.state == WriteOperationState::Filling =>
if matches!(
self.state,
WriteOperationState::Filling | WriteOperationState::Sealing
) =>
{
let Some((host_binding, expected_lease, _)) = self.grant.clone() else {
self.fail(ctx, DataPlaneError::OperationCancelled);

12
crates/data-plane/src/edge_lifecycle.rs
View file

@ -517,13 +517,11 @@ impl EdgeEstablisherState {
let worker_ring_cleanup_required = record.worker_ring_cleanup_required();
record.state = EdgeProvisionState::Stopping;
if cancel_lease {
if let Some(request_id) = request_id {
self.commands.push(EdgeCommand::CancelQueuedLease {
request_id,
edge_id,
});
}
if cancel_lease && let Some(request_id) = request_id {
self.commands.push(EdgeCommand::CancelQueuedLease {
request_id,
edge_id,
});
}
let Some(ring_id) = ring_id else {

1516
crates/data-plane/src/host.rs
View file

File diff suppressed because it is too large Load diff

5
crates/data-plane/src/lib.rs
View file

@ -7,8 +7,10 @@
pub mod arena;
pub mod blob;
pub mod blob_transfer;
pub mod bootstrap;
pub mod byte_ring;
pub mod control;
pub mod data_plane;
pub mod edge_lifecycle;
pub mod edge_runtime;
@ -16,7 +18,10 @@ pub mod edge_wire;
pub mod host;
pub mod ids;
pub mod mapped_arena;
pub mod namespace;
pub mod namespace_store;
pub mod object_record;
pub mod path;
pub mod protocol;
pub mod ring;
pub mod source;

4
crates/data-plane/src/mapped_arena.rs
View file

@ -112,6 +112,10 @@ impl MappedArena {
self.len
}
pub fn is_empty(&self) -> bool {
self.len == 0
}
pub fn base_ptr(&self) -> *const u8 {
self.ptr.as_ptr().cast_const()
}

822
crates/data-plane/src/namespace.rs Normal file
View file

@ -0,0 +1,822 @@
//! Authoritative virtual blob namespace actor and restart-tolerant client proxy.
use std::collections::{BTreeMap, HashMap};
use std::fmt;
use std::path::Path;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;
use serde::{Deserialize, Serialize};
use swactor::actor::{ActorAddress, ActorInterface, Ctx};
use swactor::runtime::{ExternalSender, Runtime};
use swactor_engine::EngineHandle;
use swactor_transport::{CodecRegistry, JsonCodec, NetworkMessage};
use crate::blob_transfer::{BlobTransferEvent, BlobTransferId};
use crate::namespace_store::{
MutationReceipt, MutationRejection, MutationRequest, NamespaceStore, NamespaceStoreError,
PersistedBinding, PersistedMutationResult, PersistedOperation,
};
use crate::path::DataPath;
use crate::source::BlobSourceIn;
pub use crate::namespace_store::{OperationId, SourceRecovery};
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct DirectoryRequestId(pub u64);
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BlobBinding {
pub source: ActorAddress,
pub length: u64,
pub revision: u64,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum NamespaceError {
PathNotFound(DataPath),
OperationConflict(OperationId),
Storage(String),
SourceRecovery(String),
DirectoryUnavailable(String),
Protocol(String),
}
impl fmt::Display for NamespaceError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::PathNotFound(path) => write!(f, "data path not found: {path}"),
Self::OperationConflict(operation) => write!(
f,
"namespace operation ID {:02x?} was reused for a different request",
operation.bytes()
),
Self::Storage(reason) => write!(f, "namespace persistence failed: {reason}"),
Self::SourceRecovery(reason) => write!(f, "namespace source recovery failed: {reason}"),
Self::DirectoryUnavailable(reason) => {
write!(f, "namespace directory is unavailable: {reason}")
}
Self::Protocol(reason) => write!(f, "namespace protocol failed: {reason}"),
}
}
}
impl std::error::Error for NamespaceError {}
impl From<NamespaceStoreError> for NamespaceError {
fn from(error: NamespaceStoreError) -> Self {
Self::Storage(error.to_string())
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum DataDirectoryIn {
Register {
request_id: DirectoryRequestId,
path: DataPath,
source: ActorAddress,
length: u64,
recovery: SourceRecovery,
operation_id: OperationId,
reply_to: ActorAddress,
},
Resolve {
request_id: DirectoryRequestId,
path: DataPath,
reply_to: ActorAddress,
},
Unregister {
request_id: DirectoryRequestId,
path: DataPath,
operation_id: OperationId,
reply_to: ActorAddress,
},
}
impl NetworkMessage for DataDirectoryIn {
fn type_tag() -> &'static str {
"data-plane.data-directory.in.v1"
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum DataDirectoryOut {
Registered {
request_id: DirectoryRequestId,
authority_epoch: u64,
result: Result<MutationReceipt, NamespaceError>,
},
Resolved {
request_id: DirectoryRequestId,
authority_epoch: u64,
result: Result<BlobBinding, NamespaceError>,
},
Unregistered {
request_id: DirectoryRequestId,
authority_epoch: u64,
result: Result<MutationReceipt, NamespaceError>,
},
}
impl DataDirectoryOut {
pub fn request_id(&self) -> DirectoryRequestId {
match self {
Self::Registered { request_id, .. }
| Self::Resolved { request_id, .. }
| Self::Unregistered { request_id, .. } => *request_id,
}
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum NamespaceRequest {
Register {
path: DataPath,
source: ActorAddress,
length: u64,
recovery: SourceRecovery,
operation_id: OperationId,
},
Resolve {
path: DataPath,
},
Unregister {
path: DataPath,
operation_id: OperationId,
},
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum NamespaceClientIn {
Request {
request: NamespaceRequest,
reply_to: ActorAddress,
},
Cancel {
reply_to: ActorAddress,
},
DirectoryReply(DataDirectoryOut),
TransferFailed {
destination: ActorAddress,
transfer_id: BlobTransferId,
reason: String,
},
Retry,
}
impl NetworkMessage for NamespaceClientIn {
fn type_tag() -> &'static str {
"data-plane.namespace-client.in.v1"
}
}
#[derive(Clone, Debug)]
enum RuntimeSource {
Available(ActorAddress),
Unavailable(String),
}
pub struct DataDirectoryActor {
store: NamespaceStore,
sources: BTreeMap<DataPath, RuntimeSource>,
authority_epoch: u64,
}
impl DataDirectoryActor {
pub fn recover(
store_path: impl AsRef<Path>,
mut recover_source: impl FnMut(&SourceRecovery, u64) -> Result<ActorAddress, NamespaceError>,
) -> Result<Self, NamespaceError> {
let mut store = NamespaceStore::open(store_path)?;
let bindings = store.snapshot().bindings.clone();
let mut sources = BTreeMap::new();
for (path, binding) in bindings {
let source = match recover_source(&binding.recovery, binding.length) {
Ok(actor) => RuntimeSource::Available(actor),
Err(error) => RuntimeSource::Unavailable(error.to_string()),
};
sources.insert(path, source);
}
let authority_epoch = store.advance_authority_epoch()?;
Ok(Self {
store,
sources,
authority_epoch,
})
}
pub fn authority_epoch(&self) -> u64 {
self.authority_epoch
}
fn replay(
&self,
operation_id: OperationId,
request: &MutationRequest,
) -> Option<Result<MutationReceipt, NamespaceError>> {
self.store
.snapshot()
.operations
.get(&operation_id)
.map(|operation| {
if &operation.request != request {
return Err(NamespaceError::OperationConflict(operation_id));
}
match &operation.result {
PersistedMutationResult::Committed(receipt) => Ok(*receipt),
PersistedMutationResult::Rejected(MutationRejection::PathNotFound(path)) => {
Err(NamespaceError::PathNotFound(path.clone()))
}
}
})
}
fn register(
&mut self,
path: DataPath,
source: ActorAddress,
length: u64,
recovery: SourceRecovery,
operation_id: OperationId,
retired: Option<ActorAddress>,
) -> Result<MutationReceipt, NamespaceError> {
let request = MutationRequest::Register {
path: path.clone(),
length,
recovery: recovery.clone(),
};
if let Some(replayed) = self.replay(operation_id, &request) {
return replayed;
}
let revision = self.store.snapshot().next_revision;
let next_revision = revision
.checked_add(1)
.filter(|revision| *revision != 0)
.ok_or(NamespaceStoreError::RevisionExhausted)?;
let receipt = MutationReceipt { revision };
let mut next = self.store.snapshot().clone();
next.next_revision = next_revision;
next.bindings.insert(
path.clone(),
PersistedBinding {
length,
revision,
recovery,
},
);
if let Some(retired) = retired
&& !next.retirements.contains(&retired)
{
next.retirements.push(retired);
}
next.operations.insert(
operation_id,
PersistedOperation {
request,
result: PersistedMutationResult::Committed(receipt),
},
);
self.store.commit(next)?;
self.sources.insert(path, RuntimeSource::Available(source));
Ok(receipt)
}
fn resolve(&self, path: &DataPath) -> Result<BlobBinding, NamespaceError> {
let persisted = self
.store
.snapshot()
.bindings
.get(path)
.ok_or_else(|| NamespaceError::PathNotFound(path.clone()))?;
match self.sources.get(path) {
Some(RuntimeSource::Available(source)) => Ok(BlobBinding {
source: *source,
length: persisted.length,
revision: persisted.revision,
}),
Some(RuntimeSource::Unavailable(reason)) => {
Err(NamespaceError::SourceRecovery(reason.clone()))
}
None => Err(NamespaceError::SourceRecovery(format!(
"binding {path} has no recovered runtime source"
))),
}
}
fn unregister(
&mut self,
path: DataPath,
operation_id: OperationId,
retired: Option<ActorAddress>,
) -> Result<MutationReceipt, NamespaceError> {
let request = MutationRequest::Unregister { path: path.clone() };
if let Some(replayed) = self.replay(operation_id, &request) {
return replayed;
}
if !self.store.snapshot().bindings.contains_key(&path) {
let mut next = self.store.snapshot().clone();
next.operations.insert(
operation_id,
PersistedOperation {
request,
result: PersistedMutationResult::Rejected(MutationRejection::PathNotFound(
path.clone(),
)),
},
);
self.store.commit(next)?;
return Err(NamespaceError::PathNotFound(path));
}
let revision = self.store.snapshot().next_revision;
let next_revision = revision
.checked_add(1)
.filter(|revision| *revision != 0)
.ok_or(NamespaceStoreError::RevisionExhausted)?;
let receipt = MutationReceipt { revision };
let mut next = self.store.snapshot().clone();
next.next_revision = next_revision;
next.bindings.remove(&path);
if let Some(retired) = retired
&& !next.retirements.contains(&retired)
{
next.retirements.push(retired);
}
next.operations.insert(
operation_id,
PersistedOperation {
request,
result: PersistedMutationResult::Committed(receipt),
},
);
self.store.commit(next)?;
self.sources.remove(&path);
Ok(receipt)
}
}
impl ActorInterface for DataDirectoryActor {
type Incoming = DataDirectoryIn;
type Response = ();
fn on_start(&mut self, ctx: &Ctx<'_>) {
for source in self.store.snapshot().retirements.iter().copied() {
let _ = ctx.send(source, BlobSourceIn::Retire);
}
}
fn handle(&mut self, ctx: &Ctx<'_>, message: DataDirectoryIn) {
match message {
DataDirectoryIn::Register {
request_id,
path,
source,
length,
recovery,
operation_id,
reply_to,
} => {
let replayed = self.store.snapshot().operations.contains_key(&operation_id);
let retired = (!replayed)
.then(|| self.sources.get(&path))
.flatten()
.and_then(|runtime_source| match runtime_source {
RuntimeSource::Available(actor) if *actor != source => Some(*actor),
RuntimeSource::Available(_) | RuntimeSource::Unavailable(_) => None,
});
let result = self.register(path, source, length, recovery, operation_id, retired);
if result.is_ok()
&& let Some(retired) = retired
{
let _ = ctx.send(retired, BlobSourceIn::Retire);
}
let _ = ctx.send(
reply_to,
NamespaceClientIn::DirectoryReply(DataDirectoryOut::Registered {
request_id,
authority_epoch: self.authority_epoch,
result,
}),
);
}
DataDirectoryIn::Resolve {
request_id,
path,
reply_to,
} => {
let result = self.resolve(&path);
let _ = ctx.send(
reply_to,
NamespaceClientIn::DirectoryReply(DataDirectoryOut::Resolved {
request_id,
authority_epoch: self.authority_epoch,
result,
}),
);
}
DataDirectoryIn::Unregister {
request_id,
path,
operation_id,
reply_to,
} => {
let replayed = self.store.snapshot().operations.contains_key(&operation_id);
let retired = (!replayed)
.then(|| self.sources.get(&path))
.flatten()
.and_then(|source| match source {
RuntimeSource::Available(actor) => Some(*actor),
RuntimeSource::Unavailable(_) => None,
});
let result = self.unregister(path, operation_id, retired);
if result.is_ok()
&& let Some(retired) = retired
{
let _ = ctx.send(retired, BlobSourceIn::Retire);
}
let _ = ctx.send(
reply_to,
NamespaceClientIn::DirectoryReply(DataDirectoryOut::Unregistered {
request_id,
authority_epoch: self.authority_epoch,
result,
}),
);
}
}
}
}
pub trait NamespaceDiscovery: Send + Sync + 'static {
fn current_directory(&self) -> Option<ActorAddress>;
}
struct PendingRequest {
request: NamespaceRequest,
reply_to: ActorAddress,
}
pub struct NamespaceClientActor {
engine: EngineHandle,
sender: ExternalSender,
discovery: Arc<dyn NamespaceDiscovery>,
retry_period: Duration,
next_request_id: u64,
pending: HashMap<DirectoryRequestId, PendingRequest>,
}
impl NamespaceClientActor {
pub fn new(
engine: EngineHandle,
sender: ExternalSender,
discovery: Arc<dyn NamespaceDiscovery>,
retry_period: Duration,
) -> Self {
Self {
engine,
sender,
discovery,
retry_period,
next_request_id: 1,
pending: HashMap::new(),
}
}
fn dispatch(&self, ctx: &Ctx<'_>, request_id: DirectoryRequestId, request: &NamespaceRequest) {
let Some(directory) = self.discovery.current_directory() else {
return;
};
let message = match request {
NamespaceRequest::Register {
path,
source,
length,
recovery,
operation_id,
} => DataDirectoryIn::Register {
request_id,
path: path.clone(),
source: *source,
length: *length,
recovery: recovery.clone(),
operation_id: *operation_id,
reply_to: ctx.self_addr(),
},
NamespaceRequest::Resolve { path } => DataDirectoryIn::Resolve {
request_id,
path: path.clone(),
reply_to: ctx.self_addr(),
},
NamespaceRequest::Unregister { path, operation_id } => DataDirectoryIn::Unregister {
request_id,
path: path.clone(),
operation_id: *operation_id,
reply_to: ctx.self_addr(),
},
};
let _ = ctx.send(directory, message);
}
}
impl ActorInterface for NamespaceClientActor {
type Incoming = NamespaceClientIn;
type Response = ();
fn on_start(&mut self, ctx: &Ctx<'_>) {
self.engine.send_every(
self.retry_period,
self.sender.clone(),
ctx.self_addr(),
NamespaceClientIn::Retry,
);
}
fn handle(&mut self, ctx: &Ctx<'_>, message: NamespaceClientIn) {
match message {
NamespaceClientIn::Request { request, reply_to } => {
let request_id = DirectoryRequestId(self.next_request_id);
let Some(next) = self
.next_request_id
.checked_add(1)
.filter(|next| *next != 0)
else {
let _ = ctx.send(
reply_to,
DataDirectoryOut::Resolved {
request_id,
authority_epoch: 0,
result: Err(NamespaceError::Protocol(
"namespace client request IDs exhausted".to_owned(),
)),
},
);
return;
};
self.next_request_id = next;
self.dispatch(ctx, request_id, &request);
self.pending
.insert(request_id, PendingRequest { request, reply_to });
}
NamespaceClientIn::Cancel { reply_to } => {
self.pending
.retain(|_, pending| pending.reply_to != reply_to);
}
NamespaceClientIn::DirectoryReply(reply) => {
if let Some(pending) = self.pending.remove(&reply.request_id()) {
let _ = ctx.send(pending.reply_to, reply);
}
}
NamespaceClientIn::TransferFailed {
destination,
transfer_id,
reason,
} => {
let _ = ctx.send(
destination,
BlobTransferEvent::Failed {
transfer_id,
reason,
},
);
}
NamespaceClientIn::Retry => {
for (request_id, pending) in &self.pending {
self.dispatch(ctx, *request_id, &pending.request);
}
}
}
}
}
struct NamespaceRequestCancellation {
runtime: Runtime,
proxy: ActorAddress,
reply_to: ActorAddress,
armed: bool,
}
impl Drop for NamespaceRequestCancellation {
fn drop(&mut self) {
if self.armed {
let _ = self.runtime.send_to(
self.proxy,
NamespaceClientIn::Cancel {
reply_to: self.reply_to,
},
);
}
}
}
#[derive(Clone)]
pub struct NamespaceClient {
runtime: Runtime,
proxy: ActorAddress,
}
impl NamespaceClient {
pub fn new(runtime: Runtime, proxy: ActorAddress) -> Self {
Self { runtime, proxy }
}
pub fn proxy(&self) -> ActorAddress {
self.proxy
}
async fn request(&self, request: NamespaceRequest) -> Result<DataDirectoryOut, NamespaceError> {
let inbox = self
.runtime
.new_inbox::<DataDirectoryOut>()
.map_err(|error| NamespaceError::DirectoryUnavailable(error.to_string()))?;
let reply_to = *inbox.addr();
self.runtime
.send_to(self.proxy, NamespaceClientIn::Request { request, reply_to })
.map_err(|error| NamespaceError::DirectoryUnavailable(error.to_string()))?;
let mut cancellation = NamespaceRequestCancellation {
runtime: self.runtime.clone(),
proxy: self.proxy,
reply_to,
armed: true,
};
let reply = inbox.recv().await;
cancellation.armed = false;
Ok(reply)
}
pub async fn register(
&self,
path: DataPath,
source: ActorAddress,
length: u64,
recovery: SourceRecovery,
operation_id: OperationId,
) -> Result<MutationReceipt, NamespaceError> {
match self
.request(NamespaceRequest::Register {
path,
source,
length,
recovery,
operation_id,
})
.await?
{
DataDirectoryOut::Registered { result, .. } => result,
other => Err(NamespaceError::Protocol(format!(
"expected register reply, received {other:?}"
))),
}
}
pub async fn resolve(&self, path: DataPath) -> Result<BlobBinding, NamespaceError> {
match self.request(NamespaceRequest::Resolve { path }).await? {
DataDirectoryOut::Resolved { result, .. } => result,
other => Err(NamespaceError::Protocol(format!(
"expected resolve reply, received {other:?}"
))),
}
}
pub async fn unregister(
&self,
path: DataPath,
operation_id: OperationId,
) -> Result<MutationReceipt, NamespaceError> {
match self
.request(NamespaceRequest::Unregister { path, operation_id })
.await?
{
DataDirectoryOut::Unregistered { result, .. } => result,
other => Err(NamespaceError::Protocol(format!(
"expected unregister reply, received {other:?}"
))),
}
}
}
#[derive(Clone)]
pub struct DirectoryClient {
runtime: Runtime,
directory: ActorAddress,
next_request_id: Arc<AtomicU64>,
}
impl DirectoryClient {
pub fn new(runtime: Runtime, directory: ActorAddress) -> Self {
Self {
runtime,
directory,
next_request_id: Arc::new(AtomicU64::new(1)),
}
}
pub fn directory(&self) -> ActorAddress {
self.directory
}
fn request_id(&self) -> DirectoryRequestId {
DirectoryRequestId(self.next_request_id.fetch_add(1, Ordering::Relaxed))
}
async fn receive(
&self,
inbox: &swactor::runtime::Inbox<NamespaceClientIn>,
) -> Result<DataDirectoryOut, NamespaceError> {
match inbox.recv().await {
NamespaceClientIn::DirectoryReply(reply) => Ok(reply),
other => Err(NamespaceError::Protocol(format!(
"expected directory reply, received {other:?}"
))),
}
}
pub async fn register(
&self,
path: DataPath,
source: ActorAddress,
length: u64,
recovery: SourceRecovery,
operation_id: OperationId,
) -> Result<MutationReceipt, NamespaceError> {
let inbox = self
.runtime
.new_inbox::<NamespaceClientIn>()
.map_err(|error| NamespaceError::DirectoryUnavailable(error.to_string()))?;
self.runtime
.send_to(
self.directory,
DataDirectoryIn::Register {
request_id: self.request_id(),
path,
source,
length,
recovery,
operation_id,
reply_to: *inbox.addr(),
},
)
.map_err(|error| NamespaceError::DirectoryUnavailable(error.to_string()))?;
match self.receive(&inbox).await? {
DataDirectoryOut::Registered { result, .. } => result,
other => Err(NamespaceError::Protocol(format!(
"expected register reply, received {other:?}"
))),
}
}
pub async fn resolve(&self, path: DataPath) -> Result<BlobBinding, NamespaceError> {
let inbox = self
.runtime
.new_inbox::<NamespaceClientIn>()
.map_err(|error| NamespaceError::DirectoryUnavailable(error.to_string()))?;
self.runtime
.send_to(
self.directory,
DataDirectoryIn::Resolve {
request_id: self.request_id(),
path,
reply_to: *inbox.addr(),
},
)
.map_err(|error| NamespaceError::DirectoryUnavailable(error.to_string()))?;
match self.receive(&inbox).await? {
DataDirectoryOut::Resolved { result, .. } => result,
other => Err(NamespaceError::Protocol(format!(
"expected resolve reply, received {other:?}"
))),
}
}
pub async fn unregister(
&self,
path: DataPath,
operation_id: OperationId,
) -> Result<MutationReceipt, NamespaceError> {
let inbox = self
.runtime
.new_inbox::<NamespaceClientIn>()
.map_err(|error| NamespaceError::DirectoryUnavailable(error.to_string()))?;
self.runtime
.send_to(
self.directory,
DataDirectoryIn::Unregister {
request_id: self.request_id(),
path,
operation_id,
reply_to: *inbox.addr(),
},
)
.map_err(|error| NamespaceError::DirectoryUnavailable(error.to_string()))?;
match self.receive(&inbox).await? {
DataDirectoryOut::Unregistered { result, .. } => result,
other => Err(NamespaceError::Protocol(format!(
"expected unregister reply, received {other:?}"
))),
}
}
}
pub fn register_namespace_codecs(registry: &mut CodecRegistry) {
registry.register::<DataDirectoryIn, _>(JsonCodec::default());
registry.register::<NamespaceClientIn, _>(JsonCodec::default());
}

407
crates/data-plane/src/namespace_store.rs Normal file
View file

@ -0,0 +1,407 @@
//! Crash-consistent durable state for the virtual blob namespace.
use std::collections::BTreeMap;
use std::fmt;
use std::fs::{self, File, OpenOptions};
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
use swactor::actor::ActorAddress;
use crate::path::DataPath;
const SCHEMA_VERSION: u32 = 1;
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct OperationId([u8; 16]);
impl OperationId {
pub const fn from_u128(value: u128) -> Self {
Self(value.to_be_bytes())
}
pub const fn bytes(self) -> [u8; 16] {
self.0
}
}
impl Serialize for OperationId {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_str(&format!("{:032x}", u128::from_be_bytes(self.0)))
}
}
impl<'de> Deserialize<'de> for OperationId {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let encoded = String::deserialize(deserializer)?;
if encoded.len() != 32 {
return Err(serde::de::Error::custom(
"namespace operation ID must contain 32 hexadecimal digits",
));
}
u128::from_str_radix(&encoded, 16)
.map(Self::from_u128)
.map_err(serde::de::Error::custom)
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum SourceRecovery {
File { path: PathBuf },
Actor { actor: ActorAddress },
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct PersistedBinding {
pub length: u64,
pub revision: u64,
pub recovery: SourceRecovery,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum MutationRequest {
Register {
path: DataPath,
length: u64,
recovery: SourceRecovery,
},
Unregister {
path: DataPath,
},
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct MutationReceipt {
pub revision: u64,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum MutationRejection {
PathNotFound(DataPath),
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum PersistedMutationResult {
Committed(MutationReceipt),
Rejected(MutationRejection),
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct PersistedOperation {
pub request: MutationRequest,
pub result: PersistedMutationResult,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct NamespaceSnapshot {
schema_version: u32,
pub authority_epoch: u64,
pub next_revision: u64,
pub bindings: BTreeMap<DataPath, PersistedBinding>,
pub operations: BTreeMap<OperationId, PersistedOperation>,
#[serde(default)]
pub retirements: Vec<ActorAddress>,
}
impl Default for NamespaceSnapshot {
fn default() -> Self {
Self {
schema_version: SCHEMA_VERSION,
authority_epoch: 0,
next_revision: 1,
bindings: BTreeMap::new(),
operations: BTreeMap::new(),
retirements: Vec::new(),
}
}
}
impl NamespaceSnapshot {
fn validate(&self) -> Result<(), NamespaceStoreError> {
if self.schema_version != SCHEMA_VERSION {
return Err(NamespaceStoreError::UnsupportedSchema {
found: self.schema_version,
supported: SCHEMA_VERSION,
});
}
if self.next_revision == 0 {
return Err(NamespaceStoreError::Corrupt(
"next namespace revision is zero".to_owned(),
));
}
if self
.bindings
.values()
.any(|binding| binding.revision == 0 || binding.revision >= self.next_revision)
{
return Err(NamespaceStoreError::Corrupt(
"binding revision is outside the committed revision range".to_owned(),
));
}
if self.operations.values().any(|operation| {
matches!(
operation.result,
PersistedMutationResult::Committed(receipt)
if receipt.revision == 0 || receipt.revision >= self.next_revision
)
}) {
return Err(NamespaceStoreError::Corrupt(
"operation revision is outside the committed revision range".to_owned(),
));
}
Ok(())
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum NamespaceStoreError {
Io(String),
Corrupt(String),
UnsupportedSchema { found: u32, supported: u32 },
RevisionExhausted,
AuthorityEpochExhausted,
}
impl fmt::Display for NamespaceStoreError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Io(reason) => write!(f, "namespace storage I/O failed: {reason}"),
Self::Corrupt(reason) => write!(f, "namespace storage is corrupt: {reason}"),
Self::UnsupportedSchema { found, supported } => write!(
f,
"unsupported namespace schema {found}; supported schema is {supported}"
),
Self::RevisionExhausted => f.write_str("namespace revision exhausted"),
Self::AuthorityEpochExhausted => f.write_str("namespace authority epoch exhausted"),
}
}
}
impl std::error::Error for NamespaceStoreError {}
impl From<std::io::Error> for NamespaceStoreError {
fn from(error: std::io::Error) -> Self {
Self::Io(error.to_string())
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum CommitFailpoint {
None,
BeforeRename,
AfterRename,
}
pub struct NamespaceStore {
path: PathBuf,
snapshot: NamespaceSnapshot,
}
impl NamespaceStore {
pub fn open(path: impl AsRef<Path>) -> Result<Self, NamespaceStoreError> {
let path = path.as_ref().to_path_buf();
let snapshot = match File::open(&path) {
Ok(mut file) => {
let mut bytes = Vec::new();
file.read_to_end(&mut bytes)?;
let snapshot: NamespaceSnapshot = serde_json::from_slice(&bytes)
.map_err(|error| NamespaceStoreError::Corrupt(error.to_string()))?;
snapshot.validate()?;
snapshot
}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
NamespaceSnapshot::default()
}
Err(error) => return Err(error.into()),
};
Ok(Self { path, snapshot })
}
pub fn snapshot(&self) -> &NamespaceSnapshot {
&self.snapshot
}
pub fn advance_authority_epoch(&mut self) -> Result<u64, NamespaceStoreError> {
let mut next = self.snapshot.clone();
next.authority_epoch = next
.authority_epoch
.checked_add(1)
.filter(|epoch| *epoch != 0)
.ok_or(NamespaceStoreError::AuthorityEpochExhausted)?;
self.commit(next)?;
Ok(self.snapshot.authority_epoch)
}
pub fn commit(&mut self, next: NamespaceSnapshot) -> Result<(), NamespaceStoreError> {
next.validate()?;
self.persist(&next)?;
self.snapshot = next;
Ok(())
}
fn persist(&self, snapshot: &NamespaceSnapshot) -> Result<(), NamespaceStoreError> {
self.persist_with_failpoint(snapshot, CommitFailpoint::None)
}
fn persist_with_failpoint(
&self,
snapshot: &NamespaceSnapshot,
failpoint: CommitFailpoint,
) -> Result<(), NamespaceStoreError> {
let parent = self.path.parent().ok_or_else(|| {
NamespaceStoreError::Io("namespace state path has no parent".to_owned())
})?;
fs::create_dir_all(parent)?;
let file_name = self.path.file_name().ok_or_else(|| {
NamespaceStoreError::Io("namespace state path has no file name".to_owned())
})?;
let temporary = parent.join(format!(".{}.tmp", file_name.to_string_lossy()));
let bytes = serde_json::to_vec(snapshot)
.map_err(|error| NamespaceStoreError::Corrupt(error.to_string()))?;
let write_result = (|| -> Result<(), NamespaceStoreError> {
let mut file = OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(&temporary)?;
file.write_all(&bytes)?;
file.sync_all()?;
if failpoint == CommitFailpoint::BeforeRename {
return Err(NamespaceStoreError::Io(
"injected failure before durable replacement".to_owned(),
));
}
fs::rename(&temporary, &self.path)?;
File::open(parent)?.sync_all()?;
if failpoint == CommitFailpoint::AfterRename {
return Err(NamespaceStoreError::Io(
"injected crash after durable replacement".to_owned(),
));
}
Ok(())
})();
if write_result.is_err() {
let _ = fs::remove_file(&temporary);
}
write_result
}
#[cfg(test)]
fn commit_with_failpoint(
&mut self,
next: NamespaceSnapshot,
failpoint: CommitFailpoint,
) -> Result<(), NamespaceStoreError> {
next.validate()?;
self.persist_with_failpoint(&next, failpoint)?;
self.snapshot = next;
Ok(())
}
}
#[cfg(test)]
mod tests {
use std::sync::atomic::{AtomicU64, Ordering};
use super::*;
static NEXT_TEMP: AtomicU64 = AtomicU64::new(1);
struct TestState {
root: PathBuf,
file: PathBuf,
}
impl TestState {
fn new() -> Self {
let sequence = NEXT_TEMP.fetch_add(1, Ordering::Relaxed);
let root = std::env::temp_dir().join(format!(
"swactor-namespace-store-{}-{sequence}",
std::process::id()
));
fs::create_dir_all(&root).expect("create test state directory");
let file = root.join("namespace.json");
Self { root, file }
}
}
impl Drop for TestState {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.root);
}
}
fn state_with_binding(store: &NamespaceStore) -> NamespaceSnapshot {
let mut next = store.snapshot().clone();
next.bindings.insert(
DataPath::parse("/models/a").unwrap(),
PersistedBinding {
length: 8,
revision: 1,
recovery: SourceRecovery::Actor {
actor: ActorAddress([7; 32]),
},
},
);
next.next_revision = 2;
next
}
#[test]
fn failure_before_rename_preserves_previous_snapshot() {
let state = TestState::new();
let mut store = NamespaceStore::open(&state.file).unwrap();
store.advance_authority_epoch().unwrap();
let next = state_with_binding(&store);
assert!(
store
.commit_with_failpoint(next, CommitFailpoint::BeforeRename)
.is_err()
);
let reopened = NamespaceStore::open(&state.file).unwrap();
assert!(reopened.snapshot().bindings.is_empty());
}
#[test]
fn crash_after_rename_recovers_new_snapshot() {
let state = TestState::new();
let mut store = NamespaceStore::open(&state.file).unwrap();
store.advance_authority_epoch().unwrap();
let next = state_with_binding(&store);
assert!(
store
.commit_with_failpoint(next, CommitFailpoint::AfterRename)
.is_err()
);
let reopened = NamespaceStore::open(&state.file).unwrap();
assert_eq!(
reopened
.snapshot()
.bindings
.get(&DataPath::parse("/models/a").unwrap())
.unwrap()
.length,
8
);
}
#[test]
fn malformed_snapshot_is_rejected() {
let state = TestState::new();
fs::write(&state.file, b"{not-json").unwrap();
assert!(matches!(
NamespaceStore::open(&state.file),
Err(NamespaceStoreError::Corrupt(_))
));
}
}

61
crates/data-plane/src/protocol.rs
View file

@ -104,7 +104,6 @@ pub enum DataPlaneError {
operation: DataOperation,
},
PathNotFound(DataPath),
PathAlreadyExists(DataPath),
SourceFailure(String),
ArenaExhausted,
Blob(BlobFailure),
@ -124,7 +123,6 @@ impl fmt::Display for DataPlaneError {
write!(f, "{operation:?} is not authorized for {path}")
}
Self::PathNotFound(path) => write!(f, "data path not found: {path}"),
Self::PathAlreadyExists(path) => write!(f, "data path already exists: {path}"),
Self::SourceFailure(reason) => write!(f, "blob source failed: {reason}"),
Self::ArenaExhausted => f.write_str("data-plane arena is exhausted"),
Self::Blob(reason) => write!(f, "blob lease failure: {reason:?}"),
@ -142,14 +140,6 @@ impl From<BlobError> for DataPlaneError {
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct PublishedBlobInfo {
pub path: DataPath,
pub binding: ActorAddress,
pub lease: BlobLease,
pub metadata: BlobMetadata,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum HostSessionIn {
Attach {
@ -163,6 +153,9 @@ pub enum HostSessionIn {
child_session: ActorAddress,
operation: ActorAddress,
},
CancelReadBlob {
operation: ActorAddress,
},
OpenWriteBlob {
path: DataPath,
length: u64,
@ -186,13 +179,6 @@ pub enum HostSessionIn {
lease_id: BlobLeaseId,
generation: u64,
},
BindingPublished {
binding: ActorAddress,
operation: ActorAddress,
path: DataPath,
lease: BlobLease,
metadata: BlobMetadata,
},
BindingFaulted {
binding: ActorAddress,
operation: ActorAddress,
@ -201,39 +187,8 @@ pub enum HostSessionIn {
BindingDone {
binding: ActorAddress,
},
InspectPublished {
path: DataPath,
reply_to: ActorAddress,
},
ReleasePublished {
path: DataPath,
},
BeginBlobSource {
path: DataPath,
metadata: BlobMetadata,
reply_to: ActorAddress,
},
BlobSourceChunk {
path: DataPath,
bytes: Vec<u8>,
},
FinishBlobSource {
path: DataPath,
},
FailBlobSource {
path: DataPath,
reason: String,
},
SourceReady {
path: DataPath,
source: ActorAddress,
lease: BlobLease,
metadata: BlobMetadata,
},
SourceFaulted {
path: DataPath,
source: ActorAddress,
error: DataPlaneError,
BindingDetached {
binding: ActorAddress,
},
ConfigureRun {
run_id: String,
@ -261,6 +216,9 @@ pub enum ChildSessionIn {
path: DataPath,
reply_to: ActorAddress,
},
CancelRead {
reply_to: ActorAddress,
},
OpenWriteBlob {
path: DataPath,
length: u64,
@ -303,4 +261,7 @@ impl NetworkMessage for ChildSessionIn {
pub fn register_data_plane_codecs(registry: &mut CodecRegistry) {
registry.register::<HostSessionIn, _>(JsonCodec::default());
registry.register::<ChildSessionIn, _>(JsonCodec::default());
crate::namespace::register_namespace_codecs(registry);
crate::blob_transfer::register_blob_transfer_codecs(registry);
crate::source::register_blob_source_codecs(registry);
}

299
crates/data-plane/src/source.rs Normal file
View file

@ -0,0 +1,299 @@
//! Persistent fixed-length blob sources.
use std::collections::HashSet;
use std::fs::{File, OpenOptions};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use serde::{Deserialize, Serialize};
use swactor::actor::{ActorAddress, ActorInterface, Ctx};
use swactor::runtime::Runtime;
use swactor_transport::{CodecRegistry, JsonCodec, NetworkMessage};
use crate::blob_transfer::{
BlobTransferCompletion, BlobTransferEvent, BlobTransferId, BlobTransferOffer,
BlobTransferSender, FileTransferRequest,
};
use crate::namespace::{NamespaceClientIn, NamespaceError};
use crate::namespace_store::SourceRecovery;
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum BlobSourceIn {
BeginTransfer {
offer: BlobTransferOffer,
},
TransferCompleted {
transfer_id: BlobTransferId,
destination: ActorAddress,
failure_proxy: Option<ActorAddress>,
result: Result<(), String>,
},
Retire,
}
impl NetworkMessage for BlobSourceIn {
fn type_tag() -> &'static str {
"data-plane.blob-source.in.v1"
}
}
struct ActorTransferCompletion {
runtime: Runtime,
source: ActorAddress,
transfer_id: BlobTransferId,
destination: ActorAddress,
failure_proxy: Option<ActorAddress>,
}
impl BlobTransferCompletion for ActorTransferCompletion {
fn complete(self: Box<Self>, result: Result<(), String>) {
let _ = self.runtime.send_to(
self.source,
BlobSourceIn::TransferCompleted {
transfer_id: self.transfer_id,
destination: self.destination,
failure_proxy: self.failure_proxy,
result,
},
);
}
}
pub trait BlobSourcePublisher: Send + Sync + 'static {
fn publish_source(&self, source: ActorAddress) -> Result<(), String>;
}
pub trait BlobSourceRetirement: Send + Sync + 'static {
fn retired(&self);
}
pub struct FileBlobSourceActor {
runtime: Runtime,
sender: Arc<dyn BlobTransferSender>,
recovery_path: Option<PathBuf>,
label: String,
file: File,
offset: u64,
length: u64,
active: HashSet<BlobTransferId>,
retiring: bool,
retirement: Option<Arc<dyn BlobSourceRetirement>>,
}
impl FileBlobSourceActor {
pub fn open(
runtime: Runtime,
sender: Arc<dyn BlobTransferSender>,
path: impl AsRef<Path>,
) -> Result<Self, NamespaceError> {
let path = path.as_ref().to_path_buf();
let file = OpenOptions::new().read(true).open(&path).map_err(|error| {
NamespaceError::SourceRecovery(format!("open {}: {error}", path.display()))
})?;
let length = file
.metadata()
.map_err(|error| {
NamespaceError::SourceRecovery(format!("stat {}: {error}", path.display()))
})?
.len();
Ok(Self {
runtime,
sender,
recovery_path: Some(path.clone()),
label: path.display().to_string(),
file,
offset: 0,
length,
active: HashSet::new(),
retiring: false,
retirement: None,
})
}
pub fn recover(
runtime: Runtime,
sender: Arc<dyn BlobTransferSender>,
path: impl AsRef<Path>,
expected_length: u64,
) -> Result<Self, NamespaceError> {
let source = Self::open(runtime, sender, path)?;
if source.length != expected_length {
return Err(NamespaceError::SourceRecovery(format!(
"recovered file {} has length {}, expected {}",
source.label, source.length, expected_length
)));
}
Ok(source)
}
pub fn from_file_region(
runtime: Runtime,
sender: Arc<dyn BlobTransferSender>,
file: File,
offset: u64,
length: u64,
retirement: Option<Arc<dyn BlobSourceRetirement>>,
) -> Result<Self, NamespaceError> {
let file_length = file
.metadata()
.map_err(|error| NamespaceError::SourceRecovery(format!("stat arena source: {error}")))?
.len();
let end = offset.checked_add(length).ok_or_else(|| {
NamespaceError::SourceRecovery("arena source range overflow".to_owned())
})?;
if end > file_length {
return Err(NamespaceError::SourceRecovery(format!(
"arena source range ends at {end}, backing length is {file_length}"
)));
}
Ok(Self {
runtime,
sender,
recovery_path: None,
label: format!("arena region {offset}..{end}"),
file,
offset,
length,
active: HashSet::new(),
retiring: false,
retirement,
})
}
pub fn length(&self) -> u64 {
self.length
}
pub fn recovery(&self) -> SourceRecovery {
SourceRecovery::File {
path: self
.recovery_path
.clone()
.expect("disk source has a recovery path"),
}
}
fn send_failure(
&self,
ctx: &Ctx<'_>,
destination: ActorAddress,
failure_proxy: Option<ActorAddress>,
transfer_id: BlobTransferId,
reason: String,
) {
if let Some(proxy) = failure_proxy {
let _ = ctx.send(
proxy,
NamespaceClientIn::TransferFailed {
destination,
transfer_id,
reason,
},
);
} else {
let _ = ctx.send(
destination,
BlobTransferEvent::Failed {
transfer_id,
reason,
},
);
}
}
fn fail_destination(&self, ctx: &Ctx<'_>, offer: &BlobTransferOffer, reason: String) {
self.send_failure(
ctx,
offer.destination,
offer.failure_proxy,
offer.transfer_id,
reason,
);
}
fn maybe_stop_retired(&mut self, ctx: &Ctx<'_>) {
if self.retiring && self.active.is_empty() {
if let Some(retirement) = self.retirement.take() {
retirement.retired();
}
ctx.stop_self();
}
}
}
impl ActorInterface for FileBlobSourceActor {
type Incoming = BlobSourceIn;
type Response = ();
fn handle(&mut self, ctx: &Ctx<'_>, message: BlobSourceIn) {
match message {
BlobSourceIn::BeginTransfer { offer } => {
if self.retiring {
self.fail_destination(ctx, &offer, "blob source is retired".to_owned());
return;
}
if !self.active.insert(offer.transfer_id) {
self.fail_destination(
ctx,
&offer,
"duplicate blob transfer identifier".to_owned(),
);
return;
}
let file = match self.file.try_clone() {
Ok(file) => file,
Err(error) => {
self.active.remove(&offer.transfer_id);
self.fail_destination(
ctx,
&offer,
format!("clone source file {}: {error}", self.label),
);
return;
}
};
let completion: Box<dyn BlobTransferCompletion> =
Box::new(ActorTransferCompletion {
runtime: self.runtime.clone(),
source: ctx.self_addr(),
transfer_id: offer.transfer_id,
destination: offer.destination,
failure_proxy: offer.failure_proxy,
});
let request = FileTransferRequest {
offer: offer.clone(),
file,
offset: self.offset,
length: self.length,
completion,
};
if let Err(error) = self.sender.start_file(request) {
self.active.remove(&offer.transfer_id);
self.fail_destination(ctx, &offer, error);
self.maybe_stop_retired(ctx);
}
}
BlobSourceIn::TransferCompleted {
transfer_id,
destination,
failure_proxy,
result,
} => {
if self.active.remove(&transfer_id) {
if let Err(reason) = result {
self.send_failure(ctx, destination, failure_proxy, transfer_id, reason);
}
self.maybe_stop_retired(ctx);
}
}
BlobSourceIn::Retire => {
self.retiring = true;
self.maybe_stop_retired(ctx);
}
}
}
}
pub fn register_blob_source_codecs(registry: &mut CodecRegistry) {
registry.register::<BlobSourceIn, _>(JsonCodec::default());
}

481
crates/data-plane/tests/actor_blob_guarantees.rs
View file

@ -1,20 +1,18 @@
#![cfg(target_os = "linux")]
use std::collections::BTreeMap;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;
use data_plane::arena::{ArenaConfig, ArenaManager, NodeId};
use data_plane::blob::{
BLOB_HEADER_LEN, Blob, BlobError, BlobLease, BlobMetadata, BlobSharedState, ContentDigest,
LeaseReleaser,
BLOB_HEADER_LEN, Blob, BlobError, BlobLease, BlobMetadata, BlobSharedState, LeaseReleaser,
};
use data_plane::bootstrap::{self, BootstrapSpec};
use data_plane::data_plane::DataPlaneBootstrap;
use data_plane::host::{BlobSource, HostDataPlaneConfig, HostDataPlaneSessionActor};
use data_plane::host::{HostDataPlaneConfig, HostDataPlaneSessionActor};
use data_plane::path::{DataPath, JobContext};
use data_plane::protocol::{DataPlaneError, HostSessionIn, JobCapability, PublishedBlobInfo};
use data_plane::protocol::{DataPlaneError, JobCapability};
use futures_lite::future::{self, FutureExt};
use swactor::Error;
use swactor::actor::ActorAddress;
@ -56,24 +54,121 @@ impl RemoteSink for BlackHoleSink {
struct Harness {
_host_engine: Engine,
_child_engine: Engine,
host_runtime: Runtime,
host_session: ActorAddress,
_temp: TempState,
bootstrap: DataPlaneBootstrap,
}
static NEXT_TEMP: AtomicU64 = AtomicU64::new(1);
struct TempState {
root: std::path::PathBuf,
}
impl TempState {
fn new() -> Self {
let sequence = NEXT_TEMP.fetch_add(1, Ordering::Relaxed);
let root = std::env::temp_dir().join(format!(
"swactor-actor-blob-{}-{sequence}",
std::process::id()
));
std::fs::create_dir_all(&root).unwrap();
Self { root }
}
}
impl Drop for TempState {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.root);
}
}
fn path(value: &str) -> DataPath {
DataPath::parse(value).expect("test path")
}
fn runtime_parts() -> (RuntimeParts, Runtime) {
let mut config = RuntimeConfig::default();
config.worker_count = 1;
let parts = RuntimeParts::new(config);
let parts = RuntimeParts::new(RuntimeConfig {
worker_count: 1,
..RuntimeConfig::default()
});
let runtime = parts.runtime().clone();
(parts, runtime)
}
struct LoopbackSender {
runtime: Runtime,
}
impl data_plane::blob_transfer::BlobTransferSender for LoopbackSender {
fn start_file(
&self,
request: data_plane::blob_transfer::FileTransferRequest,
) -> Result<(), String> {
use std::os::unix::fs::FileExt;
let mut bytes = vec![0_u8; request.length as usize];
request
.file
.read_exact_at(&mut bytes, request.offset)
.map_err(|error| error.to_string())?;
self.runtime
.send_to(
request.offer.destination,
data_plane::blob_transfer::BlobTransferEvent::Chunk {
transfer_id: request.offer.transfer_id,
bytes,
},
)
.map_err(|error| error.to_string())?;
self.runtime
.send_to(
request.offer.destination,
data_plane::blob_transfer::BlobTransferEvent::Finished {
transfer_id: request.offer.transfer_id,
},
)
.map_err(|error| error.to_string())?;
request.completion.complete(Ok(()));
Ok(())
}
}
struct DirectReceiver;
impl data_plane::blob_transfer::BlobTransferReceiver for DirectReceiver {
fn open(
&self,
destination: ActorAddress,
transfer_id: data_plane::blob_transfer::BlobTransferId,
) -> Result<data_plane::blob_transfer::BlobTransferOffer, String> {
Ok(data_plane::blob_transfer::BlobTransferOffer {
transfer_id,
destination,
failure_proxy: None,
transport: Vec::new(),
})
}
fn cancel(&self, _offer: &data_plane::blob_transfer::BlobTransferOffer) {}
}
struct StaticDiscovery(ActorAddress);
impl data_plane::namespace::NamespaceDiscovery for StaticDiscovery {
fn current_directory(&self) -> Option<ActorAddress> {
Some(self.0)
}
}
struct NoopSourceRegistrar;
impl data_plane::source::BlobSourcePublisher for NoopSourceRegistrar {
fn publish_source(&self, _source: ActorAddress) -> Result<(), String> {
Ok(())
}
}
fn harness(arena_bytes: u64) -> Harness {
let temp = TempState::new();
let mut arena = ArenaManager::boot(ArenaConfig {
node_id: NodeId(1),
reservation_ceiling: arena_bytes,
@ -97,35 +192,6 @@ fn harness(arena_bytes: u64) -> Harness {
child_runtime.set_remote_sink(Arc::new(DirectRuntimeSink {
destination: host_runtime.clone(),
}));
let mut blobs = BTreeMap::new();
blobs.insert(
path("/models/tiny-linear/weights"),
BlobSource::with_sha256(Arc::<[u8]>::from(WEIGHTS)),
);
blobs.insert(
path("/models/second"),
BlobSource::new(Arc::<[u8]>::from(b"second-blob".as_slice())),
);
let host_session = host_runtime
.spawn(
HostDataPlaneSessionActor::new(HostDataPlaneConfig {
arena,
arena_generation: ARENA_GENERATION,
session_generation: SESSION_GENERATION,
capability: CAPABILITY,
job_context: JobContext {
run_id: "run-7".to_owned(),
read_prefixes: vec![path("/models")],
write_prefixes: vec![path("/runs/run-7/results")],
},
blobs,
route_registrar: None,
})
.expect("host session config"),
)
.expect("spawn host session");
let host_engine = Engine::new(
host_parts,
TokioBackend::new(TokioConfig::default()).expect("host backend"),
@ -136,6 +202,77 @@ fn harness(arena_bytes: u64) -> Harness {
TokioBackend::new(TokioConfig::default()).expect("child backend"),
)
.expect("child engine");
let sender: Arc<dyn data_plane::blob_transfer::BlobTransferSender> = Arc::new(LoopbackSender {
runtime: host_runtime.clone(),
});
let directory_actor = data_plane::namespace::DataDirectoryActor::recover(
temp.root.join("namespace.json"),
|_record, _length| {
Err(data_plane::namespace::NamespaceError::SourceRecovery(
"unexpected recovery".to_owned(),
))
},
)
.unwrap();
let directory = host_runtime.spawn(directory_actor).unwrap();
let directory_client =
data_plane::namespace::DirectoryClient::new(host_runtime.clone(), directory);
for (logical, name, bytes) in [
("/models/tiny-linear/weights", "weights.bin", WEIGHTS),
("/models/second", "second.bin", b"second-blob".as_slice()),
] {
let file_path = temp.root.join(name);
std::fs::write(&file_path, bytes).unwrap();
let source = data_plane::source::FileBlobSourceActor::open(
host_runtime.clone(),
Arc::clone(&sender),
&file_path,
)
.unwrap();
let length = source.length();
let recovery = source.recovery();
let source = host_runtime.spawn(source).unwrap();
future::block_on(directory_client.register(
path(logical),
source,
length,
recovery,
data_plane::namespace::OperationId::from_u128(u128::from(length) + 1),
))
.unwrap();
}
let proxy = host_runtime
.spawn(data_plane::namespace::NamespaceClientActor::new(
host_engine.handle(),
host_runtime.create_sender(),
Arc::new(StaticDiscovery(directory)),
Duration::from_millis(5),
))
.unwrap();
let namespace = data_plane::namespace::NamespaceClient::new(host_runtime.clone(), proxy);
let host_session = host_runtime
.spawn(
HostDataPlaneSessionActor::new(HostDataPlaneConfig {
runtime: host_runtime.clone(),
arena,
arena_generation: ARENA_GENERATION,
session_generation: SESSION_GENERATION,
capability: CAPABILITY,
job_context: JobContext {
run_id: "run-7".to_owned(),
read_prefixes: vec![path("/models"), path("/runs/run-7/results")],
write_prefixes: vec![path("/runs/run-7/results")],
},
namespace: Some(namespace),
transfer_receiver: Some(Arc::new(DirectReceiver)),
source_sender: Some(sender),
source_publisher: Some(Arc::new(NoopSourceRegistrar)),
route_registrar: None,
})
.expect("host session config"),
)
.expect("spawn host session");
let bootstrap = future::block_on(DataPlaneBootstrap::attach(
handoff.arena_fd,
child_runtime,
@ -147,8 +284,7 @@ fn harness(arena_bytes: u64) -> Harness {
Harness {
_host_engine: host_engine,
_child_engine: child_engine,
host_runtime,
host_session,
_temp: temp,
bootstrap,
}
}
@ -160,21 +296,6 @@ impl LeaseReleaser for NoopReleaser {
fn release(&self, _lease: BlobLease) {}
}
fn inspect_published(harness: &Harness, logical: &str) -> Option<PublishedBlobInfo> {
future::block_on(async {
harness
.host_runtime
.ask::<HostSessionIn, Option<PublishedBlobInfo>>(harness.host_session, |reply_to| {
HostSessionIn::InspectPublished {
path: path(logical),
reply_to,
}
})
.expect("inspect ask")
.await
})
}
#[test]
fn attachment_without_a_host_reply_fails_on_actor_deadline() {
let mut arena = ArenaManager::boot(ArenaConfig {
@ -202,8 +323,10 @@ fn attachment_without_a_host_reply_fails_on_actor_deadline() {
ActorAddress::new_random(),
CAPABILITY,
None,
engine.handle(),
Duration::from_millis(20),
data_plane::data_plane::AttachDeadline {
engine: engine.handle(),
timeout: Duration::from_millis(20),
},
));
assert!(matches!(
result,
@ -223,7 +346,7 @@ fn routed_read_blob_maps_final_sealed_lease_without_copying() {
.expect("read blob");
assert_eq!(blob.length(), 24);
assert!(blob.digest().is_some());
assert!(blob.digest().is_none());
let lease = blob.lease();
let view = blob.map().expect("map sealed blob");
assert_eq!(view.as_ref(), WEIGHTS);
@ -304,7 +427,7 @@ fn thirty_two_concurrent_remote_opens_complete_without_cross_wiring() {
) -> future::Boxed<Vec<(usize, Blob)>> {
if count == 1 {
return async move {
let path = if first % 2 == 0 {
let path = if first.is_multiple_of(2) {
"/models/tiny-linear/weights"
} else {
"/models/second"
@ -407,7 +530,7 @@ fn path_absence_and_authorization_fail_before_blob_success() {
harness
.bootstrap
.data_plane
.read_blob_path("/runs/self/results/private"),
.read_blob_path("/runs/self/private"),
);
assert!(matches!(
unauthorized,
@ -463,21 +586,15 @@ fn write_blob_seals_once_and_abort_publishes_nothing() {
drop(view);
future::block_on(writer.seal()).expect("seal and publish");
let published = inspect_published(&harness, "/runs/self/results/blob")
.expect("clean exit publishes exactly once");
assert_eq!(published.metadata.length, 6);
// SAFETY: publication validated a sealed lease wholly inside this mapping.
let bytes = unsafe {
std::slice::from_raw_parts(
harness
.bootstrap
.arena
.base_ptr()
.add((published.lease.offset + BLOB_HEADER_LEN) as usize),
6,
)
};
assert_eq!(bytes, b"result");
let published = future::block_on(
harness
.bootstrap
.data_plane
.read_blob_path("/runs/self/results/blob"),
)
.expect("clean exit publishes exactly once");
assert_eq!(published.length(), 6);
assert_eq!(published.map().unwrap().as_ref(), b"result");
assert!(future::block_on(writer.seal()).is_err());
let mut aborted = future::block_on(
@ -491,209 +608,13 @@ fn write_blob_seals_once_and_abort_publishes_nothing() {
assert!(future::block_on(aborted.abort()).is_err());
drop(active);
future::block_on(aborted.abort()).expect("abort after view closes");
assert_eq!(
inspect_published(&harness, "/runs/self/results/aborted"),
None
);
}
#[test]
fn streamed_blob_chunks_fill_the_final_lease_before_waiting_open_completes() {
let harness = harness(4096);
let streamed = path("/models/streamed");
let payload = b"direct-final-lease";
let metadata = BlobMetadata {
length: payload.len() as u64,
digest: Some(ContentDigest::sha256(payload)),
};
future::block_on(async {
harness
.host_runtime
.ask::<HostSessionIn, Result<(), DataPlaneError>>(harness.host_session, |reply_to| {
HostSessionIn::BeginBlobSource {
path: streamed.clone(),
metadata,
reply_to,
}
})
.expect("begin source ask")
.await
.expect("source lease allocated");
let read = harness.bootstrap.data_plane.read_blob(&streamed);
futures_lite::pin!(read);
assert!(
future::poll_once(&mut read).await.is_none(),
"read remains pending until the source seals"
);
harness
.host_runtime
.send_to(
harness.host_session,
HostSessionIn::BlobSourceChunk {
path: streamed.clone(),
bytes: payload[..6].to_vec(),
},
)
.unwrap();
harness
.host_runtime
.send_to(
harness.host_session,
HostSessionIn::BlobSourceChunk {
path: streamed.clone(),
bytes: payload[6..].to_vec(),
},
)
.unwrap();
harness
.host_runtime
.send_to(
harness.host_session,
HostSessionIn::FinishBlobSource {
path: streamed.clone(),
},
)
.unwrap();
let blob = read.await.expect("sealed streamed blob");
assert_eq!(blob.map().unwrap().as_ref(), payload);
});
}
#[test]
fn read_opened_after_stream_seal_acquires_persistent_source_lease() {
let harness = harness(4096);
let streamed = path("/models/sealed-before-open");
let payload = b"sealed-first";
future::block_on(async {
harness
.host_runtime
.ask::<HostSessionIn, Result<(), DataPlaneError>>(harness.host_session, |reply_to| {
HostSessionIn::BeginBlobSource {
path: streamed.clone(),
metadata: BlobMetadata {
length: payload.len() as u64,
digest: Some(ContentDigest::sha256(payload)),
},
reply_to,
}
})
.unwrap()
.await
.unwrap();
harness
.host_runtime
.send_to(
harness.host_session,
HostSessionIn::BlobSourceChunk {
path: streamed.clone(),
bytes: payload.to_vec(),
},
)
.unwrap();
harness
.host_runtime
.send_to(
harness.host_session,
HostSessionIn::FinishBlobSource {
path: streamed.clone(),
},
)
.unwrap();
std::thread::sleep(Duration::from_millis(10));
let blob = harness
.bootstrap
.data_plane
.read_blob(&streamed)
.await
.unwrap();
assert_eq!(blob.map().unwrap().as_ref(), payload);
});
}
#[test]
fn partial_overlength_and_digest_mismatch_sources_fail_waiting_opens() {
let harness = harness(4096);
future::block_on(async {
let cases = [
(
"/models/partial",
BlobMetadata {
length: 4,
digest: None,
},
b"ab".as_slice(),
true,
),
(
"/models/overlength",
BlobMetadata {
length: 4,
digest: None,
},
b"abcde".as_slice(),
false,
),
(
"/models/bad-digest",
BlobMetadata {
length: 4,
digest: Some(ContentDigest::sha256(b"good")),
},
b"evil".as_slice(),
true,
),
];
for (name, metadata, bytes, finish) in cases {
let source = path(name);
assert!(matches!(
future::block_on(
harness
.host_runtime
.ask::<HostSessionIn, Result<(), DataPlaneError>>(
harness.host_session,
|reply_to| HostSessionIn::BeginBlobSource {
path: source.clone(),
metadata,
reply_to,
},
)
.unwrap()
.await
.unwrap();
let read = harness.bootstrap.data_plane.read_blob(&source);
futures_lite::pin!(read);
assert!(future::poll_once(&mut read).await.is_none());
std::thread::sleep(Duration::from_millis(5));
harness
.host_runtime
.send_to(
harness.host_session,
HostSessionIn::BlobSourceChunk {
path: source.clone(),
bytes: bytes.to_vec(),
},
)
.unwrap();
if finish {
harness
.host_runtime
.send_to(
harness.host_session,
HostSessionIn::FinishBlobSource {
path: source.clone(),
},
)
.unwrap();
}
let result = read.await;
assert!(
matches!(result, Err(DataPlaneError::Blob(_))),
"unexpected source failure for {name}: {result:?}"
);
}
});
.bootstrap
.data_plane
.read_blob_path("/runs/self/results/aborted")
),
Err(DataPlaneError::PathNotFound(_))
));
}

93
crates/data-plane/tests/byte_ring_guarantees.rs
View file

@ -7,7 +7,7 @@
use data_plane::arena::{ArenaConfig, ArenaManager, NodeId};
use data_plane::byte_ring::{
self, attach, install, ByteRingSpec, FlowError, HeaderError, RecordKind, Role,
self, ByteRingSpec, FlowError, HeaderError, RecordKind, Role, attach, install,
};
fn arena() -> ArenaManager {
@ -45,10 +45,7 @@ fn scribble_u64(arena: &ArenaManager, handle: &byte_ring::RingHandle, off: u64,
#[test]
fn endpoints_are_send() {
fn assert_send<T: Send>() {}
let (arena, handle) = installed(64, 1);
let producer = attach(&arena, handle, Role::Producer).expect("attach");
assert_send::<byte_ring::Endpoint>();
drop(producer);
}
// ─── install ─────────────────────────────────────────────────────────────────
@ -57,8 +54,8 @@ fn endpoints_are_send() {
#[test]
fn install_writes_valid_header_and_zeroed_data() {
use data_plane::byte_ring::{
OFF_CAPACITY, OFF_COMMIT, OFF_CONSUME, OFF_GENERATION, OFF_MAGIC, OFF_VERSION,
RING_MAGIC, RING_VERSION,
OFF_CAPACITY, OFF_COMMIT, OFF_CONSUME, OFF_GENERATION, OFF_MAGIC, OFF_VERSION, RING_MAGIC,
RING_VERSION,
};
let (arena, handle) = installed(4096, 7);
@ -83,6 +80,7 @@ fn install_writes_valid_header_and_zeroed_data() {
assert!(data.iter().all(|&b| b == 0), "fresh data region is zero");
}
#[test]
fn install_rejects_bad_specs() {
let mut arena = arena();
assert!(matches!(
@ -101,9 +99,11 @@ fn install_rejects_bad_specs() {
// ─── attach: untrusted-header containment (P5) ──────────────────────────────
type CorruptHeaderCase = (Option<(u64, u64)>, (u64, u64), HeaderError);
#[test]
fn attach_rejects_corrupt_headers_without_trusting_them() {
let cases: Vec<(Option<(u64, u64)>, (u64, u64), HeaderError)> = vec![
let cases: Vec<CorruptHeaderCase> = vec![
// (optional pre-scribble, (field, value), expected)
(
None,
@ -113,27 +113,43 @@ fn attach_rejects_corrupt_headers_without_trusting_them() {
(
None,
(byte_ring::OFF_VERSION, 2),
HeaderError::UnsupportedVersion { found: 2, supported: 1 },
HeaderError::UnsupportedVersion {
found: 2,
supported: 1,
},
),
(
None,
(byte_ring::OFF_CAPACITY, 999),
HeaderError::CapacityMismatch { header: 999, handle: 512 },
HeaderError::CapacityMismatch {
header: 999,
handle: 512,
},
),
(
None,
(byte_ring::OFF_GENERATION, 4),
HeaderError::GenerationMismatch { header: 4, handle: 3 },
HeaderError::GenerationMismatch {
header: 4,
handle: 3,
},
),
(
Some((byte_ring::OFF_CONSUME, 10)),
(byte_ring::OFF_COMMIT, 5),
HeaderError::CommitBelowConsume { commit: 5, consume: 10 },
HeaderError::CommitBelowConsume {
commit: 5,
consume: 10,
},
),
(
None,
(byte_ring::OFF_COMMIT, 600),
HeaderError::ReadableExceedsCapacity { commit: 600, consume: 0, capacity: 512 },
HeaderError::ReadableExceedsCapacity {
commit: 600,
consume: 0,
capacity: 512,
},
),
];
for (pre, (field, value), expected) in cases {
@ -164,7 +180,10 @@ fn basic_round_trip_preserves_bytes() {
.send_record(RecordKind::Data, b"weights-bytes")
.expect("send");
let received = consumer.recv_record().expect("recv");
assert_eq!(received, Some((RecordKind::Data, b"weights-bytes".to_vec())));
assert_eq!(
received,
Some((RecordKind::Data, b"weights-bytes".to_vec()))
);
assert_eq!(consumer.recv_record().expect("recv empty"), None);
}
@ -277,7 +296,10 @@ fn reserve_reports_exact_free_space_and_recovers() {
assert_eq!(
producer.reserve(50).unwrap_err(),
FlowError::InsufficientSpace { requested: 50, free: 28 }
FlowError::InsufficientSpace {
requested: 50,
free: 28
}
);
// Bytes are readable, intact, and consuming frees the space again.
@ -315,7 +337,9 @@ fn records_delimit_completion_distinctly() {
assert_eq!(consumer.recv_record().expect("post-eof"), None);
// Fault is a distinct terminal, not a second EOF.
producer.send_record(RecordKind::Fault, b"reason").expect("send");
producer
.send_record(RecordKind::Fault, b"reason")
.expect("send");
assert_eq!(
consumer.recv_record().expect("recv fault"),
Some((RecordKind::Fault, b"reason".to_vec()))
@ -358,7 +382,10 @@ fn stale_reservations_are_rejected_after_generation_change() {
scribble_u64(&arena, &handle, byte_ring::OFF_GENERATION, 2);
assert_eq!(
producer.commit(reservation).unwrap_err(),
FlowError::StaleReservation { reservation: 1, ring: 2 }
FlowError::StaleReservation {
reservation: 1,
ring: 2
}
);
}
@ -372,7 +399,10 @@ fn wrong_role_operations_are_rejected() {
assert_eq!(
consumer.reserve(8).unwrap_err(),
FlowError::RoleViolation { operation: "reserve", role: Role::Consumer }
FlowError::RoleViolation {
operation: "reserve",
role: Role::Consumer
}
);
assert_eq!(
consumer
@ -382,15 +412,24 @@ fn wrong_role_operations_are_rejected() {
generation: 1,
})
.unwrap_err(),
FlowError::RoleViolation { operation: "commit", role: Role::Consumer }
FlowError::RoleViolation {
operation: "commit",
role: Role::Consumer
}
);
assert_eq!(
producer.consume(8).unwrap_err(),
FlowError::RoleViolation { operation: "consume", role: Role::Producer }
FlowError::RoleViolation {
operation: "consume",
role: Role::Producer
}
);
assert_eq!(
producer.readable().unwrap_err(),
FlowError::RoleViolation { operation: "readable", role: Role::Producer }
FlowError::RoleViolation {
operation: "readable",
role: Role::Producer
}
);
}
@ -399,18 +438,24 @@ fn wrong_role_operations_are_rejected() {
#[test]
fn operations_revalidate_cursors_and_never_panic() {
let (arena, handle) = installed(512, 1);
let mut producer = attach(&arena, handle, Role::Producer).expect("producer");
let mut consumer = attach(&arena, handle, Role::Consumer).expect("consumer");
let producer = attach(&arena, handle, Role::Producer).expect("producer");
let consumer = attach(&arena, handle, Role::Consumer).expect("consumer");
// Corrupt the cursors mid-protocol (consume beyond commit).
scribble_u64(&arena, &handle, byte_ring::OFF_CONSUME, 5);
assert_eq!(
producer.reserve(8).unwrap_err(),
FlowError::Corrupt(HeaderError::CommitBelowConsume { commit: 0, consume: 5 })
FlowError::Corrupt(HeaderError::CommitBelowConsume {
commit: 0,
consume: 5
})
);
assert_eq!(
consumer.readable().unwrap_err(),
FlowError::Corrupt(HeaderError::CommitBelowConsume { commit: 0, consume: 5 })
FlowError::Corrupt(HeaderError::CommitBelowConsume {
commit: 0,
consume: 5
})
);
// An impossible readable span is rejected at attach, not crashed on.

38
crates/data-plane/tests/edge_runtime_guarantees.rs
View file

@ -187,10 +187,10 @@ fn arena_runtime(
(runtime, transport, boot_arena(), MockWorker::default())
}
fn find_observation<'a>(
observations: &'a [Observation],
fn find_observation(
observations: &[Observation],
predicate: impl Fn(&Observation) -> bool,
) -> Option<&'a Observation> {
) -> Option<&Observation> {
observations.iter().find(|obs| predicate(obs))
}
@ -267,13 +267,13 @@ fn outbound_edge_opens_writer_and_allocates_object_ids() {
.expect("poll");
let observations = runtime.take_observations();
assert!(matches!(
assert!(
find_observation(&observations, |obs| matches!(
obs,
Observation::EdgeReady { .. }
)),
Some(_)
));
))
.is_some()
);
assert_eq!(transport.opened, vec![EdgeId(7002)]);
assert_eq!(runtime.outbound_ring_id().map(|ring| ring.0), Some(1));
assert!(runtime.outbound_writer().is_some());
@ -296,13 +296,13 @@ fn early_stream_waits_for_recv_establishment() {
.poll(&mut transport, &mut arena, &mut worker)
.expect("poll");
let observations = runtime.take_observations();
assert!(matches!(
assert!(
find_observation(&observations, |obs| matches!(
obs,
Observation::EdgeReady { .. }
)),
Some(_)
));
))
.is_some()
);
}
// A malformed ingress record must fault: ObjectFailed observation with no
@ -321,7 +321,7 @@ fn malformed_ingress_record_is_fatal_and_reported() {
let result = runtime.poll(&mut transport, &mut arena, &mut worker);
assert!(result.is_err(), "malformed record must be fatal");
let observations = runtime.take_observations();
assert!(matches!(
assert!(
find_observation(&observations, |obs| {
matches!(
obs,
@ -330,9 +330,9 @@ fn malformed_ingress_record_is_fatal_and_reported() {
..
}
)
}),
Some(_)
));
})
.is_some()
);
}
// A worker load failure must surface ObjectFailed with the object id and
@ -351,7 +351,7 @@ fn worker_load_failure_reports_object_and_is_fatal() {
let result = runtime.poll(&mut transport, &mut arena, &mut worker);
assert!(result.is_err(), "load failure must be fatal");
let observations = runtime.take_observations();
assert!(matches!(
assert!(
find_observation(&observations, |obs| {
matches!(
obs,
@ -360,7 +360,7 @@ fn worker_load_failure_reports_object_and_is_fatal() {
..
}
)
}),
Some(_)
));
})
.is_some()
);
}

199
crates/data-plane/tests/file_source_guarantees.rs Normal file
View file

@ -0,0 +1,199 @@
use std::io::Read;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use data_plane::blob_transfer::{
BlobTransferEvent, BlobTransferId, BlobTransferOffer, BlobTransferSender, FileTransferRequest,
};
use data_plane::namespace::NamespaceError;
use data_plane::source::{BlobSourceIn, FileBlobSourceActor};
use futures_lite::future;
use parking_lot::Mutex;
use swactor::config::RuntimeConfig;
use swactor::runtime::{Runtime, RuntimeParts};
use swactor_engine::{Engine, TokioBackend, TokioConfig};
static NEXT_TEMP: AtomicU64 = AtomicU64::new(1);
struct TestFile {
path: PathBuf,
}
impl TestFile {
fn new(bytes: &[u8]) -> Self {
let sequence = NEXT_TEMP.fetch_add(1, Ordering::Relaxed);
let path = std::env::temp_dir().join(format!(
"swactor-file-source-{}-{sequence}",
std::process::id()
));
std::fs::write(&path, bytes).expect("write source fixture");
Self { path }
}
}
impl Drop for TestFile {
fn drop(&mut self) {
let _ = std::fs::remove_file(&self.path);
}
}
fn runtime() -> (Engine, Runtime) {
let parts = RuntimeParts::new(RuntimeConfig {
worker_count: 1,
..RuntimeConfig::default()
});
let runtime = parts.runtime().clone();
let engine = Engine::new(
parts,
TokioBackend::new(TokioConfig::default()).expect("tokio backend"),
)
.expect("engine");
(engine, runtime)
}
struct LoopbackSender {
runtime: Runtime,
}
impl BlobTransferSender for LoopbackSender {
fn start_file(&self, mut request: FileTransferRequest) -> Result<(), String> {
let mut bytes = Vec::new();
request
.file
.read_to_end(&mut bytes)
.map_err(|error| error.to_string())?;
if bytes.len() as u64 != request.length {
return Err("source length changed".to_owned());
}
self.runtime
.send_to(
request.offer.destination,
BlobTransferEvent::Chunk {
transfer_id: request.offer.transfer_id,
bytes,
},
)
.map_err(|error| error.to_string())?;
self.runtime
.send_to(
request.offer.destination,
BlobTransferEvent::Finished {
transfer_id: request.offer.transfer_id,
},
)
.map_err(|error| error.to_string())?;
request.completion.complete(Ok(()));
Ok(())
}
}
#[test]
fn file_source_owns_fixed_length_and_transfers_opened_file() {
let fixture = TestFile::new(b"fixed-source");
let (_engine, runtime) = runtime();
let sender = Arc::new(LoopbackSender {
runtime: runtime.clone(),
});
let source = FileBlobSourceActor::open(runtime.clone(), sender, &fixture.path)
.expect("open file source");
assert_eq!(source.length(), 12);
let destination = runtime
.new_inbox::<BlobTransferEvent>()
.expect("destination inbox");
let source = runtime.spawn(source).expect("spawn file source");
let transfer_id = BlobTransferId(41);
runtime
.send_to(
source,
BlobSourceIn::BeginTransfer {
offer: BlobTransferOffer {
transfer_id,
destination: *destination.addr(),
failure_proxy: None,
transport: Vec::new(),
},
},
)
.expect("begin source transfer");
future::block_on(async {
assert_eq!(
destination.recv().await,
BlobTransferEvent::Chunk {
transfer_id,
bytes: b"fixed-source".to_vec(),
}
);
assert_eq!(
destination.recv().await,
BlobTransferEvent::Finished { transfer_id }
);
});
}
struct HeldSender {
request: Mutex<Option<FileTransferRequest>>,
}
impl BlobTransferSender for HeldSender {
fn start_file(&self, request: FileTransferRequest) -> Result<(), String> {
*self.request.lock() = Some(request);
Ok(())
}
}
#[test]
fn retirement_does_not_cancel_an_accepted_transfer() {
let fixture = TestFile::new(b"retained");
let (_engine, runtime) = runtime();
let sender = Arc::new(HeldSender {
request: Mutex::new(None),
});
let source_actor = FileBlobSourceActor::open(runtime.clone(), sender.clone(), &fixture.path)
.expect("open file source");
let destination = runtime
.new_inbox::<BlobTransferEvent>()
.expect("destination inbox");
let source = runtime.spawn(source_actor).expect("spawn source");
let transfer_id = BlobTransferId(9);
runtime
.send_to(
source,
BlobSourceIn::BeginTransfer {
offer: BlobTransferOffer {
transfer_id,
destination: *destination.addr(),
failure_proxy: None,
transport: Vec::new(),
},
},
)
.unwrap();
runtime.send_to(source, BlobSourceIn::Retire).unwrap();
let mut request = loop {
if let Some(request) = sender.request.lock().take() {
break request;
}
std::thread::yield_now();
};
let mut bytes = Vec::new();
request.file.read_to_end(&mut bytes).unwrap();
assert_eq!(bytes, b"retained");
request.completion.complete(Ok(()));
}
#[test]
fn recovery_rejects_a_changed_file_length() {
let fixture = TestFile::new(b"changed");
let (_engine, runtime) = runtime();
let sender = Arc::new(LoopbackSender {
runtime: runtime.clone(),
});
let recovered = FileBlobSourceActor::recover(runtime, sender, &fixture.path, 99);
assert!(
matches!(recovered, Err(NamespaceError::SourceRecovery(reason)) if reason.contains("length"))
);
}

442
crates/data-plane/tests/namespace_guarantees.rs Normal file
View file

@ -0,0 +1,442 @@
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;
use data_plane::namespace::{
DataDirectoryActor, DirectoryClient, NamespaceClient, NamespaceClientActor, NamespaceClientIn,
NamespaceDiscovery, NamespaceError, OperationId, SourceRecovery,
};
use data_plane::path::DataPath;
use futures_lite::future;
use parking_lot::RwLock;
use proptest::prelude::*;
use swactor::actor::ActorAddress;
use swactor::config::RuntimeConfig;
use swactor::runtime::{Runtime, RuntimeParts};
use swactor_engine::{Engine, TokioBackend, TokioConfig};
static NEXT_TEMP: AtomicU64 = AtomicU64::new(1);
struct TempState {
root: PathBuf,
}
impl TempState {
fn new(label: &str) -> Self {
let sequence = NEXT_TEMP.fetch_add(1, Ordering::Relaxed);
let root = std::env::temp_dir().join(format!(
"swactor-namespace-{label}-{}-{sequence}",
std::process::id()
));
std::fs::create_dir_all(&root).expect("create temporary namespace directory");
Self { root }
}
fn store(&self) -> PathBuf {
self.root.join("namespace.json")
}
}
impl Drop for TempState {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.root);
}
}
struct DirectoryHarness {
engine: Engine,
runtime: Runtime,
client: DirectoryClient,
}
fn path(value: &str) -> DataPath {
DataPath::parse(value).expect("test path")
}
fn source(byte: u8) -> ActorAddress {
ActorAddress([byte; 32])
}
fn recovery(actor: ActorAddress) -> SourceRecovery {
SourceRecovery::Actor { actor }
}
fn spawn_directory(store: &Path) -> DirectoryHarness {
let parts = RuntimeParts::new(RuntimeConfig {
worker_count: 1,
..RuntimeConfig::default()
});
let runtime = parts.runtime().clone();
let actor = DataDirectoryActor::recover(store, |record, _length| match record {
SourceRecovery::Actor { actor } => Ok(*actor),
SourceRecovery::File { path } => Err(NamespaceError::SourceRecovery(format!(
"test cannot recover file source {}",
path.display()
))),
})
.expect("recover directory");
let directory = runtime.spawn(actor).expect("spawn directory actor");
let engine = Engine::new(
parts,
TokioBackend::new(TokioConfig::default()).expect("tokio backend"),
)
.expect("directory engine");
DirectoryHarness {
engine,
runtime: runtime.clone(),
client: DirectoryClient::new(runtime, directory),
}
}
#[test]
fn namespace_mutations_are_linearizable_and_durable() {
let state = TempState::new("linearizable");
let directory = spawn_directory(&state.store());
let logical = path("/models/tiny-linear/weights");
let first = source(1);
let second = source(2);
future::block_on(async {
let registered = directory
.client
.register(
logical.clone(),
first,
24,
recovery(first),
OperationId::from_u128(1),
)
.await
.expect("register first source");
assert_eq!(registered.revision, 1);
let selected_first = directory
.client
.resolve(logical.clone())
.await
.expect("resolve first source");
assert_eq!(selected_first.source, first);
assert_eq!(selected_first.length, 24);
assert_eq!(selected_first.revision, 1);
let replaced = directory
.client
.register(
logical.clone(),
second,
32,
recovery(second),
OperationId::from_u128(2),
)
.await
.expect("replace source");
assert_eq!(replaced.revision, 2);
let selected_second = directory
.client
.resolve(logical.clone())
.await
.expect("resolve replacement");
assert_eq!(selected_second.source, second);
assert_eq!(selected_second.length, 32);
assert_eq!(selected_second.revision, 2);
// A completed resolve is a binding snapshot. Later replacement cannot
// mutate the source selected by the earlier logical read.
assert_eq!(selected_first.source, first);
assert_eq!(selected_first.revision, 1);
let removed = directory
.client
.unregister(logical.clone(), OperationId::from_u128(3))
.await
.expect("unregister source");
assert_eq!(removed.revision, 3);
assert!(matches!(
directory.client.resolve(logical.clone()).await,
Err(NamespaceError::PathNotFound(found)) if found == logical
));
});
// The acknowledged removal is recovered from a fresh actor instance.
let recovered = spawn_directory(&state.store());
assert!(matches!(
future::block_on(recovered.client.resolve(logical.clone())),
Err(NamespaceError::PathNotFound(found)) if found == logical
));
}
#[test]
fn committed_mutation_retry_has_at_most_once_effect() {
let state = TempState::new("idempotent");
let logical = path("/models/a");
let actor = source(7);
let operation = OperationId::from_u128(99);
let first_runtime = spawn_directory(&state.store());
let first_receipt = future::block_on(first_runtime.client.register(
logical.clone(),
actor,
8,
recovery(actor),
operation,
))
.expect("initial registration");
assert_eq!(first_receipt.revision, 1);
drop(first_runtime);
let recovered = spawn_directory(&state.store());
let retried = future::block_on(recovered.client.register(
logical.clone(),
actor,
8,
recovery(actor),
operation,
))
.expect("retry committed registration");
assert_eq!(retried, first_receipt);
assert_eq!(
future::block_on(recovered.client.resolve(logical.clone()))
.expect("resolve recovered binding")
.revision,
1
);
let conflict = future::block_on(recovered.client.register(
logical,
source(8),
9,
recovery(source(8)),
operation,
));
assert!(matches!(conflict, Err(NamespaceError::OperationConflict(id)) if id == operation));
}
struct StaticDiscovery {
directory: Arc<RwLock<Option<ActorAddress>>>,
}
impl NamespaceDiscovery for StaticDiscovery {
fn current_directory(&self) -> Option<ActorAddress> {
*self.directory.read()
}
}
#[test]
fn unresolved_request_waits_for_recovered_authority() {
let state = TempState::new("restart");
let harness = spawn_directory(&state.store());
let logical = path("/models/restartable");
let source = source(44);
future::block_on(harness.client.register(
logical.clone(),
source,
16,
recovery(source),
OperationId::from_u128(500),
))
.expect("seed durable binding");
let old_directory = harness.client.directory();
harness.runtime.stop_actor(old_directory).unwrap();
let discovered = Arc::new(RwLock::new(None));
let proxy = harness
.runtime
.spawn(NamespaceClientActor::new(
harness.engine.handle(),
harness.runtime.create_sender(),
Arc::new(StaticDiscovery {
directory: Arc::clone(&discovered),
}),
Duration::from_millis(5),
))
.expect("spawn namespace client");
let client = NamespaceClient::new(harness.runtime.clone(), proxy);
future::block_on(async {
let mut resolving = Box::pin(client.resolve(logical.clone()));
assert!(
future::poll_once(resolving.as_mut()).await.is_none(),
"resolve must remain pending while authority is absent"
);
let recovered =
DataDirectoryActor::recover(state.store(), |record, _length| match record {
SourceRecovery::Actor { actor } => Ok(*actor),
SourceRecovery::File { path } => Err(NamespaceError::SourceRecovery(format!(
"test cannot recover file source {}",
path.display()
))),
})
.expect("recover directory state");
let recovered = harness
.runtime
.spawn(recovered)
.expect("spawn recovered directory");
*discovered.write() = Some(recovered);
harness
.runtime
.send_to(proxy, NamespaceClientIn::Retry)
.expect("trigger rediscovery retry");
let binding = resolving.await.expect("resolve after authority recovery");
assert_eq!(binding.source, source);
assert_eq!(binding.length, 16);
assert_eq!(binding.revision, 1);
});
}
#[test]
fn namespace_process_restart_helper() {
let Ok(mode) = std::env::var("SWACTOR_NAMESPACE_HELPER_MODE") else {
return;
};
let store = PathBuf::from(
std::env::var_os("SWACTOR_NAMESPACE_HELPER_STORE").expect("helper store path"),
);
let directory = spawn_directory(&store);
let logical = path("/models/process-restart");
match mode.as_str() {
"write" => {
let actor = source(55);
let receipt = future::block_on(directory.client.register(
logical,
actor,
32,
recovery(actor),
OperationId::from_u128(900),
))
.expect("helper durable registration");
assert_eq!(receipt.revision, 1);
}
"read" => {
let binding =
future::block_on(directory.client.resolve(logical)).expect("helper recovery");
assert_eq!(binding.source, source(55));
assert_eq!(binding.length, 32);
assert_eq!(binding.revision, 1);
}
other => panic!("unknown namespace helper mode {other}"),
}
}
#[test]
fn namespace_client_routes_remote_transfer_failures_to_local_binding() {
let state = TempState::new("transfer-failure");
let harness = spawn_directory(&state.store());
let proxy = harness
.runtime
.spawn(NamespaceClientActor::new(
harness.engine.handle(),
harness.runtime.create_sender(),
Arc::new(StaticDiscovery {
directory: Arc::new(RwLock::new(Some(harness.client.directory()))),
}),
Duration::from_millis(5),
))
.unwrap();
let destination = harness
.runtime
.new_inbox::<data_plane::blob_transfer::BlobTransferEvent>()
.unwrap();
let transfer_id = data_plane::blob_transfer::BlobTransferId(71);
harness
.runtime
.send_to(
proxy,
NamespaceClientIn::TransferFailed {
destination: *destination.addr(),
transfer_id,
reason: "injected source failure".to_owned(),
},
)
.unwrap();
assert_eq!(
future::block_on(destination.recv()),
data_plane::blob_transfer::BlobTransferEvent::Failed {
transfer_id,
reason: "injected source failure".to_owned(),
}
);
}
#[derive(Clone, Debug)]
struct ModelBinding {
source: ActorAddress,
length: u64,
revision: u64,
}
proptest! {
#![proptest_config(ProptestConfig {
cases: 16,
max_shrink_iters: 128,
..ProptestConfig::default()
})]
#[test]
fn legal_action_sequences_preserve_namespace_guarantees(actions in prop::collection::vec(any::<u8>(), 1..48)) {
let state = TempState::new("stateful");
let directory = spawn_directory(&state.store());
let paths = [path("/models/a"), path("/models/b"), path("/runs/7/result")];
let mut model = BTreeMap::<DataPath, ModelBinding>::new();
let mut next_revision = 1_u64;
let mut next_operation = 1_u128;
for (step, action) in actions.into_iter().enumerate() {
let logical = paths[usize::from(action) % paths.len()].clone();
match action % 4 {
0 | 1 => {
let actor = source(action.wrapping_add(step as u8).wrapping_add(1));
let length = u64::from(action) + 1;
let operation = OperationId::from_u128(next_operation);
next_operation += 1;
let receipt = future::block_on(directory.client.register(
logical.clone(),
actor,
length,
recovery(actor),
operation,
)).expect("model registration");
prop_assert_eq!(receipt.revision, next_revision);
model.insert(logical, ModelBinding { source: actor, length, revision: next_revision });
next_revision += 1;
}
2 if model.contains_key(&logical) => {
let operation = OperationId::from_u128(next_operation);
next_operation += 1;
let receipt = future::block_on(directory.client.unregister(logical.clone(), operation))
.expect("model unregistration");
prop_assert_eq!(receipt.revision, next_revision);
model.remove(&logical);
next_revision += 1;
}
_ => {
let observed = future::block_on(directory.client.resolve(logical.clone()));
match model.get(&logical) {
Some(expected) => {
let observed = observed.expect("model binding must resolve");
prop_assert_eq!(observed.source, expected.source);
prop_assert_eq!(observed.length, expected.length);
prop_assert_eq!(observed.revision, expected.revision);
}
None => prop_assert!(matches!(
observed,
Err(NamespaceError::PathNotFound(found)) if found == logical
)),
}
}
}
for (path, expected) in &model {
let observed = future::block_on(directory.client.resolve(path.clone()))
.expect("all model bindings remain resolvable");
prop_assert_eq!(observed.source, expected.source);
prop_assert_eq!(observed.length, expected.length);
prop_assert_eq!(observed.revision, expected.revision);
}
}
}
}

522
crates/data-plane/tests/namespace_host_read_guarantees.rs Normal file
View file

@ -0,0 +1,522 @@
#![cfg(target_os = "linux")]
use std::os::unix::fs::FileExt;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicU8, AtomicU64, Ordering};
use std::time::{Duration, Instant};
use data_plane::arena::{ArenaConfig, ArenaManager, NodeId};
use data_plane::blob;
use data_plane::blob_transfer::{
BlobTransferEvent, BlobTransferId, BlobTransferOffer, BlobTransferReceiver, BlobTransferSender,
FileTransferRequest,
};
use data_plane::bootstrap::{self, BootstrapSpec};
use data_plane::control::DataNamespaceService;
use data_plane::data_plane::DataPlaneBootstrap;
use data_plane::host::{HostDataPlaneConfig, HostDataPlaneSessionActor};
use data_plane::namespace::{
DirectoryClient, NamespaceClient, NamespaceClientActor, NamespaceDiscovery, NamespaceError,
OperationId,
};
use data_plane::path::{DataPath, JobContext};
use data_plane::protocol::JobCapability;
use data_plane::source::{BlobSourcePublisher, FileBlobSourceActor};
use futures_lite::future;
use parking_lot::RwLock;
use swactor::Error;
use swactor::actor::ActorAddress;
use swactor::config::RuntimeConfig;
use swactor::runtime::{RemoteSink, Runtime, RuntimeParts};
use swactor_engine::{Engine, TokioBackend, TokioConfig};
const CAPABILITY: JobCapability = JobCapability::new([0x33; 32]);
static NEXT_TEMP: AtomicU64 = AtomicU64::new(1);
struct DirectRuntimeSink {
destination: Runtime,
}
impl RemoteSink for DirectRuntimeSink {
fn send(
&self,
address: ActorAddress,
message: Box<dyn std::any::Any + Send>,
) -> Result<(), Error> {
self.destination.deliver_raw(address, message)
}
}
struct StaticDiscovery(Arc<RwLock<Option<ActorAddress>>>);
impl NamespaceDiscovery for StaticDiscovery {
fn current_directory(&self) -> Option<ActorAddress> {
*self.0.read()
}
}
struct LoopbackSender {
runtime: Runtime,
behavior: Arc<AtomicU8>,
}
impl BlobTransferSender for LoopbackSender {
fn start_file(&self, request: FileTransferRequest) -> Result<(), String> {
let mut bytes = vec![0_u8; request.length as usize];
request
.file
.read_exact_at(&mut bytes, request.offset)
.map_err(|error| error.to_string())?;
match self.behavior.load(Ordering::Acquire) {
1 => {
bytes.pop();
}
2 => bytes.push(0xff),
3 => return Err("injected source start failure".to_owned()),
_ => {}
}
self.runtime
.send_to(
request.offer.destination,
BlobTransferEvent::Chunk {
transfer_id: request.offer.transfer_id,
bytes,
},
)
.map_err(|error| error.to_string())?;
self.runtime
.send_to(
request.offer.destination,
BlobTransferEvent::Finished {
transfer_id: request.offer.transfer_id,
},
)
.map_err(|error| error.to_string())?;
request.completion.complete(Ok(()));
Ok(())
}
}
struct DirectReceiver;
impl BlobTransferReceiver for DirectReceiver {
fn open(
&self,
destination: ActorAddress,
transfer_id: BlobTransferId,
) -> Result<BlobTransferOffer, String> {
Ok(BlobTransferOffer {
transfer_id,
destination,
failure_proxy: None,
transport: Vec::new(),
})
}
fn cancel(&self, _offer: &BlobTransferOffer) {}
}
struct NoopSourceRegistrar;
impl BlobSourcePublisher for NoopSourceRegistrar {
fn publish_source(&self, _source: ActorAddress) -> Result<(), String> {
Ok(())
}
}
struct TempState {
root: PathBuf,
}
impl TempState {
fn new() -> Self {
let sequence = NEXT_TEMP.fetch_add(1, Ordering::Relaxed);
let root = std::env::temp_dir().join(format!(
"swactor-host-namespace-{}-{sequence}",
std::process::id()
));
std::fs::create_dir_all(&root).unwrap();
Self { root }
}
}
impl Drop for TempState {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.root);
}
}
fn runtime_parts() -> (RuntimeParts, Runtime) {
let parts = RuntimeParts::new(RuntimeConfig {
worker_count: 1,
..RuntimeConfig::default()
});
let runtime = parts.runtime().clone();
(parts, runtime)
}
#[test]
fn host_read_resolves_file_source_and_seals_final_arena_lease() {
let state = TempState::new();
let weights = b"namespace-backed-weights";
let source_path = state.root.join("weights.bin");
std::fs::write(&source_path, weights).unwrap();
let store_path = state.root.join("namespace.json");
let mut arena = ArenaManager::boot(ArenaConfig {
node_id: NodeId(1),
reservation_ceiling: 4096,
base_alignment: 64,
})
.unwrap();
let handoff = bootstrap::write_bootstrap(
&mut arena,
BootstrapSpec {
arena_generation: 1,
alignment: 64,
},
)
.unwrap();
let mut consumer_arena = ArenaManager::boot(ArenaConfig {
node_id: NodeId(2),
reservation_ceiling: 4096,
base_alignment: 64,
})
.unwrap();
let consumer_handoff = bootstrap::write_bootstrap(
&mut consumer_arena,
BootstrapSpec {
arena_generation: 2,
alignment: 64,
},
)
.unwrap();
let (host_parts, host_runtime) = runtime_parts();
let (child_parts, child_runtime) = runtime_parts();
host_runtime.set_remote_sink(Arc::new(DirectRuntimeSink {
destination: child_runtime.clone(),
}));
child_runtime.set_remote_sink(Arc::new(DirectRuntimeSink {
destination: host_runtime.clone(),
}));
let host_engine = Engine::new(
host_parts,
TokioBackend::new(TokioConfig::default()).unwrap(),
)
.unwrap();
let child_engine = Engine::new(
child_parts,
TokioBackend::new(TokioConfig::default()).unwrap(),
)
.unwrap();
let transfer_behavior = Arc::new(AtomicU8::new(0));
let sender: Arc<dyn BlobTransferSender> = Arc::new(LoopbackSender {
runtime: host_runtime.clone(),
behavior: Arc::clone(&transfer_behavior),
});
let source_publisher: Arc<dyn BlobSourcePublisher> = Arc::new(NoopSourceRegistrar);
let service = DataNamespaceService::recover(
host_runtime.clone(),
&store_path,
Arc::clone(&sender),
Arc::clone(&source_publisher),
)
.unwrap();
let directory = service.directory();
let direct = DirectoryClient::new(host_runtime.clone(), directory);
let logical = DataPath::parse("/models/weights").unwrap();
future::block_on(
service
.control()
.register(logical.clone(), blob::file(&source_path)),
)
.unwrap();
let discovered = Arc::new(RwLock::new(Some(directory)));
let proxy = host_runtime
.spawn(NamespaceClientActor::new(
host_engine.handle(),
host_runtime.create_sender(),
Arc::new(StaticDiscovery(Arc::clone(&discovered))),
Duration::from_millis(5),
))
.unwrap();
let namespace = NamespaceClient::new(host_runtime.clone(), proxy);
let host_session = host_runtime
.spawn(
HostDataPlaneSessionActor::new(HostDataPlaneConfig {
runtime: host_runtime.clone(),
arena,
arena_generation: 1,
session_generation: 1,
capability: CAPABILITY,
job_context: JobContext {
run_id: "run-1".to_owned(),
read_prefixes: vec![
DataPath::parse("/models").unwrap(),
DataPath::parse("/runs/run-1").unwrap(),
],
write_prefixes: vec![DataPath::parse("/runs/run-1").unwrap()],
},
namespace: Some(namespace.clone()),
transfer_receiver: Some(Arc::new(DirectReceiver)),
source_sender: Some(Arc::clone(&sender)),
source_publisher: Some(Arc::clone(&source_publisher)),
route_registrar: None,
})
.unwrap(),
)
.unwrap();
let bootstrap = future::block_on(DataPlaneBootstrap::attach(
handoff.arena_fd,
child_runtime.clone(),
host_session,
CAPABILITY,
))
.unwrap();
let consumer_host_session = host_runtime
.spawn(
HostDataPlaneSessionActor::new(HostDataPlaneConfig {
runtime: host_runtime.clone(),
arena: consumer_arena,
arena_generation: 2,
session_generation: 2,
capability: CAPABILITY,
job_context: JobContext {
run_id: "run-1".to_owned(),
read_prefixes: vec![
DataPath::parse("/models").unwrap(),
DataPath::parse("/runs/run-1").unwrap(),
],
write_prefixes: vec![],
},
namespace: Some(namespace),
transfer_receiver: Some(Arc::new(DirectReceiver)),
source_sender: Some(Arc::clone(&sender)),
source_publisher: Some(source_publisher),
route_registrar: None,
})
.unwrap(),
)
.unwrap();
let consumer = future::block_on(DataPlaneBootstrap::attach(
consumer_handoff.arena_fd,
child_runtime,
consumer_host_session,
CAPABILITY,
))
.unwrap();
let blob = future::block_on(bootstrap.data_plane.read_blob(&logical)).unwrap();
for (mode, name) in [(1_u8, "short"), (2, "oversized"), (3, "start-failure")] {
let fault_path = DataPath::parse(format!("/models/{name}")).unwrap();
let source =
FileBlobSourceActor::open(host_runtime.clone(), Arc::clone(&sender), &source_path)
.unwrap();
let length = source.length();
let recovery = source.recovery();
let source = host_runtime.spawn(source).unwrap();
future::block_on(direct.register(
fault_path.clone(),
source,
length,
recovery,
OperationId::from_u128(10 + u128::from(mode)),
))
.unwrap();
transfer_behavior.store(mode, Ordering::Release);
let result = future::block_on(bootstrap.data_plane.read_blob(&fault_path));
match mode {
1 | 2 => assert!(matches!(
result,
Err(data_plane::protocol::DataPlaneError::Blob(
data_plane::protocol::BlobFailure::Length { .. }
))
)),
3 => assert!(matches!(
result,
Err(data_plane::protocol::DataPlaneError::SourceFailure(_))
)),
_ => unreachable!(),
}
}
transfer_behavior.store(0, Ordering::Release);
let recovered_after_failures =
future::block_on(bootstrap.data_plane.read_blob(&logical)).unwrap();
assert_eq!(recovered_after_failures.map().unwrap().as_ref(), weights);
drop(recovered_after_failures);
assert_eq!(blob.length(), weights.len() as u64);
let result_path = DataPath::parse("/runs/run-1/result").unwrap();
let mut writer = future::block_on(
bootstrap
.data_plane
.write_blob(&result_path, b"first!".len() as u64),
)
.unwrap();
writer.map().unwrap().as_mut().copy_from_slice(b"first!");
future::block_on(writer.seal()).unwrap();
let first = future::block_on(bootstrap.data_plane.read_blob(&result_path)).unwrap();
assert_eq!(first.map().unwrap().as_ref(), b"first!");
let mut replacement = future::block_on(
bootstrap
.data_plane
.write_blob(&result_path, b"second".len() as u64),
)
.unwrap();
replacement
.map()
.unwrap()
.as_mut()
.copy_from_slice(b"second");
future::block_on(replacement.seal()).unwrap();
let second = future::block_on(bootstrap.data_plane.read_blob(&result_path)).unwrap();
assert_eq!(second.map().unwrap().as_ref(), b"second");
assert_eq!(first.map().unwrap().as_ref(), b"first!");
let mut aborted = future::block_on(bootstrap.data_plane.write_blob(&result_path, 5)).unwrap();
aborted.map().unwrap().as_mut().copy_from_slice(b"abort");
future::block_on(aborted.abort()).unwrap();
let after_abort = future::block_on(bootstrap.data_plane.read_blob(&result_path)).unwrap();
assert_eq!(after_abort.map().unwrap().as_ref(), b"second");
assert_eq!(blob.map().unwrap().as_ref(), weights);
let cancelled_path = DataPath::parse("/runs/run-1/cancelled").unwrap();
let mut cancelled =
future::block_on(bootstrap.data_plane.write_blob(&cancelled_path, 6)).unwrap();
cancelled.map().unwrap().as_mut().copy_from_slice(b"cancel");
*discovered.write() = None;
let mut sealing = Box::pin(cancelled.seal());
assert!(
future::block_on(future::poll_once(sealing.as_mut())).is_none(),
"seal must wait while namespace authority is absent"
);
drop(sealing);
drop(cancelled);
*discovered.write() = Some(directory);
host_runtime
.send_to(proxy, data_plane::namespace::NamespaceClientIn::Retry)
.unwrap();
std::thread::sleep(Duration::from_millis(100));
assert!(matches!(
future::block_on(direct.resolve(cancelled_path)),
Err(NamespaceError::PathNotFound(_))
));
*discovered.write() = None;
let mut cancelled_read = Box::pin(bootstrap.data_plane.read_blob(&logical));
assert!(
future::block_on(future::poll_once(cancelled_read.as_mut())).is_none(),
"read must wait while namespace authority is absent"
);
drop(cancelled_read);
*discovered.write() = Some(directory);
host_runtime
.send_to(proxy, data_plane::namespace::NamespaceClientIn::Retry)
.unwrap();
std::thread::sleep(Duration::from_millis(50));
let after_cancel = future::block_on(bootstrap.data_plane.read_blob(&logical)).unwrap();
assert_eq!(after_cancel.map().unwrap().as_ref(), weights);
drop(after_cancel);
drop(after_abort);
drop(second);
drop(first);
drop(blob);
let reclaim_path = DataPath::parse("/runs/run-1/reclaim").unwrap();
let replacement_path = DataPath::parse("/runs/run-1/reclaimed").unwrap();
let reclaim_bytes = vec![0x5a; 3_000];
let mut reclaiming = future::block_on(
bootstrap
.data_plane
.write_blob(&reclaim_path, reclaim_bytes.len() as u64),
)
.unwrap();
reclaiming
.map()
.unwrap()
.as_mut()
.copy_from_slice(&reclaim_bytes);
future::block_on(reclaiming.seal()).unwrap();
let reclaim_source = future::block_on(direct.resolve(reclaim_path.clone()))
.unwrap()
.source;
assert!(matches!(
future::block_on(
bootstrap
.data_plane
.write_blob(&replacement_path, reclaim_bytes.len() as u64)
),
Err(data_plane::protocol::DataPlaneError::ArenaExhausted)
));
future::block_on(direct.unregister(reclaim_path, OperationId::from_u128(101))).unwrap();
let deadline = Instant::now() + Duration::from_secs(2);
let mut replacement = loop {
match future::block_on(
bootstrap
.data_plane
.write_blob(&replacement_path, reclaim_bytes.len() as u64),
) {
Ok(writer) => break writer,
Err(data_plane::protocol::DataPlaneError::ArenaExhausted)
if Instant::now() < deadline =>
{
std::thread::sleep(Duration::from_millis(5));
}
Err(error) => panic!("unregister did not reclaim the published lease: {error}"),
}
};
assert!(
host_runtime
.stats()
.actors
.iter()
.all(|(actor, _)| *actor != reclaim_source),
"unregister did not reclaim the published source actor"
);
replacement
.map()
.unwrap()
.as_mut()
.copy_from_slice(&reclaim_bytes);
future::block_on(replacement.abort()).unwrap();
let published_source = future::block_on(direct.resolve(result_path.clone()))
.unwrap()
.source;
bootstrap.data_plane.close().unwrap();
let after_producer_close =
future::block_on(consumer.data_plane.read_blob(&result_path)).unwrap();
assert_eq!(
after_producer_close.map().unwrap().as_ref(),
b"second",
"a committed publication must outlive its producer session"
);
drop(after_producer_close);
future::block_on(direct.unregister(result_path.clone(), OperationId::from_u128(100))).unwrap();
assert!(matches!(
future::block_on(direct.resolve(result_path)),
Err(NamespaceError::PathNotFound(_))
));
let deadline = Instant::now() + Duration::from_secs(2);
while host_runtime
.stats()
.actors
.iter()
.any(|(actor, _)| *actor == published_source)
{
assert!(
Instant::now() < deadline,
"unregister did not reclaim the published source actor"
);
std::thread::sleep(Duration::from_millis(5));
}
consumer.data_plane.close().unwrap();
drop(bootstrap);
drop(child_engine);
drop(host_engine);
}

3
crates/distribution/Cargo.toml
View file

@ -22,3 +22,6 @@ libc = "0.2"
serde_json = "1"
proptest = "1"
swactor-engine = { path = "../engine", default-features = false }
[lints]
workspace = true

1
crates/distribution/src/directory_actor.rs
View file

@ -46,7 +46,6 @@ const BATCH: usize = 16;
/// claims stay cached so a false-dead peer can become routable again as soon as
/// SWIM reports it Alive. Claim deletion requires a future explicit tombstone or
/// owner-side lifecycle signal; raw `Tick` cadence is not a safe GC clock.
/// Everything the `DirectoryActor` receives, as one enum. Only [`Gossip`](DirectoryIn::Gossip)
/// crosses the wire (it carries the registered `DirectoryGossip` tag); the rest
/// are local control — see [`crate::messages::actor_codec_registry`].

2
crates/distribution/src/peer_auth.rs
View file

@ -133,7 +133,7 @@ impl PeerAllowList {
std::fs::create_dir_all(parent)?;
}
let json = serde_json::to_string_pretty(&file).map_err(|e| io::Error::other(e))?;
let json = serde_json::to_string_pretty(&file).map_err(io::Error::other)?;
std::fs::write(path, json)
}
}

4
crates/distribution/src/registry.rs
View file

@ -307,6 +307,10 @@ impl ClusterRegistry {
self.entries.len()
}
pub fn is_empty(&self) -> bool {
self.entries.is_empty()
}
/// Number of tombstones.
pub fn tombstone_count(&self) -> usize {
self.entries.values().filter(|e| e.tombstone).count()

4
crates/distribution/src/swim/dissemination.rs
View file

@ -6,6 +6,8 @@
//!
//! Priority ordering: Dead > Suspect > Alive (most urgent first).
use std::cmp::Reverse;
use crate::messages::MembershipUpdate;
use crate::types::{MemberState, NodeId};
@ -69,7 +71,7 @@ impl DisseminationQueue {
pub fn take(&mut self, max_count: usize) -> Vec<MembershipUpdate> {
// Sort by priority: Dead (2) > Suspect (1) > Alive (0), descending
self.entries
.sort_by(|a, b| b.update.state.priority().cmp(&a.update.state.priority()));
.sort_by_key(|entry| Reverse(entry.update.state.priority()));
let count = max_count.min(self.entries.len());
let mut result = Vec::with_capacity(count);

4
crates/distribution/src/swim/member_list.rs
View file

@ -110,6 +110,10 @@ impl MemberList {
self.members.len()
}
pub fn is_empty(&self) -> bool {
self.members.is_empty()
}
/// Apply an update. Returns `true` if the state changed (for dissemination).
///
/// SWIM merge semantics:

10
crates/distribution/src/swim/node.rs
View file

@ -190,11 +190,11 @@ impl SwimNode {
/// Alive at the same incarnation when the local entry is Dead. Removing
/// the stale Dead entry lets the fresh Alive record take effect.
pub fn clear_dead_member(&mut self, node_id: NodeId) {
if let Some(entry) = self.members.get(&node_id) {
if entry.state == MemberState::Dead {
self.members.remove(&node_id);
self.dissemination.purge_node(&node_id);
}
if let Some(entry) = self.members.get(&node_id)
&& entry.state == MemberState::Dead
{
self.members.remove(&node_id);
self.dissemination.purge_node(&node_id);
}
}

24
crates/distribution/src/swim/probe.rs
View file

@ -402,10 +402,10 @@ impl SwimProbe {
}
fn maybe_start_probe(&mut self, members: &MemberList, actions: &mut Vec<SwimAction>) {
if let Some(at) = self.next_probe_at {
if self.now < at {
return;
}
if let Some(at) = self.next_probe_at
&& self.now < at
{
return;
}
if !matches!(self.phase, ProbePhase::Idle) {
return;
@ -639,10 +639,10 @@ impl SwimProbe {
if self.config.dead_reprobe_interval == Duration::ZERO {
return;
}
if let Some(at) = self.next_reprobe_at {
if self.now < at {
return;
}
if let Some(at) = self.next_reprobe_at
&& self.now < at
{
return;
}
self.next_reprobe_at = Some(self.now + self.config.dead_reprobe_interval);
@ -750,10 +750,10 @@ impl SwimProbe {
/// At safety_sweep_interval, probe one random alive member.
fn maybe_safety_sweep(&mut self, members: &MemberList, actions: &mut Vec<SwimAction>) {
if let Some(at) = self.next_sweep_at {
if self.now < at {
return;
}
if let Some(at) = self.next_sweep_at
&& self.now < at
{
return;
}
let interval = match &self.config.probe_mode {
ProbeMode::Reactive {

11
crates/distribution/tests/routing.rs
View file

@ -804,15 +804,14 @@ mod directory_route_path {
}
// Every node considers the others alive.
for i in 0..n {
for j in 0..n {
for (i, node) in nodes.iter().enumerate() {
for (j, node_id) in ids.iter().copied().enumerate() {
if i != j {
nodes[i]
.rt
node.rt
.send_to(
nodes[i].directory,
node.directory,
DirectoryIn::Membership(MembershipChanged {
node_id: ids[j],
node_id,
state: MemberState::Alive,
incarnation: 1,
}),

75
crates/distribution/tests/swim_core.rs
View file

@ -1871,13 +1871,14 @@ mod node_behavior {
sequence,
piggyback,
} => {
if let Some(t) = self.index_of(to) {
if t != origin && !excluded.contains(&t) {
let from = self.ids[origin];
let resp = self.nodes[t].handle_ping(from, sequence, &piggyback);
self.record(t, &resp);
self.route(t, resp, excluded);
}
if let Some(t) = self.index_of(to)
&& t != origin
&& !excluded.contains(&t)
{
let from = self.ids[origin];
let resp = self.nodes[t].handle_ping(from, sequence, &piggyback);
self.record(t, &resp);
self.route(t, resp, excluded);
}
}
NodeAction::SendAck {
@ -1885,13 +1886,14 @@ mod node_behavior {
sequence,
piggyback,
} => {
if let Some(t) = self.index_of(to) {
if t != origin && !excluded.contains(&t) {
let from = self.ids[origin];
let resp = self.nodes[t].handle_ack(from, sequence, &piggyback);
self.record(t, &resp);
self.route(t, resp, excluded);
}
if let Some(t) = self.index_of(to)
&& t != origin
&& !excluded.contains(&t)
{
let from = self.ids[origin];
let resp = self.nodes[t].handle_ack(from, sequence, &piggyback);
self.record(t, &resp);
self.route(t, resp, excluded);
}
}
NodeAction::SendPingReq {
@ -1900,14 +1902,15 @@ mod node_behavior {
sequence,
piggyback,
} => {
if let Some(t) = self.index_of(relay) {
if t != origin && !excluded.contains(&t) {
let from = self.ids[origin];
let resp = self.nodes[t]
.handle_ping_req(from, target, sequence, &piggyback);
self.record(t, &resp);
self.route(t, resp, excluded);
}
if let Some(t) = self.index_of(relay)
&& t != origin
&& !excluded.contains(&t)
{
let from = self.ids[origin];
let resp =
self.nodes[t].handle_ping_req(from, target, sequence, &piggyback);
self.record(t, &resp);
self.route(t, resp, excluded);
}
}
NodeAction::ForwardAck {
@ -1916,22 +1919,24 @@ mod node_behavior {
sequence,
piggyback,
} => {
if let Some(t) = self.index_of(to) {
if t != origin && !excluded.contains(&t) {
let resp =
self.nodes[t].handle_indirect_ack(target, sequence, &piggyback);
self.record(t, &resp);
self.route(t, resp, excluded);
}
if let Some(t) = self.index_of(to)
&& t != origin
&& !excluded.contains(&t)
{
let resp =
self.nodes[t].handle_indirect_ack(target, sequence, &piggyback);
self.record(t, &resp);
self.route(t, resp, excluded);
}
}
NodeAction::SendJoinResponse { to, members } => {
if let Some(t) = self.index_of(to) {
if t != origin && !excluded.contains(&t) {
let resp = self.nodes[t].handle_join_response(members);
self.record(t, &resp);
self.route(t, resp, excluded);
}
if let Some(t) = self.index_of(to)
&& t != origin
&& !excluded.contains(&t)
{
let resp = self.nodes[t].handle_join_response(members);
self.record(t, &resp);
self.route(t, resp, excluded);
}
}
NodeAction::MembershipChanged { .. } => {}

3
crates/engine/Cargo.toml
View file

@ -15,3 +15,6 @@ tokio = { workspace = true, optional = true }
[dev-dependencies]
proptest = "1"
[lints]
workspace = true

8
crates/engine/src/engine.rs
View file

@ -18,8 +18,7 @@ use swactor::runtime::{ExternalSender, Runtime, RuntimeParts};
pub struct Engine {
/// Retained so the engine owns the runtime handle it drives for its full
/// lifetime. Core workers are moved into substrate tasks at construction.
#[allow(dead_code)]
runtime: Runtime,
_runtime: Runtime,
backend: Arc<dyn ExecutionBackend>,
}
@ -41,7 +40,10 @@ impl Engine {
// no Tokio feature gate — so core progression does not silently
// disappear when an alternate backend is used (ENGINE_SPEC.md).
crate::core_driver::install(workers, &backend);
Ok(Engine { runtime, backend })
Ok(Engine {
_runtime: runtime,
backend,
})
}
/// Return a clonable handle for scheduling engine work.

9
crates/engine/tests/common/mod.rs
View file

@ -3,8 +3,6 @@
//! Imports only public `swactor` APIs and exposes no private engine state. See
//! `ENGINE_SPEC.md`.
#![allow(dead_code)]
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::time::{Duration, Instant};
@ -23,9 +21,10 @@ pub fn default_runtime_parts() -> (RuntimeParts, Runtime) {
runtime_parts(RuntimeConfig::default())
}
pub fn runtime_parts_with_workers(worker_count: usize) -> (RuntimeParts, Runtime) {
let mut config = RuntimeConfig::default();
config.worker_count = worker_count;
runtime_parts(config)
runtime_parts(RuntimeConfig {
worker_count,
..RuntimeConfig::default()
})
}
pub fn default_parts() -> RuntimeParts {

2
crates/engine/tests/engine_contract.rs
View file

@ -10,7 +10,7 @@
//! These tests exercise the native Tokio backend specifically; the
//! non-Tokio portability proof lives in `engine_unit.rs`.
mod common;
pub mod common;
use common::*;
use std::sync::Arc;

7
crates/engine/tests/engine_unit.rs
View file

@ -5,7 +5,7 @@
//! These tests exercise internal logic directly and use the [`SteppingBackend`]
//! to prove substrate independence without Tokio (ENGINE_SPEC.md).
mod common;
pub mod common;
use common::*;
use std::sync::Arc;
@ -1318,8 +1318,7 @@ fn engine_instant_is_ordered() {
/// A tasks-only probe backend that shares a sentinel `Arc<()>` so the test can
/// observe exactly when the engine's strong backend reference is released.
struct SentinelBackend {
#[allow(dead_code)]
sentinel: Arc<()>,
_sentinel: Arc<()>,
}
impl ExecutionBackend for SentinelBackend {
@ -1347,7 +1346,7 @@ fn dropping_engine_releases_backend_even_with_live_handles() {
let engine = Engine::new(
parts,
SentinelBackend {
sentinel: sentinel.clone(),
_sentinel: sentinel.clone(),
},
)
.expect("tasks capability present");

3
crates/iroh-driver/Cargo.toml
View file

@ -27,3 +27,6 @@ parking_lot = "0.12"
[dev-dependencies]
iroh-relay = { version = "0.98", features = ["server", "test-utils"] }
[lints]
workspace = true

232
crates/iroh-driver/src/blob_transfer.rs Normal file
View file

@ -0,0 +1,232 @@
//! Iroh implementation of one-shot fixed-length blob transfer ports.
use std::collections::HashMap;
use std::os::unix::fs::FileExt;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;
use data_plane::blob_transfer::{
BlobTransferEvent, BlobTransferId, BlobTransferOffer, BlobTransferReceiver, BlobTransferSender,
FileTransferRequest,
};
use data_plane::edge_wire::WireEvent;
use iroh::EndpointAddr;
use parking_lot::Mutex;
use serde::{Deserialize, Serialize};
use swactor::actor::ActorAddress;
use swactor::runtime::Runtime;
use swactor_engine::{BlockingWorkSender, EngineHandle};
use crate::iroh_driver::EdgeConnector;
const FIRST_BLOB_EDGE_ID: u64 = 1 << 63;
const FILE_CHUNK_BYTES: usize = 64 * 1024;
const CONNECT_TIMEOUT: Duration = Duration::from_secs(30);
#[derive(Clone, Debug, Serialize, Deserialize)]
struct IrohBlobOffer {
endpoint: EndpointAddr,
edge_id: u64,
}
#[derive(Clone)]
pub struct IrohBlobTransferSender {
connector: EdgeConnector,
blocking: BlockingWorkSender,
}
impl IrohBlobTransferSender {
pub fn new(connector: EdgeConnector, engine: &EngineHandle) -> Self {
Self {
connector,
blocking: engine.blocking_work_sender(),
}
}
}
impl BlobTransferSender for IrohBlobTransferSender {
fn start_file(&self, request: FileTransferRequest) -> Result<(), String> {
let connector = self.connector.clone();
let work = Box::new(move || {
let result = send_file(connector, &request);
request.completion.complete(result);
});
self.blocking
.submit(work)
.map_err(|_| "blob transfer engine has stopped".to_owned())
}
}
fn send_file(connector: EdgeConnector, request: &FileTransferRequest) -> Result<(), String> {
let wire: IrohBlobOffer = serde_json::from_slice(&request.offer.transport)
.map_err(|error| format!("decode Iroh blob offer: {error}"))?;
let sender = connector.connect(wire.endpoint, wire.edge_id, CONNECT_TIMEOUT)?;
let mut transferred = 0_u64;
let mut chunk = vec![0_u8; FILE_CHUNK_BYTES];
while transferred < request.length {
let remaining = request.length - transferred;
let count = usize::try_from(remaining.min(FILE_CHUNK_BYTES as u64))
.expect("bounded blob chunk size");
let file_offset = request
.offset
.checked_add(transferred)
.ok_or_else(|| "blob source file offset overflow".to_owned())?;
let read = request
.file
.read_at(&mut chunk[..count], file_offset)
.map_err(|error| format!("read blob source at {file_offset}: {error}"))?;
if read == 0 {
return Err(format!(
"blob source ended after {transferred} bytes, expected {} bytes",
request.length
));
}
sender.send(chunk[..read].to_vec())?;
transferred = transferred
.checked_add(read as u64)
.ok_or_else(|| "blob source transfer offset overflow".to_owned())?;
}
drop(sender);
Ok(())
}
#[derive(Clone)]
pub struct IrohBlobTransferReceiver {
endpoint: EndpointAddr,
events: Arc<Mutex<Vec<WireEvent>>>,
destinations: Arc<Mutex<HashMap<u64, (BlobTransferId, ActorAddress)>>>,
next_edge_id: Arc<AtomicU64>,
}
impl IrohBlobTransferReceiver {
pub fn new(endpoint: EndpointAddr, events: Arc<Mutex<Vec<WireEvent>>>) -> Self {
Self {
endpoint,
events,
destinations: Arc::new(Mutex::new(HashMap::new())),
next_edge_id: Arc::new(AtomicU64::new(FIRST_BLOB_EDGE_ID)),
}
}
pub fn install_pump(
self: &Arc<Self>,
engine: &EngineHandle,
runtime: Runtime,
period: Duration,
) {
let receiver = Arc::clone(self);
let engine = engine.clone();
engine.clone().spawn(async move {
let mut interval = engine.interval(period);
loop {
(&mut interval).await;
receiver.drain(&runtime);
}
});
}
pub fn drain(&self, runtime: &Runtime) {
let mut queue = self.events.lock();
let mut remaining = Vec::with_capacity(queue.len());
for event in queue.drain(..) {
match event {
WireEvent::StreamArrived { edge_id, .. }
if self.destinations.lock().contains_key(&edge_id.0) => {}
WireEvent::BytesRead {
edge_id,
stream_id,
bytes,
} => {
let destination = self.destinations.lock().get(&edge_id.0).copied();
if let Some((transfer_id, destination)) = destination {
let _ = runtime
.send_to(destination, BlobTransferEvent::Chunk { transfer_id, bytes });
} else {
remaining.push(WireEvent::BytesRead {
edge_id,
stream_id,
bytes,
});
}
}
WireEvent::StreamEnded { edge_id, stream_id } => {
if let Some((transfer_id, destination)) =
self.destinations.lock().remove(&edge_id.0)
{
let _ = runtime
.send_to(destination, BlobTransferEvent::Finished { transfer_id });
} else {
remaining.push(WireEvent::StreamEnded { edge_id, stream_id });
}
}
WireEvent::StreamFault {
edge_id: Some(edge_id),
stream_id,
reason,
} => {
if let Some((transfer_id, destination)) =
self.destinations.lock().remove(&edge_id.0)
{
let _ = runtime.send_to(
destination,
BlobTransferEvent::Failed {
transfer_id,
reason: format!("{reason:?}"),
},
);
} else {
remaining.push(WireEvent::StreamFault {
edge_id: Some(edge_id),
stream_id,
reason,
});
}
}
event => remaining.push(event),
}
}
queue.extend(remaining);
}
}
impl BlobTransferReceiver for IrohBlobTransferReceiver {
fn open(
&self,
destination: ActorAddress,
transfer_id: BlobTransferId,
) -> Result<BlobTransferOffer, String> {
let edge_id = self.next_edge_id.fetch_add(1, Ordering::Relaxed);
if edge_id < FIRST_BLOB_EDGE_ID {
return Err("blob transfer edge identifiers exhausted".to_owned());
}
if self
.destinations
.lock()
.insert(edge_id, (transfer_id, destination))
.is_some()
{
return Err("blob transfer edge identifier collision".to_owned());
}
let transport = serde_json::to_vec(&IrohBlobOffer {
endpoint: self.endpoint.clone(),
edge_id,
})
.map_err(|error| {
self.destinations.lock().remove(&edge_id);
format!("encode Iroh blob offer: {error}")
})?;
Ok(BlobTransferOffer {
transfer_id,
destination,
failure_proxy: None,
transport,
})
}
fn cancel(&self, offer: &BlobTransferOffer) {
if let Ok(wire) = serde_json::from_slice::<IrohBlobOffer>(&offer.transport) {
self.destinations.lock().remove(&wire.edge_id);
}
}
}

2
crates/iroh-driver/src/edge_transport.rs
View file

@ -108,7 +108,7 @@ pub(crate) fn spawn_edge_send_pump(
.map_err(|_| format!("finish edge stream {edge_id}: timed out"))?
.map_err(|e| format!("finish edge stream {edge_id}: {e}"))?
{
Some(code) => return Err(format!("peer stopped edge stream {edge_id}: {code}")),
Some(code) => Err(format!("peer stopped edge stream {edge_id}: {code}")),
None => Ok(()),
}
}

171
crates/iroh-driver/src/iroh_driver.rs
View file

@ -139,14 +139,13 @@ pub fn discover_lan_ips() -> Vec<IpAddr> {
// UDP socket trick: connect to a broadcast-ish address, read local_addr
let targets: &[&str] = &["10.255.255.255:1", "192.168.255.255:1", "172.31.255.255:1"];
for target in targets {
if let Ok(sock) = std::net::UdpSocket::bind("0.0.0.0:0") {
if sock.connect(target).is_ok() {
if let Ok(local) = sock.local_addr() {
let ip = local.ip();
if !ip.is_loopback() && !ip.is_unspecified() && seen.insert(ip) {
ips.push(ip);
}
}
if let Ok(sock) = std::net::UdpSocket::bind("0.0.0.0:0")
&& sock.connect(target).is_ok()
&& let Ok(local) = sock.local_addr()
{
let ip = local.ip();
if !ip.is_loopback() && !ip.is_unspecified() && seen.insert(ip) {
ips.push(ip);
}
}
}
@ -173,10 +172,10 @@ pub fn discover_lan_ips() -> Vec<IpAddr> {
let ip = IpAddr::V6(std::net::Ipv6Addr::from(bytes));
if !ip.is_loopback() && !ip.is_unspecified() {
// Skip link-local (fe80::)
if let IpAddr::V6(v6) = ip {
if (v6.segments()[0] & 0xffc0) == 0xfe80 {
continue;
}
if let IpAddr::V6(v6) = ip
&& (v6.segments()[0] & 0xffc0) == 0xfe80
{
continue;
}
if seen.insert(ip) {
ips.push(ip);
@ -278,6 +277,35 @@ impl EdgeConnector {
}
}
/// Cloneable capability for signing local actor-location claims.
#[derive(Clone)]
pub struct ActorRegistrar {
keypair: Keypair,
}
impl ActorRegistrar {
pub fn register_actor(
&self,
actor_addr: ActorAddress,
generation: u64,
) -> distribution::types::DirectoryEntry {
self.keypair.sign_directory_entry(actor_addr, generation)
}
}
type AcceptedConnections = Arc<Mutex<Vec<(NodeId, Connection)>>>;
type OtherAcceptedConnections = Arc<Mutex<Vec<(NodeId, Vec<u8>, Connection)>>>;
type IncomingActorFrames = Arc<Mutex<Vec<(ActorAddress, String, Vec<u8>, NodeId)>>>;
pub struct ActorBridgeConfig {
pub runtime: Runtime,
pub codec: Arc<CodecRegistry>,
pub routes: HashMap<String, ActorAddress>,
pub swim: ActorAddress,
pub relay_mirror: RelayMirror,
pub route_view: RouteView,
pub outbox: Outbox,
}
/// iroh P2P network transport bridge.
///
/// Bridges the actorized distribution protocol (running on a swactor runtime)
@ -309,9 +337,9 @@ pub struct IrohDriver {
/// a dead peer would otherwise freeze the whole node for the dial budget).
dialing: Arc<Mutex<HashSet<NodeId>>>,
/// Connections accepted by the background accept loop (SWIM ALPN).
accepted_conns: Arc<Mutex<Vec<(NodeId, Connection)>>>,
accepted_conns: AcceptedConnections,
/// Connections accepted on non-SWIM ALPNs before driver-owned adapters claim them.
other_accepted_conns: Arc<Mutex<Vec<(NodeId, Vec<u8>, Connection)>>>,
other_accepted_conns: OtherAcceptedConnections,
/// Completed telemetry QUIC reads from driver-owned TELEMETRY_ALPN adapters.
telemetry_reads: Arc<Mutex<Vec<TelemetryQuicRead>>>,
/// Logical edge events emitted by driver-owned EDGE_ALPN byte pumps.
@ -326,7 +354,7 @@ pub struct IrohDriver {
/// reads from the state machine. Each entry is `(dest, type_tag, payload,
/// from)` — `dest` is the destination actor address carried on the wire
/// (`DIRECTORY.md` §5).
incoming: Arc<Mutex<Vec<(ActorAddress, String, Vec<u8>, NodeId)>>>,
incoming: IncomingActorFrames,
/// Connections whose fire-and-forget send failed; evicted (and re-dialed)
/// on the next `recv()`. Populated by the spawned send tasks.
evict: Arc<Mutex<Vec<FailedConnection>>>,
@ -467,11 +495,8 @@ impl IrohDriver {
let iroh_secret = endpoint.secret_key().to_bytes();
let keypair = Keypair::from_bytes(&iroh_secret);
// Spawn background accept loop so incoming connections are never missed
let accepted_conns: Arc<Mutex<Vec<(NodeId, Connection)>>> =
Arc::new(Mutex::new(Vec::new()));
let other_accepted_conns: Arc<Mutex<Vec<(NodeId, Vec<u8>, Connection)>>> =
Arc::new(Mutex::new(Vec::new()));
let accepted_conns: AcceptedConnections = Arc::new(Mutex::new(Vec::new()));
let other_accepted_conns: OtherAcceptedConnections = Arc::new(Mutex::new(Vec::new()));
let telemetry_reads: Arc<Mutex<Vec<TelemetryQuicRead>>> = Arc::new(Mutex::new(Vec::new()));
let edge_events: Arc<Mutex<Vec<WireEvent>>> = Arc::new(Mutex::new(Vec::new()));
{
@ -480,32 +505,26 @@ impl IrohDriver {
let swim_buf = Arc::clone(&accepted_conns);
let other_buf = Arc::clone(&other_accepted_conns);
engine.spawn(async move {
loop {
match ep.accept().await {
Some(incoming) => match incoming.await {
Ok(conn) => {
let remote_id = conn.remote_id();
let node_id = NodeId(*remote_id.as_bytes());
// Peer auth check
let allowed = match &peer_auth {
None => true,
Some(auth) => auth.lock().is_allowed(&node_id),
};
if !allowed {
conn.close(0u32.into(), b"unauthorized");
continue;
}
// Route by negotiated ALPN.
let negotiated_alpn = conn.alpn().to_vec();
if negotiated_alpn == ALPN {
swim_buf.lock().push((node_id, conn));
} else {
other_buf.lock().push((node_id, negotiated_alpn, conn));
}
}
Err(_) => {}
},
None => break, // endpoint closed
while let Some(incoming) = ep.accept().await {
if let Ok(conn) = incoming.await {
let remote_id = conn.remote_id();
let node_id = NodeId(*remote_id.as_bytes());
// Peer auth check
let allowed = match &peer_auth {
None => true,
Some(auth) => auth.lock().is_allowed(&node_id),
};
if !allowed {
conn.close(0u32.into(), b"unauthorized");
continue;
}
// Route by negotiated ALPN.
let negotiated_alpn = conn.alpn().to_vec();
if negotiated_alpn == ALPN {
swim_buf.lock().push((node_id, conn));
} else {
other_buf.lock().push((node_id, negotiated_alpn, conn));
}
}
}
});
@ -669,10 +688,10 @@ impl IrohDriver {
/// directly. IPv6 unspecified is mapped to localhost.
pub fn endpoint_addr(&self) -> EndpointAddr {
let mut addr = self.endpoint.addr();
if addr.relay_urls().next().is_none() {
if let Some(relay) = self.relay_url.clone() {
addr = addr.with_relay_url(relay);
}
if addr.relay_urls().next().is_none()
&& let Some(relay) = self.relay_url.clone()
{
addr = addr.with_relay_url(relay);
}
for sa in self.direct_addresses() {
addr = addr.with_ip_addr(sa);
@ -729,6 +748,12 @@ impl IrohDriver {
self.keypair.sign_directory_entry(actor_addr, generation)
}
pub fn actor_registrar(&self) -> ActorRegistrar {
ActorRegistrar {
keypair: self.keypair.clone(),
}
}
/// Capture the driver-owned slice of the node's observable state: identity,
/// listen address, and the directory route-view extent. The core node no
/// longer polls this (its telemetry flows over the telemetry), so this is
@ -776,7 +801,7 @@ impl IrohDriver {
.map(|(actor, host)| (*actor, *host))
.collect();
// Stable order so observers don't reshuffle each tick.
entries.sort_by(|a, b| a.0.0.cmp(&b.0.0));
entries.sort_by_key(|a| a.0.0);
entries
})
})
@ -1123,22 +1148,22 @@ impl IrohDriver {
/// and the swactor runtime — the seam by which the protocol actors send and
/// receive over iroh. Frame progression is driven by the engine-hosted
/// adapter pump ([`Self::install_actor_bridge_pump`]).
pub fn enable_actor_bridge(
&mut self,
rt: Runtime,
codec: Arc<CodecRegistry>,
routes: HashMap<String, ActorAddress>,
swim_addr: ActorAddress,
relay_mirror: RelayMirror,
route_view: RouteView,
outbox: Outbox,
) {
let self_peer_addr = peer_addr(self.node_id());
self.actor_bridge = Some(Arc::new(ActorBridge {
rt,
pub fn enable_actor_bridge(&mut self, config: ActorBridgeConfig) {
let ActorBridgeConfig {
runtime,
codec,
routes,
swim_addr,
swim,
relay_mirror,
route_view,
outbox,
} = config;
let self_peer_addr = peer_addr(self.node_id());
self.actor_bridge = Some(Arc::new(ActorBridge {
rt: runtime,
codec,
routes,
swim_addr: swim,
self_peer_addr,
relay_mirror,
route_view,
@ -1243,11 +1268,11 @@ struct AdapterPump {
engine: EngineHandle,
endpoint: Endpoint,
conns: Arc<Mutex<ConnCache>>,
incoming: Arc<Mutex<Vec<(ActorAddress, String, Vec<u8>, NodeId)>>>,
incoming: IncomingActorFrames,
evict: Arc<Mutex<Vec<FailedConnection>>>,
pending_joins: Arc<Mutex<Vec<JoinResult>>>,
accepted_conns: Arc<Mutex<Vec<(NodeId, Connection)>>>,
other_accepted_conns: Arc<Mutex<Vec<(NodeId, Vec<u8>, Connection)>>>,
accepted_conns: AcceptedConnections,
other_accepted_conns: OtherAcceptedConnections,
telemetry_reads: Arc<Mutex<Vec<TelemetryQuicRead>>>,
edge_events: Arc<Mutex<Vec<WireEvent>>>,
retain_telemetry_conns: Arc<std::sync::atomic::AtomicBool>,
@ -1293,10 +1318,8 @@ impl AdapterPump {
false
}
};
if should_redial {
if let Ok(key) = PublicKey::from_bytes(&failed.node_id.0) {
let _ = self.get_or_connect(failed.node_id, key);
}
if should_redial && let Ok(key) = PublicKey::from_bytes(&failed.node_id.0) {
let _ = self.get_or_connect(failed.node_id, key);
}
}
}
@ -1455,10 +1478,8 @@ impl AdapterPump {
.and_then(|s| s.parse::<iroh::RelayUrl>().ok())
{
Some(r)
} else if let Some(r) = self.home_relay_url() {
Some(r)
} else {
None
self.home_relay_url()
}
};

10
crates/iroh-driver/src/lib.rs
View file

@ -9,24 +9,26 @@
// work goes through `EngineHandle`.
#![deny(clippy::disallowed_methods)]
pub mod blob_transfer;
pub mod edge_transport;
pub mod endpoint_advertisement;
pub mod iroh_driver;
pub mod telemetry_transport;
pub use blob_transfer::{IrohBlobTransferReceiver, IrohBlobTransferSender};
pub use endpoint_advertisement::{
EndpointAddrMask, MVP_IROH_ENDPOINT_ADDR_MASK_ENV, advertised_endpoint,
};
pub use iroh_driver::{
ConnType, EdgeConnector, IrohDriver, IrohDriverConfig, JoinPhase, JoinStatus,
TelemetryPublishHandle, conn_type_of, discover_lan_ips,
ActorBridgeConfig, ActorRegistrar, ConnType, EdgeConnector, IrohDriver, IrohDriverConfig,
JoinPhase, JoinStatus, TelemetryPublishHandle, conn_type_of, discover_lan_ips,
};
pub use edge_transport::{EDGE_ALPN, EdgeSendHandle};
pub use telemetry_transport::{
PullCollectorHandle, TELEMETRY_ALPN, TelemetryQuicHeader, TelemetryQuicRead,
TelemetryQuicWriteStats, read_events_from_stream, read_next_event,
PullCollectorConfig, PullCollectorHandle, TELEMETRY_ALPN, TelemetryQuicHeader,
TelemetryQuicRead, TelemetryQuicWriteStats, read_events_from_stream, read_next_event,
read_next_uni_from_connection, read_pull_request, read_stream_header, read_stream_into_fanout,
spawn_connection_reader, spawn_pull_collector, spawn_pull_collector_to_actor,
spawn_pull_server, spawn_subscription_writer, write_available_subscription, write_event,

68
crates/iroh-driver/src/telemetry_transport.rs
View file

@ -144,6 +144,15 @@ impl Drop for PullCollectorCompletion {
}
}
pub struct PullCollectorConfig {
pub endpoint: Endpoint,
pub peer: EndpointAddr,
pub flow_id: [u8; 16],
pub token: Vec<u8>,
pub request: telemetry::SubscriptionRequest,
pub fanout: std::sync::Arc<telemetry::DeliveryFanout>,
}
/// Supervisor side: retain a pull subscription to a node on `TELEMETRY_ALPN`.
///
/// A transport interruption reconnects with bounded backoff. Returning after
@ -151,24 +160,10 @@ impl Drop for PullCollectorCompletion {
/// easy to trigger while several freshly-bootstrapped nodes answer at once.
pub fn spawn_pull_collector(
engine: &EngineHandle,
endpoint: Endpoint,
peer: EndpointAddr,
flow_id: [u8; 16],
token: Vec<u8>,
request: telemetry::SubscriptionRequest,
fanout: std::sync::Arc<telemetry::DeliveryFanout>,
config: PullCollectorConfig,
on_header: std::sync::mpsc::Sender<TelemetryQuicHeader>,
) -> PullCollectorHandle {
spawn_pull_collector_with_sink(
engine,
endpoint,
peer,
flow_id,
token,
request,
fanout,
PullHeaderSink::Channel(on_header),
)
spawn_pull_collector_with_sink(engine, config, PullHeaderSink::Channel(on_header))
}
/// Supervisor side variant that delivers each connection header directly to
@ -176,25 +171,11 @@ pub fn spawn_pull_collector(
/// stream identity changes domain state.
pub fn spawn_pull_collector_to_actor(
engine: &EngineHandle,
endpoint: Endpoint,
peer: EndpointAddr,
flow_id: [u8; 16],
token: Vec<u8>,
request: telemetry::SubscriptionRequest,
fanout: std::sync::Arc<telemetry::DeliveryFanout>,
config: PullCollectorConfig,
sender: ExternalSender,
actor: ActorAddress,
) -> PullCollectorHandle {
spawn_pull_collector_with_sink(
engine,
endpoint,
peer,
flow_id,
token,
request,
fanout,
PullHeaderSink::Actor { sender, actor },
)
spawn_pull_collector_with_sink(engine, config, PullHeaderSink::Actor { sender, actor })
}
enum PullHeaderSink {
@ -214,17 +195,19 @@ impl PullHeaderSink {
}
}
#[allow(clippy::too_many_arguments)]
fn spawn_pull_collector_with_sink(
engine: &EngineHandle,
endpoint: Endpoint,
peer: EndpointAddr,
flow_id: [u8; 16],
token: Vec<u8>,
request: telemetry::SubscriptionRequest,
fanout: std::sync::Arc<telemetry::DeliveryFanout>,
config: PullCollectorConfig,
on_header: PullHeaderSink,
) -> PullCollectorHandle {
let PullCollectorConfig {
endpoint,
peer,
flow_id,
token,
request,
fanout,
} = config;
let (cancellation, mut cancellation_rx) = tokio::sync::watch::channel(false);
let (completion, completion_rx) = tokio::sync::watch::channel(false);
let engine_handle = engine.clone();
@ -486,11 +469,8 @@ pub async fn read_stream_header(recv: &mut RecvStream) -> Result<TelemetryQuicHe
pub async fn read_events_from_stream(mut recv: RecvStream) -> Result<TelemetryQuicRead, BoxError> {
let header = read_header(&mut recv).await?;
let mut events = Vec::new();
loop {
match read_next_event(&mut recv, &header.stream).await? {
Some(event) => events.push(event),
None => break,
}
while let Some(event) = read_next_event(&mut recv, &header.stream).await? {
events.push(event);
}
Ok(TelemetryQuicRead { header, events })
}

153
crates/iroh-driver/tests/blob_transfer.rs Normal file
View file

@ -0,0 +1,153 @@
use std::collections::HashMap;
use std::fs::File;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, RwLock};
use std::time::{Duration, Instant};
use data_plane::blob_transfer::{
BlobTransferCompletion, BlobTransferEvent, BlobTransferId, BlobTransferReceiver,
BlobTransferSender, FileTransferRequest,
};
use iroh::RelayMode;
use iroh_driver::{
EDGE_ALPN, IrohBlobTransferReceiver, IrohBlobTransferSender, IrohDriver, IrohDriverConfig,
};
use swactor::actor::ActorAddress;
use swactor::config::RuntimeConfig;
use swactor::runtime::{Runtime, RuntimeParts};
use swactor_engine::{Engine, TokioBackend, TokioConfig};
use swactor_transport::{CodecRegistry, CodecRemoteSink, TransportRouter};
static NEXT_TEMP: AtomicU64 = AtomicU64::new(1);
struct Completion(std::sync::mpsc::Sender<Result<(), String>>);
impl BlobTransferCompletion for Completion {
fn complete(self: Box<Self>, result: Result<(), String>) {
let _ = self.0.send(result);
}
}
struct Node {
engine: Engine,
runtime: Runtime,
driver: IrohDriver,
}
fn node() -> Node {
let parts = RuntimeParts::new(RuntimeConfig::default());
let runtime = parts.runtime().clone();
let engine = Engine::new(
parts,
TokioBackend::new(TokioConfig::default()).expect("tokio backend"),
)
.expect("engine");
let mut driver = IrohDriver::with_engine(
engine.handle(),
IrohDriverConfig {
secret_key: None,
relay_mode: RelayMode::Disabled,
node: distribution::node::DistributedNodeConfig::default(),
peer_auth: None,
additional_alpns: vec![EDGE_ALPN.to_vec()],
},
)
.expect("Iroh driver");
let codecs = Arc::new(CodecRegistry::new());
let router = Arc::new(TransportRouter::new());
runtime.set_remote_sink(Arc::new(CodecRemoteSink::new(Arc::clone(&codecs), router)));
driver.enable_actor_bridge(iroh_driver::ActorBridgeConfig {
runtime: runtime.clone(),
codec: codecs,
routes: HashMap::new(),
swim: ActorAddress::default(),
relay_mirror: Arc::new(RwLock::new(HashMap::new())),
route_view: Arc::new(RwLock::new(HashMap::new())),
outbox: Arc::new(Mutex::new(Vec::new())),
});
driver.install_actor_bridge_pump(Duration::from_millis(5));
Node {
engine,
runtime,
driver,
}
}
#[test]
fn real_iroh_transfer_delivers_exact_file_bytes() {
let source = node();
let destination = node();
let receiver = Arc::new(IrohBlobTransferReceiver::new(
destination.driver.endpoint_addr(),
destination.driver.edge_events_handle(),
));
receiver.install_pump(
&destination.engine.handle(),
destination.runtime.clone(),
Duration::from_millis(5),
);
let sender =
IrohBlobTransferSender::new(source.driver.edge_connector(), &source.engine.handle());
let inbox = destination
.runtime
.new_inbox::<BlobTransferEvent>()
.expect("destination inbox");
let transfer_id = BlobTransferId(77);
let offer = receiver
.open(*inbox.addr(), transfer_id)
.expect("open transfer receiver");
let sequence = NEXT_TEMP.fetch_add(1, Ordering::Relaxed);
let path = std::env::temp_dir().join(format!(
"swactor-iroh-blob-{}-{sequence}",
std::process::id()
));
let expected = b"iroh-file-blob";
std::fs::write(&path, expected).expect("write fixture");
let (completion_tx, completion_rx) = std::sync::mpsc::channel();
sender
.start_file(FileTransferRequest {
offer,
file: File::open(&path).expect("open fixture"),
offset: 0,
length: expected.len() as u64,
completion: Box::new(Completion(completion_tx)),
})
.expect("start Iroh transfer");
completion_rx
.recv_timeout(Duration::from_secs(10))
.expect("source completion")
.expect("source transfer");
let deadline = Instant::now() + Duration::from_secs(10);
let mut received = Vec::new();
loop {
if let Some(event) = inbox.try_recv() {
match event {
BlobTransferEvent::Chunk {
transfer_id: found,
bytes,
} => {
assert_eq!(found, transfer_id);
received.extend_from_slice(&bytes);
}
BlobTransferEvent::Finished { transfer_id: found } => {
assert_eq!(found, transfer_id);
break;
}
BlobTransferEvent::Failed { reason, .. } => {
panic!("Iroh blob transfer failed: {reason}")
}
event => panic!("unexpected local blob transfer event: {event:?}"),
}
} else {
assert!(
Instant::now() < deadline,
"destination transfer deadline elapsed"
);
std::thread::sleep(Duration::from_millis(5));
}
}
assert_eq!(received, expected);
let _ = std::fs::remove_file(path);
}

Some files were not shown because too many files have changed in this diff Show more