zacheryasc/swactor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor: rename pp binaries and rewrite orchestrator

Browse source 
pp_smoke_run->pp_orchestrator (seed/vastai/hold/teardown, N stages),
pp_gpu_node->pp_worker. fleet_plugin subscribes collector SSE + folds locally;
iroh_driver drops verbose eprintln connection/relay logging.


Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
This commit is contained in:
Zachery Aaron Shores-Chmielewski 2026-05-30 11:51:22 +04:00
parent 89e9b5b587
commit c6a6b35562
34 changed files with 1517 additions and 615 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.dockerignore
View file

@ -4,7 +4,7 @@
!target/release/swactor-diag-postproc !target/release/swactor-diag-postproc
!examples/single-gpu-inference/target/release/gpu-node !examples/single-gpu-inference/target/release/gpu-node
!examples/single-gpu-inference/tinygrad_worker.py !examples/single-gpu-inference/tinygrad_worker.py
!examples/pipeline-parallel-inference/target/release/pp-gpu-node !examples/pipeline-parallel-inference/target/release/pp-worker
!examples/pipeline-parallel-inference/target/release/pp-smoke-run !examples/pipeline-parallel-inference/target/release/pp-orchestrator
!examples/pipeline-parallel-inference/pp_tinygrad_worker.py !examples/pipeline-parallel-inference/pp_tinygrad_worker.py
!examples/pipeline-parallel-inference/pp_entrypoint.sh !examples/pipeline-parallel-inference/pp_entrypoint.sh

4
crates/dashboard/src/html.rs
View file

@ -125,6 +125,7 @@ pub const ACTOR_DETAIL_HTML: &str = r##"<!DOCTYPE html>
<a href="/actors" class="nav-link">Actors</a> <a href="/actors" class="nav-link">Actors</a>
<a href="/plugin/distribution" class="nav-link">Distribution</a> <a href="/plugin/distribution" class="nav-link">Distribution</a>
<a href="/plugin/datastore" class="nav-link">Datastore</a> <a href="/plugin/datastore" class="nav-link">Datastore</a>
<a href="/plugin/vastai" class="nav-link">Fleet</a>
</nav> </nav>
</div> </div>
</div> </div>
@ -569,6 +570,7 @@ pub const ACTORS_HTML: &str = r##"<!DOCTYPE html>
<a href="/actors" class="nav-link active">Actors</a> <a href="/actors" class="nav-link active">Actors</a>
<a href="/plugin/distribution" class="nav-link">Distribution</a> <a href="/plugin/distribution" class="nav-link">Distribution</a>
<a href="/plugin/datastore" class="nav-link">Datastore</a> <a href="/plugin/datastore" class="nav-link">Datastore</a>
<a href="/plugin/vastai" class="nav-link">Fleet</a>
</nav> </nav>
</div> </div>
<div class="header-right"> <div class="header-right">
@ -1377,6 +1379,7 @@ pub const DASHBOARD_HTML: &str = r##"<!DOCTYPE html>
<a href="/actors" class="nav-link">Actors</a> <a href="/actors" class="nav-link">Actors</a>
<a href="/plugin/distribution" class="nav-link">Distribution</a> <a href="/plugin/distribution" class="nav-link">Distribution</a>
<a href="/plugin/datastore" class="nav-link">Datastore</a> <a href="/plugin/datastore" class="nav-link">Datastore</a>
<a href="/plugin/vastai" class="nav-link">Fleet</a>
</nav> </nav>
</div> </div>
<div class="header-right"> <div class="header-right">
@ -1934,6 +1937,7 @@ pub const TOPOLOGY_HTML: &str = r##"<!DOCTYPE html>
<a href="/topology" class="nav-link active">Topology</a> <a href="/topology" class="nav-link active">Topology</a>
<a href="/plugin/distribution" class="nav-link">Distribution</a> <a href="/plugin/distribution" class="nav-link">Distribution</a>
<a href="/plugin/datastore" class="nav-link">Datastore</a> <a href="/plugin/datastore" class="nav-link">Datastore</a>
<a href="/plugin/vastai" class="nav-link">Fleet</a>
</nav> </nav>
</div> </div>
</div> </div>

89
crates/distribution/src/iroh_driver.rs
View file

@ -229,13 +229,9 @@ impl IrohDriver {
match rt.block_on(start_embedded_relay(bind_addr, config.relay_public_ip)) { match rt.block_on(start_embedded_relay(bind_addr, config.relay_public_ip)) {
Ok((server, url)) => { Ok((server, url)) => {
let url_str = url.to_string(); let url_str = url.to_string();
eprintln!("Relay: embedded relay started at {url}");
(Some(server), Some(url_str), RelayMode::Custom(url.into())) (Some(server), Some(url_str), RelayMode::Custom(url.into()))
} }
Err(e) => { Err(_) => (None, None, config.relay_mode),
eprintln!("Relay: failed to start embedded relay: {e}, falling back");
(None, None, config.relay_mode)
}
} }
} }
None => (None, None, config.relay_mode), None => (None, None, config.relay_mode),
@ -301,33 +297,18 @@ impl IrohDriver {
Some(auth) => auth.lock().unwrap().is_allowed(&node_id), Some(auth) => auth.lock().unwrap().is_allowed(&node_id),
}; };
if !allowed { if !allowed {
eprintln!(
"iroh driver: rejected connection from unauthorized peer {}",
swactor::transport::hex_encode(&node_id.0[..4])
);
conn.close(0u32.into(), b"unauthorized"); conn.close(0u32.into(), b"unauthorized");
continue; continue;
} }
// Route by negotiated ALPN // Route by negotiated ALPN
let negotiated_alpn = conn.alpn(); let negotiated_alpn = conn.alpn();
if negotiated_alpn == ALPN { if negotiated_alpn == ALPN {
eprintln!(
"iroh driver: accepted SWIM connection from {}",
swactor::transport::hex_encode(&node_id.0[..4])
);
swim_buf.lock().unwrap().push((node_id, conn)); swim_buf.lock().unwrap().push((node_id, conn));
} else { } else {
eprintln!(
"iroh driver: accepted non-SWIM connection from {} (ALPN: {})",
swactor::transport::hex_encode(&node_id.0[..4]),
String::from_utf8_lossy(negotiated_alpn),
);
other_buf.lock().unwrap().push((node_id, conn)); other_buf.lock().unwrap().push((node_id, conn));
} }
} }
Err(e) => { Err(_) => {}
eprintln!("iroh driver: incoming connection error: {e}");
}
}, },
None => break, // endpoint closed None => break, // endpoint closed
} }
@ -659,7 +640,6 @@ impl IrohDriver {
}); });
} }
eprintln!("iroh driver: join attempt {attempt}/{max_attempts} connecting to {}...", seed_addr.id);
diagnostics.emit_event(DiagEvent::DialStarted { diagnostics.emit_event(DiagEvent::DialStarted {
peer: seed_node_id, peer: seed_node_id,
attempt, attempt,
@ -726,7 +706,6 @@ impl IrohDriver {
}); });
} }
eprintln!("iroh driver: join attempt {attempt}/{max_attempts} connected to {}, sending...", seed_addr.id);
let send_result: Result<(), String> = async { let send_result: Result<(), String> = async {
let mut send = conn.open_uni().await.map_err(|e| e.to_string())?; let mut send = conn.open_uni().await.map_err(|e| e.to_string())?;
let tag_len = (tag.len() as u32).to_be_bytes(); let tag_len = (tag.len() as u32).to_be_bytes();
@ -740,7 +719,6 @@ impl IrohDriver {
match send_result { match send_result {
Ok(()) => { Ok(()) => {
eprintln!("iroh driver: join attempt {attempt}/{max_attempts} sent to {}", seed_addr.id);
diagnostics.emit_event(DiagEvent::MessageSent { diagnostics.emit_event(DiagEvent::MessageSent {
peer: seed_node_id, peer: seed_node_id,
kind: tag.to_string(), kind: tag.to_string(),
@ -764,10 +742,6 @@ impl IrohDriver {
return; return;
} }
Err(e) => { Err(e) => {
eprintln!(
"iroh driver: join attempt {attempt}/{max_attempts} send error to {}: {e}",
seed_addr.id
);
diagnostics.emit_event(DiagEvent::Error { diagnostics.emit_event(DiagEvent::Error {
component: "iroh_driver".into(), component: "iroh_driver".into(),
message: format!("join send error: {e}"), message: format!("join send error: {e}"),
@ -778,10 +752,6 @@ impl IrohDriver {
} }
} }
Ok(Err(e)) => { Ok(Err(e)) => {
eprintln!(
"iroh driver: join attempt {attempt}/{max_attempts} connect error to {}: {e}",
seed_addr.id
);
let outcome = classify_dial_error_str(&e.to_string()); let outcome = classify_dial_error_str(&e.to_string());
diagnostics.emit_event(DiagEvent::DialOutcome { diagnostics.emit_event(DiagEvent::DialOutcome {
peer: seed_node_id, peer: seed_node_id,
@ -792,10 +762,6 @@ impl IrohDriver {
continue; continue;
} }
Err(_) => { Err(_) => {
eprintln!(
"iroh driver: join attempt {attempt}/{max_attempts} connect timeout to {}",
seed_addr.id
);
diagnostics.emit_event(DiagEvent::DialOutcome { diagnostics.emit_event(DiagEvent::DialOutcome {
peer: seed_node_id, peer: seed_node_id,
attempt, attempt,
@ -817,7 +783,6 @@ impl IrohDriver {
updated_at: Instant::now(), updated_at: Instant::now(),
}); });
} }
eprintln!("iroh driver: join failed after {max_attempts} attempts to {}", seed_addr.id);
}); });
} }
@ -832,9 +797,6 @@ impl IrohDriver {
// Collect completed background join connections // Collect completed background join connections
{ {
let mut pending = self.pending_joins.lock().unwrap(); let mut pending = self.pending_joins.lock().unwrap();
if !pending.is_empty() {
eprintln!("iroh driver: collecting {} pending join connection(s)", pending.len());
}
for result in pending.drain(..) { for result in pending.drain(..) {
self.connection_cache_tracker self.connection_cache_tracker
.note_dial_success(result.node_id, wall_ms_now()); .note_dial_success(result.node_id, wall_ms_now());
@ -864,7 +826,6 @@ impl IrohDriver {
let mut failure_targets: Vec<NodeId> = Vec::new(); let mut failure_targets: Vec<NodeId> = Vec::new();
for action in actions { for action in actions {
if let Err(e) = self.send_action(action) { if let Err(e) = self.send_action(action) {
eprintln!("iroh driver: send error: {e}");
let target = action_target(action); let target = action_target(action);
self.diagnostics.emit_event(DiagEvent::Error { self.diagnostics.emit_event(DiagEvent::Error {
component: "iroh_driver".into(), component: "iroh_driver".into(),
@ -882,9 +843,7 @@ impl IrohDriver {
let probe_actions = self.node.report_send_failure(target); let probe_actions = self.node.report_send_failure(target);
// Best-effort send of probe actions — no recursion on failure // Best-effort send of probe actions — no recursion on failure
for action in &probe_actions { for action in &probe_actions {
if let Err(e) = self.send_action(action) { let _ = self.send_action(action);
eprintln!("iroh driver: probe send error: {e}");
}
} }
} }
} }
@ -1230,10 +1189,6 @@ impl IrohDriver {
self.read_streams(&conn, remote_id, &mut messages).await; self.read_streams(&conn, remote_id, &mut messages).await;
} }
if !messages.is_empty() {
eprintln!("iroh driver: received {} message(s)", messages.len());
}
(messages, new_connections) (messages, new_connections)
} }
@ -1250,8 +1205,7 @@ impl IrohDriver {
Ok((tag, payload)) => { Ok((tag, payload)) => {
messages.push((tag, payload, remote_id)); messages.push((tag, payload, remote_id));
} }
Err(e) => { Err(_) => {
eprintln!("iroh driver: read error: {e}");
break; break;
} }
} }
@ -1276,18 +1230,12 @@ impl IrohDriver {
match tag { match tag {
"swactor_dist::Ping" => match serde_json::from_slice::<Ping>(payload) { "swactor_dist::Ping" => match serde_json::from_slice::<Ping>(payload) {
Ok(msg) => self.node.handle_ping(msg.from, msg.sequence, &msg.piggyback), Ok(msg) => self.node.handle_ping(msg.from, msg.sequence, &msg.piggyback),
Err(e) => { Err(_) => Vec::new(),
eprintln!("iroh driver: decode Ping: {e}");
Vec::new()
}
}, },
"swactor_dist::Ack" => match serde_json::from_slice::<Ack>(payload) { "swactor_dist::Ack" => match serde_json::from_slice::<Ack>(payload) {
Ok(msg) => self.node.handle_ack(msg.from, msg.sequence, &msg.piggyback), Ok(msg) => self.node.handle_ack(msg.from, msg.sequence, &msg.piggyback),
Err(e) => { Err(_) => Vec::new(),
eprintln!("iroh driver: decode Ack: {e}");
Vec::new()
}
}, },
"swactor_dist::PingReq" => match serde_json::from_slice::<PingReq>(payload) { "swactor_dist::PingReq" => match serde_json::from_slice::<PingReq>(payload) {
@ -1295,44 +1243,29 @@ impl IrohDriver {
self.node self.node
.handle_ping_req(msg.from, msg.target, msg.sequence, &msg.piggyback) .handle_ping_req(msg.from, msg.target, msg.sequence, &msg.piggyback)
} }
Err(e) => { Err(_) => Vec::new(),
eprintln!("iroh driver: decode PingReq: {e}");
Vec::new()
}
}, },
"swactor_dist::JoinRequest" => { "swactor_dist::JoinRequest" => {
match serde_json::from_slice::<JoinRequest>(payload) { match serde_json::from_slice::<JoinRequest>(payload) {
Ok(msg) => self.node.handle_join_request(msg.from), Ok(msg) => self.node.handle_join_request(msg.from),
Err(e) => { Err(_) => Vec::new(),
eprintln!("iroh driver: decode JoinRequest: {e}");
Vec::new()
}
} }
} }
"swactor_dist::JoinResponse" => { "swactor_dist::JoinResponse" => {
match serde_json::from_slice::<JoinResponse>(payload) { match serde_json::from_slice::<JoinResponse>(payload) {
Ok(msg) => self.node.handle_join_response(msg.members), Ok(msg) => self.node.handle_join_response(msg.members),
Err(e) => { Err(_) => Vec::new(),
eprintln!("iroh driver: decode JoinResponse: {e}");
Vec::new()
}
} }
} }
"swactor_dist::IndirectAck" => match serde_json::from_slice::<IndirectAck>(payload) { "swactor_dist::IndirectAck" => match serde_json::from_slice::<IndirectAck>(payload) {
Ok(msg) => self.node.handle_indirect_ack(msg.target, msg.sequence, &msg.piggyback), Ok(msg) => self.node.handle_indirect_ack(msg.target, msg.sequence, &msg.piggyback),
Err(e) => { Err(_) => Vec::new(),
eprintln!("iroh driver: decode IndirectAck: {e}");
Vec::new()
}
}, },
other => { _ => Vec::new(),
eprintln!("iroh driver: unknown message type: {other}");
Vec::new()
}
} }
} }

20
examples/pipeline-parallel-inference/Cargo.lock generated
View file

@ -634,6 +634,7 @@ dependencies = [
"axum", "axum",
"crossbeam-queue", "crossbeam-queue",
"ctrlc", "ctrlc",
"distribution",
"serde", "serde",
"serde_json", "serde_json",
"swactor", "swactor",
@ -2655,6 +2656,7 @@ dependencies = [
"base64", "base64",
"dashboard", "dashboard",
"distribution", "distribution",
"futures-util",
"iroh", "iroh",
"libc", "libc",
"reqwest 0.12.28", "reqwest 0.12.28",
@ -2935,6 +2937,7 @@ dependencies = [
"bytes", "bytes",
"encoding_rs", "encoding_rs",
"futures-core", "futures-core",
"futures-util",
"h2", "h2",
"http", "http",
"http-body", "http-body",
@ -2956,12 +2959,14 @@ dependencies = [
"sync_wrapper", "sync_wrapper",
"tokio", "tokio",
"tokio-native-tls", "tokio-native-tls",
"tokio-util",
"tower", "tower",
"tower-http", "tower-http",
"tower-service", "tower-service",
"url", "url",
"wasm-bindgen", "wasm-bindgen",
"wasm-bindgen-futures", "wasm-bindgen-futures",
"wasm-streams 0.4.2",
"web-sys", "web-sys",
] ]
@ -2998,7 +3003,7 @@ dependencies = [
"url", "url",
"wasm-bindgen", "wasm-bindgen",
"wasm-bindgen-futures", "wasm-bindgen-futures",
"wasm-streams", "wasm-streams 0.5.0",
"web-sys", "web-sys",
] ]
@ -4227,6 +4232,19 @@ dependencies = [
"wasmparser", "wasmparser",
] ]
[[package]]
name = "wasm-streams"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65"
dependencies = [
"futures-util",
"js-sys",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
]
[[package]] [[package]]
name = "wasm-streams" name = "wasm-streams"
version = "0.5.0" version = "0.5.0"

19
examples/pipeline-parallel-inference/Cargo.toml
View file

@ -11,24 +11,27 @@ swactor = { path = "../..", features = ["transport", "serde", "std"] }
swactor-process = { path = "../../crates/process" } swactor-process = { path = "../../crates/process" }
serde = { version = "1", features = ["derive"] } serde = { version = "1", features = ["derive"] }
serde_json = "1" serde_json = "1"
reqwest = { version = "0.12", features = ["json"] } reqwest = { version = "0.12", features = ["json", "stream"] }
futures-util = "0.3"
tokio = { version = "1", features = ["full"] } tokio = { version = "1", features = ["full"] }
distribution = { path = "../../crates/distribution", features = ["iroh", "collector"] } distribution = { path = "../../crates/distribution", features = ["iroh", "collector"] }
# Live runtime dashboard (default features = HTTP overview/actors/topology pages, # Live runtime dashboard (HTTP overview/actors/topology pages, served on
# served on localhost when PP_DASHBOARD is set). No tui/replay-viewer pulled in. # localhost when PP_DASHBOARD is set). The `live-collector` feature pulls in the
dashboard = { path = "../../crates/dashboard" } # server-side vast.ai fold (`VastaiLivePlugin`/`fold`) so the orchestrator can
# fold the collector's record stream into the Fleet tab. No tui/replay pulled in.
dashboard = { path = "../../crates/dashboard", features = ["live-collector"] }
iroh = "0.98" iroh = "0.98"
urlencoding = "2" urlencoding = "2"
base64 = "0.22" base64 = "0.22"
libc = "0.2" libc = "0.2"
[[bin]] [[bin]]
name = "pp-gpu-node" name = "pp-worker"
path = "src/bin/pp_gpu_node.rs" path = "src/bin/pp_worker.rs"
[[bin]] [[bin]]
name = "pp-smoke-run" name = "pp-orchestrator"
path = "src/bin/pp_smoke_run.rs" path = "src/bin/pp_orchestrator.rs"
[dev-dependencies] [dev-dependencies]
wiremock = "0.6" wiremock = "0.6"

4
examples/pipeline-parallel-inference/Dockerfile
View file

@ -23,8 +23,8 @@ FROM ${BASE_IMAGE}
# Pipeline binaries (this crate's target/) + diagnostics binaries (the # Pipeline binaries (this crate's target/) + diagnostics binaries (the
# workspace-root target/). All are statically linked enough that the base # workspace-root target/). All are statically linked enough that the base
# stage's libc is all they need; the worker is pure Python. # stage's libc is all they need; the worker is pure Python.
COPY examples/pipeline-parallel-inference/target/release/pp-gpu-node /usr/local/bin/pp-gpu-node COPY examples/pipeline-parallel-inference/target/release/pp-worker /usr/local/bin/pp-worker
COPY examples/pipeline-parallel-inference/target/release/pp-smoke-run /usr/local/bin/pp-smoke-run COPY examples/pipeline-parallel-inference/target/release/pp-orchestrator /usr/local/bin/pp-orchestrator
COPY target/release/swactor-diag-collector /usr/local/bin/swactor-diag-collector COPY target/release/swactor-diag-collector /usr/local/bin/swactor-diag-collector
COPY target/release/swactor-diag-postproc /usr/local/bin/swactor-diag-postproc COPY target/release/swactor-diag-postproc /usr/local/bin/swactor-diag-postproc
COPY examples/pipeline-parallel-inference/pp_tinygrad_worker.py /usr/local/share/pp_tinygrad_worker.py COPY examples/pipeline-parallel-inference/pp_tinygrad_worker.py /usr/local/share/pp_tinygrad_worker.py

49
examples/pipeline-parallel-inference/PP_N12_DEPLOY_SESSION_REPORT.md
View file

@ -1,49 +0,0 @@
# PP N=12 Deploy — Session Report
## Problems encountered (trivial → blocking)
- **GPU filter too narrow** — exact `RTX 3060` match: stage 5 hit "no offers" mid-chain. Fixed by VRAM-based selection (`PP_GPU_MIN_RAM_MB`).
- **Offer churn** — `no_such_ask`: offers vanish between search and create.
- **API rate-limit (HTTP 429)** — `provision_stage` retried with zero backoff + no inter-stage pacing, burning the candidate pool. Fixed: 429 backoff + `PP_LEASE_PACE_MS`.
- **Worker crash `Code(2)` on all 12 stages** — two root causes, both invisible at first:
- `Tensor(str)` rejected by bundled tinygrad → needed `Tensor(Path(...))`.
- Image shipped only 4 CUDA headers; NVRTC needs the full set (`vector_types.h`). Dockerfile copied a hand-picked subset.
- **Worker stderr swallowed** — StageActor buffers it into a `worker_exit_detail` diag event that needs a collector; none configured → error vanished.
- **SSH auth** — `publickey` denied initially (propagation lag), then worked on 5/12 but **persistently failed on 7/12** (pp-gpu-node holds PID 1 via `exec`; vast key-injection never ran). `vastai attach`/reboot didn't fix it.
- **`vastai execute` unusable** — "Invalid command given" (restricted command set).
- **`--redeploy` blocked** — uses the same SSH/scp, so unusable on the 7 unreachable nodes.
- **Slow/stalled image pulls** — cheap Korea GTX-10-series hosts; one fully stalled (0 bytes), triggering Phase-2 **autoreplace churn**.
- **Autoreplace not disableable** in the running binary.
- **429 on relaunch** — teardown's 12 destroys consumed the budget; absorbed by the new backoff.
## Where we spent the most time
1. **~20 min blind on the silent resolve loop** — connect-timeout SWIM noise looked like the problem but was a red herring; workers had actually crashed instantly.
2. **Getting on a node to see the real error** — SSH flakiness, restricted `execute`, local docker repro, then manual on-node run.
3. **Run #2 image-pull waiting** — many heartbeat ticks on slow/stalled pulls + the re-download after teardown.
## Observability that was clunky / insufficient
- Worker stderr + Python traceback never reach the container log (no collector) — had to reproduce locally and SSH a node to see `Code(2)`'s cause. *(Fixed: pp-gpu-node now prints abnormal-exit stderr.)*
- Resolve loop emits **nothing per-stage** — orchestrator log is just SWIM gossip for up to 20 min; no per-stage worker-ready/download visibility.
- The rich SSE diag stream (`pp_download_progress`, `worker_exit_detail`) was dead — `SWACTOR_DIAG_COLLECTOR_URL` unset.
- vast exposes **no docker-pull %** — `status_msg` only says "Pulling from"; `disk_usage` = -1.
- Connect-timeout logs were prominent but cosmetic — actively misleading.
- Node-id→stage mapping had to be derived by hand from `stage_secrets`.
## Where interaction with the live deployment was limited
- SSH worked on only 5/12 nodes; no reliable shell on the rest.
- `vastai execute` restricted; couldn't run arbitrary diagnostics via API.
- `--redeploy` (the intended fix-forward path) depends on the same broken SSH → fix-forward on live nodes was effectively impossible; had to rebuild the image + re-lease.
- Couldn't pause/disable autoreplace or see/intervene in image-pull progress.
- During the "loading" (pull) phase there's no container, so no SSH at all on the node that mattered most.
## Other notes
- **Sharded fetch works** (~1.8 GB/stage, not 18 GB) — but a stale code comment claims the full GGUF is pulled, which misled diagnosis.
- Core bugs are fixed + validated on a real GPU (stage 0 → `ready`) and in the pushed image; the remaining blocker is purely **host quality** (slow-pull hosts), not code.
- Re-leasing fresh always re-pulls image + re-downloads model; the in-place cache advantage is lost on every teardown.
- Highest-leverage follow-ups: (1) configure a diagnostics collector, (2) emit per-stage resolve/download progress to the orchestrator log, (3) host-throughput preflight or stalled-pull fast-replace, (4) fix the onstart so vast SSH-key injection survives (don't `exec` over it).
## Fixes shipped this session
- `vastai.rs`: 429 backoff in `provision_stage` (find + create paths) and inter-stage pacing (`PP_LEASE_PACE_MS`, default 600ms).
- `pp_tinygrad_worker.py`: `Tensor(gguf_path)` → `Tensor(Path(gguf_path))`.
- `Dockerfile`: copy the full CUDA include set (with a `test -f vector_types.h` build guard) instead of 4 hand-picked headers.
- `stage_actor.rs`: mirror an abnormal worker exit's stderr tail + Python traceback to pp-gpu-node's own stderr (→ container log, collector-independent).
- Image rebuilt + pushed (`zacheryasc/swactor-pp-gpu:latest`, digest `ca373d02…`); both bug fixes validated on a real GPU node (stage 0 reached `ready`).

8
examples/pipeline-parallel-inference/docker-compose.diag.yml
View file

@ -3,14 +3,14 @@
# The collector service runs in its own container with the bundles dir # The collector service runs in its own container with the bundles dir
# bind-mounted from the host so the harness can read the finalized # bind-mounted from the host so the harness can read the finalized
# tarball. Both the collector and the pp processes use the host network # tarball. Both the collector and the pp processes use the host network
# namespace, so the stage children (spawned by `pp-smoke-run` via # namespace, so the stage children (spawned by `pp-orchestrator` via
# `docker-gpu-node.sh`) share localhost reachability with the # `docker-gpu-node.sh`) share localhost reachability with the
# collector — `SWACTOR_DIAG_COLLECTOR_URL=http://127.0.0.1:9080` works # collector — `SWACTOR_DIAG_COLLECTOR_URL=http://127.0.0.1:9080` works
# uniformly from every actor in the run. # uniformly from every actor in the run.
# #
# The host network choice mirrors the existing `docker-e2e.sh` shape: # The host network choice mirrors the existing `docker-e2e.sh` shape:
# pp-smoke-run runs on the host (orchestrator) and each # pp-orchestrator runs on the host (orchestrator) and each
# `pp-gpu-node` runs in its own container under `--network host`. The # `pp-worker` runs in its own container under `--network host`. The
# collector container just adds one more service to that arrangement. # collector container just adds one more service to that arrangement.
# #
# Used by `scripts/docker-diag-e2e.sh`. Direct `docker compose up` # Used by `scripts/docker-diag-e2e.sh`. Direct `docker compose up`
@ -24,7 +24,7 @@ services:
container_name: ${PP_DIAG_COLLECTOR_NAME:-pp-diag-collector} container_name: ${PP_DIAG_COLLECTOR_NAME:-pp-diag-collector}
network_mode: host network_mode: host
# Override the image's default entrypoint (pp_entrypoint.sh, which runs # Override the image's default entrypoint (pp_entrypoint.sh, which runs
# pp-gpu-node) so this container runs the collector instead. The # pp-worker) so this container runs the collector instead. The
# diagnostics binaries ship in the same unified code image. # diagnostics binaries ship in the same unified code image.
entrypoint: /usr/local/bin/swactor-diag-collector entrypoint: /usr/local/bin/swactor-diag-collector
command: command:

22
examples/pipeline-parallel-inference/pp_entrypoint.sh
View file

@ -1,7 +1,7 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# PID-1 supervisor for the pipeline-parallel runtime container. # PID-1 supervisor for the pipeline-parallel runtime container.
# #
# The worker used to run as PID 1 (`exec pp-gpu-node`), so SSH depended on # The worker used to run as PID 1 (`exec pp-worker`), so SSH depended on
# vast's host-side helper winning a race against our exec, and a worker crash # vast's host-side helper winning a race against our exec, and a worker crash
# killed the whole container — no shell left to read the traceback. This script # killed the whole container — no shell left to read the traceback. This script
# instead owns PID 1: it brings up sshd deterministically, runs the worker as a # instead owns PID 1: it brings up sshd deterministically, runs the worker as a
@ -41,9 +41,9 @@ echo "pp-entrypoint: sshd up on :22" >&2
# ── Worker: run as a child, tee output to a file readable over SSH ─────────── # ── Worker: run as a child, tee output to a file readable over SSH ───────────
WORKER_LOG=/var/log/pp-worker.log WORKER_LOG=/var/log/pp-worker.log
echo "pp-entrypoint: launching pp-gpu-node (log -> $WORKER_LOG)" >&2 echo "pp-entrypoint: launching pp-worker (log -> $WORKER_LOG)" >&2
set -o pipefail set -o pipefail
/usr/local/bin/pp-gpu-node 2>&1 | tee "$WORKER_LOG" /usr/local/bin/pp-worker 2>&1 | tee "$WORKER_LOG"
code=${PIPESTATUS[0]} code=${PIPESTATUS[0]}
# ── Operator runbooks (manual, over SSH) ───────────────────────────────────── # ── Operator runbooks (manual, over SSH) ─────────────────────────────────────
@ -51,23 +51,23 @@ code=${PIPESTATUS[0]}
# #
# Worker hot-reload (no restart) — edit the Python in place, then SIGHUP: # Worker hot-reload (no restart) — edit the Python in place, then SIGHUP:
# scp -P <port> pp_tinygrad_worker.py root@<host>:/usr/local/share/pp_tinygrad_worker.py # scp -P <port> pp_tinygrad_worker.py root@<host>:/usr/local/share/pp_tinygrad_worker.py
# ssh <host> 'kill -HUP $(pidof pp-gpu-node)' # ssh <host> 'kill -HUP $(pidof pp-worker)'
# pp-gpu-node tears down its worker and re-execs the on-disk script; the # pp-worker tears down its worker and re-execs the on-disk script; the
# swactor process (and SWIM membership) stays up across the swap. # swactor process (and SWIM membership) stays up across the swap.
# #
# Swactor-binary swap — stop the binary, stage the new one, re-exec under # Swactor-binary swap — stop the binary, stage the new one, re-exec under
# PID 1's env (preserves STAGE/SEED_ADDR/PP_STAGE_SECRET so the node id is # PID 1's env (preserves STAGE/SEED_ADDR/PP_STAGE_SECRET so the node id is
# unchanged). `.new` staging avoids ETXTBSY on the mapped ELF: # unchanged). `.new` staging avoids ETXTBSY on the mapped ELF:
# ssh <host> 'pkill -x pp-gpu-node' # drops to the hold below # ssh <host> 'pkill -x pp-worker' # drops to the hold below
# scp -P <port> pp-gpu-node root@<host>:/usr/local/bin/pp-gpu-node.new # scp -P <port> pp-worker root@<host>:/usr/local/bin/pp-worker.new
# ssh <host> 'mv -f /usr/local/bin/pp-gpu-node.new /usr/local/bin/pp-gpu-node && \ # ssh <host> 'mv -f /usr/local/bin/pp-worker.new /usr/local/bin/pp-worker && \
# chmod +x /usr/local/bin/pp-gpu-node && \ # chmod +x /usr/local/bin/pp-worker && \
# setsid bash -c "while IFS= read -r -d \"\" kv; do export \"\$kv\"; done \ # setsid bash -c "while IFS= read -r -d \"\" kv; do export \"\$kv\"; done \
# < /proc/1/environ; exec /usr/local/bin/pp-gpu-node" \ # < /proc/1/environ; exec /usr/local/bin/pp-worker" \
# >/var/log/pp-restart.log 2>&1 </dev/null &' # >/var/log/pp-restart.log 2>&1 </dev/null &'
# #
# ── Crash policy: do NOT restart. Keep PID 1 / sshd alive for postmortem. ──── # ── Crash policy: do NOT restart. Keep PID 1 / sshd alive for postmortem. ────
echo "pp-entrypoint: pp-gpu-node exited with code $code; NOT restarting (node held for postmortem)" >&2 echo "pp-entrypoint: pp-worker exited with code $code; NOT restarting (node held for postmortem)" >&2
echo "pp-entrypoint: --- last 40 lines of $WORKER_LOG ---" >&2 echo "pp-entrypoint: --- last 40 lines of $WORKER_LOG ---" >&2
tail -n 40 "$WORKER_LOG" >&2 || true tail -n 40 "$WORKER_LOG" >&2 || true
exec sleep infinity exec sleep infinity

12
examples/pipeline-parallel-inference/profiles/example.env
View file

@ -4,7 +4,7 @@
# across the binaries and one operator's shell. Copy it to set up a run: # across the binaries and one operator's shell. Copy it to set up a run:
# #
# cp profiles/example.env profiles/local.env # untracked; put real secrets here # cp profiles/example.env profiles/local.env # untracked; put real secrets here
# PP_PROFILE=profiles/local.env cargo run --bin pp-smoke-run -- --seed ... # PP_PROFILE=profiles/local.env cargo run --bin pp-orchestrator -- --seed ...
# #
# When PP_PROFILE is unset, profiles/local.env is loaded automatically if it # When PP_PROFILE is unset, profiles/local.env is loaded automatically if it
# exists. The run scripts pick this up too (the binaries load it at startup). # exists. The run scripts pick this up too (the binaries load it at startup).
@ -25,27 +25,27 @@
#SWACTOR_IROH_RELAY_URL=https://relay.example.com #SWACTOR_IROH_RELAY_URL=https://relay.example.com
# --- Topology --------------------------------------------------------------- # --- Topology ---------------------------------------------------------------
# Number of pipeline stages (>= 2). pp-smoke-run also accepts --num-stages. # Number of pipeline stages (>= 2). pp-orchestrator also accepts --num-stages.
#NUM_STAGES=2 #NUM_STAGES=2
# --- Compute target (vast.ai / docker) -------------------------------------- # --- Compute target (vast.ai / docker) --------------------------------------
# Container image to run on each node. The run scripts already read PP_IMAGE; # Container image to run on each node. The run scripts already read PP_IMAGE;
# pp-smoke-run's --image default now reads it too. Set to your registry tag. # pp-orchestrator's --image default now reads it too. Set to your registry tag.
#PP_IMAGE=swactor-pp-gpu:latest #PP_IMAGE=swactor-pp-gpu:latest
# GPU class requested when leasing on vast.ai (pp-smoke-run --gpu overrides). # GPU class requested when leasing on vast.ai (pp-orchestrator --gpu overrides).
#PP_GPU=RTX 3060 #PP_GPU=RTX 3060
# --- Workload (what each stage computes) ------------------------------------ # --- Workload (what each stage computes) ------------------------------------
# Model identifier handed to the worker. # Model identifier handed to the worker.
#MODEL= #MODEL=
# Python worker script + interpreter (per-stage compute). WORKER_SCRIPT is the # Python worker script + interpreter (per-stage compute). WORKER_SCRIPT is the
# pp-gpu-node default; pp-smoke-run --worker ships a script to remote nodes. # pp-worker default; pp-orchestrator --worker ships a script to remote nodes.
#WORKER_SCRIPT=./pp_tinygrad_worker.py #WORKER_SCRIPT=./pp_tinygrad_worker.py
#WORKER_CMD=python3 #WORKER_CMD=python3
#PYTHON=python3 #PYTHON=python3
# Stub mode: skip the real worker, echo activations (fast local smoke runs). # Stub mode: skip the real worker, echo activations (fast local smoke runs).
#PP_WORKER_STUB=1 #PP_WORKER_STUB=1
# Inference request prompt + token budget (pp-smoke-run --prompt/--max-tokens). # Inference request prompt + token budget (pp-orchestrator --prompt/--max-tokens).
#MAX_TOKENS=64 #MAX_TOKENS=64
# --- Timeouts (seconds; sane defaults baked in — override only if needed) --- # --- Timeouts (seconds; sane defaults baked in — override only if needed) ---

2
examples/pipeline-parallel-inference/profiles/local-seed.env
View file

@ -1,7 +1,7 @@
# Local seed-mode smoke run: everything on loopback, stub workers, no relay. # Local seed-mode smoke run: everything on loopback, stub workers, no relay.
# Drives one request through a 2-stage chain without a GPU or external relay. # Drives one request through a 2-stage chain without a GPU or external relay.
# #
# PP_PROFILE=profiles/local-seed.env cargo run --bin pp-smoke-run -- --seed # PP_PROFILE=profiles/local-seed.env cargo run --bin pp-orchestrator -- --seed
# #
# (No SWACTOR_IROH_RELAY_URL: seed mode wires nodes via direct loopback addrs.) # (No SWACTOR_IROH_RELAY_URL: seed mode wires nodes via direct loopback addrs.)

10
examples/pipeline-parallel-inference/profiles/vastai.env
View file

@ -1,7 +1,7 @@
# WAN run on leased vast.ai GPUs. Copy to profiles/local.env and fill in the # WAN run on leased vast.ai GPUs. Copy to profiles/local.env and fill in the
# relay URL + image; keep that copy untracked. # relay URL + image; keep that copy untracked.
# #
# PP_PROFILE=profiles/local.env cargo run --bin pp-smoke-run -- \ # PP_PROFILE=profiles/local.env cargo run --bin pp-orchestrator -- \
# --vastai --api-key "$VASTAI_API_KEY" # --vastai --api-key "$VASTAI_API_KEY"
# #
# All nodes home onto one operator-controlled relay so they can hole-punch / # All nodes home onto one operator-controlled relay so they can hole-punch /
@ -13,6 +13,10 @@ SWACTOR_IROH_RELAY_URL=https://relay.example.com
NUM_STAGES=2 NUM_STAGES=2
PP_IMAGE=swactor-pp-gpu:latest PP_IMAGE=swactor-pp-gpu:latest
PP_GPU=RTX 3060 # GPU filters are optional. Leave both unset to let any verified 1-GPU offer
MODEL= # qualify (a 1B model fits anywhere). Set PP_GPU="RTX 3060" to pin a model, and/or
# PP_GPU_MIN_RAM_MB=8000 to require a VRAM floor.
# llama3.2:1b -> HF Llama-3.2-1B-Instruct-Q6_K.gguf (16 blocks, sharded across
# the N stages by compute_layer_range; validated key baked into the image).
MODEL=llama3.2:1b
MAX_TOKENS=64 MAX_TOKENS=64

105
examples/pipeline-parallel-inference/scripts/demo-fleet.sh
View file

@ -1,17 +1,23 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# demo-fleet.sh — one-command local mock of a vast.ai fleet, watchable live. # demo-fleet.sh — one-command local mock of a vast.ai fleet, watchable live.
# #
# Brings up, from a single command, a self-contained demo that streams the # Brings up, from a single command, a self-contained demo that mirrors the
# REAL host metrics each container measures (no synthetic data) into a live # production topology: a diagnostics collector running "off-box" (in prod, a
# browser dashboard: # VPS) and the orchestrator running locally and hosting the FULL swactor
# dashboard. The orchestrator's dashboard shows its own live swactor process
# info (overview / actors / topology / distribution / netmap) and a Fleet tab
# that pulls the vast.ai + host metrics remotely from the collector:
# #
# - swactor-diag-collector on the HOST (HTTP 9080 + UDP 9081), serving the # - swactor-diag-collector on the HOST (HTTP 9080 + UDP 9081) — the "VPS"
# live fleet board at http://127.0.0.1:9080/dashboard # sink. Each stage's in-VM monitor ships REAL host_sample + log records
# - pp-smoke-run on the HOST in --seed mode, spawning N pp-gpu-node # (no synthetic data) here; the orchestrator pushes its distribution
# containers (one per stage) via docker-gpu-node.sh, each on --network host # snapshot here too. Its own fleet board stays at http://127.0.0.1:9080/dashboard
# - pp-orchestrator on the HOST in --seed mode (PP_DASHBOARD on), spawning N
# pp-worker containers (one per stage) via docker-gpu-node.sh, each on
# --network host, and serving the full dashboard at http://127.0.0.1:9095/
# - PP_HOLD=1 keeps the cluster up after the first drive, so every stage's # - PP_HOLD=1 keeps the cluster up after the first drive, so every stage's
# in-VM monitor keeps shipping host_sample + log records (~every 5s) and # in-VM monitor keeps shipping records (~every 5s) and the dashboard
# the dashboard animates in real time. # animates in real time.
# #
# Ctrl+C (or any exit) tears everything down: stage containers, collector, # Ctrl+C (or any exit) tears everything down: stage containers, collector,
# orchestrator, and all temp files. # orchestrator, and all temp files.
@ -31,6 +37,7 @@
# PP_MAX_TOKENS decode token cap (default: 4) # PP_MAX_TOKENS decode token cap (default: 4)
# PP_BIND_HOST collector bind host (default: 127.0.0.1) # PP_BIND_HOST collector bind host (default: 127.0.0.1)
# PP_PORT collector HTTP port (default: 9080) # PP_PORT collector HTTP port (default: 9080)
# PP_DASHBOARD_PORT orchestrator dashboard HTTP port (default: 9095)
# PP_NO_OPEN if set, don't try to open the dashboard in a browser # PP_NO_OPEN if set, don't try to open the dashboard in a browser
# PP_DIAG_NETWORK docker network for stages (default: host) # PP_DIAG_NETWORK docker network for stages (default: host)
set -euo pipefail set -euo pipefail
@ -43,9 +50,13 @@ MAX_TOKENS="${PP_MAX_TOKENS:-4}"
BIND_HOST="${PP_BIND_HOST:-127.0.0.1}" BIND_HOST="${PP_BIND_HOST:-127.0.0.1}"
PORT="${PP_PORT:-9080}" PORT="${PP_PORT:-9080}"
UDP_PORT=$((PORT + 1)) UDP_PORT=$((PORT + 1))
DASH_PORT="${PP_DASHBOARD_PORT:-9095}"
CONTAINER_PREFIX="demo-fleet-stage" CONTAINER_PREFIX="demo-fleet-stage"
RUN_ID="demo-fleet-$(date +%s)" RUN_ID="demo-fleet-$(date +%s)"
DASH_URL="http://${BIND_HOST}:${PORT}/dashboard" # The full swactor dashboard is served by the orchestrator at "/"; the collector
# keeps its own standalone fleet board at :PORT/dashboard.
DASH_URL="http://${BIND_HOST}:${DASH_PORT}/"
COLLECTOR_URL="http://${BIND_HOST}:${PORT}"
if ! [[ "$NUM_STAGES" =~ ^[0-9]+$ ]] || [ "$NUM_STAGES" -lt 2 ]; then if ! [[ "$NUM_STAGES" =~ ^[0-9]+$ ]] || [ "$NUM_STAGES" -lt 2 ]; then
echo "demo-fleet: N must be an integer >= 2 (seed mode needs >=2 stages), got '$NUM_STAGES'" >&2 echo "demo-fleet: N must be an integer >= 2 (seed mode needs >=2 stages), got '$NUM_STAGES'" >&2
@ -57,26 +68,36 @@ fi
if ! docker info >/dev/null 2>&1; then if ! docker info >/dev/null 2>&1; then
echo "demo-fleet: docker daemon unreachable" >&2; exit 2 echo "demo-fleet: docker daemon unreachable" >&2; exit 2
fi fi
# Fail loudly on a clash for the orchestrator dashboard port. Its HTTP server is
# spawned on the driver's tokio runtime and `.expect()`s its bind; a collision
# panics that task silently and the run carries on with no dashboard. Catch it
# here so the user can pick a free one.
if (exec 3<>"/dev/tcp/${BIND_HOST}/${DASH_PORT}") 2>/dev/null; then
exec 3>&- 3<&-
echo "demo-fleet: dashboard port ${DASH_PORT} is already in use." \
"Pick a free one: PP_DASHBOARD_PORT=9096 $0 ${NUM_STAGES}" >&2
exit 2
fi
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
CRATE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" CRATE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
WORKSPACE_DIR="$(cd "$CRATE_DIR/../.." && pwd)" WORKSPACE_DIR="$(cd "$CRATE_DIR/../.." && pwd)"
SMOKE_RUN_BIN="$CRATE_DIR/target/release/pp-smoke-run" ORCHESTRATOR_BIN="$CRATE_DIR/target/release/pp-orchestrator"
GPU_NODE_BIN="$CRATE_DIR/target/release/pp-gpu-node" WORKER_BIN="$CRATE_DIR/target/release/pp-worker"
WORKER_PY="$CRATE_DIR/pp_tinygrad_worker.py" WORKER_PY="$CRATE_DIR/pp_tinygrad_worker.py"
COLLECTOR_BIN="$WORKSPACE_DIR/target/release/swactor-diag-collector" COLLECTOR_BIN="$WORKSPACE_DIR/target/release/swactor-diag-collector"
# ── Step 1: build release artifacts ─────────────────────────────────────── # ── Step 1: build release artifacts ───────────────────────────────────────
if [ -z "${PP_SKIP_BUILD:-}" ]; then if [ -z "${PP_SKIP_BUILD:-}" ]; then
echo "demo-fleet: cargo build pp-smoke-run + pp-gpu-node (release)" echo "demo-fleet: cargo build pp-orchestrator + pp-worker (release)"
cargo build --manifest-path "$CRATE_DIR/Cargo.toml" --release \ cargo build --manifest-path "$CRATE_DIR/Cargo.toml" --release \
--bin pp-gpu-node --bin pp-smoke-run --bin pp-worker --bin pp-orchestrator
echo "demo-fleet: cargo build swactor-diag-collector (release, --features collector)" echo "demo-fleet: cargo build swactor-diag-collector (release, --features collector)"
cargo build --manifest-path "$WORKSPACE_DIR/Cargo.toml" --release \ cargo build --manifest-path "$WORKSPACE_DIR/Cargo.toml" --release \
-p distribution --features collector --bin swactor-diag-collector -p distribution --features collector --bin swactor-diag-collector
fi fi
for f in "$SMOKE_RUN_BIN" "$GPU_NODE_BIN" "$WORKER_PY" "$COLLECTOR_BIN"; do for f in "$ORCHESTRATOR_BIN" "$WORKER_BIN" "$WORKER_PY" "$COLLECTOR_BIN"; do
[ -f "$f" ] || { echo "demo-fleet: missing $f (run without PP_SKIP_BUILD)" >&2; exit 1; } [ -f "$f" ] || { echo "demo-fleet: missing $f (run without PP_SKIP_BUILD)" >&2; exit 1; }
done done
@ -129,7 +150,7 @@ cleanup() {
trap cleanup EXIT trap cleanup EXIT
trap 'exit 130' INT TERM trap 'exit 130' INT TERM
# ── Step 3: collector on the host (serves the dashboard) ─────────────────── # ── Step 3: collector on the host (the off-box "VPS" metrics sink) ─────────
echo "demo-fleet: starting collector on ${BIND_HOST}:${PORT} (root=$COLLECTOR_ROOT)" echo "demo-fleet: starting collector on ${BIND_HOST}:${PORT} (root=$COLLECTOR_ROOT)"
"$COLLECTOR_BIN" --bind "${BIND_HOST}:${PORT}" --root "$COLLECTOR_ROOT" \ "$COLLECTOR_BIN" --bind "${BIND_HOST}:${PORT}" --root "$COLLECTOR_ROOT" \
--udp "${BIND_HOST}:${UDP_PORT}" >"$COLLECTOR_LOG" 2>&1 & --udp "${BIND_HOST}:${UDP_PORT}" >"$COLLECTOR_LOG" 2>&1 &
@ -148,38 +169,31 @@ until (echo > "/dev/tcp/${BIND_HOST}/${PORT}") >/dev/null 2>&1; do
done done
echo "demo-fleet: collector ready" echo "demo-fleet: collector ready"
# ── Step 4: announce + open the dashboard ────────────────────────────────── # ── Step 4: orchestrator (hosts the full dashboard, holds the cluster open) ─
echo
echo " ┌─────────────────────────────────────────────────────────────┐"
echo " │ Live fleet dashboard: $DASH_URL"
echo " └─────────────────────────────────────────────────────────────┘"
echo
if [ -z "${PP_NO_OPEN:-}" ]; then
if command -v xdg-open >/dev/null 2>&1; then (xdg-open "$DASH_URL" >/dev/null 2>&1 &) || true
elif command -v open >/dev/null 2>&1; then (open "$DASH_URL" >/dev/null 2>&1 &) || true
fi
fi
# ── Step 5: orchestrator (holds the cluster open, real metrics enabled) ─────
# stdin is the FIFO; we hold its write end open on fd 3 so hold_open() never # stdin is the FIFO; we hold its write end open on fd 3 so hold_open() never
# sees EOF and the cluster stays up until we tear down. # sees EOF and the cluster stays up until we tear down.
exec 3<>"$FIFO" exec 3<>"$FIFO"
echo "demo-fleet: launching orchestrator + ${NUM_STAGES} stage containers (run_id=$RUN_ID)" echo "demo-fleet: launching orchestrator + ${NUM_STAGES} stage containers (run_id=$RUN_ID)"
# The orchestrator (and, via inheritance, docker-gpu-node.sh) read these from # The orchestrator (and, via inheritance, docker-gpu-node.sh) read these from
# the environment. PP_HOLD keeps the cluster up; the SWACTOR_DIAG_* vars enable # the environment. PP_HOLD keeps the cluster up; PP_DASHBOARD makes the
# each stage's in-VM monitor so real host metrics ship to the collector. # orchestrator host the full swactor dashboard locally; the SWACTOR_DIAG_* vars
# point each stage's in-VM monitor at the collector (the off-box sink) and give
# the orchestrator the same URL to push its distribution snapshot to and to pull
# the fleet model from for its Fleet tab.
export PP_HOLD=1 export PP_HOLD=1
export PP_WORKER_STUB=1 export PP_WORKER_STUB=1
export PP_DEV=CPU export PP_DEV=CPU
export PP_IMAGE="$IMAGE" export PP_IMAGE="$IMAGE"
export PP_CONTAINER_PREFIX="$CONTAINER_PREFIX" export PP_CONTAINER_PREFIX="$CONTAINER_PREFIX"
export SWACTOR_DIAG_COLLECTOR_URL="http://${BIND_HOST}:${PORT}" export PP_DASHBOARD=1
export PP_DASHBOARD_PORT="$DASH_PORT"
export SWACTOR_DIAG_COLLECTOR_URL="$COLLECTOR_URL"
export SWACTOR_DIAG_RUN_ID="$RUN_ID" export SWACTOR_DIAG_RUN_ID="$RUN_ID"
export SWACTOR_DIAG_SPOOL_DIR="$SPOOL_DIR" export SWACTOR_DIAG_SPOOL_DIR="$SPOOL_DIR"
export SWACTOR_DIAG_UDP_ECHO="${BIND_HOST}:${UDP_PORT}" export SWACTOR_DIAG_UDP_ECHO="${BIND_HOST}:${UDP_PORT}"
[ -n "${PP_GPUS:-}" ] && export PP_GPUS [ -n "${PP_GPUS:-}" ] && export PP_GPUS
[ -n "${PP_DIAG_NETWORK:-}" ] && export PP_DIAG_NETWORK [ -n "${PP_DIAG_NETWORK:-}" ] && export PP_DIAG_NETWORK
"$SMOKE_RUN_BIN" \ "$ORCHESTRATOR_BIN" \
--seed \ --seed \
--num-stages "$NUM_STAGES" \ --num-stages "$NUM_STAGES" \
--gpu-node "$SCRIPT_DIR/docker-gpu-node.sh" \ --gpu-node "$SCRIPT_DIR/docker-gpu-node.sh" \
@ -189,6 +203,31 @@ export SWACTOR_DIAG_UDP_ECHO="${BIND_HOST}:${UDP_PORT}"
<"$FIFO" >"$ORCH_LOG" 2>&1 & <"$FIFO" >"$ORCH_LOG" 2>&1 &
ORCH_PID=$! ORCH_PID=$!
# ── Step 5: wait for the orchestrator's dashboard to bind, then announce + open
WAITED=0
until (echo > "/dev/tcp/${BIND_HOST}/${DASH_PORT}") >/dev/null 2>&1; do
if ! kill -0 "$ORCH_PID" >/dev/null 2>&1; then
echo "demo-fleet: orchestrator exited before its dashboard came up." >&2
tail -n 40 "$ORCH_LOG" >&2 || true
exit 1
fi
WAITED=$((WAITED + 1))
[ "$WAITED" -ge 30 ] && { echo "demo-fleet: orchestrator dashboard did not bind :${DASH_PORT} in 30s" >&2; tail -n 40 "$ORCH_LOG" >&2; exit 1; }
sleep 1
done
echo
echo " ┌─────────────────────────────────────────────────────────────┐"
echo " │ Full swactor dashboard: $DASH_URL"
echo " │ (overview / actors / topology / distribution / netmap / fleet)"
echo " │ Collector fleet board: ${COLLECTOR_URL}/dashboard"
echo " └─────────────────────────────────────────────────────────────┘"
echo
if [ -z "${PP_NO_OPEN:-}" ]; then
if command -v xdg-open >/dev/null 2>&1; then (xdg-open "$DASH_URL" >/dev/null 2>&1 &) || true
elif command -v open >/dev/null 2>&1; then (open "$DASH_URL" >/dev/null 2>&1 &) || true
fi
fi
# ── Step 6: wait until the cluster is converged + held open ──────────────── # ── Step 6: wait until the cluster is converged + held open ────────────────
echo "demo-fleet: waiting for the cluster to converge (first inference drive)…" echo "demo-fleet: waiting for the cluster to converge (first inference drive)…"
WAITED=0 WAITED=0

18
examples/pipeline-parallel-inference/scripts/docker-dashboard-e2e.sh
View file

@ -2,7 +2,7 @@
# docker-dashboard-e2e.sh — the docker-e2e run, held open under the live # docker-dashboard-e2e.sh — the docker-e2e run, held open under the live
# swactor dashboard, one dashboard PER STAGE. # swactor dashboard, one dashboard PER STAGE.
# #
# Brings up `N` stub-mode `pp-gpu-node` containers on localhost and drives # Brings up `N` stub-mode `pp-worker` containers on localhost and drives
# one InferenceRequest through them, exactly like `docker-e2e.sh` — but each # one InferenceRequest through them, exactly like `docker-e2e.sh` — but each
# stage serves the live swactor dashboard (PP_STAGE_DASHBOARD) and the # stage serves the live swactor dashboard (PP_STAGE_DASHBOARD) and the
# orchestrator HOLDS after the drive (PP_HOLD). The stage containers run on # orchestrator HOLDS after the drive (PP_HOLD). The stage containers run on
@ -78,8 +78,8 @@ done
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
CRATE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" CRATE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
WORKSPACE_DIR="$(cd "$CRATE_DIR/../.." && pwd)" WORKSPACE_DIR="$(cd "$CRATE_DIR/../.." && pwd)"
SMOKE_RUN_BIN="$CRATE_DIR/target/release/pp-smoke-run" ORCHESTRATOR_BIN="$CRATE_DIR/target/release/pp-orchestrator"
GPU_NODE_BIN="$CRATE_DIR/target/release/pp-gpu-node" WORKER_BIN="$CRATE_DIR/target/release/pp-worker"
WORKER_PY="$CRATE_DIR/pp_tinygrad_worker.py" WORKER_PY="$CRATE_DIR/pp_tinygrad_worker.py"
COLLECTOR_BIN="$WORKSPACE_DIR/target/release/swactor-diag-collector" COLLECTOR_BIN="$WORKSPACE_DIR/target/release/swactor-diag-collector"
POSTPROC_BIN="$WORKSPACE_DIR/target/release/swactor-diag-postproc" POSTPROC_BIN="$WORKSPACE_DIR/target/release/swactor-diag-postproc"
@ -87,20 +87,20 @@ POSTPROC_BIN="$WORKSPACE_DIR/target/release/swactor-diag-postproc"
# Step 1: build the release artifacts the docker image packages (same set # Step 1: build the release artifacts the docker image packages (same set
# docker-e2e.sh builds — the image's COPY needs the diag binaries present). # docker-e2e.sh builds — the image's COPY needs the diag binaries present).
if [ -z "${PP_SKIP_BUILD:-}" ]; then if [ -z "${PP_SKIP_BUILD:-}" ]; then
echo "docker-dashboard-e2e: building pp-gpu-node + pp-smoke-run (release)" echo "docker-dashboard-e2e: building pp-worker + pp-orchestrator (release)"
cargo build --manifest-path "$CRATE_DIR/Cargo.toml" --release \ cargo build --manifest-path "$CRATE_DIR/Cargo.toml" --release \
--bin pp-gpu-node --bin pp-smoke-run --bin pp-worker --bin pp-orchestrator
echo "docker-dashboard-e2e: building swactor-diag-{collector,postproc} (release)" echo "docker-dashboard-e2e: building swactor-diag-{collector,postproc} (release)"
cargo build --manifest-path "$WORKSPACE_DIR/Cargo.toml" --release \ cargo build --manifest-path "$WORKSPACE_DIR/Cargo.toml" --release \
-p distribution --features collector \ -p distribution --features collector \
--bin swactor-diag-collector --bin swactor-diag-postproc --bin swactor-diag-collector --bin swactor-diag-postproc
fi fi
for f in "$SMOKE_RUN_BIN" "$GPU_NODE_BIN" "$WORKER_PY" "$COLLECTOR_BIN" "$POSTPROC_BIN"; do for f in "$ORCHESTRATOR_BIN" "$WORKER_BIN" "$WORKER_PY" "$COLLECTOR_BIN" "$POSTPROC_BIN"; do
[ -f "$f" ] || { echo "docker-dashboard-e2e: missing $f" >&2; exit 1; } [ -f "$f" ] || { echo "docker-dashboard-e2e: missing $f" >&2; exit 1; }
done done
# Step 2: build the layered image (heavy CUDA base, then thin code layer). # Step 2: build the layered image (heavy CUDA base, then thin code layer).
# The stage dashboard lives in the pp-gpu-node binary baked into this image, # The stage dashboard lives in the pp-worker binary baked into this image,
# so a stale image without it will show nothing — rebuild unless you know the # so a stale image without it will show nothing — rebuild unless you know the
# current image already carries the dashboard-enabled binary. # current image already carries the dashboard-enabled binary.
if [ -z "${PP_SKIP_IMAGE_BUILD:-}" ]; then if [ -z "${PP_SKIP_IMAGE_BUILD:-}" ]; then
@ -131,7 +131,7 @@ for ((k = 0; k < NUM_STAGES; k++)); do
echo " stage ${k}: http://localhost:$((PORT_BASE + k))" echo " stage ${k}: http://localhost:$((PORT_BASE + k))"
done done
# Step 4: drive pp-smoke-run with the docker shim. The orchestrator serves its # Step 4: drive pp-orchestrator with the docker shim. The orchestrator serves its
# own dashboard (PP_DASHBOARD) — including the live SWIM distribution graph and # own dashboard (PP_DASHBOARD) — including the live SWIM distribution graph and
# message tallies — and each stage serves its own (PP_STAGE_DASHBOARD). PP_HOLD # message tallies — and each stage serves its own (PP_STAGE_DASHBOARD). PP_HOLD
# makes the orchestrator block at the end, ticking the driver so the # makes the orchestrator block at the end, ticking the driver so the
@ -146,7 +146,7 @@ PP_DASHBOARD=1 \
PP_DASHBOARD_PORT="$ORCH_PORT" \ PP_DASHBOARD_PORT="$ORCH_PORT" \
PP_STAGE_DASHBOARD=1 \ PP_STAGE_DASHBOARD=1 \
PP_STAGE_DASHBOARD_PORT_BASE="$PORT_BASE" \ PP_STAGE_DASHBOARD_PORT_BASE="$PORT_BASE" \
"$SMOKE_RUN_BIN" \ "$ORCHESTRATOR_BIN" \
--seed \ --seed \
--num-stages "$NUM_STAGES" \ --num-stages "$NUM_STAGES" \
--gpu-node "$SCRIPT_DIR/docker-gpu-node.sh" \ --gpu-node "$SCRIPT_DIR/docker-gpu-node.sh" \

30
examples/pipeline-parallel-inference/scripts/docker-diag-e2e.sh
View file

@ -3,7 +3,7 @@
# #
# Brings up: # Brings up:
# - `swactor-diag-collector` in a container (HTTP 9080 + UDP 9081) # - `swactor-diag-collector` in a container (HTTP 9080 + UDP 9081)
# - `pp-smoke-run` on the host, in seed mode with N stub-mode stage # - `pp-orchestrator` on the host, in seed mode with N stub-mode stage
# children spawned via `docker-gpu-node.sh` (each its own # children spawned via `docker-gpu-node.sh` (each its own
# container on `--network host`) # container on `--network host`)
# #
@ -95,8 +95,8 @@ CRATE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
WORKSPACE_DIR="$(cd "$CRATE_DIR/../.." && pwd)" WORKSPACE_DIR="$(cd "$CRATE_DIR/../.." && pwd)"
COMPOSE_FILE="$CRATE_DIR/docker-compose.diag.yml" COMPOSE_FILE="$CRATE_DIR/docker-compose.diag.yml"
SMOKE_RUN_BIN="$CRATE_DIR/target/release/pp-smoke-run" ORCHESTRATOR_BIN="$CRATE_DIR/target/release/pp-orchestrator"
GPU_NODE_BIN="$CRATE_DIR/target/release/pp-gpu-node" WORKER_BIN="$CRATE_DIR/target/release/pp-worker"
WORKER_PY="$CRATE_DIR/pp_tinygrad_worker.py" WORKER_PY="$CRATE_DIR/pp_tinygrad_worker.py"
COLLECTOR_BIN="$WORKSPACE_DIR/target/release/swactor-diag-collector" COLLECTOR_BIN="$WORKSPACE_DIR/target/release/swactor-diag-collector"
POSTPROC_BIN="$WORKSPACE_DIR/target/release/swactor-diag-postproc" POSTPROC_BIN="$WORKSPACE_DIR/target/release/swactor-diag-postproc"
@ -104,15 +104,15 @@ POSTPROC_BIN="$WORKSPACE_DIR/target/release/swactor-diag-postproc"
# Step 1: build release artifacts the image will package. The pp binaries # Step 1: build release artifacts the image will package. The pp binaries
# live in their own workspace; the distribution binaries live at the top. # live in their own workspace; the distribution binaries live at the top.
if [ -z "${PP_SKIP_BUILD:-}" ]; then if [ -z "${PP_SKIP_BUILD:-}" ]; then
echo "docker-diag-e2e: cargo build pp-smoke-run + pp-gpu-node (release)" echo "docker-diag-e2e: cargo build pp-orchestrator + pp-worker (release)"
cargo build --manifest-path "$CRATE_DIR/Cargo.toml" --release \ cargo build --manifest-path "$CRATE_DIR/Cargo.toml" --release \
--bin pp-gpu-node --bin pp-smoke-run --bin pp-worker --bin pp-orchestrator
echo "docker-diag-e2e: cargo build swactor-diag-{collector,postproc} (release, --features collector)" echo "docker-diag-e2e: cargo build swactor-diag-{collector,postproc} (release, --features collector)"
cargo build --manifest-path "$WORKSPACE_DIR/Cargo.toml" --release \ cargo build --manifest-path "$WORKSPACE_DIR/Cargo.toml" --release \
-p distribution --features collector \ -p distribution --features collector \
--bin swactor-diag-collector --bin swactor-diag-postproc --bin swactor-diag-collector --bin swactor-diag-postproc
fi fi
for f in "$SMOKE_RUN_BIN" "$GPU_NODE_BIN" "$WORKER_PY" "$COLLECTOR_BIN" "$POSTPROC_BIN"; do for f in "$ORCHESTRATOR_BIN" "$WORKER_BIN" "$WORKER_PY" "$COLLECTOR_BIN" "$POSTPROC_BIN"; do
[ -f "$f" ] || { echo "docker-diag-e2e: missing $f" >&2; exit 1; } [ -f "$f" ] || { echo "docker-diag-e2e: missing $f" >&2; exit 1; }
done done
@ -190,7 +190,7 @@ if [ "$USE_COMPOSE" = 1 ]; then
"${COMPOSE[@]}" -f "$COMPOSE_FILE" up -d --remove-orphans collector "${COMPOSE[@]}" -f "$COMPOSE_FILE" up -d --remove-orphans collector
else else
# --entrypoint runs the collector directly; the default image entrypoint # --entrypoint runs the collector directly; the default image entrypoint
# (pp_entrypoint.sh) would ignore these args and launch pp-gpu-node. # (pp_entrypoint.sh) would ignore these args and launch pp-worker.
docker run -d --rm \ docker run -d --rm \
--name "$COLLECTOR_NAME" \ --name "$COLLECTOR_NAME" \
--network "$DIAG_NETWORK" \ --network "$DIAG_NETWORK" \
@ -223,7 +223,7 @@ until (echo > /dev/tcp/127.0.0.1/9080) >/dev/null 2>&1; do
done done
echo "docker-diag-e2e: collector ready" echo "docker-diag-e2e: collector ready"
# Step 5: drive pp-smoke-run with diagnostics env vars set. Stage children # Step 5: drive pp-orchestrator with diagnostics env vars set. Stage children
# pick up the same vars via docker-gpu-node.sh's `-e` forwarders. # pick up the same vars via docker-gpu-node.sh's `-e` forwarders.
OUTPUT_DIR="$(mktemp -d)" OUTPUT_DIR="$(mktemp -d)"
STDOUT_LOG="$OUTPUT_DIR/stdout.log" STDOUT_LOG="$OUTPUT_DIR/stdout.log"
@ -244,7 +244,7 @@ SWACTOR_DIAG_COLLECTOR_URL="http://127.0.0.1:9080" \
SWACTOR_DIAG_RUN_ID="$RUN_ID" \ SWACTOR_DIAG_RUN_ID="$RUN_ID" \
SWACTOR_DIAG_SPOOL_DIR="$OUTPUT_DIR/spool" \ SWACTOR_DIAG_SPOOL_DIR="$OUTPUT_DIR/spool" \
SWACTOR_DIAG_UDP_ECHO="127.0.0.1:9081" \ SWACTOR_DIAG_UDP_ECHO="127.0.0.1:9081" \
"$SMOKE_RUN_BIN" \ "$ORCHESTRATOR_BIN" \
--seed \ --seed \
--num-stages "$NUM_STAGES" \ --num-stages "$NUM_STAGES" \
--gpu-node "$SCRIPT_DIR/docker-gpu-node.sh" \ --gpu-node "$SCRIPT_DIR/docker-gpu-node.sh" \
@ -252,20 +252,20 @@ SWACTOR_DIAG_UDP_ECHO="127.0.0.1:9081" \
--prompt "$PROMPT" \ --prompt "$PROMPT" \
--max-tokens "$MAX_TOKENS" \ --max-tokens "$MAX_TOKENS" \
>"$STDOUT_LOG" 2>"$STDERR_LOG" >"$STDOUT_LOG" 2>"$STDERR_LOG"
SMOKE_STATUS=$? ORCH_STATUS=$?
set -e set -e
if [ "$SMOKE_STATUS" -ne 0 ]; then if [ "$ORCH_STATUS" -ne 0 ]; then
echo "docker-diag-e2e: pp-smoke-run exited $SMOKE_STATUS" >&2 echo "docker-diag-e2e: pp-orchestrator exited $ORCH_STATUS" >&2
echo "----- stdout -----" >&2 echo "----- stdout -----" >&2
cat "$STDOUT_LOG" >&2 cat "$STDOUT_LOG" >&2
echo "----- stderr (last 60) -----" >&2 echo "----- stderr (last 60) -----" >&2
tail -n 60 "$STDERR_LOG" >&2 tail -n 60 "$STDERR_LOG" >&2
exit 1 exit 1
fi fi
echo "docker-diag-e2e: pp-smoke-run exited 0" echo "docker-diag-e2e: pp-orchestrator exited 0"
if [ -n "${PP_DIAG_VERBOSE:-}" ]; then if [ -n "${PP_DIAG_VERBOSE:-}" ]; then
echo "----- pp-smoke-run stderr (last 30) -----" echo "----- pp-orchestrator stderr (last 30) -----"
tail -n 30 "$STDERR_LOG" tail -n 30 "$STDERR_LOG"
fi fi
@ -295,7 +295,7 @@ while true; do
else else
docker logs "$COLLECTOR_NAME" 2>&1 | tail -n 50 >&2 || true docker logs "$COLLECTOR_NAME" 2>&1 | tail -n 50 >&2 || true
fi fi
echo "----- pp-smoke-run stderr (last 60) -----" >&2 echo "----- pp-orchestrator stderr (last 60) -----" >&2
tail -n 60 "$STDERR_LOG" >&2 || true tail -n 60 "$STDERR_LOG" >&2 || true
exit 1 exit 1
fi fi

26
examples/pipeline-parallel-inference/scripts/docker-e2e.sh
View file

@ -1,8 +1,8 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# docker-e2e.sh — the Stage 11 pre-deploy gate. # docker-e2e.sh — the Stage 11 pre-deploy gate.
# #
# Brings up `N` stub-mode `pp-gpu-node` containers on localhost, drives # Brings up `N` stub-mode `pp-worker` containers on localhost, drives
# one InferenceRequest through them via `pp-smoke-run --seed`, and tears # one InferenceRequest through them via `pp-orchestrator --seed`, and tears
# everything down. The image is built locally from the workspace's # everything down. The image is built locally from the workspace's
# release artifacts; no GPU, no tinygrad, no GGUF required. # release artifacts; no GPU, no tinygrad, no GGUF required.
# #
@ -63,8 +63,8 @@ fi
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
CRATE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" CRATE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
WORKSPACE_DIR="$(cd "$CRATE_DIR/../.." && pwd)" WORKSPACE_DIR="$(cd "$CRATE_DIR/../.." && pwd)"
SMOKE_RUN_BIN="$CRATE_DIR/target/release/pp-smoke-run" ORCHESTRATOR_BIN="$CRATE_DIR/target/release/pp-orchestrator"
GPU_NODE_BIN="$CRATE_DIR/target/release/pp-gpu-node" WORKER_BIN="$CRATE_DIR/target/release/pp-worker"
WORKER_PY="$CRATE_DIR/pp_tinygrad_worker.py" WORKER_PY="$CRATE_DIR/pp_tinygrad_worker.py"
COLLECTOR_BIN="$WORKSPACE_DIR/target/release/swactor-diag-collector" COLLECTOR_BIN="$WORKSPACE_DIR/target/release/swactor-diag-collector"
POSTPROC_BIN="$WORKSPACE_DIR/target/release/swactor-diag-postproc" POSTPROC_BIN="$WORKSPACE_DIR/target/release/swactor-diag-postproc"
@ -75,15 +75,15 @@ POSTPROC_BIN="$WORKSPACE_DIR/target/release/swactor-diag-postproc"
# present even for this stub run. The pp binaries live in this crate's # present even for this stub run. The pp binaries live in this crate's
# workspace; the diagnostics binaries live at the repo root. # workspace; the diagnostics binaries live at the repo root.
if [ -z "${PP_SKIP_BUILD:-}" ]; then if [ -z "${PP_SKIP_BUILD:-}" ]; then
echo "docker-e2e: building pp-gpu-node + pp-smoke-run (release)" echo "docker-e2e: building pp-worker + pp-orchestrator (release)"
cargo build --manifest-path "$CRATE_DIR/Cargo.toml" --release \ cargo build --manifest-path "$CRATE_DIR/Cargo.toml" --release \
--bin pp-gpu-node --bin pp-smoke-run --bin pp-worker --bin pp-orchestrator
echo "docker-e2e: building swactor-diag-{collector,postproc} (release, --features collector)" echo "docker-e2e: building swactor-diag-{collector,postproc} (release, --features collector)"
cargo build --manifest-path "$WORKSPACE_DIR/Cargo.toml" --release \ cargo build --manifest-path "$WORKSPACE_DIR/Cargo.toml" --release \
-p distribution --features collector \ -p distribution --features collector \
--bin swactor-diag-collector --bin swactor-diag-postproc --bin swactor-diag-collector --bin swactor-diag-postproc
fi fi
for f in "$SMOKE_RUN_BIN" "$GPU_NODE_BIN" "$WORKER_PY" "$COLLECTOR_BIN" "$POSTPROC_BIN"; do for f in "$ORCHESTRATOR_BIN" "$WORKER_BIN" "$WORKER_PY" "$COLLECTOR_BIN" "$POSTPROC_BIN"; do
[ -f "$f" ] || { echo "docker-e2e: missing $f" >&2; exit 1; } [ -f "$f" ] || { echo "docker-e2e: missing $f" >&2; exit 1; }
done done
@ -117,7 +117,7 @@ cleanup_containers() {
} }
cleanup_containers cleanup_containers
# Step 4: drive pp-smoke-run with the docker shim as its --gpu-node. # Step 4: drive pp-orchestrator with the docker shim as its --gpu-node.
# The shim consults PP_IMAGE / PP_CONTAINER_PREFIX / PP_DEV from its env. # The shim consults PP_IMAGE / PP_CONTAINER_PREFIX / PP_DEV from its env.
OUTPUT_DIR="$(mktemp -d)" OUTPUT_DIR="$(mktemp -d)"
STDOUT_LOG="$OUTPUT_DIR/stdout.log" STDOUT_LOG="$OUTPUT_DIR/stdout.log"
@ -135,7 +135,7 @@ if [ -n "${PP_REAL:-}" ]; then
PP_DEV=CUDA \ PP_DEV=CUDA \
PP_GPUS="${PP_GPUS:-all}" \ PP_GPUS="${PP_GPUS:-all}" \
PP_MODEL_CACHE_DIR="$PP_MODEL_CACHE_DIR" \ PP_MODEL_CACHE_DIR="$PP_MODEL_CACHE_DIR" \
"$SMOKE_RUN_BIN" \ "$ORCHESTRATOR_BIN" \
--seed \ --seed \
--num-stages "$NUM_STAGES" \ --num-stages "$NUM_STAGES" \
--gpu-node "$SCRIPT_DIR/docker-gpu-node.sh" \ --gpu-node "$SCRIPT_DIR/docker-gpu-node.sh" \
@ -148,7 +148,7 @@ else
PP_IMAGE="$IMAGE" \ PP_IMAGE="$IMAGE" \
PP_CONTAINER_PREFIX="$PREFIX" \ PP_CONTAINER_PREFIX="$PREFIX" \
PP_DEV=CPU \ PP_DEV=CPU \
"$SMOKE_RUN_BIN" \ "$ORCHESTRATOR_BIN" \
--seed \ --seed \
--num-stages "$NUM_STAGES" \ --num-stages "$NUM_STAGES" \
--gpu-node "$SCRIPT_DIR/docker-gpu-node.sh" \ --gpu-node "$SCRIPT_DIR/docker-gpu-node.sh" \
@ -157,11 +157,11 @@ else
--max-tokens "$MAX_TOKENS" \ --max-tokens "$MAX_TOKENS" \
>"$STDOUT_LOG" 2>"$STDERR_LOG" >"$STDOUT_LOG" 2>"$STDERR_LOG"
fi fi
SMOKE_STATUS=$? ORCH_STATUS=$?
set -e set -e
if [ $SMOKE_STATUS -ne 0 ]; then if [ $ORCH_STATUS -ne 0 ]; then
echo "docker-e2e: pp-smoke-run exited $SMOKE_STATUS" >&2 echo "docker-e2e: pp-orchestrator exited $ORCH_STATUS" >&2
echo "----- stdout -----" >&2 echo "----- stdout -----" >&2
cat "$STDOUT_LOG" >&2 cat "$STDOUT_LOG" >&2
echo "----- stderr (last 60) -----" >&2 echo "----- stderr (last 60) -----" >&2

10
examples/pipeline-parallel-inference/scripts/docker-gpu-node.sh
View file

@ -1,9 +1,9 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# docker-gpu-node.sh — shim that pp-smoke-run can spawn instead of the # docker-gpu-node.sh — shim that pp-orchestrator can spawn instead of the
# pp-gpu-node binary directly. Boots one pp-gpu-node container per stage # pp-worker binary directly. Boots one pp-worker container per stage
# on the host network so iroh can dial without NAT. # on the host network so iroh can dial without NAT.
# #
# Required env (forwarded by pp-smoke-run): # Required env (forwarded by pp-orchestrator):
# STAGE, NUM_STAGES, SEED_ADDR, SEED_DIRECT, MAX_TOKENS # STAGE, NUM_STAGES, SEED_ADDR, SEED_DIRECT, MAX_TOKENS
# Optional env (forwarded if present): # Optional env (forwarded if present):
# MODEL, PP_WORKER_STUB, WORKER_CMD, # MODEL, PP_WORKER_STUB, WORKER_CMD,
@ -80,10 +80,10 @@ mkdir -p "$CACHE_DIR"
# #
# --entrypoint runs the binary directly, bypassing the image's default # --entrypoint runs the binary directly, bypassing the image's default
# pp_entrypoint.sh (sshd + postmortem hold). That supervisor is for remote # pp_entrypoint.sh (sshd + postmortem hold). That supervisor is for remote
# vast.ai nodes; a local docker stage should exit cleanly when pp-gpu-node # vast.ai nodes; a local docker stage should exit cleanly when pp-worker
# does so --rm reaps it and the E2E's no-leftover-container check holds. # does so --rm reaps it and the E2E's no-leftover-container check holds.
exec docker run --rm --init \ exec docker run --rm --init \
--entrypoint /usr/local/bin/pp-gpu-node \ --entrypoint /usr/local/bin/pp-worker \
--name "$NAME" \ --name "$NAME" \
--network "${PP_DIAG_NETWORK:-host}" \ --network "${PP_DIAG_NETWORK:-host}" \
"${GPU_ARGS[@]}" \ "${GPU_ARGS[@]}" \

179
examples/pipeline-parallel-inference/scripts/remote-collector-fleet.sh Executable file
View file

@ -0,0 +1,179 @@
#!/usr/bin/env bash
# remote-collector-fleet.sh — like demo-fleet.sh, but the diagnostics collector
# lives OFF-box on the real VPS (prod topology) instead of on localhost.
#
# Brings up, on THIS machine, the docker stage fleet you would normally deploy
# (N stub-mode pp-worker containers on --network host) plus pp-orchestrator
# hosting the FULL swactor dashboard locally. Every stage's in-VM monitor ships
# its VastaiSample/VastaiLogs records to the REMOTE collector, and the
# orchestrator's Fleet tab subscribes to that same remote collector's SSE
# stream (/diag/stream/<run_id>) and folds the records — so the Fleet tab is
# exercised end-to-end against the production collector over the public WAN.
#
# The collector is NOT started here; it must already be running on the VPS and
# its port reachable (ufw). This is the "test the prod collector locally" path.
#
# Usage:
# examples/pipeline-parallel-inference/scripts/remote-collector-fleet.sh [N]
#
# Environment overrides:
# PP_COLLECTOR_HOST VPS host running swactor-diag-collector (default 139.59.195.69)
# PP_COLLECTOR_PORT collector HTTP port (default 9080)
# PP_COLLECTOR_UDP collector UDP echo port (default 9081)
# PP_RUN_ID diagnostics run id (default remote-fleet-<epoch>)
# PP_DIAG_IMAGE code image tag (default swactor-pp-gpu:latest)
# PP_BASE_IMAGE base image tag (default swactor-pp-base:cuda12.6)
# PP_SKIP_BUILD skip cargo build (reuse target/)
# PP_SKIP_IMAGE_BUILD skip docker image build (reuse tag)
# PP_DASHBOARD_PORT orchestrator dashboard port (default 9095)
# PP_PROMPT inference prompt (default "remote fleet demo")
# PP_MAX_TOKENS decode token cap (default 4)
# PP_NO_OPEN if set, don't open a browser
set -euo pipefail
NUM_STAGES="${1:-3}"
COLLECTOR_HOST="${PP_COLLECTOR_HOST:-139.59.195.69}"
COLLECTOR_PORT="${PP_COLLECTOR_PORT:-9080}"
COLLECTOR_UDP="${PP_COLLECTOR_UDP:-9081}"
RUN_ID="${PP_RUN_ID:-remote-fleet-$(date +%s)}"
IMAGE="${PP_DIAG_IMAGE:-swactor-pp-gpu:latest}"
BASE_IMAGE="${PP_BASE_IMAGE:-swactor-pp-base:cuda12.6}"
PROMPT="${PP_PROMPT:-remote fleet demo}"
MAX_TOKENS="${PP_MAX_TOKENS:-4}"
DASH_PORT="${PP_DASHBOARD_PORT:-9095}"
CONTAINER_PREFIX="remote-fleet-stage"
COLLECTOR_URL="http://${COLLECTOR_HOST}:${COLLECTOR_PORT}"
DASH_URL="http://127.0.0.1:${DASH_PORT}/"
if ! [[ "$NUM_STAGES" =~ ^[0-9]+$ ]] || [ "$NUM_STAGES" -lt 2 ]; then
echo "remote-fleet: N must be an integer >= 2 (seed mode needs >=2 stages), got '$NUM_STAGES'" >&2
exit 2
fi
if ! command -v docker >/dev/null 2>&1; then echo "remote-fleet: docker not on PATH" >&2; exit 2; fi
if ! docker info >/dev/null 2>&1; then echo "remote-fleet: docker daemon unreachable" >&2; exit 2; fi
if (exec 3<>"/dev/tcp/127.0.0.1/${DASH_PORT}") 2>/dev/null; then
exec 3>&- 3<&-
echo "remote-fleet: dashboard port ${DASH_PORT} in use. Pick another: PP_DASHBOARD_PORT=9096 $0 ${NUM_STAGES}" >&2
exit 2
fi
# Preflight: the remote collector must be reachable, else the Fleet tab and the
# stages' shipping will silently get nothing. Fail loudly here instead.
echo "remote-fleet: checking remote collector at ${COLLECTOR_URL} …"
if ! curl -fsS -m 8 -o /dev/null "${COLLECTOR_URL}/diag/runs"; then
echo "remote-fleet: cannot reach ${COLLECTOR_URL}/diag/runs — is the collector up and the port open (ufw)?" >&2
exit 1
fi
echo "remote-fleet: remote collector reachable."
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
CRATE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
WORKSPACE_DIR="$(cd "$CRATE_DIR/../.." && pwd)"
ORCHESTRATOR_BIN="$CRATE_DIR/target/release/pp-orchestrator"
WORKER_BIN="$CRATE_DIR/target/release/pp-worker"
WORKER_PY="$CRATE_DIR/pp_tinygrad_worker.py"
if [ -z "${PP_SKIP_BUILD:-}" ]; then
echo "remote-fleet: cargo build pp-orchestrator + pp-worker (release)"
cargo build --manifest-path "$CRATE_DIR/Cargo.toml" --release --bin pp-worker --bin pp-orchestrator
fi
for f in "$ORCHESTRATOR_BIN" "$WORKER_BIN" "$WORKER_PY"; do
[ -f "$f" ] || { echo "remote-fleet: missing $f" >&2; exit 1; }
done
if [ -z "${PP_SKIP_IMAGE_BUILD:-}" ]; then
echo "remote-fleet: docker build $BASE_IMAGE (base)"
docker build -f "$CRATE_DIR/Dockerfile.base" -t "$BASE_IMAGE" "$WORKSPACE_DIR"
echo "remote-fleet: docker build $IMAGE (code)"
docker build -f "$CRATE_DIR/Dockerfile" --build-arg "BASE_IMAGE=$BASE_IMAGE" -t "$IMAGE" "$WORKSPACE_DIR"
fi
WORKDIR="$(mktemp -d -t remote-fleet.XXXXXX)"
SPOOL_DIR="$WORKDIR/spool"
ORCH_LOG="$WORKDIR/orch.log"
FIFO="$WORKDIR/orch.stdin"
mkdir -p "$SPOOL_DIR"
mkfifo "$FIFO"
ORCH_PID=""
CLEANED=""
cleanup() {
[ -n "$CLEANED" ] && return 0
CLEANED=1
set +e
echo; echo "remote-fleet: tearing down…"
local ids
ids=$(docker ps -aq --filter "name=^${CONTAINER_PREFIX}-[0-9]+$")
[ -n "$ids" ] && docker rm -f $ids >/dev/null 2>&1
[ -n "$ORCH_PID" ] && kill "$ORCH_PID" >/dev/null 2>&1
exec 3>&- 2>/dev/null
[ -d "$WORKDIR" ] && rm -rf "$WORKDIR"
set -e
echo "remote-fleet: done."
}
trap cleanup EXIT
trap 'exit 130' INT TERM
exec 3<>"$FIFO"
echo "remote-fleet: launching orchestrator + ${NUM_STAGES} stage containers"
echo "remote-fleet: collector = ${COLLECTOR_URL} run_id = ${RUN_ID}"
export PP_HOLD=1
export PP_WORKER_STUB=1
export PP_DEV=CPU
export PP_IMAGE="$IMAGE"
export PP_CONTAINER_PREFIX="$CONTAINER_PREFIX"
export PP_DASHBOARD=1
export PP_DASHBOARD_PORT="$DASH_PORT"
export SWACTOR_DIAG_COLLECTOR_URL="$COLLECTOR_URL"
export SWACTOR_DIAG_RUN_ID="$RUN_ID"
export SWACTOR_DIAG_SPOOL_DIR="$SPOOL_DIR"
export SWACTOR_DIAG_UDP_ECHO="${COLLECTOR_HOST}:${COLLECTOR_UDP}"
"$ORCHESTRATOR_BIN" \
--seed --num-stages "$NUM_STAGES" \
--gpu-node "$SCRIPT_DIR/docker-gpu-node.sh" \
--worker "$WORKER_PY" \
--prompt "$PROMPT" --max-tokens "$MAX_TOKENS" \
<"$FIFO" >"$ORCH_LOG" 2>&1 &
ORCH_PID=$!
WAITED=0
until (echo > "/dev/tcp/127.0.0.1/${DASH_PORT}") >/dev/null 2>&1; do
if ! kill -0 "$ORCH_PID" >/dev/null 2>&1; then
echo "remote-fleet: orchestrator exited before its dashboard came up." >&2
tail -n 40 "$ORCH_LOG" >&2 || true
exit 1
fi
WAITED=$((WAITED + 1))
[ "$WAITED" -ge 30 ] && { echo "remote-fleet: dashboard did not bind :${DASH_PORT} in 30s" >&2; tail -n 40 "$ORCH_LOG" >&2; exit 1; }
sleep 1
done
echo
echo " ┌─────────────────────────────────────────────────────────────┐"
echo " │ Full swactor dashboard: $DASH_URL"
echo " │ Fleet tab pulls from remote collector: ${COLLECTOR_URL}/diag/stream/${RUN_ID}"
echo " │ Remote collector board: ${COLLECTOR_URL}/dashboard?run=${RUN_ID}"
echo " └─────────────────────────────────────────────────────────────┘"
echo
if [ -z "${PP_NO_OPEN:-}" ]; then
if command -v xdg-open >/dev/null 2>&1; then (xdg-open "$DASH_URL" >/dev/null 2>&1 &) || true
elif command -v open >/dev/null 2>&1; then (open "$DASH_URL" >/dev/null 2>&1 &) || true
fi
fi
echo "remote-fleet: waiting for the cluster to converge (first inference drive)…"
WAITED=0
until grep -q "holding cluster open" "$ORCH_LOG" 2>/dev/null; do
if ! kill -0 "$ORCH_PID" >/dev/null 2>&1; then
echo "remote-fleet: orchestrator exited before holding — drive failed." >&2
tail -n 40 "$ORCH_LOG" >&2 || true
exit 1
fi
WAITED=$((WAITED + 1))
[ "$WAITED" -ge 180 ] && { echo "remote-fleet: cluster did not converge within 180s" >&2; tail -n 40 "$ORCH_LOG" >&2; exit 1; }
sleep 1
done
RUNNING=$(docker ps -q --filter "name=^${CONTAINER_PREFIX}-[0-9]+$" | wc -l | tr -d ' ')
echo
echo "remote-fleet: ✅ fleet up — ${RUNNING}/${NUM_STAGES} stage containers shipping to ${COLLECTOR_URL}."
echo "remote-fleet: watch the Fleet tab live at $DASH_URL"
echo "remote-fleet: press Ctrl+C to tear everything down."
echo
wait "$ORCH_PID"

656
examples/pipeline-parallel-inference/src/bin/pp_smoke_run.rs → examples/pipeline-parallel-inference/src/bin/pp_orchestrator.rs
View file

File diff suppressed because it is too large Load diff

74
examples/pipeline-parallel-inference/src/bin/pp_gpu_node.rs → examples/pipeline-parallel-inference/src/bin/pp_worker.rs
View file

@ -1,4 +1,4 @@
//! pp-gpu-node — pipeline-parallel GPU inference node. //! pp-worker — pipeline-parallel GPU inference node.
//! //!
//! Boots one stage of a pipeline-parallel inference run. Reads its //! Boots one stage of a pipeline-parallel inference run. Reads its
//! configuration from the environment (set at instance-create time on //! configuration from the environment (set at instance-create time on
@ -68,7 +68,7 @@ use swactor_process::{ProcessMode, ProcessSpec};
/// SWIM name the orchestrator uses to publish the address of its /// SWIM name the orchestrator uses to publish the address of its
/// `InferenceResponse` inbox. The last stage resolves this name to learn /// `InferenceResponse` inbox. The last stage resolves this name to learn
/// where to send the final response. Defined here (and re-declared in /// where to send the final response. Defined here (and re-declared in
/// `pp-smoke-run`) so the topology module stays test-shaped; the binary /// `pp-orchestrator`) so the topology module stays test-shaped; the binary
/// is the only place that cares about this name. /// is the only place that cares about this name.
const ORCHESTRATOR_NAME: &str = "pp-orchestrator"; const ORCHESTRATOR_NAME: &str = "pp-orchestrator";
@ -88,7 +88,7 @@ fn parse_hex_node_id(s: &str) -> [u8; 32] {
fn require_env(name: &str) -> String { fn require_env(name: &str) -> String {
std::env::var(name) std::env::var(name)
.unwrap_or_else(|_| { .unwrap_or_else(|_| {
eprintln!("pp-gpu-node: env {name} is required"); eprintln!("pp-worker: env {name} is required");
std::process::exit(2); std::process::exit(2);
}) })
.trim() .trim()
@ -98,7 +98,7 @@ fn require_env(name: &str) -> String {
fn require_u32(name: &str) -> u32 { fn require_u32(name: &str) -> u32 {
let raw = require_env(name); let raw = require_env(name);
raw.parse::<u32>().unwrap_or_else(|_| { raw.parse::<u32>().unwrap_or_else(|_| {
eprintln!("pp-gpu-node: env {name}={raw:?} must be a u32"); eprintln!("pp-worker: env {name}={raw:?} must be a u32");
std::process::exit(2); std::process::exit(2);
}) })
} }
@ -118,7 +118,7 @@ fn stage_secret_from_env() -> Option<SecretKey> {
} }
if hex.len() != 64 { if hex.len() != 64 {
eprintln!( eprintln!(
"pp-gpu-node: PP_STAGE_SECRET must be 64 hex chars, got {}", "pp-worker: PP_STAGE_SECRET must be 64 hex chars, got {}",
hex.len() hex.len()
); );
std::process::exit(2); std::process::exit(2);
@ -126,7 +126,7 @@ fn stage_secret_from_env() -> Option<SecretKey> {
let mut bytes = [0u8; 32]; let mut bytes = [0u8; 32];
for (i, b) in bytes.iter_mut().enumerate() { for (i, b) in bytes.iter_mut().enumerate() {
*b = u8::from_str_radix(&hex[i * 2..i * 2 + 2], 16).unwrap_or_else(|_| { *b = u8::from_str_radix(&hex[i * 2..i * 2 + 2], 16).unwrap_or_else(|_| {
eprintln!("pp-gpu-node: PP_STAGE_SECRET is not valid hex"); eprintln!("pp-worker: PP_STAGE_SECRET is not valid hex");
std::process::exit(2); std::process::exit(2);
}); });
} }
@ -266,7 +266,7 @@ fn build_route(
fn register_name(driver: &mut IrohDriver, name: &str, addr: ActorAddress, stage: u32) { fn register_name(driver: &mut IrohDriver, name: &str, addr: ActorAddress, stage: u32) {
driver.node_mut().register_name(name.into(), addr); driver.node_mut().register_name(name.into(), addr);
diag::emit_register_name(driver, name, addr, Some(stage)); diag::emit_register_name(driver, name, addr, Some(stage));
eprintln!("pp-gpu-node: registered {name} -> {addr:?}"); eprintln!("pp-worker: registered {name} -> {addr:?}");
} }
fn resolve_or_die( fn resolve_or_die(
@ -274,10 +274,10 @@ fn resolve_or_die(
name: &str, name: &str,
timeout: Duration, timeout: Duration,
) -> (ActorAddress, String) { ) -> (ActorAddress, String) {
eprintln!("pp-gpu-node: resolving {name}..."); eprintln!("pp-worker: resolving {name}...");
resolve_name(driver, name, timeout).unwrap_or_else(|| { resolve_name(driver, name, timeout).unwrap_or_else(|| {
eprintln!( eprintln!(
"pp-gpu-node: failed to resolve {name} in {:.0}s", "pp-worker: failed to resolve {name} in {:.0}s",
timeout.as_secs_f32() timeout.as_secs_f32()
); );
std::process::exit(1); std::process::exit(1);
@ -294,16 +294,16 @@ fn add_route_or_die(
match build_route(driver, node_hex) { match build_route(driver, node_hex) {
Ok(t) => router.add_route(addr, t), Ok(t) => router.add_route(addr, t),
Err(e) => { Err(e) => {
eprintln!("pp-gpu-node: route to {label} failed: {e}"); eprintln!("pp-worker: route to {label} failed: {e}");
std::process::exit(1); std::process::exit(1);
} }
} }
} }
/// Ask the kernel to deliver `SIGTERM` to this process when its parent dies. /// Ask the kernel to deliver `SIGTERM` to this process when its parent dies.
/// Without this, a `SIGKILL` to `pp-smoke-run` would orphan its children to /// Without this, a `SIGKILL` to `pp-orchestrator` would orphan its children to
/// pid 1 and leave them running — the orchestrator's `ChainGuard::drop` runs /// pid 1 and leave them running — the orchestrator's `ChainGuard::drop` runs
/// only on graceful exit. With it, each `pp-gpu-node` dies seconds after its /// only on graceful exit. With it, each `pp-worker` dies seconds after its
/// orchestrator does, which is the `binary_e2e_orchestrator_sigkilled_*` /// orchestrator does, which is the `binary_e2e_orchestrator_sigkilled_*`
/// contract from TEST_SPEC §13.2. Linux-only; other platforms are no-ops. /// contract from TEST_SPEC §13.2. Linux-only; other platforms are no-ops.
#[cfg(target_os = "linux")] #[cfg(target_os = "linux")]
@ -322,7 +322,7 @@ fn install_parent_death_signal() {}
/// Set by the `SIGHUP` handler; polled by the pump loops to drive an /// Set by the `SIGHUP` handler; polled by the pump loops to drive an
/// in-place worker hot-reload (re-exec the on-disk worker script). An /// in-place worker hot-reload (re-exec the on-disk worker script). An
/// operator pushes a new `pp_tinygrad_worker.py` over the running one and /// operator pushes a new `pp_tinygrad_worker.py` over the running one and
/// `kill -HUP $(pidof pp-gpu-node)` to pick it up without re-leasing. /// `kill -HUP $(pidof pp-worker)` to pick it up without re-leasing.
static RELOAD_REQUESTED: AtomicBool = AtomicBool::new(false); static RELOAD_REQUESTED: AtomicBool = AtomicBool::new(false);
#[cfg(target_os = "linux")] #[cfg(target_os = "linux")]
@ -352,7 +352,7 @@ fn install_sighup_handler() {}
/// pump so both honour reloads with the same latency. /// pump so both honour reloads with the same latency.
fn drain_reload_request(rt: &Runtime, stage_actor_addr: ActorAddress) { fn drain_reload_request(rt: &Runtime, stage_actor_addr: ActorAddress) {
if RELOAD_REQUESTED.swap(false, Ordering::SeqCst) { if RELOAD_REQUESTED.swap(false, Ordering::SeqCst) {
eprintln!("pp-gpu-node: SIGHUP — reloading worker"); eprintln!("pp-worker: SIGHUP — reloading worker");
let _ = rt.send_to(stage_actor_addr, StageMsg::ReloadWorker); let _ = rt.send_to(stage_actor_addr, StageMsg::ReloadWorker);
} }
} }
@ -366,7 +366,7 @@ fn maybe_simulate_boot_delay(stage: u32) {
let secs = std::env::var("PP_BOOT_DELAY_SECS").ok().and_then(|s| s.trim().parse::<u64>().ok()); let secs = std::env::var("PP_BOOT_DELAY_SECS").ok().and_then(|s| s.trim().parse::<u64>().ok());
if let (Some(target), Some(secs)) = (target, secs) { if let (Some(target), Some(secs)) = (target, secs) {
if target == stage && secs > 0 { if target == stage && secs > 0 {
eprintln!("pp-gpu-node: simulated boot delay of {secs}s on stage {stage}"); eprintln!("pp-worker: simulated boot delay of {secs}s on stage {stage}");
std::thread::sleep(Duration::from_secs(secs)); std::thread::sleep(Duration::from_secs(secs));
} }
} }
@ -378,7 +378,7 @@ fn main() {
let num_stages = require_u32("NUM_STAGES"); let num_stages = require_u32("NUM_STAGES");
if num_stages < 2 || stage >= num_stages { if num_stages < 2 || stage >= num_stages {
eprintln!( eprintln!(
"pp-gpu-node: invalid STAGE={stage} for NUM_STAGES={num_stages} \ "pp-worker: invalid STAGE={stage} for NUM_STAGES={num_stages} \
(need NUM_STAGES >= 2 and STAGE < NUM_STAGES; N=1 is not supported)" (need NUM_STAGES >= 2 and STAGE < NUM_STAGES; N=1 is not supported)"
); );
std::process::exit(2); std::process::exit(2);
@ -441,7 +441,7 @@ fn main() {
let vastai_forwarder = _vastai_in_vm.as_ref().map(|m| m.forwarder()); let vastai_forwarder = _vastai_in_vm.as_ref().map(|m| m.forwarder());
// Stamp the bundle the moment this process announces itself, so a // Stamp the bundle the moment this process announces itself, so a
// bundle reader can tell two pp-gpu-node incarnations of the same // bundle reader can tell two pp-worker incarnations of the same
// stage apart: a manual binary swap (the operator runbook) pkills the // stage apart: a manual binary swap (the operator runbook) pkills the
// old process and setsid's a new one under the same PID-1 env, which // old process and setsid's a new one under the same PID-1 env, which
// means the same run_id + node_id, but the pid differs. The event carries that // means the same run_id + node_id, but the pid differs. The event carries that
@ -468,7 +468,7 @@ fn main() {
.map(|sa| sa.to_string()) .map(|sa| sa.to_string())
.collect(); .collect();
eprintln!( eprintln!(
"pp-gpu-node: stage {stage}/{num_stages} ({role:?}) started (node_id: {my_hex})" "pp-worker: stage {stage}/{num_stages} ({role:?}) started (node_id: {my_hex})"
); );
// PP_GPU_NODE_ADDR is printed to stdout (flushed) so a parent process // PP_GPU_NODE_ADDR is printed to stdout (flushed) so a parent process
// capturing this child's stdout can extract our addressing. The orchestrator // capturing this child's stdout can extract our addressing. The orchestrator
@ -485,7 +485,7 @@ fn main() {
let mut seed_addr = iroh::EndpointAddr::from(seed_key); let mut seed_addr = iroh::EndpointAddr::from(seed_key);
if let Some(relay) = seed_relay_env.as_deref() { if let Some(relay) = seed_relay_env.as_deref() {
if let Ok(relay_url) = relay.trim().parse::<iroh::RelayUrl>() { if let Ok(relay_url) = relay.trim().parse::<iroh::RelayUrl>() {
eprintln!("pp-gpu-node: using seed relay {relay}"); eprintln!("pp-worker: using seed relay {relay}");
seed_addr = seed_addr.with_relay_url(relay_url); seed_addr = seed_addr.with_relay_url(relay_url);
} }
} }
@ -525,7 +525,7 @@ fn main() {
peer_addr = peer_addr.with_ip_addr(sa); peer_addr = peer_addr.with_ip_addr(sa);
} }
} }
eprintln!("pp-gpu-node: also joining {label} {peer_hex}"); eprintln!("pp-worker: also joining {label} {peer_hex}");
targets.push(peer_addr); targets.push(peer_addr);
} }
}; };
@ -545,7 +545,7 @@ fn main() {
"first-stage peer", "first-stage peer",
); );
eprintln!("pp-gpu-node: joining seed {seed_hex}"); eprintln!("pp-worker: joining seed {seed_hex}");
driver.join(&join_targets); driver.join(&join_targets);
// If we ended up with a relay (vast.ai / WAN), publish it via SWIM // If we ended up with a relay (vast.ai / WAN), publish it via SWIM
@ -556,7 +556,7 @@ fn main() {
// the autoregressive feedback edge (last → first) because SWIM has // the autoregressive feedback edge (last → first) because SWIM has
// not yet probed that specific pair. // not yet probed that specific pair.
if let Some(home) = driver.home_relay_url() { if let Some(home) = driver.home_relay_url() {
eprintln!("pp-gpu-node: publishing home relay {home} to SWIM gossip"); eprintln!("pp-worker: publishing home relay {home} to SWIM gossip");
driver.node_mut().set_relay_url(Some(home.to_string())); driver.node_mut().set_relay_url(Some(home.to_string()));
} }
@ -578,10 +578,10 @@ fn main() {
.and_then(|s| s.trim().parse().ok()) .and_then(|s| s.trim().parse().ok())
.unwrap_or(1200); .unwrap_or(1200);
if !wait_for_cluster(&mut driver, Duration::from_secs(converge_secs)) { if !wait_for_cluster(&mut driver, Duration::from_secs(converge_secs)) {
eprintln!("pp-gpu-node: cluster did not converge in {converge_secs}s"); eprintln!("pp-worker: cluster did not converge in {converge_secs}s");
std::process::exit(1); std::process::exit(1);
} }
eprintln!("pp-gpu-node: cluster converged"); eprintln!("pp-worker: cluster converged");
// Create the actor runtime, codec registry, and transport router. // Create the actor runtime, codec registry, and transport router.
let mut rt = Runtime::new(RuntimeConfig::default()); let mut rt = Runtime::new(RuntimeConfig::default());
@ -617,7 +617,7 @@ fn main() {
if let Some((handle, collector, port)) = &stage_dash { if let Some((handle, collector, port)) = &stage_dash {
handle.set_runtime(Arc::clone(&rt), Arc::clone(collector)); handle.set_runtime(Arc::clone(&rt), Arc::clone(collector));
handle.start_http(driver.tokio_handle()); handle.start_http(driver.tokio_handle());
eprintln!("pp-gpu-node: stage {stage} dashboard on http://localhost:{port}"); eprintln!("pp-worker: stage {stage} dashboard on http://localhost:{port}");
} }
run_stage( run_stage(
@ -655,17 +655,17 @@ fn hold_until_worker_ready(
if let Some(status) = status_inbox.try_recv() { if let Some(status) = status_inbox.try_recv() {
match status { match status {
StageActorStatus::WorkerReady { pid } => { StageActorStatus::WorkerReady { pid } => {
eprintln!("pp-gpu-node: worker ready (pid: {pid:?})"); eprintln!("pp-worker: worker ready (pid: {pid:?})");
return; return;
} }
StageActorStatus::ProcessStarted => { StageActorStatus::ProcessStarted => {
eprintln!("pp-gpu-node: worker process started"); eprintln!("pp-worker: worker process started");
} }
StageActorStatus::ProcessExited { status } => { StageActorStatus::ProcessExited { status } => {
eprintln!( eprintln!(
"pp-gpu-node: stage-{stage} worker exited during startup: \ "pp-worker: stage-{stage} worker exited during startup: \
{status:?}; holding (SWIM alive) — push a fixed worker.py \ {status:?}; holding (SWIM alive) — push a fixed worker.py \
and `kill -HUP $(pidof pp-gpu-node)` to reload" and `kill -HUP $(pidof pp-worker)` to reload"
); );
last_warn = Instant::now(); last_warn = Instant::now();
} }
@ -674,7 +674,7 @@ fn hold_until_worker_ready(
if last_warn.elapsed() >= warn_after { if last_warn.elapsed() >= warn_after {
eprintln!( eprintln!(
"pp-gpu-node: stage-{stage} worker still not ready after {}s; \ "pp-worker: stage-{stage} worker still not ready after {}s; \
holding — SIGHUP to reload the worker script", holding — SIGHUP to reload the worker script",
warn_after.as_secs() warn_after.as_secs()
); );
@ -881,7 +881,7 @@ fn run_stage(
let (next_addr, next_hex) = let (next_addr, next_hex) =
resolve_or_die(&mut driver, &next_name, neighbor_resolve_timeout); resolve_or_die(&mut driver, &next_name, neighbor_resolve_timeout);
eprintln!( eprintln!(
"pp-gpu-node: resolved {next_name} -> {next_addr:?} on {next_hex}" "pp-worker: resolved {next_name} -> {next_addr:?} on {next_hex}"
); );
add_route_or_die(&driver, &router, next_addr, &next_hex, &next_name); add_route_or_die(&driver, &router, next_addr, &next_hex, &next_name);
rt.send_to( rt.send_to(
@ -900,7 +900,7 @@ fn run_stage(
let (next_addr, next_hex) = let (next_addr, next_hex) =
resolve_or_die(&mut driver, &next_name, neighbor_resolve_timeout); resolve_or_die(&mut driver, &next_name, neighbor_resolve_timeout);
eprintln!( eprintln!(
"pp-gpu-node: resolved {next_name} -> {next_addr:?} on {next_hex}" "pp-worker: resolved {next_name} -> {next_addr:?} on {next_hex}"
); );
add_route_or_die(&driver, &router, next_addr, &next_hex, &next_name); add_route_or_die(&driver, &router, next_addr, &next_hex, &next_name);
rt.send_to( rt.send_to(
@ -927,7 +927,7 @@ fn run_stage(
let (orch_addr, orch_hex) = let (orch_addr, orch_hex) =
resolve_or_die(&mut driver, ORCHESTRATOR_NAME, neighbor_resolve_timeout); resolve_or_die(&mut driver, ORCHESTRATOR_NAME, neighbor_resolve_timeout);
eprintln!( eprintln!(
"pp-gpu-node: resolved {feedback_name}={feedback_addr:?} on \ "pp-worker: resolved {feedback_name}={feedback_addr:?} on \
{feedback_hex}, orch={orch_addr:?} on {orch_hex}" {feedback_hex}, orch={orch_addr:?} on {orch_hex}"
); );
add_route_or_die( add_route_or_die(
@ -978,7 +978,7 @@ fn main_pump(
msg_pump: ActorMessagePump, msg_pump: ActorMessagePump,
stage_actor_addr: ActorAddress, stage_actor_addr: ActorAddress,
) { ) {
eprintln!("pp-gpu-node: entering main pump loop"); eprintln!("pp-worker: entering main pump loop");
loop { loop {
driver.recv(); driver.recv();
driver.tick(); driver.tick();
@ -989,10 +989,10 @@ fn main_pump(
if let Some(status) = status_inbox.try_recv() { if let Some(status) = status_inbox.try_recv() {
match status { match status {
StageActorStatus::ProcessExited { status } => { StageActorStatus::ProcessExited { status } => {
eprintln!("pp-gpu-node: worker exited: {status:?}"); eprintln!("pp-worker: worker exited: {status:?}");
eprintln!("pp-gpu-node: keeping SWIM alive for diagnostics"); eprintln!("pp-worker: keeping SWIM alive for diagnostics");
} }
other => eprintln!("pp-gpu-node: status: {other:?}"), other => eprintln!("pp-worker: status: {other:?}"),
} }
} }
std::thread::sleep(Duration::from_millis(20)); std::thread::sleep(Duration::from_millis(20));

4
examples/pipeline-parallel-inference/src/diag.rs
View file

@ -1,5 +1,5 @@
//! Wire `crates/distribution` diagnostics into `pp-smoke-run` and //! Wire `crates/distribution` diagnostics into `pp-orchestrator` and
//! `pp-gpu-node` from environment variables. //! `pp-worker` from environment variables.
//! //!
//! Reading `SWACTOR_DIAG_COLLECTOR_URL` is the opt-in switch. When it is //! Reading `SWACTOR_DIAG_COLLECTOR_URL` is the opt-in switch. When it is
//! unset (or empty) `install_from_env` returns `None` and the binary //! unset (or empty) `install_from_env` returns `None` and the binary

29
examples/pipeline-parallel-inference/src/dist_page.html
View file

@ -7,10 +7,14 @@
<style> <style>
* { margin: 0; padding: 0; box-sizing: border-box; } * { margin: 0; padding: 0; box-sizing: border-box; }
body { font-family: 'Menlo','Consolas','Monaco',monospace; background:#0f1117; color:#e0e0e0; font-size:13px; } body { font-family: 'Menlo','Consolas','Monaco',monospace; background:#0f1117; color:#e0e0e0; font-size:13px; }
.header { display:flex; align-items:center; gap:16px; padding:10px 16px; background:#161922; border-bottom:1px solid #262b38; } .header { display:flex; align-items:center; justify-content:space-between; padding:12px 20px; background:#161822; border-bottom:1px solid #2a2d3e; }
.header h1 { font-size:15px; font-weight:600; color:#7aa2f7; } .header h1 { font-size:16px; font-weight:600; color:#fff; }
.nav a { color:#9aa5b1; text-decoration:none; margin-right:14px; } .header-left { display:flex; align-items:center; }
.nav a:hover, .nav a.active { color:#7aa2f7; } .nav-links { display:flex; gap:4px; margin-left:20px; }
.nav-link { color:#888; text-decoration:none; font-size:12px; padding:4px 10px; border-radius:3px; transition:color 0.2s; }
.nav-link:hover { color:#e0e0e0; }
.nav-link.active { color:#fff; background:#2a2d3e; }
.header-right { display:flex; align-items:center; gap:12px; }
.wrap { padding:16px; display:grid; grid-template-columns: 1fr 1fr; gap:16px; } .wrap { padding:16px; display:grid; grid-template-columns: 1fr 1fr; gap:16px; }
.card { background:#161922; border:1px solid #262b38; border-radius:6px; padding:12px; } .card { background:#161922; border:1px solid #262b38; border-radius:6px; padding:12px; }
.card h2 { font-size:13px; color:#c0caf5; margin-bottom:10px; border-bottom:1px solid #262b38; padding-bottom:6px; } .card h2 { font-size:13px; color:#c0caf5; margin-bottom:10px; border-bottom:1px solid #262b38; padding-bottom:6px; }
@ -33,13 +37,18 @@
</head> </head>
<body> <body>
<div class="header"> <div class="header">
<h1>Orchestrator · Distribution</h1> <div class="header-left">
<div class="nav"> <h1>Swactor Runtime Dashboard</h1>
<a href="/">Overview</a> <nav class="nav-links">
<a href="/actors">Actors</a> <a href="/" class="nav-link">Overview</a>
<a href="/plugin/distribution" class="active">Distribution</a> <a href="/actors" class="nav-link">Actors</a>
<a href="/topology" class="nav-link">Topology</a>
<a href="/plugin/distribution" class="nav-link active">Distribution</a>
<a href="/plugin/netmap" class="nav-link">Net map</a>
<a href="/plugin/vastai" class="nav-link">Fleet</a>
</nav>
</div> </div>
<span id="node" class="mono" style="margin-left:auto"></span> <div class="header-right"><span id="node" class="mono"></span></div>
</div> </div>
<div class="wrap"> <div class="wrap">

206
examples/pipeline-parallel-inference/src/fleet_page.html Normal file
View file

@ -0,0 +1,206 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Swactor Runtime Dashboard — Fleet</title>
<style>
/* Reuses the Swactor Runtime Dashboard layout/CSS (panels + stat-cards). */
* { margin: 0; padding: 0; box-sizing: border-box; }
body { font-family: 'Menlo', 'Consolas', 'Monaco', monospace; background: #0f1117; color: #e0e0e0; font-size: 13px; }
.header {
display: flex; align-items: center; justify-content: space-between;
padding: 12px 20px; background: #161822; border-bottom: 1px solid #2a2d3e;
}
.header h1 { font-size: 16px; font-weight: 600; color: #fff; }
.header-left { display: flex; align-items: center; }
.nav-links { display: flex; gap: 4px; margin-left: 20px; }
.nav-link {
color: #888; text-decoration: none; font-size: 12px;
padding: 4px 10px; border-radius: 3px; transition: color 0.2s;
}
.nav-link:hover { color: #e0e0e0; }
.nav-link.active { color: #fff; background: #2a2d3e; }
.header-right { display: flex; align-items: center; gap: 12px; }
.mono { color: #9aa5b1; }
.muted { color: #5c6370; font-style: italic; }
.grid { display: grid; grid-template-columns: 1fr; gap: 12px; padding: 12px; }
.panel {
background: #161822; border: 1px solid #2a2d3e; border-radius: 6px;
padding: 14px; overflow: hidden;
}
.panel h2 { font-size: 12px; color: #888; text-transform: uppercase; letter-spacing: 1px; margin-bottom: 10px; }
.stats-cards { display: grid; grid-template-columns: repeat(4, 1fr); gap: 10px; }
.stat-card { background: #1c1f2e; border-radius: 4px; padding: 10px; text-align: center; }
.stat-card .value { font-size: 22px; font-weight: 700; color: #fff; }
.stat-card .label { font-size: 10px; color: #888; text-transform: uppercase; margin-top: 2px; }
/* Per-stage panels laid out in a responsive grid. */
.stages-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(320px, 1fr)); gap: 12px; }
.stage-head { display: flex; align-items: center; justify-content: space-between; margin-bottom: 4px; }
.stage-head .name { font-size: 14px; font-weight: 700; color: #fff; }
.stage-sub { font-size: 11px; margin-bottom: 10px; }
.stage .stats-cards { grid-template-columns: repeat(3, 1fr); gap: 8px; }
.stage .stat-card .value { font-size: 18px; }
.pill { display: inline-flex; align-items: center; gap: 5px; padding: 2px 9px; border-radius: 20px; font-size: 11px; font-weight: 600; background: #0f1117; }
.pill .dot { width: 7px; height: 7px; border-radius: 50%; }
.statusmsg { margin-top: 9px; font-size: 11px; color: #fbbf24; min-height: 0; }
.statusmsg.err { color: #f87171; }
.log-wrap { max-height: 320px; overflow-y: auto; }
.log-wrap table { width: 100%; border-collapse: collapse; }
.log-wrap td { padding: 3px 8px; text-align: left; border-bottom: 1px solid #1c1f2e; font-size: 11px; vertical-align: top; }
.log-wrap td.t { color: #5c6370; white-space: nowrap; }
.log-wrap td.s { font-weight: 600; white-space: nowrap; }
.log-wrap td.msg { white-space: pre-wrap; word-break: break-word; }
.log-wrap tr.stderr td.msg { color: #f87171; }
::-webkit-scrollbar { width: 6px; }
::-webkit-scrollbar-track { background: #0f1117; }
::-webkit-scrollbar-thumb { background: #2a2d3e; border-radius: 3px; }
</style>
</head>
<body>
<div class="header">
<div class="header-left">
<h1>Swactor Runtime Dashboard</h1>
<nav class="nav-links">
<a href="/" class="nav-link">Overview</a>
<a href="/actors" class="nav-link">Actors</a>
<a href="/topology" class="nav-link">Topology</a>
<a href="/plugin/distribution" class="nav-link">Distribution</a>
<a href="/plugin/netmap" class="nav-link">Net map</a>
<a href="/plugin/vastai" class="nav-link active">Fleet</a>
</nav>
</div>
<div class="header-right"><span id="run" class="mono"></span></div>
</div>
<div class="grid">
<div class="panel">
<h2>Fleet</h2>
<div class="stats-cards">
<div class="stat-card"><div class="value" id="s-running">—</div><div class="label">Running</div></div>
<div class="stat-card"><div class="value" id="s-gpu">—</div><div class="label">Fleet GPU</div></div>
<div class="stat-card"><div class="value" id="s-cost">—</div><div class="label">Spend</div></div>
<div class="stat-card"><div class="value" id="s-stages">—</div><div class="label">Stages</div></div>
</div>
</div>
<div class="panel">
<h2>Stages</h2>
<div class="stages-grid" id="stages"><div class="muted">waiting for vast.ai records…</div></div>
</div>
<div class="panel">
<h2>Logs</h2>
<div class="log-wrap"><table><tbody id="logs"></tbody></table></div>
</div>
</div>
<script>
const $ = (id) => document.getElementById(id);
const STAGE_COLORS = ["#4a90e2","#f06292","#ffb74d","#81c784","#ba68c8","#4dd0e1","#aed581","#ff8a65"];
function stageColor(idx){ return STAGE_COLORS[idx % STAGE_COLORS.length]; }
function statusInfo(status){
switch(status){
case "running": return {color:"#4ade80",label:"running",err:false};
case "loading": return {color:"#fbbf24",label:"loading",err:false};
case "created": return {color:"#fbbf24",label:"created",err:false};
case "offline": return {color:"#9ca3af",label:"offline",err:true};
case "exited": return {color:"#f87171",label:"exited",err:true};
default: return {color:"#6b7280",label:status||"—",err:false};
}
}
function fmtClock(ms){ const s=Math.floor(ms/1000),m=Math.floor(s/60); return `${String(m).padStart(2,"0")}:${String(s%60).padStart(2,"0")}`; }
function fmtBytesRate(bps){ if(bps==null)return "—"; const u=["B/s","KB/s","MB/s","GB/s"]; let i=0,v=bps; while(v>=1024&&i<u.length-1){v/=1024;i++;} return `${v.toFixed(v<10?1:0)} ${u[i]}`; }
function fmtUSD(v){ if(v==null)return "—"; return `$${v.toFixed(v<1?4:2)}`; }
function escapeHtml(s){ return s.replace(/[&<>]/g,(c)=>({"&":"&amp;","<":"&lt;",">":"&gt;"}[c])); }
// View-time selectors over the already-folded model (always at the latest time).
function stageStateAt(stage, t){
const lastAtOrBefore=(arr)=>{ let r=null; for(const x of arr){ if(x.at<=t) r=x; else break; } return r; };
const inst=lastAtOrBefore(stage.instances);
const samp=lastAtOrBefore(stage.samples);
let activeContract=null;
for(const c of (stage.contractList||[])){ if((c.leasedAt??0)<=t && (c.endAt==null||c.endAt>t)) activeContract=c; }
return {inst,samp,activeContract};
}
function sumCost(s,t){ const byC=new Map(); for(const o of s.instances){ if(o.at<=t && o.cost!=null) byC.set(o.contractId,o.cost); } let c=0; for(const v of byC.values()) c+=v; return c; }
function card(label,val){ return `<div class="stat-card"><div class="value">${val}</div><div class="label">${label}</div></div>`; }
let model=null;
function renderStages(){
const t = model.tEnd || 0;
let running=0,gpuSum=0,gpuN=0,costSum=0;
const panels=model.stages.map((s)=>{
const {inst,samp,activeContract}=stageStateAt(s,t);
const status=inst?inst.status:(activeContract?"loading":"—");
const si=statusInfo(status);
if(si.label==="running") running++;
const util=samp?samp.util:(inst?inst.gpuUtil:null);
const temp=samp?samp.temp:(inst?inst.gpuTemp:null);
const vram=samp?samp.vram:null;
const vramTot=samp?samp.vramTotal:(inst&&inst.gpuRam?inst.gpuRam*1024:null);
const cpu=samp?samp.cpu:(inst?inst.cpuUtil:null);
if(si.label==="running"&&util!=null){gpuSum+=util;gpuN++;}
const cost=sumCost(s,t); costSum+=cost;
const vramTxt=(vram!=null&&vramTot)?(vram/1024).toFixed(1)+"/"+(vramTot/1024).toFixed(0)+"G":(vram!=null?(vram/1024).toFixed(1)+"G":"—");
return `<div class="panel stage">
<div class="stage-head">
<span class="name" style="color:${s.color}">${s.label}</span>
<span class="pill"><span class="dot" style="background:${si.color}"></span>${si.label}</span>
</div>
<div class="stage-sub muted">${s.gpuName||"CPU"}${s.geo?" · "+s.geo:""} · ${fmtUSD(cost)} @ ${fmtUSD(s.dph)}/h</div>
<div class="stats-cards">
${card("GPU", util!=null?util.toFixed(0)+"%":"—")}
${card("VRAM", vramTxt)}
${card("Temp", temp!=null?temp.toFixed(0)+"°C":"—")}
${card("CPU", cpu!=null?cpu.toFixed(0)+"%":"—")}
${card("Power", samp&&samp.power!=null?samp.power.toFixed(0)+"W":"—")}
${card("Net↓", samp?fmtBytesRate(samp.netRx):"—")}
</div>
<div class="statusmsg ${si.err?"err":""}">${inst&&inst.statusMsg?escapeHtml(inst.statusMsg):""}</div>
</div>`;
});
$("stages").innerHTML=panels.join("")||'<div class="muted">no stages yet</div>';
$("s-running").textContent=`${running}/${model.stages.length}`;
$("s-stages").textContent=model.stages.length;
$("s-cost").textContent=fmtUSD(costSum);
$("s-gpu").textContent=gpuN?(gpuSum/gpuN).toFixed(0)+"%":"—";
}
function renderLogs(){
const t = model.tEnd || 0;
const tail=(model.allLogs||[]).filter(l=>l.at<=t).slice(-80);
$("logs").innerHTML=tail.map((l)=>
`<tr class="${l.stream==="stderr"?"stderr":""}">
<td class="t">${fmtClock(l.at)}</td>
<td class="s" style="color:${stageColor(l.stage)}">stage-${l.stage}</td>
<td class="msg">${escapeHtml(l.text)}</td>
</tr>`).join("");
}
function onFleetModel(m){
model=m;
$("run").textContent=(m.meta&&m.meta.run_id)?("run "+m.meta.run_id):"";
renderStages();
renderLogs();
}
// ── Data source ──────────────────────────────────────────────────────────────
// The orchestrator's RemoteVastaiPlugin re-serves the collector's folded fleet
// model. Seed from the one-shot API, then stream live updates over SSE (`vastai`).
fetch('/api/plugin/vastai/model').then(r=>r.json()).then(function(m){ if(m) onFleetModel(m); }).catch(function(){});
var es=new EventSource('/events');
es.addEventListener('vastai', function(e){ try{ onFleetModel(JSON.parse(e.data)); }catch(err){} });
es.addEventListener('done', function(){ es.close(); });
window.addEventListener('beforeunload', function(){ es.close(); });
</script>
</body>
</html>

159
examples/pipeline-parallel-inference/src/fleet_plugin.rs
View file

@ -1,71 +1,135 @@
//! Orchestrator "fleet" dashboard plugin — a remote-sourced vast.ai view. //! Orchestrator "fleet" dashboard plugin — a remote-sourced vast.ai view.
//! //!
//! In production the diagnostics collector runs on a VPS: the stage containers //! In production the diagnostics collector runs on a VPS: the stage containers
//! ship their vast.ai/host-metric records to it, and it folds them into a fleet //! ship their vast.ai/host-metric records to it. The orchestrator runs locally
//! board model. The orchestrator runs locally and hosts the *full* swactor //! and hosts the *full* swactor dashboard (overview / actors / topology /
//! dashboard (overview / actors / topology / distribution / netmap). To surface //! distribution / netmap). To surface the fleet alongside those live views, this
//! the fleet alongside those live views, this plugin **pulls** the collector's //! plugin **subscribes** to the collector's raw record stream
//! already-folded model (`GET {collector}/api/plugin/vastai/model`) ~1/s and //! (`GET {collector}/diag/stream/{run_id}`, an SSE feed of `LiveRecord`s) and
//! re-serves it verbatim under the same `"vastai"` name — so the Fleet page //! folds the vast.ai records locally — reusing the dashboard's own server-side
//! renders identically to the standalone collector board, with no extra ingest. //! [`VastaiLivePlugin`] fold — then serves the result same-origin at
//! `/api/plugin/vastai/model` and over `/events` (the `vastai` event), so the
//! Fleet page renders without any cross-origin calls to the collector.
//! //!
//! It is the read mirror of the orchestrator's distribution broadcaster //! It is the read mirror of the orchestrator's distribution broadcaster
//! ([`crate::dist_broadcast`]), which *pushes* its snapshot to the same //! ([`crate::dist_broadcast`]), which *pushes* its snapshot to the same
//! collector. The poll loop tolerates an unreachable collector: a failed fetch //! collector. The stream loop reconnects on drops and tolerates an unreachable
//! leaves the last good model in place, so a transient blip never blanks the UI. //! collector: until records arrive, the Fleet tab simply waits.
use std::collections::HashMap; use std::collections::HashMap;
use std::sync::{Arc, Mutex}; use std::sync::Arc;
use std::time::Duration; use std::time::Duration;
use dashboard::live_collector::VastaiLivePlugin;
use dashboard::plugin::{DashboardPlugin, PluginResponse}; use dashboard::plugin::{DashboardPlugin, PluginResponse};
use distribution::diagnostics::collector::protocol::{LiveRecord, RecordKind};
use futures_util::StreamExt;
/// The Fleet page. Reuses the standalone collector board's Fleet renderer but /// Rebuild a [`LiveRecord`] from one stream frame's `data:` JSON. `LiveRecord` is
/// with the orchestrator dashboard's nav, so it sits beside the live tabs. /// a serialize-only wire type, so we parse the fields by hand. Returns `None` for
/// non-vast.ai kinds (e.g. the pushed dist snapshot) so we skip them cheaply.
fn parse_vastai_record(data: &str) -> Option<LiveRecord> {
let v: serde_json::Value = serde_json::from_str(data).ok()?;
let kind = RecordKind::parse(v.get("kind")?.as_str()?)?;
if !kind.is_vastai() {
return None;
}
Some(LiveRecord {
run_id: v.get("run_id").and_then(|x| x.as_str()).unwrap_or("").to_string(),
node_id: v.get("node_id").and_then(|x| x.as_str()).unwrap_or("").to_string(),
kind,
recv_ms: v.get("recv_ms").and_then(|x| x.as_u64()).unwrap_or(0),
seq: v.get("seq").and_then(|x| x.as_u64()).unwrap_or(0),
body: v.get("body").cloned().unwrap_or(serde_json::Value::Null),
})
}
/// The Fleet page. Reuses the Swactor Runtime Dashboard layout/CSS so it sits
/// beside the live tabs (no scrubber / Live control — this is a live-only view).
const FLEET_HTML: &str = include_str!("fleet_page.html"); const FLEET_HTML: &str = include_str!("fleet_page.html");
/// Read-only dashboard plugin backing `/plugin/vastai`. Holds the last good fleet /// Dashboard plugin (name `"vastai"`) backing `/plugin/vastai`. Wraps the
/// model JSON pulled from the remote collector; serves it to the SSE stream /// dashboard's [`VastaiLivePlugin`] (which buffers records and serves the fold)
/// (event `vastai`) and the seed API (`/api/plugin/vastai/model`). /// and feeds it from the remote collector's record stream; overrides only the
/// HTML page so the Fleet tab wears the dashboard chrome.
pub struct RemoteVastaiPlugin { pub struct RemoteVastaiPlugin {
/// Last successfully fetched fleet model JSON (the collector's folded board). inner: Arc<VastaiLivePlugin>,
cached: Arc<Mutex<Option<String>>>,
} }
impl RemoteVastaiPlugin { impl RemoteVastaiPlugin {
pub fn new() -> Self { pub fn new() -> Self {
Self { Self {
cached: Arc::new(Mutex::new(None)), inner: Arc::new(VastaiLivePlugin::new()),
} }
} }
/// Spawn the poll loop on `rt`. Every ~1s it GETs /// Subscribe to the collector's `/diag/stream/{run_id}` SSE feed and fold each
/// `{collector_url}/api/plugin/vastai/model`; a successful response with a /// vast.ai `LiveRecord` into the inner plugin. Reconnects every ~2s on drop or
/// non-`null` body replaces the cache, anything else (error, non-2xx, empty, /// while the collector is unreachable. Ends when `rt`'s runtime is dropped at
/// `null`) leaves the last good model untouched. The task ends when `rt`'s /// end of run. `run_id` must match what the stages ship under
/// runtime is dropped at end of run. Cadence matches [`crate::dist_broadcast`]. /// (`SWACTOR_DIAG_RUN_ID`).
pub fn spawn_poller(&self, rt: &tokio::runtime::Handle, collector_url: String) { pub fn spawn_stream(&self, rt: &tokio::runtime::Handle, collector_url: String, run_id: String) {
let url = format!( let url = format!(
"{}/api/plugin/vastai/model", "{}/diag/stream/{}",
collector_url.trim_end_matches('/') collector_url.trim_end_matches('/'),
run_id
); );
let cached = Arc::clone(&self.cached); let inner = Arc::clone(&self.inner);
eprintln!("pp-orchestrator: fleet tab streaming vast.ai records from {url}");
rt.spawn(async move { rt.spawn(async move {
let http = reqwest::Client::new(); let http = reqwest::Client::new();
let mut ticker = tokio::time::interval(Duration::from_secs(1)); // Log the first record and the first error once, so a blank Fleet tab
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); // is easy to localize (collector empty vs. stream unreachable) without
// spamming the orchestrator log.
let mut logged_first = false;
let mut logged_err = false;
loop { loop {
ticker.tick().await; match http.get(&url).send().await {
let body = match http.get(&url).send().await { Ok(resp) if resp.status().is_success() => {
Ok(resp) if resp.status().is_success() => resp.text().await.ok(), let mut stream = resp.bytes_stream();
_ => None, let mut buf: Vec<u8> = Vec::new();
}; while let Some(chunk) = stream.next().await {
if let Some(body) = body { let Ok(chunk) = chunk else { break };
let trimmed = body.trim(); buf.extend_from_slice(&chunk);
if !trimmed.is_empty() && trimmed != "null" { // SSE frames are separated by a blank line. Parse on
*cached.lock().unwrap() = Some(body); // byte boundaries so a chunk split mid-frame is safe.
while let Some(idx) = buf.windows(2).position(|w| w == b"\n\n") {
let frame: Vec<u8> = buf.drain(..idx + 2).collect();
let Ok(text) = std::str::from_utf8(&frame) else {
continue;
};
for line in text.lines() {
let Some(data) = line.strip_prefix("data:") else {
continue;
};
let data = data.trim_start();
if let Some(rec) = parse_vastai_record(data) {
inner.ingest(&rec);
if !logged_first {
logged_first = true;
eprintln!(
"pp-orchestrator: fleet tab is receiving vast.ai records from the collector"
);
}
}
}
}
}
}
Ok(resp) => {
if !logged_err {
logged_err = true;
eprintln!("pp-orchestrator: fleet stream got HTTP {}", resp.status());
}
}
Err(e) => {
if !logged_err {
logged_err = true;
eprintln!("pp-orchestrator: fleet stream error (collector reachable?): {e}");
}
} }
} }
tokio::time::sleep(Duration::from_secs(2)).await;
} }
}); });
} }
@ -83,28 +147,17 @@ impl DashboardPlugin for RemoteVastaiPlugin {
} }
fn snapshot_json(&self) -> Option<String> { fn snapshot_json(&self) -> Option<String> {
self.cached.lock().unwrap().clone() self.inner.snapshot_json()
} }
fn handle_request( fn handle_request(
&self, &self,
method: &str, method: &str,
path: &str, path: &str,
_query: &HashMap<String, String>, query: &HashMap<String, String>,
_body: &[u8], body: &[u8],
) -> PluginResponse { ) -> PluginResponse {
// The page seeds from `/api/plugin/vastai/model` (the route requires a self.inner.handle_request(method, path, query, body)
// non-empty trailing segment), matching the standalone collector board.
match (method, path) {
("GET", "" | "model") => PluginResponse::json(
self.cached
.lock()
.unwrap()
.clone()
.unwrap_or_else(|| "null".into()),
),
_ => PluginResponse::not_found(),
}
} }
fn html_page(&self) -> Option<&str> { fn html_page(&self) -> Option<&str> {

1
examples/pipeline-parallel-inference/src/lib.rs
View file

@ -1,6 +1,7 @@
pub mod diag; pub mod diag;
pub mod dist_broadcast; pub mod dist_broadcast;
pub mod dist_plugin; pub mod dist_plugin;
pub mod fleet_plugin;
pub mod messages; pub mod messages;
pub mod netmap_plugin; pub mod netmap_plugin;
pub mod profile; pub mod profile;

30
examples/pipeline-parallel-inference/src/netmap_page.html
View file

@ -7,10 +7,14 @@
<style> <style>
* { margin: 0; padding: 0; box-sizing: border-box; } * { margin: 0; padding: 0; box-sizing: border-box; }
body { font-family: 'Menlo','Consolas','Monaco',monospace; background:#0f1117; color:#e0e0e0; font-size:13px; } body { font-family: 'Menlo','Consolas','Monaco',monospace; background:#0f1117; color:#e0e0e0; font-size:13px; }
.header { display:flex; align-items:center; gap:16px; padding:10px 16px; background:#161922; border-bottom:1px solid #262b38; } .header { display:flex; align-items:center; justify-content:space-between; padding:12px 20px; background:#161822; border-bottom:1px solid #2a2d3e; }
.header h1 { font-size:15px; font-weight:600; color:#7aa2f7; } .header h1 { font-size:16px; font-weight:600; color:#fff; }
.nav a { color:#9aa5b1; text-decoration:none; margin-right:14px; } .header-left { display:flex; align-items:center; }
.nav a:hover, .nav a.active { color:#7aa2f7; } .nav-links { display:flex; gap:4px; margin-left:20px; }
.nav-link { color:#888; text-decoration:none; font-size:12px; padding:4px 10px; border-radius:3px; transition:color 0.2s; }
.nav-link:hover { color:#e0e0e0; }
.nav-link.active { color:#fff; background:#2a2d3e; }
.header-right { display:flex; align-items:center; gap:12px; }
.wrap { padding:16px; display:grid; grid-template-columns: 1fr; gap:16px; } .wrap { padding:16px; display:grid; grid-template-columns: 1fr; gap:16px; }
.card { background:#161922; border:1px solid #262b38; border-radius:6px; padding:12px; } .card { background:#161922; border:1px solid #262b38; border-radius:6px; padding:12px; }
.card h2 { font-size:13px; color:#c0caf5; margin-bottom:10px; border-bottom:1px solid #262b38; padding-bottom:6px; } .card h2 { font-size:13px; color:#c0caf5; margin-bottom:10px; border-bottom:1px solid #262b38; padding-bottom:6px; }
@ -33,14 +37,18 @@
</head> </head>
<body> <body>
<div class="header"> <div class="header">
<h1>Orchestrator · Net map</h1> <div class="header-left">
<div class="nav"> <h1>Swactor Runtime Dashboard</h1>
<a href="/">Overview</a> <nav class="nav-links">
<a href="/actors">Actors</a> <a href="/" class="nav-link">Overview</a>
<a href="/plugin/distribution">Distribution</a> <a href="/actors" class="nav-link">Actors</a>
<a href="/plugin/netmap" class="active">Net map</a> <a href="/topology" class="nav-link">Topology</a>
<a href="/plugin/distribution" class="nav-link">Distribution</a>
<a href="/plugin/netmap" class="nav-link active">Net map</a>
<a href="/plugin/vastai" class="nav-link">Fleet</a>
</nav>
</div> </div>
<span id="node" class="mono" style="margin-left:auto"></span> <div class="header-right"><span id="node" class="mono"></span></div>
</div> </div>
<div class="wrap"> <div class="wrap">

10
examples/pipeline-parallel-inference/src/orchestrator.rs
View file

@ -1,4 +1,4 @@
//! Orchestrator helpers for `pp-smoke-run`. //! Orchestrator helpers for `pp-orchestrator`.
//! //!
//! Extracted from the binary so the spawn-chain and convergence-wait logic //! Extracted from the binary so the spawn-chain and convergence-wait logic
//! can be unit-tested without provisioning child processes or driving a //! can be unit-tested without provisioning child processes or driving a
@ -17,7 +17,7 @@
//! //!
//! Tests inject a fake command builder (e.g. `sh -c "echo PP_GPU_NODE_ADDR //! Tests inject a fake command builder (e.g. `sh -c "echo PP_GPU_NODE_ADDR
//! <hex> <direct>; sleep 60"`) so the chain can be exercised end-to-end //! <hex> <direct>; sleep 60"`) so the chain can be exercised end-to-end
//! without `pp-gpu-node` on disk. //! without `pp-worker` on disk.
use std::io::{BufRead, BufReader}; use std::io::{BufRead, BufReader};
use std::process::{Child, ChildStdout, Command, Stdio}; use std::process::{Child, ChildStdout, Command, Stdio};
@ -118,7 +118,7 @@ impl Drop for ChainGuard {
// to the container's PID 1 and only then does the container exit // to the container's PID 1 and only then does the container exit
// and `--rm` clean up. A bare SIGKILL bypasses that proxy and // and `--rm` clean up. A bare SIGKILL bypasses that proxy and
// orphans the container. For the no-wrapper host case the cost is // orphans the container. For the no-wrapper host case the cost is
// ~tens of ms — pp-gpu-node has no SIGTERM handler so it exits // ~tens of ms — pp-worker has no SIGTERM handler so it exits
// immediately on receipt. // immediately on receipt.
// //
// The budget is *per stage*, not shared across the whole chain. // The budget is *per stage*, not shared across the whole chain.
@ -148,7 +148,7 @@ impl Drop for ChainGuard {
// child already exited (we ignore the error either way). // child already exited (we ignore the error either way).
let _ = stage.child.kill(); let _ = stage.child.kill();
let _ = stage.child.wait(); let _ = stage.child.wait();
eprintln!("pp-smoke-run: stopped stage {} child pid {pid}", stage.stage); eprintln!("pp-orchestrator: stopped stage {} child pid {pid}", stage.stage);
} }
} }
} }
@ -307,7 +307,7 @@ fn read_stage_address(
}); });
rx.recv_timeout(timeout).map_err(|_| { rx.recv_timeout(timeout).map_err(|_| {
eprintln!( eprintln!(
"pp-smoke-run: stage {stage} did not announce PP_GPU_NODE_ADDR within {:.0}s", "pp-orchestrator: stage {stage} did not announce PP_GPU_NODE_ADDR within {:.0}s",
timeout.as_secs_f32() timeout.as_secs_f32()
); );
}) })

2
examples/pipeline-parallel-inference/src/relay_config.rs
View file

@ -1,6 +1,6 @@
//! Picks the iroh `RelayMode` from process environment. //! Picks the iroh `RelayMode` from process environment.
//! //!
//! `SWACTOR_IROH_RELAY_URL` — when set, both pp-smoke-run and pp-gpu-node //! `SWACTOR_IROH_RELAY_URL` — when set, both pp-orchestrator and pp-worker
//! use `RelayMode::Custom(<url>)` instead of the canary default. The //! use `RelayMode::Custom(<url>)` instead of the canary default. The
//! orchestrator's [`DiagEnv`](crate::vastai::DiagEnv) propagates the same //! orchestrator's [`DiagEnv`](crate::vastai::DiagEnv) propagates the same
//! var into every rented container so the whole cluster homes onto one //! var into every rented container so the whole cluster homes onto one

10
examples/pipeline-parallel-inference/src/stage_actor.rs
View file

@ -121,7 +121,7 @@ pub enum StageActorStatus {
/// this enum by the bridge actors; `Process` is adapted from the worker /// this enum by the bridge actors; `Process` is adapted from the worker
/// subprocess; `SetNeighbors` and `Reset` are control messages. /// subprocess; `SetNeighbors` and `Reset` are control messages.
/// ///
/// `SetNeighbors` is a one-shot setup message used by the `pp-gpu-node` /// `SetNeighbors` is a one-shot setup message used by the `pp-worker`
/// binary to inject the resolved addresses of neighbouring stages and the /// binary to inject the resolved addresses of neighbouring stages and the
/// orchestrator after SWIM gossip has propagated them. Each field is /// orchestrator after SWIM gossip has propagated them. Each field is
/// optional; only the fields relevant to the actor's role need to be set. /// optional; only the fields relevant to the actor's role need to be set.
@ -143,7 +143,7 @@ pub enum StageMsg {
/// Tear down the running worker subprocess and spawn a fresh one, /// Tear down the running worker subprocess and spawn a fresh one,
/// re-exec'ing the on-disk worker script so an edited /// re-exec'ing the on-disk worker script so an edited
/// `pp_tinygrad_worker.py` is picked up without restarting the node. /// `pp_tinygrad_worker.py` is picked up without restarting the node.
/// Driven by a `SIGHUP` to `pp-gpu-node`. /// Driven by a `SIGHUP` to `pp-worker`.
ReloadWorker, ReloadWorker,
} }
@ -1184,18 +1184,18 @@ impl ActorInterface for StageActor {
// stderr. The worker's stderr is otherwise consumed here and // stderr. The worker's stderr is otherwise consumed here and
// only re-emitted on the `worker_exit_detail` diagnostics // only re-emitted on the `worker_exit_detail` diagnostics
// event, which is invisible when no collector is configured // event, which is invisible when no collector is configured
// (the common bare-deploy case). pp-gpu-node's stderr is // (the common bare-deploy case). pp-worker's stderr is
// captured by the container log, so this makes a crashed // captured by the container log, so this makes a crashed
// worker self-diagnosing without a diagnostics backend. // worker self-diagnosing without a diagnostics backend.
if !normal_exit { if !normal_exit {
eprintln!( eprintln!(
"pp-gpu-node: worker exited abnormally (code={exit_code:?} signal={signal:?}); stderr tail:" "pp-worker: worker exited abnormally (code={exit_code:?} signal={signal:?}); stderr tail:"
); );
for line in &stderr_tail { for line in &stderr_tail {
eprintln!(" worker| {line}"); eprintln!(" worker| {line}");
} }
if let Some(tb) = traceback.as_deref() { if let Some(tb) = traceback.as_deref() {
eprintln!("pp-gpu-node: worker python traceback:\n{tb}"); eprintln!("pp-worker: worker python traceback:\n{tb}");
} }
} }

238
examples/pipeline-parallel-inference/src/vastai.rs
View file

@ -15,6 +15,7 @@
//! //!
//! All functions accept a `base_url` so tests can point at a wiremock server. //! All functions accept a `base_url` so tests can point at a wiremock server.
use std::io::{IsTerminal, Write};
use std::time::Duration; use std::time::Duration;
use reqwest::Client; use reqwest::Client;
@ -28,7 +29,7 @@ pub struct InstanceInfo {
/// Diagnostics env-var bundle forwarded to rented stage containers. /// Diagnostics env-var bundle forwarded to rented stage containers.
/// ///
/// `pp-gpu-node::diag::install_from_env` reads `SWACTOR_DIAG_*` on boot /// `pp-worker::diag::install_from_env` reads `SWACTOR_DIAG_*` on boot
/// inside each container to decide whether to enable the aggregator and /// inside each container to decide whether to enable the aggregator and
/// where to ship to. The orchestrator-side caller of [`lease_chain`] /// where to ship to. The orchestrator-side caller of [`lease_chain`]
/// builds this from its own process env (typically the same vars the /// builds this from its own process env (typically the same vars the
@ -111,6 +112,12 @@ pub struct Offer {
/// that gouge on bandwidth, since bandwidth price is a per-host policy. /// that gouge on bandwidth, since bandwidth price is a per-host policy.
#[serde(default)] #[serde(default)]
pub host_id: Option<u64>, pub host_id: Option<u64>,
/// vast.ai host verification state: `"verified"`, `"unverified"` (never
/// tested), or `"deverified"` (was verified, then failed vast's checks).
/// Deverified hosts recurrently fail CDI GPU-device injection at container
/// start despite a high `reliability2`, so they are dropped by default.
#[serde(default)]
pub verification: Option<String>,
} }
/// Connection details for a running instance. /// Connection details for a running instance.
@ -237,6 +244,118 @@ fn next_eligible_offer<'a>(
}) })
} }
/// The offers [`lease_chain`] would rent on the happy path: the cheapest
/// `num_stages` on distinct hosts, in stage order (stage 0 = cheapest). Mirrors
/// the distinct-host draw in [`next_eligible_offer`] (a `None` host id is never
/// deduped, matching that helper). Actual picks can differ only if a create
/// fails and the chain falls through to the next survivor — so the confirmed
/// cost is the floor, not a ceiling.
fn plan_picks(pool: &[Offer], num_stages: u32) -> Vec<&Offer> {
let mut picks: Vec<&Offer> = Vec::with_capacity(num_stages as usize);
let mut used: std::collections::HashSet<u64> = std::collections::HashSet::new();
for o in pool {
if picks.len() == num_stages as usize {
break;
}
if let Some(h) = o.host_id {
if !used.insert(h) {
continue; // host already claimed by an earlier pick
}
}
picks.push(o);
}
picks
}
/// `PP_ASSUME_YES`: skip the interactive lease confirmation (for scripted / CI
/// runs that intend to rent without a human at the keyboard). Truthy = any
/// non-empty value other than `0` / `false` / `no`.
fn assume_yes() -> bool {
std::env::var("PP_ASSUME_YES")
.ok()
.map(|v| {
let v = v.trim().to_ascii_lowercase();
!v.is_empty() && v != "0" && v != "false" && v != "no"
})
.unwrap_or(false)
}
/// Print the planned lease + its hourly cost and, when running interactively,
/// require an explicit `y`/`N` before any instance is created.
///
/// This is the cost guardrail: the offer search can legitimately land on an
/// expensive card when the cheap pool is thin, and an unconfirmed lease once put
/// a 3-stage run onto an A100. The prompt is the last gate before money is spent.
///
/// To keep scripted runs and the (HTTP-mocked) test suite unaffected, the
/// confirmation is *skipped* (the lease proceeds) when `PP_ASSUME_YES` is set or
/// when stdin is not a TTY — there is no human to answer in those cases. The cost
/// summary is always logged regardless.
fn confirm_lease(pool: &[Offer], num_stages: u32, cost: &CostModel) -> Result<(), String> {
let picks = plan_picks(pool, num_stages);
let total_dph: f64 = picks.iter().map(|o| o.dph_total).sum();
let total_eff: f64 = picks.iter().map(|o| cost.effective_price(o)).sum();
eprintln!(
"pp-orchestrator: lease plan — {num_stages} stage(s), cheapest on distinct hosts:"
);
for (i, o) in picks.iter().enumerate() {
eprintln!(
" stage {i} {:<14} {:>8} ${:.3}/hr [{}] host {}",
o.gpu_name,
o.gpu_ram
.map(|r| format!("{:.0}MB", r))
.unwrap_or_else(|| "?MB".into()),
o.dph_total,
o.geolocation.as_deref().unwrap_or("?"),
o.host_id
.map(|h| h.to_string())
.unwrap_or_else(|| "?".into()),
);
}
if picks.len() < num_stages as usize {
eprintln!(
" WARNING: only {} distinct-host offer(s) available for {num_stages} stage(s) — \
the lease will likely fail to fill the chain.",
picks.len(),
);
}
let eff_note = if (total_eff - total_dph).abs() > 1e-6 {
format!(" (image-pull priced in: ${total_eff:.3}/hr eff)")
} else {
String::new()
};
eprintln!(
" TOTAL ${total_dph:.3}/hr (~${:.2}/day){eff_note}",
total_dph * 24.0,
);
if assume_yes() {
eprintln!("pp-orchestrator: PP_ASSUME_YES set — proceeding without confirmation");
return Ok(());
}
if !std::io::stdin().is_terminal() {
eprintln!(
"pp-orchestrator: stdin is not a TTY — proceeding without interactive confirmation \
(set PP_ASSUME_YES=1 to silence this)"
);
return Ok(());
}
eprint!("Proceed with renting these {num_stages} instance(s)? [y/N]: ");
let _ = std::io::stderr().flush();
let mut line = String::new();
std::io::stdin()
.read_line(&mut line)
.map_err(|e| format!("failed to read lease confirmation: {e}"))?;
let ans = line.trim().to_ascii_lowercase();
if ans == "y" || ans == "yes" {
Ok(())
} else {
Err("operator declined the lease (cost not confirmed); no instances were created".into())
}
}
/// Build the ranked survivor pool for a heterogeneous PP lease in a single /// Build the ranked survivor pool for a heterogeneous PP lease in a single
/// query. Replaces the old per-stage `find_offer` / `find_offer_chain`. /// query. Replaces the old per-stage `find_offer` / `find_offer_chain`.
/// ///
@ -253,40 +372,55 @@ pub async fn select_offer_pool(
gpu_name: &str, gpu_name: &str,
num_stages: u32, num_stages: u32,
) -> Result<Vec<Offer>, String> { ) -> Result<Vec<Offer>, String> {
// Hard gates expressed server-side. reliability2 >= 0.995 and // Hard gates expressed server-side. reliability2 (PP_MIN_RELIABILITY,
// cuda_max_good >= 12.6 (our CUDA-12.6 base image) drop hosts that // default 0.95 — "semi-reliable", NOT near-perfect) and cuda_max_good >=
// recurrently fail container init / CDI device injection; num_gpus == 1 // 12.6 (our CUDA-12.6 base image) drop hosts that recurrently fail
// keeps us from renting a multi-GPU rig per stage. Network speed can't be // container init / CDI device injection; num_gpus == 1 keeps us from
// probed before renting, so we trust vast.ai's measured inet figures and // renting a multi-GPU rig per stage. Network speed can't be probed before
// gate on a configurable minimum (PP_MIN_INET_DOWN_MBPS, default 100; the // renting, so we trust vast.ai's measured inet figures and gate on a
// upload gate is off by default). // configurable minimum (PP_MIN_INET_DOWN_MBPS, default 100; the upload
// gate is off by default).
let mut query = serde_json::json!({ let mut query = serde_json::json!({
"rentable": {"eq": true}, "rentable": {"eq": true},
"rented": {"eq": false}, "rented": {"eq": false},
"reliability2": {"gte": 0.995}, "reliability2": {"gte": env_min_reliability()},
"cuda_max_good": {"gte": 12.6}, "cuda_max_good": {"gte": 12.6},
"verified": {"eq": true},
"direct_port_count": {"gte": 1}, "direct_port_count": {"gte": 1},
"num_gpus": {"eq": 1}, "num_gpus": {"eq": 1},
"inet_down": {"gte": env_min_inet_down_mbps()}, "inet_down": {"gte": env_min_inet_down_mbps()},
// Cap the response so one query covers N distinct hosts even after the // vast.ai treats `limit` as a SCAN BUDGET (machines examined in the
// per-model cheap drop, without paging. // engine's default high-perf-first order), NOT a result cap: a small
"limit": 512, // limit returns *fewer* matches because it never reaches the cheap
// commodity hosts that rank low. Empirically `limit:512` returned ~184
// ram>=8000 offers while `limit:5000` returned ~1639 — the missing
// ~1450 included the cheap 3090/3060 supply, so a small limit alone
// skews the pool toward datacenter cards. Set high so the survivor pool
// reflects the whole market.
"limit": 5000,
}); });
if let Some(up) = env_min_inet_up_mbps() { if let Some(up) = env_min_inet_up_mbps() {
query["inet_up"] = serde_json::json!({"gte": up}); query["inet_up"] = serde_json::json!({"gte": up});
} }
// VRAM mode spans a heterogeneous card set (each PP stage is an independent // vast.ai's `verified` flag means the host passed vast's own datacenter
// process exchanging fp16 hidden state, so stages need not share a model — // vetting. AND'd with the other gates it discarded ~90% of supply — almost
// only enough VRAM for their block slice). Model-name mode is the historical // every cheap consumer 3090/3060 is unverified — so it is OFF by default and
// default and implicitly bounds cost to that one cheap model. // reliability2 (above) carries the quality floor. Opt back in with
match env_min_gpu_ram_mb() { // PP_REQUIRE_VERIFIED=1 for a vetted-hosts-only pool.
Some(min_ram) => { if env_require_verified() {
query["gpu_ram"] = serde_json::json!({"gte": min_ram}); query["verified"] = serde_json::json!({"eq": true});
} }
None => { // GPU selection is two independent, optional filters — neither is required.
query["gpu_name"] = serde_json::json!({"eq": gpu_name}); // A VRAM floor (PP_GPU_MIN_RAM_MB) spans a heterogeneous card set (each PP
} // stage is an independent process exchanging fp16 hidden state, so stages
// need not share a model — only enough VRAM for their block slice). A model
// pin (PP_GPU / `gpu_name`) restricts to one model. Unset both → the GPU
// itself isn't filtered and the quality gates above + cost ranking pick the
// host.
if let Some(min_ram) = env_min_gpu_ram_mb() {
query["gpu_ram"] = serde_json::json!({"gte": min_ram});
}
if !gpu_name.is_empty() {
query["gpu_name"] = serde_json::json!({"eq": gpu_name});
} }
let url = format!( let url = format!(
"{base_url}/api/v0/bundles/?q={}", "{base_url}/api/v0/bundles/?q={}",
@ -311,8 +445,16 @@ pub async fn select_offer_pool(
.map_err(|e| format!("select_offer_pool parse failed: {e}"))?; .map_err(|e| format!("select_offer_pool parse failed: {e}"))?;
// Post-filter in Rust: drop unknown/Chinese geolocations (Docker Hub and // Post-filter in Rust: drop unknown/Chinese geolocations (Docker Hub and
// iroh relays are unreachable from behind the Great Firewall) and any // iroh relays are unreachable from behind the Great Firewall), any
// blacklisted host (providers caught gouging on bandwidth). // blacklisted host (providers caught gouging on bandwidth), and — by
// default — `deverified` hosts. vast.ai deverifies a host after it fails
// vast's own checks; in practice these recurrently fail CDI GPU-device
// injection at container start ("unresolvable CDI devices …/gpu=0") even
// though their `reliability2` stays ~0.99, which is why the reliability
// gate alone does not catch them. `unverified` (never-tested) hosts are
// kept — they hold the cheap consumer-GPU supply and usually start fine.
// PP_REQUIRE_VERIFIED already restricts the query to verified-only, in
// which case this filter is a no-op.
let blacklist = blacklisted_host_ids(); let blacklist = blacklisted_host_ids();
let reachable: Vec<Offer> = body let reachable: Vec<Offer> = body
.offers .offers
@ -323,6 +465,7 @@ pub async fn select_offer_pool(
.map_or(false, |g| !g.to_uppercase().contains("CN")) .map_or(false, |g| !g.to_uppercase().contains("CN"))
}) })
.filter(|o| o.host_id.map_or(true, |h| !blacklist.contains(&h))) .filter(|o| o.host_id.map_or(true, |h| !blacklist.contains(&h)))
.filter(|o| o.verification.as_deref() != Some("deverified"))
.collect(); .collect();
let cost = CostModel::from_env(); let cost = CostModel::from_env();
@ -348,10 +491,10 @@ pub async fn select_offer_pool(
Ok(pool) Ok(pool)
} }
/// `PP_GPU_MIN_RAM_MB`: when set to a positive integer, the offer search /// `PP_GPU_MIN_RAM_MB`: when set to a positive integer, adds a VRAM floor
/// selects cards by VRAM (`gpu_ram >= N` MB) instead of by exact GPU model, /// (`gpu_ram >= N` MB) to the offer search, enabling a heterogeneous cluster.
/// enabling a heterogeneous cluster. Unset / blank / zero → model-name mode /// Independent of the optional `PP_GPU` model pin; unset / blank / zero → no
/// (the historical default, unchanged). /// VRAM filter.
fn env_min_gpu_ram_mb() -> Option<u64> { fn env_min_gpu_ram_mb() -> Option<u64> {
std::env::var("PP_GPU_MIN_RAM_MB") std::env::var("PP_GPU_MIN_RAM_MB")
.ok() .ok()
@ -371,6 +514,31 @@ fn env_min_inet_down_mbps() -> f64 {
.unwrap_or(100.0) .unwrap_or(100.0)
} }
/// `PP_MIN_RELIABILITY`: minimum vast.ai `reliability2` an offer must carry.
/// The old hardcoded 0.995, combined with the `verified` gate, admitted almost
/// only datacenter rigs (A100/H100) — every cheap consumer 3090/3060 sits at
/// 0.95–0.99 and/or is unverified, so the two gates AND'd together left zero
/// cheap cards and the lease was forced onto an expensive datacenter card.
/// Default 0.95 ("semi-reliable"); clamped to [0, 1]; 0 disables the gate.
fn env_min_reliability() -> f64 {
std::env::var("PP_MIN_RELIABILITY")
.ok()
.and_then(|s| s.trim().parse::<f64>().ok())
.filter(|&v| (0.0..=1.0).contains(&v))
.unwrap_or(0.95)
}
/// `PP_REQUIRE_VERIFIED`: when truthy (`1`/`true`/`yes`, case-insensitive),
/// restrict the search to vast.ai-verified hosts. Default off — the verified
/// flag AND'd with the other gates excluded nearly all cheap consumer GPUs, so
/// reliability2 carries the quality floor and unvetted hosts are admitted.
fn env_require_verified() -> bool {
std::env::var("PP_REQUIRE_VERIFIED")
.ok()
.map(|s| matches!(s.trim().to_ascii_lowercase().as_str(), "1" | "true" | "yes"))
.unwrap_or(false)
}
/// `PP_MIN_INET_UP_MBPS`: optional minimum reported upload speed (Mbps). /// `PP_MIN_INET_UP_MBPS`: optional minimum reported upload speed (Mbps).
/// Default unset / 0 → no upload-speed gate. /// Default unset / 0 → no upload-speed gate.
fn env_min_inet_up_mbps() -> Option<f64> { fn env_min_inet_up_mbps() -> Option<f64> {
@ -677,7 +845,7 @@ pub async fn create_instance(
// Pin this stage's iroh identity so it survives a restart: re-read from // Pin this stage's iroh identity so it survives a restart: re-read from
// PID 1's env on restart, the stage keeps the same node id and the // PID 1's env on restart, the stage keeps the same node id and the
// pipeline name registry stays valid. See // pipeline name registry stays valid. See
// pp-gpu-node::stage_secret_from_env. // pp-worker::stage_secret_from_env.
if let Some(secret) = stage_secret { if let Some(secret) = stage_secret {
env["PP_STAGE_SECRET"] = serde_json::Value::String(secret.to_string()); env["PP_STAGE_SECRET"] = serde_json::Value::String(secret.to_string());
} }
@ -717,7 +885,7 @@ pub async fn create_instance(
// Run the PID-1 supervisor (not the worker directly): it brings up // Run the PID-1 supervisor (not the worker directly): it brings up
// sshd deterministically and keeps the container — and the shell — // sshd deterministically and keeps the container — and the shell —
// alive if the worker crashes. No `exec` of the worker: the supervisor // alive if the worker crashes. No `exec` of the worker: the supervisor
// owns PID 1 and runs pp-gpu-node as a child. // owns PID 1 and runs pp-worker as a child.
"onstart": "/usr/local/bin/pp_entrypoint.sh 2>&1", "onstart": "/usr/local/bin/pp_entrypoint.sh 2>&1",
// Every stage fetch()s the FULL gguf (whole file mmap'd by // Every stage fetch()s the FULL gguf (whole file mmap'd by
// from_gguf), regardless of which layers it runs. qwen3:30b-a3b // from_gguf), regardless of which layers it runs. qwen3:30b-a3b
@ -1135,6 +1303,11 @@ pub async fn lease_chain(
.await .await
.map_err(|e| format!("lease_chain: {e}"))?; .map_err(|e| format!("lease_chain: {e}"))?;
// Cost guardrail: show what we're about to rent and (interactively) require
// a y/N before spending money. Runs before any create_instance, so a decline
// is a clean abort with nothing leased. Skipped for non-TTY / PP_ASSUME_YES.
confirm_lease(&pool, num_stages, &CostModel::from_env())?;
let mut tried_offer_ids: Vec<u64> = Vec::new(); let mut tried_offer_ids: Vec<u64> = Vec::new();
let mut created: Vec<InstanceInfo> = Vec::with_capacity(num_stages as usize); let mut created: Vec<InstanceInfo> = Vec::with_capacity(num_stages as usize);
// Host ids leased by this chain, owned here so it survives across both // Host ids leased by this chain, owned here so it survives across both
@ -1297,6 +1470,7 @@ mod tests {
inet_down_cost_per_tb: 0.0, inet_down_cost_per_tb: 0.0,
inet_up_cost_per_tb: 0.0, inet_up_cost_per_tb: 0.0,
host_id: Some(host), host_id: Some(host),
verification: Some("verified".to_string()),
} }
} }

68
examples/pipeline-parallel-inference/tests/t_binary.rs
View file

@ -1,4 +1,4 @@
//! T-binary: drive the actual `pp-smoke-run --seed` and `pp-gpu-node` //! T-binary: drive the actual `pp-orchestrator --seed` and `pp-worker`
//! binaries as child processes. TEST_SPEC §13 (stub workers) and §14 //! binaries as child processes. TEST_SPEC §13 (stub workers) and §14
//! (real tinygrad workers, `#[ignore]`). //! (real tinygrad workers, `#[ignore]`).
//! //!
@ -18,8 +18,8 @@ use std::sync::{Arc, Mutex};
use std::thread; use std::thread;
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
const SMOKE_RUN_BIN: &str = env!("CARGO_BIN_EXE_pp-smoke-run"); const ORCHESTRATOR_BIN: &str = env!("CARGO_BIN_EXE_pp-orchestrator");
const GPU_NODE_BIN: &str = env!("CARGO_BIN_EXE_pp-gpu-node"); const WORKER_BIN: &str = env!("CARGO_BIN_EXE_pp-worker");
/// Default per-test budget for an N=5 stub-mode happy path: cluster build + /// Default per-test budget for an N=5 stub-mode happy path: cluster build +
/// SWIM convergence + sequential worker boots + a short decode loop. Tests /// SWIM convergence + sequential worker boots + a short decode loop. Tests
@ -156,7 +156,7 @@ fn wait_for_n_children(parent_pid: u32, expected: usize, timeout: Duration) -> O
} }
} }
/// Extract the response text from `pp-smoke-run`'s stdout. The orchestrator /// Extract the response text from `pp-orchestrator`'s stdout. The orchestrator
/// prints the response between two banner lines: /// prints the response between two banner lines:
/// ///
/// ```text /// ```text
@ -194,14 +194,14 @@ struct SmokeRunOpts {
boot_delay_secs: Option<u32>, boot_delay_secs: Option<u32>,
} }
/// Spawn `pp-smoke-run --seed` with the given options. Returns the spawned /// Spawn `pp-orchestrator --seed` with the given options. Returns the spawned
/// process plus a shared log buffer that captures every stdout / stderr /// process plus a shared log buffer that captures every stdout / stderr
/// line from `pp-smoke-run` AND every `pp-gpu-node` child (children inherit /// line from `pp-orchestrator` AND every `pp-worker` child (children inherit
/// the orchestrator's stderr fd, so their messages land in the same buffer). /// the orchestrator's stderr fd, so their messages land in the same buffer).
fn spawn_smoke_run(opts: &SmokeRunOpts) -> (Child, LogBuffer, LogBuffer) { fn spawn_smoke_run(opts: &SmokeRunOpts) -> (Child, LogBuffer, LogBuffer) {
assert!(opts.num_stages >= 2); assert!(opts.num_stages >= 2);
let worker = worker_script(); let worker = worker_script();
let mut cmd = Command::new(SMOKE_RUN_BIN); let mut cmd = Command::new(ORCHESTRATOR_BIN);
cmd.arg("--seed") cmd.arg("--seed")
.arg("--num-stages") .arg("--num-stages")
.arg(opts.num_stages.to_string()) .arg(opts.num_stages.to_string())
@ -210,7 +210,7 @@ fn spawn_smoke_run(opts: &SmokeRunOpts) -> (Child, LogBuffer, LogBuffer) {
.arg("--max-tokens") .arg("--max-tokens")
.arg(opts.max_tokens.to_string()) .arg(opts.max_tokens.to_string())
.arg("--gpu-node") .arg("--gpu-node")
.arg(GPU_NODE_BIN) .arg(WORKER_BIN)
.arg("--worker") .arg("--worker")
.arg(&worker) .arg(&worker)
.stdout(Stdio::piped()) .stdout(Stdio::piped())
@ -228,13 +228,13 @@ fn spawn_smoke_run(opts: &SmokeRunOpts) -> (Child, LogBuffer, LogBuffer) {
cmd.env_remove("PP_BOOT_DELAY_SECS"); cmd.env_remove("PP_BOOT_DELAY_SECS");
} }
let mut child = cmd.spawn().expect("spawn pp-smoke-run"); let mut child = cmd.spawn().expect("spawn pp-orchestrator");
let stdout = child.stdout.take().expect("piped stdout"); let stdout = child.stdout.take().expect("piped stdout");
let stderr = child.stderr.take().expect("piped stderr"); let stderr = child.stderr.take().expect("piped stderr");
let stdout_buf = LogBuffer::default(); let stdout_buf = LogBuffer::default();
let stderr_buf = LogBuffer::default(); let stderr_buf = LogBuffer::default();
drain_stdout(stdout, "pp-smoke-run", stdout_buf.clone()); drain_stdout(stdout, "pp-orchestrator", stdout_buf.clone());
drain_stderr(stderr, "pp-smoke-run", stderr_buf.clone()); drain_stderr(stderr, "pp-orchestrator", stderr_buf.clone());
(child, stdout_buf, stderr_buf) (child, stdout_buf, stderr_buf)
} }
@ -251,13 +251,13 @@ fn run_to_completion(opts: &SmokeRunOpts) -> RunOutcome {
.unwrap_or_else(|| { .unwrap_or_else(|| {
let _ = smoke.kill(); let _ = smoke.kill();
let _ = smoke.wait(); let _ = smoke.wait();
panic!("pp-smoke-run did not spawn {n} pp-gpu-node children within 60s") panic!("pp-orchestrator did not spawn {n} pp-worker children within 60s")
}); });
let status = wait_with_timeout(&mut smoke, HAPPY_PATH_TIMEOUT).unwrap_or_else(|| { let status = wait_with_timeout(&mut smoke, HAPPY_PATH_TIMEOUT).unwrap_or_else(|| {
let _ = smoke.kill(); let _ = smoke.kill();
let _ = smoke.wait(); let _ = smoke.wait();
panic!("pp-smoke-run did not exit within {:?}", HAPPY_PATH_TIMEOUT); panic!("pp-orchestrator did not exit within {:?}", HAPPY_PATH_TIMEOUT);
}); });
RunOutcome { RunOutcome {
status, status,
@ -278,7 +278,7 @@ impl RunOutcome {
fn require_success(&self) { fn require_success(&self) {
assert!( assert!(
self.status.success(), self.status.success(),
"pp-smoke-run exited with {:?}\n--- stdout ---\n{}\n--- stderr (last 40 lines) ---\n{}", "pp-orchestrator exited with {:?}\n--- stdout ---\n{}\n--- stderr (last 40 lines) ---\n{}",
self.status, self.status,
self.stdout.join("\n"), self.stdout.join("\n"),
self.stderr self.stderr
@ -295,7 +295,7 @@ impl RunOutcome {
fn require_response_non_empty(&self) -> String { fn require_response_non_empty(&self) -> String {
let response = extract_response(&self.stdout).unwrap_or_else(|| { let response = extract_response(&self.stdout).unwrap_or_else(|| {
panic!( panic!(
"pp-smoke-run stdout missing response banner; got:\n{}", "pp-orchestrator stdout missing response banner; got:\n{}",
self.stdout.join("\n") self.stdout.join("\n")
) )
}); });
@ -310,7 +310,7 @@ impl RunOutcome {
for pid in &self.pre_exit_pids { for pid in &self.pre_exit_pids {
assert!( assert!(
!pid_alive(*pid), !pid_alive(*pid),
"pp-gpu-node child pid {pid} still alive after pp-smoke-run exit" "pp-worker child pid {pid} still alive after pp-orchestrator exit"
); );
} }
} }
@ -436,7 +436,7 @@ fn binary_e2e_all_stages_register_pp_stage_index_names() {
// §13.2 — Failure / cleanup paths // §13.2 — Failure / cleanup paths
// ─────────────────────────────────────────────────────────────────────── // ───────────────────────────────────────────────────────────────────────
/// Spawn `pp-smoke-run` at `num_stages`, wait until every stage child is /// Spawn `pp-orchestrator` at `num_stages`, wait until every stage child is
/// visible, kill `stage_to_kill`, and assert the orchestrator exits /// visible, kill `stage_to_kill`, and assert the orchestrator exits
/// non-zero with no surviving stage children. /// non-zero with no surviving stage children.
fn kill_stage_and_expect_failure(num_stages: u32, stage_to_kill: u32) { fn kill_stage_and_expect_failure(num_stages: u32, stage_to_kill: u32) {
@ -448,7 +448,7 @@ fn kill_stage_and_expect_failure(num_stages: u32, stage_to_kill: u32) {
.unwrap_or_else(|| { .unwrap_or_else(|| {
let _ = smoke.kill(); let _ = smoke.kill();
let _ = smoke.wait(); let _ = smoke.wait();
panic!("pp-smoke-run did not spawn {num_stages} children within 60s"); panic!("pp-orchestrator did not spawn {num_stages} children within 60s");
}); });
let victim = pids[stage_to_kill as usize]; let victim = pids[stage_to_kill as usize];
@ -457,16 +457,16 @@ fn kill_stage_and_expect_failure(num_stages: u32, stage_to_kill: u32) {
let exit = wait_with_timeout(&mut smoke, Duration::from_secs(120)).unwrap_or_else(|| { let exit = wait_with_timeout(&mut smoke, Duration::from_secs(120)).unwrap_or_else(|| {
let _ = smoke.kill(); let _ = smoke.kill();
let _ = smoke.wait(); let _ = smoke.wait();
panic!("pp-smoke-run did not exit within 120s after killing stage {stage_to_kill}"); panic!("pp-orchestrator did not exit within 120s after killing stage {stage_to_kill}");
}); });
assert!( assert!(
!exit.success(), !exit.success(),
"pp-smoke-run should fail when stage {stage_to_kill} (pid {victim}) is killed, got {exit:?}" "pp-orchestrator should fail when stage {stage_to_kill} (pid {victim}) is killed, got {exit:?}"
); );
for pid in &pids { for pid in &pids {
assert!( assert!(
!pid_alive(*pid), !pid_alive(*pid),
"stage child pid {pid} still alive after pp-smoke-run exit" "stage child pid {pid} still alive after pp-orchestrator exit"
); );
} }
} }
@ -532,9 +532,9 @@ fn binary_e2e_no_orphaned_processes_after_failed_exit_3() {
#[test] #[test]
#[ignore] #[ignore]
fn binary_e2e_orchestrator_sigkilled_children_die_within_timeout() { fn binary_e2e_orchestrator_sigkilled_children_die_within_timeout() {
// SIGKILL `pp-smoke-run` itself once its children are up. The kernel // SIGKILL `pp-orchestrator` itself once its children are up. The kernel
// delivers `SIGTERM` to each pp-gpu-node (via PR_SET_PDEATHSIG, set in // delivers `SIGTERM` to each pp-worker (via PR_SET_PDEATHSIG, set in
// pp-gpu-node's main), and each pp-gpu-node then dies — which also // pp-worker's main), and each pp-worker then dies — which also
// closes its Python worker's stdin, making the worker exit on EOF. // closes its Python worker's stdin, making the worker exit on EOF.
let opts = stub_opts(3); let opts = stub_opts(3);
let (mut smoke, _stdout, _stderr) = spawn_smoke_run(&opts); let (mut smoke, _stdout, _stderr) = spawn_smoke_run(&opts);
@ -543,7 +543,7 @@ fn binary_e2e_orchestrator_sigkilled_children_die_within_timeout() {
.unwrap_or_else(|| { .unwrap_or_else(|| {
let _ = smoke.kill(); let _ = smoke.kill();
let _ = smoke.wait(); let _ = smoke.wait();
panic!("pp-smoke-run did not spawn 3 children within 60s"); panic!("pp-orchestrator did not spawn 3 children within 60s");
}); });
sigkill(smoke_pid); sigkill(smoke_pid);
@ -554,7 +554,7 @@ fn binary_e2e_orchestrator_sigkilled_children_die_within_timeout() {
}); });
assert!( assert!(
cleaned_up, cleaned_up,
"pp-gpu-node children {pids:?} still alive 10s after pp-smoke-run SIGKILL; \ "pp-worker children {pids:?} still alive 10s after pp-orchestrator SIGKILL; \
per-pid alive states: {:?}", per-pid alive states: {:?}",
pids.iter().map(|p| (p, pid_alive(*p))).collect::<Vec<_>>() pids.iter().map(|p| (p, pid_alive(*p))).collect::<Vec<_>>()
); );
@ -595,7 +595,7 @@ fn binary_e2e_orchestrator_can_resolve_pp_entry_after_n_stages_register() {
#[test] #[test]
#[ignore] #[ignore]
fn binary_e2e_pp_smoke_run_handles_slow_middle_stage_boot() { fn binary_e2e_pp_orchestrator_handles_slow_middle_stage_boot() {
let opts = SmokeRunOpts { let opts = SmokeRunOpts {
num_stages: 4, num_stages: 4,
prompt: "Say hello".into(), prompt: "Say hello".into(),
@ -612,7 +612,7 @@ fn binary_e2e_pp_smoke_run_handles_slow_middle_stage_boot() {
#[test] #[test]
#[ignore] #[ignore]
fn binary_e2e_pp_smoke_run_handles_slow_last_stage_boot() { fn binary_e2e_pp_orchestrator_handles_slow_last_stage_boot() {
let opts = SmokeRunOpts { let opts = SmokeRunOpts {
num_stages: 4, num_stages: 4,
prompt: "Say hello".into(), prompt: "Say hello".into(),
@ -675,18 +675,18 @@ fn real_tinygrad_at(num_stages: u32) {
.unwrap_or_else(|| { .unwrap_or_else(|| {
let _ = smoke.kill(); let _ = smoke.kill();
let _ = smoke.wait(); let _ = smoke.wait();
panic!("pp-smoke-run did not spawn {num_stages} children within 120s"); panic!("pp-orchestrator did not spawn {num_stages} children within 120s");
}); });
let status = wait_with_timeout(&mut smoke, Duration::from_secs(1200)).unwrap_or_else(|| { let status = wait_with_timeout(&mut smoke, Duration::from_secs(1200)).unwrap_or_else(|| {
let _ = smoke.kill(); let _ = smoke.kill();
let _ = smoke.wait(); let _ = smoke.wait();
panic!("pp-smoke-run did not finish within 20m"); panic!("pp-orchestrator did not finish within 20m");
}); });
let stdout_lines = stdout.lines(); let stdout_lines = stdout.lines();
let stderr_joined = stderr.joined(); let stderr_joined = stderr.joined();
assert!( assert!(
status.success(), status.success(),
"pp-smoke-run exited {status:?}\n--- stdout ---\n{}\n--- stderr (tail) ---\n{}", "pp-orchestrator exited {status:?}\n--- stdout ---\n{}\n--- stderr (tail) ---\n{}",
stdout_lines.join("\n"), stdout_lines.join("\n"),
stderr_joined stderr_joined
.lines() .lines()
@ -708,7 +708,7 @@ fn real_tinygrad_at(num_stages: u32) {
for pid in pre_exit_pids { for pid in pre_exit_pids {
assert!( assert!(
!pid_alive(pid), !pid_alive(pid),
"pp-gpu-node child pid {pid} still alive after pp-smoke-run exit" "pp-worker child pid {pid} still alive after pp-orchestrator exit"
); );
} }
} }
@ -764,9 +764,9 @@ fn binary_e2e_real_tinygrad_response_matches_single_node_for_say_hello() {
let status = wait_with_timeout(&mut smoke, Duration::from_secs(1200)).unwrap_or_else(|| { let status = wait_with_timeout(&mut smoke, Duration::from_secs(1200)).unwrap_or_else(|| {
let _ = smoke.kill(); let _ = smoke.kill();
let _ = smoke.wait(); let _ = smoke.wait();
panic!("pp-smoke-run did not finish within 20m"); panic!("pp-orchestrator did not finish within 20m");
}); });
assert!(status.success(), "pp-smoke-run exited {status:?}"); assert!(status.success(), "pp-orchestrator exited {status:?}");
let response = extract_response(&stdout.lines()).expect("missing response banner"); let response = extract_response(&stdout.lines()).expect("missing response banner");
assert_eq!( assert_eq!(
response.trim(), response.trim(),

28
examples/pipeline-parallel-inference/tests/t_docker.rs
View file

@ -3,7 +3,7 @@
//! and verify clean teardown. //! and verify clean teardown.
//! //!
//! TEST_SPEC §13b. Mirrors §13's happy path and at least one failure //! TEST_SPEC §13b. Mirrors §13's happy path and at least one failure
//! scenario, but with each `pp-gpu-node` running inside its own //! scenario, but with each `pp-worker` running inside its own
//! container instead of as a host process. //! container instead of as a host process.
//! //!
//! Every test in this file is `#[ignore]`d and requires a working //! Every test in this file is `#[ignore]`d and requires a working
@ -144,7 +144,7 @@ fn wait_until_prefix_drains(prefix: &str, timeout: Duration) {
} }
/// Poll for "no leftover stage containers" with a short grace window. /// Poll for "no leftover stage containers" with a short grace window.
/// A clean `pp-smoke-run` exit triggers `docker run --rm` teardown on /// A clean `pp-orchestrator` exit triggers `docker run --rm` teardown on
/// each shim, but the daemon-side delete is not synchronous with the /// each shim, but the daemon-side delete is not synchronous with the
/// CLI's exit, so we give the daemon a moment to catch up before we /// CLI's exit, so we give the daemon a moment to catch up before we
/// call the run dirty. /// call the run dirty.
@ -359,9 +359,9 @@ fn cargo_build_release_once() {
.arg(crate_dir().join("Cargo.toml")) .arg(crate_dir().join("Cargo.toml"))
.arg("--release") .arg("--release")
.arg("--bin") .arg("--bin")
.arg("pp-gpu-node") .arg("pp-worker")
.arg("--bin") .arg("--bin")
.arg("pp-smoke-run") .arg("pp-orchestrator")
.status() .status()
.expect("invoke cargo build"); .expect("invoke cargo build");
assert!(status.success(), "cargo build --release failed"); assert!(status.success(), "cargo build --release failed");
@ -570,7 +570,7 @@ fn docker_e2e_premature_container_exit_fails_fast() {
.expect("docker build (code)"); .expect("docker build (code)");
assert!(build.success(), "docker build (code) failed"); assert!(build.success(), "docker build (code) failed");
let smoke_bin = crate_dir().join("target/release/pp-smoke-run"); let smoke_bin = crate_dir().join("target/release/pp-orchestrator");
let worker_py = crate_dir().join("pp_tinygrad_worker.py"); let worker_py = crate_dir().join("pp_tinygrad_worker.py");
assert!(smoke_bin.exists() && worker_py.exists()); assert!(smoke_bin.exists() && worker_py.exists());
@ -593,10 +593,10 @@ fn docker_e2e_premature_container_exit_fails_fast() {
.stdout(Stdio::piped()) .stdout(Stdio::piped())
.stderr(Stdio::piped()) .stderr(Stdio::piped())
.spawn() .spawn()
.expect("spawn pp-smoke-run with docker shim"); .expect("spawn pp-orchestrator with docker shim");
// Drain the child's stderr (and stdout) into in-memory buffers via // Drain the child's stderr (and stdout) into in-memory buffers via
// background reader threads. Without this any pp-smoke-run / shim // background reader threads. Without this any pp-orchestrator / shim
// diagnostic message is swallowed and the test gives the reader // diagnostic message is swallowed and the test gives the reader
// nothing actionable on failure. // nothing actionable on failure.
let stderr_buf = spawn_stream_collector(child.stderr.take().expect("child stderr piped")); let stderr_buf = spawn_stream_collector(child.stderr.take().expect("child stderr piped"));
@ -610,8 +610,8 @@ fn docker_e2e_premature_container_exit_fails_fast() {
let _ = child.wait(); let _ = child.wait();
panic!( panic!(
"3 stage containers did not start within 180s\n\ "3 stage containers did not start within 180s\n\
--- pp-smoke-run stderr (tail) ---\n{}\n\ --- pp-orchestrator stderr (tail) ---\n{}\n\
--- pp-smoke-run stdout (tail) ---\n{}", --- pp-orchestrator stdout (tail) ---\n{}",
tail_lines(&stderr_buf.snapshot(), 80), tail_lines(&stderr_buf.snapshot(), 80),
tail_lines(&stdout_buf.snapshot(), 40), tail_lines(&stdout_buf.snapshot(), 40),
); );
@ -634,15 +634,15 @@ fn docker_e2e_premature_container_exit_fails_fast() {
// the docker-shim child exit promptly). // the docker-shim child exit promptly).
let deadline = Instant::now() + Duration::from_secs(60); let deadline = Instant::now() + Duration::from_secs(60);
let status = loop { let status = loop {
match child.try_wait().expect("try_wait pp-smoke-run") { match child.try_wait().expect("try_wait pp-orchestrator") {
Some(s) => break s, Some(s) => break s,
None if Instant::now() >= deadline => { None if Instant::now() >= deadline => {
let _ = child.kill(); let _ = child.kill();
let _ = child.wait(); let _ = child.wait();
panic!( panic!(
"pp-smoke-run did not exit within 60s of killing a container\n\ "pp-orchestrator did not exit within 60s of killing a container\n\
--- pp-smoke-run stderr (tail) ---\n{}\n\ --- pp-orchestrator stderr (tail) ---\n{}\n\
--- pp-smoke-run stdout (tail) ---\n{}", --- pp-orchestrator stdout (tail) ---\n{}",
tail_lines(&stderr_buf.snapshot(), 80), tail_lines(&stderr_buf.snapshot(), 80),
tail_lines(&stdout_buf.snapshot(), 40), tail_lines(&stdout_buf.snapshot(), 40),
); );
@ -653,7 +653,7 @@ fn docker_e2e_premature_container_exit_fails_fast() {
assert!( assert!(
!status.success(), !status.success(),
"expected non-zero exit after container kill, got {status:?}\n\ "expected non-zero exit after container kill, got {status:?}\n\
--- pp-smoke-run stderr (tail) ---\n{}", --- pp-orchestrator stderr (tail) ---\n{}",
tail_lines(&stderr_buf.snapshot(), 80), tail_lines(&stderr_buf.snapshot(), 80),
); );

2
examples/pipeline-parallel-inference/tests/t_integration.rs
View file

@ -10,7 +10,7 @@
//! to stage 0, and so on until EOS or `max_tokens`. The last stage emits the //! to stage 0, and so on until EOS or `max_tokens`. The last stage emits the
//! final `InferenceResponse` back to the orchestrator. //! final `InferenceResponse` back to the orchestrator.
//! //!
//! The same shape the binary uses (see `pp_smoke_run.rs`), minus the child //! The same shape the binary uses (see `pp_orchestrator.rs`), minus the child
//! subprocesses — actors live in this test process and addresses are wired up //! subprocesses — actors live in this test process and addresses are wired up
//! directly without SWIM resolution. //! directly without SWIM resolution.

4
examples/pipeline-parallel-inference/tests/t_orchestrator.rs
View file

@ -2,7 +2,7 @@
//! lease chain (mocked HTTP), and the convergence-wait helper. //! lease chain (mocked HTTP), and the convergence-wait helper.
//! //!
//! None of these tests touch real iroh, real vast.ai, or the //! None of these tests touch real iroh, real vast.ai, or the
//! `pp-gpu-node` binary. The spawn chain is exercised with a `sh -c` //! `pp-worker` binary. The spawn chain is exercised with a `sh -c`
//! "fake child" that prints `PP_GPU_NODE_ADDR ...` and then sleeps; the //! "fake child" that prints `PP_GPU_NODE_ADDR ...` and then sleeps; the
//! vast.ai chain uses `wiremock`; the convergence-wait helper is a //! vast.ai chain uses `wiremock`; the convergence-wait helper is a
//! pure function fed a closure. //! pure function fed a closure.
@ -151,7 +151,7 @@ fn spawn_chain_propagates_each_stage_peer_direct_to_successor() {
} }
#[test] #[test]
fn spawn_chain_reads_pp_gpu_node_addr_in_order() { fn spawn_chain_reads_addr_announcement_in_order() {
let tmp = TempDir::new("reads-in-order"); let tmp = TempDir::new("reads-in-order");
let pids_file = tmp.child("pids"); let pids_file = tmp.child("pids");