2026-08-17 22:51:08 +00:00
|
|
|
use std::process::{Command, ExitCode};
|
|
|
|
|
use std::time::Instant;
|
2026-02-15 17:03:31 +00:00
|
|
|
|
|
|
|
|
struct TestStep {
|
|
|
|
|
label: &'static str,
|
|
|
|
|
args: &'static [&'static str],
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-25 12:30:18 +00:00
|
|
|
const BASIC_TESTS: &[TestStep] = &[
|
|
|
|
|
TestStep {
|
|
|
|
|
label: "root crate",
|
|
|
|
|
args: &["test"],
|
|
|
|
|
},
|
|
|
|
|
TestStep {
|
2026-08-15 08:17:48 +00:00
|
|
|
label: "telemetry",
|
|
|
|
|
args: &["test", "-p", "telemetry"],
|
2026-06-25 12:30:18 +00:00
|
|
|
},
|
|
|
|
|
TestStep {
|
|
|
|
|
label: "distribution",
|
|
|
|
|
args: &["test", "-p", "distribution"],
|
|
|
|
|
},
|
|
|
|
|
TestStep {
|
|
|
|
|
label: "iroh-driver",
|
|
|
|
|
args: &["test", "-p", "iroh-driver"],
|
|
|
|
|
},
|
|
|
|
|
TestStep {
|
refactor: `mvp-system` is now a standalone app, `myelin`
Promote the `mvp-system` workspace library crate to a standalone application at `apps/myelin`, rebranding the MVP system along with its binaries, node image, and spec.
- workspace `Cargo.toml`: swap member `crates/mvp-system` -> `apps/myelin` and drop `apps` from `exclude` so the app joins the workspace
- `apps/myelin/Cargo.toml`: declare package `myelin` with `autobins = false` and explicit `[[bin]]` targets `myelin-worker`, `myelin-orchestrator`, `myelin-chat`
- `apps/myelin/src`: move the whole `mvp-system` source tree and rebrand module surfaces (`chat/mod.rs`, `prompt/mod.rs`); add `bin/chat.rs` (`myelin::run_chat_from_args`) and delete the old `mvp_chat.rs`
- `apps/myelin/node-image`: relocate the worker image assets from `apps/mvp-node/` (Dockerfile, Dockerfile.base, tinygrad_worker.py, entrypoint, e2e script) and rename `MVP_SYSTEM_SPEC.md` -> `MYELIN_SPEC.md`
- `xtask`: rewrite build/reference paths for the rename (~1000-line churn); add `crates/dashboard/ACTOR_PANEL_SPEC.md`
Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-08-01 09:46:55 +00:00
|
|
|
label: "myelin",
|
|
|
|
|
args: &["test", "-p", "myelin"],
|
2026-06-25 12:30:18 +00:00
|
|
|
},
|
|
|
|
|
TestStep {
|
|
|
|
|
label: "swactor-process",
|
|
|
|
|
args: &["test", "-p", "swactor-process"],
|
|
|
|
|
},
|
|
|
|
|
TestStep {
|
|
|
|
|
label: "swactor-transport",
|
|
|
|
|
args: &["test", "-p", "swactor-transport"],
|
|
|
|
|
},
|
|
|
|
|
TestStep {
|
|
|
|
|
label: "dashboard",
|
|
|
|
|
args: &["test", "-p", "dashboard"],
|
|
|
|
|
},
|
|
|
|
|
TestStep {
|
|
|
|
|
label: "swactor-vastai",
|
|
|
|
|
args: &["test", "-p", "swactor-vastai"],
|
|
|
|
|
},
|
|
|
|
|
TestStep {
|
|
|
|
|
label: "xtask",
|
|
|
|
|
args: &["test", "-p", "xtask"],
|
|
|
|
|
},
|
|
|
|
|
];
|
2026-02-24 09:12:28 +00:00
|
|
|
|
2026-06-25 12:30:18 +00:00
|
|
|
fn cargo_bin() -> String {
|
|
|
|
|
option_env!("CARGO")
|
|
|
|
|
.map(str::to_string)
|
|
|
|
|
.unwrap_or_else(|| "cargo".to_string())
|
2026-02-15 17:03:31 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn print_usage() {
|
|
|
|
|
println!(
|
|
|
|
|
"\
|
2026-07-05 09:59:51 +00:00
|
|
|
USAGE: cargo xtask <command>
|
2026-02-15 17:03:31 +00:00
|
|
|
|
2026-06-25 12:30:18 +00:00
|
|
|
COMMANDS:
|
demo: rename xtask demo command; dashboard-established data-plane edges
Rename `cargo xtask provisioning-reconciler-demo` to `cargo xtask demo`
(CLI dispatch, help, child re-exec argv, launch spec strings, module dir
xtask/src/provisioning_demo -> xtask/src/demo).
Add iteration-1 data-plane edges, established from Fleet Control:
- Fleet Control "edge" button -> POST /control/edge (new
ControlCommand::EstablishEdge) -> supervisor actor resolves the node's
advertised EndpointAddr (stashed in NodeRuntime by AnnounceActor) and
provisions a real outbound EdgeRuntime (arena ring lease, recorder
WorkerPort, EDGE_ALPN send pump) in a new edge pump thread.
- Node gains EDGE_ALPN, an actor bridge decoding EdgeProvision gossip,
and a NodeEdgeAgent that provisions its (single) inbound edge, polls
it, mirrors observations onto the node.edge telemetry channel
(render-only), and answers EdgeAck gossip which terminates the
supervisor's provision retries. Node teardown replaces its inbound on
re-provision; supervisor replaces sessions per node and tears them
down on node exit/replacement/shutdown.
- The edge pump runs on the engine's blocking pool with sole session
ownership (commands in, state mirror + feed lines out): the connect
handshake blocks its thread and must not run on a Tokio worker or
share a lock with the actor. Connects are bounded (10s) so a dead
node faults its session instead of wedging edge polling.
- iroh-driver: retain_telemetry_connections() opts an application out
of the driver-owned TELEMETRY_ALPN ingress so the node's pull server
can drain those connections itself (the actor-bridge pump would
otherwise claim them).
- Dashboard: edges array in the reconciler snapshot, per-node edge
badges and edge button in Fleet Control, node_edges render mirror.
2026-08-16 20:11:20 +00:00
|
|
|
demo [--port n] [--nodes n] [--docker]
|
2026-08-17 22:51:08 +00:00
|
|
|
Run the visual provisioning-reconciler demo.
|
|
|
|
|
check-telemetry-isolation
|
|
|
|
|
Verify no frame types appear in control-plane modules.
|
|
|
|
|
test Run the non-binding repository test barrier."
|
2026-02-15 17:03:31 +00:00
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-25 12:30:18 +00:00
|
|
|
fn run_step(step: &TestStep) -> bool {
|
|
|
|
|
println!("\n=== {} ===", step.label);
|
|
|
|
|
println!(" cargo {}", step.args.join(" "));
|
|
|
|
|
println!();
|
2026-02-15 17:03:31 +00:00
|
|
|
|
2026-06-25 12:30:18 +00:00
|
|
|
match Command::new(cargo_bin()).args(step.args).status() {
|
|
|
|
|
Ok(status) => status.success(),
|
|
|
|
|
Err(error) => {
|
|
|
|
|
eprintln!("Failed to execute cargo: {error}");
|
|
|
|
|
false
|
2026-02-15 17:03:31 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-25 12:30:18 +00:00
|
|
|
fn run_tests() -> ExitCode {
|
2026-02-15 17:03:31 +00:00
|
|
|
let start = Instant::now();
|
2026-06-25 12:30:18 +00:00
|
|
|
for (index, step) in BASIC_TESTS.iter().enumerate() {
|
|
|
|
|
if !run_step(step) {
|
|
|
|
|
eprintln!(
|
|
|
|
|
"\n--- FAILED after {:.1}s ({index} passed, 1 failed) ---",
|
|
|
|
|
start.elapsed().as_secs_f64()
|
|
|
|
|
);
|
|
|
|
|
return ExitCode::from(1);
|
2026-02-15 17:03:31 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
println!(
|
2026-06-25 12:30:18 +00:00
|
|
|
"\n--- All {} step(s) passed in {:.1}s ---",
|
|
|
|
|
BASIC_TESTS.len(),
|
|
|
|
|
start.elapsed().as_secs_f64()
|
2026-02-15 17:03:31 +00:00
|
|
|
);
|
2026-06-25 12:30:18 +00:00
|
|
|
ExitCode::SUCCESS
|
2026-02-15 17:03:31 +00:00
|
|
|
}
|
2026-02-16 15:39:27 +00:00
|
|
|
|
2026-08-17 22:51:08 +00:00
|
|
|
/// Verify that control-plane modules never import telemetry frame/read-side
|
|
|
|
|
/// types. They may emit through the producer API only.
|
|
|
|
|
fn check_telemetry_isolation() -> ExitCode {
|
|
|
|
|
const CONTROL_DIRS: &[&str] = &[
|
|
|
|
|
"apps/myelin/src/orchestration",
|
|
|
|
|
"crates/distribution/src",
|
|
|
|
|
"crates/data-plane/src",
|
|
|
|
|
"crates/provisioning/src",
|
|
|
|
|
];
|
|
|
|
|
const FORBIDDEN: &[&str] = &[
|
|
|
|
|
"telemetry::frame::",
|
|
|
|
|
"telemetry::store::",
|
|
|
|
|
"telemetry::ingest::",
|
|
|
|
|
"telemetry::views::",
|
|
|
|
|
"telemetry::transport::",
|
|
|
|
|
"CollectedTelemetryFrame",
|
|
|
|
|
];
|
2026-07-05 09:59:51 +00:00
|
|
|
|
2026-08-17 22:51:08 +00:00
|
|
|
let mut files = Vec::new();
|
|
|
|
|
for dir in CONTROL_DIRS {
|
|
|
|
|
collect_rs_files(dir, &mut files);
|
feat: mvp-chat benchmarking
Add end-to-end timing instrumentation and an xtask benchmark report for mvp-chat runs.
- benchmark_observability: add a shared stamping module — stamp(component) emitting schema/pid/monotonic+wall ms from a process-global start and sequence counter, plus unix_ms_now() — stamped onto every mvp-chat/orchestrator/worker-node event and frame-archive record
- mvp-chat: thread a run_id (new --run-id, defaults to 1) through config and the orchestrator CLI, add per-phase started/ready/failed emits for ensure_orch_binary/ensure_worker_binary/prepare_node_image, and a prompt_complete record carrying tokens_generated/elapsed_ms/final_text bytes
- orchestrator/worker-node: stamp bootstrap and prompt events, add arrival_unix_ms to archived frames, propagate MVP_RUN_ID/MVP_LOGICAL_NODE_ID/MVP_STAGE_INDEX into the tinygrad worker, default the device to CPU for the process provider, emit a prompt_rpc started span, and drop the MVP_TINYGRAD_TEST_MODE passthrough
- tinygrad_worker.py: stamp every control() event and tag it with run/node/stage env, add a CPU:X86 fallback when clang is absent, and remove the test_mode() short-circuits
- xtask: replace the flat dump-log fact assertions with a benchmark report builder (build_benchmark_report) that requires named spans (prepare_runtime, ensure_*_binary, weights_loaded, prompt_rpc) and emits per-prompt first-token/decode/tokens-per-second latency; wrap the cargo run in XtaskBenchmark synthetic frames and pass a unix-ms --run-id
Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-23 05:35:04 +00:00
|
|
|
}
|
|
|
|
|
|
2026-08-17 22:51:08 +00:00
|
|
|
let mut found = false;
|
|
|
|
|
for file in &files {
|
|
|
|
|
let Ok(src) = std::fs::read_to_string(file) else {
|
|
|
|
|
continue;
|
|
|
|
|
};
|
|
|
|
|
for (lineno, line) in src.lines().enumerate() {
|
|
|
|
|
for pattern in FORBIDDEN {
|
|
|
|
|
if line.contains(pattern) {
|
|
|
|
|
eprintln!(
|
|
|
|
|
"telemetry-isolation violation: {file}:{}: {}",
|
|
|
|
|
lineno + 1,
|
|
|
|
|
line.trim()
|
|
|
|
|
);
|
|
|
|
|
found = true;
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-07-05 09:59:51 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-17 22:51:08 +00:00
|
|
|
if found {
|
2026-07-24 19:05:11 +00:00
|
|
|
eprintln!(
|
2026-08-17 22:51:08 +00:00
|
|
|
"\ntelemetry-isolation: control-plane code must not import frame types \
|
|
|
|
|
or read-side modules. Use the telemetry producer API for emission."
|
2026-07-24 19:05:11 +00:00
|
|
|
);
|
2026-08-17 22:51:08 +00:00
|
|
|
ExitCode::from(1)
|
feat: mvp-chat benchmarking
Add end-to-end timing instrumentation and an xtask benchmark report for mvp-chat runs.
- benchmark_observability: add a shared stamping module — stamp(component) emitting schema/pid/monotonic+wall ms from a process-global start and sequence counter, plus unix_ms_now() — stamped onto every mvp-chat/orchestrator/worker-node event and frame-archive record
- mvp-chat: thread a run_id (new --run-id, defaults to 1) through config and the orchestrator CLI, add per-phase started/ready/failed emits for ensure_orch_binary/ensure_worker_binary/prepare_node_image, and a prompt_complete record carrying tokens_generated/elapsed_ms/final_text bytes
- orchestrator/worker-node: stamp bootstrap and prompt events, add arrival_unix_ms to archived frames, propagate MVP_RUN_ID/MVP_LOGICAL_NODE_ID/MVP_STAGE_INDEX into the tinygrad worker, default the device to CPU for the process provider, emit a prompt_rpc started span, and drop the MVP_TINYGRAD_TEST_MODE passthrough
- tinygrad_worker.py: stamp every control() event and tag it with run/node/stage env, add a CPU:X86 fallback when clang is absent, and remove the test_mode() short-circuits
- xtask: replace the flat dump-log fact assertions with a benchmark report builder (build_benchmark_report) that requires named spans (prepare_runtime, ensure_*_binary, weights_loaded, prompt_rpc) and emits per-prompt first-token/decode/tokens-per-second latency; wrap the cargo run in XtaskBenchmark synthetic frames and pass a unix-ms --run-id
Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-23 05:35:04 +00:00
|
|
|
} else {
|
2026-08-17 22:51:08 +00:00
|
|
|
println!("telemetry-isolation: OK — no frame types in control-plane modules.");
|
|
|
|
|
ExitCode::SUCCESS
|
2026-07-22 07:50:53 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-17 22:51:08 +00:00
|
|
|
fn collect_rs_files(dir: &str, out: &mut Vec<String>) {
|
|
|
|
|
let Ok(entries) = std::fs::read_dir(dir) else {
|
|
|
|
|
return;
|
feat: working 7B inferenced over 4 pipeline stages
Land end-to-end canonical benchmark observability and a synthetic datastream-connectivity preflight across the orchestrator, chat, worker-node, and Python tinygrad worker, plus an xtask validator, so a 4-stage 7B pipeline run is fully diagnosable.
- benchmark_observability: expand stamp() with canonical producer fields (producer_component/instance_id/process_id/sequence, wall_clock_unix_ms, monotonic_ms, clock_source) and schema_version so every event shares one envelope shape
- orchestrator_app + bin/{mvp_chat,worker_node}: stamp OrchBootstrap/OrchPromptEvent/ChatProgress/NodeEvent/SamplerHealth with the canonical fields plus span_id/parent_span_id, and add a 4-phase synthetic datastream preflight (ProducerConfigured/Connected/SyntheticEventSent/Observed) plus an endpoint_config_snapshot event on each process
- apps/mvp-node/tinygrad_worker: add apply_canonical_envelope()/datastream_endpoint_snapshot() and emit_python_datastream_preflight() mirroring the Rust preflight, and enrich benchmark_stamp() with the same producer fields
- bin/worker_node: pass MVP_DATASTREAM_ENDPOINT_ID/MVP_BENCHMARK_PRODUCER_INSTANCE/MVP_IROH_ENDPOINT_ADDR_MASK/MVP_IROH_RELAY_MODE env to the spawned tinygrad worker so its stamps identify the stage
- bin/mvp_chat: add --pipeline-parallel as an alias for --pipeline-stages (with a duplicate-guard) and bump recursion_limit
- xtask: add a benchmark-observability validator (ValidatorFinding/BenchmarkValidation, validate_benchmark_observability, canonical-stamp and stage/edge checks, evidence + gap-report builders) with tests for missing python datastream connectivity, wrong run_id, and missing span_id
Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-27 06:47:17 +00:00
|
|
|
};
|
2026-08-17 22:51:08 +00:00
|
|
|
for entry in entries.flatten() {
|
|
|
|
|
let path = entry.path();
|
|
|
|
|
if path.is_dir() {
|
|
|
|
|
if let Some(path) = path.to_str() {
|
|
|
|
|
collect_rs_files(path, out);
|
|
|
|
|
}
|
|
|
|
|
} else if path.extension().is_some_and(|extension| extension == "rs")
|
|
|
|
|
&& let Some(path) = path.to_str()
|
|
|
|
|
{
|
|
|
|
|
out.push(path.to_owned());
|
feat: successful 8 stage pipeline parallel run, more metrics
Complete an 8-stage pipeline-parallel run over VastAI by provisioning stages high-to-low, adding per-stage/per-step metrics, host anti-colocation, and provider state-timeout guardrails.
- orchestrator_app: select the next weight-load stage by max index (provision stages high-to-low for parallel spread), add a throttled "loaded N of M; waiting on stage X" stage_provision_wait headline, and surface min_compute_cap/state_timeout_secs in the config dump.
- orchestrator_app: enrich pipeline_token_in/out and tokenizer_decode events with token_count/token_ids/generated_index.
- worker_node: add timing metrics across the data path (helper_execute_ms, egress_ring_read_ms, send_ms, ingress_ring_write_ms, object_load_ms), refactor take_complete_ingress_record into IngressRecordBytes (object_id/sequence/extent/flags), and emit a new object_loaded event.
- vastai_provisioning: track leased host_ids and blacklist already-leased hosts in later ProvisionRequests so stages don't co-locate, and tag SSH-bootstrap retry logs with the attempt number.
- tools/vastai: add min_compute_cap (PP_MIN_COMPUTE_CAP) filter/search query and a LifecyclePolicy state_timeout (PP_STATE_TIMEOUT_SECS) that fails instances stuck in a non-running status instead of polling forever.
- xtask: raise the check timeout to 1800s/30s grace, drop --skip-rebuild for VastAI, aggregate per-stage StepExecuted metrics, add a vastai summary section, and write failure artifacts on abort.
Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-25 16:04:57 +00:00
|
|
|
}
|
2026-07-22 07:50:53 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-17 22:51:08 +00:00
|
|
|
mod demo;
|
2026-08-15 14:10:52 +00:00
|
|
|
|
2026-05-21 19:48:02 +00:00
|
|
|
fn main() -> ExitCode {
|
2026-06-25 12:30:18 +00:00
|
|
|
let mut args = std::env::args().skip(1);
|
2026-07-05 09:59:51 +00:00
|
|
|
match args.next().as_deref() {
|
enforce datastream telemetry-only invariant: ban frame types from control code
The datastream is metrics/logging only; control decisions must never branch
on a frame. This was a recurring cultural problem with no structural
enforcement. This change makes it a compile-time and CI-enforced fact.
datastream crate (lib.rs):
- Stop re-exporting Frame, DatastreamEvent, FrameDelivery at crate root.
is now a compile error (E0425). These types live
only in datastream::frame::* and are documented as the observer surface.
- Safe identity types (ChannelId, StreamId, Position, Record, etc.) remain
re-exported at root for producer-side callers.
orchestration/app.rs:
- Extracted all frame-touching code (CollectedDatastreamFrame,
drain_datastream_connections, update_load_progress_from_frame,
drain_frames, archive_collected_frame, pump, OrchDatastream,
DashboardSupport) into two new observability modules:
frame_collector.rs and orch_datastream.rs.
- The orchestrator now interacts through a FrameCollector whose
drain/drain_with_progress methods take closures; it never names Frame,
DatastreamEvent, or CollectedDatastreamFrame.
- StageLoadProgress (the one control-relevant signal previously scraped
from frame payloads) is extracted inside FrameCollector and handed to
the control loop as plain data.
xtask:
- New check-telemetry-isolation command scans control-plane modules
(orchestration/, distribution/, data-plane/, provisioning/) for
forbidden frame-type references and fails the build if any are found.
Verified: workspace builds (myelin + dashboard feature), datastream 29
tests pass, myelin 64 lib tests pass, check-telemetry-isolation passes
clean.
Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-08-12 12:14:57 +00:00
|
|
|
Some("check-telemetry-isolation") => check_telemetry_isolation(),
|
2026-07-05 09:59:51 +00:00
|
|
|
Some("test") if args.next().is_none() => run_tests(),
|
2026-08-17 22:51:08 +00:00
|
|
|
Some("demo") => demo::run(&args.collect::<Vec<_>>()),
|
2026-07-05 09:59:51 +00:00
|
|
|
Some("help" | "--help" | "-h") | None => {
|
2026-06-25 12:30:18 +00:00
|
|
|
print_usage();
|
2026-05-21 19:48:02 +00:00
|
|
|
ExitCode::SUCCESS
|
|
|
|
|
}
|
2026-06-25 12:30:18 +00:00
|
|
|
_ => {
|
2026-02-24 09:12:28 +00:00
|
|
|
print_usage();
|
2026-05-21 19:48:02 +00:00
|
|
|
ExitCode::from(1)
|
2026-02-16 15:39:27 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|