2026-08-22 17:18:25 +00:00
|
|
|
use std::path::{Path, PathBuf};
|
|
|
|
|
use std::process::{Command, ExitCode, Stdio};
|
2026-08-17 22:51:08 +00:00
|
|
|
use std::time::Instant;
|
2026-02-15 17:03:31 +00:00
|
|
|
|
|
|
|
|
struct TestStep {
|
|
|
|
|
label: &'static str,
|
|
|
|
|
args: &'static [&'static str],
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-22 17:18:25 +00:00
|
|
|
const TEST_STEPS: &[TestStep] = &[
|
2026-06-25 12:30:18 +00:00
|
|
|
TestStep {
|
2026-08-22 17:18:25 +00:00
|
|
|
label: "strict workspace lint",
|
|
|
|
|
args: &["lint"],
|
2026-06-25 12:30:18 +00:00
|
|
|
},
|
|
|
|
|
TestStep {
|
2026-08-22 17:18:25 +00:00
|
|
|
label: "all Rust tests (60s per-test timeout)",
|
|
|
|
|
args: &["nextest", "run", "--workspace", "--all-features"],
|
2026-06-25 12:30:18 +00:00
|
|
|
},
|
|
|
|
|
TestStep {
|
2026-08-22 17:18:25 +00:00
|
|
|
label: "all Rust doctests",
|
|
|
|
|
args: &[
|
|
|
|
|
"test",
|
|
|
|
|
"--workspace",
|
|
|
|
|
"--all-features",
|
|
|
|
|
"--doc",
|
|
|
|
|
"--exclude",
|
|
|
|
|
"python",
|
|
|
|
|
"--exclude",
|
|
|
|
|
"wasm-runtime",
|
|
|
|
|
],
|
2026-06-25 12:30:18 +00:00
|
|
|
},
|
|
|
|
|
];
|
2026-02-24 09:12:28 +00:00
|
|
|
|
2026-06-25 12:30:18 +00:00
|
|
|
fn cargo_bin() -> String {
|
|
|
|
|
option_env!("CARGO")
|
|
|
|
|
.map(str::to_string)
|
|
|
|
|
.unwrap_or_else(|| "cargo".to_string())
|
2026-02-15 17:03:31 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn print_usage() {
|
|
|
|
|
println!(
|
|
|
|
|
"\
|
2026-07-05 09:59:51 +00:00
|
|
|
USAGE: cargo xtask <command>
|
2026-02-15 17:03:31 +00:00
|
|
|
|
2026-06-25 12:30:18 +00:00
|
|
|
COMMANDS:
|
demo: rename xtask demo command; dashboard-established data-plane edges
Rename `cargo xtask provisioning-reconciler-demo` to `cargo xtask demo`
(CLI dispatch, help, child re-exec argv, launch spec strings, module dir
xtask/src/provisioning_demo -> xtask/src/demo).
Add iteration-1 data-plane edges, established from Fleet Control:
- Fleet Control "edge" button -> POST /control/edge (new
ControlCommand::EstablishEdge) -> supervisor actor resolves the node's
advertised EndpointAddr (stashed in NodeRuntime by AnnounceActor) and
provisions a real outbound EdgeRuntime (arena ring lease, recorder
WorkerPort, EDGE_ALPN send pump) in a new edge pump thread.
- Node gains EDGE_ALPN, an actor bridge decoding EdgeProvision gossip,
and a NodeEdgeAgent that provisions its (single) inbound edge, polls
it, mirrors observations onto the node.edge telemetry channel
(render-only), and answers EdgeAck gossip which terminates the
supervisor's provision retries. Node teardown replaces its inbound on
re-provision; supervisor replaces sessions per node and tears them
down on node exit/replacement/shutdown.
- The edge pump runs on the engine's blocking pool with sole session
ownership (commands in, state mirror + feed lines out): the connect
handshake blocks its thread and must not run on a Tokio worker or
share a lock with the actor. Connects are bounded (10s) so a dead
node faults its session instead of wedging edge polling.
- iroh-driver: retain_telemetry_connections() opts an application out
of the driver-owned TELEMETRY_ALPN ingress so the node's pull server
can drain those connections itself (the actor-bridge pump would
otherwise claim them).
- Dashboard: edges array in the reconciler snapshot, per-node edge
badges and edge button in Fleet Control, node_edges render mirror.
2026-08-16 20:11:20 +00:00
|
|
|
demo [--port n] [--nodes n] [--docker]
|
2026-08-17 22:51:08 +00:00
|
|
|
Run the visual provisioning-reconciler demo.
|
|
|
|
|
check-telemetry-isolation
|
|
|
|
|
Verify no frame types appear in control-plane modules.
|
2026-08-22 17:18:25 +00:00
|
|
|
test Run strict lint plus every Rust and Python test."
|
2026-02-15 17:03:31 +00:00
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-22 17:18:25 +00:00
|
|
|
fn run_step(step: &TestStep, python: &Path) -> bool {
|
2026-06-25 12:30:18 +00:00
|
|
|
println!("\n=== {} ===", step.label);
|
|
|
|
|
println!(" cargo {}", step.args.join(" "));
|
|
|
|
|
println!();
|
2026-02-15 17:03:31 +00:00
|
|
|
|
2026-08-22 17:18:25 +00:00
|
|
|
match swactor_process::command_status(
|
|
|
|
|
Command::new(cargo_bin())
|
|
|
|
|
.args(step.args)
|
|
|
|
|
.env("PYO3_PYTHON", python),
|
|
|
|
|
) {
|
2026-06-25 12:30:18 +00:00
|
|
|
Ok(status) => status.success(),
|
|
|
|
|
Err(error) => {
|
|
|
|
|
eprintln!("Failed to execute cargo: {error}");
|
|
|
|
|
false
|
2026-02-15 17:03:31 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-25 12:30:18 +00:00
|
|
|
fn run_tests() -> ExitCode {
|
2026-02-15 17:03:31 +00:00
|
|
|
let start = Instant::now();
|
2026-08-22 17:18:25 +00:00
|
|
|
let Some(python) = PythonTestTools::discover() else {
|
|
|
|
|
return ExitCode::from(1);
|
|
|
|
|
};
|
|
|
|
|
if !nextest_available() {
|
|
|
|
|
return ExitCode::from(1);
|
|
|
|
|
}
|
|
|
|
|
if !check_telemetry_isolation() {
|
|
|
|
|
return ExitCode::from(1);
|
|
|
|
|
}
|
|
|
|
|
for (index, step) in TEST_STEPS.iter().enumerate() {
|
|
|
|
|
if !run_step(step, &python.python) {
|
2026-06-25 12:30:18 +00:00
|
|
|
eprintln!(
|
|
|
|
|
"\n--- FAILED after {:.1}s ({index} passed, 1 failed) ---",
|
|
|
|
|
start.elapsed().as_secs_f64()
|
|
|
|
|
);
|
|
|
|
|
return ExitCode::from(1);
|
2026-02-15 17:03:31 +00:00
|
|
|
}
|
|
|
|
|
}
|
2026-08-22 17:18:25 +00:00
|
|
|
if !python.run() {
|
|
|
|
|
eprintln!(
|
|
|
|
|
"\n--- FAILED after {:.1}s (Python tests failed) ---",
|
|
|
|
|
start.elapsed().as_secs_f64()
|
|
|
|
|
);
|
|
|
|
|
return ExitCode::from(1);
|
|
|
|
|
}
|
2026-02-15 17:03:31 +00:00
|
|
|
|
|
|
|
|
println!(
|
2026-06-25 12:30:18 +00:00
|
|
|
"\n--- All {} step(s) passed in {:.1}s ---",
|
2026-08-22 17:18:25 +00:00
|
|
|
TEST_STEPS.len() + 1,
|
2026-06-25 12:30:18 +00:00
|
|
|
start.elapsed().as_secs_f64()
|
2026-02-15 17:03:31 +00:00
|
|
|
);
|
2026-06-25 12:30:18 +00:00
|
|
|
ExitCode::SUCCESS
|
2026-02-15 17:03:31 +00:00
|
|
|
}
|
2026-02-16 15:39:27 +00:00
|
|
|
|
2026-08-22 17:18:25 +00:00
|
|
|
fn nextest_available() -> bool {
|
|
|
|
|
let available = swactor_process::command_status(
|
|
|
|
|
Command::new(cargo_bin())
|
|
|
|
|
.args(["nextest", "--version"])
|
|
|
|
|
.stdout(Stdio::null())
|
|
|
|
|
.stderr(Stdio::null()),
|
|
|
|
|
)
|
|
|
|
|
.is_ok_and(|status| status.success());
|
|
|
|
|
if !available {
|
|
|
|
|
eprintln!("cargo-nextest is required; install it from https://nexte.st/docs/installation/");
|
|
|
|
|
}
|
|
|
|
|
available
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
struct PythonTestTools {
|
|
|
|
|
directory: PathBuf,
|
|
|
|
|
maturin: PathBuf,
|
|
|
|
|
python: PathBuf,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
impl PythonTestTools {
|
|
|
|
|
fn discover() -> Option<Self> {
|
|
|
|
|
let directory = workspace_root().join("crates/bindings/python");
|
|
|
|
|
let tools = Self {
|
|
|
|
|
maturin: directory.join(".venv/bin/maturin"),
|
|
|
|
|
python: directory.join(".venv/bin/python"),
|
|
|
|
|
directory,
|
|
|
|
|
};
|
|
|
|
|
if tools.maturin.is_file() && tools.python.is_file() {
|
|
|
|
|
Some(tools)
|
|
|
|
|
} else {
|
|
|
|
|
eprintln!(
|
|
|
|
|
"Python test environment is missing; run `uv sync --project {}` first",
|
|
|
|
|
tools.directory.display()
|
|
|
|
|
);
|
|
|
|
|
None
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn run(&self) -> bool {
|
|
|
|
|
println!("\n=== all Python tests ===");
|
|
|
|
|
let built = swactor_process::command_status(
|
|
|
|
|
Command::new(&self.maturin)
|
|
|
|
|
.current_dir(&self.directory)
|
|
|
|
|
.arg("develop")
|
|
|
|
|
.env("PYO3_PYTHON", &self.python),
|
|
|
|
|
)
|
|
|
|
|
.is_ok_and(|status| status.success());
|
|
|
|
|
if !built {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
swactor_process::command_status(
|
|
|
|
|
Command::new(&self.python)
|
|
|
|
|
.current_dir(&self.directory)
|
|
|
|
|
.args([
|
|
|
|
|
"-m",
|
|
|
|
|
"pytest",
|
|
|
|
|
"-q",
|
|
|
|
|
"--timeout=60",
|
|
|
|
|
"tests/test_bootstrap.py",
|
|
|
|
|
"../../../tests/test_python.py",
|
|
|
|
|
]),
|
|
|
|
|
)
|
|
|
|
|
.is_ok_and(|status| status.success())
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn workspace_root() -> PathBuf {
|
|
|
|
|
Path::new(env!("CARGO_MANIFEST_DIR"))
|
|
|
|
|
.parent()
|
|
|
|
|
.expect("xtask must live directly under the workspace root")
|
|
|
|
|
.to_path_buf()
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-17 22:51:08 +00:00
|
|
|
/// Verify that control-plane modules never import telemetry frame/read-side
|
|
|
|
|
/// types. They may emit through the producer API only.
|
2026-08-22 17:18:25 +00:00
|
|
|
fn check_telemetry_isolation() -> bool {
|
2026-08-17 22:51:08 +00:00
|
|
|
const CONTROL_DIRS: &[&str] = &[
|
|
|
|
|
"apps/myelin/src/orchestration",
|
|
|
|
|
"crates/distribution/src",
|
|
|
|
|
"crates/data-plane/src",
|
|
|
|
|
"crates/provisioning/src",
|
|
|
|
|
];
|
|
|
|
|
const FORBIDDEN: &[&str] = &[
|
|
|
|
|
"telemetry::frame::",
|
|
|
|
|
"telemetry::store::",
|
|
|
|
|
"telemetry::ingest::",
|
|
|
|
|
"telemetry::views::",
|
|
|
|
|
"telemetry::transport::",
|
|
|
|
|
"CollectedTelemetryFrame",
|
|
|
|
|
];
|
2026-07-05 09:59:51 +00:00
|
|
|
|
2026-08-17 22:51:08 +00:00
|
|
|
let mut files = Vec::new();
|
|
|
|
|
for dir in CONTROL_DIRS {
|
|
|
|
|
collect_rs_files(dir, &mut files);
|
feat: mvp-chat benchmarking
Add end-to-end timing instrumentation and an xtask benchmark report for mvp-chat runs.
- benchmark_observability: add a shared stamping module — stamp(component) emitting schema/pid/monotonic+wall ms from a process-global start and sequence counter, plus unix_ms_now() — stamped onto every mvp-chat/orchestrator/worker-node event and frame-archive record
- mvp-chat: thread a run_id (new --run-id, defaults to 1) through config and the orchestrator CLI, add per-phase started/ready/failed emits for ensure_orch_binary/ensure_worker_binary/prepare_node_image, and a prompt_complete record carrying tokens_generated/elapsed_ms/final_text bytes
- orchestrator/worker-node: stamp bootstrap and prompt events, add arrival_unix_ms to archived frames, propagate MVP_RUN_ID/MVP_LOGICAL_NODE_ID/MVP_STAGE_INDEX into the tinygrad worker, default the device to CPU for the process provider, emit a prompt_rpc started span, and drop the MVP_TINYGRAD_TEST_MODE passthrough
- tinygrad_worker.py: stamp every control() event and tag it with run/node/stage env, add a CPU:X86 fallback when clang is absent, and remove the test_mode() short-circuits
- xtask: replace the flat dump-log fact assertions with a benchmark report builder (build_benchmark_report) that requires named spans (prepare_runtime, ensure_*_binary, weights_loaded, prompt_rpc) and emits per-prompt first-token/decode/tokens-per-second latency; wrap the cargo run in XtaskBenchmark synthetic frames and pass a unix-ms --run-id
Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-23 05:35:04 +00:00
|
|
|
}
|
|
|
|
|
|
2026-08-17 22:51:08 +00:00
|
|
|
let mut found = false;
|
|
|
|
|
for file in &files {
|
|
|
|
|
let Ok(src) = std::fs::read_to_string(file) else {
|
|
|
|
|
continue;
|
|
|
|
|
};
|
|
|
|
|
for (lineno, line) in src.lines().enumerate() {
|
|
|
|
|
for pattern in FORBIDDEN {
|
|
|
|
|
if line.contains(pattern) {
|
|
|
|
|
eprintln!(
|
|
|
|
|
"telemetry-isolation violation: {file}:{}: {}",
|
|
|
|
|
lineno + 1,
|
|
|
|
|
line.trim()
|
|
|
|
|
);
|
|
|
|
|
found = true;
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-07-05 09:59:51 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-17 22:51:08 +00:00
|
|
|
if found {
|
2026-07-24 19:05:11 +00:00
|
|
|
eprintln!(
|
2026-08-17 22:51:08 +00:00
|
|
|
"\ntelemetry-isolation: control-plane code must not import frame types \
|
|
|
|
|
or read-side modules. Use the telemetry producer API for emission."
|
2026-07-24 19:05:11 +00:00
|
|
|
);
|
2026-08-22 17:18:25 +00:00
|
|
|
false
|
feat: mvp-chat benchmarking
Add end-to-end timing instrumentation and an xtask benchmark report for mvp-chat runs.
- benchmark_observability: add a shared stamping module — stamp(component) emitting schema/pid/monotonic+wall ms from a process-global start and sequence counter, plus unix_ms_now() — stamped onto every mvp-chat/orchestrator/worker-node event and frame-archive record
- mvp-chat: thread a run_id (new --run-id, defaults to 1) through config and the orchestrator CLI, add per-phase started/ready/failed emits for ensure_orch_binary/ensure_worker_binary/prepare_node_image, and a prompt_complete record carrying tokens_generated/elapsed_ms/final_text bytes
- orchestrator/worker-node: stamp bootstrap and prompt events, add arrival_unix_ms to archived frames, propagate MVP_RUN_ID/MVP_LOGICAL_NODE_ID/MVP_STAGE_INDEX into the tinygrad worker, default the device to CPU for the process provider, emit a prompt_rpc started span, and drop the MVP_TINYGRAD_TEST_MODE passthrough
- tinygrad_worker.py: stamp every control() event and tag it with run/node/stage env, add a CPU:X86 fallback when clang is absent, and remove the test_mode() short-circuits
- xtask: replace the flat dump-log fact assertions with a benchmark report builder (build_benchmark_report) that requires named spans (prepare_runtime, ensure_*_binary, weights_loaded, prompt_rpc) and emits per-prompt first-token/decode/tokens-per-second latency; wrap the cargo run in XtaskBenchmark synthetic frames and pass a unix-ms --run-id
Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-23 05:35:04 +00:00
|
|
|
} else {
|
2026-08-17 22:51:08 +00:00
|
|
|
println!("telemetry-isolation: OK — no frame types in control-plane modules.");
|
2026-08-22 17:18:25 +00:00
|
|
|
true
|
2026-07-22 07:50:53 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-17 22:51:08 +00:00
|
|
|
fn collect_rs_files(dir: &str, out: &mut Vec<String>) {
|
|
|
|
|
let Ok(entries) = std::fs::read_dir(dir) else {
|
|
|
|
|
return;
|
feat: working 7B inferenced over 4 pipeline stages
Land end-to-end canonical benchmark observability and a synthetic datastream-connectivity preflight across the orchestrator, chat, worker-node, and Python tinygrad worker, plus an xtask validator, so a 4-stage 7B pipeline run is fully diagnosable.
- benchmark_observability: expand stamp() with canonical producer fields (producer_component/instance_id/process_id/sequence, wall_clock_unix_ms, monotonic_ms, clock_source) and schema_version so every event shares one envelope shape
- orchestrator_app + bin/{mvp_chat,worker_node}: stamp OrchBootstrap/OrchPromptEvent/ChatProgress/NodeEvent/SamplerHealth with the canonical fields plus span_id/parent_span_id, and add a 4-phase synthetic datastream preflight (ProducerConfigured/Connected/SyntheticEventSent/Observed) plus an endpoint_config_snapshot event on each process
- apps/mvp-node/tinygrad_worker: add apply_canonical_envelope()/datastream_endpoint_snapshot() and emit_python_datastream_preflight() mirroring the Rust preflight, and enrich benchmark_stamp() with the same producer fields
- bin/worker_node: pass MVP_DATASTREAM_ENDPOINT_ID/MVP_BENCHMARK_PRODUCER_INSTANCE/MVP_IROH_ENDPOINT_ADDR_MASK/MVP_IROH_RELAY_MODE env to the spawned tinygrad worker so its stamps identify the stage
- bin/mvp_chat: add --pipeline-parallel as an alias for --pipeline-stages (with a duplicate-guard) and bump recursion_limit
- xtask: add a benchmark-observability validator (ValidatorFinding/BenchmarkValidation, validate_benchmark_observability, canonical-stamp and stage/edge checks, evidence + gap-report builders) with tests for missing python datastream connectivity, wrong run_id, and missing span_id
Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-27 06:47:17 +00:00
|
|
|
};
|
2026-08-17 22:51:08 +00:00
|
|
|
for entry in entries.flatten() {
|
|
|
|
|
let path = entry.path();
|
|
|
|
|
if path.is_dir() {
|
|
|
|
|
if let Some(path) = path.to_str() {
|
|
|
|
|
collect_rs_files(path, out);
|
|
|
|
|
}
|
|
|
|
|
} else if path.extension().is_some_and(|extension| extension == "rs")
|
|
|
|
|
&& let Some(path) = path.to_str()
|
|
|
|
|
{
|
|
|
|
|
out.push(path.to_owned());
|
feat: successful 8 stage pipeline parallel run, more metrics
Complete an 8-stage pipeline-parallel run over VastAI by provisioning stages high-to-low, adding per-stage/per-step metrics, host anti-colocation, and provider state-timeout guardrails.
- orchestrator_app: select the next weight-load stage by max index (provision stages high-to-low for parallel spread), add a throttled "loaded N of M; waiting on stage X" stage_provision_wait headline, and surface min_compute_cap/state_timeout_secs in the config dump.
- orchestrator_app: enrich pipeline_token_in/out and tokenizer_decode events with token_count/token_ids/generated_index.
- worker_node: add timing metrics across the data path (helper_execute_ms, egress_ring_read_ms, send_ms, ingress_ring_write_ms, object_load_ms), refactor take_complete_ingress_record into IngressRecordBytes (object_id/sequence/extent/flags), and emit a new object_loaded event.
- vastai_provisioning: track leased host_ids and blacklist already-leased hosts in later ProvisionRequests so stages don't co-locate, and tag SSH-bootstrap retry logs with the attempt number.
- tools/vastai: add min_compute_cap (PP_MIN_COMPUTE_CAP) filter/search query and a LifecyclePolicy state_timeout (PP_STATE_TIMEOUT_SECS) that fails instances stuck in a non-running status instead of polling forever.
- xtask: raise the check timeout to 1800s/30s grace, drop --skip-rebuild for VastAI, aggregate per-stage StepExecuted metrics, add a vastai summary section, and write failure artifacts on abort.
Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-25 16:04:57 +00:00
|
|
|
}
|
2026-07-22 07:50:53 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-17 22:51:08 +00:00
|
|
|
mod demo;
|
2026-08-15 14:10:52 +00:00
|
|
|
|
2026-05-21 19:48:02 +00:00
|
|
|
fn main() -> ExitCode {
|
2026-06-25 12:30:18 +00:00
|
|
|
let mut args = std::env::args().skip(1);
|
2026-07-05 09:59:51 +00:00
|
|
|
match args.next().as_deref() {
|
2026-08-22 17:18:25 +00:00
|
|
|
Some("check-telemetry-isolation") => {
|
|
|
|
|
if check_telemetry_isolation() {
|
|
|
|
|
ExitCode::SUCCESS
|
|
|
|
|
} else {
|
|
|
|
|
ExitCode::from(1)
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-07-05 09:59:51 +00:00
|
|
|
Some("test") if args.next().is_none() => run_tests(),
|
2026-08-17 22:51:08 +00:00
|
|
|
Some("demo") => demo::run(&args.collect::<Vec<_>>()),
|
2026-07-05 09:59:51 +00:00
|
|
|
Some("help" | "--help" | "-h") | None => {
|
2026-06-25 12:30:18 +00:00
|
|
|
print_usage();
|
2026-05-21 19:48:02 +00:00
|
|
|
ExitCode::SUCCESS
|
|
|
|
|
}
|
2026-06-25 12:30:18 +00:00
|
|
|
_ => {
|
2026-02-24 09:12:28 +00:00
|
|
|
print_usage();
|
2026-05-21 19:48:02 +00:00
|
|
|
ExitCode::from(1)
|
2026-02-16 15:39:27 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|