swactor/xtask/src/main.rs

278 lines
7.5 KiB
Rust
Raw Normal View History

use std::path::{Path, PathBuf};
use std::process::{Command, ExitCode, Stdio};
use std::time::Instant;
struct TestStep {
label: &'static str,
args: &'static [&'static str],
}
const TEST_STEPS: &[TestStep] = &[
TestStep {
label: "strict workspace lint",
args: &["lint"],
},
TestStep {
label: "all Rust tests (60s per-test timeout)",
args: &["nextest", "run", "--workspace", "--all-features"],
},
TestStep {
label: "all Rust doctests",
args: &[
"test",
"--workspace",
"--all-features",
"--doc",
"--exclude",
"python",
"--exclude",
"wasm-runtime",
],
},
];
fn cargo_bin() -> String {
option_env!("CARGO")
.map(str::to_string)
.unwrap_or_else(|| "cargo".to_string())
}
fn print_usage() {
println!(
"\
USAGE: cargo xtask <command>
COMMANDS:
demo: rename xtask demo command; dashboard-established data-plane edges Rename `cargo xtask provisioning-reconciler-demo` to `cargo xtask demo` (CLI dispatch, help, child re-exec argv, launch spec strings, module dir xtask/src/provisioning_demo -> xtask/src/demo). Add iteration-1 data-plane edges, established from Fleet Control: - Fleet Control "edge" button -> POST /control/edge (new ControlCommand::EstablishEdge) -> supervisor actor resolves the node's advertised EndpointAddr (stashed in NodeRuntime by AnnounceActor) and provisions a real outbound EdgeRuntime (arena ring lease, recorder WorkerPort, EDGE_ALPN send pump) in a new edge pump thread. - Node gains EDGE_ALPN, an actor bridge decoding EdgeProvision gossip, and a NodeEdgeAgent that provisions its (single) inbound edge, polls it, mirrors observations onto the node.edge telemetry channel (render-only), and answers EdgeAck gossip which terminates the supervisor's provision retries. Node teardown replaces its inbound on re-provision; supervisor replaces sessions per node and tears them down on node exit/replacement/shutdown. - The edge pump runs on the engine's blocking pool with sole session ownership (commands in, state mirror + feed lines out): the connect handshake blocks its thread and must not run on a Tokio worker or share a lock with the actor. Connects are bounded (10s) so a dead node faults its session instead of wedging edge polling. - iroh-driver: retain_telemetry_connections() opts an application out of the driver-owned TELEMETRY_ALPN ingress so the node's pull server can drain those connections itself (the actor-bridge pump would otherwise claim them). - Dashboard: edges array in the reconciler snapshot, per-node edge badges and edge button in Fleet Control, node_edges render mirror.
2026-08-16 20:11:20 +00:00
demo [--port n] [--nodes n] [--docker]
Run the visual provisioning-reconciler demo.
check-telemetry-isolation
Verify no frame types appear in control-plane modules.
test Run strict lint plus every Rust and Python test."
);
}
fn run_step(step: &TestStep, python: &Path) -> bool {
println!("\n=== {} ===", step.label);
println!(" cargo {}", step.args.join(" "));
println!();
match swactor_process::command_status(
Command::new(cargo_bin())
.args(step.args)
.env("PYO3_PYTHON", python),
) {
Ok(status) => status.success(),
Err(error) => {
eprintln!("Failed to execute cargo: {error}");
false
}
}
}
fn run_tests() -> ExitCode {
let start = Instant::now();
let Some(python) = PythonTestTools::discover() else {
return ExitCode::from(1);
};
if !nextest_available() {
return ExitCode::from(1);
}
if !check_telemetry_isolation() {
return ExitCode::from(1);
}
for (index, step) in TEST_STEPS.iter().enumerate() {
if !run_step(step, &python.python) {
eprintln!(
"\n--- FAILED after {:.1}s ({index} passed, 1 failed) ---",
start.elapsed().as_secs_f64()
);
return ExitCode::from(1);
}
}
if !python.run() {
eprintln!(
"\n--- FAILED after {:.1}s (Python tests failed) ---",
start.elapsed().as_secs_f64()
);
return ExitCode::from(1);
}
println!(
"\n--- All {} step(s) passed in {:.1}s ---",
TEST_STEPS.len() + 1,
start.elapsed().as_secs_f64()
);
ExitCode::SUCCESS
}
fn nextest_available() -> bool {
let available = swactor_process::command_status(
Command::new(cargo_bin())
.args(["nextest", "--version"])
.stdout(Stdio::null())
.stderr(Stdio::null()),
)
.is_ok_and(|status| status.success());
if !available {
eprintln!("cargo-nextest is required; install it from https://nexte.st/docs/installation/");
}
available
}
struct PythonTestTools {
directory: PathBuf,
maturin: PathBuf,
python: PathBuf,
}
impl PythonTestTools {
fn discover() -> Option<Self> {
let directory = workspace_root().join("crates/bindings/python");
let tools = Self {
maturin: directory.join(".venv/bin/maturin"),
python: directory.join(".venv/bin/python"),
directory,
};
if tools.maturin.is_file() && tools.python.is_file() {
Some(tools)
} else {
eprintln!(
"Python test environment is missing; run `uv sync --project {}` first",
tools.directory.display()
);
None
}
}
fn run(&self) -> bool {
println!("\n=== all Python tests ===");
let built = swactor_process::command_status(
Command::new(&self.maturin)
.current_dir(&self.directory)
.arg("develop")
.env("PYO3_PYTHON", &self.python),
)
.is_ok_and(|status| status.success());
if !built {
return false;
}
swactor_process::command_status(
Command::new(&self.python)
.current_dir(&self.directory)
.args([
"-m",
"pytest",
"-q",
"--timeout=60",
"tests/test_bootstrap.py",
"../../../tests/test_python.py",
]),
)
.is_ok_and(|status| status.success())
}
}
fn workspace_root() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.expect("xtask must live directly under the workspace root")
.to_path_buf()
}
/// Verify that control-plane modules never import telemetry frame/read-side
/// types. They may emit through the producer API only.
fn check_telemetry_isolation() -> bool {
const CONTROL_DIRS: &[&str] = &[
"apps/myelin/src/orchestration",
"crates/distribution/src",
"crates/data-plane/src",
"crates/provisioning/src",
];
const FORBIDDEN: &[&str] = &[
"telemetry::frame::",
"telemetry::store::",
"telemetry::ingest::",
"telemetry::views::",
"telemetry::transport::",
"CollectedTelemetryFrame",
];
let mut files = Vec::new();
for dir in CONTROL_DIRS {
collect_rs_files(dir, &mut files);
feat: mvp-chat benchmarking Add end-to-end timing instrumentation and an xtask benchmark report for mvp-chat runs. - benchmark_observability: add a shared stamping module — stamp(component) emitting schema/pid/monotonic+wall ms from a process-global start and sequence counter, plus unix_ms_now() — stamped onto every mvp-chat/orchestrator/worker-node event and frame-archive record - mvp-chat: thread a run_id (new --run-id, defaults to 1) through config and the orchestrator CLI, add per-phase started/ready/failed emits for ensure_orch_binary/ensure_worker_binary/prepare_node_image, and a prompt_complete record carrying tokens_generated/elapsed_ms/final_text bytes - orchestrator/worker-node: stamp bootstrap and prompt events, add arrival_unix_ms to archived frames, propagate MVP_RUN_ID/MVP_LOGICAL_NODE_ID/MVP_STAGE_INDEX into the tinygrad worker, default the device to CPU for the process provider, emit a prompt_rpc started span, and drop the MVP_TINYGRAD_TEST_MODE passthrough - tinygrad_worker.py: stamp every control() event and tag it with run/node/stage env, add a CPU:X86 fallback when clang is absent, and remove the test_mode() short-circuits - xtask: replace the flat dump-log fact assertions with a benchmark report builder (build_benchmark_report) that requires named spans (prepare_runtime, ensure_*_binary, weights_loaded, prompt_rpc) and emits per-prompt first-token/decode/tokens-per-second latency; wrap the cargo run in XtaskBenchmark synthetic frames and pass a unix-ms --run-id Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-23 05:35:04 +00:00
}
let mut found = false;
for file in &files {
let Ok(src) = std::fs::read_to_string(file) else {
continue;
};
for (lineno, line) in src.lines().enumerate() {
for pattern in FORBIDDEN {
if line.contains(pattern) {
eprintln!(
"telemetry-isolation violation: {file}:{}: {}",
lineno + 1,
line.trim()
);
found = true;
}
}
}
}
if found {
eprintln!(
"\ntelemetry-isolation: control-plane code must not import frame types \
or read-side modules. Use the telemetry producer API for emission."
);
false
feat: mvp-chat benchmarking Add end-to-end timing instrumentation and an xtask benchmark report for mvp-chat runs. - benchmark_observability: add a shared stamping module — stamp(component) emitting schema/pid/monotonic+wall ms from a process-global start and sequence counter, plus unix_ms_now() — stamped onto every mvp-chat/orchestrator/worker-node event and frame-archive record - mvp-chat: thread a run_id (new --run-id, defaults to 1) through config and the orchestrator CLI, add per-phase started/ready/failed emits for ensure_orch_binary/ensure_worker_binary/prepare_node_image, and a prompt_complete record carrying tokens_generated/elapsed_ms/final_text bytes - orchestrator/worker-node: stamp bootstrap and prompt events, add arrival_unix_ms to archived frames, propagate MVP_RUN_ID/MVP_LOGICAL_NODE_ID/MVP_STAGE_INDEX into the tinygrad worker, default the device to CPU for the process provider, emit a prompt_rpc started span, and drop the MVP_TINYGRAD_TEST_MODE passthrough - tinygrad_worker.py: stamp every control() event and tag it with run/node/stage env, add a CPU:X86 fallback when clang is absent, and remove the test_mode() short-circuits - xtask: replace the flat dump-log fact assertions with a benchmark report builder (build_benchmark_report) that requires named spans (prepare_runtime, ensure_*_binary, weights_loaded, prompt_rpc) and emits per-prompt first-token/decode/tokens-per-second latency; wrap the cargo run in XtaskBenchmark synthetic frames and pass a unix-ms --run-id Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-23 05:35:04 +00:00
} else {
println!("telemetry-isolation: OK — no frame types in control-plane modules.");
true
}
}
fn collect_rs_files(dir: &str, out: &mut Vec<String>) {
let Ok(entries) = std::fs::read_dir(dir) else {
return;
feat: working 7B inferenced over 4 pipeline stages Land end-to-end canonical benchmark observability and a synthetic datastream-connectivity preflight across the orchestrator, chat, worker-node, and Python tinygrad worker, plus an xtask validator, so a 4-stage 7B pipeline run is fully diagnosable. - benchmark_observability: expand stamp() with canonical producer fields (producer_component/instance_id/process_id/sequence, wall_clock_unix_ms, monotonic_ms, clock_source) and schema_version so every event shares one envelope shape - orchestrator_app + bin/{mvp_chat,worker_node}: stamp OrchBootstrap/OrchPromptEvent/ChatProgress/NodeEvent/SamplerHealth with the canonical fields plus span_id/parent_span_id, and add a 4-phase synthetic datastream preflight (ProducerConfigured/Connected/SyntheticEventSent/Observed) plus an endpoint_config_snapshot event on each process - apps/mvp-node/tinygrad_worker: add apply_canonical_envelope()/datastream_endpoint_snapshot() and emit_python_datastream_preflight() mirroring the Rust preflight, and enrich benchmark_stamp() with the same producer fields - bin/worker_node: pass MVP_DATASTREAM_ENDPOINT_ID/MVP_BENCHMARK_PRODUCER_INSTANCE/MVP_IROH_ENDPOINT_ADDR_MASK/MVP_IROH_RELAY_MODE env to the spawned tinygrad worker so its stamps identify the stage - bin/mvp_chat: add --pipeline-parallel as an alias for --pipeline-stages (with a duplicate-guard) and bump recursion_limit - xtask: add a benchmark-observability validator (ValidatorFinding/BenchmarkValidation, validate_benchmark_observability, canonical-stamp and stage/edge checks, evidence + gap-report builders) with tests for missing python datastream connectivity, wrong run_id, and missing span_id Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-27 06:47:17 +00:00
};
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
if let Some(path) = path.to_str() {
collect_rs_files(path, out);
}
} else if path.extension().is_some_and(|extension| extension == "rs")
&& let Some(path) = path.to_str()
{
out.push(path.to_owned());
feat: successful 8 stage pipeline parallel run, more metrics Complete an 8-stage pipeline-parallel run over VastAI by provisioning stages high-to-low, adding per-stage/per-step metrics, host anti-colocation, and provider state-timeout guardrails. - orchestrator_app: select the next weight-load stage by max index (provision stages high-to-low for parallel spread), add a throttled "loaded N of M; waiting on stage X" stage_provision_wait headline, and surface min_compute_cap/state_timeout_secs in the config dump. - orchestrator_app: enrich pipeline_token_in/out and tokenizer_decode events with token_count/token_ids/generated_index. - worker_node: add timing metrics across the data path (helper_execute_ms, egress_ring_read_ms, send_ms, ingress_ring_write_ms, object_load_ms), refactor take_complete_ingress_record into IngressRecordBytes (object_id/sequence/extent/flags), and emit a new object_loaded event. - vastai_provisioning: track leased host_ids and blacklist already-leased hosts in later ProvisionRequests so stages don't co-locate, and tag SSH-bootstrap retry logs with the attempt number. - tools/vastai: add min_compute_cap (PP_MIN_COMPUTE_CAP) filter/search query and a LifecyclePolicy state_timeout (PP_STATE_TIMEOUT_SECS) that fails instances stuck in a non-running status instead of polling forever. - xtask: raise the check timeout to 1800s/30s grace, drop --skip-rebuild for VastAI, aggregate per-stage StepExecuted metrics, add a vastai summary section, and write failure artifacts on abort. Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-25 16:04:57 +00:00
}
}
}
mod demo;
fn main() -> ExitCode {
let mut args = std::env::args().skip(1);
match args.next().as_deref() {
Some("check-telemetry-isolation") => {
if check_telemetry_isolation() {
ExitCode::SUCCESS
} else {
ExitCode::from(1)
}
}
Some("test") if args.next().is_none() => run_tests(),
Some("demo") => demo::run(&args.collect::<Vec<_>>()),
Some("help" | "--help" | "-h") | None => {
print_usage();
ExitCode::SUCCESS
}
_ => {
print_usage();
ExitCode::from(1)
}
}
}