zacheryasc/swactor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: working 7B inferenced over 4 pipeline stages

Browse source
This commit is contained in:
Zachery Aaron Shores-Chmielewski 2026-07-27 10:47:17 +04:00
parent a5736730b5
commit 155741bedb
6 changed files with 1753 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

95
apps/mvp-node/tinygrad_worker.py
View file

@ -149,13 +149,30 @@ def env_flag(name: str, default: bool = True) -> bool:
def benchmark_stamp() -> dict[str, Any]:
global _benchmark_seq
_benchmark_seq += 1
pid = os.getpid()
wall_ms = time.time_ns() // 1_000_000
mono_ms = int((time.monotonic() - _benchmark_start) * 1000)
return {
"schema": BENCHMARK_SCHEMA,
"schema_version": BENCHMARK_SCHEMA,
"component": "tinygrad-worker",
"pid": os.getpid(),
"producer_component": "tinygrad-worker",
"producer_instance_id": os.environ.get(
"MVP_BENCHMARK_PRODUCER_INSTANCE",
f"tinygrad-worker:{pid}",
),
"producer_process_id": pid,
"pid": pid,
"seq": _benchmark_seq,
"wall_unix_ms": time.time_ns() // 1_000_000,
"mono_ms": int((time.monotonic() - _benchmark_start) * 1000),
"producer_sequence": _benchmark_seq,
"wall_unix_ms": wall_ms,
"wall_clock_unix_ms": wall_ms,
"mono_ms": mono_ms,
"monotonic_ms": mono_ms,
"clock_source": {
"wall": "time.time_ns_unix_ms",
"monotonic": "time.monotonic_process_elapsed_ms",
},
}
@ -169,8 +186,45 @@ def env_int(name: str) -> int | None:
return None
def datastream_endpoint_snapshot() -> dict[str, Any]:
return {
"role": "python-worker-stdio-json-bridge",
"transport": "stdout-json-lines",
"endpoint_identity": os.environ.get("MVP_DATASTREAM_ENDPOINT_ID", "worker-stdio-bridge"),
"configured_source": "worker-node-env",
"resolved_source": "TinygradWorker::spawn environment",
"authentication_present": False,
"tls_present": False,
"relay_mode": os.environ.get("MVP_IROH_RELAY_MODE"),
"endpoint_addr_mask": os.environ.get("MVP_IROH_ENDPOINT_ADDR_MASK"),
"connectivity_result": "configured",
}
def apply_canonical_envelope(event: dict[str, Any]) -> None:
benchmark = event.setdefault("benchmark", benchmark_stamp())
event.setdefault("schema_version", BENCHMARK_SCHEMA)
event.setdefault("event_type", event.get("type"))
event.setdefault("event_name", event.get("phase", event.get("type")))
event.setdefault("producer_component", benchmark.get("producer_component", "tinygrad-worker"))
event.setdefault("producer_instance_id", benchmark.get("producer_instance_id"))
event.setdefault("producer_process_id", benchmark.get("producer_process_id", os.getpid()))
event.setdefault("producer_sequence", benchmark.get("producer_sequence", benchmark.get("seq")))
event.setdefault("wall_clock_unix_ms", benchmark.get("wall_clock_unix_ms", benchmark.get("wall_unix_ms")))
event.setdefault("monotonic_ms", benchmark.get("monotonic_ms", benchmark.get("mono_ms")))
event.setdefault("clock_source", benchmark.get("clock_source"))
event.setdefault("datastream_endpoint", datastream_endpoint_snapshot())
event.setdefault(
"span_id",
f"{event.get('producer_instance_id')}:{event.get('producer_sequence')}:{event.get('event_name')}",
)
if "parent_span_id" not in event:
request_id = event.get("request_id")
event["parent_span_id"] = f"request:{request_id}" if request_id is not None else None
def control(**event: Any) -> None:
event.setdefault("benchmark", benchmark_stamp())
apply_canonical_envelope(event)
if (run_id := env_int("MVP_RUN_ID")) is not None:
event.setdefault("run_id", run_id)
if (node_id := env_int("MVP_LOGICAL_NODE_ID")) is not None:
@ -1273,6 +1327,37 @@ def shutdown_worker(_: dict[str, Any]) -> None:
raise SystemExit(0)
def emit_python_datastream_preflight() -> None:
endpoint = datastream_endpoint_snapshot()
control(
type="PythonDatastreamConfigured",
phase="PythonDatastreamConfigured",
status="configured",
endpoint=endpoint,
)
control(
type="PythonDatastreamConnected",
phase="PythonDatastreamConnected",
status="ready",
endpoint=endpoint,
)
synthetic_id = f"python-{os.getpid()}-{_benchmark_seq + 1}"
control(
type="PythonDatastreamSyntheticEventSent",
phase="PythonDatastreamSyntheticEventSent",
status="sent",
endpoint=endpoint,
synthetic_id=synthetic_id,
)
control(
type="PythonDatastreamSyntheticEventObserved",
phase="PythonDatastreamSyntheticEventObserved",
status="observed",
endpoint=endpoint,
synthetic_id=synthetic_id,
)
HANDLERS = {
"InitializeWorker": initialize,
"ConfigureRole": configure_role,
@ -1288,6 +1373,8 @@ HANDLERS = {
"ShutdownWorker": shutdown_worker,
}
emit_python_datastream_preflight()
for raw in sys.stdin:
if not raw.strip():
continue

17
crates/mvp-system/src/benchmark_observability.rs
View file

@ -21,12 +21,25 @@ pub fn stamp(component: &'static str) -> Value {
let start = BENCHMARK_START.get_or_init(Instant::now);
let mono_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
let seq = BENCHMARK_SEQ.fetch_add(1, Ordering::Relaxed);
let pid = std::process::id();
let wall_ms = unix_ms_now();
json!({
"schema": BENCHMARK_SCHEMA,
"schema_version": BENCHMARK_SCHEMA,
"component": component,
"pid": std::process::id(),
"producer_component": component,
"producer_instance_id": format!("{component}:{pid}"),
"producer_process_id": pid,
"pid": pid,
"seq": seq,
"wall_unix_ms": unix_ms_now(),
"producer_sequence": seq,
"wall_unix_ms": wall_ms,
"wall_clock_unix_ms": wall_ms,
"mono_ms": mono_ms,
"monotonic_ms": mono_ms,
"clock_source": {
"wall": "system_unix_ms",
"monotonic": "process_elapsed_ms"
},
})
}

95
crates/mvp-system/src/bin/mvp_chat.rs
View file

@ -1,3 +1,5 @@
#![recursion_limit = "256"]
use std::collections::BTreeMap;
use std::fs::{self, File, OpenOptions};
use std::io::{self, BufRead, BufReader, IsTerminal, Write};
@ -47,7 +49,7 @@ OPTIONS:
--process | --docker | --vastai
Select the runtime provider
--config <path> Load config overlay
--pipeline-stages <count> Number of pipeline stages
--pipeline-stages|--pipeline-parallel <count>
--relay-mode <mode> Relay mode: default or disabled
--relay-url <url> Custom relay URL passed to mvp-orchestrator
--endpoint-addr-mask <mask> Endpoint address mask: full or relay-only
@ -154,6 +156,7 @@ where
}),
);
progress.emit_benchmark_envelope(&config);
progress.emit_endpoint_config_snapshot(&config);
confirm_vastai_if_needed(&config)?;
let prepare_runtime_started = Instant::now();
progress.emit(
@ -411,12 +414,25 @@ impl ChatDatastream {
fn emit(&mut self, channel: &str, phase: &str, status: &str, detail: Value) {
let id = self.channel_by_name(channel);
let benchmark = benchmark_observability::stamp("mvp-chat");
let payload = serde_json::to_vec(&json!({
"schema_version": benchmark["schema_version"].clone(),
"type": "ChatProgress",
"event_type": "ChatProgress",
"event_name": phase,
"phase": phase,
"status": status,
"run_id": self.run_id,
"benchmark": benchmark_observability::stamp("mvp-chat"),
"producer_component": benchmark["producer_component"].clone(),
"producer_instance_id": benchmark["producer_instance_id"].clone(),
"producer_process_id": benchmark["producer_process_id"].clone(),
"producer_sequence": benchmark["producer_sequence"].clone(),
"wall_clock_unix_ms": benchmark["wall_clock_unix_ms"].clone(),
"monotonic_ms": benchmark["monotonic_ms"].clone(),
"clock_source": benchmark["clock_source"].clone(),
"span_id": format!("mvp-chat:{}:{}:{phase}", self.run_id, benchmark["producer_sequence"]),
"parent_span_id": Value::Null,
"benchmark": benchmark,
"detail": detail,
}))
.expect("serialize mvp-chat progress event");
@ -426,12 +442,25 @@ impl ChatDatastream {
fn emit_benchmark_envelope(&mut self, config: &Config) {
let id = self.channel_by_name(CHAT_BENCHMARK_CHANNEL);
let benchmark = benchmark_observability::stamp("mvp-chat");
let payload = serde_json::to_vec(&json!({
"schema_version": benchmark["schema_version"].clone(),
"type": "BenchmarkRunEnvelope",
"event_type": "BenchmarkRunEnvelope",
"event_name": "run_envelope",
"phase": "run_envelope",
"status": "ready",
"run_id": self.run_id,
"benchmark": benchmark_observability::stamp("mvp-chat"),
"producer_component": benchmark["producer_component"].clone(),
"producer_instance_id": benchmark["producer_instance_id"].clone(),
"producer_process_id": benchmark["producer_process_id"].clone(),
"producer_sequence": benchmark["producer_sequence"].clone(),
"wall_clock_unix_ms": benchmark["wall_clock_unix_ms"].clone(),
"monotonic_ms": benchmark["monotonic_ms"].clone(),
"clock_source": benchmark["clock_source"].clone(),
"span_id": format!("mvp-chat:{}:{}:run_envelope", self.run_id, benchmark["producer_sequence"]),
"parent_span_id": Value::Null,
"benchmark": benchmark,
"detail": {
"scenario": "mvp-chat",
"detail_level": "benchmark_observability_v1",
@ -487,6 +516,56 @@ impl ChatDatastream {
self.flush();
}
fn emit_endpoint_config_snapshot(&mut self, config: &Config) {
let endpoint = json!({
"role": "chat-frame-archive",
"transport": "datastream-frame-log",
"configured": config.datastream_frame_log.is_some(),
"archive_path": config.datastream_frame_log.as_ref().map(|path| path.to_string_lossy().to_string()),
});
let runtime_endpoint = json!({
"provider": config.provider.as_str(),
"relay_mode": config.relay_mode.as_deref(),
"relay_configured": config.relay_url.is_some(),
"endpoint_addr_mask": config.endpoint_addr_mask.as_str(),
});
let synthetic_id = format!("mvp-chat-{}-datastream-preflight", self.run_id);
for (phase, status) in [
("DatastreamProducerConfigured", "configured"),
("DatastreamProducerConnected", "ready"),
("DatastreamSyntheticEventSent", "sent"),
("DatastreamSyntheticEventObserved", "observed"),
] {
self.emit(
CHAT_BENCHMARK_CHANNEL,
phase,
status,
json!({
"producer": "mvp-chat",
"producer_class": "rust-chat",
"synthetic_id": synthetic_id,
"datastream_endpoint": endpoint,
"runtime_endpoint": runtime_endpoint,
}),
);
}
self.emit(
CHAT_BENCHMARK_CHANNEL,
"endpoint_config_snapshot",
"ready",
json!({
"producer": "mvp-chat",
"expected_producers": ["mvp-chat", "mvp-orchestrator", "mvp-worker-node", "tinygrad-worker"],
"datastream_endpoint": endpoint,
"runtime_endpoint": runtime_endpoint,
"connectivity_preflight": {
"status": "configured",
"canonical_datastream_required": true,
},
}),
);
}
fn flush(&mut self) {
let stream = self.stream.clone();
for frame in self.endpoint.mux().drain() {
@ -1034,7 +1113,10 @@ impl ParsedArgs {
"--config" => {
parsed.config_path = Some(PathBuf::from(next_arg(&mut args, "--config")?))
}
"--pipeline-stages" => {
"--pipeline-stages" | "--pipeline-parallel" => {
if parsed.pipeline_stages.is_some() {
return Err("pipeline stage count was provided more than once".to_owned());
}
parsed.pipeline_stages =
Some(parse_pipeline_stages_value(&mut args, arg.as_str())?)
}
@ -2583,6 +2665,11 @@ mod tests {
assert!(help.help);
let short_help = ParsedArgs::parse(strings(&["-h"])).expect("short help parses");
assert!(short_help.help);
let alias = ParsedArgs::parse(strings(&["--vastai", "--pipeline-parallel", "4"]))
.expect("pipeline-parallel alias parses");
assert_eq!(alias.provider, Some(ProviderKind::VastAi));
assert_eq!(alias.pipeline_stages, Some(4));
}
#[test]

78
crates/mvp-system/src/bin/worker_node.rs
View file

@ -74,14 +74,27 @@ fn node_event_payload(
status: &str,
detail: Value,
) -> Value {
let benchmark = benchmark_observability::stamp("mvp-worker-node");
json!({
"schema_version": benchmark["schema_version"].clone(),
"type":"NodeEvent",
"event_type":"NodeEvent",
"event_name":phase,
"phase":phase,
"status":status,
"run_id":config.run_id,
"node_id":config.logical_node_id,
"stage_index":config.stage_index,
"benchmark":benchmark_observability::stamp("mvp-worker-node"),
"producer_component":benchmark["producer_component"].clone(),
"producer_instance_id":benchmark["producer_instance_id"].clone(),
"producer_process_id":benchmark["producer_process_id"].clone(),
"producer_sequence":benchmark["producer_sequence"].clone(),
"wall_clock_unix_ms":benchmark["wall_clock_unix_ms"].clone(),
"monotonic_ms":benchmark["monotonic_ms"].clone(),
"clock_source":benchmark["clock_source"].clone(),
"span_id":format!("mvp-worker-node:{}:{}:{}", config.run_id, benchmark["producer_sequence"], phase),
"parent_span_id":Value::Null,
"benchmark":benchmark,
"detail":detail,
})
}
@ -444,18 +457,31 @@ fn sampler_health_payload(
status: &str,
detail: Value,
) -> Value {
let benchmark = benchmark_observability::stamp("mvp-worker-node");
json!({
"schema_version":benchmark["schema_version"].clone(),
"type":"SamplerHealth",
"event_type":"SamplerHealth",
"event_name":"host_sampler_health",
"schema":"mvp.node.sampler.health.v1",
"run_id":context.run_id,
"node_id":context.node_id,
"stage_index":context.stage_index,
"producer_component":benchmark["producer_component"].clone(),
"producer_instance_id":benchmark["producer_instance_id"].clone(),
"producer_process_id":benchmark["producer_process_id"].clone(),
"producer_sequence":benchmark["producer_sequence"].clone(),
"wall_clock_unix_ms":benchmark["wall_clock_unix_ms"].clone(),
"monotonic_ms":benchmark["monotonic_ms"].clone(),
"clock_source":benchmark["clock_source"].clone(),
"span_id":format!("mvp-worker-node:{}:{}:host_sampler_health", context.run_id, benchmark["producer_sequence"]),
"parent_span_id":Value::Null,
"phase":"host_sampler_health",
"status":status,
"sampler":sampler,
"sample_channel":sample_channel,
"detail":detail,
"benchmark":benchmark_observability::stamp("mvp-worker-node"),
"benchmark":benchmark,
})
}
@ -1827,6 +1853,35 @@ fn run() -> Result<(), String> {
"ready",
json!({"actor":datastream_publisher,"name":DATASTREAM_PUBLISHER_NAME,"subscription_transport":"iroh"}),
);
let worker_synthetic_id = format!(
"mvp-worker-node-{}-{}-datastream-preflight",
config.logical_node_id, config.stage_index
);
for (phase, status) in [
("DatastreamProducerConfigured", "configured"),
("DatastreamProducerConnected", "ready"),
("DatastreamSyntheticEventSent", "sent"),
("DatastreamSyntheticEventObserved", "observed"),
] {
emit_node_event(
&mut datastream,
&config,
NODE_BOOTSTRAP_CHANNEL,
phase,
status,
json!({
"producer":"mvp-worker-node",
"producer_class":"rust-worker-node",
"synthetic_id":worker_synthetic_id,
"datastream_endpoint":{
"role":"worker-node-iroh-publisher",
"transport":"iroh-datastream",
"endpoint_addr_mask":config.endpoint_addr_mask.as_str(),
"relay_mode":format!("{:?}", config.relay_mode),
},
}),
);
}
let mut debug_join_rx = match &config.debug_join_socket {
Some(path) => {
match spawn_debug_join_listener(tokio.handle().clone(), PathBuf::from(path)) {
@ -3890,6 +3945,25 @@ impl TinygradWorker {
.env("MVP_STAGE_INDEX", config.stage_index.to_string())
.env("MVP_ARENA_FD", arena_fd.to_string())
.env("MVP_ARENA_BYTES", config.arena_bytes.to_string())
.env(
"MVP_DATASTREAM_ENDPOINT_ID",
format!(
"worker-node-{}-stage-{}-stdio-bridge",
config.logical_node_id, config.stage_index
),
)
.env(
"MVP_BENCHMARK_PRODUCER_INSTANCE",
format!(
"tinygrad-worker:{}:{}",
config.logical_node_id, config.stage_index
),
)
.env(
"MVP_IROH_ENDPOINT_ADDR_MASK",
config.endpoint_addr_mask.as_str(),
)
.env("MVP_IROH_RELAY_MODE", format!("{:?}", config.relay_mode))
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())

93
crates/mvp-system/src/orchestrator_app.rs
View file

@ -166,6 +166,53 @@ where
"provider_config":config.provider_datastream_detail(),
}),
);
orch_datastream.emit_bootstrap(
None,
config.run_id,
config.node_id,
"datastream_preflight",
"configured",
json!({
"producer":"mvp-orchestrator",
"datastream_endpoint":{
"role":"orchestrator-frame-archive",
"transport":"datastream-frame-log",
"configured":config.datastream_frame_log.is_some(),
"archive_path":config.datastream_frame_log.as_ref().map(|path| path.to_string_lossy().to_string()),
},
"expected_worker_producers":["mvp-worker-node","tinygrad-worker"],
"provider":config.provider.as_str(),
"pipeline_stages":config.pipeline_stages,
"endpoint_addr_mask":config.endpoint_addr_mask.as_str(),
"provider_config":config.provider_datastream_detail(),
}),
);
let orch_synthetic_id = format!("mvp-orchestrator-{}-datastream-preflight", config.run_id);
for (phase, status) in [
("DatastreamProducerConfigured", "configured"),
("DatastreamProducerConnected", "ready"),
("DatastreamSyntheticEventSent", "sent"),
("DatastreamSyntheticEventObserved", "observed"),
] {
orch_datastream.emit_bootstrap(
None,
config.run_id,
config.node_id,
phase,
status,
json!({
"producer":"mvp-orchestrator",
"producer_class":"rust-orchestrator",
"synthetic_id":orch_synthetic_id,
"datastream_endpoint":{
"role":"orchestrator-frame-archive",
"transport":"datastream-frame-log",
"configured":config.datastream_frame_log.is_some(),
"archive_path":config.datastream_frame_log.as_ref().map(|path| path.to_string_lossy().to_string()),
},
}),
);
}
drain_orch_stdio_capture(
orch_stdio_rx.as_ref(),
&mut orch_datastream,
@ -253,6 +300,22 @@ where
"ready",
json!({"endpoint":coordinator_endpoint.clone(),"has_relay":coordinator_endpoint.relay_urls().next().is_some(),"direct_addr_count":coordinator_endpoint.ip_addrs().count(),"relay_mode":format!("{:?}", config.relay.mode),"endpoint_addr_mask":config.endpoint_addr_mask.as_str()}),
);
orch_datastream.emit_bootstrap(
None,
config.run_id,
config.node_id,
"endpoint_config_snapshot",
"ready",
json!({
"producer":"mvp-orchestrator",
"coordinator_endpoint":coordinator_endpoint.clone(),
"has_relay":coordinator_endpoint.relay_urls().next().is_some(),
"direct_addr_count":coordinator_endpoint.ip_addrs().count(),
"relay_mode":format!("{:?}", config.relay.mode),
"endpoint_addr_mask":config.endpoint_addr_mask.as_str(),
"connectivity_preflight":"ready",
}),
);
let stack = DistributionRuntimeStack::new_with_codecs(
driver.node_id(),
DistributedNodeConfig::default(),
@ -4057,13 +4120,26 @@ impl OrchDatastream {
status: &str,
detail: Value,
) {
let benchmark = benchmark_observability::stamp("mvp-orchestrator");
let payload = serde_json::to_vec(&json!({
"schema_version": benchmark["schema_version"].clone(),
"type":"OrchBootstrap",
"event_type":"OrchBootstrap",
"event_name":phase,
"phase":phase,
"status":status,
"run_id":run_id,
"node_id":node_id,
"benchmark":benchmark_observability::stamp("mvp-orchestrator"),
"producer_component":benchmark["producer_component"].clone(),
"producer_instance_id":benchmark["producer_instance_id"].clone(),
"producer_process_id":benchmark["producer_process_id"].clone(),
"producer_sequence":benchmark["producer_sequence"].clone(),
"wall_clock_unix_ms":benchmark["wall_clock_unix_ms"].clone(),
"monotonic_ms":benchmark["monotonic_ms"].clone(),
"clock_source":benchmark["clock_source"].clone(),
"span_id":format!("mvp-orchestrator:{run_id}:{}:{phase}", benchmark["producer_sequence"]),
"parent_span_id":Value::Null,
"benchmark":benchmark,
"detail":detail,
}))
.expect("serialize orch bootstrap event");
@ -4080,14 +4156,27 @@ impl OrchDatastream {
status: &str,
detail: Value,
) {
let benchmark = benchmark_observability::stamp("mvp-orchestrator");
let payload = serde_json::to_vec(&json!({
"schema_version": benchmark["schema_version"].clone(),
"type":"OrchPromptEvent",
"event_type":"OrchPromptEvent",
"event_name":phase,
"phase":phase,
"status":status,
"run_id":run_id,
"node_id":node_id,
"request_id":request_id,
"benchmark":benchmark_observability::stamp("mvp-orchestrator"),
"producer_component":benchmark["producer_component"].clone(),
"producer_instance_id":benchmark["producer_instance_id"].clone(),
"producer_process_id":benchmark["producer_process_id"].clone(),
"producer_sequence":benchmark["producer_sequence"].clone(),
"wall_clock_unix_ms":benchmark["wall_clock_unix_ms"].clone(),
"monotonic_ms":benchmark["monotonic_ms"].clone(),
"clock_source":benchmark["clock_source"].clone(),
"span_id":format!("mvp-orchestrator:{run_id}:{request_id}:{}:{phase}", benchmark["producer_sequence"]),
"parent_span_id":format!("request:{request_id}"),
"benchmark":benchmark,
"detail":detail,
}))
.expect("serialize orch prompt event");

1422
xtask/src/main.rs
View file

File diff suppressed because it is too large Load diff