stash
This commit is contained in:
parent
2ea137a66f
commit
d7759ac5b8
36 changed files with 4594 additions and 53 deletions
|
|
@ -1,5 +1,6 @@
|
|||
*
|
||||
!target/x86_64-unknown-linux-musl/release/swactor
|
||||
!target/x86_64-unknown-linux-musl/release/swactor-datastream-collector
|
||||
!target/release/swactor-diag-collector
|
||||
!target/release/swactor-diag-postproc
|
||||
!examples/single-gpu-inference/target/release/gpu-node
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
FROM scratch
|
||||
COPY target/x86_64-unknown-linux-musl/release/swactor /swactor
|
||||
COPY target/x86_64-unknown-linux-musl/release/swactor-datastream-collector /collector
|
||||
ENTRYPOINT ["/swactor"]
|
||||
|
|
|
|||
|
|
@ -32,6 +32,11 @@ replay-viewer = ["dep:anyhow", "dep:distribution", "distribution/collector"]
|
|||
# SSE UI folding the vast.ai fleet telemetry and the orchestrator's
|
||||
# distribution snapshot server-side.
|
||||
live-collector = ["dep:distribution", "distribution/collector"]
|
||||
# Datastream-backed dashboard: bind the demo's UDP telemetry sink, demux the
|
||||
# per-node frames, and drive the live HTTP dashboard. Only needs the
|
||||
# `distribution::datastream` module (not feature-gated there), so it stays free
|
||||
# of the heavier collector/iroh stack.
|
||||
datastream = ["dep:distribution"]
|
||||
|
||||
[[bin]]
|
||||
name = "swactor-tui"
|
||||
|
|
@ -43,6 +48,11 @@ name = "dashboard_collector"
|
|||
path = "src/bin/dashboard_collector.rs"
|
||||
required-features = ["live-collector"]
|
||||
|
||||
[[bin]]
|
||||
name = "swactor-datastream-dashboard"
|
||||
path = "src/bin/datastream_dashboard.rs"
|
||||
required-features = ["datastream"]
|
||||
|
||||
[[example]]
|
||||
name = "replay_viewer"
|
||||
path = "examples/replay_viewer.rs"
|
||||
|
|
|
|||
86
crates/dashboard/src/bin/datastream_dashboard.rs
Normal file
86
crates/dashboard/src/bin/datastream_dashboard.rs
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
//! Datastream-backed dashboard: bind the demo cluster's UDP telemetry sink
|
||||
//! (the same wire `swactor-datastream-collector` reads), demultiplex the
|
||||
//! per-node frames, and serve the live HTTP/browser dashboard from them.
|
||||
//!
|
||||
//! Drop-in for the `collector` service in the datastream demo: same UDP bind,
|
||||
//! but instead of printing frames it renders them in the existing dashboard UI.
|
||||
//!
|
||||
//! Usage: `swactor-datastream-dashboard [--bind HOST:PORT] [--port HTTP_PORT] [--node FILTER]`
|
||||
//! --bind UDP sink to listen on (default `0.0.0.0:7700`)
|
||||
//! --port HTTP dashboard port (default `9090`)
|
||||
//! --node show the node whose id/region/role matches FILTER (default: first seen)
|
||||
|
||||
use dashboard::datastream_source::run_datastream_ingest;
|
||||
use dashboard::{start_dashboard, DashboardConfig};
|
||||
use tracing_subscriber::filter::LevelFilter;
|
||||
use tracing_subscriber::layer::SubscriberExt;
|
||||
use tracing_subscriber::util::SubscriberInitExt;
|
||||
|
||||
fn main() {
|
||||
let args = Args::parse();
|
||||
|
||||
let dashboard = start_dashboard(DashboardConfig {
|
||||
port: args.port,
|
||||
..Default::default()
|
||||
});
|
||||
// Route process-output / membership tracing into the activity log panel.
|
||||
// Cap at INFO so the panel shows our events, not tokio/mio TRACE internals.
|
||||
tracing_subscriber::registry()
|
||||
.with(dashboard.layer())
|
||||
.with(LevelFilter::INFO)
|
||||
.init();
|
||||
dashboard.start_http_standalone();
|
||||
|
||||
eprintln!(
|
||||
"datastream dashboard: serving on container port {} (UDP sink {}) — \
|
||||
browse via the published host port:\n \
|
||||
/ Overview / Actors (single node)\n \
|
||||
/plugin/distribution SWIM connection graph (all nodes)\n \
|
||||
/plugin/vastai Fleet table (all nodes)",
|
||||
args.port, args.bind
|
||||
);
|
||||
|
||||
// Blocks forever, demuxing frames into the dashboard's pushed stats.
|
||||
if let Err(e) = run_datastream_ingest(&args.bind, args.node.as_deref(), &dashboard) {
|
||||
eprintln!("datastream dashboard: fatal: failed to bind {}: {e}", args.bind);
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
struct Args {
|
||||
bind: String,
|
||||
port: u16,
|
||||
node: Option<String>,
|
||||
}
|
||||
|
||||
impl Args {
|
||||
fn parse() -> Self {
|
||||
let mut bind = std::env::var("SWACTOR_DATASTREAM_BIND").ok();
|
||||
let mut port: u16 = 9090;
|
||||
let mut node: Option<String> = None;
|
||||
|
||||
let mut it = std::env::args().skip(1);
|
||||
while let Some(arg) = it.next() {
|
||||
match arg.as_str() {
|
||||
"--bind" => bind = it.next(),
|
||||
"--port" => port = it.next().and_then(|v| v.parse().ok()).unwrap_or(port),
|
||||
"--node" => node = it.next(),
|
||||
other => {
|
||||
if let Some(v) = other.strip_prefix("--bind=") {
|
||||
bind = Some(v.to_string());
|
||||
} else if let Some(v) = other.strip_prefix("--port=") {
|
||||
port = v.parse().unwrap_or(port);
|
||||
} else if let Some(v) = other.strip_prefix("--node=") {
|
||||
node = Some(v.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Self {
|
||||
bind: bind.unwrap_or_else(|| "0.0.0.0:7700".to_string()),
|
||||
port,
|
||||
node,
|
||||
}
|
||||
}
|
||||
}
|
||||
871
crates/dashboard/src/datastream_source.rs
Normal file
871
crates/dashboard/src/datastream_source.rs
Normal file
|
|
@ -0,0 +1,871 @@
|
|||
//! Datastream → dashboard adapter.
|
||||
//!
|
||||
//! Binds the UDP sink the demo cluster ships to (the same wire the dumb
|
||||
//! `swactor-datastream-collector` reads), **demultiplexes** the per-node frames,
|
||||
//! and drives the dashboard's *existing* views from them — no bespoke UI:
|
||||
//!
|
||||
//! * the single-node **Overview / Actors** page (`/`) via a synthesized
|
||||
//! [`RuntimeStats`] (each datastream channel becomes one synthetic actor row);
|
||||
//! * the canonical **Distribution** connection-graph page
|
||||
//! (`/plugin/distribution`, [`crate::DISTRIBUTION_PAGE_HTML`]) via a
|
||||
//! [`DistributionNodeSnapshot`] rebuilt from the selected node's membership —
|
||||
//! so it renders the exact SWIM graph a live node shows;
|
||||
//! * a cross-node **Fleet** table (`/plugin/vastai`) served in the same
|
||||
//! dashboard chrome (nav bar + palette), not a separate app.
|
||||
//!
|
||||
//! Process output and membership transitions are emitted as `tracing` events so
|
||||
//! they flow through the dashboard's activity-log path.
|
||||
//!
|
||||
//! A node is only shown while it is *live* (has shipped a frame within
|
||||
//! [`NODE_TTL`]); a node that stops streaming drops out of every view, so a
|
||||
//! restarted/departed node leaves no ghost in the graph or the fleet table.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::net::UdpSocket;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use distribution::datastream::catalog::{
|
||||
self, IdentityRecord, LifecycleCost, MembershipTransition, Record, ResourceSample,
|
||||
RuntimeStats as DsRuntimeStats, TransportInternals,
|
||||
};
|
||||
use distribution::datastream::wire::decode_delivery;
|
||||
use distribution::snapshot::{DistributionNodeSnapshot, MemberInfo};
|
||||
|
||||
use swactor::actor::ActorAddress;
|
||||
use swactor::stats::{ActorInfo, RuntimeStats, TickTiming, WorkerInfo};
|
||||
|
||||
use crate::plugin::{DashboardPlugin, PluginResponse};
|
||||
use crate::{DashboardHandle, DISTRIBUTION_PAGE_HTML};
|
||||
|
||||
/// A node counts as live — shown in the graph and fleet table — if it has
|
||||
/// streamed a frame within this window. Nodes ship resource/runtime/transport
|
||||
/// samples every ~1s, so a node silent past this has left the cluster; its
|
||||
/// `models` entry lingers but is filtered out of every view (no ghost nodes).
|
||||
const NODE_TTL: Duration = Duration::from_secs(8);
|
||||
|
||||
/// Per-node accumulator: the latest value seen on each typed channel plus
|
||||
/// running membership/process state. One of these drives the display.
|
||||
#[derive(Default)]
|
||||
struct DatastreamModel {
|
||||
identity: Option<IdentityRecord>,
|
||||
resource: Option<ResourceSample>,
|
||||
runtime: Option<DsRuntimeStats>,
|
||||
transport: Option<TransportInternals>,
|
||||
lifecycle: Option<LifecycleCost>,
|
||||
/// peer node-id → latest liveness state.
|
||||
membership: HashMap<String, String>,
|
||||
/// last membership transition, formatted for display.
|
||||
last_transition: Option<String>,
|
||||
/// proc label → (line count, last line).
|
||||
procs: HashMap<String, (u64, String)>,
|
||||
first_seen: Option<Instant>,
|
||||
last_seen: Option<Instant>,
|
||||
}
|
||||
|
||||
/// A log line to surface through the dashboard's activity path.
|
||||
enum LogEvent {
|
||||
Info(String),
|
||||
Warn(String),
|
||||
}
|
||||
|
||||
impl DatastreamModel {
|
||||
/// Fold one frame's channel/payload into the model, returning any activity
|
||||
/// log events it produced (process output, membership transitions).
|
||||
fn update(&mut self, channel: &str, payload: &[u8]) -> Vec<LogEvent> {
|
||||
let now = Instant::now();
|
||||
self.first_seen.get_or_insert(now);
|
||||
self.last_seen = Some(now);
|
||||
let mut events = Vec::new();
|
||||
|
||||
match channel {
|
||||
catalog::IDENTITY => {
|
||||
if let Ok(r) = IdentityRecord::decode(payload) {
|
||||
self.identity = Some(r);
|
||||
}
|
||||
}
|
||||
catalog::HOST_RESOURCE => {
|
||||
if let Ok(r) = ResourceSample::decode(payload) {
|
||||
self.resource = Some(r);
|
||||
}
|
||||
}
|
||||
catalog::RUNTIME_STATS => {
|
||||
if let Ok(r) = DsRuntimeStats::decode(payload) {
|
||||
self.runtime = Some(r);
|
||||
}
|
||||
}
|
||||
catalog::TRANSPORT_INTERNALS => {
|
||||
if let Ok(r) = TransportInternals::decode(payload) {
|
||||
self.transport = Some(r);
|
||||
}
|
||||
}
|
||||
catalog::LIFECYCLE_COST => {
|
||||
if let Ok(r) = LifecycleCost::decode(payload) {
|
||||
self.lifecycle = Some(r);
|
||||
}
|
||||
}
|
||||
catalog::MEMBERSHIP => {
|
||||
if let Ok(t) = MembershipTransition::decode(payload) {
|
||||
// Display the short id; key the membership map by the full
|
||||
// id so the views can resolve it to a friendly label.
|
||||
let line = format!("{}: {} → {}", short_id(&t.peer), t.from, t.to);
|
||||
self.membership.insert(t.peer.clone(), t.to.clone());
|
||||
self.last_transition = Some(line.clone());
|
||||
events.push(LogEvent::Info(format!("membership {line}")));
|
||||
}
|
||||
}
|
||||
// Raw-text process output: `proc.<label>.{stdout,stderr}`.
|
||||
_ if channel.starts_with("proc.") => {
|
||||
let line = String::from_utf8_lossy(payload).into_owned();
|
||||
if let Some((label, is_err)) = parse_proc_channel(channel) {
|
||||
let entry = self.procs.entry(label.to_string()).or_default();
|
||||
entry.0 += 1;
|
||||
entry.1 = line.clone();
|
||||
let tagged = format!("[{label}] {line}");
|
||||
events.push(if is_err {
|
||||
LogEvent::Warn(tagged)
|
||||
} else {
|
||||
LogEvent::Info(tagged)
|
||||
});
|
||||
}
|
||||
}
|
||||
// Unknown channel — ignored for display (still demuxed cleanly).
|
||||
_ => {}
|
||||
}
|
||||
events
|
||||
}
|
||||
|
||||
/// Has this node streamed a frame within [`NODE_TTL`]?
|
||||
fn is_live(&self, now: Instant) -> bool {
|
||||
self.last_seen
|
||||
.map(|t| now.duration_since(t) < NODE_TTL)
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// A friendly label for this node: `region · short-id` once its identity is
|
||||
/// known, else just the short id.
|
||||
fn label(&self, node_id: &str) -> String {
|
||||
match &self.identity {
|
||||
Some(idr) if !idr.region.is_empty() => {
|
||||
format!("{} · {}", idr.region, short_id(node_id))
|
||||
}
|
||||
_ => short_id(node_id),
|
||||
}
|
||||
}
|
||||
|
||||
/// Total process-output lines seen across all labels.
|
||||
fn total_proc_lines(&self) -> u64 {
|
||||
self.procs.values().map(|(n, _)| *n).sum()
|
||||
}
|
||||
|
||||
/// The peers this node currently sees that are themselves still live, as
|
||||
/// `(peer_id, state)`. Filtering by liveness drops stale incarnations a
|
||||
/// lost SWIM `dead` transition would otherwise leave stuck at `alive`.
|
||||
fn live_peers<'a>(&'a self, live: &HashSet<String>) -> Vec<(&'a String, &'a String)> {
|
||||
self.membership
|
||||
.iter()
|
||||
.filter(|(peer, _)| live.contains(peer.as_str()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Synthesize the dashboard's native stats from the accumulated datastream.
|
||||
fn to_runtime_stats(&self) -> RuntimeStats {
|
||||
let ds_rt = self.runtime.clone().unwrap_or(DsRuntimeStats {
|
||||
actors_live: 0,
|
||||
mailbox_depth: 0,
|
||||
scheduled_tasks: 0,
|
||||
});
|
||||
|
||||
// One synthetic worker = this node.
|
||||
let worker = WorkerInfo {
|
||||
id: 0,
|
||||
num_actors: ds_rt.actors_live as usize,
|
||||
mailbox_depth: ds_rt.mailbox_depth as usize,
|
||||
messages_processed: self.total_proc_lines(),
|
||||
local_sends: 0,
|
||||
cross_sends: 0,
|
||||
inbox_sends: 0,
|
||||
type_mismatches: 0,
|
||||
panics: 0,
|
||||
messages_dropped: 0,
|
||||
restarts: 0,
|
||||
stops: 0,
|
||||
};
|
||||
|
||||
// Each datastream channel becomes one synthetic actor row, with columns
|
||||
// and the per-actor breakdown chart repurposed to show its values.
|
||||
let mut actor_details: Vec<ActorInfo> = Vec::new();
|
||||
|
||||
if let Some(r) = &self.resource {
|
||||
actor_details.push(synth_actor(
|
||||
"host.resource",
|
||||
Some(format!(
|
||||
"cpu {:.0}% mem {}/{}MB",
|
||||
r.cpu_pct, r.mem_used_mb, r.mem_total_mb
|
||||
)),
|
||||
self.total_proc_lines(),
|
||||
vec![
|
||||
("cpu_pct".to_string(), r.cpu_pct.round() as u64),
|
||||
("mem_used_mb".to_string(), r.mem_used_mb as u64),
|
||||
("mem_total_mb".to_string(), r.mem_total_mb as u64),
|
||||
("gpu_pct".to_string(), r.gpu_pct.round() as u64),
|
||||
("disk_used_gb".to_string(), r.disk_used_gb as u64),
|
||||
("net_rx_kbps".to_string(), r.net_rx_kbps as u64),
|
||||
("net_tx_kbps".to_string(), r.net_tx_kbps as u64),
|
||||
],
|
||||
));
|
||||
}
|
||||
|
||||
if let Some(t) = &self.transport {
|
||||
actor_details.push(synth_actor(
|
||||
"transport.internals",
|
||||
Some(format!(
|
||||
"relay {} · {} direct · {} relayed",
|
||||
if t.relay_connected { "up" } else { "down" },
|
||||
t.direct_peers,
|
||||
t.relay_peers
|
||||
)),
|
||||
0,
|
||||
vec![
|
||||
("relay_connected".to_string(), t.relay_connected as u64),
|
||||
("direct_peers".to_string(), t.direct_peers as u64),
|
||||
("relay_peers".to_string(), t.relay_peers as u64),
|
||||
("rtt_ms_p50".to_string(), t.rtt_ms_p50 as u64),
|
||||
],
|
||||
));
|
||||
}
|
||||
|
||||
if !self.membership.is_empty() || self.last_transition.is_some() {
|
||||
let mut counts: HashMap<&str, u64> = HashMap::new();
|
||||
for state in self.membership.values() {
|
||||
*counts.entry(state.as_str()).or_default() += 1;
|
||||
}
|
||||
let breakdown = counts
|
||||
.into_iter()
|
||||
.map(|(k, v)| (k.to_string(), v))
|
||||
.collect();
|
||||
actor_details.push(synth_actor(
|
||||
"membership",
|
||||
self.last_transition.clone(),
|
||||
self.membership.len() as u64,
|
||||
breakdown,
|
||||
));
|
||||
}
|
||||
|
||||
// One row per process label, newest line as "last message".
|
||||
let mut procs: Vec<(&String, &(u64, String))> = self.procs.iter().collect();
|
||||
procs.sort_by(|a, b| a.0.cmp(b.0));
|
||||
for (label, (count, last)) in procs {
|
||||
actor_details.push(synth_actor(
|
||||
&format!("proc.{label}"),
|
||||
Some(last.clone()),
|
||||
*count,
|
||||
vec![("lines".to_string(), *count)],
|
||||
));
|
||||
}
|
||||
|
||||
let actors = actor_details
|
||||
.iter()
|
||||
.map(|a| (a.address, a.worker_id))
|
||||
.collect();
|
||||
|
||||
let uptime_ms = self
|
||||
.lifecycle
|
||||
.as_ref()
|
||||
.map(|l| l.uptime_s * 1000)
|
||||
.unwrap_or_else(|| {
|
||||
self.first_seen
|
||||
.map(|t| t.elapsed().as_millis() as u64)
|
||||
.unwrap_or(0)
|
||||
});
|
||||
|
||||
RuntimeStats {
|
||||
num_workers: 1,
|
||||
uptime_ms,
|
||||
actors,
|
||||
workers: vec![worker],
|
||||
actor_details,
|
||||
// No per-tick phase timing on the wire; one empty worker entry keeps
|
||||
// the phase bars degrading to a single fill rather than panicking.
|
||||
tick_timings: vec![Vec::<TickTiming>::new()],
|
||||
}
|
||||
}
|
||||
|
||||
/// Does this node match the optional selection filter? Matches against the
|
||||
/// node id and (once known) the identity region/role.
|
||||
fn matches(&self, node_id: &str, filter: &str) -> bool {
|
||||
if node_id.contains(filter) {
|
||||
return true;
|
||||
}
|
||||
match &self.identity {
|
||||
Some(id) => {
|
||||
id.region == filter || format!("{:?}", id.role).eq_ignore_ascii_case(filter)
|
||||
}
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Rebuild a [`DistributionNodeSnapshot`] for this node from the demuxed
|
||||
/// stream, so the canonical Distribution page renders its SWIM connection
|
||||
/// graph exactly as it would for a live node. Only live peers are included
|
||||
/// (`peer_labels`/`live` are the fleet-wide label map and live set).
|
||||
///
|
||||
/// Fields the datastream does not carry (routing table, cache, directory,
|
||||
/// registry) are honest zeros — the graph and membership panel, which is all
|
||||
/// this view exists to show, are driven entirely by `members`.
|
||||
fn dist_snapshot(
|
||||
&self,
|
||||
node_id: &str,
|
||||
peer_labels: &HashMap<String, String>,
|
||||
live: &HashSet<String>,
|
||||
) -> DistributionNodeSnapshot {
|
||||
let mut members: Vec<MemberInfo> = self
|
||||
.live_peers(live)
|
||||
.into_iter()
|
||||
.map(|(peer, state)| MemberInfo {
|
||||
node_id: peer.clone(),
|
||||
addr: None,
|
||||
state: state.clone(),
|
||||
incarnation: 0,
|
||||
is_authorized: None,
|
||||
label: None,
|
||||
relay_url: None,
|
||||
node_name: Some(peer_labels.get(peer).cloned().unwrap_or_else(|| short_id(peer))),
|
||||
})
|
||||
.collect();
|
||||
members.sort_by(|a, b| a.node_id.cmp(&b.node_id));
|
||||
|
||||
let count = |want: &str| members.iter().filter(|m| m.state == want).count();
|
||||
let (alive_count, suspect_count, dead_count) =
|
||||
(count("alive"), count("suspect"), count("dead"));
|
||||
|
||||
DistributionNodeSnapshot {
|
||||
node_id: node_id.to_string(),
|
||||
listen_addr: None,
|
||||
members,
|
||||
alive_count,
|
||||
suspect_count,
|
||||
dead_count,
|
||||
routing_table_size: 0,
|
||||
routing_buckets: Vec::new(),
|
||||
routing_neighbors: Vec::new(),
|
||||
cache_size: 0,
|
||||
cache_entries: Vec::new(),
|
||||
directory_entry_count: 0,
|
||||
repair_queue_size: 0,
|
||||
registry_size: 0,
|
||||
registry_tombstones: 0,
|
||||
registry_entries: Vec::new(),
|
||||
recent_probe_targets: Vec::new(),
|
||||
peer_auth_mode: "open".into(),
|
||||
authorized_peer_count: None,
|
||||
node_name: Some(peer_labels.get(node_id).cloned().unwrap_or_else(|| short_id(node_id))),
|
||||
invite_code: None,
|
||||
// `relay_url` means "this node runs an embedded relay server" (it
|
||||
// draws a teal ring in the graph) — not "is connected to a relay".
|
||||
// The datastream doesn't carry that, so leave it unset.
|
||||
relay_url: None,
|
||||
version: None,
|
||||
join_statuses: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// A compact per-node summary row for the fleet table.
|
||||
fn node_summary(
|
||||
&self,
|
||||
node_id: &str,
|
||||
expected_peers: usize,
|
||||
selected: bool,
|
||||
live: &HashSet<String>,
|
||||
) -> serde_json::Value {
|
||||
let peers = self.live_peers(live);
|
||||
let count = |want: &str| peers.iter().filter(|(_, s)| s.as_str() == want).count() as u32;
|
||||
let (alive, suspect, dead) = (count("alive"), count("suspect"), count("dead"));
|
||||
let seen = peers.len();
|
||||
// "Converged" from this node's vantage: it sees every other live node,
|
||||
// all alive (no suspect/dead).
|
||||
let converged = suspect == 0 && dead == 0 && seen >= expected_peers && expected_peers > 0;
|
||||
|
||||
let r = self.resource.as_ref();
|
||||
let t = self.transport.as_ref();
|
||||
let rt = self.runtime.as_ref();
|
||||
let id = self.identity.as_ref();
|
||||
let last_proc = self
|
||||
.procs
|
||||
.values()
|
||||
.map(|(_, line)| line.clone())
|
||||
.last()
|
||||
.unwrap_or_default();
|
||||
|
||||
serde_json::json!({
|
||||
"id": node_id,
|
||||
"short": short_id(node_id),
|
||||
"region": id.map(|i| i.region.clone()).unwrap_or_default(),
|
||||
"role": id.map(|i| format!("{:?}", i.role).to_lowercase()).unwrap_or_default(),
|
||||
"selected": selected,
|
||||
"cpu_pct": r.map(|r| r.cpu_pct.round() as u32).unwrap_or(0),
|
||||
"mem_used_mb": r.map(|r| r.mem_used_mb).unwrap_or(0),
|
||||
"mem_total_mb": r.map(|r| r.mem_total_mb).unwrap_or(0),
|
||||
"gpu_pct": r.map(|r| r.gpu_pct.round() as u32).unwrap_or(0),
|
||||
"actors_live": rt.map(|r| r.actors_live).unwrap_or(0),
|
||||
"mailbox_depth": rt.map(|r| r.mailbox_depth).unwrap_or(0),
|
||||
"relay_connected": t.map(|t| t.relay_connected).unwrap_or(false),
|
||||
"direct_peers": t.map(|t| t.direct_peers).unwrap_or(0),
|
||||
"relay_peers": t.map(|t| t.relay_peers).unwrap_or(0),
|
||||
"rtt_ms_p50": t.map(|t| t.rtt_ms_p50).unwrap_or(0),
|
||||
"alive": alive,
|
||||
"suspect": suspect,
|
||||
"dead": dead,
|
||||
"converged": converged,
|
||||
"proc_lines": self.total_proc_lines(),
|
||||
"last_proc": last_proc,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// `node-id → "region · short-id"` for every demuxed node.
|
||||
fn build_labels(models: &HashMap<String, DatastreamModel>) -> HashMap<String, String> {
|
||||
models.iter().map(|(id, m)| (id.clone(), m.label(id))).collect()
|
||||
}
|
||||
|
||||
/// The set of node ids currently live (streamed within [`NODE_TTL`]).
|
||||
fn live_set(models: &HashMap<String, DatastreamModel>, now: Instant) -> HashSet<String> {
|
||||
models
|
||||
.iter()
|
||||
.filter(|(_, m)| m.is_live(now))
|
||||
.map(|(id, _)| id.clone())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Build the Fleet-table JSON model from every *live* demuxed node.
|
||||
fn fleet_json(
|
||||
models: &HashMap<String, DatastreamModel>,
|
||||
selected: Option<&str>,
|
||||
live: &HashSet<String>,
|
||||
) -> String {
|
||||
let node_count = live.len();
|
||||
let expected_peers = node_count.saturating_sub(1);
|
||||
let mut nodes: Vec<serde_json::Value> = models
|
||||
.iter()
|
||||
.filter(|(id, _)| live.contains(id.as_str()))
|
||||
.map(|(id, m)| m.node_summary(id, expected_peers, selected == Some(id.as_str()), live))
|
||||
.collect();
|
||||
// Stable order: region then short id, so rows don't jump around.
|
||||
nodes.sort_by(|a, b| {
|
||||
(a["region"].as_str(), a["short"].as_str())
|
||||
.cmp(&(b["region"].as_str(), b["short"].as_str()))
|
||||
});
|
||||
// Whole fleet converged once every node has converged and there's >1 node.
|
||||
let converged =
|
||||
node_count > 1 && nodes.iter().all(|n| n["converged"].as_bool() == Some(true));
|
||||
|
||||
serde_json::json!({
|
||||
"node_count": node_count,
|
||||
"converged": converged,
|
||||
"nodes": nodes,
|
||||
})
|
||||
.to_string()
|
||||
}
|
||||
|
||||
/// Plugin backed by a shared cache string: serves a fixed HTML page, emits its
|
||||
/// cache on the SSE stream under `name`, and answers `GET /api/plugin/{name}`.
|
||||
/// Used for both the Fleet table (`vastai`) and the Distribution graph
|
||||
/// (`distribution`); each is fed by [`run_datastream_ingest`].
|
||||
struct CachePlugin {
|
||||
name: &'static str,
|
||||
page: &'static str,
|
||||
cache: Arc<Mutex<Option<String>>>,
|
||||
}
|
||||
|
||||
impl CachePlugin {
|
||||
fn new(name: &'static str, page: &'static str, cache: Arc<Mutex<Option<String>>>) -> Self {
|
||||
Self { name, page, cache }
|
||||
}
|
||||
}
|
||||
|
||||
impl DashboardPlugin for CachePlugin {
|
||||
fn name(&self) -> &str {
|
||||
self.name
|
||||
}
|
||||
|
||||
fn snapshot_json(&self) -> Option<String> {
|
||||
self.cache.lock().unwrap().clone()
|
||||
}
|
||||
|
||||
fn handle_request(
|
||||
&self,
|
||||
method: &str,
|
||||
path: &str,
|
||||
_query: &HashMap<String, String>,
|
||||
_body: &[u8],
|
||||
) -> PluginResponse {
|
||||
match (method, path) {
|
||||
("GET", "" | "model" | "snapshot") => {
|
||||
PluginResponse::json(self.cache.lock().unwrap().clone().unwrap_or_else(|| "{}".into()))
|
||||
}
|
||||
// The Distribution page has Re-peer / dismiss buttons; in this
|
||||
// read-only datastream view they are inert (acknowledged, no-op).
|
||||
("POST", "rejoin" | "clear_status") => PluginResponse::json(r#"{"ok":true}"#.into()),
|
||||
_ => PluginResponse::not_found(),
|
||||
}
|
||||
}
|
||||
|
||||
fn html_page(&self) -> Option<&str> {
|
||||
Some(self.page)
|
||||
}
|
||||
}
|
||||
|
||||
/// Minimal `peers` plugin so the Distribution page's `fetch('/api/plugin/peers')`
|
||||
/// resolves cleanly (open auth, no managed peer list) instead of 404ing. Silent
|
||||
/// on the SSE stream.
|
||||
struct PeersStub;
|
||||
|
||||
impl DashboardPlugin for PeersStub {
|
||||
fn name(&self) -> &str {
|
||||
"peers"
|
||||
}
|
||||
fn snapshot_json(&self) -> Option<String> {
|
||||
None
|
||||
}
|
||||
fn handle_request(
|
||||
&self,
|
||||
method: &str,
|
||||
path: &str,
|
||||
_query: &HashMap<String, String>,
|
||||
_body: &[u8],
|
||||
) -> PluginResponse {
|
||||
match (method, path) {
|
||||
("GET", "" | "list") => PluginResponse::json(r#"{"mode":"open","peers":[]}"#.into()),
|
||||
_ => PluginResponse::not_found(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Build one synthetic actor row with a deterministic address from its name.
|
||||
///
|
||||
/// `mailbox_depth` is always 0: a datastream channel is not a real mailbox, and
|
||||
/// the dashboard's WarningDetector flags any row with `depth > 0` whose
|
||||
/// `messages_processed` is flat as a "stalled actor". The channel's real values
|
||||
/// live in `last_msg_type` and the `message_type_counts` breakdown instead.
|
||||
fn synth_actor(
|
||||
name: &str,
|
||||
last_msg_type: Option<String>,
|
||||
messages_processed: u64,
|
||||
message_type_counts: Vec<(String, u64)>,
|
||||
) -> ActorInfo {
|
||||
ActorInfo {
|
||||
address: addr_from(name),
|
||||
worker_id: 0,
|
||||
mailbox_depth: 0,
|
||||
last_msg_type,
|
||||
messages_processed,
|
||||
poisoned: false,
|
||||
name: Some(name.to_string()),
|
||||
message_type_counts,
|
||||
}
|
||||
}
|
||||
|
||||
/// Deterministic 32-byte address from a channel name (FNV-1a in the first 8
|
||||
/// bytes — which is what the address Hash/Display use — plus the name splatted
|
||||
/// after for readability). Stable across frames so sparklines accumulate.
|
||||
fn addr_from(name: &str) -> ActorAddress {
|
||||
let mut bytes = [0u8; 32];
|
||||
let mut hash: u64 = 0xcbf2_9ce4_8422_2325;
|
||||
for b in name.bytes() {
|
||||
hash ^= b as u64;
|
||||
hash = hash.wrapping_mul(0x0100_0000_01b3);
|
||||
}
|
||||
bytes[..8].copy_from_slice(&hash.to_le_bytes());
|
||||
for (i, b) in name.bytes().take(24).enumerate() {
|
||||
bytes[8 + i] = b;
|
||||
}
|
||||
ActorAddress(bytes)
|
||||
}
|
||||
|
||||
/// Split `proc.<label>.stdout` / `proc.<label>.stderr` into `(label, is_stderr)`.
|
||||
fn parse_proc_channel(channel: &str) -> Option<(&str, bool)> {
|
||||
let rest = channel.strip_prefix("proc.")?;
|
||||
if let Some(label) = rest.strip_suffix(".stderr") {
|
||||
Some((label, true))
|
||||
} else {
|
||||
rest.strip_suffix(".stdout").map(|label| (label, false))
|
||||
}
|
||||
}
|
||||
|
||||
/// First 8 chars of an id — the readable short form used throughout the views.
|
||||
fn short_id(id: &str) -> String {
|
||||
id[..id.len().min(8)].to_string()
|
||||
}
|
||||
|
||||
/// Bind the UDP datastream sink, demux frames, and drive the dashboard's views.
|
||||
/// Blocks forever, like the dumb collector's `main`.
|
||||
///
|
||||
/// Registers three plugins fed off the demuxed stream:
|
||||
/// * `distribution` — the canonical SWIM connection-graph page, snapshotting
|
||||
/// the selected node;
|
||||
/// * `vastai` (the nav's "Fleet") — the cross-node telemetry table;
|
||||
/// * `peers` — an open-auth stub so the graph page's peer fetch resolves.
|
||||
///
|
||||
/// The single-node Overview/Actors page is driven via `handle.set_stats`.
|
||||
pub fn run_datastream_ingest(
|
||||
bind: &str,
|
||||
node_filter: Option<&str>,
|
||||
handle: &DashboardHandle,
|
||||
) -> std::io::Result<()> {
|
||||
let sock = UdpSocket::bind(bind)?;
|
||||
eprintln!("datastream dashboard: listening on {bind} (one frame per datagram)");
|
||||
|
||||
// Distribution graph (selected node) and Fleet table (all live nodes), each
|
||||
// a shared cache the ingest loop refreshes and the SSE loop fans out.
|
||||
let dist_cache: Arc<Mutex<Option<String>>> = Arc::new(Mutex::new(None));
|
||||
let fleet_cache: Arc<Mutex<Option<String>>> = Arc::new(Mutex::new(None));
|
||||
handle.register_plugin(Arc::new(CachePlugin::new(
|
||||
"distribution",
|
||||
DISTRIBUTION_PAGE_HTML,
|
||||
Arc::clone(&dist_cache),
|
||||
)) as Arc<dyn DashboardPlugin>);
|
||||
handle.register_plugin(Arc::new(CachePlugin::new(
|
||||
"vastai",
|
||||
FLEET_HTML,
|
||||
Arc::clone(&fleet_cache),
|
||||
)) as Arc<dyn DashboardPlugin>);
|
||||
// The base dashboard nav has a Datastore link; the datastream carries no
|
||||
// datastore, so serve an honest "not available" page in-chrome rather than
|
||||
// 404ing. Empty cache → silent on the SSE stream.
|
||||
handle.register_plugin(Arc::new(CachePlugin::new(
|
||||
"datastore",
|
||||
DATASTORE_HTML,
|
||||
Arc::new(Mutex::new(None)),
|
||||
)) as Arc<dyn DashboardPlugin>);
|
||||
handle.register_plugin(Arc::new(PeersStub) as Arc<dyn DashboardPlugin>);
|
||||
|
||||
let mut models: HashMap<String, DatastreamModel> = HashMap::new();
|
||||
let mut selected: Option<String> = None;
|
||||
// 64 KiB comfortably exceeds a UDP datagram; a frame never spans datagrams.
|
||||
let mut buf = vec![0u8; 64 * 1024];
|
||||
|
||||
loop {
|
||||
let (n, _src) = match sock.recv_from(&mut buf) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
eprintln!("datastream dashboard: recv error: {e}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let (stream, frame) = match decode_delivery(&buf[..n]) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
eprintln!("datastream dashboard: dropped malformed datagram ({n} B): {e:?}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let node = stream.node.as_str().to_string();
|
||||
let model = models.entry(node.clone()).or_default();
|
||||
let events = model.update(frame.channel.as_str(), &frame.payload);
|
||||
|
||||
// Pick the display node: first matching the filter, else first seen.
|
||||
if selected.is_none() {
|
||||
let qualifies = match node_filter {
|
||||
Some(f) => model.matches(&node, f),
|
||||
None => true,
|
||||
};
|
||||
if qualifies {
|
||||
eprintln!("datastream dashboard: displaying node {}", short_id(&node));
|
||||
selected = Some(node.clone());
|
||||
}
|
||||
}
|
||||
|
||||
if selected.as_deref() == Some(node.as_str()) {
|
||||
// Surface process output / membership through the activity log.
|
||||
for ev in events {
|
||||
match ev {
|
||||
LogEvent::Info(m) => tracing::info!(target: "datastream", "{m}"),
|
||||
LogEvent::Warn(m) => tracing::warn!(target: "datastream", "{m}"),
|
||||
}
|
||||
}
|
||||
handle.set_stats(model.to_runtime_stats());
|
||||
}
|
||||
|
||||
// Refresh both views from the current live nodes on each frame.
|
||||
let now = Instant::now();
|
||||
let live = live_set(&models, now);
|
||||
let labels = build_labels(&models);
|
||||
*fleet_cache.lock().unwrap() = Some(fleet_json(&models, selected.as_deref(), &live));
|
||||
if let Some(sel) = selected.as_deref() {
|
||||
if let Some(m) = models.get(sel) {
|
||||
let snap = m.dist_snapshot(sel, &labels, &live);
|
||||
if let Ok(json) = serde_json::to_string(&snap) {
|
||||
*dist_cache.lock().unwrap() = Some(json);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Cross-node **Fleet** table, served in the dashboard's own chrome (the same
|
||||
/// header / nav bar / palette as the Overview and Distribution pages, so it is a
|
||||
/// section of the one app — not a separate UI). Renders the `vastai` SSE event
|
||||
/// built by [`fleet_json`]: one row per live node with its resource / runtime /
|
||||
/// transport telemetry and a fleet-wide SWIM-convergence pill.
|
||||
const FLEET_HTML: &str = r#"<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Swactor Runtime – Fleet</title>
|
||||
<style>
|
||||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||
body { font-family: 'Menlo', 'Consolas', 'Monaco', monospace; background: #0f1117; color: #e0e0e0; font-size: 13px; }
|
||||
.header { display: flex; align-items: center; justify-content: space-between;
|
||||
padding: 12px 20px; background: #161822; border-bottom: 1px solid #2a2d3e; }
|
||||
.header-left { display: flex; align-items: center; }
|
||||
.header h1 { font-size: 16px; font-weight: 600; color: #fff; }
|
||||
.status-dot { width: 10px; height: 10px; border-radius: 50%; background: #4caf50;
|
||||
display: inline-block; margin-left: 8px; vertical-align: middle; }
|
||||
.status-dot.disconnected { background: #f44336; }
|
||||
.status-dot.done { background: #ff9800; }
|
||||
.nav-links { display: flex; gap: 4px; margin-left: 20px; }
|
||||
.nav-link { color: #888; text-decoration: none; font-size: 12px;
|
||||
padding: 4px 10px; border-radius: 3px; transition: color 0.2s; }
|
||||
.nav-link:hover { color: #e0e0e0; }
|
||||
.nav-link.active { color: #fff; background: #2a2d3e; }
|
||||
.header-right { display: flex; align-items: center; gap: 12px; }
|
||||
.pill { padding: 3px 12px; border-radius: 999px; font-size: 12px; font-weight: 600; }
|
||||
.pill.ok { background: #14361f; color: #4ade80; }
|
||||
.pill.warn { background: #3a2d12; color: #fbbf24; }
|
||||
.content { padding: 16px 20px; }
|
||||
table { width: 100%; border-collapse: collapse; }
|
||||
th, td { text-align: left; padding: 7px 12px; border-bottom: 1px solid #1f2230; white-space: nowrap; }
|
||||
th { color: #888; font-weight: 600; font-size: 11px; text-transform: uppercase; letter-spacing: .5px; }
|
||||
td.num { text-align: right; font-variant-numeric: tabular-nums; }
|
||||
tr.sel td { background: #161e2e; }
|
||||
.tag { padding: 1px 8px; border-radius: 4px; background: #1c1f2e; color: #888; font-size: 11px; }
|
||||
.tag.coordinator { background: #1c2d4a; color: #79c0ff; }
|
||||
.tag.worker { background: #1a2e2e; color: #56d4dd; }
|
||||
.bar { display: inline-block; width: 64px; height: 7px; background: #1c1f2e; border-radius: 4px;
|
||||
overflow: hidden; vertical-align: middle; margin-right: 6px; }
|
||||
.bar > i { display: block; height: 100%; background: #4ade80; }
|
||||
.bar > i.hi { background: #fbbf24; } .bar > i.crit { background: #f87171; }
|
||||
.st { padding: 1px 6px; border-radius: 4px; font-weight: 600; font-size: 11px; }
|
||||
.st.suspect { background: #3a2d12; color: #fbbf24; } .st.dead { background: #3a1518; color: #f87171; }
|
||||
.muted { color: #666; }
|
||||
.last { max-width: 380px; overflow: hidden; text-overflow: ellipsis; color: #999; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="header">
|
||||
<div class="header-left">
|
||||
<h1>Swactor Runtime Dashboard <span id="statusDot" class="status-dot disconnected"></span></h1>
|
||||
<nav class="nav-links">
|
||||
<a href="/" class="nav-link">Overview</a>
|
||||
<a href="/actors" class="nav-link">Actors</a>
|
||||
<a href="/plugin/distribution" class="nav-link">Distribution</a>
|
||||
<a href="/plugin/datastore" class="nav-link">Datastore</a>
|
||||
<a href="/plugin/vastai" class="nav-link active">Fleet</a>
|
||||
</nav>
|
||||
</div>
|
||||
<div class="header-right">
|
||||
<span id="conv" class="pill warn">SWIM: …</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="content">
|
||||
<table>
|
||||
<thead><tr>
|
||||
<th>node</th><th>region</th><th>role</th><th>CPU</th><th>mem</th>
|
||||
<th class="num">actors</th><th class="num">mbox</th><th>transport</th>
|
||||
<th class="num">peers</th><th class="num">proc</th><th>last line</th>
|
||||
</tr></thead>
|
||||
<tbody id="rows"></tbody>
|
||||
</table>
|
||||
<p id="empty" class="muted" style="margin-top:12px;">waiting for telemetry…</p>
|
||||
</div>
|
||||
<script>
|
||||
(function () {
|
||||
var state = { node_count: 0, converged: false, nodes: [] };
|
||||
function esc(s){ return String(s==null?"":s).replace(/[&<>]/g, function(c){
|
||||
return {"&":"&","<":"<",">":">"}[c]; }); }
|
||||
function bar(pct){ var c = pct>=90?"crit":pct>=70?"hi":"";
|
||||
return '<span class="bar"><i class="'+c+'" style="width:'+Math.min(100,pct)+'%"></i></span>'+pct+'%'; }
|
||||
|
||||
function render(){
|
||||
var conv = document.getElementById("conv");
|
||||
if (state.node_count > 1 && state.converged){ conv.className="pill ok"; conv.textContent="SWIM: converged ("+state.node_count+" nodes)"; }
|
||||
else { conv.className="pill warn"; conv.textContent="SWIM: converging ("+state.node_count+" nodes)"; }
|
||||
|
||||
var rows = state.nodes.map(function(n){
|
||||
var peers = n.alive + (n.suspect?(' <span class="st suspect">'+n.suspect+'</span>'):'')
|
||||
+ (n.dead?(' <span class="st dead">'+n.dead+'</span>'):'');
|
||||
return '<tr class="'+(n.selected?"sel":"")+'">'
|
||||
+ '<td>'+esc(n.short)+(n.selected?' <span class="muted">(shown)</span>':'')+'</td>'
|
||||
+ '<td>'+esc(n.region)+'</td>'
|
||||
+ '<td><span class="tag '+esc(n.role)+'">'+esc(n.role||"?")+'</span></td>'
|
||||
+ '<td>'+bar(n.cpu_pct)+'</td>'
|
||||
+ '<td>'+n.mem_used_mb+'/'+n.mem_total_mb+'MB</td>'
|
||||
+ '<td class="num">'+n.actors_live+'</td>'
|
||||
+ '<td class="num">'+n.mailbox_depth+'</td>'
|
||||
+ '<td>'+(n.relay_connected?'relay ':'')+n.direct_peers+'d/'+n.relay_peers+'r</td>'
|
||||
+ '<td class="num">'+peers+'</td>'
|
||||
+ '<td class="num">'+n.proc_lines+'</td>'
|
||||
+ '<td class="last">'+esc(n.last_proc)+'</td>'
|
||||
+ '</tr>';
|
||||
}).join("");
|
||||
document.getElementById("rows").innerHTML = rows;
|
||||
document.getElementById("empty").style.display = state.nodes.length ? "none" : "block";
|
||||
}
|
||||
|
||||
function onModel(m){ if(!m||!m.nodes) return; state = m; render(); }
|
||||
|
||||
fetch("/api/plugin/vastai/model").then(function(r){return r.json();}).then(onModel).catch(function(){});
|
||||
var dot = document.getElementById("statusDot");
|
||||
var es = new EventSource("/events");
|
||||
es.onopen = function(){ dot.className = "status-dot"; };
|
||||
es.onerror = function(){ dot.className = "status-dot disconnected"; };
|
||||
es.addEventListener("vastai", function(e){ try { onModel(JSON.parse(e.data)); } catch(err){} });
|
||||
es.addEventListener("done", function(){ dot.className = "status-dot done"; es.close(); });
|
||||
window.addEventListener("beforeunload", function(){ es.close(); });
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>"#;
|
||||
|
||||
/// Honest in-chrome placeholder for the Datastore nav link: the datastream demo
|
||||
/// ships no datastore telemetry, so rather than 404 the link, explain that.
|
||||
const DATASTORE_HTML: &str = r#"<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Swactor Runtime – Datastore</title>
|
||||
<style>
|
||||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||
body { font-family: 'Menlo', 'Consolas', 'Monaco', monospace; background: #0f1117; color: #e0e0e0; font-size: 13px; }
|
||||
.header { display: flex; align-items: center; padding: 12px 20px; background: #161822; border-bottom: 1px solid #2a2d3e; }
|
||||
.header h1 { font-size: 16px; font-weight: 600; color: #fff; }
|
||||
.nav-links { display: flex; gap: 4px; margin-left: 20px; }
|
||||
.nav-link { color: #888; text-decoration: none; font-size: 12px; padding: 4px 10px; border-radius: 3px; }
|
||||
.nav-link:hover { color: #e0e0e0; }
|
||||
.nav-link.active { color: #fff; background: #2a2d3e; }
|
||||
.note { margin: 80px auto; max-width: 520px; text-align: center; color: #888; line-height: 1.6; }
|
||||
.note b { color: #cbd5e1; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="header">
|
||||
<h1>Swactor Runtime Dashboard</h1>
|
||||
<nav class="nav-links">
|
||||
<a href="/" class="nav-link">Overview</a>
|
||||
<a href="/actors" class="nav-link">Actors</a>
|
||||
<a href="/plugin/distribution" class="nav-link">Distribution</a>
|
||||
<a href="/plugin/datastore" class="nav-link active">Datastore</a>
|
||||
<a href="/plugin/vastai" class="nav-link">Fleet</a>
|
||||
</nav>
|
||||
</div>
|
||||
<div class="note">
|
||||
<p><b>No datastore in this view.</b></p>
|
||||
<p>This dashboard is fed by the per-node telemetry <b>datastream</b>, which does not
|
||||
carry datastore contents. See <a href="/plugin/distribution" class="nav-link">Distribution</a>
|
||||
for the cluster connection graph or <a href="/plugin/vastai" class="nav-link">Fleet</a> for per-node telemetry.</p>
|
||||
</div>
|
||||
</body>
|
||||
</html>"#;
|
||||
|
|
@ -873,7 +873,9 @@
|
|||
function setToken(t) { dashToken = t; }
|
||||
|
||||
function fetchPeers() {
|
||||
fetch('/api/plugin/peers')
|
||||
// The plugin API route requires a non-empty sub-path ('' 404s), and the
|
||||
// peers plugin answers GET .../list — so query that, not the bare name.
|
||||
fetch('/api/plugin/peers/list')
|
||||
.then(function(r) { return r.json(); })
|
||||
.then(function(d) { renderPeers(d); })
|
||||
.catch(function() {});
|
||||
|
|
@ -15,6 +15,15 @@ pub mod tui;
|
|||
#[cfg(feature = "live-collector")]
|
||||
pub mod live_collector;
|
||||
|
||||
#[cfg(feature = "datastream")]
|
||||
pub mod datastream_source;
|
||||
|
||||
/// The canonical Distribution page (the SWIM connection-graph view). Owned by
|
||||
/// the dashboard crate so every front-end that serves it — a live node's
|
||||
/// `DistributionPlugin` and the datastream dashboard — renders the exact same
|
||||
/// page and chrome, fed by a [`distribution::snapshot::DistributionNodeSnapshot`].
|
||||
pub const DISTRIBUTION_PAGE_HTML: &str = include_str!("distribution_page.html");
|
||||
|
||||
use std::io;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
|
@ -106,6 +115,9 @@ pub struct DashboardHandle {
|
|||
store: Arc<EventStore>,
|
||||
runtime: Arc<Mutex<Option<Arc<Runtime>>>>,
|
||||
collector: Arc<Mutex<Option<Arc<StatsCollector>>>>,
|
||||
/// Externally pushed stats, used when no live `Runtime` is attached (e.g. a
|
||||
/// datastream-backed source synthesizes `RuntimeStats` and pushes them here).
|
||||
pushed_stats: Arc<Mutex<Option<RuntimeStats>>>,
|
||||
shutdown: Arc<AtomicBool>,
|
||||
shutdown_notify: Arc<tokio::sync::Notify>,
|
||||
stats_timeline: Arc<ArrayQueue<TimestampedStats>>,
|
||||
|
|
@ -141,6 +153,13 @@ impl DashboardHandle {
|
|||
*self.collector.lock().unwrap() = Some(collector);
|
||||
}
|
||||
|
||||
/// Push a stats snapshot from an external source (latest wins). When no live
|
||||
/// `Runtime` is attached, the SSE loop serves these to the dashboard exactly
|
||||
/// as if they came from a runtime. Used by the datastream source.
|
||||
pub fn set_stats(&self, stats: RuntimeStats) {
|
||||
*self.pushed_stats.lock().unwrap() = Some(stats);
|
||||
}
|
||||
|
||||
/// Whether trace recording is enabled.
|
||||
pub fn is_recording(&self) -> bool {
|
||||
self.recording
|
||||
|
|
@ -205,6 +224,7 @@ impl DashboardHandle {
|
|||
store: Arc::clone(&self.store),
|
||||
runtime: Arc::clone(&self.runtime),
|
||||
collector: Arc::clone(&self.collector),
|
||||
pushed_stats: Arc::clone(&self.pushed_stats),
|
||||
shutdown: Arc::clone(&self.shutdown),
|
||||
shutdown_notify: Arc::clone(&self.shutdown_notify),
|
||||
history: Arc::clone(&self.history),
|
||||
|
|
@ -252,6 +272,7 @@ pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle {
|
|||
));
|
||||
let runtime: Arc<Mutex<Option<Arc<Runtime>>>> = Arc::new(Mutex::new(None));
|
||||
let collector: Arc<Mutex<Option<Arc<StatsCollector>>>> = Arc::new(Mutex::new(None));
|
||||
let pushed_stats: Arc<Mutex<Option<RuntimeStats>>> = Arc::new(Mutex::new(None));
|
||||
let shutdown = Arc::new(AtomicBool::new(false));
|
||||
let shutdown_notify = Arc::new(tokio::sync::Notify::new());
|
||||
let stats_timeline = Arc::new(ArrayQueue::new(config.record_stats_capacity.max(1)));
|
||||
|
|
@ -292,6 +313,7 @@ pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle {
|
|||
store,
|
||||
runtime,
|
||||
collector,
|
||||
pushed_stats,
|
||||
shutdown,
|
||||
shutdown_notify,
|
||||
stats_timeline,
|
||||
|
|
|
|||
|
|
@ -40,6 +40,8 @@ pub(crate) struct AppState {
|
|||
pub store: Arc<EventStore>,
|
||||
pub runtime: Arc<Mutex<Option<Arc<Runtime>>>>,
|
||||
pub collector: Arc<Mutex<Option<Arc<StatsCollector>>>>,
|
||||
/// Externally pushed stats, served when no live `Runtime` is attached.
|
||||
pub pushed_stats: Arc<Mutex<Option<swactor::stats::RuntimeStats>>>,
|
||||
pub shutdown: Arc<AtomicBool>,
|
||||
pub shutdown_notify: Arc<tokio::sync::Notify>,
|
||||
pub history: Arc<DashboardHistory>,
|
||||
|
|
@ -163,6 +165,7 @@ async fn handle_live_sse(
|
|||
// Send stats if runtime is available
|
||||
{
|
||||
let maybe_rt = state.runtime.lock().unwrap().clone();
|
||||
let maybe_pushed = state.pushed_stats.lock().unwrap().clone();
|
||||
if let Some(rt) = maybe_rt {
|
||||
let mut stats = rt.stats();
|
||||
if let Some(col) = state.collector.lock().unwrap().as_ref() {
|
||||
|
|
@ -185,6 +188,32 @@ async fn handle_live_sse(
|
|||
}
|
||||
|
||||
// Send topology every 5th tick (~1/sec)
|
||||
tick_count += 1;
|
||||
if tick_count.is_multiple_of(5) {
|
||||
let topo = topology::worker_topology(&stats);
|
||||
if let Ok(tjson) = serde_json::to_string(&topo)
|
||||
&& tx.send(format_sse("topology", &tjson)).await.is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
} else if let Some(stats) = maybe_pushed {
|
||||
// No live runtime: serve externally pushed stats (e.g. a
|
||||
// datastream source). These already carry full actor_details,
|
||||
// so no collector/name enrichment is needed.
|
||||
state.history.record(&stats);
|
||||
|
||||
let warnings = warning_detector.check(&stats);
|
||||
if !warnings.is_empty()
|
||||
&& let Ok(wjson) = serde_json::to_string(&warnings)
|
||||
&& tx.send(format_sse("warnings", &wjson)).await.is_err() {
|
||||
return;
|
||||
}
|
||||
|
||||
let json = serde_json::to_string(&stats).unwrap();
|
||||
if tx.send(format_sse("stats", &json)).await.is_err() {
|
||||
return;
|
||||
}
|
||||
|
||||
tick_count += 1;
|
||||
if tick_count.is_multiple_of(5) {
|
||||
let topo = topology::worker_topology(&stats);
|
||||
|
|
|
|||
|
|
@ -1,49 +0,0 @@
|
|||
# Distribution Crate — Design Notes
|
||||
|
||||
Design decisions behind non-obvious mechanisms in the distribution crate.
|
||||
|
||||
---
|
||||
|
||||
## Transmit Budget (dissemination.rs)
|
||||
|
||||
The transmit budget controls how many times a membership update gets piggybacked onto
|
||||
protocol messages before being evicted from the dissemination queue.
|
||||
|
||||
It is computed as **`Λ * ceil(log₂(n))`** where `Λ` (lambda) is a configurable multiplier
|
||||
and `n` is the cluster size. The logarithmic scaling ensures that in a 10-node cluster
|
||||
each update is sent ~4Λ times, while in a 1000-node cluster it gets ~10Λ sends — enough
|
||||
redundancy for epidemic-style convergence without flooding the network.
|
||||
|
||||
Each time an update is piggybacked onto a Ping or Ack message, its remaining budget
|
||||
decrements by 1. When the budget reaches zero the update is evicted from the queue.
|
||||
Higher-priority updates (e.g. deaths) are piggybacked first, so critical state changes
|
||||
propagate faster than routine alive announcements.
|
||||
|
||||
## Re-Replication (kademlia/repair.rs — RepairQueue)
|
||||
|
||||
In the Kademlia directory, each actor's location entry is STOREd on the `r` closest nodes
|
||||
(by XOR distance to the actor address). When one of those replica holders dies, the
|
||||
replication factor drops below `r`.
|
||||
|
||||
**Re-replication** restores the target replication factor: surviving nodes that detect the
|
||||
death extract all directory entries the dead node held and re-STORE them on the
|
||||
next-closest node that didn't already have a copy.
|
||||
|
||||
In practice: `RepairQueue::on_node_death()` pulls all entries authored by the dead node
|
||||
from the local `DirectoryShard` and queues them. The node's tick loop drains the queue
|
||||
and issues STORE RPCs to the new r-closest nodes, restoring the replication invariant.
|
||||
|
||||
## Periodic Republish (kademlia/repair.rs — RepublishTracker)
|
||||
|
||||
Topology churn — nodes joining and leaving — gradually shifts which nodes are "r-closest"
|
||||
to a given actor address in XOR space. Without periodic republishing:
|
||||
|
||||
- A new node that joins *closer* to an actor than existing replicas would never learn
|
||||
about that actor's entry.
|
||||
- Entries could become stranded on nodes that are no longer among the closest, making
|
||||
lookups slower or requiring more hops.
|
||||
|
||||
`RepublishTracker` has each node periodically re-STORE the directory entries for its own
|
||||
locally-spawned actors at a configurable interval. This ensures entries migrate to the
|
||||
current r-closest nodes as the topology evolves, without waiting for a failure event to
|
||||
trigger repair.
|
||||
226
crates/distribution/src/datastream/catalog.rs
Normal file
226
crates/distribution/src/datastream/catalog.rs
Normal file
|
|
@ -0,0 +1,226 @@
|
|||
//! The channel catalog (spec §6): the kinds of observation the datastream
|
||||
//! carries, as channels, plus the codecs that interpret typed channels.
|
||||
//!
|
||||
//! This is the schema contract between producers and views. The *pipe*
|
||||
//! never consults it — only producers (to tag bytes) and views (to decode)
|
||||
//! do. Adding a channel here, or teaching a view a new codec, changes
|
||||
//! nothing in the mux, transport, ingest, or store (spec §6.3, §9.3).
|
||||
//!
|
||||
//! Typed channels use JSON as their codec. JSON is forgiving by design:
|
||||
//! decoding ignores unknown fields and `#[serde(default)]` fills missing
|
||||
//! ones, so a producer and a consumer can evolve a record independently
|
||||
//! (spec §6.3 version skew). A typed channel decodes to a record; a
|
||||
//! raw-text channel's "codec" is the identity and a view treats it as
|
||||
//! lines (spec §4.2).
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::frame::ChannelId;
|
||||
|
||||
// ── Typed channel ids (spec §6.1) ──────────────────────────────────────
|
||||
//
|
||||
// Each is a stable token. They are `&'static str` constants, not an enum,
|
||||
// so that "unknown channel" is simply "an id with no entry here" and a
|
||||
// newer producer's channel still lands whole in the store (spec §6.3).
|
||||
|
||||
/// Identity / boot — emitted first in a stream; identifies the node and
|
||||
/// its context so the consumer can attribute the stream. Event-driven.
|
||||
pub const IDENTITY: &str = "identity";
|
||||
/// Host / resource samples — periodic snapshots of machine resources.
|
||||
pub const HOST_RESOURCE: &str = "host.resource";
|
||||
/// Transport internals — the node's connectivity to peers and the relay.
|
||||
pub const TRANSPORT_INTERNALS: &str = "transport.internals";
|
||||
/// Membership / liveness — the node's view of which peers are alive,
|
||||
/// suspect, or dead, and transitions thereof. Event-driven.
|
||||
pub const MEMBERSHIP: &str = "membership";
|
||||
/// Runtime stats — the node's own actor-runtime metrics. Periodic.
|
||||
pub const RUNTIME_STATS: &str = "runtime.stats";
|
||||
/// Provider / lifecycle / cost — coarse lifecycle and cost facts about the
|
||||
/// node as a rented resource.
|
||||
pub const LIFECYCLE_COST: &str = "provider.lifecycle";
|
||||
|
||||
/// Which standard stream a span of process output came from (spec §6.2).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ProcStream {
|
||||
Stdout,
|
||||
Stderr,
|
||||
}
|
||||
|
||||
impl ProcStream {
|
||||
fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
ProcStream::Stdout => "stdout",
|
||||
ProcStream::Stderr => "stderr",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A raw-text process-output channel (spec §6.2). This is a *family*
|
||||
/// parameterized by process label and stream, so managing a new process
|
||||
/// introduces channels without defining new channel *types*:
|
||||
/// `proc.<label>.stdout` / `proc.<label>.stderr`.
|
||||
pub fn process_output(label: &str, stream: ProcStream) -> ChannelId {
|
||||
ChannelId::new(format!("proc.{label}.{}", stream.as_str()))
|
||||
}
|
||||
|
||||
/// How a view should treat a channel's bytes, decided by the catalog at
|
||||
/// read time. An id the catalog does not know is [`ChannelKind::Opaque`]
|
||||
/// and degrades to raw bytes (spec §6.3, §9.3).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ChannelKind {
|
||||
/// Decodes to a structured record under a JSON codec.
|
||||
Typed,
|
||||
/// Opaque text; a view treats it as lines.
|
||||
Text,
|
||||
/// Unknown to this consumer; retained and shown as raw bytes.
|
||||
Opaque,
|
||||
}
|
||||
|
||||
/// Classify a channel id. Known typed ids and the `proc.*` text family are
|
||||
/// recognized; everything else is [`ChannelKind::Opaque`].
|
||||
pub fn classify(channel: &ChannelId) -> ChannelKind {
|
||||
let id = channel.as_str();
|
||||
match id {
|
||||
IDENTITY | HOST_RESOURCE | TRANSPORT_INTERNALS | MEMBERSHIP | RUNTIME_STATS
|
||||
| LIFECYCLE_COST => ChannelKind::Typed,
|
||||
_ if id.starts_with("proc.") => ChannelKind::Text,
|
||||
_ => ChannelKind::Opaque,
|
||||
}
|
||||
}
|
||||
|
||||
// ── Typed records (realistic shapes for spec §6.1 channels) ────────────
|
||||
//
|
||||
// These define the bytes a producer actually emits and a metric view
|
||||
// actually decodes. They carry `#[serde(default)]` so a missing field
|
||||
// decodes to a default (version skew, spec §6.3). Unknown fields are
|
||||
// ignored by serde_json on decode for the same reason.
|
||||
|
||||
/// A typed channel record: a record knows its own channel and round-trips
|
||||
/// through the JSON codec. `decode(encode(r)) == r` is the codec contract
|
||||
/// (testing spec §3, §5).
|
||||
pub trait Record: Serialize + for<'de> Deserialize<'de> + Sized {
|
||||
/// The channel this record is carried on.
|
||||
const CHANNEL: &'static str;
|
||||
|
||||
/// The channel id this record is carried on.
|
||||
fn channel() -> ChannelId {
|
||||
ChannelId::new(Self::CHANNEL)
|
||||
}
|
||||
|
||||
/// Encode this record to its opaque payload bytes.
|
||||
fn encode(&self) -> Vec<u8> {
|
||||
// Records are plain data; JSON serialization of them cannot fail.
|
||||
serde_json::to_vec(self).expect("record serializes to JSON")
|
||||
}
|
||||
|
||||
/// Decode a payload back into the record. Fails (gracefully) if the
|
||||
/// bytes are not this record's shape — a view degrades to raw bytes
|
||||
/// (spec §9.3) rather than propagating the error.
|
||||
fn decode(payload: &[u8]) -> Result<Self, serde_json::Error> {
|
||||
serde_json::from_slice(payload)
|
||||
}
|
||||
}
|
||||
|
||||
/// The node's coarse role in the fleet.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Role {
|
||||
Worker,
|
||||
Coordinator,
|
||||
Relay,
|
||||
}
|
||||
|
||||
/// Identity / boot record (spec §6.1).
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct IdentityRecord {
|
||||
pub node: String,
|
||||
pub role: Role,
|
||||
pub region: String,
|
||||
/// The lifetime this stream belongs to (spec §8.4), echoed in-band so a
|
||||
/// view can confirm attribution.
|
||||
#[serde(default)]
|
||||
pub life: u64,
|
||||
}
|
||||
|
||||
/// Host / resource sample (spec §6.1). The metric projection (spec §9.2)
|
||||
/// decodes a series of these.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct ResourceSample {
|
||||
#[serde(default)]
|
||||
pub cpu_pct: f32,
|
||||
#[serde(default)]
|
||||
pub mem_used_mb: u32,
|
||||
#[serde(default)]
|
||||
pub mem_total_mb: u32,
|
||||
#[serde(default)]
|
||||
pub gpu_pct: f32,
|
||||
#[serde(default)]
|
||||
pub disk_used_gb: u32,
|
||||
#[serde(default)]
|
||||
pub net_rx_kbps: u32,
|
||||
#[serde(default)]
|
||||
pub net_tx_kbps: u32,
|
||||
}
|
||||
|
||||
/// Transport-internals record (spec §6.1).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct TransportInternals {
|
||||
#[serde(default)]
|
||||
pub relay_connected: bool,
|
||||
#[serde(default)]
|
||||
pub direct_peers: u32,
|
||||
#[serde(default)]
|
||||
pub relay_peers: u32,
|
||||
#[serde(default)]
|
||||
pub rtt_ms_p50: u32,
|
||||
}
|
||||
|
||||
/// Membership / liveness transition (spec §6.1).
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct MembershipTransition {
|
||||
pub peer: String,
|
||||
pub from: String,
|
||||
pub to: String,
|
||||
#[serde(default)]
|
||||
pub reason: String,
|
||||
}
|
||||
|
||||
/// Runtime-stats record (spec §6.1).
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct RuntimeStats {
|
||||
#[serde(default)]
|
||||
pub actors_live: u32,
|
||||
#[serde(default)]
|
||||
pub mailbox_depth: u32,
|
||||
#[serde(default)]
|
||||
pub scheduled_tasks: u32,
|
||||
}
|
||||
|
||||
/// Provider / lifecycle / cost record (spec §6.1).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct LifecycleCost {
|
||||
pub phase: String,
|
||||
#[serde(default)]
|
||||
pub cost_usd_per_hr: f32,
|
||||
#[serde(default)]
|
||||
pub uptime_s: u64,
|
||||
}
|
||||
|
||||
impl Record for IdentityRecord {
|
||||
const CHANNEL: &'static str = IDENTITY;
|
||||
}
|
||||
impl Record for ResourceSample {
|
||||
const CHANNEL: &'static str = HOST_RESOURCE;
|
||||
}
|
||||
impl Record for TransportInternals {
|
||||
const CHANNEL: &'static str = TRANSPORT_INTERNALS;
|
||||
}
|
||||
impl Record for MembershipTransition {
|
||||
const CHANNEL: &'static str = MEMBERSHIP;
|
||||
}
|
||||
impl Record for RuntimeStats {
|
||||
const CHANNEL: &'static str = RUNTIME_STATS;
|
||||
}
|
||||
impl Record for LifecycleCost {
|
||||
const CHANNEL: &'static str = LIFECYCLE_COST;
|
||||
}
|
||||
191
crates/distribution/src/datastream/frame.rs
Normal file
191
crates/distribution/src/datastream/frame.rs
Normal file
|
|
@ -0,0 +1,191 @@
|
|||
//! Core data model: the framed, channel-multiplexed stream (spec §4).
|
||||
//!
|
||||
//! These are the values that ride the seams a test observes (testing spec
|
||||
//! §2): a [`Frame`] crosses the mux→transport boundary, and a [`StreamId`]
|
||||
//! keys the reconstructed stream at ingest. Everything here is a plain
|
||||
//! value type with no behavior — the behavior lives in the mux, ingest,
|
||||
//! store, and views.
|
||||
|
||||
use std::fmt;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// A position assigned by a node's mux (spec §5.2).
|
||||
///
|
||||
/// Positions are **monotonic** and **gap-free** within a single node's
|
||||
/// stream: the mux never reuses one and never skips one in its numbering.
|
||||
/// A position that is assigned but never delivered surfaces downstream as
|
||||
/// a missing position — a detectable gap (spec §5.3, §7.5).
|
||||
///
|
||||
/// Across nodes, positions are **not** comparable (spec §4.3): they order
|
||||
/// frames within one node only.
|
||||
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
|
||||
pub struct Position(pub u64);
|
||||
|
||||
impl fmt::Display for Position {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{}", self.0)
|
||||
}
|
||||
}
|
||||
|
||||
/// The stable identity of a channel — the named lane a frame's bytes
|
||||
/// belong to (spec §4.2, §3).
|
||||
///
|
||||
/// It is an opaque token: the pipe (mux, transport, ingest, store) never
|
||||
/// interprets it. Only a *view* resolves it, through the catalog
|
||||
/// ([`crate::datastream::catalog`]), into a codec. A token with no
|
||||
/// registered codec is still carried and stored whole, then decoded later
|
||||
/// (spec §6.3) — which is why this type is open (any string) rather than a
|
||||
/// closed enum: a new channel is a new id, no pipe code changes.
|
||||
#[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
|
||||
pub struct ChannelId(Arc<str>);
|
||||
|
||||
impl ChannelId {
|
||||
/// Construct a channel id from any string-like value.
|
||||
pub fn new(id: impl AsRef<str>) -> Self {
|
||||
ChannelId(Arc::from(id.as_ref()))
|
||||
}
|
||||
|
||||
/// The id as a string slice.
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&str> for ChannelId {
|
||||
fn from(s: &str) -> Self {
|
||||
ChannelId::new(s)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<String> for ChannelId {
|
||||
fn from(s: String) -> Self {
|
||||
ChannelId::new(s)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ChannelId {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for ChannelId {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "ChannelId({:?})", &self.0)
|
||||
}
|
||||
}
|
||||
|
||||
/// The stable identity of a node that produces a stream (spec §4.4, §8.1).
|
||||
///
|
||||
/// Opaque to the pipe. In a deployment this is whatever durable id the
|
||||
/// system already assigns a machine (e.g. its public key); tests use
|
||||
/// readable names. A frame records the producing *node*, never a producer
|
||||
/// identity within it (spec §4.4).
|
||||
#[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
|
||||
pub struct NodeId(Arc<str>);
|
||||
|
||||
impl NodeId {
|
||||
/// Construct a node id from any string-like value.
|
||||
pub fn new(id: impl AsRef<str>) -> Self {
|
||||
NodeId(Arc::from(id.as_ref()))
|
||||
}
|
||||
|
||||
/// The id as a string slice.
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&str> for NodeId {
|
||||
fn from(s: &str) -> Self {
|
||||
NodeId::new(s)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for NodeId {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for NodeId {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "NodeId({:?})", &self.0)
|
||||
}
|
||||
}
|
||||
|
||||
/// A lifetime discriminator distinguishing a node's incarnations (spec
|
||||
/// §8.4). A node that dies and is re-rented starts a new lifetime, so its
|
||||
/// fresh stream does not collide with or append to its prior life.
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)]
|
||||
pub struct Lifetime(pub u64);
|
||||
|
||||
/// Identifies exactly one stored stream: a node plus the life it was
|
||||
/// produced in (spec §8.4).
|
||||
///
|
||||
/// This is the ingest key. Two streams with the same [`NodeId`] but
|
||||
/// different [`Lifetime`] are different streams and MUST NOT merge — that
|
||||
/// is what lets a re-incarnated node not append to its prior life.
|
||||
#[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)]
|
||||
pub struct StreamId {
|
||||
/// Which node produced the stream.
|
||||
pub node: NodeId,
|
||||
/// Which life of that node.
|
||||
pub life: Lifetime,
|
||||
}
|
||||
|
||||
impl StreamId {
|
||||
/// Construct a stream id from a node and a lifetime.
|
||||
pub fn new(node: impl Into<NodeId>, life: Lifetime) -> Self {
|
||||
StreamId { node: node.into(), life }
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for StreamId {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{}#{}", self.node, self.life.0)
|
||||
}
|
||||
}
|
||||
|
||||
/// The unit the mux emits (spec §4.1): bytes tagged with a channel and a
|
||||
/// position.
|
||||
///
|
||||
/// The `payload` is **opaque** to everything between the producer and a
|
||||
/// view — the mux, the transport, and storage treat it as bytes and never
|
||||
/// interpret it (spec §4.1). A typed event and a log line are the same
|
||||
/// kind of thing here: bytes on a channel.
|
||||
///
|
||||
/// Per the `// USER:` annotation on spec §4.1/§5.2 there is no per-frame
|
||||
/// wall-clock timestamp: frames are ordered and correlated by position
|
||||
/// alone.
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct Frame {
|
||||
/// The lane these bytes belong to.
|
||||
pub channel: ChannelId,
|
||||
/// The mux-assigned position within the node's stream.
|
||||
pub position: Position,
|
||||
/// The opaque payload bytes.
|
||||
pub payload: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Frame {
|
||||
/// Assemble a frame from its parts.
|
||||
pub fn new(channel: impl Into<ChannelId>, position: Position, payload: Vec<u8>) -> Self {
|
||||
Frame { channel: channel.into(), position, payload }
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for Frame {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
// Render the payload as text when it is valid UTF-8 (log lines,
|
||||
// JSON records both are) so debug output is readable; fall back to
|
||||
// a byte count for genuinely binary payloads.
|
||||
let mut dbg = f.debug_struct("Frame");
|
||||
dbg.field("channel", &self.channel).field("position", &self.position);
|
||||
match std::str::from_utf8(&self.payload) {
|
||||
Ok(text) => dbg.field("payload", &text),
|
||||
Err(_) => dbg.field("payload", &format_args!("<{} bytes>", self.payload.len())),
|
||||
};
|
||||
dbg.finish()
|
||||
}
|
||||
}
|
||||
58
crates/distribution/src/datastream/ingest.rs
Normal file
58
crates/distribution/src/datastream/ingest.rs
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
//! Consumer ingest: reconstruct each node's stream from deliveries (spec
|
||||
//! §8.1).
|
||||
//!
|
||||
//! The consumer receives frames from many nodes, in any order, some never
|
||||
//! arriving, and reconstructs each node's stream — keyed by stream id,
|
||||
//! ordered by position — into the [`Store`]. Ingest is deliberately thin:
|
||||
//! it routes a delivery to its stream and records the frame whole. It MUST
|
||||
//! NOT thin, aggregate, decode-and-discard, or truncate (spec §8.2), and it
|
||||
//! never inspects a channel or a payload, so a channel it cannot decode is
|
||||
//! retained exactly like any other (spec §8.3).
|
||||
//!
|
||||
//! Reconstruction is by position, not arrival: out-of-order deliveries land
|
||||
//! in order in the store, and a position delivered twice collapses to one
|
||||
//! (spec §7.5). Two lives of one node are different stream ids and never
|
||||
//! merge (spec §8.4).
|
||||
|
||||
use super::store::Store;
|
||||
use super::transport::Delivery;
|
||||
|
||||
/// The single consumer toward which all telemetry flows (spec §2). It owns
|
||||
/// the stored streams and grows them as deliveries arrive.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Consumer {
|
||||
store: Store,
|
||||
}
|
||||
|
||||
impl Consumer {
|
||||
/// A consumer with an empty store.
|
||||
pub fn new() -> Self {
|
||||
Consumer::default()
|
||||
}
|
||||
|
||||
/// Accept one delivery: route it to its stream and record the frame.
|
||||
/// Returns `true` if the frame was new (a duplicate position is
|
||||
/// ignored, keeping the first — the carrier cannot fabricate content,
|
||||
/// spec §9).
|
||||
pub fn accept(&mut self, delivery: Delivery) -> bool {
|
||||
let Delivery { stream, frame } = delivery;
|
||||
self.store.stream_mut(&stream).record(frame)
|
||||
}
|
||||
|
||||
/// Accept a batch of deliveries, in whatever order they arrive.
|
||||
pub fn ingest(&mut self, deliveries: impl IntoIterator<Item = Delivery>) {
|
||||
for delivery in deliveries {
|
||||
self.accept(delivery);
|
||||
}
|
||||
}
|
||||
|
||||
/// The stored streams — the source of truth for every view (spec §8.2).
|
||||
pub fn store(&self) -> &Store {
|
||||
&self.store
|
||||
}
|
||||
|
||||
/// Consume the consumer, yielding its store.
|
||||
pub fn into_store(self) -> Store {
|
||||
self.store
|
||||
}
|
||||
}
|
||||
49
crates/distribution/src/datastream/mod.rs
Normal file
49
crates/distribution/src/datastream/mod.rs
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
//! The per-node telemetry **datastream** (see `DATASTREAM_SPEC.md`).
|
||||
//!
|
||||
//! A deliberately dumb pipe: producers dump bytes tagged by channel, a
|
||||
//! single per-node mux interleaves them into one ordered stream, a
|
||||
//! best-effort transport carries that stream to the one consumer, ingest
|
||||
//! reconstructs each node's stream by position, and views are read-time
|
||||
//! projections over the stored stream. Nothing between a producer and a
|
||||
//! view interprets the payload.
|
||||
//!
|
||||
//! ```text
|
||||
//! producers (typed + text)
|
||||
//! │ bytes tagged by channel → [`catalog`]
|
||||
//! ▼
|
||||
//! per-node MUX → [`mux::Mux`]
|
||||
//! │ one ordered stream of [`Frame`]s
|
||||
//! ▼
|
||||
//! best-effort transport → [`transport`]
|
||||
//! │ delivery: frames, maybe dropped/reordered/delayed
|
||||
//! ▼
|
||||
//! consumer INGEST → [`ingest::Consumer`]
|
||||
//! │ complete stream, stored whole
|
||||
//! ▼
|
||||
//! stored STREAM (truth) → [`store`]
|
||||
//! │ read-time only
|
||||
//! ▼
|
||||
//! VIEWS → [`views`]
|
||||
//! ```
|
||||
//!
|
||||
//! The data model ([`frame`]) and the wire envelope ([`wire`]) are the
|
||||
//! seams a test observes; the catalog ([`catalog`]) is the schema contract
|
||||
//! between producers and views. See `DATASTREAM_TESTING_SPEC.md` for how
|
||||
//! the pipe is verified.
|
||||
|
||||
pub mod catalog;
|
||||
pub mod frame;
|
||||
pub mod ingest;
|
||||
pub mod mux;
|
||||
pub mod source;
|
||||
pub mod store;
|
||||
pub mod transport;
|
||||
pub mod views;
|
||||
pub mod wire;
|
||||
|
||||
pub use frame::{ChannelId, Frame, Lifetime, NodeId, Position, StreamId};
|
||||
pub use ingest::Consumer;
|
||||
pub use mux::Mux;
|
||||
pub use store::{GapSpan, Store, StoredStream};
|
||||
pub use transport::{Delivery, Reorder, ScriptedTransport, StreamScript};
|
||||
pub use views::{Body, LogEntry, MergedFrame};
|
||||
111
crates/distribution/src/datastream/mux.rs
Normal file
111
crates/distribution/src/datastream/mux.rs
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
//! The per-node mux: the single ordering authority (spec §5).
|
||||
//!
|
||||
//! Every producer on a node submits its bytes — tagged with a channel — to
|
||||
//! one mux, and the mux interleaves them into the node's single ordered
|
||||
//! stream. Because all channels pass through one assigner, a structured
|
||||
//! event and a log line emitted close together have a well-defined relative
|
||||
//! order (spec §5.1): that is what makes the one-timeline guarantee real.
|
||||
//!
|
||||
//! Two invariants do the heavy lifting:
|
||||
//!
|
||||
//! * **Gap-free, monotonic numbering** (spec §5.2). The mux hands out
|
||||
//! positions `0, 1, 2, …` with an atomic counter — never reused, never
|
||||
//! skipped. Numbering is independent of delivery: assigning a position
|
||||
//! does not mean the frame is, or ever will be, delivered.
|
||||
//! * **A drop is a missing position, never a renumber** (spec §5.3). The
|
||||
//! mux buffers within a bound to smooth bursts; on overflow it drops the
|
||||
//! frame. But the position was already consumed, so the drop surfaces
|
||||
//! downstream as a detectable gap (spec §7.5) rather than a silent
|
||||
//! renumbering.
|
||||
//!
|
||||
//! Submission is non-blocking in spirit: the only shared section is an
|
||||
//! O(1) counter bump and a push onto a bounded queue, so telemetry never
|
||||
//! stalls the node's real work (spec §5.3).
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
use super::frame::{ChannelId, Frame, Position, StreamId};
|
||||
|
||||
/// A node's single position authority and outgoing telemetry buffer.
|
||||
///
|
||||
/// One mux belongs to one stream — one life of one node (spec §8.4). It is
|
||||
/// `Send + Sync`: producers on different threads may submit concurrently
|
||||
/// and the mux serializes them into one position order (spec §5.3).
|
||||
pub struct Mux {
|
||||
stream: StreamId,
|
||||
next: AtomicU64,
|
||||
dropped: AtomicU64,
|
||||
capacity: usize,
|
||||
buffer: Mutex<VecDeque<Frame>>,
|
||||
}
|
||||
|
||||
impl Mux {
|
||||
/// Create a mux for `stream` with a bounded outgoing buffer. When more
|
||||
/// than `capacity` frames are waiting to be drained, further
|
||||
/// submissions are dropped (spec §5.3) — but still consume a position.
|
||||
pub fn new(stream: StreamId, capacity: usize) -> Self {
|
||||
Mux {
|
||||
stream,
|
||||
next: AtomicU64::new(0),
|
||||
dropped: AtomicU64::new(0),
|
||||
capacity,
|
||||
buffer: Mutex::new(VecDeque::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a mux whose buffer never overflows. Useful when a caller
|
||||
/// drains promptly and wants every assigned frame retained.
|
||||
pub fn unbounded(stream: StreamId) -> Self {
|
||||
Mux::new(stream, usize::MAX)
|
||||
}
|
||||
|
||||
/// The stream this mux produces (spec §8.4 ingest key).
|
||||
pub fn stream_id(&self) -> &StreamId {
|
||||
&self.stream
|
||||
}
|
||||
|
||||
/// Submit opaque bytes on a channel. Assigns and returns the next
|
||||
/// position. The frame is buffered for the transport to drain, or
|
||||
/// dropped on overflow — either way the returned position is consumed,
|
||||
/// so a drop becomes a missing position downstream (spec §5.3).
|
||||
///
|
||||
/// The mux never inspects `payload`; it is opaque (spec §4.1).
|
||||
pub fn submit(&self, channel: impl Into<ChannelId>, payload: Vec<u8>) -> Position {
|
||||
// Assign first, unconditionally: numbering is independent of
|
||||
// whether the frame survives the buffer (spec §5.2).
|
||||
let position = Position(self.next.fetch_add(1, Ordering::Relaxed));
|
||||
let frame = Frame { channel: channel.into(), position, payload };
|
||||
|
||||
let mut buffer = self.buffer.lock().expect("mux buffer poisoned");
|
||||
if buffer.len() < self.capacity {
|
||||
buffer.push_back(frame);
|
||||
} else {
|
||||
// Overflow: drop the frame that does not fit. Its position is
|
||||
// already spent, so it will read as a gap, not a renumber.
|
||||
self.dropped.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
position
|
||||
}
|
||||
|
||||
/// Pull all currently buffered frames, in the order they were buffered,
|
||||
/// emptying the buffer. The transport drains the mux's outgoing stream
|
||||
/// this way.
|
||||
pub fn drain(&self) -> Vec<Frame> {
|
||||
let mut buffer = self.buffer.lock().expect("mux buffer poisoned");
|
||||
buffer.drain(..).collect()
|
||||
}
|
||||
|
||||
/// How many positions have been assigned — the gap-free high-water mark
|
||||
/// (spec §5.2). Equal to the number of `submit` calls.
|
||||
pub fn assigned(&self) -> u64 {
|
||||
self.next.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// How many frames have been dropped on overflow (spec §5.3). Each
|
||||
/// dropped frame is one missing position downstream.
|
||||
pub fn dropped(&self) -> u64 {
|
||||
self.dropped.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
216
crates/distribution/src/datastream/source.rs
Normal file
216
crates/distribution/src/datastream/source.rs
Normal file
|
|
@ -0,0 +1,216 @@
|
|||
//! Producer-side helpers: map a node's live state into the typed records of
|
||||
//! the [`catalog`](super::catalog).
|
||||
//!
|
||||
//! These turn data the node already has — host counters, a runtime-stats
|
||||
//! snapshot, the membership view — into the records a producer submits to its
|
||||
//! mux. Nothing here touches the pipe; it is pure "live state in, record out"
|
||||
//! so it can be unit-tested without a transport, and it deliberately carries
|
||||
//! no dependency on the cluster transport (iroh) so it builds with any feature
|
||||
//! set.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::time::Instant;
|
||||
|
||||
use super::catalog::{IdentityRecord, MembershipTransition, ResourceSample, Role};
|
||||
|
||||
/// Build the identity record a node emits first on its stream (spec §6.1).
|
||||
pub fn identity_record(node: &str, role: Role, region: &str, life: u64) -> IdentityRecord {
|
||||
IdentityRecord {
|
||||
node: node.to_string(),
|
||||
role,
|
||||
region: region.to_string(),
|
||||
life,
|
||||
}
|
||||
}
|
||||
|
||||
/// Samples host CPU usage across calls. CPU percent is a rate, so it needs two
|
||||
/// observations to compute; the first call seeds the baseline and reports 0.
|
||||
#[derive(Default)]
|
||||
pub struct CpuSampler {
|
||||
prev: Option<(u64, Instant)>,
|
||||
}
|
||||
|
||||
impl CpuSampler {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// CPU busy percent since the previous call, in `[0, 100*ncpu]` clamped to
|
||||
/// `[0, 100]`. Returns 0 on the first call (no baseline yet) and on any
|
||||
/// platform where `/proc/stat` is unavailable.
|
||||
pub fn sample(&mut self) -> f32 {
|
||||
let now = Instant::now();
|
||||
let busy = match read_proc_stat_busy_jiffies() {
|
||||
Some(j) => j,
|
||||
None => return 0.0,
|
||||
};
|
||||
let pct = match self.prev {
|
||||
Some((prev_busy, prev_at)) => {
|
||||
let elapsed = now.duration_since(prev_at).as_secs_f64();
|
||||
if elapsed <= 0.0 {
|
||||
0.0
|
||||
} else {
|
||||
// Jiffies are USER_HZ (typically 100/s) per CPU. Normalize by
|
||||
// wall time and the tick rate to get a busy fraction.
|
||||
let delta = busy.saturating_sub(prev_busy) as f64;
|
||||
let hz = clock_ticks_per_sec();
|
||||
let frac = delta / (hz * elapsed);
|
||||
(frac * 100.0).clamp(0.0, 100.0) as f32
|
||||
}
|
||||
}
|
||||
None => 0.0,
|
||||
};
|
||||
self.prev = Some((busy, now));
|
||||
pct
|
||||
}
|
||||
}
|
||||
|
||||
/// Read a host-resource sample: CPU busy percent (via `sampler`) and memory
|
||||
/// from `/proc/meminfo`. Fields we have no source for stay at their default
|
||||
/// (honest zeros), and on a non-Linux host the whole sample is the default.
|
||||
pub fn read_host_resource(sampler: &mut CpuSampler) -> ResourceSample {
|
||||
let cpu_pct = sampler.sample();
|
||||
let (mem_total_mb, mem_used_mb) = read_meminfo_mb().unwrap_or((0, 0));
|
||||
// Fields we have no host source for stay at honest zeros.
|
||||
ResourceSample {
|
||||
cpu_pct,
|
||||
mem_used_mb,
|
||||
mem_total_mb,
|
||||
gpu_pct: 0.0,
|
||||
disk_used_gb: 0,
|
||||
net_rx_kbps: 0,
|
||||
net_tx_kbps: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Tracks the last-seen membership state per peer and emits a transition only
|
||||
/// when a peer's state changes — membership is an event-driven channel
|
||||
/// (spec §6.1), so a steady cluster produces no frames.
|
||||
#[derive(Default)]
|
||||
pub struct MembershipTracker {
|
||||
last: HashMap<String, String>,
|
||||
}
|
||||
|
||||
impl MembershipTracker {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Diff the current `(peer, state)` view against the last one. A newly seen
|
||||
/// peer transitions from `"unknown"`; a peer whose state is unchanged
|
||||
/// produces nothing.
|
||||
pub fn diff(&mut self, members: &[(String, String)]) -> Vec<MembershipTransition> {
|
||||
let mut out = Vec::new();
|
||||
for (peer, state) in members {
|
||||
let changed = match self.last.get(peer) {
|
||||
Some(prev) => prev != state,
|
||||
None => true,
|
||||
};
|
||||
if changed {
|
||||
let from = self
|
||||
.last
|
||||
.get(peer)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| "unknown".to_string());
|
||||
out.push(MembershipTransition {
|
||||
peer: peer.clone(),
|
||||
from,
|
||||
to: state.clone(),
|
||||
reason: String::new(),
|
||||
});
|
||||
self.last.insert(peer.clone(), state.clone());
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
// ── Host counters (Linux /proc) ────────────────────────────────────────────
|
||||
|
||||
fn clock_ticks_per_sec() -> f64 {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
// SAFETY: sysconf is a pure lookup with no preconditions.
|
||||
let hz = unsafe { libc::sysconf(libc::_SC_CLK_TCK) };
|
||||
if hz > 0 {
|
||||
return hz as f64;
|
||||
}
|
||||
}
|
||||
100.0
|
||||
}
|
||||
|
||||
/// Total non-idle jiffies from the aggregate `cpu` line of `/proc/stat`.
|
||||
fn read_proc_stat_busy_jiffies() -> Option<u64> {
|
||||
let text = std::fs::read_to_string("/proc/stat").ok()?;
|
||||
let line = text.lines().next()?; // the aggregate "cpu ..." line
|
||||
let mut it = line.split_whitespace();
|
||||
if it.next()? != "cpu" {
|
||||
return None;
|
||||
}
|
||||
// user nice system idle iowait irq softirq steal guest guest_nice
|
||||
let vals: Vec<u64> = it.filter_map(|t| t.parse::<u64>().ok()).collect();
|
||||
if vals.len() < 4 {
|
||||
return None;
|
||||
}
|
||||
let total: u64 = vals.iter().sum();
|
||||
let idle = vals[3] + vals.get(4).copied().unwrap_or(0); // idle + iowait
|
||||
Some(total.saturating_sub(idle))
|
||||
}
|
||||
|
||||
/// `(MemTotal, MemTotal - MemAvailable)` in MiB from `/proc/meminfo`.
|
||||
fn read_meminfo_mb() -> Option<(u32, u32)> {
|
||||
let text = std::fs::read_to_string("/proc/meminfo").ok()?;
|
||||
let mut total_kb = None;
|
||||
let mut avail_kb = None;
|
||||
for line in text.lines() {
|
||||
if let Some(rest) = line.strip_prefix("MemTotal:") {
|
||||
total_kb = rest.split_whitespace().next().and_then(|v| v.parse::<u64>().ok());
|
||||
} else if let Some(rest) = line.strip_prefix("MemAvailable:") {
|
||||
avail_kb = rest.split_whitespace().next().and_then(|v| v.parse::<u64>().ok());
|
||||
}
|
||||
}
|
||||
let total = total_kb?;
|
||||
let avail = avail_kb.unwrap_or(total);
|
||||
let used = total.saturating_sub(avail);
|
||||
Some(((total / 1024) as u32, (used / 1024) as u32))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
#[cfg_attr(not(target_os = "linux"), ignore)]
|
||||
fn host_resource_reports_real_memory_on_linux() {
|
||||
// On a real Linux host the machine has some memory; a sample that
|
||||
// reported zero total would mean we never read the host at all.
|
||||
let mut sampler = CpuSampler::new();
|
||||
let sample = read_host_resource(&mut sampler);
|
||||
assert!(sample.mem_total_mb > 0, "expected to read MemTotal from the host");
|
||||
assert!(
|
||||
sample.mem_used_mb <= sample.mem_total_mb,
|
||||
"used memory cannot exceed total"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn membership_emits_only_when_a_peer_changes_state() {
|
||||
let mut tracker = MembershipTracker::new();
|
||||
|
||||
// First sighting of a peer is a transition from the unknown state.
|
||||
let first = tracker.diff(&[("peer-a".into(), "alive".into())]);
|
||||
assert_eq!(first.len(), 1);
|
||||
assert_eq!(first[0].from, "unknown");
|
||||
assert_eq!(first[0].to, "alive");
|
||||
|
||||
// Re-reporting the same state is silence — a steady cluster is quiet.
|
||||
let steady = tracker.diff(&[("peer-a".into(), "alive".into())]);
|
||||
assert!(steady.is_empty(), "no transition when nothing changed");
|
||||
|
||||
// A genuine state change surfaces, carrying the prior state.
|
||||
let changed = tracker.diff(&[("peer-a".into(), "suspect".into())]);
|
||||
assert_eq!(changed.len(), 1);
|
||||
assert_eq!(changed[0].from, "alive");
|
||||
assert_eq!(changed[0].to, "suspect");
|
||||
}
|
||||
}
|
||||
160
crates/distribution/src/datastream/store.rs
Normal file
160
crates/distribution/src/datastream/store.rs
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
//! The stored stream — the consumer's source of truth (spec §8).
|
||||
//!
|
||||
//! Storage holds each node's **complete** stream, whole and append-only.
|
||||
//! Nothing is thinned, aggregated, decoded-and-discarded, or truncated at
|
||||
//! ingest (spec §8.2); everything a view ever shows is derived from here
|
||||
//! (spec §9.1). Frames on channels the consumer cannot decode are kept as
|
||||
//! opaque bytes, in order, alongside the rest (spec §8.3) — the store never
|
||||
//! looks at a channel or a payload.
|
||||
//!
|
||||
//! A [`StoredStream`] is keyed in the [`Store`] by [`StreamId`] — node plus
|
||||
//! lifetime — so a re-incarnated node does not append to its prior life
|
||||
//! (spec §8.4).
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use super::frame::{Frame, Position, StreamId};
|
||||
|
||||
/// One node's reconstructed stream: its frames in position order, whole.
|
||||
///
|
||||
/// Backed by a position-keyed map so out-of-order arrivals land in order
|
||||
/// and a position seen twice collapses to one (the carrier may not
|
||||
/// fabricate content, spec §9). Gaps are not stored — they are *derived*
|
||||
/// at read time from the positions that are present (spec §9.1).
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct StoredStream {
|
||||
frames: BTreeMap<u64, Frame>,
|
||||
}
|
||||
|
||||
impl StoredStream {
|
||||
/// An empty stream.
|
||||
pub fn new() -> Self {
|
||||
StoredStream::default()
|
||||
}
|
||||
|
||||
/// Record a delivered frame. Idempotent by position: the first frame
|
||||
/// seen for a position wins and is never mutated (append-only,
|
||||
/// spec §8.2). Returns `true` if this was the first time the position
|
||||
/// was seen.
|
||||
pub fn record(&mut self, frame: Frame) -> bool {
|
||||
match self.frames.entry(frame.position.0) {
|
||||
std::collections::btree_map::Entry::Vacant(slot) => {
|
||||
slot.insert(frame);
|
||||
true
|
||||
}
|
||||
std::collections::btree_map::Entry::Occupied(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// The stored frames, in position order.
|
||||
pub fn frames(&self) -> impl Iterator<Item = &Frame> {
|
||||
self.frames.values()
|
||||
}
|
||||
|
||||
/// The stored frames cloned into a vector, in position order. Handy for
|
||||
/// asserting against the reference model.
|
||||
pub fn to_vec(&self) -> Vec<Frame> {
|
||||
self.frames.values().cloned().collect()
|
||||
}
|
||||
|
||||
/// How many frames are stored.
|
||||
pub fn len(&self) -> usize {
|
||||
self.frames.len()
|
||||
}
|
||||
|
||||
/// Whether the stream has no frames yet.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.frames.is_empty()
|
||||
}
|
||||
|
||||
/// The frame at an exact position, if stored.
|
||||
pub fn at(&self, position: Position) -> Option<&Frame> {
|
||||
self.frames.get(&position.0)
|
||||
}
|
||||
|
||||
/// The **interior** gaps — runs of positions assigned between the first
|
||||
/// and last delivered frame but never delivered (spec §7.5, §8), each as
|
||||
/// one [`GapSpan`].
|
||||
///
|
||||
/// Cost is O(stored frames), never O(gap size): it walks adjacent stored
|
||||
/// positions and reads each span's endpoints from them, rather than
|
||||
/// enumerating the (possibly enormous) range in between. A stream that
|
||||
/// brackets a huge interior gap — what a long consumer outage produces
|
||||
/// (spec §7.4), or a single wild position from a corrupt datagram — still
|
||||
/// surfaces in work proportional to the frames held, not to `u64::MAX`.
|
||||
///
|
||||
/// Only interior gaps are knowable: a position lost *after* the last
|
||||
/// delivered frame leaves no bracketing frame to reveal it, so it shows
|
||||
/// up as the stream simply ending (spec §7.4 node death), not a gap.
|
||||
pub fn gap_spans(&self) -> Vec<GapSpan> {
|
||||
let mut spans = Vec::new();
|
||||
let mut prev: Option<u64> = None;
|
||||
for &p in self.frames.keys() {
|
||||
if let Some(q) = prev
|
||||
&& p > q + 1
|
||||
{
|
||||
spans.push(GapSpan { start: q + 1, end: p - 1 });
|
||||
}
|
||||
prev = Some(p);
|
||||
}
|
||||
spans
|
||||
}
|
||||
}
|
||||
|
||||
/// A contiguous run of missing positions surfaced in a stored stream
|
||||
/// (spec §7.5). Inclusive on both ends.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct GapSpan {
|
||||
/// First missing position.
|
||||
pub start: u64,
|
||||
/// Last missing position.
|
||||
pub end: u64,
|
||||
}
|
||||
|
||||
impl GapSpan {
|
||||
/// How many positions the gap spans (always at least one).
|
||||
pub fn count(&self) -> u64 {
|
||||
self.end - self.start + 1
|
||||
}
|
||||
}
|
||||
|
||||
/// All stored streams at the consumer, keyed by [`StreamId`] (spec §8.4).
|
||||
///
|
||||
/// Two streams with the same node but different lifetime are distinct keys
|
||||
/// and never merge.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct Store {
|
||||
streams: BTreeMap<StreamId, StoredStream>,
|
||||
}
|
||||
|
||||
impl Store {
|
||||
/// An empty store.
|
||||
pub fn new() -> Self {
|
||||
Store::default()
|
||||
}
|
||||
|
||||
/// The stored stream for a node's life, if any frames have landed.
|
||||
pub fn stream(&self, id: &StreamId) -> Option<&StoredStream> {
|
||||
self.streams.get(id)
|
||||
}
|
||||
|
||||
/// The stored stream for a node's life, creating an empty one if needed.
|
||||
pub fn stream_mut(&mut self, id: &StreamId) -> &mut StoredStream {
|
||||
self.streams.entry(id.clone()).or_default()
|
||||
}
|
||||
|
||||
/// Every stream id the store holds, in a stable order.
|
||||
pub fn stream_ids(&self) -> impl Iterator<Item = &StreamId> {
|
||||
self.streams.keys()
|
||||
}
|
||||
|
||||
/// How many distinct streams the store holds.
|
||||
pub fn len(&self) -> usize {
|
||||
self.streams.len()
|
||||
}
|
||||
|
||||
/// Whether the store holds no streams.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.streams.is_empty()
|
||||
}
|
||||
}
|
||||
159
crates/distribution/src/datastream/transport.rs
Normal file
159
crates/distribution/src/datastream/transport.rs
Normal file
|
|
@ -0,0 +1,159 @@
|
|||
//! Transport: best-effort carriage of a node's stream to the one consumer
|
||||
//! (spec §7), and a scripted in-process carrier for offline tests (testing
|
||||
//! spec §2, §9).
|
||||
//!
|
||||
//! A [`Delivery`] is the value on the transport→ingest seam: which stream a
|
||||
//! frame belongs to, and the frame. A real carrier rides the connections
|
||||
//! the system already maintains (spec §7.1); a test replaces it with the
|
||||
//! [`ScriptedTransport`] here, whose faults are chosen by the scenario and
|
||||
//! stay inside the **envelope** (testing spec §9): a carrier may *deliver*,
|
||||
//! *drop*, *reorder*, or *delay*, and it MUST NOT corrupt a payload,
|
||||
//! fabricate a frame, or alter a position.
|
||||
//!
|
||||
//! Under position-ordering a *delay* is indistinguishable from a *reorder*
|
||||
//! (a delayed frame simply arrives later), so the envelope's delay is
|
||||
//! covered by [`Reorder`]. Everything the scripted carrier produces is a
|
||||
//! reordered subsequence of what was sent — never a superset, never a
|
||||
//! mutation — which is exactly the property the real-transport conformance
|
||||
//! check pins (testing spec §9).
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use super::frame::{Frame, StreamId};
|
||||
|
||||
/// A frame as the consumer receives it from the transport (testing spec §2
|
||||
/// seam): tagged with the stream it belongs to.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Delivery {
|
||||
/// Which node's life produced the frame (spec §8.1 ingest key).
|
||||
pub stream: StreamId,
|
||||
/// The carried frame, its payload and position untouched.
|
||||
pub frame: Frame,
|
||||
}
|
||||
|
||||
impl Delivery {
|
||||
/// Pair a stream id with a frame.
|
||||
pub fn new(stream: StreamId, frame: Frame) -> Self {
|
||||
Delivery { stream, frame }
|
||||
}
|
||||
}
|
||||
|
||||
/// How the surviving frames of a stream are reordered on arrival. This is
|
||||
/// the envelope's *reorder* (and *delay*) axis (testing spec §9); each
|
||||
/// variant is a permutation of the survivors, never adding or dropping.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub enum Reorder {
|
||||
/// Delivered in the order sent.
|
||||
#[default]
|
||||
InOrder,
|
||||
/// Delivered fully reversed — the deepest reorder a stream admits.
|
||||
Reversed,
|
||||
/// Each consecutive run of `width` frames is reversed, bounding how far
|
||||
/// out of order any frame can arrive (reorder depth ≤ `width`).
|
||||
Windows(usize),
|
||||
/// An explicit permutation: `delivered[i] = survivors[indices[i]]`.
|
||||
/// `indices` should be a permutation of `0..survivors.len()`; entries
|
||||
/// out of range are skipped so a mis-authored vector cannot panic.
|
||||
Permutation(Vec<usize>),
|
||||
}
|
||||
|
||||
/// The faults a scripted carrier applies to one stream (testing spec §9
|
||||
/// envelope). Drops and reorders only — payloads and positions are never
|
||||
/// touched.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct StreamScript {
|
||||
/// Positions the carrier never delivers. This one set expresses a
|
||||
/// single drop, a total-loss span, and a consumer outage alike — every
|
||||
/// position produced during the loss is simply listed here.
|
||||
pub dropped: BTreeSet<u64>,
|
||||
/// How the surviving frames are reordered on arrival.
|
||||
pub reorder: Reorder,
|
||||
}
|
||||
|
||||
impl StreamScript {
|
||||
/// A clean carrier: deliver everything, in order.
|
||||
pub fn perfect() -> Self {
|
||||
StreamScript::default()
|
||||
}
|
||||
|
||||
/// Drop exactly these positions, otherwise deliver in order.
|
||||
pub fn dropping(positions: impl IntoIterator<Item = u64>) -> Self {
|
||||
StreamScript { dropped: positions.into_iter().collect(), reorder: Reorder::InOrder }
|
||||
}
|
||||
|
||||
/// Set the reorder behavior (builder style).
|
||||
pub fn with_reorder(mut self, reorder: Reorder) -> Self {
|
||||
self.reorder = reorder;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// A scripted, in-process transport (testing spec §2). It is a pure,
|
||||
/// deterministic transform from what a node *sent* to what the consumer is
|
||||
/// *delivered* — the entanglement of real wires replaced by a script so a
|
||||
/// run completes in microseconds and returns the same result every time.
|
||||
pub struct ScriptedTransport;
|
||||
|
||||
impl ScriptedTransport {
|
||||
/// Carry one node's sent frames to the consumer under `script`,
|
||||
/// returning the deliveries in arrival order. Dropped positions are
|
||||
/// removed; the survivors are reordered; nothing else changes.
|
||||
pub fn carry(stream: &StreamId, sent: &[Frame], script: &StreamScript) -> Vec<Delivery> {
|
||||
let survivors: Vec<Frame> =
|
||||
sent.iter().filter(|f| !script.dropped.contains(&f.position.0)).cloned().collect();
|
||||
let ordered = reorder(survivors, &script.reorder);
|
||||
ordered.into_iter().map(|frame| Delivery::new(stream.clone(), frame)).collect()
|
||||
}
|
||||
|
||||
/// Carry several nodes' streams and interleave their deliveries in a
|
||||
/// fixed round-robin, the way one wire would multiplex many senders.
|
||||
/// Cross-node order is meaningless (spec §4.3); this just proves ingest
|
||||
/// routes by stream id, not by arrival.
|
||||
pub fn carry_all(streams: &[(StreamId, Vec<Frame>, StreamScript)]) -> Vec<Delivery> {
|
||||
let per_stream: Vec<Vec<Delivery>> =
|
||||
streams.iter().map(|(id, sent, script)| Self::carry(id, sent, script)).collect();
|
||||
round_robin(per_stream)
|
||||
}
|
||||
}
|
||||
|
||||
fn reorder(mut survivors: Vec<Frame>, reorder: &Reorder) -> Vec<Frame> {
|
||||
match reorder {
|
||||
Reorder::InOrder => survivors,
|
||||
Reorder::Reversed => {
|
||||
survivors.reverse();
|
||||
survivors
|
||||
}
|
||||
Reorder::Windows(width) => {
|
||||
let width = (*width).max(1);
|
||||
let mut out = Vec::with_capacity(survivors.len());
|
||||
for chunk in survivors.chunks(width) {
|
||||
out.extend(chunk.iter().rev().cloned());
|
||||
}
|
||||
out
|
||||
}
|
||||
Reorder::Permutation(indices) => indices
|
||||
.iter()
|
||||
.filter_map(|&i| survivors.get(i).cloned())
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn round_robin(mut lists: Vec<Vec<Delivery>>) -> Vec<Delivery> {
|
||||
// Reverse each so we can pop from the back cheaply while preserving the
|
||||
// per-stream arrival order.
|
||||
for list in &mut lists {
|
||||
list.reverse();
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
let mut any = true;
|
||||
while any {
|
||||
any = false;
|
||||
for list in &mut lists {
|
||||
if let Some(d) = list.pop() {
|
||||
out.push(d);
|
||||
any = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
177
crates/distribution/src/datastream/views.rs
Normal file
177
crates/distribution/src/datastream/views.rs
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
//! Views: read-time projections over a stored stream (spec §9).
|
||||
//!
|
||||
//! Everything a consumer shows is computed here, at read time, from the
|
||||
//! stored stream alone (spec §9.1) — nothing is pre-projected at ingest.
|
||||
//! That is what lets a merged log, a metric series, a tail/grep, and a
|
||||
//! replay all be views of the same complete record. Adding or changing a
|
||||
//! view changes nothing in producers, channels, or storage (spec §9.3):
|
||||
//! every function here takes only a [`StoredStream`].
|
||||
//!
|
||||
//! A view decodes a typed channel via its catalog codec; over a channel it
|
||||
//! cannot decode — an unknown id, or typed bytes that do not parse — it
|
||||
//! **degrades to raw bytes** rather than failing (spec §9.3).
|
||||
//!
|
||||
//! The four named projections of spec §9.2 are:
|
||||
//!
|
||||
//! * [`merged_log`] / [`replay`] — the single timeline across all channels,
|
||||
//! in position order, with surfaced gaps;
|
||||
//! * [`metric_series`] — decode one typed channel into a time series;
|
||||
//! * [`tail`], [`grep`], [`filter`] — windowed / predicate-restricted views.
|
||||
|
||||
use std::fmt;
|
||||
|
||||
use super::catalog::{self, ChannelKind, Record};
|
||||
use super::frame::{ChannelId, Frame, Position};
|
||||
use super::store::{GapSpan, StoredStream};
|
||||
|
||||
/// One entry on the merged timeline: either a frame or a surfaced gap.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum LogEntry {
|
||||
/// A stored frame, its payload decoded for display.
|
||||
Frame(MergedFrame),
|
||||
/// A run of positions that were assigned but never delivered (spec
|
||||
/// §7.5), surfaced rather than silently concatenated across.
|
||||
Gap(GapSpan),
|
||||
}
|
||||
|
||||
/// A frame as the merged log presents it: where it sits on the timeline,
|
||||
/// which channel it came from, and its payload decoded as far as the
|
||||
/// catalog allows.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct MergedFrame {
|
||||
/// The frame's position on the node's single timeline.
|
||||
pub position: Position,
|
||||
/// The channel the bytes belong to.
|
||||
pub channel: ChannelId,
|
||||
/// The payload, decoded per the channel's codec (or degraded to bytes).
|
||||
pub body: Body,
|
||||
}
|
||||
|
||||
/// A payload decoded as far as the catalog allows (spec §9.3).
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum Body {
|
||||
/// A typed channel decoded to a structured value.
|
||||
Record(serde_json::Value),
|
||||
/// A raw-text channel as its line(s).
|
||||
Text(String),
|
||||
/// A channel the consumer cannot decode — an unknown id, or typed bytes
|
||||
/// that failed to parse — kept as raw bytes (graceful degradation).
|
||||
Raw(Vec<u8>),
|
||||
}
|
||||
|
||||
/// Decode a payload for display, degrading gracefully (spec §9.3): a typed
|
||||
/// channel whose bytes do not parse, and any unknown channel, fall back to
|
||||
/// raw bytes instead of failing.
|
||||
pub fn decode_body(channel: &ChannelId, payload: &[u8]) -> Body {
|
||||
match catalog::classify(channel) {
|
||||
ChannelKind::Typed => match serde_json::from_slice::<serde_json::Value>(payload) {
|
||||
Ok(value) => Body::Record(value),
|
||||
Err(_) => Body::Raw(payload.to_vec()),
|
||||
},
|
||||
ChannelKind::Text => match std::str::from_utf8(payload) {
|
||||
Ok(text) => Body::Text(text.to_string()),
|
||||
Err(_) => Body::Raw(payload.to_vec()),
|
||||
},
|
||||
ChannelKind::Opaque => Body::Raw(payload.to_vec()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the merged timeline: every stored frame in position order, with a
|
||||
/// [`LogEntry::Gap`] inserted wherever an interior position is missing.
|
||||
fn timeline(stream: &StoredStream) -> Vec<LogEntry> {
|
||||
let mut out = Vec::with_capacity(stream.len());
|
||||
let mut prev: Option<u64> = None;
|
||||
for frame in stream.frames() {
|
||||
let pos = frame.position.0;
|
||||
if let Some(p) = prev
|
||||
&& pos > p + 1
|
||||
{
|
||||
out.push(LogEntry::Gap(GapSpan { start: p + 1, end: pos - 1 }));
|
||||
}
|
||||
out.push(LogEntry::Frame(MergedFrame {
|
||||
position: frame.position,
|
||||
channel: frame.channel.clone(),
|
||||
body: decode_body(&frame.channel, &frame.payload),
|
||||
}));
|
||||
prev = Some(pos);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// **Full merged log view** (spec §9.2): the single timeline across all
|
||||
/// channels, in position order, typed records and text lines interleaved,
|
||||
/// with gaps surfaced.
|
||||
pub fn merged_log(stream: &StoredStream) -> Vec<LogEntry> {
|
||||
timeline(stream)
|
||||
}
|
||||
|
||||
/// **Replay** (spec §9.2): reconstruct the timeline after the fact, as if
|
||||
/// observed live. It is the merged log presented in position order as a
|
||||
/// stream — the same complete record, walked front to back the way a live
|
||||
/// observer would have seen it arrive.
|
||||
pub fn replay(stream: &StoredStream) -> impl Iterator<Item = LogEntry> {
|
||||
timeline(stream).into_iter()
|
||||
}
|
||||
|
||||
/// **Typed / metric projection** (spec §9.2): decode one typed channel into
|
||||
/// a time series of `(position, record)`, in position order. Frames whose
|
||||
/// bytes do not parse as `R` are skipped — the projection degrades rather
|
||||
/// than failing (spec §9.3); they remain visible as raw bytes in the merged
|
||||
/// log.
|
||||
pub fn metric_series<R: Record>(stream: &StoredStream) -> Vec<(Position, R)> {
|
||||
let channel = R::channel();
|
||||
stream
|
||||
.frames()
|
||||
.filter(|f| f.channel == channel)
|
||||
.filter_map(|f| R::decode(&f.payload).ok().map(|record| (f.position, record)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// **Tail** (spec §9.2): the last `n` frames in position order (fewer if
|
||||
/// the stream is shorter).
|
||||
pub fn tail(stream: &StoredStream, n: usize) -> Vec<Frame> {
|
||||
let all = stream.to_vec();
|
||||
let start = all.len().saturating_sub(n);
|
||||
all[start..].to_vec()
|
||||
}
|
||||
|
||||
/// **Filter** (spec §9.2): the frames matching a predicate, in position
|
||||
/// order.
|
||||
pub fn filter<F>(stream: &StoredStream, predicate: F) -> Vec<Frame>
|
||||
where
|
||||
F: Fn(&Frame) -> bool,
|
||||
{
|
||||
stream.frames().filter(|f| predicate(f)).cloned().collect()
|
||||
}
|
||||
|
||||
/// **Grep** (spec §9.2): the frames whose payload, read as text, contains
|
||||
/// `needle`. Works uniformly across typed channels (their JSON bytes) and
|
||||
/// text channels (their lines); binary payloads simply do not match.
|
||||
pub fn grep(stream: &StoredStream, needle: &str) -> Vec<Frame> {
|
||||
filter(stream, |f| String::from_utf8_lossy(&f.payload).contains(needle))
|
||||
}
|
||||
|
||||
// ── Human-readable rendering ───────────────────────────────────────────
|
||||
|
||||
impl fmt::Display for Body {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Body::Record(value) => write!(f, "{value}"),
|
||||
Body::Text(text) => f.write_str(text),
|
||||
Body::Raw(bytes) => write!(f, "<{} opaque bytes>", bytes.len()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for LogEntry {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
LogEntry::Frame(frame) => {
|
||||
write!(f, "#{:<4} [{}] {}", frame.position, frame.channel, frame.body)
|
||||
}
|
||||
LogEntry::Gap(span) => {
|
||||
write!(f, "#{:<4} ── gap: {} position(s) missing ──", span.start, span.count())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
121
crates/distribution/src/datastream/wire.rs
Normal file
121
crates/distribution/src/datastream/wire.rs
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
//! The transport envelope: how a delivered frame is encoded as bytes for
|
||||
//! a real carrier (spec §7, testing spec §9).
|
||||
//!
|
||||
//! This is the **only** place the transport touches a frame as bytes. The
|
||||
//! envelope is self-describing and length-prefixed so it can be framed on
|
||||
//! any byte transport (a datagram, a length-delimited stream). It MUST
|
||||
//! round-trip exactly — the conformance check (testing spec §9) leans on
|
||||
//! "payloads byte-identical, positions intact" — and it MUST fail
|
||||
//! gracefully on a truncated or malformed buffer rather than panic, since
|
||||
//! a best-effort carrier can hand us anything.
|
||||
//!
|
||||
//! Layout (all integers little-endian):
|
||||
//!
|
||||
//! ```text
|
||||
//! node_len: u32 | node: utf8[node_len]
|
||||
//! life: u64
|
||||
//! position: u64
|
||||
//! chan_len: u32 | channel: utf8[chan_len]
|
||||
//! pay_len: u32 | payload: bytes[pay_len]
|
||||
//! ```
|
||||
|
||||
use super::frame::{ChannelId, Frame, Lifetime, NodeId, Position, StreamId};
|
||||
|
||||
/// Why a buffer could not be decoded as an envelope. A carrier that
|
||||
/// receives one of these drops the datagram (best-effort) rather than
|
||||
/// crashing the consumer.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum WireError {
|
||||
/// The buffer ended before a declared field was complete.
|
||||
Truncated,
|
||||
/// A length prefix claimed more bytes than the buffer holds.
|
||||
BadLength,
|
||||
/// A string field was not valid UTF-8.
|
||||
NotUtf8,
|
||||
/// Bytes remained after a complete envelope. One datagram carries
|
||||
/// exactly one frame, so a trailing tail is a malformed buffer.
|
||||
TrailingBytes,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for WireError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
WireError::Truncated => f.write_str("envelope truncated"),
|
||||
WireError::BadLength => f.write_str("envelope length prefix exceeds buffer"),
|
||||
WireError::NotUtf8 => f.write_str("envelope string field is not valid UTF-8"),
|
||||
WireError::TrailingBytes => f.write_str("bytes remain after a complete envelope"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for WireError {}
|
||||
|
||||
/// Encode a delivery — a `(StreamId, Frame)` — into a self-describing byte
|
||||
/// buffer.
|
||||
pub fn encode_delivery(stream: &StreamId, frame: &Frame) -> Vec<u8> {
|
||||
let node = stream.node.as_str().as_bytes();
|
||||
let channel = frame.channel.as_str().as_bytes();
|
||||
let mut out = Vec::with_capacity(4 + node.len() + 8 + 8 + 4 + channel.len() + 4 + frame.payload.len());
|
||||
put_bytes(&mut out, node);
|
||||
out.extend_from_slice(&stream.life.0.to_le_bytes());
|
||||
out.extend_from_slice(&frame.position.0.to_le_bytes());
|
||||
put_bytes(&mut out, channel);
|
||||
put_bytes(&mut out, &frame.payload);
|
||||
out
|
||||
}
|
||||
|
||||
/// Decode a delivery previously produced by [`encode_delivery`]. Returns a
|
||||
/// [`WireError`] on any malformed buffer instead of panicking.
|
||||
pub fn decode_delivery(buf: &[u8]) -> Result<(StreamId, Frame), WireError> {
|
||||
let mut cur = Cursor { buf, pos: 0 };
|
||||
let node = cur.take_str()?;
|
||||
let life = Lifetime(cur.take_u64()?);
|
||||
let position = Position(cur.take_u64()?);
|
||||
let channel = cur.take_str()?;
|
||||
let payload = cur.take_bytes()?.to_vec();
|
||||
if cur.pos != buf.len() {
|
||||
return Err(WireError::TrailingBytes);
|
||||
}
|
||||
let stream = StreamId::new(NodeId::new(node), life);
|
||||
let frame = Frame::new(ChannelId::new(channel), position, payload);
|
||||
Ok((stream, frame))
|
||||
}
|
||||
|
||||
fn put_bytes(out: &mut Vec<u8>, bytes: &[u8]) {
|
||||
out.extend_from_slice(&(bytes.len() as u32).to_le_bytes());
|
||||
out.extend_from_slice(bytes);
|
||||
}
|
||||
|
||||
struct Cursor<'a> {
|
||||
buf: &'a [u8],
|
||||
pos: usize,
|
||||
}
|
||||
|
||||
impl<'a> Cursor<'a> {
|
||||
fn take(&mut self, n: usize) -> Result<&'a [u8], WireError> {
|
||||
let end = self.pos.checked_add(n).ok_or(WireError::BadLength)?;
|
||||
let slice = self.buf.get(self.pos..end).ok_or(WireError::Truncated)?;
|
||||
self.pos = end;
|
||||
Ok(slice)
|
||||
}
|
||||
|
||||
fn take_u32(&mut self) -> Result<u32, WireError> {
|
||||
let b = self.take(4)?;
|
||||
Ok(u32::from_le_bytes([b[0], b[1], b[2], b[3]]))
|
||||
}
|
||||
|
||||
fn take_u64(&mut self) -> Result<u64, WireError> {
|
||||
let b = self.take(8)?;
|
||||
Ok(u64::from_le_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]]))
|
||||
}
|
||||
|
||||
fn take_bytes(&mut self) -> Result<&'a [u8], WireError> {
|
||||
let len = self.take_u32()? as usize;
|
||||
self.take(len)
|
||||
}
|
||||
|
||||
fn take_str(&mut self) -> Result<&'a str, WireError> {
|
||||
let bytes = self.take_bytes()?;
|
||||
std::str::from_utf8(bytes).map_err(|_| WireError::NotUtf8)
|
||||
}
|
||||
}
|
||||
|
|
@ -10,5 +10,6 @@ pub mod registry;
|
|||
pub mod node_metadata;
|
||||
pub mod snapshot;
|
||||
pub mod diagnostics;
|
||||
pub mod datastream;
|
||||
#[cfg(feature = "iroh")]
|
||||
pub mod iroh_driver;
|
||||
|
|
|
|||
243
crates/distribution/tests/datastream_support/mod.rs
Normal file
243
crates/distribution/tests/datastream_support/mod.rs
Normal file
|
|
@ -0,0 +1,243 @@
|
|||
//! Shared support for the datastream tests (testing spec §2, §3).
|
||||
//!
|
||||
//! Two things live here, both deliberately separate from the system under
|
||||
//! test so a test never asserts the code against itself:
|
||||
//!
|
||||
//! * the **payload library** — realistic record shapes and log lines, the
|
||||
//! bytes the pipe will actually carry (testing spec §5: "never
|
||||
//! placeholder text"); and
|
||||
//! * the **reference model** — the spec's rules restated as small, total
|
||||
//! functions over sequences (testing spec §3). It is the trusted oracle:
|
||||
//! every test's `expected` is derived from it, never captured from a run.
|
||||
//! It is written naively on purpose (collect, sort, scan) so a reader can
|
||||
//! confirm it against the spec by eye, while the pipe computes the same
|
||||
//! answers the long way.
|
||||
//!
|
||||
//! This module is `#[path]`-included into more than one test binary, so
|
||||
//! some items are unused in some of them.
|
||||
#![allow(dead_code)]
|
||||
|
||||
use distribution::datastream::catalog::{
|
||||
self, IdentityRecord, LifecycleCost, MembershipTransition, ProcStream, Record, ResourceSample,
|
||||
Role, RuntimeStats, TransportInternals,
|
||||
};
|
||||
use distribution::datastream::frame::{ChannelId, Frame, Position, StreamId};
|
||||
use distribution::datastream::mux::Mux;
|
||||
use distribution::datastream::store::GapSpan;
|
||||
pub use distribution::datastream::transport::Delivery;
|
||||
|
||||
/// An in-process node (testing spec §2 — the faked machine boundary): a
|
||||
/// *real* mux plus the producers that feed it. Producers push realistic
|
||||
/// bytes in at the producer seam; [`Node::sent`] takes the mux's ordered
|
||||
/// output stream — the value on the mux→transport seam. Only the machine
|
||||
/// boundary is faked; the mux is the production code.
|
||||
pub struct Node {
|
||||
stream: StreamId,
|
||||
mux: Mux,
|
||||
}
|
||||
|
||||
impl Node {
|
||||
/// Stand up a node for a given stream (node identity + lifetime).
|
||||
pub fn new(stream: StreamId) -> Self {
|
||||
Node { mux: Mux::unbounded(stream.clone()), stream }
|
||||
}
|
||||
|
||||
/// The stream this node produces.
|
||||
pub fn stream_id(&self) -> &StreamId {
|
||||
&self.stream
|
||||
}
|
||||
|
||||
/// A producer emits a typed record on its own channel (spec §6.1).
|
||||
pub fn emit<R: Record>(&self, record: &R) -> Position {
|
||||
self.mux.submit(R::channel(), record.encode())
|
||||
}
|
||||
|
||||
/// A producer emits a line of raw process output (spec §6.2).
|
||||
pub fn emit_text(&self, label: &str, stream: ProcStream, line: &str) -> Position {
|
||||
self.mux.submit(catalog::process_output(label, stream), line.as_bytes().to_vec())
|
||||
}
|
||||
|
||||
/// A producer emits bytes on a channel the consumer may not know
|
||||
/// (spec §6.3) — opaque to everything until a view learns the channel.
|
||||
pub fn emit_opaque(&self, channel: &str, bytes: &[u8]) -> Position {
|
||||
self.mux.submit(ChannelId::new(channel), bytes.to_vec())
|
||||
}
|
||||
|
||||
/// Take the node's ordered output stream (drains the mux).
|
||||
pub fn sent(&self) -> Vec<Frame> {
|
||||
self.mux.drain()
|
||||
}
|
||||
}
|
||||
|
||||
/// Realistic payloads, drawn on by both the verified vectors (testing spec
|
||||
/// §5) and the deployment scenario (testing spec §8). Nothing here is
|
||||
/// placeholder text.
|
||||
pub mod payloads {
|
||||
use super::*;
|
||||
|
||||
/// A boot/identity record for a node.
|
||||
pub fn identity(node: &str, life: u64) -> IdentityRecord {
|
||||
IdentityRecord {
|
||||
node: node.to_string(),
|
||||
role: Role::Worker,
|
||||
region: "us-east-1".to_string(),
|
||||
life,
|
||||
}
|
||||
}
|
||||
|
||||
/// A plausible resource sample; `tick` nudges the values so a series is
|
||||
/// not constant.
|
||||
pub fn resource(tick: u64) -> ResourceSample {
|
||||
ResourceSample {
|
||||
cpu_pct: 12.5 + (tick % 7) as f32 * 3.0,
|
||||
mem_used_mb: 2048 + (tick % 5) as u32 * 128,
|
||||
mem_total_mb: 16384,
|
||||
gpu_pct: (tick % 4) as f32 * 25.0,
|
||||
disk_used_gb: 40 + (tick % 3) as u32,
|
||||
net_rx_kbps: 900 + (tick % 11) as u32 * 30,
|
||||
net_tx_kbps: 300 + (tick % 13) as u32 * 20,
|
||||
}
|
||||
}
|
||||
|
||||
/// A transport-internals snapshot.
|
||||
pub fn transport(tick: u64) -> TransportInternals {
|
||||
TransportInternals {
|
||||
relay_connected: !tick.is_multiple_of(9),
|
||||
direct_peers: 2 + (tick % 3) as u32,
|
||||
relay_peers: 1,
|
||||
rtt_ms_p50: 18 + (tick % 5) as u32 * 4,
|
||||
}
|
||||
}
|
||||
|
||||
/// A membership transition between two peers' states.
|
||||
pub fn membership(peer: &str, from: &str, to: &str) -> MembershipTransition {
|
||||
MembershipTransition {
|
||||
peer: peer.to_string(),
|
||||
from: from.to_string(),
|
||||
to: to.to_string(),
|
||||
reason: "probe timeout".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// A runtime-stats record.
|
||||
pub fn runtime(tick: u64) -> RuntimeStats {
|
||||
RuntimeStats {
|
||||
actors_live: 30 + (tick % 6) as u32,
|
||||
mailbox_depth: (tick % 17) as u32,
|
||||
scheduled_tasks: 4 + (tick % 3) as u32,
|
||||
}
|
||||
}
|
||||
|
||||
/// A provider/lifecycle/cost record.
|
||||
pub fn lifecycle(phase: &str, uptime_s: u64) -> LifecycleCost {
|
||||
LifecycleCost { phase: phase.to_string(), cost_usd_per_hr: 1.27, uptime_s }
|
||||
}
|
||||
|
||||
/// A realistic line of process output (without trailing newline).
|
||||
pub fn log_line(label: &str, tick: u64) -> String {
|
||||
format!("[{label}] step {tick} loss=0.{:03} lr=3e-4", 250 - (tick % 200))
|
||||
}
|
||||
}
|
||||
|
||||
/// The reference model: the spec's rules as plain total functions over
|
||||
/// sequences. No transport, no storage, no concurrency, no time.
|
||||
pub mod reference {
|
||||
use super::*;
|
||||
|
||||
/// The frames a consumer was delivered for one stream, in arrival
|
||||
/// order — the raw material reconstruction works over.
|
||||
pub fn delivered_frames(stream: &StreamId, deliveries: &[Delivery]) -> Vec<Frame> {
|
||||
deliveries
|
||||
.iter()
|
||||
.filter(|d| &d.stream == stream)
|
||||
.map(|d| d.frame.clone())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Reconstruction (spec §8.1, testing spec §3): "keep the frames that
|
||||
/// were delivered, in position order." Duplicates of a position
|
||||
/// collapse to one (the carrier may not fabricate content, spec §9, so
|
||||
/// a repeat carries identical bytes).
|
||||
pub fn reconstruct(delivered: &[Frame]) -> Vec<Frame> {
|
||||
let mut frames: Vec<Frame> = Vec::new();
|
||||
for f in delivered {
|
||||
if !frames.iter().any(|seen| seen.position == f.position) {
|
||||
frames.push(f.clone());
|
||||
}
|
||||
}
|
||||
frames.sort_by_key(|f| f.position);
|
||||
frames
|
||||
}
|
||||
|
||||
/// The surfaced gaps as spans: the **interior** runs of positions missing
|
||||
/// between the first and last delivered position (spec §7.5, §8). A
|
||||
/// consumer can only detect gaps it has bracketing frames for; positions
|
||||
/// lost after the last delivered frame are invisible and manifest as the
|
||||
/// stream ending (spec §7.4 node death = truncation, not a gap).
|
||||
///
|
||||
/// Walks the sorted delivered positions — O(frames), never the gap size —
|
||||
/// so the oracle agrees with the store on the cheap path even across a
|
||||
/// near-`u64::MAX` gap.
|
||||
pub fn gap_spans(delivered: &[Frame]) -> Vec<GapSpan> {
|
||||
let recon = reconstruct(delivered);
|
||||
let mut spans = Vec::new();
|
||||
let mut prev: Option<u64> = None;
|
||||
for f in &recon {
|
||||
let p = f.position.0;
|
||||
if let Some(q) = prev
|
||||
&& p > q + 1
|
||||
{
|
||||
spans.push(GapSpan { start: q + 1, end: p - 1 });
|
||||
}
|
||||
prev = Some(p);
|
||||
}
|
||||
spans
|
||||
}
|
||||
|
||||
/// One structural item on the merged timeline (testing spec §3: "all
|
||||
/// stored frames in position order, channels interleaved"). This is the
|
||||
/// *structure* of the merged log — order and surfaced gaps — decoupled
|
||||
/// from how each payload is rendered for display, which is a separate
|
||||
/// §9.3 concern the tests assert on its own.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum TimelineItem {
|
||||
Frame { position: u64, channel: String },
|
||||
Gap { start: u64, end: u64 },
|
||||
}
|
||||
|
||||
/// The merged log oracle (spec §9.2): every delivered frame in position
|
||||
/// order, channels interleaved, with an interior gap surfaced wherever a
|
||||
/// position is missing. Written naively so it is obviously the spec.
|
||||
pub fn merged_log(delivered: &[Frame]) -> Vec<TimelineItem> {
|
||||
let frames = reconstruct(delivered);
|
||||
let mut out = Vec::new();
|
||||
let mut prev: Option<u64> = None;
|
||||
for f in &frames {
|
||||
let pos = f.position.0;
|
||||
if let Some(p) = prev
|
||||
&& pos > p + 1
|
||||
{
|
||||
out.push(TimelineItem::Gap { start: p + 1, end: pos - 1 });
|
||||
}
|
||||
out.push(TimelineItem::Frame { position: pos, channel: f.channel.to_string() });
|
||||
prev = Some(pos);
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
/// Convenience: build a frame on a typed channel from a record.
|
||||
pub fn typed_frame<R: Record>(record: &R, position: u64) -> Frame {
|
||||
Frame::new(R::channel(), Position(position), record.encode())
|
||||
}
|
||||
|
||||
/// Convenience: build a frame on a raw-text process-output channel.
|
||||
pub fn text_frame(label: &str, stream: ProcStream, line: &str, position: u64) -> Frame {
|
||||
Frame::new(catalog::process_output(label, stream), Position(position), line.as_bytes().to_vec())
|
||||
}
|
||||
|
||||
/// Convenience: a frame on a channel id the consumer does not know — an
|
||||
/// opaque channel (spec §6.3).
|
||||
pub fn opaque_frame(id: &str, payload: &[u8], position: u64) -> Frame {
|
||||
Frame::new(ChannelId::new(id), Position(position), payload.to_vec())
|
||||
}
|
||||
990
crates/distribution/tests/t_datastream.rs
Normal file
990
crates/distribution/tests/t_datastream.rs
Normal file
|
|
@ -0,0 +1,990 @@
|
|||
//! Datastream verification harness (see `DATASTREAM_TESTING_SPEC.md`).
|
||||
//!
|
||||
//! The tests climb the ladder of testing spec §4: Kind I verified vectors
|
||||
//! (one rule), Kind II seam tests (one boundary), Kind III the full mock
|
||||
//! (the assembled pipe), Kind IV a deployment simulation. Every expected
|
||||
//! answer is derived from the reference model in [`support::reference`],
|
||||
//! never captured from the system under test (testing spec §3). The real-
|
||||
//! I/O checks of testing spec §9–§10 live in `t_datastream_realio.rs`.
|
||||
|
||||
#[path = "datastream_support/mod.rs"]
|
||||
mod support;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use distribution::datastream::catalog::{
|
||||
self, ChannelKind, IdentityRecord, LifecycleCost, ProcStream, Record, ResourceSample, Role,
|
||||
};
|
||||
use distribution::datastream::frame::{ChannelId, Frame, Lifetime, NodeId, Position, StreamId};
|
||||
use distribution::datastream::ingest::Consumer;
|
||||
use distribution::datastream::mux::Mux;
|
||||
use distribution::datastream::store::{GapSpan, StoredStream};
|
||||
use distribution::datastream::transport::{Delivery, Reorder, ScriptedTransport, StreamScript};
|
||||
use distribution::datastream::views::{self, Body, LogEntry};
|
||||
use distribution::datastream::wire::{decode_delivery, encode_delivery, WireError};
|
||||
|
||||
use support::reference::TimelineItem;
|
||||
use support::{payloads, reference, Node};
|
||||
|
||||
/// The frames of `sent` that survive dropping `dropped`, in send order —
|
||||
/// the scenario's delivered set, derived without running the pipe.
|
||||
fn surviving(sent: &[Frame], dropped: &[u64]) -> Vec<Frame> {
|
||||
let drop: std::collections::BTreeSet<u64> = dropped.iter().copied().collect();
|
||||
sent.iter().filter(|f| !drop.contains(&f.position.0)).cloned().collect()
|
||||
}
|
||||
|
||||
/// A node that has emitted a realistic spread of channels: identity, two
|
||||
/// resource samples, a transport snapshot, a membership transition, runtime
|
||||
/// stats, two lines of process output, and a lifecycle record.
|
||||
fn busy_node(stream: &StreamId) -> Vec<Frame> {
|
||||
let node = Node::new(stream.clone());
|
||||
node.emit(&payloads::identity(stream.node.as_str(), stream.life.0)); // 0
|
||||
node.emit(&payloads::resource(0)); // 1
|
||||
node.emit_text("trainer", ProcStream::Stdout, &payloads::log_line("trainer", 0)); // 2
|
||||
node.emit(&payloads::transport(1)); // 3
|
||||
node.emit(&payloads::membership("node-beta", "alive", "suspect")); // 4
|
||||
node.emit(&payloads::resource(1)); // 5
|
||||
node.emit(&payloads::runtime(2)); // 6
|
||||
node.emit_text("trainer", ProcStream::Stderr, "WARN cuda oom, retrying"); // 7
|
||||
node.emit(&payloads::lifecycle("running", 1800)); // 8
|
||||
node.emit(&payloads::resource(2)); // 9
|
||||
node.sent()
|
||||
}
|
||||
|
||||
/// Carry a node's sent frames through the scripted transport and ingest the
|
||||
/// result, returning the consumer (whose store the views read).
|
||||
fn consume(stream: &StreamId, sent: &[Frame], script: StreamScript) -> Consumer {
|
||||
let delivered = ScriptedTransport::carry(stream, sent, &script);
|
||||
let mut consumer = Consumer::new();
|
||||
consumer.ingest(delivered);
|
||||
consumer
|
||||
}
|
||||
|
||||
/// Project a view's merged log down to its structural skeleton — order and
|
||||
/// surfaced gaps — so it can be compared to the reference oracle.
|
||||
fn structure(entries: &[LogEntry]) -> Vec<TimelineItem> {
|
||||
entries
|
||||
.iter()
|
||||
.map(|e| match e {
|
||||
LogEntry::Frame(mf) => {
|
||||
TimelineItem::Frame { position: mf.position.0, channel: mf.channel.to_string() }
|
||||
}
|
||||
LogEntry::Gap(span) => TimelineItem::Gap { start: span.start, end: span.end },
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn test_stream() -> StreamId {
|
||||
StreamId::new(NodeId::new("node-alpha"), Lifetime(1))
|
||||
}
|
||||
|
||||
// ── Kind I — verified vectors (testing spec §5) ────────────────────────
|
||||
//
|
||||
// Each targets a single rule in isolation and asserts a *relationship*
|
||||
// (round-trip, byte-equality) rather than a captured literal, so the
|
||||
// vector cannot be authored wrong and survives a refactor.
|
||||
|
||||
/// The codec round-trips: decoding what was encoded yields the original
|
||||
/// record (testing spec §3, §5), for every typed channel in the catalog
|
||||
/// (spec §6.1), on realistic payloads.
|
||||
#[test]
|
||||
fn codec_round_trips_every_typed_channel() {
|
||||
assert_round_trip(&payloads::identity("node-alpha", 1));
|
||||
assert_round_trip(&payloads::resource(3));
|
||||
assert_round_trip(&payloads::transport(4));
|
||||
assert_round_trip(&payloads::membership("node-beta", "alive", "suspect"));
|
||||
assert_round_trip(&payloads::runtime(5));
|
||||
assert_round_trip(&payloads::lifecycle("running", 3600));
|
||||
}
|
||||
|
||||
fn assert_round_trip<R: Record + PartialEq + std::fmt::Debug>(record: &R) {
|
||||
let decoded = R::decode(&record.encode()).expect("round-trips");
|
||||
assert_eq!(&decoded, record, "decode(encode(r)) must equal r");
|
||||
}
|
||||
|
||||
/// A typed channel's codec tolerates version skew (spec §6.3): a record
|
||||
/// written by a newer producer (extra field) and one written by an older
|
||||
/// producer (missing field) both still decode.
|
||||
#[test]
|
||||
fn typed_codec_tolerates_version_skew() {
|
||||
// Newer producer: an extra, unknown field. The consumer ignores it.
|
||||
let with_extra = br#"{"node":"node-x","role":"worker","region":"eu-west-1",
|
||||
"life":2,"future_field":{"nested":true}}"#;
|
||||
let decoded = IdentityRecord::decode(with_extra).expect("unknown field ignored");
|
||||
assert_eq!(decoded.node, "node-x");
|
||||
assert_eq!(decoded.life, 2);
|
||||
|
||||
// Older producer: a sample that predates several fields. The missing
|
||||
// ones decode to their defaults rather than failing.
|
||||
let sparse = br#"{"cpu_pct":42.0}"#;
|
||||
let decoded = ResourceSample::decode(sparse).expect("missing fields default");
|
||||
assert_eq!(decoded.cpu_pct, 42.0);
|
||||
assert_eq!(decoded.mem_used_mb, 0);
|
||||
assert_eq!(decoded.net_tx_kbps, 0);
|
||||
}
|
||||
|
||||
/// The transport envelope round-trips exactly (testing spec §9): a decoded
|
||||
/// delivery equals the one encoded — payload bytes byte-identical, channel
|
||||
/// and position intact — across realistic, unicode, empty, and binary
|
||||
/// payloads.
|
||||
#[test]
|
||||
fn wire_envelope_round_trips() {
|
||||
let stream = StreamId::new(NodeId::new("node-ünïcode-Ω"), Lifetime(7));
|
||||
let cases = vec![
|
||||
support::typed_frame(&payloads::resource(2), 0),
|
||||
support::text_frame("trainer", ProcStream::Stdout, "loss=0.0312 lr=3e-4", 1),
|
||||
// Empty payload — a real edge (a channel may emit a zero-length span).
|
||||
Frame::new(ChannelId::new("proc.trainer.stderr"), Position(2), Vec::new()),
|
||||
// Binary payload on an opaque channel — bytes the consumer cannot read.
|
||||
support::opaque_frame("sensor.raw", &[0u8, 255, 1, 254, 128, 0, 0, 7], 3),
|
||||
// Unicode channel id and payload.
|
||||
Frame::new(ChannelId::new("proc.café.stdout"), Position(4), "café ☕".as_bytes().to_vec()),
|
||||
];
|
||||
|
||||
for frame in &cases {
|
||||
let bytes = encode_delivery(&stream, frame);
|
||||
let (out_stream, out_frame) = decode_delivery(&bytes).expect("decodes");
|
||||
assert_eq!(out_stream, stream, "stream id intact");
|
||||
assert_eq!(&out_frame, frame, "frame intact (channel, position, payload bytes)");
|
||||
assert_eq!(out_frame.payload, frame.payload, "payload byte-identical");
|
||||
}
|
||||
}
|
||||
|
||||
/// A best-effort carrier can hand the consumer anything; a truncated or
|
||||
/// malformed envelope must fail gracefully (a `WireError`), never panic.
|
||||
#[test]
|
||||
fn wire_envelope_rejects_malformed_buffers_without_panicking() {
|
||||
let stream = StreamId::new(NodeId::new("node-a"), Lifetime(1));
|
||||
let frame = support::typed_frame(&payloads::resource(1), 9);
|
||||
let good = encode_delivery(&stream, &frame);
|
||||
|
||||
// Every proper prefix is incomplete and must be rejected as an error.
|
||||
for cut in 0..good.len() {
|
||||
match decode_delivery(&good[..cut]) {
|
||||
Err(_) => {}
|
||||
Ok(_) => panic!("truncated buffer of {cut} bytes decoded as if whole"),
|
||||
}
|
||||
}
|
||||
// The full buffer still decodes.
|
||||
assert!(decode_delivery(&good).is_ok());
|
||||
|
||||
// A length prefix that overruns the buffer is a clean error.
|
||||
let mut lying = Vec::new();
|
||||
lying.extend_from_slice(&u32::MAX.to_le_bytes()); // claims 4 GiB of node id
|
||||
assert_eq!(decode_delivery(&lying), Err(WireError::Truncated));
|
||||
|
||||
// One datagram carries exactly one frame: trailing bytes are rejected,
|
||||
// so a malformed concatenation cannot be silently half-read.
|
||||
let mut trailing = good.clone();
|
||||
trailing.push(0xFF);
|
||||
assert_eq!(decode_delivery(&trailing), Err(WireError::TrailingBytes));
|
||||
}
|
||||
|
||||
/// Opaque retention is byte-exact (spec §6.3, §8.3): a channel the
|
||||
/// consumer does not recognize classifies as opaque, and its bytes survive
|
||||
/// the envelope unchanged — the relationship "stored bytes equal submitted
|
||||
/// bytes" (testing spec §5).
|
||||
#[test]
|
||||
fn opaque_channel_classifies_and_preserves_bytes() {
|
||||
let id = ChannelId::new("v2.gpu.thermals"); // not in the catalog
|
||||
assert_eq!(catalog::classify(&id), ChannelKind::Opaque);
|
||||
|
||||
let payload: Vec<u8> = (0u8..=255).cycle().take(1000).collect();
|
||||
let frame = Frame::new(id, Position(42), payload.clone());
|
||||
let stream = StreamId::new(NodeId::new("node-z"), Lifetime(3));
|
||||
|
||||
let (_, out) = decode_delivery(&encode_delivery(&stream, &frame)).expect("decodes");
|
||||
assert_eq!(out.payload, payload, "opaque bytes retained whole, byte-identical");
|
||||
}
|
||||
|
||||
/// The catalog classifies known channels correctly and treats the
|
||||
/// process-output family (spec §6.2) as text without enumerating labels.
|
||||
#[test]
|
||||
fn catalog_classifies_known_and_text_family() {
|
||||
assert_eq!(catalog::classify(&ChannelId::new(catalog::IDENTITY)), ChannelKind::Typed);
|
||||
assert_eq!(catalog::classify(&ChannelId::new(catalog::HOST_RESOURCE)), ChannelKind::Typed);
|
||||
// A process introduced at runtime gets text channels for free.
|
||||
let out = catalog::process_output("inference-server", ProcStream::Stdout);
|
||||
assert_eq!(out.as_str(), "proc.inference-server.stdout");
|
||||
assert_eq!(catalog::classify(&out), ChannelKind::Text);
|
||||
}
|
||||
|
||||
/// A record carries its own channel (spec §6.1 fixed identity), and that
|
||||
/// channel classifies as typed.
|
||||
#[test]
|
||||
fn records_name_their_own_typed_channel() {
|
||||
assert_eq!(IdentityRecord::channel().as_str(), catalog::IDENTITY);
|
||||
assert_eq!(ResourceSample::channel().as_str(), catalog::HOST_RESOURCE);
|
||||
assert_eq!(catalog::classify(&IdentityRecord::channel()), ChannelKind::Typed);
|
||||
// Role enum encodes in snake_case as the wire expects.
|
||||
let r = IdentityRecord { node: "n".into(), role: Role::Coordinator, region: "r".into(), life: 0 };
|
||||
let json = String::from_utf8(r.encode()).unwrap();
|
||||
assert!(json.contains("\"coordinator\""), "role serializes snake_case: {json}");
|
||||
}
|
||||
|
||||
// ── The mux: position authority (spec §5) ──────────────────────────────
|
||||
|
||||
/// Kind I (testing spec §5) — position assignment is monotonic and
|
||||
/// gap-free (spec §5.2): with no overflow, K submissions number `0..K`
|
||||
/// exactly, and `submit` returns each position in order.
|
||||
#[test]
|
||||
fn mux_numbers_monotonic_and_gap_free() {
|
||||
let mux = Mux::unbounded(test_stream());
|
||||
let k = 64u64;
|
||||
for i in 0..k {
|
||||
let pos = mux.submit(catalog::HOST_RESOURCE, payloads::resource(i).encode());
|
||||
assert_eq!(pos, Position(i), "submit returns the next position, in order");
|
||||
}
|
||||
assert_eq!(mux.assigned(), k, "assigned == number of submissions (never skips)");
|
||||
assert_eq!(mux.dropped(), 0, "no overflow, nothing dropped");
|
||||
|
||||
let positions: Vec<u64> = mux.drain().iter().map(|f| f.position.0).collect();
|
||||
assert_eq!(positions, (0..k).collect::<Vec<_>>(), "emitted positions are 0..k, gap-free");
|
||||
}
|
||||
|
||||
/// Kind II (testing spec §6) — across the mux seam, every submission
|
||||
/// appears as exactly one frame, byte-identical, in one interleaved order.
|
||||
/// A typed event and a log line share the single timeline (spec §5.1).
|
||||
#[test]
|
||||
fn mux_seam_preserves_every_submission_byte_identical() {
|
||||
let mux = Mux::unbounded(test_stream());
|
||||
|
||||
// A realistic interleaving of typed records and raw process output —
|
||||
// the same kind of thing on one stream (spec §4.2).
|
||||
let submissions: Vec<(ChannelId, Vec<u8>)> = vec![
|
||||
(catalog::IDENTITY.into(), payloads::identity("node-alpha", 1).encode()),
|
||||
(
|
||||
catalog::process_output("trainer", ProcStream::Stdout),
|
||||
payloads::log_line("trainer", 0).into_bytes(),
|
||||
),
|
||||
(catalog::HOST_RESOURCE.into(), payloads::resource(1).encode()),
|
||||
(catalog::MEMBERSHIP.into(), payloads::membership("node-beta", "alive", "suspect").encode()),
|
||||
(
|
||||
catalog::process_output("trainer", ProcStream::Stderr),
|
||||
b"WARN cuda oom, retrying".to_vec(),
|
||||
),
|
||||
(catalog::RUNTIME_STATS.into(), payloads::runtime(2).encode()),
|
||||
];
|
||||
|
||||
for (channel, payload) in &submissions {
|
||||
mux.submit(channel.clone(), payload.clone());
|
||||
}
|
||||
|
||||
let frames = mux.drain();
|
||||
assert_eq!(frames.len(), submissions.len(), "exactly one frame per submission — none lost");
|
||||
for (i, (frame, (channel, payload))) in frames.iter().zip(&submissions).enumerate() {
|
||||
assert_eq!(frame.position, Position(i as u64), "interleaved in submission order, gap-free");
|
||||
assert_eq!(&frame.channel, channel, "channel tag preserved crossing the seam");
|
||||
assert_eq!(&frame.payload, payload, "payload bytes byte-identical crossing the seam");
|
||||
}
|
||||
}
|
||||
|
||||
/// Spec §5.3 — on overflow the mux drops the frame, but its position is
|
||||
/// already spent, so the loss surfaces as a missing position (a detectable
|
||||
/// gap), never a silent renumber. The reference-model gap oracle confirms
|
||||
/// the interior gap.
|
||||
#[test]
|
||||
fn mux_overflow_drops_surface_as_a_gap_not_a_renumber() {
|
||||
let mux = Mux::new(test_stream(), 2); // tiny buffer
|
||||
|
||||
mux.submit(catalog::HOST_RESOURCE, payloads::resource(0).encode()); // pos 0 -> buffered
|
||||
mux.submit(catalog::HOST_RESOURCE, payloads::resource(1).encode()); // pos 1 -> buffered
|
||||
mux.submit(catalog::HOST_RESOURCE, payloads::resource(2).encode()); // pos 2 -> OVERFLOW, dropped
|
||||
let first = mux.drain(); // empties the buffer
|
||||
mux.submit(catalog::HOST_RESOURCE, payloads::resource(3).encode()); // pos 3 -> buffered
|
||||
let second = mux.drain();
|
||||
|
||||
assert_eq!(mux.assigned(), 4, "every submission consumed a position — numbering never skips");
|
||||
assert_eq!(mux.dropped(), 1, "exactly the overflowing frame was dropped");
|
||||
|
||||
let mut emitted: Vec<Frame> = first;
|
||||
emitted.extend(second);
|
||||
let positions: Vec<u64> = emitted.iter().map(|f| f.position.0).collect();
|
||||
assert_eq!(positions, vec![0, 1, 3], "dropped position 2 is simply absent, others not renumbered");
|
||||
|
||||
// The dropped position reads as an interior gap, exactly what the
|
||||
// consumer will later surface (spec §7.5).
|
||||
assert_eq!(
|
||||
reference::gap_spans(&emitted),
|
||||
vec![GapSpan { start: 2, end: 2 }],
|
||||
"the drop is a detectable gap at position 2"
|
||||
);
|
||||
}
|
||||
|
||||
/// Spec §5.3 — the mux serializes concurrent producers into one order:
|
||||
/// every position is assigned exactly once across threads, with no
|
||||
/// duplicate and no gap. This is the single-ordering-authority guarantee.
|
||||
#[test]
|
||||
fn mux_serializes_concurrent_producers_without_collision() {
|
||||
let mux = Arc::new(Mux::unbounded(test_stream()));
|
||||
let threads = 8u64;
|
||||
let per_thread = 500u64;
|
||||
|
||||
let handles: Vec<_> = (0..threads)
|
||||
.map(|t| {
|
||||
let mux = Arc::clone(&mux);
|
||||
std::thread::spawn(move || {
|
||||
let mut mine = Vec::with_capacity(per_thread as usize);
|
||||
for i in 0..per_thread {
|
||||
// Each thread is a distinct producer writing real bytes.
|
||||
let pos = mux.submit(catalog::RUNTIME_STATS, payloads::runtime(t * 1000 + i).encode());
|
||||
mine.push(pos.0);
|
||||
}
|
||||
mine
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
let mut assigned: Vec<u64> = handles.into_iter().flat_map(|h| h.join().unwrap()).collect();
|
||||
let total = threads * per_thread;
|
||||
assert_eq!(mux.assigned(), total);
|
||||
assert_eq!(mux.dropped(), 0, "unbounded mux drops nothing");
|
||||
|
||||
assigned.sort_unstable();
|
||||
assert_eq!(assigned, (0..total).collect::<Vec<_>>(), "each position assigned exactly once");
|
||||
|
||||
// The buffered frames carry the same complete set of positions.
|
||||
let mut emitted: Vec<u64> = mux.drain().iter().map(|f| f.position.0).collect();
|
||||
emitted.sort_unstable();
|
||||
assert_eq!(emitted, (0..total).collect::<Vec<_>>(), "no frame lost, no position duplicated");
|
||||
}
|
||||
|
||||
// ── Transport seam + ingest + store (spec §7, §8) ──────────────────────
|
||||
|
||||
/// Build a realistic single-node stream the way a node would: drive the
|
||||
/// real mux with a mix of typed records, raw process output, and a channel
|
||||
/// the consumer does not know, then take its output.
|
||||
fn realistic_stream(stream: &StreamId) -> Vec<Frame> {
|
||||
let mux = Mux::unbounded(stream.clone());
|
||||
mux.submit(catalog::IDENTITY, payloads::identity(stream.node.as_str(), stream.life.0).encode());
|
||||
mux.submit(catalog::HOST_RESOURCE, payloads::resource(0).encode());
|
||||
// A channel this consumer cannot decode — must still be retained whole.
|
||||
mux.submit(ChannelId::new("v2.gpu.thermals"), vec![0xDE, 0xAD, 0xBE, 0xEF]);
|
||||
mux.submit(
|
||||
catalog::process_output("trainer", ProcStream::Stdout),
|
||||
payloads::log_line("trainer", 0).into_bytes(),
|
||||
);
|
||||
mux.submit(catalog::HOST_RESOURCE, payloads::resource(1).encode());
|
||||
mux.submit(catalog::MEMBERSHIP, payloads::membership("node-beta", "alive", "suspect").encode());
|
||||
mux.drain()
|
||||
}
|
||||
|
||||
/// Kind II (testing spec §6) — across ingest, every delivered frame appears
|
||||
/// in the stored stream byte-identical, an undecodable channel is retained
|
||||
/// whole, and the gap left by a dropped position is surfaced. Both sides of
|
||||
/// the seam are checked against the reference model.
|
||||
#[test]
|
||||
fn ingest_seam_stores_every_delivered_frame_and_surfaces_gaps() {
|
||||
let stream = test_stream();
|
||||
let sent = realistic_stream(&stream); // positions 0..6
|
||||
|
||||
// The carrier drops one resource sample (interior loss) and scrambles
|
||||
// arrival order within windows — both inside the envelope (spec §9).
|
||||
let script = StreamScript::dropping([4]).with_reorder(Reorder::Windows(3));
|
||||
let delivered = ScriptedTransport::carry(&stream, &sent, &script);
|
||||
|
||||
let mut consumer = Consumer::new();
|
||||
consumer.ingest(delivered.clone());
|
||||
let stored = consumer.store().stream(&stream).expect("stream stored");
|
||||
|
||||
// Derive the trusted answer from the reference model, never from the run.
|
||||
let delivered_frames = reference::delivered_frames(&stream, &delivered);
|
||||
assert_eq!(
|
||||
stored.to_vec(),
|
||||
reference::reconstruct(&delivered_frames),
|
||||
"stored stream equals the reference reconstruction (every delivered frame, in position order)"
|
||||
);
|
||||
assert_eq!(
|
||||
stored.gap_spans(),
|
||||
reference::gap_spans(&delivered_frames),
|
||||
"the dropped position is surfaced as a gap"
|
||||
);
|
||||
assert_eq!(
|
||||
stored.gap_spans(),
|
||||
vec![GapSpan { start: 4, end: 4 }],
|
||||
"specifically position 4 is missing"
|
||||
);
|
||||
|
||||
// The undecodable channel landed whole, byte-identical (spec §8.3).
|
||||
let opaque = stored.at(Position(2)).expect("opaque frame retained");
|
||||
assert_eq!(opaque.channel, ChannelId::new("v2.gpu.thermals"));
|
||||
assert_eq!(opaque.payload, vec![0xDE, 0xAD, 0xBE, 0xEF], "opaque bytes retained whole");
|
||||
assert_eq!(catalog::classify(&opaque.channel), ChannelKind::Opaque);
|
||||
}
|
||||
|
||||
/// Spec §7.5 — the consumer reconstructs by position, not by arrival: even
|
||||
/// when the carrier delivers a stream fully reversed, the stored stream is
|
||||
/// in position order and identical to the in-order case.
|
||||
#[test]
|
||||
fn ingest_reconstructs_by_position_not_arrival_order() {
|
||||
let stream = test_stream();
|
||||
let sent = realistic_stream(&stream);
|
||||
|
||||
let reversed = ScriptedTransport::carry(
|
||||
&stream,
|
||||
&sent,
|
||||
&StreamScript::perfect().with_reorder(Reorder::Reversed),
|
||||
);
|
||||
// The carrier really did reverse arrival: first delivered is last sent.
|
||||
assert_eq!(reversed.first().unwrap().frame.position, sent.last().unwrap().position);
|
||||
|
||||
let mut consumer = Consumer::new();
|
||||
consumer.ingest(reversed);
|
||||
let stored = consumer.store().stream(&stream).unwrap();
|
||||
|
||||
assert_eq!(stored.to_vec(), sent, "reconstruction restores the sent order from reversed arrival");
|
||||
}
|
||||
|
||||
/// Spec §9 (no fabrication) — a position delivered twice collapses to one
|
||||
/// stored frame; ingest is idempotent and append-only.
|
||||
#[test]
|
||||
fn ingest_is_idempotent_on_duplicate_positions() {
|
||||
let stream = test_stream();
|
||||
let frame = support::typed_frame(&payloads::resource(0), 0);
|
||||
|
||||
let mut consumer = Consumer::new();
|
||||
consumer.accept(Delivery::new(stream.clone(), frame.clone()));
|
||||
let was_new = consumer.accept(Delivery::new(stream.clone(), frame.clone()));
|
||||
|
||||
assert!(!was_new, "a repeated position is not recorded a second time");
|
||||
assert_eq!(consumer.store().stream(&stream).unwrap().len(), 1, "exactly one frame stored");
|
||||
}
|
||||
|
||||
/// Spec §8.4 — a node identity reused across lifetimes does not merge: each
|
||||
/// life is its own stored stream, even though both begin at position 0 with
|
||||
/// an identity frame.
|
||||
#[test]
|
||||
fn reincarnated_node_does_not_merge_with_prior_life() {
|
||||
let node = "node-worker-7";
|
||||
let life1 = StreamId::new(NodeId::new(node), Lifetime(1));
|
||||
let life2 = StreamId::new(NodeId::new(node), Lifetime(2)); // same node, new life
|
||||
let other = StreamId::new(NodeId::new("node-worker-8"), Lifetime(1));
|
||||
|
||||
let s1 = realistic_stream(&life1);
|
||||
let s2 = realistic_stream(&life2);
|
||||
let s3 = realistic_stream(&other);
|
||||
|
||||
// Interleaved on one wire; ingest must route by stream id alone.
|
||||
let delivered = ScriptedTransport::carry_all(&[
|
||||
(life1.clone(), s1.clone(), StreamScript::perfect()),
|
||||
(life2.clone(), s2.clone(), StreamScript::perfect()),
|
||||
(other.clone(), s3.clone(), StreamScript::perfect()),
|
||||
]);
|
||||
|
||||
let mut consumer = Consumer::new();
|
||||
consumer.ingest(delivered);
|
||||
let store = consumer.store();
|
||||
|
||||
assert_eq!(store.len(), 3, "three distinct streams, none merged");
|
||||
assert_eq!(store.stream(&life1).unwrap().to_vec(), s1);
|
||||
assert_eq!(store.stream(&life2).unwrap().to_vec(), s2);
|
||||
assert_eq!(store.stream(&other).unwrap().to_vec(), s3);
|
||||
|
||||
// Both lives have a position-0 identity frame; they did not collide.
|
||||
let id1 = store.stream(&life1).unwrap().at(Position(0)).unwrap();
|
||||
let id2 = store.stream(&life2).unwrap().at(Position(0)).unwrap();
|
||||
assert_ne!(id1.payload, id2.payload, "each life's identity frame is its own");
|
||||
assert_eq!(
|
||||
IdentityRecord::decode(&id1.payload).unwrap().life,
|
||||
1,
|
||||
"life-1 identity attributes to lifetime 1"
|
||||
);
|
||||
assert_eq!(IdentityRecord::decode(&id2.payload).unwrap().life, 2);
|
||||
}
|
||||
|
||||
// ── Views: read-time projections (spec §9) ─────────────────────────────
|
||||
|
||||
/// Kind II (testing spec §6) — the merged log view (spec §9.2) is every
|
||||
/// stored frame in position order, channels interleaved, with the dropped
|
||||
/// position surfaced as a gap. Its structure equals the reference oracle.
|
||||
#[test]
|
||||
fn merged_log_matches_oracle_with_surfaced_gap() {
|
||||
let stream = test_stream();
|
||||
let sent = realistic_stream(&stream); // positions 0..6, mixed channels
|
||||
let script = StreamScript::dropping([3]).with_reorder(Reorder::Reversed);
|
||||
let delivered = ScriptedTransport::carry(&stream, &sent, &script);
|
||||
|
||||
let mut consumer = Consumer::new();
|
||||
consumer.ingest(delivered.clone());
|
||||
let stored = consumer.store().stream(&stream).unwrap();
|
||||
|
||||
let log = views::merged_log(stored);
|
||||
let expected = reference::merged_log(&reference::delivered_frames(&stream, &delivered));
|
||||
assert_eq!(structure(&log), expected, "merged log = timeline in position order with gap surfaced");
|
||||
|
||||
// The gap sits between the bracketing frames, not at the end.
|
||||
assert!(
|
||||
log.iter().any(|e| matches!(e, LogEntry::Gap(g) if g.start == 3 && g.end == 3)),
|
||||
"position 3 is surfaced as an interior gap"
|
||||
);
|
||||
}
|
||||
|
||||
/// Kind I (testing spec §5) — a view decodes each channel via its codec and
|
||||
/// degrades to raw bytes over anything it cannot decode (spec §9.3): a typed
|
||||
/// channel decodes to a record, a text channel to lines, an unknown channel
|
||||
/// to bytes, and a typed channel carrying garbage degrades rather than
|
||||
/// failing.
|
||||
#[test]
|
||||
fn view_decodes_each_channel_and_degrades_gracefully() {
|
||||
// Typed channel → structured record.
|
||||
let resource = payloads::resource(2);
|
||||
match views::decode_body(&ResourceSample::channel(), &resource.encode()) {
|
||||
Body::Record(value) => {
|
||||
assert!(value.get("cpu_pct").is_some(), "typed channel decodes to its record fields");
|
||||
}
|
||||
other => panic!("typed channel should decode to a record, got {other:?}"),
|
||||
}
|
||||
|
||||
// Raw-text channel → its line.
|
||||
let line = payloads::log_line("trainer", 7);
|
||||
let text_channel = catalog::process_output("trainer", ProcStream::Stdout);
|
||||
assert_eq!(views::decode_body(&text_channel, line.as_bytes()), Body::Text(line.clone()));
|
||||
|
||||
// Unknown channel → raw bytes (graceful, spec §6.3/§9.3).
|
||||
let opaque = ChannelId::new("v2.gpu.thermals");
|
||||
assert_eq!(views::decode_body(&opaque, &[0xDE, 0xAD]), Body::Raw(vec![0xDE, 0xAD]));
|
||||
|
||||
// Typed channel, garbage bytes → degrades to raw, never panics or drops.
|
||||
let garbage = b"not-json{oops".to_vec();
|
||||
assert_eq!(
|
||||
views::decode_body(&ResourceSample::channel(), &garbage),
|
||||
Body::Raw(garbage.clone()),
|
||||
"a typed channel that fails to parse degrades to bytes (spec §9.3)"
|
||||
);
|
||||
}
|
||||
|
||||
/// Kind I (testing spec §5) — the metric projection (spec §9.2) decodes one
|
||||
/// typed channel into a time series, in position order, leaving the other
|
||||
/// channels untouched. Expected is the records that were emitted.
|
||||
#[test]
|
||||
fn metric_projection_decodes_one_typed_channel_into_a_series() {
|
||||
let stream = test_stream();
|
||||
let mux = Mux::unbounded(stream.clone());
|
||||
mux.submit(catalog::IDENTITY, payloads::identity("node-alpha", 1).encode());
|
||||
|
||||
let mut expected: Vec<(Position, ResourceSample)> = Vec::new();
|
||||
for i in 0..5 {
|
||||
let sample = payloads::resource(i);
|
||||
let pos = mux.submit(catalog::HOST_RESOURCE, sample.encode());
|
||||
expected.push((pos, sample));
|
||||
// Interleave an unrelated channel — the projection must ignore it.
|
||||
mux.submit(catalog::MEMBERSHIP, payloads::membership("p", "alive", "dead").encode());
|
||||
}
|
||||
let sent = mux.drain();
|
||||
|
||||
let consumer = consume(&stream, &sent, StreamScript::perfect());
|
||||
let series = views::metric_series::<ResourceSample>(consumer.store().stream(&stream).unwrap());
|
||||
|
||||
assert_eq!(series, expected, "the resource series is exactly the samples emitted, in order");
|
||||
}
|
||||
|
||||
/// Kind I (testing spec §5) — tail, grep, and filter are windowed /
|
||||
/// predicate views over the stream (spec §9.2).
|
||||
#[test]
|
||||
fn tail_grep_and_filter_restrict_the_stream() {
|
||||
let stream = test_stream();
|
||||
let sent = realistic_stream(&stream); // id, resource, opaque, proc, resource, membership
|
||||
let consumer = consume(&stream, &sent, StreamScript::perfect());
|
||||
let stored = consumer.store().stream(&stream).unwrap();
|
||||
|
||||
// Tail: the last two frames by position.
|
||||
let last_two = views::tail(stored, 2);
|
||||
assert_eq!(last_two, stored.to_vec()[4..].to_vec(), "tail(2) is the final two frames in order");
|
||||
|
||||
// Grep: only the membership transition mentions "suspect".
|
||||
let hits = views::grep(stored, "suspect");
|
||||
assert_eq!(hits.len(), 1, "exactly one frame matches the needle");
|
||||
assert_eq!(hits[0].channel, ChannelId::new(catalog::MEMBERSHIP));
|
||||
|
||||
// Filter: restrict to a single channel.
|
||||
let resources = views::filter(stored, |f| f.channel == ResourceSample::channel());
|
||||
assert_eq!(resources.len(), 2, "two resource samples in the stream");
|
||||
assert!(resources.iter().all(|f| f.channel == ResourceSample::channel()));
|
||||
}
|
||||
|
||||
/// Boundary — views over an empty stream are empty, and a single-frame
|
||||
/// stream has no gaps. Views must not panic at the edges.
|
||||
#[test]
|
||||
fn views_handle_empty_and_singleton_streams() {
|
||||
let empty = StoredStream::new();
|
||||
assert!(views::merged_log(&empty).is_empty());
|
||||
assert!(views::tail(&empty, 5).is_empty());
|
||||
assert!(views::metric_series::<ResourceSample>(&empty).is_empty());
|
||||
assert_eq!(views::replay(&empty).count(), 0);
|
||||
|
||||
let mut one = StoredStream::new();
|
||||
one.record(support::typed_frame(&payloads::resource(0), 0));
|
||||
let log = views::merged_log(&one);
|
||||
assert_eq!(log.len(), 1, "one frame, one entry");
|
||||
assert!(matches!(log[0], LogEntry::Frame(_)), "no spurious gap around a lone frame");
|
||||
}
|
||||
|
||||
/// Resource safety (spec §7.4/§7.5) — surfacing a gap costs O(stored frames),
|
||||
/// never O(gap size). A long consumer outage, or a single wild position from
|
||||
/// a corrupt best-effort datagram, can leave the store bracketing an enormous
|
||||
/// interior gap. Surfacing it must be one span computed from the bracketing
|
||||
/// frames, not an enumeration of the missing range (which would hang / OOM
|
||||
/// the single consumer). The reference oracle agrees on the same cheap path.
|
||||
#[test]
|
||||
fn gap_surfacing_is_bounded_by_frame_count_not_gap_size() {
|
||||
// The largest interior gap a u64 position space admits.
|
||||
let mut extreme = StoredStream::new();
|
||||
extreme.record(support::typed_frame(&payloads::resource(0), 0));
|
||||
let high = Frame::new(ResourceSample::channel(), Position(u64::MAX), payloads::resource(1).encode());
|
||||
extreme.record(high.clone());
|
||||
|
||||
// Were this O(gap size), the next line would never return.
|
||||
assert_eq!(
|
||||
extreme.gap_spans(),
|
||||
vec![GapSpan { start: 1, end: u64::MAX - 1 }],
|
||||
"one span covers the whole interior gap"
|
||||
);
|
||||
assert_eq!(
|
||||
reference::gap_spans(&extreme.to_vec()),
|
||||
extreme.gap_spans(),
|
||||
"oracle agrees on the cheap path"
|
||||
);
|
||||
|
||||
// The merged log over the same store is also O(frames): two frames with
|
||||
// a single gap span between them, not a billion entries.
|
||||
let log = views::merged_log(&extreme);
|
||||
assert_eq!(log.len(), 3, "two frames and one gap span");
|
||||
assert!(matches!(log[1], LogEntry::Gap(_)), "the gap sits between the bracketing frames");
|
||||
|
||||
// A realistic long outage (millions of dropped positions) is just as cheap.
|
||||
let mut outage = StoredStream::new();
|
||||
outage.record(support::typed_frame(&payloads::resource(0), 0));
|
||||
for p in 5_000_000u64..5_000_003 {
|
||||
outage.record(support::typed_frame(&payloads::resource(p), p));
|
||||
}
|
||||
assert_eq!(
|
||||
outage.gap_spans(),
|
||||
vec![GapSpan { start: 1, end: 4_999_999 }],
|
||||
"the outage is one surfaced span, the stream resumes after it"
|
||||
);
|
||||
}
|
||||
|
||||
// ── Kind III — the full mock (testing spec §7) ─────────────────────────
|
||||
//
|
||||
// Each test assembles the ENTIRE pipe — producers → real mux → scripted
|
||||
// transport seam → real ingest → store → merged-log view — and drives it
|
||||
// with one verified vector, checking the view (the consumer-facing end)
|
||||
// against the reference oracle. The oracle's `expected` is derived from the
|
||||
// scenario (emitted frames + the script's drops), never from the run, and
|
||||
// is blind to delivery order — which is precisely the property the pipe
|
||||
// must satisfy.
|
||||
|
||||
/// Adversarial vector — reordering as deep as the envelope allows. Delivered
|
||||
/// fully reversed, the assembled pipe still reconstructs the exact timeline:
|
||||
/// reconstruction does not assume arrival order (spec §7.5).
|
||||
#[test]
|
||||
fn full_mock_survives_deepest_reorder() {
|
||||
let id = StreamId::new(NodeId::new("node-mock-1"), Lifetime(1));
|
||||
let sent = busy_node(&id);
|
||||
|
||||
let script = StreamScript::perfect().with_reorder(Reorder::Reversed);
|
||||
let delivered = ScriptedTransport::carry(&id, &sent, &script);
|
||||
|
||||
let mut consumer = Consumer::new();
|
||||
consumer.ingest(delivered);
|
||||
let stored = consumer.store().stream(&id).unwrap();
|
||||
let log = views::merged_log(stored);
|
||||
|
||||
assert_eq!(
|
||||
structure(&log),
|
||||
reference::merged_log(&surviving(&sent, &[])),
|
||||
"merged log equals the oracle despite the deepest reorder"
|
||||
);
|
||||
assert!(stored.gap_spans().is_empty(), "nothing dropped — no gaps");
|
||||
assert_eq!(stored.to_vec(), sent, "full timeline restored from fully reversed arrival");
|
||||
}
|
||||
|
||||
/// Adversarial vector — total loss of a span. A whole contiguous run of
|
||||
/// positions never arrives (and survivors are reordered too); the view
|
||||
/// surfaces one gap and the stream resumes after it, without replaying lost
|
||||
/// history (spec §7.4, §7.5).
|
||||
#[test]
|
||||
fn full_mock_surfaces_total_span_loss_and_resumes() {
|
||||
let id = StreamId::new(NodeId::new("node-mock-2"), Lifetime(1));
|
||||
let sent = busy_node(&id); // positions 0..10
|
||||
let lost = [4u64, 5, 6, 7]; // an entire span vanishes
|
||||
|
||||
let script = StreamScript::dropping(lost).with_reorder(Reorder::Windows(3));
|
||||
let delivered = ScriptedTransport::carry(&id, &sent, &script);
|
||||
|
||||
let mut consumer = Consumer::new();
|
||||
consumer.ingest(delivered);
|
||||
let stored = consumer.store().stream(&id).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
structure(&views::merged_log(stored)),
|
||||
reference::merged_log(&surviving(&sent, &lost)),
|
||||
"view equals the oracle with the span surfaced as a gap"
|
||||
);
|
||||
assert_eq!(
|
||||
stored.gap_spans(),
|
||||
vec![GapSpan { start: 4, end: 7 }],
|
||||
"the lost span is one surfaced gap, not silent concatenation"
|
||||
);
|
||||
|
||||
// The stream resumes with the original post-gap frames — nothing from
|
||||
// the lost span reappears, and history is not replayed.
|
||||
assert!(stored.frames().all(|f| !lost.contains(&f.position.0)), "lost span absent");
|
||||
let after: Vec<Frame> = stored.frames().filter(|f| f.position.0 >= 8).cloned().collect();
|
||||
let original_after: Vec<Frame> = sent.iter().filter(|f| f.position.0 >= 8).cloned().collect();
|
||||
assert_eq!(after, original_after, "resumes at position 8 with the originals, in order");
|
||||
}
|
||||
|
||||
/// Adversarial vector — a channel the consumer cannot decode. Its bytes are
|
||||
/// a perfectly valid record, but on a channel the catalog does not know:
|
||||
/// the pipe stores it whole, a view degrades it to raw bytes now, and it
|
||||
/// decodes later once the channel is learned (spec §6.3, §8.3).
|
||||
#[test]
|
||||
fn full_mock_stores_undecodable_channel_and_decodes_it_later() {
|
||||
let id = StreamId::new(NodeId::new("node-mock-3"), Lifetime(1));
|
||||
let node = Node::new(id.clone());
|
||||
node.emit(&payloads::identity("node-mock-3", 1)); // 0
|
||||
// A future host-metric channel this consumer has never heard of, whose
|
||||
// payload happens to be a valid resource sample.
|
||||
let future = payloads::resource(3);
|
||||
let future_channel = "v2.future.host_metric";
|
||||
node.emit_opaque(future_channel, &future.encode()); // 1
|
||||
node.emit(&payloads::runtime(1)); // 2
|
||||
let sent = node.sent();
|
||||
|
||||
let delivered = ScriptedTransport::carry(&id, &sent, &StreamScript::perfect());
|
||||
let mut consumer = Consumer::new();
|
||||
consumer.ingest(delivered);
|
||||
let stored = consumer.store().stream(&id).unwrap();
|
||||
|
||||
// Now: the view cannot decode it, so it degrades to raw bytes (§9.3),
|
||||
// but the frame is present and whole.
|
||||
let log = views::merged_log(stored);
|
||||
let entry = log
|
||||
.iter()
|
||||
.find_map(|e| match e {
|
||||
LogEntry::Frame(mf) if mf.position == Position(1) => Some(mf),
|
||||
_ => None,
|
||||
})
|
||||
.expect("the unknown-channel frame is on the timeline");
|
||||
assert!(matches!(entry.body, Body::Raw(_)), "unknown channel degrades to raw bytes now");
|
||||
|
||||
let raw = stored.at(Position(1)).unwrap();
|
||||
assert_eq!(raw.channel, ChannelId::new(future_channel), "retained on its own channel");
|
||||
|
||||
// Later: once the channel is learned, the stored bytes decode to the
|
||||
// original record — nothing was lost at ingest.
|
||||
let decoded = ResourceSample::decode(&raw.payload).expect("decodes once the channel is known");
|
||||
assert_eq!(decoded, future, "the opaque bytes were the record all along");
|
||||
}
|
||||
|
||||
/// Adversarial vector — a node identity reused across lifetimes. Two lives
|
||||
/// of one node flow into the one consumer (one delivered reversed); at the
|
||||
/// view end they are two separate timelines that never merge (spec §8.4).
|
||||
#[test]
|
||||
fn full_mock_reused_identity_does_not_merge_at_the_view() {
|
||||
let node = "node-recycled";
|
||||
let life1 = StreamId::new(NodeId::new(node), Lifetime(1));
|
||||
let life2 = StreamId::new(NodeId::new(node), Lifetime(2));
|
||||
|
||||
let s1 = busy_node(&life1);
|
||||
let s2 = busy_node(&life2);
|
||||
|
||||
let delivered = ScriptedTransport::carry_all(&[
|
||||
(life1.clone(), s1.clone(), StreamScript::perfect()),
|
||||
(life2.clone(), s2.clone(), StreamScript::perfect().with_reorder(Reorder::Reversed)),
|
||||
]);
|
||||
|
||||
let mut consumer = Consumer::new();
|
||||
consumer.ingest(delivered);
|
||||
let store = consumer.store();
|
||||
assert_eq!(store.len(), 2, "two lives, two streams — never merged");
|
||||
|
||||
let log1 = views::merged_log(store.stream(&life1).unwrap());
|
||||
let log2 = views::merged_log(store.stream(&life2).unwrap());
|
||||
assert_eq!(structure(&log1), reference::merged_log(&surviving(&s1, &[])));
|
||||
assert_eq!(structure(&log2), reference::merged_log(&surviving(&s2, &[])));
|
||||
|
||||
// Each life's position-0 identity frame is its own, correctly attributed.
|
||||
let id1 = IdentityRecord::decode(&store.stream(&life1).unwrap().at(Position(0)).unwrap().payload)
|
||||
.unwrap();
|
||||
let id2 = IdentityRecord::decode(&store.stream(&life2).unwrap().at(Position(0)).unwrap().payload)
|
||||
.unwrap();
|
||||
assert_eq!((id1.life, id2.life), (1, 2), "identities attribute to their own lifetimes");
|
||||
}
|
||||
|
||||
/// Envelope coverage (testing spec §9) — an explicit arrival permutation,
|
||||
/// distinct from full reversal: the carrier delivers evens then odds. The
|
||||
/// carrier applies exactly that permutation (no fabrication or loss), and
|
||||
/// the pipe reconstructs the send order regardless.
|
||||
#[test]
|
||||
fn full_mock_reconstructs_under_explicit_permutation() {
|
||||
let id = StreamId::new(NodeId::new("node-perm"), Lifetime(1));
|
||||
let sent = busy_node(&id);
|
||||
let n = sent.len();
|
||||
|
||||
// Deliver all even indices first, then all odd ones.
|
||||
let perm: Vec<usize> =
|
||||
(0..n).filter(|i| i.is_multiple_of(2)).chain((0..n).filter(|i| !i.is_multiple_of(2))).collect();
|
||||
let script = StreamScript::perfect().with_reorder(Reorder::Permutation(perm.clone()));
|
||||
let delivered = ScriptedTransport::carry(&id, &sent, &script);
|
||||
|
||||
// The carrier delivered exactly the scripted permutation of positions.
|
||||
let arrival: Vec<u64> = delivered.iter().map(|d| d.frame.position.0).collect();
|
||||
let expected_arrival: Vec<u64> = perm.iter().map(|&i| sent[i].position.0).collect();
|
||||
assert_eq!(arrival, expected_arrival, "delivered in the scripted permutation, nothing added or lost");
|
||||
|
||||
let mut consumer = Consumer::new();
|
||||
consumer.ingest(delivered);
|
||||
let stored = consumer.store().stream(&id).unwrap();
|
||||
assert_eq!(stored.to_vec(), sent, "reconstruction restores send order from the permutation");
|
||||
assert_eq!(
|
||||
structure(&views::merged_log(stored)),
|
||||
reference::merged_log(&surviving(&sent, &[])),
|
||||
"merged log equals the oracle"
|
||||
);
|
||||
}
|
||||
|
||||
// ── Kind IV — the deployment simulation (testing spec §8) ──────────────
|
||||
|
||||
/// Record the virtual tick at which a position was emitted (positions are
|
||||
/// contiguous from 0, so the tick vector is indexed by position).
|
||||
fn at_tick(ticks: &mut Vec<u64>, tick: u64, pos: Position) {
|
||||
assert_eq!(pos.0 as usize, ticks.len(), "positions emitted contiguously");
|
||||
ticks.push(tick);
|
||||
}
|
||||
|
||||
/// The positions a node emitted during an outage window — the frames lost
|
||||
/// while the consumer was absent (spec §7.4).
|
||||
fn emitted_during(tick_of: &[u64], outage_ticks: &[u64]) -> Vec<u64> {
|
||||
tick_of
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, t)| outage_ticks.contains(t))
|
||||
.map(|(pos, _)| pos as u64)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A scenario shaped like a real run (testing spec §8): three nodes boot and
|
||||
/// emit on real channels at real cadence over a virtual clock; the carrier
|
||||
/// drops one frame; the consumer is absent for a span; one node dies; one
|
||||
/// finalizes. Every fault is in the §9 envelope. The end-to-end answer is
|
||||
/// the full merged-log view over the consumer's store, **derived from the
|
||||
/// scenario by the reference model** — so the expected log already carries
|
||||
/// the right gaps, the dead node already ends at its last delivered frame,
|
||||
/// and the outage span is already absent. The simulation passes only if the
|
||||
/// assembled pipe reproduces every node's log exactly.
|
||||
#[test]
|
||||
fn kind_iv_deployment_simulation() {
|
||||
const OUTAGE: [u64; 3] = [4, 5, 6]; // ticks the consumer is absent
|
||||
|
||||
// ── Node A: lives the whole run and finalizes at t8 ────────────────
|
||||
let id_a = StreamId::new(NodeId::new("node-a"), Lifetime(1));
|
||||
let a = Node::new(id_a.clone());
|
||||
let mut a_ticks = Vec::new();
|
||||
at_tick(&mut a_ticks, 0, a.emit(&payloads::identity("node-a", 1)));
|
||||
for t in 1..=5 {
|
||||
at_tick(&mut a_ticks, t, a.emit(&payloads::resource(t)));
|
||||
}
|
||||
at_tick(&mut a_ticks, 7, a.emit(&payloads::resource(7))); // resumes after the outage
|
||||
at_tick(&mut a_ticks, 8, a.emit(&payloads::lifecycle("finalized", 7200))); // finalize
|
||||
let sent_a = a.sent();
|
||||
|
||||
// ── Node B: lives, and also emits membership transitions ───────────
|
||||
let id_b = StreamId::new(NodeId::new("node-b"), Lifetime(1));
|
||||
let b = Node::new(id_b.clone());
|
||||
let mut b_ticks = Vec::new();
|
||||
at_tick(&mut b_ticks, 0, b.emit(&payloads::identity("node-b", 1)));
|
||||
at_tick(&mut b_ticks, 1, b.emit(&payloads::resource(1)));
|
||||
at_tick(&mut b_ticks, 2, b.emit(&payloads::resource(2)));
|
||||
at_tick(&mut b_ticks, 2, b.emit(&payloads::membership("node-c", "alive", "suspect")));
|
||||
at_tick(&mut b_ticks, 3, b.emit(&payloads::resource(3)));
|
||||
at_tick(&mut b_ticks, 4, b.emit(&payloads::resource(4)));
|
||||
at_tick(&mut b_ticks, 4, b.emit(&payloads::membership("node-c", "suspect", "dead")));
|
||||
at_tick(&mut b_ticks, 5, b.emit(&payloads::resource(5)));
|
||||
at_tick(&mut b_ticks, 7, b.emit(&payloads::resource(7)));
|
||||
at_tick(&mut b_ticks, 8, b.emit(&payloads::runtime(8)));
|
||||
let sent_b = b.sent();
|
||||
|
||||
// ── Node C: dies at t6 (emits nothing after t5) ────────────────────
|
||||
let id_c = StreamId::new(NodeId::new("node-c"), Lifetime(1));
|
||||
let c = Node::new(id_c.clone());
|
||||
let mut c_ticks = Vec::new();
|
||||
at_tick(&mut c_ticks, 0, c.emit(&payloads::identity("node-c", 1)));
|
||||
for t in 1..=5 {
|
||||
at_tick(&mut c_ticks, t, c.emit(&payloads::resource(t)));
|
||||
}
|
||||
// t6: dies. Nothing more is emitted.
|
||||
let sent_c = c.sent();
|
||||
|
||||
// ── Faults, all inside the §9 envelope ─────────────────────────────
|
||||
// The carrier drops A's t3 resource frame (a known single drop).
|
||||
let mut drop_a = emitted_during(&a_ticks, &OUTAGE);
|
||||
drop_a.push(3);
|
||||
let drop_b = emitted_during(&b_ticks, &OUTAGE);
|
||||
let drop_c = emitted_during(&c_ticks, &OUTAGE);
|
||||
|
||||
// Each node's surviving frames may also arrive reordered, at varying
|
||||
// depth — reconstruction must not care.
|
||||
let delivered = ScriptedTransport::carry_all(&[
|
||||
(id_a.clone(), sent_a.clone(), StreamScript::dropping(drop_a.clone())),
|
||||
(
|
||||
id_b.clone(),
|
||||
sent_b.clone(),
|
||||
StreamScript::dropping(drop_b.clone()).with_reorder(Reorder::Windows(3)),
|
||||
),
|
||||
(
|
||||
id_c.clone(),
|
||||
sent_c.clone(),
|
||||
StreamScript::dropping(drop_c.clone()).with_reorder(Reorder::Reversed),
|
||||
),
|
||||
]);
|
||||
|
||||
let mut consumer = Consumer::new();
|
||||
consumer.ingest(delivered);
|
||||
let store = consumer.store();
|
||||
|
||||
// ── The end-to-end answer: each node's full merged-log view equals the
|
||||
// reference model's, derived from the scenario alone. ─────────────
|
||||
assert_eq!(store.len(), 3, "three independent streams; positions not comparable across them");
|
||||
for (id, sent, dropped) in [
|
||||
(&id_a, &sent_a, &drop_a),
|
||||
(&id_b, &sent_b, &drop_b),
|
||||
(&id_c, &sent_c, &drop_c),
|
||||
] {
|
||||
let log = views::merged_log(store.stream(id).unwrap());
|
||||
assert_eq!(
|
||||
structure(&log),
|
||||
reference::merged_log(&surviving(sent, dropped)),
|
||||
"merged log for {id} matches the oracle derived from the scenario"
|
||||
);
|
||||
}
|
||||
|
||||
// ── Targeted reads of the scenario's signature properties ──────────
|
||||
let a_stored = store.stream(&id_a).unwrap();
|
||||
let b_stored = store.stream(&id_b).unwrap();
|
||||
let c_stored = store.stream(&id_c).unwrap();
|
||||
|
||||
// A: the carrier drop (t3) and the outage (t4–5) coalesce into one
|
||||
// surfaced gap, then the stream resumes — without replaying history.
|
||||
assert_eq!(a_stored.gap_spans(), vec![GapSpan { start: 3, end: 5 }], "A: one surfaced gap");
|
||||
let finalize = a_stored.frames().last().unwrap();
|
||||
assert_eq!(
|
||||
LifecycleCost::decode(&finalize.payload).unwrap().phase,
|
||||
"finalized",
|
||||
"A's run ends with the finalize frame, delivered after the outage"
|
||||
);
|
||||
|
||||
// B: its outage span (t4–5 emissions) is one surfaced gap; it resumes.
|
||||
assert_eq!(b_stored.gap_spans().len(), 1, "B: a single outage gap");
|
||||
assert!(b_stored.frames().count() > 5, "B resumed and kept producing after the outage");
|
||||
|
||||
// C: the dead node's stream ends at its last delivered frame — its
|
||||
// outage-lost t4–5 frames are trailing loss (truncation), NOT a gap.
|
||||
assert!(c_stored.gap_spans().is_empty(), "C: dead node truncates, no trailing gap");
|
||||
assert_eq!(
|
||||
c_stored.frames().last().unwrap().position,
|
||||
Position(3),
|
||||
"C ends at its last delivered position (t3)"
|
||||
);
|
||||
}
|
||||
152
crates/distribution/tests/t_datastream_realio.rs
Normal file
152
crates/distribution/tests/t_datastream_realio.rs
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
//! The two real-I/O checks that license trusting the offline work
|
||||
//! (`DATASTREAM_TESTING_SPEC.md` §9–§10).
|
||||
//!
|
||||
//! Every offline test in `t_datastream.rs` replaces the transport with a
|
||||
//! script. That substitution is honest only if the *real* transport never
|
||||
//! does anything the script cannot express. These two tests are the only
|
||||
//! ones that pay the cost of real I/O, and they are what convert "we
|
||||
//! reasoned about the transport" into "we verified it":
|
||||
//!
|
||||
//! * the **envelope-conformance check** (§9) sends a stream over a real
|
||||
//! local socket and asserts the carrier never leaves the envelope —
|
||||
//! whatever it delivers is a reordered subsequence of what was sent,
|
||||
//! payloads byte-identical, positions intact. It asserts *no* delivery,
|
||||
//! so loss is allowed and it cannot be flaky;
|
||||
//! * the **wiring smoke** (§10) stands a node and a consumer up over the
|
||||
//! real socket and requires that *some* frames arrive and reconstruct —
|
||||
//! proving the path is actually connected, not that it is complete.
|
||||
//!
|
||||
//! The "real transport" here is a loopback UDP datagram socket: real OS I/O,
|
||||
//! best-effort like the production carrier, carrying the very same
|
||||
//! [`encode_delivery`] envelope a deployed transport would. The production
|
||||
//! code under test is real throughout — the mux, the wire envelope, ingest,
|
||||
//! the store, and reconstruction; only the carrier is a local stand-in for
|
||||
//! the deployed one.
|
||||
|
||||
#[path = "datastream_support/mod.rs"]
|
||||
mod support;
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::net::UdpSocket;
|
||||
use std::time::Duration;
|
||||
|
||||
use distribution::datastream::catalog::ProcStream;
|
||||
use distribution::datastream::frame::{Frame, Lifetime, NodeId, StreamId};
|
||||
use distribution::datastream::ingest::Consumer;
|
||||
use distribution::datastream::transport::Delivery;
|
||||
use distribution::datastream::wire::{decode_delivery, encode_delivery};
|
||||
|
||||
use support::{payloads, Node};
|
||||
|
||||
/// A realistic node stream, produced by the real mux.
|
||||
fn build_stream() -> (StreamId, Vec<Frame>) {
|
||||
let id = StreamId::new(NodeId::new("node-real"), Lifetime(1));
|
||||
let node = Node::new(id.clone());
|
||||
node.emit(&payloads::identity("node-real", 1));
|
||||
for tick in 0..20 {
|
||||
node.emit(&payloads::resource(tick));
|
||||
}
|
||||
node.emit_text("trainer", ProcStream::Stdout, "epoch 1 complete");
|
||||
node.emit(&payloads::membership("node-x", "alive", "suspect"));
|
||||
(id, node.sent())
|
||||
}
|
||||
|
||||
/// Carry a node's frames to a consumer over a real loopback UDP socket,
|
||||
/// each frame as one [`encode_delivery`] datagram, and return what the
|
||||
/// consumer actually received. Best-effort: send and receive errors are
|
||||
/// treated as loss, never as failures.
|
||||
fn carry_over_real_socket(stream: &StreamId, frames: &[Frame]) -> Vec<Delivery> {
|
||||
let consumer = UdpSocket::bind("127.0.0.1:0").expect("bind consumer socket");
|
||||
consumer.set_read_timeout(Some(Duration::from_millis(300))).expect("set timeout");
|
||||
let consumer_addr = consumer.local_addr().expect("consumer addr");
|
||||
|
||||
let node = UdpSocket::bind("127.0.0.1:0").expect("bind node socket");
|
||||
for frame in frames {
|
||||
let datagram = encode_delivery(stream, frame);
|
||||
// A send failure (e.g. a full socket buffer) is just loss.
|
||||
let _ = node.send_to(&datagram, consumer_addr);
|
||||
}
|
||||
|
||||
// Drain whatever is waiting; stop on the first read timeout.
|
||||
let mut delivered = Vec::new();
|
||||
let mut buf = vec![0u8; 64 * 1024];
|
||||
loop {
|
||||
match consumer.recv_from(&mut buf) {
|
||||
Ok((n, _)) => {
|
||||
if let Ok((s, frame)) = decode_delivery(&buf[..n]) {
|
||||
delivered.push(Delivery::new(s, frame));
|
||||
}
|
||||
}
|
||||
Err(ref e)
|
||||
if e.kind() == std::io::ErrorKind::WouldBlock
|
||||
|| e.kind() == std::io::ErrorKind::TimedOut =>
|
||||
{
|
||||
break;
|
||||
}
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
delivered
|
||||
}
|
||||
|
||||
/// §9 conformance — the real transport never leaves the envelope: whatever
|
||||
/// it delivers is a reordered subsequence of what was sent, byte-identical,
|
||||
/// positions intact. Asserts no specific delivery, so loss is tolerated and
|
||||
/// the test is not flaky.
|
||||
#[test]
|
||||
fn real_transport_stays_within_the_envelope() {
|
||||
let (id, sent) = build_stream();
|
||||
let delivered = carry_over_real_socket(&id, &sent);
|
||||
|
||||
let by_position: HashMap<u64, &Frame> = sent.iter().map(|f| (f.position.0, f)).collect();
|
||||
let mut seen = HashSet::new();
|
||||
for d in &delivered {
|
||||
assert_eq!(d.stream, id, "the carrier did not alter the stream id");
|
||||
let original = by_position
|
||||
.get(&d.frame.position.0)
|
||||
.expect("a delivered position was never sent — the carrier fabricated a frame");
|
||||
assert_eq!(
|
||||
&d.frame, *original,
|
||||
"payload byte-identical, channel and position intact — no corruption or alteration"
|
||||
);
|
||||
assert!(seen.insert(d.frame.position.0), "no duplicate — a subsequence has no repeats");
|
||||
}
|
||||
// Deliberately no assertion on how many arrived: loss is within the
|
||||
// envelope, so the check is sound without requiring delivery.
|
||||
}
|
||||
|
||||
/// §10 wiring smoke — telemetry is actually plugged in: a node's stream,
|
||||
/// produced by the real mux and carried over a real socket, reaches a real
|
||||
/// consumer and reconstructs. Loose by design (best-effort): it requires
|
||||
/// *some* frames to arrive, not all, and tolerates loss and reorder.
|
||||
#[test]
|
||||
fn wiring_smoke_some_frames_arrive_and_reconstruct() {
|
||||
let (id, sent) = build_stream();
|
||||
|
||||
// node (real mux output) → real socket → real ingest → store.
|
||||
let delivered = carry_over_real_socket(&id, &sent);
|
||||
let mut consumer = Consumer::new();
|
||||
consumer.ingest(delivered);
|
||||
|
||||
let stored = consumer
|
||||
.store()
|
||||
.stream(&id)
|
||||
.expect("the path is connected: the node's frames reached the consumer");
|
||||
assert!(!stored.is_empty(), "some frames arrived over the real transport");
|
||||
|
||||
// Whatever arrived reconstructs correctly: each stored frame is the
|
||||
// original at that position, and the store is in position order.
|
||||
let by_position: HashMap<u64, &Frame> = sent.iter().map(|f| (f.position.0, f)).collect();
|
||||
let mut prev: Option<u64> = None;
|
||||
for frame in stored.frames() {
|
||||
assert_eq!(
|
||||
frame,
|
||||
*by_position.get(&frame.position.0).expect("only sent frames arrive"),
|
||||
"a reconstructed frame is the original, byte-identical"
|
||||
);
|
||||
if let Some(p) = prev {
|
||||
assert!(frame.position.0 > p, "reconstructed in position order");
|
||||
}
|
||||
prev = Some(frame.position.0);
|
||||
}
|
||||
}
|
||||
|
|
@ -29,3 +29,7 @@ relay = ["iroh", "distribution/relay"]
|
|||
[[bin]]
|
||||
name = "swactor"
|
||||
path = "src/main.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "swactor-datastream-collector"
|
||||
path = "src/bin/swactor-datastream-collector.rs"
|
||||
|
|
|
|||
78
crates/node/src/bin/swactor-datastream-collector.rs
Normal file
78
crates/node/src/bin/swactor-datastream-collector.rs
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
//! The datastream consumer, standalone: bind a UDP socket, decode every
|
||||
//! delivery that arrives, and print each frame to stdout the instant it lands.
|
||||
//!
|
||||
//! This is the "raw stream to stdout" end of the per-node telemetry datastream
|
||||
//! (see `distribution/DATASTREAM_SPEC.md`). It is deliberately the dumbest
|
||||
//! possible consumer: one datagram carries one frame
|
||||
//! ([`encode_delivery`](distribution::datastream::wire::encode_delivery)), so
|
||||
//! we decode and print in arrival order — loss and reorder show up as they
|
||||
//! happen on the wire, which is exactly what you want when watching a live
|
||||
//! cluster. No store, no views, no dashboard.
|
||||
//!
|
||||
//! Usage: `swactor-datastream-collector [--bind HOST:PORT]`
|
||||
//! (defaults to `$SWACTOR_DATASTREAM_BIND` or `0.0.0.0:7700`).
|
||||
|
||||
use std::io::Write;
|
||||
use std::net::UdpSocket;
|
||||
|
||||
use distribution::datastream::views::decode_body;
|
||||
use distribution::datastream::wire::decode_delivery;
|
||||
|
||||
fn main() {
|
||||
let bind = resolve_bind();
|
||||
let sock = match UdpSocket::bind(&bind) {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
eprintln!("datastream collector: failed to bind {bind}: {e}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
eprintln!("datastream collector listening on {bind} (one frame per datagram)");
|
||||
|
||||
// 64 KiB comfortably exceeds a UDP datagram; a frame never spans datagrams.
|
||||
let mut buf = vec![0u8; 64 * 1024];
|
||||
let stdout = std::io::stdout();
|
||||
loop {
|
||||
match sock.recv_from(&mut buf) {
|
||||
Ok((n, _src)) => match decode_delivery(&buf[..n]) {
|
||||
Ok((stream, frame)) => {
|
||||
let body = decode_body(&frame.channel, &frame.payload);
|
||||
let node = stream.node.as_str();
|
||||
let short = &node[..node.len().min(8)];
|
||||
let mut out = stdout.lock();
|
||||
// arrival order — no grouping, no buffering.
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"{short}#{life} #{pos:<5} [{chan}] {body}",
|
||||
life = stream.life.0,
|
||||
pos = frame.position.0,
|
||||
chan = frame.channel,
|
||||
);
|
||||
let _ = out.flush();
|
||||
}
|
||||
Err(e) => eprintln!("datastream collector: dropped malformed datagram ({n} B): {e:?}"),
|
||||
},
|
||||
Err(e) => eprintln!("datastream collector: recv error: {e}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve the bind address: `--bind HOST:PORT`, else `$SWACTOR_DATASTREAM_BIND`,
|
||||
/// else `0.0.0.0:7700`.
|
||||
fn resolve_bind() -> String {
|
||||
let mut args = std::env::args().skip(1);
|
||||
while let Some(arg) = args.next() {
|
||||
match arg.as_str() {
|
||||
"--bind" => {
|
||||
if let Some(v) = args.next() {
|
||||
return v;
|
||||
}
|
||||
}
|
||||
other if other.starts_with("--bind=") => {
|
||||
return other["--bind=".len()..].to_string();
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
std::env::var("SWACTOR_DATASTREAM_BIND").unwrap_or_else(|_| "0.0.0.0:7700".to_string())
|
||||
}
|
||||
|
|
@ -22,6 +22,14 @@ use distribution::peer_auth::PeerAllowList;
|
|||
use distribution::snapshot::DistributionNodeSnapshot;
|
||||
use distribution::swim::probe::SwimConfig;
|
||||
|
||||
use distribution::datastream::catalog::{
|
||||
self, ProcStream, Record, Role, RuntimeStats as DsRuntimeStats, TransportInternals,
|
||||
};
|
||||
use distribution::datastream::frame::{Lifetime, NodeId as DsNodeId, StreamId};
|
||||
use distribution::datastream::mux::Mux;
|
||||
use distribution::datastream::source::{self, CpuSampler, MembershipTracker};
|
||||
use distribution::datastream::wire::encode_delivery;
|
||||
|
||||
use dashboard::collector::StatsCollector;
|
||||
use dashboard::{start_dashboard, DashboardConfig};
|
||||
|
||||
|
|
@ -57,6 +65,19 @@ enum Subcmd {
|
|||
/// Invite code (base58-encoded node ID)
|
||||
code: String,
|
||||
},
|
||||
/// Internal: a dummy child workload that prints lines forever.
|
||||
///
|
||||
/// The node spawns itself with this subcommand as the "real process" each
|
||||
/// node drives; its stdout/stderr are captured into the datastream's
|
||||
/// `proc.<label>.{stdout,stderr}` channels. It runs until killed.
|
||||
DummyWorkload {
|
||||
/// Milliseconds between output lines
|
||||
#[arg(long, default_value = "1000")]
|
||||
interval_ms: u64,
|
||||
/// Label used to name the process-output channels
|
||||
#[arg(long, default_value = "workload")]
|
||||
label: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Parser)]
|
||||
|
|
@ -136,6 +157,40 @@ struct Args {
|
|||
/// Bind address for embedded relay server
|
||||
#[arg(long, default_value = "0.0.0.0")]
|
||||
relay_bind: String,
|
||||
|
||||
/// Relay host(s) for cluster discovery (repeatable). Overrides config.
|
||||
#[arg(long)]
|
||||
relay_hosts: Vec<String>,
|
||||
|
||||
/// Enable per-node telemetry datastream emission
|
||||
#[arg(long)]
|
||||
datastream: bool,
|
||||
|
||||
/// Ship datastream frames to this UDP collector (host:port). Implies --datastream.
|
||||
#[arg(long)]
|
||||
datastream_collector: Option<String>,
|
||||
|
||||
/// Region label reported in the datastream identity record
|
||||
#[arg(long, default_value = "local")]
|
||||
datastream_region: String,
|
||||
|
||||
/// Do not spawn the dummy child workload (whose output feeds proc.* channels)
|
||||
#[arg(long)]
|
||||
no_datastream_child: bool,
|
||||
|
||||
/// Label for the dummy child workload process
|
||||
#[arg(long, default_value = "workload")]
|
||||
datastream_child_label: String,
|
||||
}
|
||||
|
||||
/// Datastream emission configuration, resolved from flags/env in `main`.
|
||||
struct DatastreamOpts {
|
||||
enabled: bool,
|
||||
collector: Option<String>,
|
||||
region: String,
|
||||
child: bool,
|
||||
child_label: String,
|
||||
life: u64,
|
||||
}
|
||||
|
||||
// ── Dummy actor ──────────────────────────────────────────────────────────
|
||||
|
|
@ -260,6 +315,10 @@ fn main() {
|
|||
install::uninstall();
|
||||
return;
|
||||
}
|
||||
Subcmd::DummyWorkload { interval_ms, label } => {
|
||||
run_dummy_workload(*interval_ms, label);
|
||||
return;
|
||||
}
|
||||
Subcmd::Name { .. } | Subcmd::Invite | Subcmd::Join { .. } => {
|
||||
/* handled after config/identity is loaded */
|
||||
}
|
||||
|
|
@ -350,7 +409,31 @@ fn main() {
|
|||
} else {
|
||||
cfg.relay_bind.unwrap_or_else(|| args.relay_bind.clone())
|
||||
};
|
||||
let relay_hosts = cfg.relay_hosts.unwrap_or_default();
|
||||
let relay_hosts = if !args.relay_hosts.is_empty() {
|
||||
args.relay_hosts.clone()
|
||||
} else {
|
||||
cfg.relay_hosts.unwrap_or_default()
|
||||
};
|
||||
|
||||
// Datastream telemetry options (all inert unless --datastream / --datastream-collector)
|
||||
let datastream_enabled = args.datastream || args.datastream_collector.is_some();
|
||||
let ds_opts = DatastreamOpts {
|
||||
enabled: datastream_enabled,
|
||||
collector: args.datastream_collector.clone(),
|
||||
region: args.datastream_region.clone(),
|
||||
child: datastream_enabled && !args.no_datastream_child,
|
||||
child_label: args.datastream_child_label.clone(),
|
||||
life: std::env::var("SWACTOR_LIFETIME")
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or_else(|| {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or(0)
|
||||
}),
|
||||
};
|
||||
|
||||
// Signal handler — second Ctrl+C forces immediate exit
|
||||
{
|
||||
let stop = Arc::clone(&stop);
|
||||
|
|
@ -609,6 +692,8 @@ fn main() {
|
|||
&relay_bind,
|
||||
relay_port,
|
||||
relay_hosts,
|
||||
node_hex.clone(),
|
||||
ds_opts,
|
||||
);
|
||||
|
||||
#[cfg(not(feature = "iroh"))]
|
||||
|
|
@ -644,11 +729,16 @@ fn run_iroh(
|
|||
relay_bind: &str,
|
||||
relay_port: u16,
|
||||
relay_hosts: Vec<String>,
|
||||
node_hex: String,
|
||||
ds_opts: DatastreamOpts,
|
||||
) {
|
||||
use distribution::iroh_driver::{IrohDriver, IrohDriverConfig};
|
||||
use iroh::{RelayMode, SecretKey};
|
||||
use swactor::std::RuntimeNaming;
|
||||
|
||||
// A node that was given no seed to join is the cluster's coordinator/seed.
|
||||
let is_seed = seed_node_id.is_none();
|
||||
|
||||
// Evaluate relay candidacy and determine embedded relay bind address
|
||||
#[cfg(feature = "relay")]
|
||||
let (embedded_relay_bind, relay_public_ip) = if relay_enabled {
|
||||
|
|
@ -792,6 +882,20 @@ fn run_iroh(
|
|||
dash.start_http(driver.tokio_handle());
|
||||
eprintln!("Dashboard at http://0.0.0.0:{dashboard_port}");
|
||||
|
||||
// Datastream telemetry: create the per-node mux, emit identity, spawn the
|
||||
// dummy child workload, and prepare the UDP shipping socket.
|
||||
let mut ds_rt = if ds_opts.enabled {
|
||||
eprintln!(
|
||||
"Datastream: enabled (collector: {}, region: {}, child: {})",
|
||||
ds_opts.collector.as_deref().unwrap_or("<none>"),
|
||||
ds_opts.region,
|
||||
ds_opts.child,
|
||||
);
|
||||
Some(setup_datastream(&ds_opts, &node_hex, is_seed))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Main loop
|
||||
let mut round: u64 = 0;
|
||||
while !stop.load(Ordering::Relaxed) {
|
||||
|
|
@ -874,6 +978,62 @@ fn run_iroh(
|
|||
let mut snap = driver.snapshot();
|
||||
snap.node_name = Some(node_name.clone());
|
||||
|
||||
// Datastream emission: periodic samples (~1s), event-driven membership,
|
||||
// then drain and ship every frame to the collector (one per datagram).
|
||||
if let Some(ds) = ds_rt.as_mut() {
|
||||
if round % 10 == 0 {
|
||||
ds.mux.submit(
|
||||
catalog::HOST_RESOURCE,
|
||||
source::read_host_resource(&mut ds.cpu).encode(),
|
||||
);
|
||||
|
||||
let rs = handle.runtime.stats();
|
||||
let runtime_rec = DsRuntimeStats {
|
||||
actors_live: rs.actors.len() as u32,
|
||||
mailbox_depth: rs.workers.iter().map(|w| w.mailbox_depth as u32).sum(),
|
||||
scheduled_tasks: rs.workers.iter().map(|w| w.num_actors as u32).sum(),
|
||||
};
|
||||
ds.mux.submit(catalog::RUNTIME_STATS, runtime_rec.encode());
|
||||
|
||||
let transport = TransportInternals {
|
||||
relay_connected: driver.home_relay_url().is_some(),
|
||||
direct_peers: snap.members.iter().filter(|m| m.state == "alive").count() as u32,
|
||||
relay_peers: snap.members.iter().filter(|m| m.relay_url.is_some()).count() as u32,
|
||||
rtt_ms_p50: 0,
|
||||
};
|
||||
ds.mux.submit(catalog::TRANSPORT_INTERNALS, transport.encode());
|
||||
}
|
||||
|
||||
// Key the membership view by the stable node id, not the friendly
|
||||
// name: a peer's `node_name` resolves only after its metadata
|
||||
// arrives, so keying on it would emit the same peer twice (once by
|
||||
// id before the name is known, once by name after) and double-count
|
||||
// it downstream. The id never changes; the consumer maps it to a
|
||||
// friendly label for display.
|
||||
let members: Vec<(String, String)> = snap
|
||||
.members
|
||||
.iter()
|
||||
.map(|m| (m.node_id.clone(), m.state.clone()))
|
||||
.collect();
|
||||
for transition in ds.membership.diff(&members) {
|
||||
ds.mux.submit(catalog::MEMBERSHIP, transition.encode());
|
||||
}
|
||||
|
||||
// Drain unconditionally to bound the buffer; ship when we have a
|
||||
// resolved collector address (re-resolve lazily if it was down).
|
||||
let frames = ds.mux.drain();
|
||||
if let Some(sock) = ds.udp.as_ref() {
|
||||
if ds.collector_addr.is_none() && round % 10 == 0 {
|
||||
ds.collector_addr = ds.collector_spec.as_deref().and_then(resolve_addr);
|
||||
}
|
||||
if let Some(addr) = ds.collector_addr {
|
||||
for frame in &frames {
|
||||
let _ = sock.send_to(&encode_delivery(&ds.stream_id, frame), addr);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build rich invite code: <base58>#<addr1>,<addr2>@<relay_url>
|
||||
{
|
||||
let direct_addrs = driver.direct_addresses();
|
||||
|
|
@ -960,10 +1120,143 @@ fn run_iroh(
|
|||
thread::sleep(Duration::from_millis(100));
|
||||
}
|
||||
|
||||
// Stop the dummy child workload, if any.
|
||||
if let Some(ds) = ds_rt.as_mut() {
|
||||
if let Some(child) = ds.child.as_mut() {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
}
|
||||
}
|
||||
|
||||
// Driver shutdown handles embedded relay cleanup automatically
|
||||
driver.shutdown();
|
||||
}
|
||||
|
||||
// ── Datastream telemetry ───────────────────────────────────────────────────
|
||||
|
||||
/// Live state a node carries to emit and ship its telemetry datastream.
|
||||
struct DatastreamRuntime {
|
||||
stream_id: StreamId,
|
||||
mux: Arc<Mux>,
|
||||
udp: Option<std::net::UdpSocket>,
|
||||
collector_addr: Option<std::net::SocketAddr>,
|
||||
collector_spec: Option<String>,
|
||||
cpu: CpuSampler,
|
||||
membership: MembershipTracker,
|
||||
child: Option<std::process::Child>,
|
||||
}
|
||||
|
||||
/// Build the per-node datastream runtime: a mux keyed by this node's stream id,
|
||||
/// an immediate identity frame, the outgoing UDP socket, and the dummy child.
|
||||
fn setup_datastream(opts: &DatastreamOpts, node_hex: &str, is_seed: bool) -> DatastreamRuntime {
|
||||
let stream_id = StreamId::new(DsNodeId::new(node_hex), Lifetime(opts.life));
|
||||
let mux = Arc::new(Mux::new(stream_id.clone(), 4096));
|
||||
|
||||
// Identity is emitted first so the consumer can attribute the stream.
|
||||
let role = if is_seed { Role::Coordinator } else { Role::Worker };
|
||||
mux.submit(
|
||||
catalog::IDENTITY,
|
||||
source::identity_record(node_hex, role, &opts.region, opts.life).encode(),
|
||||
);
|
||||
|
||||
let udp = if opts.collector.is_some() {
|
||||
match std::net::UdpSocket::bind("0.0.0.0:0") {
|
||||
Ok(s) => Some(s),
|
||||
Err(e) => {
|
||||
eprintln!("Datastream: failed to bind UDP socket: {e}");
|
||||
None
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let collector_addr = opts.collector.as_deref().and_then(resolve_addr);
|
||||
|
||||
let child = if opts.child {
|
||||
spawn_workload_child(&mux, &opts.child_label)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
DatastreamRuntime {
|
||||
stream_id,
|
||||
mux,
|
||||
udp,
|
||||
collector_addr,
|
||||
collector_spec: opts.collector.clone(),
|
||||
cpu: CpuSampler::new(),
|
||||
membership: MembershipTracker::new(),
|
||||
child,
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawn the node binary as its own dummy child workload and pump the child's
|
||||
/// stdout/stderr into the mux as `proc.<label>.{stdout,stderr}` text frames.
|
||||
fn spawn_workload_child(mux: &Arc<Mux>, label: &str) -> Option<std::process::Child> {
|
||||
use std::io::BufRead;
|
||||
use std::process::{Command, Stdio};
|
||||
|
||||
let exe = std::env::current_exe().ok()?;
|
||||
let mut child = Command::new(exe)
|
||||
.arg("dummy-workload")
|
||||
.arg("--label")
|
||||
.arg(label)
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.map_err(|e| eprintln!("Datastream: failed to spawn child workload: {e}"))
|
||||
.ok()?;
|
||||
|
||||
if let Some(out) = child.stdout.take() {
|
||||
let mux = Arc::clone(mux);
|
||||
let channel = catalog::process_output(label, ProcStream::Stdout);
|
||||
thread::spawn(move || {
|
||||
for line in std::io::BufReader::new(out).lines().map_while(Result::ok) {
|
||||
mux.submit(channel.clone(), line.into_bytes());
|
||||
}
|
||||
});
|
||||
}
|
||||
if let Some(err) = child.stderr.take() {
|
||||
let mux = Arc::clone(mux);
|
||||
let channel = catalog::process_output(label, ProcStream::Stderr);
|
||||
thread::spawn(move || {
|
||||
for line in std::io::BufReader::new(err).lines().map_while(Result::ok) {
|
||||
mux.submit(channel.clone(), line.into_bytes());
|
||||
}
|
||||
});
|
||||
}
|
||||
Some(child)
|
||||
}
|
||||
|
||||
/// Resolve a `host:port` collector spec to a single socket address.
|
||||
fn resolve_addr(spec: &str) -> Option<std::net::SocketAddr> {
|
||||
use std::net::ToSocketAddrs;
|
||||
spec.to_socket_addrs().ok()?.next()
|
||||
}
|
||||
|
||||
/// The dummy child workload: print a line to stdout every `interval_ms`, and an
|
||||
/// occasional stderr line, until killed. This is the real OS process each node
|
||||
/// drives; its output is what flows on the datastream's `proc.*` channels.
|
||||
fn run_dummy_workload(interval_ms: u64, label: &str) {
|
||||
use std::io::Write;
|
||||
let interval = Duration::from_millis(interval_ms.max(1));
|
||||
let mut n: u64 = 0;
|
||||
loop {
|
||||
n += 1;
|
||||
{
|
||||
let mut out = std::io::stdout().lock();
|
||||
let _ = writeln!(out, "tick {n} — {label} working");
|
||||
let _ = out.flush();
|
||||
}
|
||||
if n % 5 == 0 {
|
||||
let mut err = std::io::stderr().lock();
|
||||
let _ = writeln!(err, "warn: {label} synthetic backpressure at tick {n}");
|
||||
let _ = err.flush();
|
||||
}
|
||||
thread::sleep(interval);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
fn spawn_actors(
|
||||
|
|
|
|||
|
|
@ -10,8 +10,9 @@ use dashboard::plugin::{DashboardPlugin, PluginResponse};
|
|||
use dashboard::JoinPeerInfo;
|
||||
use distribution::snapshot::DistributionNodeSnapshot;
|
||||
|
||||
/// HTML page for the distribution plugin.
|
||||
const DISTRIBUTION_HTML: &str = include_str!("distribution_page.html");
|
||||
/// HTML page for the distribution plugin. Owned by the `dashboard` crate so the
|
||||
/// live node and the datastream dashboard serve the identical page.
|
||||
const DISTRIBUTION_HTML: &str = dashboard::DISTRIBUTION_PAGE_HTML;
|
||||
|
||||
/// Dashboard plugin that exposes distribution node snapshots.
|
||||
pub struct DistributionPlugin {
|
||||
|
|
|
|||
96
tests/docker/DATASTREAM_DEMO.md
Normal file
96
tests/docker/DATASTREAM_DEMO.md
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
# Datastream demo cluster
|
||||
|
||||
Spins up a small cluster of real `swactor` nodes in Docker, each driving a dummy
|
||||
child process, all shipping their per-node telemetry **datastream** to a single
|
||||
collector that prints the raw stream — live, frame-by-frame — to stdout.
|
||||
|
||||
This is the "see the new metrics scheme working" demo. No dashboard; just the
|
||||
raw stream.
|
||||
|
||||
## One-liner
|
||||
|
||||
```sh
|
||||
./tests/docker/datastream-demo.sh
|
||||
```
|
||||
|
||||
Builds everything, brings the cluster up in the foreground, and streams the raw
|
||||
datastream to your terminal. Ctrl-C tears it down. (Everything below is the
|
||||
manual breakdown of what that script does.)
|
||||
|
||||
## What runs
|
||||
|
||||
| Service | Role |
|
||||
|-------------|-------------------------------------------------------------------|
|
||||
| `collector` | Binds UDP `:7700`, decodes each delivery, prints it on arrival |
|
||||
| `relay` | `swactor-iroh-relay` so nodes can find each other over the bridge |
|
||||
| `seed` | Coordinator node (fixed identity), drives a dummy workload |
|
||||
| `node-2/3` | Worker nodes that join the seed, each drives a dummy workload |
|
||||
|
||||
Each node emits `identity` once, then `host.resource` / `runtime.stats` /
|
||||
`transport.internals` every ~1s, `membership` transitions as peers come and go,
|
||||
and its dummy child's stdout/stderr as `proc.workload.{stdout,stderr}`.
|
||||
|
||||
## Run it
|
||||
|
||||
From the repo root (the image is multi-stage — Docker compiles the binaries
|
||||
itself, so you need nothing on the host but Docker):
|
||||
|
||||
```sh
|
||||
# 1. Build the images (first build compiles the workspace; later builds cache)
|
||||
# and bring the cluster up in the foreground so the collector's stream is
|
||||
# visible in the aggregated log.
|
||||
docker compose -f tests/docker/docker-compose.datastream.yml build
|
||||
docker compose -f tests/docker/docker-compose.datastream.yml up
|
||||
|
||||
# 2. Ctrl-C to stop, then clean up:
|
||||
docker compose -f tests/docker/docker-compose.datastream.yml down
|
||||
```
|
||||
|
||||
Watch the `collector-1 | ...` lines. Each is one frame:
|
||||
|
||||
```
|
||||
collector-1 | e8d77206#1780337699 #0 [identity] {"node":"e8d7...","region":"seed","role":"coordinator",...}
|
||||
collector-1 | e8d77206#1780337699 #1 [proc.workload.stdout] tick 1 — workload working
|
||||
collector-1 | e8d77206#1780337699 #2 [host.resource] {"cpu_pct":3.5,"mem_total_mb":31741,"mem_used_mb":4019,...}
|
||||
collector-1 | e8d77206#1780337699 #3 [runtime.stats] {"actors_live":7,"mailbox_depth":0,"scheduled_tasks":7}
|
||||
collector-1 | e8d77206#1780337699 #4 [transport.internals] {"direct_peers":2,"relay_connected":true,...}
|
||||
collector-1 | e98b5ff5#1780337699 #7 [membership] {"from":"unknown","peer":"...","to":"alive"}
|
||||
```
|
||||
|
||||
The prefix is `<node-id-prefix>#<lifetime>`; `#N` is the position within that
|
||||
node's stream (monotonic, gap-free at the source — gaps in the printed sequence
|
||||
mean datagrams were lost in transit, which is expected for a best-effort UDP
|
||||
carrier).
|
||||
|
||||
## Local (non-Docker) version
|
||||
|
||||
You don't need Docker to see the stream. Run a collector and a couple of nodes
|
||||
on localhost; they ship over loopback UDP:
|
||||
|
||||
```sh
|
||||
cargo build -p node --bin swactor --bin swactor-datastream-collector
|
||||
|
||||
./target/debug/swactor-datastream-collector --bind 127.0.0.1:7700 &
|
||||
|
||||
./target/debug/swactor --identity-dir /tmp/dsA --dashboard-port 9101 --no-relay \
|
||||
--datastream --datastream-collector 127.0.0.1:7700 --datastream-region A --actors 2 &
|
||||
./target/debug/swactor --identity-dir /tmp/dsB --dashboard-port 9102 --no-relay \
|
||||
--datastream --datastream-collector 127.0.0.1:7700 --datastream-region B --actors 3 &
|
||||
```
|
||||
|
||||
(Two standalone nodes with `--no-relay` and no seed won't discover each other,
|
||||
so `membership` stays quiet — that's expected. Every other channel streams.)
|
||||
|
||||
## Flags
|
||||
|
||||
| Flag | Meaning |
|
||||
|-------------------------------|----------------------------------------------------------|
|
||||
| `--datastream` | Enable telemetry emission (off by default) |
|
||||
| `--datastream-collector H:P` | UDP collector address; implies `--datastream` |
|
||||
| `--datastream-region R` | Region label in the `identity` record |
|
||||
| `--no-datastream-child` | Don't spawn the dummy child workload |
|
||||
| `--datastream-child-label L` | Name for the `proc.<L>.*` channels (default `workload`) |
|
||||
|
||||
The lifetime discriminator is taken from `$SWACTOR_LIFETIME` if set, else the
|
||||
current UNIX seconds — bump it across restarts so a re-incarnated node starts a
|
||||
fresh stream.
|
||||
37
tests/docker/Dockerfile.datastream
Normal file
37
tests/docker/Dockerfile.datastream
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
# Datastream demo image (multi-stage, self-contained).
|
||||
#
|
||||
# The production image (../../Dockerfile) is `FROM scratch` over a static musl
|
||||
# binary. This demo image instead compiles the binaries *inside* Docker and
|
||||
# runs them on a slim glibc base. Building in the same Debian release as the
|
||||
# runtime keeps glibc versions matched, so the image is portable regardless of
|
||||
# the host's glibc — and you don't need a Rust toolchain on the host at all.
|
||||
#
|
||||
# Build context is the repo root (see docker-compose.datastream.yml). A
|
||||
# sibling Dockerfile.datastream.dockerignore keeps the build context to the
|
||||
# source tree (excludes target/, .git/).
|
||||
FROM rust:1-bookworm AS builder
|
||||
WORKDIR /src
|
||||
COPY . .
|
||||
RUN cargo build --release -p node -p distribution \
|
||||
--bin swactor --bin swactor-datastream-collector --bin swactor-iroh-relay \
|
||||
&& cargo build --release -p dashboard --features datastream \
|
||||
--bin swactor-datastream-dashboard
|
||||
|
||||
FROM debian:bookworm-slim
|
||||
|
||||
# ca-certificates is handy for iroh's default relays; harmless otherwise.
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=builder /src/target/release/swactor /usr/local/bin/swactor
|
||||
COPY --from=builder /src/target/release/swactor-datastream-collector /usr/local/bin/swactor-datastream-collector
|
||||
COPY --from=builder /src/target/release/swactor-iroh-relay /usr/local/bin/swactor-iroh-relay
|
||||
COPY --from=builder /src/target/release/swactor-datastream-dashboard /usr/local/bin/swactor-datastream-dashboard
|
||||
|
||||
# Bake the demo config and the fixed seed identity into the image so the
|
||||
# cluster is fully self-contained — no bind mounts (which are fragile across
|
||||
# daemon configurations). The seed identity is a throwaway demo key.
|
||||
COPY tests/docker/datastream-node.toml /etc/swactor/node.toml
|
||||
COPY tests/docker/identities/seed /seed-identity
|
||||
|
||||
ENTRYPOINT ["swactor"]
|
||||
10
tests/docker/Dockerfile.datastream.dockerignore
Normal file
10
tests/docker/Dockerfile.datastream.dockerignore
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
# Build context for the multi-stage datastream demo image. Unlike the root
|
||||
# .dockerignore (which excludes everything for the prebuilt-binary scratch
|
||||
# image), this build compiles from source inside Docker, so it needs the source
|
||||
# tree — just not the heavy build artifacts or VCS metadata.
|
||||
target
|
||||
**/target
|
||||
.git
|
||||
identity
|
||||
auth
|
||||
*.log
|
||||
30
tests/docker/datastream-dashboard-demo.sh
Executable file
30
tests/docker/datastream-dashboard-demo.sh
Executable file
|
|
@ -0,0 +1,30 @@
|
|||
#!/usr/bin/env bash
|
||||
# Datastream demo, browser-dashboard edition: build, bring the cluster up, and
|
||||
# serve the live dashboard from the demuxed telemetry stream.
|
||||
#
|
||||
# ./tests/docker/datastream-dashboard-demo.sh
|
||||
#
|
||||
# Same cluster and data as ./datastream-demo.sh — relay, seed, two workers, each
|
||||
# shipping its telemetry datastream — but the UDP sink is the HTTP dashboard
|
||||
# (swactor-datastream-dashboard) instead of the stdout collector. It demuxes the
|
||||
# frames and renders one node in the existing browser UI.
|
||||
#
|
||||
# Open the dashboard once the cluster is up (port printed below; default 18080).
|
||||
# Ctrl-C tears it down. Set DASH_PORT=<port> if the default host port is taken.
|
||||
# (First build is slow — it compiles the workspace; later runs are cached.)
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/../.."
|
||||
|
||||
DASH_PORT="${DASH_PORT:-18080}"
|
||||
export DASH_PORT
|
||||
|
||||
COMPOSE="docker compose \
|
||||
-f tests/docker/docker-compose.datastream.yml \
|
||||
-f tests/docker/docker-compose.datastream.dashboard.yml"
|
||||
|
||||
$COMPOSE build
|
||||
trap '$COMPOSE down' EXIT INT TERM
|
||||
echo
|
||||
echo " Dashboard: http://localhost:${DASH_PORT}"
|
||||
echo
|
||||
$COMPOSE up
|
||||
18
tests/docker/datastream-demo.sh
Executable file
18
tests/docker/datastream-demo.sh
Executable file
|
|
@ -0,0 +1,18 @@
|
|||
#!/usr/bin/env bash
|
||||
# One-liner datastream demo: build, bring the cluster up, stream to stdout.
|
||||
#
|
||||
# ./tests/docker/datastream-demo.sh
|
||||
#
|
||||
# The image is multi-stage — Docker compiles the binaries inside the builder
|
||||
# stage, so you need nothing on the host but Docker. Brings the cluster up in
|
||||
# the foreground so the collector's live, frame-by-frame stream prints to your
|
||||
# terminal; Ctrl-C tears it back down. (First build is slow — it compiles the
|
||||
# workspace; later runs are cached.)
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/../.."
|
||||
|
||||
COMPOSE="docker compose -f tests/docker/docker-compose.datastream.yml"
|
||||
|
||||
$COMPOSE build
|
||||
trap '$COMPOSE down' EXIT INT TERM
|
||||
$COMPOSE up
|
||||
6
tests/docker/datastream-node.toml
Normal file
6
tests/docker/datastream-node.toml
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
# Minimal node config for the datastream demo cluster.
|
||||
# Keeps containers deterministic: no auth, no embedded relay, in-memory
|
||||
# datastore. Cluster discovery is driven by CLI flags in the compose file.
|
||||
dashboard_port = 9090
|
||||
relay = false
|
||||
auth = false
|
||||
25
tests/docker/docker-compose.datastream.dashboard.yml
Normal file
25
tests/docker/docker-compose.datastream.dashboard.yml
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
# Dashboard overlay for the datastream demo.
|
||||
#
|
||||
# Use *together* with the base compose file so the same relay/seed/worker nodes
|
||||
# (and their telemetry) are the data source — only the sink changes. The base
|
||||
# `collector` service (which prints frames to stdout) is replaced by
|
||||
# `swactor-datastream-dashboard`, which binds the same UDP port (7700), demuxes
|
||||
# the per-node frames, and serves the live browser dashboard on :9090.
|
||||
#
|
||||
# docker compose \
|
||||
# -f tests/docker/docker-compose.datastream.yml \
|
||||
# -f tests/docker/docker-compose.datastream.dashboard.yml up
|
||||
#
|
||||
# Then open http://localhost:18080. (Or just run ./tests/docker/datastream-dashboard-demo.sh)
|
||||
# Override the host port with DASH_PORT=<port> if 18080 is taken too.
|
||||
|
||||
services:
|
||||
collector:
|
||||
entrypoint: ["swactor-datastream-dashboard"]
|
||||
# Bind the same UDP sink the nodes ship to, serve the dashboard on :9090,
|
||||
# and show the seed node. Drop --node to display the first node seen instead.
|
||||
command: ["--bind", "0.0.0.0:7700", "--port", "9090", "--node", "seed"]
|
||||
# Host port is overridable (DASH_PORT); defaults to 18080 since 9090 is a
|
||||
# commonly-occupied port. Container port is always the dashboard's 9090.
|
||||
ports:
|
||||
- "${DASH_PORT:-18080}:9090"
|
||||
113
tests/docker/docker-compose.datastream.yml
Normal file
113
tests/docker/docker-compose.datastream.yml
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
# Datastream demo cluster.
|
||||
#
|
||||
# A relay, a UDP collector, a seed swactor node, and two workers — every node
|
||||
# drives a dummy child process and ships its telemetry datastream to the
|
||||
# collector, whose stdout is the live, frame-by-frame raw stream.
|
||||
#
|
||||
# Run it (from the repo root), foreground so the collector's stream is visible:
|
||||
#
|
||||
# cargo build --release -p node -p distribution \
|
||||
# --bin swactor --bin swactor-datastream-collector --bin swactor-iroh-relay
|
||||
# docker compose -f tests/docker/docker-compose.datastream.yml build
|
||||
# docker compose -f tests/docker/docker-compose.datastream.yml up
|
||||
#
|
||||
# Watch the `collector-1 | ...` lines. Ctrl-C then `down` to clean up.
|
||||
#
|
||||
# Nodes converge over the standalone relay (10.0.3.2): each node runs with the
|
||||
# embedded relay off (--no-relay) and points --relay-hosts at the relay so iroh
|
||||
# selects RelayMode::Custom. The seed's node id below is baked from the
|
||||
# committed identity in ./identities/seed/node.key.json.
|
||||
|
||||
x-node-build: &node-build
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: tests/docker/Dockerfile.datastream
|
||||
|
||||
services:
|
||||
collector:
|
||||
<<: *node-build
|
||||
entrypoint: ["swactor-datastream-collector"]
|
||||
command: ["--bind", "0.0.0.0:7700"]
|
||||
networks:
|
||||
datastream:
|
||||
ipv4_address: 10.0.3.5
|
||||
|
||||
relay:
|
||||
<<: *node-build
|
||||
entrypoint: ["swactor-iroh-relay"]
|
||||
command: ["--bind", "0.0.0.0:7843", "--public-host", "10.0.3.2"]
|
||||
networks:
|
||||
datastream:
|
||||
ipv4_address: 10.0.3.2
|
||||
|
||||
seed:
|
||||
<<: *node-build
|
||||
command:
|
||||
- "--config=/etc/swactor/node.toml"
|
||||
- "--identity-dir=/seed-identity"
|
||||
- "--no-relay"
|
||||
- "--relay-port=7843"
|
||||
- "--relay-hosts=10.0.3.2"
|
||||
- "--datastream"
|
||||
- "--datastream-collector=collector:7700"
|
||||
- "--datastream-region=seed"
|
||||
- "--actors=2"
|
||||
networks:
|
||||
datastream:
|
||||
ipv4_address: 10.0.3.10
|
||||
ports:
|
||||
- "9091:9090"
|
||||
depends_on:
|
||||
- collector
|
||||
- relay
|
||||
|
||||
node-2:
|
||||
<<: *node-build
|
||||
command:
|
||||
- "--config=/etc/swactor/node.toml"
|
||||
- "--seed-node-id=G2hRm2rWTFCqyeVnStJg72ysBiTvE19NTTPjdSEXhFK6"
|
||||
- "--no-relay"
|
||||
- "--relay-port=7843"
|
||||
- "--relay-hosts=10.0.3.2"
|
||||
- "--datastream"
|
||||
- "--datastream-collector=collector:7700"
|
||||
- "--datastream-region=worker"
|
||||
- "--actors=3"
|
||||
networks:
|
||||
datastream:
|
||||
ipv4_address: 10.0.3.11
|
||||
ports:
|
||||
- "9092:9090"
|
||||
depends_on:
|
||||
- collector
|
||||
- relay
|
||||
- seed
|
||||
|
||||
node-3:
|
||||
<<: *node-build
|
||||
command:
|
||||
- "--config=/etc/swactor/node.toml"
|
||||
- "--seed-node-id=G2hRm2rWTFCqyeVnStJg72ysBiTvE19NTTPjdSEXhFK6"
|
||||
- "--no-relay"
|
||||
- "--relay-port=7843"
|
||||
- "--relay-hosts=10.0.3.2"
|
||||
- "--datastream"
|
||||
- "--datastream-collector=collector:7700"
|
||||
- "--datastream-region=worker"
|
||||
- "--actors=2"
|
||||
networks:
|
||||
datastream:
|
||||
ipv4_address: 10.0.3.12
|
||||
ports:
|
||||
- "9093:9090"
|
||||
depends_on:
|
||||
- collector
|
||||
- relay
|
||||
- seed
|
||||
|
||||
networks:
|
||||
datastream:
|
||||
driver: bridge
|
||||
ipam:
|
||||
config:
|
||||
- subnet: 10.0.3.0/24
|
||||
3
tests/docker/identities/seed/node.key.json
Normal file
3
tests/docker/identities/seed/node.key.json
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
{
|
||||
"secret_key_hex": "539734dd2b487aaedec2b04300eacf9d6bf0feab7a5e2134956c8b8561a5b5d7"
|
||||
}
|
||||
Loading…
Reference in a new issue