diff --git a/apps/pipeline-parallel-inference/src/bin/pp_orchestrator.rs b/apps/pipeline-parallel-inference/src/bin/pp_orchestrator.rs index 8c5a158..ca5a697 100644 --- a/apps/pipeline-parallel-inference/src/bin/pp_orchestrator.rs +++ b/apps/pipeline-parallel-inference/src/bin/pp_orchestrator.rs @@ -54,7 +54,7 @@ use dashboard::collector::StatsCollector; use dashboard::datastream_source::{fleet_cache_plugin, FleetView}; use dashboard::{start_dashboard, DashboardConfig}; -use pipeline_parallel_inference::dist_plugin::{DistDashPlugin, MsgCounts, SharedSnapshot}; +use pipeline_parallel_inference::dist_plugin::{DistDashPlugin, SharedSnapshot}; use pipeline_parallel_inference::netmap_plugin::{spawn_conn_poller, ConnTracker, NetmapPlugin}; use pipeline_parallel_inference::iroh_transport::{ ActorMessagePump, IrohActorTransport, ACTOR_ALPN, @@ -500,12 +500,6 @@ fn run_seed(args: &Args) -> i32 { // its snapshot cell exists whenever the in-process dashboard is on. let want_dist = std::env::var_os("PP_DASHBOARD").is_some(); - // Orchestrator dashboard message tallies. The per-message wire tally used - // to decorate the driver's diagnostics emitter; that emitter was removed - // from the engine, so the counts stay empty (the distribution page still - // renders membership/routing, just without live message counters). - let msg_counts = Arc::new(MsgCounts::default()); - let my_id = cluster.node_id(); let my_hex: String = my_id.0.iter().map(|b| format!("{:02x}", b)).collect(); let direct: Vec = cluster.driver.direct_addresses().to_vec(); @@ -545,19 +539,15 @@ fn run_seed(args: &Args) -> i32 { // plugins ("Distribution"/"Netmap" nav tabs) against the shared cell. if let (Some((handle, collector, port)), Some(cached)) = (&dashboard, &dist_cached) { handle.set_runtime(Arc::clone(&rt), Arc::clone(collector)); - handle.register_plugin(Arc::new(DistDashPlugin::new( - Arc::clone(cached), - Arc::clone(&msg_counts), - ))); + handle.register_plugin(Arc::new(DistDashPlugin::new(Arc::clone(cached)))); // Net map plugin: a live connection/bandwidth graph. Shares the - // cached snapshot and message tallies; a background poller keeps its - // transport map fresh by querying the iroh endpoint directly. The - // poller's stop flag rides the process lifetime (the driver's tokio - // runtime is torn down at end of run, aborting the task). + // cached snapshot; a background poller keeps its transport map fresh + // by querying the iroh endpoint directly. The poller's stop flag + // rides the process lifetime (the driver's tokio runtime is torn + // down at end of run, aborting the task). let conn_tracker = Arc::new(ConnTracker::default()); handle.register_plugin(Arc::new(NetmapPlugin::new( Arc::clone(cached), - Arc::clone(&msg_counts), Arc::clone(&conn_tracker), ))); let poll_stop = Arc::new(AtomicBool::new(false)); @@ -1499,12 +1489,6 @@ fn run_vastai(args: &Args) -> i32 { // snapshot cell exists whenever the in-process dashboard is on. let want_dist = std::env::var_os("PP_DASHBOARD").is_some(); - // Orchestrator dashboard message tallies. The per-message wire tally used - // to decorate the driver's diagnostics emitter; that emitter was removed - // from the engine, so the counts stay empty (the distribution page still - // renders membership/routing, just without live message counters). - let msg_counts = Arc::new(MsgCounts::default()); - // Per-cluster drive counter, emitted on pp_drive_start / pp_drive_end so // the bundle reader can slice the interleaved event stream by attempt. // One-shot and --hold each drive exactly once per process, so this is 1. @@ -1581,19 +1565,15 @@ fn run_vastai(args: &Args) -> i32 { // plugins ("Distribution"/"Netmap" nav tabs) against the shared cell. if let (Some((handle, collector, port)), Some(cached)) = (&dashboard, &dist_cached) { handle.set_runtime(Arc::clone(&rt), Arc::clone(collector)); - handle.register_plugin(Arc::new(DistDashPlugin::new( - Arc::clone(cached), - Arc::clone(&msg_counts), - ))); - // Net map plugin: a live connection/bandwidth graph. Shares the - // cached snapshot and message tallies; a background poller keeps its - // transport map fresh by querying the iroh endpoint directly. The - // poller's stop flag rides the process lifetime (the driver's tokio - // runtime is torn down at end of run, aborting the task). + handle.register_plugin(Arc::new(DistDashPlugin::new(Arc::clone(cached)))); + // Net map plugin: a live connection/bandwidth graph. Shares the cached + // snapshot; a background poller keeps its transport map fresh by querying + // the iroh endpoint directly. The poller's stop flag rides the process + // lifetime (the driver's tokio runtime is torn down at end of run, + // aborting the task). let conn_tracker = Arc::new(ConnTracker::default()); handle.register_plugin(Arc::new(NetmapPlugin::new( Arc::clone(cached), - Arc::clone(&msg_counts), Arc::clone(&conn_tracker), ))); let poll_stop = Arc::new(AtomicBool::new(false)); diff --git a/apps/pipeline-parallel-inference/src/bin/pp_worker.rs b/apps/pipeline-parallel-inference/src/bin/pp_worker.rs index a5df11d..85c0b1a 100644 --- a/apps/pipeline-parallel-inference/src/bin/pp_worker.rs +++ b/apps/pipeline-parallel-inference/src/bin/pp_worker.rs @@ -992,7 +992,26 @@ fn main_pump( let members = fleet::members_to_pairs(&cluster.members_raw()); let runtime = fleet::runtime_stats(&cluster.rt); let relay_connected = cluster.driver.home_relay_url().is_some(); - fleet.tick(&members, runtime, relay_connected, 0); + + // Real membership transitions (with cause) from the SWIM observer — + // submitted before tick() so they drain this cycle (replaces the + // emitter's reason-less member-list diff). + for t in cluster.drain_swim_transitions() { + fleet.submit_membership(&fleet::membership_transition(&t)); + } + // Consolidated distribution state: registry + location cache + recent + // probe targets + directory route count. + let dist = fleet::build_dist_state( + &cluster.registry_snapshot(), + &cluster.location_cache_entries(), + &cluster.swim_recent_targets(), + cluster.driver.directory_route_count() as u32, + ); + fleet.submit_dist_state(&dist); + // Worker-runtime counters (the deep slice behind runtime.stats). + fleet.submit_worker_counters(&fleet::worker_counters(&cluster.rt)); + + fleet.tick(&members, runtime, relay_connected, 0, cluster.swim_rtt_p50()); } if let Some(status) = status_inbox.try_recv() { diff --git a/apps/pipeline-parallel-inference/src/cluster.rs b/apps/pipeline-parallel-inference/src/cluster.rs index b4209e4..8c087fc 100644 --- a/apps/pipeline-parallel-inference/src/cluster.rs +++ b/apps/pipeline-parallel-inference/src/cluster.rs @@ -39,10 +39,14 @@ use distribution::directory_actor::{DirectoryActor, DirectoryIn}; use distribution::iroh_driver::{IrohDriver, IrohDriverConfig}; use distribution::node::DistributedNodeConfig; use distribution::node_metadata_actor::{MetadataActor, MetadataIn, RelayInfo}; -use distribution::registry_actor::{NameResolved, RegistryActor, RegistryIn}; -use distribution::snapshot::{DistributionNodeSnapshot, MemberInfo}; +use distribution::registry::RegistrySnapshot; +use distribution::registry_actor::{NameResolved, RegistryActor, RegistryIn, RegistryView}; +use distribution::snapshot::{ + CacheEntryInfo, DistributionNodeSnapshot, MemberInfo, RegistryEntryInfo, +}; use distribution::swim::actor::{MembershipChanged, SwimActor, SwimIn}; use distribution::swim::member_list::MemberList; +use distribution::swim::telemetry::{ObservedTransition, SwimTelemetry}; use distribution::transport_bridge::{ IrohPeerDirectory, IrohRouteBinder, Outbox, RelayMirror, RouteView, RouteViewTransport, }; @@ -89,6 +93,11 @@ pub struct ClusterNode { membership_mirror: Arc>, relay_mirror: RelayMirror, _route_view: RouteView, + /// Read-mirror of the cluster registry, for the `dist.state` telemetry. + registry_view: RegistryView, + /// Production SWIM observer: probe RTT, recent probe targets, and membership + /// transitions (with cause), for `transport.internals` / `membership` / `dist.state`. + swim_telemetry: Arc, /// Reused per `resolve_name` call to avoid leaking inbox addresses in the /// runtime's inbox registry. resolve_inbox: Inbox, @@ -168,21 +177,24 @@ impl ClusterNode { Arc::clone(&outbox), )); + // Telemetry mirrors observed by the fleet emitter (same wiring as the + // standalone node): the SWIM observer (probe RTT, recent targets, + // transitions-with-cause) and the registry read-mirror. + let swim_telemetry = SwimTelemetry::new(); + let registry_view: RegistryView = Arc::new(RwLock::new(RegistrySnapshot::default())); + // The four protocol actors. let swim_addr = rt - .spawn(SwimActor::new( - node_id, - swim_config, - Instant::now(), - peer_directory.clone(), - )) + .spawn( + SwimActor::new(node_id, swim_config, Instant::now(), peer_directory.clone()) + .with_observer(Box::new(Arc::clone(&swim_telemetry))), + ) .expect("spawn SwimActor"); let registry_addr = rt - .spawn(RegistryActor::new( - node_id, - registry_config, - peer_directory.clone(), - )) + .spawn( + RegistryActor::new(node_id, registry_config, peer_directory.clone()) + .with_view(Arc::clone(®istry_view)), + ) .expect("spawn RegistryActor"); let metadata_addr = rt .spawn(MetadataActor::new( @@ -266,11 +278,59 @@ impl ClusterNode { membership_mirror, relay_mirror, _route_view: route_view, + registry_view, + swim_telemetry, resolve_inbox, relay_inbox, } } + // ── Telemetry accessors (for the fleet emitter) ───────────────────────── + + /// A consistent snapshot of the cluster registry (size / tombstones / entries). + pub fn registry_snapshot(&self) -> RegistrySnapshot { + self.registry_view.read().expect("registry view poisoned").clone() + } + + /// Median SWIM probe round-trip time (ms); `0` until a probe completes. + pub fn swim_rtt_p50(&self) -> u32 { + self.swim_telemetry.rtt_ms_p50() + } + + /// Recent SWIM probe targets (most recent last). + pub fn swim_recent_targets(&self) -> Vec { + self.swim_telemetry.recent_targets() + } + + /// Drain the membership transitions captured since the last call (each carries + /// a real cause string). + pub fn drain_swim_transitions(&self) -> Vec { + self.swim_telemetry.drain_transitions() + } + + /// This node's location cache: the remote `(actor, host)` pairs it knows, from + /// the directory route-view **and** the registry's live bindings (the demo + /// resolves neighbours by name, so the registry is its location directory). + /// Self-hosted and tombstoned entries are excluded; deduplicated by address. + pub fn location_cache_entries(&self) -> Vec<(ActorAddress, NodeId)> { + let self_id = self.driver.node_id(); + let mut seen = std::collections::HashSet::new(); + let mut out: Vec<(ActorAddress, NodeId)> = Vec::new(); + for (addr, host) in self.driver.location_cache_entries() { + if seen.insert(addr) { + out.push((addr, host)); + } + } + let registry = self.registry_view.read().expect("registry view poisoned"); + for e in ®istry.entries { + if !e.tombstone && e.node_id != self_id && seen.insert(e.actor_addr) { + out.push((e.actor_addr, e.node_id)); + } + } + out.sort_by(|a, b| a.0.0.cmp(&b.0.0)); + out + } + // ── Driver passthroughs ───────────────────────────────────────────────── pub fn node_id(&self) -> NodeId { @@ -449,9 +509,12 @@ impl ClusterNode { pub fn snapshot(&self) -> DistributionNodeSnapshot { let mut snap = self.driver.snapshot(); - // Members + counts from the SWIM mirror. + // Members + counts from the SWIM mirror, each labelled with the cause of + // its most recent liveness transition from the production SWIM observer + // (a non-draining read — the fleet emitter still owns the drain). let mirror = self.membership_mirror.lock().unwrap(); let relays = self.relay_mirror.read().ok(); + let reasons = self.swim_telemetry.last_reasons(); let mut members: Vec = Vec::with_capacity(mirror.len()); let (mut alive, mut suspect, mut dead) = (0usize, 0usize, 0usize); for entry in mirror.all_members() { @@ -482,6 +545,7 @@ impl ClusterNode { label: None, relay_url, node_name: None, + reason: reasons.get(&entry.node_id).map(|r| r.to_string()), }); } snap.members = members; @@ -499,6 +563,39 @@ impl ClusterNode { } } + // Registry (name directory), location cache, and recent probe targets, + // from this node's telemetry mirrors — the `dist.state` fields the + // Distribution page renders. (`driver.snapshot()` fills only the + // directory route count.) + let hex = |bytes: &[u8]| -> String { bytes.iter().map(|b| format!("{b:02x}")).collect() }; + let registry = self.registry_snapshot(); + snap.registry_size = registry.size; + snap.registry_tombstones = registry.tombstones; + snap.registry_entries = registry + .entries + .iter() + .map(|e| RegistryEntryInfo { + name: e.name.clone(), + actor_addr: hex(&e.actor_addr.0), + node_id: hex(&e.node_id.0), + tombstone: e.tombstone, + }) + .collect(); + let cache = self.location_cache_entries(); + snap.cache_size = cache.len(); + snap.cache_entries = cache + .iter() + .map(|(addr, host)| CacheEntryInfo { + actor_addr: hex(&addr.0), + node_id: hex(&host.0), + }) + .collect(); + snap.recent_probe_targets = self + .swim_recent_targets() + .iter() + .map(|t| hex(&t.0)) + .collect(); + snap } } diff --git a/apps/pipeline-parallel-inference/src/dist_page.html b/apps/pipeline-parallel-inference/src/dist_page.html index d9772f9..e0e64af 100644 --- a/apps/pipeline-parallel-inference/src/dist_page.html +++ b/apps/pipeline-parallel-inference/src/dist_page.html @@ -64,16 +64,6 @@ -
-

Messages (this node, wire)

-
-
0Sent
-
0Received
-
0Sent bytes
-
0Recv bytes
-
-
-

SWIM membership graph

@@ -83,26 +73,11 @@

Members

- - + +
StateName / NodeAddrInc
waiting for snapshot…
StateName / NodeAddrIncReason
waiting for snapshot…
-
-

Messages by kind

- - - -
KindSentRecv
no traffic yet
-
- -
-

Messages by peer

- - - -
PeerNameSentRecv
no traffic yet
-