refactor(mvp-system): move swim telemetry into distribution stack

- Fold swim_recent_probe_targets/swim_probe_event_record/membership_transition into
  DistributionRuntimeStack.
- De-generify vastai approval (drop VastAiApproval trait).
- Slim worker_node_runtime and orchestration/app.


Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>

crates/datastream/src/publisher_actor.rs

@@ -76,7 +76,5 @@ impl ActorInterface for DatastreamPublisherActor {
 
 /// Register JSON encoding for remote datastream publisher messages.
 pub fn register_datastream_publisher_codec(registry: &mut CodecRegistry) {
-    registry.register::<DatastreamPublisherMsg, _>(
-        JsonCodec::<DatastreamPublisherMsg>::default(),
-    );
+    registry.register::<DatastreamPublisherMsg, _>(JsonCodec::<DatastreamPublisherMsg>::default());
 }

crates/iroh-driver/src/lib.rs

@@ -10,13 +10,13 @@ pub mod edge_transport;
 pub mod endpoint_advertisement;
 pub mod iroh_driver;
 
+pub use endpoint_advertisement::{
+    EndpointAddrMask, MVP_IROH_ENDPOINT_ADDR_MASK_ENV, advertised_endpoint,
+};
 pub use iroh_driver::{
     ConnType, DatastreamPublishHandle, IrohDriver, IrohDriverConfig, JoinPhase, JoinStatus,
     conn_type_of, discover_lan_ips,
 };
-pub use endpoint_advertisement::{
-    MVP_IROH_ENDPOINT_ADDR_MASK_ENV, EndpointAddrMask, advertised_endpoint,
-};
 
 pub use edge_transport::{EDGE_ALPN, EdgeSendHandle, EdgeTransportEvent, EdgeTransportFault};
 

crates/mvp-system/src/chat/runtime.rs

@@ -30,11 +30,11 @@ use crate::node_provisioning::{ProviderKind, provider_kind};
 use crate::observability::{benchmark, frame_archive::FrameArchive};
 use crate::orchestration::config::ResolvedVastAiConfig;
 use crate::prompt::rpc::{PromptEvent, SubmitPrompt, write_json_line};
-use iroh_driver::EndpointAddrMask;
 use crate::{
     DEFAULT_PIPELINE_CACHED_MODEL_FILE, DEFAULT_PIPELINE_CACHED_MODEL_ID,
     DEFAULT_PIPELINE_CACHED_MODEL_MAX_CONTEXT, DEFAULT_PIPELINE_CACHED_MODEL_REPO,
 };
+use iroh_driver::EndpointAddrMask;
 
 const DEFAULT_CONFIG_PATH: &str = ".config/config.toml";
 const DEFAULT_RPC_ADDR: &str = "127.0.0.1:19777";
@@ -150,8 +150,7 @@ where
     );
     progress.emit_benchmark_envelope(&config);
     progress.emit_endpoint_config_snapshot(&config);
-    let mut approval = StdinVastAiApproval;
-    confirm_vastai_if_needed_with_approval(&config, &mut approval)?;
+    confirm_vastai_if_needed(&config)?;
     let prepare_runtime_started = Instant::now();
     progress.emit(
         CHAT_RUNTIME_CHANNEL,
@@ -1264,19 +1263,24 @@ fn first_non_empty<const N: usize>(values: [Option<String>; N]) -> Option<String
         .find(|value| !value.is_empty())
 }
 
-trait VastAiApproval {
-    fn stdin_is_terminal(&self) -> bool;
-    fn ask(&mut self) -> Result<bool, String>;
+fn confirm_vastai_if_needed(config: &Config) -> Result<(), String> {
+    if config.vastai.is_none() {
+        return Ok(());
+    }
+    if config.vastai_yes {
+        return Ok(());
+    }
+    if !io::stdin().is_terminal() {
+        return Err("Vast.ai rental requires --yes when stdin is not a terminal".to_owned());
+    }
+    if prompt_vastai_approval()? {
+        Ok(())
+    } else {
+        Err("Vast.ai rental declined".to_owned())
+    }
 }
 
-struct StdinVastAiApproval;
-
-impl VastAiApproval for StdinVastAiApproval {
-    fn stdin_is_terminal(&self) -> bool {
-        io::stdin().is_terminal()
-    }
-
-    fn ask(&mut self) -> Result<bool, String> {
+fn prompt_vastai_approval() -> Result<bool, String> {
     #[cfg(test)]
     {
         let mut input = std::io::Cursor::new(Vec::<u8>::new());
@@ -1291,30 +1295,6 @@ impl VastAiApproval for StdinVastAiApproval {
         ask_vastai_approval(&mut input, &mut output)
     }
 }
-}
-
-fn confirm_vastai_if_needed_with_approval<A>(
-    config: &Config,
-    approval: &mut A,
-) -> Result<(), String>
-where
-    A: VastAiApproval,
-{
-    if config.vastai.is_none() {
-        return Ok(());
-    }
-    if config.vastai_yes {
-        return Ok(());
-    }
-    if !approval.stdin_is_terminal() {
-        return Err("Vast.ai rental requires --yes when stdin is not a terminal".to_owned());
-    }
-    if approval.ask()? {
-        Ok(())
-    } else {
-        Err("Vast.ai rental declined".to_owned())
-    }
-}
 
 fn ask_vastai_approval<R, W>(input: &mut R, output: &mut W) -> Result<bool, String>
 where

crates/mvp-system/src/lib.rs

@@ -43,12 +43,12 @@ mod run_fsm;
 mod run_plan;
 
 mod chat;
+mod codecs;
 mod node;
 mod observability;
 mod orchestration;
 mod prompt;
 mod staging;
-mod codecs;
 
 #[cfg(test)]
 mod tests;

crates/mvp-system/src/orchestration/distribution_stack.rs

@@ -9,7 +9,7 @@
 
 use std::collections::HashMap;
 use std::sync::{Arc, Mutex, RwLock};
-use std::time::Instant;
+use std::time::{Duration, Instant};
 
 use swactor::actor::{ActorAddress, ActorInterface};
 use swactor::config::RuntimeConfig;
@@ -28,6 +28,8 @@ use distribution::swim::actor::{MembershipChanged, SwimActor, SwimIn};
 use distribution::swim::member_list::MemberList;
 use distribution::swim::probe::SwimConfig;
 use distribution::swim::telemetry::{ObservedProbeEvent, ObservedTransition, SwimTelemetry};
+use distribution::telemetry::MembershipTransition;
+use distribution::telemetry::SwimProbeEvent;
 use distribution::transport_bridge::{
     Outbox, OutboxPeerDirectory, OutboxRouteBinder, RelayMirror, RouteView, RouteViewTransport,
 };
@@ -237,6 +239,70 @@ impl DistributionRuntimeStack {
     pub(crate) fn drain_swim_probe_events(&self) -> Vec<ObservedProbeEvent> {
         self.swim_telemetry.drain_probe_events()
     }
+
+    pub(crate) fn swim_recent_probe_targets(&self) -> Vec<String> {
+        self.swim_telemetry
+            .recent_targets()
+            .into_iter()
+            .map(|node_id| format!("{:?}", node_id))
+            .collect()
+    }
+
+    pub(crate) fn swim_probe_event_record(
+        &self,
+        event: ObservedProbeEvent,
+        local_phase: &str,
+    ) -> SwimProbeEvent {
+        let config = &self.swim_config;
+        let budget_ms = event.budget_ms;
+        SwimProbeEvent {
+            event: event.event.to_owned(),
+            target: format!("{:?}", event.target),
+            sequence: event.sequence,
+            kind: event.kind.to_owned(),
+            rtt_ms: event.rtt_ms,
+            budget_ms,
+            budget_ticks: budget_ms,
+            last_ack_age_ms: event.last_ack_age.map(duration_ms_u64),
+            consecutive_timeouts: event.consecutive_timeouts,
+            recent_probe_targets: self.swim_recent_probe_targets(),
+            member_state: self
+                .member_state(event.target)
+                .map(|state| format!("{:?}", state)),
+            local_phase: local_phase.to_owned(),
+            probe_interval_ms: duration_ms_u64(config.probe_interval),
+            probe_timeout_ms: duration_ms_u64(config.probe_timeout),
+            indirect_probes: u32::try_from(config.indirect_probes).unwrap_or(u32::MAX),
+            suspicion_timeout_ms: duration_ms_u64(config.suspicion_timeout),
+            dead_reprobe_interval_ms: duration_ms_u64(config.dead_reprobe_interval),
+            probe_mode: format!("{:?}", config.probe_mode),
+            lifeguard_enabled: config.lifeguard.is_some(),
+        }
+    }
+    pub(crate) fn membership_transition(
+        &self,
+        transition: &ObservedTransition,
+    ) -> MembershipTransition {
+        MembershipTransition {
+            peer: format!("{:?}", transition.peer),
+            from: transition
+                .from
+                .map(|state| format!("{:?}", state))
+                .unwrap_or_default(),
+            to: format!("{:?}", transition.to),
+            reason: transition.reason.to_owned(),
+            last_ack_age_ms: transition.last_ack_age.map(duration_ms_u64),
+            consecutive_timeouts: transition.consecutive_timeouts,
+            recent_probe_targets: self.swim_recent_probe_targets(),
+            member_state: self
+                .member_state(transition.peer)
+                .map(|state| format!("{:?}", state)),
+        }
+    }
+}
+
+pub(crate) fn duration_ms_u64(duration: Duration) -> u64 {
+    u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
 }
 
 struct MembershipFanout {

crates/transport/src/json_codec.rs

@@ -6,8 +6,8 @@
 
 use std::marker::PhantomData;
 
-use serde::Serialize;
 use serde::de::DeserializeOwned;
+use serde::Serialize;
 use swactor::Error;
 
 use crate::codec::Codec;