//! The demo provider seam: real node-role children behind the reconciler's //! effect executor. //! //! `DemoProvider` is the demo's `ProvisionPlugin` — create spawns a //! `swactor-process` actor whose child re-execs this binary in node role //! (`--demo-node`), stop goes through the real process-actor command path. //! `DemoBackend` adapts the plugin to `NodeManagerCommand` effects, mirroring //! the conformance kit's adapter semantics (attempt-keyed identity, adoption //! on retry). //! //! Actor spawning needs an actor context, so spawn requests travel over a //! channel to the supervisor actor; the plugin call blocks for the reply on //! an executor blocking thread. //! //! Children run with stdio null, so the supervisor learns a node's iroh //! identity and liveness from the wire announce (see `node.rs`): the //! [`AnnounceActor`] routes it to the owning bootstrap actor and records //! `last_announce_ms` in the [`NodeManager`] registry. use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; use std::time::{Duration, SystemTime}; use swactor::actor::{ActorAddress, ActorInterface, Ctx}; use swactor::runtime::ExternalSender; use swactor_process::{ExitStatus, ProcessOutput}; use provisioning::executor::{EffectBackend, EffectError}; use provisioning::node::{ BootstrapSessionId, CreateLeaseResult, DestroyHandle, LeaseFacts, NodeManagerCommand, ProviderKind, ProviderLeaseId, SshEndpoint, }; use provisioning::plugin::{NodeProvisionSpec, PluginNodeHandle, PluginSink, ProvisionPlugin}; use provisioning::reconciler::{OperationId, OperationOutcome, PlannedEffect}; use telemetry::{ChannelContent, StreamDescriptor, TelemetryEndpoint, TelemetryProducer}; /// How long a blocking plugin call waits for the supervisor actor. pub const BACKEND_WAIT: Duration = Duration::from_secs(20); /// One provisioned node's runtime facts. #[derive(Clone)] pub struct NodeRuntime { pub attempt: u64, pub logical_node: String, /// The bootstrap actor that owns this attempt's lifecycle. pub bootstrap: ActorAddress, pub pid: Option, pub exited: Option, pub spawn_failed: Option, /// Wall-clock ms of the last wire announce from the node (None until /// the first announce). The node re-announces every heartbeat period, /// so staleness here means the control-plane path is dead. pub last_announce_ms: Option, /// Serde-serialized `iroh::EndpointAddr` from the node's announce — /// what the supervisor dials for telemetry pulls and demo edges. pub endpoint_addr: Option, } /// Spawn request from the plugin (blocking thread) to the supervisor actor. #[derive(Clone)] pub struct SpawnNodeRequest { pub attempt: u64, pub logical_node: String, pub reply: std::sync::mpsc::Sender>, } /// Shared node registry + spawn queue: hub between executor blocking threads, /// relay actors, and the supervisor actor. #[derive(Clone, Default)] pub struct NodeManager { inner: Arc>, } #[derive(Default)] struct NodeManagerInner { nodes: BTreeMap, spawn_tx: Option>, } impl NodeManager { pub fn new() -> Self { Self::default() } pub fn set_spawn_channel(&self, sender: std::sync::mpsc::Sender) { self.inner.lock().expect("node manager").spawn_tx = Some(sender); } pub fn request_spawn(&self, attempt: u64, logical_node: String) -> Result { let (reply_tx, reply_rx) = std::sync::mpsc::channel(); let request = SpawnNodeRequest { attempt, logical_node, reply: reply_tx, }; { let inner = self.inner.lock().expect("node manager"); let sender = inner .spawn_tx .as_ref() .ok_or_else(|| "supervisor spawn channel not installed".to_owned())?; sender .send(request) .map_err(|_| "supervisor actor gone".to_owned())?; } reply_rx .recv_timeout(BACKEND_WAIT) .map_err(|_| "timed out waiting for node spawn".to_owned())? } pub fn register(&self, runtime: NodeRuntime) { self.inner .lock() .expect("node manager") .nodes .insert(runtime.attempt, runtime); } pub fn get(&self, attempt: u64) -> Option { self.inner .lock() .expect("node manager") .nodes .get(&attempt) .cloned() } /// All registered runtimes (snapshot for liveness sweeps). pub fn nodes(&self) -> Vec { self.inner .lock() .expect("node manager") .nodes .values() .cloned() .collect() } pub fn find_by_stream_node(&self, node: &str) -> Option { self.inner .lock() .expect("node manager") .nodes .values() .find(|runtime| runtime.logical_node == node) .cloned() } fn update(&self, attempt: u64, mutate: impl FnOnce(&mut NodeRuntime)) { if let Some(runtime) = self .inner .lock() .expect("node manager") .nodes .get_mut(&attempt) { mutate(runtime); } } pub fn set_pid(&self, attempt: u64, pid: u32) { self.update(attempt, |runtime| runtime.pid = Some(pid)); } pub fn set_exited(&self, attempt: u64, status: ExitStatus) { self.update(attempt, |runtime| runtime.exited = Some(status)); } pub fn set_spawn_failed(&self, attempt: u64, reason: String) { self.update(attempt, |runtime| runtime.spawn_failed = Some(reason)); } pub fn set_announce(&self, attempt: u64, at_ms: u64) { self.update(attempt, |runtime| runtime.last_announce_ms = Some(at_ms)); } pub fn set_endpoint(&self, attempt: u64, endpoint_addr_json: String) { self.update(attempt, |runtime| { runtime.endpoint_addr = Some(endpoint_addr_json) }); } pub fn remove(&self, attempt: u64) { self.inner .lock() .expect("node manager") .nodes .remove(&attempt); } } /// The demo `ProvisionPlugin`: resources are node-role child processes. pub struct DemoProvider { manager: NodeManager, } impl DemoProvider { pub fn new(manager: NodeManager) -> Self { Self { manager } } } impl ProvisionPlugin for DemoProvider { fn create_node( &mut self, spec: NodeProvisionSpec, _sink: PluginSink, ) -> Result { let attempt = spec.attempt_id; if let Some(runtime) = self.manager.get(attempt) { // Adoption: the child for this attempt already exists. return Ok(PluginNodeHandle { id: attempt, provider_process_id: runtime.pid, }); } let logical_node = spec .env .iter() .find(|(key, _)| key == "DEMO_LOGICAL_NODE") .map(|(_, value)| value.clone()) .ok_or_else(|| "spec missing DEMO_LOGICAL_NODE".to_owned())?; let runtime = self.manager.request_spawn(attempt, logical_node)?; Ok(PluginNodeHandle { id: attempt, provider_process_id: runtime.pid, }) } fn start_bootstrap(&mut self, _handle: &PluginNodeHandle) -> Result<(), String> { Ok(()) } fn cancel_bootstrap(&mut self, _handle: &PluginNodeHandle) -> Result<(), String> { Ok(()) } fn complete_bootstrap(&mut self, _handle: &PluginNodeHandle) -> Result<(), String> { Ok(()) } fn stop_node(&mut self, handle: &PluginNodeHandle) -> Result<(), String> { // The backend path sends the process Stop command before calling // here; direct plugin-level stop just drops the registration. self.manager.remove(handle.id); Ok(()) } } /// Per-node telemetry: one endpoint/producer per provisioned node so each /// lands on its own dashboard stream (one fleet card per node). Carries the /// stream descriptor's origin/label so the dashboard can classify the stream /// (the frame path itself has no catalog). pub struct NodeTelemetry { pub endpoint: TelemetryEndpoint, pub producer: TelemetryProducer, pub origin: &'static str, pub label: String, } impl NodeTelemetry { pub fn new(logical_node: &str, life: u64) -> Self { let stream = telemetry::frame::StreamId::new( telemetry::frame::NodeId::new(logical_node), telemetry::frame::Lifetime(life), ); let endpoint = TelemetryEndpoint::with_descriptor( StreamDescriptor { stream, label: Some(format!("demo node {logical_node}")), origin: telemetry::frame::StreamOrigin::RemoteNode, }, 256, 16, ); let producer = endpoint.producer(); Self { endpoint, producer, origin: "remote_node", label: format!("demo node {logical_node}"), } } } /// Relay actor: folds process-actor lifecycle reports into the registry. pub struct NodeRelayActor { manager: NodeManager, attempt: u64, } impl NodeRelayActor { pub fn new(manager: NodeManager, attempt: u64) -> Self { Self { manager, attempt } } } impl ActorInterface for NodeRelayActor { type Incoming = ProcessOutput; type Response = (); fn handle(&mut self, _ctx: &Ctx, output: ProcessOutput) { match output { ProcessOutput::Started { pid } => self.manager.set_pid(self.attempt, pid), ProcessOutput::Exited { status } => self.manager.set_exited(self.attempt, status), ProcessOutput::SpawnFailed { error } => { eprintln!("demo node {}: spawn failed: {error}", self.attempt); self.manager .set_spawn_failed(self.attempt, error.to_string()); } ProcessOutput::Error { error } => { eprintln!("demo node {}: process actor error: {error}", self.attempt); } } } } /// Supervisor-side announce relay. The iroh actor bridge decodes inbound /// [`NodeAnnounce`] gossip frames and routes them here by wire tag; this /// actor correlates by attempt token and forwards to the owning bootstrap /// actor ([`provisioning::BootstrapMsg::Announce`]) — the handoff from /// bootstrap to control plane. Announces for unknown attempts (deregistered /// lease, stale container from a dead attempt) are dropped with a log line. pub struct AnnounceActor { manager: NodeManager, sender: ExternalSender, } impl AnnounceActor { pub fn new(manager: NodeManager, sender: ExternalSender) -> Self { Self { manager, sender } } } impl ActorInterface for AnnounceActor { type Incoming = crate::demo::node::NodeAnnounce; type Response = (); fn handle(&mut self, _ctx: &Ctx, announce: crate::demo::node::NodeAnnounce) { self.manager.set_announce(announce.attempt, announce.at_ms); self.manager .set_endpoint(announce.attempt, announce.endpoint_addr_json.clone()); let Some(runtime) = self.manager.get(announce.attempt) else { let key = &announce.key_hex; eprintln!( "demo: announce for unknown attempt {} (key {}…) dropped", announce.attempt, &key[..8.min(key.len())] ); return; }; let identity = provisioning::NodeIdentity { attempt: announce.attempt, logical_node: announce.logical_node, key_hex: announce.key_hex, transport_addr: announce.endpoint_addr_json, }; let _ = self.sender.send_to( runtime.bootstrap, provisioning::BootstrapMsg::Announce(identity), ); } } /// Demo lease/session identity (attempt-encoded, kit conventions). pub fn demo_lease(attempt: u64) -> LeaseFacts { let provider = ProviderKind::new("demo"); let lease_id = ProviderLeaseId(format!("demo-lease-{attempt}")); LeaseFacts { destroy_handle: DestroyHandle { provider: provider.clone(), lease_id: lease_id.clone(), provider_contract_id: format!("demo-contract-{attempt}"), }, provider, lease_id, provider_contract_id: format!("demo-contract-{attempt}"), offer_id: None, provider_metadata: BTreeMap::new(), } } pub fn demo_endpoint() -> SshEndpoint { SshEndpoint { host: "127.0.0.1".to_owned(), port: 0, user: "demo".to_owned(), auth_ref: "demo-key".to_owned(), } } /// Session identity: one bootstrap session per attempt (production /// convention — the supervisor's close path addresses it the same way). pub fn session_id_for(operation: OperationId) -> BootstrapSessionId { BootstrapSessionId(operation.attempt.0) } fn runtime_exited(manager: &NodeManager, attempt: u64) -> bool { manager .get(attempt) .is_some_and(|runtime| runtime.exited.is_some()) } /// The demo `EffectBackend`: routes effects through the plugin. pub struct DemoBackend { /// The plugin lives behind a mutex: `EffectBackend::execute` is `&self` /// while `ProvisionPlugin` methods are `&mut self` (kit adapter shape). pub plugin: Arc>, pub manager: NodeManager, pub sender: ExternalSender, } impl DemoBackend { fn handle_for(&self, attempt: u64) -> PluginNodeHandle { let pid = self.manager.get(attempt).and_then(|r| r.pid); PluginNodeHandle { id: attempt, provider_process_id: pid, } } } impl EffectBackend for DemoBackend { fn execute(&self, effect: &PlannedEffect) -> Result { let attempt = effect.operation.attempt.0; match &effect.command { NodeManagerCommand::CreateLease(request) => { let spec = NodeProvisionSpec { run_id: request.spec.run_id.0, node_id: attempt, attempt_id: attempt, stage_index: None, image: request.spec.shape.image.clone(), env: vec![( "DEMO_LOGICAL_NODE".to_owned(), request.spec.logical_node_id.0.clone(), )], args: Vec::new(), mounts: Vec::new(), }; self.plugin .lock() .expect("demo provider") .create_node(spec, null_sink()) .map_err(EffectError::definite)?; Ok(OperationOutcome::LeaseCreated(CreateLeaseResult { lease: demo_lease(attempt), endpoint: Some(demo_endpoint()), })) } NodeManagerCommand::LookupEndpoint(_) => { Ok(OperationOutcome::EndpointLookup(Some(demo_endpoint()))) } NodeManagerCommand::StartBootstrap(_) => { let handle = self.handle_for(attempt); self.plugin .lock() .expect("demo provider") .start_bootstrap(&handle) .map_err(EffectError::definite)?; Ok(OperationOutcome::BootstrapStarted { session_id: session_id_for(effect.operation), }) } NodeManagerCommand::BootstrapConvergenceObserved { .. } => { let handle = self.handle_for(attempt); self.plugin .lock() .expect("demo provider") .complete_bootstrap(&handle) .map_err(EffectError::definite)?; Ok(OperationOutcome::BootstrapConvergenceAccepted) } NodeManagerCommand::CancelBootstrap { .. } => { let handle = self.handle_for(attempt); self.plugin .lock() .expect("demo provider") .cancel_bootstrap(&handle) .map_err(EffectError::definite)?; Ok(OperationOutcome::BootstrapCancelled) } NodeManagerCommand::DestroyLease(_) => { let handle = self.handle_for(attempt); // stop_node needs the sender; route it explicitly. stop_node_with_sender(&self.plugin, &self.manager, &handle, &self.sender) .map_err(EffectError::definite)?; Ok(OperationOutcome::LeaseDestroyed) } } } } fn stop_node_with_sender( plugin: &Arc>, manager: &NodeManager, handle: &PluginNodeHandle, sender: &ExternalSender, ) -> Result<(), String> { if let Some(runtime) = manager.get(handle.id) { if runtime.pid.is_some() && runtime.exited.is_none() { let _ = sender.send_to( runtime.bootstrap, provisioning::BootstrapMsg::Stop { kill_after: Some(Duration::from_secs(1)), }, ); } let deadline = std::time::Instant::now() + BACKEND_WAIT; while !runtime_exited(manager, handle.id) { if std::time::Instant::now() > deadline { break; } std::thread::sleep(Duration::from_millis(50)); } } plugin.lock().expect("demo provider").stop_node(handle) } fn null_sink() -> PluginSink { struct NullSink; impl provisioning::plugin::PluginObservationSink for NullSink { fn observe(&self, _observation: provisioning::plugin::PluginObservation) {} } PluginSink::new(Arc::new(NullSink)) } /// Register per-node supplementary channels; returns the `node.status` /// channel id for the supervisor's liveness heartbeats. pub fn register_node_channels(producer: &TelemetryProducer) -> telemetry::ChannelId { producer.register_channel( "node.status", ChannelContent::JsonRecord { schema: Some("demo.node.status.v1".to_owned()), }, ) } pub fn unix_ms(now: SystemTime) -> u64 { now.duration_since(SystemTime::UNIX_EPOCH) .map(|duration| duration.as_millis() as u64) .unwrap_or(0) }