#![allow(dead_code)] #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct RunId(pub u64); #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct NodeId(pub u64); #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct StageRef { pub stage_index: u32, pub node_id: NodeId, } #[derive(Clone, Debug, PartialEq, Eq)] pub struct RunPlan { pub run_id: RunId, pub stages: Vec, } impl RunPlan { pub fn test_linear(run_id: RunId, stages: Vec) -> Self { Self { run_id, stages } } pub fn stage_nodes(&self) -> Vec { self.stages.iter().map(|stage| stage.node_id).collect() } } #[derive(Clone, Debug, PartialEq, Eq)] pub struct RunConfig { pub run_id: RunId, pub max_tokens: u64, pub prompt: Vec, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct SamplingData { pub source_sequence: u64, } #[derive(Clone, Debug, PartialEq, Eq)] pub enum TokenObjectPayload { Prompt { tokens: Vec, }, Decode { token_id: u32, sampling: SamplingData, }, } #[derive(Clone, Debug, PartialEq, Eq)] pub struct TokenObjectInjection { pub sequence: u64, pub payload: TokenObjectPayload, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum StageFaultReason { WorkerCrashed, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum EndpointKind { TokenIn, TokenOut, } #[derive(Clone, Debug, PartialEq, Eq)] pub enum RunEvent { PoolReady { nodes: Vec, }, PlanAvailable(RunPlan), StageReady { run_id: RunId, stage_index: u32, }, TokenInEndpointReady, TokenOutEndpointReady, TokenReceived { sequence: u64, token_id: u32, eos: bool, }, StageFault { run_id: RunId, stage_index: u32, reason: StageFaultReason, }, EndpointFault { run_id: RunId, endpoint: EndpointKind, }, OperatorStop { run_id: RunId, }, MembershipLost { run_id: RunId, node_id: NodeId, }, StageStopped { run_id: RunId, stage_index: u32, }, TokenEndpointsStopped, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum RunFaultReason { StageFault { stage_index: u32, reason: StageFaultReason, }, EndpointFault { endpoint: EndpointKind, }, MembershipLost { node_id: NodeId, }, UnknownStageReady { stage_index: u32, }, } #[derive(Clone, Debug, PartialEq, Eq)] pub enum LifecycleEvent { RunRejected { run_id: RunId, reason: RunFaultReason, }, RunFaulted { run_id: RunId, reason: RunFaultReason, }, RunCompleted { run_id: RunId, }, RunOperatorStopped { run_id: RunId, }, RunTornDown { run_id: RunId, }, } #[derive(Clone, Debug, PartialEq, Eq)] pub struct StageProvision { pub run_id: RunId, pub stage_index: u32, pub node_id: NodeId, } #[derive(Clone, Debug, PartialEq, Eq)] pub enum RunCommand { ProvisionStage { provision: StageProvision, }, CreateTokenInEndpoint { run_id: RunId, }, CreateTokenOutEndpoint { run_id: RunId, }, InjectTokenObject { run_id: RunId, object: TokenObjectInjection, }, StopRun { run_id: RunId, stage_index: u32, }, TearDownTokenEndpoints { run_id: RunId, }, } pub type OrchestratorHarness = OrchestratorRun; pub struct OrchestratorRun { config: RunConfig, plan: Option, pool_ready: bool, provisioned: bool, token_in_ready: bool, token_out_ready: bool, ready_stages: std::collections::BTreeSet, injected_sequences: Vec, expected_token_sequence: u64, events: Vec, commands: Vec, terminal: bool, teardown_started: bool, stopped_stages: std::collections::BTreeSet, token_endpoints_stopped: bool, } impl OrchestratorRun { pub fn new(config: RunConfig) -> Self { Self { config, plan: None, pool_ready: false, provisioned: false, token_in_ready: false, token_out_ready: false, ready_stages: std::collections::BTreeSet::new(), injected_sequences: Vec::new(), expected_token_sequence: 0, events: Vec::new(), commands: Vec::new(), terminal: false, teardown_started: false, stopped_stages: std::collections::BTreeSet::new(), token_endpoints_stopped: false, } } pub fn observe(&mut self, event: RunEvent) { match event { RunEvent::PoolReady { .. } => { self.pool_ready = true; self.maybe_provision(); } RunEvent::PlanAvailable(plan) => { self.plan = Some(plan); self.maybe_provision(); } RunEvent::StageReady { run_id, stage_index, } => { if run_id != self.config.run_id || !self.plan_has_stage(stage_index) { self.fault(RunFaultReason::UnknownStageReady { stage_index }); return; } self.ready_stages.insert(stage_index); self.maybe_inject_initial(); } RunEvent::TokenInEndpointReady => { self.token_in_ready = true; self.maybe_inject_initial(); } RunEvent::TokenOutEndpointReady => { self.token_out_ready = true; self.maybe_inject_initial(); } RunEvent::TokenReceived { sequence, token_id, eos, } => { if self.terminal { return; } if sequence != self.expected_token_sequence { return; } self.expected_token_sequence += 1; if eos { self.complete(); } else if (self.injected_sequences.len() as u64) < self.config.max_tokens { self.inject_decode(sequence + 1, token_id, sequence); } else { self.complete(); } } RunEvent::StageFault { run_id, stage_index, reason, } if run_id == self.config.run_id => { self.fault(RunFaultReason::StageFault { stage_index, reason, }); } RunEvent::EndpointFault { run_id, endpoint } if run_id == self.config.run_id => { self.fault(RunFaultReason::EndpointFault { endpoint }); } RunEvent::OperatorStop { run_id } if run_id == self.config.run_id => { self.operator_stop(); } RunEvent::MembershipLost { run_id, node_id } if run_id == self.config.run_id => { self.fault(RunFaultReason::MembershipLost { node_id }); } RunEvent::StageStopped { run_id, stage_index, } if run_id == self.config.run_id => { self.stopped_stages.insert(stage_index); self.maybe_torn_down(); } RunEvent::TokenEndpointsStopped => { self.token_endpoints_stopped = true; self.maybe_torn_down(); } RunEvent::StageFault { .. } | RunEvent::EndpointFault { .. } | RunEvent::OperatorStop { .. } | RunEvent::MembershipLost { .. } | RunEvent::StageStopped { .. } => {} } } pub fn advance_time_ms(&mut self, _delta: u64) {} pub fn commands(&self) -> &[RunCommand] { &self.commands } pub fn events(&self) -> &[LifecycleEvent] { &self.events } pub fn injected_sequences(&self) -> Vec { self.injected_sequences.clone() } fn maybe_provision(&mut self) { if !self.pool_ready || self.provisioned { return; } let Some(plan) = &self.plan else { return; }; self.provisioned = true; for stage in &plan.stages { self.commands.push(RunCommand::ProvisionStage { provision: StageProvision { run_id: plan.run_id, stage_index: stage.stage_index, node_id: stage.node_id, }, }); } self.commands.push(RunCommand::CreateTokenInEndpoint { run_id: self.config.run_id, }); self.commands.push(RunCommand::CreateTokenOutEndpoint { run_id: self.config.run_id, }); } fn maybe_inject_initial(&mut self) { if self.terminal || !self.provisioned || !self.injected_sequences.is_empty() { return; } if self.token_in_ready && self.token_out_ready && self.all_stages_ready() { self.inject_prompt(0); } } fn inject_prompt(&mut self, sequence: u64) { self.inject(TokenObjectInjection { sequence, payload: TokenObjectPayload::Prompt { tokens: self.config.prompt.clone(), }, }); } fn inject_decode(&mut self, sequence: u64, token_id: u32, source_sequence: u64) { self.inject(TokenObjectInjection { sequence, payload: TokenObjectPayload::Decode { token_id, sampling: SamplingData { source_sequence }, }, }); } fn inject(&mut self, object: TokenObjectInjection) { if self.terminal { return; } self.injected_sequences.push(object.sequence); self.commands.push(RunCommand::InjectTokenObject { run_id: self.config.run_id, object, }); } fn complete(&mut self) { if self.terminal { return; } self.terminal = true; self.events.push(LifecycleEvent::RunCompleted { run_id: self.config.run_id, }); self.start_teardown(); } fn operator_stop(&mut self) { if self.terminal { return; } self.terminal = true; self.events.push(LifecycleEvent::RunOperatorStopped { run_id: self.config.run_id, }); self.start_teardown(); } fn fault(&mut self, reason: RunFaultReason) { if self.terminal { return; } self.terminal = true; self.events.push(LifecycleEvent::RunFaulted { run_id: self.config.run_id, reason, }); self.start_teardown(); } fn start_teardown(&mut self) { if self.teardown_started { return; } self.teardown_started = true; if let Some(plan) = &self.plan { for stage in &plan.stages { self.commands.push(RunCommand::StopRun { run_id: self.config.run_id, stage_index: stage.stage_index, }); } } self.commands.push(RunCommand::TearDownTokenEndpoints { run_id: self.config.run_id, }); } fn maybe_torn_down(&mut self) { if !self.teardown_started || !self.token_endpoints_stopped { return; } if !self.all_stages_stopped() { return; } self.mark_torn_down(); } fn mark_torn_down(&mut self) { if !self .events .iter() .any(|event| matches!(event, LifecycleEvent::RunTornDown { .. })) { self.events.push(LifecycleEvent::RunTornDown { run_id: self.config.run_id, }); } } fn plan_has_stage(&self, stage_index: u32) -> bool { self.plan.as_ref().is_some_and(|plan| { plan.stages .iter() .any(|stage| stage.stage_index == stage_index) }) } fn all_stages_ready(&self) -> bool { self.plan.as_ref().is_some_and(|plan| { plan.stages .iter() .all(|stage| self.ready_stages.contains(&stage.stage_index)) }) } fn all_stages_stopped(&self) -> bool { self.plan.as_ref().is_some_and(|plan| { plan.stages .iter() .all(|stage| self.stopped_stages.contains(&stage.stage_index)) }) } }