swactor/src/worker.rs

1125 lines
40 KiB
Rust
Raw Normal View History

use crate::Instant;
use std::any::Any;
use std::cell::RefCell;
use std::collections::{HashMap, VecDeque};
use std::sync::Arc;
use std::sync::atomic::Ordering;
use crate::Error;
use crate::actor::{
ActorAddress, AnyActor, ContextInner, Ctx, Environment, ExitValue, ResumeSignal, SpawnRequest,
StopReason, StopSignal, StopWithSignal, SystemInfo,
};
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
use crate::admin::{
ActorStatus, ActorSummary, AdminCommand, AdminError, AdminResult, InspectActorResponse,
ListActorsResponse, OperationResult,
};
use crate::channel::Receiver;
use crate::delivery::{AddrBuildHasher, AddrMap, Envelope, TickContext};
use crate::stats::{ActorSnapshot, TickTiming, WorkerStats};
use crate::extension::WorkerExtension;
/// Whether an actor should be skipped during `tick_all`.
pub(crate) fn should_skip_actor(poisoned: bool, stopping: bool, suspended: bool) -> bool {
poisoned || stopping || suspended
}
/// Whether `on_stop` should fire for an actor being cleaned up.
pub(crate) fn is_on_stop_eligible(stopping: bool, poisoned: bool) -> bool {
stopping && !poisoned
}
/// Determine the `StopReason` for a dead actor based on its flags.
pub(crate) fn determine_stop_reason(poisoned: bool, has_exit_value: bool) -> StopReason {
if poisoned {
StopReason::Panicked
} else if has_exit_value {
StopReason::Completed
} else {
StopReason::Normal
}
}
/// Route a message: try local pool first, then deposit for pending-spawn actors,
/// then inbox_registry for external receivers.
fn route_to_pool_or_remote(
pool: &mut ActorPool,
tc: &TickContext,
dest: ActorAddress,
msg: Box<dyn Any + Send>,
) {
if pool.contains(&dest) {
pool.deliver(&dest, msg);
} else if tc.address_map.contains(&dest) {
// Actor exists but not yet in pool (pending spawn) — deposit for next tick
tc.transfer_tx.send(Envelope::new(dest, msg));
} else {
let _ = tc.route_nonlocal(dest, msg);
}
}
// ─── Worker ─────────────────────────────────────────────────────────────────
/// A worker owns a set of actors and processes them via tick_once.
pub(crate) struct Worker {
pub(crate) pool: ActorPool,
transfer_rx: Receiver<Envelope>,
spawn_rx: Receiver<SpawnRequest>,
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
admin_rx: Receiver<AdminCommand>,
stats: Arc<WorkerStats>,
/// Reusable scratch buffer for building per-actor snapshots.
snapshot_buf: Vec<ActorSnapshot>,
/// Per-worker extension (e.g., timer wheel). Created by RuntimeExtension factory.
pub(crate) worker_ext: Option<Box<dyn WorkerExtension>>,
/// True if the previous tick did work — ensures one full tick follows a productive
/// tick so pending_local messages delivered to mailboxes get drained.
has_backlog: bool,
}
impl Worker {
pub(crate) fn new(
transfer_rx: Receiver<Envelope>,
spawn_rx: Receiver<SpawnRequest>,
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
admin_rx: Receiver<AdminCommand>,
stats: Arc<WorkerStats>,
) -> Self {
Self {
pool: ActorPool::new(),
transfer_rx,
spawn_rx,
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
admin_rx,
stats,
snapshot_buf: Vec::new(),
worker_ext: None,
has_backlog: false,
}
}
pub(crate) fn has_work(&self) -> bool {
self.has_backlog
|| !self.spawn_rx.is_empty()
|| !self.transfer_rx.is_empty()
|| !self.admin_rx.is_empty()
|| self
.worker_ext
.as_ref()
.map_or(false, |e| e.has_pending_work())
}
/// Run one iteration of the worker loop. Returns `true` if any work was done.
/// Drain the spawn queue, inserting new actors into the pool.
/// Used in phases 1 and 4 of tick_once.
fn drain_spawns(&mut self, tc: &TickContext) -> bool {
let mut did_work = false;
#[cfg(feature = "tracing")]
let mut spawn_count: usize = 0;
while let Some(mut req) = self.spawn_rx.try_recv() {
if let Some(ext) = tc.extension {
req.env = ext.on_spawn(
req.addr,
req.parent,
req.env,
tc.created_at.elapsed().as_millis() as u64,
);
}
self.pool.insert(req);
#[cfg(feature = "tracing")]
{
spawn_count += 1;
}
did_work = true;
}
#[cfg(feature = "tracing")]
if spawn_count > 0 {
tracing::debug!(
worker_id = 0,
count = spawn_count,
"worker.spawns_drained"
);
}
did_work
}
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
fn drain_admin(&mut self, tc: &TickContext) -> bool {
let mut did_work = false;
while let Some(cmd) = self.admin_rx.try_recv() {
did_work = true;
self.apply_admin_command(tc, cmd);
}
did_work
}
fn send_admin_reply<T: crate::actor::Message>(
tc: &TickContext,
reply_to: ActorAddress,
result: AdminResult<T>,
) {
let _ = tc.inbox_registry.try_deliver(reply_to, Box::new(result));
}
fn apply_admin_command(&mut self, tc: &TickContext, cmd: AdminCommand) {
match cmd {
AdminCommand::ListActors { acc } => {
let mut local = Vec::new();
self.pool.actor_summaries_into(&mut local);
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
{
let mut summaries = acc.summaries.lock();
summaries.extend(local);
}
if acc.remaining.fetch_sub(1, Ordering::AcqRel) == 1 {
let actors = {
let mut summaries = acc.summaries.lock();
std::mem::take(&mut *summaries)
};
Self::send_admin_reply(tc, acc.reply_to, Ok(ListActorsResponse { actors }));
}
}
AdminCommand::InspectActor { actor, reply_to } => {
let result = self
.pool
.actor_summary(actor)
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
.map(|summary| InspectActorResponse { summary });
Self::send_admin_reply(tc, reply_to, result);
}
AdminCommand::GetActorState {
actor,
reply_to,
get,
not_found,
} => {
let boxed = match self.pool.get_actor_erased(actor) {
Some(erased) => get(actor, erased, erased.metadata()),
None => not_found(actor),
};
let _ = tc.inbox_registry.try_deliver(reply_to, boxed);
}
AdminCommand::ReplaceActorState {
actor,
reply_to,
replace,
} => {
let result = match self.pool.get_actor_erased_mut(actor) {
Some(erased) => {
let metadata = erased.metadata();
replace(erased, metadata)
}
None => Err(AdminError::ActorNotFound { actor }),
};
Self::send_admin_reply(tc, reply_to, result);
}
AdminCommand::StopActor { actor, reply_to } => {
let result = self.pool.stop_actor_admin(actor, &self.stats);
Self::send_admin_reply(tc, reply_to, result);
}
AdminCommand::SuspendActor { actor, reply_to } => {
let result = self.pool.suspend_actor_admin(actor);
Self::send_admin_reply(tc, reply_to, result);
}
AdminCommand::ResumeActor { actor, reply_to } => {
let result = self.pool.resume_actor_admin(actor);
Self::send_admin_reply(tc, reply_to, result);
}
}
}
/// Phase 7: clean up dead actors, deliver death notifications, GC extension state.
fn cleanup_dead_actors(&mut self, tc: &TickContext) -> bool {
let cleanup_pending: RefCell<Vec<(ActorAddress, Box<dyn Any + Send>)>> =
RefCell::new(Vec::new());
let cleanup_stops: RefCell<Vec<ActorAddress>> = RefCell::new(Vec::new());
let cleanup_stop_withs: RefCell<Vec<(ActorAddress, ExitValue)>> = RefCell::new(Vec::new());
let cleanup_suspends: RefCell<Vec<ActorAddress>> = RefCell::new(Vec::new());
let cleanup_requests: RefCell<Vec<Box<dyn Any + Send>>> = RefCell::new(Vec::new());
let dead = {
let cleanup_ctx = WorkerContext {
tc,
pending_local: &cleanup_pending,
stop_requests: &cleanup_stops,
stop_with_values: &cleanup_stop_withs,
suspend_requests: &cleanup_suspends,
worker_requests: &cleanup_requests,
stats: &self.stats,
};
self.pool.cleanup_dead(&cleanup_ctx)
};
let had_dead = !dead.is_empty();
if had_dead {
for (addr, _, _) in &dead {
tc.address_map.remove(addr);
}
if let Some(ext) = tc.extension {
let notifications = ext.on_actor_death(&dead);
let dead_addrs: Vec<_> = dead.iter().map(|(a, _, _)| *a).collect();
ext.cleanup_dead(&dead_addrs);
for (dest, msg) in notifications {
route_to_pool_or_remote(&mut self.pool, tc, dest, msg);
}
}
self.stats
.num_actors
.store(self.pool.len(), Ordering::Relaxed);
}
// Deliver any messages sent during on_stop callbacks
for (addr, msg) in cleanup_pending.into_inner() {
self.pool.deliver(&addr, msg);
}
// GC per-worker extension state for dead actors
if let Some(ext) = &mut self.worker_ext {
let dead_addrs: Vec<ActorAddress> = dead.iter().map(|(a, _, _)| *a).collect();
ext.gc_dead(&dead_addrs);
}
had_dead
}
pub(crate) fn tick_once(&mut self, tc: &TickContext) -> bool {
#[cfg(feature = "tracing")]
let _span = tracing::trace_span!("worker.tick", worker_id = 0).entered();
// Fast idle path: skip the entire tick when nothing could have changed.
// Cost: ~3 atomic loads, zero syscalls, zero actor iteration.
if !self.has_work() {
return false;
}
let mut did_work = false;
let t0 = Instant::now();
// 1. Drain spawn queue → add actors to pool
did_work |= self.drain_spawns(tc);
let t1 = Instant::now();
// 2. Drain transfer queue → deliver envelopes to actors
while let Some(envelope) = self.transfer_rx.try_recv() {
let dest = envelope.dest();
let payload = envelope.into_payload();
self.pool.deliver(&dest, payload);
did_work = true;
}
let t2 = Instant::now();
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
// 3. Drain admin queue → inspect or mutate worker-owned slots before handlers
did_work |= self.drain_admin(tc);
// 4. Fire per-worker extension (e.g., timers) → deliver before tick_all
let ext_msgs: Vec<_> = self
.worker_ext
.as_mut()
.map(|ext| ext.on_tick())
.unwrap_or_default();
for (dest, msg) in ext_msgs {
route_to_pool_or_remote(&mut self.pool, tc, dest, msg);
did_work = true;
}
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
// 5. Tick all actors with WorkerContext
let pending_local: RefCell<Vec<(ActorAddress, Box<dyn Any + Send>)>> =
RefCell::new(Vec::new());
let stop_requests: RefCell<Vec<ActorAddress>> = RefCell::new(Vec::new());
let stop_with_values: RefCell<Vec<(ActorAddress, ExitValue)>> = RefCell::new(Vec::new());
let suspend_requests: RefCell<Vec<ActorAddress>> = RefCell::new(Vec::new());
let worker_requests: RefCell<Vec<Box<dyn Any + Send>>> = RefCell::new(Vec::new());
let processed;
{
let worker_ctx = WorkerContext {
tc,
pending_local: &pending_local,
stop_requests: &stop_requests,
stop_with_values: &stop_with_values,
suspend_requests: &suspend_requests,
worker_requests: &worker_requests,
stats: &self.stats,
};
processed = self.pool.tick_all(
&worker_ctx,
&self.stats,
tc.config.actor_message_budget,
);
if processed > 0 {
did_work = true;
}
}
let t3 = Instant::now();
#[cfg(feature = "tracing")]
if processed > 0 {
tracing::debug!(
worker_id = 0,
messages_processed = processed,
"worker.tick_all"
);
}
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
// 6. Drain spawn queue again — actors spawned during step 5
// must be in the pool before pending_local delivery.
did_work |= self.drain_spawns(tc);
let t4 = Instant::now();
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
// 7. Drain pending_local buffer → deliver to local actors
let pending = pending_local.into_inner();
if !pending.is_empty() {
did_work = true;
}
for (addr, msg) in pending {
self.pool.deliver(&addr, msg);
}
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
// 7.5. Process worker extension requests from handlers (e.g., timer scheduling)
if let Some(ext) = &mut self.worker_ext {
for request in worker_requests.into_inner() {
ext.handle_request(request);
}
}
let t5 = Instant::now();
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
// 8. Publish stats (skip entirely when idle to avoid allocation + mutex)
if did_work {
self.stats
.num_actors
.store(self.pool.len(), Ordering::Relaxed);
self.stats
.total_mailbox_depth
.store(self.pool.total_mailbox_depth(), Ordering::Relaxed);
self.stats
.messages_processed
.fetch_add(processed as u64, Ordering::Relaxed);
if let Some(hook) = tc.stats_hook {
self.pool.mailbox_depths_into(&mut self.snapshot_buf);
hook.on_tick(0, &self.snapshot_buf);
}
}
let t6 = Instant::now();
// Record tick timing
let timing = TickTiming {
phase_us: [
t1.duration_since(t0).as_micros() as u64,
t2.duration_since(t1).as_micros() as u64,
t3.duration_since(t2).as_micros() as u64,
t4.duration_since(t3).as_micros() as u64,
t5.duration_since(t4).as_micros() as u64,
t6.duration_since(t5).as_micros() as u64,
],
messages_processed: processed,
did_work,
};
self.stats.push_tick_timing(timing);
#[cfg(feature = "tracing")]
if did_work {
tracing::debug!(
worker_id = 0,
num_actors = self.pool.len(),
mailbox_depth = self.pool.total_mailbox_depth(),
messages_processed = processed,
"worker.stats"
);
}
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
// 9. Clean up poisoned and stopping actors
did_work |= self.cleanup_dead_actors(tc);
self.has_backlog = did_work;
did_work
}
}
/// The `ContextInner` impl for in-worker sends.
///
/// All sends to local actors are buffered in `pending_local` (delivered after
/// the current tick round). Non-local addresses route to inbox_registry or remote.
struct WorkerContext<'a> {
tc: &'a TickContext<'a>,
pending_local: &'a RefCell<Vec<(ActorAddress, Box<dyn Any + Send>)>>,
stop_requests: &'a RefCell<Vec<ActorAddress>>,
stop_with_values: &'a RefCell<Vec<(ActorAddress, ExitValue)>>,
suspend_requests: &'a RefCell<Vec<ActorAddress>>,
worker_requests: &'a RefCell<Vec<Box<dyn Any + Send>>>,
stats: &'a WorkerStats,
}
impl ContextInner for WorkerContext<'_> {
fn send_any(&self, addr: ActorAddress, msg: Box<dyn Any + Send>) -> Result<(), Error> {
if self.tc.address_map.contains(&addr) {
self.stats.local_sends.fetch_add(1, Ordering::Relaxed);
self.pending_local.borrow_mut().push((addr, msg));
Ok(())
} else {
self.stats.inbox_sends.fetch_add(1, Ordering::Relaxed);
self.tc.route_nonlocal(addr, msg)
}
}
fn spawn_any(&self, request: SpawnRequest) {
self.tc.address_map.insert(request.addr);
self.tc.spawn_tx.send(request);
}
fn request_stop(&self, addr: ActorAddress) {
self.stop_requests.borrow_mut().push(addr);
}
fn request_stop_with(&self, addr: ActorAddress, value: ExitValue) {
self.stop_with_values.borrow_mut().push((addr, value));
}
fn request_suspend(&self, addr: ActorAddress) {
self.suspend_requests.borrow_mut().push(addr);
}
fn request_resume(&self, addr: ActorAddress) {
self.pending_local
.borrow_mut()
.push((addr, Box::new(ResumeSignal)));
}
fn post_worker_request(&self, request: Box<dyn Any + Send>) {
self.worker_requests.borrow_mut().push(request);
}
fn extension(&self) -> Option<&dyn crate::extension::RuntimeExtension> {
self.tc.extension
}
fn process_output_observer(
&self,
) -> Option<std::sync::Arc<dyn crate::process_observer::ProcessOutputObserver>> {
self.tc.process_output_observer.cloned()
}
fn system_info(&self) -> SystemInfo {
SystemInfo {
worker_id: 0,
num_workers: 1,
total_actors: self.tc.worker_stats.num_actors.load(Ordering::Relaxed),
uptime_ms: self.tc.created_at.elapsed().as_millis() as u64,
}
}
}
struct ActorSlot {
mailbox: VecDeque<Box<dyn Any + Send>>,
actor: Box<dyn AnyActor>,
poisoned: bool,
/// Graceful stop requested (via StopSignal).
stopping: bool,
/// Whether on_start has been called for this actor.
started: bool,
/// Actor is suspended — messages queue but are not processed.
suspended: bool,
last_msg_type: Option<&'static str>,
messages_processed: u64,
/// Per-message-type counters (bounded to 32 entries).
msg_type_counts: HashMap<&'static str, u64>,
/// Address of the actor that spawned this one, or `None` for externally-spawned actors.
parent_addr: Option<ActorAddress>,
/// Inherited environment from parent (or empty for runtime-spawned actors).
env: Environment,
/// Typed exit value set by `ctx.stop_with()`.
exit_value: Option<ExitValue>,
}
/// Per-worker actor storage. Owns per-actor mailboxes.
pub(crate) struct ActorPool {
actors: AddrMap<ActorSlot>,
}
impl ActorPool {
pub fn new() -> Self {
Self {
actors: HashMap::with_hasher(AddrBuildHasher),
}
}
pub fn insert(&mut self, req: SpawnRequest) {
self.actors.insert(
req.addr,
ActorSlot {
mailbox: VecDeque::with_capacity(16),
actor: req.actor,
poisoned: false,
stopping: false,
started: false,
suspended: false,
last_msg_type: None,
messages_processed: 0,
msg_type_counts: HashMap::new(),
parent_addr: req.parent,
env: req.env,
exit_value: None,
},
);
}
/// Deliver a type-erased message to the actor at `addr`.
/// Returns `true` if the actor exists (message enqueued; type check deferred to tick).
pub fn deliver(&mut self, addr: &ActorAddress, msg: Box<dyn Any + Send>) -> bool {
if let Some(slot) = self.actors.get_mut(addr) {
// Intercept control signals for suspended actors: they skip tick_all
// so we must handle resume/stop at delivery time.
if slot.suspended {
if msg.is::<ResumeSignal>() {
slot.suspended = false;
return true;
}
if msg.is::<StopSignal>() {
slot.stopping = true;
slot.mailbox.clear();
return true;
}
if msg.is::<StopWithSignal>() {
if let Ok(sig) = msg.downcast::<StopWithSignal>() {
slot.exit_value = Some(sig.0);
}
slot.stopping = true;
slot.mailbox.clear();
return true;
}
}
slot.mailbox.push_back(msg);
true
} else {
false
}
}
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
fn get_actor_erased(&self, addr: ActorAddress) -> Option<&dyn AnyActor> {
self.actors.get(&addr).map(|slot| slot.actor.as_ref())
}
fn get_actor_erased_mut(
&mut self,
addr: ActorAddress,
) -> Option<&mut (dyn AnyActor + 'static)> {
self.actors.get_mut(&addr).map(|slot| slot.actor.as_mut())
}
fn actor_summary_from_slot(address: ActorAddress, slot: &ActorSlot) -> ActorSummary {
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
let metadata = slot.actor.metadata();
ActorSummary {
address,
actor_type: metadata.actor_type_name,
message_type: metadata.message_type_name,
worker_id: 0,
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
parent: slot.parent_addr,
mailbox_depth: slot.mailbox.len(),
status: ActorStatus {
started: slot.started,
suspended: slot.suspended,
stopping: slot.stopping,
poisoned: slot.poisoned,
},
last_message_type: slot.last_msg_type,
messages_handled: slot.messages_processed,
}
}
fn actor_summary(&self, addr: ActorAddress) -> AdminResult<ActorSummary> {
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
self.actors
.get(&addr)
.map(|slot| Self::actor_summary_from_slot(addr, slot))
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
.ok_or(AdminError::ActorNotFound { actor: addr })
}
fn actor_summaries_into(&self, out: &mut Vec<ActorSummary>) {
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
out.clear();
out.extend(
self.actors
.iter()
.map(|(&addr, slot)| Self::actor_summary_from_slot(addr, slot)),
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
);
}
fn suspend_actor_admin(&mut self, addr: ActorAddress) -> AdminResult<OperationResult> {
match self.actors.get_mut(&addr) {
Some(slot) => {
slot.suspended = true;
Ok(OperationResult { applied: true })
}
None => Err(AdminError::ActorNotFound { actor: addr }),
}
}
fn resume_actor_admin(&mut self, addr: ActorAddress) -> AdminResult<OperationResult> {
match self.actors.get_mut(&addr) {
Some(slot) => {
slot.suspended = false;
Ok(OperationResult { applied: true })
}
None => Err(AdminError::ActorNotFound { actor: addr }),
}
}
fn stop_actor_admin(
&mut self,
addr: ActorAddress,
stats: &WorkerStats,
) -> AdminResult<OperationResult> {
match self.actors.get_mut(&addr) {
Some(slot) => {
if !slot.stopping {
stats.stops.fetch_add(1, Ordering::Relaxed);
}
slot.stopping = true;
slot.mailbox.clear();
Ok(OperationResult { applied: true })
}
None => Err(AdminError::ActorNotFound { actor: addr }),
}
}
/// Tick all actors in the pool. Returns the number of messages processed.
///
/// Each actor processes up to `budget` messages per tick (0 = unlimited).
/// This prevents a single hot actor from starving others on the same worker.
fn tick_all(
&mut self,
wctx: &WorkerContext<'_>,
stats: &WorkerStats,
budget: usize,
) -> usize {
let mut count = 0;
for (&addr, slot) in self.actors.iter_mut() {
if should_skip_actor(slot.poisoned, slot.stopping, slot.suspended) {
// Discard all messages for poisoned/stopping actors (not suspended — those queue)
if slot.poisoned || slot.stopping {
slot.mailbox.clear();
}
continue;
}
debug_assert!(
!slot.poisoned && !slot.stopping && !slot.suspended,
"G4: non-processable actor reached processing"
);
#[cfg(feature = "tracing")]
let _actor_span = tracing::trace_span!("actor.tick", actor_addr = %addr).entered();
// Snapshot self-stats before creating Ctx
let snap_processed = slot.messages_processed;
let snap_depth = slot.mailbox.len();
let mut snap_type_counts: Vec<(&'static str, u64)> =
slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect();
snap_type_counts.sort_by(|a, b| b.1.cmp(&a.1));
let ctx = Ctx::new(
wctx,
addr,
slot.parent_addr,
slot.env.clone(),
snap_processed,
snap_depth,
snap_type_counts,
);
// Call on_start once, before first message
if !slot.started {
let start_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
slot.actor.on_start(&ctx);
}));
slot.started = true;
if start_result.is_err() {
stats.panics.fetch_add(1, Ordering::Relaxed);
eprintln!("swactor: actor {addr} panicked in on_start — poisoned");
#[cfg(feature = "tracing")]
tracing::error!(actor_addr = %addr, "actor.on_start_panicked");
slot.poisoned = true;
slot.mailbox.clear();
continue;
}
// Check if on_start requested stop or stop_with
{
let stops = wctx.stop_requests.borrow();
if !stops.is_empty() && stops.contains(&addr) {
drop(stops);
slot.stopping = true;
stats.stops.fetch_add(1, Ordering::Relaxed);
slot.mailbox.clear();
// Check for stop_with value
let mut sws = wctx.stop_with_values.borrow_mut();
if let Some(pos) = sws.iter().position(|(a, _)| *a == addr) {
let (_, val) = sws.swap_remove(pos);
slot.exit_value = Some(val);
}
continue;
}
}
// Check if on_start requested stop_with (without plain stop)
{
let mut sws = wctx.stop_with_values.borrow_mut();
if let Some(pos) = sws.iter().position(|(a, _)| *a == addr) {
let (_, val) = sws.swap_remove(pos);
slot.exit_value = Some(val);
slot.stopping = true;
stats.stops.fetch_add(1, Ordering::Relaxed);
slot.mailbox.clear();
continue;
}
}
// Check if on_start requested suspend
{
let suspends = wctx.suspend_requests.borrow();
if !suspends.is_empty() && suspends.contains(&addr) {
drop(suspends);
slot.suspended = true;
continue;
}
}
}
let mut actor_count = 0usize;
while let Some(msg) = slot.mailbox.pop_front() {
// Intercept StopSignal (from external runtime.stop_actor)
if msg.is::<StopSignal>() {
slot.stopping = true;
stats.stops.fetch_add(1, Ordering::Relaxed);
slot.mailbox.clear();
#[cfg(feature = "tracing")]
tracing::info!(actor_addr = %addr, "actor.stop_requested");
break;
}
// Intercept StopWithSignal (from external runtime)
if msg.is::<StopWithSignal>() {
if let Ok(sig) = msg.downcast::<StopWithSignal>() {
slot.exit_value = Some(sig.0);
}
slot.stopping = true;
stats.stops.fetch_add(1, Ordering::Relaxed);
slot.mailbox.clear();
break;
}
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
slot.actor.handle_any(&ctx, msg)
}));
match result {
Ok(None) => {
stats.type_mismatches.fetch_add(1, Ordering::Relaxed);
}
Err(_) => {
stats.panics.fetch_add(1, Ordering::Relaxed);
slot.mailbox.clear();
eprintln!(
"swactor: actor {addr} panicked — poisoned, future messages will be discarded"
);
#[cfg(feature = "tracing")]
tracing::error!(actor_addr = %addr, "actor.panicked");
slot.poisoned = true;
slot.mailbox.clear();
break;
}
Ok(Some(type_name)) => {
slot.last_msg_type = Some(type_name);
slot.messages_processed += 1;
// Track per-type counts (bounded to 32 distinct types)
if slot.msg_type_counts.len() < 32
|| slot.msg_type_counts.contains_key(type_name)
{
*slot.msg_type_counts.entry(type_name).or_insert(0) += 1;
}
}
}
count += 1;
actor_count += 1;
// Check if handler requested self-stop or stop_with
{
let stops = wctx.stop_requests.borrow();
let has_stop = !stops.is_empty() && stops.contains(&addr);
drop(stops);
let mut sws = wctx.stop_with_values.borrow_mut();
let sw_pos = sws.iter().position(|(a, _)| *a == addr);
if has_stop || sw_pos.is_some() {
if let Some(pos) = sw_pos {
let (_, val) = sws.swap_remove(pos);
slot.exit_value = Some(val);
}
drop(sws);
slot.stopping = true;
stats.stops.fetch_add(1, Ordering::Relaxed);
slot.mailbox.clear();
break;
}
drop(sws);
}
// Check if handler requested suspend
{
let suspends = wctx.suspend_requests.borrow();
if !suspends.is_empty() && suspends.contains(&addr) {
drop(suspends);
slot.suspended = true;
break; // stop processing this actor's messages this tick
}
}
if budget > 0 && actor_count >= budget {
break;
}
}
}
count
}
pub fn len(&self) -> usize {
self.actors.len()
}
pub fn contains(&self, addr: &ActorAddress) -> bool {
self.actors.contains_key(addr)
}
pub fn total_mailbox_depth(&self) -> usize {
self.actors.values().map(|slot| slot.mailbox.len()).sum()
}
/// Remove poisoned and stopping actors, returning their addresses, stop reasons,
/// and optional exit values.
/// Called after tick_all so the caller can clean up the address map.
///
/// For stopping actors: calls `on_stop()` before removal (wrapped in catch_unwind).
/// For poisoned actors: `on_stop()` is NOT called (state may be corrupt).
pub fn cleanup_dead(
&mut self,
inner: &dyn ContextInner,
) -> Vec<(ActorAddress, StopReason, Option<ExitValue>)> {
let dead_addrs: Vec<ActorAddress> = self
.actors
.iter()
.filter(|(_, slot)| slot.poisoned || slot.stopping)
.map(|(&addr, _)| addr)
.collect();
let mut dead = Vec::with_capacity(dead_addrs.len());
for addr in dead_addrs {
if let Some(mut slot) = self.actors.remove(&addr) {
let reason = determine_stop_reason(slot.poisoned, slot.exit_value.is_some());
// Call on_stop for gracefully stopping actors only
debug_assert!(
slot.poisoned || slot.stopping,
"G4: non-dead actor reached cleanup_dead"
);
if is_on_stop_eligible(slot.stopping, slot.poisoned) {
let mut type_counts: Vec<(&'static str, u64)> =
slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect();
type_counts.sort_by(|a, b| b.1.cmp(&a.1));
let ctx = Ctx::new(
inner,
addr,
slot.parent_addr,
slot.env.clone(),
slot.messages_processed,
slot.mailbox.len(),
type_counts,
);
let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
slot.actor.on_stop(&ctx);
}));
}
dead.push((addr, reason, slot.exit_value.take()));
// slot is dropped here — actor resources freed
}
}
dead
}
/// Fill `out` with per-actor snapshots, reusing the existing allocation.
pub fn mailbox_depths_into(&self, out: &mut Vec<ActorSnapshot>) {
out.clear();
out.extend(self.actors.iter().map(|(&addr, slot)| {
let mut type_counts: Vec<(&'static str, u64)> =
slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect();
type_counts.sort_by(|a, b| b.1.cmp(&a.1));
let metadata = slot.actor.metadata();
ActorSnapshot {
address: addr,
mailbox_depth: slot.mailbox.len(),
last_msg_type: slot.last_msg_type,
actor_type: Some(metadata.actor_type_name),
message_type: Some(metadata.message_type_name),
messages_processed: slot.messages_processed,
poisoned: slot.poisoned,
message_type_counts: type_counts,
}
}));
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::actor::StopReason;
#[test]
fn lifecycle_decision_helpers_cover_all_inputs() {
let mut skip_combinations = 0;
for poisoned in [false, true] {
for stopping in [false, true] {
for suspended in [false, true] {
assert_eq!(
should_skip_actor(poisoned, stopping, suspended),
poisoned || stopping || suspended
);
skip_combinations += 1;
}
}
}
assert_eq!(skip_combinations, 8);
let mut on_stop_combinations = 0;
for stopping in [false, true] {
for poisoned in [false, true] {
assert_eq!(
is_on_stop_eligible(stopping, poisoned),
stopping && !poisoned
);
on_stop_combinations += 1;
}
}
assert_eq!(on_stop_combinations, 4);
let mut reason_combinations = 0;
for poisoned in [false, true] {
for has_exit_value in [false, true] {
let expected = if poisoned {
StopReason::Panicked
} else if has_exit_value {
StopReason::Completed
} else {
StopReason::Normal
};
assert_eq!(determine_stop_reason(poisoned, has_exit_value), expected);
reason_combinations += 1;
}
}
assert_eq!(reason_combinations, 4);
}
#[test]
fn bounded_lifecycle_model_preserves_callback_invariants() {
#[derive(Clone, Copy)]
enum Step {
Tick,
RequestStop,
Poison,
Suspend,
Resume,
Cleanup,
}
#[derive(Clone, Copy, Default)]
struct ActorProbe {
started: bool,
stopping: bool,
poisoned: bool,
suspended: bool,
removed: bool,
on_start_count: u8,
handle_count: u8,
on_stop_count: u8,
handled_after_on_stop: bool,
}
impl ActorProbe {
fn apply(&mut self, step: Step) {
if self.removed {
return;
}
match step {
Step::Tick => {
let eligible =
!should_skip_actor(self.poisoned, self.stopping, self.suspended);
if !self.started && eligible {
self.started = true;
self.on_start_count += 1;
}
if self.started && eligible {
if self.on_stop_count > 0 {
self.handled_after_on_stop = true;
}
self.handle_count += 1;
}
}
Step::RequestStop => {
self.stopping = true;
}
Step::Poison => {
self.poisoned = true;
}
Step::Suspend => {
self.suspended = true;
}
Step::Resume => {
self.suspended = false;
}
Step::Cleanup => {
if self.stopping || self.poisoned {
if is_on_stop_eligible(self.stopping, self.poisoned) {
self.on_stop_count += 1;
}
self.removed = true;
}
}
}
}
fn assert_invariants(self) {
assert!(self.on_start_count <= 1, "on_start fired more than once");
assert!(self.on_stop_count <= 1, "on_stop fired more than once");
assert!(
!self.handled_after_on_stop,
"handle fired after on_stop cleanup"
);
if self.handle_count > 0 {
assert!(self.started, "handle fired before on_start");
}
if self.on_stop_count > 0 {
assert!(self.removed, "on_stop fired without cleanup");
assert!(!self.poisoned, "poisoned actor ran on_stop");
}
}
}
fn walk(depth: usize, state: ActorProbe, checked: &mut usize) {
state.assert_invariants();
*checked += 1;
if depth == 0 {
return;
}
for step in [
Step::Tick,
Step::RequestStop,
Step::Poison,
Step::Suspend,
Step::Resume,
Step::Cleanup,
] {
let mut next = state;
next.apply(step);
walk(depth - 1, next, checked);
}
}
let mut checked = 0;
walk(6, ActorProbe::default(), &mut checked);
assert_eq!(checked, 55_987);
}
}