feat: actor direct control via runtime admin
Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
This commit is contained in:
parent
f54f62b491
commit
7627ef3e18
6 changed files with 1238 additions and 12 deletions
39
src/actor.rs
39
src/actor.rs
|
|
@ -443,6 +443,22 @@ impl<A: ActorInterface> Actor<A> {
|
||||||
pub fn new(inner: A) -> Self {
|
pub fn new(inner: A) -> Self {
|
||||||
Self { inner }
|
Self { inner }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn inner(&self) -> &A {
|
||||||
|
&self.inner
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn replace_inner(&mut self, inner: A) {
|
||||||
|
self.inner = inner;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub struct ActorTypeMetadata {
|
||||||
|
pub actor_type_id: TypeId,
|
||||||
|
pub actor_type_name: &'static str,
|
||||||
|
pub message_type_id: TypeId,
|
||||||
|
pub message_type_name: &'static str,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Trait for type-erased actors — single-message handler.
|
/// Trait for type-erased actors — single-message handler.
|
||||||
|
|
@ -456,6 +472,12 @@ pub trait AnyActor: Send {
|
||||||
|
|
||||||
/// Called on graceful stop, before removal. See [`ActorInterface::on_stop`].
|
/// Called on graceful stop, before removal. See [`ActorInterface::on_stop`].
|
||||||
fn on_stop(&mut self, _ctx: &Ctx) {}
|
fn on_stop(&mut self, _ctx: &Ctx) {}
|
||||||
|
|
||||||
|
fn metadata(&self) -> ActorTypeMetadata;
|
||||||
|
|
||||||
|
fn as_any(&self) -> &dyn Any;
|
||||||
|
|
||||||
|
fn as_any_mut(&mut self) -> &mut dyn Any;
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<A> AnyActor for Actor<A>
|
impl<A> AnyActor for Actor<A>
|
||||||
|
|
@ -493,6 +515,23 @@ where
|
||||||
fn on_stop(&mut self, ctx: &Ctx) {
|
fn on_stop(&mut self, ctx: &Ctx) {
|
||||||
self.inner.on_stop(ctx);
|
self.inner.on_stop(ctx);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn metadata(&self) -> ActorTypeMetadata {
|
||||||
|
ActorTypeMetadata {
|
||||||
|
actor_type_id: TypeId::of::<A>(),
|
||||||
|
actor_type_name: std::any::type_name::<A>(),
|
||||||
|
message_type_id: TypeId::of::<A::Incoming>(),
|
||||||
|
message_type_name: std::any::type_name::<A::Incoming>(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn as_any(&self) -> &dyn Any {
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
fn as_any_mut(&mut self) -> &mut dyn Any {
|
||||||
|
self
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Unique token identifying a monitor subscription.
|
/// Unique token identifying a monitor subscription.
|
||||||
|
|
|
||||||
163
src/admin.rs
Normal file
163
src/admin.rs
Normal file
|
|
@ -0,0 +1,163 @@
|
||||||
|
use crate::actor::{ActorAddress, ActorInterface, ActorTypeMetadata, AnyActor, Message};
|
||||||
|
use crate::runtime::{Inbox, Runtime};
|
||||||
|
use parking_lot::Mutex;
|
||||||
|
use std::any::Any;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::AtomicUsize;
|
||||||
|
|
||||||
|
pub type AdminResult<T> = Result<T, AdminError>;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub enum AdminError {
|
||||||
|
ActorNotFound {
|
||||||
|
actor: ActorAddress,
|
||||||
|
},
|
||||||
|
AddressMismatch {
|
||||||
|
requested: ActorAddress,
|
||||||
|
snapshot: ActorAddress,
|
||||||
|
},
|
||||||
|
TypeMismatch {
|
||||||
|
expected_actor_type: &'static str,
|
||||||
|
expected_message_type: &'static str,
|
||||||
|
actual_actor_type: &'static str,
|
||||||
|
actual_message_type: &'static str,
|
||||||
|
},
|
||||||
|
Timeout,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct RuntimeAdmin<'a> {
|
||||||
|
pub(crate) runtime: &'a Runtime,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct Admin<T: Message> {
|
||||||
|
pub(crate) inbox: Inbox<AdminResult<T>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct OperationResult {
|
||||||
|
pub applied: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct ActorStatus {
|
||||||
|
pub started: bool,
|
||||||
|
pub suspended: bool,
|
||||||
|
pub stopping: bool,
|
||||||
|
pub poisoned: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct ActorSummary {
|
||||||
|
pub address: ActorAddress,
|
||||||
|
pub actor_type: &'static str,
|
||||||
|
pub message_type: &'static str,
|
||||||
|
pub worker_id: usize,
|
||||||
|
pub parent: Option<ActorAddress>,
|
||||||
|
pub mailbox_depth: usize,
|
||||||
|
pub status: ActorStatus,
|
||||||
|
pub last_message_type: Option<&'static str>,
|
||||||
|
pub messages_handled: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct ListActorsResponse {
|
||||||
|
pub actors: Vec<ActorSummary>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct InspectActorResponse {
|
||||||
|
pub summary: ActorSummary,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct ActorStateSnapshot<A> {
|
||||||
|
pub actor: ActorAddress,
|
||||||
|
pub actor_type: &'static str,
|
||||||
|
pub message_type: &'static str,
|
||||||
|
pub actor_instance: A,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct GetActorStateResponse<A> {
|
||||||
|
pub state: ActorStateSnapshot<A>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<A: ActorInterface> ActorStateSnapshot<A> {
|
||||||
|
pub fn new(actor: ActorAddress, actor_instance: A) -> Self {
|
||||||
|
Self {
|
||||||
|
actor,
|
||||||
|
actor_type: std::any::type_name::<A>(),
|
||||||
|
message_type: std::any::type_name::<A::Incoming>(),
|
||||||
|
actor_instance,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T: Message> Admin<T> {
|
||||||
|
pub(crate) fn new(inbox: Inbox<AdminResult<T>>) -> Self {
|
||||||
|
Self { inbox }
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn try_recv(&self) -> Option<AdminResult<T>> {
|
||||||
|
self.inbox.try_recv()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn recv_ticking(&self, rt: &Runtime, max_ticks: usize) -> AdminResult<T> {
|
||||||
|
for _ in 0..max_ticks {
|
||||||
|
rt.tick();
|
||||||
|
if let Some(resp) = self.inbox.try_recv() {
|
||||||
|
return resp;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(AdminError::Timeout)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn reply_addr(&self) -> &ActorAddress {
|
||||||
|
self.inbox.addr()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) type AdminBoxedReply = Box<dyn Any + Send>;
|
||||||
|
|
||||||
|
pub(crate) enum AdminCommand {
|
||||||
|
ListActors {
|
||||||
|
acc: Arc<ListActorsAccumulator>,
|
||||||
|
},
|
||||||
|
InspectActor {
|
||||||
|
actor: ActorAddress,
|
||||||
|
reply_to: ActorAddress,
|
||||||
|
},
|
||||||
|
GetActorState {
|
||||||
|
actor: ActorAddress,
|
||||||
|
reply_to: ActorAddress,
|
||||||
|
get: Box<
|
||||||
|
dyn FnOnce(ActorAddress, &dyn AnyActor, ActorTypeMetadata) -> AdminBoxedReply + Send,
|
||||||
|
>,
|
||||||
|
not_found: Box<dyn FnOnce(ActorAddress) -> AdminBoxedReply + Send>,
|
||||||
|
},
|
||||||
|
ReplaceActorState {
|
||||||
|
actor: ActorAddress,
|
||||||
|
reply_to: ActorAddress,
|
||||||
|
replace: Box<
|
||||||
|
dyn FnOnce(&mut dyn AnyActor, ActorTypeMetadata) -> AdminResult<OperationResult> + Send,
|
||||||
|
>,
|
||||||
|
},
|
||||||
|
StopActor {
|
||||||
|
actor: ActorAddress,
|
||||||
|
reply_to: ActorAddress,
|
||||||
|
},
|
||||||
|
SuspendActor {
|
||||||
|
actor: ActorAddress,
|
||||||
|
reply_to: ActorAddress,
|
||||||
|
},
|
||||||
|
ResumeActor {
|
||||||
|
actor: ActorAddress,
|
||||||
|
reply_to: ActorAddress,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) struct ListActorsAccumulator {
|
||||||
|
pub(crate) remaining: AtomicUsize,
|
||||||
|
pub(crate) summaries: Mutex<Vec<ActorSummary>>,
|
||||||
|
pub(crate) reply_to: ActorAddress,
|
||||||
|
}
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
pub mod actor;
|
pub mod actor;
|
||||||
|
pub mod admin;
|
||||||
pub mod extension;
|
pub mod extension;
|
||||||
pub mod process_observer;
|
pub mod process_observer;
|
||||||
pub mod worker;
|
pub mod worker;
|
||||||
|
|
|
||||||
243
src/runtime.rs
243
src/runtime.rs
|
|
@ -1,15 +1,19 @@
|
||||||
use crate::Instant;
|
use crate::Instant;
|
||||||
use std::any::Any;
|
use std::any::{Any, TypeId};
|
||||||
use std::cell::RefCell;
|
use std::cell::RefCell;
|
||||||
use std::sync::atomic::{AtomicBool, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||||
use std::sync::{Arc, OnceLock};
|
use std::sync::{Arc, OnceLock};
|
||||||
use std::thread::Thread;
|
use std::thread::Thread;
|
||||||
#[cfg(not(target_arch = "wasm32"))]
|
#[cfg(not(target_arch = "wasm32"))]
|
||||||
use std::thread::{self, JoinHandle};
|
use std::thread::{self, JoinHandle};
|
||||||
|
|
||||||
use crate::actor::{
|
use crate::actor::{
|
||||||
Actor, ActorAddress, ActorInterface, AnyActor, Environment, ExitValue, Message, ResumeSignal,
|
Actor, ActorAddress, ActorInterface, ActorTypeMetadata, AnyActor, Environment, ExitValue,
|
||||||
SpawnRequest, StopSignal, StopWithSignal, SystemInfo,
|
Message, ResumeSignal, SpawnRequest, StopSignal, StopWithSignal, SystemInfo,
|
||||||
|
};
|
||||||
|
use crate::admin::{
|
||||||
|
ActorStateSnapshot, Admin, AdminCommand, AdminError, AdminResult, GetActorStateResponse,
|
||||||
|
InspectActorResponse, ListActorsAccumulator, ListActorsResponse, OperationResult, RuntimeAdmin,
|
||||||
};
|
};
|
||||||
use crate::channel::{Receiver, Sender};
|
use crate::channel::{Receiver, Sender};
|
||||||
// Re-export config types so existing code using `runtime::RuntimeConfig` still works
|
// Re-export config types so existing code using `runtime::RuntimeConfig` still works
|
||||||
|
|
@ -106,6 +110,7 @@ pub struct Runtime {
|
||||||
extension: Option<Arc<dyn RuntimeExtension>>,
|
extension: Option<Arc<dyn RuntimeExtension>>,
|
||||||
transfer_txs: Vec<Sender<Envelope>>,
|
transfer_txs: Vec<Sender<Envelope>>,
|
||||||
spawn_txs: Vec<Sender<SpawnRequest>>,
|
spawn_txs: Vec<Sender<SpawnRequest>>,
|
||||||
|
admin_txs: Vec<Sender<AdminCommand>>,
|
||||||
placement: Placement,
|
placement: Placement,
|
||||||
is_running: AtomicBool,
|
is_running: AtomicBool,
|
||||||
worker_stats: Vec<Arc<WorkerStats>>,
|
worker_stats: Vec<Arc<WorkerStats>>,
|
||||||
|
|
@ -221,6 +226,7 @@ impl Runtime {
|
||||||
|
|
||||||
let mut transfer_txs = Vec::with_capacity(num_workers);
|
let mut transfer_txs = Vec::with_capacity(num_workers);
|
||||||
let mut spawn_txs = Vec::with_capacity(num_workers);
|
let mut spawn_txs = Vec::with_capacity(num_workers);
|
||||||
|
let mut admin_txs = Vec::with_capacity(num_workers);
|
||||||
let mut worker_stats = Vec::with_capacity(num_workers);
|
let mut worker_stats = Vec::with_capacity(num_workers);
|
||||||
let mut workers = Vec::with_capacity(num_workers);
|
let mut workers = Vec::with_capacity(num_workers);
|
||||||
|
|
||||||
|
|
@ -233,9 +239,19 @@ impl Runtime {
|
||||||
let spawn_tx = spawn_rx.new_sender();
|
let spawn_tx = spawn_rx.new_sender();
|
||||||
spawn_txs.push(spawn_tx);
|
spawn_txs.push(spawn_tx);
|
||||||
|
|
||||||
|
let admin_rx = Receiver::<AdminCommand>::new(config.channel_buffer_size);
|
||||||
|
let admin_tx = admin_rx.new_sender();
|
||||||
|
admin_txs.push(admin_tx);
|
||||||
|
|
||||||
let stats = Arc::new(WorkerStats::new());
|
let stats = Arc::new(WorkerStats::new());
|
||||||
worker_stats.push(stats.clone());
|
worker_stats.push(stats.clone());
|
||||||
workers.push(Worker::new(WorkerId(i), transfer_rx, spawn_rx, stats));
|
workers.push(Worker::new(
|
||||||
|
WorkerId(i),
|
||||||
|
transfer_rx,
|
||||||
|
spawn_rx,
|
||||||
|
admin_rx,
|
||||||
|
stats,
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
let placement = Placement::new(num_workers, worker_stats.clone());
|
let placement = Placement::new(num_workers, worker_stats.clone());
|
||||||
|
|
@ -250,6 +266,7 @@ impl Runtime {
|
||||||
extension: None,
|
extension: None,
|
||||||
transfer_txs,
|
transfer_txs,
|
||||||
spawn_txs,
|
spawn_txs,
|
||||||
|
admin_txs,
|
||||||
placement,
|
placement,
|
||||||
is_running: AtomicBool::new(false),
|
is_running: AtomicBool::new(false),
|
||||||
worker_stats,
|
worker_stats,
|
||||||
|
|
@ -342,6 +359,10 @@ impl Runtime {
|
||||||
self.extension.as_deref()
|
self.extension.as_deref()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn admin(&self) -> RuntimeAdmin<'_> {
|
||||||
|
RuntimeAdmin { runtime: self }
|
||||||
|
}
|
||||||
|
|
||||||
/// Send a request and get a handle for the response.
|
/// Send a request and get a handle for the response.
|
||||||
///
|
///
|
||||||
/// Creates a temporary inbox, calls `msg_builder` with the inbox's address
|
/// Creates a temporary inbox, calls `msg_builder` with the inbox's address
|
||||||
|
|
@ -590,6 +611,218 @@ impl Runtime {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl RuntimeAdmin<'_> {
|
||||||
|
fn new_admin<T: Message>(&self) -> Result<(Admin<T>, ActorAddress), Error> {
|
||||||
|
let inbox = self.runtime.new_inbox::<AdminResult<T>>()?;
|
||||||
|
let reply_to = *inbox.addr();
|
||||||
|
Ok((Admin::new(inbox), reply_to))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ready<T: Message>(&self, result: AdminResult<T>) -> Result<Admin<T>, Error> {
|
||||||
|
let (admin, reply_to) = self.new_admin::<T>()?;
|
||||||
|
let _ = self
|
||||||
|
.runtime
|
||||||
|
.inbox_registry
|
||||||
|
.try_deliver(reply_to, Box::new(result));
|
||||||
|
Ok(admin)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn list_actors(&self) -> Result<Admin<ListActorsResponse>, Error> {
|
||||||
|
let (admin, reply_to) = self.new_admin::<ListActorsResponse>()?;
|
||||||
|
let acc = Arc::new(ListActorsAccumulator {
|
||||||
|
remaining: AtomicUsize::new(self.runtime.admin_txs.len()),
|
||||||
|
summaries: parking_lot::Mutex::new(Vec::new()),
|
||||||
|
reply_to,
|
||||||
|
});
|
||||||
|
|
||||||
|
for (idx, tx) in self.runtime.admin_txs.iter().enumerate() {
|
||||||
|
tx.send(AdminCommand::ListActors { acc: acc.clone() });
|
||||||
|
notify_worker(&self.runtime.worker_threads, idx);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(admin)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn inspect_actor(&self, actor: ActorAddress) -> Result<Admin<InspectActorResponse>, Error> {
|
||||||
|
let Some(wid) = self.runtime.address_map.lookup(&actor) else {
|
||||||
|
return self.ready::<InspectActorResponse>(Err(AdminError::ActorNotFound { actor }));
|
||||||
|
};
|
||||||
|
let (admin, reply_to) = self.new_admin::<InspectActorResponse>()?;
|
||||||
|
let worker_idx = wid.as_usize();
|
||||||
|
self.runtime.admin_txs[worker_idx].send(AdminCommand::InspectActor { actor, reply_to });
|
||||||
|
notify_worker(&self.runtime.worker_threads, worker_idx);
|
||||||
|
Ok(admin)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_actor_state<A>(
|
||||||
|
&self,
|
||||||
|
actor: ActorAddress,
|
||||||
|
) -> Result<Admin<GetActorStateResponse<A>>, Error>
|
||||||
|
where
|
||||||
|
A: ActorInterface + Clone + Sync,
|
||||||
|
{
|
||||||
|
let Some(wid) = self.runtime.address_map.lookup(&actor) else {
|
||||||
|
return self
|
||||||
|
.ready::<GetActorStateResponse<A>>(Err(AdminError::ActorNotFound { actor }));
|
||||||
|
};
|
||||||
|
let (admin, reply_to) = self.new_admin::<GetActorStateResponse<A>>()?;
|
||||||
|
|
||||||
|
let get = Box::new(
|
||||||
|
|actor: ActorAddress,
|
||||||
|
erased: &dyn AnyActor,
|
||||||
|
metadata: ActorTypeMetadata|
|
||||||
|
-> Box<dyn Any + Send> {
|
||||||
|
let expected_actor_type = std::any::type_name::<A>();
|
||||||
|
let expected_message_type = std::any::type_name::<A::Incoming>();
|
||||||
|
if metadata.actor_type_id != TypeId::of::<A>()
|
||||||
|
|| metadata.message_type_id != TypeId::of::<A::Incoming>()
|
||||||
|
{
|
||||||
|
return Box::new(Err::<GetActorStateResponse<A>, AdminError>(
|
||||||
|
AdminError::TypeMismatch {
|
||||||
|
expected_actor_type,
|
||||||
|
expected_message_type,
|
||||||
|
actual_actor_type: metadata.actor_type_name,
|
||||||
|
actual_message_type: metadata.message_type_name,
|
||||||
|
},
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
let Some(typed) = erased.as_any().downcast_ref::<Actor<A>>() else {
|
||||||
|
return Box::new(Err::<GetActorStateResponse<A>, AdminError>(
|
||||||
|
AdminError::TypeMismatch {
|
||||||
|
expected_actor_type,
|
||||||
|
expected_message_type,
|
||||||
|
actual_actor_type: metadata.actor_type_name,
|
||||||
|
actual_message_type: metadata.message_type_name,
|
||||||
|
},
|
||||||
|
));
|
||||||
|
};
|
||||||
|
|
||||||
|
Box::new(Ok::<GetActorStateResponse<A>, AdminError>(
|
||||||
|
GetActorStateResponse {
|
||||||
|
state: ActorStateSnapshot {
|
||||||
|
actor,
|
||||||
|
actor_type: metadata.actor_type_name,
|
||||||
|
message_type: metadata.message_type_name,
|
||||||
|
actor_instance: typed.inner().clone(),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
))
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let not_found = Box::new(|actor| {
|
||||||
|
Box::new(Err::<GetActorStateResponse<A>, AdminError>(
|
||||||
|
AdminError::ActorNotFound { actor },
|
||||||
|
)) as Box<dyn Any + Send>
|
||||||
|
});
|
||||||
|
|
||||||
|
let worker_idx = wid.as_usize();
|
||||||
|
self.runtime.admin_txs[worker_idx].send(AdminCommand::GetActorState {
|
||||||
|
actor,
|
||||||
|
reply_to,
|
||||||
|
get,
|
||||||
|
not_found,
|
||||||
|
});
|
||||||
|
notify_worker(&self.runtime.worker_threads, worker_idx);
|
||||||
|
Ok(admin)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn replace_actor_state<A>(
|
||||||
|
&self,
|
||||||
|
actor: ActorAddress,
|
||||||
|
state: ActorStateSnapshot<A>,
|
||||||
|
) -> Result<Admin<OperationResult>, Error>
|
||||||
|
where
|
||||||
|
A: ActorInterface,
|
||||||
|
{
|
||||||
|
if state.actor != actor {
|
||||||
|
return self.ready::<OperationResult>(Err(AdminError::AddressMismatch {
|
||||||
|
requested: actor,
|
||||||
|
snapshot: state.actor,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
let Some(wid) = self.runtime.address_map.lookup(&actor) else {
|
||||||
|
return self.ready::<OperationResult>(Err(AdminError::ActorNotFound { actor }));
|
||||||
|
};
|
||||||
|
let (admin, reply_to) = self.new_admin::<OperationResult>()?;
|
||||||
|
|
||||||
|
let actor_instance = state.actor_instance;
|
||||||
|
let replace = Box::new(
|
||||||
|
move |erased: &mut dyn AnyActor,
|
||||||
|
metadata: ActorTypeMetadata|
|
||||||
|
-> AdminResult<OperationResult> {
|
||||||
|
let expected_actor_type = std::any::type_name::<A>();
|
||||||
|
let expected_message_type = std::any::type_name::<A::Incoming>();
|
||||||
|
if metadata.actor_type_id != TypeId::of::<A>()
|
||||||
|
|| metadata.message_type_id != TypeId::of::<A::Incoming>()
|
||||||
|
{
|
||||||
|
return Err(AdminError::TypeMismatch {
|
||||||
|
expected_actor_type,
|
||||||
|
expected_message_type,
|
||||||
|
actual_actor_type: metadata.actor_type_name,
|
||||||
|
actual_message_type: metadata.message_type_name,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let Some(typed) = erased.as_any_mut().downcast_mut::<Actor<A>>() else {
|
||||||
|
return Err(AdminError::TypeMismatch {
|
||||||
|
expected_actor_type,
|
||||||
|
expected_message_type,
|
||||||
|
actual_actor_type: metadata.actor_type_name,
|
||||||
|
actual_message_type: metadata.message_type_name,
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
typed.replace_inner(actor_instance);
|
||||||
|
Ok(OperationResult { applied: true })
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
let worker_idx = wid.as_usize();
|
||||||
|
self.runtime.admin_txs[worker_idx].send(AdminCommand::ReplaceActorState {
|
||||||
|
actor,
|
||||||
|
reply_to,
|
||||||
|
replace,
|
||||||
|
});
|
||||||
|
notify_worker(&self.runtime.worker_threads, worker_idx);
|
||||||
|
Ok(admin)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn stop_actor(&self, actor: ActorAddress) -> Result<Admin<OperationResult>, Error> {
|
||||||
|
let Some(wid) = self.runtime.address_map.lookup(&actor) else {
|
||||||
|
return self.ready::<OperationResult>(Err(AdminError::ActorNotFound { actor }));
|
||||||
|
};
|
||||||
|
let (admin, reply_to) = self.new_admin::<OperationResult>()?;
|
||||||
|
let worker_idx = wid.as_usize();
|
||||||
|
self.runtime.admin_txs[worker_idx].send(AdminCommand::StopActor { actor, reply_to });
|
||||||
|
notify_worker(&self.runtime.worker_threads, worker_idx);
|
||||||
|
Ok(admin)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn suspend_actor(&self, actor: ActorAddress) -> Result<Admin<OperationResult>, Error> {
|
||||||
|
let Some(wid) = self.runtime.address_map.lookup(&actor) else {
|
||||||
|
return self.ready::<OperationResult>(Err(AdminError::ActorNotFound { actor }));
|
||||||
|
};
|
||||||
|
let (admin, reply_to) = self.new_admin::<OperationResult>()?;
|
||||||
|
let worker_idx = wid.as_usize();
|
||||||
|
self.runtime.admin_txs[worker_idx].send(AdminCommand::SuspendActor { actor, reply_to });
|
||||||
|
notify_worker(&self.runtime.worker_threads, worker_idx);
|
||||||
|
Ok(admin)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn resume_actor(&self, actor: ActorAddress) -> Result<Admin<OperationResult>, Error> {
|
||||||
|
let Some(wid) = self.runtime.address_map.lookup(&actor) else {
|
||||||
|
return self.ready::<OperationResult>(Err(AdminError::ActorNotFound { actor }));
|
||||||
|
};
|
||||||
|
let (admin, reply_to) = self.new_admin::<OperationResult>()?;
|
||||||
|
let worker_idx = wid.as_usize();
|
||||||
|
self.runtime.admin_txs[worker_idx].send(AdminCommand::ResumeActor { actor, reply_to });
|
||||||
|
notify_worker(&self.runtime.worker_threads, worker_idx);
|
||||||
|
Ok(admin)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Wake a parked worker thread so it can process new work.
|
/// Wake a parked worker thread so it can process new work.
|
||||||
/// No-op if the thread handle hasn't been registered yet (single-threaded tick mode).
|
/// No-op if the thread handle hasn't been registered yet (single-threaded tick mode).
|
||||||
#[inline]
|
#[inline]
|
||||||
|
|
|
||||||
196
src/worker.rs
196
src/worker.rs
|
|
@ -11,6 +11,10 @@ use crate::actor::{
|
||||||
ActorAddress, AnyActor, ContextInner, Ctx, Environment, ExitValue, ResumeSignal, SpawnRequest,
|
ActorAddress, AnyActor, ContextInner, Ctx, Environment, ExitValue, ResumeSignal, SpawnRequest,
|
||||||
StopReason, StopSignal, StopWithSignal, SystemInfo,
|
StopReason, StopSignal, StopWithSignal, SystemInfo,
|
||||||
};
|
};
|
||||||
|
use crate::admin::{
|
||||||
|
ActorStatus, ActorSummary, AdminCommand, AdminError, AdminResult, InspectActorResponse,
|
||||||
|
ListActorsResponse, OperationResult,
|
||||||
|
};
|
||||||
use crate::channel::Receiver;
|
use crate::channel::Receiver;
|
||||||
use crate::delivery::{AddrBuildHasher, AddrMap, Envelope, TickContext, WorkerId};
|
use crate::delivery::{AddrBuildHasher, AddrMap, Envelope, TickContext, WorkerId};
|
||||||
use crate::stats::{ActorSnapshot, TickTiming, WorkerStats};
|
use crate::stats::{ActorSnapshot, TickTiming, WorkerStats};
|
||||||
|
|
@ -69,6 +73,7 @@ pub(crate) struct Worker {
|
||||||
pub(crate) pool: ActorPool,
|
pub(crate) pool: ActorPool,
|
||||||
transfer_rx: Receiver<Envelope>,
|
transfer_rx: Receiver<Envelope>,
|
||||||
spawn_rx: Receiver<SpawnRequest>,
|
spawn_rx: Receiver<SpawnRequest>,
|
||||||
|
admin_rx: Receiver<AdminCommand>,
|
||||||
stats: Arc<WorkerStats>,
|
stats: Arc<WorkerStats>,
|
||||||
/// Reusable scratch buffer for building per-actor snapshots.
|
/// Reusable scratch buffer for building per-actor snapshots.
|
||||||
snapshot_buf: Vec<ActorSnapshot>,
|
snapshot_buf: Vec<ActorSnapshot>,
|
||||||
|
|
@ -84,6 +89,7 @@ impl Worker {
|
||||||
id: WorkerId,
|
id: WorkerId,
|
||||||
transfer_rx: Receiver<Envelope>,
|
transfer_rx: Receiver<Envelope>,
|
||||||
spawn_rx: Receiver<SpawnRequest>,
|
spawn_rx: Receiver<SpawnRequest>,
|
||||||
|
admin_rx: Receiver<AdminCommand>,
|
||||||
stats: Arc<WorkerStats>,
|
stats: Arc<WorkerStats>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
|
|
@ -91,6 +97,7 @@ impl Worker {
|
||||||
pool: ActorPool::new(),
|
pool: ActorPool::new(),
|
||||||
transfer_rx,
|
transfer_rx,
|
||||||
spawn_rx,
|
spawn_rx,
|
||||||
|
admin_rx,
|
||||||
stats,
|
stats,
|
||||||
snapshot_buf: Vec::new(),
|
snapshot_buf: Vec::new(),
|
||||||
worker_ext: None,
|
worker_ext: None,
|
||||||
|
|
@ -132,6 +139,88 @@ impl Worker {
|
||||||
did_work
|
did_work
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn drain_admin(&mut self, tc: &TickContext) -> bool {
|
||||||
|
let mut did_work = false;
|
||||||
|
while let Some(cmd) = self.admin_rx.try_recv() {
|
||||||
|
did_work = true;
|
||||||
|
self.apply_admin_command(tc, cmd);
|
||||||
|
}
|
||||||
|
did_work
|
||||||
|
}
|
||||||
|
|
||||||
|
fn send_admin_reply<T: crate::actor::Message>(
|
||||||
|
tc: &TickContext,
|
||||||
|
reply_to: ActorAddress,
|
||||||
|
result: AdminResult<T>,
|
||||||
|
) {
|
||||||
|
let _ = tc.inbox_registry.try_deliver(reply_to, Box::new(result));
|
||||||
|
}
|
||||||
|
|
||||||
|
fn apply_admin_command(&mut self, tc: &TickContext, cmd: AdminCommand) {
|
||||||
|
match cmd {
|
||||||
|
AdminCommand::ListActors { acc } => {
|
||||||
|
let mut local = Vec::new();
|
||||||
|
self.pool.actor_summaries_into(self.id, &mut local);
|
||||||
|
{
|
||||||
|
let mut summaries = acc.summaries.lock();
|
||||||
|
summaries.extend(local);
|
||||||
|
}
|
||||||
|
if acc.remaining.fetch_sub(1, Ordering::AcqRel) == 1 {
|
||||||
|
let actors = {
|
||||||
|
let mut summaries = acc.summaries.lock();
|
||||||
|
std::mem::take(&mut *summaries)
|
||||||
|
};
|
||||||
|
Self::send_admin_reply(tc, acc.reply_to, Ok(ListActorsResponse { actors }));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
AdminCommand::InspectActor { actor, reply_to } => {
|
||||||
|
let result = self
|
||||||
|
.pool
|
||||||
|
.actor_summary(self.id, actor)
|
||||||
|
.map(|summary| InspectActorResponse { summary });
|
||||||
|
Self::send_admin_reply(tc, reply_to, result);
|
||||||
|
}
|
||||||
|
AdminCommand::GetActorState {
|
||||||
|
actor,
|
||||||
|
reply_to,
|
||||||
|
get,
|
||||||
|
not_found,
|
||||||
|
} => {
|
||||||
|
let boxed = match self.pool.get_actor_erased(actor) {
|
||||||
|
Some(erased) => get(actor, erased, erased.metadata()),
|
||||||
|
None => not_found(actor),
|
||||||
|
};
|
||||||
|
let _ = tc.inbox_registry.try_deliver(reply_to, boxed);
|
||||||
|
}
|
||||||
|
AdminCommand::ReplaceActorState {
|
||||||
|
actor,
|
||||||
|
reply_to,
|
||||||
|
replace,
|
||||||
|
} => {
|
||||||
|
let result = match self.pool.get_actor_erased_mut(actor) {
|
||||||
|
Some(erased) => {
|
||||||
|
let metadata = erased.metadata();
|
||||||
|
replace(erased, metadata)
|
||||||
|
}
|
||||||
|
None => Err(AdminError::ActorNotFound { actor }),
|
||||||
|
};
|
||||||
|
Self::send_admin_reply(tc, reply_to, result);
|
||||||
|
}
|
||||||
|
AdminCommand::StopActor { actor, reply_to } => {
|
||||||
|
let result = self.pool.stop_actor_admin(actor, &self.stats);
|
||||||
|
Self::send_admin_reply(tc, reply_to, result);
|
||||||
|
}
|
||||||
|
AdminCommand::SuspendActor { actor, reply_to } => {
|
||||||
|
let result = self.pool.suspend_actor_admin(actor);
|
||||||
|
Self::send_admin_reply(tc, reply_to, result);
|
||||||
|
}
|
||||||
|
AdminCommand::ResumeActor { actor, reply_to } => {
|
||||||
|
let result = self.pool.resume_actor_admin(actor);
|
||||||
|
Self::send_admin_reply(tc, reply_to, result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Phase 7: clean up dead actors, deliver death notifications, GC extension state.
|
/// Phase 7: clean up dead actors, deliver death notifications, GC extension state.
|
||||||
fn cleanup_dead_actors(&mut self, tc: &TickContext) -> bool {
|
fn cleanup_dead_actors(&mut self, tc: &TickContext) -> bool {
|
||||||
let cleanup_pending: RefCell<Vec<(ActorAddress, Box<dyn Any + Send>)>> =
|
let cleanup_pending: RefCell<Vec<(ActorAddress, Box<dyn Any + Send>)>> =
|
||||||
|
|
@ -197,6 +286,7 @@ impl Worker {
|
||||||
if !self.has_backlog
|
if !self.has_backlog
|
||||||
&& self.spawn_rx.is_empty()
|
&& self.spawn_rx.is_empty()
|
||||||
&& self.transfer_rx.is_empty()
|
&& self.transfer_rx.is_empty()
|
||||||
|
&& self.admin_rx.is_empty()
|
||||||
&& !self
|
&& !self
|
||||||
.worker_ext
|
.worker_ext
|
||||||
.as_ref()
|
.as_ref()
|
||||||
|
|
@ -221,7 +311,10 @@ impl Worker {
|
||||||
}
|
}
|
||||||
let t2 = Instant::now();
|
let t2 = Instant::now();
|
||||||
|
|
||||||
// 2.5. Fire per-worker extension (e.g., timers) → deliver before tick_all
|
// 3. Drain admin queue → inspect or mutate worker-owned slots before handlers
|
||||||
|
did_work |= self.drain_admin(tc);
|
||||||
|
|
||||||
|
// 4. Fire per-worker extension (e.g., timers) → deliver before tick_all
|
||||||
let ext_msgs: Vec<_> = self
|
let ext_msgs: Vec<_> = self
|
||||||
.worker_ext
|
.worker_ext
|
||||||
.as_mut()
|
.as_mut()
|
||||||
|
|
@ -232,7 +325,7 @@ impl Worker {
|
||||||
did_work = true;
|
did_work = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3. Tick all actors with WorkerContext
|
// 5. Tick all actors with WorkerContext
|
||||||
let pending_local: RefCell<Vec<(ActorAddress, Box<dyn Any + Send>)>> =
|
let pending_local: RefCell<Vec<(ActorAddress, Box<dyn Any + Send>)>> =
|
||||||
RefCell::new(Vec::new());
|
RefCell::new(Vec::new());
|
||||||
let stop_requests: RefCell<Vec<ActorAddress>> = RefCell::new(Vec::new());
|
let stop_requests: RefCell<Vec<ActorAddress>> = RefCell::new(Vec::new());
|
||||||
|
|
@ -275,12 +368,12 @@ impl Worker {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 4. Drain spawn queue again — actors spawned during step 3
|
// 6. Drain spawn queue again — actors spawned during step 5
|
||||||
// must be in the pool before pending_local delivery.
|
// must be in the pool before pending_local delivery.
|
||||||
did_work |= self.drain_spawns(tc);
|
did_work |= self.drain_spawns(tc);
|
||||||
let t4 = Instant::now();
|
let t4 = Instant::now();
|
||||||
|
|
||||||
// 5. Drain pending_local buffer → deliver to local actors
|
// 7. Drain pending_local buffer → deliver to local actors
|
||||||
let pending = pending_local.into_inner();
|
let pending = pending_local.into_inner();
|
||||||
if !pending.is_empty() {
|
if !pending.is_empty() {
|
||||||
did_work = true;
|
did_work = true;
|
||||||
|
|
@ -289,7 +382,7 @@ impl Worker {
|
||||||
self.pool.deliver(&addr, msg);
|
self.pool.deliver(&addr, msg);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 5.5. Process worker extension requests from handlers (e.g., timer scheduling)
|
// 7.5. Process worker extension requests from handlers (e.g., timer scheduling)
|
||||||
if let Some(ext) = &mut self.worker_ext {
|
if let Some(ext) = &mut self.worker_ext {
|
||||||
for request in worker_requests.into_inner() {
|
for request in worker_requests.into_inner() {
|
||||||
ext.handle_request(request);
|
ext.handle_request(request);
|
||||||
|
|
@ -298,7 +391,7 @@ impl Worker {
|
||||||
|
|
||||||
let t5 = Instant::now();
|
let t5 = Instant::now();
|
||||||
|
|
||||||
// 6. Publish stats (skip entirely when idle to avoid allocation + mutex)
|
// 8. Publish stats (skip entirely when idle to avoid allocation + mutex)
|
||||||
if did_work {
|
if did_work {
|
||||||
self.stats
|
self.stats
|
||||||
.num_actors
|
.num_actors
|
||||||
|
|
@ -344,7 +437,7 @@ impl Worker {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 7. Clean up poisoned and stopping actors
|
// 9. Clean up poisoned and stopping actors
|
||||||
did_work |= self.cleanup_dead_actors(tc);
|
did_work |= self.cleanup_dead_actors(tc);
|
||||||
|
|
||||||
self.has_backlog = did_work;
|
self.has_backlog = did_work;
|
||||||
|
|
@ -545,6 +638,95 @@ impl ActorPool {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn get_actor_erased(&self, addr: ActorAddress) -> Option<&dyn AnyActor> {
|
||||||
|
self.actors.get(&addr).map(|slot| slot.actor.as_ref())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_actor_erased_mut(
|
||||||
|
&mut self,
|
||||||
|
addr: ActorAddress,
|
||||||
|
) -> Option<&mut (dyn AnyActor + 'static)> {
|
||||||
|
self.actors.get_mut(&addr).map(|slot| slot.actor.as_mut())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn actor_summary_from_slot(
|
||||||
|
worker_id: WorkerId,
|
||||||
|
address: ActorAddress,
|
||||||
|
slot: &ActorSlot,
|
||||||
|
) -> ActorSummary {
|
||||||
|
let metadata = slot.actor.metadata();
|
||||||
|
ActorSummary {
|
||||||
|
address,
|
||||||
|
actor_type: metadata.actor_type_name,
|
||||||
|
message_type: metadata.message_type_name,
|
||||||
|
worker_id: worker_id.as_usize(),
|
||||||
|
parent: slot.parent_addr,
|
||||||
|
mailbox_depth: slot.mailbox.len(),
|
||||||
|
status: ActorStatus {
|
||||||
|
started: slot.started,
|
||||||
|
suspended: slot.suspended,
|
||||||
|
stopping: slot.stopping,
|
||||||
|
poisoned: slot.poisoned,
|
||||||
|
},
|
||||||
|
last_message_type: slot.last_msg_type,
|
||||||
|
messages_handled: slot.messages_processed,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn actor_summary(&self, worker_id: WorkerId, addr: ActorAddress) -> AdminResult<ActorSummary> {
|
||||||
|
self.actors
|
||||||
|
.get(&addr)
|
||||||
|
.map(|slot| Self::actor_summary_from_slot(worker_id, addr, slot))
|
||||||
|
.ok_or(AdminError::ActorNotFound { actor: addr })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn actor_summaries_into(&self, worker_id: WorkerId, out: &mut Vec<ActorSummary>) {
|
||||||
|
out.clear();
|
||||||
|
out.extend(
|
||||||
|
self.actors
|
||||||
|
.iter()
|
||||||
|
.map(|(&addr, slot)| Self::actor_summary_from_slot(worker_id, addr, slot)),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn suspend_actor_admin(&mut self, addr: ActorAddress) -> AdminResult<OperationResult> {
|
||||||
|
match self.actors.get_mut(&addr) {
|
||||||
|
Some(slot) => {
|
||||||
|
slot.suspended = true;
|
||||||
|
Ok(OperationResult { applied: true })
|
||||||
|
}
|
||||||
|
None => Err(AdminError::ActorNotFound { actor: addr }),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn resume_actor_admin(&mut self, addr: ActorAddress) -> AdminResult<OperationResult> {
|
||||||
|
match self.actors.get_mut(&addr) {
|
||||||
|
Some(slot) => {
|
||||||
|
slot.suspended = false;
|
||||||
|
Ok(OperationResult { applied: true })
|
||||||
|
}
|
||||||
|
None => Err(AdminError::ActorNotFound { actor: addr }),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stop_actor_admin(
|
||||||
|
&mut self,
|
||||||
|
addr: ActorAddress,
|
||||||
|
stats: &WorkerStats,
|
||||||
|
) -> AdminResult<OperationResult> {
|
||||||
|
match self.actors.get_mut(&addr) {
|
||||||
|
Some(slot) => {
|
||||||
|
if !slot.stopping {
|
||||||
|
stats.stops.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
slot.stopping = true;
|
||||||
|
slot.mailbox.clear();
|
||||||
|
Ok(OperationResult { applied: true })
|
||||||
|
}
|
||||||
|
None => Err(AdminError::ActorNotFound { actor: addr }),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Tick all actors in the pool. Returns the number of messages processed.
|
/// Tick all actors in the pool. Returns the number of messages processed.
|
||||||
///
|
///
|
||||||
/// Each actor processes up to `budget` messages per tick (0 = unlimited).
|
/// Each actor processes up to `budget` messages per tick (0 = unlimited).
|
||||||
|
|
|
||||||
608
tests/runtime_admin.rs
Normal file
608
tests/runtime_admin.rs
Normal file
|
|
@ -0,0 +1,608 @@
|
||||||
|
//! Runtime Admin API tests — inventory, typed actor state, lifecycle control, and scheduling.
|
||||||
|
|
||||||
|
mod common;
|
||||||
|
use common::*;
|
||||||
|
|
||||||
|
use std::collections::HashSet;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use swactor::admin::{ActorStateSnapshot, AdminError, OperationResult};
|
||||||
|
use swactor::config::RuntimeConfig;
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct AddAndReport {
|
||||||
|
delta: usize,
|
||||||
|
reply_to: ActorAddress,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct ReplaceProbe {
|
||||||
|
value: usize,
|
||||||
|
started: Arc<AtomicUsize>,
|
||||||
|
stopped: Arc<AtomicUsize>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ActorInterface for ReplaceProbe {
|
||||||
|
type Incoming = AddAndReport;
|
||||||
|
type Response = Count;
|
||||||
|
|
||||||
|
fn on_start(&mut self, _ctx: &Ctx) {
|
||||||
|
self.started.fetch_add(1, Ordering::SeqCst);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn on_stop(&mut self, _ctx: &Ctx) {
|
||||||
|
self.stopped.fetch_add(1, Ordering::SeqCst);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn handle(&mut self, ctx: &Ctx, msg: AddAndReport) {
|
||||||
|
self.value += msg.delta;
|
||||||
|
let _ = ctx.send(msg.reply_to, Count(self.value));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct WrongProbe;
|
||||||
|
|
||||||
|
impl ActorInterface for WrongProbe {
|
||||||
|
type Incoming = Ping;
|
||||||
|
type Response = Pong;
|
||||||
|
|
||||||
|
fn handle(&mut self, _ctx: &Ctx, _msg: Ping) {}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct StopProbe {
|
||||||
|
started: Arc<AtomicUsize>,
|
||||||
|
handled: Arc<AtomicUsize>,
|
||||||
|
stopped: Arc<AtomicUsize>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ActorInterface for StopProbe {
|
||||||
|
type Incoming = Ping;
|
||||||
|
type Response = Pong;
|
||||||
|
|
||||||
|
fn on_start(&mut self, _ctx: &Ctx) {
|
||||||
|
self.started.fetch_add(1, Ordering::SeqCst);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn on_stop(&mut self, _ctx: &Ctx) {
|
||||||
|
self.stopped.fetch_add(1, Ordering::SeqCst);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn handle(&mut self, ctx: &Ctx, msg: Ping) {
|
||||||
|
self.handled.fetch_add(1, Ordering::SeqCst);
|
||||||
|
let _ = ctx.send(msg.reply_to, Pong);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn poll_admin<T: swactor::actor::Message>(
|
||||||
|
admin: &swactor::admin::Admin<T>,
|
||||||
|
timeout: Duration,
|
||||||
|
) -> Option<swactor::admin::AdminResult<T>> {
|
||||||
|
let start = Instant::now();
|
||||||
|
while start.elapsed() < timeout {
|
||||||
|
if let Some(value) = admin.try_recv() {
|
||||||
|
return Some(value);
|
||||||
|
}
|
||||||
|
std::thread::sleep(Duration::from_millis(5));
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
fn poll_inbox<M: swactor::actor::Message>(inbox: &Inbox<M>, timeout: Duration) -> Option<M> {
|
||||||
|
let start = Instant::now();
|
||||||
|
while start.elapsed() < timeout {
|
||||||
|
if let Some(value) = inbox.try_recv() {
|
||||||
|
return Some(value);
|
||||||
|
}
|
||||||
|
std::thread::sleep(Duration::from_millis(5));
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
fn operation_applied() -> OperationResult {
|
||||||
|
OperationResult { applied: true }
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ask_recv_ticking_delivers_reply_through_runtime_inbox() {
|
||||||
|
let rt = std_runtime(RuntimeConfig::default());
|
||||||
|
let actor = rt.spawn(SelfAddrActor).unwrap();
|
||||||
|
|
||||||
|
let ask = rt
|
||||||
|
.ask::<WhoAreYou, MyAddr>(actor, |reply_to| WhoAreYou { reply_to })
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
ask.try_recv(),
|
||||||
|
None,
|
||||||
|
"ask reply is not available before ticking"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
ask.recv_ticking(&rt, 5).unwrap(),
|
||||||
|
MyAddr(actor),
|
||||||
|
"recv_ticking drives the runtime inbox reply path"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn admin_list_and_inspect_report_actor_slot_metadata() {
|
||||||
|
let rt = std_runtime(RuntimeConfig::default());
|
||||||
|
let ping_pong = rt.spawn(PingPongActor).unwrap();
|
||||||
|
let counter = rt.spawn(CounterActor { count: 0 }).unwrap();
|
||||||
|
rt.tick();
|
||||||
|
|
||||||
|
let count_inbox = rt.new_inbox::<Count>().unwrap();
|
||||||
|
rt.send_to(
|
||||||
|
counter,
|
||||||
|
Increment {
|
||||||
|
reply_to: *count_inbox.addr(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
rt.send_to(
|
||||||
|
counter,
|
||||||
|
Increment {
|
||||||
|
reply_to: *count_inbox.addr(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(tick_until_recv(&rt, &count_inbox, 5), Some(Count(1)));
|
||||||
|
assert_eq!(tick_until_recv(&rt, &count_inbox, 5), Some(Count(2)));
|
||||||
|
|
||||||
|
let response = rt
|
||||||
|
.admin()
|
||||||
|
.list_actors()
|
||||||
|
.unwrap()
|
||||||
|
.recv_ticking(&rt, 5)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
response
|
||||||
|
.actors
|
||||||
|
.iter()
|
||||||
|
.filter(|summary| summary.address == ping_pong)
|
||||||
|
.count(),
|
||||||
|
1,
|
||||||
|
"ping-pong actor appears exactly once in inventory"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
response
|
||||||
|
.actors
|
||||||
|
.iter()
|
||||||
|
.filter(|summary| summary.address == counter)
|
||||||
|
.count(),
|
||||||
|
1,
|
||||||
|
"counter actor appears exactly once in inventory"
|
||||||
|
);
|
||||||
|
|
||||||
|
let counter_summary = response
|
||||||
|
.actors
|
||||||
|
.iter()
|
||||||
|
.find(|summary| summary.address == counter)
|
||||||
|
.expect("counter summary missing");
|
||||||
|
|
||||||
|
assert_eq!(counter_summary.worker_id, 0);
|
||||||
|
assert_eq!(counter_summary.parent, None);
|
||||||
|
assert_eq!(counter_summary.mailbox_depth, 0);
|
||||||
|
assert!(counter_summary.status.started);
|
||||||
|
assert!(!counter_summary.status.suspended);
|
||||||
|
assert!(!counter_summary.status.stopping);
|
||||||
|
assert!(!counter_summary.status.poisoned);
|
||||||
|
assert_eq!(counter_summary.messages_handled, 2);
|
||||||
|
assert!(counter_summary.actor_type.ends_with("CounterActor"));
|
||||||
|
assert!(counter_summary.message_type.ends_with("Increment"));
|
||||||
|
|
||||||
|
let inspect = rt
|
||||||
|
.admin()
|
||||||
|
.inspect_actor(counter)
|
||||||
|
.unwrap()
|
||||||
|
.recv_ticking(&rt, 5)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(inspect.summary, *counter_summary);
|
||||||
|
|
||||||
|
let missing = ActorAddress::new_random();
|
||||||
|
let missing_result = rt
|
||||||
|
.admin()
|
||||||
|
.inspect_actor(missing)
|
||||||
|
.unwrap()
|
||||||
|
.recv_ticking(&rt, 5);
|
||||||
|
assert!(
|
||||||
|
matches!(missing_result, Err(AdminError::ActorNotFound { actor }) if actor == missing),
|
||||||
|
"missing actor is reported through AdminResult"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn admin_get_and_replace_actor_state_preserves_slot_metadata() {
|
||||||
|
let rt = std_runtime(RuntimeConfig::default());
|
||||||
|
let started = Arc::new(AtomicUsize::new(0));
|
||||||
|
let stopped = Arc::new(AtomicUsize::new(0));
|
||||||
|
let addr = rt
|
||||||
|
.spawn(ReplaceProbe {
|
||||||
|
value: 1,
|
||||||
|
started: started.clone(),
|
||||||
|
stopped: stopped.clone(),
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
rt.tick();
|
||||||
|
|
||||||
|
assert_eq!(started.load(Ordering::SeqCst), 1);
|
||||||
|
assert_eq!(stopped.load(Ordering::SeqCst), 0);
|
||||||
|
|
||||||
|
let count_inbox = rt.new_inbox::<Count>().unwrap();
|
||||||
|
rt.send_to(
|
||||||
|
addr,
|
||||||
|
AddAndReport {
|
||||||
|
delta: 1,
|
||||||
|
reply_to: *count_inbox.addr(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(tick_until_recv(&rt, &count_inbox, 5), Some(Count(2)));
|
||||||
|
|
||||||
|
let state = rt
|
||||||
|
.admin()
|
||||||
|
.get_actor_state::<ReplaceProbe>(addr)
|
||||||
|
.unwrap()
|
||||||
|
.recv_ticking(&rt, 5)
|
||||||
|
.unwrap()
|
||||||
|
.state;
|
||||||
|
assert_eq!(state.actor, addr);
|
||||||
|
assert!(state.actor_type.ends_with("ReplaceProbe"));
|
||||||
|
assert!(state.message_type.ends_with("AddAndReport"));
|
||||||
|
assert_eq!(state.actor_instance.value, 2);
|
||||||
|
|
||||||
|
let replacement = ActorStateSnapshot::new(
|
||||||
|
addr,
|
||||||
|
ReplaceProbe {
|
||||||
|
value: 100,
|
||||||
|
started: started.clone(),
|
||||||
|
stopped: stopped.clone(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let replace_result = rt
|
||||||
|
.admin()
|
||||||
|
.replace_actor_state::<ReplaceProbe>(addr, replacement)
|
||||||
|
.unwrap()
|
||||||
|
.recv_ticking(&rt, 5)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(replace_result, operation_applied());
|
||||||
|
assert_eq!(
|
||||||
|
started.load(Ordering::SeqCst),
|
||||||
|
1,
|
||||||
|
"replacement does not call on_start"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
stopped.load(Ordering::SeqCst),
|
||||||
|
0,
|
||||||
|
"replacement does not call on_stop"
|
||||||
|
);
|
||||||
|
|
||||||
|
rt.send_to(
|
||||||
|
addr,
|
||||||
|
AddAndReport {
|
||||||
|
delta: 1,
|
||||||
|
reply_to: *count_inbox.addr(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(tick_until_recv(&rt, &count_inbox, 5), Some(Count(101)));
|
||||||
|
|
||||||
|
let summary = rt
|
||||||
|
.admin()
|
||||||
|
.inspect_actor(addr)
|
||||||
|
.unwrap()
|
||||||
|
.recv_ticking(&rt, 5)
|
||||||
|
.unwrap()
|
||||||
|
.summary;
|
||||||
|
assert_eq!(summary.address, addr);
|
||||||
|
assert_eq!(summary.worker_id, 0);
|
||||||
|
assert_eq!(
|
||||||
|
summary.messages_handled, 2,
|
||||||
|
"state replacement preserves slot-owned message counters"
|
||||||
|
);
|
||||||
|
|
||||||
|
let stop_result = rt
|
||||||
|
.admin()
|
||||||
|
.stop_actor(addr)
|
||||||
|
.unwrap()
|
||||||
|
.recv_ticking(&rt, 5)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(stop_result, operation_applied());
|
||||||
|
rt.tick();
|
||||||
|
assert_eq!(stopped.load(Ordering::SeqCst), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn admin_replace_rejects_wrong_actor_type_and_wrong_snapshot_address() {
|
||||||
|
let rt = std_runtime(RuntimeConfig::default());
|
||||||
|
let started = Arc::new(AtomicUsize::new(0));
|
||||||
|
let stopped = Arc::new(AtomicUsize::new(0));
|
||||||
|
let addr = rt
|
||||||
|
.spawn(ReplaceProbe {
|
||||||
|
value: 10,
|
||||||
|
started: started.clone(),
|
||||||
|
stopped: stopped.clone(),
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
rt.tick();
|
||||||
|
|
||||||
|
let wrong_type_snapshot = ActorStateSnapshot::new(addr, WrongProbe);
|
||||||
|
let wrong_type = rt
|
||||||
|
.admin()
|
||||||
|
.replace_actor_state::<WrongProbe>(addr, wrong_type_snapshot)
|
||||||
|
.unwrap()
|
||||||
|
.recv_ticking(&rt, 5);
|
||||||
|
assert!(
|
||||||
|
matches!(wrong_type, Err(AdminError::TypeMismatch { .. })),
|
||||||
|
"wrong concrete actor type is rejected"
|
||||||
|
);
|
||||||
|
|
||||||
|
let wrong_addr = ActorAddress::new_random();
|
||||||
|
let wrong_addr_snapshot = ActorStateSnapshot::new(
|
||||||
|
wrong_addr,
|
||||||
|
ReplaceProbe {
|
||||||
|
value: 50,
|
||||||
|
started: started.clone(),
|
||||||
|
stopped: stopped.clone(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let wrong_address = rt
|
||||||
|
.admin()
|
||||||
|
.replace_actor_state::<ReplaceProbe>(addr, wrong_addr_snapshot)
|
||||||
|
.unwrap()
|
||||||
|
.recv_ticking(&rt, 5);
|
||||||
|
assert!(
|
||||||
|
matches!(wrong_address, Err(AdminError::AddressMismatch { requested, snapshot }) if requested == addr && snapshot == wrong_addr),
|
||||||
|
"snapshot address must match the target address"
|
||||||
|
);
|
||||||
|
|
||||||
|
let count_inbox = rt.new_inbox::<Count>().unwrap();
|
||||||
|
rt.send_to(
|
||||||
|
addr,
|
||||||
|
AddAndReport {
|
||||||
|
delta: 1,
|
||||||
|
reply_to: *count_inbox.addr(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
tick_until_recv(&rt, &count_inbox, 5),
|
||||||
|
Some(Count(11)),
|
||||||
|
"failed replacements do not mutate the original actor state"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn admin_suspend_queues_messages_until_resume() {
|
||||||
|
let rt = std_runtime(RuntimeConfig::default());
|
||||||
|
let counter = Arc::new(AtomicUsize::new(0));
|
||||||
|
let addr = rt
|
||||||
|
.spawn(CountingPingActor {
|
||||||
|
counter: counter.clone(),
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
rt.tick();
|
||||||
|
|
||||||
|
let suspend_result = rt
|
||||||
|
.admin()
|
||||||
|
.suspend_actor(addr)
|
||||||
|
.unwrap()
|
||||||
|
.recv_ticking(&rt, 5)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(suspend_result, operation_applied());
|
||||||
|
|
||||||
|
let pong_inbox = rt.new_inbox::<Pong>().unwrap();
|
||||||
|
for _ in 0..3 {
|
||||||
|
rt.send_to(
|
||||||
|
addr,
|
||||||
|
Ping {
|
||||||
|
reply_to: *pong_inbox.addr(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
tick_n(&rt, 5);
|
||||||
|
assert_eq!(counter.load(Ordering::SeqCst), 0);
|
||||||
|
assert_eq!(pong_inbox.try_recv(), None);
|
||||||
|
|
||||||
|
let suspended = rt
|
||||||
|
.admin()
|
||||||
|
.inspect_actor(addr)
|
||||||
|
.unwrap()
|
||||||
|
.recv_ticking(&rt, 5)
|
||||||
|
.unwrap()
|
||||||
|
.summary;
|
||||||
|
assert!(suspended.status.suspended);
|
||||||
|
assert_eq!(suspended.mailbox_depth, 3);
|
||||||
|
|
||||||
|
let resume_result = rt
|
||||||
|
.admin()
|
||||||
|
.resume_actor(addr)
|
||||||
|
.unwrap()
|
||||||
|
.recv_ticking(&rt, 5)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resume_result, operation_applied());
|
||||||
|
for _ in 0..3 {
|
||||||
|
assert_eq!(tick_until_recv(&rt, &pong_inbox, 5), Some(Pong));
|
||||||
|
}
|
||||||
|
assert_eq!(counter.load(Ordering::SeqCst), 3);
|
||||||
|
|
||||||
|
let resumed = rt
|
||||||
|
.admin()
|
||||||
|
.inspect_actor(addr)
|
||||||
|
.unwrap()
|
||||||
|
.recv_ticking(&rt, 5)
|
||||||
|
.unwrap()
|
||||||
|
.summary;
|
||||||
|
assert!(!resumed.status.suspended);
|
||||||
|
assert_eq!(resumed.mailbox_depth, 0);
|
||||||
|
assert_eq!(resumed.messages_handled, 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn admin_stop_clears_pending_mailbox_without_calling_handle() {
|
||||||
|
let rt = std_runtime(RuntimeConfig::default());
|
||||||
|
let started = Arc::new(AtomicUsize::new(0));
|
||||||
|
let handled = Arc::new(AtomicUsize::new(0));
|
||||||
|
let stopped = Arc::new(AtomicUsize::new(0));
|
||||||
|
let addr = rt
|
||||||
|
.spawn(StopProbe {
|
||||||
|
started: started.clone(),
|
||||||
|
handled: handled.clone(),
|
||||||
|
stopped: stopped.clone(),
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
rt.tick();
|
||||||
|
assert_eq!(started.load(Ordering::SeqCst), 1);
|
||||||
|
|
||||||
|
let pong_inbox = rt.new_inbox::<Pong>().unwrap();
|
||||||
|
for _ in 0..5 {
|
||||||
|
rt.send_to(
|
||||||
|
addr,
|
||||||
|
Ping {
|
||||||
|
reply_to: *pong_inbox.addr(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
let stop_result = rt
|
||||||
|
.admin()
|
||||||
|
.stop_actor(addr)
|
||||||
|
.unwrap()
|
||||||
|
.recv_ticking(&rt, 5)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(stop_result, operation_applied());
|
||||||
|
rt.tick();
|
||||||
|
|
||||||
|
assert_eq!(handled.load(Ordering::SeqCst), 0);
|
||||||
|
assert_eq!(stopped.load(Ordering::SeqCst), 1);
|
||||||
|
assert_eq!(pong_inbox.try_recv(), None);
|
||||||
|
assert!(
|
||||||
|
rt.send_to(
|
||||||
|
addr,
|
||||||
|
Ping {
|
||||||
|
reply_to: *pong_inbox.addr(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.is_err(),
|
||||||
|
"admin-stopped actor is removed from normal send routing"
|
||||||
|
);
|
||||||
|
|
||||||
|
let inspect = rt.admin().inspect_actor(addr).unwrap().recv_ticking(&rt, 5);
|
||||||
|
assert!(
|
||||||
|
matches!(inspect, Err(AdminError::ActorNotFound { actor }) if actor == addr),
|
||||||
|
"admin-stopped actor is no longer inspectable"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn threaded_admin_suspend_resume_wakes_parked_worker() {
|
||||||
|
let rt = std_runtime(RuntimeConfig {
|
||||||
|
num_threads: 2,
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
let counter = Arc::new(AtomicUsize::new(0));
|
||||||
|
let addr = rt
|
||||||
|
.spawn(CountingPingActor {
|
||||||
|
counter: counter.clone(),
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
let pong_inbox = rt.new_inbox::<Pong>().unwrap();
|
||||||
|
|
||||||
|
let handle = rt.run().unwrap();
|
||||||
|
std::thread::sleep(Duration::from_millis(50));
|
||||||
|
|
||||||
|
let suspended = handle.runtime.admin().suspend_actor(addr).unwrap();
|
||||||
|
let suspended = poll_admin(&suspended, Duration::from_secs(1));
|
||||||
|
|
||||||
|
handle
|
||||||
|
.runtime
|
||||||
|
.send_to(
|
||||||
|
addr,
|
||||||
|
Ping {
|
||||||
|
reply_to: *pong_inbox.addr(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let pong_while_suspended = poll_inbox(&pong_inbox, Duration::from_millis(100));
|
||||||
|
let count_while_suspended = counter.load(Ordering::SeqCst);
|
||||||
|
|
||||||
|
let resumed = handle.runtime.admin().resume_actor(addr).unwrap();
|
||||||
|
let resumed = poll_admin(&resumed, Duration::from_secs(1));
|
||||||
|
let pong_after_resume = poll_inbox(&pong_inbox, Duration::from_secs(1));
|
||||||
|
let final_count = counter.load(Ordering::SeqCst);
|
||||||
|
|
||||||
|
handle.shutdown();
|
||||||
|
handle.join();
|
||||||
|
|
||||||
|
assert_eq!(suspended, Some(Ok(operation_applied())));
|
||||||
|
assert_eq!(pong_while_suspended, None);
|
||||||
|
assert_eq!(count_while_suspended, 0);
|
||||||
|
assert_eq!(resumed, Some(Ok(operation_applied())));
|
||||||
|
assert_eq!(pong_after_resume, Some(Pong));
|
||||||
|
assert_eq!(final_count, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn admin_list_actors_aggregates_all_workers() {
|
||||||
|
let rt = std_runtime(RuntimeConfig {
|
||||||
|
num_threads: 4,
|
||||||
|
max_actors: 100,
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
let mut addrs = Vec::new();
|
||||||
|
for _ in 0..16 {
|
||||||
|
addrs.push(rt.spawn(CounterActor { count: 0 }).unwrap());
|
||||||
|
}
|
||||||
|
|
||||||
|
let handle = rt.run().unwrap();
|
||||||
|
let start = Instant::now();
|
||||||
|
let mut response = None;
|
||||||
|
while start.elapsed() < Duration::from_secs(1) {
|
||||||
|
let admin = handle.runtime.admin().list_actors().unwrap();
|
||||||
|
if let Some(Ok(list)) = poll_admin(&admin, Duration::from_millis(100)) {
|
||||||
|
if list.actors.len() == addrs.len() {
|
||||||
|
response = Some(list);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
std::thread::sleep(Duration::from_millis(5));
|
||||||
|
}
|
||||||
|
|
||||||
|
handle.shutdown();
|
||||||
|
handle.join();
|
||||||
|
|
||||||
|
let response = response.expect("admin list did not observe all spawned actors within timeout");
|
||||||
|
let expected: HashSet<_> = addrs.iter().copied().collect();
|
||||||
|
let actual: HashSet<_> = response
|
||||||
|
.actors
|
||||||
|
.iter()
|
||||||
|
.map(|summary| summary.address)
|
||||||
|
.collect();
|
||||||
|
assert_eq!(actual, expected);
|
||||||
|
for addr in &addrs {
|
||||||
|
assert_eq!(
|
||||||
|
response
|
||||||
|
.actors
|
||||||
|
.iter()
|
||||||
|
.filter(|summary| summary.address == *addr)
|
||||||
|
.count(),
|
||||||
|
1,
|
||||||
|
"actor {addr} appears exactly once in aggregated list"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let worker_ids: HashSet<_> = response
|
||||||
|
.actors
|
||||||
|
.iter()
|
||||||
|
.map(|summary| summary.worker_id)
|
||||||
|
.collect();
|
||||||
|
assert!(
|
||||||
|
worker_ids.len() >= 2,
|
||||||
|
"aggregation should include actors from at least two workers, got {worker_ids:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue