From 8c68a59b45dec6ce1afe9bfa71990d4fcd271360 Mon Sep 17 00:00:00 2001 From: Developer Date: Thu, 12 Feb 2026 13:42:23 +0000 Subject: [PATCH] docs: update architecture docs for Cycles 10-16 Reflect current state of tick_once (8 phases), TickContext (3 new registries + stats_hook + worker_threads), TimerWheel, lifecycle hooks, cleanup_dead with StopReason/Down notifications/registry cleanup, and expanded Ctx/Runtime public API. Co-Authored-By: Claude Opus 4.6 --- docs/runtime.md | 127 ++++++++++++++++++++------- docs/worker-thread.md | 195 ++++++++++++++++++++++++++++++++---------- 2 files changed, 247 insertions(+), 75 deletions(-) diff --git a/docs/runtime.md b/docs/runtime.md index 9ca1c13..b554e09 100644 --- a/docs/runtime.md +++ b/docs/runtime.md @@ -14,10 +14,13 @@ messages. │ │ │ ┌─ Shared State (lives on Arc) ──────────────────────────────┐ │ │ │ │ │ -│ │ address_map: Arc -- actor -> worker lookup │ │ -│ │ inbox_registry: Arc -- external inbox delivery │ │ -│ │ placement: Placement -- round-robin worker picker │ │ -│ │ worker_stats: Vec> -- atomic stat counters │ │ +│ │ address_map: Arc -- actor -> worker lookup │ │ +│ │ inbox_registry: Arc -- external inbox delivery │ │ +│ │ name_registry: Arc -- name -> address lookup │ │ +│ │ monitor_registry: Arc -- death watch subscripts │ │ +│ │ group_registry: Arc -- pub-sub actor groups │ │ +│ │ placement: Placement -- load-aware worker picker │ │ +│ │ worker_stats: Vec> -- atomic stat counters │ │ │ │ │ │ │ └─────────────────────────────────────────────────────────────────────┘ │ │ │ @@ -28,14 +31,8 @@ messages. │ │ │ │ │ └─────────────────────────────────────────────────────────────────────┘ │ │ │ -│ ┌─ Mode ──────────────────────────────────────────────────────────────┐ │ -│ │ │ │ -│ │ SINGLE-THREADED: single_worker: Some(RefCell) │ │ -│ │ MULTI-THREADED: pending_workers: Some(Vec) │ │ -│ │ │ │ -│ │ After run() is called, both are None — workers move to threads. │ │ -│ │ │ │ -│ └─────────────────────────────────────────────────────────────────────┘ │ +│ tick_workers: RefCell> -- for tick(); run() drains these │ +│ worker_threads: Vec> -- for waking parked workers │ │ │ └───────────────────────────────────────────────────────────────────────────┘ ``` @@ -80,9 +77,21 @@ only way for actors to interact with the outside world. │ │ │ ┌─ Public API ────────────────────────────────────────────────────────┐ │ │ │ │ │ -│ │ ctx.self_addr() -> ActorAddress │ │ -│ │ ctx.send(addr, msg) -> Result<(), Error> │ │ -│ │ ctx.spawn(actor) -> Result │ │ +│ │ ctx.self_addr() -> ActorAddress │ │ +│ │ ctx.send(addr, msg) -> Result<(), Error> │ │ +│ │ ctx.spawn(actor) -> Result │ │ +│ │ ctx.spawn_named(name, actor) -> Result │ │ +│ │ ctx.spawn_restartable(a, f, max) -> Result │ │ +│ │ ctx.stop_self() │ │ +│ │ ctx.where_is(name) -> Option │ │ +│ │ ctx.monitor(target) -> MonitorRef │ │ +│ │ ctx.demonitor(mref) │ │ +│ │ ctx.join_group(group) │ │ +│ │ ctx.leave_group(group) │ │ +│ │ ctx.publish(group, msg) -> usize │ │ +│ │ ctx.group_members(group) -> Vec │ │ +│ │ ctx.send_after_ticks(addr, msg, n) │ │ +│ │ ctx.send_interval_ticks(addr, msg, period) │ │ │ │ │ │ │ └─────────────────────────────────────────────────────────────────────┘ │ │ │ @@ -130,21 +139,73 @@ handler, etc.) receive typed messages from actors. │ if let Some(msg) = inbox.try_recv() { ... } │ │ │ └────────────────────────────────────────────────────────────────────────┘ +``` - ┌─ Delivery Path ────────────────────────────────────────────────────────┐ - │ │ - │ actor calls ctx.send(inbox_addr, response) │ - │ │ │ - │ v │ - │ address_map.lookup(inbox_addr) → None (inboxes aren't actors) │ - │ │ │ - │ v │ - │ inbox_registry.try_deliver(addr, msg) │ - │ │ │ - │ v │ - │ downcast Box → M, push into Receiver │ - │ │ - └────────────────────────────────────────────────────────────────────────┘ +## Ask — Typed Request-Response + +`Ask` wraps an `Inbox` for convenient request-response: + +``` + let response: Pong = rt.ask(actor, |reply_to| Ping { reply_to })? + .recv_ticking(&rt, 10)?; // tick until response or timeout +``` + +## Named Actors + +Actors can be spawned with a registered name for discovery: + +``` + let addr = rt.spawn_named("coordinator", my_actor)?; + let found = rt.where_is("coordinator"); // -> Some(addr) + // Names are auto-unregistered when the actor dies. +``` + +## Actor Monitoring (Death Watch) + +Subscribe to death notifications via `ctx.monitor()`: + +``` + let mref = ctx.monitor(target_addr); + // When target dies, a Down { addr, reason } message arrives in + // the watcher's normal handle() method. No special callback needed. +``` + +`StopReason`: `Normal` (graceful stop) | `Panicked` (panic, not restartable) + +## Actor Groups (Pub-Sub) + +Named groups for broadcast messaging: + +``` + ctx.join_group("workers"); + ctx.publish("workers", StatusUpdate { ... }); // all members receive it + // Members auto-removed on death. Groups auto-deleted when empty. +``` + +## Lifecycle Hooks + +``` + fn on_start(&mut self, ctx: &Ctx) {} -- called once before first message + fn on_stop(&mut self, ctx: &Ctx) {} -- called on graceful stop (not panic) +``` + +## Actor Recovery + +Factory-based restart after panic: + +``` + rt.spawn_restartable(actor, || MyActor::new(), 3)?; + // On panic: mailbox cleared, factory creates fresh instance, up to 3 times. + // After max_restarts: permanently poisoned. +``` + +## Per-Worker Timers + +Deterministic tick-counting timers (not wall-clock): + +``` + ctx.send_after_ticks(addr, msg, 5); // one-shot: fires after 5 ticks + ctx.send_interval_ticks(addr, msg, 10); // repeating: every 10 ticks ``` ## RuntimeHandle @@ -171,12 +232,18 @@ Returned by `run()`. Holds `Arc` and the thread `JoinHandle`s. ┌─ RuntimeStats ────────────────────────────────────────────────────────────┐ │ │ │ num_workers: usize │ +│ uptime_ms: u64 │ │ actors: Vec<(ActorAddress, worker_id)> -- from AddressMap snapshot │ │ workers: Vec │ │ ├─ id: usize │ │ ├─ num_actors: usize -- from atomic counter │ │ ├─ mailbox_depth: usize -- total queued messages │ -│ └─ messages_processed: u64 -- cumulative count │ +│ ├─ messages_processed: u64 -- cumulative count │ +│ ├─ messages_dropped: u64 -- overflow drops │ +│ ├─ panics: u64 │ +│ ├─ restarts: u64 │ +│ └─ stops: u64 │ +│ tick_timings: Vec> -- per-phase timing data │ │ │ └───────────────────────────────────────────────────────────────────────────┘ ``` diff --git a/docs/worker-thread.md b/docs/worker-thread.md index bb909da..3916f9e 100644 --- a/docs/worker-thread.md +++ b/docs/worker-thread.md @@ -48,6 +48,12 @@ │ │ │ │ │ └──────────────────────────────────────────────────────────────────┘ │ │ │ +│ ┌─ TimerWheel ────────────────────────────────────────────────────┐ │ +│ │ current_tick: u64 │ │ +│ │ once_timers: Vec -- fire_at, dest, msg │ │ +│ │ interval_timers: Vec -- period, dest, clone_msg │ │ +│ └──────────────────────────────────────────────────────────────────┘ │ +│ │ └────────────────────────────────────────────────────────────────────────┘ ``` @@ -58,12 +64,17 @@ Lives on `Arc`, shared read-only across all worker threads. ``` ┌─ TickContext<'a> ──────────────────────────────────────────────────────┐ │ │ -│ address_map: &AddressMap -- ActorAddress -> WorkerId lookup │ -│ transfer_txs: &[Sender] -- one Sender per worker (cross-send) │ -│ spawn_txs: &[Sender] -- one Sender per worker (spawn reqs) │ -│ placement: &Placement -- round-robin next-worker picker │ -│ inbox_registry: &InboxRegistry -- external Inbox receivers │ -│ config: &RuntimeConfig -- waterlevel, backoff params, etc. │ +│ address_map: &AddressMap -- ActorAddress -> WorkerId │ +│ transfer_txs: &[Sender] -- one Sender per worker │ +│ spawn_txs: &[Sender] -- one Sender per worker │ +│ placement: &Placement -- load-aware worker picker │ +│ inbox_registry: &InboxRegistry -- external Inbox receivers │ +│ name_registry: &NameRegistry -- String -> ActorAddress │ +│ monitor_registry: &MonitorRegistry -- death watch subscriptions │ +│ group_registry: &GroupRegistry -- pub-sub actor groups │ +│ config: &RuntimeConfig -- budget, backoff, etc. │ +│ stats_hook: Option<&dyn Hook> -- per-tick stats callback │ +│ worker_threads: &[OnceLock] -- for unpark on send/spawn │ │ │ └────────────────────────────────────────────────────────────────────────┘ ``` @@ -89,11 +100,13 @@ Lives on `Arc`, shared read-only across all worker threads. │ v v │ │ │ idle = 0 idle++ │ │ │ │ │ │ │ -│ │ ┌────┴────────────────────────┐ │ │ -│ │ │ idle < spin_thr: spin │ │ │ -│ │ │ idle < yield_thr: yield │ │ │ -│ │ │ else: sleep(incr, capped) │ │ │ -│ │ └─────────────────────────┬───┘ │ │ +│ │ ┌────┴──────────────────────────────┐ │ │ +│ │ │ idle < spin_thr: spin │ │ │ +│ │ │ idle < yield_thr: yield_now │ │ │ +│ │ │ else: park_timeout(incr, capped) │ │ │ +│ │ │ (instant wake via Thread::unpark │ │ │ +│ │ │ when send/spawn targets worker) │ │ │ +│ │ └──────────────────────────────┬─────┘ │ │ │ │ │ │ │ │ └──────────┬───────────────────┘ │ │ │ │ │ │ @@ -102,7 +115,7 @@ Lives on `Arc`, shared read-only across all worker threads. └────────────────────────────────────────────────────────────────────────┘ ``` -## Tick Once (four phases) +## Tick Once (eight phases) ``` ┌─ tick_once ────────────────────────────────────────────────────────────┐ @@ -114,12 +127,9 @@ Lives on `Arc`, shared read-only across all worker threads. │ │ │ ││ │ │ v ││ │ │ pool.insert(addr, actor) ││ -│ │ │ ││ -│ │ v ││ -│ │ ActorSlot { ││ -│ │ mailbox: VecDeque::new() ││ -│ │ actor: ││ -│ │ } ││ +│ │ started: false ││ +│ │ stopping: false ││ +│ │ mailbox_capacity: from config ││ │ │ ││ │ └────────────────────────────────────────────────────────────────────┘│ │ │ │ @@ -131,10 +141,25 @@ Lives on `Arc`, shared read-only across all worker threads. │ │ │ ││ │ │ v ││ │ │ pool.deliver(&dest, payload) ││ -│ │ │ ││ -│ │ v ││ -│ │ slot.mailbox.push_back(msg) ││ -│ │ (untyped; type check at handle time) ││ +│ │ (enforces mailbox_capacity; ││ +│ │ drop newest/oldest on overflow) ││ +│ │ ││ +│ └────────────────────────────────────────────────────────────────────┘│ +│ │ │ +│ v │ +│ PHASE 2.5 --- Fire Due Timers │ +│ ┌────────────────────────────────────────────────────────────────────┐│ +│ │ ││ +│ │ timers.fire() (advances tick counter, collects due messages) ││ +│ │ │ ││ +│ │ v ││ +│ │ for (dest, msg) in timer_msgs: ││ +│ │ ┌──────────────┬──────────────┬─────────────────┐ ││ +│ │ │ local actor │ other worker │ inbox/unknown │ ││ +│ │ │ │ │ │ ││ +│ │ │ pool.deliver │ transfer_tx │ inbox_registry │ ││ +│ │ │ │ + unpark │ .try_deliver() │ ││ +│ │ └──────────────┴──────────────┴─────────────────┘ ││ │ │ ││ │ └────────────────────────────────────────────────────────────────────┘│ │ │ │ @@ -144,25 +169,35 @@ Lives on `Arc`, shared read-only across all worker threads. │ │ ││ │ │ ┌─ WorkerContext (on stack) ─────────────────────────────────┐ ││ │ │ │ implements ContextInner │ ││ -│ │ │ owns pending_local: RefCell)>> │ ││ +│ │ │ pending_local: RefCell)>> │ ││ +│ │ │ stop_requests: RefCell> │ ││ +│ │ │ timer_requests: RefCell> │ ││ │ │ └────────────────────────────────────────────────────────────┘ ││ │ │ ││ │ │ for each (addr, slot) in pool: ││ +│ │ if poisoned or stopping → clear mailbox, skip ││ │ │ ││ -│ │ ┌─ drain_count ──────────────────────────────────────────┐ ││ -│ │ │ len = slot.mailbox.len() │ ││ -│ │ │ len < waterlevel --> n = len (drain all) │ ││ -│ │ │ len >= waterlevel --> n = len / 2 (backpressure) │ ││ -│ │ └────────────────────────────────────────────────────────┘ ││ +│ │ ┌─ on_start (once per actor) ──────────────────────────────┐ ││ +│ │ │ if !slot.started: │ ││ +│ │ │ catch_unwind(actor.on_start(&ctx)) │ ││ +│ │ │ panic → poisoned (immediate, no messages) │ ││ +│ │ │ ok → started = true │ ││ +│ │ └──────────────────────────────────────────────────────────┘ ││ │ │ ││ -│ │ ctx = Ctx { inner: &worker_ctx, self_addr: addr } ││ -│ │ ││ -│ │ repeat n times: ││ -│ │ msg = slot.mailbox.pop_front() ││ -│ │ slot.actor.handle_any(&ctx, msg) ││ -│ │ │ ││ -│ │ │ actor calls ctx.send() or ctx.spawn() ││ -│ │ v ││ +│ │ ┌─ message loop (budget-limited) ──────────────────────────┐ ││ +│ │ │ repeat up to `budget` times (budget=0 → unlimited): │ ││ +│ │ │ msg = slot.mailbox.pop_front() │ ││ +│ │ │ │ ││ +│ │ │ if msg is StopSignal: │ ││ +│ │ │ slot.stopping = true; clear mailbox; break │ ││ +│ │ │ │ ││ +│ │ │ catch_unwind(actor.handle_any(&ctx, msg)) │ ││ +│ │ │ panic → try_restart (factory) or poison │ ││ +│ │ │ ok → count += 1 │ ││ +│ │ │ │ ││ +│ │ │ if stop_requests contains addr: │ ││ +│ │ │ slot.stopping = true; clear mailbox; break │ ││ +│ │ └──────────────────────────────────────────────────────────┘ ││ │ │ ││ │ │ ┌─ WorkerContext routes ─────────────────────────────────────┐ ││ │ │ │ │ ││ @@ -171,29 +206,91 @@ Lives on `Arc`, shared read-only across all worker threads. │ │ │ │ same worker │ other worker │ unknown addr │ │ ││ │ │ │ │ │ │ │ │ ││ │ │ │ │ pending_ │ transfer_tx │ inbox_registry │ │ ││ -│ │ │ │ local.push()│ [wid].send()│ .try_deliver() │ │ ││ +│ │ │ │ local.push()│ + unpark │ .try_deliver() │ │ ││ │ │ │ └──────────────┴──────────────┴─────────────────┘ │ ││ │ │ │ │ ││ │ │ │ spawn_any(addr, actor): │ ││ -│ │ │ wid = placement.next_worker() │ ││ +│ │ │ wid = placement.next_worker() (load-aware) │ ││ │ │ │ address_map.insert(addr, wid) │ ││ -│ │ │ spawn_txs[wid].send((addr, actor)) │ ││ +│ │ │ spawn_txs[wid].send((addr, actor)) + unpark │ ││ +│ │ │ │ ││ +│ │ │ request_stop(addr): → stop_requests.push(addr) │ ││ +│ │ │ schedule_timer(req): → timer_requests.push(req) │ ││ +│ │ │ where_is(name): → name_registry.lookup(name) │ ││ +│ │ │ monitor(w, t): → monitor_registry.register(w, t) │ ││ +│ │ │ join_group(a, g): → group_registry.join(g, a) │ ││ │ │ │ │ ││ │ │ └────────────────────────────────────────────────────────────┘ ││ │ │ ││ │ └────────────────────────────────────────────────────────────────────┘│ │ │ │ │ v │ -│ PHASE 4 --- Drain Pending Local │ +│ PHASE 4 --- Drain Spawn Queue (again) │ +│ ┌────────────────────────────────────────────────────────────────────┐│ +│ │ Actors spawned during phase 3 must be in the pool before ││ +│ │ pending_local delivery (phase 5). ││ +│ └────────────────────────────────────────────────────────────────────┘│ +│ │ │ +│ v │ +│ PHASE 5 --- Drain Pending Local │ │ ┌────────────────────────────────────────────────────────────────────┐│ │ │ ││ │ │ for (addr, msg) in pending_local.into_inner(): ││ │ │ pool.deliver(&addr, msg) ││ -│ │ --> slot.mailbox.push_back(msg) ││ -│ │ ││ │ │ these sit in the mailbox until NEXT tick ││ │ │ ││ │ └────────────────────────────────────────────────────────────────────┘│ +│ │ │ +│ v │ +│ PHASE 5.5 --- Drain Timer Requests │ +│ ┌────────────────────────────────────────────────────────────────────┐│ +│ │ ││ +│ │ for request in timer_requests: ││ +│ │ Once { dest, msg, ticks } → timers.add_once(dest, msg, ticks) ││ +│ │ Interval { dest, msg, p } → timers.add_interval(dest, msg, p) ││ +│ │ ││ +│ └────────────────────────────────────────────────────────────────────┘│ +│ │ │ +│ v │ +│ PHASE 6 --- Publish Stats │ +│ ┌────────────────────────────────────────────────────────────────────┐│ +│ │ ││ +│ │ if did_work: ││ +│ │ stats.num_actors, total_mailbox_depth, messages_processed ││ +│ │ stats.messages_dropped (if any overflow drops) ││ +│ │ stats_hook.on_tick(worker_id, snapshots) if configured ││ +│ │ ││ +│ │ record TickTiming (6-element phase_us array + processed + flag) ││ +│ │ ││ +│ └────────────────────────────────────────────────────────────────────┘│ +│ │ │ +│ v │ +│ PHASE 7 --- Cleanup Dead Actors │ +│ ┌────────────────────────────────────────────────────────────────────┐│ +│ │ ││ +│ │ pool.cleanup_dead() → Vec<(ActorAddress, StopReason)> ││ +│ │ stopping actors: call on_stop(&ctx) before removal ││ +│ │ poisoned actors: skip on_stop (state may be corrupt) ││ +│ │ ││ +│ │ for each dead (addr, reason): ││ +│ │ address_map.remove(&addr) ││ +│ │ name_registry.unregister_by_addr(&addr) ││ +│ │ group_registry.cleanup(&addr) ││ +│ │ ││ +│ │ deliver any messages sent during on_stop callbacks ││ +│ │ ││ +│ │ emit Down notifications for monitored dead actors: ││ +│ │ for (addr, reason) in dead: ││ +│ │ watchers = monitor_registry.take_monitors(&addr) ││ +│ │ for each watcher: route Down { addr, reason } ││ +│ │ same-worker → pool.deliver ││ +│ │ cross-worker → transfer_tx + unpark ││ +│ │ inbox → inbox_registry.try_deliver ││ +│ │ monitor_registry.remove_watcher(&addr) ││ +│ │ ││ +│ │ timers.gc_dead_intervals(dead_addrs) ││ +│ │ ││ +│ └────────────────────────────────────────────────────────────────────┘│ │ │ └────────────────────────────────────────────────────────────────────────┘ ``` @@ -400,7 +497,9 @@ Who holds what: ``` ┌─ User Code ────────────────────────────────────────────────────────────┐ -│ rt.spawn() rt.send_to() inbox.try_recv() rt.shutdown() │ +│ rt.spawn() rt.send_to() inbox.try_recv() rt.shutdown() │ +│ rt.spawn_named() rt.ask() rt.where_is() rt.stop_actor()│ +│ rt.join_group() rt.publish_to() rt.group_members() │ └────┬──────────────────┬──────────────────┬──────────────────┬──────────┘ │ │ ^ │ v v │ v @@ -408,10 +507,16 @@ Who holds what: │ │ │ ┌───────────┐ ┌────────────┐ ┌─────────────┐ ┌────────────┐ │ │ │AddressMap │ │ Placement │ │InboxRegistry│ │ is_running │ │ -│ │ addr->wid │ │ round-robin│ │ addr->Sender│ │ AtomicBool │ │ +│ │ addr->wid │ │ load-aware │ │ addr->Sender│ │ AtomicBool │ │ │ └─────┬─────┘ └──────┬─────┘ └──────┬──────┘ └──────┬─────┘ │ │ │ │ │ │ │ -│ ┌─────┴───────────────┴──────────────┴───────────────┴────────────────┐ │ +│ ┌─────────────┐ ┌──────────────┐ ┌─────────────┐ │ +│ │NameRegistry │ │MonitorRegist.│ │GroupRegistry │ │ +│ │ name->addr │ │ watched-> │ │ group->addrs │ │ +│ │ addr->name │ │ watchers │ │ addr->groups │ │ +│ └─────┬───────┘ └──────┬───────┘ └──────┬───────┘ │ +│ │ │ │ │ +│ ┌─────┴───────────────┴──────────────┴───────────────────────────────┐ │ │ │ TickContext (borrows all above) │ │ │ └──────────────────────────┬──────────────────────────────────────────┘ │ │ │ │