swactor/src/guarantees/correspondence.rs

711 lines
27 KiB
Rust
Raw Normal View History

//! Correspondence tests: verify that kani bounded mirrors agree with
//! the real runtime.
//!
//! These are the drift detectors. If someone changes `tick_all`'s skip logic
//! or `Supervisor::handle_down`'s restart decision without updating the kani
//! mirrors, these tests fail.
//!
//! How they work:
//! - Drive the same inputs through BOTH the mirror logic AND the real runtime
//! - Assert they agree on observable outcomes
//! - Property-based (proptest) for coverage across the input space
use std::sync::Arc;
use proptest::prelude::*;
use crate::actor::{ActorAddress, ActorInterface, StopReason};
use crate::config::RuntimeConfig;
use crate::runtime::{Ctx, Runtime};
use crate::std::{
ChildSpec, RestartPolicy, StdExtension, Supervisor, SupervisorStrategy,
};
// ─── Helpers ────────────────────────────────────────────────────────────────
fn std_runtime() -> Runtime {
Runtime::new(RuntimeConfig::default()).with_extension(Arc::new(StdExtension::new()))
}
fn tick_many(rt: &Runtime, n: usize) {
for _ in 0..n {
rt.tick();
}
}
// ═══════════════════════════════════════════════════════════════════════════
// G4 Correspondence: lifecycle mirror vs real runtime
// ═══════════════════════════════════════════════════════════════════════════
// ─── Mirror (duplicated from g4_lifecycle.rs for cfg(test) visibility) ────
/// Whether the mirror predicts handle will be called for an actor with
/// these flags during tick_all.
fn mirror_should_process(poisoned: bool, stopping: bool, suspended: bool) -> bool {
// From g4_lifecycle.rs tick(): skip if poisoned || stopping || suspended
!poisoned && !stopping && !suspended
}
/// Whether the mirror predicts on_stop will be called during cleanup_dead.
fn mirror_should_on_stop(stopping: bool, poisoned: bool) -> bool {
// From g4_lifecycle.rs cleanup(): on_stop fires only when stopping && !poisoned
stopping && !poisoned
}
// ─── Real runtime actors for G4 correspondence ──────────────────────────
#[derive(Clone, Debug)]
struct Ping;
#[derive(Clone, Debug)]
struct HandleCalled(#[allow(dead_code)] ActorAddress);
#[derive(Clone, Debug)]
struct OnStopCalled(#[allow(dead_code)] ActorAddress);
/// Actor that reports when handle is called.
struct HandleReporter {
report_to: ActorAddress,
}
impl ActorInterface for HandleReporter {
type Incoming = Ping;
type Response = ();
fn handle(&mut self, ctx: &Ctx, _msg: Ping) {
let _ = ctx.send(self.report_to, HandleCalled(ctx.self_addr()));
}
fn on_stop(&mut self, ctx: &Ctx) {
let _ = ctx.send(self.report_to, OnStopCalled(ctx.self_addr()));
}
}
/// Actor that panics in on_start.
struct OnStartPanicker {
report_to: ActorAddress,
}
impl ActorInterface for OnStartPanicker {
type Incoming = Ping;
type Response = ();
fn on_start(&mut self, _ctx: &Ctx) {
panic!("intentional on_start panic");
}
fn handle(&mut self, ctx: &Ctx, _msg: Ping) {
let _ = ctx.send(self.report_to, HandleCalled(ctx.self_addr()));
}
fn on_stop(&mut self, ctx: &Ctx) {
let _ = ctx.send(self.report_to, OnStopCalled(ctx.self_addr()));
}
}
/// Actor that panics on first handle call.
struct HandlePanicker {
report_to: ActorAddress,
}
impl ActorInterface for HandlePanicker {
type Incoming = Ping;
type Response = ();
fn handle(&mut self, _ctx: &Ctx, _msg: Ping) {
panic!("intentional handle panic");
}
fn on_stop(&mut self, ctx: &Ctx) {
let _ = ctx.send(self.report_to, OnStopCalled(ctx.self_addr()));
}
}
// ─── G4 Property Tests ─────────────────────────────────────────────────────
proptest! {
#![proptest_config(ProptestConfig::with_cases(80))]
/// G4 correspondence: for a healthy actor (not poisoned, not stopping, not
/// suspended), the mirror predicts handle is called — the real runtime must
/// agree.
#[test]
fn g4_healthy_actor_handle_called(
msg_count in 1usize..=20,
) {
let rt = Runtime::new(RuntimeConfig::default());
let report_inbox = rt.new_inbox::<HandleCalled>().unwrap();
let report_addr = *report_inbox.addr();
let addr = rt.spawn(HandleReporter { report_to: report_addr }).unwrap();
rt.tick(); // on_start
// Mirror prediction: healthy actor should process messages
let mirror_predicts_process = mirror_should_process(false, false, false);
prop_assert!(mirror_predicts_process, "mirror must predict processing for healthy actor");
// Send messages and tick
for _ in 0..msg_count {
rt.send_to(addr, Ping).unwrap();
}
tick_many(&rt, msg_count + 5);
// Real runtime: check handle was called
let mut handle_count = 0;
while report_inbox.try_recv().is_some() {
handle_count += 1;
}
prop_assert_eq!(handle_count, msg_count, "runtime must call handle for each message");
}
/// G4 correspondence: a poisoned actor (panicked in on_start) must not
/// have handle called and must not have on_stop called.
#[test]
fn g4_poisoned_actor_no_handle_no_on_stop(
msg_count in 1usize..=10,
) {
let rt = Runtime::new(RuntimeConfig::default());
let handle_inbox = rt.new_inbox::<HandleCalled>().unwrap();
let stop_inbox = rt.new_inbox::<OnStopCalled>().unwrap();
let handle_addr = *handle_inbox.addr();
// Mirror predictions for poisoned actor
let mirror_predicts_process = mirror_should_process(true, false, false);
prop_assert!(!mirror_predicts_process, "mirror must predict NO processing for poisoned actor");
let mirror_predicts_on_stop = mirror_should_on_stop(false, true);
prop_assert!(!mirror_predicts_on_stop, "mirror must predict NO on_stop for poisoned actor");
let addr = rt.spawn(OnStartPanicker { report_to: handle_addr }).unwrap();
rt.tick(); // on_start panics → poisoned, cleanup removes from address map
// Send messages — actor is already removed, sends fail (expected)
for _ in 0..msg_count {
let _ = rt.send_to(addr, Ping);
}
tick_many(&rt, msg_count + 5);
// Real runtime: handle must NOT have been called
let handle_count: usize = std::iter::from_fn(|| handle_inbox.try_recv()).count();
prop_assert_eq!(handle_count, 0, "poisoned actor must not call handle");
// Real runtime: on_stop must NOT have been called
let stop_count: usize = std::iter::from_fn(|| stop_inbox.try_recv()).count();
prop_assert_eq!(stop_count, 0, "poisoned actor must not call on_stop");
}
/// G4 correspondence: a stopping actor must not have handle called,
/// but must have on_stop called exactly once.
#[test]
fn g4_stopping_actor_no_handle_yes_on_stop(
msg_count in 1usize..=10,
) {
let rt = Runtime::new(RuntimeConfig::default());
let handle_inbox = rt.new_inbox::<HandleCalled>().unwrap();
let stop_inbox = rt.new_inbox::<OnStopCalled>().unwrap();
let handle_addr = *handle_inbox.addr();
let _stop_addr = *stop_inbox.addr();
// Mirror predictions
let mirror_predicts_process = mirror_should_process(false, true, false);
prop_assert!(!mirror_predicts_process, "mirror must predict NO processing for stopping actor");
let mirror_predicts_on_stop = mirror_should_on_stop(true, false);
prop_assert!(mirror_predicts_on_stop, "mirror must predict on_stop for stopping && !poisoned");
let addr = rt.spawn(HandleReporter { report_to: handle_addr }).unwrap();
rt.tick(); // on_start
// Request stop
rt.stop_actor(addr).unwrap();
rt.tick(); // processes stop
// Send messages after stop (should be discarded or fail)
for _ in 0..msg_count {
let _ = rt.send_to(addr, Ping);
}
tick_many(&rt, 5);
// Drain handle reports — get only reports from this actor
let handle_count: usize = std::iter::from_fn(|| handle_inbox.try_recv()).count();
// The actor might process the StopSignal before any Ping arrives,
// or some pings might arrive before the stop signal. The key property:
// after stopping flag is set, no more handles are called.
// We verify this indirectly: messages sent after stop_actor aren't processed.
// on_stop must have been called exactly once
// The stop report goes to handle_addr — we need a separate inbox for stop
// Actually HandleReporter sends OnStopCalled to report_to (same addr).
// Let's just verify the actor is gone.
let _ = handle_count; // used above
// Respawn with proper report addresses
let rt2 = Runtime::new(RuntimeConfig::default());
let h_inbox = rt2.new_inbox::<HandleCalled>().unwrap();
let s_inbox = rt2.new_inbox::<OnStopCalled>().unwrap();
struct DualReporter {
handle_to: ActorAddress,
stop_to: ActorAddress,
}
impl ActorInterface for DualReporter {
type Incoming = Ping;
type Response = ();
fn handle(&mut self, ctx: &Ctx, _msg: Ping) {
let _ = ctx.send(self.handle_to, HandleCalled(ctx.self_addr()));
}
fn on_stop(&mut self, ctx: &Ctx) {
let _ = ctx.send(self.stop_to, OnStopCalled(ctx.self_addr()));
}
}
let addr2 = rt2.spawn(DualReporter {
handle_to: *h_inbox.addr(),
stop_to: *s_inbox.addr(),
}).unwrap();
rt2.tick(); // on_start
// Stop immediately, then send messages
rt2.stop_actor(addr2).unwrap();
for _ in 0..msg_count {
let _ = rt2.send_to(addr2, Ping);
}
tick_many(&rt2, 5);
// Messages sent after stop_actor should not be handled
let h_count: usize = std::iter::from_fn(|| h_inbox.try_recv()).count();
prop_assert_eq!(h_count, 0, "stopping actor must not call handle for messages sent after stop");
// on_stop must fire exactly once
let s_count: usize = std::iter::from_fn(|| s_inbox.try_recv()).count();
prop_assert_eq!(s_count, 1, "stopping actor must call on_stop exactly once");
}
/// G4 correspondence: a handle-panicked actor must not call on_stop.
#[test]
fn g4_handle_panic_poisons_no_on_stop(
_dummy in 0usize..1,
) {
let rt = Runtime::new(RuntimeConfig::default());
let stop_inbox = rt.new_inbox::<OnStopCalled>().unwrap();
let addr = rt.spawn(HandlePanicker { report_to: *stop_inbox.addr() }).unwrap();
rt.tick(); // on_start
// Send one message to trigger panic
rt.send_to(addr, Ping).unwrap();
tick_many(&rt, 5);
// Mirror prediction: poisoned actor gets no on_stop
let mirror_predicts_on_stop = mirror_should_on_stop(false, true);
prop_assert!(!mirror_predicts_on_stop);
// Real runtime: on_stop must NOT fire
let stop_count: usize = std::iter::from_fn(|| stop_inbox.try_recv()).count();
prop_assert_eq!(stop_count, 0, "handle-panicked actor must not call on_stop");
}
}
// ═══════════════════════════════════════════════════════════════════════════
// G10 Correspondence: restart mirror vs real supervisor
// ═══════════════════════════════════════════════════════════════════════════
// ─── Mirror (duplicated from g10_supervisor.rs for cfg(test) visibility) ──
/// Mirror specification of should_restart — independent of production code.
fn mirror_should_restart(policy: RestartPolicy, reason: StopReason) -> bool {
match policy {
RestartPolicy::Permanent => true,
RestartPolicy::Transient => reason == StopReason::Panicked,
RestartPolicy::Temporary => false,
}
}
// ─── Real runtime actors for G10 correspondence ─────────────────────────
#[derive(Clone, Debug)]
struct ChildStarted(ActorAddress);
#[derive(Clone, Debug)]
#[allow(dead_code)]
struct DownReport {
dead: ActorAddress,
reason: StopReason,
}
/// Actor that panics on receiving Ping — used to trigger Panicked death.
struct PanicOnPing;
impl ActorInterface for PanicOnPing {
type Incoming = Ping;
type Response = ();
fn handle(&mut self, _ctx: &Ctx, _msg: Ping) {
panic!("intentional child panic");
}
}
/// Actor that does nothing — used as a normal child.
struct IdleChild;
impl ActorInterface for IdleChild {
type Incoming = Ping;
type Response = ();
fn handle(&mut self, _ctx: &Ctx, _msg: Ping) {}
}
/// Proptest strategy for RestartPolicy.
fn arb_restart_policy() -> impl Strategy<Value = RestartPolicy> {
prop_oneof![
Just(RestartPolicy::Permanent),
Just(RestartPolicy::Transient),
Just(RestartPolicy::Temporary),
]
}
/// Proptest strategy for StopReason (only Normal and Panicked are relevant).
fn arb_stop_reason() -> impl Strategy<Value = StopReason> {
prop_oneof![
Just(StopReason::Normal),
Just(StopReason::Panicked),
]
}
/// Proptest strategy for SupervisorStrategy.
fn arb_strategy() -> impl Strategy<Value = SupervisorStrategy> {
prop_oneof![
Just(SupervisorStrategy::OneForOne),
Just(SupervisorStrategy::OneForAll),
Just(SupervisorStrategy::RestForOne),
]
}
// ─── G10 Property Tests ────────────────────────────────────────────────────
proptest! {
#![proptest_config(ProptestConfig::with_cases(60))]
/// G10 correspondence: the mirror's should_restart must agree with
/// the real supervisor's restart decision for all policy x reason combos.
///
/// We observe the real supervisor's behavior by:
/// 1. Spawning a supervisor with one child of the given policy
/// 2. Killing the child with the given reason (panic or normal stop)
/// 3. Checking whether the supervisor restarted the child
#[test]
fn g10_should_restart_matches_supervisor(
policy in arb_restart_policy(),
reason in arb_stop_reason(),
) {
let rt = std_runtime();
let report_inbox = rt.new_inbox::<ChildStarted>().unwrap();
let report_addr = *report_inbox.addr();
// Track child spawns via a shared counter
let spawn_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let spawn_count_clone = spawn_count.clone();
let spec = ChildSpec::new(
"test-child",
policy,
move |ctx| {
spawn_count_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
let addr = if reason == StopReason::Panicked {
ctx.spawn(PanicOnPing)?
} else {
ctx.spawn(IdleChild)?
};
let _ = ctx.send(report_addr, ChildStarted(addr));
Ok(addr)
},
);
let sup = Supervisor::new(SupervisorStrategy::OneForOne, 10, vec![spec]);
let _sup_addr = rt.spawn(sup).unwrap();
// Tick to start supervisor and child
tick_many(&rt, 3);
// Get child address from spawn report
let child_started = report_inbox.try_recv();
prop_assert!(child_started.is_some(), "child must have started");
let child_addr = child_started.unwrap().0;
let initial_spawns = spawn_count.load(std::sync::atomic::Ordering::SeqCst);
prop_assert_eq!(initial_spawns, 1, "exactly one child spawn initially");
// Kill child according to reason
match reason {
StopReason::Panicked => {
// Send message to trigger panic
rt.send_to(child_addr, Ping).unwrap();
}
StopReason::Normal | StopReason::Completed => {
// Normal stop
rt.stop_actor(child_addr).unwrap();
}
}
// Tick enough for supervisor to process Down and potentially restart
tick_many(&rt, 10);
// Check if child was restarted
let final_spawns = spawn_count.load(std::sync::atomic::Ordering::SeqCst);
let was_restarted = final_spawns > initial_spawns;
// Mirror prediction
let mirror_predicts_restart = mirror_should_restart(policy, reason);
prop_assert_eq!(
was_restarted, mirror_predicts_restart,
"mirror predicts restart={} but runtime restarted={} for policy={:?} reason={:?}",
mirror_predicts_restart, was_restarted, policy, reason
);
}
/// G10 correspondence: OneForOne strategy restarts only the dead child.
/// Mirror predicts only dead_idx restarted; runtime must agree.
#[test]
fn g10_one_for_one_restarts_only_dead(
num_children in 2usize..=4,
dead_idx_raw in 0usize..4,
) {
let dead_idx = dead_idx_raw % num_children;
let rt = std_runtime();
let report_inbox = rt.new_inbox::<ChildStarted>().unwrap();
let report_addr = *report_inbox.addr();
// Track per-child spawn counts
let spawn_counts: Vec<Arc<std::sync::atomic::AtomicUsize>> =
(0..num_children).map(|_| Arc::new(std::sync::atomic::AtomicUsize::new(0))).collect();
let specs: Vec<ChildSpec> = (0..num_children)
.map(|i| {
let counter = spawn_counts[i].clone();
let is_dead_child = i == dead_idx;
let report = report_addr;
ChildSpec::new(
format!("child-{}", i),
RestartPolicy::Permanent,
move |ctx| {
counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
let addr = if is_dead_child {
ctx.spawn(PanicOnPing)?
} else {
ctx.spawn(IdleChild)?
};
let _ = ctx.send(report, ChildStarted(addr));
Ok(addr)
},
)
})
.collect();
let sup = Supervisor::new(SupervisorStrategy::OneForOne, 10, specs);
let _sup_addr = rt.spawn(sup).unwrap();
tick_many(&rt, 5);
// Collect initial child addresses
let mut child_addrs = Vec::new();
while let Some(ChildStarted(addr)) = report_inbox.try_recv() {
child_addrs.push(addr);
}
prop_assert_eq!(child_addrs.len(), num_children, "all children must start");
// Record initial spawn counts
let initial_counts: Vec<usize> = spawn_counts.iter()
.map(|c| c.load(std::sync::atomic::Ordering::SeqCst))
.collect();
// Kill the designated child via panic
rt.send_to(child_addrs[dead_idx], Ping).unwrap();
tick_many(&rt, 10);
// Check which children were restarted
let final_counts: Vec<usize> = spawn_counts.iter()
.map(|c| c.load(std::sync::atomic::Ordering::SeqCst))
.collect();
for i in 0..num_children {
let restarted = final_counts[i] > initial_counts[i];
if i == dead_idx {
// Mirror: OneForOne restarts only dead child (Permanent policy)
prop_assert!(restarted,
"OneForOne: dead child {} must be restarted", i);
} else {
// Mirror: other children untouched
prop_assert!(!restarted,
"OneForOne: non-dead child {} must NOT be restarted", i);
}
}
}
/// G10 correspondence: Temporary policy never restarts, regardless of
/// strategy or death reason.
#[test]
fn g10_temporary_never_restarts(
strategy in arb_strategy(),
reason in arb_stop_reason(),
) {
let rt = std_runtime();
let spawn_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let spawn_count_clone = spawn_count.clone();
let spec = ChildSpec::new(
"temp-child",
RestartPolicy::Temporary,
move |ctx| {
spawn_count_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
let addr = if reason == StopReason::Panicked {
ctx.spawn(PanicOnPing)?
} else {
ctx.spawn(IdleChild)?
};
Ok(addr)
},
);
let report_inbox = rt.new_inbox::<ChildStarted>().unwrap();
let report_addr = *report_inbox.addr();
// For strategies that need multiple children, add idle permanent children
let mut specs = vec![spec];
for i in 0..2 {
let report = report_addr;
specs.push(ChildSpec::new(
format!("filler-{}", i),
RestartPolicy::Permanent,
move |ctx| {
let addr = ctx.spawn(IdleChild)?;
let _ = ctx.send(report, ChildStarted(addr));
Ok(addr)
},
));
}
let sup = Supervisor::new(strategy, 10, specs);
let _sup_addr = rt.spawn(sup).unwrap();
tick_many(&rt, 5);
let initial = spawn_count.load(std::sync::atomic::Ordering::SeqCst);
prop_assert_eq!(initial, 1, "temp child started once");
// Mirror prediction: Temporary → never restart
let mirror_predicts = mirror_should_restart(RestartPolicy::Temporary, reason);
prop_assert!(!mirror_predicts, "mirror must predict no restart for Temporary");
// Kill via the appropriate mechanism — but we need the child addr.
// We can get it from runtime stats or by tracking it. Since the factory
// already ran, we need to find the child. Let's just verify through
// spawn_count that no second spawn happens after death.
// The child is the first one spawned. We can trigger its death
// by sending it a stop or a message to panic.
// For simplicity, stop the supervisor — temporary children won't be restarted
// even if they die. The key assertion: spawn_count stays at 1.
// Actually we need to kill just the child, not the supervisor.
// Since we can't easily get the child address from outside, let's
// restructure to track it:
let rt2 = std_runtime();
let child_inbox = rt2.new_inbox::<ChildStarted>().unwrap();
let child_report = *child_inbox.addr();
let spawn_count2 = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let sc2 = spawn_count2.clone();
let spec2 = ChildSpec::new(
"temp-child",
RestartPolicy::Temporary,
move |ctx| {
sc2.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
let addr = if reason == StopReason::Panicked {
ctx.spawn(PanicOnPing)?
} else {
ctx.spawn(IdleChild)?
};
let _ = ctx.send(child_report, ChildStarted(addr));
Ok(addr)
},
);
let sup2 = Supervisor::new(strategy, 10, vec![spec2]);
let _sup_addr2 = rt2.spawn(sup2).unwrap();
tick_many(&rt2, 5);
let child_addr = child_inbox.try_recv().expect("child must start").0;
let init2 = spawn_count2.load(std::sync::atomic::Ordering::SeqCst);
// Kill child
match reason {
StopReason::Panicked => {
rt2.send_to(child_addr, Ping).unwrap();
}
_ => {
rt2.stop_actor(child_addr).unwrap();
}
}
tick_many(&rt2, 10);
let final2 = spawn_count2.load(std::sync::atomic::Ordering::SeqCst);
prop_assert_eq!(final2, init2,
"Temporary child must NOT be restarted: spawns before={} after={} strategy={:?} reason={:?}",
init2, final2, strategy, reason);
}
/// G10 correspondence: Transient + Normal stop → no restart.
/// Transient + Panicked → restart. Mirror must agree with runtime.
#[test]
fn g10_transient_restart_only_on_panic(
reason in arb_stop_reason(),
) {
let rt = std_runtime();
let child_inbox = rt.new_inbox::<ChildStarted>().unwrap();
let child_report = *child_inbox.addr();
let spawn_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let sc = spawn_count.clone();
let spec = ChildSpec::new(
"transient-child",
RestartPolicy::Transient,
move |ctx| {
sc.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
let addr = if reason == StopReason::Panicked {
ctx.spawn(PanicOnPing)?
} else {
ctx.spawn(IdleChild)?
};
let _ = ctx.send(child_report, ChildStarted(addr));
Ok(addr)
},
);
let sup = Supervisor::new(SupervisorStrategy::OneForOne, 10, vec![spec]);
let _sup_addr = rt.spawn(sup).unwrap();
tick_many(&rt, 5);
let child_addr = child_inbox.try_recv().expect("child must start").0;
let initial = spawn_count.load(std::sync::atomic::Ordering::SeqCst);
// Kill child
match reason {
StopReason::Panicked => {
rt.send_to(child_addr, Ping).unwrap();
}
_ => {
rt.stop_actor(child_addr).unwrap();
}
}
tick_many(&rt, 10);
let final_count = spawn_count.load(std::sync::atomic::Ordering::SeqCst);
let was_restarted = final_count > initial;
let mirror_predicts = mirror_should_restart(RestartPolicy::Transient, reason);
prop_assert_eq!(was_restarted, mirror_predicts,
"Transient: mirror predicts restart={} but runtime restarted={} for reason={:?}",
mirror_predicts, was_restarted, reason);
}
}