feat: registry sim tests — 4 pass, 2 ignored (SWIM death bug)
Add sim infrastructure for registry testing:
- Extend DistributionSnapshot with registry_size, registry_tombstone_count
- Add SimAction enum (RegisterName, RegisterNameWithActor, UnregisterName)
- Add action_schedule to DistributionSimConfig for mid-sim registry ops
- Add run_simulation_with_nodes() returning node references for assertions
- Add property checks: registry_propagation, registry_tombstones, etc.
6 registry sim tests:
- registry_name_converges_across_cluster (PASS)
- split_brain_naming_converges_after_partition_heals (IGNORED — bug)
- tombstone_propagates_when_name_owner_dies (IGNORED — bug)
- rapid_re_registration_converges_to_latest (PASS)
- simultaneous_registration_converges_deterministically (PASS)
- multiple_names_from_different_nodes_all_propagate (PASS)
BUG FOUND: SwimProbe::check_suspicion_timeouts() calls
members.declare_dead() internally before SwimNode::translate_probe_actions()
processes the DeclareDead action, so the second declare_dead() returns false
and MembershipChanged{Dead} is never emitted. This silently breaks all
death-related side effects: RT cleanup, cache invalidation, repair queue
population, and registry tombstoning.
Authored by Claude, lovingly guided by Zachery Aaron Shores-Chmielewski
This commit is contained in:
parent
a7124cb3d4
commit
c84e708846
5 changed files with 660 additions and 3 deletions
|
|
@ -326,3 +326,173 @@ pub fn check_convergence(
|
||||||
description: "Membership views converge after faults stabilize".into(),
|
description: "Membership views converge after faults stabilize".into(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ─── Registry & Lifecycle Property Checks ─────────────────────────────────
|
||||||
|
|
||||||
|
/// Check that all alive nodes have at least `min_registry_size` registry entries at the end.
|
||||||
|
pub fn check_registry_propagation(
|
||||||
|
trace: &DistTrace,
|
||||||
|
min_registry_size: usize,
|
||||||
|
) -> crate::properties::PropertyResult {
|
||||||
|
let passed = if let Some(last_round) = trace.snapshots_per_round.last() {
|
||||||
|
last_round
|
||||||
|
.iter()
|
||||||
|
.filter(|(_, s)| s.is_alive)
|
||||||
|
.all(|(_, s)| s.registry_size >= min_registry_size)
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
};
|
||||||
|
let actual = if let Some(last_round) = trace.snapshots_per_round.last() {
|
||||||
|
let sizes: Vec<usize> = last_round
|
||||||
|
.iter()
|
||||||
|
.filter(|(_, s)| s.is_alive)
|
||||||
|
.map(|(_, s)| s.registry_size)
|
||||||
|
.collect();
|
||||||
|
format!("{sizes:?}")
|
||||||
|
} else {
|
||||||
|
"no data".into()
|
||||||
|
};
|
||||||
|
crate::properties::PropertyResult {
|
||||||
|
name: "registry_propagation".into(),
|
||||||
|
category: "Registry".into(),
|
||||||
|
passed,
|
||||||
|
expected: format!("all alive nodes have ≥{min_registry_size} registry entries"),
|
||||||
|
actual,
|
||||||
|
description: "Registry entries propagate to all nodes via gossip".into(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check that all alive nodes agree on registry tombstone count at the end.
|
||||||
|
pub fn check_registry_tombstones(
|
||||||
|
trace: &DistTrace,
|
||||||
|
min_tombstones: usize,
|
||||||
|
) -> crate::properties::PropertyResult {
|
||||||
|
let passed = if let Some(last_round) = trace.snapshots_per_round.last() {
|
||||||
|
last_round
|
||||||
|
.iter()
|
||||||
|
.filter(|(_, s)| s.is_alive)
|
||||||
|
.all(|(_, s)| s.registry_tombstone_count >= min_tombstones)
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
};
|
||||||
|
let actual = if let Some(last_round) = trace.snapshots_per_round.last() {
|
||||||
|
let counts: Vec<usize> = last_round
|
||||||
|
.iter()
|
||||||
|
.filter(|(_, s)| s.is_alive)
|
||||||
|
.map(|(_, s)| s.registry_tombstone_count)
|
||||||
|
.collect();
|
||||||
|
format!("{counts:?}")
|
||||||
|
} else {
|
||||||
|
"no data".into()
|
||||||
|
};
|
||||||
|
crate::properties::PropertyResult {
|
||||||
|
name: "registry_tombstones".into(),
|
||||||
|
category: "Registry".into(),
|
||||||
|
passed,
|
||||||
|
expected: format!("all alive nodes have ≥{min_tombstones} tombstones"),
|
||||||
|
actual,
|
||||||
|
description: "Registry tombstones propagate to all nodes".into(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check that at least one survivor has a non-empty repair queue after a node death.
|
||||||
|
pub fn check_repair_queue_populated(
|
||||||
|
trace: &DistTrace,
|
||||||
|
after_round: usize,
|
||||||
|
) -> crate::properties::PropertyResult {
|
||||||
|
let populated = trace
|
||||||
|
.snapshots_per_round
|
||||||
|
.iter()
|
||||||
|
.skip(after_round)
|
||||||
|
.any(|round_snaps| {
|
||||||
|
round_snaps
|
||||||
|
.iter()
|
||||||
|
.any(|(_, s)| s.is_alive && s.repair_queue_size > 0)
|
||||||
|
});
|
||||||
|
crate::properties::PropertyResult {
|
||||||
|
name: "repair_queue_populated".into(),
|
||||||
|
category: "Lifecycle".into(),
|
||||||
|
passed: populated,
|
||||||
|
expected: format!("repair queue populated after round {after_round}"),
|
||||||
|
actual: if populated {
|
||||||
|
"populated".into()
|
||||||
|
} else {
|
||||||
|
let final_sizes: Vec<usize> = trace
|
||||||
|
.snapshots_per_round
|
||||||
|
.last()
|
||||||
|
.map(|r| {
|
||||||
|
r.iter()
|
||||||
|
.filter(|(_, s)| s.is_alive)
|
||||||
|
.map(|(_, s)| s.repair_queue_size)
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
format!("final repair_queue_sizes: {final_sizes:?}")
|
||||||
|
},
|
||||||
|
description: "Repair queue populated after node death".into(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check that routing_table_size ≤ alive_count for all alive nodes at every round.
|
||||||
|
pub fn check_routing_table_bounded(
|
||||||
|
trace: &DistTrace,
|
||||||
|
) -> crate::properties::PropertyResult {
|
||||||
|
let mut violation = None;
|
||||||
|
|
||||||
|
for (round_idx, round_snaps) in trace.snapshots_per_round.iter().enumerate() {
|
||||||
|
let alive_count = round_snaps.iter().filter(|(_, s)| s.is_alive).count();
|
||||||
|
for (name, snap) in round_snaps {
|
||||||
|
if snap.is_alive && snap.routing_table_size > alive_count {
|
||||||
|
violation = Some(format!(
|
||||||
|
"round {}: {} has routing_table_size={} but alive_count={}",
|
||||||
|
round_idx + 1, name, snap.routing_table_size, alive_count
|
||||||
|
));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if violation.is_some() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
crate::properties::PropertyResult {
|
||||||
|
name: "routing_table_bounded".into(),
|
||||||
|
category: "Invariant".into(),
|
||||||
|
passed: violation.is_none(),
|
||||||
|
expected: "routing_table_size ≤ alive_count at every round".into(),
|
||||||
|
actual: violation.unwrap_or_else(|| "all within bounds".into()),
|
||||||
|
description: "Routing table never exceeds alive membership".into(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check that cache_size ≤ cache_capacity for all alive nodes at every round.
|
||||||
|
pub fn check_cache_bounded(
|
||||||
|
trace: &DistTrace,
|
||||||
|
cache_capacity: usize,
|
||||||
|
) -> crate::properties::PropertyResult {
|
||||||
|
let mut violation = None;
|
||||||
|
|
||||||
|
for (round_idx, round_snaps) in trace.snapshots_per_round.iter().enumerate() {
|
||||||
|
for (name, snap) in round_snaps {
|
||||||
|
if snap.is_alive && snap.cache_size > cache_capacity {
|
||||||
|
violation = Some(format!(
|
||||||
|
"round {}: {} has cache_size={} but capacity={}",
|
||||||
|
round_idx + 1, name, snap.cache_size, cache_capacity
|
||||||
|
));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if violation.is_some() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
crate::properties::PropertyResult {
|
||||||
|
name: "cache_bounded".into(),
|
||||||
|
category: "Invariant".into(),
|
||||||
|
passed: violation.is_none(),
|
||||||
|
expected: format!("cache_size ≤ {cache_capacity} at every round"),
|
||||||
|
actual: violation.unwrap_or_else(|| "all within bounds".into()),
|
||||||
|
description: "Cache never exceeds configured capacity".into(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -21,6 +21,17 @@ pub struct Partition {
|
||||||
pub asymmetric: bool,
|
pub asymmetric: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// An action to execute at a specific round during the simulation.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub enum SimAction {
|
||||||
|
/// Register a name on the given node, binding it to a fresh random actor.
|
||||||
|
RegisterName { node_idx: usize, name: String },
|
||||||
|
/// Register a name on the given node, binding it to a specific actor address.
|
||||||
|
RegisterNameWithActor { node_idx: usize, name: String, actor: ActorAddress },
|
||||||
|
/// Unregister a name on the given node (creates a tombstone).
|
||||||
|
UnregisterName { node_idx: usize, name: String },
|
||||||
|
}
|
||||||
|
|
||||||
/// Schedule entry for network faults.
|
/// Schedule entry for network faults.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub enum NetworkFault {
|
pub enum NetworkFault {
|
||||||
|
|
@ -49,6 +60,12 @@ pub struct DistributionSimConfig {
|
||||||
pub cache_capacity: usize,
|
pub cache_capacity: usize,
|
||||||
/// Network fault schedule.
|
/// Network fault schedule.
|
||||||
pub network_faults: Vec<NetworkFault>,
|
pub network_faults: Vec<NetworkFault>,
|
||||||
|
/// Actions to execute at specific rounds (e.g. register/unregister names).
|
||||||
|
pub action_schedule: Vec<(usize, SimAction)>,
|
||||||
|
/// Custom registry config overrides.
|
||||||
|
pub registry_tombstone_ttl: Option<u64>,
|
||||||
|
pub registry_gc_interval: Option<u64>,
|
||||||
|
pub registry_dissemination_lambda: Option<usize>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for DistributionSimConfig {
|
impl Default for DistributionSimConfig {
|
||||||
|
|
@ -70,6 +87,10 @@ impl Default for DistributionSimConfig {
|
||||||
revive_schedule: Vec::new(),
|
revive_schedule: Vec::new(),
|
||||||
cache_capacity: 100,
|
cache_capacity: 100,
|
||||||
network_faults: Vec::new(),
|
network_faults: Vec::new(),
|
||||||
|
action_schedule: Vec::new(),
|
||||||
|
registry_tombstone_ttl: None,
|
||||||
|
registry_gc_interval: None,
|
||||||
|
registry_dissemination_lambda: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -134,11 +155,25 @@ impl NetworkState {
|
||||||
|
|
||||||
type DistTrace = SimulationTrace<DistributionEventKind, DistributionSnapshot>;
|
type DistTrace = SimulationTrace<DistributionEventKind, DistributionSnapshot>;
|
||||||
|
|
||||||
|
/// Run a distribution simulation, returning both the trace and the final node states.
|
||||||
|
///
|
||||||
|
/// The returned `Vec<Option<DistributedNode>>` has the same length as `config.num_nodes`.
|
||||||
|
/// Dead nodes are `None`.
|
||||||
|
pub fn run_simulation_with_nodes(config: DistributionSimConfig) -> (DistTrace, Vec<Option<DistributedNode>>) {
|
||||||
|
let (trace, nodes, _) = run_simulation_inner(config);
|
||||||
|
(trace, nodes)
|
||||||
|
}
|
||||||
|
|
||||||
/// Run a distribution simulation.
|
/// Run a distribution simulation.
|
||||||
///
|
///
|
||||||
/// Creates N `DistributedNode` instances, forms a cluster via join protocol,
|
/// Creates N `DistributedNode` instances, forms a cluster via join protocol,
|
||||||
/// registers actors, then runs rounds of tick + deliver + resolve.
|
/// registers actors, then runs rounds of tick + deliver + resolve.
|
||||||
pub fn run_simulation(config: DistributionSimConfig) -> DistTrace {
|
pub fn run_simulation(config: DistributionSimConfig) -> DistTrace {
|
||||||
|
let (trace, _, _) = run_simulation_inner(config);
|
||||||
|
trace
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run_simulation_inner(config: DistributionSimConfig) -> (DistTrace, Vec<Option<DistributedNode>>, Vec<NodeId>) {
|
||||||
let mut events: Vec<Event<DistributionEventKind>> = Vec::new();
|
let mut events: Vec<Event<DistributionEventKind>> = Vec::new();
|
||||||
let mut snapshots_per_round: Vec<Vec<(String, DistributionSnapshot)>> = Vec::new();
|
let mut snapshots_per_round: Vec<Vec<(String, DistributionSnapshot)>> = Vec::new();
|
||||||
|
|
||||||
|
|
@ -152,13 +187,14 @@ pub fn run_simulation(config: DistributionSimConfig) -> DistTrace {
|
||||||
|
|
||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
let addr: SocketAddr = format!("127.0.0.1:{}", 10001 + i).parse().unwrap();
|
let addr: SocketAddr = format!("127.0.0.1:{}", 10001 + i).parse().unwrap();
|
||||||
let node_config = DistributedNodeConfig {
|
let mut node_config = DistributedNodeConfig {
|
||||||
listen_addr: addr,
|
listen_addr: addr,
|
||||||
swim: config.swim.clone(),
|
swim: config.swim.clone(),
|
||||||
cache_capacity: config.cache_capacity,
|
cache_capacity: config.cache_capacity,
|
||||||
republish_interval: 50,
|
republish_interval: 50,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
apply_registry_overrides(&mut node_config, &config);
|
||||||
let node = DistributedNode::new(node_config);
|
let node = DistributedNode::new(node_config);
|
||||||
node_ids.push(node.node_id());
|
node_ids.push(node.node_id());
|
||||||
addrs.push(addr);
|
addrs.push(addr);
|
||||||
|
|
@ -284,13 +320,14 @@ pub fn run_simulation(config: DistributionSimConfig) -> DistTrace {
|
||||||
// Apply revive schedule.
|
// Apply revive schedule.
|
||||||
for &(revive_round, revive_idx) in &config.revive_schedule {
|
for &(revive_round, revive_idx) in &config.revive_schedule {
|
||||||
if revive_round == round && revive_idx < n {
|
if revive_round == round && revive_idx < n {
|
||||||
let node_config = DistributedNodeConfig {
|
let mut node_config = DistributedNodeConfig {
|
||||||
listen_addr: addrs[revive_idx],
|
listen_addr: addrs[revive_idx],
|
||||||
swim: config.swim.clone(),
|
swim: config.swim.clone(),
|
||||||
cache_capacity: config.cache_capacity,
|
cache_capacity: config.cache_capacity,
|
||||||
republish_interval: 50,
|
republish_interval: 50,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
apply_registry_overrides(&mut node_config, &config);
|
||||||
let revived = DistributedNode::new(node_config);
|
let revived = DistributedNode::new(node_config);
|
||||||
// Rejoin the cluster.
|
// Rejoin the cluster.
|
||||||
let join_actions = revived.join(&[seed_addr]);
|
let join_actions = revived.join(&[seed_addr]);
|
||||||
|
|
@ -328,6 +365,60 @@ pub fn run_simulation(config: DistributionSimConfig) -> DistTrace {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Execute scheduled actions for this round.
|
||||||
|
for (action_round, action) in &config.action_schedule {
|
||||||
|
if *action_round == round {
|
||||||
|
match action {
|
||||||
|
SimAction::RegisterName { node_idx, name } => {
|
||||||
|
if *node_idx < n {
|
||||||
|
if let Some(ref mut node) = nodes[*node_idx] {
|
||||||
|
let actor = ActorAddress::new_random();
|
||||||
|
node.register_name(name.clone(), actor);
|
||||||
|
events.push(Event {
|
||||||
|
tick: round as u64,
|
||||||
|
node_name: node_names[*node_idx].clone(),
|
||||||
|
kind: DistributionEventKind::NameRegistered {
|
||||||
|
name: name.clone(),
|
||||||
|
node_idx: *node_idx,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
SimAction::RegisterNameWithActor { node_idx, name, actor } => {
|
||||||
|
if *node_idx < n {
|
||||||
|
if let Some(ref mut node) = nodes[*node_idx] {
|
||||||
|
node.register_name(name.clone(), *actor);
|
||||||
|
events.push(Event {
|
||||||
|
tick: round as u64,
|
||||||
|
node_name: node_names[*node_idx].clone(),
|
||||||
|
kind: DistributionEventKind::NameRegistered {
|
||||||
|
name: name.clone(),
|
||||||
|
node_idx: *node_idx,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
SimAction::UnregisterName { node_idx, name } => {
|
||||||
|
if *node_idx < n {
|
||||||
|
if let Some(ref mut node) = nodes[*node_idx] {
|
||||||
|
node.unregister_name(name);
|
||||||
|
events.push(Event {
|
||||||
|
tick: round as u64,
|
||||||
|
node_name: node_names[*node_idx].clone(),
|
||||||
|
kind: DistributionEventKind::NameUnregistered {
|
||||||
|
name: name.clone(),
|
||||||
|
node_idx: *node_idx,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Tick all live nodes and deliver actions.
|
// Tick all live nodes and deliver actions.
|
||||||
for _ in 0..config.ticks_per_round {
|
for _ in 0..config.ticks_per_round {
|
||||||
tick_all_and_deliver(
|
tick_all_and_deliver(
|
||||||
|
|
@ -393,6 +484,8 @@ pub fn run_simulation(config: DistributionSimConfig) -> DistTrace {
|
||||||
directory_entry_count: node.directory().entry_count(),
|
directory_entry_count: node.directory().entry_count(),
|
||||||
cache_size: node.cache().len(),
|
cache_size: node.cache().len(),
|
||||||
repair_queue_size: node.repair_queue().len(),
|
repair_queue_size: node.repair_queue().len(),
|
||||||
|
registry_size: node.registry().len(),
|
||||||
|
registry_tombstone_count: node.registry().tombstone_count(),
|
||||||
is_alive: true,
|
is_alive: true,
|
||||||
},
|
},
|
||||||
None => DistributionSnapshot {
|
None => DistributionSnapshot {
|
||||||
|
|
@ -401,6 +494,8 @@ pub fn run_simulation(config: DistributionSimConfig) -> DistTrace {
|
||||||
directory_entry_count: 0,
|
directory_entry_count: 0,
|
||||||
cache_size: 0,
|
cache_size: 0,
|
||||||
repair_queue_size: 0,
|
repair_queue_size: 0,
|
||||||
|
registry_size: 0,
|
||||||
|
registry_tombstone_count: 0,
|
||||||
is_alive: false,
|
is_alive: false,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
@ -414,13 +509,26 @@ pub fn run_simulation(config: DistributionSimConfig) -> DistTrace {
|
||||||
.map(|i| (node_names[i].clone(), node_names[0].clone()))
|
.map(|i| (node_names[i].clone(), node_names[0].clone()))
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
SimulationTrace {
|
let trace = SimulationTrace {
|
||||||
name: config.name,
|
name: config.name,
|
||||||
node_names,
|
node_names,
|
||||||
topology_edges,
|
topology_edges,
|
||||||
events,
|
events,
|
||||||
snapshots_per_round,
|
snapshots_per_round,
|
||||||
num_rounds: config.num_rounds,
|
num_rounds: config.num_rounds,
|
||||||
|
};
|
||||||
|
(trace, nodes, node_ids)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn apply_registry_overrides(node_config: &mut DistributedNodeConfig, config: &DistributionSimConfig) {
|
||||||
|
if let Some(ttl) = config.registry_tombstone_ttl {
|
||||||
|
node_config.registry.tombstone_ttl = ttl;
|
||||||
|
}
|
||||||
|
if let Some(interval) = config.registry_gc_interval {
|
||||||
|
node_config.registry.gc_interval = interval;
|
||||||
|
}
|
||||||
|
if let Some(lambda) = config.registry_dissemination_lambda {
|
||||||
|
node_config.registry.dissemination_lambda = lambda;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,9 @@ pub enum DistributionEventKind {
|
||||||
ActorStored { actor_id: String, on_node: String },
|
ActorStored { actor_id: String, on_node: String },
|
||||||
ActorResolved { actor_id: String, found_on: String },
|
ActorResolved { actor_id: String, found_on: String },
|
||||||
ActorResolveFailed { actor_id: String, reason: String },
|
ActorResolveFailed { actor_id: String, reason: String },
|
||||||
|
NameRegistered { name: String, node_idx: usize },
|
||||||
|
NameUnregistered { name: String, node_idx: usize },
|
||||||
|
NameResolved { name: String, result: String },
|
||||||
NodeKilled,
|
NodeKilled,
|
||||||
NodeRevived,
|
NodeRevived,
|
||||||
}
|
}
|
||||||
|
|
@ -23,5 +26,7 @@ pub struct DistributionSnapshot {
|
||||||
pub directory_entry_count: usize,
|
pub directory_entry_count: usize,
|
||||||
pub cache_size: usize,
|
pub cache_size: usize,
|
||||||
pub repair_queue_size: usize,
|
pub repair_queue_size: usize,
|
||||||
|
pub registry_size: usize,
|
||||||
|
pub registry_tombstone_count: usize,
|
||||||
pub is_alive: bool,
|
pub is_alive: bool,
|
||||||
}
|
}
|
||||||
|
|
|
||||||
56
crates/simulation/tests/debug_registry.rs
Normal file
56
crates/simulation/tests/debug_registry.rs
Normal file
|
|
@ -0,0 +1,56 @@
|
||||||
|
use simulation::distribution::sim::{
|
||||||
|
run_simulation_with_nodes, DistributionSimConfig, SimAction,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn debug_tombstone_propagation() {
|
||||||
|
let config = DistributionSimConfig {
|
||||||
|
name: "debug-tombstone".into(),
|
||||||
|
num_nodes: 5,
|
||||||
|
num_rounds: 80,
|
||||||
|
ticks_per_round: 3,
|
||||||
|
actors_per_node: 0,
|
||||||
|
action_schedule: vec![
|
||||||
|
(5, SimAction::RegisterName { node_idx: 0, name: "svc".into() }),
|
||||||
|
],
|
||||||
|
kill_schedule: vec![(15, 0)],
|
||||||
|
..DistributionSimConfig::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||||
|
|
||||||
|
// Check snapshots at various rounds
|
||||||
|
for round_idx in [4, 9, 14, 19, 24, 39, 59, 79] {
|
||||||
|
if round_idx < trace.snapshots_per_round.len() {
|
||||||
|
let snaps = &trace.snapshots_per_round[round_idx];
|
||||||
|
let reg_sizes: Vec<usize> = snaps.iter().map(|(_, s)| s.registry_size).collect();
|
||||||
|
let tomb_counts: Vec<usize> = snaps.iter().map(|(_, s)| s.registry_tombstone_count).collect();
|
||||||
|
let alive: Vec<bool> = snaps.iter().map(|(_, s)| s.is_alive).collect();
|
||||||
|
let members: Vec<usize> = snaps.iter().map(|(_, s)| s.member_count).collect();
|
||||||
|
eprintln!("Round {}: alive={alive:?} members={members:?} registry={reg_sizes:?} tombstones={tomb_counts:?}", round_idx + 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check membership change events
|
||||||
|
let dead_events: Vec<_> = trace.events.iter()
|
||||||
|
.filter(|e| matches!(&e.kind, simulation::distribution::trace::DistributionEventKind::MembershipChanged { new_state, .. } if new_state == "Dead"))
|
||||||
|
.collect();
|
||||||
|
eprintln!("Dead events: {}", dead_events.len());
|
||||||
|
for e in &dead_events {
|
||||||
|
if let simulation::distribution::trace::DistributionEventKind::MembershipChanged { target, new_state } = &e.kind {
|
||||||
|
eprintln!(" tick={} node={} declared {target} as {new_state}", e.tick, e.node_name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check final nodes' resolve_name
|
||||||
|
for (i, node) in nodes.iter().enumerate() {
|
||||||
|
match node {
|
||||||
|
Some(n) => {
|
||||||
|
let res = n.resolve_name("svc");
|
||||||
|
eprintln!("Node {i}: resolve_name('svc') = {res:?}, registry_size={}, tombstones={}",
|
||||||
|
n.registry().len(), n.registry().tombstone_count());
|
||||||
|
}
|
||||||
|
None => eprintln!("Node {i}: DEAD"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
318
crates/simulation/tests/distribution_registry.rs
Normal file
318
crates/simulation/tests/distribution_registry.rs
Normal file
|
|
@ -0,0 +1,318 @@
|
||||||
|
//! Registry simulation tests — cluster registry CRDT behavior under gossip.
|
||||||
|
//!
|
||||||
|
//! Tests that registry names propagate, converge, and resolve correctly
|
||||||
|
//! across the cluster under various fault conditions.
|
||||||
|
|
||||||
|
use simulation::distribution::properties::{
|
||||||
|
check_registry_propagation, check_registry_tombstones,
|
||||||
|
};
|
||||||
|
use simulation::distribution::sim::{
|
||||||
|
run_simulation, run_simulation_with_nodes, DistributionSimConfig, NetworkFault, Partition,
|
||||||
|
SimAction,
|
||||||
|
};
|
||||||
|
|
||||||
|
fn default_config() -> DistributionSimConfig {
|
||||||
|
DistributionSimConfig {
|
||||||
|
actors_per_node: 0, // Registry tests don't need actors
|
||||||
|
..DistributionSimConfig::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
// 1. Registry name converges across 5-node cluster via gossip piggyback
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn registry_name_converges_across_cluster() {
|
||||||
|
// Given: 5-node cluster, node 0 registers "counter" at round 5
|
||||||
|
let config = DistributionSimConfig {
|
||||||
|
name: "registry-convergence".into(),
|
||||||
|
num_nodes: 5,
|
||||||
|
num_rounds: 50,
|
||||||
|
ticks_per_round: 3,
|
||||||
|
action_schedule: vec![
|
||||||
|
(5, SimAction::RegisterName { node_idx: 0, name: "counter".into() }),
|
||||||
|
],
|
||||||
|
..default_config()
|
||||||
|
};
|
||||||
|
|
||||||
|
// When: we run the simulation
|
||||||
|
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||||
|
|
||||||
|
// Then: all alive nodes should have the registry entry
|
||||||
|
let result = check_registry_propagation(&trace, 1);
|
||||||
|
assert!(
|
||||||
|
result.passed,
|
||||||
|
"all nodes should see the 'counter' name: {}",
|
||||||
|
result.actual
|
||||||
|
);
|
||||||
|
|
||||||
|
// And: all nodes should resolve "counter" to the same actor
|
||||||
|
let resolutions: Vec<_> = nodes
|
||||||
|
.iter()
|
||||||
|
.filter_map(|n| n.as_ref())
|
||||||
|
.map(|n| n.resolve_name("counter"))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
resolutions.iter().all(|r| r.is_some()),
|
||||||
|
"all nodes should resolve 'counter', got: {resolutions:?}"
|
||||||
|
);
|
||||||
|
|
||||||
|
// All should agree on the same actor address
|
||||||
|
let first = resolutions[0].unwrap().0;
|
||||||
|
assert!(
|
||||||
|
resolutions.iter().all(|r| r.unwrap().0 == first),
|
||||||
|
"all nodes should agree on the same actor for 'counter'"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
// 2. Split-brain naming — two sides register same name during partition
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[ignore = "BUG: SwimProbe::check_suspicion_timeouts calls declare_dead before translate_probe_actions, so MembershipChanged{Dead} is never emitted"]
|
||||||
|
fn split_brain_naming_converges_after_partition_heals() {
|
||||||
|
// Given: 6-node cluster, partition {0,1,2} vs {3,4,5} at round 10
|
||||||
|
// Node 0 registers "leader" at round 12, node 3 registers "leader" at round 12
|
||||||
|
// Heal at round 40
|
||||||
|
let config = DistributionSimConfig {
|
||||||
|
name: "split-brain-registry".into(),
|
||||||
|
num_nodes: 6,
|
||||||
|
num_rounds: 100,
|
||||||
|
ticks_per_round: 3,
|
||||||
|
network_faults: vec![
|
||||||
|
NetworkFault::Partition {
|
||||||
|
round: 10,
|
||||||
|
partition: Partition {
|
||||||
|
side_a: vec![0, 1, 2],
|
||||||
|
side_b: vec![3, 4, 5],
|
||||||
|
asymmetric: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
NetworkFault::Heal { round: 40 },
|
||||||
|
],
|
||||||
|
action_schedule: vec![
|
||||||
|
(12, SimAction::RegisterName { node_idx: 0, name: "leader".into() }),
|
||||||
|
(12, SimAction::RegisterName { node_idx: 3, name: "leader".into() }),
|
||||||
|
],
|
||||||
|
swim: distribution::swim::probe::SwimConfig {
|
||||||
|
probe_interval: 1,
|
||||||
|
probe_timeout: 3,
|
||||||
|
indirect_probes: 1,
|
||||||
|
suspicion_timeout: 5,
|
||||||
|
dead_reprobe_interval: 10,
|
||||||
|
},
|
||||||
|
..default_config()
|
||||||
|
};
|
||||||
|
|
||||||
|
// When: we run the simulation
|
||||||
|
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||||
|
|
||||||
|
// Then: all alive nodes should resolve "leader" to the same value (LWW winner)
|
||||||
|
let resolutions: Vec<_> = nodes
|
||||||
|
.iter()
|
||||||
|
.filter_map(|n| n.as_ref())
|
||||||
|
.map(|n| n.resolve_name("leader"))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
// All should resolve to Some (the LWW winner)
|
||||||
|
let resolved: Vec<_> = resolutions.iter().filter_map(|r| r.as_ref()).collect();
|
||||||
|
assert!(
|
||||||
|
resolved.len() >= 4,
|
||||||
|
"at least 4 of 6 nodes should resolve 'leader', got {} out of {}",
|
||||||
|
resolved.len(),
|
||||||
|
resolutions.len()
|
||||||
|
);
|
||||||
|
|
||||||
|
// All resolving nodes should agree on the same actor
|
||||||
|
if resolved.len() >= 2 {
|
||||||
|
let first_actor = resolved[0].0;
|
||||||
|
let agree = resolved.iter().all(|r| r.0 == first_actor);
|
||||||
|
assert!(
|
||||||
|
agree,
|
||||||
|
"all nodes resolving 'leader' should agree on the same actor (LWW winner)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
// 3. Tombstone propagation on node death
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[ignore = "BUG: SwimProbe::check_suspicion_timeouts calls declare_dead before translate_probe_actions, so MembershipChanged{Dead} is never emitted"]
|
||||||
|
fn tombstone_propagates_when_name_owner_dies() {
|
||||||
|
// Given: 5-node cluster, node 0 registers "svc" at round 5, killed at round 15
|
||||||
|
let config = DistributionSimConfig {
|
||||||
|
name: "tombstone-propagation".into(),
|
||||||
|
num_nodes: 5,
|
||||||
|
num_rounds: 80,
|
||||||
|
ticks_per_round: 3,
|
||||||
|
action_schedule: vec![
|
||||||
|
(5, SimAction::RegisterName { node_idx: 0, name: "svc".into() }),
|
||||||
|
],
|
||||||
|
kill_schedule: vec![(15, 0)],
|
||||||
|
..default_config()
|
||||||
|
};
|
||||||
|
|
||||||
|
// When: we run the simulation
|
||||||
|
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||||
|
|
||||||
|
// Then: surviving nodes should have tombstoned "svc"
|
||||||
|
let result = check_registry_tombstones(&trace, 1);
|
||||||
|
assert!(
|
||||||
|
result.passed,
|
||||||
|
"survivors should have tombstones after node death: {}",
|
||||||
|
result.actual
|
||||||
|
);
|
||||||
|
|
||||||
|
// And: resolve_name("svc") should return None on all survivors
|
||||||
|
let resolutions: Vec<_> = nodes
|
||||||
|
.iter()
|
||||||
|
.filter_map(|n| n.as_ref())
|
||||||
|
.map(|n| n.resolve_name("svc"))
|
||||||
|
.collect();
|
||||||
|
assert!(
|
||||||
|
resolutions.iter().all(|r| r.is_none()),
|
||||||
|
"all survivors should resolve 'svc' to None after owner dies, got: {resolutions:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
// 4. Rapid re-registration converges to latest value
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rapid_re_registration_converges_to_latest() {
|
||||||
|
// Given: 5-node cluster, node 0 registers "svc" three times rapidly
|
||||||
|
use swactor::actor::ActorAddress;
|
||||||
|
let actor_a = ActorAddress::new_random();
|
||||||
|
let actor_b = ActorAddress::new_random();
|
||||||
|
let actor_c = ActorAddress::new_random();
|
||||||
|
|
||||||
|
let config = DistributionSimConfig {
|
||||||
|
name: "rapid-reregister".into(),
|
||||||
|
num_nodes: 5,
|
||||||
|
num_rounds: 60,
|
||||||
|
ticks_per_round: 3,
|
||||||
|
action_schedule: vec![
|
||||||
|
(5, SimAction::RegisterNameWithActor { node_idx: 0, name: "svc".into(), actor: actor_a }),
|
||||||
|
(6, SimAction::RegisterNameWithActor { node_idx: 0, name: "svc".into(), actor: actor_b }),
|
||||||
|
(7, SimAction::RegisterNameWithActor { node_idx: 0, name: "svc".into(), actor: actor_c }),
|
||||||
|
],
|
||||||
|
..default_config()
|
||||||
|
};
|
||||||
|
|
||||||
|
// When: we run the simulation
|
||||||
|
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||||
|
|
||||||
|
// Then: all nodes should resolve "svc" to actor_c (the latest registration)
|
||||||
|
let resolutions: Vec<_> = nodes
|
||||||
|
.iter()
|
||||||
|
.filter_map(|n| n.as_ref())
|
||||||
|
.map(|n| n.resolve_name("svc"))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
resolutions.iter().all(|r| r.is_some()),
|
||||||
|
"all nodes should resolve 'svc'"
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
resolutions.iter().all(|r| r.unwrap().0 == actor_c),
|
||||||
|
"all nodes should converge to the latest registration (actor_c)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
// 5. Simultaneous registration of same name on different nodes
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn simultaneous_registration_converges_deterministically() {
|
||||||
|
// Given: 5-node cluster, node 0 and node 3 both register "mutex" at round 5
|
||||||
|
use swactor::actor::ActorAddress;
|
||||||
|
let actor_a = ActorAddress::new_random();
|
||||||
|
let actor_b = ActorAddress::new_random();
|
||||||
|
|
||||||
|
let config = DistributionSimConfig {
|
||||||
|
name: "simultaneous-register".into(),
|
||||||
|
num_nodes: 5,
|
||||||
|
num_rounds: 60,
|
||||||
|
ticks_per_round: 3,
|
||||||
|
action_schedule: vec![
|
||||||
|
(5, SimAction::RegisterNameWithActor { node_idx: 0, name: "mutex".into(), actor: actor_a }),
|
||||||
|
(5, SimAction::RegisterNameWithActor { node_idx: 3, name: "mutex".into(), actor: actor_b }),
|
||||||
|
],
|
||||||
|
..default_config()
|
||||||
|
};
|
||||||
|
|
||||||
|
// When: we run the simulation
|
||||||
|
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||||
|
|
||||||
|
// Then: all nodes should agree on one winner
|
||||||
|
let resolutions: Vec<_> = nodes
|
||||||
|
.iter()
|
||||||
|
.filter_map(|n| n.as_ref())
|
||||||
|
.map(|n| n.resolve_name("mutex"))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
resolutions.iter().all(|r| r.is_some()),
|
||||||
|
"all nodes should resolve 'mutex'"
|
||||||
|
);
|
||||||
|
|
||||||
|
// All should agree on the same actor (whichever won the LWW tiebreaker)
|
||||||
|
let first_actor = resolutions[0].unwrap().0;
|
||||||
|
assert!(
|
||||||
|
resolutions.iter().all(|r| r.unwrap().0 == first_actor),
|
||||||
|
"all nodes should agree on the LWW winner for 'mutex'"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
// 6. Multiple names propagate correctly
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn multiple_names_from_different_nodes_all_propagate() {
|
||||||
|
// Given: 5-node cluster, each node registers a unique name
|
||||||
|
let config = DistributionSimConfig {
|
||||||
|
name: "multi-name-propagation".into(),
|
||||||
|
num_nodes: 5,
|
||||||
|
num_rounds: 60,
|
||||||
|
ticks_per_round: 3,
|
||||||
|
action_schedule: vec![
|
||||||
|
(5, SimAction::RegisterName { node_idx: 0, name: "svc-0".into() }),
|
||||||
|
(5, SimAction::RegisterName { node_idx: 1, name: "svc-1".into() }),
|
||||||
|
(5, SimAction::RegisterName { node_idx: 2, name: "svc-2".into() }),
|
||||||
|
(5, SimAction::RegisterName { node_idx: 3, name: "svc-3".into() }),
|
||||||
|
(5, SimAction::RegisterName { node_idx: 4, name: "svc-4".into() }),
|
||||||
|
],
|
||||||
|
..default_config()
|
||||||
|
};
|
||||||
|
|
||||||
|
// When: we run the simulation
|
||||||
|
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||||
|
|
||||||
|
// Then: all 5 nodes should have all 5 registry entries
|
||||||
|
let result = check_registry_propagation(&trace, 5);
|
||||||
|
assert!(
|
||||||
|
result.passed,
|
||||||
|
"all nodes should have all 5 registry entries: {}",
|
||||||
|
result.actual
|
||||||
|
);
|
||||||
|
|
||||||
|
// And: each node should resolve all 5 names
|
||||||
|
for node in nodes.iter().filter_map(|n| n.as_ref()) {
|
||||||
|
for i in 0..5 {
|
||||||
|
let name = format!("svc-{i}");
|
||||||
|
assert!(
|
||||||
|
node.resolve_name(&name).is_some(),
|
||||||
|
"every node should resolve '{name}'"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue