c84e708846
Add sim infrastructure for registry testing:
- Extend DistributionSnapshot with registry_size, registry_tombstone_count
- Add SimAction enum (RegisterName, RegisterNameWithActor, UnregisterName)
- Add action_schedule to DistributionSimConfig for mid-sim registry ops
- Add run_simulation_with_nodes() returning node references for assertions
- Add property checks: registry_propagation, registry_tombstones, etc.
6 registry sim tests:
- registry_name_converges_across_cluster (PASS)
- split_brain_naming_converges_after_partition_heals (IGNORED — bug)
- tombstone_propagates_when_name_owner_dies (IGNORED — bug)
- rapid_re_registration_converges_to_latest (PASS)
- simultaneous_registration_converges_deterministically (PASS)
- multiple_names_from_different_nodes_all_propagate (PASS)
BUG FOUND: SwimProbe::check_suspicion_timeouts() calls
members.declare_dead() internally before SwimNode::translate_probe_actions()
processes the DeclareDead action, so the second declare_dead() returns false
and MembershipChanged{Dead} is never emitted. This silently breaks all
death-related side effects: RT cleanup, cache invalidation, repair queue
population, and registry tombstoning.
Authored by Claude, lovingly guided by Zachery Aaron Shores-Chmielewski
56 lines
2.3 KiB
Rust
56 lines
2.3 KiB
Rust
use simulation::distribution::sim::{
|
|
run_simulation_with_nodes, DistributionSimConfig, SimAction,
|
|
};
|
|
|
|
#[test]
|
|
fn debug_tombstone_propagation() {
|
|
let config = DistributionSimConfig {
|
|
name: "debug-tombstone".into(),
|
|
num_nodes: 5,
|
|
num_rounds: 80,
|
|
ticks_per_round: 3,
|
|
actors_per_node: 0,
|
|
action_schedule: vec![
|
|
(5, SimAction::RegisterName { node_idx: 0, name: "svc".into() }),
|
|
],
|
|
kill_schedule: vec![(15, 0)],
|
|
..DistributionSimConfig::default()
|
|
};
|
|
|
|
let (trace, nodes) = run_simulation_with_nodes(config);
|
|
|
|
// Check snapshots at various rounds
|
|
for round_idx in [4, 9, 14, 19, 24, 39, 59, 79] {
|
|
if round_idx < trace.snapshots_per_round.len() {
|
|
let snaps = &trace.snapshots_per_round[round_idx];
|
|
let reg_sizes: Vec<usize> = snaps.iter().map(|(_, s)| s.registry_size).collect();
|
|
let tomb_counts: Vec<usize> = snaps.iter().map(|(_, s)| s.registry_tombstone_count).collect();
|
|
let alive: Vec<bool> = snaps.iter().map(|(_, s)| s.is_alive).collect();
|
|
let members: Vec<usize> = snaps.iter().map(|(_, s)| s.member_count).collect();
|
|
eprintln!("Round {}: alive={alive:?} members={members:?} registry={reg_sizes:?} tombstones={tomb_counts:?}", round_idx + 1);
|
|
}
|
|
}
|
|
|
|
// Check membership change events
|
|
let dead_events: Vec<_> = trace.events.iter()
|
|
.filter(|e| matches!(&e.kind, simulation::distribution::trace::DistributionEventKind::MembershipChanged { new_state, .. } if new_state == "Dead"))
|
|
.collect();
|
|
eprintln!("Dead events: {}", dead_events.len());
|
|
for e in &dead_events {
|
|
if let simulation::distribution::trace::DistributionEventKind::MembershipChanged { target, new_state } = &e.kind {
|
|
eprintln!(" tick={} node={} declared {target} as {new_state}", e.tick, e.node_name);
|
|
}
|
|
}
|
|
|
|
// Check final nodes' resolve_name
|
|
for (i, node) in nodes.iter().enumerate() {
|
|
match node {
|
|
Some(n) => {
|
|
let res = n.resolve_name("svc");
|
|
eprintln!("Node {i}: resolve_name('svc') = {res:?}, registry_size={}, tombstones={}",
|
|
n.registry().len(), n.registry().tombstone_count());
|
|
}
|
|
None => eprintln!("Node {i}: DEAD"),
|
|
}
|
|
}
|
|
}
|