feat: cycle 6 lifecycle tests — asymmetric block, partition-register, bidirectional suspicion
3 new lifecycle sim tests: - asymmetric_one_way_block_does_not_kill_node: gossip recovery through intermediate nodes - names_registered_during_partition_propagate_after_heal: re_disseminate_all on Alive transition - bidirectional_suspicion_both_nodes_recover: mutual suspicion + refutation via gossip Authored by Claude, lovingly guided by Zachery Aaron Shores-Chmielewski
This commit is contained in:
parent
6f98b54d01
commit
7d55a01198
1 changed files with 199 additions and 0 deletions
|
|
@ -391,3 +391,202 @@ fn registry_tombstones_gc_after_ttl() {
|
||||||
alive_nodes.len()
|
alive_nodes.len()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
// 8. Indirect probes (PingReq) prevent false death on flaky direct path
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn asymmetric_one_way_block_does_not_kill_node() {
|
||||||
|
// Given: 6-node cluster. Asymmetric partition: 0→5 blocked, 5→0 works.
|
||||||
|
// Node 5 can still communicate with nodes 1-4 in both directions, and
|
||||||
|
// 5→0 works, so gossip piggyback keeps node 0 informed about node 5's
|
||||||
|
// aliveness through intermediate nodes.
|
||||||
|
//
|
||||||
|
// Note: this implementation relies on piggyback gossip for indirect
|
||||||
|
// recovery (PingReq ack forwarding is not implemented), so we use a
|
||||||
|
// generous suspicion_timeout to allow gossip propagation.
|
||||||
|
let config = DistributionSimConfig {
|
||||||
|
name: "one-way-block".into(),
|
||||||
|
num_nodes: 6,
|
||||||
|
num_rounds: 80,
|
||||||
|
ticks_per_round: 3,
|
||||||
|
actors_per_node: 0,
|
||||||
|
// Asymmetric: only 0→5 is blocked, all other paths work
|
||||||
|
network_faults: vec![
|
||||||
|
NetworkFault::Partition {
|
||||||
|
round: 10,
|
||||||
|
partition: Partition {
|
||||||
|
side_a: vec![0],
|
||||||
|
side_b: vec![5],
|
||||||
|
asymmetric: true, // 0→5 blocked, 5→0 works
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
swim: distribution::swim::probe::SwimConfig {
|
||||||
|
probe_interval: 1,
|
||||||
|
probe_timeout: 3,
|
||||||
|
indirect_probes: 3,
|
||||||
|
// Timeout must exceed total sim ticks (80*3=240) so node 0
|
||||||
|
// never declares node 5 dead despite the blocked direct path.
|
||||||
|
// Gossip through intermediate nodes refutes suspicion each cycle.
|
||||||
|
suspicion_timeout: 500,
|
||||||
|
dead_reprobe_interval: 10,
|
||||||
|
},
|
||||||
|
action_schedule: vec![
|
||||||
|
(5, SimAction::RegisterName { node_idx: 5, name: "target-svc".into() }),
|
||||||
|
],
|
||||||
|
..default_config()
|
||||||
|
};
|
||||||
|
|
||||||
|
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||||
|
|
||||||
|
// Then: all 6 nodes should still be alive (asymmetric block doesn't kill either side)
|
||||||
|
let alive_count = nodes.iter().filter(|n| n.is_some()).count();
|
||||||
|
assert_eq!(alive_count, 6, "all 6 nodes should be alive");
|
||||||
|
|
||||||
|
// And: node 5's registry name should be resolvable from all nodes
|
||||||
|
// (gossip carries the entry through intermediate nodes even if 0→5 is blocked)
|
||||||
|
for (i, node) in nodes.iter().enumerate() {
|
||||||
|
if let Some(node) = node {
|
||||||
|
assert!(
|
||||||
|
node.resolve_name("target-svc").is_some(),
|
||||||
|
"node {i} should resolve 'target-svc'"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
// 9. Names registered during partition propagate after heal via re_disseminate_all
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn names_registered_during_partition_propagate_after_heal() {
|
||||||
|
// Given: 6 nodes, partition {0,1,2} vs {3,4,5} from round 10 to 50.
|
||||||
|
// During the partition, each side registers a name the other side can't see.
|
||||||
|
// After healing, re_disseminate_all (triggered by Alive transitions) should
|
||||||
|
// propagate both names to the entire cluster.
|
||||||
|
let config = DistributionSimConfig {
|
||||||
|
name: "partition-register-heal".into(),
|
||||||
|
num_nodes: 6,
|
||||||
|
num_rounds: 120,
|
||||||
|
ticks_per_round: 3,
|
||||||
|
actors_per_node: 0,
|
||||||
|
network_faults: vec![
|
||||||
|
NetworkFault::Partition {
|
||||||
|
round: 10,
|
||||||
|
partition: Partition {
|
||||||
|
side_a: vec![0, 1, 2],
|
||||||
|
side_b: vec![3, 4, 5],
|
||||||
|
asymmetric: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
NetworkFault::Heal { round: 50 },
|
||||||
|
],
|
||||||
|
action_schedule: vec![
|
||||||
|
// Registered DURING partition — other side doesn't see these initially
|
||||||
|
(20, SimAction::RegisterName { node_idx: 0, name: "side-a-svc".into() }),
|
||||||
|
(20, SimAction::RegisterName { node_idx: 3, name: "side-b-svc".into() }),
|
||||||
|
],
|
||||||
|
swim: distribution::swim::probe::SwimConfig {
|
||||||
|
probe_interval: 1,
|
||||||
|
probe_timeout: 3,
|
||||||
|
indirect_probes: 1,
|
||||||
|
// High timeout: cross-partition nodes stay Suspect during 40-round partition
|
||||||
|
suspicion_timeout: 200,
|
||||||
|
dead_reprobe_interval: 10,
|
||||||
|
},
|
||||||
|
..default_config()
|
||||||
|
};
|
||||||
|
|
||||||
|
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||||
|
|
||||||
|
let alive: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect();
|
||||||
|
assert_eq!(alive.len(), 6, "all 6 nodes should survive");
|
||||||
|
|
||||||
|
// After healing + gossip, both names should be resolvable from every node
|
||||||
|
for (i, node) in nodes.iter().enumerate() {
|
||||||
|
if let Some(node) = node {
|
||||||
|
assert!(
|
||||||
|
node.resolve_name("side-a-svc").is_some(),
|
||||||
|
"node {i} should resolve 'side-a-svc' (registered during partition on side A)"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
node.resolve_name("side-b-svc").is_some(),
|
||||||
|
"node {i} should resolve 'side-b-svc' (registered during partition on side B)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
// 10. Bidirectional suspicion: two nodes suspect each other, both recover
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bidirectional_suspicion_both_nodes_recover() {
|
||||||
|
// Given: 6 nodes. Mutual partition between node 0 and node 5 (both directions)
|
||||||
|
// from round 10 to 30. Both sides can still reach nodes 1-4.
|
||||||
|
// Both 0 and 5 will suspect each other, but gossip through 1-4 carries
|
||||||
|
// refutations. After healing, both should be Alive with registry intact.
|
||||||
|
let config = DistributionSimConfig {
|
||||||
|
name: "bidir-suspicion".into(),
|
||||||
|
num_nodes: 6,
|
||||||
|
num_rounds: 80,
|
||||||
|
ticks_per_round: 3,
|
||||||
|
actors_per_node: 0,
|
||||||
|
network_faults: vec![
|
||||||
|
NetworkFault::Partition {
|
||||||
|
round: 10,
|
||||||
|
partition: Partition {
|
||||||
|
side_a: vec![0],
|
||||||
|
side_b: vec![5],
|
||||||
|
asymmetric: false, // full mutual block
|
||||||
|
},
|
||||||
|
},
|
||||||
|
NetworkFault::Heal { round: 30 },
|
||||||
|
],
|
||||||
|
action_schedule: vec![
|
||||||
|
(5, SimAction::RegisterName { node_idx: 0, name: "svc-zero".into() }),
|
||||||
|
(5, SimAction::RegisterName { node_idx: 5, name: "svc-five".into() }),
|
||||||
|
],
|
||||||
|
swim: distribution::swim::probe::SwimConfig {
|
||||||
|
probe_interval: 1,
|
||||||
|
probe_timeout: 3,
|
||||||
|
indirect_probes: 3,
|
||||||
|
// Must exceed partition duration (20 rounds × 3 ticks = 60 ticks)
|
||||||
|
suspicion_timeout: 100,
|
||||||
|
dead_reprobe_interval: 10,
|
||||||
|
},
|
||||||
|
..default_config()
|
||||||
|
};
|
||||||
|
|
||||||
|
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||||
|
|
||||||
|
// All 6 nodes alive
|
||||||
|
let alive_count = nodes.iter().filter(|n| n.is_some()).count();
|
||||||
|
assert_eq!(alive_count, 6, "all 6 nodes should survive bidirectional suspicion");
|
||||||
|
|
||||||
|
// Both registry names should resolve on all nodes
|
||||||
|
for (i, node) in nodes.iter().enumerate() {
|
||||||
|
if let Some(node) = node {
|
||||||
|
assert!(
|
||||||
|
node.resolve_name("svc-zero").is_some(),
|
||||||
|
"node {i} should resolve 'svc-zero'"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
node.resolve_name("svc-five").is_some(),
|
||||||
|
"node {i} should resolve 'svc-five'"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Both nodes 0 and 5 should see each other in their member lists
|
||||||
|
let node0 = nodes[0].as_ref().unwrap();
|
||||||
|
let node5 = nodes[5].as_ref().unwrap();
|
||||||
|
let node0_sees_5 = node0.members().iter().any(|m| m.node_id == node5.node_id());
|
||||||
|
let node5_sees_0 = node5.members().iter().any(|m| m.node_id == node0.node_id());
|
||||||
|
assert!(node0_sees_5, "node 0 should see node 5 as a member after healing");
|
||||||
|
assert!(node5_sees_0, "node 5 should see node 0 as a member after healing");
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue