From 6f98b54d01f24783f73a1b799dec479981d8e936 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Feb 2026 09:35:54 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20cycle=205=20lifecycle=20tests=20?= =?UTF-8?q?=E2=80=94=20partition+death=20combined,=20registry=20GC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New tests: - partition_then_death_during_partition_then_heal (combined lifecycle scenario) - registry_tombstones_gc_after_ttl (tombstone GC with short TTL + clock advancement) Total: 15 registry + 7 lifecycle + 9 property = 31 new distribution sim tests. Authored by Claude, lovingly guided by Zachery Aaron Shores-Chmielewski --- .../tests/distribution_lifecycle.rs | 158 +++++++++++++++++- 1 file changed, 157 insertions(+), 1 deletion(-) diff --git a/crates/simulation/tests/distribution_lifecycle.rs b/crates/simulation/tests/distribution_lifecycle.rs index fb9f31f..1c7dc6b 100644 --- a/crates/simulation/tests/distribution_lifecycle.rs +++ b/crates/simulation/tests/distribution_lifecycle.rs @@ -10,7 +10,7 @@ use simulation::distribution::properties::{ check_repair_queue_populated, check_routing_table_bounded, }; use simulation::distribution::sim::{ - run_simulation, run_simulation_with_nodes, DistributionSimConfig, NetworkFault, Partition, + run_simulation_with_nodes, DistributionSimConfig, NetworkFault, Partition, SimAction, }; fn default_config() -> DistributionSimConfig { @@ -235,3 +235,159 @@ fn routing_table_bounded_by_alive_count() { result.actual ); } + +// ──────────────────────────────────────────────────────────────────────────── +// 6. Combined: partition + death during partition + heal + verify +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +fn partition_then_death_during_partition_then_heal() { + // Given: 6 nodes, partition at r=10, node 2 (side A) killed during partition + // at r=20, heal at r=40. Node 2 had actors and a registry name. + // Use high suspicion_timeout so cross-partition nodes stay Suspect (not Dead), + // while within-partition death of node 2 is detected after timeout expires. + let config = DistributionSimConfig { + name: "partition-death-heal".into(), + num_nodes: 6, + num_rounds: 150, + ticks_per_round: 3, + actors_per_node: 2, + network_faults: vec![ + NetworkFault::Partition { + round: 10, + partition: Partition { + side_a: vec![0, 1, 2], + side_b: vec![3, 4, 5], + asymmetric: false, + }, + }, + NetworkFault::Heal { round: 40 }, + ], + action_schedule: vec![ + (5, SimAction::RegisterName { node_idx: 2, name: "doomed-svc".into() }), + (5, SimAction::RegisterName { node_idx: 4, name: "stable-svc".into() }), + ], + kill_schedule: vec![(20, 2)], + swim: distribution::swim::probe::SwimConfig { + probe_interval: 1, + probe_timeout: 3, + indirect_probes: 1, + // >90 ticks (30 rounds × 3 ticks) so cross-partition nodes stay Suspect during + // the 30-round partition. Node 2 (truly dead) gets declared dead ~33 rounds + // after kill, well after partition heals. + suspicion_timeout: 100, + dead_reprobe_interval: 10, + }, + ..default_config() + }; + + let (_trace, nodes) = run_simulation_with_nodes(config); + + let survivors: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect(); + assert_eq!(survivors.len(), 5, "5 of 6 should survive"); + + // "doomed-svc" should be tombstoned (owner node 2 died) + for node in &survivors { + assert!( + node.resolve_name("doomed-svc").is_none(), + "doomed-svc should be tombstoned after owner died during partition" + ); + } + + // "stable-svc" should still resolve (node 4 alive throughout) + for node in &survivors { + assert!( + node.resolve_name("stable-svc").is_some(), + "stable-svc should resolve (owner survived partition)" + ); + } + + // After partition heal + dead reprobe, routing tables should recover + // (at least 4 entries for each surviving node) + for node in &survivors { + assert!( + node.routing_table().len() >= 3, + "surviving node should have ≥3 RT entries after partition heals, got {}", + node.routing_table().len() + ); + } +} + +// ──────────────────────────────────────────────────────────────────────────── +// 7. Registry GC: tombstones are garbage-collected after TTL +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +fn registry_tombstones_gc_after_ttl() { + // Given: short tombstone TTL and GC interval, register then unregister a name. + // Then do many more register/unregister operations to advance the logical clock + // (which is used for TTL comparison). After enough clock advancement, the + // original tombstone should be garbage-collected. + let config = DistributionSimConfig { + name: "registry-gc".into(), + num_nodes: 5, + num_rounds: 80, + ticks_per_round: 3, + actors_per_node: 0, + // Very short GC: TTL=5 logical clock ticks, GC runs every 3 ticks + registry_tombstone_ttl: Some(5), + registry_gc_interval: Some(3), + action_schedule: vec![ + (5, SimAction::RegisterName { node_idx: 0, name: "ephemeral".into() }), + (10, SimAction::UnregisterName { node_idx: 0, name: "ephemeral".into() }), + // Additional operations to advance the logical clock past the TTL + (15, SimAction::RegisterName { node_idx: 1, name: "churn-1".into() }), + (16, SimAction::RegisterName { node_idx: 2, name: "churn-2".into() }), + (17, SimAction::RegisterName { node_idx: 3, name: "churn-3".into() }), + (18, SimAction::RegisterName { node_idx: 4, name: "churn-4".into() }), + (19, SimAction::RegisterName { node_idx: 1, name: "churn-5".into() }), + (20, SimAction::RegisterName { node_idx: 2, name: "churn-6".into() }), + ], + ..default_config() + }; + + let (trace, nodes) = run_simulation_with_nodes(config); + + // Shortly after unregister (round 12), tombstones should exist + let mid_tombstones: usize = trace.snapshots_per_round + .get(11) // round 12 + .map(|snaps| { + snaps.iter() + .filter(|(_, s)| s.is_alive) + .map(|(_, s)| s.registry_tombstone_count) + .sum() + }) + .unwrap_or(0); + + assert!( + mid_tombstones > 0, + "tombstones should exist shortly after unregister" + ); + + // After many more register operations advance the clock, the "ephemeral" tombstone + // should be GC'd (its age exceeds TTL=5 in logical clock terms) + let alive_nodes: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect(); + + // Check that "ephemeral" resolves to None on all nodes (whether GC'd or still tombstoned) + for node in &alive_nodes { + assert!( + node.resolve_name("ephemeral").is_none(), + "ephemeral should not resolve (tombstoned or GC'd)" + ); + } + + // At least some nodes should have GC'd the tombstone (clock advanced past TTL) + let nodes_with_ephemeral_tombstone: usize = alive_nodes + .iter() + .filter(|n| { + n.registry().entries().any(|e| e.name == "ephemeral" && e.tombstone) + }) + .count(); + + assert!( + nodes_with_ephemeral_tombstone < alive_nodes.len(), + "at least some nodes should have GC'd the 'ephemeral' tombstone, but {} of {} still have it", + nodes_with_ephemeral_tombstone, + alive_nodes.len() + ); +}