crates/simulation/tests/distribution_lifecycle.rs

@@ -10,7 +10,7 @@ use simulation::distribution::properties::{
     check_repair_queue_populated, check_routing_table_bounded,
 };
 use simulation::distribution::sim::{
-    run_simulation, run_simulation_with_nodes, DistributionSimConfig, NetworkFault, Partition,
+    run_simulation_with_nodes, DistributionSimConfig, NetworkFault, Partition, SimAction,
 };
 
 fn default_config() -> DistributionSimConfig {
@@ -235,3 +235,159 @@ fn routing_table_bounded_by_alive_count() {
         result.actual
     );
 }
+
+// ────────────────────────────────────────────────────────────────────────────
+// 6. Combined: partition + death during partition + heal + verify
+// ────────────────────────────────────────────────────────────────────────────
+
+#[test]
+fn partition_then_death_during_partition_then_heal() {
+    // Given: 6 nodes, partition at r=10, node 2 (side A) killed during partition
+    // at r=20, heal at r=40. Node 2 had actors and a registry name.
+    // Use high suspicion_timeout so cross-partition nodes stay Suspect (not Dead),
+    // while within-partition death of node 2 is detected after timeout expires.
+    let config = DistributionSimConfig {
+        name: "partition-death-heal".into(),
+        num_nodes: 6,
+        num_rounds: 150,
+        ticks_per_round: 3,
+        actors_per_node: 2,
+        network_faults: vec![
+            NetworkFault::Partition {
+                round: 10,
+                partition: Partition {
+                    side_a: vec![0, 1, 2],
+                    side_b: vec![3, 4, 5],
+                    asymmetric: false,
+                },
+            },
+            NetworkFault::Heal { round: 40 },
+        ],
+        action_schedule: vec![
+            (5, SimAction::RegisterName { node_idx: 2, name: "doomed-svc".into() }),
+            (5, SimAction::RegisterName { node_idx: 4, name: "stable-svc".into() }),
+        ],
+        kill_schedule: vec![(20, 2)],
+        swim: distribution::swim::probe::SwimConfig {
+            probe_interval: 1,
+            probe_timeout: 3,
+            indirect_probes: 1,
+            // >90 ticks (30 rounds × 3 ticks) so cross-partition nodes stay Suspect during
+            // the 30-round partition. Node 2 (truly dead) gets declared dead ~33 rounds
+            // after kill, well after partition heals.
+            suspicion_timeout: 100,
+            dead_reprobe_interval: 10,
+        },
+        ..default_config()
+    };
+
+    let (_trace, nodes) = run_simulation_with_nodes(config);
+
+    let survivors: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect();
+    assert_eq!(survivors.len(), 5, "5 of 6 should survive");
+
+    // "doomed-svc" should be tombstoned (owner node 2 died)
+    for node in &survivors {
+        assert!(
+            node.resolve_name("doomed-svc").is_none(),
+            "doomed-svc should be tombstoned after owner died during partition"
+        );
+    }
+
+    // "stable-svc" should still resolve (node 4 alive throughout)
+    for node in &survivors {
+        assert!(
+            node.resolve_name("stable-svc").is_some(),
+            "stable-svc should resolve (owner survived partition)"
+        );
+    }
+
+    // After partition heal + dead reprobe, routing tables should recover
+    // (at least 4 entries for each surviving node)
+    for node in &survivors {
+        assert!(
+            node.routing_table().len() >= 3,
+            "surviving node should have ≥3 RT entries after partition heals, got {}",
+            node.routing_table().len()
+        );
+    }
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+// 7. Registry GC: tombstones are garbage-collected after TTL
+// ────────────────────────────────────────────────────────────────────────────
+
+#[test]
+fn registry_tombstones_gc_after_ttl() {
+    // Given: short tombstone TTL and GC interval, register then unregister a name.
+    // Then do many more register/unregister operations to advance the logical clock
+    // (which is used for TTL comparison). After enough clock advancement, the
+    // original tombstone should be garbage-collected.
+    let config = DistributionSimConfig {
+        name: "registry-gc".into(),
+        num_nodes: 5,
+        num_rounds: 80,
+        ticks_per_round: 3,
+        actors_per_node: 0,
+        // Very short GC: TTL=5 logical clock ticks, GC runs every 3 ticks
+        registry_tombstone_ttl: Some(5),
+        registry_gc_interval: Some(3),
+        action_schedule: vec![
+            (5, SimAction::RegisterName { node_idx: 0, name: "ephemeral".into() }),
+            (10, SimAction::UnregisterName { node_idx: 0, name: "ephemeral".into() }),
+            // Additional operations to advance the logical clock past the TTL
+            (15, SimAction::RegisterName { node_idx: 1, name: "churn-1".into() }),
+            (16, SimAction::RegisterName { node_idx: 2, name: "churn-2".into() }),
+            (17, SimAction::RegisterName { node_idx: 3, name: "churn-3".into() }),
+            (18, SimAction::RegisterName { node_idx: 4, name: "churn-4".into() }),
+            (19, SimAction::RegisterName { node_idx: 1, name: "churn-5".into() }),
+            (20, SimAction::RegisterName { node_idx: 2, name: "churn-6".into() }),
+        ],
+        ..default_config()
+    };
+
+    let (trace, nodes) = run_simulation_with_nodes(config);
+
+    // Shortly after unregister (round 12), tombstones should exist
+    let mid_tombstones: usize = trace.snapshots_per_round
+        .get(11) // round 12
+        .map(|snaps| {
+            snaps.iter()
+                .filter(|(_, s)| s.is_alive)
+                .map(|(_, s)| s.registry_tombstone_count)
+                .sum()
+        })
+        .unwrap_or(0);
+
+    assert!(
+        mid_tombstones > 0,
+        "tombstones should exist shortly after unregister"
+    );
+
+    // After many more register operations advance the clock, the "ephemeral" tombstone
+    // should be GC'd (its age exceeds TTL=5 in logical clock terms)
+    let alive_nodes: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect();
+
+    // Check that "ephemeral" resolves to None on all nodes (whether GC'd or still tombstoned)
+    for node in &alive_nodes {
+        assert!(
+            node.resolve_name("ephemeral").is_none(),
+            "ephemeral should not resolve (tombstoned or GC'd)"
+        );
+    }
+
+    // At least some nodes should have GC'd the tombstone (clock advanced past TTL)
+    let nodes_with_ephemeral_tombstone: usize = alive_nodes
+        .iter()
+        .filter(|n| {
+            n.registry().entries().any(|e| e.name == "ephemeral" && e.tombstone)
+        })
+        .count();
+
+    assert!(
+        nodes_with_ephemeral_tombstone < alive_nodes.len(),
+        "at least some nodes should have GC'd the 'ephemeral' tombstone, but {} of {} still have it",
+        nodes_with_ephemeral_tombstone,
+        alive_nodes.len()
+    );
+}