Zachery Aaron Shores-Chmielewski
3cf0ae5947
Remove co-dependencies for different modules found in `crates` and migrate the development history to a new repository. The docs were stale, and largely not getting used, so simply deleted for now. When code stabilizes more, they will become useful again. Co-authored-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com> Co-committed-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
753 lines
31 KiB
Rust
753 lines
31 KiB
Rust
#![cfg(feature = "distribution")]
|
|
//! Registry simulation tests — cluster registry CRDT behavior under gossip.
|
|
//!
|
|
//! Tests that registry names propagate, converge, and resolve correctly
|
|
//! across the cluster under various fault conditions.
|
|
|
|
use simulation::distribution::properties::{
|
|
check_registry_propagation, check_registry_tombstones,
|
|
};
|
|
use simulation::distribution::sim::{
|
|
run_simulation_with_nodes, DistributionSimConfig, DistTrace, NetworkFault, Partition,
|
|
SimAction,
|
|
};
|
|
|
|
fn maybe_save_trace(trace: &DistTrace) {
|
|
if let Ok(dir) = std::env::var("SWACTOR_TRACE_DIR") {
|
|
std::fs::create_dir_all(&dir).ok();
|
|
let filename = format!(
|
|
"{}/{}.trace.json",
|
|
dir,
|
|
trace.name.to_lowercase().replace(' ', "_").replace(['(', ')'], "")
|
|
);
|
|
let json = serde_json::to_string_pretty(trace).expect("trace serialization failed");
|
|
std::fs::write(&filename, json).expect("trace write failed");
|
|
}
|
|
}
|
|
|
|
fn default_config() -> DistributionSimConfig {
|
|
DistributionSimConfig {
|
|
actors_per_node: 0, // Registry tests don't need actors
|
|
..DistributionSimConfig::default()
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
// 1. Registry name converges across 5-node cluster via gossip piggyback
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
|
|
#[test]
|
|
fn registry_name_converges_across_cluster() {
|
|
// Given: 5-node cluster, node 0 registers "counter" at round 5
|
|
let config = DistributionSimConfig {
|
|
name: "registry-convergence".into(),
|
|
num_nodes: 5,
|
|
num_rounds: 50,
|
|
ticks_per_round: 3,
|
|
action_schedule: vec![
|
|
(5, SimAction::RegisterName { node_idx: 0, name: "counter".into() }),
|
|
],
|
|
..default_config()
|
|
};
|
|
|
|
// When: we run the simulation
|
|
let (trace, nodes) = run_simulation_with_nodes(config);
|
|
maybe_save_trace(&trace);
|
|
|
|
// Then: all alive nodes should have the registry entry
|
|
let result = check_registry_propagation(&trace, 1);
|
|
assert!(
|
|
result.passed,
|
|
"all nodes should see the 'counter' name: {}",
|
|
result.actual
|
|
);
|
|
|
|
// And: all nodes should resolve "counter" to the same actor
|
|
let resolutions: Vec<_> = nodes
|
|
.iter()
|
|
.filter_map(|n| n.as_ref())
|
|
.map(|n| n.resolve_name("counter"))
|
|
.collect();
|
|
|
|
assert!(
|
|
resolutions.iter().all(|r| r.is_some()),
|
|
"all nodes should resolve 'counter', got: {resolutions:?}"
|
|
);
|
|
|
|
// All should agree on the same actor address
|
|
let first = resolutions[0].unwrap().0;
|
|
assert!(
|
|
resolutions.iter().all(|r| r.unwrap().0 == first),
|
|
"all nodes should agree on the same actor for 'counter'"
|
|
);
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
// 2. Split-brain naming — two sides register same name during partition
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
|
|
#[test]
|
|
fn split_brain_naming_converges_after_partition_heals() {
|
|
// Given: 6-node cluster, partition {0,1,2} vs {3,4,5} at round 10
|
|
// Node 0 registers "leader" at round 12, node 3 registers "leader" at round 12
|
|
// Heal at round 40
|
|
let config = DistributionSimConfig {
|
|
name: "split-brain-registry".into(),
|
|
num_nodes: 6,
|
|
num_rounds: 100,
|
|
ticks_per_round: 3,
|
|
network_faults: vec![
|
|
NetworkFault::Partition {
|
|
round: 10,
|
|
partition: Partition {
|
|
side_a: vec![0, 1, 2],
|
|
side_b: vec![3, 4, 5],
|
|
asymmetric: false,
|
|
},
|
|
},
|
|
NetworkFault::Heal { round: 40 },
|
|
],
|
|
action_schedule: vec![
|
|
(12, SimAction::RegisterName { node_idx: 0, name: "leader".into() }),
|
|
(12, SimAction::RegisterName { node_idx: 3, name: "leader".into() }),
|
|
],
|
|
swim: distribution::swim::probe::SwimConfig {
|
|
probe_interval: 1,
|
|
probe_timeout: 3,
|
|
indirect_probes: 1,
|
|
// High enough that no node reaches Dead during the 30-round partition.
|
|
// Nodes go Suspect → back to Alive when partition heals, triggering
|
|
// re_disseminate_all which propagates both sides' registry entries.
|
|
suspicion_timeout: 200,
|
|
dead_reprobe_interval: 10,
|
|
},
|
|
..default_config()
|
|
};
|
|
|
|
// When: we run the simulation
|
|
let (trace, nodes) = run_simulation_with_nodes(config);
|
|
maybe_save_trace(&trace);
|
|
|
|
// Then: all alive nodes should resolve "leader" to the same value (LWW winner)
|
|
let resolutions: Vec<_> = nodes
|
|
.iter()
|
|
.filter_map(|n| n.as_ref())
|
|
.map(|n| n.resolve_name("leader"))
|
|
.collect();
|
|
|
|
// All should resolve to Some (the LWW winner)
|
|
let resolved: Vec<_> = resolutions.iter().filter_map(|r| r.as_ref()).collect();
|
|
assert!(
|
|
resolved.len() >= 4,
|
|
"at least 4 of 6 nodes should resolve 'leader', got {} out of {}",
|
|
resolved.len(),
|
|
resolutions.len()
|
|
);
|
|
|
|
// All resolving nodes should agree on the same actor
|
|
if resolved.len() >= 2 {
|
|
let first_actor = resolved[0].0;
|
|
let agree = resolved.iter().all(|r| r.0 == first_actor);
|
|
assert!(
|
|
agree,
|
|
"all nodes resolving 'leader' should agree on the same actor (LWW winner)"
|
|
);
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
// 3. Tombstone propagation on node death
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
|
|
#[test]
|
|
fn tombstone_propagates_when_name_owner_dies() {
|
|
// Given: 5-node cluster, node 0 registers "svc" at round 5, killed at round 15
|
|
let config = DistributionSimConfig {
|
|
name: "tombstone-propagation".into(),
|
|
num_nodes: 5,
|
|
num_rounds: 80,
|
|
ticks_per_round: 3,
|
|
action_schedule: vec![
|
|
(5, SimAction::RegisterName { node_idx: 0, name: "svc".into() }),
|
|
],
|
|
kill_schedule: vec![(15, 0)],
|
|
..default_config()
|
|
};
|
|
|
|
// When: we run the simulation
|
|
let (trace, nodes) = run_simulation_with_nodes(config);
|
|
maybe_save_trace(&trace);
|
|
|
|
// Then: surviving nodes should have tombstoned "svc"
|
|
let result = check_registry_tombstones(&trace, 1);
|
|
assert!(
|
|
result.passed,
|
|
"survivors should have tombstones after node death: {}",
|
|
result.actual
|
|
);
|
|
|
|
// And: resolve_name("svc") should return None on all survivors
|
|
let resolutions: Vec<_> = nodes
|
|
.iter()
|
|
.filter_map(|n| n.as_ref())
|
|
.map(|n| n.resolve_name("svc"))
|
|
.collect();
|
|
assert!(
|
|
resolutions.iter().all(|r| r.is_none()),
|
|
"all survivors should resolve 'svc' to None after owner dies, got: {resolutions:?}"
|
|
);
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
// 4. Rapid re-registration converges to latest value
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
|
|
#[test]
|
|
fn rapid_re_registration_converges_to_latest() {
|
|
// Given: 5-node cluster, node 0 registers "svc" three times rapidly
|
|
use swactor::actor::ActorAddress;
|
|
let actor_a = ActorAddress::new_random();
|
|
let actor_b = ActorAddress::new_random();
|
|
let actor_c = ActorAddress::new_random();
|
|
|
|
let config = DistributionSimConfig {
|
|
name: "rapid-reregister".into(),
|
|
num_nodes: 5,
|
|
num_rounds: 60,
|
|
ticks_per_round: 3,
|
|
action_schedule: vec![
|
|
(5, SimAction::RegisterNameWithActor { node_idx: 0, name: "svc".into(), actor: actor_a }),
|
|
(6, SimAction::RegisterNameWithActor { node_idx: 0, name: "svc".into(), actor: actor_b }),
|
|
(7, SimAction::RegisterNameWithActor { node_idx: 0, name: "svc".into(), actor: actor_c }),
|
|
],
|
|
..default_config()
|
|
};
|
|
|
|
// When: we run the simulation
|
|
let (trace, nodes) = run_simulation_with_nodes(config);
|
|
maybe_save_trace(&trace);
|
|
|
|
// Then: all nodes should resolve "svc" to actor_c (the latest registration)
|
|
let resolutions: Vec<_> = nodes
|
|
.iter()
|
|
.filter_map(|n| n.as_ref())
|
|
.map(|n| n.resolve_name("svc"))
|
|
.collect();
|
|
|
|
assert!(
|
|
resolutions.iter().all(|r| r.is_some()),
|
|
"all nodes should resolve 'svc'"
|
|
);
|
|
|
|
assert!(
|
|
resolutions.iter().all(|r| r.unwrap().0 == actor_c),
|
|
"all nodes should converge to the latest registration (actor_c)"
|
|
);
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
// 5. Simultaneous registration of same name on different nodes
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
|
|
#[test]
|
|
fn simultaneous_registration_converges_deterministically() {
|
|
// Given: 5-node cluster, node 0 and node 3 both register "mutex" at round 5
|
|
use swactor::actor::ActorAddress;
|
|
let actor_a = ActorAddress::new_random();
|
|
let actor_b = ActorAddress::new_random();
|
|
|
|
let config = DistributionSimConfig {
|
|
name: "simultaneous-register".into(),
|
|
num_nodes: 5,
|
|
num_rounds: 60,
|
|
ticks_per_round: 3,
|
|
action_schedule: vec![
|
|
(5, SimAction::RegisterNameWithActor { node_idx: 0, name: "mutex".into(), actor: actor_a }),
|
|
(5, SimAction::RegisterNameWithActor { node_idx: 3, name: "mutex".into(), actor: actor_b }),
|
|
],
|
|
..default_config()
|
|
};
|
|
|
|
// When: we run the simulation
|
|
let (trace, nodes) = run_simulation_with_nodes(config);
|
|
maybe_save_trace(&trace);
|
|
|
|
// Then: all nodes should agree on one winner
|
|
let resolutions: Vec<_> = nodes
|
|
.iter()
|
|
.filter_map(|n| n.as_ref())
|
|
.map(|n| n.resolve_name("mutex"))
|
|
.collect();
|
|
|
|
assert!(
|
|
resolutions.iter().all(|r| r.is_some()),
|
|
"all nodes should resolve 'mutex'"
|
|
);
|
|
|
|
// All should agree on the same actor (whichever won the LWW tiebreaker)
|
|
let first_actor = resolutions[0].unwrap().0;
|
|
assert!(
|
|
resolutions.iter().all(|r| r.unwrap().0 == first_actor),
|
|
"all nodes should agree on the LWW winner for 'mutex'"
|
|
);
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
// 6. Multiple names propagate correctly
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
|
|
#[test]
|
|
fn multiple_names_from_different_nodes_all_propagate() {
|
|
// Given: 5-node cluster, each node registers a unique name
|
|
let config = DistributionSimConfig {
|
|
name: "multi-name-propagation".into(),
|
|
num_nodes: 5,
|
|
num_rounds: 60,
|
|
ticks_per_round: 3,
|
|
action_schedule: vec![
|
|
(5, SimAction::RegisterName { node_idx: 0, name: "svc-0".into() }),
|
|
(5, SimAction::RegisterName { node_idx: 1, name: "svc-1".into() }),
|
|
(5, SimAction::RegisterName { node_idx: 2, name: "svc-2".into() }),
|
|
(5, SimAction::RegisterName { node_idx: 3, name: "svc-3".into() }),
|
|
(5, SimAction::RegisterName { node_idx: 4, name: "svc-4".into() }),
|
|
],
|
|
..default_config()
|
|
};
|
|
|
|
// When: we run the simulation
|
|
let (trace, nodes) = run_simulation_with_nodes(config);
|
|
maybe_save_trace(&trace);
|
|
|
|
// Then: all 5 nodes should have all 5 registry entries
|
|
let result = check_registry_propagation(&trace, 5);
|
|
assert!(
|
|
result.passed,
|
|
"all nodes should have all 5 registry entries: {}",
|
|
result.actual
|
|
);
|
|
|
|
// And: each node should resolve all 5 names
|
|
for node in nodes.iter().filter_map(|n| n.as_ref()) {
|
|
for i in 0..5 {
|
|
let name = format!("svc-{i}");
|
|
assert!(
|
|
node.resolve_name(&name).is_some(),
|
|
"every node should resolve '{name}'"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
// 7. Explicit unregister propagates to all nodes
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
|
|
#[test]
|
|
fn explicit_unregister_propagates_to_all_nodes() {
|
|
// Given: 5-node cluster, node 0 registers "svc" at round 5, unregisters at round 15
|
|
let config = DistributionSimConfig {
|
|
name: "explicit-unregister".into(),
|
|
num_nodes: 5,
|
|
num_rounds: 60,
|
|
ticks_per_round: 3,
|
|
action_schedule: vec![
|
|
(5, SimAction::RegisterName { node_idx: 0, name: "svc".into() }),
|
|
(15, SimAction::UnregisterName { node_idx: 0, name: "svc".into() }),
|
|
],
|
|
..default_config()
|
|
};
|
|
|
|
let (trace, nodes) = run_simulation_with_nodes(config);
|
|
maybe_save_trace(&trace);
|
|
|
|
// Then: all nodes should resolve "svc" to None (tombstoned)
|
|
for (i, node) in nodes.iter().filter_map(|n| n.as_ref()).enumerate() {
|
|
assert!(
|
|
node.resolve_name("svc").is_none(),
|
|
"node {i} should resolve 'svc' to None after unregister"
|
|
);
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
// 8. Re-registration after tombstone overwrites the tombstone
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
|
|
#[test]
|
|
fn re_registration_after_tombstone_succeeds() {
|
|
// Given: node 0 registers "svc", then it's killed (tombstoned),
|
|
// then node 1 re-registers "svc" with a new actor
|
|
use swactor::actor::ActorAddress;
|
|
let new_actor = ActorAddress::new_random();
|
|
|
|
let config = DistributionSimConfig {
|
|
name: "re-register-after-tombstone".into(),
|
|
num_nodes: 5,
|
|
num_rounds: 100,
|
|
ticks_per_round: 3,
|
|
action_schedule: vec![
|
|
(5, SimAction::RegisterName { node_idx: 0, name: "svc".into() }),
|
|
// Node 1 re-registers "svc" well after node 0 dies and tombstone propagates
|
|
(50, SimAction::RegisterNameWithActor { node_idx: 1, name: "svc".into(), actor: new_actor }),
|
|
],
|
|
kill_schedule: vec![(15, 0)],
|
|
..default_config()
|
|
};
|
|
|
|
let (trace, nodes) = run_simulation_with_nodes(config);
|
|
maybe_save_trace(&trace);
|
|
|
|
// Then: all survivors should resolve "svc" to the new actor from node 1
|
|
let resolutions: Vec<_> = nodes
|
|
.iter()
|
|
.filter_map(|n| n.as_ref())
|
|
.map(|n| n.resolve_name("svc"))
|
|
.collect();
|
|
|
|
assert!(
|
|
resolutions.iter().all(|r| r.is_some()),
|
|
"all survivors should resolve 'svc' to the new registration, got: {resolutions:?}"
|
|
);
|
|
|
|
assert!(
|
|
resolutions.iter().all(|r| r.unwrap().0 == new_actor),
|
|
"all survivors should resolve 'svc' to the new actor"
|
|
);
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
// 9. Multiple names from same node, kill node, all tombstoned
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
|
|
#[test]
|
|
fn all_names_tombstoned_when_owner_dies() {
|
|
// Given: node 0 registers 3 names, then is killed
|
|
let config = DistributionSimConfig {
|
|
name: "multi-name-tombstone".into(),
|
|
num_nodes: 5,
|
|
num_rounds: 80,
|
|
ticks_per_round: 3,
|
|
action_schedule: vec![
|
|
(5, SimAction::RegisterName { node_idx: 0, name: "alpha".into() }),
|
|
(5, SimAction::RegisterName { node_idx: 0, name: "beta".into() }),
|
|
(5, SimAction::RegisterName { node_idx: 0, name: "gamma".into() }),
|
|
],
|
|
kill_schedule: vec![(15, 0)],
|
|
..default_config()
|
|
};
|
|
|
|
let (trace, nodes) = run_simulation_with_nodes(config);
|
|
maybe_save_trace(&trace);
|
|
|
|
// Then: all survivors should resolve all 3 names to None
|
|
for node in nodes.iter().filter_map(|n| n.as_ref()) {
|
|
for name in &["alpha", "beta", "gamma"] {
|
|
assert!(
|
|
node.resolve_name(name).is_none(),
|
|
"all names should be tombstoned after owner dies, but '{}' still resolves",
|
|
name
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
// 10. Graceful leave tombstones the leaving node's names
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
|
|
#[test]
|
|
fn graceful_leave_tombstones_registry_names() {
|
|
// Given: node 0 registers "svc", then does a graceful leave
|
|
let config = DistributionSimConfig {
|
|
name: "graceful-leave-registry".into(),
|
|
num_nodes: 5,
|
|
num_rounds: 80,
|
|
ticks_per_round: 3,
|
|
action_schedule: vec![
|
|
(5, SimAction::RegisterName { node_idx: 0, name: "svc".into() }),
|
|
(20, SimAction::GracefulLeave { node_idx: 0 }),
|
|
],
|
|
..default_config()
|
|
};
|
|
|
|
let (trace, nodes) = run_simulation_with_nodes(config);
|
|
maybe_save_trace(&trace);
|
|
|
|
// Then: all survivors should resolve "svc" to None (tombstoned via death notification)
|
|
let resolutions: Vec<_> = nodes
|
|
.iter()
|
|
.filter_map(|n| n.as_ref())
|
|
.map(|n| n.resolve_name("svc"))
|
|
.collect();
|
|
|
|
assert!(
|
|
resolutions.iter().all(|r| r.is_none()),
|
|
"all survivors should resolve 'svc' to None after graceful leave, got: {resolutions:?}"
|
|
);
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
// 11. Piggyback contention — kills + registrations compete for bandwidth
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
|
|
#[test]
|
|
fn piggyback_contention_both_propagate() {
|
|
// Given: 10-node cluster, kill 2 nodes + register 3 names simultaneously
|
|
// Both membership death updates and registry entries share piggyback bandwidth
|
|
let config = DistributionSimConfig {
|
|
name: "piggyback-contention".into(),
|
|
num_nodes: 10,
|
|
num_rounds: 100,
|
|
ticks_per_round: 3,
|
|
actors_per_node: 0,
|
|
action_schedule: vec![
|
|
(10, SimAction::RegisterName { node_idx: 0, name: "alpha".into() }),
|
|
(10, SimAction::RegisterName { node_idx: 3, name: "beta".into() }),
|
|
(10, SimAction::RegisterName { node_idx: 6, name: "gamma".into() }),
|
|
],
|
|
kill_schedule: vec![(10, 2), (10, 5)],
|
|
..default_config()
|
|
};
|
|
|
|
let (trace, nodes) = run_simulation_with_nodes(config);
|
|
maybe_save_trace(&trace);
|
|
|
|
// Then: all survivors should have all 3 registry names
|
|
let result = check_registry_propagation(&trace, 3);
|
|
assert!(
|
|
result.passed,
|
|
"all 3 names should propagate despite contention with death updates: {}",
|
|
result.actual
|
|
);
|
|
|
|
// And: all survivors should resolve all 3 names
|
|
for node in nodes.iter().filter_map(|n| n.as_ref()) {
|
|
for name in &["alpha", "beta", "gamma"] {
|
|
assert!(
|
|
node.resolve_name(name).is_some(),
|
|
"survivor should resolve '{}' despite piggyback contention",
|
|
name
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
// 12. Registry convergence under message loss
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
|
|
#[test]
|
|
fn registry_converges_despite_message_loss() {
|
|
// Given: 5-node cluster with a loss window during name registration,
|
|
// then clean connectivity for convergence.
|
|
let config = DistributionSimConfig {
|
|
name: "registry-under-loss".into(),
|
|
num_nodes: 5,
|
|
num_rounds: 100,
|
|
ticks_per_round: 3,
|
|
registry_dissemination_lambda: Some(5),
|
|
network_faults: vec![
|
|
// Loss window: 30% drops during registration phase
|
|
NetworkFault::SetDropRate { round: 3, rate: 0.30 },
|
|
// Restore clean connectivity, forcing convergence via Alive transitions
|
|
NetworkFault::SetDropRate { round: 30, rate: 0.0 },
|
|
],
|
|
action_schedule: vec![
|
|
(5, SimAction::RegisterName { node_idx: 0, name: "alpha".into() }),
|
|
(5, SimAction::RegisterName { node_idx: 2, name: "beta".into() }),
|
|
(5, SimAction::RegisterName { node_idx: 4, name: "gamma".into() }),
|
|
],
|
|
swim: distribution::swim::probe::SwimConfig {
|
|
probe_interval: 1,
|
|
probe_timeout: 3,
|
|
indirect_probes: 3,
|
|
// High timeout prevents false deaths during the loss window.
|
|
suspicion_timeout: 200,
|
|
dead_reprobe_interval: 15,
|
|
},
|
|
..default_config()
|
|
};
|
|
|
|
let (trace, nodes) = run_simulation_with_nodes(config);
|
|
maybe_save_trace(&trace);
|
|
|
|
// Then: all surviving nodes should resolve all 3 names despite packet loss
|
|
let alive_nodes: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect();
|
|
for node in &alive_nodes {
|
|
for name in &["alpha", "beta", "gamma"] {
|
|
assert!(
|
|
node.resolve_name(name).is_some(),
|
|
"all nodes should resolve '{}' despite 20% message loss",
|
|
name
|
|
);
|
|
}
|
|
}
|
|
|
|
// All should agree on the same actor for each name
|
|
for name in &["alpha", "beta", "gamma"] {
|
|
let first = alive_nodes[0].resolve_name(name).unwrap().0;
|
|
assert!(
|
|
alive_nodes.iter().all(|n| n.resolve_name(name).unwrap().0 == first),
|
|
"all nodes should agree on actor for '{}' despite message loss",
|
|
name
|
|
);
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
// 13. Multiple name-owning nodes killed simultaneously
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
|
|
#[test]
|
|
fn simultaneous_kill_of_multiple_name_owners() {
|
|
// Given: 7-node cluster, nodes 0-2 each own a name, all 3 killed at round 15
|
|
let config = DistributionSimConfig {
|
|
name: "multi-owner-kill".into(),
|
|
num_nodes: 7,
|
|
num_rounds: 80,
|
|
ticks_per_round: 3,
|
|
action_schedule: vec![
|
|
(5, SimAction::RegisterName { node_idx: 0, name: "svc-a".into() }),
|
|
(5, SimAction::RegisterName { node_idx: 1, name: "svc-b".into() }),
|
|
(5, SimAction::RegisterName { node_idx: 2, name: "svc-c".into() }),
|
|
],
|
|
kill_schedule: vec![(15, 0), (15, 1), (15, 2)],
|
|
..default_config()
|
|
};
|
|
|
|
let (trace, nodes) = run_simulation_with_nodes(config);
|
|
maybe_save_trace(&trace);
|
|
|
|
// Then: all 4 survivors should resolve all 3 names to None (tombstoned)
|
|
let survivors: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect();
|
|
assert_eq!(survivors.len(), 4, "4 of 7 nodes should survive");
|
|
|
|
for node in &survivors {
|
|
for name in &["svc-a", "svc-b", "svc-c"] {
|
|
assert!(
|
|
node.resolve_name(name).is_none(),
|
|
"survivor should resolve '{}' to None after owner died",
|
|
name
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
// 14. Name owner suspected but recovers — registry entry survives
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
|
|
#[test]
|
|
fn suspected_name_owner_recovers_and_registry_survives() {
|
|
// Given: 5-node cluster, node 0 registers "svc",
|
|
// then a brief partition isolates node 0 (becomes Suspect, recovers before Dead)
|
|
let config = DistributionSimConfig {
|
|
name: "suspect-recovery-registry".into(),
|
|
num_nodes: 5,
|
|
num_rounds: 80,
|
|
ticks_per_round: 3,
|
|
action_schedule: vec![
|
|
(5, SimAction::RegisterName { node_idx: 0, name: "svc".into() }),
|
|
],
|
|
// Brief partition: isolate node 0 for 10 rounds (not long enough to reach Dead)
|
|
network_faults: vec![
|
|
NetworkFault::Partition {
|
|
round: 15,
|
|
partition: Partition {
|
|
side_a: vec![0],
|
|
side_b: vec![1, 2, 3, 4],
|
|
asymmetric: false,
|
|
},
|
|
},
|
|
NetworkFault::Heal { round: 25 },
|
|
],
|
|
swim: distribution::swim::probe::SwimConfig {
|
|
probe_interval: 1,
|
|
probe_timeout: 3,
|
|
indirect_probes: 1,
|
|
// Node stays Suspect during the 10-round partition (30 ticks < 200)
|
|
suspicion_timeout: 200,
|
|
dead_reprobe_interval: 10,
|
|
},
|
|
..default_config()
|
|
};
|
|
|
|
let (trace, nodes) = run_simulation_with_nodes(config);
|
|
maybe_save_trace(&trace);
|
|
|
|
// Then: all nodes should still resolve "svc" (node 0 never died, only suspected)
|
|
let resolutions: Vec<_> = nodes
|
|
.iter()
|
|
.filter_map(|n| n.as_ref())
|
|
.map(|n| n.resolve_name("svc"))
|
|
.collect();
|
|
|
|
assert!(
|
|
resolutions.iter().all(|r| r.is_some()),
|
|
"all nodes should resolve 'svc' after owner recovers from Suspect, got: {resolutions:?}"
|
|
);
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
// 15. Rapid churn: interleaved kills and registrations
|
|
// ────────────────────────────────────────────────────────────────────────────
|
|
|
|
#[test]
|
|
fn registry_correct_under_rapid_churn() {
|
|
// Given: 8-node cluster with interleaved kills and registrations
|
|
let config = DistributionSimConfig {
|
|
name: "rapid-churn-registry".into(),
|
|
num_nodes: 8,
|
|
num_rounds: 100,
|
|
ticks_per_round: 3,
|
|
action_schedule: vec![
|
|
(5, SimAction::RegisterName { node_idx: 0, name: "svc-0".into() }),
|
|
(5, SimAction::RegisterName { node_idx: 1, name: "svc-1".into() }),
|
|
(8, SimAction::RegisterName { node_idx: 4, name: "svc-4".into() }),
|
|
// Node 3 registers AFTER nodes 0 and 1 are killed
|
|
(20, SimAction::RegisterName { node_idx: 3, name: "svc-3".into() }),
|
|
// Node 5 takes over "svc-0" after original owner dies
|
|
(30, SimAction::RegisterName { node_idx: 5, name: "svc-0".into() }),
|
|
],
|
|
kill_schedule: vec![(10, 0), (10, 1), (25, 2)],
|
|
..default_config()
|
|
};
|
|
|
|
let (trace, nodes) = run_simulation_with_nodes(config);
|
|
maybe_save_trace(&trace);
|
|
|
|
let survivors: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect();
|
|
assert_eq!(survivors.len(), 5, "5 of 8 nodes should survive");
|
|
|
|
// svc-0 should resolve to node 5's re-registration (not tombstoned)
|
|
for node in &survivors {
|
|
assert!(
|
|
node.resolve_name("svc-0").is_some(),
|
|
"svc-0 should resolve (re-registered by node 5 after owner death)"
|
|
);
|
|
}
|
|
|
|
// svc-1 should be tombstoned (node 1 died, no re-registration)
|
|
for node in &survivors {
|
|
assert!(
|
|
node.resolve_name("svc-1").is_none(),
|
|
"svc-1 should be tombstoned (owner died, not re-registered)"
|
|
);
|
|
}
|
|
|
|
// svc-3 should resolve (registered after kills, node 3 alive)
|
|
for node in &survivors {
|
|
assert!(
|
|
node.resolve_name("svc-3").is_some(),
|
|
"svc-3 should resolve (registered after kills)"
|
|
);
|
|
}
|
|
|
|
// svc-4 should resolve (node 4 alive throughout)
|
|
for node in &survivors {
|
|
assert!(
|
|
node.resolve_name("svc-4").is_some(),
|
|
"svc-4 should resolve (owner alive throughout)"
|
|
);
|
|
}
|
|
}
|