diff --git a/crates/distribution/src/swim/probe.rs b/crates/distribution/src/swim/probe.rs index 31750da..a85deea 100644 --- a/crates/distribution/src/swim/probe.rs +++ b/crates/distribution/src/swim/probe.rs @@ -5,7 +5,7 @@ use std::collections::VecDeque; -use crate::types::NodeId; +use crate::types::{MemberState, NodeId}; use super::member_list::MemberList; @@ -342,8 +342,26 @@ impl SwimProbe { .collect(); for node_id in expired { - if members.declare_dead(node_id) { + // Only declare dead if still suspect. A refutation (Alive with + // higher incarnation) clears the suspect state in MemberList; + // honour that by dropping the stale timer instead of killing the node. + let still_suspect = members + .get(&node_id) + .is_some_and(|e| e.state == MemberState::Suspect); + + if still_suspect && members.declare_dead(node_id) { actions.push(SwimAction::DeclareDead(node_id)); + // If we're currently probing the dead node, cancel immediately + // so we can probe live members on this same tick. + match &self.phase { + ProbePhase::WaitingDirectAck { target, .. } + | ProbePhase::WaitingIndirectAck { target, .. } + if *target == node_id => + { + self.phase = ProbePhase::Idle; + } + _ => {} + } } self.cancel_suspicion_timer(node_id); } diff --git a/crates/distribution/tests/registry.rs b/crates/distribution/tests/registry.rs index 19d6760..1b92681 100644 --- a/crates/distribution/tests/registry.rs +++ b/crates/distribution/tests/registry.rs @@ -309,24 +309,37 @@ fn node_death_tombstones_entries() { b.register_name("b-service".into(), actor); // Propagate B's registration to A and C via mesh gossip. + // Deliver to each target separately so responses carry the correct sender_id. for _ in 0..5 { let actions = b.tick(); - let mut nodes = vec![(a_id, &mut a), (c_id, &mut c)]; - let responses = deliver_actions(&actions, b_id, &mut nodes); - let mut nodes = vec![(b_id, &mut b)]; - let _ = deliver_actions(&responses, a_id, &mut nodes); + let mut t = vec![(a_id, &mut a)]; + let resp_a = deliver_actions(&actions, b_id, &mut t); + let mut t = vec![(c_id, &mut c)]; + let resp_c = deliver_actions(&actions, b_id, &mut t); + let mut t = vec![(b_id, &mut b)]; + let _ = deliver_actions(&resp_a, a_id, &mut t); + let mut t = vec![(b_id, &mut b)]; + let _ = deliver_actions(&resp_c, c_id, &mut t); let actions = a.tick(); - let mut nodes = vec![(b_id, &mut b), (c_id, &mut c)]; - let responses = deliver_actions(&actions, a_id, &mut nodes); - let mut nodes = vec![(a_id, &mut a)]; - let _ = deliver_actions(&responses, b_id, &mut nodes); + let mut t = vec![(b_id, &mut b)]; + let resp_b = deliver_actions(&actions, a_id, &mut t); + let mut t = vec![(c_id, &mut c)]; + let resp_c = deliver_actions(&actions, a_id, &mut t); + let mut t = vec![(a_id, &mut a)]; + let _ = deliver_actions(&resp_b, b_id, &mut t); + let mut t = vec![(a_id, &mut a)]; + let _ = deliver_actions(&resp_c, c_id, &mut t); let actions = c.tick(); - let mut nodes = vec![(a_id, &mut a), (b_id, &mut b)]; - let responses = deliver_actions(&actions, c_id, &mut nodes); - let mut nodes = vec![(c_id, &mut c)]; - let _ = deliver_actions(&responses, a_id, &mut nodes); + let mut t = vec![(a_id, &mut a)]; + let resp_a = deliver_actions(&actions, c_id, &mut t); + let mut t = vec![(b_id, &mut b)]; + let resp_b = deliver_actions(&actions, c_id, &mut t); + let mut t = vec![(c_id, &mut c)]; + let _ = deliver_actions(&resp_a, a_id, &mut t); + let mut t = vec![(c_id, &mut c)]; + let _ = deliver_actions(&resp_b, b_id, &mut t); } assert_eq!(a.resolve_name("b-service"), Some((actor, b_id)));