//! SWIM tuning harness. //! //! Runs the gossip-flap reproduction, the three N3 calibration scenarios, //! and a synthesised §10.3 library property under a configurable SWIM //! `kind_config`. Prints one line of NDJSON per scenario per config: //! //! ```text //! {"scenario": "gossip_flap", "config": {...}, "verdicts": [...], //! "metrics": {"self_incarnation_peak": 12, "relay_queue_peak_bytes": 0, ...}} //! ``` //! //! Invocation //! //! ```text //! cargo run --release --example swim_tune -- \ //! --probe_interval_ns 1000000000 \ //! --probe_timeout_ns 350000000 \ //! --suspicion_timeout_ns 8000000000 \ //! --indirect_ping_fanout 3 \ //! --dead_reprobe_interval_ns 5000000000 //! ``` //! //! Each flag is optional; omitted flags use the production default //! (which the binary derives from `SwimConfig::default()` translated //! through the scenario's tick period). The CLI is positional/loose //! on purpose — this is an internal sweep tool, not a stable interface. //! //! `--mode baseline` strips SWIM kind_config overrides from the scenario //! so the live `SwimConfig::default()` values take effect. `--mode tuned` //! (the default) injects the supplied knobs into every SWIM peer's //! kind_config. use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use distribution::swim::probe::SwimConfig; use simulation::bundle::VecWriter; use simulation::engine::Engine; use simulation::evaluator::{EventLine, Outcome, SnapshotEntry, SnapshotIndex, evaluate}; use simulation::network::Network; use simulation::scenario::{ Assertion, AssertionKind, DefaultTick, HostKindRegistry, Link, LinkPolicy, Peer, Scenario, load_from_path, }; use simulation::swim_host::SwimHostFactory; #[derive(Debug, Clone, Copy)] struct Knobs { probe_interval_ns: Option, probe_timeout_ns: Option, suspicion_timeout_ns: Option, indirect_ping_fanout: Option, dead_reprobe_interval_ns: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Mode { /// Strip kind_config overrides so SwimConfig::default() takes /// effect (used to capture the *current* production defaults). Baseline, /// Inject the supplied knobs into every SWIM peer's kind_config. Tuned, } fn parse_args() -> (Mode, Knobs, Option) { let mut knobs = Knobs { probe_interval_ns: None, probe_timeout_ns: None, suspicion_timeout_ns: None, indirect_ping_fanout: None, dead_reprobe_interval_ns: None, }; let mut mode = Mode::Tuned; let mut scenario: Option = None; let args: Vec = std::env::args().skip(1).collect(); let mut i = 0usize; while i < args.len() { let a = &args[i]; i += 1; let mut take = || { let v = args.get(i).cloned().expect("value"); i += 1; v }; match a.as_str() { "--mode" => { mode = match take().as_str() { "baseline" => Mode::Baseline, "tuned" => Mode::Tuned, other => panic!("--mode must be baseline|tuned, got {other}"), }; } "--scenario" => scenario = Some(take()), "--probe_interval_ns" => knobs.probe_interval_ns = Some(take().parse().unwrap()), "--probe_timeout_ns" => knobs.probe_timeout_ns = Some(take().parse().unwrap()), "--suspicion_timeout_ns" => knobs.suspicion_timeout_ns = Some(take().parse().unwrap()), "--indirect_ping_fanout" => knobs.indirect_ping_fanout = Some(take().parse().unwrap()), "--dead_reprobe_interval_ns" => { knobs.dead_reprobe_interval_ns = Some(take().parse().unwrap()) } other => panic!("unknown arg {other}"), } } (mode, knobs, scenario) } fn registry() -> HostKindRegistry { HostKindRegistry::with_swim() } fn cargo_root() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) } fn load(rel: &str) -> Scenario { let path = cargo_root().join(rel); load_from_path(&path, ®istry()).expect("scenario validates") } /// Mutate every `swim` peer's `kind_config` according to mode + knobs. /// /// - Baseline strips probe_interval_ns / probe_timeout_ns / /// suspicion_timeout_ns / indirect_ping_fanout / /// dead_reprobe_interval_ns so `SwimHost::config_from_kind` falls /// through to SwimConfig::default()-derived values. /// - Tuned writes the supplied knobs and removes the rest (so the /// adapter's tick-period fallback gives default-equivalent values). fn apply_knobs(scenario: &mut Scenario, mode: Mode, knobs: Knobs) { if mode == Mode::Baseline { // Baseline runs the scenario exactly as it sits on disk. The // §8 validator requires probe_interval_ns and // suspicion_timeout_ns, so we cannot blanket-strip; the // scenarios' own kind_config values are the "before" picture. return; } for peer in &mut scenario.peers { if peer.kind != "swim" { continue; } if let Some(v) = knobs.probe_interval_ns { peer.kind_config .insert("probe_interval_ns".into(), toml::Value::Integer(v as i64)); } if let Some(v) = knobs.probe_timeout_ns { peer.kind_config .insert("probe_timeout_ns".into(), toml::Value::Integer(v as i64)); } if let Some(v) = knobs.suspicion_timeout_ns { peer.kind_config.insert( "suspicion_timeout_ns".into(), toml::Value::Integer(v as i64), ); } if let Some(v) = knobs.indirect_ping_fanout { peer.kind_config.insert( "indirect_ping_fanout".into(), toml::Value::Integer(v as i64), ); } if let Some(v) = knobs.dead_reprobe_interval_ns { peer.kind_config.insert( "dead_reprobe_interval_ns".into(), toml::Value::Integer(v as i64), ); } } } #[derive(serde::Serialize)] struct ScenarioReport { scenario: String, verdicts: Vec, metrics: Metrics, } #[derive(serde::Serialize)] struct VerdictBrief { name: String, kind: &'static str, outcome: String, } #[derive(serde::Serialize, Default)] struct Metrics { /// Peak self_incarnation across any snapshot. self_incarnation_peak: u64, /// Peak relay enqueued_bytes seen via replay of relay events. relay_queue_peak_bytes: u64, /// Largest piggybacked message_send `bytes` value over the run. message_size_peak: u64, /// Earliest convergence time across observers (ns from t=0). None /// if no snapshot witnessed agreement. convergence_observed_ns: Option, /// Number of state_transition events into Suspect across the run. suspect_events: u64, /// Number of state_transition events into Dead across the run. dead_events: u64, /// Number of state_transition events into Alive across the run. alive_events: u64, } fn run_scenario_report(name: &str, mut scen: Scenario, mode: Mode, knobs: Knobs) -> ScenarioReport { apply_knobs(&mut scen, mode, knobs); let writer = VecWriter::default(); let network = Network::new(&scen); let mut engine = Engine::new(&scen, network, writer); engine.register_factory(Box::new(SwimHostFactory)); engine.register_factory(Box::new(simulation::stage_host::StageHostFactory)); engine.auto_install_hosts(); engine.set_pop_budget(2_000_000); let _ = engine.run(); let records = engine.into_writer().records; let (events, snapshots) = records_to_eval_inputs(&records); let verdicts = evaluate(&scen, &events, &snapshots); let metrics = collect_metrics(&events, &snapshots, &scen); ScenarioReport { scenario: name.to_string(), verdicts: verdicts .iter() .map(|v| VerdictBrief { name: v.name.clone(), kind: v.kind, outcome: outcome_word(&v.outcome).to_string(), }) .collect(), metrics, } } fn outcome_word(o: &Outcome) -> &'static str { match o { Outcome::Pass => "PASS", Outcome::Fail => "FAIL", Outcome::Inconclusive => "INCONCLUSIVE", } } fn records_to_eval_inputs( records: &[simulation::bundle::BundleRecord], ) -> (Vec, SnapshotIndex) { use simulation::bundle::BundleRecord; let mut events = Vec::new(); let mut idx = SnapshotIndex::default(); let mut line_idx = 0usize; let mut seq_by_host: BTreeMap = BTreeMap::new(); for rec in records { match rec { BundleRecord::Event(e) => { events.push(EventLine::from_event_record(e, line_idx)); line_idx += 1; } BundleRecord::Mutation(m) => { events.push(EventLine::from_mutation_record(m, line_idx)); line_idx += 1; } BundleRecord::Snapshot(s) => { let seq = seq_by_host.entry(s.host_id.clone()).or_insert(0); let entry = SnapshotEntry::from_snapshot_record(s, *seq); *seq += 1; idx.by_host.entry(s.host_id.clone()).or_default().push(entry); } } } (events, idx) } fn collect_metrics(events: &[EventLine], snaps: &SnapshotIndex, scen: &Scenario) -> Metrics { let mut m = Metrics::default(); // Snapshot-derived: self_incarnation peak. for list in snaps.by_host.values() { for s in list { if s.self_incarnation > m.self_incarnation_peak { m.self_incarnation_peak = s.self_incarnation; } } } // Relay queue peak: replay enqueue/dequeue in time order. let mut relay_events: Vec<&EventLine> = events .iter() .filter(|e| { e.kind_tag == "relay" && (e.event["kind"] == "relay_enqueue" || e.event["kind"] == "relay_dequeue") }) .collect(); relay_events.sort_by(|a, b| { a.virtual_time_ns .cmp(&b.virtual_time_ns) .then(a.line_idx.cmp(&b.line_idx)) }); let mut relay_depths: BTreeMap = BTreeMap::new(); for e in &relay_events { let relay = e.event["relay"].as_str().unwrap_or("").to_string(); let bl = e.event["byte_len"].as_u64().unwrap_or(0); let entry = relay_depths.entry(relay).or_insert(0); match e.event["kind"].as_str() { Some("relay_enqueue") => { *entry = entry.saturating_add(bl); if *entry > m.relay_queue_peak_bytes { m.relay_queue_peak_bytes = *entry; } } Some("relay_dequeue") => { *entry = entry.saturating_sub(bl); } _ => {} } } for e in events { if e.event["kind"] == "message_send" { let bytes = e.event["bytes"].as_u64().unwrap_or(0); if bytes > m.message_size_peak { m.message_size_peak = bytes; } } if e.event["kind"] == "state_transition" { match e.event["to"].as_str() { Some("Suspect") => m.suspect_events += 1, Some("Dead") => m.dead_events += 1, Some("Alive") => m.alive_events += 1, _ => {} } } } // Convergence: earliest snapshot time at which every observer's // membership view of every other peer agrees. We approximate by // checking each observer's full snapshot list and looking for the // smallest virtual_time_ns where all observers agree on every // subject's `state`. let peers: Vec = scen .peers .iter() .filter(|p| p.kind == "swim") .map(|p| p.id.clone()) .collect(); let mut all_times: std::collections::BTreeSet = std::collections::BTreeSet::new(); for p in &peers { if let Some(list) = snaps.by_host.get(p) { for s in list { all_times.insert(s.virtual_time_ns); } } } for t in all_times { let mut converged = true; 'outer: for subject in &peers { let mut last: Option = None; for observer in &peers { if observer == subject { continue; } let Some(list) = snaps.by_host.get(observer) else { converged = false; break 'outer; }; let snap = list.iter().filter(|s| s.virtual_time_ns <= t).next_back(); let Some(snap) = snap else { converged = false; break 'outer; }; let state = snap .members .get(subject) .map(|mv| mv.state.clone()) .unwrap_or_else(|| "Unknown".to_string()); if let Some(prev) = &last { if prev != &state { converged = false; break 'outer; } } else { last = Some(state); } } } if converged && !peers.is_empty() { m.convergence_observed_ns = Some(t); break; } } m } // ────────────────────────────────────────────────────────────────────── // Synthesised gossip-flap library property (§10.3 primary scorer). // // 3-peer mesh, 60 ms link latency, 15 ms jitter, 0.5 % loss, 20 s // duration, snapshots every 2 s — matches `gossip_flap.toml`'s shape // but built in code so we can vary the SWIM kind_config per run without // disturbing the on-disk scenario. A passing tuning brings // self_incarnation_bounded into Pass on this scenario. // ────────────────────────────────────────────────────────────────────── fn gossip_flap_property_scenario(mode: Mode, knobs: Knobs) -> Scenario { let mut kind_config = toml::value::Table::new(); // Baseline values mirror the on-disk gossip_flap.toml's kind_config. // The §8 validator requires probe_interval_ns and suspicion_timeout_ns // to be present, so we always seed them; tuned mode overrides. kind_config.insert( "probe_interval_ns".into(), toml::Value::Integer(500_000_000), ); kind_config.insert("probe_timeout_ns".into(), toml::Value::Integer(100_000_000)); kind_config.insert( "suspicion_timeout_ns".into(), toml::Value::Integer(2_000_000_000), ); kind_config.insert("indirect_ping_fanout".into(), toml::Value::Integer(3)); if mode == Mode::Tuned { if let Some(v) = knobs.probe_interval_ns { kind_config.insert("probe_interval_ns".into(), toml::Value::Integer(v as i64)); } if let Some(v) = knobs.probe_timeout_ns { kind_config.insert("probe_timeout_ns".into(), toml::Value::Integer(v as i64)); } if let Some(v) = knobs.suspicion_timeout_ns { kind_config.insert( "suspicion_timeout_ns".into(), toml::Value::Integer(v as i64), ); } if let Some(v) = knobs.indirect_ping_fanout { kind_config.insert( "indirect_ping_fanout".into(), toml::Value::Integer(v as i64), ); } if let Some(v) = knobs.dead_reprobe_interval_ns { kind_config.insert( "dead_reprobe_interval_ns".into(), toml::Value::Integer(v as i64), ); } } let peers_ids = ["orchestrator", "worker_a", "worker_b"]; let peers: Vec = peers_ids .iter() .map(|id| Peer { id: (*id).into(), kind: "swim".into(), kind_config: kind_config.clone(), initial_state: "alive".into(), tick_period_ns_override: None, }) .collect(); let policy = LinkPolicy { latency_ns: 60_000_000, jitter_stddev_ns: 15_000_000, loss_prob_ppm: 5_000, reorder_prob_ppm: 0, bandwidth_bps: 25_000_000, cold_dial_penalty_ns: 200_000_000, cache_warm_after_ns: 200_000_000, cache_invalidate_after_idle_ns: 10_000_000_000, }; let mut links = Vec::new(); for a in &peers_ids { for b in &peers_ids { if a == b { continue; } links.push(Link { from: (*a).into(), to: (*b).into(), policy, }); } } let mut snapshots = Vec::new(); for at_ns in [2_000_000_000u64, 4_000_000_000, 6_000_000_000, 8_000_000_000, 10_000_000_000, 12_000_000_000, 14_000_000_000, 16_000_000_000, 18_000_000_000, 19_500_000_000] { snapshots.push(simulation::scenario::Snapshot { at_ns }); } let assertions = vec![ Assertion { kind: AssertionKind::SelfIncarnationBounded { peer: "orchestrator".into(), max_value: 2, }, }, Assertion { kind: AssertionKind::SelfIncarnationBounded { peer: "worker_a".into(), max_value: 2, }, }, Assertion { kind: AssertionKind::SelfIncarnationBounded { peer: "worker_b".into(), max_value: 2, }, }, Assertion { kind: AssertionKind::ConvergenceAfter { after_ns: 0, within_ns: 10_000_000_000, peers: peers_ids.iter().map(|s| (*s).into()).collect(), }, }, Assertion { kind: AssertionKind::MessageSizeBounded { message_kind: "swactor_dist::Ping".into(), max_bytes: 4_096, }, }, ]; let scen = Scenario { name: "gossip_flap_property".into(), seed: 42, duration_ns: 20_000_000_000, early_terminate_on_all_assertions_resolved: false, default_tick: DefaultTick { period_ns: 50_000_000 }, default_link: policy, peers, relays: Vec::new(), links, mutations: Vec::new(), snapshots, assertions, routes: Vec::new(), }; // Round-trip through the loader to populate routes etc. let text = simulation::scenario::to_toml(&scen); simulation::scenario::load_from_str( Path::new("property://gossip_flap.toml"), &text, ®istry(), ) .expect("synthesised scenario validates") } fn main() { let (mode, knobs, only) = parse_args(); // Emit the effective SwimConfig::default() once so the operator // sees what "baseline" actually means in tick-units. let defaults = SwimConfig::default(); eprintln!( "[meta] SwimConfig::default = {{ probe_interval: {:?}, probe_timeout: {:?}, suspicion_timeout: {:?}, indirect_probes: {}, dead_reprobe_interval: {:?} }}", defaults.probe_interval, defaults.probe_timeout, defaults.suspicion_timeout, defaults.indirect_probes, defaults.dead_reprobe_interval, ); eprintln!("[meta] mode={mode:?} knobs={knobs:?}"); // Scenarios scored by the sweep. The four named calibration // scenarios match the on-disk corpus; `n3_1779733878_repro` is // the deployment-mirror added by the SWIM retune // (`SWIM_RETUNE_REPORT.md` §3); `gossip_flap_property` is the // synthesised §10.3 library property the prior tune scored // against (kept as a regression guard so the retune does not // unwind the prior tune's order-of-magnitude collapse of // `self_incarnation_peak`). let scenarios: Vec<(&str, Scenario)> = vec![ ( "gossip_flap_repro", load("scenarios/reproduction/gossip_flap.toml"), ), ( "n3_own_relay_stub", load("scenarios/calibration/n3_own_relay_stub.toml"), ), ( "n3_own_relay_real_worker", load("scenarios/calibration/n3_own_relay_real_worker.toml"), ), ( "n3_canary_relay_real_worker", load("scenarios/calibration/n3_canary_relay_real_worker.toml"), ), ( "n3_1779733878_repro", load("scenarios/calibration/n3_1779733878_repro.toml"), ), ( "gossip_flap_property", gossip_flap_property_scenario(mode, knobs), ), ]; for (name, scen) in scenarios { if let Some(only_name) = &only { if name != only_name { continue; } } let report = run_scenario_report(name, scen, mode, knobs); let line = serde_json::to_string(&report).expect("ScenarioReport serialises by construction"); println!("{line}"); } }