swactor/crates/distribution/tests/t_diag_kernel_counters.rs

//! Spec §11 (kernel network counters, gap 11).
//!
//! Tier-3 host scrape carries UDP-layer counters from `/proc/net/snmp`
//! and per-interface byte/packet/drop/error counters from
//! `/proc/net/dev`. All counters are best-effort `Option`s: absent on
//! non-Linux, absent when the file can't be read, never silently zero.
//! The post-processor highlights any node whose UDP-drop or
//! interface-drop deltas are non-zero across the run window.

#![cfg(feature = "collector")]

use std::fs;

use distribution::diagnostics::identity::Identity;
use distribution::diagnostics::postproc::{render_summary, Bundle};
use distribution::diagnostics::snapshot::{
    Snapshot, SnapshotBody, SnapshotTrigger, Tier3DnsResolution, Tier3HostNetwork, Tier3HostState,
    Tier3Interface, Tier3InterfaceCounters, Tier3UdpKernelStats,
};
use distribution::diagnostics::Role;
use distribution::types::NodeId;

#[test]
fn snapshot_carries_kernel_counters_as_options_and_roundtrips() {
    let host = Tier3HostState {
        network: Some(Tier3HostNetwork {
            interfaces: vec![Tier3Interface {
                name: "eth0".into(),
                addresses: vec!["10.0.0.1".into()],
                mtu: Some(1500),
                up: true,
                counters: Some(Tier3InterfaceCounters {
                    rx_bytes: 1000,
                    rx_packets: 10,
                    rx_errors: 0,
                    rx_dropped: 0,
                    tx_bytes: 2000,
                    tx_packets: 20,
                    tx_errors: 0,
                    tx_dropped: 0,
                }),
            }],
            udp_kernel_stats: Some(Tier3UdpKernelStats {
                in_datagrams: Some(50),
                no_ports: Some(0),
                in_errors: Some(0),
                out_datagrams: Some(100),
                rcvbuf_errors: Some(0),
                sndbuf_errors: None,
            }),
            refreshed_at_ms: 1234,
            ..Tier3HostNetwork::default()
        }),
        dns: Vec::<Tier3DnsResolution>::new(),
        scraped_at_ms: 1234,
    };
    let s = serde_json::to_string(&host).unwrap();
    let back: Tier3HostState = serde_json::from_str(&s).unwrap();
    let net = back.network.expect("network present");
    let udp = net.udp_kernel_stats.expect("udp_kernel_stats present");
    assert_eq!(udp.in_datagrams, Some(50));
    assert_eq!(udp.sndbuf_errors, None, "missing fields must remain absent, never zero");
    let iface = &net.interfaces[0];
    let counters = iface.counters.as_ref().expect("counters present");
    assert_eq!(counters.rx_bytes, 1000);
    assert_eq!(counters.tx_packets, 20);
}

#[test]
fn old_snapshot_without_kernel_counters_still_parses() {
    // Spec §1: additive evolution. An old bundle (no `udp_kernel_stats`
    // / no `counters` per interface) must still parse cleanly through
    // the new schema.
    let json = serde_json::json!({
        "network": {
            "interfaces": [
                { "name": "lo", "addresses": [], "up": true }
            ],
            "refreshed_at_ms": 7
        },
        "dns": [],
        "scraped_at_ms": 7
    });
    let parsed: Tier3HostState = serde_json::from_value(json).unwrap();
    let net = parsed.network.expect("network present");
    assert!(net.udp_kernel_stats.is_none(), "old bundle: udp counters absent");
    assert!(net.interfaces[0].counters.is_none(), "old bundle: iface counters absent");
}

#[test]
fn postproc_surfaces_nodes_with_rising_drop_counters() {
    // Build a tiny in-memory bundle with two snapshots; the second one
    // shows a non-zero delta for udp.no_ports and for eth0.rx_dropped.
    let tmp = tempdir();
    let path = write_bundle_with_two_snapshots(tmp.path());
    let bundle = Bundle::parse_path(&path).expect("parse bundle");
    let md = render_summary(&bundle);
    assert!(
        md.contains("## Kernel network drops"),
        "summary must include the kernel-drops section; got:\n{md}",
    );
    assert!(
        md.contains("udp.no_ports +5"),
        "summary must call out the udp.no_ports delta (+5); got:\n{md}",
    );
    assert!(
        md.contains("eth0.rx_dropped +12"),
        "summary must call out the interface drop delta; got:\n{md}",
    );
}

#[test]
fn postproc_says_nothing_when_drops_stayed_at_zero() {
    let tmp = tempdir();
    let path = write_bundle_with_clean_counters(tmp.path());
    let bundle = Bundle::parse_path(&path).expect("parse bundle");
    let md = render_summary(&bundle);
    assert!(
        md.contains("No non-zero UDP/interface drop deltas observed."),
        "summary must say drops were clean; got:\n{md}",
    );
}

fn write_bundle_with_two_snapshots(dir: &std::path::Path) -> std::path::PathBuf {
    let node_hex = "11".repeat(32);
    let id = Identity::new(node_id_from_hex(&node_hex), Role::stage(), "run-counters");

    let snap0 = make_snapshot(&id, 1000, 0, 100, 0);
    let snap1 = make_snapshot(&id, 2000, 5, 200, 12);

    write_bundle(
        dir,
        "run-counters",
        &node_hex,
        "stage-0",
        &[snap0, snap1],
    )
}

fn write_bundle_with_clean_counters(dir: &std::path::Path) -> std::path::PathBuf {
    let node_hex = "22".repeat(32);
    let id = Identity::new(node_id_from_hex(&node_hex), Role::stage(), "run-clean");
    let snap0 = make_snapshot(&id, 1000, 0, 100, 0);
    let snap1 = make_snapshot(&id, 2000, 0, 200, 0);
    write_bundle(dir, "run-clean", &node_hex, "stage-0", &[snap0, snap1])
}

fn make_snapshot(
    id: &Identity,
    wall_ms: u64,
    no_ports: u64,
    in_datagrams: u64,
    rx_dropped: u64,
) -> Snapshot {
    Snapshot {
        identity: id.clone(),
        run_id: id.run_id.clone(),
        snapshot_id: format!("snap-{wall_ms}"),
        wall_ms,
        monotonic_seq: wall_ms,
        trigger: SnapshotTrigger::Periodic,
        body: SnapshotBody {
            host: Some(Tier3HostState {
                network: Some(Tier3HostNetwork {
                    interfaces: vec![Tier3Interface {
                        name: "eth0".into(),
                        addresses: Vec::new(),
                        mtu: None,
                        up: true,
                        counters: Some(Tier3InterfaceCounters {
                            rx_dropped,
                            ..Tier3InterfaceCounters::default()
                        }),
                    }],
                    udp_kernel_stats: Some(Tier3UdpKernelStats {
                        no_ports: Some(no_ports),
                        in_datagrams: Some(in_datagrams),
                        out_datagrams: Some(0),
                        in_errors: Some(0),
                        rcvbuf_errors: Some(0),
                        sndbuf_errors: Some(0),
                    }),
                    refreshed_at_ms: wall_ms,
                    ..Tier3HostNetwork::default()
                }),
                dns: Vec::new(),
                scraped_at_ms: wall_ms,
            }),
            ..SnapshotBody::default()
        },
    }
}

fn write_bundle(
    dir: &std::path::Path,
    run_id: &str,
    node_hex: &str,
    label: &str,
    snapshots: &[Snapshot],
) -> std::path::PathBuf {
    let tarball = dir.join(format!("{run_id}.tar.gz"));
    let f = fs::File::create(&tarball).unwrap();
    let gz = flate2::write::GzEncoder::new(f, flate2::Compression::default());
    let mut tar = tar::Builder::new(gz);

    let manifest = serde_json::json!({
        "run_id": run_id,
        "run_start_collector_ms": 1,
        "run_end_collector_ms": 9000,
        "finalize_received": true,
        "nodes": [
            { "node_id_hex": node_hex, "label": label, "role": "stage", "stage_index": 0, "boot_recorded": true, "event_batches": 0, "snapshots": snapshots.len() as u64, "finalize_recorded": true },
        ],
    });
    append_bytes(
        &mut tar,
        &format!("{run_id}/MANIFEST.json"),
        &serde_json::to_vec_pretty(&manifest).unwrap(),
    );
    let boot = serde_json::json!({
        "node_id_hex": node_hex,
        "node_id_short": &node_hex[..8],
        "role": "stage",
        "stage_index": 0,
        "stage_count": 1,
        "run_id": run_id,
        "process_start_unix_ms": 1,
        "boot_sequence": 0,
    });
    append_bytes(
        &mut tar,
        &format!("{run_id}/{label}/boot.json"),
        &serde_json::to_vec_pretty(&boot).unwrap(),
    );
    for (i, snap) in snapshots.iter().enumerate() {
        let body = serde_json::to_vec_pretty(snap).unwrap();
        append_bytes(
            &mut tar,
            &format!("{run_id}/{label}/snapshots/snapshot-{:06}.json", i + 1),
            &body,
        );
    }

    tar.finish().unwrap();
    tarball
}

fn append_bytes(
    tar: &mut tar::Builder<flate2::write::GzEncoder<fs::File>>,
    dst: &str,
    bytes: &[u8],
) {
    let mut header = tar::Header::new_gnu();
    header.set_size(bytes.len() as u64);
    header.set_mode(0o644);
    header.set_mtime(0);
    header.set_entry_type(tar::EntryType::Regular);
    header.set_cksum();
    tar.append_data(&mut header, dst, bytes).unwrap();
}

fn node_id_from_hex(hex: &str) -> NodeId {
    let mut out = [0u8; 32];
    for (i, pair) in hex.as_bytes().chunks_exact(2).enumerate() {
        let hi = match pair[0] {
            b'0'..=b'9' => pair[0] - b'0',
            b'a'..=b'f' => pair[0] - b'a' + 10,
            _ => 0,
        };
        let lo = match pair[1] {
            b'0'..=b'9' => pair[1] - b'0',
            b'a'..=b'f' => pair[1] - b'a' + 10,
            _ => 0,
        };
        out[i] = (hi << 4) | lo;
    }
    NodeId(out)
}

struct TempDir {
    path: std::path::PathBuf,
}

impl TempDir {
    fn path(&self) -> &std::path::Path {
        &self.path
    }
}

impl Drop for TempDir {
    fn drop(&mut self) {
        let _ = fs::remove_dir_all(&self.path);
    }
}

fn tempdir() -> TempDir {
    let mut path = std::env::temp_dir();
    let n: u32 = std::time::SystemTime::now()
        .duration_since(std::time::UNIX_EPOCH)
        .map(|d| (d.as_nanos() as u32) ^ std::process::id())
        .unwrap_or(0);
    path.push(format!("swactor-counters-{n:x}"));
    fs::create_dir_all(&path).unwrap();
    TempDir { path }
}