swactor/examples/pipeline-parallel-inference/tests/spec_probes.rs

//! Adversarial spec-anchored probes built per §4/§5 contracts of
//! `PP_DEPLOY_FIX_SPEC.md`. Written to break the implementation, not echo it.
//!
//! Run with:
//!   \
//!     cargo test --test spec_probes -- --nocapture

use pipeline_parallel_inference::orchestrator::{
    spawn_chain, stage_roster_event_fields, resolve_roster, ChainGuard,
    StageRosterEntry, SpawnChainError,
};

use std::time::Duration;

// ─── §4.5 stage roster event fields ─────────────────────────────────

#[test]
fn s45_roster_lists_every_stage_in_index_order_with_required_fields() {
    let roster = vec![
        StageRosterEntry {
            stage_index: 0,
            node_id_hex: "00".repeat(32),
            node_id_short: "00".repeat(4),
        },
        StageRosterEntry {
            stage_index: 1,
            node_id_hex: "ff".repeat(32),
            node_id_short: "ff".repeat(4),
        },
        StageRosterEntry {
            stage_index: 2,
            node_id_hex: "aa".repeat(32),
            node_id_short: "aa".repeat(4),
        },
    ];
    let fields = stage_roster_event_fields(7, &roster);
    assert_eq!(fields["drive_seq"], 7);
    let stages = fields["stages"].as_array().unwrap();
    assert_eq!(stages.len(), 3);
    for (k, s) in stages.iter().enumerate() {
        // spec §4.5 fields
        assert!(s["stage_index"].as_u64().is_some());
        assert!(s["node_id_hex"].as_str().is_some());
        assert!(s["node_id_short"].as_str().is_some());
        // ordered by stage_index ascending
        assert_eq!(s["stage_index"].as_u64().unwrap(), k as u64);
    }
}

#[test]
fn s45_drive_seq_changes_per_drive() {
    // Just confirm: the helper takes drive_seq as a parameter, so the
    // orchestrator can vary it per drive (spec §4.5: "emitted on every
    // drive").
    let roster = vec![StageRosterEntry {
        stage_index: 0,
        node_id_hex: "00".repeat(32),
        node_id_short: "00".repeat(4),
    }];
    assert_ne!(
        stage_roster_event_fields(1, &roster),
        stage_roster_event_fields(2, &roster),
    );
}

// ─── §4.6 pipeline-wired ────────────────────────────────────────────
//
// Negative side: resolve_roster MUST NOT report all-resolved until every
// pp-stage-K is up. Without all stages resolved, an emitter cannot emit
// pp_pipeline_wired (spec §4.6 negative space).

#[test]
fn s46_resolve_roster_times_out_when_a_stage_never_registers() {
    // Stage 1 (out of 3) is never resolvable.
    let started_at = std::time::Instant::now();
    let result = resolve_roster(
        3,
        Duration::from_millis(150),
        Duration::from_millis(10),
        |k| if k == 1 { None } else { Some("ab".repeat(32)) },
    );
    let elapsed = started_at.elapsed();
    match result {
        Err(e) => {
            let msg = format!("{e}");
            assert!(msg.contains("[1]"), "missing-stages list must name stage 1: {msg}");
            assert!(elapsed >= Duration::from_millis(100), "must wait full budget: {elapsed:?}");
        }
        Ok(r) => panic!("expected timeout, got Ok({:?})", r),
    }
}

// ─── §4.9 spawn-chain partial-failure cleanup ───────────────────────
//
// spawn_chain treats each stage independently: a failure on one stage
// MUST NOT leak the subprocesses already spawned for earlier stages, and
// the chain's "did all succeed" predicate is what gates downstream drive.

#[test]
fn s49_independent_per_host_attempt_is_visible_in_chain_guard() {
    // Spawn_chain treats each stage independently in the sense that
    // failure rolls back what was spawned. The negative case: if stage 1
    // is the one that fails, stage 0 (already spawned) is killed (no
    // orphaned subprocess) — proxy for §4.9's "per-host result" guarantee:
    // operation does not leak processes if any one host fails.
    let mut spawn_count = 0;
    let result = spawn_chain(3, Duration::from_millis(200), |_ctx| {
        spawn_count += 1;
        let mut cmd = std::process::Command::new("sh");
        if spawn_count == 2 {
            // Force stage 1 to never announce
            cmd.arg("-c").arg("sleep 10");
        } else {
            cmd.arg("-c").arg(
                "echo PP_GPU_NODE_ADDR aabbccddeeff0011223344556677889900aabbccddeeff00112233445566778899 127.0.0.1:9999; \
                 sleep 10",
            );
        }
        cmd
    });
    // stage 1 timeout aborts
    match result {
        Err(SpawnChainError::AddressTimeout { stage, .. }) => {
            assert_eq!(stage, 1, "expected timeout on stage 1, got {stage}");
        }
        other => panic!("expected AddressTimeout(stage=1), got {other:?}"),
    }
}

// ─── §4.5 negative space: roster entry fields ───────────────────────

#[test]
fn s45_stages_list_carries_stage_index_node_id_hex_node_id_short() {
    let r = vec![StageRosterEntry {
        stage_index: 42,
        node_id_hex: "deadbeef".repeat(8),
        node_id_short: "dead".repeat(2),
    }];
    let v = stage_roster_event_fields(1, &r);
    let entry = &v["stages"][0];
    let keys: std::collections::HashSet<&str> = entry
        .as_object()
        .unwrap()
        .keys()
        .map(|s| s.as_str())
        .collect();
    // Spec §4.5: at minimum these three fields
    for f in ["stage_index", "node_id_hex", "node_id_short"] {
        assert!(keys.contains(f), "missing field {f} in roster entry");
    }
}

// ─── §5.2 distinct-host selection ───────────────────────────────────
//
// The host-throughput preflight prototype (PP_PREFLIGHT_HF) was removed; its
// only live behavior — never leasing two stages on the same physical host — is
// now unconditional in the lease's distinct-host pick. That invariant is
// covered by the `next_eligible_offer` scenario tests in
// `src/vastai.rs` (no two draws share a host_id) and the orchestrator's
// `lease_chain_finds_n_distinct_offers`.