swactor/apps/old-pipeline-parallel-inference/tests/t_binary.rs

//! T-binary: drive the actual `pp-orchestrator --seed` and `pp-worker`
//! binaries as child processes. TEST_SPEC §13 (stub workers) and §14
//! (real tinygrad workers, `#[ignore]`).
//!
//! Every test in this file is `#[ignore]`d. Each one spawns at least
//! `N + 1` real processes (one orchestrator, `N` stages, and each stage's
//! Python worker) so even the fastest case is too heavy to run in the
//! default `cargo test` guard. They are the deploy-readiness gate; run
//! them with:
//!
//! ```text
//! cargo test -p pipeline-parallel-inference --test t_binary -- --ignored
//! ```

use std::io::{BufRead, BufReader};
use std::process::{Child, ChildStderr, ChildStdout, Command, Stdio};
use std::sync::{Arc, Mutex};
use std::thread;
use std::time::{Duration, Instant};

const ORCHESTRATOR_BIN: &str = env!("CARGO_BIN_EXE_pp-orchestrator");
const WORKER_BIN: &str = env!("CARGO_BIN_EXE_pp-worker");

/// Default per-test budget for an N=5 stub-mode happy path: cluster build +
/// SWIM convergence + sequential worker boots + a short decode loop. Tests
/// that intentionally push past convergence (slow-boot, kill scenarios)
/// override this locally.
const HAPPY_PATH_TIMEOUT: Duration = Duration::from_secs(180);

fn worker_script() -> String {
    format!("{}/pp_tinygrad_worker.py", env!("CARGO_MANIFEST_DIR"))
}

/// A thread-safe sink for stdout / stderr lines. The drainer threads append
/// every line they read; tests `lines()` to read a snapshot. The vector
/// preserves arrival order so timing-sensitive assertions (e.g. "pp-entry
/// registered after all pp-stage-X") can compare positions.
#[derive(Clone, Default)]
struct LogBuffer {
    lines: Arc<Mutex<Vec<String>>>,
}

impl LogBuffer {
    fn push(&self, line: String) {
        self.lines.lock().unwrap().push(line);
    }

    fn lines(&self) -> Vec<String> {
        self.lines.lock().unwrap().clone()
    }

    fn joined(&self) -> String {
        self.lines().join("\n")
    }

    fn first_index_containing(&self, needle: &str) -> Option<usize> {
        self.lines().iter().position(|l| l.contains(needle))
    }
}

fn drain_stdout(stdout: ChildStdout, label: &'static str, buf: LogBuffer) {
    thread::spawn(move || {
        let reader = BufReader::new(stdout);
        for line in reader.lines().map_while(Result::ok) {
            eprintln!("[{label} OUT] {line}");
            buf.push(line);
        }
    });
}

fn drain_stderr(stderr: ChildStderr, label: &'static str, buf: LogBuffer) {
    thread::spawn(move || {
        let reader = BufReader::new(stderr);
        for line in reader.lines().map_while(Result::ok) {
            eprintln!("[{label} ERR] {line}");
            buf.push(line);
        }
    });
}

/// Wait for `child` to exit, polling every 100ms. Kills the child and
/// returns `None` if the timeout elapses; the caller is then responsible
/// for finishing the kill (`child.wait()`).
fn wait_with_timeout(child: &mut Child, timeout: Duration) -> Option<std::process::ExitStatus> {
    let start = Instant::now();
    loop {
        match child.try_wait() {
            Ok(Some(s)) => return Some(s),
            Ok(None) => {
                if start.elapsed() > timeout {
                    return None;
                }
                thread::sleep(Duration::from_millis(100));
            }
            Err(e) => panic!("try_wait error: {e}"),
        }
    }
}

/// List direct child pids of `pid` by reading `/proc/<pid>/task/<pid>/children`.
/// Returns an empty list if the file is missing or unreadable. Sorted ascending
/// so `pids[i]` corresponds to the i-th-spawned stage (Linux assigns rising
/// pids and `spawn_chain` is sequential).
fn child_pids(pid: u32) -> Vec<u32> {
    let path = format!("/proc/{pid}/task/{pid}/children");
    let mut v: Vec<u32> = std::fs::read_to_string(&path)
        .ok()
        .map(|s| {
            s.split_whitespace()
                .filter_map(|p| p.parse::<u32>().ok())
                .collect()
        })
        .unwrap_or_default();
    v.sort();
    v
}

/// True iff the kernel still has `pid` in a non-zombie state. Used after
/// the orchestrator has been waited on, or while inspecting orphaned
/// grandchildren. A `Z (zombie)` entry counts as dead because the
/// process is finished — it just has not been `wait()`ed on yet, which
/// happens lazily when whatever subreaper inherits the orphan claims it.
fn pid_alive(pid: u32) -> bool {
    match std::fs::read_to_string(format!("/proc/{pid}/status")) {
        Ok(s) => !s
            .lines()
            .any(|l| l.starts_with("State:") && l.contains('Z')),
        Err(_) => false,
    }
}

/// Wait until `pred()` returns true or `timeout` elapses. Returns true on
/// success.
fn wait_until(timeout: Duration, mut pred: impl FnMut() -> bool) -> bool {
    let start = Instant::now();
    while start.elapsed() < timeout {
        if pred() {
            return true;
        }
        thread::sleep(Duration::from_millis(50));
    }
    pred()
}

/// Wait for the orchestrator's `/proc/<pid>` child-list to contain at least
/// `expected` pids. Returns the sorted list when it does, or `None` on timeout.
fn wait_for_n_children(parent_pid: u32, expected: usize, timeout: Duration) -> Option<Vec<u32>> {
    let mut last: Vec<u32> = Vec::new();
    if wait_until(timeout, || {
        last = child_pids(parent_pid);
        last.len() >= expected
    }) {
        Some(child_pids(parent_pid))
    } else {
        None
    }
}

/// Extract the response text from `pp-orchestrator`'s stdout. The orchestrator
/// prints the response between two banner lines:
///
/// ```text
/// === pipeline-parallel Inference Response ===
/// <text>
/// ============================================
/// ```
///
/// Children's `print!` output from the chain spawner can be interleaved with
/// the banners on stdout. We collect every line between the markers and the
/// caller's assertion is responsible for narrowing further (the only line
/// the orchestrator's `println!` produces between them is the response text
/// itself, so the body is the response plus zero or more spurious child
/// stdout lines).
fn extract_response(lines: &[String]) -> Option<String> {
    const HEADER: &str = "=== pipeline-parallel Inference Response ===";
    const FOOTER: &str = "============================================";
    let header_idx = lines.iter().position(|l| l == HEADER)?;
    let footer_idx = lines
        .iter()
        .skip(header_idx + 1)
        .position(|l| l == FOOTER)?;
    let body: Vec<&str> = lines[header_idx + 1..header_idx + 1 + footer_idx]
        .iter()
        .map(|s| s.as_str())
        .collect();
    Some(body.join("\n"))
}

#[derive(Clone, Debug, Default)]
struct SmokeRunOpts {
    num_stages: u32,
    prompt: String,
    max_tokens: u32,
    stub: bool,
    /// Stage index that should sleep `boot_delay_secs` before initialising.
    boot_delay_stage: Option<u32>,
    boot_delay_secs: Option<u32>,
}

/// Spawn `pp-orchestrator --seed` with the given options. Returns the spawned
/// process plus a shared log buffer that captures every stdout / stderr
/// line from `pp-orchestrator` AND every `pp-worker` child (children inherit
/// the orchestrator's stderr fd, so their messages land in the same buffer).
fn spawn_smoke_run(opts: &SmokeRunOpts) -> (Child, LogBuffer, LogBuffer) {
    assert!(opts.num_stages >= 2);
    let worker = worker_script();
    let mut cmd = Command::new(ORCHESTRATOR_BIN);
    cmd.arg("--seed")
        .arg("--num-stages")
        .arg(opts.num_stages.to_string())
        .arg("--prompt")
        .arg(&opts.prompt)
        .arg("--max-tokens")
        .arg(opts.max_tokens.to_string())
        .arg("--gpu-node")
        .arg(WORKER_BIN)
        .arg("--worker")
        .arg(&worker)
        .stdout(Stdio::piped())
        .stderr(Stdio::piped());
    if opts.stub {
        cmd.env("PP_WORKER_STUB", "1");
    } else {
        cmd.env_remove("PP_WORKER_STUB");
    }
    if let (Some(s), Some(d)) = (opts.boot_delay_stage, opts.boot_delay_secs) {
        cmd.env("PP_BOOT_DELAY_STAGE", s.to_string())
            .env("PP_BOOT_DELAY_SECS", d.to_string());
    } else {
        cmd.env_remove("PP_BOOT_DELAY_STAGE");
        cmd.env_remove("PP_BOOT_DELAY_SECS");
    }

    let mut child = cmd.spawn().expect("spawn pp-orchestrator");
    let stdout = child.stdout.take().expect("piped stdout");
    let stderr = child.stderr.take().expect("piped stderr");
    let stdout_buf = LogBuffer::default();
    let stderr_buf = LogBuffer::default();
    drain_stdout(stdout, "pp-orchestrator", stdout_buf.clone());
    drain_stderr(stderr, "pp-orchestrator", stderr_buf.clone());
    (child, stdout_buf, stderr_buf)
}

/// One-shot stub-mode happy-path runner. Spawns at `num_stages`, waits for
/// completion, panics on timeout, and returns `(exit_status, stdout_lines,
/// stderr_lines, pre_exit_pids)`. The pre-exit pids are the stage children's
/// pids captured while the orchestrator was still alive — used by callers
/// to verify post-exit cleanup.
fn run_to_completion(opts: &SmokeRunOpts) -> RunOutcome {
    let (mut smoke, stdout, stderr) = spawn_smoke_run(opts);
    let smoke_pid = smoke.id();
    let n = opts.num_stages as usize;
    let pre_exit_pids =
        wait_for_n_children(smoke_pid, n, Duration::from_secs(60)).unwrap_or_else(|| {
            let _ = smoke.kill();
            let _ = smoke.wait();
            panic!("pp-orchestrator did not spawn {n} pp-worker children within 60s")
        });

    let status = wait_with_timeout(&mut smoke, HAPPY_PATH_TIMEOUT).unwrap_or_else(|| {
        let _ = smoke.kill();
        let _ = smoke.wait();
        panic!(
            "pp-orchestrator did not exit within {:?}",
            HAPPY_PATH_TIMEOUT
        );
    });
    RunOutcome {
        status,
        stdout: stdout.lines(),
        stderr: stderr.lines(),
        pre_exit_pids,
    }
}

struct RunOutcome {
    status: std::process::ExitStatus,
    stdout: Vec<String>,
    stderr: Vec<String>,
    pre_exit_pids: Vec<u32>,
}

impl RunOutcome {
    fn require_success(&self) {
        assert!(
            self.status.success(),
            "pp-orchestrator exited with {:?}\n--- stdout ---\n{}\n--- stderr (last 40 lines) ---\n{}",
            self.status,
            self.stdout.join("\n"),
            self.stderr
                .iter()
                .rev()
                .take(40)
                .rev()
                .cloned()
                .collect::<Vec<_>>()
                .join("\n"),
        );
    }

    fn require_response_non_empty(&self) -> String {
        let response = extract_response(&self.stdout).unwrap_or_else(|| {
            panic!(
                "pp-orchestrator stdout missing response banner; got:\n{}",
                self.stdout.join("\n")
            )
        });
        assert!(
            !response.trim().is_empty(),
            "response text must be non-empty, got {response:?}"
        );
        response
    }

    fn require_no_orphans(&self) {
        for pid in &self.pre_exit_pids {
            assert!(
                !pid_alive(*pid),
                "pp-worker child pid {pid} still alive after pp-orchestrator exit"
            );
        }
    }
}

/// Default stub-mode happy-path options. `max_tokens=4` keeps the decode
/// loop short; the response text is deterministic per stub-worker seed.
fn stub_opts(num_stages: u32) -> SmokeRunOpts {
    SmokeRunOpts {
        num_stages,
        prompt: "Say hello".into(),
        max_tokens: 4,
        stub: true,
        boot_delay_stage: None,
        boot_delay_secs: None,
    }
}

/// SIGKILL a pid via `kill -9`. Returns once the kernel has accepted the
/// signal (the target may not yet have been reaped).
fn sigkill(pid: u32) {
    let status = Command::new("kill")
        .arg("-9")
        .arg(pid.to_string())
        .status()
        .expect("kill -9");
    assert!(status.success(), "kill -9 {pid} failed: {status:?}");
}

/// Stub-worker accumulated text format: `tokens: [<id> <id> ...]`. The Last
/// stage builds this in `StageActor::detokenize_stub`; matching it here lets
/// the §13.1 token-count test extract the count without parsing freeform text.
fn stub_token_count(response: &str) -> Option<usize> {
    let inner = response.lines().find_map(|l| {
        l.trim()
            .strip_prefix("tokens: [")
            .and_then(|s| s.strip_suffix("]"))
    })?;
    Some(inner.split_whitespace().count())
}

// ───────────────────────────────────────────────────────────────────────
// §13.1 — Happy paths
// ───────────────────────────────────────────────────────────────────────

fn happy_path_at(num_stages: u32) {
    let outcome = run_to_completion(&stub_opts(num_stages));
    outcome.require_success();
    outcome.require_response_non_empty();
    outcome.require_no_orphans();
}

#[test]
#[ignore]
fn binary_e2e_n_stub_workers_returns_response_2() {
    happy_path_at(2);
}

#[test]
#[ignore]
fn binary_e2e_n_stub_workers_returns_response_3() {
    happy_path_at(3);
}

#[test]
#[ignore]
fn binary_e2e_n_stub_workers_returns_response_5() {
    happy_path_at(5);
}

#[test]
#[ignore]
fn binary_e2e_response_contains_accumulated_token_count() {
    let opts = SmokeRunOpts {
        num_stages: 5,
        prompt: "Say hello".into(),
        max_tokens: 6,
        stub: true,
        ..SmokeRunOpts::default()
    };
    let outcome = run_to_completion(&opts);
    outcome.require_success();
    let response = outcome.require_response_non_empty();
    let count = stub_token_count(&response).unwrap_or_else(|| {
        panic!("response does not match stub `tokens: [...]` format:\n{response}")
    });
    assert_eq!(
        count, opts.max_tokens as usize,
        "stub response should contain exactly max_tokens ids; got {count} in:\n{response}"
    );
}

#[test]
#[ignore]
fn binary_e2e_orchestrator_registers_pp_orchestrator_name() {
    let outcome = run_to_completion(&stub_opts(3));
    outcome.require_success();
    let stderr = outcome.stderr.join("\n");
    assert!(
        stderr.contains("registered pp-orchestrator"),
        "expected `registered pp-orchestrator` in stderr; got:\n{stderr}"
    );
    // The last stage cannot have sent its InferenceResponse without first
    // resolving pp-orchestrator on the iroh side. The response banner on
    // stdout is the user-visible confirmation that the round-trip closed.
    outcome.require_response_non_empty();
}

#[test]
#[ignore]
fn binary_e2e_all_stages_register_pp_stage_index_names() {
    let outcome = run_to_completion(&stub_opts(4));
    outcome.require_success();
    let stderr = outcome.stderr.join("\n");
    for i in 0..4 {
        let needle = format!("registered pp-stage-{i}");
        assert!(
            stderr.contains(&needle),
            "expected `{needle}` in stderr; full stderr:\n{stderr}"
        );
    }
}

// ───────────────────────────────────────────────────────────────────────
// §13.2 — Failure / cleanup paths
// ───────────────────────────────────────────────────────────────────────

/// Spawn `pp-orchestrator` at `num_stages`, wait until every stage child is
/// visible, kill `stage_to_kill`, and assert the orchestrator exits
/// non-zero with no surviving stage children.
fn kill_stage_and_expect_failure(num_stages: u32, stage_to_kill: u32) {
    assert!(stage_to_kill < num_stages);
    let opts = stub_opts(num_stages);
    let (mut smoke, _stdout, _stderr) = spawn_smoke_run(&opts);
    let smoke_pid = smoke.id();
    let pids = wait_for_n_children(smoke_pid, num_stages as usize, Duration::from_secs(60))
        .unwrap_or_else(|| {
            let _ = smoke.kill();
            let _ = smoke.wait();
            panic!("pp-orchestrator did not spawn {num_stages} children within 60s");
        });
    let victim = pids[stage_to_kill as usize];

    sigkill(victim);

    let exit = wait_with_timeout(&mut smoke, Duration::from_secs(120)).unwrap_or_else(|| {
        let _ = smoke.kill();
        let _ = smoke.wait();
        panic!("pp-orchestrator did not exit within 120s after killing stage {stage_to_kill}");
    });
    assert!(
        !exit.success(),
        "pp-orchestrator should fail when stage {stage_to_kill} (pid {victim}) is killed, got {exit:?}"
    );
    for pid in &pids {
        assert!(
            !pid_alive(*pid),
            "stage child pid {pid} still alive after pp-orchestrator exit"
        );
    }
}

#[test]
#[ignore]
fn binary_e2e_first_stage_killed_orchestrator_exits_nonzero_2() {
    kill_stage_and_expect_failure(2, 0);
}

#[test]
#[ignore]
fn binary_e2e_first_stage_killed_orchestrator_exits_nonzero_3() {
    kill_stage_and_expect_failure(3, 0);
}

#[test]
#[ignore]
fn binary_e2e_first_stage_killed_orchestrator_exits_nonzero_5() {
    kill_stage_and_expect_failure(5, 0);
}

#[test]
#[ignore]
fn binary_e2e_middle_stage_killed_orchestrator_exits_nonzero() {
    kill_stage_and_expect_failure(5, 2);
}

#[test]
#[ignore]
fn binary_e2e_last_stage_killed_orchestrator_exits_nonzero_2() {
    kill_stage_and_expect_failure(2, 1);
}

#[test]
#[ignore]
fn binary_e2e_last_stage_killed_orchestrator_exits_nonzero_3() {
    kill_stage_and_expect_failure(3, 2);
}

#[test]
#[ignore]
fn binary_e2e_last_stage_killed_orchestrator_exits_nonzero_5() {
    kill_stage_and_expect_failure(5, 4);
}

#[test]
#[ignore]
fn binary_e2e_no_orphaned_processes_after_clean_exit_3() {
    let outcome = run_to_completion(&stub_opts(3));
    outcome.require_success();
    outcome.require_no_orphans();
}

#[test]
#[ignore]
fn binary_e2e_no_orphaned_processes_after_failed_exit_3() {
    // Reuse the kill helper, which already asserts no orphans. Asserting
    // separately here would just re-run the same scenario.
    kill_stage_and_expect_failure(3, 1);
}

#[test]
#[ignore]
fn binary_e2e_orchestrator_sigkilled_children_die_within_timeout() {
    // SIGKILL `pp-orchestrator` itself once its children are up. The kernel
    // delivers `SIGTERM` to each pp-worker (via PR_SET_PDEATHSIG, set in
    // pp-worker's main), and each pp-worker then dies — which also
    // closes its Python worker's stdin, making the worker exit on EOF.
    let opts = stub_opts(3);
    let (mut smoke, _stdout, _stderr) = spawn_smoke_run(&opts);
    let smoke_pid = smoke.id();
    let pids = wait_for_n_children(smoke_pid, 3, Duration::from_secs(60)).unwrap_or_else(|| {
        let _ = smoke.kill();
        let _ = smoke.wait();
        panic!("pp-orchestrator did not spawn 3 children within 60s");
    });

    sigkill(smoke_pid);
    let _ = smoke.wait();

    let cleaned_up = wait_until(Duration::from_secs(10), || {
        pids.iter().all(|p| !pid_alive(*p))
    });
    assert!(
        cleaned_up,
        "pp-worker children {pids:?} still alive 10s after pp-orchestrator SIGKILL; \
         per-pid alive states: {:?}",
        pids.iter().map(|p| (p, pid_alive(*p))).collect::<Vec<_>>()
    );
}

// ───────────────────────────────────────────────────────────────────────
// §13.3 — Boot-order edge cases
// ───────────────────────────────────────────────────────────────────────

#[test]
#[ignore]
fn binary_e2e_orchestrator_can_resolve_pp_entry_after_n_stages_register() {
    let outcome = run_to_completion(&stub_opts(5));
    outcome.require_success();
    let stderr_buf = LogBuffer::default();
    for l in &outcome.stderr {
        stderr_buf.push(l.clone());
    }
    // pp-entry is registered by stage 0 only *after* it has resolved its
    // next neighbour (which in turn requires stage 1's pp-stage-1 to have
    // been registered, and so on). Walk the log: the line that announces
    // `registered pp-entry` must appear *after* every `registered
    // pp-stage-{i}` line for i in 0..N.
    let entry_idx = stderr_buf
        .first_index_containing("registered pp-entry")
        .expect("missing `registered pp-entry` in stderr");
    for i in 0..5 {
        let needle = format!("registered pp-stage-{i}");
        let idx = stderr_buf
            .first_index_containing(&needle)
            .unwrap_or_else(|| panic!("missing `{needle}` in stderr"));
        assert!(
            idx < entry_idx,
            "expected `{needle}` to be registered before pp-entry (got {idx} vs {entry_idx})"
        );
    }
}

#[test]
#[ignore]
fn binary_e2e_pp_orchestrator_handles_slow_middle_stage_boot() {
    let opts = SmokeRunOpts {
        num_stages: 4,
        prompt: "Say hello".into(),
        max_tokens: 4,
        stub: true,
        boot_delay_stage: Some(2),
        boot_delay_secs: Some(30),
    };
    let outcome = run_to_completion(&opts);
    outcome.require_success();
    outcome.require_response_non_empty();
    outcome.require_no_orphans();
}

#[test]
#[ignore]
fn binary_e2e_pp_orchestrator_handles_slow_last_stage_boot() {
    let opts = SmokeRunOpts {
        num_stages: 4,
        prompt: "Say hello".into(),
        max_tokens: 4,
        stub: true,
        boot_delay_stage: Some(3),
        boot_delay_secs: Some(30),
    };
    let outcome = run_to_completion(&opts);
    outcome.require_success();
    outcome.require_response_non_empty();
    outcome.require_no_orphans();
}

// ───────────────────────────────────────────────────────────────────────
// §14 — Real-tinygrad binary E2E (gated, very slow)
// ───────────────────────────────────────────────────────────────────────
//
// Requirements:
// * `python3` with `tinygrad` importable (a `.venv/bin/python` works if
//   exported via `WORKER_CMD`).
// * `clang` on PATH — tinygrad's CPU backend compiles kernels with it.
// * `~/.cache/tinygrad/downloads/` containing `llama3.2:1b`, or network to
//   fetch it on first run.
//
// Run explicitly:
//   `cargo test -p pipeline-parallel-inference --test t_binary \
//       binary_e2e_real_tinygrad -- --ignored --nocapture --test-threads=1`

/// §14 tests require `numpy`, `tinygrad`, `clang`, and the `llama3.2:1b`
/// GGUF to be reachable from the worker process. None of that is guaranteed
/// even with `--ignored`, so each §14 test opts in via this env var. Set
/// `PP_TINYGRAD_E2E=1` to actually run them.
fn skip_unless_tinygrad_opted_in() -> bool {
    if std::env::var("PP_TINYGRAD_E2E").ok().as_deref() != Some("1") {
        eprintln!(
            "skipping: §14 real-tinygrad tests are off by default; \
             set PP_TINYGRAD_E2E=1 to run them"
        );
        return true;
    }
    false
}

fn real_tinygrad_at(num_stages: u32) {
    if skip_unless_tinygrad_opted_in() {
        return;
    }
    let opts = SmokeRunOpts {
        num_stages,
        prompt: "Say hello".into(),
        max_tokens: 4,
        stub: false,
        ..SmokeRunOpts::default()
    };
    let (mut smoke, stdout, stderr) = spawn_smoke_run(&opts);
    let smoke_pid = smoke.id();
    let pre_exit_pids =
        wait_for_n_children(smoke_pid, num_stages as usize, Duration::from_secs(120))
            .unwrap_or_else(|| {
                let _ = smoke.kill();
                let _ = smoke.wait();
                panic!("pp-orchestrator did not spawn {num_stages} children within 120s");
            });
    let status = wait_with_timeout(&mut smoke, Duration::from_secs(1200)).unwrap_or_else(|| {
        let _ = smoke.kill();
        let _ = smoke.wait();
        panic!("pp-orchestrator did not finish within 20m");
    });
    let stdout_lines = stdout.lines();
    let stderr_joined = stderr.joined();
    assert!(
        status.success(),
        "pp-orchestrator exited {status:?}\n--- stdout ---\n{}\n--- stderr (tail) ---\n{}",
        stdout_lines.join("\n"),
        stderr_joined
            .lines()
            .rev()
            .take(40)
            .collect::<Vec<_>>()
            .into_iter()
            .rev()
            .collect::<Vec<_>>()
            .join("\n"),
    );
    let response = extract_response(&stdout_lines).unwrap_or_else(|| {
        panic!(
            "missing response banner; stdout:\n{}",
            stdout_lines.join("\n")
        )
    });
    assert!(
        !response.trim().is_empty(),
        "real-tinygrad response must be non-empty, got {response:?}"
    );
    for pid in pre_exit_pids {
        assert!(
            !pid_alive(pid),
            "pp-worker child pid {pid} still alive after pp-orchestrator exit"
        );
    }
}

#[test]
#[ignore]
fn binary_e2e_real_tinygrad_two_stage_returns_response() {
    real_tinygrad_at(2);
}

#[test]
#[ignore]
fn binary_e2e_real_tinygrad_three_stage_returns_response() {
    real_tinygrad_at(3);
}

#[test]
#[ignore]
fn binary_e2e_real_tinygrad_four_stage_returns_response() {
    real_tinygrad_at(4);
}

/// §14: the printed response at `N=3` should match what
/// `Transformer.generate()` produces single-node. The full equivalence
/// machinery lives in `t_integration.rs::§12`; this test is the same
/// assertion escalated through the binary. Skipped unless
/// `PP_SINGLE_NODE_REFERENCE` is set to the precomputed reference text.
#[test]
#[ignore]
fn binary_e2e_real_tinygrad_response_matches_single_node_for_say_hello() {
    if skip_unless_tinygrad_opted_in() {
        return;
    }
    let reference = match std::env::var("PP_SINGLE_NODE_REFERENCE") {
        Ok(s) if !s.trim().is_empty() => s,
        _ => {
            eprintln!(
                "skipping: PP_SINGLE_NODE_REFERENCE is not set. Run the §12 \
                 single-node reference script and pass the output text via \
                 PP_SINGLE_NODE_REFERENCE=\"...\""
            );
            return;
        }
    };
    let opts = SmokeRunOpts {
        num_stages: 3,
        prompt: "Say hello".into(),
        max_tokens: 4,
        stub: false,
        ..SmokeRunOpts::default()
    };
    let (mut smoke, stdout, _stderr) = spawn_smoke_run(&opts);
    let status = wait_with_timeout(&mut smoke, Duration::from_secs(1200)).unwrap_or_else(|| {
        let _ = smoke.kill();
        let _ = smoke.wait();
        panic!("pp-orchestrator did not finish within 20m");
    });
    assert!(status.success(), "pp-orchestrator exited {status:?}");
    let response = extract_response(&stdout.lines()).expect("missing response banner");
    assert_eq!(
        response.trim(),
        reference.trim(),
        "binary response at N=3 should match single-node generate() output"
    );
}