swactor/crates/ci/src/local_runner.rs

//! LocalRunner actor: executes job commands directly on the host via shell.
//!
//! Short-lived actor, one per job. Spawned by LocalCoordinator when a job
//! is ready to execute.

use std::io::BufRead;
use std::process::{Child, Command, Stdio};
use std::sync::{Arc, Mutex};

use swactor::actor::{ActorAddress, ActorInterface, Ctx};

use crate::local_coordinator::LocalCoordinatorMsg;
use crate::{JobComplete, JobFailure, JobProgress, JobSuccess, LocalStartJob};

/// Messages the LocalRunner can receive.
#[derive(Debug, Clone)]
pub enum LocalRunnerMsg {
    /// Begin executing the job (sent to self in on_start).
    Execute,
    /// Simulated: job completed (for testing without real shell).
    SimComplete(Result<(), String>),
}

/// LocalRunner actor state.
pub struct LocalRunner {
    coordinator_addr: ActorAddress,
    start_job: LocalStartJob,
}

impl LocalRunner {
    pub fn new(coordinator_addr: ActorAddress, start_job: LocalStartJob) -> Self {
        Self {
            coordinator_addr,
            start_job,
        }
    }

    /// Execute all commands in the job definition, streaming output back.
    fn execute(&self, ctx: &Ctx) {
        let job_id = &self.start_job.job_id;
        let work_dir = &self.start_job.work_dir;
        let timeout_secs = self.start_job.job_def.timeout_secs;

        eprintln!(
            "[runner] job {}/{} starting ({} commands, timeout {}s, workdir {})",
            job_id.pipeline_id.0,
            job_id.job_name,
            self.start_job.job_def.run.len(),
            timeout_secs,
            work_dir,
        );

        for cmd_str in &self.start_job.job_def.run {
            eprintln!("[runner] exec: {cmd_str}");

            // Send progress: command being run.
            let _ = ctx.send(
                self.coordinator_addr,
                LocalCoordinatorMsg::JobProgress(JobProgress {
                    job_id: job_id.clone(),
                    output_line: format!("$ {cmd_str}"),
                }),
            );

            let child_result = Command::new("sh")
                .arg("-c")
                .arg(cmd_str)
                .current_dir(work_dir)
                .envs(&self.start_job.env_overrides)
                .envs(&self.start_job.job_def.env)
                .stdout(Stdio::piped())
                .stderr(Stdio::piped())
                .spawn();

            let mut child = match child_result {
                Ok(c) => c,
                Err(e) => {
                    eprintln!("[runner] spawn failed: {e}");
                    let _ = ctx.send(
                        self.coordinator_addr,
                        LocalCoordinatorMsg::JobComplete(JobComplete {
                            job_id: job_id.clone(),
                            result: Err(JobFailure::ExecError(e.to_string())),
                            artifacts: Vec::new(),
                        }),
                    );
                    ctx.stop_self();
                    return;
                }
            };

            // Timeout: spawn a thread that kills the child after timeout_secs.
            let child_id = child.id();
            let kill_flag = Arc::new(Mutex::new(false));
            let kill_flag_clone = Arc::clone(&kill_flag);
            let (done_tx, done_rx) = std::sync::mpsc::channel::<()>();
            let timeout_handle = std::thread::spawn(move || {
                if done_rx
                    .recv_timeout(std::time::Duration::from_secs(timeout_secs))
                    .is_err()
                {
                    *kill_flag_clone.lock().unwrap() = true;
                    // Actually kill the child process so the pipe readers unblock.
                    let _ = std::process::Command::new("kill")
                        .args(["-9", &child_id.to_string()])
                        .status();
                }
            });

            // Read stdout and stderr concurrently to avoid pipe-buffer deadlock.
            let (last_lines, timed_out) =
                drain_child_output(&mut child, job_id, ctx, self.coordinator_addr, &kill_flag);

            let status = child.wait();

            // Signal timeout thread that we're done.
            let _ = done_tx.send(());
            let _ = timeout_handle.join();

            if timed_out || *kill_flag.lock().unwrap() {
                eprintln!("[runner] command timed out after {timeout_secs}s");
                let _ = ctx.send(
                    self.coordinator_addr,
                    LocalCoordinatorMsg::JobComplete(JobComplete {
                        job_id: job_id.clone(),
                        result: Err(JobFailure::Timeout),
                        artifacts: Vec::new(),
                    }),
                );
                ctx.stop_self();
                return;
            }

            match status {
                Ok(exit) if exit.success() => {
                    eprintln!("[runner] command succeeded");
                }
                Ok(exit) => {
                    let exit_code = exit.code().unwrap_or(-1);
                    eprintln!("[runner] command failed (exit {exit_code})");
                    for line in &last_lines {
                        eprintln!("[runner]   {line}");
                    }
                    let _ = ctx.send(
                        self.coordinator_addr,
                        LocalCoordinatorMsg::JobComplete(JobComplete {
                            job_id: job_id.clone(),
                            result: Err(JobFailure::CommandFailed {
                                exit_code,
                                last_lines,
                            }),
                            artifacts: Vec::new(),
                        }),
                    );
                    ctx.stop_self();
                    return;
                }
                Err(e) => {
                    eprintln!("[runner] wait failed: {e}");
                    let _ = ctx.send(
                        self.coordinator_addr,
                        LocalCoordinatorMsg::JobComplete(JobComplete {
                            job_id: job_id.clone(),
                            result: Err(JobFailure::ExecError(e.to_string())),
                            artifacts: Vec::new(),
                        }),
                    );
                    ctx.stop_self();
                    return;
                }
            }
        }

        eprintln!(
            "[runner] job {}/{} passed",
            job_id.pipeline_id.0, job_id.job_name
        );
        let _ = ctx.send(
            self.coordinator_addr,
            LocalCoordinatorMsg::JobComplete(JobComplete {
                job_id: job_id.clone(),
                result: Ok(JobSuccess),
                artifacts: Vec::new(),
            }),
        );
        ctx.stop_self();
    }
}

/// Drain stdout and stderr from a child process concurrently.
///
/// Spawns a background thread for stderr so that both pipes are consumed
/// in parallel, preventing the classic pipe-buffer deadlock where the child
/// blocks writing to a full stderr while the parent blocks reading stdout.
///
/// Returns (last_lines, timed_out).
fn drain_child_output(
    child: &mut Child,
    job_id: &crate::JobId,
    ctx: &Ctx,
    coordinator_addr: ActorAddress,
    kill_flag: &Arc<Mutex<bool>>,
) -> (Vec<String>, bool) {
    let stdout = child.stdout.take();
    let stderr = child.stderr.take();

    // Collect stderr on a background thread.
    let stderr_job_id = job_id.clone();
    let stderr_kill = Arc::clone(kill_flag);
    let stderr_handle = std::thread::spawn(move || {
        let mut lines = Vec::new();
        if let Some(stderr) = stderr {
            let reader = std::io::BufReader::new(stderr);
            for line in reader.lines() {
                if *stderr_kill.lock().unwrap() {
                    break;
                }
                if let Ok(line) = line {
                    lines.push(line);
                }
            }
        }
        lines
    });

    // Read stdout on the current thread, streaming progress.
    let mut last_lines: Vec<String> = Vec::new();
    if let Some(stdout) = stdout {
        let reader = std::io::BufReader::new(stdout);
        for line in reader.lines() {
            if let Ok(line) = line {
                let _ = ctx.send(
                    coordinator_addr,
                    LocalCoordinatorMsg::JobProgress(JobProgress {
                        job_id: job_id.clone(),
                        output_line: line.clone(),
                    }),
                );
                last_lines.push(line);
                if last_lines.len() > 50 {
                    last_lines.remove(0);
                }
            }
        }
    }

    // Join stderr thread and stream its lines as progress.
    let timed_out = *kill_flag.lock().unwrap();
    let stderr_lines = stderr_handle.join().unwrap_or_default();
    for line in &stderr_lines {
        let _ = ctx.send(
            coordinator_addr,
            LocalCoordinatorMsg::JobProgress(JobProgress {
                job_id: stderr_job_id.clone(),
                output_line: format!("[stderr] {line}"),
            }),
        );
    }

    // Merge stderr into last_lines tail.
    for line in stderr_lines {
        last_lines.push(line);
        if last_lines.len() > 50 {
            last_lines.remove(0);
        }
    }

    (last_lines, timed_out)
}

impl ActorInterface for LocalRunner {
    type Incoming = LocalRunnerMsg;
    type Response = ();

    fn on_start(&mut self, ctx: &Ctx) {
        let _ = ctx.send(ctx.self_addr(), LocalRunnerMsg::Execute);
    }

    fn handle(&mut self, ctx: &Ctx, msg: LocalRunnerMsg) {
        match msg {
            LocalRunnerMsg::Execute => {
                self.execute(ctx);
            }
            LocalRunnerMsg::SimComplete(result) => {
                let complete = JobComplete {
                    job_id: self.start_job.job_id.clone(),
                    result: match result {
                        Ok(()) => Ok(JobSuccess),
                        Err(msg) => Err(JobFailure::CommandFailed {
                            exit_code: 1,
                            last_lines: vec![msg],
                        }),
                    },
                    artifacts: Vec::new(),
                };
                let _ = ctx.send(
                    self.coordinator_addr,
                    LocalCoordinatorMsg::JobComplete(complete),
                );
                ctx.stop_self();
            }
        }
    }
}