Architecture enforcement: - Install a repository-owned rustc wrapper for ordinary cargo check, build, and test commands. Resolve compiler item identities so renamed imports and helper wrappers cannot hide spawning, timing, blocking, polling, thread, or runtime-driving capabilities. - Define the execution-owner crates and reject dependencies from those substrates back into Myelin policy. Add compile-pass and compile-fail contracts for actor helpers, execution owners, test waits, forbidden capabilities, suppression attempts, and owner dependency inversions. Execution ownership: - Add engine-owned actor timers with cancellation and generation identity, then migrate lifecycle deadlines and protocol ticks off application tasks. Keep networking, process output, telemetry, and blocking provider calls in their approved I/O substrates. - Move process spawn, wait, signal, Unix listener, and output-following mechanics into swactor-process. Isolate Vast.ai blocking HTTP mechanics behind its adapter while actors retain retry, recovery, and provisioning decisions. Myelin control flow: - Rework manual control, worker lifecycle, provisioning, provider recovery, job deployment, distribution, edge orchestration, and shutdown as actor state transitions and typed effects. Preserve durable provider adoption and command outcomes across graceful and abrupt restarts. - Replace controller loops and timer-forwarding tasks with actor messages; leave substrate tasks as cancellable observation streams with no durable policy state. Properties and resource ownership: - Add deterministic engine and component properties, a stateful mock-VastAI lifecycle model, persisted regression cases, controlled fault injection, and a bounded nightly workflow covering restart and teardown behavior. - Terminate reply observers, cancel telemetry collectors, bound dashboard projections, and release child observers, file descriptors, process records, and inode-verified Unix sockets on every terminal path. Verified with the compiler-policy contracts, 105 Myelin library tests, 32 swactor-process tests, telemetry cancellation contracts, randomized stateful restart cases, cargo check, and formatting checks.
929 lines
28 KiB
Rust
929 lines
28 KiB
Rust
// NOTE: This module is currently excluded from compilation (its `mod` entry
|
|
// was removed from orchestration/mod.rs). Backburner code — deprecated for
|
|
// now but not yet staged for pruning. Re-wire or delete when node-image
|
|
// provisioning is revisited.
|
|
|
|
use std::collections::{BTreeMap, BTreeSet};
|
|
use std::fs::{self, File};
|
|
use std::io::{BufRead, BufReader, Read};
|
|
use std::path::{Path, PathBuf};
|
|
use std::process::{Command, Stdio};
|
|
use std::sync::mpsc;
|
|
use std::thread;
|
|
use std::time::{Duration, Instant};
|
|
|
|
const NODE_IMAGE_CONTENT_INPUTS: &[&str] = &["apps/myelin/node-image/Dockerfile"];
|
|
|
|
const BASE_IMAGE_SOURCE_INPUTS: &[&str] = &[
|
|
"apps/myelin/node-image/Dockerfile.base",
|
|
"apps/myelin/node-image/myelin_entrypoint.sh",
|
|
];
|
|
const NODE_IMAGE_TAG_LABEL: &str = "org.swactor.myelin.node-image-tag";
|
|
const NODE_IMAGE_SOURCE_HASH_LABEL: &str = "org.swactor.myelin.node.source-hash";
|
|
const NODE_IMAGE_AGENT_HASH_LABEL: &str = "org.swactor.myelin.node.agent-hash";
|
|
const NODE_IMAGE_BASE_HASH_LABEL: &str = "org.swactor.myelin.node.base-hash";
|
|
const BASE_IMAGE_SOURCE_HASH_LABEL: &str = "org.swactor.myelin.base.source-hash";
|
|
|
|
pub(super) struct NodeImageRequest {
|
|
pub(super) requested_image: String,
|
|
pub(super) base_image: String,
|
|
pub(super) node_bin: PathBuf,
|
|
pub(super) requires_registry_image: bool,
|
|
pub(super) extra_tag: Option<String>,
|
|
pub(super) force_refresh: bool,
|
|
pub(super) enabled: bool,
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub(super) enum NodeImageProgressEventKind {
|
|
ImageReference {
|
|
role: String,
|
|
image_ref: String,
|
|
},
|
|
CommandStarted {
|
|
program: String,
|
|
args: Vec<String>,
|
|
},
|
|
CommandStdout {
|
|
line: String,
|
|
},
|
|
CommandStderr {
|
|
line: String,
|
|
},
|
|
CommandExited {
|
|
status: String,
|
|
code: Option<i32>,
|
|
success: bool,
|
|
},
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub(super) struct NodeImageProgressEvent {
|
|
pub(super) command_label: Option<String>,
|
|
pub(super) image_ref: Option<String>,
|
|
pub(super) elapsed_ms: Option<u128>,
|
|
pub(super) kind: NodeImageProgressEventKind,
|
|
}
|
|
|
|
pub(super) trait NodeImageProgressSink {
|
|
fn emit(&mut self, event: NodeImageProgressEvent);
|
|
}
|
|
|
|
pub(super) fn prepare_node_image_with_progress(
|
|
request: NodeImageRequest,
|
|
progress: Option<&mut dyn NodeImageProgressSink>,
|
|
) -> Result<String, String> {
|
|
let mut progress = progress;
|
|
prepare_node_image_inner(request, &mut progress)
|
|
}
|
|
|
|
fn prepare_node_image_inner(
|
|
request: NodeImageRequest,
|
|
progress: &mut Option<&mut dyn NodeImageProgressSink>,
|
|
) -> Result<String, String> {
|
|
emit_image_reference(progress, "requested", &request.requested_image);
|
|
if !request.enabled {
|
|
return Ok(request.requested_image);
|
|
}
|
|
|
|
emit_image_reference(progress, "base", &request.base_image);
|
|
let root = workspace_root()?;
|
|
let image = ImageName::parse(&request.requested_image)?;
|
|
let first_repository_component = image
|
|
.repository
|
|
.split('/')
|
|
.next()
|
|
.unwrap_or(&image.repository);
|
|
let registry_reachable = image.repository.contains('/')
|
|
|| first_repository_component.contains('.')
|
|
|| first_repository_component.contains(':')
|
|
|| first_repository_component == "localhost";
|
|
if request.requires_registry_image && !registry_reachable {
|
|
return Err(format!(
|
|
"VastAI node image {:?} must include a registry namespace",
|
|
image.repository
|
|
));
|
|
}
|
|
|
|
run_status_command(
|
|
&root,
|
|
"cargo",
|
|
&["build", "--quiet", "-p", "myelin", "--bin", "myelin-worker"],
|
|
"build myelin-worker",
|
|
None,
|
|
progress,
|
|
)?;
|
|
|
|
let base_hash = content_hash_for_inputs(&root, BASE_IMAGE_SOURCE_INPUTS)?;
|
|
let image_content_hash = node_image_content_hash(&root, &request.node_bin, &base_hash)?;
|
|
let tag = image_version_tag(&root, &image_content_hash)?;
|
|
let image_ref = image.ref_for_tag(&tag);
|
|
emit_image_reference(progress, "resolved", &image_ref);
|
|
let expected_node_labels = vec![
|
|
(NODE_IMAGE_TAG_LABEL, tag.as_str()),
|
|
(NODE_IMAGE_SOURCE_HASH_LABEL, image_content_hash.as_str()),
|
|
(NODE_IMAGE_AGENT_HASH_LABEL, image_content_hash.as_str()),
|
|
(NODE_IMAGE_BASE_HASH_LABEL, base_hash.as_str()),
|
|
];
|
|
let expected_base_labels = vec![(BASE_IMAGE_SOURCE_HASH_LABEL, base_hash.as_str())];
|
|
let alias_tags = alias_tags(&image, request.extra_tag.as_deref(), &tag)?;
|
|
for alias in alias_refs(&image, &alias_tags) {
|
|
emit_image_reference(progress, "alias", &alias);
|
|
}
|
|
let remote_required = request.requires_registry_image;
|
|
|
|
let local_image_matches = docker_image_labels_match(&root, &image_ref, &expected_node_labels)?;
|
|
let remote_available = remote_required && docker_manifest_exists(&root, &image_ref);
|
|
if !request.force_refresh && remote_required && remote_available {
|
|
ensure_aliases_for_remote(progress, &root, &image_ref, &image, &alias_tags)?;
|
|
prune_old_dirty_images(&root, &image, &tag);
|
|
return Ok(image_ref);
|
|
}
|
|
if !request.force_refresh && remote_required && local_image_matches {
|
|
ensure_aliases_local(progress, &root, &image_ref, &image, &alias_tags)?;
|
|
push_image(progress, &root, &image_ref)?;
|
|
for alias in alias_refs(&image, &alias_tags) {
|
|
push_image(progress, &root, &alias)?;
|
|
}
|
|
prune_old_dirty_images(&root, &image, &tag);
|
|
return Ok(image_ref);
|
|
}
|
|
if !request.force_refresh && !remote_required && local_image_matches {
|
|
ensure_aliases_local(progress, &root, &image_ref, &image, &alias_tags)?;
|
|
prune_old_dirty_images(&root, &image, &tag);
|
|
return Ok(image_ref);
|
|
}
|
|
let base_image_matches =
|
|
docker_image_labels_match(&root, &request.base_image, &expected_base_labels)?;
|
|
if !base_image_matches {
|
|
let base_source_hash_label = format!("{BASE_IMAGE_SOURCE_HASH_LABEL}={base_hash}");
|
|
run_status_command(
|
|
&root,
|
|
"docker",
|
|
&[
|
|
"build",
|
|
"-f",
|
|
"apps/myelin/node-image/Dockerfile.base",
|
|
"--label",
|
|
base_source_hash_label.as_str(),
|
|
"-t",
|
|
request.base_image.as_str(),
|
|
".",
|
|
],
|
|
"build myelin node base image",
|
|
Some(&request.base_image),
|
|
progress,
|
|
)?;
|
|
}
|
|
|
|
let node_bin = {
|
|
let full = if request.node_bin.is_absolute() {
|
|
request.node_bin.to_path_buf()
|
|
} else {
|
|
root.join(&request.node_bin)
|
|
};
|
|
relative_path(&root, &full).map(|relative| relative.to_string_lossy().to_string())
|
|
}?;
|
|
let mut build_args = vec![
|
|
"build".to_owned(),
|
|
"-f".to_owned(),
|
|
"apps/myelin/node-image/Dockerfile".to_owned(),
|
|
"--build-arg".to_owned(),
|
|
format!("BASE_IMAGE={}", request.base_image),
|
|
"--build-arg".to_owned(),
|
|
format!("MYELIN_NODE_BIN={node_bin}"),
|
|
];
|
|
for (key, value) in &expected_node_labels {
|
|
build_args.push("--label".to_owned());
|
|
build_args.push(format!("{key}={value}"));
|
|
}
|
|
build_args.extend(["-t".to_owned(), image_ref.clone(), ".".to_owned()]);
|
|
run_status_command(
|
|
&root,
|
|
"docker",
|
|
&build_args.iter().map(String::as_str).collect::<Vec<_>>(),
|
|
"build myelin node image",
|
|
Some(&image_ref),
|
|
progress,
|
|
)?;
|
|
ensure_aliases_local(progress, &root, &image_ref, &image, &alias_tags)?;
|
|
|
|
if remote_required {
|
|
push_image(progress, &root, &image_ref)?;
|
|
for alias in alias_refs(&image, &alias_tags) {
|
|
push_image(progress, &root, &alias)?;
|
|
}
|
|
}
|
|
|
|
prune_old_dirty_images(&root, &image, &tag);
|
|
Ok(image_ref)
|
|
}
|
|
|
|
fn workspace_root() -> Result<PathBuf, String> {
|
|
let output = swactor_process::command_output(&mut Command::new("git")
|
|
.args(["rev-parse", "--show-toplevel"])
|
|
.current_dir(env!("CARGO_MANIFEST_DIR"))
|
|
.stdin(Stdio::null()))
|
|
.map_err(|e| format!("locate repository root with git: {e}"))?;
|
|
if !output.status.success() {
|
|
return Err(format!(
|
|
"git rev-parse --show-toplevel failed with {}: {}",
|
|
output.status,
|
|
String::from_utf8_lossy(&output.stderr).trim()
|
|
));
|
|
}
|
|
Ok(PathBuf::from(
|
|
String::from_utf8_lossy(&output.stdout).trim(),
|
|
))
|
|
}
|
|
|
|
fn image_version_tag(root: &Path, image_content_hash: &str) -> Result<String, String> {
|
|
if git_capture(root, &["status", "--porcelain"])?
|
|
.trim()
|
|
.is_empty()
|
|
{
|
|
let sha = git_capture(root, &["rev-parse", "--short=12", "HEAD"])?;
|
|
Ok(format!("git-{}", sha.trim()))
|
|
} else {
|
|
Ok(format!("dirty-{image_content_hash}"))
|
|
}
|
|
}
|
|
|
|
fn git_capture(root: &Path, args: &[&str]) -> Result<String, String> {
|
|
let output = swactor_process::command_output(&mut Command::new("git")
|
|
.current_dir(root)
|
|
.args(args)
|
|
.stdin(Stdio::null()))
|
|
.map_err(|e| format!("run git {}: {e}", args.join(" ")))?;
|
|
if output.status.success() {
|
|
Ok(String::from_utf8_lossy(&output.stdout).to_string())
|
|
} else {
|
|
Err(format!(
|
|
"git {} failed with {}: {}",
|
|
args.join(" "),
|
|
output.status,
|
|
String::from_utf8_lossy(&output.stderr).trim()
|
|
))
|
|
}
|
|
}
|
|
|
|
fn node_image_content_hash(
|
|
root: &Path,
|
|
node_bin: &Path,
|
|
base_hash: &str,
|
|
) -> Result<String, String> {
|
|
let mut files = Vec::new();
|
|
for input in NODE_IMAGE_CONTENT_INPUTS {
|
|
let path = root.join(input);
|
|
collect_hash_inputs(root, &path, &mut files)?;
|
|
}
|
|
let node_bin = if node_bin.is_absolute() {
|
|
node_bin.to_path_buf()
|
|
} else {
|
|
root.join(node_bin)
|
|
};
|
|
files.push(relative_path(root, &node_bin)?);
|
|
files.sort();
|
|
files.dedup();
|
|
hash_relative_files_with_salts(root, files, &[("base", base_hash)])
|
|
}
|
|
|
|
fn content_hash_for_inputs(root: &Path, inputs: &[&str]) -> Result<String, String> {
|
|
let mut files = Vec::new();
|
|
for input in inputs {
|
|
let path = root.join(input);
|
|
collect_hash_inputs(root, &path, &mut files)?;
|
|
}
|
|
files.sort();
|
|
files.dedup();
|
|
hash_relative_files_with_salts(root, files, &[])
|
|
}
|
|
|
|
fn hash_relative_files_with_salts(
|
|
root: &Path,
|
|
files: Vec<PathBuf>,
|
|
salts: &[(&str, &str)],
|
|
) -> Result<String, String> {
|
|
let mut hasher = blake3::Hasher::new();
|
|
for (key, value) in salts {
|
|
hasher.update(key.as_bytes());
|
|
hasher.update(b"\0");
|
|
hasher.update(value.as_bytes());
|
|
hasher.update(b"\0");
|
|
}
|
|
for relative in files {
|
|
let full = root.join(&relative);
|
|
hasher.update(relative.to_string_lossy().as_bytes());
|
|
hasher.update(b"\0");
|
|
hash_file_content(root, &full, &mut hasher)?;
|
|
hasher.update(b"\0");
|
|
}
|
|
let hash = hasher.finalize().to_hex().to_string();
|
|
Ok(hash[..16].to_owned())
|
|
}
|
|
|
|
fn hash_file_content(root: &Path, path: &Path, hasher: &mut blake3::Hasher) -> Result<(), String> {
|
|
let display = display_workspace_path(root, path);
|
|
let mut file = File::open(path).map_err(|e| format!("open {display}: {e}"))?;
|
|
let mut buf = [0_u8; 64 * 1024];
|
|
loop {
|
|
let n = file
|
|
.read(&mut buf)
|
|
.map_err(|e| format!("read {display}: {e}"))?;
|
|
if n == 0 {
|
|
break;
|
|
}
|
|
hasher.update(&buf[..n]);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn collect_hash_inputs(root: &Path, path: &Path, out: &mut Vec<PathBuf>) -> Result<(), String> {
|
|
if !path.exists() {
|
|
return Ok(());
|
|
}
|
|
let display = display_workspace_path(root, path);
|
|
let metadata = fs::metadata(path).map_err(|e| format!("stat {display}: {e}"))?;
|
|
if metadata.is_file() {
|
|
if !matches!(path.extension().and_then(|ext| ext.to_str()), Some("pyc")) {
|
|
out.push(relative_path(root, path)?);
|
|
}
|
|
return Ok(());
|
|
}
|
|
if !metadata.is_dir()
|
|
|| matches!(
|
|
path.file_name().and_then(|name| name.to_str()),
|
|
Some(".git" | "target" | "__pycache__")
|
|
)
|
|
{
|
|
return Ok(());
|
|
}
|
|
let entries = fs::read_dir(path).map_err(|e| format!("read dir {display}: {e}"))?;
|
|
for entry in entries {
|
|
let entry = entry.map_err(|e| format!("read dir entry {display}: {e}"))?;
|
|
collect_hash_inputs(root, &entry.path(), out)?;
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn relative_path(root: &Path, path: &Path) -> Result<PathBuf, String> {
|
|
path.strip_prefix(root).map(Path::to_path_buf).map_err(|e| {
|
|
format!(
|
|
"make {} relative to {}: {e}",
|
|
display_workspace_path(root, path),
|
|
"."
|
|
)
|
|
})
|
|
}
|
|
|
|
fn display_workspace_path(root: &Path, path: &Path) -> String {
|
|
match path.strip_prefix(root) {
|
|
Ok(relative) if relative.as_os_str().is_empty() => ".".to_owned(),
|
|
Ok(relative) => format!("./{}", relative.display()),
|
|
Err(_) => path.display().to_string(),
|
|
}
|
|
}
|
|
|
|
fn alias_tags(
|
|
image: &ImageName,
|
|
extra_tag: Option<&str>,
|
|
version_tag: &str,
|
|
) -> Result<BTreeSet<String>, String> {
|
|
let mut tags = BTreeSet::new();
|
|
if let Some(tag) = image.requested_tag.as_deref() {
|
|
insert_alias_tag(&mut tags, tag, version_tag)?;
|
|
}
|
|
if let Some(tag) = extra_tag {
|
|
insert_alias_tag(&mut tags, tag, version_tag)?;
|
|
}
|
|
Ok(tags)
|
|
}
|
|
|
|
fn insert_alias_tag(
|
|
tags: &mut BTreeSet<String>,
|
|
tag: &str,
|
|
version_tag: &str,
|
|
) -> Result<(), String> {
|
|
let tag = tag.trim();
|
|
if tag.is_empty() {
|
|
return Err("node image tag must not be empty".to_owned());
|
|
}
|
|
if tag != version_tag {
|
|
tags.insert(tag.to_owned());
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn ensure_aliases_local(
|
|
progress: &mut Option<&mut dyn NodeImageProgressSink>,
|
|
root: &Path,
|
|
source_ref: &str,
|
|
image: &ImageName,
|
|
alias_tags: &BTreeSet<String>,
|
|
) -> Result<(), String> {
|
|
for alias in alias_refs(image, alias_tags) {
|
|
if alias != source_ref {
|
|
run_status_command(
|
|
root,
|
|
"docker",
|
|
&["tag", source_ref, &alias],
|
|
"tag myelin node image",
|
|
Some(&alias),
|
|
progress,
|
|
)?;
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn ensure_aliases_for_remote(
|
|
progress: &mut Option<&mut dyn NodeImageProgressSink>,
|
|
root: &Path,
|
|
source_ref: &str,
|
|
image: &ImageName,
|
|
alias_tags: &BTreeSet<String>,
|
|
) -> Result<bool, String> {
|
|
if alias_tags.is_empty() {
|
|
return Ok(false);
|
|
}
|
|
if !docker_image_exists(root, source_ref) {
|
|
run_status_command(
|
|
root,
|
|
"docker",
|
|
&["pull", source_ref],
|
|
"pull myelin node image",
|
|
Some(source_ref),
|
|
progress,
|
|
)?;
|
|
}
|
|
ensure_aliases_local(progress, root, source_ref, image, alias_tags)?;
|
|
for alias in alias_refs(image, alias_tags) {
|
|
push_image(progress, root, &alias)?;
|
|
}
|
|
Ok(true)
|
|
}
|
|
|
|
fn alias_refs(image: &ImageName, alias_tags: &BTreeSet<String>) -> Vec<String> {
|
|
alias_tags
|
|
.iter()
|
|
.map(|tag| image.ref_for_tag(tag))
|
|
.collect()
|
|
}
|
|
|
|
fn push_image(
|
|
progress: &mut Option<&mut dyn NodeImageProgressSink>,
|
|
root: &Path,
|
|
image_ref: &str,
|
|
) -> Result<(), String> {
|
|
run_status_command(
|
|
root,
|
|
"docker",
|
|
&["push", image_ref],
|
|
"push myelin node image",
|
|
Some(image_ref),
|
|
progress,
|
|
)
|
|
}
|
|
|
|
fn docker_image_labels_match(
|
|
root: &Path,
|
|
image_ref: &str,
|
|
expected: &[(&str, &str)],
|
|
) -> Result<bool, String> {
|
|
let Some(labels) = docker_image_labels(root, image_ref)? else {
|
|
return Ok(false);
|
|
};
|
|
Ok(expected
|
|
.iter()
|
|
.all(|(key, value)| labels.get(*key).map(String::as_str) == Some(*value)))
|
|
}
|
|
|
|
fn prune_old_dirty_images(root: &Path, image: &ImageName, keep_tag: &str) {
|
|
let prune_enabled = std::env::var("MYELIN_NODE_IMAGE_PRUNE")
|
|
.map(|value| {
|
|
let value = value.trim().to_ascii_lowercase();
|
|
!matches!(value.as_str(), "0" | "false" | "no" | "off")
|
|
})
|
|
.unwrap_or(true);
|
|
if !prune_enabled {
|
|
return;
|
|
}
|
|
|
|
let tags = match docker_image_tags(root, &image.repository) {
|
|
Ok(tags) => tags,
|
|
Err(error) => {
|
|
eprintln!("myelin-node-image: prune old dirty images skipped: {error}");
|
|
return;
|
|
}
|
|
};
|
|
|
|
let keep_old = std::env::var("MYELIN_NODE_IMAGE_PRUNE_KEEP")
|
|
.ok()
|
|
.and_then(|value| value.trim().parse::<usize>().ok())
|
|
.unwrap_or(3);
|
|
let mut retained_old = 0_usize;
|
|
for (repository, tag) in tags {
|
|
if repository != image.repository
|
|
|| !tag.starts_with("dirty-")
|
|
|| tag == keep_tag
|
|
|| tag == "<none>"
|
|
{
|
|
continue;
|
|
}
|
|
|
|
let image_ref = image.ref_for_tag(&tag);
|
|
let Ok(Some(labels)) = docker_image_labels(root, &image_ref) else {
|
|
continue;
|
|
};
|
|
if labels.get(NODE_IMAGE_TAG_LABEL).map(String::as_str) != Some(tag.as_str())
|
|
|| !labels.contains_key(NODE_IMAGE_SOURCE_HASH_LABEL)
|
|
|| !labels.contains_key(NODE_IMAGE_AGENT_HASH_LABEL)
|
|
|| !labels.contains_key(NODE_IMAGE_BASE_HASH_LABEL)
|
|
{
|
|
continue;
|
|
}
|
|
if docker_image_has_container(root, &image_ref) {
|
|
eprintln!(
|
|
"myelin-node-image: prune old dirty image {image_ref} skipped: container exists"
|
|
);
|
|
continue;
|
|
}
|
|
if retained_old < keep_old {
|
|
retained_old += 1;
|
|
continue;
|
|
}
|
|
|
|
eprintln!("myelin-node-image: prune old dirty image {image_ref}");
|
|
if let Err(error) = docker_image_remove(root, &image_ref) {
|
|
eprintln!("myelin-node-image: prune old dirty image {image_ref} skipped: {error}");
|
|
}
|
|
}
|
|
}
|
|
|
|
fn emit_image_reference(
|
|
progress: &mut Option<&mut dyn NodeImageProgressSink>,
|
|
role: &str,
|
|
image_ref: &str,
|
|
) {
|
|
emit_progress(
|
|
progress,
|
|
NodeImageProgressEvent {
|
|
command_label: None,
|
|
image_ref: Some(image_ref.to_owned()),
|
|
elapsed_ms: None,
|
|
kind: NodeImageProgressEventKind::ImageReference {
|
|
role: role.to_owned(),
|
|
image_ref: image_ref.to_owned(),
|
|
},
|
|
},
|
|
);
|
|
}
|
|
|
|
fn emit_progress(
|
|
progress: &mut Option<&mut dyn NodeImageProgressSink>,
|
|
event: NodeImageProgressEvent,
|
|
) {
|
|
if let Some(sink) = progress.as_deref_mut() {
|
|
sink.emit(event);
|
|
}
|
|
}
|
|
|
|
fn emit_command_progress(
|
|
progress: &mut Option<&mut dyn NodeImageProgressSink>,
|
|
label: &str,
|
|
image_ref: Option<&str>,
|
|
elapsed_ms: u128,
|
|
kind: NodeImageProgressEventKind,
|
|
) {
|
|
emit_progress(
|
|
progress,
|
|
NodeImageProgressEvent {
|
|
command_label: Some(label.to_owned()),
|
|
image_ref: image_ref.map(str::to_owned),
|
|
elapsed_ms: Some(elapsed_ms),
|
|
kind,
|
|
},
|
|
);
|
|
}
|
|
|
|
enum CommandOutputLine {
|
|
Stdout(String),
|
|
Stderr(String),
|
|
}
|
|
|
|
fn spawn_line_reader<R>(
|
|
reader: R,
|
|
to_line: fn(String) -> CommandOutputLine,
|
|
tx: mpsc::Sender<CommandOutputLine>,
|
|
) -> thread::JoinHandle<()>
|
|
where
|
|
R: Read + Send + 'static,
|
|
{
|
|
thread::spawn(move || {
|
|
for line in BufReader::new(reader).lines().map_while(Result::ok) {
|
|
if tx.send(to_line(line)).is_err() {
|
|
break;
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
fn drain_command_lines(
|
|
rx: &mpsc::Receiver<CommandOutputLine>,
|
|
progress: &mut Option<&mut dyn NodeImageProgressSink>,
|
|
label: &str,
|
|
image_ref: Option<&str>,
|
|
started: Instant,
|
|
) {
|
|
while let Ok(line) = rx.try_recv() {
|
|
let kind = match line {
|
|
CommandOutputLine::Stdout(line) => NodeImageProgressEventKind::CommandStdout { line },
|
|
CommandOutputLine::Stderr(line) => NodeImageProgressEventKind::CommandStderr { line },
|
|
};
|
|
emit_command_progress(
|
|
progress,
|
|
label,
|
|
image_ref,
|
|
started.elapsed().as_millis(),
|
|
kind,
|
|
);
|
|
}
|
|
}
|
|
|
|
fn run_status_command(
|
|
root: &Path,
|
|
program: &str,
|
|
args: &[&str],
|
|
label: &str,
|
|
image_ref: Option<&str>,
|
|
progress: &mut Option<&mut dyn NodeImageProgressSink>,
|
|
) -> Result<(), String> {
|
|
let args: Vec<String> = args.iter().map(|arg| (*arg).to_owned()).collect();
|
|
eprintln!("myelin-node-image: {label}");
|
|
if progress.is_none() {
|
|
let status = swactor_process::command_status(&mut Command::new(program)
|
|
.current_dir(root)
|
|
.args(&args)
|
|
.stdin(Stdio::null())
|
|
.stdout(Stdio::inherit())
|
|
.stderr(Stdio::inherit()))
|
|
.map_err(|e| format!("run {label}: {e}"))?;
|
|
return if status.success() {
|
|
Ok(())
|
|
} else {
|
|
Err(format!("{label} failed with {status}"))
|
|
};
|
|
}
|
|
|
|
let started = Instant::now();
|
|
emit_command_progress(
|
|
progress,
|
|
label,
|
|
image_ref,
|
|
0,
|
|
NodeImageProgressEventKind::CommandStarted {
|
|
program: program.to_owned(),
|
|
args: args.to_vec(),
|
|
},
|
|
);
|
|
let mut child = match swactor_process::command_spawn(&mut Command::new(program)
|
|
.current_dir(root)
|
|
.args(&args)
|
|
.stdin(Stdio::null())
|
|
.stdout(Stdio::piped())
|
|
.stderr(Stdio::piped()))
|
|
{
|
|
Ok(child) => child,
|
|
Err(error) => {
|
|
emit_command_progress(
|
|
progress,
|
|
label,
|
|
image_ref,
|
|
started.elapsed().as_millis(),
|
|
NodeImageProgressEventKind::CommandExited {
|
|
status: format!("spawn error: {error}"),
|
|
code: None,
|
|
success: false,
|
|
},
|
|
);
|
|
return Err(format!("run {label}: {error}"));
|
|
}
|
|
};
|
|
|
|
let (tx, rx) = mpsc::channel();
|
|
let mut readers = Vec::new();
|
|
if let Some(stdout) = child.stdout.take() {
|
|
readers.push(spawn_line_reader(
|
|
stdout,
|
|
CommandOutputLine::Stdout,
|
|
tx.clone(),
|
|
));
|
|
}
|
|
if let Some(stderr) = child.stderr.take() {
|
|
readers.push(spawn_line_reader(
|
|
stderr,
|
|
CommandOutputLine::Stderr,
|
|
tx.clone(),
|
|
));
|
|
}
|
|
drop(tx);
|
|
|
|
let status = loop {
|
|
match swactor_process::child_try_wait(&mut child) {
|
|
Ok(Some(status)) => break status,
|
|
Ok(None) => {
|
|
drain_command_lines(&rx, progress, label, image_ref, started);
|
|
thread::sleep(Duration::from_millis(10));
|
|
}
|
|
Err(error) => {
|
|
drain_command_lines(&rx, progress, label, image_ref, started);
|
|
emit_command_progress(
|
|
progress,
|
|
label,
|
|
image_ref,
|
|
started.elapsed().as_millis(),
|
|
NodeImageProgressEventKind::CommandExited {
|
|
status: format!("wait error: {error}"),
|
|
code: None,
|
|
success: false,
|
|
},
|
|
);
|
|
return Err(format!("run {label}: {error}"));
|
|
}
|
|
}
|
|
};
|
|
for reader in readers {
|
|
let _ = reader.join();
|
|
}
|
|
drain_command_lines(&rx, progress, label, image_ref, started);
|
|
let status_text = status.to_string();
|
|
let success = status.success();
|
|
emit_command_progress(
|
|
progress,
|
|
label,
|
|
image_ref,
|
|
started.elapsed().as_millis(),
|
|
NodeImageProgressEventKind::CommandExited {
|
|
status: status_text.clone(),
|
|
code: status.code(),
|
|
success,
|
|
},
|
|
);
|
|
if success {
|
|
Ok(())
|
|
} else {
|
|
Err(format!("{label} failed with {status_text}"))
|
|
}
|
|
}
|
|
|
|
fn docker_image_exists(root: &Path, image_ref: &str) -> bool {
|
|
swactor_process::command_status(&mut Command::new("docker")
|
|
.current_dir(root)
|
|
.args(["image", "inspect", image_ref])
|
|
.stdin(Stdio::null())
|
|
.stdout(Stdio::null())
|
|
.stderr(Stdio::null()))
|
|
.map(|status| status.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
fn docker_image_labels(
|
|
root: &Path,
|
|
image_ref: &str,
|
|
) -> Result<Option<BTreeMap<String, String>>, String> {
|
|
let output = swactor_process::command_output(&mut Command::new("docker")
|
|
.current_dir(root)
|
|
.args([
|
|
"image",
|
|
"inspect",
|
|
"--format",
|
|
"{{ json .Config.Labels }}",
|
|
image_ref,
|
|
])
|
|
.stdin(Stdio::null()))
|
|
.map_err(|e| format!("inspect docker image {image_ref}: {e}"))?;
|
|
if !output.status.success() {
|
|
return Ok(None);
|
|
}
|
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
|
let labels: Option<BTreeMap<String, String>> = serde_json::from_str(stdout.trim())
|
|
.map_err(|e| format!("parse docker labels for {image_ref}: {e}"))?;
|
|
Ok(Some(labels.unwrap_or_default()))
|
|
}
|
|
|
|
fn docker_manifest_exists(root: &Path, image_ref: &str) -> bool {
|
|
swactor_process::command_status(&mut Command::new("docker")
|
|
.current_dir(root)
|
|
.args(["manifest", "inspect", image_ref])
|
|
.stdin(Stdio::null())
|
|
.stdout(Stdio::null())
|
|
.stderr(Stdio::null()))
|
|
.map(|status| status.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
fn docker_image_has_container(root: &Path, image_ref: &str) -> bool {
|
|
swactor_process::command_output(&mut Command::new("docker")
|
|
.current_dir(root)
|
|
.args([
|
|
"ps",
|
|
"-a",
|
|
"--filter",
|
|
&format!("ancestor={image_ref}"),
|
|
"--format",
|
|
"{{.ID}}",
|
|
])
|
|
.stdin(Stdio::null()))
|
|
.map(|output| output.status.success() && !output.stdout.is_empty())
|
|
.unwrap_or(true)
|
|
}
|
|
|
|
fn docker_image_tags(root: &Path, repository: &str) -> Result<Vec<(String, String)>, String> {
|
|
let output = swactor_process::command_output(&mut Command::new("docker")
|
|
.current_dir(root)
|
|
.args([
|
|
"image",
|
|
"ls",
|
|
"--format",
|
|
"{{.Repository}}\t{{.Tag}}",
|
|
repository,
|
|
])
|
|
.stdin(Stdio::null()))
|
|
.map_err(|error| format!("docker image ls failed: {error}"))?;
|
|
if !output.status.success() {
|
|
return Err(format!("docker image ls failed with {}", output.status));
|
|
}
|
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
|
Ok(stdout
|
|
.lines()
|
|
.filter_map(|line| {
|
|
let (repository, tag) = line.split_once('\t')?;
|
|
Some((repository.to_owned(), tag.to_owned()))
|
|
})
|
|
.collect())
|
|
}
|
|
|
|
fn docker_image_remove(root: &Path, image_ref: &str) -> Result<(), String> {
|
|
let status = swactor_process::command_status(&mut Command::new("docker")
|
|
.current_dir(root)
|
|
.args(["image", "rm", image_ref])
|
|
.stdin(Stdio::null())
|
|
.stdout(Stdio::null())
|
|
.stderr(Stdio::null()))
|
|
.map_err(|error| format!("docker image rm failed: {error}"))?;
|
|
if status.success() {
|
|
Ok(())
|
|
} else {
|
|
Err(format!("docker image rm failed with {status}"))
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug)]
|
|
struct ImageName {
|
|
repository: String,
|
|
requested_tag: Option<String>,
|
|
}
|
|
|
|
impl ImageName {
|
|
fn parse(raw: &str) -> Result<Self, String> {
|
|
let raw = raw.trim();
|
|
if raw.is_empty() {
|
|
return Err("node image must not be empty".to_owned());
|
|
}
|
|
if raw.contains('@') {
|
|
return Err(format!(
|
|
"node image {raw:?} uses a digest; use a repository/tag base for image preparation"
|
|
));
|
|
}
|
|
let last_slash = raw.rfind('/');
|
|
let last_colon = raw.rfind(':');
|
|
let has_tag = match (last_slash, last_colon) {
|
|
(_, None) => false,
|
|
(None, Some(_)) => true,
|
|
(Some(slash), Some(colon)) => colon > slash,
|
|
};
|
|
let (repository, requested_tag) = if has_tag {
|
|
let colon = last_colon.expect("has tag colon");
|
|
let repository = raw[..colon].to_owned();
|
|
let tag = raw[colon + 1..].to_owned();
|
|
if tag.is_empty() {
|
|
return Err(format!("node image {raw:?} has an empty tag"));
|
|
}
|
|
(repository, Some(tag))
|
|
} else {
|
|
(raw.to_owned(), None)
|
|
};
|
|
if repository.is_empty() {
|
|
return Err(format!("node image {raw:?} has an empty repository"));
|
|
}
|
|
Ok(Self {
|
|
repository,
|
|
|
|
requested_tag,
|
|
})
|
|
}
|
|
|
|
fn ref_for_tag(&self, tag: &str) -> String {
|
|
format!("{}:{tag}", self.repository)
|
|
}
|
|
}
|