From e8a44c366666dd3aea7ef063980fe6797024abdd Mon Sep 17 00:00:00 2001 From: Zachery Aaron Shores-Chmielewski Date: Wed, 15 Jul 2026 12:49:56 +0400 Subject: [PATCH] mid refactor to clean up all the unclear and badly distributed logic, add specs --- Cargo.lock | 11 + crates/mvp-system/Cargo.toml | 1 + crates/mvp-system/specs/mvp_chat.md | 758 +++++ crates/mvp-system/specs/orchestrator.md | 2730 ++++++++++++++++ crates/mvp-system/src/bin/mvp_chat.rs | 3027 ++++++++---------- crates/mvp-system/src/config.rs | 10 +- crates/mvp-system/tests/one_node_chat_e2e.rs | 3 +- xtask/src/main.rs | 2 +- 8 files changed, 4829 insertions(+), 1713 deletions(-) create mode 100644 crates/mvp-system/specs/mvp_chat.md create mode 100644 crates/mvp-system/specs/orchestrator.md diff --git a/Cargo.lock b/Cargo.lock index f7ba83f..47912b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2484,6 +2484,7 @@ dependencies = [ "parking_lot", "serde", "serde_json", + "signal-hook", "swactor", "swactor-transport", "swactor-vastai", @@ -4147,6 +4148,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + [[package]] name = "signal-hook-registry" version = "1.4.8" diff --git a/crates/mvp-system/Cargo.toml b/crates/mvp-system/Cargo.toml index 321d405..74c1928 100644 --- a/crates/mvp-system/Cargo.toml +++ b/crates/mvp-system/Cargo.toml @@ -27,6 +27,7 @@ toml = "0.8" [target.'cfg(target_os = "linux")'.dependencies] libc = "0.2" +signal-hook = "0.3" [dev-dependencies] iroh-relay = { version = "0.98", features = ["server", "test-utils"] } diff --git a/crates/mvp-system/specs/mvp_chat.md b/crates/mvp-system/specs/mvp_chat.md new file mode 100644 index 0000000..8278cb2 --- /dev/null +++ b/crates/mvp-system/specs/mvp_chat.md @@ -0,0 +1,758 @@ +# MVP Chat Wrapper Fixed Specification + +**Status:** draft target behavior for `mvp-chat`. + +This document describes the intended public contract. Implementation details that +remain in the source but were marked out of scope are not part of this fixed +contract. + +--- + +## 1. Purpose + +`mvp-chat` is the user-facing process that starts an MVP runtime and attaches an +interactive prompt session to it. + +The wrapper is responsible for: + +- accepting the approved public inputs; +- resolving provider and runtime launch configuration; +- preparing required local runtime artifacts through Cargo unless rebuilds are + explicitly skipped; +- starting the orchestrator through the approved orchestrator launch contract; +- waiting until the runtime can accept prompt requests; +- running the interactive prompt loop; +- notifying the runtime to shut down on normal exit or interruption; +- reporting errors clearly to the user. + +`mvp-chat` is not responsible for: + +- model inference quality; +- worker internals; +- node-image construction internals; +- orchestrator argument naming; +- provider API details beyond the inputs needed to request a provider-backed + runtime; +- dashboard rendering; +- non-Linux behavior. + +--- + +## 2. Supported Platform + +This specification covers Linux only. + +Linux signal handling, child-process lifecycle, Cargo artifact discovery, and +runtime shutdown semantics are the only supported platform behavior. Non-Linux +behavior is out of scope until explicitly specified. + +--- + +## 3. Public Input Surface + +`mvp-chat` accepts inputs only from the surfaces listed in this section. +Commented-out or implementation-only inputs from the earlier draft are pruned +from the public contract. + +### 3.1 Process Arguments + +Provider selectors: + +- `--process` +- `--docker` +- `--vastai` + +General flags: + +- `--config ` +- `--yes` +- `-y` +- `--pipeline-stages ` +- `--dump-logs` +- `--dump-logs=` +- `--cached-model` +- `--cached-model=` +- `--skip-rebuild` + +Any process argument that is not one of the listed flags or a required or +attached value for one of those flags is a configuration error. + +`--process`, `--docker`, and `--vastai` are mutually exclusive. Supplying more +than one provider selector is a configuration error. + +If no provider selector is supplied, the provider is `process`. + +`--pipeline-stages ` accepts a positive integer. Zero and invalid values +are configuration errors. + +`--dump-logs` writes the consolidated log stream to the default file +`mvp-chat.log` in the current working directory. + +`--dump-logs=` writes the same stream to ``. The `` value is +the literal string after `=`, may start with `-`, and may be relative or +absolute. Relative paths are resolved relative to the current working directory. +`--dump-logs=` is a configuration error. + +`--dump-logs ` is not accepted. Without `--dump-logs`, logs are not stored +in a file. + +`--cached-model` enables cached-model use and discovers a cached model from +`.model-cache/`. + +`--cached-model=` uses `` as the cached model file. The `` +value is the literal string after `=`, may start with `-`, and may be relative +or absolute. Relative paths are resolved relative to the current working +directory. `--cached-model=` is a configuration error. + +`--cached-model ` is not accepted. Without `--cached-model`, cached-model +use is disabled. + +`--skip-rebuild` prevents `mvp-chat` from invoking Cargo builds. If a required +runtime artifact is unavailable while rebuilds are skipped, preparation fails +with a clear error. + +### 3.2 Environment Variables + +The public configuration environment surface is limited to secret material. + +Accepted environment variable: + +- `VAST_API_KEY` + +`VAST_API_KEY` supplies the Vast.ai API key when the selected provider is +`vastai`. The value is trimmed before validation; an unset, empty, or +whitespace-only value is treated as missing. + +No other environment variable is part of the public `mvp-chat` configuration +contract. Normal inherited process environment, such as the environment used by +Cargo or child processes, is ordinary OS execution context rather than +`mvp-chat` configuration. + +### 3.3 Configuration File + +`mvp-chat` reads configuration from a TOML file. + +The config path is: + +- `--config `, when supplied; +- otherwise `.config/config.toml` relative to the current working directory. + +A supplied `--config ` is required to be a readable file and parse as +TOML. The default `.config/config.toml` is optional and is read only when it +exists as a file. If the default config file exists but cannot be read or parsed, +configuration fails; if the default path is absent or not a file, built-in +defaults are used. + +The fixed spec accepts only active behavior fields. Unused schema fields from the +earlier draft are pruned. Unknown top-level TOML tables and unknown fields inside +accepted tables are configuration errors. + +Accepted provider field: + +- `[provider].kind` + +`[provider].kind` is trimmed before validation. After trimming, accepted values +are case-sensitive and exactly `process`, `docker`, or `vastai`. + +Accepted runtime fields: + +- `[runtime].pipeline_stages` +- `[runtime].max_tokens` + +`[runtime].max_tokens` sets the maximum number of tokens requested for each +prompt submission. It must be a positive integer. + +Accepted observability fields: + +- `[observability].dump_logs` +- `[observability].dump_log_path` + +Accepted image fields: + +- `[image].node` +- `[image].tag` + +`[image].node` names the desired worker node image. + +For provider `docker`, `[image].node` may name a local or remote image. + +For provider `vastai`, `[image].node` must name a remote registry image that the +provider can pull. + +`[image].tag` may provide an additional human-selected tag or alias for an image +prepared by `mvp-chat`. It does not replace the resolved image reference used for +freshness or content identity. + +Accepted Vast.ai fields: + +- `[vastai].relay_url` +- `[vastai].bootstrap_command` +- `[vastai].gpu_name` +- `[vastai].min_gpu_ram_mb` +- `[vastai].min_down_mbps` +- `[vastai].min_up_mbps` +- `[vastai].min_reliability` +- `[vastai].require_verified` +- `[vastai].disk_gb` +- `[vastai].onstart` +- `[vastai].ssh_identity` + +`[vastai].ssh_identity` is a filesystem path to an SSH private-key identity file +used for provider bootstrap access. It is configuration, not a secret-value +environment variable. + +### 3.4 Standard Input + +Accepted standard input: + +- interactive prompt lines; +- EOF or input disconnection; +- standard-input read error during prompt input; +- Vast.ai rental approval response when approval is required. + +Prompt lines drive the prompt loop. EOF, input disconnection, and standard-input +read errors during prompt input end the prompt loop cleanly and trigger normal +runtime cleanup. There are no prompt text commands for exiting. + +While waiting for prompt input, `mvp-chat` must still respond to EOF, input +disconnection, standard-input read errors, `SIGINT`, and `SIGTERM`. The exact +input-read mechanism is an implementation detail. + +### 3.5 Signals + +Accepted Linux signals: + +- `SIGINT` +- `SIGTERM` + +Both request controlled shutdown. + +### 3.6 Filesystem Inputs + +Filesystem inputs: + +- current working directory; +- config file selected by Section 3.3; +- `.model-cache/` when `--cached-model` is supplied; +- cached model file path when `--cached-model=` is supplied; +- Cargo workspace files needed by Cargo to build or locate runtime artifacts; +- Cargo target artifacts for the orchestrator and worker; +- optional Vast.ai SSH identity file path from config; +- optional dump-log output path parent directories. + +Additional filesystem inputs for image preparation: + +- node image Dockerfile; +- node image build context; +- worker binary artifact included in the image; +- source files used to determine image freshness; +- local Docker image metadata. + +The current working directory is the root for relative paths. + +When `--cached-model` is supplied, `mvp-chat` reads the direct files in +`.model-cache/`, filters for model files accepted by the runtime, sorts the +remaining files alphabetically by filename, and selects the first file. Failure +to read `.model-cache/`, read a direct directory entry, or stat a direct entry is +a preparation error. Direct entries that can be statted but do not match the +cached-model predicate are ignored. If no usable cached model is present, +preparation fails with a clear error. + +When `--cached-model=` is supplied, `mvp-chat` validates that the path is a +usable cached model file. A usable cached model path must resolve to a regular +file whose extension is `.gguf`, matched case-insensitively. If it is missing, +not a regular file, or not accepted by this predicate, preparation fails with a +clear error. Accepted cached-model paths are canonicalized before being included +in the runtime launch request. + +### 3.7 Network and Runtime Inputs + +Network inputs: + +- prompt RPC readiness outcome; +- prompt RPC stream records; +- orchestrator progress events received through the progress datastream. + +Additional provider/image inputs: + +- Docker daemon responses when checking local image availability; +- registry responses when checking remote image availability; +- registry responses when pushing images for remote providers. + +Prompt RPC stream records are newline-delimited prompt events: + +- text delta; +- request completion; +- request fault. + +The detailed progress-event channel schema is not specified here. This spec only +requires that `mvp-chat` receive enough progress information to present the +user-facing progress states defined in Section 7. + +### 3.8 Child-Process Inputs + +Child-process inputs observed by `mvp-chat`: + +- orchestrator process start success or failure; +- orchestrator process readiness result; +- orchestrator process exit before readiness; +- failure to notify or wait for orchestrator shutdown. + +The OS process APIs used to observe these states are implementation details. The +observable contract is the resulting success, failure, or controlled shutdown. + +--- + +## 4. Provider Selection + +`mvp-chat` has no public runtime-profile concept. The public provider choices +are: + +- `process` +- `docker` +- `vastai` + +Provider resolution order: + +1. CLI provider selector. +2. TOML `[provider].kind`. +3. default `process`. + +The accepted provider values are exactly: + +- `process` +- `docker` +- `vastai` + +Compatibility aliases may exist in implementation, but they are not part of the +fixed public contract. + +If `vastai` is selected, required Vast.ai configuration must be present before +offer preview, approval, or launch. Missing required Vast.ai configuration is a +configuration error. + +--- + +## 5. Configuration Resolution + +Configuration is resolved from: + +- process arguments; +- TOML configuration; +- approved secret environment variables; +- fixed defaults. + +Process arguments override TOML where both define the same behavior. + +The only approved environment override is `VAST_API_KEY` for the Vast.ai API key. +Other configuration must come from process arguments, TOML, fixed defaults, or +filesystem discovery. + +Default values: + +- provider: `process`; +- pipeline stages: `1`; +- max tokens: `64`; +- dump logs: disabled; +- cached model: disabled unless `--cached-model` is supplied; +- rebuild: enabled unless `--skip-rebuild` is supplied. + +Invalid values must fail before runtime preparation begins. + +`[runtime].max_tokens` must be greater than zero. Zero and invalid values are +configuration errors detected before runtime preparation. + +For provider `process`, no node image is required. + +For provider `docker`, an image reference is required. It may be local or remote. + +For provider `vastai`, an image reference is required and must be a remote +registry image. + +If a provider requires an image and no valid image reference is configured, +configuration fails before runtime preparation. + +--- + +## 6. Runtime and Image Artifact Preparation + +### 6.1 Cargo Runtime Artifacts + +`mvp-chat` obtains the orchestrator and worker artifacts through Cargo. + +The wrapper must not infer the orchestrator or worker path by changing the file +name of the current executable. + +The current working directory is the artifact root and must be available. The +default orchestrator artifact path is `target/debug/mvp-orchestrator` under that +directory. The default worker artifact path is `target/debug/mvp-worker-node` +under that directory. No fallback artifact root is defined. + +Unless `--skip-rebuild` is supplied, `mvp-chat` may invoke Cargo to make required +artifacts available. + +Approved Cargo builds: + +- `cargo build --quiet -p mvp-system --bin mvp-orchestrator` +- `cargo build --quiet -p mvp-system --bin mvp-worker-node` + +When `--skip-rebuild` is supplied: + +- `mvp-chat` must not invoke Cargo builds; +- required Cargo artifacts must already be available; +- missing Cargo artifacts are preparation errors. + +Worker binary behavior mirrors orchestrator binary behavior: both are resolved +through Cargo artifacts, both honor `--skip-rebuild`, and both fail clearly when +required artifacts are unavailable. + +### 6.2 Node Image Preparation + +Node image preparation applies only to provider-backed runtimes: + +- `docker` +- `vastai` + +Provider `process` does not require a node image. + +For provider-backed runtimes, `mvp-chat` performs node image resolution before +launching the orchestrator. Node image resolution consumes: + +- the selected provider; +- `[image].node`; +- optional `[image].tag`; +- rebuild policy from `--skip-rebuild`; +- the worker binary artifact selected for this run; +- the approved node-image Dockerfile and build context; +- source files and metadata used to determine image freshness; +- Docker daemon observations for local images; +- registry observations for remote images. + +Node image resolution produces the resolved image reference included in the +orchestrator launch request. + +For provider `docker`, the resolved image must be runnable by the local Docker +daemon. The image may be local or remote. + +For provider `vastai`, the resolved image must be pullable by the remote +provider and must include a registry/repository namespace. Local-only image names +are invalid. + +When rebuilds are enabled, `mvp-chat` must determine whether the requested image +is already acceptable for the selected provider and current runtime inputs. If no +acceptable image is available, `mvp-chat` may build, tag, push, and validate an +image as required by the selected provider. + +An acceptable prepared image is one that: + +- is usable by the selected provider; +- was prepared from the approved node-image Dockerfile and build context; +- includes the selected worker binary artifact; +- is not stale with respect to the freshness inputs used by the + image-preparation contract; +- has any configured `[image].tag` alias applied when applicable. + +When `--skip-rebuild` is supplied, `mvp-chat` must not build, tag, or push +images. It must use only existing image artifacts and fail clearly if the +required image is missing, stale, unavailable, or unsuitable for the selected +provider. + +The exact freshness algorithm, metadata format, Docker commands, cache policy, +and registry authentication mechanics are owned by the node-image preparation +contract. + +### 6.3 Cached Models and Images + +Cached model selection is independent from node image preparation unless an +approved image-preparation contract explicitly says otherwise. + +By default, `mvp-chat` treats cached models as runtime inputs, not as image +contents. It must not silently bake cached models into prepared images. + +--- + +## 7. Progress, Logs, and Datastream + +Normal runtime logs are consolidated into one `mvp-chat` log stream. + +Without `--dump-logs`, the stream is not stored in a file. + +With `--dump-logs`, the stream is written to the default file defined in Section +3.1. + +With `--dump-logs=`, the stream is written to the specified path according +to the path parsing rules in Section 3.1. + +`mvp-chat` must not create hidden startup archive files as part of the public +contract. + +Progress observation uses datastream, not archive-file polling. + +The intended topology is: + +- the orchestrator exposes or publishes runtime progress; +- `mvp-chat` subscribes to orchestrator progress; +- `mvp-chat` exposes a process datastream endpoint for dashboard and user-facing + observers. + +Detailed progress channel names and payload schemas are out of scope. They belong +in a datastream/progress contract. + +The user-facing progress model must eventually define visible transitions for +runtime startup. Until that model is approved, this spec only fixes prompt-loop +output in Section 10 and keeps non-prompt progress output deferred. + +--- + +## 8. Vast.ai Behavior + +When provider is not `vastai`, Vast.ai config and approval are not used. + +When provider is `vastai`, required configuration must be present before any +offer preview or launch. + +Required Vast.ai inputs: + +- API key from `VAST_API_KEY`; +- relay URL from config; +- node image reference from config; +- bootstrap command when required by the provider contract. + +Optional Vast.ai selection inputs: + +- GPU name; +- minimum GPU RAM; +- minimum downlink bandwidth; +- minimum uplink bandwidth; +- minimum reliability; +- verified-host requirement; +- disk size; +- onstart command; +- SSH identity file path. + +If approval is required and `--yes` is not supplied, `mvp-chat` asks the user for +approval through the terminal. Only `y` and `yes`, after trimming and +case-folding, approve the rental. Any other answer declines. + +If `--yes` or `-y` is supplied, approval is accepted non-interactively after +required configuration is validated. + +If approval is required but standard input is not interactive, `mvp-chat` fails +unless `--yes` or `-y` is supplied. + +--- + +## 9. Orchestrator Launch and Shutdown + +Exact orchestrator argv is out of scope until the orchestrator launch contract is +specified. + +`mvp-chat` is responsible for handing the resolved runtime request to the +orchestrator through that approved launch contract. + +The semantic launch request must include, as applicable: + +- selected provider; +- resolved node image reference for provider-backed runtimes; +- pipeline stage count; +- cached model selection result; +- dump-log configuration; +- provider-specific runtime configuration; +- prompt RPC connection information required by the orchestrator contract; +- progress datastream connection information required by the orchestrator + contract. + +The orchestrator launch contract must define the prompt RPC address shared by +the orchestrator and `mvp-chat`. The default prompt RPC address is +`127.0.0.1:19777` unless the orchestrator launch contract provides another +address. + +The resolved node image reference is the image the orchestrator must use for the +provider-backed node. Exact argv or wire encoding remains owned by the +orchestrator launch contract. + +The wrapper starts the orchestrator as a child runtime process or through the +approved successor mechanism. + +On shutdown, `mvp-chat` must notify the orchestrator process to stop. This spec +does not expose stdin or any specific control string as the normative shutdown +mechanism. The shutdown mechanism is owned by the orchestrator contract. + +Shutdown must be idempotent from the user's perspective. Normal prompt exit, +input EOF, startup interruption, and signal interruption must not leave the +runtime running when `mvp-chat` can notify it. + +--- + +## 10. Prompt Loop + +The prompt loop accepts user prompt lines from standard input. + +For each cycle, `mvp-chat` must: + +- display a prompt marker; +- read one line of input; +- remove trailing whitespace from the input line before prompt handling, while + preserving leading whitespace; +- exit cleanly for EOF, input disconnection, or standard-input read error; +- ignore prompts that are empty after whitespace trimming; +- submit non-empty prompts to prompt RPC; +- display that decoding has started; +- stream response text as prompt RPC events arrive; +- return to the prompt marker after completion or prompt fault. + +Prompt RPC submissions carry: + +- request id; +- prompt text; +- max token limit resolved from `mvp-chat` configuration. + +`mvp-chat` submits at most one prompt at a time on its prompt RPC connection. It +waits for a terminal `Done` or `Fault` event before submitting the next prompt. +Prompt RPC guarantees that response events on the connection belong to the active +request and arrive in order. + +The prompt-loop output states are: + +- waiting for prompt; +- prompt submitted; +- decoding; +- streaming response; +- request completed; +- request faulted; +- prompt loop exited. + +Prompt-loop user output goes to standard output unless it is an actual wrapper +error. Expected model faults are prompt-loop results, not wrapper diagnostics. + +A fixed TCP read timeout is not part of the contract. The implementation must +remain interruptible, but this spec does not require a specific timeout-based +mechanism. + +--- + +## 11. Public Output Surface + +### 11.1 Exit Codes + +Exit code `0` means clean completion or controlled interrupted shutdown. + +Exit code `1` means configuration failure, preparation failure, startup failure, +prompt RPC failure, or another wrapper error. + +### 11.2 Standard Output + +Standard output is for expected user-facing behavior. + +Standard output includes: + +- prompt marker; +- decoding marker; +- response prefix; +- response text; +- prompt-loop completion formatting; +- expected prompt fault display; +- Vast.ai approval prompt when interactive approval is required. + +Non-prompt startup progress output is deferred until the progress event model is +approved. + +### 11.3 Standard Error + +Standard error is for actual wrapper errors and exceptional diagnostics. + +Standard error must not be used for ordinary status messages such as successful +provider selection, normal build status, normal cached-model selection, or normal +prompt-loop events. + +Errors must be clear enough for the user to identify the failed input or failed +runtime phase. + +Image preparation errors must be displayed clearly when image preparation fails. + +Image-preparation errors include: + +- missing required image reference; +- invalid image reference; +- required rebuild skipped; +- local image unavailable; +- remote image unavailable; +- image build failure; +- image tag failure; +- image push failure. + +### 11.4 Filesystem Outputs + +Filesystem outputs are limited to: + +- Cargo build artifacts when rebuilds are enabled; +- dump-log file when `--dump-logs` is supplied; +- local Docker image layers when image rebuilds are allowed; +- local Docker image tags or aliases when image rebuilds are allowed; +- image build cache entries when image rebuilds are allowed; +- provider/runtime artifacts owned by external contracts, if those contracts are + invoked. + +`mvp-chat` MUST NOT create unspecified filesystem outputs. + +### 11.5 Network Outputs + +Network-visible outputs: + +- prompt RPC connection attempts; +- prompt RPC prompt submissions; +- `mvp-chat` datastream endpoint for dashboard/user observers when progress + observation is active. + +Additional network-visible outputs when preparing remote images: + +- registry manifest checks; +- image layer uploads; +- image manifest or tag pushes. + +Prompt RPC submissions are newline-delimited request messages according to the +prompt RPC contract. The prompt RPC contract owns the exact wire schema. + +### 11.6 Child Runtime Outputs + +Outputs to the child runtime are limited to the approved orchestrator launch and +shutdown contracts. + +This spec does not define exact argv names, stdin control strings, or private +orchestrator flags. + +--- + +## 12. Error Handling + +Configuration errors must be detected before runtime preparation where possible. + +Preparation errors must identify the missing artifact, invalid file, failed Cargo +operation, or invalid provider configuration. + +Startup errors must identify the failed startup phase when progress information +is available. + +Prompt RPC errors must identify whether connection, write, read, protocol, or +remote closure failed. + +Controlled shutdown is not an error. + +Errors are displayed clearly to the user and cause nonzero exit unless the error +occurs during a controlled shutdown path defined as successful by this spec. + +--- + +## 13. Out of Scope + +Out of scope for this document: + +- path display formatting as a standalone contract; +- exact orchestrator argv; +- detailed datastream channel schemas; +- non-Linux support; +- Dockerfile contents; +- base-image implementation details; +- registry authentication UX beyond clear preparation errors; +- image optimization policy; +- image garbage-collection policy; diff --git a/crates/mvp-system/specs/orchestrator.md b/crates/mvp-system/specs/orchestrator.md new file mode 100644 index 0000000..beddf4e --- /dev/null +++ b/crates/mvp-system/specs/orchestrator.md @@ -0,0 +1,2730 @@ +# MVP Orchestrator Fixed Specification + +**Status:** draft outline for `mvp-orchestrator`. + +This document will define the intended public and runtime contract for the MVP orchestrator. Section contents are intentionally left for section-by-section review. + +--- + +## 1. Purpose and Contract Boundary + +This specification defines the observable contract of `mvp-orchestrator`: what it accepts, what it emits, how it moves a runtime from launch to shutdown, and what behavior callers may rely on. + +The orchestrator is responsible for turning a resolved launch request into a running MVP worker runtime. That responsibility includes: + +- Resolving runtime configuration from fixed defaults, optional TOML config, environment variables, and process arguments. + +- Preparing provider prerequisites that must exist before workers can start, including Vast.ai SSH identity registration when the Vast.ai provider is selected. + +- Initializing the local orchestration runtime: Tokio, Iroh transport, the Swactor distribution stack, actor codecs, local reply actors, datastream production, and optional dashboard publication. + +- Building the execution shape for the run. For direct execution this is a single worker stage. For planned execution this is a pipeline run plan derived from cached or locally inspectable model metadata. + +- Provisioning worker nodes through the selected provider. The orchestrator supplies each worker with its image, logical node id, stage index, environment, mounts, coordinator endpoint, and orchestrator actor address. + +- Driving runtime readiness. The orchestrator waits for worker runtime-ready reports, SWIM membership, actor route ownership, runtime-ready acknowledgements, stage provisioning, and weight-loaded reports before accepting prompt work. + +- Serving prompt requests through prompt RPC. In direct mode it forwards prompt requests to the worker node agent. In pipeline mode it coordinates tokenizer encode/decode work and token-edge traffic between the orchestrator and pipeline stages. + +- Emitting runtime observations. Bootstrap progress, prompt progress, provisioning events, provider logs, worker stdout/stderr, orchestrator stdio capture, SWIM transitions, stage-route checks, node datastream frames, dashboard frames, and optional frame-archive records are all orchestrator outputs. + +- Shutting down controlled runtimes. On stop, shutdown, prompt-serving completion, or fatal error, the orchestrator stops provider-owned worker nodes before exiting when it has enough state to do so. + +The orchestrator is not responsible for model quality, worker-node internals, tokenizer implementation details, provider implementation internals, Docker image construction, Vast.ai marketplace behavior, dashboard rendering semantics, or the interactive user-facing prompt loop owned by `mvp-chat`. + +--- + +## 2. Input Channels + +The orchestrator accepts input through configuration files, environment variables, process arguments, filesystem paths, prompt/control RPC, Swactor inbox delivery, provisioning datastream observations, and Iroh datastream connections. + +Configuration inputs describe the requested runtime. Runtime inputs report what workers, providers, and network peers do after launch. Prompt text and RPC control inputs enter only through prompt/control RPC. Downstream shutdown propagation uses Swactor runtime and actor delivery, not standard input. + +### 2.1 Configuration File + +The orchestrator reads configuration from the path provided by `--runtime-config ` when that argument is present. Otherwise, it reads `.config/config.toml` if it exists. + +The config file is optional when the default path is used and absent. A provided config path must exist, be readable, and contain valid TOML. + +Accepted TOML configuration: + +- `[runtime]`: `profile`, `run_id`, `node_id`, `stage_index`, `layer_end_exclusive`, `pipeline_stages` +- `[provider]`: `kind` +- `[image]`: `node` +- `[relay]`: `mode`, `url` +- `[prompt]`: `rpc_addr` +- `[model]`: `id`, `gguf_local_path`, `gguf_repo`, `gguf_file`, `gguf_revision`, `tokenizer_local_path`, `max_context` +- `[docker]`: `gpus` +- `[observability]`: `datastream_frame_log` +- `[vastai]`: `image`, `api_key`, `bootstrap_command`, `disk_gb`, `ssh_user`, `confirm_lease`, `onstart`, `ssh_identity`, `gpu_name`, `min_gpu_ram_mb`, `min_down_mbps`, `min_up_mbps`, `min_reliability`, `require_verified`, `poll_interval_secs` + +TOML values are configuration inputs, not protocol messages. Unsupported values, malformed values, or invalid combinations are configuration errors. + +### 2.2 Environment Variables + +Environment variables provide secrets and tokens that should not be written into configuration files. Environment variables are not a general runtime configuration surface: runtime identity, provider selection, worker image, relay configuration, prompt serving, observability, model paths, and pipeline shape are configured through TOML or the accepted process arguments. + +Accepted environment inputs: + +- provider credentials: + - `VASTAI_API_KEY` + +- model and tokenizer credentials: + - `HF_TOKEN` + +`VASTAI_API_KEY` supplies the Vast.ai API key when the Vast.ai provider is selected. If both `[vastai].api_key` and `VASTAI_API_KEY` are present, `VASTAI_API_KEY` is used. + +`HF_TOKEN` is passed only to runtime paths that need Hugging Face authentication for model or tokenizer access. + +Unsupported `MVP_*` environment variables are not accepted orchestrator configuration inputs. + +Environment values are parsed only when the selected runtime path needs them. Missing required secret values, malformed secret values, or unsupported environment configuration are configuration errors. + +### 2.3 Process Arguments + +Process arguments are a small launch-time convenience surface. Values not listed here must be configured through TOML. + +Accepted process arguments: + +- configuration: + - `--runtime-config ` + +- provider: + - `--provider ` + +- worker launch: + - `--worker-bin ` + +- runtime shape: + - `--pipeline-stages ` + +- Vast.ai: + - `--vastai-ssh-identity ` + +`--runtime-config` selects the TOML file used for this invocation. Relative paths are resolved against the current working directory. + +The other accepted process arguments override the corresponding TOML values for this invocation only. + +No process argument is accepted for runtime identity, model or tokenizer selection, relay settings, prompt serving, observability, Docker GPU settings, Vast.ai API keys, Vast.ai lease parameters, or Vast.ai search parameters. + +Unknown arguments, missing argument values, unreadable config paths, invalid TOML, invalid numbers, unsupported provider names, unsupported runtime profiles, and invalid paths required by the selected runtime are configuration errors. + + +### 2.4 Filesystem Inputs + +Filesystem inputs are paths that the orchestrator reads or validates while preparing the runtime: + +- default or configured TOML configuration path +- current executable path, used to derive the default worker binary path +- configured worker binary path +- configured cached model host path +- default pipeline cached model path +- configured local GGUF path +- configured local tokenizer path +- configured Vast.ai SSH identity path +- current working directory for relative paths +- model cache paths exposed to workers + +A missing file is an error only when the selected runtime path requires that file. + +### 2.5 Prompt and Control RPC Input + +Prompt/control RPC input is newline-delimited JSON over TCP. + +Prompt request datatype: + +```text +SubmitPrompt { + request_id: u64, + prompt_text: String, + max_tokens: Option, +} +``` + +`max_tokens` is optional. When omitted or set to `0`, the orchestrator does not impose an arbitrary generated-token cap. Generation still ends on model EOS, context/window exhaustion, worker fault, client disconnect, shutdown, or runtime/model limits. When positive, `max_tokens` is the maximum number of generated completion tokens the orchestrator permits for that request. + +The RPC channel may also carry a shutdown control request. After accepted, downstream shutdown signals sent to actors, providers, and worker-runtime components are delivered through the Swactor runtime and actor planes. + +A valid request is enqueued as prompt work. A malformed request is a prompt RPC protocol error for that connection. + +A prompt RPC TCP connection supports at most one in-flight `SubmitPrompt` request +at a time. The client must not submit another prompt on the same connection until +the previous request has produced a terminal `Done` or `Fault` event. Submitting +a second prompt before the active prompt reaches a terminal event is a prompt RPC +protocol error for that connection. + +### 2.6 Actor Message Input Channel + +Actor-message input reaches the orchestrator through two surfaces: + +- **Iroh actor plane surface**: remote actor messages arrive through the Iroh actor bridge and are handled by the Swactor runtime and registered actor routes. + +- **Swactor inbox surface**: process-visible actor messages are delivered into local Swactor inboxes owned and drained by the orchestrator process. + +This section defines the input surfaces only. Actor message schemas belong in the `Actors` section. + +### 2.7 Provisioning Datastream Observations + +Provisioner actor subsystems publish node lifecycle and log observations as datastream records. The orchestrator receives these records through datastream ingestion, not through a provider plugin control surface. + +Observation payload datatype: + +```text +PluginObservation +``` + +Accepted datastream observation kinds: + +- worker stdout line +- worker stderr line +- provider log line +- provider datastream frame +- node exited +- provisioning failed + +Node exit and provisioning-failed observations in this channel are datastream records only. They do not drive lifecycle decisions by themselves. Startup, prompt-serving, and shutdown failures are driven by actor-plane reports, provider actor results, prompt/control RPC shutdown, or runtime state checks. + +### 2.8 Datastream Input + +Worker datastream input arrives over the datastream ALPN. + +Accepted datastream inputs: + +- stream header +- channel declaration +- frame +- stream end + +Frame payloads are opaque bytes at this boundary. The orchestrator records channel name, channel id, stream id, frame position, and payload, then forwards the frame to configured sinks. + +--- + +## 3. Output Channels + +The orchestrator emits output through process status, stdio log streams, provisional prompt output, Swactor actor delivery, Iroh transport, datastream frames, optional dashboard publication, and optional frame archives. + +This section defines output channels only. Detailed actor message schemas belong in the `Actors` section. Detailed datastream record contents belong in `Datastream and Logs`. + +### 3.1 Process Exit Status + +The orchestrator exits with process status: + +```text +0 +``` + +for successful completion or controlled shutdown. + +The orchestrator exits with process status: + +```text +1 +``` + +for configuration failure, startup failure, provisioning failure, prompt-serving failure, provider-stop failure, or any other fatal orchestrator error. + +### 3.2 Standard Output and Standard Error + +Standard output and standard error are log streams only. They are not user-facing status protocols, report channels, control channels, signaling channels, or fatal-error reporting contracts. + +On Linux, after stdio capture is installed, orchestrator stdout and stderr lines are redirected into the orchestrator log/datastream path as log observations. Before capture is installed, any stdout or stderr bytes are still logs only and are not part of the orchestrator contract. + +Provisioner and provider logs are owned by their managing actor subsystems and enter the orchestrator datastream through those subsystem publications. + +Structured runtime state, lifecycle progress, failures, prompt progress, worker logs, provider logs, and datastream frames are emitted through datastream, RPC, or actor-plane outputs, not through standard output or standard error. + +### 3.3 Prompt RPC Output + +Prompt RPC output is provisional and intentionally not specified here. + +The prompt interface is being moved from RPC/TCP response streams to Swactor messaging. This section will be rewritten with the Swactor prompt output contract when that migration is specified. + +### 3.4 Swactor Actor Output Channel + +The orchestrator has one actor-output surface: its owned local Swactor runtime. + +All actor messages emitted by the orchestrator process are submitted to that runtime. Swactor owns routing. If the destination is local, the runtime routes locally. If the destination is remote, the runtime uses the Iroh actor bridge. + +The orchestrator does not maintain separate local and remote actor outboxes. + +Process-owned Swactor inboxes may be supplied as reply addresses for reports, prompt events, tokenizer events, or other actor responses. Those inboxes are input queues drained by the orchestrator process; they are not a second actor-output channel. + +Outgoing actor traffic includes: + +- runtime-ready acknowledgements to worker node agents; +- stage provisioning commands; +- direct prompt inference commands; +- tokenizer encode requests; +- tokenizer decode requests; +- datastream subscription requests; +- shutdown or stop commands to managed runtime actors. + +This section defines the actor-output surface only. Message schemas belong in the `Actors` section. + + + + +### 3.9 Datastream Output + +The orchestrator datastream is the process-owned observation stream for a single orchestrator run. + +The orchestrator datastream is responsible for collecting: + +- records produced directly by the orchestrator process; +- captured orchestrator stdout and stderr, converted into log records; +- datastreams published by provider/provisioner actor subsystems; +- datastreams published by provisioned worker processes and managed nodes. + +Datastream records are observational. They do not carry lifecycle authority, actor commands, provider control, shutdown control, prompt control, or readiness gates. Lifecycle decisions are driven through the Swactor actor/control plane and runtime state checks. + +The orchestrator directly owns these datastream channels: + +- `mvp.orch.bootstrap` + - Orchestrator startup, configuration resolution, runtime initialization, readiness waiting, prompt-service availability, shutdown progress, and process-exit observations. + +- `mvp.orch.prompt` + - Prompt-serving observations owned by the orchestrator process, such as prompt accepted, prompt dispatched, prompt completed, prompt faulted, or prompt cancelled. + +- `mvp.orch.stdio.stdout` + - Captured stdout lines emitted by the orchestrator process after stdio capture is installed. + +- `mvp.orch.stdio.stderr` + - Captured stderr lines emitted by the orchestrator process after stdio capture is installed. + +- `mvp.swim.membership` + - SWIM membership observations visible to the orchestrator runtime. + +- `mvp.orch.stage_route` + - Stage route and actor-route ownership observations used to explain readiness and routing state. + +The orchestrator datastream also collects downstream datastream publications. These channels are not directly authored by the orchestrator process, but they must be published into the orchestrator datastream for observation: + +- provisioning lifecycle event channels published by provider/provisioner actor subsystems; +- provisioning log channels published by provider/provisioner actor subsystems; +- worker log channels published by provisioned processes or managed nodes; +- worker-defined datastream channels declared by provisioned processes or managed nodes; +- provider-defined diagnostic channels published by provider/provisioner actor subsystems. + +Collected downstream frames preserve their source identity, stream identity, channel name, channel id, frame position, and payload bytes. The orchestrator may add collection metadata, but it must not rewrite downstream payloads into prompt output, lifecycle control, or actor messages. + +Provisioning events and logs are managed by provider/provisioner actor subsystems. The orchestrator datastream is responsible for collecting and publishing those records as observations, not for interpreting them as control signals. + +Worker and node datastream frames collected by the orchestrator are forwarded to configured observability sinks, such as datastream subscribers, dashboard subscribers, or frame archives. + + +### 3.11 Filesystem Output + +Filesystem output is optional and exists only when datastream frame logging is configured. + +When enabled, the orchestrator starts a file-log sink task. That task subscribes to the orchestrator datastream endpoint and appends received datastream frames to the configured file. + +The file-log sink is a datastream subscriber. It does not own lifecycle state, does not emit control signals, and does not change runtime behavior when absent. + +Frame archive output is JSON lines appended to the configured path. + +Frame archive records include: + +```text +FrameArchiveRecord { + arrival_seq, + source, + stream, + channel, + channel_id, + position, + payload, +} +``` + +The file-log sink may create parent directories for the configured frame archive path. + +Without datastream frame logging, the file-log sink is not started and no frame archive is created. + + +--- + +## 4. Runtime Lifecycle + +The orchestrator lifecycle is a single owned run: resolve configuration, initialize local runtime services, provision workers, wait for readiness, serve prompts, stop workers, and exit. + +Lifecycle summary: + +```text +configure +-> prepare provider prerequisites +-> initialize observability +-> initialize local runtime +-> build optional run plan +-> provision workers +-> wait for runtime readiness +-> acknowledge runtime readiness +-> provision stages +-> wait for weights loaded +-> bind prompt RPC +-> serve prompts +-> stop provider-owned workers +-> exit +``` + +A fatal error may terminate the lifecycle at any phase. Once worker handles are owned by the provisioned-cluster guard, dropping the guard attempts to stop all remaining workers. + +### 4.1 Configuration Phase + +The orchestrator first resolves its effective configuration from defaults, optional TOML, environment variables, and process arguments. + +This phase also validates configuration that must be known before runtime setup, including provider kind, prompt RPC bind address, pipeline stage count, cached model path, relay configuration, and Vast.ai-specific requirements. + +If the selected provider is Vast.ai, the orchestrator prepares the SSH identity before installing stdio capture. Preparation includes resolving the identity path, deriving the public key, ensuring the key is registered with the Vast.ai account, and recording the prepared identity in the resolved provider config. + +Failure in this phase exits before workers are started. + +### 4.2 Observability Startup Phase + +The orchestrator installs its own stdout/stderr capture and creates the orchestrator datastream. + +The first bootstrap records describe the resolved configuration and runtime setup progress. If frame archive logging is configured, the frame archive is opened during datastream initialization. + +From this point forward, ordinary orchestrator stdout and stderr lines are captured as log records rather than treated as terminal UI. + +### 4.3 Local Runtime Initialization Phase + +The orchestrator initializes the local runtime services required to coordinate the cluster: + +- Tokio runtime; +- Iroh driver; +- distribution runtime stack; +- actor codecs; +- actor bridge; +- datastream collector; +- optional dashboard support; +- local Swactor inboxes for orchestrator reports, prompt replies, and tokenizer replies; +- local orchestrator actor; +- stop-listener thread for standard-input control. + +The Iroh driver and Swactor runtime are pumped together throughout later phases. Actor messages, transport traffic, SWIM state, route ownership, datastream connections, and prompt work only progress while the orchestrator pump loop is running. + +### 4.4 Run Planning Phase + +The orchestrator enters planned execution when either cached-model execution is selected or more than one pipeline stage is requested. + +In planned execution, the orchestrator reads locally inspectable GGUF metadata and builds a run plan. The plan determines stage placement, layer ranges, token edges, activation edges, object sizes, and ring sizes. + +In direct execution, no run plan is built. The runtime is treated as a single worker stage. + +A planning failure exits before workers are started. + +### 4.5 Worker Provisioning Phase + +The orchestrator builds a provider-specific provisioner and computes one `NodeProvisionSpec` per worker. + +Direct execution provisions one worker. + +Planned pipeline execution provisions one worker per planned stage. + +For each worker, the orchestrator emits provider-start progress, calls the provider plugin, captures provider observations, and stores the returned worker handle. If one worker fails to start, workers already started in that provisioning attempt are stopped before the error is returned. + +Once all workers are started, the provisioned-cluster guard owns the worker handles. + +### 4.6 Runtime Readiness Phase + +After workers are started, the orchestrator waits for runtime readiness. + +A worker is not considered ready when it merely reports its endpoint and actor addresses. The readiness barrier requires: + +- a matching runtime-ready report for the active run; +- SWIM membership showing the worker node as alive; +- actor route ownership showing the node actor is reachable through that worker; +- no provider failure or premature node exit. + +For planned pipeline execution, every expected stage worker must pass this readiness barrier. + +For direct execution, the single expected worker must pass this readiness barrier. + +### 4.7 Runtime-Ready Acknowledgement Phase + +After readiness barriers pass, the orchestrator acknowledges each worker’s runtime-ready report. + +The acknowledgement is sent to the worker node agent. The orchestrator also attempts to subscribe to the worker datastream publisher when the datastream publisher route is available. + +Acknowledgements are retried until all expected acknowledgement reports arrive or the acknowledgement timeout expires. + +Failure to receive required acknowledgements is a startup failure. + +### 4.8 Stage Provisioning Phase + +After worker readiness is acknowledged, the orchestrator provisions stages. + +In direct execution, the orchestrator sends a single stage provisioning command to the worker node agent. + +In planned pipeline execution, the orchestrator provisions stages from the run plan. Stage provisioning includes model identity, GGUF source, tokenizer source, layer range, stage index, stage count, inbound edge information, outbound edge information, object specs, ring specs, and consumer endpoint information. + +Stage provisioning is complete only after the corresponding weight-loaded reports are observed. + +### 4.9 Weight Loading Phase + +After stage provisioning begins, the orchestrator waits for workers to report that weights are loaded. + +Direct execution waits for the single configured stage. + +Planned pipeline execution loads stages sequentially. The orchestrator sends or resends stage provisioning for the next unloaded pipeline stage, waits for its weight-ready report, then advances to the next stage. + +A stage fault during weight loading is a startup failure. + +A provider failure or worker exit during weight loading is a startup failure. + +### 4.10 Prompt RPC Startup Phase + +The prompt RPC listener is created only after workers are ready and weights are loaded. + +When prompt RPC binds successfully, the orchestrator emits prompt-loop readiness. At that point external clients may submit prompt requests. + +Prompt RPC bind failure is a startup failure. + +### 4.11 Prompt Serving Phase + +Prompt serving is the steady-state runtime phase. + +During prompt serving, the orchestrator repeatedly: + +- pumps Iroh and Swactor runtime work; +- drains provider observations; +- drains worker datastream frames; +- drains captured orchestrator stdio; +- checks for shutdown control input; +- accepts at most one active prompt request; +- forwards prompt work through direct or pipeline execution; +- streams prompt events back to the prompt RPC client. + +In direct execution, prompt work is sent to the worker node agent as an inference command. + +In planned pipeline execution, prompt work is encoded by the tokenizer actor, sent into the token-in edge, received from the token-out edge, decoded by the tokenizer actor, and streamed back as prompt RPC events. + +Prompt serving continues until shutdown is requested or a fatal runtime error occurs. + +### 4.12 Shutdown Phase + +Shutdown begins when prompt serving returns successfully or with an error. + +The orchestrator emits provider-stop progress and calls `stop_node` for every provisioned worker handle. Handles are stopped in guard-owned order until none remain. The first provider-stop error is preserved. + +If prompt serving succeeded and provider stop succeeded, the orchestrator emits an orchestrator-exit success record and exits successfully. + +If prompt serving failed, provider stop failed, or both failed, the orchestrator exits with failure after attempting worker cleanup. + +### 4.13 Cleanup Guarantee + +The orchestrator owns provider worker handles through `ProvisionedClusterGuard`. + +The guard attempts to stop remaining workers on explicit shutdown and again on drop if any handles remain. This makes worker cleanup best-effort even when the lifecycle exits through an error path. + +Cleanup is best-effort, not proof that external provider resources were removed. Provider failures during cleanup are reported when they occur through the explicit provider-stop path. + +--- + +## 5. Core Runtime Dataflow + +The orchestrator is a coordinator. It does not perform model inference itself. It transforms configuration into worker launch requests, worker reports into lifecycle decisions, prompt requests into actor or pipeline commands, and runtime observations into datastream/log outputs. + +The core dataflow has five paths: + +```text +configuration -> resolved runtime request -> worker provisioning + +worker reports -> readiness/stage/prompt decisions + +prompt RPC -> prompt execution -> prompt RPC events + +worker/provider/orchestrator observations -> datastream/log sinks + +shutdown request/error -> provider stop -> process exit +``` + +### 5.1 Runtime Pump + +The runtime advances through an explicit pump loop. + +Each pump cycle performs the same basic work: + +```text +tick protocol actors +-> move inbound Iroh actor messages into Swactor +-> run local Swactor work once +-> drain Swactor outbound actor messages into Iroh +-> accept datastream connections +``` + +This pump is the coordination boundary between the local Swactor runtime and Iroh transport. Actor delivery, SWIM state, route ownership, datastream connection handling, and remote worker reports only progress while the orchestrator is pumping. + +### 5.2 Configuration to Provisioning + +Configuration data enters through TOML, environment variables, process arguments, and defaults. + +The orchestrator resolves those inputs into one effective runtime request: + +```text +defaults +-> optional TOML overlay +-> environment overlay +-> process argument overlay +-> Config +``` + +The resolved `Config` drives: + +- provider selection; +- worker image selection; +- worker binary selection for process provider; +- Docker GPU configuration; +- relay configuration; +- prompt RPC bind address; +- model and tokenizer source selection; +- cached model selection; +- Vast.ai provisioning settings; +- observability settings. + +For direct execution, the `Config` creates one `NodeProvisionSpec`. + +For planned pipeline execution, the `Config` first creates a run plan, then creates one `NodeProvisionSpec` per planned stage. + +### 5.3 Run Plan Dataflow + +Planned execution starts from locally inspectable GGUF metadata. + +```text +cached/local GGUF +-> planning metadata +-> model facts +-> run plan +-> stage provisioning wires +``` + +The run plan determines: + +- stage count; +- worker node ids; +- stage indexes; +- layer ranges; +- token-in edge; +- activation edges; +- token-out edge; +- object specs; +- ring specs; +- consumer endpoints. + +The run plan is used twice: + +- before provisioning, to decide which workers to start; +- after readiness, to derive the stage provisioning commands sent to workers. + +Direct execution skips this path. + +### 5.4 Provisioning Dataflow + +Worker provisioning flows from the orchestrator to the selected provider plugin. + +```text +Config / RunPlan +-> NodeProvisionSpec +-> ProvisionPlugin::start_node +-> worker runtime +-> PluginObservation / OrchestratorReport +``` + +The `NodeProvisionSpec` carries the data the provider needs to start a worker: + +```text +NodeProvisionSpec { + run_id, + node_id, + stage_index, + image, + env, + args, + mounts, +} +``` + +The provider returns a worker handle. The orchestrator stores that handle in the provisioned-cluster guard. + +After the worker has passed readiness acknowledgement, the orchestrator calls `complete_bootstrap` for that handle. + +During shutdown, the same handle is passed to `stop_node`. + +### 5.5 Worker Readiness Dataflow + +Worker readiness reaches the orchestrator through actor reports. + +```text +worker node agent +-> orchestrator actor +-> orchestrator report inbox +-> readiness wait loop +``` + +A runtime-ready report contains the worker endpoint, node actor address, datastream publisher address, stage index, and readiness id. + +The orchestrator does not treat the report alone as sufficient. It combines the report with local runtime state: + +```text +runtime-ready report ++ SWIM member is alive ++ actor route owner matches worker += worker ready +``` + +After that barrier passes, the orchestrator sends runtime-ready acknowledgement back to the worker node agent. + +### 5.6 Stage Provisioning Dataflow + +Stage provisioning flows from the orchestrator to worker node agents after readiness acknowledgement. + +Direct execution: + +```text +Config +-> StageProvisionWire +-> NodeAgentMsg::ProvisionStage +-> worker loads stage +-> WeightsReady or StageFault report +``` + +Planned execution: + +```text +RunPlan + worker readiness map +-> StageProvisionWire for stage N +-> NodeAgentMsg::ProvisionStage +-> worker loads stage N +-> WeightsReady or StageFault report +-> next stage +``` + +Pipeline stages are loaded sequentially. The orchestrator advances to the next unloaded stage only after the active stage reports weights ready. + +### 5.7 Prompt Dataflow: Direct Mode + +Direct prompt execution is used when no planned pipeline runtime is active. + +```text +prompt RPC SubmitPrompt +-> PromptWork queue +-> serve_prompts loop +-> NodeAgentMsg::InferPrompt +-> worker prompt execution +-> PromptEvent inbox +-> prompt RPC response stream +``` + +The orchestrator accepts at most one active prompt request at a time. + +Prompt events whose request id does not match the active prompt are dropped from the active prompt flow. + +Terminal prompt events clear the active prompt and allow the next prompt request to begin. + +### 5.8 Prompt Dataflow: Pipeline Mode + +Pipeline prompt execution is used for planned pipeline runtimes. + +```text +prompt RPC SubmitPrompt +-> PromptWork queue +-> tokenizer encode actor +-> token-in edge +-> pipeline stages +-> token-out edge +-> tokenizer decode actor +-> prompt RPC response stream +``` + +The pipeline prompt runtime keeps the active prompt state, generated token list, final text buffer, token sequence number, pending tokenizer encode state, and pending tokenizer decode state. + +Prompt text is first sent to the tokenizer encode actor. + +Encoded tokens are sent to the first pipeline stage over the token-in edge. + +Generated token records return over the token-out edge. + +Each generated token is sent to the tokenizer decode actor. + +Decoded text is emitted to the prompt RPC client as `TextDelta`. + +The prompt completes when the pipeline reports EOS or when the request reaches its max-token limit. + +### 5.9 Observability Dataflow + +Observability has three sources: + +```text +orchestrator internal events +provider observations +worker datastream frames +``` + +Orchestrator internal events become bootstrap or prompt datastream records. + +Provider observations become provisioning events or provisioning log records. + +Worker datastream frames are collected over the datastream ALPN. + +Configured observability sinks receive frames from these sources: + +```text +orchestrator datastream +-> dashboard sink, if enabled +-> frame archive, if configured + +worker datastream +-> dashboard sink, if enabled +-> frame archive, if configured +``` + +Prompt RPC does not receive observability frames. Prompt RPC receives only prompt events. + +### 5.10 Shutdown Dataflow + +Shutdown begins from either control input or a fatal lifecycle result. + +```text +stdin stop/shutdown/quit +or prompt-serving error +or provider/runtime failure +-> serve_prompts returns +-> provider_stop starts +-> stop_node for each worker handle +-> process exit +``` + +The provisioned-cluster guard owns cleanup. Explicit shutdown drains the guard by calling `stop_node` for each worker handle. If the guard is dropped with handles still present, it attempts best-effort cleanup. + +The process exit result is computed from prompt-serving result and provider-stop result. + +--- + +## 6. Actors + +Actors are the orchestrator control plane. They carry runtime state transitions, worker commands, prompt commands, tokenizer commands, and readiness reports. + +Actor messages are distinct from prompt RPC records, provider plugin observations, datastream frames, worker stdout/stderr logs, and pipeline token-edge bytes. + +Actor delivery is asynchronous. Messages only make progress while the orchestrator pumps the Swactor runtime and Iroh driver. + +### 6.1 Actor Runtime + +The orchestrator creates a local Swactor runtime as part of the distribution runtime stack. + +The runtime is connected to Iroh through the actor bridge: + +```text +Swactor runtime +<-> actor bridge +<-> Iroh driver +<-> remote worker node actors +``` + +Local actor messages stay inside the process. + +Remote actor messages are serialized with registered codecs and delivered through the Iroh actor bridge. + +Actor route availability is part of worker readiness. A worker node is not ready for orchestration until the route owner for its node actor matches the worker’s Iroh node identity. + +The actor runtime has these operational states from the orchestrator’s perspective: + +- **initializing**: codecs, local actors, and actor bridge are being registered; +- **pumping**: inbound Iroh messages, local actor work, and outbound actor messages are advanced by the orchestrator pump loop; +- **stopped by process exit**: actor progress ends when the orchestrator process exits. + +There is no independent actor scheduler contract outside the orchestrator pump loop. + +### 6.2 Actor Addresses + +Actors are addressed by `ActorAddress`. + +The orchestrator uses actor addresses for: + +- the local orchestrator actor; +- the local orchestrator report inbox; +- the local prompt reply inbox; +- the local tokenizer reply inbox; +- each worker node agent actor; +- each worker datastream publisher actor. + +Worker actor addresses are learned from runtime-ready reports. The orchestrator does not construct remote worker actor addresses by convention. + +An actor address becomes usable for remote delivery only after the route view reports that the address is owned by the expected worker node. + +### 6.3 Local Inbox Actors + +The orchestrator creates local inbox actors for receiving reports and replies. + +Local inbox actors are queue endpoints. Their state is the set of messages not yet drained by the orchestrator loop. + +The local inbox actors are: + +- **orchestrator report inbox** + Holds `OrchestratorReport` values emitted by the orchestrator actor. + +- **prompt reply inbox** + Holds `PromptEvent` values emitted by direct prompt inference. + +- **tokenizer reply inbox** + Holds `TokenizerEvent` values emitted by tokenizer encode/decode work in pipeline mode. + +Inbox states are: + +- **empty**: no pending messages; +- **pending**: one or more messages are queued; +- **drained**: the orchestrator loop has consumed all currently available messages. + +Inbox actors do not own lifecycle decisions. They only buffer messages until the orchestrator loop drains them. + +### 6.4 Orchestrator Actor + +The orchestrator actor wraps the run-level FSM. + +It receives `OrchestratorMsg` actor messages and may emit `OrchestratorReport` actor messages to the orchestrator report inbox. + +State owned by the orchestrator actor: + +```text +OrchestratorActor { + core: OrchestratorRun, + report_to: Option, + command_cursor, + event_cursor, +} +``` + +The `OrchestratorRun` state includes: + +```text +OrchestratorRun { + config, + plan, + pool_ready, + provisioned, + token_in_ready, + token_out_ready, + ready_stages, + injected_sequences, + expected_token_sequence, + events, + commands, + terminal, + teardown_started, + stopped_stages, + token_endpoints_stopped, +} +``` + +Logical states: + +- **waiting for plan and pool**: the actor has a run config but cannot provision until required plan/pool facts are observed. +- **provisioned**: the FSM has emitted or recorded stage provisioning intent. +- **waiting for token endpoints**: the actor has not yet observed both token-in and token-out endpoint readiness. +- **waiting for stage readiness**: the actor is collecting ready stage indexes. +- **generating tokens**: the actor tracks injected token sequences and expected returned token sequence. +- **terminal**: the actor has observed run completion or a run fault. Further token receipt does not advance generation. +- **tearing down**: the actor has begun teardown and waits for stage stops and token endpoint teardown. +- **torn down**: all required stopped-stage and token-endpoint stopped facts have been observed. + +Incoming actor messages to `OrchestratorActor` are variants of `OrchestratorMsg`. + +Common incoming observations: + +```text +OrchestratorMsg::ObserveNodeRuntimeReady { ... } +OrchestratorMsg::ObserveNodeRuntimeReadyAck { ... } +OrchestratorMsg::ObserveWeightsReady { ... } +OrchestratorMsg::ObserveStageReady { ... } +OrchestratorMsg::ObserveStageFault { ... } +OrchestratorMsg::ObserveStageStopped { ... } +OrchestratorMsg::ObserveTokenReceived { ... } +OrchestratorMsg::ObserveEndpointFault { ... } +OrchestratorMsg::ObserveTokenEndpointsStopped +``` + +These are typed actor messages delivered by Swactor. When sent by a remote worker, delivery passes through the Iroh actor bridge. + +Reports emitted by `OrchestratorActor` are variants of `OrchestratorReport`. + +Common report messages: + +```text +OrchestratorReport::NodeRuntimeReady { ... } +OrchestratorReport::NodeRuntimeReadyAck { ... } +OrchestratorReport::WeightsReady { ... } +OrchestratorReport::StageReady { ... } +OrchestratorReport::StageFault { ... } +OrchestratorReport::Command(...) +OrchestratorReport::Lifecycle(...) +``` + +These are typed actor messages sent by `OrchestratorActor` to the local orchestrator report inbox. The orchestrator process drains that inbox and uses the reports to advance startup, provisioning, weight-loading, and failure handling. + +The orchestrator binary uses direct readiness, acknowledgement, weight, and fault reports as lifecycle gates. FSM command and lifecycle reports remain part of the actor surface, but they are not the primary startup gates in the current binary. + +### 6.5 Worker Node Agent Actor + +Each worker has a node agent actor. + +The node agent actor is the orchestrator’s primary control target on a worker. It receives stage provisioning, prompt, tokenizer, and readiness acknowledgement commands. + +State owned by the node agent actor: + +```text +NodeAgentActor { + core: StageController, + orchestrator: ActorAddress, + report_to: Option, + inbound_edge: Option, + outbound_edge: Option, + command_cursor, + event_cursor, +} +``` + +The node agent’s `StageController` state includes: + +```text +StageController { + provision, + worker_ready, + weights_ready, + inbound_ready, + outbound_ready, + stage_ready_emitted, + busy, + expected_sequence, + active_input, + commands, + events, + faulted, + stopped, + stopping_run, + local_edges_stopped, + worker_rings_quiesced, + release_reset_requested, + device_objects_released, + worker_role_reset, +} +``` + +Logical states: + +- **unprovisioned**: no stage provision has been accepted. +- **provisioning**: a valid `ProvisionStage` message has been accepted. The stage controller has emitted commands to establish inbound edge, establish outbound edge, configure worker role, and load weights. +- **waiting for stage readiness**: the stage has provision data but has not yet observed every readiness prerequisite. +- **ready idle**: worker ready, weights ready, inbound edge ready, and outbound edge ready have all been observed. `StageReady` has been emitted. No step is active. +- **busy executing step**: a valid inbound object has arrived with the expected sequence. The controller has emitted an execute-step command and is waiting for completion or failure. +- **faulted**: the actor has observed an unauthorized provision, sequence violation, worker crash, step failure, object failure, output fault, or edge fault. Faulted stages do not accept new execution work. +- **stopping**: a stop has been requested for the run. The actor has emitted stop/release/reset commands and waits for local edges, worker rings, device objects, and worker role reset facts. +- **stopped**: all stop prerequisites have been observed and `StageStopped` has been emitted. + +Important orchestrator-sent messages: + +```text +NodeAgentMsg::RuntimeReadyAck { ... } +NodeAgentMsg::ProvisionStage(StageProvisionWire) +NodeAgentMsg::InferPrompt { ... } +NodeAgentMsg::EncodePrompt { ... } +NodeAgentMsg::DecodeTokens { ... } +``` + +Important worker-side or stage-side observations accepted by the node agent: + +```text +NodeAgentMsg::RuntimeLoaded { ... } +NodeAgentMsg::MarkWeightsReady { ... } +NodeAgentMsg::MarkInboundEdgeReady { ... } +NodeAgentMsg::MarkOutboundEdgeReady { ... } +NodeAgentMsg::ObjectLoaded { ... } +NodeAgentMsg::StepCompleted { ... } +NodeAgentMsg::WorkerCrashed +NodeAgentMsg::StopRun { ... } +NodeAgentMsg::LocalEdgesStopped { ... } +NodeAgentMsg::WorkerRingsQuiesced { ... } +NodeAgentMsg::DeviceObjectsReleased { ... } +NodeAgentMsg::WorkerRoleReset { ... } +``` + +Important outputs to the orchestrator actor: + +```text +OrchestratorMsg::ObserveNodeRuntimeReady { ... } +OrchestratorMsg::ObserveNodeRuntimeReadyAck { ... } +OrchestratorMsg::ObserveWeightsReady { ... } +OrchestratorMsg::ObserveStageReady { ... } +OrchestratorMsg::ObserveStageFault { ... } +OrchestratorMsg::ObserveStageStopped { ... } +``` + +These outputs are actor messages sent to `OrchestratorActor`, not datastream records or prompt RPC events. + +### 6.6 Stage Provision Message + +Stage provisioning is carried by `StageProvisionWire`. + +```text +StageProvisionWire { + run_id, + authorized_orchestrator, + node_id, + stage_index, + stage_count, + layer_start, + layer_end_exclusive, + inbound_edge_id, + outbound_edge_id, + inbound_edge, + outbound_edge, + model_id, + gguf_source, + tokenizer, +} +``` + +For direct execution, inbound and outbound edge details may be absent. + +For planned pipeline execution, inbound and outbound edge details describe the token or activation edge assigned by the run plan. + +A node agent accepts a stage provision only when: + +- `authorized_orchestrator` matches the expected orchestrator identity; +- `node_id` matches the local worker node id. + +Invalid provisioning faults the stage. + +### 6.7 Datastream Publisher Actor + +Each worker reports a datastream publisher actor address. + +The datastream publisher actor accepts: + +```text +DatastreamPublisherMsg::Subscribe(DatastreamSubscribe) +``` + +Subscription request shape: + +```text +DatastreamSubscribe { + collector, + request, + flow_id, + token, +} +``` + +State owned by the publisher actor: + +```text +DatastreamPublisherActor { + endpoint, + on_subscribe, +} +``` + +Logical states: + +- **waiting for subscription**: the actor owns a datastream endpoint and waits for subscribe messages. +- **subscription accepted**: a subscribe message has been accepted. The actor creates a local datastream subscription from the endpoint and passes it to the transport-specific `on_subscribe` callback. + +The publisher actor does not itself stream bytes. It creates the subscription and hands it to transport code that writes datastream frames. + +### 6.8 Provisioner Actor + +The codebase defines a `ProvisionerActor`, but the orchestrator binary covered by this spec provisions workers directly through `ProvisionPlugin`. + +Therefore, the `ProvisionerActor` is not part of the current orchestrator runtime lifecycle. + +If a future orchestrator path uses it, its state and message contracts must be specified before it becomes part of this document’s active contract. + +### 6.9 Prompt Actor Flow + +Direct prompt mode uses actor messages for worker prompt execution. + +```text +SubmitPrompt +-> PromptWork +-> NodeAgentMsg::InferPrompt +-> worker +-> PromptEvent +-> prompt reply inbox +-> prompt RPC stream +``` + +State involved in this flow: + +- prompt-serving loop tracks the active prompt; +- prompt reply inbox buffers `PromptEvent`; +- node agent actor receives `InferPrompt`; +- worker prompt implementation produces prompt events. + +The `reply_to` address in `InferPrompt` is the local prompt reply inbox. + +Prompt events with a mismatched request id are dropped from the active prompt flow. + +### 6.10 Pipeline Tokenizer Actor Flow + +Pipeline prompt mode uses actor messages for tokenizer work and edge transport for generated tokens. + +```text +SubmitPrompt +-> PromptWork +-> NodeAgentMsg::EncodePrompt +-> TokenizerEvent::PromptEncoded +-> token-in edge +-> token-out edge +-> NodeAgentMsg::DecodeTokens +-> TokenizerEvent::TokensDecoded +-> prompt RPC stream +``` + +State involved in this flow: + +- pipeline prompt runtime tracks active prompt state; +- tokenizer reply inbox buffers `TokenizerEvent`; +- encode actor receives `EncodePrompt`; +- decode actor receives `DecodeTokens`; +- token-edge transport carries generated token records; +- prompt RPC stream receives decoded text. + +The encode actor is the first-stage node actor. + +The decode actor is the final-stage node actor. + +The `reply_to` address for tokenizer messages is the local tokenizer reply inbox. + +Tokenizer faults become prompt faults for the active request. + +### 6.11 Actor Delivery Contracts + +Actor send failure is a runtime error for the phase that attempted the send. + +Actor delivery is not synchronous execution. A successful send means the message was accepted by the local runtime for delivery, not that the remote worker has acted on it. + +Remote actor delivery requires: + +- Iroh transport progress; +- actor bridge progress; +- route availability; +- SWIM membership state sufficient for route ownership. + +The orchestrator must keep pumping runtime and transport while waiting for actor-driven results. + +### 6.12 Actor Message Filtering + +Actor reports are accepted only when they match the active orchestration context. + +Lifecycle reports are filtered by `run_id`. + +Worker readiness reports are filtered by expected `node_id`. + +Stage reports are filtered by expected `stage_index`. + +Prompt and tokenizer events are filtered by active `request_id`. + +Mismatched reports do not advance the active lifecycle or prompt state. + +--- + +## 7. Subcomponents and Behaviors + +This section describes the in-process subcomponents that implement orchestration behavior. + +Actors are covered in `Actors`. Datastream record schemas are covered in `Datastream and Logs`. This section focuses on non-actor runtime components and the state they own. + +### 7.1 Configuration Resolver + +The configuration resolver turns defaults, TOML, environment variables, and process arguments into one `Config`. + +Owned state: + +```text +Config { + config_profile: RuntimeConfigProfile, + provider: ProviderKind, + image: String, + docker_gpus: String, + rpc_bind: SocketAddr, + run_id: u64, + node_id: u64, + stage_index: u32, + layer_end_exclusive: Option, + pipeline_stages: u32, + model_id: String, + gguf_source: GgufSource, + tokenizer: TokenizerSource, + default_max_tokens: u32, + dashboard: bool, + max_context: Option, + relay: RelayRuntimeConfig, + vastai: Option, + cached_model: Option, + worker_bin: Option, + datastream_frame_log: Option, +} +``` + +Defaults: + +- `config_profile` defaults to `Local`. +- `provider` defaults from `config_profile`: + - `Local` uses `Process`. + - `Deploy` uses `VastAi`. +- `image` defaults to `swactor-mvp-node:latest`. +- `docker_gpus` defaults to `all`. +- `rpc_bind` defaults to `127.0.0.1:19777`. +- `run_id` defaults to `1`. +- `node_id` defaults to `1`. +- `stage_index` defaults to `0`. +- `layer_end_exclusive` defaults to absent. +- `pipeline_stages` defaults to `1`. +- `default_max_tokens` defaults to `64`. +- `dashboard` defaults to disabled. +- `max_context` defaults to absent. +- `vastai` defaults to absent unless the resolved provider is `VastAi`. +- `cached_model` defaults to absent. +- `worker_bin` defaults to absent. +- `datastream_frame_log` defaults to absent. + +`model_id`, `gguf_source`, and `tokenizer` are resolved model inputs. This section records their typed presence in `Config`, but does not define a hardcoded default model contract. + +Behavior: + +- starts from fixed defaults; +- overlays optional TOML; +- overlays environment variables; +- overlays process arguments; +- validates provider-specific constraints; +- resolves relay configuration; +- resolves cached model configuration; +- rejects invalid pipeline stage counts; +- rejects unsupported provider/profile values; +- rejects malformed prompt RPC bind addresses. + +The resolver is the only component that should interpret raw configuration strings. Later components receive typed runtime state. + +### 7.2 Cached Model Resolver + +The cached model resolver validates host-local cached model paths and converts them into worker-visible paths. + +Owned state: + +```text +CachedModelConfig { + host_path, + container_path, +} +``` + +Behavior: + +- canonicalizes the host path; +- requires the host path to point to a file; +- derives a container path under the cached model container directory; +- exposes the host path to process workers; +- exposes the container path to Docker workers; +- enables planned execution when cached-model execution is selected. + +Cached model paths are supported only for process and Docker providers. + +### 7.3 Vast.ai Runtime Preparation + +Vast.ai runtime preparation resolves provider-specific launch requirements before workers are started. + +Owned state: + +```text +VastAiRuntimeConfig { + api_key, + provisioning, + bootstrap_command, + ssh_identity, + ssh_public_key, + ssh_public_fingerprint, +} +``` + +Behavior: + +- requires an API key when Vast.ai is selected; +- resolves the SSH identity path; +- verifies the SSH identity file exists; +- derives the public key from the identity; +- ensures the public key is registered with the Vast.ai account; +- records the prepared identity and public-key metadata into provider config; +- requires a bootstrap command before constructing the Vast.ai provisioner. + +Failure in this component stops startup before worker provisioning begins. + +### 7.4 Run Planner + +The run planner is used only for planned execution. + +Planned execution is selected when: + +```text +cached_model is present +or pipeline_stages > 1 +``` + +Behavior: + +- reads locally inspectable GGUF metadata; +- converts metadata into model facts; +- rejects pipeline stage counts larger than model layer count; +- computes activation ring size; +- computes token ring size; +- assigns fixed linear stage placement; +- produces a `RunPlan`. + +The run plan drives: + +- number of workers; +- stage indexes; +- logical node ids; +- layer ranges; +- token-in edge; +- activation edges; +- token-out edge; +- object specs; +- ring specs; +- stage provisioning payloads. + +Direct execution skips this component. + +### 7.5 Provisioner Builder + +The provisioner builder constructs the provider implementation used to start and stop workers. + +Behavior by provider: + +- **process** + Resolves the worker binary path and requires it to exist. + +- **Docker** + Constructs a local Docker provisioner using the configured container name prefix. + +- **Vast.ai** + Requires prepared Vast.ai config, API key, bootstrap command, and SSH identity; constructs a Vast.ai provisioning plugin backed by the Vast.ai client and SSH launcher. + +The orchestrator does not use the `ProvisionerActor` in the current binary. It calls the selected `ProvisionPlugin` directly. + +### 7.6 Provisioned Cluster Guard + +The provisioned-cluster guard owns worker handles after successful provider startup. + +Owned state: + +```text +ProvisionedClusterGuard { + provisioner, + handles, +} +``` + +Behavior: + +- stores each returned provider handle; +- calls `complete_bootstrap` for all handles after runtime-ready acknowledgement succeeds; +- calls `stop_node` for each handle during explicit shutdown; +- preserves the first stop error; +- attempts best-effort cleanup on drop if handles remain. + +The guard is the ownership boundary for worker cleanup. Once a handle is in the guard, the orchestrator is responsible for attempting to stop it. + +### 7.7 Runtime Stack and Iroh Driver + +The runtime stack and Iroh driver provide transport, actor delivery, routing, SWIM membership, and datastream connection acceptance. + +Owned state is split across: + +- `IrohDriver`; +- `DistributionRuntimeStack`; +- route view; +- relay mirror; +- SWIM actor state; +- actor bridge routes; +- runtime outbox. + +Behavior: + +- registers actor and datastream codecs; +- enables the actor bridge; +- registers local actor routes; +- pumps inbound Iroh messages into actors; +- pumps local actor runtime work; +- drains outbound actor messages to Iroh; +- accepts datastream connections; +- exposes route owner and member state checks used by readiness barriers. + +This subcomponent is not autonomous. It advances only when the orchestrator calls the pump function. + +### 7.8 Prompt RPC Server + +The prompt RPC server accepts external prompt submissions over TCP. + +Owned state: + +```text +Prompt RPC listener { + bind_addr, + work_tx, + default_max_tokens, +} +``` + +Behavior: + +- binds the configured prompt RPC address; +- spawns an accept loop; +- spawns one handler thread per accepted connection; +- reads newline-delimited `SubmitPrompt` JSON; +- applies default max tokens when request max tokens is zero; +- sends accepted work into the prompt work queue; +- writes newline-delimited `PromptEvent` JSON back to the client; +- stops writing for a request after `Done` or `Fault`. + +Prompt RPC starts only after runtime readiness and weight loading have completed. + +### 7.9 Prompt Serving Loop + +The prompt serving loop is the steady-state coordinator after prompt RPC is ready. + +Owned state: + +```text +serve_prompts { + active: Option, + optional pipeline runtime, +} +``` + +Behavior: + +- pumps actor and transport runtime; +- drains provider observations; +- drains worker datastream frames; +- drains captured orchestrator stdio; +- checks for stop requests; +- accepts prompt work only when no prompt is active; +- sends direct prompt work to the worker node agent in direct mode; +- delegates prompt work to `PipelinePromptRuntime` in pipeline mode; +- forwards matching prompt events to the prompt RPC client; +- drops prompt events for non-active request ids; +- clears active prompt state on terminal prompt event. + +The prompt serving loop enforces the one-active-prompt rule. + +### 7.10 Pipeline Prompt Runtime + +The pipeline prompt runtime coordinates prompt execution for planned pipeline mode. + +Owned state: + +```text +PipelinePromptRuntime { + token_in_edge_id, + token_out_edge_id, + token_spec, + token_out_spec, + token_in_sender, + recv_rx, + recv_tx, + recv_buffer, + tokenizer_encode_actor, + tokenizer_decode_actor, + tokenizer_reply_to, + pending_encode, + pending_decode, + next_sequence, + generated_tokens, + final_text, + active, + started_at, +} +``` + +Behavior: + +- starts one active prompt; +- requests tokenizer encode for prompt text; +- sends encoded prompt tokens over the token-in edge; +- receives generated token records from the token-out edge; +- validates token sequence order; +- requests tokenizer decode for each generated token; +- emits text deltas to the prompt RPC client; +- appends decoded text to final text; +- stops on EOS or max-token limit; +- sends `Done` on successful completion; +- sends `Fault` on tokenizer failure or runtime error; +- clears active prompt state after terminal output. + +The pipeline prompt runtime owns prompt-generation state, not worker stage state. + +### 7.11 Pipeline Token Sender and Receiver + +Pipeline token transport is handled by token sender, receiver, and acceptor helpers. + +Behavior: + +- token sender opens a unidirectional stream to the first stage endpoint; +- token sender writes encoded token-in records; +- token acceptor accepts incoming pipeline edge connections; +- token receiver reads bytes from accepted streams; +- received bytes are buffered until complete token records can be decoded. + +The token transport carries bytes. The pipeline prompt runtime owns record sequencing and prompt semantics. + +### 7.12 Orchestrator Datastream + +The orchestrator datastream component emits runtime observations produced by the orchestrator itself. + +Owned state: + +```text +OrchDatastream { + stream, + endpoint, + producer, + channels, + channel_names, + archive, +} +``` + +Behavior: + +- creates the orchestrator stream id; +- registers core channels; +- emits bootstrap records; +- emits prompt records; +- emits provisioning events; +- emits provisioning log records; +- emits arbitrary channel payloads from provider observations; +- flushes frames to dashboard if enabled; +- writes frames to the frame archive if configured. + +The orchestrator datastream is the primary structured observation path for orchestrator-owned events. + +### 7.13 Datastream Frame Archive + +The frame archive records datastream frames to a JSON-lines file when configured. + +Owned state: + +```text +FrameArchive { + file, + next_seq, +} +``` + +Behavior: + +- creates parent directories for the configured path when needed; +- opens the archive file in append mode; +- records frames with an arrival sequence; +- records source, stream, channel, channel id, position, and payload; +- encodes UTF-8 payloads as text; +- encodes non-UTF-8 payloads as bytes; +- flushes after each record. + +Without configured datastream frame logging, this component is absent. + +### 7.14 Orchestrator Stdio Capture + +The stdio capture component redirects orchestrator stdout and stderr into provisioning log records. + +Owned state: + +```text +optional mpsc receiver of captured stdio lines +``` + +Behavior: + +- on Linux, redirects stdout and stderr through pipes; +- spawns reader threads for captured stdout and stderr; +- converts captured lines into `OrchStdioLine`; +- drains captured lines into the orchestrator datastream as log records; +- on non-capturing targets, may be absent. + +Captured orchestrator stdio is observability data. It is not a terminal UI contract. + +### 7.15 Dashboard Support + +Dashboard support is an optional sink for datastream frames. + +Owned state: + +```text +optional DashboardSupport +``` + +Behavior: + +- starts only when dashboard support is enabled; +- receives frames from orchestrator datastream flushes; +- receives frames collected from worker datastream streams; +- publishes frames to the dashboard handle; +- does not affect runtime correctness when absent. + +Dashboard output is derived from datastream frames and does not own lifecycle state. + +### 7.16 Stop Listener + +The stop listener watches standard input for shutdown control lines. + +Owned state: + +```text +mpsc receiver of stop notifications +``` + +Behavior: + +- runs in a background thread; +- reads standard input line by line; +- trims each line; +- accepts `stop`, `shutdown`, or `quit` case-insensitively; +- sends one stop notification; +- causes wait loops or prompt serving to exit through controlled shutdown paths. + +The stop listener is not a prompt input path. + +--- + +## 8. Datastream and Logs + +Datastream is the orchestrator’s structured observation path. Logs are represented as datastream records, not as terminal UI. + +This section defines the datastream and log channels used by the orchestrator process. It does not define prompt RPC payloads, actor message schemas, or provider command protocols. + +### 8.1 Datastream Model + +A datastream is an ordered stream of frames. + +Each frame has: + +```text +Frame { + channel, + position, + payload, +} +``` + +A channel gives meaning to the payload. The datastream transport itself treats payloads as opaque bytes. + +The orchestrator uses datastream for: + +- bootstrap progress; +- prompt progress; +- provisioning events; +- worker stdout/stderr logs; +- provider logs; +- captured orchestrator stdout/stderr logs; +- SWIM membership observations; +- stage route observations; +- worker-emitted datastream frames; +- dashboard publication; +- optional frame archive output. + +Datastream records are observational. They do not drive prompt RPC response text, actor delivery, or provider lifecycle by themselves. + +### 8.2 Orchestrator Datastream + +The orchestrator creates its own datastream at startup. + +Its stream identity is tied to the orchestrator and the active run id. + +The orchestrator datastream owns: + +```text +OrchDatastream { + stream, + endpoint, + producer, + channels, + channel_names, + archive, +} +``` + +The orchestrator datastream registers core channels, emits records into those channels, flushes produced frames to configured sinks, and records frames to the archive when frame logging is enabled. + +### 8.3 Core Orchestrator Channels + +The orchestrator emits these core channels: + +```text +mvp.orch.bootstrap +mvp.orch.prompt +mvp.swim.membership +mvp.orch.stage_route +mvp.provisioning.events +mvp.provisioning.logs.node..stdout +mvp.provisioning.logs.node..stderr +mvp.provisioning.logs.node..provider +``` + +`mvp.orch.bootstrap` carries orchestrator lifecycle progress. + +`mvp.orch.prompt` carries prompt-serving progress. + +`mvp.swim.membership` carries observed membership transitions. + +`mvp.orch.stage_route` carries route checks during pipeline stage provisioning. + +`mvp.provisioning.events` carries node provisioning lifecycle events. + +`mvp.provisioning.logs.node..` carries stdout, stderr, or provider log lines for a node id. + +Provider-supplied datastream frames may create additional channels by name. Worker datastream frames may also use worker-defined channel names. + +### 8.4 Bootstrap Records + +Bootstrap records use this envelope: + +```text +OrchBootstrap { + type: "OrchBootstrap", + phase, + status, + run_id, + node_id, + detail, +} +``` + +`phase` identifies the lifecycle area being reported. + +`status` identifies the transition or outcome, such as: + +```text +started +ready +failed +sent +observed +``` + +`detail` is phase-specific JSON. + +Bootstrap records are emitted for runtime setup, provider start/stop, node specs, readiness waiting, stage provisioning, weight loading, prompt RPC readiness, shutdown, and process exit. + +### 8.5 Prompt Records + +Prompt records use this envelope: + +```text +OrchPromptEvent { + type: "OrchPromptEvent", + phase, + status, + run_id, + node_id, + request_id, + detail, +} +``` + +Prompt records describe orchestration progress for a prompt request. They are not the prompt response stream. + +Prompt record phases include: + +- prompt work observed; +- direct prompt send started/ready/failed; +- direct prompt event observed/dropped; +- prompt complete; +- pipeline tokenizer encode started/ready; +- pipeline token-in started/ready; +- pipeline token-out observed; +- pipeline tokenizer decode started/ready. + +Prompt response text is emitted through prompt RPC as `PromptEvent`. Prompt datastream records are diagnostic and observational. + +### 8.6 Provisioning Event Records + +Provisioning lifecycle events are emitted on: + +```text +mvp.provisioning.events +``` + +Record shape: + +```text +MvpProvisionEventRecord { + event: ProvisionEvent, +} +``` + +Provision event shape: + +```text +ProvisionEvent { + run_id, + node_id, + kind, + provider, + message, +} +``` + +Accepted event kinds: + +```text +ProvisionStart +NodeLive +ProvisionFailed +NodeStopped +``` + +Provisioning events are emitted when provider startup begins, nodes become live, provider startup fails, or nodes stop. + +### 8.7 Provisioning Log Records + +Provisioning logs are emitted on node-specific log channels: + +```text +mvp.provisioning.logs.node..stdout +mvp.provisioning.logs.node..stderr +mvp.provisioning.logs.node..provider +``` + +Record shape: + +```text +MvpProvisionLogRecord { + line: ProvisionLogLine, +} +``` + +Log line shape: + +```text +ProvisionLogLine { + run_id, + node_id, + stream, + line, +} +``` + +Accepted log streams: + +```text +Stdout +Stderr +Provider +``` + +Worker stdout, worker stderr, provider log lines, and captured orchestrator stdout/stderr are represented through this log record format. + +Log lines are observational. They are not parsed as commands. + +### 8.8 Orchestrator Stdio Logs + +After stdio capture is installed, orchestrator stdout and stderr are redirected into log records. + +Captured stdout becomes a provisioning log record with stream `Stdout`. + +Captured stderr becomes a provisioning log record with stream `Stderr`. + +The capture path is used so startup/runtime diagnostics appear in the same datastream/log stream as worker and provider logs. + +Fatal errors before capture may still appear on process stderr. + +### 8.9 Provider Observation Logs + +Provider plugin observations are converted into datastream output. + +Conversion rules: + +- `StdoutLine` becomes a provisioning stdout log record. +- `StderrLine` becomes a provisioning stderr log record. +- `ProviderLine` becomes a provisioning provider log record. +- `DatastreamFrame` is emitted to the supplied channel as a raw payload. +- `Exited` becomes a provisioning node-stopped event. +- `Failed` becomes a provisioning failed event. + +Provider observation logs keep provider output visible without making provider stdout/stderr a direct user interface contract. + +### 8.10 Worker Datastream Collection + +Worker datastream frames arrive over the datastream ALPN. + +The orchestrator accepts datastream connections and reads: + +- stream headers; +- channel declarations; +- frame deliveries; +- stream end notifications. + +For each frame, the orchestrator records: + +- source stream id; +- channel name; +- channel id; +- frame position; +- payload bytes. + +Collected worker frames are forwarded to configured sinks: + +```text +worker datastream frame +-> dashboard, if enabled +-> frame archive, if configured +``` + +Worker datastream frames are not re-emitted through prompt RPC. + +### 8.11 Dashboard Sink + +The dashboard receives datastream frames when dashboard support is enabled. + +The dashboard sink consumes frames from: + +- orchestrator datastream flushes; +- collected worker datastream frames. + +Dashboard state is derived from datastream frames. The dashboard is not the source of runtime truth. + +If dashboard support is disabled, the orchestrator still runs and emits datastream frames to other configured sinks. + +### 8.12 Frame Archive + +The frame archive is enabled by datastream frame log configuration. + +Frame archive output is JSON lines. + +Archive record shape: + +```text +FrameArchiveRecord { + arrival_seq, + source, + stream, + channel, + channel_id, + position, + payload, +} +``` + +`arrival_seq` is assigned by the archive and increases for each archived frame. + +`position` is the frame position inside its source datastream. + +`source` identifies the ingestion path, such as orchestrator-originated frames, node bootstrap stdio frames, or node cluster datastream frames. + +Payload encoding is recorded as either: + +```text +{ encoding: "utf8", value: } +``` + +or: + +```text +{ encoding: "bytes", value: } +``` + +The archive may create parent directories for the configured path. + +Without frame logging, the frame archive component is absent. + +### 8.13 Ordering and Scope + +Frame ordering is local to its stream and channel position. + +Archive `arrival_seq` is the archive’s observed arrival order, not a global runtime ordering guarantee. + +Datastream frames from different streams may interleave. + +Log line ordering is preserved only to the extent that the producing stream, capture pipe, provider observation channel, and archive arrival order preserve it. + +### 8.14 Secret Handling + +Secret values must not be emitted as datastream payloads or log lines by orchestrator-owned records. + +The orchestrator may emit secret presence as metadata, such as whether a Vast.ai API key is configured. + +External tools and providers may produce output outside the orchestrator’s control. The orchestrator should avoid copying secret values into structured records when it handles provider errors. + +### 8.15 Datastream Non-Goals + +Datastream is not: + +- prompt RPC; +- actor transport; +- provider control; +- worker stdin; +- an ordering authority across all runtime systems; +- a replacement for lifecycle gates. + +Lifecycle gates are driven by explicit actor reports, provider results, process/control inputs, and runtime state checks. Datastream records explain what happened; they do not by themselves make the runtime ready, failed, or stopped. + +--- + +## 9. Behavioral Contracts + +Behavioral contracts are runtime invariants that callers, wrappers, tests, and maintainers may rely on. + +### 9.1 Configuration Must Resolve Before Runtime Startup + +The orchestrator must resolve configuration before it initializes the runtime stack or starts workers. + +Configuration failure must prevent worker provisioning. + +Configuration failures include: + +- unknown process arguments; +- missing process-argument values; +- unsupported runtime profile; +- unsupported provider; +- invalid prompt RPC bind address; +- invalid pipeline stage count; +- invalid cached model path when cached model execution is selected; +- missing worker binary for process provider; +- missing Vast.ai requirements when Vast.ai is selected. + +The mock provider is not a supported orchestrator runtime provider. + +### 9.2 Provider Constraints Must Be Enforced Before Provisioning + +Provider-specific constraints must be checked before workers are started. + +Contracts: + +- process provider requires a local worker binary; +- Docker provider may use Docker GPU and mount settings; +- cached model host paths are supported only for process and Docker providers; +- Vast.ai requires prepared API and SSH configuration; +- Vast.ai does not support multi-stage pipeline provisioning in the current orchestrator contract. + +A provider constraint failure must stop startup before node provisioning. + +### 9.3 Planned Execution Requires Local Model Metadata + +Planned execution requires locally inspectable GGUF metadata before provisioning. + +Planned execution is selected when: + +```text +cached_model is present +or pipeline_stages > 1 +``` + +The orchestrator must reject planned execution when it cannot inspect the selected GGUF metadata locally. + +The orchestrator must reject a pipeline stage count greater than the model layer count. + +### 9.4 Prompt RPC Must Start After Runtime Readiness + +Prompt RPC must not be advertised or bound as ready until workers are ready and weights are loaded. + +Required prerequisites: + +```text +workers started +runtime-ready reports received +SWIM membership alive +actor routes owned by expected workers +runtime-ready acknowledgements completed +stage provisioning sent +weights loaded +``` + +If these prerequisites fail, prompt RPC startup must not be reported as ready. + +### 9.5 Worker Runtime Readiness Requires More Than a Worker Report + +A worker runtime-ready report is necessary but not sufficient. + +A worker is ready only when all readiness facts are true: + +```text +matching NodeRuntimeReady report ++ SWIM member state is Alive ++ route owner for node actor matches worker node += worker ready +``` + +For planned pipeline execution, every expected worker must satisfy this barrier. + +For direct execution, the single expected worker must satisfy this barrier. + +### 9.6 Runtime-Ready Acknowledgement Must Complete + +After readiness barriers pass, the orchestrator must send runtime-ready acknowledgements to every expected worker. + +The acknowledgement must include: + +```text +run_id +node_id +stage_index +readiness_id +``` + +The orchestrator retries acknowledgements until all expected acknowledgement reports arrive or the acknowledgement timeout expires. + +Timeout is a startup failure. + +### 9.7 Stage Provisioning Must Follow Readiness + +Stage provisioning must occur after worker readiness and runtime-ready acknowledgement. + +Direct execution provisions one stage. + +Planned pipeline execution provisions stages from the run plan. + +Stage provisioning must include the model identity, tokenizer source, layer range, stage index, stage count, and edge wiring needed by that stage. + +A stage fault during provisioning or weight loading is a startup failure. + +### 9.8 Pipeline Weight Loading Is Sequential + +In planned pipeline execution, stages are weight-loaded sequentially. + +The orchestrator must not advance to the next unloaded stage until the active stage reports weights ready. + +If a stage faults while loading weights, pipeline startup fails. + +If a worker exits while loading weights, pipeline startup fails. + +### 9.9 Prompt Serving Allows One Active Prompt + +The orchestrator accepts at most one active prompt at a time. + +While a prompt is active: + +- additional prompt work remains queued; +- direct prompt events are matched by request id; +- pipeline tokenizer events are matched by request id; +- mismatched prompt or tokenizer events are ignored or dropped for the active prompt. + +A terminal prompt event clears active prompt state. + +### 9.10 Prompt Terminal Events End a Request + +A prompt request ends with exactly one terminal outcome: + +```text +Done +Fault +``` + +`TextDelta` is non-terminal. + +After `Done` or `Fault`, the prompt RPC response stream for that request is complete. + +Expected model or prompt failures should be represented as prompt `Fault` events, not as orchestrator process errors, unless the orchestration path itself failed. + +### 9.11 Direct Prompt Mode Must Use Node Actor Inference + +In direct prompt mode, the orchestrator must send prompt work to the worker node actor as an inference command. + +Direct prompt flow: + +```text +SubmitPrompt +-> NodeAgentMsg::InferPrompt +-> PromptEvent +-> prompt RPC stream +``` + +The prompt reply actor address must be supplied as `reply_to`. + +### 9.12 Pipeline Prompt Mode Must Use Tokenizer and Token Edges + +In pipeline prompt mode, prompt text must flow through tokenizer encode, token-in edge, pipeline stages, token-out edge, tokenizer decode, and prompt RPC. + +Pipeline prompt flow: + +```text +SubmitPrompt +-> EncodePrompt +-> PromptEncoded +-> token-in edge +-> token-out edge +-> DecodeTokens +-> TokensDecoded +-> prompt RPC stream +``` + +The first-stage node actor is the tokenizer encode actor. + +The final-stage node actor is the tokenizer decode actor. + +Tokenizer failures become prompt faults for the active request. + +### 9.13 Pipeline Token Sequence Must Be Monotonic + +Pipeline token output records must arrive in the expected sequence order. + +The orchestrator tracks the next expected token sequence. + +If a received token record sequence does not equal the expected sequence, prompt serving fails. + +Sequence validation protects prompt output ordering and prevents feeding token feedback out of order. + +### 9.14 Runtime Pumping Is Required for Progress + +Actor delivery, Iroh transport, SWIM membership, route ownership, datastream connection acceptance, provider observation draining, and prompt progress require the orchestrator pump loop to run. + +A wait loop must keep pumping runtime work while waiting for actor or transport-driven facts. + +A blocking wait that does not pump runtime work violates the runtime model. + +### 9.15 Provider Failures Are Fatal in Active Runtime Phases + +Provider observations can fail startup or prompt serving. + +Contracts: + +- provider failure before readiness is a startup failure; +- worker exit before readiness is a startup failure; +- worker exit while loading weights is a startup failure; +- worker exit during prompt serving is a runtime failure; +- provider stop failure is a shutdown failure. + +Provider stdout/stderr/provider log lines are observational and do not by themselves indicate failure. + +### 9.16 Shutdown Must Attempt Worker Cleanup + +Once worker handles are owned by the provisioned-cluster guard, the orchestrator must attempt to stop all remaining workers on shutdown. + +Shutdown cleanup is best-effort. + +The orchestrator preserves and reports the first provider-stop error from explicit shutdown. + +The guard also attempts cleanup on drop if handles remain. + +Cleanup success does not prove that all external provider resources were removed; it only proves that the orchestrator’s provider stop calls completed successfully. + +### 9.17 Stop Commands Are Controlled Shutdown Requests + +The accepted standard-input stop commands are: + +```text +stop +shutdown +quit +``` + +They are trimmed and compared case-insensitively. + +A stop command requests controlled shutdown. It is not a prompt request. + +### 9.18 Datastream Is Observational + +Datastream records do not make runtime state true. + +Readiness, provisioning, prompt completion, failure, and shutdown are driven by actor reports, provider results, runtime state checks, prompt events, and explicit control inputs. + +Datastream records may describe those transitions, but they are not lifecycle gates. + +### 9.19 Actor Reports Must Match Active Context + +Actor reports must match the active run and expected target before they can advance lifecycle state. + +Filtering rules: + +- run-scoped reports must match `run_id`; +- worker readiness reports must match expected `node_id`; +- stage reports must match expected `stage_index`; +- prompt events must match active `request_id`; +- tokenizer events must match active `request_id`. + +Mismatched reports are ignored or dropped for the active lifecycle path. + +### 9.20 Secrets Must Not Be Emitted by Orchestrator-Owned Records + +Orchestrator-owned datastream records and logs must not emit secret values. + +Allowed secret-related output is limited to presence metadata, such as whether a Vast.ai API key is configured. + +Provider tools may emit output outside the orchestrator’s control. When the orchestrator handles provider errors, it should redact or avoid copying secret values into structured records. + +--- + +## 10. Error Handling + +The orchestrator treats errors as phase-specific failures. Each failure should identify the phase that failed and the concrete operation or runtime condition that failed. + +Top-level process error format: + +```text +mvp-orchestrator: +``` + +Top-level process exit code: + +```text +0 = successful completion or controlled shutdown +1 = configuration, startup, provisioning, prompt-serving, shutdown, or runtime failure +``` + +### 10.1 Error Propagation Model + +Most orchestrator operations return: + +```text +Result<(), String> +``` + +or a typed success value with `String` error: + +```text +Result +``` + +The top-level `run()` function propagates the first unrecovered fatal error. + +When prompt serving has already started, shutdown combines two results: + +```text +prompt-serving result +provider-stop result +``` + +The process succeeds only when both succeed. + +If prompt serving fails, the orchestrator still attempts provider stop. + +If provider stop fails, the first provider-stop error is preserved and returned. + +### 10.2 Configuration Errors + +Configuration errors happen before worker provisioning. + +Configuration errors include: + +- unreadable or invalid TOML; +- unsupported runtime profile; +- unsupported provider; +- unknown process argument; +- missing process-argument value; +- invalid integer value; +- invalid floating-point value; +- invalid boolean value; +- invalid prompt RPC bind address; +- pipeline stage count of zero; +- pipeline stage count unsupported by the selected provider; +- missing process worker binary; +- invalid cached model path; +- cached model path selected with unsupported provider; +- missing required Vast.ai API key; +- missing required Vast.ai bootstrap command; +- missing or invalid Vast.ai SSH identity. + +Configuration errors must stop startup before workers are provisioned. + +### 10.3 Runtime Initialization Errors + +Runtime initialization errors happen while creating local orchestration services. + +Runtime initialization errors include: + +- failure to install stdio capture; +- failure to open the datastream frame archive; +- failure to create the Tokio runtime; +- failure to create the Iroh driver; +- failure to create the distribution runtime stack; +- failure to create local actor inboxes; +- failure to spawn the local orchestrator actor; +- failure to register local actors with the actor bridge; +- failure to start dashboard support; +- failure to create the pipeline edge endpoint. + +When possible, runtime initialization failures emit a bootstrap failure record before returning the error. + +### 10.4 Planning Errors + +Planning errors happen before workers are started. + +Planning errors include: + +- selected GGUF source is remote when local inspection is required; +- selected local GGUF path does not exist or is not a file; +- GGUF metadata cannot be read; +- GGUF metadata cannot be converted into model facts; +- requested pipeline stage count exceeds model layer count; +- activation ring sizing overflows; +- token ring sizing overflows; +- run planner rejects the requested placement or model facts. + +Planning errors stop startup before provider provisioning. + +### 10.5 Provider Preparation Errors + +Provider preparation errors happen before or during provider construction. + +Provider preparation errors include: + +- process provider worker binary missing; +- Docker provisioner setup failure; +- Vast.ai API client construction failure; +- Vast.ai SSH identity resolution failure; +- Vast.ai public-key derivation failure; +- Vast.ai account key lookup failure; +- Vast.ai account key registration failure; +- missing Vast.ai bootstrap command; +- missing prepared Vast.ai SSH identity. + +Provider preparation errors stop startup before workers are started. + +### 10.6 Provisioning Errors + +Provisioning errors happen while starting workers. + +Provisioning errors include: + +- provider `start_node` returns an error; +- provider reports `Failed`; +- provider reports worker `Exited` before readiness; +- worker process exits before ready; +- Docker container exits before ready; +- remote provider bootstrap fails before ready. + +If one worker fails to start after earlier workers started in the same provisioning attempt, the orchestrator must stop the already-started workers before returning the provisioning error. + +Once the provisioned-cluster guard owns worker handles, the guard is responsible for cleanup attempts. + +### 10.7 Runtime Readiness Errors + +Runtime readiness errors happen while waiting for workers to become usable. + +Readiness errors include: + +- shutdown requested while waiting for node ready; +- provider failure while waiting for node ready; +- worker exit while waiting for node ready; +- missing expected runtime-ready report; +- runtime-ready report for unexpected run or node; +- SWIM membership never reaches required alive state; +- actor route owner never matches expected worker; +- runtime-ready acknowledgement timeout; +- failure to send runtime-ready acknowledgement. + +Readiness wait loops must keep pumping actor and transport runtime while waiting. + +Some readiness waits do not have a fixed timeout. They end only when readiness succeeds, a shutdown request arrives, or a failure is observed. + +### 10.8 Stage Provisioning and Weight Loading Errors + +Stage provisioning and weight loading errors happen after runtime readiness and before prompt RPC readiness. + +Errors include: + +- unplanned single-stage execution missing required layer range; +- failure to send `ProvisionStage`; +- missing runtime-ready state for a planned stage; +- missing consumer endpoint for a planned edge; +- failure to derive stage provisioning from the run plan; +- stage fault while loading weights; +- worker exit while loading weights; +- provider failure while loading weights; +- shutdown requested while loading weights. + +A stage fault during weight loading is a startup failure. + +Prompt RPC must not be reported ready after a stage provisioning or weight-loading failure. + +### 10.9 Prompt RPC Errors + +Prompt RPC errors happen while binding, reading, writing, or forwarding prompt work. + +Errors include: + +- failure to bind the configured prompt RPC socket; +- failure to read the bound socket address; +- failure to clone a prompt TCP stream; +- malformed prompt request JSON; +- prompt work queue stopped; +- failure to serialize a prompt response; +- failure to write a prompt response; +- failure to flush a prompt response. + +Prompt RPC bind failure is a startup failure. + +Malformed prompt request handling is scoped to the client connection. It does not by itself require the orchestrator process to fail unless it stops prompt serving or exposes a runtime error. + +### 10.10 Prompt Serving Errors + +Prompt serving errors happen after prompt RPC is ready. + +Errors include: + +- provider reports failure; +- provider reports worker exit; +- actor send failure for direct prompt inference; +- actor send failure for tokenizer encode/decode; +- pipeline token-in sender stops; +- pipeline token sequence violation; +- token record decode failure; +- shutdown channel behavior that prevents controlled exit. + +Expected prompt-level faults are not prompt-serving errors. + +Prompt-level faults include: + +- worker returns `PromptEvent::Fault`; +- tokenizer returns `TokenizerEvent::Fault`. + +Prompt-level faults should be returned to the prompt RPC client as prompt `Fault` events for the active request. + +### 10.11 Datastream and Log Errors + +Datastream and log errors are split into startup errors and best-effort observation errors. + +Startup datastream/log errors include: + +- failure to create parent directories for the configured frame archive path; +- failure to open the configured frame archive file; +- failure to initialize the orchestrator datastream endpoint. + +These are startup failures. + +Best-effort observation errors include: + +- malformed datastream frame from a node; +- closed datastream connection; +- per-frame archive write failure after archive open; +- dashboard publication failure, when the dashboard sink can drop or reject frames without affecting runtime state. + +Best-effort observation errors should not change lifecycle state unless the code explicitly treats them as fatal. + +### 10.12 Actor Delivery Errors + +Actor delivery errors happen when sending through the Swactor runtime fails. + +Actor send failures are phase errors. + +Examples: + +- failure to send runtime-ready acknowledgement; +- failure to send datastream subscription request; +- failure to send stage provisioning; +- failure to send direct prompt inference; +- failure to send tokenizer encode request; +- failure to send tokenizer decode request. + +The error belongs to the lifecycle phase that attempted the send. + +A successful send means the actor runtime accepted the message for delivery. It does not prove the remote actor processed the message. + +### 10.13 Provider Stop Errors + +Provider stop errors happen during explicit shutdown or guard cleanup. + +Explicit provider stop behavior: + +- stop every remaining worker handle; +- preserve the first stop error; +- continue attempting to stop remaining handles; +- return the first stop error after all handles have been attempted. + +Drop cleanup behavior: + +- attempt to stop remaining handles; +- ignore stop errors because drop cannot return them. + +Provider stop failure makes the orchestrator exit with failure unless a prior fatal error is already being reported. + +### 10.14 Controlled Shutdown + +Controlled shutdown is requested by standard input control words: + +```text +stop +shutdown +quit +``` + +Controlled shutdown is not an error by itself. + +A controlled shutdown succeeds only if prompt serving exits cleanly and provider stop succeeds. + +If controlled shutdown is requested while startup is waiting for readiness or weight loading, the wait loop returns a shutdown-requested error for that startup phase. + +### 10.15 Error Reporting Through Datastream + +When the orchestrator has a datastream available, it should emit failure records for the phase that failed. + +Failure records should include: + +- phase; +- status `failed`; +- run id; +- node id when applicable; +- provider when applicable; +- error string or reason. + +Datastream failure records are diagnostic. The actual process result is still determined by returned errors and provider stop result. + +### 10.16 Secret Redaction + +Error messages and datastream failure records must avoid exposing configured secrets. + +Secrets include: + +- Vast.ai API keys; +- Hugging Face tokens; +- SSH private-key material; +- provider credentials. + +Secret presence may be reported. Secret values must not be copied into orchestrator-owned records. + +--- + +## 11. Out of Scope + +This document defines the orchestrator contract. It does not define every subsystem the orchestrator calls, hosts, or observes. + +Out of scope: + +- `mvp-chat` behavior, including interactive terminal UX, wrapper argument parsing, image preparation, rebuild policy, and user-facing prompt formatting. + +- Worker-node internals, including model execution, TinyGrad helper behavior, CUDA behavior, worker process command protocol, weight loading implementation, tensor allocation, and device cleanup details. + +- Model quality, sampling quality, tokenizer correctness, generated text quality, or semantic correctness of model outputs. + +- GGUF format semantics beyond the orchestrator’s need to inspect metadata for planned execution. + +- Docker image construction, Dockerfile contents, registry authentication, image freshness, image tagging policy, image push behavior, and image garbage collection. + +- Docker daemon behavior beyond the provider result and observations returned to the orchestrator. + +- Vast.ai marketplace semantics, offer selection quality, billing behavior, host reliability, remote image pull behavior, and remote shell behavior beyond provider success, failure, logs, and bootstrap status. + +- SSH protocol details, SSH agent behavior, host key policy, key generation UX, and remote shell semantics beyond the orchestrator’s use of a configured identity and provider bootstrap launcher. + +- Iroh protocol internals, relay implementation details, NAT traversal behavior, transport congestion behavior, and cryptographic details beyond the actor and datastream connectivity required by this contract. + +- Swactor runtime internals beyond actor addressing, message delivery through the local runtime, and Iroh actor bridge integration used by the orchestrator. + +- Datastream library internals beyond the frame, channel, record, dashboard, and archive behavior stated in this document. + +- Dashboard rendering semantics, dashboard UI layout, dashboard persistence, and dashboard query APIs. + +- Full security threat model, authentication model, authorization model, secret storage policy, or audit-log policy. + +- External provider resource cleanup guarantees after the orchestrator has issued its provider stop calls. + +- Cross-process supervision outside the orchestrator process. + +- Long-term compatibility guarantees for implementation-private phase names, debug details, or non-contract telemetry fields. + +- Performance guarantees, latency targets, throughput targets, GPU utilization targets, and prompt generation speed. + +- Retry policies not explicitly stated in this document. + +- Recovery after orchestrator process crash. + +- Multi-run orchestration in a single process. diff --git a/crates/mvp-system/src/bin/mvp_chat.rs b/crates/mvp-system/src/bin/mvp_chat.rs index 9f16d55..371547e 100644 --- a/crates/mvp-system/src/bin/mvp_chat.rs +++ b/crates/mvp-system/src/bin/mvp_chat.rs @@ -1,61 +1,54 @@ -use std::collections::HashSet; -use std::fs::{self, File}; -use std::io::{self, BufRead, BufReader, IsTerminal, Read, Seek, SeekFrom, Write}; +use std::fs; +use std::io::{self, BufRead, BufReader, IsTerminal, Write}; use std::net::{Shutdown, TcpStream}; +#[cfg(all(target_os = "linux", not(test)))] +use std::os::unix::process::CommandExt; use std::path::{Path, PathBuf}; -use std::process::{Child, ChildStdin, Command, ExitCode, Stdio}; +use std::process::{Child, Command, ExitCode, Stdio}; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::mpsc; +use std::sync::{Mutex, mpsc}; use std::thread; -use std::time::{Duration, SystemTime}; +use std::time::{Duration, Instant}; + +use serde::Deserialize; +#[cfg(target_os = "linux")] +use signal_hook::consts::signal::{SIGINT, SIGTERM}; +#[cfg(target_os = "linux")] +use signal_hook::iterator::Signals; use mvp_system::config as chat_config; use mvp_system::config::ResolvedVastAiConfig; -use mvp_system::node_image::{NodeImageProvider, NodeImageRequest, prepare_node_image}; +use mvp_system::node_image::{ + NodeImageProvider, NodeImageRequest, PreparedNodeImage, prepare_node_image, +}; use mvp_system::node_provisioning::ProviderKind; use mvp_system::prompt_rpc::{PromptEvent, SubmitPrompt, write_json_line}; -use mvp_system::vastai_offer_preview::{OfferPreview, OfferPreviewer, VastAiOfferPreviewer}; -use serde_json::Value; - -#[cfg(target_os = "linux")] -use std::os::unix::process::CommandExt; const DEFAULT_RPC_ADDR: &str = "127.0.0.1:19777"; -const DEFAULT_NODE_IMAGE: &str = "swactor-mvp-node:latest"; const BASE_NODE_IMAGE: &str = "swactor-mvp-node-base:cuda12.6"; -const MVP_RUNTIME_CONFIG_ENV: &str = "MVP_RUNTIME_CONFIG"; -const DEFAULT_CACHED_MODEL_FILE: &str = "SmolLM2-135M-Instruct.Q4_0.gguf"; const REPO_MODEL_CACHE_DIR: &str = ".model-cache"; const DEFAULT_MAX_TOKENS: u32 = 64; -const ORCH_REBUILD_INPUTS: &[&str] = &[ - "Cargo.lock", - "Cargo.toml", - "src", - "crates/datastream/Cargo.toml", - "crates/datastream/src", - "crates/dashboard/Cargo.toml", - "crates/dashboard/src", - "crates/distribution/Cargo.toml", - "crates/distribution/src", - "crates/iroh-driver/Cargo.toml", - "crates/iroh-driver/src", - "crates/mvp-system/Cargo.toml", - "crates/mvp-system/src", - "crates/transport/Cargo.toml", - "crates/transport/src", - "tools/vastai/Cargo.toml", - "tools/vastai/src", -]; -const CHAT_READ_TIMEOUT: Duration = Duration::from_millis(100); +const ORCH_SHUTDOWN_GRACE_MS: u64 = 5_000; +const ORCH_SHUTDOWN_POLL_MS: u64 = 50; + +#[derive(Debug)] +enum PromptInput { + Line(String), + Closed, + StopRequested, +} static STOP_REQUESTED: AtomicBool = AtomicBool::new(false); -static STOP_ACKNOWLEDGED: AtomicBool = AtomicBool::new(false); +static PROMPT_STOP_TX: Mutex>> = Mutex::new(None); pub fn run_from_args(args: I) -> ExitCode where I: IntoIterator, { - install_signal_handlers(); + if let Err(error) = install_signal_handlers() { + eprintln!("mvp-chat: {error}"); + return ExitCode::from(1); + } match run(args) { Ok(()) => ExitCode::SUCCESS, Err(error) => { @@ -69,68 +62,118 @@ fn run(args: I) -> Result<(), String> where I: IntoIterator, { - let mut config = Config::from_args(args)?; - confirm_vastai_if_needed(&config, VastAiOfferPreviewer)?; + let config = Config::from_args(args)?; + confirm_vastai_if_needed(&config)?; let image_ref = prepare_runtime(&config)?; - let frame_log = configure_progress_frame_log(&mut config)?; - if let Some(path) = &config.datastream_frame_log { - eprintln!( - "mvp-chat: dumping datastream frames to {}", - display_user_path(path) - ); - } - let mut progress = StartupProgress::new(&frame_log); let mut orch = OrchChild::spawn(&config, &image_ref)?; - let rpc_addr = match orch.wait_ready(config.rpc_addr.clone(), &mut progress) { + let rpc_addr = match orch.wait_ready(config.rpc_addr.clone()) { Ok(addr) => addr, Err(_) if STOP_REQUESTED.load(Ordering::SeqCst) => { - acknowledge_stop(); - let _ = orch.shutdown(true); - report_interrupt_shutdown(); + orch.shutdown(); return Ok(()); } - Err(error) => { - progress.poll(); - return Err(progress.failure_summary().unwrap_or(error)); - } + Err(error) => return Err(error), }; - progress.poll(); - println!("model successfully loaded."); let result = run_chat_loop(&rpc_addr, config.max_tokens); - let interrupted = STOP_REQUESTED.load(Ordering::SeqCst); - let _ = orch.shutdown(interrupted); - if interrupted { - report_interrupt_shutdown(); - } + orch.shutdown(); result } struct Config { orch_bin: PathBuf, worker_bin: PathBuf, - orch_args: Vec, rpc_addr: String, node_image: String, - config_profile: RuntimeConfigProfile, provider: ProviderKind, - relay_mode: iroh::RelayMode, - relay_url: Option, - max_tokens: u32, - dashboard: bool, - build_image: bool, image_tag: Option, - push_image: bool, - force_image_refresh: bool, cached_model: Option, datastream_frame_log: Option, vastai_yes: bool, vastai: Option, - model_id: Option, - gguf_repo: Option, - gguf_file: Option, - gguf_revision: Option, - max_context: Option, pipeline_stages: u32, + max_tokens: u32, + skip_rebuild: bool, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +struct ChatTomlConfig { + provider: ChatProviderConfig, + runtime: ChatRuntimeConfig, + observability: ChatObservabilityConfig, + image: ChatImageConfig, + vastai: ChatVastAiConfig, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +struct ChatProviderConfig { + kind: Option, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +struct ChatRuntimeConfig { + pipeline_stages: Option, + max_tokens: Option, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +struct ChatObservabilityConfig { + dump_logs: Option, + dump_log_path: Option, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +struct ChatImageConfig { + node: Option, + tag: Option, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +struct ChatVastAiConfig { + relay_url: Option, + bootstrap_command: Option, + gpu_name: Option, + min_gpu_ram_mb: Option, + min_down_mbps: Option, + min_up_mbps: Option, + min_reliability: Option, + require_verified: Option, + disk_gb: Option, + onstart: Option, + ssh_identity: Option, +} + +#[derive(Clone, Debug)] +struct LoadedChatTomlConfig { + overlay: ChatTomlConfig, +} + +fn load_chat_config(path: Option<&Path>) -> Result { + let overlay = match path { + Some(path) => { + let text = fs::read_to_string(path) + .map_err(|e| format!("read config {}: {e}", path.display()))?; + toml::from_str::(&text) + .map_err(|e| format!("parse config {}: {e}", path.display()))? + } + None => { + let default = Path::new(chat_config::DEFAULT_CONFIG_PATH); + if !default.is_file() { + ChatTomlConfig::default() + } else { + let text = fs::read_to_string(default) + .map_err(|e| format!("read config {}: {e}", default.display()))?; + toml::from_str::(&text) + .map_err(|e| format!("parse config {}: {e}", default.display()))? + } + } + }; + Ok(LoadedChatTomlConfig { overlay }) } impl Config { @@ -139,159 +182,69 @@ impl Config { I: IntoIterator, { let args = ParsedArgs::parse(provided_args)?; - let loaded = chat_config::TomlConfigOverlay::load(args.config_path.as_deref())?; + let loaded = load_chat_config(args.config_path.as_deref())?; let toml = loaded.overlay; - let profile = if args.vastai { - RuntimeConfigProfile::Deploy - } else { - RuntimeConfigProfile::from_env()? - }; - let provider = - provider_from_sources(args.provider, toml.provider.kind.as_deref(), profile)?; - let relay_mode = relay_mode_from_sources(toml.relay.mode.as_deref())?; - let relay_url = first_non_empty([ - env_optional("MVP_IROH_RELAY_URL"), - env_optional("SWACTOR_IROH_RELAY_URL"), - toml.relay.url.clone(), - ]); - let node_image = first_non_empty([ - args.image, - env_optional("MVP_NODE_IMAGE"), - if args.vastai { - toml.vastai.image.clone() - } else { - None - }, - toml.image.node.clone(), - Some(DEFAULT_NODE_IMAGE.to_owned()), - ]) - .expect("default image is non-empty"); - let max_tokens = args - .max_tokens - .or(env_u32_optional("MVP_PROMPT_MAX_TOKENS")?) - .or(toml.prompt.max_tokens) - .unwrap_or(DEFAULT_MAX_TOKENS); - let dashboard = args - .dashboard - .or(env_bool_optional("MVP_DASHBOARD")?) - .or(toml.prompt.dashboard) - .unwrap_or(true); - let build_image = args - .build_image - .or(env_bool_optional("MVP_BUILD_NODE_IMAGE")?) - .or(toml.image.build) - .unwrap_or(true); - let push_image = args - .push_image - .or(env_bool_optional("MVP_PUSH_NODE_IMAGE")?) - .or(toml.image.push) - .unwrap_or(false); - let force_image_refresh = args - .force_image_refresh - .or(env_bool_optional("MVP_FORCE_NODE_IMAGE_REFRESH")?) - .or(toml.image.force_refresh) - .unwrap_or(false); - let image_tag = first_non_empty([ - args.image_tag, - env_optional("MVP_NODE_IMAGE_TAG"), - toml.image.tag.clone(), - ]); - let rpc_addr = first_non_empty([ - args.rpc_addr, - env_optional("MVP_PROMPT_RPC_ADDR"), - env_optional("MVP_PROMPT_RPC_BIND"), - toml.prompt.rpc_addr.clone(), - Some(DEFAULT_RPC_ADDR.to_owned()), - ]) - .expect("default RPC address is non-empty"); - let datastream_frame_log = match (args.dump_logs, args.datastream_frame_log) { - (true, Some(_)) => { - return Err("--dump-logs cannot be combined with --datastream-frame-log; use one datastream log destination".to_owned()); - } - (true, None) => Some(PathBuf::from("mvp-chat.log")), - (false, explicit) => explicit - .or_else(|| env_optional("MVP_DATASTREAM_FRAME_LOG").map(PathBuf::from)) - .or_else(|| { - toml.observability - .datastream_frame_log - .clone() - .map(PathBuf::from) - }), - }; - let model_id = first_non_empty([env_optional("MVP_MODEL_ID"), toml.model.id.clone()]); - let gguf_repo = - first_non_empty([env_optional("MVP_GGUF_REPO"), toml.model.gguf_repo.clone()]); - let gguf_file = - first_non_empty([env_optional("MVP_GGUF_FILE"), toml.model.gguf_file.clone()]); - let gguf_revision = first_non_empty([ - env_optional("MVP_GGUF_REVISION"), - toml.model.gguf_revision.clone(), - ]); - let max_context = env_u32_optional("MVP_MAX_CONTEXT")?.or(toml.model.max_context); + let provider = provider_from_sources(args.provider, toml.provider.kind.as_deref())?; + let node_image = first_non_empty([toml.image.node.clone()]).unwrap_or_default(); + if provider != ProviderKind::Process && node_image.is_empty() { + return Err("node image is required for docker or vastai provider".to_owned()); + } let pipeline_stages = args .pipeline_stages - .or(env_u32_optional("MVP_PIPELINE_STAGES")?) .or(toml.runtime.pipeline_stages) .unwrap_or(1); if pipeline_stages == 0 { return Err("--pipeline-stages must be greater than 0".to_owned()); } - if provider == ProviderKind::VastAi && pipeline_stages > 1 { - return Err( - "pipeline stages greater than 1 are only supported with provider=docker; --vastai cannot be combined with -N/--pipeline-stages > 1".to_owned(), - ); + let max_tokens = toml.runtime.max_tokens.unwrap_or(DEFAULT_MAX_TOKENS); + if max_tokens == 0 { + return Err("[runtime].max_tokens must be greater than 0".to_owned()); } - let cached_model = match ( - args.cached_model, - toml.docker.cached_model_host_path.clone(), - ) { - (Some(cached_model), _) => Some(cached_model), - (None, Some(path)) => Some(CachedModelConfig::from_arg(Some(path))?), - (None, None) => None, + let cached_model = args + .cached_model + .map(CachedModelConfig::from_source) + .transpose()?; + let datastream_frame_log = if args.dump_logs { + Some( + args.dump_log_path + .unwrap_or_else(|| PathBuf::from("mvp-chat.log")), + ) + } else if toml.observability.dump_logs.unwrap_or(false) { + Some( + first_non_empty([toml.observability.dump_log_path.clone()]) + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from("mvp-chat.log")), + ) + } else { + None }; - let vastai = if args.vastai { - Some(resolve_vastai_config( - &toml.vastai, - &node_image, - relay_url.clone(), - )?) + let vastai = if provider == ProviderKind::VastAi { + Some(resolve_vastai_config(&toml.vastai, &node_image)?) } else { None }; Ok(Self { - orch_bin: args.orch_bin.unwrap_or(default_orch_bin()?), - worker_bin: args.worker_bin.unwrap_or(node_bin_for_current_profile()?), - orch_args: args.orch_args, - rpc_addr, + orch_bin: default_orch_bin()?, + worker_bin: node_bin_for_current_profile()?, + rpc_addr: DEFAULT_RPC_ADDR.to_owned(), node_image, - config_profile: profile, provider, - relay_mode, - relay_url, - max_tokens, - dashboard, - build_image, - image_tag, - push_image, - force_image_refresh, + image_tag: first_non_empty([toml.image.tag.clone()]), cached_model, datastream_frame_log, vastai_yes: args.vastai_yes, - model_id, - gguf_repo, - gguf_file, - gguf_revision, - max_context, pipeline_stages, + max_tokens, vastai, + skip_rebuild: args.skip_rebuild, }) } + // The orchestrator launch spec is still pending. These flags are the current adapter; + // adjust this mapping when the approved orchestrator launch contract is finalized. fn orchestrator_cli_args(&self, image_ref: &str) -> Vec { let mut args = vec![ - "--runtime-config".to_owned(), - self.config_profile.as_str().to_owned(), "--provider".to_owned(), self.provider.as_str().to_owned(), "--image".to_owned(), @@ -302,40 +255,13 @@ impl Config { self.max_tokens.to_string(), "--pipeline-stages".to_owned(), self.pipeline_stages.to_string(), - if self.dashboard { - "--dashboard".to_owned() - } else { - "--no-dashboard".to_owned() - }, ]; - if let Some(relay_url) = &self.relay_url { - args.extend(["--relay-url".to_owned(), relay_url.clone()]); - } - args.extend([ - "--relay-mode".to_owned(), - relay_mode_env_value(&self.relay_mode).to_owned(), - ]); if self.provider == ProviderKind::Process { args.extend([ "--worker-bin".to_owned(), self.worker_bin.to_string_lossy().to_string(), ]); } - if let Some(model_id) = &self.model_id { - args.extend(["--model-id".to_owned(), model_id.clone()]); - } - if let Some(repo) = &self.gguf_repo { - args.extend(["--gguf-repo".to_owned(), repo.clone()]); - } - if let Some(file) = &self.gguf_file { - args.extend(["--gguf-file".to_owned(), file.clone()]); - } - if let Some(revision) = &self.gguf_revision { - args.extend(["--gguf-revision".to_owned(), revision.clone()]); - } - if let Some(max_context) = self.max_context { - args.extend(["--max-context".to_owned(), max_context.to_string()]); - } if let Some(cached_model) = &self.cached_model { args.extend([ "--cached-model-host-path".to_owned(), @@ -403,28 +329,24 @@ impl Config { #[derive(Default, Debug)] struct ParsedArgs { - vastai: bool, provider: Option, vastai_yes: bool, config_path: Option, - orch_bin: Option, - worker_bin: Option, - rpc_addr: Option, - image: Option, - max_tokens: Option, pipeline_stages: Option, - datastream_frame_log: Option, dump_logs: bool, - dashboard: Option, - build_image: Option, - image_tag: Option, - push_image: Option, - force_image_refresh: Option, - cached_model: Option, - orch_args: Vec, + dump_log_path: Option, + skip_rebuild: bool, + cached_model: Option, } -const PROVIDER_SELECTOR_CONFLICT: &str = "conflicting provider selectors; use exactly one of --process, --docker, --vastai, or --provider "; +#[derive(Clone, Debug, PartialEq, Eq)] +enum CachedModelSource { + Discover, + Path(PathBuf), +} + +const PROVIDER_SELECTOR_CONFLICT: &str = + "conflicting provider selectors; use exactly one of --process, --docker, or --vastai"; impl ParsedArgs { fn set_provider_selector(&mut self, provider: ProviderKind) -> Result<(), String> { @@ -443,58 +365,42 @@ impl ParsedArgs { let mut args = provided_args.into_iter().peekable(); while let Some(arg) = args.next() { match arg.as_str() { - "--vastai" => { - parsed.vastai = true; - parsed.set_provider_selector(ProviderKind::VastAi)?; - } - "--process" | "--local-process" => { - parsed.set_provider_selector(ProviderKind::Process)? - } + "--vastai" => parsed.set_provider_selector(ProviderKind::VastAi)?, + "--process" => parsed.set_provider_selector(ProviderKind::Process)?, "--docker" => parsed.set_provider_selector(ProviderKind::Docker)?, - "--provider" => { - let provider = ProviderKind::parse_deploy(&next_arg(&mut args, "--provider")?)?; - parsed.set_provider_selector(provider)?; - } "--yes" | "-y" => parsed.vastai_yes = true, "--config" => { parsed.config_path = Some(PathBuf::from(next_arg(&mut args, "--config")?)) } - "--orch-bin" => { - parsed.orch_bin = Some(PathBuf::from(next_arg(&mut args, "--orch-bin")?)) - } - "--worker-bin" => { - parsed.worker_bin = Some(PathBuf::from(next_arg(&mut args, "--worker-bin")?)) - } - "--addr" => parsed.rpc_addr = Some(next_arg(&mut args, "--addr")?), - "--image" => parsed.image = Some(next_arg(&mut args, "--image")?), - "--max-tokens" => parsed.max_tokens = Some(parse_next(&mut args, "--max-tokens")?), - "-N" | "--pipeline-stages" => { + "--pipeline-stages" => { parsed.pipeline_stages = Some(parse_pipeline_stages_value(&mut args, arg.as_str())?) } - "--datastream-frame-log" => { - parsed.datastream_frame_log = Some(PathBuf::from(next_arg( - &mut args, - "--datastream-frame-log", - )?)); + "--dump-logs" => { + parsed.dump_logs = true; + } + value if value.starts_with("--dump-logs=") => { + let path = value.strip_prefix("--dump-logs=").expect("prefix checked"); + if path.is_empty() { + return Err("--dump-logs path must not be empty".to_owned()); + } + parsed.dump_logs = true; + parsed.dump_log_path = Some(PathBuf::from(path)); } - "--dump-logs" => parsed.dump_logs = true, - "--dashboard" => parsed.dashboard = Some(true), - "--no-dashboard" => parsed.dashboard = Some(false), - "--no-build-image" => parsed.build_image = Some(false), - "--image-tag" => parsed.image_tag = Some(next_arg(&mut args, "--image-tag")?), - "--push-image" => parsed.push_image = Some(true), - "--no-push-image" => parsed.push_image = Some(false), - "--force-image-refresh" => parsed.force_image_refresh = Some(true), "--cached-model" => { - let path = args.next_if(|value| !value.starts_with('-')); - parsed.cached_model = Some(CachedModelConfig::from_arg(path)?); + parsed.cached_model = Some(CachedModelSource::Discover); } - "--" => { - parsed.orch_args.extend(args); - break; + value if value.starts_with("--cached-model=") => { + let path = value + .strip_prefix("--cached-model=") + .expect("prefix checked"); + if path.is_empty() { + return Err("--cached-model path must not be empty".to_owned()); + } + parsed.cached_model = Some(CachedModelSource::Path(PathBuf::from(path))); } - other => parsed.orch_args.push(other.to_owned()), + "--skip-rebuild" => parsed.skip_rebuild = true, + other => return Err(format!("unsupported mvp-chat argument {other:?}")), } } Ok(parsed) @@ -502,37 +408,23 @@ impl ParsedArgs { } fn resolve_vastai_config( - file: &chat_config::VastAiConfig, + file: &ChatVastAiConfig, node_image: &str, - relay_url: Option, ) -> Result { ResolvedVastAiConfig { - api_key: first_non_empty([ - env_optional("MVP_VASTAI_API_KEY"), - env_optional("VAST_API_KEY"), - file.api_key.clone(), - ]) - .unwrap_or_default(), - relay_url: relay_url.unwrap_or_default(), + api_key: first_non_empty([env_optional("VAST_API_KEY")]).unwrap_or_default(), + relay_url: first_non_empty([file.relay_url.clone()]).unwrap_or_default(), image: node_image.to_owned(), - bootstrap_command: first_non_empty([ - env_optional("MVP_VASTAI_BOOTSTRAP_COMMAND"), - file.bootstrap_command.clone(), - ]) - .unwrap_or_default(), - disk_gb: env_u32_optional("MVP_VASTAI_DISK_GB")?.or(file.disk_gb), - gpu_name: first_non_empty([env_optional("MVP_VASTAI_GPU_NAME"), file.gpu_name.clone()]), - min_gpu_ram_mb: env_u64_optional("MVP_VASTAI_MIN_GPU_RAM_MB")?.or(file.min_gpu_ram_mb), - min_down_mbps: env_f64_optional("MVP_VASTAI_MIN_DOWN_MBPS")?.or(file.min_down_mbps), - min_up_mbps: env_f64_optional("MVP_VASTAI_MIN_UP_MBPS")?.or(file.min_up_mbps), - min_reliability: env_f64_optional("MVP_VASTAI_MIN_RELIABILITY")?.or(file.min_reliability), - require_verified: env_bool_optional("MVP_VASTAI_REQUIRE_VERIFIED")? - .or(file.require_verified), - onstart: first_non_empty([env_optional("MVP_VASTAI_ONSTART"), file.onstart.clone()]), - ssh_identity: first_non_empty([ - env_optional("MVP_VASTAI_SSH_IDENTITY"), - file.ssh_identity.clone(), - ]), + bootstrap_command: first_non_empty([file.bootstrap_command.clone()]).unwrap_or_default(), + disk_gb: file.disk_gb, + gpu_name: first_non_empty([file.gpu_name.clone()]), + min_gpu_ram_mb: file.min_gpu_ram_mb, + min_down_mbps: file.min_down_mbps, + min_up_mbps: file.min_up_mbps, + min_reliability: file.min_reliability, + require_verified: file.require_verified, + onstart: first_non_empty([file.onstart.clone()]), + ssh_identity: first_non_empty([file.ssh_identity.clone()]), } .validate() } @@ -541,48 +433,75 @@ fn first_non_empty(values: [Option; N]) -> Option(config: &Config, previewer: P) -> Result<(), String> +fn confirm_vastai_if_needed(config: &Config) -> Result<(), String> { + let mut approval = StdinVastAiApproval; + confirm_vastai_if_needed_with_approval(config, &mut approval) +} + +trait VastAiApproval { + fn stdin_is_terminal(&self) -> bool; + fn ask(&mut self) -> Result; +} + +struct StdinVastAiApproval; + +impl VastAiApproval for StdinVastAiApproval { + fn stdin_is_terminal(&self) -> bool { + io::stdin().is_terminal() + } + + fn ask(&mut self) -> Result { + #[cfg(test)] + { + let mut input = std::io::Cursor::new(Vec::::new()); + let mut output = io::sink(); + ask_vastai_approval(&mut input, &mut output) + } + #[cfg(not(test))] + { + let stdin = io::stdin(); + let mut input = stdin.lock(); + let mut output = io::stdout(); + ask_vastai_approval(&mut input, &mut output) + } + } +} + +fn confirm_vastai_if_needed_with_approval( + config: &Config, + approval: &mut A, +) -> Result<(), String> where - P: OfferPreviewer, + A: VastAiApproval, { - let Some(vastai) = &config.vastai else { - return Ok(()); - }; - eprintln!("mvp-chat: checking Vast.ai offers..."); - let preview = previewer.preview(&vastai.api_key, &vastai.selection_policy())?; - print_offer_preview(&preview); - if config.vastai_yes { - eprintln!("mvp-chat: --yes supplied; skipping Vast.ai rental prompt"); + if config.vastai.is_none() { return Ok(()); } - if !io::stdin().is_terminal() { + if config.vastai_yes { + return Ok(()); + } + if !approval.stdin_is_terminal() { return Err("Vast.ai rental requires --yes when stdin is not a terminal".to_owned()); } - if ask_vastai_approval()? { + if approval.ask()? { Ok(()) } else { Err("Vast.ai rental declined".to_owned()) } } -fn print_offer_preview(preview: &OfferPreview) { - let ram = preview - .gpu_ram_mb - .map(|mb| format!(", {mb} MB VRAM")) - .unwrap_or_default(); - eprintln!( - "mvp-chat: best Vast.ai offer {}{} at ${:.3}/hr", - preview.gpu_name, ram, preview.dollars_per_hour - ); -} - -fn ask_vastai_approval() -> Result { - eprint!("Rent 1 Vast.ai node? [y/N]: "); - io::stderr() +fn ask_vastai_approval(input: &mut R, output: &mut W) -> Result +where + R: BufRead, + W: Write, +{ + write!(output, "Rent 1 Vast.ai node? [y/N]: ") + .map_err(|e| format!("write Vast.ai approval prompt: {e}"))?; + output .flush() .map_err(|e| format!("flush Vast.ai approval prompt: {e}"))?; let mut line = String::new(); - io::stdin() + input .read_line(&mut line) .map_err(|e| format!("read Vast.ai approval: {e}"))?; Ok(parse_approval(&line)) @@ -592,272 +511,20 @@ fn parse_approval(input: &str) -> bool { matches!(input.trim().to_ascii_lowercase().as_str(), "y" | "yes") } -struct FrameLogConfig { - path: PathBuf, - start_offset: u64, - remove_on_drop: bool, -} - -fn configure_progress_frame_log(config: &mut Config) -> Result { - if let Some(path) = &config.datastream_frame_log { - let start_offset = fs::metadata(path) - .map(|metadata| metadata.len()) - .unwrap_or(0); - return Ok(FrameLogConfig { - path: path.clone(), - start_offset, - remove_on_drop: false, - }); - } - - let path = PathBuf::from("target") - .join("mvp-chat") - .join(format!("startup-{}.frames.jsonl", std::process::id())); - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .map_err(|e| format!("create startup frame log dir {}: {e}", parent.display()))?; - } - File::create(&path).map_err(|e| format!("create startup frame log {}: {e}", path.display()))?; - config.datastream_frame_log = Some(path.clone()); - Ok(FrameLogConfig { - path, - start_offset: 0, - remove_on_drop: true, - }) -} - -impl Drop for FrameLogConfig { - fn drop(&mut self) { - if self.remove_on_drop { - let _ = fs::remove_file(&self.path); - } - } -} - -struct StartupProgress { - path: PathBuf, - offset: u64, - partial: String, - printed: HashSet, - last_error_line: Option, - last_failure: Option, - last_download_bucket: Option, - provider: Option, -} - -impl StartupProgress { - fn new(frame_log: &FrameLogConfig) -> Self { - Self { - path: frame_log.path.clone(), - offset: frame_log.start_offset, - partial: String::new(), - printed: HashSet::new(), - last_error_line: None, - last_failure: None, - last_download_bucket: None, - provider: None, - } - } - - fn poll(&mut self) { - let mut file = match File::open(&self.path) { - Ok(file) => file, - Err(_) => return, - }; - if file.seek(SeekFrom::Start(self.offset)).is_err() { - return; - } - let mut chunk = String::new(); - if file.read_to_string(&mut chunk).is_err() || chunk.is_empty() { - return; - } - self.offset += chunk.as_bytes().len() as u64; - self.partial.push_str(&chunk); - while let Some(newline) = self.partial.find('\n') { - let line: String = self.partial.drain(..=newline).collect(); - let line = line.trim(); - if !line.is_empty() { - self.observe_archive_line(line); - } - } - } - - fn failure_summary(&self) -> Option { - if let Some(line) = &self.last_error_line { - return Some(format!("node provisioning failed: {line}")); - } - self.last_failure.clone() - } - - fn observe_archive_line(&mut self, line: &str) { - let Ok(record) = serde_json::from_str::(line) else { - return; - }; - let Some(channel) = record.get("channel").and_then(Value::as_str) else { - return; - }; - let Some(payload_text) = record - .get("payload") - .and_then(|payload| payload.get("value")) - .and_then(Value::as_str) - else { - return; - }; - let Ok(payload) = serde_json::from_str::(payload_text) else { - return; - }; - - if channel == "mvp.orch.bootstrap" { - self.observe_bootstrap(&payload); - } else if channel == "mvp.provisioning.events" { - self.observe_provision_event(&payload); - } else if channel == "mvp.worker.weights" { - self.observe_worker_weights(&payload); - } else if channel.starts_with("mvp.provisioning.logs.") { - self.observe_provision_log(&payload); - } - } - - fn observe_bootstrap(&mut self, payload: &Value) { - let phase = payload.get("phase").and_then(Value::as_str).unwrap_or(""); - let status = payload.get("status").and_then(Value::as_str).unwrap_or(""); - if phase == "config" && status == "ready" { - if let Some(provider) = payload - .get("detail") - .and_then(|detail| detail.get("provider")) - .and_then(Value::as_str) - { - self.provider = Some(provider.to_owned()); - } - } - if status == "failed" { - let error = payload - .get("detail") - .and_then(|detail| detail.get("error")) - .and_then(Value::as_str) - .unwrap_or("unknown error"); - self.last_failure = Some(format!("{phase} failed: {error}")); - return; - } - match (phase, status) { - ("provider_start", "started") => self.print_provider_start(), - ("node_runtime_ready", "started") => { - self.print_once("node_runtime_ready_started", "waiting for node runtime") - } - ("node_runtime_ready", "ready") => { - self.print_once("node_runtime_ready", "node runtime ready") - } - ("stage_provision", "started") => { - self.print_once("stage_provision", "configuring model stage") - } - ("weights_loaded", "started") => { - self.print_once("weights_loaded_started", "loading model") - } - ("prompt_rpc", "ready") | ("prompt_loop", "ready") => { - self.print_once("prompt_ready", "prompt RPC ready") - } - _ => {} - } - } - - fn observe_provision_event(&mut self, payload: &Value) { - let Some(event) = payload.get("event") else { - return; - }; - match event.get("kind").and_then(Value::as_str).unwrap_or("") { - "ProvisionStart" => self.print_provider_start(), - "NodeLive" => self.print_once("node_runtime_ready", "node runtime ready"), - "ProvisionFailed" => { - let message = event - .get("message") - .and_then(Value::as_str) - .unwrap_or("provisioning failed"); - self.last_failure = Some(format!("node provisioning failed: {message}")); - } - _ => {} - } - } - - fn observe_worker_weights(&mut self, payload: &Value) { - match payload.get("type").and_then(Value::as_str).unwrap_or("") { - "GgufDownloadStarted" => { - self.print_once("download_started", "downloading model weights") - } - "GgufDownloadProgress" => { - let done = payload - .get("bytes_done") - .and_then(Value::as_u64) - .unwrap_or(0); - let total = payload - .get("bytes_total") - .and_then(Value::as_u64) - .unwrap_or(0); - if total == 0 { - return; - } - let pct = done.saturating_mul(100).saturating_div(total).min(100); - let bucket = pct / 10; - if self.last_download_bucket != Some(bucket) { - self.last_download_bucket = Some(bucket); - eprintln!("mvp-chat: downloading model weights {pct}%"); - } - } - _ => {} - } - } - - fn observe_provision_log(&mut self, payload: &Value) { - let Some(line) = payload - .get("line") - .and_then(|line| line.get("line")) - .and_then(Value::as_str) - .map(str::trim) - .filter(|line| !line.is_empty()) - else { - return; - }; - if line.starts_with("docker:") || line.contains("Error response") || line.contains("error") - { - self.last_error_line = Some(line.to_owned()); - } - } - - fn print_provider_start(&mut self) { - match self.provider.as_deref().unwrap_or("provider") { - "process" => self.print_once("starting_process_node", "starting process node"), - "docker" => self.print_once("starting_docker_node", "starting docker node"), - "vastai" => self.print_once("starting_vastai_node", "starting Vast.ai node"), - other => self.print_once( - format!("starting_{other}_node"), - format!("starting {other} node"), - ), - } - } - - fn print_once(&mut self, key: impl Into, message: impl AsRef) { - if self.printed.insert(key.into()) { - eprintln!("mvp-chat: {}", message.as_ref()); - } - } -} - struct OrchChild { child: Child, - stdin: Option, cleaned: bool, } impl OrchChild { fn spawn(config: &Config, image_ref: &str) -> Result { let mut command = Command::new(&config.orch_bin); - let mut orch_args = config.orchestrator_cli_args(image_ref); - orch_args.extend(config.orch_args.clone()); command - .args(&orch_args) - .stdin(Stdio::piped()) + .args(config.orchestrator_cli_args(image_ref)) + .stdin(Stdio::null()) .stdout(Stdio::null()) .stderr(Stdio::null()); - #[cfg(target_os = "linux")] + #[cfg(all(target_os = "linux", not(test)))] unsafe { command.pre_exec(|| { if libc::setpgid(0, 0) == 0 { @@ -867,27 +534,18 @@ impl OrchChild { } }); } - let display_orch = display_user_path(&config.orch_bin); - let mut child = command + let child = command .spawn() - .map_err(|e| format!("spawn {display_orch}: {e}"))?; - let stdin = child.stdin.take(); + .map_err(|e| format!("spawn {}: {e}", config.orch_bin.display()))?; Ok(Self { child, - stdin, cleaned: false, }) } - fn wait_ready( - &mut self, - rpc_addr: String, - progress: &mut StartupProgress, - ) -> Result { + fn wait_ready(&mut self, rpc_addr: String) -> Result { loop { - progress.poll(); if STOP_REQUESTED.load(Ordering::SeqCst) { - acknowledge_stop(); return Err("interrupted before orchestrator became ready".to_owned()); } match TcpStream::connect(&rpc_addr) { @@ -909,10 +567,6 @@ impl OrchChild { .try_wait() .map_err(|e| format!("poll orchestrator: {e}"))? { - progress.poll(); - if let Some(failure) = progress.failure_summary() { - return Err(failure); - } return Err(format!( "orchestrator exited before prompt RPC ready: {status}" )); @@ -921,109 +575,180 @@ impl OrchChild { } } - fn shutdown(&mut self, _interrupt: bool) -> bool { + // The orchestrator shutdown spec is still pending. Replace this with the approved + // shutdown contract when it is finalized; do not add private stdin commands here. + fn shutdown(&mut self) { if self.cleaned { - return false; + return; } self.cleaned = true; - if let Some(mut stdin) = self.stdin.take() { - let _ = writeln!(stdin, "shutdown"); - let _ = stdin.flush(); + if matches!(self.child.try_wait(), Ok(Some(_))) { + return; + } + + #[cfg(target_os = "linux")] + let _ = signal_orch_process_group(&self.child, libc::SIGTERM); + + let grace = Duration::from_millis(ORCH_SHUTDOWN_GRACE_MS); + let poll = Duration::from_millis(ORCH_SHUTDOWN_POLL_MS); + let started = Instant::now(); + while started.elapsed() < grace { + match self.child.try_wait() { + Ok(Some(_)) => { + let _ = self.child.wait(); + return; + } + Ok(None) | Err(_) => thread::sleep(poll), + } + } + + #[cfg(target_os = "linux")] + { + if signal_orch_process_group(&self.child, libc::SIGKILL).is_err() { + let _ = self.child.kill(); + } + } + #[cfg(not(target_os = "linux"))] + { + let _ = self.child.kill(); } let _ = self.child.wait(); - false } } impl Drop for OrchChild { fn drop(&mut self) { - let _ = self.shutdown(false); + self.shutdown(); } } +#[cfg(target_os = "linux")] +fn signal_orch_process_group(child: &Child, signal: libc::c_int) -> io::Result<()> { + let result = unsafe { libc::kill(-(child.id() as libc::pid_t), signal) }; + if result == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } +} + +type PrepareNodeImageFn = fn(NodeImageRequest) -> Result; + fn prepare_runtime(config: &Config) -> Result { + prepare_runtime_with(config, prepare_node_image) +} + +fn prepare_runtime_with( + config: &Config, + prepare_node_image_fn: PrepareNodeImageFn, +) -> Result { ensure_orch_binary(config)?; if config.provider == ProviderKind::Process { ensure_worker_binary(config)?; - eprintln!( - "mvp-chat: using local worker process {}", - display_user_path(&config.worker_bin) - ); return Ok(config.node_image.clone()); } - if !config.build_image { - eprintln!("mvp-chat: skipping node image preparation (--no-build-image)"); + if config.skip_rebuild { + ensure_worker_binary(config)?; return Ok(config.node_image.clone()); } - if let Some(cached_model) = &config.cached_model { - eprintln!( - "mvp-chat: using cached model {}", - cached_model.display_path.display() - ); - } - let prepared = prepare_node_image(NodeImageRequest { + let prepared = prepare_node_image_fn(NodeImageRequest { requested_image: config.node_image.clone(), base_image: BASE_NODE_IMAGE.to_owned(), node_bin: node_bin_for_current_profile()?, provider: node_image_provider(config.provider)?, extra_tag: config.image_tag.clone(), - push: config.push_image, - force_refresh: config.force_image_refresh, + push: false, + force_refresh: false, enabled: true, })?; - eprintln!( - "mvp-chat: using node image {} ({})", - prepared.image_ref, prepared.tag - ); Ok(prepared.image_ref) } +fn stdin_prompt_events() -> mpsc::Receiver { + let (tx, rx) = mpsc::channel(); + if STOP_REQUESTED.load(Ordering::SeqCst) { + let _ = tx.send(PromptInput::StopRequested); + } + if let Ok(mut stop_tx) = PROMPT_STOP_TX.lock() { + *stop_tx = Some(tx.clone()); + } + thread::spawn(move || { + let stdin = io::stdin(); + for line in stdin.lock().lines() { + match line { + Ok(line) => { + if tx.send(PromptInput::Line(line)).is_err() { + return; + } + } + Err(_) => { + let _ = tx.send(PromptInput::Closed); + return; + } + } + } + let _ = tx.send(PromptInput::Closed); + }); + rx +} + fn run_chat_loop(addr: &str, max_tokens: u32) -> Result<(), String> { + run_chat_loop_with_input(addr, max_tokens, stdin_prompt_events()) +} + +fn run_chat_loop_with_input( + addr: &str, + max_tokens: u32, + input_rx: mpsc::Receiver, +) -> Result<(), String> { let mut stream = TcpStream::connect(addr).map_err(|e| format!("connect prompt RPC {addr}: {e}"))?; - stream - .set_read_timeout(Some(CHAT_READ_TIMEOUT)) - .map_err(|e| format!("set prompt RPC read timeout: {e}"))?; - let mut reader = BufReader::new( + let reader = BufReader::new( stream .try_clone() .map_err(|e| format!("clone prompt RPC stream: {e}"))?, ); - let (input_tx, input_rx) = mpsc::channel::(); - thread::spawn(move || { - let stdin = io::stdin(); - for line in stdin.lock().lines().map_while(Result::ok) { - if input_tx.send(line).is_err() { - break; - } - } - }); + run_chat_session(&mut stream, reader, input_rx, max_tokens) +} + +fn run_chat_session( + writer: &mut W, + reader: R, + input_rx: mpsc::Receiver, + max_tokens: u32, +) -> Result<(), String> +where + R: BufRead, + W: Write, +{ + let mut output = io::stdout(); + run_chat_session_with_output(writer, reader, input_rx, max_tokens, &mut output) +} + +fn run_chat_session_with_output( + writer: &mut W, + mut reader: R, + input_rx: mpsc::Receiver, + max_tokens: u32, + output: &mut O, +) -> Result<(), String> +where + R: BufRead, + W: Write, + O: Write, +{ let mut next_request_id = 1_u64; - eprintln!("mvp-chat: Ctrl-C cleans up the orchestrator and provider node; /exit exits cleanly"); loop { if STOP_REQUESTED.load(Ordering::SeqCst) { - acknowledge_stop(); return Ok(()); } - print!("prompt:> "); - io::stdout() - .flush() - .map_err(|e| format!("flush prompt: {e}"))?; - let prompt = loop { - if STOP_REQUESTED.load(Ordering::SeqCst) { - acknowledge_stop(); - return Ok(()); - } - match input_rx.recv_timeout(Duration::from_millis(100)) { - Ok(line) => break line.trim_end().to_owned(), - Err(mpsc::RecvTimeoutError::Timeout) => continue, - Err(mpsc::RecvTimeoutError::Disconnected) => return Ok(()), - } + write!(output, "prompt:> ").map_err(|e| format!("write prompt: {e}"))?; + output.flush().map_err(|e| format!("flush prompt: {e}"))?; + let prompt = match input_rx.recv() { + Ok(PromptInput::Line(line)) => line.trim_end().to_owned(), + Ok(PromptInput::Closed | PromptInput::StopRequested) | Err(_) => return Ok(()), }; - if prompt.eq_ignore_ascii_case("/quit") || prompt.eq_ignore_ascii_case("/exit") { - return Ok(()); - } if prompt.trim().is_empty() { continue; } @@ -1031,92 +756,71 @@ fn run_chat_loop(addr: &str, max_tokens: u32) -> Result<(), String> { let request_id = next_request_id; next_request_id = next_request_id.wrapping_add(1).max(1); write_json_line( - &mut stream, + writer, &SubmitPrompt { request_id, prompt_text: prompt, max_tokens, }, )?; - println!("decoding..."); + writeln!(output, "decoding...").map_err(|e| format!("write decoding marker: {e}"))?; let mut response_started = false; loop { if STOP_REQUESTED.load(Ordering::SeqCst) { - acknowledge_stop(); return Ok(()); } let mut line = String::new(); match reader.read_line(&mut line) { Ok(0) => return Err("prompt RPC closed".to_owned()), Ok(_) => {} - Err(error) - if matches!( - error.kind(), - std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut - ) => - { - continue; - } - Err(error) => return Err(format!("read prompt event: {error}")), + Err(error) => return Err(format!("read prompt RPC event: {error}")), } let event = serde_json::from_str::(&line) - .map_err(|e| format!("parse prompt event: {e}"))?; + .map_err(|e| format!("parse prompt RPC event: {e}"))?; + let seen = event.request_id(); + if seen != request_id { + return Err(format!( + "prompt RPC protocol error: response request_id {seen} does not match active request_id {request_id}" + )); + } match event { - PromptEvent::TextDelta { - request_id: seen, - text, - } if seen == request_id => { + PromptEvent::TextDelta { text, .. } => { if !response_started { - print!("Response: "); + write!(output, "Response: ") + .map_err(|e| format!("write response prefix: {e}"))?; response_started = true; } - print!("{text}"); - io::stdout() + write!(output, "{text}").map_err(|e| format!("write response text: {e}"))?; + output .flush() .map_err(|e| format!("flush response text: {e}"))?; } - PromptEvent::Done { - request_id: seen, .. - } if seen == request_id => { + PromptEvent::Done { .. } => { if response_started { - println!(); + writeln!(output).map_err(|e| format!("write response terminator: {e}"))?; } else { - println!("Response: "); + writeln!(output, "Response: ") + .map_err(|e| format!("write empty response: {e}"))?; } break; } - PromptEvent::Fault { - request_id: seen, - error, - } if seen == request_id => { - eprintln!("error: {error}"); + PromptEvent::Fault { error, .. } => { + writeln!(output, "error: {error}") + .map_err(|e| format!("write prompt fault: {e}"))?; break; } - _ => {} } } } } fn default_orch_bin() -> Result { - if let Some(path) = std::env::var_os("MVP_ORCH_BIN") { - return Ok(PathBuf::from(path)); - } - let mut path = std::env::current_exe().map_err(|e| format!("current exe: {e}"))?; - path.set_file_name("mvp-orchestrator"); - Ok(path) + Ok(artifact_root().join("target/debug/mvp-orchestrator")) } fn node_bin_for_current_profile() -> Result { - let mut path = std::env::current_exe().map_err(|e| format!("current exe: {e}"))?; - path.set_file_name("mvp-worker-node"); - let cwd = std::env::current_dir().map_err(|e| format!("current dir: {e}"))?; - if let Ok(relative) = path.strip_prefix(&cwd) { - Ok(relative.to_path_buf()) - } else { - Ok(path) - } + Ok(artifact_root().join("target/debug/mvp-worker-node")) } fn cargo_command() -> &'static str { @@ -1124,30 +828,9 @@ fn cargo_command() -> &'static str { } fn ensure_orch_binary(config: &Config) -> Result<(), String> { - let default_orch = default_orch_bin()?; - if config.orch_bin != default_orch { - if config.orch_bin.is_file() { - let display_orch = display_workspace_path(&workspace_root(), &config.orch_bin); - eprintln!( - "mvp-chat: using custom orchestrator binary {display_orch}; skipping cargo build" - ); - return Ok(()); - } - let display_orch = display_workspace_path(&workspace_root(), &config.orch_bin); - return Err(format!( - "custom orchestrator binary {display_orch} does not exist" - )); + if config.skip_rebuild { + return ensure_existing_artifact(&config.orch_bin, "mvp-orchestrator"); } - - let root = workspace_root(); - let rebuild_needed = orch_rebuild_needed(&config.orch_bin, &root, ORCH_REBUILD_INPUTS)?; - let dashboard_feature_stale = - config.dashboard && orch_local_e2e_marker_stale(&config.orch_bin, &root)?; - if !rebuild_needed && !dashboard_feature_stale { - eprintln!("mvp-chat: mvp-orchestrator is up to date; skipping cargo build"); - return Ok(()); - } - run_status( cargo_command(), &[ @@ -1155,39 +838,17 @@ fn ensure_orch_binary(config: &Config) -> Result<(), String> { "--quiet", "-p", "mvp-system", - "--features", - "local-e2e", "--bin", "mvp-orchestrator", ], "build mvp-orchestrator", - )?; - write_orch_local_e2e_marker(&config.orch_bin, &root) + ) } fn ensure_worker_binary(config: &Config) -> Result<(), String> { - let default_worker = node_bin_for_current_profile()?; - if config.worker_bin != default_worker { - if config.worker_bin.is_file() { - let display_worker = display_workspace_path(&workspace_root(), &config.worker_bin); - eprintln!( - "mvp-chat: using custom worker binary {display_worker}; skipping cargo build" - ); - return Ok(()); - } - let display_worker = display_workspace_path(&workspace_root(), &config.worker_bin); - return Err(format!( - "custom worker binary {display_worker} does not exist" - )); + if config.skip_rebuild { + return ensure_existing_artifact(&config.worker_bin, "mvp-worker-node"); } - - let root = workspace_root(); - let rebuild_needed = orch_rebuild_needed(&config.worker_bin, &root, ORCH_REBUILD_INPUTS)?; - if !rebuild_needed { - eprintln!("mvp-chat: mvp-worker-node is up to date; skipping cargo build"); - return Ok(()); - } - run_status( cargo_command(), &[ @@ -1202,121 +863,19 @@ fn ensure_worker_binary(config: &Config) -> Result<(), String> { ) } -fn orch_rebuild_needed(bin: &Path, root: &Path, inputs: &[&str]) -> Result { - if !bin.is_file() { - return Ok(true); +fn ensure_existing_artifact(path: &PathBuf, label: &str) -> Result<(), String> { + let metadata = fs::metadata(path) + .map_err(|e| format!("missing required {label} artifact {}: {e}", path.display()))?; + if !metadata.is_file() { + return Err(format!( + "missing required {label} artifact {}; not a file", + path.display() + )); } - let bin_mtime = modified_time(root, bin)?; - for input in inputs { - let path = root.join(input); - if latest_mtime(root, &path)? > bin_mtime { - return Ok(true); - } - } - Ok(false) -} - -fn orch_local_e2e_marker(bin: &Path) -> PathBuf { - let mut marker = bin.to_path_buf(); - let file_name = bin - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or("mvp-orchestrator"); - marker.set_file_name(format!("{file_name}.local-e2e")); - marker -} - -fn orch_binary_fingerprint(bin: &Path, root: &Path) -> Result { - let display = display_workspace_path(root, bin); - let metadata = fs::metadata(bin).map_err(|e| format!("stat {display}: {e}"))?; - let modified = metadata - .modified() - .map_err(|e| format!("modified time {display}: {e}"))?; - let modified_ns = modified - .duration_since(std::time::UNIX_EPOCH) - .map_err(|e| format!("modified time before Unix epoch for {display}: {e}"))? - .as_nanos(); - Ok(format!( - "local-e2e\nlen={}\nmodified_ns={modified_ns}\n", - metadata.len() - )) -} - -fn orch_local_e2e_marker_stale(bin: &Path, root: &Path) -> Result { - if !bin.is_file() { - return Ok(true); - } - let marker = orch_local_e2e_marker(bin); - if !marker.is_file() { - return Ok(true); - } - let expected = orch_binary_fingerprint(bin, root)?; - let actual = fs::read_to_string(&marker).unwrap_or_default(); - Ok(actual != expected) -} - -fn write_orch_local_e2e_marker(bin: &Path, root: &Path) -> Result<(), String> { - let marker = orch_local_e2e_marker(bin); - let display = display_workspace_path(root, &marker); - let fingerprint = orch_binary_fingerprint(bin, root)?; - fs::write(&marker, fingerprint).map_err(|e| format!("write {display}: {e}")) -} - -fn latest_mtime(root: &Path, path: &Path) -> Result { - let display = display_workspace_path(root, path); - let metadata = fs::metadata(path).map_err(|e| format!("stat {display}: {e}"))?; - let mut latest = metadata - .modified() - .map_err(|e| format!("modified time {display}: {e}"))?; - if metadata.is_dir() { - for entry in fs::read_dir(path).map_err(|e| format!("read dir {display}: {e}"))? { - let entry = entry.map_err(|e| format!("read dir entry {display}: {e}"))?; - let entry_mtime = latest_mtime(root, &entry.path())?; - if entry_mtime > latest { - latest = entry_mtime; - } - } - } - Ok(latest) -} - -fn modified_time(root: &Path, path: &Path) -> Result { - let display = display_workspace_path(root, path); - fs::metadata(path) - .map_err(|e| format!("stat {display}: {e}"))? - .modified() - .map_err(|e| format!("modified time {display}: {e}")) -} - -fn display_workspace_path(root: &Path, path: &Path) -> String { - match path.strip_prefix(root) { - Ok(relative) if relative.as_os_str().is_empty() => ".".to_owned(), - Ok(relative) => format!("./{}", relative.display()), - Err(_) => path.display().to_string(), - } -} - -fn display_user_path(path: &Path) -> String { - let root = workspace_root(); - if let Ok(relative) = path.strip_prefix(&root) { - if relative.as_os_str().is_empty() { - return ".".to_owned(); - } - return format!("./{}", relative.display()); - } - if let Ok(cwd) = std::env::current_dir() { - if let Ok(relative) = path.strip_prefix(&cwd) { - if relative.as_os_str().is_empty() { - return ".".to_owned(); - } - return format!("./{}", relative.display()); - } - } - path.display().to_string() + Ok(()) } fn run_status(program: &str, args: &[&str], label: &str) -> Result<(), String> { - eprintln!("mvp-chat: {label}"); let status = Command::new(program) .args(args) .stdin(Stdio::null()) @@ -1331,126 +890,113 @@ fn run_status(program: &str, args: &[&str], label: &str) -> Result<(), String> { } } -extern "C" fn request_stop(_: libc::c_int) { - STOP_REQUESTED.store(true, Ordering::SeqCst); -} - -fn install_signal_handlers() { +fn install_signal_handlers() -> Result<(), String> { #[cfg(target_os = "linux")] - unsafe { - libc::signal(libc::SIGINT, request_stop as *const () as usize); - libc::signal(libc::SIGTERM, request_stop as *const () as usize); + { + let mut signals = + Signals::new([SIGINT, SIGTERM]).map_err(|e| format!("install signal handlers: {e}"))?; + thread::spawn(move || { + for _ in signals.forever() { + STOP_REQUESTED.store(true, Ordering::SeqCst); + if let Ok(stop_tx) = PROMPT_STOP_TX.lock() { + if let Some(tx) = stop_tx.as_ref() { + let _ = tx.send(PromptInput::StopRequested); + } + } + } + }); } -} - -fn acknowledge_stop() { - if !STOP_ACKNOWLEDGED.swap(true, Ordering::SeqCst) { - eprintln!("mvp-chat: Ctrl-C received; stopping runtime..."); - } -} - -fn report_interrupt_shutdown() { - eprintln!("mvp-chat: runtime stopped"); + Ok(()) } #[derive(Clone, Debug)] struct CachedModelConfig { host_path: PathBuf, - display_path: PathBuf, } impl CachedModelConfig { - fn from_arg(path: Option) -> Result { - let requested = path - .map(PathBuf::from) - .unwrap_or_else(default_cached_model_path); - let host_path = requested - .canonicalize() - .map_err(|e| format!("resolve --cached-model path {}: {e}", requested.display()))?; - let metadata = fs::metadata(&host_path) - .map_err(|e| format!("stat cached model {}: {e}", requested.display()))?; - if !metadata.is_file() { + fn from_source(source: CachedModelSource) -> Result { + match source { + CachedModelSource::Discover => Self::discover(), + CachedModelSource::Path(path) => Self::from_path(path), + } + } + + fn from_path(path: PathBuf) -> Result { + let metadata = fs::metadata(&path) + .map_err(|e| format!("stat cached model {}: {e}", path.display()))?; + if !is_accepted_cached_model_file(&path, &metadata) { return Err(format!( - "--cached-model must point at a file: {}", - requested.display() + "cached model {} must be a regular .gguf file", + path.display() )); } - Ok(Self { - host_path, - display_path: requested, - }) + let host_path = path + .canonicalize() + .map_err(|e| format!("resolve cached model {}: {e}", path.display()))?; + Ok(Self { host_path }) } -} -fn default_cached_model_path() -> PathBuf { - PathBuf::from(".") - .join(REPO_MODEL_CACHE_DIR) - .join(DEFAULT_CACHED_MODEL_FILE) -} - -fn workspace_root() -> PathBuf { - let git_root = Command::new("git") - .args(["rev-parse", "--show-toplevel"]) - .stdin(Stdio::null()) - .output(); - if let Ok(output) = git_root { - if output.status.success() { - return PathBuf::from(String::from_utf8_lossy(&output.stdout).trim()); + fn discover() -> Result { + let cache_dir = PathBuf::from(REPO_MODEL_CACHE_DIR); + let entries = fs::read_dir(&cache_dir) + .map_err(|e| format!("discover cached model in {}: {e}", cache_dir.display()))?; + let mut candidates = Vec::new(); + for entry in entries { + let entry = entry + .map_err(|e| format!("read cached model entry in {}: {e}", cache_dir.display()))?; + let path = entry.path(); + let metadata = entry + .metadata() + .map_err(|e| format!("stat cached model candidate {}: {e}", path.display()))?; + if is_accepted_cached_model_file(&path, &metadata) { + candidates.push(path); + } } + candidates.sort_by(|left, right| left.file_name().cmp(&right.file_name())); + let requested = candidates.into_iter().next().ok_or_else(|| { + format!( + "discover cached model in {}: no usable cached model files found", + cache_dir.display() + ) + })?; + let host_path = requested + .canonicalize() + .map_err(|e| format!("resolve cached model {}: {e}", requested.display()))?; + Ok(Self { host_path }) } +} + +fn is_accepted_cached_model_file(path: &Path, metadata: &fs::Metadata) -> bool { + metadata.is_file() + && path + .extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| ext.eq_ignore_ascii_case("gguf")) +} + +fn artifact_root() -> PathBuf { std::env::current_dir().expect("current directory is available") } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum RuntimeConfigProfile { - Local, - Deploy, -} - -impl RuntimeConfigProfile { - fn from_env() -> Result { - match env_optional(MVP_RUNTIME_CONFIG_ENV).as_deref() { - None | Some("local") => Ok(Self::Local), - Some("deploy") => Ok(Self::Deploy), - Some(other) => Err(format!( - "unsupported {MVP_RUNTIME_CONFIG_ENV}={other:?}; use local or deploy" - )), - } - } - - fn as_str(self) -> &'static str { - match self { - Self::Local => "local", - Self::Deploy => "deploy", - } - } - - fn default_provider(self) -> ProviderKind { - match self { - Self::Local => ProviderKind::Process, - Self::Deploy => ProviderKind::VastAi, - } - } -} - fn provider_from_sources( cli_provider: Option, toml_provider: Option<&str>, - config_profile: RuntimeConfigProfile, ) -> Result { if let Some(provider) = cli_provider { return Ok(provider); } - if let Some(value) = env_optional("MVP_NODE_PROVIDER") { - return ProviderKind::parse_deploy(&value); - } - if let Some(value) = env_optional("MVP_PROVIDER") { - return ProviderKind::parse_deploy(&value); - } if let Some(value) = toml_provider { - return ProviderKind::parse_deploy(value); + return match value.trim() { + "process" => Ok(ProviderKind::Process), + "docker" => Ok(ProviderKind::Docker), + "vastai" => Ok(ProviderKind::VastAi), + other => Err(format!( + "unsupported provider {other:?}; use process, docker, or vastai" + )), + }; } - Ok(config_profile.default_provider()) + Ok(ProviderKind::Process) } fn env_optional(name: &str) -> Option { @@ -1460,27 +1006,6 @@ fn env_optional(name: &str) -> Option { .filter(|value| !value.is_empty()) } -fn relay_mode_from_sources(config_value: Option<&str>) -> Result { - match env_optional("MVP_IROH_RELAY_MODE") - .as_deref() - .or(config_value) - .unwrap_or("default") - { - "disabled" => Ok(iroh::RelayMode::Disabled), - "default" => Ok(iroh::RelayMode::Default), - other => Err(format!( - "unsupported relay mode {other:?}; use disabled or default" - )), - } -} - -fn relay_mode_env_value(mode: &iroh::RelayMode) -> &'static str { - match mode { - iroh::RelayMode::Disabled => "disabled", - _ => "default", - } -} - fn node_image_provider(provider: ProviderKind) -> Result { match provider { ProviderKind::Docker => Ok(NodeImageProvider::Docker), @@ -1490,45 +1015,6 @@ fn node_image_provider(provider: ProviderKind) -> Result Result, String> { - match env_optional(name) { - None => Ok(None), - Some(value) => match value.to_ascii_lowercase().as_str() { - "1" | "true" | "yes" | "on" => Ok(Some(true)), - "0" | "false" | "no" | "off" => Ok(Some(false)), - _ => Err(format!( - "invalid {name}={value:?}; use 1/0, true/false, yes/no, or on/off" - )), - }, - } -} - -fn env_u64_optional(name: &str) -> Result, String> { - env_parse_optional(name) -} - -fn env_u32_optional(name: &str) -> Result, String> { - env_parse_optional(name) -} - -fn env_f64_optional(name: &str) -> Result, String> { - env_parse_optional(name) -} - -fn env_parse_optional(name: &str) -> Result, String> -where - T: std::str::FromStr, - T::Err: std::fmt::Display, -{ - match env_optional(name) { - Some(value) => value - .parse::() - .map(Some) - .map_err(|e| format!("invalid {name}={value:?}: {e}")), - None => Ok(None), - } -} - fn next_arg(args: &mut impl Iterator, name: &str) -> Result { args.next() .ok_or_else(|| format!("missing value after {name}")) @@ -1563,46 +1049,61 @@ fn main() -> ExitCode { #[cfg(test)] mod tests { use super::*; - use std::ffi::OsString; + + use std::ffi::{OsStr, OsString}; + use std::io::{Cursor, Read}; + #[cfg(target_os = "linux")] + use std::os::unix::process::CommandExt; + use std::path::Path; + use std::sync::Mutex; use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering}; - use std::time::Instant; - static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0); + static PROCESS_STATE_LOCK: Mutex<()> = Mutex::new(()); + static NEXT_TEMP_ID: AtomicU64 = AtomicU64::new(1); - static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); - const ENV_KEYS: &[&str] = &[ - "MVP_BUILD_NODE_IMAGE", - "MVP_DASHBOARD", - "MVP_DATASTREAM_FRAME_LOG", - "MVP_FORCE_NODE_IMAGE_REFRESH", - "MVP_GGUF_FILE", - "MVP_GGUF_REPO", - "MVP_GGUF_REVISION", - "MVP_IROH_RELAY_MODE", - "MVP_IROH_RELAY_URL", - "MVP_MAX_CONTEXT", - "MVP_MODEL_ID", - "MVP_NODE_IMAGE", - "MVP_NODE_IMAGE_TAG", - "MVP_NODE_PROVIDER", - "MVP_PIPELINE_STAGES", - "MVP_PROMPT_MAX_TOKENS", - "MVP_PROMPT_RPC_ADDR", - "MVP_PROMPT_RPC_BIND", - "MVP_PROVIDER", - "MVP_PUSH_NODE_IMAGE", - "MVP_RUNTIME_CONFIG", - "SWACTOR_IROH_RELAY_URL", - "VASTAI_API_KEY", - ]; + const PROCESS_ENV_KEYS: &[&str] = + &["VAST_API_KEY", "MVP_PIPELINE_STAGES", "MVP_RUNTIME_CONFIG"]; - struct RestoreEnv { - saved: Vec<(&'static str, Option)>, + struct TempDir { + path: PathBuf, } - impl Drop for RestoreEnv { + impl TempDir { + fn new(label: &str) -> Self { + let id = NEXT_TEMP_ID.fetch_add(1, AtomicOrdering::SeqCst); + let path = std::env::temp_dir().join(format!( + "mvp-chat-test-{}-{}-{}", + std::process::id(), + id, + label + )); + if path.exists() { + fs::remove_dir_all(&path).expect("remove stale temp dir"); + } + fs::create_dir_all(&path).expect("create temp dir"); + Self { path } + } + + fn path(&self) -> &Path { + &self.path + } + } + + impl Drop for TempDir { fn drop(&mut self) { - for (key, value) in &self.saved { + let _ = fs::remove_dir_all(&self.path); + } + } + + struct RestoreProcessState { + saved_env: Vec<(&'static str, Option)>, + saved_cwd: PathBuf, + } + + impl Drop for RestoreProcessState { + fn drop(&mut self) { + let _ = std::env::set_current_dir(&self.saved_cwd); + for (key, value) in &self.saved_env { match value { Some(value) => unsafe { std::env::set_var(key, value) }, None => unsafe { std::env::remove_var(key) }, @@ -1611,724 +1112,836 @@ mod tests { } } - fn with_clean_env(test: impl FnOnce() -> T) -> T { - let _lock = ENV_LOCK + fn with_process_state( + settings: &[(&'static str, Option<&str>)], + cwd: Option<&Path>, + test: impl FnOnce() -> T, + ) -> T { + let _lock = PROCESS_STATE_LOCK .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); - let saved = ENV_KEYS + let saved_env = PROCESS_ENV_KEYS .iter() - .map(|&key| (key, std::env::var_os(key))) + .map(|key| (*key, std::env::var_os(key))) .collect::>(); - for key in ENV_KEYS { + for key in PROCESS_ENV_KEYS { unsafe { std::env::remove_var(key) }; } - let _restore = RestoreEnv { saved }; + for (key, value) in settings { + match value { + Some(value) => unsafe { std::env::set_var(key, value) }, + None => unsafe { std::env::remove_var(key) }, + } + } + let saved_cwd = std::env::current_dir().expect("current directory"); + if let Some(cwd) = cwd { + std::env::set_current_dir(cwd).expect("set test current directory"); + } + let _restore = RestoreProcessState { + saved_env, + saved_cwd, + }; test() } - struct TempWorkspace { - root: PathBuf, + fn strings(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() } - impl TempWorkspace { - fn new(name: &str) -> Self { - let counter = TEMP_COUNTER.fetch_add(1, AtomicOrdering::Relaxed); - let root = std::env::temp_dir() - .join(format!("mvp-chat-{name}-{}-{counter}", std::process::id())); - let _ = fs::remove_dir_all(&root); - fs::create_dir_all(&root).expect("create temp workspace"); - Self { root } - } - - fn path(&self, relative: &str) -> PathBuf { - self.root.join(relative) - } - - fn write(&self, relative: &str, contents: &[u8]) -> PathBuf { - let path = self.path(relative); - if let Some(parent) = path.parent() { - fs::create_dir_all(parent).expect("create temp parent directory"); - } - fs::write(&path, contents).expect("write temp file"); - path - } - } - - impl Drop for TempWorkspace { - fn drop(&mut self) { - let _ = fs::remove_dir_all(&self.root); - } - } - - fn write_file_newer_than(path: &Path, contents: &[u8], older_than: SystemTime) { + fn write_config(dir: &TempDir, name: &str, text: &str) -> PathBuf { + let path = dir.path().join(name); if let Some(parent) = path.parent() { - fs::create_dir_all(parent).expect("create temp parent directory"); - } - - let started = Instant::now(); - loop { - fs::write(path, contents).expect("write temp file"); - let mtime = fs::metadata(path) - .expect("stat temp file") - .modified() - .expect("read temp file mtime"); - if mtime > older_than { - return; - } - assert!( - started.elapsed() <= Duration::from_secs(3), - "filesystem did not record a newer mtime for {}", - path.display() - ); - thread::sleep(Duration::from_millis(20)); + fs::create_dir_all(parent).expect("create config parent"); } + fs::write(&path, text).expect("write config"); + path } - struct FakeOfferPreviewer; - - impl OfferPreviewer for FakeOfferPreviewer { - fn preview( - &self, - _api_key: &str, - _policy: &mvp_system::vastai_offer_preview::SelectionPolicy, - ) -> Result { - Ok(OfferPreview { - offer_id: 7, - host_id: Some(8), - gpu_name: "RTX 4090".to_owned(), - gpu_ram_mb: Some(24_000), - dollars_per_hour: 0.42, - }) - } - } - - fn valid_vastai_config() -> ResolvedVastAiConfig { - ResolvedVastAiConfig { - api_key: "vast-key".to_owned(), - relay_url: "https://relay.example.com".to_owned(), - image: "ghcr.io/swactor/mvp-node:latest".to_owned(), - bootstrap_command: "/usr/local/bin/mvp-node".to_owned(), - disk_gb: Some(80), - gpu_name: Some("RTX 4090".to_owned()), - min_gpu_ram_mb: Some(16_000), - min_down_mbps: Some(100.0), - min_up_mbps: Some(25.0), - min_reliability: Some(0.98), - require_verified: Some(true), - onstart: None, - ssh_identity: Some("~/.ssh/swactor_vastai_ed25519".to_owned()), - } - } - - fn vastai_yes_chat_config() -> Config { + fn base_config(provider: ProviderKind) -> Config { Config { - orch_bin: PathBuf::from("mvp-orchestrator"), - worker_bin: PathBuf::from("mvp-worker-node"), - orch_args: Vec::new(), + orch_bin: PathBuf::from("/tmp/mvp-orchestrator"), + worker_bin: PathBuf::from("/tmp/mvp-worker-node"), rpc_addr: DEFAULT_RPC_ADDR.to_owned(), - node_image: "ghcr.io/swactor/mvp-node:latest".to_owned(), - config_profile: RuntimeConfigProfile::Deploy, - provider: ProviderKind::VastAi, - relay_mode: iroh::RelayMode::Default, - relay_url: Some("https://relay.example.com".to_owned()), - max_tokens: 128, - pipeline_stages: 1, - dashboard: false, - build_image: false, - image_tag: Some("trial".to_owned()), - push_image: true, - force_image_refresh: false, + node_image: "docker.io/acme/node:latest".to_owned(), + provider, + image_tag: None, cached_model: None, datastream_frame_log: None, - vastai_yes: true, - model_id: None, - gguf_repo: None, - gguf_file: None, - gguf_revision: None, - max_context: None, - vastai: Some(valid_vastai_config()), - } - } - - fn assert_arg_value(args: &[String], flag: &str, expected: &str) { - let flag_index = args - .iter() - .position(|arg| arg == flag) - .unwrap_or_else(|| panic!("missing CLI flag {flag}; args={args:?}")); - assert_eq!( - args.get(flag_index + 1).map(String::as_str), - Some(expected), - "unexpected value for CLI flag {flag}; args={args:?}" - ); - } - - fn assert_flag(args: &[String], flag: &str) { - assert!( - args.iter().any(|arg| arg == flag), - "missing CLI flag {flag}; args={args:?}" - ); - } - - #[test] - fn orchestrator_cli_args_cover_wrapper_launch_config() { - let config = Config { - orch_bin: PathBuf::from("mvp-orchestrator"), - worker_bin: PathBuf::from("mvp-worker-node"), - orch_args: Vec::new(), - rpc_addr: "127.0.0.1:20123".to_owned(), - node_image: "docker.io/example/config-node:ignored".to_owned(), - config_profile: RuntimeConfigProfile::Local, - provider: ProviderKind::Docker, - relay_mode: iroh::RelayMode::Default, - relay_url: Some("https://relay.example.com".to_owned()), - max_tokens: 37, - pipeline_stages: 3, - dashboard: false, - build_image: false, - image_tag: None, - push_image: false, - force_image_refresh: false, - cached_model: Some(CachedModelConfig { - host_path: PathBuf::from("/var/cache/swactor/model.gguf"), - display_path: PathBuf::from("model.gguf"), - }), - datastream_frame_log: Some(PathBuf::from("/tmp/mvp-chat-frames.jsonl")), vastai_yes: false, vastai: None, - model_id: Some("wrapper-model".to_owned()), - gguf_repo: Some("example/wrapper-repo".to_owned()), - gguf_file: Some("wrapper-model.gguf".to_owned()), - gguf_revision: None, - max_context: Some(768), - }; - - let args = config.orchestrator_cli_args("docker.io/example/prepared-node:latest"); - - assert_arg_value(&args, "--runtime-config", "local"); - assert_arg_value(&args, "--provider", "docker"); - assert_arg_value(&args, "--image", "docker.io/example/prepared-node:latest"); - assert_arg_value(&args, "--rpc-bind", "127.0.0.1:20123"); - assert_arg_value(&args, "--max-tokens", "37"); - assert_arg_value(&args, "--pipeline-stages", "3"); - assert_flag(&args, "--no-dashboard"); - assert_arg_value(&args, "--relay-url", "https://relay.example.com"); - assert_arg_value(&args, "--model-id", "wrapper-model"); - assert_arg_value(&args, "--gguf-repo", "example/wrapper-repo"); - assert_arg_value(&args, "--gguf-file", "wrapper-model.gguf"); - assert_arg_value(&args, "--max-context", "768"); - assert_arg_value( - &args, - "--cached-model-host-path", - "/var/cache/swactor/model.gguf", - ); - assert_arg_value( - &args, - "--datastream-frame-log", - "/tmp/mvp-chat-frames.jsonl", - ); - } - - #[test] - fn local_cached_model_defaults_to_process_and_forwards_worker_bin() { - with_clean_env(|| { - let workspace = TempWorkspace::new("process-default-cached-model"); - let config_path = workspace.write("config.toml", b""); - let model_path = workspace.write("cached-model.gguf", b"fake cached model"); - let model_path_str = model_path.to_str().expect("temp model path is utf8"); - let config_path_str = config_path.to_str().expect("temp config path is utf8"); - - let config = Config::from_args( - [ - "--cached-model", - model_path_str, - "-N", - "3", - "--config", - config_path_str, - "--orch-bin", - "/tmp/mvp-orchestrator", - ] - .into_iter() - .map(str::to_owned), - ) - .unwrap_or_else(|error| panic!("process default cached config parses: {error}")); - - let canonical_model = model_path - .canonicalize() - .expect("canonicalize cached model fixture"); - assert_eq!(config.provider, ProviderKind::Process); - assert_eq!(config.pipeline_stages, 3); - assert_eq!( - config.cached_model.as_ref().map(|cached| &cached.host_path), - Some(&canonical_model) - ); - - let args = config.orchestrator_cli_args("swactor-mvp-node:latest"); - assert_arg_value(&args, "--provider", "process"); - assert_arg_value(&args, "--pipeline-stages", "3"); - assert_arg_value( - &args, - "--cached-model-host-path", - canonical_model.to_str().expect("canonical path is utf8"), - ); - assert_arg_value( - &args, - "--worker-bin", - config.worker_bin.to_str().expect("worker bin path is utf8"), - ); - }); - } - - #[test] - fn docker_selector_keeps_cached_model_forwarding_for_docker() { - with_clean_env(|| { - let workspace = TempWorkspace::new("docker-cached-model"); - let config_path = workspace.write("config.toml", b""); - let model_path = workspace.write("cached-model.gguf", b"fake cached model"); - let config = Config::from_args( - [ - "--docker", - "--cached-model", - model_path.to_str().expect("temp model path is utf8"), - "--config", - config_path.to_str().expect("temp config path is utf8"), - "--orch-bin", - "/tmp/mvp-orchestrator", - ] - .into_iter() - .map(str::to_owned), - ) - .unwrap_or_else(|error| panic!("docker cached config parses: {error}")); - - let canonical_model = model_path - .canonicalize() - .expect("canonicalize cached model fixture"); - assert_eq!(config.provider, ProviderKind::Docker); - - let args = config.orchestrator_cli_args("docker.io/example/prepared-node:latest"); - assert_arg_value(&args, "--provider", "docker"); - assert_arg_value( - &args, - "--cached-model-host-path", - canonical_model.to_str().expect("canonical path is utf8"), - ); - assert!( - !args.iter().any(|arg| arg == "--worker-bin"), - "docker launch must not forward --worker-bin: {args:?}" - ); - }); - } - - #[test] - fn parsed_args_rejects_conflicting_provider_selectors() { - for args in [ - vec!["--docker", "--process"], - vec!["--provider", "process", "--vastai"], - ] { - let error = ParsedArgs::parse(args.iter().copied().map(str::to_owned)) - .expect_err("conflicting provider selectors must fail"); - - assert_eq!(error, PROVIDER_SELECTOR_CONFLICT); - } - } - - #[test] - fn provider_selection_precedence_is_cli_env_toml_default() { - with_clean_env(|| { - let workspace = TempWorkspace::new("provider-precedence"); - let config_path = workspace.write("config.toml", b"[provider]\nkind = \"docker\"\n"); - let config_path_str = config_path.to_str().expect("temp config path is utf8"); - - let toml_config = Config::from_args( - [ - "--config", - config_path_str, - "--orch-bin", - "/tmp/mvp-orchestrator", - ] - .into_iter() - .map(str::to_owned), - ) - .expect("TOML provider config parses"); - assert_eq!(toml_config.provider, ProviderKind::Docker); - - unsafe { - std::env::set_var("MVP_PROVIDER", "vastai"); - } - let mvp_provider_config = Config::from_args( - [ - "--config", - config_path_str, - "--orch-bin", - "/tmp/mvp-orchestrator", - ] - .into_iter() - .map(str::to_owned), - ) - .expect("MVP_PROVIDER config parses"); - assert_eq!(mvp_provider_config.provider, ProviderKind::VastAi); - - unsafe { - std::env::set_var("MVP_NODE_PROVIDER", "process"); - } - let node_provider_config = Config::from_args( - [ - "--config", - config_path_str, - "--orch-bin", - "/tmp/mvp-orchestrator", - ] - .into_iter() - .map(str::to_owned), - ) - .expect("MVP_NODE_PROVIDER config parses"); - assert_eq!(node_provider_config.provider, ProviderKind::Process); - - let cli_config = Config::from_args( - [ - "--docker", - "--config", - config_path_str, - "--orch-bin", - "/tmp/mvp-orchestrator", - ] - .into_iter() - .map(str::to_owned), - ) - .expect("CLI provider config parses"); - assert_eq!(cli_config.provider, ProviderKind::Docker); - }); - - with_clean_env(|| { - let workspace = TempWorkspace::new("provider-default"); - let config_path = workspace.write("config.toml", b""); - let default_config = Config::from_args( - [ - "--config", - config_path.to_str().expect("temp config path is utf8"), - "--orch-bin", - "/tmp/mvp-orchestrator", - ] - .into_iter() - .map(str::to_owned), - ) - .expect("default provider config parses"); - assert_eq!(default_config.provider, ProviderKind::Process); - }); - } - - #[test] - fn config_forwards_short_pipeline_stages_alias_to_orchestrator() { - with_clean_env(|| { - let workspace = TempWorkspace::new("short-pipeline-stages-forward"); - let config_path = workspace.write("config.toml", b""); - let config = Config::from_args( - [ - "-N", - "6", - "--config", - config_path.to_str().expect("temp config path is utf8"), - "--orch-bin", - "/tmp/mvp-orchestrator", - ] - .into_iter() - .map(str::to_owned), - ) - .unwrap_or_else(|error| panic!("-N should resolve wrapper config: {error}")); - - let args = config.orchestrator_cli_args("docker.io/example/prepared-node:latest"); - - assert_arg_value(&args, "--pipeline-stages", "6"); - }); - } - - #[test] - fn orchestrator_cli_args_cover_vastai_config() { - let mut vastai = valid_vastai_config(); - vastai.onstart = Some("echo preparing vastai node".to_owned()); - let config = Config { - orch_bin: PathBuf::from("mvp-orchestrator"), - worker_bin: PathBuf::from("mvp-worker-node"), - orch_args: Vec::new(), - rpc_addr: DEFAULT_RPC_ADDR.to_owned(), - node_image: "ghcr.io/swactor/mvp-node:latest".to_owned(), - config_profile: RuntimeConfigProfile::Deploy, - provider: ProviderKind::VastAi, - relay_mode: iroh::RelayMode::Default, - relay_url: Some(vastai.relay_url.clone()), - max_tokens: 128, pipeline_stages: 1, - dashboard: false, - build_image: false, - image_tag: Some("trial".to_owned()), - push_image: true, - force_image_refresh: false, - cached_model: None, - datastream_frame_log: None, - vastai_yes: true, - vastai: Some(vastai), - model_id: None, - gguf_repo: None, - gguf_file: None, - gguf_revision: None, - max_context: None, - }; - - let args = config.orchestrator_cli_args("ghcr.io/swactor/mvp-node:latest"); - - assert_arg_value(&args, "--vastai-api-key", "vast-key"); - assert_arg_value( - &args, - "--vastai-bootstrap-command", - "/usr/local/bin/mvp-node", - ); - assert_flag(&args, "--no-vastai-confirm-lease"); - assert_arg_value(&args, "--vastai-disk-gb", "80"); - assert_arg_value(&args, "--vastai-gpu-name", "RTX 4090"); - assert_arg_value(&args, "--vastai-min-gpu-ram-mb", "16000"); - assert_arg_value(&args, "--vastai-min-down-mbps", "100"); - assert_arg_value(&args, "--vastai-min-up-mbps", "25"); - assert_arg_value(&args, "--vastai-min-reliability", "0.98"); - assert_flag(&args, "--vastai-require-verified"); - assert_arg_value(&args, "--vastai-onstart", "echo preparing vastai node"); - assert_arg_value( - &args, - "--vastai-ssh-identity", - "~/.ssh/swactor_vastai_ed25519", - ); - } - - #[test] - fn parsed_args_accepts_pipeline_stages_aliases() { - for (flag, value) in [("-N", 3), ("--pipeline-stages", 4)] { - let parsed = ParsedArgs::parse(vec![flag.to_owned(), value.to_string()]) - .unwrap_or_else(|error| panic!("{flag} {value} should parse: {error}")); - - assert_eq!( - parsed.pipeline_stages, - Some(value), - "{flag} must set stage count" - ); + max_tokens: DEFAULT_MAX_TOKENS, + skip_rebuild: true, } } - #[test] - fn parsed_cached_model_flag_does_not_consume_following_short_flag() { - let workspace = TempWorkspace::new("cached-model-short-flag"); - workspace.write( - ".model-cache/SmolLM2-135M-Instruct.Q4_0.gguf", - b"fake cached model", - ); - let previous_cwd = std::env::current_dir().expect("current dir is available"); - std::env::set_current_dir(&workspace.root).expect("enter temp workspace"); - let parsed = - ParsedArgs::parse(["--cached-model", "-N", "3"].into_iter().map(str::to_owned)); - std::env::set_current_dir(previous_cwd).expect("restore current dir"); - let parsed = parsed.expect("cached model flag before -N parses"); - - assert!(parsed.cached_model.is_some()); - assert_eq!(parsed.pipeline_stages, Some(3)); - } - - #[test] - fn parsed_args_rejects_invalid_pipeline_stages_values() { - for (args, expected) in [ - (vec!["-N"], "missing value after -N"), - ( - vec!["--pipeline-stages"], - "missing value after --pipeline-stages", - ), - (vec!["-N", "many"], "invalid -N=\"many\""), - ( - vec!["--pipeline-stages", "many"], - "invalid --pipeline-stages=\"many\"", - ), - (vec!["-N", "0"], "-N must be greater than 0"), - ( - vec!["--pipeline-stages", "0"], - "--pipeline-stages must be greater than 0", - ), - ] { - let error = match ParsedArgs::parse(args.iter().copied().map(str::to_owned)) { - Ok(_) => panic!("invalid pipeline stage flag must be rejected"), - Err(error) => error, - }; - - assert!( - error.contains(expected), - "error {error:?} should contain {expected:?} for args {args:?}" - ); + fn valid_vastai() -> ResolvedVastAiConfig { + ResolvedVastAiConfig { + api_key: "secret".to_owned(), + relay_url: "https://relay.example".to_owned(), + image: "docker.io/acme/node:latest".to_owned(), + bootstrap_command: "boot".to_owned(), + disk_gb: None, + gpu_name: None, + min_gpu_ram_mb: None, + min_down_mbps: None, + min_up_mbps: None, + min_reliability: None, + require_verified: None, + onstart: None, + ssh_identity: None, } } - #[test] - fn config_rejects_vastai_pipeline_stages_count_before_launch() { - with_clean_env(|| { - let workspace = TempWorkspace::new("vastai-pipeline-rejected"); - let config_path = workspace.write("config.toml", b""); - let error = match Config::from_args( - [ - "--vastai", - "-N", - "2", - "--config", - config_path.to_str().expect("temp config path is utf8"), - ] - .into_iter() - .map(str::to_owned), - ) { - Ok(_) => panic!("Vast.ai pipeline stage count above one must fail before launch"), - Err(error) => error, - }; - - assert!( - error.contains("--vastai cannot be combined with -N/--pipeline-stages > 1"), - "unexpected error: {error}" - ); - }); + fn channel_lines(lines: &[&str]) -> mpsc::Receiver { + let (tx, rx) = mpsc::channel(); + for line in lines { + tx.send(PromptInput::Line((*line).to_owned())) + .expect("send input line"); + } + drop(tx); + rx } - #[test] - fn approval_parser_accepts_only_y_or_yes() { - for (input, expected) in [ - ("y", true), - ("Y", true), - (" yes ", true), - ("YES", true), - ("", false), - ("n", false), - ("no", false), - ("yeah", false), - ("yep", false), - ("yes please", false), - ] { - assert_eq!(parse_approval(input), expected, "approval input {input:?}"); + fn event_reader(events: &[PromptEvent]) -> Cursor> { + let mut bytes = Vec::new(); + for event in events { + serde_json::to_writer(&mut bytes, event).expect("serialize prompt event"); + bytes.push(b'\n'); + } + Cursor::new(bytes) + } + + fn done(request_id: u64) -> PromptEvent { + PromptEvent::Done { + request_id, + final_text: String::new(), + tokens_generated: 0, + elapsed_ms: 0, } } - #[test] - fn parsed_args_handles_vastai_yes_and_config_path() { - let parsed = ParsedArgs::parse( - [ - "--vastai", - "--yes", - "--config", - "/tmp/mvp-chat-config.toml", - "--", - "--orchestrator-flag", - ] - .into_iter() - .map(str::to_owned), - ) - .expect("vastai flags parse"); + fn submitted_prompts(bytes: &[u8]) -> Vec { + String::from_utf8(bytes.to_vec()) + .expect("submitted prompts are UTF-8") + .lines() + .map(|line| serde_json::from_str(line).expect("submitted prompt JSON")) + .collect() + } - assert!(parsed.vastai); + #[test] + fn parsed_args_accepts_public_flags() { + let parsed = ParsedArgs::parse(strings(&[ + "--docker", + "--yes", + "--config", + "chat.toml", + "--pipeline-stages", + "3", + "--dump-logs=logs.ndjson", + "--cached-model", + "--skip-rebuild", + ])) + .expect("public args parse"); + + assert_eq!(parsed.provider, Some(ProviderKind::Docker)); assert!(parsed.vastai_yes); - assert_eq!( - parsed.config_path.as_deref(), - Some(Path::new("/tmp/mvp-chat-config.toml")) - ); - assert_eq!(parsed.orch_args, vec!["--orchestrator-flag"]); + assert_eq!(parsed.config_path, Some(PathBuf::from("chat.toml"))); + assert_eq!(parsed.pipeline_stages, Some(3)); + assert!(parsed.dump_logs); + assert_eq!(parsed.dump_log_path, Some(PathBuf::from("logs.ndjson"))); + assert_eq!(parsed.cached_model, Some(CachedModelSource::Discover)); + assert!(parsed.skip_rebuild); } #[test] - fn parsed_args_and_config_dump_logs_select_default_log_file() { - let parsed = ParsedArgs::parse(["--dump-logs"].into_iter().map(str::to_owned)) - .expect("--dump-logs parses"); + fn parsed_args_accepts_cached_model_path() { + let parsed = ParsedArgs::parse(strings(&["--cached-model=/tmp/model.gguf"])) + .expect("cached model path parses"); + + assert_eq!( + parsed.cached_model, + Some(CachedModelSource::Path(PathBuf::from("/tmp/model.gguf"))) + ); + } + + #[test] + fn parsed_args_accepts_cached_model_equals_path_with_dash_prefix() { + let parsed = ParsedArgs::parse(strings(&["--cached-model=-model.gguf"])) + .expect("cached model path parses"); + + assert_eq!( + parsed.cached_model, + Some(CachedModelSource::Path(PathBuf::from("-model.gguf"))) + ); + } + + #[test] + fn parsed_args_accepts_dump_logs_equals_path_with_dash_prefix() { + let parsed = ParsedArgs::parse(strings(&["--dump-logs=-logs.ndjson"])) + .expect("dump log path parses"); assert!(parsed.dump_logs); + assert_eq!(parsed.dump_log_path, Some(PathBuf::from("-logs.ndjson"))); + } - let config = Config::from_args( - ["--dump-logs", "--orch-bin", "/tmp/mvp-orchestrator"] - .into_iter() - .map(str::to_owned), - ) - .expect("--dump-logs resolves chat config"); + #[test] + fn parsed_args_rejects_conflicts_and_pruned_inputs() { + for args in [ + vec!["--process", "--docker"], + vec!["-N", "2"], + vec!["--pipeline-stages", "0"], + vec!["--pipeline-stages", "many"], + vec!["--config"], + vec!["--dump-logs", "logs.ndjson"], + vec!["--dump-logs="], + vec!["--cached-model", "/tmp/model.gguf"], + vec!["--cached-model="], + vec!["--"], + ] { + assert!( + ParsedArgs::parse(strings(&args)).is_err(), + "args should fail: {args:?}" + ); + } + } - assert_eq!( - config.datastream_frame_log.as_deref(), - Some(Path::new("mvp-chat.log")) + #[test] + fn config_resolution_uses_defaults_toml_and_cli_precedence() { + let temp = TempDir::new("config-resolution"); + + with_process_state( + &[ + ("MVP_PIPELINE_STAGES", Some("9")), + ("MVP_RUNTIME_CONFIG", Some("local")), + ], + Some(temp.path()), + || { + let defaults = Config::from_args(Vec::::new()).expect("defaults resolve"); + assert_eq!(defaults.provider, ProviderKind::Process); + assert_eq!(defaults.pipeline_stages, 1); + assert!(defaults.datastream_frame_log.is_none()); + assert!(defaults.cached_model.is_none()); + assert!(defaults.vastai.is_none()); + assert!(!defaults.skip_rebuild); + + let config_path = write_config( + &temp, + "chat.toml", + r#" +[provider] +kind = "docker" + +[runtime] +pipeline_stages = 2 + +[observability] +dump_logs = true +dump_log_path = "toml.log" + +[image] +node = "docker.io/acme/node:toml" +tag = " alias " +"#, + ); + let config_arg = config_path.to_string_lossy().into_owned(); + let config = Config::from_args(strings(&[ + "--config", + config_arg.as_str(), + "--process", + "--pipeline-stages", + "4", + "--dump-logs=cli.log", + ])) + .expect("config resolves"); + + assert_eq!(config.provider, ProviderKind::Process); + assert_eq!(config.pipeline_stages, 4); + assert_eq!(config.datastream_frame_log, Some(PathBuf::from("cli.log"))); + assert_eq!(config.node_image, "docker.io/acme/node:toml"); + assert_eq!(config.image_tag, Some("alias".to_owned())); + }, ); } #[test] - fn dump_logs_conflicts_with_explicit_datastream_frame_log() { - let result = Config::from_args( - [ - "--dump-logs", - "--datastream-frame-log", - "/tmp/frames.jsonl", - "--orch-bin", - "/tmp/mvp-orchestrator", - ] - .into_iter() - .map(str::to_owned), + fn config_max_tokens_drives_orchestrator_args_and_submit_prompt() { + let temp = TempDir::new("config-max-tokens"); + let config_path = write_config( + &temp, + "chat.toml", + r#" +[runtime] +max_tokens = 12 +"#, ); - let error = match result { - Ok(_) => panic!("conflicting datastream log destinations must fail"), - Err(error) => error, + + with_process_state(&[], Some(temp.path()), || { + let config_arg = config_path.to_string_lossy().into_owned(); + let config = Config::from_args(strings(&["--config", config_arg.as_str()])) + .expect("max_tokens config resolves"); + assert_eq!(config.max_tokens, 12); + + let args = config.orchestrator_cli_args("resolved-image"); + let max_tokens_arg = args + .windows(2) + .find(|pair| pair[0] == "--max-tokens") + .map(|pair| pair[1].as_str()); + assert_eq!(max_tokens_arg, Some("12"), "{args:?}"); + + let mut rpc_writer = Vec::new(); + let reader = event_reader(&[done(1)]); + let input = channel_lines(&["hello"]); + let mut output = Vec::new(); + run_chat_session_with_output( + &mut rpc_writer, + reader, + input, + config.max_tokens, + &mut output, + ) + .expect("prompt loop completes"); + + assert_eq!( + submitted_prompts(&rpc_writer), + vec![SubmitPrompt { + request_id: 1, + prompt_text: "hello".to_owned(), + max_tokens: 12, + }] + ); + }); + } + + #[test] + fn config_rejects_zero_max_tokens() { + let temp = TempDir::new("config-zero-max-tokens"); + let config_path = write_config( + &temp, + "chat.toml", + r#" +[runtime] +max_tokens = 0 +"#, + ); + + with_process_state(&[], Some(temp.path()), || { + let config_arg = config_path.to_string_lossy().into_owned(); + assert!(Config::from_args(strings(&["--config", config_arg.as_str()])).is_err()); + }); + } + + #[test] + fn config_rejects_out_of_spec_sections() { + let temp = TempDir::new("config-strict-surface"); + let config_path = write_config( + &temp, + "chat.toml", + r#" +[prompt] +max_tokens = 7 +"#, + ); + + with_process_state(&[], Some(temp.path()), || { + let config_arg = config_path.to_string_lossy().into_owned(); + assert!(Config::from_args(strings(&["--config", config_arg.as_str()])).is_err()); + }); + } + #[test] + fn config_rejects_invalid_pipeline_provider_and_missing_images() { + let temp = TempDir::new("config-rejections"); + + with_process_state(&[], Some(temp.path()), || { + let zero_pipeline = write_config( + &temp, + "zero-pipeline.toml", + r#" +[runtime] +pipeline_stages = 0 +"#, + ); + let zero_pipeline_arg = zero_pipeline.to_string_lossy().into_owned(); + assert!(Config::from_args(strings(&["--config", zero_pipeline_arg.as_str()])).is_err()); + + let invalid_provider = write_config( + &temp, + "invalid-provider.toml", + r#" +[provider] +kind = "mock" +"#, + ); + let invalid_provider_arg = invalid_provider.to_string_lossy().into_owned(); + assert!( + Config::from_args(strings(&["--config", invalid_provider_arg.as_str()])).is_err() + ); + + assert!(Config::from_args(strings(&["--docker"])).is_err()); + assert!(Config::from_args(strings(&["--vastai"])).is_err()); + }); + } + + #[test] + fn vastai_config_requires_secret_relay_bootstrap_and_remote_image() { + let missing_secret = TempDir::new("vastai-missing-secret"); + let missing_secret_config = write_config( + &missing_secret, + "chat.toml", + r#" +[provider] +kind = "vastai" + +[image] +node = "docker.io/acme/node:latest" + +[vastai] +relay_url = "https://relay.example" +bootstrap_command = "boot" +"#, + ); + with_process_state(&[], Some(missing_secret.path()), || { + let config_arg = missing_secret_config.to_string_lossy().into_owned(); + assert!(Config::from_args(strings(&["--config", config_arg.as_str()])).is_err()); + }); + + let missing_relay = TempDir::new("vastai-missing-relay"); + let missing_relay_config = write_config( + &missing_relay, + "chat.toml", + r#" +[provider] +kind = "vastai" + +[image] +node = "docker.io/acme/node:latest" + +[vastai] +bootstrap_command = "boot" +"#, + ); + with_process_state( + &[("VAST_API_KEY", Some("secret"))], + Some(missing_relay.path()), + || { + let config_arg = missing_relay_config.to_string_lossy().into_owned(); + assert!(Config::from_args(strings(&["--config", config_arg.as_str()])).is_err()); + }, + ); + + let local_image = TempDir::new("vastai-local-image"); + let local_image_config = write_config( + &local_image, + "chat.toml", + r#" +[provider] +kind = "vastai" + +[image] +node = "local-node:latest" + +[vastai] +relay_url = "https://relay.example" +bootstrap_command = "boot" +"#, + ); + with_process_state( + &[("VAST_API_KEY", Some("secret"))], + Some(local_image.path()), + || { + let config_arg = local_image_config.to_string_lossy().into_owned(); + assert!(Config::from_args(strings(&["--config", config_arg.as_str()])).is_err()); + }, + ); + + let valid = TempDir::new("vastai-valid"); + let valid_config = write_config( + &valid, + "chat.toml", + r#" +[provider] +kind = "vastai" + +[image] +node = "docker.io/acme/node:latest" + +[vastai] +relay_url = "https://relay.example" +bootstrap_command = "boot" +"#, + ); + with_process_state( + &[("VAST_API_KEY", Some("secret"))], + Some(valid.path()), + || { + let config_arg = valid_config.to_string_lossy().into_owned(); + let config = Config::from_args(strings(&["--config", config_arg.as_str()])) + .expect("valid Vast.ai config resolves"); + let vastai = config.vastai.as_ref().expect("resolved Vast.ai config"); + assert_eq!(vastai.api_key, "secret"); + assert_eq!(vastai.relay_url, "https://relay.example"); + assert_eq!(vastai.bootstrap_command, "boot"); + assert_eq!(vastai.image, "docker.io/acme/node:latest"); + }, + ); + } + + struct MockApproval { + terminal: bool, + answer: Result, + } + + impl VastAiApproval for MockApproval { + fn stdin_is_terminal(&self) -> bool { + self.terminal + } + + fn ask(&mut self) -> Result { + self.answer.clone() + } + } + + #[test] + fn parse_approval_accepts_only_yes_variants() { + for value in ["y", "Y", " yes \n", "YeS"] { + assert!(parse_approval(value), "{value:?} should approve"); + } + for value in ["", "n", "no", "yep", " yes please"] { + assert!(!parse_approval(value), "{value:?} should decline"); + } + } + + #[test] + fn vastai_approval_is_used_only_when_required() { + let process = base_config(ProviderKind::Process); + let mut approval = MockApproval { + terminal: false, + answer: Err("should not ask".to_owned()), + }; + confirm_vastai_if_needed_with_approval(&process, &mut approval) + .expect("non-Vast.ai skips approval"); + + let mut yes_config = base_config(ProviderKind::VastAi); + yes_config.vastai = Some(valid_vastai()); + yes_config.vastai_yes = true; + let mut approval = MockApproval { + terminal: false, + answer: Err("should not ask".to_owned()), + }; + confirm_vastai_if_needed_with_approval(&yes_config, &mut approval) + .expect("--yes skips approval prompt"); + + let mut non_terminal = base_config(ProviderKind::VastAi); + non_terminal.vastai = Some(valid_vastai()); + let mut approval = MockApproval { + terminal: false, + answer: Err("should not ask".to_owned()), + }; + assert!(confirm_vastai_if_needed_with_approval(&non_terminal, &mut approval).is_err()); + + let mut accepted = base_config(ProviderKind::VastAi); + accepted.vastai = Some(valid_vastai()); + let mut approval = MockApproval { + terminal: true, + answer: Ok(true), + }; + confirm_vastai_if_needed_with_approval(&accepted, &mut approval) + .expect("interactive approval accepts"); + + let mut declined = base_config(ProviderKind::VastAi); + declined.vastai = Some(valid_vastai()); + let mut approval = MockApproval { + terminal: true, + answer: Ok(false), + }; + assert!(confirm_vastai_if_needed_with_approval(&declined, &mut approval).is_err()); + } + + #[test] + fn cached_model_discovery_selects_first_sorted_gguf_file() { + let temp = TempDir::new("cached-model-selects"); + let cache_dir = temp.path().join(".model-cache"); + fs::create_dir_all(&cache_dir).expect("create cache dir"); + fs::write(cache_dir.join("z.gguf"), b"z").expect("write z model"); + fs::write(cache_dir.join("a.gguf"), b"a").expect("write a model"); + fs::write(cache_dir.join("ignored.txt"), b"ignored").expect("write ignored file"); + fs::create_dir(cache_dir.join("0.gguf")).expect("create ignored directory"); + + with_process_state(&[], Some(temp.path()), || { + let cached = CachedModelConfig::discover().expect("cached model discovered"); + assert_eq!(cached.host_path.file_name(), Some(OsStr::new("a.gguf"))); + }); + } + + #[test] + fn cached_model_discovery_errors_when_no_usable_model_exists() { + let missing = TempDir::new("cached-model-missing"); + with_process_state(&[], Some(missing.path()), || { + assert!(CachedModelConfig::discover().is_err()); + }); + + let empty = TempDir::new("cached-model-empty"); + let cache_dir = empty.path().join(".model-cache"); + fs::create_dir_all(&cache_dir).expect("create cache dir"); + fs::write(cache_dir.join("ignored.txt"), b"ignored").expect("write ignored file"); + fs::create_dir(cache_dir.join("not-a-file.gguf")).expect("create ignored directory"); + with_process_state(&[], Some(empty.path()), || { + assert!(CachedModelConfig::discover().is_err()); + }); + } + + #[test] + fn cached_model_path_resolves_regular_gguf_file() { + let temp = TempDir::new("cached-model-path"); + let model = temp.path().join("chosen.gguf"); + fs::write(&model, b"model").expect("write chosen model"); + let model_arg = model.to_string_lossy().into_owned(); + + with_process_state(&[], Some(temp.path()), || { + let parsed = ParsedArgs::parse(strings(&[&format!("--cached-model={model_arg}")])) + .expect("cached model path parses"); + assert_eq!( + parsed.cached_model, + Some(CachedModelSource::Path(PathBuf::from(model_arg.as_str()))) + ); + + let config = Config::from_args(strings(&[&format!("--cached-model={model_arg}")])) + .expect("cached model path resolves"); + assert_eq!( + config.cached_model.unwrap().host_path.file_name(), + Some(OsStr::new("chosen.gguf")) + ); + + let upper_model = temp.path().join("upper.GGUF"); + fs::write(&upper_model, b"model").expect("write uppercase model"); + let upper = CachedModelConfig::from_path(upper_model) + .expect("uppercase cached model extension resolves"); + assert_eq!(upper.host_path.file_name(), Some(OsStr::new("upper.GGUF"))); + }); + } + + fn panic_prepare_node_image(_: NodeImageRequest) -> Result { + panic!("image preparer must not be called when --skip-rebuild is set") + } + + #[test] + fn skip_rebuild_requires_existing_artifacts_and_skips_image_preparation() { + let temp = TempDir::new("skip-rebuild"); + let orch_bin = temp.path().join("mvp-orchestrator"); + let worker_bin = temp.path().join("mvp-worker-node"); + let mut config = base_config(ProviderKind::Docker); + config.skip_rebuild = true; + config.orch_bin = orch_bin.clone(); + config.worker_bin = worker_bin.clone(); + config.node_image = "docker.io/acme/node:latest".to_owned(); + + assert!(prepare_runtime_with(&config, panic_prepare_node_image).is_err()); + + fs::write(&orch_bin, b"orch").expect("write orchestrator artifact"); + assert!(prepare_runtime_with(&config, panic_prepare_node_image).is_err()); + + fs::write(&worker_bin, b"worker").expect("write worker artifact"); + let image_ref = prepare_runtime_with(&config, panic_prepare_node_image) + .expect("skip rebuild uses existing artifacts"); + assert_eq!(image_ref, "docker.io/acme/node:latest"); + } + + #[test] + fn artifact_roots_use_current_directory() { + let temp = TempDir::new("artifact-root"); + + with_process_state(&[], Some(temp.path()), || { + let path = default_orch_bin().expect("default orchestrator path resolves"); + assert!(path.starts_with(temp.path()), "{path:?}"); + assert!(path.ends_with("target/debug/mvp-orchestrator"), "{path:?}"); + }); + } + + #[cfg(target_os = "linux")] + #[test] + fn orch_child_shutdown_sends_sigterm_to_process_group() { + let temp = TempDir::new("orch-shutdown"); + let flag_path = temp.path().join("term.flag"); + let mut command = Command::new("sh"); + command + .args([ + "-c", + "trap 'echo term > \"$1\"; exit 0' TERM; while true; do sleep 1; done", + "sh", + ]) + .arg(&flag_path) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + unsafe { + command.pre_exec(|| { + if libc::setpgid(0, 0) == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } + }); + } + let child = command.spawn().expect("spawn signal test child"); + thread::sleep(Duration::from_millis(100)); + let mut orch = OrchChild { + child, + cleaned: false, }; + orch.shutdown(); + + assert!(flag_path.exists(), "SIGTERM trap should write flag"); + } + + #[test] + fn prompt_loop_exits_cleanly_and_ignores_empty_prompts() { + let mut rpc_writer = Vec::new(); + let reader = event_reader(&[]); + let input = channel_lines(&["", " "]); + let mut output = Vec::new(); + + run_chat_session_with_output(&mut rpc_writer, reader, input, 7, &mut output) + .expect("prompt loop exits"); + + assert!(rpc_writer.is_empty()); + let output = String::from_utf8(output).expect("output is UTF-8"); + assert_eq!(output.matches("prompt:> ").count(), 3, "{output:?}"); + assert!(!output.contains("decoding..."), "{output:?}"); + } + + #[test] + fn prompt_loop_submits_prompts_streams_text_and_increments_request_ids() { + let mut rpc_writer = Vec::new(); + let reader = event_reader(&[ + PromptEvent::TextDelta { + request_id: 1, + text: "hi".to_owned(), + }, + done(1), + PromptEvent::TextDelta { + request_id: 2, + text: "bye".to_owned(), + }, + done(2), + ]); + let input = channel_lines(&["hello\n", "again"]); + let mut output = Vec::new(); + + run_chat_session_with_output(&mut rpc_writer, reader, input, 7, &mut output) + .expect("prompt loop completes"); + assert_eq!( - error, - "--dump-logs cannot be combined with --datastream-frame-log; use one datastream log destination" + submitted_prompts(&rpc_writer), + vec![ + SubmitPrompt { + request_id: 1, + prompt_text: "hello".to_owned(), + max_tokens: 7, + }, + SubmitPrompt { + request_id: 2, + prompt_text: "again".to_owned(), + max_tokens: 7, + }, + ] + ); + assert_eq!( + String::from_utf8(output).expect("output is UTF-8"), + "prompt:> decoding...\nResponse: hi\nprompt:> decoding...\nResponse: bye\nprompt:> " ); } #[test] - fn confirm_vastai_if_needed_accepts_yes_without_terminal_approval() { - let config = vastai_yes_chat_config(); + fn prompt_loop_rejects_mismatched_response_request_id() { + let mut rpc_writer = Vec::new(); + let reader = event_reader(&[PromptEvent::TextDelta { + request_id: 99, + text: "wrong".to_owned(), + }]); + let input = channel_lines(&["hello"]); + let mut output = Vec::new(); - confirm_vastai_if_needed(&config, FakeOfferPreviewer) - .expect("--yes accepts the previewed Vast.ai rental"); + let error = run_chat_session_with_output(&mut rpc_writer, reader, input, 7, &mut output) + .expect_err("mismatched request id fails"); + + assert!(error.contains("prompt RPC protocol error"), "{error}"); + let prompts = submitted_prompts(&rpc_writer); + assert_eq!(prompts.len(), 1); + assert_eq!(prompts[0].request_id, 1); } #[test] - fn orch_rebuild_missing_binary_requires_rebuild() { - let workspace = TempWorkspace::new("missing-binary"); - workspace.write("src/main.rs", b"fn main() {}\n"); - let bin = workspace.path("target/debug/mvp-orchestrator"); + fn prompt_loop_fault_is_expected_prompt_result() { + let mut rpc_writer = Vec::new(); + let reader = event_reader(&[PromptEvent::Fault { + request_id: 1, + error: "boom".to_owned(), + }]); + let input = channel_lines(&["bad"]); + let mut output = Vec::new(); - let needed = orch_rebuild_needed(&bin, &workspace.root, &["src/main.rs"]) - .expect("missing binary check succeeds"); + run_chat_session_with_output(&mut rpc_writer, reader, input, 7, &mut output) + .expect("fault is a prompt result"); - assert!(needed, "missing orchestrator binary must trigger rebuild"); + assert_eq!(submitted_prompts(&rpc_writer).len(), 1); + let output = String::from_utf8(output).expect("output is UTF-8"); + assert!(output.contains("error: boom\n"), "{output:?}"); + } + + struct FailingBufRead; + + impl Read for FailingBufRead { + fn read(&mut self, _buf: &mut [u8]) -> io::Result { + Err(io::Error::new(io::ErrorKind::Other, "reader failed")) + } + } + + impl BufRead for FailingBufRead { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + Err(io::Error::new(io::ErrorKind::Other, "reader failed")) + } + + fn consume(&mut self, _amt: usize) {} } #[test] - fn orch_rebuild_binary_newer_than_input_skips_rebuild() { - let workspace = TempWorkspace::new("fresh-binary"); - let input = workspace.write("src/main.rs", b"fn main() {}\n"); - let input_mtime = modified_time(&workspace.root, &input).expect("read input mtime"); - let bin = workspace.path("target/debug/mvp-orchestrator"); - write_file_newer_than(&bin, b"orchestrator binary\n", input_mtime); + fn prompt_loop_reports_prompt_rpc_errors() { + let mut rpc_writer = Vec::new(); + let input = channel_lines(&["hello"]); + let mut output = Vec::new(); + let error = run_chat_session_with_output( + &mut rpc_writer, + Cursor::new(Vec::new()), + input, + 7, + &mut output, + ) + .expect_err("closed RPC fails"); + assert!(error.contains("prompt RPC closed"), "{error}"); - let needed = orch_rebuild_needed(&bin, &workspace.root, &["src/main.rs"]) - .expect("fresh binary check succeeds"); + let mut rpc_writer = Vec::new(); + let input = channel_lines(&["hello"]); + let mut output = Vec::new(); + let error = run_chat_session_with_output( + &mut rpc_writer, + Cursor::new(b"not-json\n".to_vec()), + input, + 7, + &mut output, + ) + .expect_err("malformed event fails"); + assert!(error.contains("parse prompt RPC event"), "{error}"); + let mut rpc_writer = Vec::new(); + let input = channel_lines(&["hello"]); + let mut output = Vec::new(); + let error = + run_chat_session_with_output(&mut rpc_writer, FailingBufRead, input, 7, &mut output) + .expect_err("read error fails"); assert!( - !needed, - "binary newer than every tracked input must skip rebuild" - ); - } - - #[test] - fn orch_rebuild_nested_directory_input_newer_than_binary_requires_rebuild() { - let workspace = TempWorkspace::new("nested-newer-input"); - let nested_input = workspace.write("src/nested/orchestrator.rs", b"old source\n"); - let src_mtime = - latest_mtime(&workspace.root, &workspace.path("src")).expect("read source tree mtime"); - let bin = workspace.path("target/debug/mvp-orchestrator"); - write_file_newer_than(&bin, b"orchestrator binary\n", src_mtime); - let bin_mtime = modified_time(&workspace.root, &bin).expect("read binary mtime"); - write_file_newer_than(&nested_input, b"new source\n", bin_mtime); - - let needed = orch_rebuild_needed(&bin, &workspace.root, &["src"]) - .expect("stale binary check succeeds"); - - assert!( - needed, - "newer file inside a tracked directory must trigger rebuild" + error.contains("read prompt RPC event: reader failed"), + "{error}" ); } } diff --git a/crates/mvp-system/src/config.rs b/crates/mvp-system/src/config.rs index 374b8fb..008f14b 100644 --- a/crates/mvp-system/src/config.rs +++ b/crates/mvp-system/src/config.rs @@ -6,8 +6,9 @@ use swactor_vastai::SelectionPolicy; pub const DEFAULT_CONFIG_PATH: &str = ".config/config.toml"; -/// Transient serde target for `.config/config.toml`. -/// This is not runtime state; it exists only to interpret TOML fields and overlay them onto hardcoded defaults. +/// Shared overlay for legacy and multi-binary configuration parsing. This accepts +/// fields outside the fixed `mvp-chat` public config surface; `mvp-chat` uses a +/// bin-local strict config loader instead. #[derive(Clone, Debug, Default, Deserialize, PartialEq)] #[serde(default)] pub struct TomlConfigOverlay { @@ -86,6 +87,8 @@ pub struct DockerConfigOverlay { #[derive(Clone, Debug, Default, Deserialize, PartialEq)] #[serde(default)] pub struct ObservabilityConfigOverlay { + pub dump_logs: Option, + pub dump_log_path: Option, pub datastream_frame_log: Option, } @@ -94,6 +97,7 @@ pub struct ObservabilityConfigOverlay { pub struct VastAiConfig { pub api_key: Option, pub image: Option, + pub relay_url: Option, pub bootstrap_command: Option, pub disk_gb: Option, pub ssh_user: Option, @@ -170,7 +174,7 @@ impl TomlConfigOverlay { impl ResolvedVastAiConfig { pub fn validate(self) -> Result { - require_non_empty("VASTAI_API_KEY", &self.api_key)?; + require_non_empty("VAST_API_KEY", &self.api_key)?; require_non_empty("relay.url", &self.relay_url)?; require_non_empty("vastai.image", &self.image)?; require_non_empty("vastai.bootstrap_command", &self.bootstrap_command)?; diff --git a/crates/mvp-system/tests/one_node_chat_e2e.rs b/crates/mvp-system/tests/one_node_chat_e2e.rs index f391b76..4eddd9b 100644 --- a/crates/mvp-system/tests/one_node_chat_e2e.rs +++ b/crates/mvp-system/tests/one_node_chat_e2e.rs @@ -101,8 +101,7 @@ fn one_node_chat_process_cached_model_e2e() { command .current_dir(&root) .args(["mvp-chat", "--cached-model"]) - .arg(&cached_model) - .args(["-N", "2", "--dump-logs"]) + .args(["--pipeline-stages", "2", "--dump-logs"]) .env("MVP_RUNTIME_CONFIG", "local") .env("MVP_IROH_RELAY_MODE", "disabled") .env(DOCKER_CONTAINER_PREFIX_ENV, &container_prefix) diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 9136339..d53c0b5 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -61,7 +61,7 @@ fn print_usage() { USAGE: cargo xtask COMMANDS: - mvp-chat [--process|--docker|--vastai] [-N n] [--cached-model [path]] [args...] Run the human chat wrapper against the real orchestrator/worker bins. + mvp-chat [--process|--docker|--vastai] [--pipeline-stages n] [--cached-model] [-- args...] Run the human chat wrapper against the real orchestrator/worker bins. test Run all basic non-binding tests. This includes the root crate with `cargo test` plus each non-binding repository package with `cargo test -p`. Feature-gated E2E/bin tests are intentionally excluded."