diff --git a/Cargo.lock b/Cargo.lock index 0699444..e6fe324 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2920,6 +2920,13 @@ dependencies = [ "uuid", ] +[[package]] +name = "mvp-system" +version = "0.1.0" +dependencies = [ + "libc", +] + [[package]] name = "n0-error" version = "0.1.3" @@ -5370,6 +5377,7 @@ dependencies = [ "crossbeam-queue", "crossbeam-utils", "getrandom 0.2.17", + "mvp-system", "proptest", "proptest-state-machine", "serde", diff --git a/Cargo.toml b/Cargo.toml index a92fc9a..c9e8b58 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "crates/distribution", "crates/datastream", "crates/datastore", + "crates/mvp-system", "crates/node", "tests/docker", "tests/integration", @@ -56,6 +57,11 @@ unexpected_cfgs = { level = "allow", check-cfg = ['cfg(kani)'] } criterion = { version = "0.5", features = ["html_reports"] } proptest = "1" proptest-state-machine = "0.3" +mvp-system = { path = "crates/mvp-system" } + +[[test]] +name = "arena_manager_guarantees" +path = "tests/mvp_system/arena_manager_guarantees.rs" [[bench]] name = "runtime_benchmarks" diff --git a/GPU_WORKER_INTERFACE_SPEC.md b/GPU_WORKER_INTERFACE_SPEC.md index 2cb445f..9966122 100644 --- a/GPU_WORKER_INTERFACE_SPEC.md +++ b/GPU_WORKER_INTERFACE_SPEC.md @@ -187,8 +187,6 @@ outside payload ownership. Ring cursors are the authority. ### 4.4 Device Bridge -USER: If it is determined this is an outsized amount of engineering work, we will defer and require objects to be complete before gpu io. - The worker must have a backend-specific device bridge capable of range copies: ```text @@ -445,7 +443,6 @@ may be available. The worker must reload ring cursors from shared memory. Duplicate hints may be coalesced. ### 7.7 ExecuteStep -USER: Why do we need this explicit? Why doesnt it fall out naturally from the I/O? This whole thing is async message driven, so why do we need a start message? ```rust struct ExecuteStep { @@ -654,8 +651,6 @@ all copy events for that object have completed. committed to the egress ring. It does not mean the remote node received it. ### 8.5 Step Events -USER: Why is this here? It seems useless. - ```rust StepCompleted { diff --git a/MVP_SYSTEM_SPEC.md b/MVP_SYSTEM_SPEC.md new file mode 100644 index 0000000..ef86624 --- /dev/null +++ b/MVP_SYSTEM_SPEC.md @@ -0,0 +1,2332 @@ +# MVP System Specification + +**Status:** draft consolidated system specification. + +This document is the single MVP reference for the swactor GGUF pipeline system. +It folds the system behavior previously split across the orchestration, +ring/backpressure, and GPU-worker drafts into one end-to-end specification. + +The focused root specs may remain as design history while this document is being +stabilized. For the MVP drafting process, this document is the place where the +complete system, its components, guarantees, and systemic behavior are described +together. + +--- + +## 1. Purpose + +The MVP is a trusted distributed ML runtime for one linear GGUF inference +pipeline over a rented pool of GPU nodes. + +The first deployment shape is: + +```text +operator provisions rented GPU nodes + -> each node runs a barebones Docker image + -> the image contains swactor runtime + tinygrad/CUDA worker + -> nodes join membership and reach SWIM convergence + -> orchestrator assigns pipeline roles and edges + -> nodes download/load assigned GGUF weights + -> nodes provision arena-backed data edges + -> nodes report readiness + -> orchestrator injects a "hello world" prompt + -> pipeline returns tokens + -> run completes or faults and tears down +``` + +The document is intentionally about the first complete feature, not a broad +graph engine. Later features can generalize from this system after the MVP +behavior is stable. + +--- + +## 2. Scope + +In scope: + +- trusted node pool boot and readiness +- SWIM membership convergence as the planning gate +- orchestrator authority, planning, staging, faults, and teardown +- one linear GGUF pipeline +- stage/layer assignment and edge id assignment +- stage-local weight download, load, and bind readiness +- arena-backed rings and iroh/QUIC edge transport +- Rust node process to Python/tinygrad worker process control +- GPU worker object loading, step execution, and object production +- prompt injection, prefill, decode, token return +- completed, faulted, and operator-stopped terminal outcomes +- observability events needed for behavioral contracts and tests + +Out of scope: + +- arbitrary graph IR +- automatic placement optimization +- batching, speculative decoding, continuous serving +- re-placement after churn or node failure +- trustless verification, incentives, Sybil defense, adversarial tensor checks +- production artifact layout and cache eviction policy +- VastAI account automation, bidding, billing, or provider-specific control + machinery +- high-performance host pinning policy beyond the safety contract + +--- + +## 3. Design Commitments + +Nodes are trusted. Payload content is trusted after structural checks. The +system does not defend against malicious tensor values or adversarial peers. + +The orchestrator is the run authority. It owns placement, run planning, +provisioning, prompt injection, terminal outcome, and teardown. + +swactor owns the control plane. Actors establish, observe, and tear down +components. Actors do not move tensor payload bytes. + +Payload bytes move through shared-memory rings and iroh/QUIC streams. The hot +path is ring cursors, wake hints, and byte-pump tasks, not actor mailboxes. + +The GPU worker is a supervised process, not a separate swactor runtime. It maps +the shared arena, uses a native ring helper, owns device allocations, and runs +tinygrad role code when explicitly commanded. + +Compute is explicit. `ObjectLoaded` means data is ready on device. +`ExecuteStep` is the local compute admission command. Loading an object does not +automatically run tinygrad. + +Backpressure is absence of writable ring space plus QUIC flow control. There is +no actor-level credit protocol, RTS/CTS exchange, or per-range acknowledgement. + +One persistent iroh uni-stream carries all objects for one edge. The stream +starts with an `edge_id` preamble and then object records. + +The MVP permits transfer and device upload/download to overlap with network +streaming. It does not permit compute to observe a logical object until the +whole object has loaded. + +--- + +## 4. Trust And Authority Model + +The orchestrator owns: + +- intended node pool +- resource inventory used by placement +- run ids +- model and runtime configuration +- tokenizer/vocabulary facts +- stage count and stage order +- stage-to-node assignment +- layer range assignment +- edge id assignment +- object specs and ring specs +- weight source assignment +- global readiness barrier +- prompt tokenization and prompt injection +- output token consumption +- EOS and `max_tokens` stop policy +- run-level fault and teardown decisions + +Each node owns: + +- stable node identity for the run +- node process lifecycle +- GPU worker process lifecycle +- local arena and arena leases +- local iroh endpoint and byte pumps +- SWIM participant +- local stage controller +- local edge establishment records +- assigned weight download/load work +- local worker/device resources + +Nodes do not rewire the graph. A node accepts run provisioning only from the +authorized orchestrator and rejects edge data or provisioning that does not match +its active run plan. + +Other stages cannot redirect a node to a new peer, edge id, or layer range. + +Resource inventory is known to the orchestrator before placement in the MVP. A +node may report boot health and readiness, but there is no distributed +capability negotiation protocol after boot. + +--- + +## 5. System Topology + +Each GPU node runs one Rust node process and one Python/tinygrad GPU worker +process: + +```text +GPU node container +------------------ +Rust node process + swactor runtime + SWIM participant + iroh endpoint and driver + ArenaManager + EdgeEstablisher + Tx/Rx edge actors + GpuWorkerCtl + StageController + +Python/tinygrad worker process + mapped shared arena + native ring helper + device bridge + role code + device allocations and KV/state +``` + +The orchestrator may run in its own container or on one of the provisioned GPU +machines. It is a control participant and token endpoint. It does not run model +compute for the MVP. + +When the orchestrator participates in token edges over the data plane, it has a +stable `node_id`, an iroh endpoint, and local token edge endpoints like any +other producer or consumer node. Co-locating the orchestrator with a GPU node is +an implementation choice, not a change to edge semantics. + +The system has two network-facing planes: + +- membership/control observation: SWIM provides node liveness/membership state + used by the orchestrator as the pool readiness gate +- data movement: iroh/QUIC carries persistent uni-streams for data edges, + addressed by `(node_id, edge_id)` + +Provisioning, readiness, faults, and teardown are swactor messages. Tensor bytes +do not travel in actor messages. + +--- + +## 6. Identifiers + +Identifier types are schematic. Concrete Rust APIs may wrap these as newtypes. + +```rust +struct RunId(u64); +struct NodeId(u64); +struct RoleId(u64); +struct StageIndex(u32); +struct EdgeId(u64); +struct RingId(u64); +struct ObjectId(u64); +struct Sequence(u64); +struct StepId(u64); +struct WorkerGeneration(u64); +``` + +`EdgeId` is unique within a run and assigned only by the orchestrator. + +`RingId` is unique for the node lifetime. Arena ranges may be reused after +quiescence, but ring ids are not reused. + +`DeviceObjectHandle` is opaque to the Rust node process: + +```rust +struct DeviceObjectHandle { + worker_generation: u64, + id: u64, +} +``` + +A device handle is valid only in the worker generation that created it. Worker +restart invalidates all prior handles, roles, rings, and in-flight steps. + +--- + +## 7. System Lifecycle + +The complete MVP lifecycle is: + +```text +Deployment + -> NodeBoot + -> MembershipConvergence + -> RunPlanning + -> StageProvisioning + -> WeightLoading + -> EdgeProvisioning + -> GlobalReadiness + -> PromptInjection + -> PipelineExecution + -> CompletionOrFault + -> Teardown + -> Done +``` + +Phase ownership: + +- node boot is local to each node +- membership convergence is observed by the orchestrator +- run planning is owned by the orchestrator +- local provisioning is owned by each node +- global readiness is owned by the orchestrator +- execution progress is driven by object arrival at stages +- terminal outcome and teardown are owned by the orchestrator + +There is no broadcast start message. The first prompt object written by the +orchestrator after the readiness barrier starts execution. + +--- + +## 8. Node Boot + +At container start, each node runs: + +```text +ContainerStarting + -> NodeProcessStarting + -> ArenaReady + -> WorkerReady + -> TransportReady + -> MembershipJoining + -> NodeAvailable +``` + +`NodeAvailable` means: + +- Rust node process is alive +- swactor runtime can receive control messages +- node has a stable `node_id` known to the orchestrator +- arena is created and mapped in the node process +- GPU worker has emitted `WorkerReady` +- iroh endpoint is initialized and associated with the node identity +- SWIM participant has joined or is joining the intended pool +- node can accept run provisioning + +`NodeAvailable` does not mean weights are present, a role is configured, or any +run edge is established. + +If implementation defers worker startup until run provisioning, the same +run-level gate must still hold: a stage cannot report `StageReady` before its +worker is ready and assigned weights are usable. + +Boot failure before `ArenaReady` or `WorkerReady` means the node is unavailable. +No run edge should be provisioned to that node. + +--- + +## 9. Membership And Pool Readiness + +The orchestrator starts run planning only after the intended candidate pool is +ready. + +For the MVP, `PoolReady` means: + +- every node in the intended candidate pool is known to the orchestrator +- every candidate node is live in the SWIM membership view +- every candidate node has reported `NodeAvailable` +- every candidate node has data-plane identity/endpoint material the + orchestrator can provision into edges +- no candidate node is currently suspect or faulted in the orchestrator view +- the pool view has remained stable for the configured convergence window + +The convergence window is an implementation parameter. + +Nodes do not compute placement and do not need to agree on graph state. SWIM is a +membership/liveness input to the orchestrator, not a distributed graph protocol. + +If pool readiness is lost before a `RunPlan` is committed, the orchestrator keeps +waiting or aborts before provisioning according to local policy. + +If pool readiness is lost after provisioning begins, the run faults. The MVP does +not re-place an active run after a node disappears. + +--- + +## 10. GGUF Pipeline Workload + +The MVP workload is one linear pipeline: + +```text +orchestrator --tokens--> stage 0 --activations--> stage 1 --activations--> +... --activations--> stage N-1 --tokens--> orchestrator +``` + +The orchestrator is a token endpoint and control participant. It does not run GPU +compute. + +Each stage owns a contiguous GGUF layer range. A stage receives typed input +objects, executes its assigned layer range, and writes typed output objects. + +Stage `0` consumes token objects from the orchestrator and produces activation +objects. Interior stages consume and produce activation objects. The last stage +consumes activation objects and produces token objects for the orchestrator. + +The last stage samples or otherwise produces token ids as part of its worker +step if sampling is delegated to the stage. If sampling is not delegated, the +worker output object must contain enough logits/token data for the orchestrator +to apply the configured policy. The `RunPlan` must state which policy is used. + +--- + +## 11. Run Plan + +The orchestrator builds exactly one `RunPlan` before provisioning: + +```rust +struct RunPlan { + run_id: RunId, + model: GgufModelPlan, + runtime: RuntimePlan, + stages: Vec, + edges: Vec, + max_tokens: u32, +} + +struct GgufModelPlan { + model_id: String, + gguf_source: GgufSource, + num_layers: u32, + hidden_dim: u32, + dtype_family: DTypeFamily, + dtype_width_bytes: u32, + max_seq_len: u32, + eos_token_id: u32, + tokenizer: TokenizerSource, +} + +struct RuntimePlan { + prompt: PromptSource, + sampling: SamplingPolicy, + token_output_policy: TokenOutputPolicy, +} +``` + +`gguf_source` may identify a whole GGUF file, a pre-split shard collection, a +cache key, a local path, or an HTTP/object-store artifact. The system contract is +the assigned layer range and readiness after that range is usable by the worker. + +Each stage receives one contiguous layer range: + +```rust +struct StagePlan { + run_id: RunId, + stage_index: u32, + stage_count: u32, + node_id: NodeId, + gguf_source: GgufSource, + layer_start: u32, + layer_end_exclusive: u32, + inbound_edge: EdgeId, + outbound_edge: EdgeId, +} +``` + +Each edge has exactly one producer and one consumer: + +```rust +struct EdgePlan { + run_id: RunId, + edge_id: EdgeId, + kind: EdgeKind, + producer: EdgeEndpoint, + consumer: EdgeEndpoint, + object_spec: ObjectSpec, + ring_spec: RingSpec, +} + +enum EdgeKind { + TokenIn, + Activation, + TokenOut, +} + +enum EdgeEndpoint { + Orchestrator { node_id: NodeId }, + Stage { node_id: NodeId, stage_index: u32 }, +} +``` + +Layer assignment guarantees: + +- stage layer ranges are contiguous +- stage layer ranges do not overlap +- the union of stage layer ranges covers the intended GGUF block range +- every stage has exactly one assigned range + +Edge assignment guarantees: + +- every edge id is unique within the run +- token-in is produced by the orchestrator and consumed by stage `0` +- token-out is produced by stage `N - 1` and consumed by the orchestrator +- activation edge `i` is produced by stage `i` and consumed by stage `i + 1` +- stages never derive edge ids from names, layer ranges, peer ids, or hashes + +--- + +## 12. Provisioning Messages + +The orchestrator sends one stage provision message to each stage node: + +```rust +struct ProvisionStage { + run_id: RunId, + stage_index: u32, + stage_count: u32, + gguf_source: GgufSource, + layer_start: u32, + layer_end_exclusive: u32, + inbound: InboundEdgeProvision, + outbound: OutboundEdgeProvision, + model: StageModelFacts, + runtime: StageRuntimeFacts, +} + +struct InboundEdgeProvision { + edge_id: EdgeId, + kind: EdgeKind, + object_spec: ObjectSpec, + ring_spec: RingSpec, +} + +struct OutboundEdgeProvision { + edge_id: EdgeId, + kind: EdgeKind, + consumer_node_id: NodeId, + object_spec: ObjectSpec, + ring_spec: RingSpec, +} + +struct StageModelFacts { + model_id: String, + hidden_dim: u32, + dtype_family: DTypeFamily, + dtype_width_bytes: u32, + max_seq_len: u32, +} + +struct StageRuntimeFacts { + role_id: RoleId, + input_port: PortId, + output_port: PortId, + sampling: Option, +} +``` + +The inbound edge is established locally as a receive edge. The outbound edge is +established locally as a send edge to `consumer_node_id`. + +For the last stage, `consumer_node_id` is the orchestrator node. + +Provisioning fan-out is parallel. There is no Tx-to-Rx actor handshake and no +peer-to-peer endpoint exchange. + +--- + +## 13. Orchestrator FSM + +The orchestrator has one run-level FSM: + +```text +Planning + build RunPlan + validate layer ranges, edge ids, object specs, node ids + -> Provisioning + +Provisioning + send ProvisionStage to every stage node + create local token-in producer + create local token-out consumer + -> WaitingReady + +WaitingReady + on StageReady for every stage and local token endpoints ready + -> Running + on StageFault, endpoint fault, membership loss, or timeout + -> Faulted + +Running + inject prompt token object sequence 0 + consume token objects from token-out edge in sequence order + after token sequence k: + if EOS or max_tokens reached -> Completed + else inject token object sequence k + 1 + on StageFault, edge fault, endpoint fault, membership loss, or timeout + -> Faulted + +Completed + stop injecting tokens + finalize output text or token stream + -> TearingDown + +Faulted + stop injecting tokens + record one run-level failure reason + -> TearingDown + +TearingDown + send StopRun to all provisioned stages + tear down local token endpoints + wait for StageStopped from every stage or timeout + -> Done + +Done + terminal +``` + +The orchestrator does not issue per-stage `ExecuteStep` during a run. After +prompt injection, stage execution is driven by object arrival and local stage +readiness. + +--- + +## 14. StageController FSM + +Each provisioned stage has one node-local StageController. It is control-path +only: it watches worker and edge events, issues worker commands, and reports +run-level events. It does not move payload bytes. + +```text +Unprovisioned + on ProvisionStage from authorized orchestrator + validate run_id and stage assignment + -> Preparing + +Preparing + configure local worker role path + start GGUF shard/range load and bind + establish inbound receive edge + establish outbound send edge + when worker configured, weights bound, and both edge ends ready + emit StageReady + -> Ready + on required setup failure + emit StageFault + -> Faulted + +Ready + on inbound ObjectLoaded(sequence = s) + if s is next expected sequence and local state is ready + issue ExecuteStep for sequence s + -> Executing + else + emit StageFault(sequence violation) + -> Faulted + on StopRun + -> Stopping + +Executing + worker runs exactly one step for the loaded input object + worker writes output object to outbound edge with same sequence + on StepCompleted + release per-step input handles according to policy + -> Ready + on StepFailed, ObjectFailed, output fault, or worker crash + emit StageFault + -> Faulted + +Faulted + reject new run work + wait for StopRun + -> Stopping + +Stopping + stop local edges + release per-run device objects + stop or reset worker role state according to local policy + emit StageStopped + -> Stopped + +Stopped + terminal for this run +``` + +The MVP allows one active `ExecuteStep` per stage. + +--- + +## 15. Weight Lifecycle + +Weights are stage-local persistent state for the run. + +The StageController starts weight work from the assigned `GgufSource` and layer +range. A stage may download: + +- a whole GGUF and load only its range +- one or more physical shards containing its range +- a cached artifact that already exists on the node + +The physical loading mechanism may be: + +- worker startup configuration +- a `ConfigureRole` command +- a local loader path owned by the StageController +- weight objects moved through the same object/ring machinery + +The system-visible contract is `WeightsReady` before `StageReady`. + +`WeightsReady` means: + +- assigned artifact bytes are locally available or already cached +- assigned layer range has been validated against the run plan +- GPU worker has loaded or bound the range needed for execution +- failures in download, parse, device allocation, or binding have surfaced as + stage faults + +Warm model reuse across runs is deferred. + +--- + +## 16. Execution Semantics + +Sequence `0` is prefill: + +```text +orchestrator writes prompt token object sequence 0 +stage 0 executes prefill over prompt tokens +stage 0 writes activation sequence 0 +each interior stage executes prefill over activation sequence 0 +last stage executes prefill and writes token sequence 0 +orchestrator consumes token sequence 0 +``` + +Decode sequences are `1..`: + +```text +orchestrator writes one-token object sequence k +stage 0 executes decode for sequence k +each downstream stage executes decode for sequence k +last stage writes token sequence k +orchestrator consumes token sequence k +``` + +For every stage: + +- inbound object sequence equals outbound object sequence +- a stage cannot execute before weights are loaded and bound +- a stage cannot execute before inbound object is loaded +- a stage cannot produce to an edge that is not ready +- `ObjectLoaded` is data readiness, not compute completion +- `ObjectProduced` is output object committed to egress ring +- `StepCompleted` is the compute transaction terminal success event + +The orchestrator writes sequence `k + 1` only after consuming output token +sequence `k` and deciding the run should continue. + +--- + +## 17. Object Specs And Object Records + +Objects are logical payloads on an edge: token batches, activations, weights, or +model shards. + +Object specs are role-known validation contracts: + +```rust +struct ObjectSpec { + kind: ObjectKind, + max_extent: u64, + dtype_family: DTypeFamily, + dtype_width_bytes: u32, + shape: ShapeRule, + layout: LayoutRule, + alignment: u32, + sequence_policy: SequencePolicy, +} +``` + +The stream carries object records: + +```text +ObjectHeader +payload bytes, exactly header.extent bytes +``` + +The MVP header is fixed-size: + +```rust +struct ObjectHeader { + magic: u32, + version: u16, + header_len: u16, + object_id: u64, + sequence: u64, + extent: u64, + flags: u32, + reserved: u32, +} +``` + +The header supplies runtime facts: object id, sequence, extent, and flags. +`ObjectSpec` supplies dtype, shape/layout family, max extent, alignment, and +sequence policy. + +Activation maximum extent: + +```text +max_extent = max_seq_len * hidden_dim * dtype_width_bytes +``` + +For prefill, `extent` may cover many token rows. For decode, `extent` may cover +one row/token. `extent <= ObjectSpec.max_extent`. + +The worker rejects a record before exposing a device object if: + +- magic or version is unsupported +- header length is malformed +- `extent > ObjectSpec.max_extent` +- extent violates alignment/layout rules +- sequence violates the edge ordering policy +- ring or stream closes before `extent` bytes arrive + +Payload content itself is trusted. The worker does not inspect tensor values. + +--- + +## 18. Ring Specs And Layouts + +`RingSpec` is the requested operating shape for a ring: + +```rust +struct RingSpec { + data_capacity: u64, + alignment: u32, + direction: RingDirection, + host_pinning: HostPinning, + wake_coalescing: WakeCoalescing, +} + +enum RingDirection { + Ingress, + Egress, +} + +enum HostPinning { + Pageable, + PinnedRequired, +} + +enum WakeCoalescing { + PendingBit, + ReadySet, +} +``` + +`RingLayout` is the arena-relative layout minted by ArenaManager after a lease: + +```rust +struct RingLayout { + ring_id: RingId, + arena_offset: u64, + total_len: u64, + header_offset: u64, + data_offset: u64, + data_capacity: u64, + alignment: u32, +} +``` + +`RingLayout` never contains process-local pointers. Each process derives local +pointers from its own arena mapping base plus arena offsets. + +The ring header and data region must be aligned for cross-process atomics and +backend copy requirements. Host pinning is optional for the MVP, but if a range +is pinned, it cannot be unpinned or returned to the free list until ring +quiescence. + +--- + +## 19. Arena Model + +Each node owns one sparse host-memory arena. + +The arena is a `memfd`, created by the Rust node process, truncated to a large +sparse ceiling, and mapped once in the Rust process. The GPU worker maps the same +fd once. Neither process remaps or resizes the arena during node lifetime. + +The arena is address space. It does not define flow control and does not define +object ownership. + +The ArenaManager owns: + +- arena fd +- node process mapping base pointer +- reservation ceiling +- arena free list +- pending lease queue +- table of live ring leases + +The ArenaManager does not own: + +- worker process +- QUIC stream +- pump task +- object parser +- payload byte + +The ArenaManager only mints stable offsets and releases ranges after proof of +quiescence. + +ArenaManager FSM: + +```text +Booting + on ConstructArena{ceiling} + -> Ready if memfd, truncate, mmap succeed + -> Failed if any boot resource fails + +Ready + on LeaseRing{request_id, requester, edge_id, direction, ring_spec} + -> RingLeased if a range fits + -> queue request if satisfiable but no current range fits + -> RingLeaseRejected if request can never fit in the ceiling + +Ready + on CancelLease{request_id} + remove queued request if not yet leased + +Ready + on ReleaseRing{ring_id, proof} + return range to free list + retry queued leases serially + +Ready + on Shutdown + -> ShuttingDown + +ShuttingDown + reject new leases +``` + +Temporary arena pressure is represented by a queued lease request. Permanent +impossibility is explicit through `RingLeaseRejected`. + +Two live leases cannot overlap because lease/release operations pass through one +ArenaManager mailbox. + +--- + +## 20. Ring Contract + +A ring is a bounded single-producer/single-consumer byte stream backed by an +arena lease. + +Rings define flow control for every payload-moving boundary: + +- QUIC ingress stream -> host ingress ring -> worker -> GPU memory +- GPU memory -> worker -> host egress ring -> QUIC egress stream +- disk reader -> host ring -> worker/GPU memory, if used + +Each ring has exactly one producer and one consumer. + +Ingress: + +```text +producer = recv-pump +consumer = GPU worker +``` + +Egress: + +```text +producer = GPU worker +consumer = send-pump +``` + +Fan-in and fan-out are not ring features. They are represented by multiple rings +or by a higher-level component that owns one side of a ring. + +The shared arena ABI is not a Rust collection. It stores offsets, cursors, state +bits, wake bits, and payload bytes. It never stores process-local pointers. + +Ring header: + +```rust +#[repr(C, align(64))] +struct RingHeader { + magic: u32, + version: u16, + header_len: u16, + ring_id: u64, + capacity: u64, + commit: AtomicU64, + consume: AtomicU64, + state: AtomicU32, + wake: AtomicU32, +} +``` + +`commit` is the first byte after the committed readable prefix. Bytes with +logical positions `< commit` are valid for the consumer. + +`consume` is the first byte not yet released by the consumer. Bytes with logical +positions `< consume` are free for producer reuse. + +The producer keeps a local `write` cursor. `write` is the first byte after the +producer's reserved or in-progress write prefix. + +Readable interval: + +```text +consume .. commit +``` + +Reserved but not readable interval: + +```text +commit .. write +``` + +Physical index: + +```text +physical_index = cursor % capacity +``` + +The ring is empty when `consume == commit`. The ring is full when: + +```text +write - consume == capacity +``` + +Cursor values are monotonic logical byte positions. + +--- + +## 21. Ring Producer, Consumer, And Wake Rules + +Producer rule: + +- acquire `consume` +- compute free space as `capacity - (write - consume)` +- reserve a contiguous physical span by advancing local `write` +- write bytes into that span +- publish new `commit` with release ordering after bytes are valid +- send or coalesce `RingReadable { ring_id }` + +Consumer rule: + +- acquire `commit` +- read bytes in `consume .. commit` +- greedily drain all bytes it can use +- publish new `consume` with release ordering after bytes are safe to release +- send or coalesce `RingWritable { ring_id }` if producer may be unblocked + +Wake hints are edge-trigger hints, not ownership: + +```rust +RingReadable { ring_id: RingId } +RingWritable { ring_id: RingId } +``` + +The receiver of a wake must reload cursors from shared memory. Wake hints carry +no byte ranges, byte counts, host pointers, or free-space counts. + +Duplicate wake hints may be coalesced only while a pending bit, ready-set entry, +or equivalent durable scheduler state still makes the ring discoverable. Losing +the only transition from empty to readable or full to writable is a liveness bug. + +If asynchronous DMA reads host ring memory, the consumer cannot advance +`consume` until the DMA no longer depends on those bytes. + +Safety: + +- consumer cannot read unwritten bytes because producer publishes `commit` only + after writing +- producer cannot overwrite unread bytes because free space is computed from + consumer-owned `consume` +- wraparound cannot alias stale bytes because ownership uses monotonic logical + cursors +- stale wake/control events cannot alias replacement rings because `RingId` is + unique for node lifetime + +--- + +## 22. Native Ring Helper + +Both Rust hot-path code and the Python worker access process-crossing rings +through the same native implementation. + +Python does not implement shared atomics, wrap arithmetic, span calculation, or +cursor publication directly. + +The helper exposes operations equivalent to: + +```text +ring_open(arena_base, RingLayout) -> RingHandle +ring_close(handle) +ring_readable_span(handle) -> ptr, len +ring_advance_consume(handle, len) +ring_writable_span(handle) -> ptr, len +ring_advance_commit(handle, len) +ring_state(handle) -> state +``` + +Returned pointers are process-local addresses derived from the caller's mapped +arena base plus arena offsets. + +Ring cursors must use atomic acquire/release semantics across the process +boundary. + +--- + +## 23. Edge Stream Protocol + +Each edge uses one persistent iroh/QUIC uni-stream from producer node to +consumer node. + +Wire shape: + +```text +[edge_id preamble] +[object record] +[object record] +... +``` + +The receiver's edge-demux reader consumes the fixed-width `edge_id` preamble and +hands the stream to the driver rendezvous. After the preamble, the recv-pump is +byte-blind. It copies stream bytes into the ingress ring and advances `commit`. +The worker parses object records from the ring. + +The stream is persistent for the edge. It is not opened per object. + +QUIC owns reliability and stream ordering. The MVP does not add app-level +fragment hashes, striping, resume, or per-object streams. + +--- + +## 24. Driver And Transport + +The driver is the node's swactor-to-iroh boundary. It owns: + +- one iroh endpoint +- connection cache +- edge ALPN +- edge demux +- receive stream rendezvous +- send/recv pump tasks + +Example edge ALPN: + +```text +swactor/edge/1 +``` + +Connections are cached per `(peer_node_id, ALPN)`. All edges between the same +node pair and ALPN reuse the same connection. Each edge has one persistent +uni-stream within that connection. + +Driver inbound messages: + +```rust +EstablishSend { + edge_id: EdgeId, + rx_node_id: NodeId, + ring_id: RingId, + ring_layout: RingLayout, + tx_addr: ActorAddress, +} + +EstablishRecv { + edge_id: EdgeId, + ring_id: RingId, + ring_layout: RingLayout, + rx_addr: ActorAddress, +} + +StreamArrived { + edge_id: EdgeId, + stream: RecvStream, +} + +StopEdge { + edge_id: EdgeId, +} +``` + +Driver outbound events: + +```rust +DriverEdgeReady { edge_id: EdgeId } +StreamClosed { edge_id: EdgeId } +StreamFault { edge_id: EdgeId, reason: StreamFaultReason } +PumpStopped { edge_id: EdgeId, ring_id: RingId } +``` + +No per-object, per-range, or fragment messages exist in the driver hot path. + +--- + +## 25. Receive Demux And Pump FSMs + +A recv-pump needs two resources: + +- local receive establishment state, including ingress ring +- arriving QUIC stream + +They may arrive in either order. The driver stores both halves: + +```text +recv_specs: HashMap +pending_streams: HashMap +``` + +On `EstablishRecv`, if a pending stream exists, the driver spawns the recv-pump. +Otherwise it stores the spec. + +On `StreamArrived`, if a recv spec exists, the driver spawns the recv-pump. +Otherwise it stores the stream. + +This removes the need for an inter-end readiness handshake. If a stream arrives +before local receive establishment, it waits in `pending_streams`; because no +recv-pump reads from it, QUIC flow control eventually stalls the sender. + +Recv-pump FSM: + +```text +WaitingForSpecAndStream + -> Streaming when ring and stream are both present + +Streaming + read free ring span + read QUIC bytes into that span + advance commit + send/coalesce RingReadable{ring_id} to worker + repeat + +Backpressured + entered when no ring free space exists + wait for RingWritable{ring_id} + return to Streaming + +Closed + entered on stream EOF or edge teardown + +Faulted + entered on read error, protocol edge failure, or ring fault +``` + +The recv-pump does not parse `ObjectHeader` and does not know object boundaries. +EOF alignment is classified by the worker parser, not by the pump. + +Send-pump FSM: + +```text +WaitingForConnection + ensure or await cached edge-ALPN connection + +WaitingForBytes + wait for RingReadable{ring_id} + +OpenStream + open one uni-stream + write edge_id preamble + -> Streaming + +Streaming + acquire commit + write committed egress bytes to QUIC + advance consume after bytes are accepted by write_all + send/coalesce RingWritable{ring_id} + repeat + +Backpressured + write_all is pending because network/QUIC flow control is slow + keep ownership of unread ring bytes until write completes + +Closed/Faulted + emit coarse driver event +``` + +--- + +## 26. Edge Establishment + +Establishment is local actor setup plus transport rendezvous. Remote edge ends +do not exchange actor messages. + +For each edge, the orchestrator provisions the producer and consumer: + +```rust +struct ProvisionTx { + edge_id: EdgeId, + rx_node_id: NodeId, + object_spec: ObjectSpec, + ring_spec: RingSpec, +} + +struct ProvisionRx { + edge_id: EdgeId, + object_spec: ObjectSpec, + ring_spec: RingSpec, +} +``` + +The producer needs consumer `node_id`, not consumer actor address. The consumer +needs shared `edge_id`, not producer actor address. + +Each node has one EdgeEstablisher actor. It owns per-edge records: + +```rust +struct EdgeRecord { + edge_id: EdgeId, + direction: RingDirection, + state: EdgeProvisionState, + lease_request_id: Option, + ring_id: Option, + local_edge_actor: ActorAddress, +} +``` + +Record FSM: + +```text +New + on ProvisionTx/ProvisionRx + spawn local Tx/Rx edge actor + create LeaseRequestId + send LeaseRing to ArenaManager + -> WaitingForLease + +WaitingForLease + on RingLeased matching request_id + record ring_id + send InstallRing to GpuWorkerCtl or token endpoint + -> WaitingForWorkerRing + on RingLeaseRejected matching request_id + notify local edge actor failure + -> Failed + on StopEdge + send CancelLease + notify local edge actor stopped + -> Stopped + +WaitingForWorkerRing + on RingInstalled + send EstablishSend/EstablishRecv to Driver + -> WaitingForDriver + on RingFault or StopEdge + -> Stopping + +WaitingForDriver + on DriverEdgeReady + notify local Tx/Rx actor Ready + -> Ready + on StreamFault/RingFault/StopEdge + -> Stopping + +Ready + hot path runs without EdgeEstablisher + coarse object/fault events may pass through + +Stopping + stop pump if one exists + uninstall worker ring if installed + wait for quiescence proofs + release ring if leased + -> Stopped +``` + +Stale events for stopped records are ignored except that an unused fresh lease +granted after cancellation must be released without installing worker or pump +state. + +--- + +## 27. Tx And Rx Edge Actors + +Tx and Rx actors are role-facing lifecycle gates. + +Tx actor state: + +```text +Provisioning -> Ready -> Producing -> Stopping -> Stopped + -> Faulted +``` + +Rx actor state: + +```text +Provisioning -> Ready -> LoadingObject -> ObjectReady -> Stopping -> Stopped + -> Faulted +``` + +They receive coarse events: + +```rust +EdgeReady { edge_id: EdgeId } +ObjectLoaded { edge_id: EdgeId, object_id: ObjectId, device_handle: DeviceObjectHandle } +ObjectProduced { edge_id: EdgeId, object_id: ObjectId } +ObjectFailed { edge_id: EdgeId, object_id: Option, reason: ObjectFailure } +StreamFault { edge_id: EdgeId, reason: StreamFaultReason } +StopEdge { edge_id: EdgeId } +``` + +They do not receive: + +- bytes +- host pointers +- per-range readiness +- consumed ranges +- free-space events +- flow-control credits + +--- + +## 28. GPU Worker Process Boundary + +Each node has one `GpuWorkerCtl` actor. It owns: + +- worker process spawn and termination through `swactor_process` +- worker generation numbering +- arena fd inheritance or passing setup +- process actor address and notification bridge +- table of installed rings for current worker generation +- routing worker events to EdgeEstablisher, Driver, Tx/Rx actors, and role layer +- crash detection and crash fanout +- restart policy + +`GpuWorkerCtl` does not own: + +- arena leases +- QUIC streams +- ring payload bytes +- device allocations +- tinygrad execution +- graph placement + +The worker process owns: + +- mapped arena view +- worker-side ring handles +- per-ring parser/producer state +- device allocations +- device object handles +- tinygrad role state +- KV cache and worker-internal state +- host-to-device and device-to-host copy scheduling +- copy completion tracking + +The public Rust-side interface is a normal actor message enum: + +```rust +enum GpuWorkerCtlMsg { + StartWorker, + InstallRing(InstallRing), + UninstallRing(UninstallRing), + RingReadable { ring_id: RingId }, + RingWritable { ring_id: RingId }, + ConfigureRole(ConfigureRole), + ExecuteStep(ExecuteStep), + ReleaseDeviceObject { device_handle: DeviceObjectHandle }, + ShutdownWorker(ShutdownWorker), + Process(ProcessNotification), +} +``` + +`Process(ProcessNotification)` is delivered by a small `ProcessBridge` actor. +`GpuWorkerCtl` sends process input through `ProcessCommand::WriteStdin`. + +The process adapter may use newline-delimited JSON for worker commands/events. +This is an adapter, not a second distributed protocol. + +Adapter rules: + +- one command/event JSON object per line +- stdout is reserved for worker events +- stderr is reserved for logs and diagnostics +- payload bytes are forbidden in JSON +- invalid JSON or unknown event shape is a worker/process fault + +Worker environment: + +```text +SWACTOR_ARENA_FD memfd for the shared arena +SWACTOR_ARENA_BYTES arena reservation ceiling +``` + +--- + +## 29. GPU Worker Commands + +`InitializeWorker`: + +```rust +struct InitializeWorker { + worker_generation: WorkerGeneration, + arena_ceiling: u64, + required_ring_helper_abi: u16, + backend: JsonValue, +} +``` + +Maps the arena, initializes native helper and backend, then emits `WorkerReady` +or `WorkerFatal`. + +`ConfigureRole` is optional: + +```rust +struct ConfigureRole { + role_id: RoleId, + config: JsonValue, +} +``` + +`config` is low-frequency app metadata and must not carry payload bytes. + +`InstallRing`: + +```rust +struct InstallRing { + ring_id: RingId, + edge_id: EdgeId, + port_id: PortId, + direction: RingDirection, + layout: RingLayout, + object_spec: ObjectSpec, +} +``` + +`edge_id` and `port_id` are required so worker events can be reported in +graph-facing terms. + +`UninstallRing`: + +```rust +struct UninstallRing { + ring_id: RingId, + reason: UninstallReason, +} +``` + +The worker removes ring state, waits for copy lifetimes to end, closes the +helper handle, and emits `RingQuiesced`. + +Wake hints: + +```rust +RingReadable { ring_id: RingId } +RingWritable { ring_id: RingId } +``` + +`ExecuteStep`: + +```rust +struct ExecuteStep { + role_id: RoleId, + step_id: StepId, + inputs: Vec, + outputs: Vec, + runtime: JsonValue, + release_inputs_after: bool, +} + +struct InputBinding { + port_id: PortId, + object_id: ObjectId, + sequence: Sequence, + device_handle: DeviceObjectHandle, +} + +struct OutputBinding { + port_id: PortId, + ring_id: RingId, + object_id: ObjectId, + sequence: Sequence, + extent: u64, + flags: u32, +} +``` + +Execution requirements: + +1. Validate role is available. +2. Validate input handles belong to current worker generation. +3. Wrap inputs as tinygrad-compatible views. +4. Run role code. +5. Validate returned outputs against declared output bindings. +6. Write each output object to the named egress ring. +7. Emit `ObjectProduced` after each full output object is committed. +8. Emit `StepCompleted` after all declared outputs are produced and role state + updates are complete. + +`ReleaseDeviceObject` frees a device object after no compute or copy event still +references it. + +`ShutdownWorker`: + +```rust +struct ShutdownWorker { + mode: ShutdownMode, +} + +enum ShutdownMode { + Graceful, + AbortInFlight, +} +``` + +Deferred worker commands: + +- generic command accepted/rejected acks +- `BindDeviceObject` / `UnbindDeviceObject` +- `CancelStep` +- `AbortObject` +- `Ping` +- role module/factory provisioning + +--- + +## 30. GPU Worker Events + +Worker lifecycle: + +```rust +WorkerReady { + pid: u32, + worker_generation: WorkerGeneration, + ring_helper_abi: u16, + backend: JsonValue, +} + +WorkerFatal { + reason: WorkerFatalReason, +} + +WorkerStopped { + reason: WorkerStoppedReason, +} +``` + +Role events, only if `ConfigureRole` is used: + +```rust +RoleConfigured { role_id: RoleId } +RoleFailed { role_id: RoleId, reason: RoleFailure } +``` + +Ring events: + +```rust +RingInstalled { ring_id: RingId, edge_id: EdgeId, port_id: PortId } +RingFault { ring_id: RingId, edge_id: EdgeId, port_id: PortId, reason: RingFaultReason } +RingQuiesced { ring_id: RingId } +``` + +Object events: + +```rust +ObjectLoaded { + ring_id: RingId, + edge_id: EdgeId, + port_id: PortId, + object_id: ObjectId, + sequence: Sequence, + extent: u64, + device_handle: DeviceObjectHandle, +} + +ObjectProduced { + ring_id: RingId, + edge_id: EdgeId, + port_id: PortId, + object_id: ObjectId, + sequence: Sequence, + extent: u64, +} + +ObjectFailed { + ring_id: RingId, + edge_id: EdgeId, + port_id: PortId, + object_id: Option, + sequence: Option, + reason: ObjectFailure, +} +``` + +Step events: + +```rust +StepCompleted { role_id: RoleId, step_id: StepId } +StepFailed { role_id: RoleId, step_id: StepId, reason: StepFailure } +``` + +Device lifetime: + +```rust +DeviceObjectReleased { device_handle: DeviceObjectHandle } +ReleaseFailed { device_handle: DeviceObjectHandle, reason: ReleaseFailure } +``` + +Wake hints emitted by worker: + +- ingress rings: `RingWritable` after advancing `consume` +- egress rings: `RingReadable` after advancing `commit` + +`GpuWorkerCtl` may synthesize `WorkerCrashed` and `RingFault` after process exit, +process error, or stdout control-stream failure. + +--- + +## 31. GpuWorkerCtl And Worker FSMs + +`GpuWorkerCtl` FSM: + +```text +NotStarted + on StartWorker -> Spawning + +Spawning + spawn ProcessActor with ProcessSpec + spawn ProcessBridge + subscribe bridge to ProcessActor + wait for ProcessNotification::Started + send InitializeWorker through ProcessCommand::WriteStdin + -> Initializing + +Initializing + on WorkerReady -> Running + on WorkerFatal/process exit/timeout -> Failed + +Running + on actor command -> validate state, write worker command, update tables + on worker event -> route event + on ShutdownWorker -> Stopping + on process exit/error -> Crashed + +Stopping + send ShutdownWorker if process alive + wait for WorkerStopped and process exit + on timeout -> Killing + +Killing + close/kill ProcessActor according to policy + reap process notification + mark installed rings faulted + -> Stopped + +Crashed + mark current-generation device handles invalid + mark installed rings faulted + ask driver to stop pumps for installed rings + wait for teardown to release rings + -> Failed or Restarting + +Restarting + increment worker_generation + -> Spawning +``` + +Worker process FSM: + +```text +Booting + read environment + wait for InitializeWorker on stdin + +Initializing + map arena + initialize native helper + initialize backend/tinygrad + emit WorkerReady + -> Running + +Running + on InstallRing -> install ring state + on RingReadable/RingWritable -> reload cursors and advance ring FSMs + on ExecuteStep -> run explicit role step subject to backpressure + on ReleaseDeviceObject -> release handle when safe + on ShutdownWorker -> Draining + +Draining + reject new work + finish or abort in-flight work according to mode + quiesce rings + emit WorkerStopped + exit + +Fatal + emit WorkerFatal if possible + exit non-zero +``` + +The worker greedily makes progress after relevant commands or wake hints: drain +readable ingress prefixes, advance egress output if writable space exists, +observe copy completions, and emit resulting events. + +--- + +## 32. Worker Ring FSMs + +Ingress ring FSM: + +```text +Uninstalled + on InstallRing(direction = Ingress) -> NeedHeader + +NeedHeader + on committed bytes < header_len -> wait + on complete header -> validate + valid -> allocate device object -> NeedPayload + invalid -> Faulted + +NeedPayload + copy committed payload prefixes to device allocation + advance consume only after copied bytes are safe to release + if copied == extent -> ObjectComplete + on close before copied == extent -> Faulted + +ObjectComplete + wait for copy completion + emit ObjectLoaded + -> NeedHeader + +Faulted + emit ObjectFailed or RingFault + stop consuming until UninstallRing + +Uninstalling + stop consuming + wait for copy lifetimes to end + emit RingQuiesced + -> Uninstalled +``` + +Egress ring FSM: + +```text +Uninstalled + on InstallRing(direction = Egress) -> WaitingForOutput + +WaitingForOutput + on ExecuteStep output binding naming this ring -> NeedWritableForHeader + +NeedWritableForHeader + wait for writable span + write ObjectHeader + advance commit + emit RingReadable + -> NeedWritableForPayload + +NeedWritableForPayload + copy device payload prefixes into writable ring spans + advance commit only after host bytes are valid + emit/coalesce RingReadable + if produced == extent -> ObjectProduced + -> WaitingForOutput + +Faulted + emit RingFault + stop producing until UninstallRing + +Uninstalling + stop accepting outputs + wait for copy lifetimes to end + emit RingQuiesced + -> Uninstalled +``` + +The egress producer may block an `ExecuteStep` while waiting for egress ring +space. This is the intended backpressure path. + +--- + +## 33. Device Bridge + +The worker must provide a backend-specific device bridge: + +```text +alloc_device(ObjectSpec, extent) -> DeviceAllocation +free_device(DeviceAllocation) +host_to_device(arena_ptr, len, DeviceAllocation, device_offset) -> CopyEvent +device_to_host(DeviceAllocation, device_offset, arena_ptr, len) -> CopyEvent +copy_event_complete(CopyEvent) -> bool +wrap_for_tinygrad(DeviceAllocation, TensorViewSpec) -> tinygrad object +``` + +The canonical contract supports range copies so objects may be larger than a +ring. Ingress and egress can then stream prefixes through bounded host rings +while the logical object lives in device memory. + +If an implementation uses a constrained complete-object fallback, that mode must +be explicit and must preserve system-visible object, sequence, readiness, and +backpressure semantics. It must also constrain object/ring sizing accordingly. + +For synchronous copies: + +```text +copy returns -> safe to advance relevant cursor +``` + +For asynchronous copies: + +```text +start copy -> record CopyEvent and host/device ranges +event complete -> advance cursor or release device object +``` + +--- + +## 34. Ingress And Egress Flow + +Ingress path: + +```text +remote worker/GPU -> remote egress ring -> QUIC -> local ingress ring + -> local worker -> local GPU memory -> ObjectLoaded +``` + +Receiving sequence: + +1. Driver demux reads stream `edge_id` preamble. +2. Driver pairs stream with local ingress ring. +3. Recv-pump waits for free ring space. +4. Recv-pump reads QUIC bytes into free ring span. +5. Recv-pump advances `commit`. +6. Recv-pump sends/coalesces `RingReadable`. +7. Worker wakes and reads `consume..commit`. +8. Worker parses object headers when enough committed bytes exist. +9. Worker allocates device object after header validation. +10. Worker copies committed payload prefixes into device memory. +11. Worker advances `consume` after copies are safe to release. +12. Recv-pump sees free space and continues. +13. Worker emits `ObjectLoaded` after full object and copy completion. + +Egress path: + +```text +local GPU memory -> local worker -> local egress ring -> QUIC + -> remote ingress ring -> remote worker/GPU +``` + +Sending sequence: + +1. Worker receives or creates compute result in GPU memory. +2. Worker creates `ObjectHeader` according to `ObjectSpec`. +3. Worker waits for free egress ring space. +4. Worker writes and commits header bytes. +5. Worker copies payload bytes from device into writable ring spans. +6. Worker advances `commit` as host bytes become valid. +7. Worker sends/coalesces `RingReadable`. +8. Send-pump opens persistent uni-stream on first bytes. +9. Send-pump writes `edge_id` preamble once. +10. Send-pump writes committed bytes to QUIC. +11. Send-pump advances `consume` after `write_all` accepts bytes. +12. Send-pump sends/coalesces `RingWritable`. + +An object may be larger than the ring. The ring is a transfer window, not object +storage. Compute visibility still begins only at `ObjectLoaded`. + +--- + +## 35. Backpressure + +No credits or per-range acknowledgements exist. + +Worker slow on ingress: + +```text +worker slow + -> ingress consume does not advance + -> recv-pump sees no free ingress space + -> recv-pump stops reading QUIC + -> QUIC stream flow control stalls remote sender + -> sender send-pump stops draining egress ring + -> sender worker stalls on egress free space +``` + +Network slow on egress: + +```text +network slow + -> send-pump write_all remains pending + -> egress consume does not advance + -> worker sees no free egress space + -> worker stops producing more bytes into egress ring +``` + +Arena pressure: + +```text +arena temporarily exhausted + -> ArenaManager queues LeaseRing + -> EdgeEstablisher record remains WaitingForLease + -> no worker ring is installed + -> no pump is spawned + -> no hot-path state exists for that edge +``` + +No data is dropped because each layer stops before overwriting unread bytes. + +--- + +## 36. Fault Semantics + +A run has one terminal outcome: + +- completed +- faulted +- operator-stopped before completion + +Fault sources include: + +- node unavailable before or during run +- membership loss for a required node +- provisioning rejection +- arena boot failure +- oversized ring request +- temporary pressure timeout +- weight download/load/bind failure +- edge establishment failure +- malformed object header +- EOF mid-object +- stream fault +- pump failure +- ring fault +- worker fatal error +- worker crash +- device OOM +- device copy failure +- sequence violation +- step failure +- teardown timeout + +MVP recovery policy is fail-stop at the run level. The orchestrator records the +first run-level failure reason, stops injecting tokens, and begins teardown. It +does not re-place the run. + +After a stage faults, it rejects new run work until stopped. + +--- + +## 37. Error Taxonomy + +Ring faults: + +```rust +enum RingFaultReason { + UnsupportedRingVersion, + RingLayoutInvalid, + RingStateInvalid, + DeviceOutOfMemory, + DeviceCopyFailed, + SequenceViolation, + HeaderMalformed, + WorkerProcessExited, + WorkerShuttingDown, + WorkerInternal, +} +``` + +Object failures: + +```rust +enum ObjectFailure { + HeaderMalformed, + UnsupportedObjectVersion, + ExtentExceedsMax, + ExtentAlignmentInvalid, + SequenceViolation, + DeviceAllocationFailed, + DeviceCopyFailed, + EofMidObject, +} +``` + +Step failures: + +```rust +enum StepFailure { + RoleUnavailable, + InvalidInputHandle, + InvalidOutputRing, + TinygradError, + DeviceOutOfMemory, + OutputExtentInvalid, + OutputCopyFailed, + WorkerProcessExited, + WorkerShuttingDown, +} +``` + +Worker fatal reasons: + +```rust +enum WorkerFatalReason { + ArenaMapFailed, + RingHelperAbiMismatch, + BackendInitFailed, + MalformedControlMessage, + UnhandledException, +} +``` + +Other control reasons: + +```rust +enum UninstallReason { + EdgeStopped, + RingFaulted, + WorkerShutdown, +} + +enum WorkerStoppedReason { + Graceful, + AbortInFlight, + Fatal, +} +``` + +--- + +## 38. Teardown + +Teardown starts after completion, fault, or operator stop. + +Orchestrator teardown: + +1. Stop injecting tokens. +2. Send `StopRun` to every provisioned stage. +3. Stop local token endpoints. +4. Wait for `StageStopped` from every stage or timeout. +5. Record run teardown completion. + +Stage teardown: + +1. Stop accepting new run work. +2. Stop local edges. +3. Stop driver pumps. +4. Uninstall worker rings. +5. Release arena leases after quiescence proof. +6. Release per-run device objects. +7. Stop or reset worker role state according to local policy. +8. Emit `StageStopped`. + +Edge teardown FSM: + +```text +Ready + on StopEdge or fault -> StoppingPump + +StoppingPump + driver stops recv/send pump + driver emits PumpStopped + -> StoppingWorkerRing + +StoppingWorkerRing + GpuWorkerCtl sends UninstallRing + worker removes ring state + worker waits for in-flight copies or process death + worker emits RingQuiesced + -> ReleasingArena + +ReleasingArena + EdgeEstablisher sends ReleaseRing{proof} + ArenaManager returns lease to free list + -> Stopped +``` + +Arena release is safe only after: + +- driver pump stopped +- worker ring quiesced or worker process reaped +- copy lifetimes ended or owning process is gone + +Teardown is required after success and fault. + +--- + +## 39. System Guarantees + +Authority: + +- orchestrator is the only component that assigns topology, stages, layer + ranges, edge ids, and object specs +- nodes reject provisioning and edge data that do not match active run plan + +Readiness: + +- orchestrator does not inject prompt before global readiness barrier +- a stage does not report `StageReady` before worker, weights, inbound edge, + outbound edge, and StageController are ready + +Weight use: + +- a stage cannot execute before assigned weights are loaded and bound +- weight loading failures become stage faults + +Edge identity: + +- every edge id is unique within run +- every edge has one producer and one consumer +- data plane is addressed by `(node_id, edge_id)` +- remote actor addresses are not needed for data flow + +Sequence: + +- prefill is sequence `0` +- decode sequences are strictly increasing +- a stage executes sequence `s` only after loading inbound object sequence `s` +- stage output object uses sequence `s` +- orchestrator injects `s + 1` only after consuming output token sequence `s` +- workers do not invent graph-visible object ids or sequence numbers + +Payload isolation: + +- payload bytes never travel in actor messages, process control messages, logs, + or JSON command/event lines +- actors carry identities, lifecycle events, wake hints, and opaque handles + +Compute visibility: + +- compute receives only complete logical objects +- `ObjectLoaded` is emitted only after valid header, exact extent copy, copy + completion, and device handle creation + +Backpressure: + +- slow worker, slow network, or downstream stall propagates by ring and QUIC + flow control +- unread bytes are not overwritten + +Terminal outcome: + +- each run records exactly one terminal outcome +- after terminal outcome begins, no new run work is accepted except teardown + +Quiescence: + +- arena memory cannot be reused under a live pump, worker ring, or copy + operation +- release requires teardown proof + +Worker restart: + +- restart creates a new generation +- old device handles, roles, rings, and steps are invalid +- roles and rings must be reinstalled + +--- + +## 40. Observability Surface + +The system must emit stable lifecycle events for behavioral contracts and tests. +Event transport and storage are implementation details. + +Required event identities: + +- `run_id` +- `node_id` +- `stage_index` +- `edge_id` +- `ring_id` +- `object_id` +- `sequence` +- `step_id` +- `worker_generation` + +Required lifecycle events: + +```text +node_started +node_available +node_faulted +pool_ready +run_planned +stage_provision_started +weights_download_started +weights_downloaded +weights_loaded +edge_provision_started +edge_ready +stage_ready +readiness_barrier_passed +prompt_injected +object_loaded +execute_step_started +object_produced +step_completed +token_received +run_completed +run_faulted +stop_run_sent +stage_stopped +run_torn_down +``` + +Fault events must include a stable reason enum and the component that detected +the fault. Tests and operators should not need to scrape free-form logs to +determine lifecycle progress. + +--- + +## 41. Behavioral Contract Inventory + +The next specification layer should extract API surfaces and behavioral +contracts for these components and required behaviors: + +- node boot lifecycle +- membership/SWIM convergence +- resource inventory and run planner +- orchestrator run FSM +- StageController +- stage-local weight loading path +- orchestrator token endpoint behavior +- ArenaManager +- shared ring helper/ABI +- EdgeEstablisher +- Tx and Rx edge actors +- iroh driver and recv/send pumps +- GpuWorkerCtl +- GPU worker process adapter +- GPU worker ingress parser +- GPU worker egress producer +- device bridge +- observability/event surface + +Required test layers: + +1. Component behavioral contract tests. +2. Local end-to-end mock tests without real networking. +3. Local Docker cluster tests with real ports and real networking. + +VastAI smoke/integration validation comes after local Docker cluster behavior is +stable. + +--- + +## 42. Canonical MVP Scenario + +The first end-to-end scenario is one "hello world" prompt over a rented GPU +pool: + +1. Operator provisions `N` GPU nodes with the barebones swactor + tinygrad/CUDA + image. +2. Every node starts the Rust node process and GPU worker process. +3. Nodes join SWIM membership. +4. Orchestrator observes `PoolReady`. +5. Orchestrator builds a linear GGUF `RunPlan`. +6. Orchestrator provisions every stage. +7. Each stage downloads and loads assigned weights. +8. Each stage provisions inbound and outbound edges. +9. Every stage reports `StageReady`. +10. Orchestrator observes the global readiness barrier. +11. Orchestrator tokenizes and injects prompt as token object sequence `0`. +12. Stages execute prefill and return token sequence `0`. +13. Orchestrator continues decode until EOS or `max_tokens`. +14. Orchestrator records `run_completed`. +15. Orchestrator tears down edges, worker run state, and token endpoints. +16. Orchestrator records `run_torn_down`. + +Success requires one terminal run outcome and every arena range leased by the run +to be quiesced or released by teardown. + +--- + +## 43. Deferred + +- automatic placement optimization +- arbitrary graph execution +- multi-input joins and fan-out beyond explicit role-layer handling +- multiple concurrent runs on one stage chain +- batching, speculative decoding, continuous serving +- warm model/weight reuse across prompts +- production artifact layout and weight cache eviction +- re-placement after node failure +- trustless verification or adversarial payload defense +- host pinning and asynchronous DMA performance policy +- removing backend copy limitations +- VastAI-specific provisioning automation diff --git a/crates/mvp-system/Cargo.toml b/crates/mvp-system/Cargo.toml new file mode 100644 index 0000000..4b770e2 --- /dev/null +++ b/crates/mvp-system/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "mvp-system" +version = "0.1.0" +edition = "2024" +publish = false + +[target.'cfg(target_os = "linux")'.dependencies] +libc = "0.2" diff --git a/crates/mvp-system/src/lib.rs b/crates/mvp-system/src/lib.rs new file mode 100644 index 0000000..7a18f05 --- /dev/null +++ b/crates/mvp-system/src/lib.rs @@ -0,0 +1 @@ +pub mod arena_manager; diff --git a/specs/mvp_system/arena_manager_contract.md b/specs/mvp_system/arena_manager_contract.md new file mode 100644 index 0000000..5114b65 --- /dev/null +++ b/specs/mvp_system/arena_manager_contract.md @@ -0,0 +1,58 @@ +# ArenaManager Contract + +This document defines the behavioral contract for node-local arena management. +The `ArenaManager` mints stable arena-relative ring layouts and releases ranges +only after quiescence proof. + +## Arena Formation + +- The node constructs one sparse arena. +- The arena has one reservation ceiling. +- The arena mapping is stable for the node lifetime. +- Arena layouts contain offsets, not process-local pointers. +- Arena boot failure emits a typed arena fault. + +## Ring Lease + +- `LeaseRing` either emits `RingLeased`, queues the request, or emits + `RingLeaseRejected`. +- A satisfiable request may queue under temporary arena pressure. +- A request that can never fit within the ceiling is rejected. +- A live lease has one `RingId`. +- `RingId` is unique for the node lifetime. + +## Layout Safety + +- Live leases do not overlap. +- Every lease lies within the arena ceiling. +- Every lease satisfies the requested alignment. +- Header and data offsets are stable for the lease lifetime. +- A layout never exposes process-local pointers. + +## Cancellation + +- `CancelLease` removes a queued request that has not been leased. +- A canceled queued request does not later install worker or pump state. +- If a fresh lease races with cancellation, it is released without becoming + hot-path state. + +## Release + +- `ReleaseRing` requires quiescence proof. +- The arena manager does not infer quiescence. +- Released ranges may be reused after release. +- Ranges are not reused while a live pump, worker ring, or copy operation still + owns them. + +## Shutdown + +- Shutdown rejects new leases. +- Shutdown does not corrupt existing live lease records. +- Shutdown does not release live ranges without proof. + +## Test Direction + +Tests should issue deterministic lease, cancel, release, and shutdown messages +and inspect public lease events. Successful tests should assert non-overlap, +alignment, reuse after release, and queue retry. Failure tests should assert +oversized rejection, canceled lease suppression, and no release without proof. diff --git a/specs/mvp_system/device_bridge_contract.md b/specs/mvp_system/device_bridge_contract.md new file mode 100644 index 0000000..f36ed89 --- /dev/null +++ b/specs/mvp_system/device_bridge_contract.md @@ -0,0 +1,53 @@ +# Device Bridge Contract + +This document defines the behavioral contract for the backend-specific device +bridge used by the GPU worker. The bridge is the worker's boundary between host +ring memory, device memory, and tinygrad-compatible views. + +## Allocation + +- `alloc_device(ObjectSpec, extent)` creates a device allocation suitable for + the object spec and extent. +- Allocation failure is reported as device allocation failure. +- Allocations are tied to the current worker generation. +- `free_device` releases an allocation after no compute or copy event depends + on it. + +## Host To Device + +- `host_to_device` copies exactly the requested host range to the requested + device range. +- For synchronous copies, return means bytes are safe to release. +- For asynchronous copies, completion of the copy event means bytes are safe to + release. +- Copy failure is reported as device copy failure. + +## Device To Host + +- `device_to_host` copies exactly the requested device range to the requested + host range. +- For synchronous copies, return means host bytes are valid. +- For asynchronous copies, completion of the copy event means host bytes are + valid. +- Copy failure is reported as output copy failure or device copy failure. + +## Tinygrad View + +- `wrap_for_tinygrad` creates a tinygrad-compatible view over a device + allocation. +- The view matches the tensor view spec used by the role. +- Invalid view shape or dtype fails the step. + +## Lifetime + +- Device allocations are not freed while compute uses them. +- Device allocations are not freed while copy events use them. +- Worker restart invalidates all prior device handles. +- Old-generation handles are rejected. + +## Test Direction + +Tests should use a fake backend to observe allocation, copy, completion, view, +and free calls. Success tests should assert exact ranges and safe cursor-release +points. Fault tests should inject allocation failure, copy failure, invalid +view, and old-generation handles. diff --git a/specs/mvp_system/driver_pumps_contract.md b/specs/mvp_system/driver_pumps_contract.md new file mode 100644 index 0000000..79527c5 --- /dev/null +++ b/specs/mvp_system/driver_pumps_contract.md @@ -0,0 +1,64 @@ +# Driver And Pump Contract + +This document defines the behavioral contract for the iroh driver, edge demux, +and send/recv pumps. The driver owns tokio, connections, streams, demux, and +byte-pump tasks. + +## Driver Authority + +- The driver owns one iroh endpoint. +- The driver owns connection caching. +- The driver owns edge ALPN. +- The driver owns edge-id stream demux. +- The driver owns send and recv pump tasks. +- Actors do not poll stream futures directly in the MVP. + +## Connection And Stream Shape + +- Each edge uses one persistent uni-stream. +- The stream starts with an `edge_id` preamble. +- After the preamble, the stream carries object records as bytes. +- The stream is not opened per object. +- Connections may be cached per `(peer_node_id, ALPN)`. + +## Receive Rendezvous + +- `EstablishRecv` may arrive before the stream. +- The stream may arrive before `EstablishRecv`. +- The driver stores whichever half arrives first. +- The recv pump starts only after both receive spec and stream exist. +- A pending stream is not read before the receive spec exists. + +## Recv Pump + +- The recv pump is byte-blind after edge demux. +- It does not parse `ObjectHeader`. +- It copies QUIC bytes into free ingress ring spans. +- It advances `commit` after bytes are in the ring. +- It emits or coalesces `RingReadable`. +- If no ring space exists, it stops reading and waits for `RingWritable`. + +## Send Pump + +- The send pump opens one uni-stream for the edge. +- It writes the `edge_id` preamble once. +- It writes committed egress ring bytes to QUIC. +- It advances `consume` only after `write_all` accepts bytes. +- It emits or coalesces `RingWritable`. +- If network or QUIC flow control stalls, it keeps ownership of unread ring + bytes until the write completes. + +## Fault And Stop + +- Read error emits `StreamFault`. +- Write error emits `StreamFault`. +- Protocol edge failure emits `StreamFault`. +- `StopEdge` stops the corresponding pump. +- Stopped pumps emit `PumpStopped`. + +## Test Direction + +Tests should use mock streams and rings to drive driver messages and pump wake +events. Successful tests should assert rendezvous in both arrival orders, +preamble once, byte-blind copying, cursor advancement after I/O acceptance, and +backpressure. Fault tests should inject read/write errors and stop races. diff --git a/specs/mvp_system/edge_establisher_contract.md b/specs/mvp_system/edge_establisher_contract.md new file mode 100644 index 0000000..d968718 --- /dev/null +++ b/specs/mvp_system/edge_establisher_contract.md @@ -0,0 +1,61 @@ +# EdgeEstablisher Contract + +This document defines the behavioral contract for local edge establishment. It +turns `ProvisionTx` or `ProvisionRx` into a local edge actor, arena lease, +worker ring installation, driver establishment, and ready/fault outcome. + +## Provisioning + +- Each node has one `EdgeEstablisher`. +- `ProvisionTx` creates a local send edge record. +- `ProvisionRx` creates a local receive edge record. +- The producer needs the consumer `node_id`. +- The consumer needs the shared `edge_id`. +- Remote actor addresses are not required for data flow. + +## Lease Flow + +- Establishment requests one arena lease per edge end. +- `RingLeased` matching the request advances the edge record. +- `RingLeaseRejected` matching the request fails the edge record. +- Stale lease events for stopped records do not install worker or pump state. +- An unused fresh lease granted after cancellation is released. + +## Worker Ring Installation + +- The edge establishes driver state only after worker or token endpoint ring + installation succeeds. +- `RingInstalled` matching the edge advances establishment. +- `RingFault` before readiness stops establishment. +- Ring installation uses the `ObjectSpec` and `RingSpec` from provisioning. + +## Driver Establishment + +- Send edges call `EstablishSend` with `edge_id`, consumer `node_id`, and local + ring layout. +- Receive edges call `EstablishRecv` with `edge_id` and local ring layout. +- `DriverEdgeReady` marks the local edge actor ready. +- Stream and pump behavior are driver-owned after readiness. + +## Ready And Hot Path + +- Ready means local actor, arena lease, worker/token ring, and driver state are + installed. +- The hot path runs without `EdgeEstablisher`. +- The establisher observes coarse ready, stopped, and fault events only. + +## Stopping + +- `StopEdge` cancels queued leases. +- `StopEdge` stops pumps if present. +- `StopEdge` uninstalls worker rings if installed. +- The arena lease is released only after quiescence proof. +- `Stopped` is terminal for that edge record. + +## Test Direction + +Tests should drive the public establishment FSM with lease, worker, driver, and +stop events. Successful tests should assert the order lease -> install -> +driver -> ready. Fault tests should inject lease rejection, ring fault, driver +fault, stale events, and stop races and assert no hot-path state survives +incorrectly. diff --git a/specs/mvp_system/gpu_worker_ctl_contract.md b/specs/mvp_system/gpu_worker_ctl_contract.md new file mode 100644 index 0000000..54f0575 --- /dev/null +++ b/specs/mvp_system/gpu_worker_ctl_contract.md @@ -0,0 +1,55 @@ +# GpuWorkerCtl Contract + +This document defines the behavioral contract for the Rust-side GPU worker +controller. `GpuWorkerCtl` owns the supervised worker process boundary and +routes worker events to local control components. + +## Worker Lifecycle + +- `StartWorker` spawns the process actor and process bridge. +- After process start, `GpuWorkerCtl` sends `InitializeWorker`. +- `WorkerReady` moves the controller to running. +- `WorkerFatal`, process exit, or initialization timeout moves the controller + to failed or crashed state. +- Worker generation increments on restart. + +## Command Routing + +- In running state, valid actor commands are serialized to the worker. +- `InstallRing` is sent only for the current worker generation. +- `ExecuteStep` is sent only with current-generation handles. +- `ReleaseDeviceObject` is sent only for current-generation handles. +- Payload bytes are never sent through worker control messages. + +## Event Routing + +- Worker events are parsed from process stdout. +- `RingInstalled` routes to edge establishment. +- `ObjectLoaded` routes to Rx or role layer. +- `ObjectProduced` routes to Tx or role layer. +- `StepCompleted` and `StepFailed` route to the StageController. +- Wake events route to the driver or worker side as appropriate. + +## Crash Behavior + +- Worker crash invalidates old device handles. +- Worker crash invalidates installed roles, rings, and in-flight steps. +- The controller synthesizes ring faults for installed rings. +- The controller asks the driver to stop pumps for affected rings. +- Old-generation handles are rejected after restart. + +## Shutdown + +- Graceful shutdown sends `ShutdownWorker`. +- Timeout escalates according to local process policy. +- Shutdown marks installed rings faulted or quiesced according to observed + worker/process outcome. +- `WorkerStopped` is observed before terminal stopped when graceful shutdown + succeeds. + +## Test Direction + +Tests should use a fake process adapter and public controller messages. Success +tests should assert start -> initialize -> ready, command serialization, and +event routing. Fault tests should inject malformed stdout, worker fatal, process +exit, restart, and old-generation handles and assert invalidation and fanout. diff --git a/specs/mvp_system/gpu_worker_egress_producer_contract.md b/specs/mvp_system/gpu_worker_egress_producer_contract.md new file mode 100644 index 0000000..23d7d5e --- /dev/null +++ b/specs/mvp_system/gpu_worker_egress_producer_contract.md @@ -0,0 +1,50 @@ +# GPU Worker Egress Producer Contract + +This document defines the behavioral contract for worker-side egress +production. The worker writes object records to an egress ring from explicit +`ExecuteStep` output bindings. + +## Output Admission + +- Egress production starts only after `InstallRing(direction = Egress)`. +- The worker writes output only for an `ExecuteStep` output binding naming that + ring. +- The output binding supplies object id, sequence, extent, and flags. +- The worker does not invent graph-visible object ids or sequence numbers. + +## Header Production + +- The worker creates an `ObjectHeader` according to the edge `ObjectSpec`. +- Header bytes are written before payload bytes. +- `commit` advances only after header bytes are valid. +- `RingReadable` is emitted or coalesced after committed header bytes. + +## Payload Production + +- The worker copies exactly `extent` bytes from device to the egress ring. +- `commit` advances only after host bytes are valid. +- Objects larger than the ring may stream through bounded ring spans. +- If no writable span exists, `ExecuteStep` may block on egress backpressure. + +## ObjectProduced + +- `ObjectProduced` is emitted after the full output object is committed. +- `StepCompleted` is emitted only after all declared outputs are produced and + role state updates are complete. +- `ObjectProduced` includes ring id, edge id, port id, object id, sequence, and + extent. + +## Fault Behavior + +- Invalid output ring fails the step. +- Output extent violation fails the step. +- Device copy failure fails the step or faults the ring. +- Worker shutdown rejects or aborts output production according to shutdown + mode. + +## Test Direction + +Tests should drive `ExecuteStep` with fake device outputs and bounded egress +rings. Success tests should assert header-before-payload, exact extent, cursor +publication, `ObjectProduced`, and `StepCompleted` ordering. Fault tests should +cover invalid output ring, invalid extent, backpressure, and copy failure. diff --git a/specs/mvp_system/gpu_worker_ingress_parser_contract.md b/specs/mvp_system/gpu_worker_ingress_parser_contract.md new file mode 100644 index 0000000..3b39d0f --- /dev/null +++ b/specs/mvp_system/gpu_worker_ingress_parser_contract.md @@ -0,0 +1,53 @@ +# GPU Worker Ingress Parser Contract + +This document defines the behavioral contract for worker-side ingress parsing. +The worker parses object records from an ingress ring and emits `ObjectLoaded` +only after a complete logical object is available on device. + +## Ring Admission + +- Ingress parsing starts only after `InstallRing(direction = Ingress)`. +- The worker reloads cursors after `RingReadable`. +- The parser consumes committed bytes only. +- The parser does not read uncommitted bytes. + +## Header Validation + +- The parser waits until a complete `ObjectHeader` is committed. +- Unsupported magic rejects the object. +- Unsupported version rejects the object. +- Malformed header length rejects the object. +- `extent > ObjectSpec.max_extent` rejects the object. +- Extent alignment or layout violation rejects the object. +- Sequence violation rejects the object. + +## Payload Loading + +- Payload content values are trusted. +- The parser copies exactly `extent` payload bytes to device memory. +- `consume` advances only after copied bytes are safe to release. +- Objects larger than the ring may stream through bounded ring spans. +- EOF before the full payload faults the object. + +## ObjectLoaded + +- `ObjectLoaded` is emitted only after valid header, exact extent copy, copy + completion, and device handle creation. +- `ObjectLoaded` includes ring id, edge id, port id, object id, sequence, + extent, and device handle. +- The emitted handle belongs to the current worker generation. + +## Fault Behavior + +- Header malformed emits `ObjectFailed`. +- Extent exceeds max emits `ObjectFailed`. +- Sequence violation emits `ObjectFailed`. +- Device allocation or copy failure emits `ObjectFailed` or `RingFault`. +- After ring fault, the worker stops consuming until uninstall. + +## Test Direction + +Tests should write object records through the public ring helper and observe +worker events. Success tests should assert `ObjectLoaded` only after complete +payload and copy completion. Fault tests should cover malformed headers, +oversized extent, bad sequence, EOF mid-object, and copy failure. diff --git a/specs/mvp_system/gpu_worker_process_adapter_contract.md b/specs/mvp_system/gpu_worker_process_adapter_contract.md new file mode 100644 index 0000000..e892c0f --- /dev/null +++ b/specs/mvp_system/gpu_worker_process_adapter_contract.md @@ -0,0 +1,45 @@ +# GPU Worker Process Adapter Contract + +This document defines the behavioral contract for the stdin/stdout adapter +between `GpuWorkerCtl` and the Python/tinygrad worker process. The adapter is +not a distributed protocol. + +## Control Stream Shape + +- Commands are one JSON object per stdin line. +- Events are one JSON object per stdout line. +- Stderr is reserved for logs and diagnostics. +- Payload bytes are forbidden in JSON commands. +- Payload bytes are forbidden in JSON events. + +## Initialization + +- The worker reads arena environment variables. +- The worker waits for `InitializeWorker`. +- The worker maps the arena. +- The worker initializes the native ring helper. +- The worker initializes backend/tinygrad. +- Success emits `WorkerReady`. +- Failure emits `WorkerFatal` if possible and exits non-zero. + +## Parsing + +- Invalid JSON is a worker/process fault. +- Unknown event shape is a worker/process fault. +- Unsupported helper ABI emits `WorkerFatal`. +- Stderr output alone is diagnostic and does not define lifecycle state. + +## Command Discipline + +- `InstallRing` installs worker-side ring state. +- Wake hints reload ring cursors. +- `ExecuteStep` runs explicit role compute. +- `ReleaseDeviceObject` releases handles when safe. +- `ShutdownWorker` moves the worker toward draining. + +## Test Direction + +Tests should drive the adapter with fake stdin/stdout lines. Successful tests +should assert one-line command/event framing and initialization ordering. Fault +tests should inject malformed JSON, unknown events, ABI mismatch, and payload +bytes in control messages. diff --git a/specs/mvp_system/membership_pool_readiness_contract.md b/specs/mvp_system/membership_pool_readiness_contract.md new file mode 100644 index 0000000..798c684 --- /dev/null +++ b/specs/mvp_system/membership_pool_readiness_contract.md @@ -0,0 +1,46 @@ +# Membership And Pool Readiness Contract + +This document defines the behavioral contract for the orchestrator's membership +gate. SWIM is an input to pool readiness; it is not a distributed graph +agreement protocol. + +## Pool Readiness Formation + +- The orchestrator emits `PoolReady` only for the intended candidate pool. +- `PoolReady` requires every candidate node to be known to the orchestrator. +- `PoolReady` requires every candidate node to be live in the SWIM view. +- `PoolReady` requires every candidate node to have emitted `NodeAvailable`. +- `PoolReady` requires every candidate node to have data-plane identity material. +- `PoolReady` requires no candidate node to be suspect or faulted. +- `PoolReady` requires the pool view to remain stable for the configured + convergence window. + +## Planning Gate + +- Run planning starts only after `PoolReady`. +- If pool readiness is lost before a `RunPlan` is committed, the orchestrator + waits or aborts according to local policy. +- Nodes do not need to agree on graph state before planning. +- Nodes do not compute placement from SWIM state. + +## Loss After Provisioning + +- Membership loss for a required node after provisioning begins faults the run. +- The MVP does not re-place an active run after membership loss. +- A suspect or faulted required node is treated as unavailable for the active + run. + +## Authority + +- SWIM reports membership and liveness facts. +- The orchestrator decides pool readiness. +- The orchestrator owns the candidate pool definition. +- SWIM does not assign stages, edges, layers, or object specs. + +## Test Direction + +Tests should feed deterministic membership and node-availability observations +into the orchestrator readiness gate. Successful tests should assert that +`PoolReady` appears only after all required facts and the convergence window. +Failure tests should remove or suspect one candidate node and assert no +`PoolReady`, or a run fault if provisioning has already begun. diff --git a/specs/mvp_system/node_boot_lifecycle_contract.md b/specs/mvp_system/node_boot_lifecycle_contract.md new file mode 100644 index 0000000..239ad02 --- /dev/null +++ b/specs/mvp_system/node_boot_lifecycle_contract.md @@ -0,0 +1,56 @@ +# Node Boot Lifecycle Contract + +This document defines the behavioral contract for node boot. It covers the +observable transition from container start to `NodeAvailable`, and the failure +cases that keep a node out of the candidate run pool. + +## Boot Formation + +- A node boot attempt starts when the node process is launched in the intended + pool. +- A boot attempt emits either `NodeAvailable` or a typed node boot fault. +- It never emits `NodeAvailable` before required local resources are ready. +- It never allows run provisioning to race ahead of node availability. + +## Required Readiness Facts + +- The Rust node process is alive. +- The swactor runtime can receive control messages. +- The node has a stable `node_id` known to the orchestrator. +- The arena is created and mapped in the node process. +- The GPU worker is ready, or the implementation has an explicit deferred + worker-start policy with the same run-level readiness guarantee. +- The iroh endpoint is initialized and bound to the node identity. +- The SWIM participant has joined or is joining the intended pool. +- The node can accept run provisioning. + +## Non-Readiness Facts + +- `NodeAvailable` does not mean weights are present. +- `NodeAvailable` does not mean a role is configured. +- `NodeAvailable` does not mean run edges are established. +- `NodeAvailable` does not mean the node has been selected by a `RunPlan`. + +## Failure Behavior + +- Arena construction failure rejects node availability. +- Worker startup failure rejects node availability unless worker startup is + explicitly deferred. +- Transport endpoint failure rejects node availability. +- Missing or invalid node identity rejects node availability. +- A node that is boot-faulted is not eligible for run planning. + +## Authority + +- The node owns local boot resources. +- The orchestrator owns whether a booted node is part of the intended candidate + pool. +- A node does not self-assign stages, edges, or layer ranges during boot. + +## Test Direction + +Tests should drive the boot component through public resource outcomes and +observe emitted lifecycle events. Successful tests should assert that +`NodeAvailable` appears only after the required readiness facts. Failure tests +should inject one failed resource at a time and assert a typed boot fault with +no `NodeAvailable`. diff --git a/specs/mvp_system/observability_surface_contract.md b/specs/mvp_system/observability_surface_contract.md new file mode 100644 index 0000000..8c4c6ba --- /dev/null +++ b/specs/mvp_system/observability_surface_contract.md @@ -0,0 +1,54 @@ +# Observability Surface Contract + +This document defines the behavioral contract for lifecycle and fault events +used by tests and operators. Observability transport and storage are +implementation details. + +## Event Identity + +- Required events include `run_id` when run-scoped. +- Required events include `node_id` when node-scoped. +- Required events include `stage_index` when stage-scoped. +- Required events include `edge_id` when edge-scoped. +- Required events include `ring_id` when ring-scoped. +- Object events include `object_id` and `sequence`. +- Step events include `step_id`. +- Worker events include `worker_generation`. + +## Lifecycle Events + +- Node boot emits `node_started` and `node_available` or `node_faulted`. +- Pool readiness emits `pool_ready`. +- Planning emits `run_planned`. +- Stage provisioning emits `stage_provision_started`. +- Weight work emits `weights_download_started`, `weights_downloaded`, and + `weights_loaded` when those phases occur. +- Edge provisioning emits `edge_provision_started` and `edge_ready`. +- Stage readiness emits `stage_ready`. +- The global barrier emits `readiness_barrier_passed`. +- Prompt injection emits `prompt_injected`. +- Execution emits `object_loaded`, `execute_step_started`, `object_produced`, + `step_completed`, and `token_received`. +- Terminal run state emits `run_completed` or `run_faulted`. +- Teardown emits `stop_run_sent`, `stage_stopped`, and `run_torn_down`. + +## Fault Events + +- Fault events include a stable reason enum. +- Fault events include the component that detected the fault. +- Tests do not need to scrape logs to determine lifecycle progress. +- Free-form logs may add diagnostics but do not replace structured events. + +## Ordering + +- Events reflect the same ordering guarantees as the component contracts. +- `prompt_injected` cannot precede `readiness_barrier_passed`. +- `stage_ready` cannot precede required local readiness. +- `run_torn_down` cannot precede teardown completion. +- A run emits exactly one terminal outcome event. + +## Test Direction + +Tests should subscribe to the stable event stream and assert event identities, +reason enums, and ordering. Contract tests should not depend on log text, +transport implementation, storage backend, or event batching policy. diff --git a/specs/mvp_system/orchestrator_run_fsm_contract.md b/specs/mvp_system/orchestrator_run_fsm_contract.md new file mode 100644 index 0000000..10a571d --- /dev/null +++ b/specs/mvp_system/orchestrator_run_fsm_contract.md @@ -0,0 +1,67 @@ +# Orchestrator Run FSM Contract + +This document defines the behavioral contract for the orchestrator run FSM. It +covers how a valid `RunPlan` becomes provisioning, readiness, prompt injection, +execution, terminal outcome, and teardown. + +## Planning And Provisioning + +- The orchestrator plans only after `PoolReady`. +- It provisions only from a valid `RunPlan`. +- It sends exactly one `ProvisionStage` to every planned stage. +- It does not provision unknown stages. +- It does not provision nodes outside the committed plan. +- It creates local token-in and token-out endpoints according to the plan. + +## Readiness Barrier + +- The orchestrator does not inject the prompt before every planned stage reports + `StageReady`. +- The orchestrator does not inject the prompt before local token endpoints are + ready. +- Duplicate `StageReady` does not advance readiness twice. +- `StageReady` from an unknown stage rejects or faults. +- `StageReady` for a different run rejects or faults. + +## Execution Drive + +- Prompt injection is the start signal. +- There is no separate broadcast start. +- The orchestrator injects sequence `0` first. +- It injects sequence `k + 1` only after consuming token sequence `k`. +- It stops injecting after EOS. +- It stops injecting after `max_tokens`. + +## Fault Behavior + +- `StageFault` before readiness faults the run. +- `StageFault` during execution faults the run. +- Membership loss for a required node faults the run. +- Endpoint fault faults the run. +- Timeout faults the run. +- The first run-level failure reason is retained. +- Later failure reasons do not replace the recorded terminal reason. + +## Terminal Outcome + +- Each run records exactly one terminal outcome. +- `Completed` and `Faulted` are mutually exclusive. +- Operator stop before completion records the operator-stopped outcome. +- After terminal outcome begins, no new prompt or token work is accepted. +- Teardown is required after success, fault, and operator stop. + +## Teardown + +- The orchestrator sends `StopRun` to every provisioned stage. +- It tears down local token endpoints. +- It waits for `StageStopped` from every provisioned stage or teardown timeout. +- It emits `run_torn_down` exactly once. +- `run_torn_down` is emitted only after teardown has reached its terminal state. + +## Test Direction + +Tests should drive the orchestrator with public events and observe emitted +commands, lifecycle events, and terminal outcome. Successful tests should prove +the readiness barrier, sequence injection rule, and teardown after completion. +Fault tests should inject one fault source at a time and assert exactly one +run-level terminal outcome and teardown. diff --git a/specs/mvp_system/orchestrator_token_endpoint_contract.md b/specs/mvp_system/orchestrator_token_endpoint_contract.md new file mode 100644 index 0000000..25d4397 --- /dev/null +++ b/specs/mvp_system/orchestrator_token_endpoint_contract.md @@ -0,0 +1,44 @@ +# Orchestrator Token Endpoint Contract + +This document defines the behavioral contract for orchestrator-owned token +edges. The orchestrator is a data-plane participant for token-in and token-out, +but it does not run model compute. + +## Endpoint Formation + +- The orchestrator creates the token-in producer from the committed plan. +- The orchestrator creates the token-out consumer from the committed plan. +- Token endpoints use the same edge semantics as stage endpoints. +- The orchestrator endpoint has a stable `node_id`. +- Co-location with a GPU node does not change token edge semantics. + +## Prompt Injection + +- Prompt injection happens only after the global readiness barrier. +- Prompt injection writes token object sequence `0`. +- Prompt injection is the only run start signal. +- The token-in object conforms to the token `ObjectSpec`. + +## Token Consumption + +- Token-out consumption observes token objects in sequence order. +- The orchestrator consumes token sequence `k` before deciding whether to inject + sequence `k + 1`. +- EOS stops further injection. +- `max_tokens` stops further injection. +- Out-of-order token output faults the run. + +## Fault Behavior + +- Token-in endpoint fault faults the run. +- Token-out endpoint fault faults the run. +- Malformed token object faults the run. +- Token sequence violation faults the run. +- Token endpoint teardown failure contributes to run teardown failure. + +## Test Direction + +Tests should drive the endpoint through prompt injection and token-return +traces. Successful tests should assert sequence `0` first and `k + 1` only +after token `k`. Fault tests should inject endpoint fault, malformed token, and +out-of-order token output and assert run fault. diff --git a/specs/mvp_system/resource_inventory_contract.md b/specs/mvp_system/resource_inventory_contract.md new file mode 100644 index 0000000..815687e --- /dev/null +++ b/specs/mvp_system/resource_inventory_contract.md @@ -0,0 +1,37 @@ +# Resource Inventory Contract + +This document defines the behavioral contract for the MVP resource inventory +used by planning. The inventory is an orchestrator-owned input to `RunPlan` +formation, not a distributed negotiation protocol. + +## Inventory Formation + +- The orchestrator owns the intended node pool. +- The orchestrator owns the resource inventory used for placement. +- Each inventory entry is tied to a known `node_id`. +- Inventory facts are available before planning starts. +- A candidate node may report boot health and readiness, but does not negotiate + graph placement after boot. + +## Planning Input + +- The planner receives a candidate pool and placement input from the + orchestrator. +- The planner rejects placement that names nodes outside the candidate pool. +- The planner rejects missing or duplicate stage assignments. +- The planner does not mutate the candidate pool. +- The planner does not derive hidden nodes outside the inventory. + +## Authority + +- Resource inventory determines what the planner is allowed to place onto. +- `RunPlan` determines what actually gets placed. +- Nodes do not advertise new placement facts during run planning. +- Stages do not reinterpret inventory after provisioning. + +## Test Direction + +Tests should treat inventory as public planner input. Successful tests should +assert that every planned stage node comes from the inventory. Rejection tests +should name unknown nodes, duplicate stage assignments, or incomplete placement +facts and assert typed planner rejection. diff --git a/specs/mvp_system/run_plan_contract.md b/specs/mvp_system/run_plan_contract.md new file mode 100644 index 0000000..72ff292 --- /dev/null +++ b/specs/mvp_system/run_plan_contract.md @@ -0,0 +1,79 @@ +# RunPlan Contract + +This document defines the behavioral contract for MVP `RunPlan` formation and +projection. It is concerned with externally observable planner behavior: the +planner inputs, the emitted `RunPlan`, derived provisioning messages, and typed +rejections. + +The planner is the first contract target because the orchestrator is the run +authority. Every downstream component receives its stage, layer, edge, and +object obligations from the committed plan. + +## RunPlan Formation + +- Given model facts, runtime config, intended node pool, stage count, and + placement input, the planner either emits exactly one `RunPlan` or a typed + rejection. +- It never emits a partial plan. +- It never mutates the candidate pool or derives hidden topology outside the + plan. + +## Layer Assignment + +- Stage ranges are contiguous. +- Stage ranges do not overlap. +- The union covers the intended GGUF block range. +- Every stage has exactly one non-empty assigned range unless the spec + explicitly allows empty ranges. +- Stage indices are `0..stage_count-1`. + +## Edge Assignment + +- Every edge id is unique within the run. +- Every edge has exactly one producer and one consumer. +- Token-in edge is `orchestrator -> stage 0`. +- Activation edge `i` is `stage i -> stage i + 1`. +- Token-out edge is `stage N-1 -> orchestrator`. +- Stages do not derive edge ids from names, hashes, layer ranges, or peers. + +## Provisioning Projection + +- For each `StagePlan`, deriving `ProvisionStage` is deterministic. +- A stage receives only its own layer range. +- A stage receives exactly one inbound and one outbound edge provision. +- Outbound provision contains the consumer `node_id`. +- Inbound provision does not require producer actor address. +- No provision message contains remote actor addresses for data flow. + +## Object/Ring Spec Consistency + +- Every edge has an `ObjectSpec` and `RingSpec`. +- Activation `max_extent` follows the model facts: + `max_seq_len * hidden_dim * dtype_width_bytes`. +- Token edges use token object specs, activation edges use activation object + specs. +- Specs are copied consistently into stage provisions. +- Invalid extent, alignment, dtype width, or unsupported shape/layout rejects + the plan. + +## Authority / Rejection + +- Unknown node id rejects. +- Duplicate stage assignment rejects. +- Missing stage rejects. +- Invalid stage count rejects. +- Edge endpoint mismatch rejects. +- Model facts inconsistent with requested stage layout reject. +- The planner is the only source of stage, layer, edge, and object-spec + assignment. + +## Test Direction + +The tests for this contract should assert guarantees over planner inputs, +planner outputs, derived provisioning messages, and typed rejections. They +should not assert planner internals, placement heuristics, helper APIs, +allocation strategy, or private data structures. + +Successful-plan tests should inspect the returned `RunPlan` and derived +`ProvisionStage` values. Rejection tests should feed contradictory inputs and +assert that no plan is emitted. diff --git a/specs/mvp_system/shared_ring_helper_abi_contract.md b/specs/mvp_system/shared_ring_helper_abi_contract.md new file mode 100644 index 0000000..72a3bfb --- /dev/null +++ b/specs/mvp_system/shared_ring_helper_abi_contract.md @@ -0,0 +1,62 @@ +# Shared Ring Helper ABI Contract + +This document defines the behavioral contract for the shared ring ABI and native +helper. Both Rust hot-path code and the Python worker access process-crossing +rings through this helper. + +## Ring Identity + +- A ring is a bounded single-producer/single-consumer byte stream. +- Each ring has exactly one producer and one consumer. +- `RingId` is unique for the node lifetime. +- Stale wake events cannot alias replacement rings. + +## Cursor Contract + +- `commit` is the first byte after the committed readable prefix. +- `consume` is the first byte not yet released by the consumer. +- Producer-local `write` does not expose bytes to the consumer. +- Cursor values are monotonic logical byte positions. +- Physical indices are derived by `cursor % capacity`. + +## Producer Rules + +- The producer computes free space from acquired `consume`. +- The producer does not reserve beyond ring capacity. +- The producer writes bytes before publishing `commit`. +- Publishing `commit` uses release ordering. +- After publishing readable bytes, the producer sends or coalesces + `RingReadable`. + +## Consumer Rules + +- The consumer computes readable bytes from acquired `commit`. +- The consumer does not read beyond committed bytes. +- The consumer advances `consume` only after bytes are safe to release. +- Publishing `consume` uses release ordering. +- After releasing space, the consumer sends or coalesces `RingWritable`. + +## Wake Rules + +- Wake hints are edge-trigger hints. +- Wake hints carry no byte ranges, counts, pointers, or credits. +- Receivers reload cursors from shared memory. +- Duplicate wakes may be coalesced only while durable scheduler state still + makes the ring discoverable. +- Losing the only empty-to-readable or full-to-writable transition is a liveness + bug. + +## Helper ABI + +- Python does not implement shared atomics directly. +- Python does not implement wrap arithmetic directly. +- Returned pointers are process-local addresses derived from arena base plus + arena offsets. +- Helper operations use acquire/release semantics across the process boundary. + +## Test Direction + +Tests should use the public helper operations to exercise wraparound, full, +empty, publish, consume, and wake behavior. Safety tests should assert no +unwritten reads and no unread overwrite. Liveness tests should assert that wake +coalescing cannot hide a discoverable readable or writable transition. diff --git a/specs/mvp_system/stage_controller_contract.md b/specs/mvp_system/stage_controller_contract.md new file mode 100644 index 0000000..61ba72b --- /dev/null +++ b/specs/mvp_system/stage_controller_contract.md @@ -0,0 +1,70 @@ +# StageController Contract + +This document defines the behavioral contract for a provisioned stage's +`StageController`. The controller is control-path only: it observes setup, +worker, edge, and object events, and issues worker commands. + +## Provisioning + +- A stage starts unprovisioned. +- It accepts `ProvisionStage` only from the authorized orchestrator. +- It validates `run_id`, `stage_index`, stage count, layer range, and edge + provisions before setup. +- Invalid provisioning emits `StageFault`. +- A stage does not rewire inbound or outbound edges. + +## Preparation + +- The controller configures the local worker role path. +- It starts assigned weight download, load, or bind work. +- It establishes the inbound receive edge. +- It establishes the outbound send edge. +- It reports `StageReady` only after worker readiness, weight readiness, inbound + edge readiness, and outbound edge readiness. + +## Execution Admission + +- A ready stage executes only after inbound `ObjectLoaded`. +- The inbound object sequence must equal the next expected sequence. +- The stage issues exactly one `ExecuteStep` for each accepted inbound object. +- The output binding uses the same sequence as the input object. +- The MVP allows one active `ExecuteStep` per stage. + +## Sequence Safety + +- Sequence `0` is accepted as prefill. +- Decode sequences are strictly increasing after sequence `0`. +- Duplicate sequence faults the stage. +- Skipped sequence faults the stage. +- Out-of-order sequence faults the stage. + +## Completion + +- `StepCompleted` returns the stage to ready-for-next-object state. +- The controller releases per-step input handles according to policy. +- The stage does not report compute completion before the worker reports + `StepCompleted`. + +## Fault Behavior + +- Worker crash faults the stage. +- `StepFailed` faults the stage. +- `ObjectFailed` faults the stage. +- Output edge fault faults the stage. +- Sequence violation faults the stage. +- After fault, the stage rejects new run work until stopped. + +## Stopping + +- On `StopRun`, the controller stops local edges. +- It releases per-run device objects. +- It stops or resets worker role state according to local policy. +- It emits `StageStopped` once teardown reaches the local terminal state. + +## Test Direction + +Tests should drive the controller with public setup and worker/edge events. +Successful tests should assert `StageReady` ordering, one `ExecuteStep` per +accepted object, and same-sequence output binding. Fault tests should inject +invalid sequence, setup failure, worker crash, and step failure and assert +`StageFault` followed by rejection of new work. diff --git a/specs/mvp_system/tx_rx_edge_actor_contract.md b/specs/mvp_system/tx_rx_edge_actor_contract.md new file mode 100644 index 0000000..dbdb017 --- /dev/null +++ b/specs/mvp_system/tx_rx_edge_actor_contract.md @@ -0,0 +1,53 @@ +# Tx And Rx Edge Actor Contract + +This document defines the behavioral contract for Tx and Rx edge actors. They +are role-facing lifecycle gates and never carry payload bytes. + +## Actor Role + +- A Tx actor represents the producer side of one edge. +- An Rx actor represents the consumer side of one edge. +- Each edge actor is tied to one `edge_id`. +- Edge actors receive lifecycle and object events only. +- Edge actors do not receive bytes, pointers, ranges, credits, or free-space + counts. + +## Tx Lifecycle + +- Tx starts in provisioning. +- `EdgeReady` moves Tx to ready. +- Producing is allowed only after ready. +- `ObjectProduced` reports committed output object identity. +- Stream fault or object fault moves Tx to faulted. +- `StopEdge` moves Tx toward stopped. + +## Rx Lifecycle + +- Rx starts in provisioning. +- `EdgeReady` moves Rx to ready. +- `ObjectLoaded` reports complete logical input object identity and handle. +- Rx exposes loaded objects to the role layer only after `ObjectLoaded`. +- Object failure or stream fault moves Rx to faulted. +- `StopEdge` moves Rx toward stopped. + +## Payload Isolation + +- Payload bytes never travel in edge actor messages. +- Host pointers never travel in edge actor messages. +- Flow-control details never travel in edge actor messages. +- Edge actors traffic only identities, lifecycle events, opaque handles, and + coarse faults. + +## Fault Behavior + +- `ObjectFailed` faults the corresponding edge actor. +- `StreamFault` faults the corresponding edge actor. +- Stale events for stopped actors are ignored. +- Events for a mismatched `edge_id` reject or fault according to local policy. + +## Test Direction + +Tests should drive Tx and Rx actors with public lifecycle and object events. +Successful tests should assert readiness before production/loading and payload +isolation in message types. Fault tests should inject stream and object faults +and assert no further run work is accepted before stop. diff --git a/specs/mvp_system/weight_lifecycle_contract.md b/specs/mvp_system/weight_lifecycle_contract.md new file mode 100644 index 0000000..7c6e803 --- /dev/null +++ b/specs/mvp_system/weight_lifecycle_contract.md @@ -0,0 +1,45 @@ +# Weight Lifecycle Contract + +This document defines the behavioral contract for stage-local weight work. +Weights are persistent run state for a stage and must be usable before the +stage reports `StageReady`. + +## Assignment + +- The stage receives its weight source from `ProvisionStage`. +- The stage receives exactly one assigned layer range. +- The stage validates the assigned layer range against the run plan. +- The stage does not load layers outside its assigned range as graph-visible + ownership. + +## Loading + +- A stage may download a whole GGUF and load only its range. +- A stage may download physical shards containing its range. +- A stage may use a cached artifact that already exists on the node. +- The physical loading mechanism is implementation-defined. +- The system-visible outcome is `WeightsReady` or `StageFault`. + +## WeightsReady + +- `WeightsReady` requires assigned artifact bytes to be locally available or + cached. +- `WeightsReady` requires the assigned layer range to be validated. +- `WeightsReady` requires the worker to have loaded or bound the range needed + for execution. +- `WeightsReady` happens before `StageReady`. + +## Failure Behavior + +- Download failure faults the stage. +- Parse failure faults the stage. +- Device allocation failure faults the stage. +- Binding failure faults the stage. +- Invalid layer range faults the stage. + +## Test Direction + +Tests should treat weight loading as a stage-local black box with observable +events. Successful tests should assert `WeightsReady` before `StageReady`. +Failure tests should inject download, parse, allocation, and bind failures and +assert `StageFault` with no `StageReady`. diff --git a/tests/mvp_system/arena_manager_guarantees.rs b/tests/mvp_system/arena_manager_guarantees.rs new file mode 100644 index 0000000..1fbfb73 --- /dev/null +++ b/tests/mvp_system/arena_manager_guarantees.rs @@ -0,0 +1,266 @@ +//! Black-box contract tests for MVP ArenaManager behavior. +//! +//! These tests intentionally know only the public arena-manager surface: +//! +//! - arena boot configuration +//! - lease, cancel, release, quiescence, and shutdown requests in +//! - lease events, rejected requests, released ranges, and faults out +//! +//! They assert the guarantees in +//! `specs/mvp_system/arena_manager_contract.md`. + +use mvp_system::arena_manager as arena; + +// A small deterministic arena makes overlap, alignment, queueing, and reuse +// proofs easy to inspect. The concrete mmap strategy remains outside the test. +fn arena_config() -> arena::ArenaConfig { + arena::ArenaConfig { + node_id: arena::NodeId(10), + reservation_ceiling: 4096, + base_alignment: 64, + } +} + +// Lease requests name size and alignment only. They do not request pointers or +// private allocator slots, which keeps layout authority inside ArenaManager. +fn lease_request(request_id: u64, bytes: u64, alignment: u64) -> arena::LeaseRing { + arena::LeaseRing { + request_id: arena::LeaseRequestId(request_id), + ring_spec: arena::RingSpec { + header_bytes: 128, + data_bytes: bytes, + alignment, + }, + } +} + +// The harness exposes public lease events and lease snapshots. Tests use those +// snapshots only after a RingLeased event, so private allocator state remains +// unobservable. +fn new_arena() -> arena::ArenaManagerHarness { + arena::ArenaManagerHarness::boot(arena_config()).expect("test arena must boot") +} + +// This helper proves two public layouts are disjoint using half-open ranges. +// It is more useful than comparing offsets directly because allocator choice is +// intentionally implementation-defined. +fn assert_non_overlapping(left: &arena::RingLayout, right: &arena::RingLayout) { + let left_range = left.start_offset..left.end_offset; + let right_range = right.start_offset..right.end_offset; + assert!( + left_range.end <= right_range.start || right_range.end <= left_range.start, + "live leases overlap: {left:?} and {right:?}" + ); +} + +// This proves arena boot creates one stable sparse arena with one reservation +// ceiling, offset-only layouts, and typed boot failure. +#[test] +fn arena_boot_creates_stable_offset_only_layout_domain() { + // Boot a valid arena. + let mut harness = new_arena(); + + // Lease one ring so the public layout can be inspected. + harness.request(arena::ArenaRequest::LeaseRing(lease_request(1, 512, 64))); + let lease = harness + .events() + .iter() + .find_map(|event| match event { + arena::ArenaEvent::RingLeased { lease } => Some(lease), + _ => None, + }) + .expect("valid lease must be granted"); + + // Layout facts are arena offsets and stay under the reservation ceiling. + assert!(lease.layout.start_offset < arena_config().reservation_ceiling); + assert!(lease.layout.end_offset <= arena_config().reservation_ceiling); + assert!(matches!(lease.layout.pointer, arena::LayoutPointer::NoProcessPointer)); + + // Boot failure is typed and emits no usable arena. + let failed = arena::ArenaManagerHarness::boot(arena::ArenaConfig { + reservation_ceiling: 0, + ..arena_config() + }); + assert!(matches!( + failed, + Err(arena::ArenaFault::InvalidReservationCeiling) + )); +} + +// This proves LeaseRing either leases, queues, or rejects, and that live RingId +// values are unique for the node lifetime. +#[test] +fn lease_requests_grant_queue_or_reject_with_unique_ring_ids() { + // Fill most of the arena with one live lease. + let mut harness = new_arena(); + harness.request(arena::ArenaRequest::LeaseRing(lease_request(1, 3072, 64))); + harness.request(arena::ArenaRequest::LeaseRing(lease_request(2, 1024, 64))); + + // A satisfiable request under pressure may queue instead of rejecting. + assert!(harness.events().iter().any(|event| { + matches!( + event, + arena::ArenaEvent::RingLeaseQueued { + request_id: arena::LeaseRequestId(2) + } + ) + })); + + // A request larger than the reservation ceiling must reject. + harness.request(arena::ArenaRequest::LeaseRing(lease_request(3, 8192, 64))); + assert!(harness.events().iter().any(|event| { + matches!( + event, + arena::ArenaEvent::RingLeaseRejected { + request_id: arena::LeaseRequestId(3), + reason: arena::RingLeaseRejection::CannotFitWithinCeiling, + } + ) + })); + + // Granted RingId values must be unique among all live leases. + let ids = harness + .live_leases() + .iter() + .map(|lease| lease.ring_id) + .collect::>(); + assert_eq!(ids.len(), harness.live_leases().len()); +} + +// This proves live leases are non-overlapping, in-bounds, aligned, and stable +// for the lease lifetime. +#[test] +fn live_layouts_are_non_overlapping_aligned_in_bounds_and_stable() { + // Lease two rings with explicit alignment requirements. + let mut harness = new_arena(); + harness.request(arena::ArenaRequest::LeaseRing(lease_request(1, 512, 64))); + harness.request(arena::ArenaRequest::LeaseRing(lease_request(2, 512, 128))); + + // Read the public lease snapshots. + let leases = harness.live_leases().to_vec(); + assert_eq!(leases.len(), 2); + + // Prove non-overlap and in-bounds without constraining allocator placement. + assert_non_overlapping(&leases[0].layout, &leases[1].layout); + for lease in &leases { + assert!(lease.layout.end_offset <= arena_config().reservation_ceiling); + assert_eq!(lease.layout.start_offset % lease.requested_alignment, 0); + } + + // Re-observing the same live lease must not change offsets. + let before = leases[0].layout.clone(); + let after = harness + .lookup_lease(leases[0].ring_id) + .expect("live lease must be lookupable") + .layout + .clone(); + assert_eq!(after, before); +} + +// This proves CancelLease removes queued work and suppresses later hot-path +// installation, including a fresh lease that races with cancellation. +#[test] +fn cancelled_queued_lease_never_installs_hot_path_state() { + // Fill the arena and queue a second satisfiable lease. + let mut harness = new_arena(); + harness.request(arena::ArenaRequest::LeaseRing(lease_request(1, 3072, 64))); + harness.request(arena::ArenaRequest::LeaseRing(lease_request(2, 1024, 64))); + + // Cancel the queued request before it is leased. + harness.request(arena::ArenaRequest::CancelLease { + request_id: arena::LeaseRequestId(2), + }); + + // Releasing pressure must not install worker or pump state for the canceled + // request. + let live = harness.live_leases()[0].ring_id; + harness.request(arena::ArenaRequest::ReleaseRing { + ring_id: live, + proof: arena::QuiescenceProof::verified(), + }); + assert!(!harness.commands().iter().any(|command| { + matches!( + command, + arena::ArenaCommand::InstallWorkerOrPumpState { + request_id: arena::LeaseRequestId(2), + .. + } + ) + })); + + // If a fresh lease was produced during the race, it must be released instead + // of becoming hot-path state. + assert!(harness.events().iter().any(|event| { + matches!( + event, + arena::ArenaEvent::CancelledFreshLeaseReleased { + request_id: arena::LeaseRequestId(2) + } + ) + })); +} + +// This proves ranges are released only with quiescence proof, are not reused +// while live work owns them, and may be reused after release. +#[test] +fn release_requires_quiescence_and_reuse_happens_only_after_release() { + // Lease one ring and record its range. + let mut harness = new_arena(); + harness.request(arena::ArenaRequest::LeaseRing(lease_request(1, 1024, 64))); + let first = harness.live_leases()[0].clone(); + + // Release without proof must reject and keep the range live. + harness.request(arena::ArenaRequest::ReleaseRing { + ring_id: first.ring_id, + proof: arena::QuiescenceProof::missing(), + }); + assert!(harness.lookup_lease(first.ring_id).is_some()); + + // A second lease while the first is live must not overlap the first range. + harness.request(arena::ArenaRequest::LeaseRing(lease_request(2, 1024, 64))); + let second = harness + .live_leases() + .iter() + .find(|lease| lease.ring_id != first.ring_id) + .expect("second live lease must exist") + .clone(); + assert_non_overlapping(&first.layout, &second.layout); + + // Verified quiescence allows release, after which reuse is legal. + harness.request(arena::ArenaRequest::ReleaseRing { + ring_id: first.ring_id, + proof: arena::QuiescenceProof::verified(), + }); + harness.request(arena::ArenaRequest::LeaseRing(lease_request(3, 1024, 64))); + assert!(harness.events().iter().any(|event| { + matches!(event, arena::ArenaEvent::RingLeased { .. }) + })); +} + +// This proves shutdown rejects new leases, preserves live lease records, and +// does not release live ranges without quiescence proof. +#[test] +fn shutdown_rejects_new_leases_without_corrupting_live_records() { + // Create a live lease before shutdown. + let mut harness = new_arena(); + harness.request(arena::ArenaRequest::LeaseRing(lease_request(1, 512, 64))); + let live_before = harness.live_leases().to_vec(); + + // Shut the arena down. + harness.request(arena::ArenaRequest::Shutdown); + + // New leases are rejected after shutdown. + harness.request(arena::ArenaRequest::LeaseRing(lease_request(2, 512, 64))); + assert!(harness.events().iter().any(|event| { + matches!( + event, + arena::ArenaEvent::RingLeaseRejected { + request_id: arena::LeaseRequestId(2), + reason: arena::RingLeaseRejection::ArenaShuttingDown, + } + ) + })); + + // Existing live lease records remain intact until proof-backed release. + assert_eq!(harness.live_leases(), live_before.as_slice()); +} diff --git a/tests/mvp_system/device_bridge_guarantees.rs b/tests/mvp_system/device_bridge_guarantees.rs new file mode 100644 index 0000000..d8c6f21 --- /dev/null +++ b/tests/mvp_system/device_bridge_guarantees.rs @@ -0,0 +1,240 @@ +//! Black-box contract tests for MVP device bridge behavior. +//! +//! These tests intentionally know only the public device-bridge surface: +//! +//! - allocation, host/device copy, view, compute-use, restart, and free requests +//! in +//! - fake backend calls, completion events, handles, and failures out +//! +//! They assert the guarantees in +//! `specs/mvp_system/device_bridge_contract.md`. + +use mvp_system::device_bridge as device; + +// The object spec is small and aligned so exact range checks are readable. The +// bridge remains free to choose backend-specific allocation details. +fn object_spec() -> device::ObjectSpec { + device::ObjectSpec { + max_extent: 16, + alignment: 4, + dtype: device::DType::U32, + shape: device::Shape::Vector, + } +} + +// The harness records fake backend calls and public bridge outcomes. Tests do +// not inspect device memory or tinygrad internals. +fn new_bridge() -> device::DeviceBridgeHarness { + device::DeviceBridgeHarness::new(device::WorkerGeneration(1)) +} + +// This helper allocates one current-generation device object for copy and view +// tests. It keeps tests on the public allocation path. +fn allocated_bridge() -> (device::DeviceBridgeHarness, device::DeviceHandle) { + let mut harness = new_bridge(); + let handle = harness + .alloc_device(object_spec(), 8) + .expect("valid allocation must succeed"); + (harness, handle) +} + +// This proves alloc_device creates a current-generation allocation suitable for +// the object spec and extent, and allocation failure is reported as device +// allocation failure. +#[test] +fn allocation_creates_current_generation_handle_or_typed_failure() { + // Allocate a valid object. + let mut harness = new_bridge(); + let handle = harness + .alloc_device(object_spec(), 8) + .expect("valid allocation must succeed"); + + // The handle is tied to the current worker generation. + assert_eq!(handle.generation, device::WorkerGeneration(1)); + assert!(harness.backend_calls().iter().any(|call| { + matches!( + call, + device::BackendCall::Alloc { + spec, + extent: 8, + .. + } if *spec == object_spec() + ) + })); + + // Backend allocation failure becomes typed allocation failure. + harness.inject_backend_failure(device::BackendFailure::AllocationFailed); + let failure = harness + .alloc_device(object_spec(), 8) + .expect_err("backend allocation failure must surface"); + assert_eq!(failure, device::DeviceError::AllocationFailed); +} + +// This proves host_to_device copies exactly the requested host range to the +// requested device range, and bytes become safe to release only after the copy +// returns or the asynchronous completion event fires. +#[test] +fn host_to_device_copies_exact_range_and_defers_release_until_safe() { + // Allocate a device object and request an asynchronous copy. + let (mut harness, handle) = allocated_bridge(); + let copy = harness.host_to_device( + device::HostRange { offset: 4, len: 8 }, + device::DeviceRange { handle, offset: 0, len: 8 }, + device::CopyMode::Async, + ).expect("copy request must be accepted"); + + // The backend sees the exact ranges. + assert!(harness.backend_calls().iter().any(|call| { + matches!( + call, + device::BackendCall::HostToDevice { + host: device::HostRange { offset: 4, len: 8 }, + device: device::DeviceRange { offset: 0, len: 8, .. }, + .. + } + ) + })); + + // Async copy request alone is not a release proof. + assert!(!harness.safe_to_release_host(copy)); + + // Completion makes bytes safe to release. + harness.observe(device::DeviceEvent::CopyCompleted { copy }); + assert!(harness.safe_to_release_host(copy)); +} + +// This proves device_to_host copies exactly the requested device range to the +// requested host range, and host bytes become valid only after return or copy +// completion. +#[test] +fn device_to_host_copies_exact_range_and_defers_host_validity_until_safe() { + // Allocate a device object and request an asynchronous device-to-host copy. + let (mut harness, handle) = allocated_bridge(); + let copy = harness.device_to_host( + device::DeviceRange { handle, offset: 0, len: 8 }, + device::HostRange { offset: 12, len: 8 }, + device::CopyMode::Async, + ).expect("copy request must be accepted"); + + // The backend sees the exact ranges. + assert!(harness.backend_calls().iter().any(|call| { + matches!( + call, + device::BackendCall::DeviceToHost { + device: device::DeviceRange { offset: 0, len: 8, .. }, + host: device::HostRange { offset: 12, len: 8 }, + .. + } + ) + })); + + // Host bytes are invalid until completion. + assert!(!harness.host_bytes_valid(copy)); + harness.observe(device::DeviceEvent::CopyCompleted { copy }); + assert!(harness.host_bytes_valid(copy)); +} + +// This proves wrap_for_tinygrad creates a compatible view matching the role +// tensor spec, and invalid view shape or dtype fails the step. +#[test] +fn tinygrad_view_matches_role_tensor_spec_or_fails_step() { + // Allocate a valid object and wrap it for a matching role tensor view. + let (mut harness, handle) = allocated_bridge(); + let view = harness + .wrap_for_tinygrad( + handle, + device::TensorViewSpec { + dtype: device::DType::U32, + shape: device::Shape::Vector, + }, + ) + .expect("matching tensor view must succeed"); + assert_eq!(view.dtype, device::DType::U32); + assert_eq!(view.shape, device::Shape::Vector); + + // Invalid dtype fails the step at the bridge boundary. + let failure = harness + .wrap_for_tinygrad( + handle, + device::TensorViewSpec { + dtype: device::DType::F16, + shape: device::Shape::Vector, + }, + ) + .expect_err("invalid dtype must fail"); + assert_eq!(failure, device::DeviceError::InvalidViewDType); +} + +// This proves allocations are not freed while compute or copy events depend on +// them, free_device releases after dependencies clear, restart invalidates old +// handles, and old-generation handles are rejected. +#[test] +fn lifetime_blocks_free_until_dependencies_clear_and_rejects_old_generation() { + // Allocate a handle and mark it used by compute and copy. + let (mut harness, handle) = allocated_bridge(); + harness.observe(device::DeviceEvent::ComputeStarted { + handle, + step_id: device::StepId(77), + }); + let copy = harness.host_to_device( + device::HostRange { offset: 0, len: 8 }, + device::DeviceRange { handle, offset: 0, len: 8 }, + device::CopyMode::Async, + ).expect("copy request must be accepted"); + + // Free is blocked while compute/copy depends on the allocation. + assert_eq!( + harness.free_device(handle), + Err(device::DeviceError::AllocationStillInUse) + ); + + // Complete dependencies, then free succeeds. + harness.observe(device::DeviceEvent::ComputeCompleted { + handle, + step_id: device::StepId(77), + }); + harness.observe(device::DeviceEvent::CopyCompleted { copy }); + assert_eq!(harness.free_device(handle), Ok(())); + assert!(harness.backend_calls().iter().any(|call| { + matches!(call, device::BackendCall::Free { freed } if *freed == handle) + })); + + // Restart invalidates prior handles. + harness.observe(device::DeviceEvent::WorkerRestarted { + generation: device::WorkerGeneration(2), + }); + assert_eq!( + harness.free_device(handle), + Err(device::DeviceError::OldGenerationHandle) + ); +} + +// This proves copy and view backend failures surface as typed copy or step +// failures rather than ambiguous panics or logs. +#[test] +fn backend_copy_and_view_failures_are_typed() { + // Copy failure is reported as device copy failure. + let (mut harness, handle) = allocated_bridge(); + harness.inject_backend_failure(device::BackendFailure::CopyFailed); + let copy_failure = harness + .host_to_device( + device::HostRange { offset: 0, len: 8 }, + device::DeviceRange { handle, offset: 0, len: 8 }, + device::CopyMode::Sync, + ) + .expect_err("copy failure must surface"); + assert_eq!(copy_failure, device::DeviceError::DeviceCopyFailed); + + // View failure is reported as a step-visible invalid view. + harness.inject_backend_failure(device::BackendFailure::InvalidView); + let view_failure = harness + .wrap_for_tinygrad( + handle, + device::TensorViewSpec { + dtype: device::DType::U32, + shape: device::Shape::Vector, + }, + ) + .expect_err("invalid backend view must surface"); + assert_eq!(view_failure, device::DeviceError::InvalidTensorView); +} diff --git a/tests/mvp_system/driver_pumps_guarantees.rs b/tests/mvp_system/driver_pumps_guarantees.rs new file mode 100644 index 0000000..e31f106 --- /dev/null +++ b/tests/mvp_system/driver_pumps_guarantees.rs @@ -0,0 +1,259 @@ +//! Black-box contract tests for MVP driver and pump behavior. +//! +//! These tests intentionally know only the public driver surface: +//! +//! - establish-send/recv, incoming stream, wake, stop, and I/O outcomes in +//! - stream opens, ring cursor updates, wake hints, faults, and stopped events out +//! +//! They assert the guarantees in +//! `specs/mvp_system/driver_pumps_contract.md`. + +use mvp_system::driver_pumps as driver; + +// A driver config names one endpoint and one ALPN. Tests do not expose tokio +// tasks, connection internals, or stream futures to actors. +fn driver_config() -> driver::DriverConfig { + driver::DriverConfig { + local_node_id: driver::NodeId(10), + alpn: driver::Alpn("swactor-edge-mvp".into()), + } +} + +// A send spec describes one persistent uni-stream for one edge to one peer. +// The pump still owns when and how bytes leave the egress ring. +fn send_spec() -> driver::EstablishSend { + driver::EstablishSend { + edge_id: driver::EdgeId(7001), + peer_node_id: driver::NodeId(11), + layout: driver::RingLayout::test_egress(), + } +} + +// A recv spec describes one edge and one ingress ring. It may arrive before or +// after the network stream, which is the receive rendezvous guarantee. +fn recv_spec() -> driver::EstablishRecv { + driver::EstablishRecv { + edge_id: driver::EdgeId(7001), + layout: driver::RingLayout::test_ingress(), + } +} + +// The harness gives tests mock stream and ring observations while keeping the +// driver as the owner of demux and byte-pump behavior. +fn new_driver() -> driver::DriverHarness { + driver::DriverHarness::new(driver_config()) +} + +// This helper extracts the byte transcript for a stream. It proves preamble and +// object bytes by observing writes accepted by the mock stream, not by peeking +// into pump internals. +fn written_bytes(harness: &driver::DriverHarness, edge_id: driver::EdgeId) -> Vec { + harness + .stream_writes(edge_id) + .iter() + .flat_map(|write| write.bytes.clone()) + .collect() +} + +// This proves the driver owns endpoint, connection cache, ALPN, stream demux, +// and send/recv pump tasks, while actors do not poll stream futures directly. +#[test] +fn driver_owns_endpoint_connection_demux_and_pump_tasks() { + // Create the driver and establish a send edge. + let mut harness = new_driver(); + harness.observe(driver::DriverEvent::EstablishSend(send_spec())); + + // The driver creates or reuses a connection under its endpoint and ALPN. + assert!(harness.commands().iter().any(|command| { + matches!( + command, + driver::DriverCommand::OpenOrReuseConnection { + peer_node_id: driver::NodeId(11), + alpn: driver::Alpn(ref value), + .. + } if value == "swactor-edge-mvp" + ) + })); + + // Pump tasks are driver-owned. + assert!(harness.commands().iter().any(|command| { + matches!(command, driver::DriverCommand::SpawnSendPump { edge_id: driver::EdgeId(7001), .. }) + })); + + // Actor commands must not expose stream polling. + assert!(!harness.actor_messages().iter().any(|message| { + matches!(message, driver::ActorMessage::PollStreamFuture { .. }) + })); +} + +// This proves each edge uses one persistent uni-stream, writes an edge-id +// preamble once, carries object records as bytes after the preamble, and does +// not open one stream per object. +#[test] +fn send_stream_is_persistent_with_single_edge_preamble() { + // Establish one send edge and make two committed egress records readable. + let mut harness = new_driver(); + harness.observe(driver::DriverEvent::EstablishSend(send_spec())); + harness.observe(driver::DriverEvent::EgressBytesCommitted { + edge_id: driver::EdgeId(7001), + bytes: b"obj0".to_vec(), + }); + harness.observe(driver::DriverEvent::RingReadable { + edge_id: driver::EdgeId(7001), + }); + harness.observe(driver::DriverEvent::EgressBytesCommitted { + edge_id: driver::EdgeId(7001), + bytes: b"obj1".to_vec(), + }); + harness.observe(driver::DriverEvent::RingReadable { + edge_id: driver::EdgeId(7001), + }); + + // Only one stream is opened for the edge. + let stream_opens = harness + .commands() + .iter() + .filter(|command| matches!(command, driver::DriverCommand::OpenUniStream { .. })) + .count(); + assert_eq!(stream_opens, 1); + + // The edge preamble appears once before object bytes. + let bytes = written_bytes(&harness, driver::EdgeId(7001)); + assert!(bytes.starts_with(&driver::encode_edge_preamble(driver::EdgeId(7001)))); + let preamble_count = driver::count_preamble_occurrences(&bytes, driver::EdgeId(7001)); + assert_eq!(preamble_count, 1); +} + +// This proves receive rendezvous works in both arrival orders and that a +// pending stream is not read before the receive spec exists. +#[test] +fn recv_rendezvous_starts_pump_only_after_spec_and_stream_exist() { + // Spec first, stream second. + let mut spec_first = new_driver(); + spec_first.observe(driver::DriverEvent::EstablishRecv(recv_spec())); + assert!(!spec_first.commands().iter().any(|command| { + matches!(command, driver::DriverCommand::SpawnRecvPump { .. }) + })); + spec_first.observe(driver::DriverEvent::IncomingUniStream { + edge_id: driver::EdgeId(7001), + stream_id: driver::StreamId(1), + }); + assert!(spec_first.commands().iter().any(|command| { + matches!(command, driver::DriverCommand::SpawnRecvPump { edge_id: driver::EdgeId(7001), .. }) + })); + + // Stream first, spec second. + let mut stream_first = new_driver(); + stream_first.observe(driver::DriverEvent::IncomingUniStream { + edge_id: driver::EdgeId(7001), + stream_id: driver::StreamId(2), + }); + assert!(!stream_first.stream_reads_started(driver::StreamId(2))); + stream_first.observe(driver::DriverEvent::EstablishRecv(recv_spec())); + assert!(stream_first.stream_reads_started(driver::StreamId(2))); +} + +// This proves the recv pump is byte-blind after demux, copies QUIC bytes into +// ingress ring spans, advances commit after copy, emits readable wakes, and +// stops reading under backpressure. +#[test] +fn recv_pump_copies_bytes_without_parsing_and_respects_backpressure() { + // Rendezvous a receive pump. + let mut harness = new_driver(); + harness.observe(driver::DriverEvent::IncomingUniStream { + edge_id: driver::EdgeId(7001), + stream_id: driver::StreamId(1), + }); + harness.observe(driver::DriverEvent::EstablishRecv(recv_spec())); + + // Deliver bytes that happen to look like an object header. The pump must + // copy them blindly, not parse them. + harness.observe(driver::DriverEvent::StreamBytesRead { + edge_id: driver::EdgeId(7001), + bytes: driver::fake_object_header_bytes(), + }); + assert!(!harness.events().iter().any(|event| { + matches!(event, driver::DriverEventOut::ObjectHeaderParsed { .. }) + })); + assert!(harness.ring_commit(driver::EdgeId(7001)) > 0); + assert!(harness.wake_hints().iter().any(|wake| { + matches!(wake, driver::WakeHint::RingReadable { edge_id: driver::EdgeId(7001) }) + })); + + // With no ring space, the pump stops reading and waits for RingWritable. + harness.observe(driver::DriverEvent::IngressRingFull { + edge_id: driver::EdgeId(7001), + }); + assert!(!harness.is_reading_stream(driver::EdgeId(7001))); + harness.observe(driver::DriverEvent::RingWritable { + edge_id: driver::EdgeId(7001), + }); + assert!(harness.is_reading_stream(driver::EdgeId(7001))); +} + +// This proves the send pump writes committed egress bytes, advances consume +// only after write_all accepts bytes, emits writable wakes, and keeps ownership +// of unread bytes while network flow control stalls. +#[test] +fn send_pump_advances_consume_only_after_write_acceptance() { + // Establish a send pump and make bytes readable. + let mut harness = new_driver(); + harness.observe(driver::DriverEvent::EstablishSend(send_spec())); + harness.observe(driver::DriverEvent::EgressBytesCommitted { + edge_id: driver::EdgeId(7001), + bytes: b"payload".to_vec(), + }); + harness.observe(driver::DriverEvent::NetworkStalled { + edge_id: driver::EdgeId(7001), + }); + + // Stalled network keeps ownership of unread ring bytes. + assert_eq!(harness.ring_consume(driver::EdgeId(7001)), 0); + + // Once write_all accepts the bytes, consume advances and writable is hinted. + harness.observe(driver::DriverEvent::WriteAllAccepted { + edge_id: driver::EdgeId(7001), + byte_count: 7, + }); + assert_eq!(harness.ring_consume(driver::EdgeId(7001)), 7); + assert!(harness.wake_hints().iter().any(|wake| { + matches!(wake, driver::WakeHint::RingWritable { edge_id: driver::EdgeId(7001) }) + })); +} + +// This proves read, write, protocol, and stop outcomes are surfaced as +// StreamFault or PumpStopped events. +#[test] +fn driver_faults_and_stop_emit_stream_fault_or_pump_stopped() { + // Read error faults the receive edge. + let mut recv = new_driver(); + recv.observe(driver::DriverEvent::IncomingUniStream { + edge_id: driver::EdgeId(7001), + stream_id: driver::StreamId(1), + }); + recv.observe(driver::DriverEvent::EstablishRecv(recv_spec())); + recv.observe(driver::DriverEvent::ReadError { + edge_id: driver::EdgeId(7001), + }); + assert!(recv.events().iter().any(|event| { + matches!(event, driver::DriverEventOut::StreamFault { edge_id: driver::EdgeId(7001), .. }) + })); + + // Write error faults the send edge. + let mut send = new_driver(); + send.observe(driver::DriverEvent::EstablishSend(send_spec())); + send.observe(driver::DriverEvent::WriteError { + edge_id: driver::EdgeId(7001), + }); + assert!(send.events().iter().any(|event| { + matches!(event, driver::DriverEventOut::StreamFault { edge_id: driver::EdgeId(7001), .. }) + })); + + // StopEdge stops the corresponding pump. + send.observe(driver::DriverEvent::StopEdge { + edge_id: driver::EdgeId(7001), + }); + assert!(send.events().iter().any(|event| { + matches!(event, driver::DriverEventOut::PumpStopped { edge_id: driver::EdgeId(7001), .. }) + })); +} diff --git a/tests/mvp_system/edge_establisher_guarantees.rs b/tests/mvp_system/edge_establisher_guarantees.rs new file mode 100644 index 0000000..7c896fe --- /dev/null +++ b/tests/mvp_system/edge_establisher_guarantees.rs @@ -0,0 +1,299 @@ +//! Black-box contract tests for MVP edge establishment. +//! +//! These tests intentionally know only the public EdgeEstablisher surface: +//! +//! - `ProvisionTx`, `ProvisionRx`, lease, ring-install, driver, stop, and fault +//! events in +//! - arena, worker/token, driver, ready, fault, and release commands out +//! +//! They assert the guarantees in +//! `specs/mvp_system/edge_establisher_contract.md`. + +use mvp_system::edge_establisher as edge; + +// A send provision carries the consumer node id because the driver must know +// where to send. It deliberately carries no remote actor address. +fn tx_provision() -> edge::ProvisionTx { + edge::ProvisionTx { + run_id: edge::RunId(7), + edge_id: edge::EdgeId(7001), + local_node_id: edge::NodeId(10), + consumer_node_id: edge::NodeId(11), + object_spec: edge::ObjectSpec::test_activation(), + ring_spec: edge::RingSpec::test_activation(), + } +} + +// A receive provision needs the shared edge id and local layout information +// after leasing; it does not need producer actor addressing for data flow. +fn rx_provision() -> edge::ProvisionRx { + edge::ProvisionRx { + run_id: edge::RunId(7), + edge_id: edge::EdgeId(7001), + local_node_id: edge::NodeId(11), + object_spec: edge::ObjectSpec::test_activation(), + ring_spec: edge::RingSpec::test_activation(), + } +} + +// The harness records only public establishment outputs. Tests never inspect a +// private edge record; they infer it from commands and lifecycle events. +fn new_establisher() -> edge::EdgeEstablisherHarness { + edge::EdgeEstablisherHarness::new(edge::NodeId(10)) +} + +// This helper drives a successful lease and ring install for the tx edge. It is +// used by driver and stop tests to stay on the public establishment path. +fn leased_and_installed_tx() -> edge::EdgeEstablisherHarness { + let mut harness = new_establisher(); + harness.observe(edge::EdgeEvent::ProvisionTx(tx_provision())); + harness.observe(edge::EdgeEvent::RingLeased { + request_id: edge::LeaseRequestId(1), + ring_id: edge::RingId(8001), + layout: edge::RingLayout::test_layout(0), + }); + harness.observe(edge::EdgeEvent::RingInstalled { + edge_id: edge::EdgeId(7001), + ring_id: edge::RingId(8001), + }); + harness +} + +// This proves ProvisionTx and ProvisionRx create local edge records with the +// required addressing facts and without remote actor addresses. +#[test] +fn provisioning_creates_local_edge_records_without_remote_actor_addresses() { + // Provision both sides through public messages. + let mut tx = new_establisher(); + tx.observe(edge::EdgeEvent::ProvisionTx(tx_provision())); + let mut rx = edge::EdgeEstablisherHarness::new(edge::NodeId(11)); + rx.observe(edge::EdgeEvent::ProvisionRx(rx_provision())); + + // Tx must request a local lease and remember the consumer node id for the + // later driver command. + assert!(tx.commands().iter().any(|command| { + matches!( + command, + edge::EdgeCommand::LeaseRing { + edge_id: edge::EdgeId(7001), + .. + } + ) + })); + assert_eq!( + tx.local_record(edge::EdgeId(7001)).unwrap().peer_node_id, + Some(edge::NodeId(11)) + ); + + // Rx must create a receive record keyed by edge id. + assert!(rx.local_record(edge::EdgeId(7001)).is_some()); + + // Data-flow provisioning must not require remote actor addresses. + for record in [tx.local_record(edge::EdgeId(7001)).unwrap(), rx.local_record(edge::EdgeId(7001)).unwrap()] { + assert!(record.remote_actor_address.is_none()); + } +} + +// This proves lease events advance or fail only matching records, stale lease +// events for stopped records do not install state, and unused fresh leases after +// cancellation are released. +#[test] +fn lease_flow_matches_records_and_suppresses_stale_or_cancelled_leases() { + // Start one tx record and send a mismatched lease event. + let mut harness = new_establisher(); + harness.observe(edge::EdgeEvent::ProvisionTx(tx_provision())); + harness.observe(edge::EdgeEvent::RingLeased { + request_id: edge::LeaseRequestId(99), + ring_id: edge::RingId(8999), + layout: edge::RingLayout::test_layout(0), + }); + + // Mismatched lease must not install worker or pump state. + assert!(!harness.commands().iter().any(|command| { + matches!(command, edge::EdgeCommand::InstallWorkerRing { .. }) + })); + + // A matching rejection faults the record. + harness.observe(edge::EdgeEvent::RingLeaseRejected { + request_id: edge::LeaseRequestId(1), + reason: edge::RingLeaseRejection::CannotFit, + }); + assert!(harness.events().iter().any(|event| { + matches!( + event, + edge::EdgeLifecycleEvent::EdgeFaulted { + edge_id: edge::EdgeId(7001), + .. + } + ) + })); + + // Stop and then deliver a fresh lease; it must be released, not installed. + harness.observe(edge::EdgeEvent::StopEdge { + edge_id: edge::EdgeId(7001), + }); + harness.observe(edge::EdgeEvent::RingLeased { + request_id: edge::LeaseRequestId(1), + ring_id: edge::RingId(8001), + layout: edge::RingLayout::test_layout(0), + }); + assert!(harness.commands().iter().any(|command| { + matches!( + command, + edge::EdgeCommand::ReleaseArenaLease { + ring_id: edge::RingId(8001), + .. + } + ) + })); +} + +// This proves driver state is established only after worker or token endpoint +// ring installation succeeds, using the ObjectSpec and RingSpec from +// provisioning. +#[test] +fn worker_ring_install_precedes_driver_establishment_and_uses_provision_specs() { + // Provision and lease a tx edge. + let mut harness = new_establisher(); + let provision = tx_provision(); + harness.observe(edge::EdgeEvent::ProvisionTx(provision.clone())); + harness.observe(edge::EdgeEvent::RingLeased { + request_id: edge::LeaseRequestId(1), + ring_id: edge::RingId(8001), + layout: edge::RingLayout::test_layout(0), + }); + + // Before ring installation, no driver command may be issued. + assert!(!harness.commands().iter().any(|command| { + matches!(command, edge::EdgeCommand::EstablishSend { .. }) + })); + + // Matching ring installation advances establishment. + harness.observe(edge::EdgeEvent::RingInstalled { + edge_id: edge::EdgeId(7001), + ring_id: edge::RingId(8001), + }); + + // The worker install command must carry the provisioned object/ring specs. + assert!(harness.commands().iter().any(|command| { + matches!( + command, + edge::EdgeCommand::InstallWorkerRing { + object_spec, + ring_spec, + .. + } if *object_spec == provision.object_spec && *ring_spec == provision.ring_spec + ) + })); + + // Now the driver may be established. + assert!(harness.commands().iter().any(|command| { + matches!(command, edge::EdgeCommand::EstablishSend { edge_id: edge::EdgeId(7001), .. }) + })); +} + +// This proves send and receive driver establishment uses the correct public +// arguments, and DriverEdgeReady is the readiness boundary for the local actor. +#[test] +fn driver_ready_marks_local_edge_actor_ready() { + // Drive tx through lease and ring install. + let mut tx = leased_and_installed_tx(); + + // EstablishSend must include edge id, consumer node id, and local layout. + assert!(tx.commands().iter().any(|command| { + matches!( + command, + edge::EdgeCommand::EstablishSend { + edge_id: edge::EdgeId(7001), + consumer_node_id: edge::NodeId(11), + layout, + } if *layout == edge::RingLayout::test_layout(0) + ) + })); + + // The edge is not ready until DriverEdgeReady arrives. + assert!(!tx.events().iter().any(|event| { + matches!(event, edge::EdgeLifecycleEvent::EdgeReady { .. }) + })); + tx.observe(edge::EdgeEvent::DriverEdgeReady { + edge_id: edge::EdgeId(7001), + }); + assert!(tx.events().iter().any(|event| { + matches!( + event, + edge::EdgeLifecycleEvent::EdgeReady { + edge_id: edge::EdgeId(7001), + .. + } + ) + })); + + // After readiness, stream and pump behavior belongs to the driver; the + // establisher should not emit hot-path byte commands. + assert!(!tx.commands().iter().any(|command| { + matches!(command, edge::EdgeCommand::CopyHotPathBytes { .. }) + })); +} + +// This proves StopEdge cancels queued leases, stops pumps, uninstalls worker +// rings, releases arena lease only after proof, and makes Stopped terminal. +#[test] +fn stop_edge_tears_down_local_state_and_terminal_stopped_ignores_late_events() { + // Drive an edge to ready. + let mut harness = leased_and_installed_tx(); + harness.observe(edge::EdgeEvent::DriverEdgeReady { + edge_id: edge::EdgeId(7001), + }); + + // Stop the edge. + harness.observe(edge::EdgeEvent::StopEdge { + edge_id: edge::EdgeId(7001), + }); + + // Stop commands must cover queued lease, pump, and worker ring cleanup. + assert!(harness.commands().iter().any(|command| { + matches!(command, edge::EdgeCommand::CancelQueuedLease { .. }) + })); + assert!(harness.commands().iter().any(|command| { + matches!(command, edge::EdgeCommand::StopPump { .. }) + })); + assert!(harness.commands().iter().any(|command| { + matches!(command, edge::EdgeCommand::UninstallWorkerRing { .. }) + })); + + // Arena release is withheld until quiescence proof arrives. + assert!(!harness.commands().iter().any(|command| { + matches!(command, edge::EdgeCommand::ReleaseArenaLease { .. }) + })); + harness.observe(edge::EdgeEvent::QuiescenceProven { + ring_id: edge::RingId(8001), + }); + assert!(harness.commands().iter().any(|command| { + matches!( + command, + edge::EdgeCommand::ReleaseArenaLease { + ring_id: edge::RingId(8001), + .. + } + ) + })); + + // Stopped is terminal: later stale events cannot revive readiness. + harness.observe(edge::EdgeEvent::Stopped { + edge_id: edge::EdgeId(7001), + }); + let ready_before = harness + .events() + .iter() + .filter(|event| matches!(event, edge::EdgeLifecycleEvent::EdgeReady { .. })) + .count(); + harness.observe(edge::EdgeEvent::DriverEdgeReady { + edge_id: edge::EdgeId(7001), + }); + let ready_after = harness + .events() + .iter() + .filter(|event| matches!(event, edge::EdgeLifecycleEvent::EdgeReady { .. })) + .count(); + assert_eq!(ready_after, ready_before); +} diff --git a/tests/mvp_system/gpu_worker_ctl_guarantees.rs b/tests/mvp_system/gpu_worker_ctl_guarantees.rs new file mode 100644 index 0000000..8a7f425 --- /dev/null +++ b/tests/mvp_system/gpu_worker_ctl_guarantees.rs @@ -0,0 +1,266 @@ +//! Black-box contract tests for MVP GpuWorkerCtl behavior. +//! +//! These tests intentionally know only the public controller surface: +//! +//! - start, process, stdout event, actor command, crash, restart, and shutdown +//! observations in +//! - serialized worker commands, routed events, faults, and stopped events out +//! +//! They assert the guarantees in +//! `specs/mvp_system/gpu_worker_ctl_contract.md`. + +use mvp_system::gpu_worker_ctl as ctl; + +// A test controller config names one worker process boundary. It does not grant +// tests access to child-process internals or Python implementation details. +fn worker_config() -> ctl::WorkerConfig { + ctl::WorkerConfig { + node_id: ctl::NodeId(10), + arena_env: ctl::ArenaEnv::test_default(), + initialization_timeout_ms: 1_000, + } +} + +// The harness uses a fake process adapter at the public boundary. It lets tests +// observe serialized commands and routed events without inspecting controller +// state or private supervision tasks. +fn new_controller() -> ctl::GpuWorkerCtlHarness { + ctl::GpuWorkerCtlHarness::new(worker_config()) +} + +// This helper starts the worker and drives it to running through public process +// observations. Tests that need a running worker all use the same path. +fn running_controller() -> ctl::GpuWorkerCtlHarness { + let mut harness = new_controller(); + harness.observe(ctl::WorkerCtlEvent::StartWorker); + harness.observe(ctl::WorkerCtlEvent::ProcessStarted { + pid: ctl::ProcessId(1234), + }); + harness.observe(ctl::WorkerCtlEvent::WorkerReady { + generation: ctl::WorkerGeneration(1), + }); + harness +} + +// The command fixture carries identities and handles but no payload bytes. It +// exercises all command families that should be serialized only while running. +fn current_generation_commands() -> Vec { + vec![ + ctl::ActorCommand::InstallRing { + generation: ctl::WorkerGeneration(1), + ring_id: ctl::RingId(8001), + }, + ctl::ActorCommand::ExecuteStep { + generation: ctl::WorkerGeneration(1), + step_id: ctl::StepId(9001), + input: ctl::DeviceHandle::new(ctl::WorkerGeneration(1), 42), + }, + ctl::ActorCommand::ReleaseDeviceObject { + generation: ctl::WorkerGeneration(1), + handle: ctl::DeviceHandle::new(ctl::WorkerGeneration(1), 42), + }, + ] +} + +// This proves StartWorker spawns the process boundary, then sends +// InitializeWorker, and WorkerReady moves the controller to running. Fatal, +// process exit, and timeout move it to failed or crashed. +#[test] +fn worker_lifecycle_runs_start_initialize_ready_and_fault_paths() { + // Start the worker process. + let mut harness = new_controller(); + harness.observe(ctl::WorkerCtlEvent::StartWorker); + assert!(harness.commands().iter().any(|command| { + matches!(command, ctl::WorkerCtlCommand::SpawnProcessActor { .. }) + })); + + // Process start triggers InitializeWorker. + harness.observe(ctl::WorkerCtlEvent::ProcessStarted { + pid: ctl::ProcessId(1234), + }); + assert!(harness.serialized_worker_commands().iter().any(|command| { + matches!(command, ctl::WorkerCommand::InitializeWorker { .. }) + })); + + // WorkerReady emits a running lifecycle event. + harness.observe(ctl::WorkerCtlEvent::WorkerReady { + generation: ctl::WorkerGeneration(1), + }); + assert!(harness.events().iter().any(|event| { + matches!(event, ctl::WorkerCtlOut::WorkerRunning { generation: ctl::WorkerGeneration(1) }) + })); + + // Initialization timeout in a fresh controller is terminal failure. + let mut timed_out = new_controller(); + timed_out.observe(ctl::WorkerCtlEvent::StartWorker); + timed_out.advance_time_ms(1_001); + assert!(timed_out.events().iter().any(|event| { + matches!(event, ctl::WorkerCtlOut::WorkerFailed { reason: ctl::WorkerFailure::InitializationTimeout, .. }) + })); +} + +// This proves valid actor commands are serialized only in running state, only +// for the current generation, and never carry payload bytes. +#[test] +fn command_routing_requires_running_current_generation_and_is_payload_free() { + // Drive the controller to running. + let mut harness = running_controller(); + + // Send every current-generation command. + for command in current_generation_commands() { + harness.observe(ctl::WorkerCtlEvent::ActorCommand(command)); + } + + // All valid commands are serialized to the worker. + assert_eq!(harness.serialized_worker_commands().len(), current_generation_commands().len() + 1); + + // Serialized commands must not contain payload bytes. + for command in harness.serialized_worker_commands() { + match command { + ctl::WorkerCommand::InitializeWorker { .. } + | ctl::WorkerCommand::InstallRing { .. } + | ctl::WorkerCommand::ExecuteStep { .. } + | ctl::WorkerCommand::ReleaseDeviceObject { .. } + | ctl::WorkerCommand::ShutdownWorker { .. } => {} + ctl::WorkerCommand::PayloadBytes { .. } => { + panic!("worker control command carried payload bytes: {command:?}") + } + } + } + + // Old-generation handles must be rejected after restart. + harness.observe(ctl::WorkerCtlEvent::RestartRequested); + harness.observe(ctl::WorkerCtlEvent::ProcessStarted { + pid: ctl::ProcessId(1235), + }); + harness.observe(ctl::WorkerCtlEvent::WorkerReady { + generation: ctl::WorkerGeneration(2), + }); + harness.observe(ctl::WorkerCtlEvent::ActorCommand(ctl::ActorCommand::ExecuteStep { + generation: ctl::WorkerGeneration(1), + step_id: ctl::StepId(9002), + input: ctl::DeviceHandle::new(ctl::WorkerGeneration(1), 42), + })); + assert!(harness.events().iter().any(|event| { + matches!(event, ctl::WorkerCtlOut::CommandRejected { reason: ctl::CommandRejection::OldGenerationHandle, .. }) + })); +} + +// This proves worker events parsed from stdout route to the correct local +// control component. +#[test] +fn parsed_worker_events_route_to_their_control_owners() { + // Start from a running worker. + let mut harness = running_controller(); + + // Deliver each worker event family through stdout parsing. + harness.observe(ctl::WorkerCtlEvent::StdoutEvent(ctl::WorkerEvent::RingInstalled { + ring_id: ctl::RingId(8001), + })); + harness.observe(ctl::WorkerCtlEvent::StdoutEvent(ctl::WorkerEvent::ObjectLoaded { + object_id: ctl::ObjectId(9000), + sequence: 0, + })); + harness.observe(ctl::WorkerCtlEvent::StdoutEvent(ctl::WorkerEvent::ObjectProduced { + object_id: ctl::ObjectId(9001), + sequence: 0, + })); + harness.observe(ctl::WorkerCtlEvent::StdoutEvent(ctl::WorkerEvent::StepCompleted { + step_id: ctl::StepId(77), + })); + harness.observe(ctl::WorkerCtlEvent::StdoutEvent(ctl::WorkerEvent::RingReadable { + ring_id: ctl::RingId(8001), + })); + + // Routing is proven by destination commands/events, not private dispatch + // tables. + assert!(harness.routed().iter().any(|route| matches!(route, ctl::RoutedEvent::ToEdgeEstablisher(_)))); + assert!(harness.routed().iter().any(|route| matches!(route, ctl::RoutedEvent::ToRxOrRole(_)))); + assert!(harness.routed().iter().any(|route| matches!(route, ctl::RoutedEvent::ToTxOrRole(_)))); + assert!(harness.routed().iter().any(|route| matches!(route, ctl::RoutedEvent::ToStageController(_)))); + assert!(harness.routed().iter().any(|route| matches!(route, ctl::RoutedEvent::ToDriverOrWorkerSide(_)))); +} + +// This proves worker crash invalidates old handles, roles, rings, in-flight +// steps, synthesizes ring faults, asks the driver to stop pumps, and increments +// generation on restart. +#[test] +fn crash_invalidates_generation_state_and_fans_out_faults() { + // Install one ring and start one step in generation 1. + let mut harness = running_controller(); + harness.observe(ctl::WorkerCtlEvent::ActorCommand(ctl::ActorCommand::InstallRing { + generation: ctl::WorkerGeneration(1), + ring_id: ctl::RingId(8001), + })); + harness.observe(ctl::WorkerCtlEvent::ActorCommand(ctl::ActorCommand::ExecuteStep { + generation: ctl::WorkerGeneration(1), + step_id: ctl::StepId(9001), + input: ctl::DeviceHandle::new(ctl::WorkerGeneration(1), 42), + })); + + // Crash the worker process. + harness.observe(ctl::WorkerCtlEvent::ProcessExited { + status: ctl::ExitStatus::Signal(9), + }); + + // Installed rings fault and affected pumps are stopped. + assert!(harness.events().iter().any(|event| { + matches!(event, ctl::WorkerCtlOut::RingFaulted { ring_id: ctl::RingId(8001), .. }) + })); + assert!(harness.commands().iter().any(|command| { + matches!(command, ctl::WorkerCtlCommand::StopDriverPump { ring_id: ctl::RingId(8001), .. }) + })); + + // Restart increments worker generation. + harness.observe(ctl::WorkerCtlEvent::RestartRequested); + harness.observe(ctl::WorkerCtlEvent::ProcessStarted { + pid: ctl::ProcessId(1235), + }); + harness.observe(ctl::WorkerCtlEvent::WorkerReady { + generation: ctl::WorkerGeneration(2), + }); + assert_eq!(harness.current_generation(), ctl::WorkerGeneration(2)); +} + +// This proves graceful shutdown sends ShutdownWorker, observes WorkerStopped +// before terminal stopped, and classifies installed rings according to the +// observed worker/process outcome. +#[test] +fn graceful_shutdown_sends_worker_shutdown_and_reaches_terminal_stopped() { + // Start from a running worker with an installed ring. + let mut harness = running_controller(); + harness.observe(ctl::WorkerCtlEvent::ActorCommand(ctl::ActorCommand::InstallRing { + generation: ctl::WorkerGeneration(1), + ring_id: ctl::RingId(8001), + })); + + // Request graceful shutdown. + harness.observe(ctl::WorkerCtlEvent::ShutdownRequested); + assert!(harness.serialized_worker_commands().iter().any(|command| { + matches!(command, ctl::WorkerCommand::ShutdownWorker { .. }) + })); + + // WorkerStopped must precede terminal stopped. + harness.observe(ctl::WorkerCtlEvent::WorkerStopped { + generation: ctl::WorkerGeneration(1), + }); + harness.observe(ctl::WorkerCtlEvent::ProcessExited { + status: ctl::ExitStatus::Code(0), + }); + let worker_stopped_pos = harness + .events() + .iter() + .position(|event| matches!(event, ctl::WorkerCtlOut::WorkerStopped { .. })) + .expect("WorkerStopped must be observed"); + let terminal_pos = harness + .events() + .iter() + .position(|event| matches!(event, ctl::WorkerCtlOut::TerminalStopped { .. })) + .expect("terminal stopped must be observed"); + assert!(worker_stopped_pos < terminal_pos); + + // Rings are marked quiesced on graceful stop. + assert!(harness.events().iter().any(|event| { + matches!(event, ctl::WorkerCtlOut::RingQuiesced { ring_id: ctl::RingId(8001), .. }) + })); +} diff --git a/tests/mvp_system/gpu_worker_egress_producer_guarantees.rs b/tests/mvp_system/gpu_worker_egress_producer_guarantees.rs new file mode 100644 index 0000000..3014c45 --- /dev/null +++ b/tests/mvp_system/gpu_worker_egress_producer_guarantees.rs @@ -0,0 +1,253 @@ +//! Black-box contract tests for MVP GPU worker egress production. +//! +//! These tests intentionally know only the public worker egress surface: +//! +//! - `InstallRing`, `ExecuteStep` output bindings, device-copy outcomes, +//! backpressure, and shutdown events in +//! - committed ring bytes, cursor publication, `ObjectProduced`, +//! `StepCompleted`, and step failures out +//! +//! They assert the guarantees in +//! `specs/mvp_system/gpu_worker_egress_producer_contract.md`. + +use mvp_system::gpu_worker_egress_producer as egress; + +// A valid egress ring config supplies the edge object spec and current worker +// generation. The producer remains free to choose copy scheduling internally. +fn egress_ring() -> egress::InstallRing { + egress::InstallRing { + ring_id: egress::RingId(8002), + edge_id: egress::EdgeId(7002), + port_id: egress::PortId("out".into()), + direction: egress::RingDirection::Egress, + object_spec: egress::ObjectSpec { + max_extent: 16, + alignment: 4, + layout: egress::ObjectLayout::Token, + }, + generation: egress::WorkerGeneration(1), + } +} + +// The harness exposes egress ring writes and worker events, not private output +// queues, device kernels, or role internals. +fn new_producer() -> egress::EgressProducerHarness { + egress::EgressProducerHarness::new(egress::WorkerGeneration(1)) +} + +// This helper installs the output ring through the public worker command path. +fn installed_producer() -> egress::EgressProducerHarness { + let mut harness = new_producer(); + harness.observe(egress::WorkerEgressEvent::InstallRing(egress_ring())); + harness +} + +// A valid output binding carries object identity, sequence, extent, flags, and +// target ring. The worker must not invent these graph-visible facts. +fn output_binding(sequence: u64, extent: u64) -> egress::OutputBinding { + egress::OutputBinding { + ring_id: egress::RingId(8002), + object_id: egress::ObjectId(9000 + sequence), + sequence, + extent, + flags: egress::ObjectFlags::default(), + device_source: egress::DeviceHandle::new(egress::WorkerGeneration(1), 40 + sequence), + } +} + +// This proves egress production starts only after InstallRing, only for +// ExecuteStep output bindings naming that ring, and never invents object ids or +// sequence numbers. +#[test] +fn output_admission_requires_installed_ring_and_explicit_binding() { + // Execute before InstallRing must not write output. + let mut not_installed = new_producer(); + not_installed.observe(egress::WorkerEgressEvent::ExecuteStep { + step_id: egress::StepId(77), + outputs: vec![output_binding(0, 8)], + }); + assert_eq!(not_installed.committed_bytes(egress::RingId(8002)).len(), 0); + + // Install the ring and execute with a binding that names it. + let mut harness = installed_producer(); + harness.observe(egress::WorkerEgressEvent::ExecuteStep { + step_id: egress::StepId(77), + outputs: vec![output_binding(0, 8)], + }); + + // The pending output identity must match the binding exactly. + let pending = harness.pending_outputs(); + assert_eq!(pending[0].object_id, egress::ObjectId(9000)); + assert_eq!(pending[0].sequence, 0); + assert_eq!(pending[0].extent, 8); +} + +// This proves the worker creates a valid ObjectHeader from ObjectSpec, writes +// header bytes before payload bytes, advances commit only after valid header +// bytes, and emits readable wake after committed header bytes. +#[test] +fn header_is_written_and_committed_before_payload() { + // Start one egress output. + let mut harness = installed_producer(); + harness.observe(egress::WorkerEgressEvent::ExecuteStep { + step_id: egress::StepId(77), + outputs: vec![output_binding(0, 8)], + }); + + // Complete header production but not payload copy. + harness.observe(egress::WorkerEgressEvent::HeaderReady { + object_id: egress::ObjectId(9000), + }); + + // The committed prefix must decode as a header for the configured spec. + let committed = harness.committed_bytes(egress::RingId(8002)); + let header = egress::ObjectHeader::decode(committed).expect("header must decode"); + assert_eq!(header.object_id, egress::ObjectId(9000)); + assert_eq!(header.sequence, 0); + assert_eq!(header.extent, 8); + + // Payload bytes are not committed before the payload copy is valid. + assert_eq!(harness.committed_payload_bytes(egress::RingId(8002)), 0); + assert!(harness.wake_hints().iter().any(|wake| { + matches!(wake, egress::WakeHint::RingReadable { ring_id: egress::RingId(8002) }) + })); +} + +// This proves payload production copies exactly extent bytes from device to the +// egress ring, advances commit only after host bytes are valid, and blocks on +// egress backpressure without dropping ownership. +#[test] +fn payload_copy_is_exact_extent_and_respects_backpressure() { + // Start one output with extent 8. + let mut harness = installed_producer(); + harness.observe(egress::WorkerEgressEvent::ExecuteStep { + step_id: egress::StepId(77), + outputs: vec![output_binding(0, 8)], + }); + harness.observe(egress::WorkerEgressEvent::HeaderReady { + object_id: egress::ObjectId(9000), + }); + + // Backpressure prevents committing payload bytes. + harness.observe(egress::WorkerEgressEvent::EgressRingFull { + ring_id: egress::RingId(8002), + }); + harness.observe(egress::WorkerEgressEvent::DeviceToHostCopyCompleted { + object_id: egress::ObjectId(9000), + byte_count: 4, + }); + assert_eq!(harness.committed_payload_bytes(egress::RingId(8002)), 0); + + // Once writable, the full exact extent can commit. + harness.observe(egress::WorkerEgressEvent::RingWritable { + ring_id: egress::RingId(8002), + }); + harness.observe(egress::WorkerEgressEvent::DeviceToHostCopyCompleted { + object_id: egress::ObjectId(9000), + byte_count: 8, + }); + assert_eq!(harness.committed_payload_bytes(egress::RingId(8002)), 8); +} + +// This proves ObjectProduced is emitted after the full output object is +// committed, and StepCompleted is emitted only after all declared outputs are +// produced and role state updates are complete. +#[test] +fn object_produced_precedes_step_completed_after_all_outputs() { + // Execute a step with two outputs. + let mut harness = installed_producer(); + harness.observe(egress::WorkerEgressEvent::InstallRing(egress::InstallRing { + ring_id: egress::RingId(8003), + edge_id: egress::EdgeId(7003), + port_id: egress::PortId("out2".into()), + ..egress_ring() + })); + harness.observe(egress::WorkerEgressEvent::ExecuteStep { + step_id: egress::StepId(77), + outputs: vec![output_binding(0, 8), egress::OutputBinding { + ring_id: egress::RingId(8003), + object_id: egress::ObjectId(9100), + sequence: 0, + extent: 8, + flags: egress::ObjectFlags::default(), + device_source: egress::DeviceHandle::new(egress::WorkerGeneration(1), 55), + }], + }); + + // Produce only the first output and prove StepCompleted is still absent. + harness.complete_output(egress::ObjectId(9000)); + assert!(!harness.events().iter().any(|event| { + matches!(event, egress::WorkerEgressOut::StepCompleted { .. }) + })); + + // Produce the second output and complete role state update. + harness.complete_output(egress::ObjectId(9100)); + harness.observe(egress::WorkerEgressEvent::RoleStateUpdated { + step_id: egress::StepId(77), + }); + + // Both object-produced events precede StepCompleted. + let first_object_pos = harness + .events() + .iter() + .position(|event| matches!(event, egress::WorkerEgressOut::ObjectProduced { object_id: egress::ObjectId(9000), .. })) + .expect("first object produced"); + let second_object_pos = harness + .events() + .iter() + .position(|event| matches!(event, egress::WorkerEgressOut::ObjectProduced { object_id: egress::ObjectId(9100), .. })) + .expect("second object produced"); + let completed_pos = harness + .events() + .iter() + .position(|event| matches!(event, egress::WorkerEgressOut::StepCompleted { step_id: egress::StepId(77), .. })) + .expect("step completed"); + assert!(first_object_pos < completed_pos); + assert!(second_object_pos < completed_pos); +} + +// This proves invalid output ring, extent violation, device copy failure, and +// shutdown reject or abort egress production with visible step/ring faults. +#[test] +fn egress_faults_are_visible_and_suppress_success_events() { + // Invalid output ring fails the step. + let mut invalid_ring = installed_producer(); + invalid_ring.observe(egress::WorkerEgressEvent::ExecuteStep { + step_id: egress::StepId(77), + outputs: vec![egress::OutputBinding { + ring_id: egress::RingId(9999), + ..output_binding(0, 8) + }], + }); + assert!(invalid_ring.events().iter().any(|event| { + matches!(event, egress::WorkerEgressOut::StepFailed { reason: egress::StepFailureReason::InvalidOutputRing, .. }) + })); + + // Extent violation fails the step. + let mut bad_extent = installed_producer(); + bad_extent.observe(egress::WorkerEgressEvent::ExecuteStep { + step_id: egress::StepId(78), + outputs: vec![output_binding(0, 32)], + }); + assert!(bad_extent.events().iter().any(|event| { + matches!(event, egress::WorkerEgressOut::StepFailed { reason: egress::StepFailureReason::OutputExtentViolation, .. }) + })); + + // Copy failure faults the ring or fails the step, but must not emit + // ObjectProduced. + let mut copy_failed = installed_producer(); + copy_failed.observe(egress::WorkerEgressEvent::ExecuteStep { + step_id: egress::StepId(79), + outputs: vec![output_binding(0, 8)], + }); + copy_failed.observe(egress::WorkerEgressEvent::DeviceCopyFailed { + object_id: egress::ObjectId(9000), + }); + assert!(copy_failed.events().iter().any(|event| { + matches!(event, egress::WorkerEgressOut::StepFailed { .. }) + || matches!(event, egress::WorkerEgressOut::RingFault { .. }) + })); + assert!(!copy_failed.events().iter().any(|event| { + matches!(event, egress::WorkerEgressOut::ObjectProduced { .. }) + })); +} diff --git a/tests/mvp_system/gpu_worker_ingress_parser_guarantees.rs b/tests/mvp_system/gpu_worker_ingress_parser_guarantees.rs new file mode 100644 index 0000000..2bf536e --- /dev/null +++ b/tests/mvp_system/gpu_worker_ingress_parser_guarantees.rs @@ -0,0 +1,287 @@ +//! Black-box contract tests for MVP GPU worker ingress parsing. +//! +//! These tests intentionally know only the public worker ingress surface: +//! +//! - `InstallRing`, ring-readable wakes, committed ring bytes, EOF, and device +//! copy outcomes in +//! - cursor changes, `ObjectLoaded`, `ObjectFailed`, and `RingFault` out +//! +//! They assert the guarantees in +//! `specs/mvp_system/gpu_worker_ingress_parser_contract.md`. + +use mvp_system::gpu_worker_ingress_parser as ingress; + +// A valid ingress ring config supplies the edge object spec and current worker +// generation. The parser remains a black box behind ring helper operations. +fn ingress_ring() -> ingress::InstallRing { + ingress::InstallRing { + ring_id: ingress::RingId(8001), + edge_id: ingress::EdgeId(7001), + port_id: ingress::PortId("in".into()), + direction: ingress::RingDirection::Ingress, + object_spec: ingress::ObjectSpec { + max_extent: 16, + alignment: 4, + layout: ingress::ObjectLayout::Token, + }, + generation: ingress::WorkerGeneration(1), + } +} + +// The harness observes only worker-facing ring and device events. Tests do not +// parse private parser state or device allocation internals. +fn new_parser() -> ingress::IngressParserHarness { + ingress::IngressParserHarness::new(ingress::WorkerGeneration(1)) +} + +// A valid object record is constructed through the public helper encoder so the +// test proves parser behavior rather than hard-coded header bytes. +fn valid_record(sequence: u64, extent: u64) -> Vec { + ingress::ObjectRecordBuilder::new(ingress_ring().object_spec) + .object_id(ingress::ObjectId(9000 + sequence)) + .sequence(sequence) + .extent(extent) + .payload(vec![7; extent as usize]) + .encode() +} + +// This helper installs the ingress ring through the public command path. +fn installed_parser() -> ingress::IngressParserHarness { + let mut harness = new_parser(); + harness.observe(ingress::WorkerIngressEvent::InstallRing(ingress_ring())); + harness +} + +// This proves ingress parsing starts only after InstallRing, reloads cursors +// after RingReadable, consumes committed bytes only, and does not read +// uncommitted bytes. +#[test] +fn parser_reads_only_committed_bytes_after_ingress_ring_install() { + // Write committed bytes before install; the parser must not consume them. + let mut harness = new_parser(); + harness.write_committed_bytes(ingress::RingId(8001), valid_record(0, 8)); + harness.observe(ingress::WorkerIngressEvent::RingReadable { + ring_id: ingress::RingId(8001), + }); + assert_eq!(harness.consume_cursor(ingress::RingId(8001)), 0); + + // Install the ingress ring and make committed bytes readable. + harness.observe(ingress::WorkerIngressEvent::InstallRing(ingress_ring())); + harness.observe(ingress::WorkerIngressEvent::RingReadable { + ring_id: ingress::RingId(8001), + }); + assert!(harness.cursor_reload_count(ingress::RingId(8001)) > 0); + + // Uncommitted bytes must not be read. + harness.write_uncommitted_bytes(ingress::RingId(8001), vec![1, 2, 3, 4]); + let before = harness.consume_cursor(ingress::RingId(8001)); + harness.observe(ingress::WorkerIngressEvent::RingReadable { + ring_id: ingress::RingId(8001), + }); + assert_eq!(harness.consume_cursor(ingress::RingId(8001)), before); +} + +// This proves malformed or unsupported headers, max-extent violations, +// alignment/layout violations, and sequence violations emit ObjectFailed. +#[test] +fn header_validation_rejects_invalid_object_records() { + // Each record corrupts one header fact in an installed parser. + let cases = vec![ + ( + ingress::ObjectRecordBuilder::new(ingress_ring().object_spec) + .unsupported_magic() + .encode(), + ingress::ObjectFailureReason::UnsupportedMagic, + ), + ( + ingress::ObjectRecordBuilder::new(ingress_ring().object_spec) + .unsupported_version() + .encode(), + ingress::ObjectFailureReason::UnsupportedVersion, + ), + ( + ingress::ObjectRecordBuilder::new(ingress_ring().object_spec) + .malformed_header_length() + .encode(), + ingress::ObjectFailureReason::MalformedHeaderLength, + ), + ( + ingress::ObjectRecordBuilder::new(ingress_ring().object_spec) + .extent(32) + .payload(vec![0; 32]) + .encode(), + ingress::ObjectFailureReason::ExtentExceedsMax, + ), + ( + ingress::ObjectRecordBuilder::new(ingress_ring().object_spec) + .extent(6) + .payload(vec![0; 6]) + .encode(), + ingress::ObjectFailureReason::ExtentAlignmentViolation, + ), + ]; + + for (record, expected_reason) in cases { + // Use a fresh installed parser for each invalid object. + let mut harness = installed_parser(); + harness.write_committed_bytes(ingress::RingId(8001), record); + harness.observe(ingress::WorkerIngressEvent::RingReadable { + ring_id: ingress::RingId(8001), + }); + + // The parser must emit a typed object failure. + assert!(harness.events().iter().any(|event| { + matches!( + event, + ingress::WorkerIngressOut::ObjectFailed { + reason, + .. + } if *reason == expected_reason + ) + })); + } +} + +// This proves the parser copies exactly extent payload bytes to device memory, +// advances consume only after bytes are safe to release, can stream objects +// larger than the ring through bounded spans, and faults EOF before full payload. +#[test] +fn payload_loading_is_exact_extent_and_release_after_copy_completion() { + // Install the parser and provide a valid record. + let mut harness = installed_parser(); + harness.write_committed_bytes(ingress::RingId(8001), valid_record(0, 8)); + harness.observe(ingress::WorkerIngressEvent::RingReadable { + ring_id: ingress::RingId(8001), + }); + + // Before device copy completion, consume must not advance past safe bytes. + let before_copy_complete = harness.consume_cursor(ingress::RingId(8001)); + assert_eq!(before_copy_complete, 0); + + // Complete the device copy and prove exact extent was copied. + harness.observe(ingress::WorkerIngressEvent::DeviceCopyCompleted { + object_id: ingress::ObjectId(9000), + byte_count: 8, + }); + assert_eq!( + harness.device_copy_log().last().unwrap().byte_count, + 8 + ); + assert!(harness.consume_cursor(ingress::RingId(8001)) > before_copy_complete); + + // EOF mid-object faults the object. + let mut eof = installed_parser(); + eof.write_committed_bytes( + ingress::RingId(8001), + ingress::ObjectRecordBuilder::new(ingress_ring().object_spec) + .object_id(ingress::ObjectId(9010)) + .sequence(0) + .extent(12) + .partial_payload(vec![1, 2, 3]) + .encode(), + ); + eof.observe(ingress::WorkerIngressEvent::Eof { + ring_id: ingress::RingId(8001), + }); + assert!(eof.events().iter().any(|event| { + matches!( + event, + ingress::WorkerIngressOut::ObjectFailed { + reason: ingress::ObjectFailureReason::EofBeforeFullPayload, + .. + } + ) + })); +} + +// This proves ObjectLoaded is emitted only after valid header, exact extent +// copy, copy completion, device handle creation, and with current-generation +// identity fields. +#[test] +fn object_loaded_requires_complete_valid_object_and_current_handle() { + // Install and parse one valid record. + let mut harness = installed_parser(); + harness.write_committed_bytes(ingress::RingId(8001), valid_record(0, 8)); + harness.observe(ingress::WorkerIngressEvent::RingReadable { + ring_id: ingress::RingId(8001), + }); + + // No ObjectLoaded may appear before device copy completion and handle + // creation. + assert!(!harness.events().iter().any(|event| { + matches!(event, ingress::WorkerIngressOut::ObjectLoaded { .. }) + })); + + // Complete device work. + harness.observe(ingress::WorkerIngressEvent::DeviceCopyCompleted { + object_id: ingress::ObjectId(9000), + byte_count: 8, + }); + harness.observe(ingress::WorkerIngressEvent::DeviceHandleCreated { + object_id: ingress::ObjectId(9000), + handle: ingress::DeviceHandle::new(ingress::WorkerGeneration(1), 42), + }); + + // ObjectLoaded includes the required identities and current generation. + assert!(harness.events().iter().any(|event| { + matches!( + event, + ingress::WorkerIngressOut::ObjectLoaded { + ring_id: ingress::RingId(8001), + edge_id: ingress::EdgeId(7001), + port_id, + object_id: ingress::ObjectId(9000), + sequence: 0, + extent: 8, + handle, + } if port_id == &ingress::PortId("in".into()) + && handle.generation == ingress::WorkerGeneration(1) + ) + })); +} + +// This proves parser faults cover sequence violations and device allocation or +// copy failures, and after RingFault the worker stops consuming until uninstall. +#[test] +fn sequence_and_device_failures_fault_and_ring_fault_stops_consumption() { + // Parse sequence 0 successfully. + let mut harness = installed_parser(); + harness.write_committed_bytes(ingress::RingId(8001), valid_record(0, 8)); + harness.observe(ingress::WorkerIngressEvent::RingReadable { + ring_id: ingress::RingId(8001), + }); + harness.observe(ingress::WorkerIngressEvent::DeviceCopyCompleted { + object_id: ingress::ObjectId(9000), + byte_count: 8, + }); + harness.observe(ingress::WorkerIngressEvent::DeviceHandleCreated { + object_id: ingress::ObjectId(9000), + handle: ingress::DeviceHandle::new(ingress::WorkerGeneration(1), 42), + }); + + // Repeating sequence 0 violates sequence safety. + harness.write_committed_bytes(ingress::RingId(8001), valid_record(0, 8)); + harness.observe(ingress::WorkerIngressEvent::RingReadable { + ring_id: ingress::RingId(8001), + }); + assert!(harness.events().iter().any(|event| { + matches!( + event, + ingress::WorkerIngressOut::ObjectFailed { + reason: ingress::ObjectFailureReason::SequenceViolation, + .. + } + ) + })); + + // RingFault stops consumption until uninstall. + let before_fault = harness.consume_cursor(ingress::RingId(8001)); + harness.observe(ingress::WorkerIngressEvent::RingFault { + ring_id: ingress::RingId(8001), + }); + harness.write_committed_bytes(ingress::RingId(8001), valid_record(1, 8)); + harness.observe(ingress::WorkerIngressEvent::RingReadable { + ring_id: ingress::RingId(8001), + }); + assert_eq!(harness.consume_cursor(ingress::RingId(8001)), before_fault); +} diff --git a/tests/mvp_system/gpu_worker_process_adapter_guarantees.rs b/tests/mvp_system/gpu_worker_process_adapter_guarantees.rs new file mode 100644 index 0000000..5a58305 --- /dev/null +++ b/tests/mvp_system/gpu_worker_process_adapter_guarantees.rs @@ -0,0 +1,200 @@ +//! Black-box contract tests for the MVP GPU worker process adapter. +//! +//! These tests intentionally know only the public stdin/stdout adapter surface: +//! +//! - command JSON lines written to stdin +//! - stdout JSON event lines and stderr diagnostic lines read back +//! - adapter events and process faults out +//! +//! They assert the guarantees in +//! `specs/mvp_system/gpu_worker_process_adapter_contract.md`. + +use mvp_system::gpu_worker_process_adapter as adapter; + +// The adapter config supplies arena environment variables and an ABI version. +// Tests do not assume how Python maps the arena or initializes tinygrad. +fn adapter_config() -> adapter::AdapterConfig { + adapter::AdapterConfig { + arena_fd: adapter::ArenaFd(3), + arena_bytes: 4096, + helper_abi_version: adapter::HelperAbiVersion(1), + } +} + +// The harness exposes fake stdin/stdout/stderr lines at the process boundary. +// It does not expose worker internals or controller routing. +fn new_adapter() -> adapter::ProcessAdapterHarness { + adapter::ProcessAdapterHarness::new(adapter_config()) +} + +// Valid control commands carry identities and handles, never payload bytes. +// The command list exercises framing without depending on a specific JSON key +// order. +fn valid_commands() -> Vec { + vec![ + adapter::WorkerCommand::InitializeWorker { + helper_abi_version: adapter::HelperAbiVersion(1), + }, + adapter::WorkerCommand::InstallRing { + ring_id: adapter::RingId(8001), + }, + adapter::WorkerCommand::ExecuteStep { + step_id: adapter::StepId(9001), + }, + adapter::WorkerCommand::ReleaseDeviceObject { + handle: adapter::DeviceHandle(42), + }, + adapter::WorkerCommand::ShutdownWorker, + ] +} + +// This helper proves payload bytes are absent from JSON values by checking the +// public command/event representation before it crosses the process boundary. +fn assert_no_payload_bytes(value: &adapter::JsonLine) { + assert!( + !value.contains_key("payload") + && !value.contains_key("bytes") + && !value.contains_key("data"), + "control JSON carried payload bytes: {value:?}" + ); +} + +// This proves commands and events are one JSON object per line, stderr is only +// diagnostics, and payload bytes are forbidden in control JSON. +#[test] +fn control_stream_is_line_framed_json_without_payload_bytes() { + // Send every valid command through the adapter. + let mut harness = new_adapter(); + for command in valid_commands() { + harness.send_command(command); + } + + // Every stdin write must be exactly one JSON object followed by one newline. + for line in harness.stdin_lines() { + assert!(line.ends_with('\n')); + let parsed = adapter::JsonLine::parse(line).expect("stdin line must parse"); + assert!(parsed.is_object()); + assert_no_payload_bytes(&parsed); + } + + // Stdout events are also one JSON object per line. + harness.receive_stdout_line(r#"{"type":"WorkerReady","generation":1}"#); + let event = harness.events().last().expect("stdout event must parse"); + assert!(matches!(event, adapter::AdapterEvent::WorkerReady { .. })); + + // Stderr alone is diagnostic and does not define lifecycle state. + harness.receive_stderr_line("loading tinygrad backend"); + assert!(!harness.events().iter().any(|event| { + matches!(event, adapter::AdapterEvent::WorkerFatal { .. }) + })); +} + +// This proves worker initialization reads arena environment, waits for +// InitializeWorker, maps the arena, initializes the helper and backend, and then +// emits WorkerReady; failures emit WorkerFatal if possible and exit non-zero. +#[test] +fn initialization_order_is_env_initialize_map_helper_backend_ready() { + // Start the worker process with arena environment. + let mut harness = new_adapter(); + harness.start_worker_process(); + + // Before InitializeWorker, no mapping or backend initialization may occur. + assert!(!harness.worker_actions().iter().any(|action| { + matches!(action, adapter::WorkerAction::MapArena { .. }) + || matches!(action, adapter::WorkerAction::InitializeBackend { .. }) + })); + + // Send InitializeWorker and observe ordered worker actions. + harness.send_command(adapter::WorkerCommand::InitializeWorker { + helper_abi_version: adapter::HelperAbiVersion(1), + }); + let actions = harness.worker_actions(); + let env_pos = actions + .iter() + .position(|action| matches!(action, adapter::WorkerAction::ReadArenaEnvironment { .. })) + .expect("worker must read arena env"); + let map_pos = actions + .iter() + .position(|action| matches!(action, adapter::WorkerAction::MapArena { .. })) + .expect("worker must map arena"); + let helper_pos = actions + .iter() + .position(|action| matches!(action, adapter::WorkerAction::InitializeRingHelper { .. })) + .expect("worker must init helper"); + let backend_pos = actions + .iter() + .position(|action| matches!(action, adapter::WorkerAction::InitializeBackend { .. })) + .expect("worker must init backend"); + assert!(env_pos < map_pos && map_pos < helper_pos && helper_pos < backend_pos); + + // Successful initialization emits WorkerReady. + harness.receive_stdout_line(r#"{"type":"WorkerReady","generation":1}"#); + assert!(harness.events().iter().any(|event| { + matches!(event, adapter::AdapterEvent::WorkerReady { generation: adapter::WorkerGeneration(1) }) + })); + + // Initialization failure emits WorkerFatal if possible and exits non-zero. + let mut failed = new_adapter(); + failed.start_worker_process(); + failed.inject_initialization_failure(adapter::InitializationFailure::BackendUnavailable); + assert!(failed.events().iter().any(|event| { + matches!(event, adapter::AdapterEvent::WorkerFatal { .. }) + })); + assert_ne!(failed.exit_status(), Some(adapter::ExitStatus::Code(0))); +} + +// This proves invalid JSON, unknown event shapes, and unsupported helper ABI are +// worker/process faults, while stderr output alone is not lifecycle state. +#[test] +fn parsing_and_abi_errors_fault_the_worker_process() { + // Invalid JSON faults the adapter. + let mut invalid_json = new_adapter(); + invalid_json.receive_stdout_line("{not-json"); + assert!(invalid_json.events().iter().any(|event| { + matches!(event, adapter::AdapterEvent::ProcessFault { reason: adapter::ProcessFaultReason::InvalidJson, .. }) + })); + + // Unknown event shape faults the adapter. + let mut unknown = new_adapter(); + unknown.receive_stdout_line(r#"{"type":"NotAWorkerEvent"}"#); + assert!(unknown.events().iter().any(|event| { + matches!(event, adapter::AdapterEvent::ProcessFault { reason: adapter::ProcessFaultReason::UnknownEventShape, .. }) + })); + + // Unsupported helper ABI emits WorkerFatal. + let mut abi = new_adapter(); + abi.send_command(adapter::WorkerCommand::InitializeWorker { + helper_abi_version: adapter::HelperAbiVersion(999), + }); + assert!(abi.events().iter().any(|event| { + matches!(event, adapter::AdapterEvent::WorkerFatal { reason: adapter::WorkerFatalReason::UnsupportedHelperAbi, .. }) + })); +} + +// This proves command discipline: InstallRing, wake hints, ExecuteStep, +// ReleaseDeviceObject, and ShutdownWorker are accepted only as control messages +// and payload-bearing control JSON is rejected. +#[test] +fn command_discipline_rejects_payload_bearing_control_messages() { + // Valid control commands are accepted and framed. + let mut harness = new_adapter(); + for command in valid_commands() { + harness.send_command(command); + } + assert_eq!(harness.command_rejections().len(), 0); + + // Payload bytes in a control command are rejected at the adapter boundary. + harness.send_raw_json_command(r#"{"type":"ExecuteStep","step_id":1,"payload":[1,2,3]}"#); + assert!(harness.command_rejections().iter().any(|rejection| { + matches!(rejection.reason, adapter::CommandRejectionReason::PayloadBytesForbidden) + })); + + // Wake hints reload cursors; they do not carry byte ranges or credits. + harness.send_command(adapter::WorkerCommand::RingReadable { + ring_id: adapter::RingId(8001), + }); + let wake_line = harness.stdin_lines().last().expect("wake command must be written"); + let parsed = adapter::JsonLine::parse(wake_line).expect("wake line must parse"); + assert_no_payload_bytes(&parsed); + assert!(!parsed.contains_key("range") && !parsed.contains_key("credits")); +} diff --git a/tests/mvp_system/membership_pool_readiness_guarantees.rs b/tests/mvp_system/membership_pool_readiness_guarantees.rs new file mode 100644 index 0000000..fe268a5 --- /dev/null +++ b/tests/mvp_system/membership_pool_readiness_guarantees.rs @@ -0,0 +1,223 @@ +//! Black-box contract tests for MVP membership and pool readiness. +//! +//! These tests intentionally know only the public readiness-gate surface: +//! +//! - a fixed orchestrator-owned candidate pool +//! - membership, availability, and identity observations +//! - emitted pool readiness, planning, and run fault events +//! +//! They assert the guarantees in +//! `specs/mvp_system/membership_pool_readiness_contract.md`. + +use mvp_system::membership_pool_readiness as membership; + +// Three nodes are enough to prove all-node quantification without hiding behind +// a single-node special case. The orchestrator owns this pool; SWIM only reports +// liveness facts about it. +fn candidate_pool() -> Vec { + vec![ + membership::NodeId(10), + membership::NodeId(11), + membership::NodeId(12), + ] +} + +// The convergence window is short but non-zero so tests can prove PoolReady is +// not emitted at the instant the last fact arrives. +fn readiness_config() -> membership::ReadinessConfig { + membership::ReadinessConfig { + pool_id: membership::PoolId("mvp-test-pool".into()), + convergence_window_ms: 500, + } +} + +// This helper provides every required public fact for one node. Tests use it to +// build complete and deliberately incomplete pool views without observing any +// private readiness bookkeeping. +fn report_node_ready( + gate: &mut membership::ReadinessGateHarness, + node_id: membership::NodeId, +) { + gate.observe(membership::Observation::NodeKnown { node_id }); + gate.observe(membership::Observation::SwimLive { node_id }); + gate.observe(membership::Observation::NodeAvailable { node_id }); + gate.observe(membership::Observation::DataPlaneIdentityReady { node_id }); +} + +// The test harness records public events as a transcript. This helper counts a +// specific event so duplicate readiness or duplicate faults are visible at the +// contract boundary. +fn count_pool_ready(events: &[membership::ReadinessEvent]) -> usize { + events + .iter() + .filter(|event| matches!(event, membership::ReadinessEvent::PoolReady { .. })) + .count() +} + +// This proves PoolReady requires every candidate node, every required fact, no +// suspect or faulted candidates, and a stable convergence window. +#[test] +fn pool_ready_requires_complete_stable_candidate_pool() { + // Build the readiness gate with an orchestrator-owned candidate pool. + let pool = candidate_pool(); + let mut gate = membership::ReadinessGateHarness::new(readiness_config(), pool.clone()); + + // Report complete facts for every node except the final candidate. + report_node_ready(&mut gate, pool[0]); + report_node_ready(&mut gate, pool[1]); + gate.advance_time_ms(1_000); + + // A partial pool must not become ready, even after time passes. + assert_eq!(count_pool_ready(gate.events()), 0); + + // Report the final candidate, then prove the convergence window still + // matters by advancing less than the configured duration. + report_node_ready(&mut gate, pool[2]); + gate.advance_time_ms(499); + assert_eq!(count_pool_ready(gate.events()), 0); + + // After the window, PoolReady must describe exactly the intended pool. + gate.advance_time_ms(1); + let ready = gate + .events() + .iter() + .find_map(|event| match event { + membership::ReadinessEvent::PoolReady { pool } => Some(pool), + _ => None, + }) + .expect("complete stable pool must emit PoolReady"); + + // Compare as sets so ordering is not part of the behavioral contract. + let observed = ready.iter().copied().collect::>(); + let expected = pool.iter().copied().collect::>(); + assert_eq!(observed, expected); +} + +// This proves suspect, faulted, missing, or identity-less candidates prevent +// PoolReady rather than being silently ignored. +#[test] +fn unavailable_candidate_prevents_pool_ready() { + // Each case withholds or poisons one readiness fact for node 12. + let cases = [ + membership::Observation::SwimSuspect { + node_id: membership::NodeId(12), + }, + membership::Observation::NodeFaulted { + node_id: membership::NodeId(12), + }, + membership::Observation::DataPlaneIdentityMissing { + node_id: membership::NodeId(12), + }, + ]; + + for poisoned_fact in cases { + // Start each run from a fresh gate so one poisoned fact is responsible + // for the absence of readiness. + let pool = candidate_pool(); + let mut gate = membership::ReadinessGateHarness::new(readiness_config(), pool.clone()); + + // Make every candidate otherwise known and available. + for node_id in &pool { + report_node_ready(&mut gate, *node_id); + } + + // Poison the final candidate and wait past convergence. + gate.observe(poisoned_fact.clone()); + gate.advance_time_ms(1_000); + + // PoolReady must be absent because one required candidate is no longer + // available to the intended pool. + assert_eq!(count_pool_ready(gate.events()), 0); + } +} + +// This proves run planning is gated by PoolReady. SWIM observations alone do +// not start placement, and loss before a committed RunPlan prevents planning +// from racing ahead. +#[test] +fn planning_starts_only_after_pool_ready_and_stops_if_readiness_is_lost() { + // Create a gate and ask the orchestrator to plan before readiness. + let pool = candidate_pool(); + let mut gate = membership::ReadinessGateHarness::new(readiness_config(), pool.clone()); + gate.request_run_planning(membership::RunRequest::new(membership::RunId(7))); + + // With no PoolReady event, there must be no planning command. + assert!(!gate.commands().iter().any(|command| { + matches!(command, membership::ReadinessCommand::StartPlanning { .. }) + })); + + // Satisfy readiness, then immediately lose a required node before plan + // commit. The policy may wait or abort, but it must not commit placement. + for node_id in &pool { + report_node_ready(&mut gate, *node_id); + } + gate.advance_time_ms(500); + gate.observe(membership::Observation::SwimLost { node_id: pool[1] }); + + // No plan commitment command may be emitted from an unstable pool view. + assert!(!gate.commands().iter().any(|command| { + matches!(command, membership::ReadinessCommand::CommitRunPlan { .. }) + })); +} + +// This proves membership loss after provisioning is a run fault, not active +// re-placement. The MVP run keeps its committed topology until teardown. +#[test] +fn required_node_loss_after_provisioning_faults_without_replacement() { + // Drive the pool to ready and mark a run as provisioned from that pool. + let pool = candidate_pool(); + let mut gate = membership::ReadinessGateHarness::new(readiness_config(), pool.clone()); + for node_id in &pool { + report_node_ready(&mut gate, *node_id); + } + gate.advance_time_ms(500); + gate.observe(membership::Observation::RunProvisioned { + run_id: membership::RunId(7), + required_nodes: pool.clone(), + }); + + // Lose one required node during the active run. + gate.observe(membership::Observation::SwimLost { node_id: pool[1] }); + + // The run must fault with a membership reason. + assert!(gate.events().contains(&membership::ReadinessEvent::RunFaulted { + run_id: membership::RunId(7), + reason: membership::RunFaultReason::RequiredNodeLost { + node_id: pool[1], + }, + })); + + // Re-placement would violate the committed-plan authority boundary. + assert!(!gate.commands().iter().any(|command| { + matches!(command, membership::ReadinessCommand::RecomputePlacement { .. }) + })); +} + +// This proves SWIM authority is limited to membership and liveness. It may +// report facts, but it must not assign stages, edges, layers, or object specs. +#[test] +fn swim_observations_do_not_create_graph_assignments() { + // Feed a complete, stable membership view. + let pool = candidate_pool(); + let mut gate = membership::ReadinessGateHarness::new(readiness_config(), pool.clone()); + for node_id in &pool { + report_node_ready(&mut gate, *node_id); + } + gate.advance_time_ms(500); + + // Walk emitted commands and reject any graph-assignment side effect. + for command in gate.commands() { + match command { + membership::ReadinessCommand::EmitPoolReady { .. } + | membership::ReadinessCommand::StartPlanning { .. } + | membership::ReadinessCommand::WaitForStability { .. } + | membership::ReadinessCommand::AbortPendingRun { .. } => {} + membership::ReadinessCommand::AssignStage { .. } + | membership::ReadinessCommand::AssignEdge { .. } + | membership::ReadinessCommand::AssignLayerRange { .. } + | membership::ReadinessCommand::AssignObjectSpec { .. } => { + panic!("membership gate emitted graph assignment: {command:?}") + } + } + } +} diff --git a/tests/mvp_system/node_boot_lifecycle_guarantees.rs b/tests/mvp_system/node_boot_lifecycle_guarantees.rs new file mode 100644 index 0000000..f045752 --- /dev/null +++ b/tests/mvp_system/node_boot_lifecycle_guarantees.rs @@ -0,0 +1,204 @@ +//! Black-box contract tests for MVP node boot lifecycle. +//! +//! These tests intentionally know only the public node-boot surface: +//! +//! - `BootHarness::launch(config)` +//! - public resource outcomes delivered into the harness +//! - lifecycle events and provisioning admission observed from the harness +//! +//! They assert the guarantees in +//! `specs/mvp_system/node_boot_lifecycle_contract.md`. + +use mvp_system::node_boot_lifecycle as boot; + +// A complete boot config lets the tests focus on ordering and failure behavior. +// The concrete process, transport, arena, worker, and SWIM implementations are +// outside this contract; the harness exposes only their observable outcomes. +fn valid_boot_config() -> boot::BootConfig { + boot::BootConfig { + node_id: boot::NodeId(10), + intended_pool_id: boot::PoolId("mvp-test-pool".into()), + worker_policy: boot::WorkerStartPolicy::StartBeforeAvailable, + } +} + +// The required facts are deliberately listed as public resource outcomes. That +// makes the proof about the boot barrier rather than about any private boot FSM +// state or the order in which an implementation happens to initialize resources. +fn required_readiness_facts() -> Vec { + vec![ + boot::ResourceOutcome::RustProcessAlive, + boot::ResourceOutcome::RuntimeAcceptingControl, + boot::ResourceOutcome::StableNodeIdKnown(boot::NodeId(10)), + boot::ResourceOutcome::ArenaMapped, + boot::ResourceOutcome::GpuWorkerReady, + boot::ResourceOutcome::TransportEndpointBound, + boot::ResourceOutcome::SwimJoiningPool, + boot::ResourceOutcome::ProvisioningReceiverOpen, + ] +} + +// Fault injection is one-per-resource so each failure must be explained by the +// resource that failed. This prevents a broad "boot failed" bucket from hiding +// which readiness fact was not satisfied. +fn boot_fault_cases() -> Vec<(boot::ResourceOutcome, boot::BootFaultKind)> { + vec![ + ( + boot::ResourceOutcome::ArenaFault, + boot::BootFaultKind::ArenaConstructionFailed, + ), + ( + boot::ResourceOutcome::GpuWorkerFault, + boot::BootFaultKind::WorkerStartupFailed, + ), + ( + boot::ResourceOutcome::TransportEndpointFault, + boot::BootFaultKind::TransportEndpointFailed, + ), + ( + boot::ResourceOutcome::InvalidNodeIdentity, + boot::BootFaultKind::InvalidNodeIdentity, + ), + ] +} + +// This proves NodeAvailable is emitted only after every required local boot +// resource is ready, and that a boot attempt resolves to one availability event +// rather than a partial intermediate state. +#[test] +fn node_available_waits_for_all_required_readiness_facts() { + // Launch starts one public boot attempt in the intended pool. + let mut harness = boot::BootHarness::launch(valid_boot_config()); + + // Drive all but the final readiness fact and prove no prefix is sufficient. + let mut facts = required_readiness_facts(); + let final_fact = facts.pop().expect("fixture has a final fact"); + for fact in facts { + harness.observe(fact); + assert!(!harness.events().contains(&boot::LifecycleEvent::NodeAvailable { + node_id: boot::NodeId(10), + })); + } + + // Once the final required fact arrives, availability becomes observable. + harness.observe(final_fact); + + // Count the public event, not a private state bit; duplicate availability + // would make the boot attempt ambiguous to the orchestrator. + let available_count = harness + .events() + .iter() + .filter(|event| { + matches!( + event, + boot::LifecycleEvent::NodeAvailable { + node_id + } if *node_id == boot::NodeId(10) + ) + }) + .count(); + assert_eq!(available_count, 1); +} + +// This proves run provisioning cannot race ahead of node availability. The +// orchestrator can use NodeAvailable as the public admission point without +// inspecting node-local boot state. +#[test] +fn provisioning_is_rejected_until_node_available_is_observed() { + // Launch the node and create a provision message that would be valid after + // boot finishes. + let mut harness = boot::BootHarness::launch(valid_boot_config()); + let provision = boot::ProvisionRequest::for_node(boot::NodeId(10)); + + // Before readiness, provisioning must be rejected at the public boundary. + assert_eq!( + harness.try_accept_provisioning(provision.clone()), + Err(boot::ProvisioningAdmissionRejection::NodeNotAvailable) + ); + + // Complete readiness through observable facts only. + for fact in required_readiness_facts() { + harness.observe(fact); + } + + // After NodeAvailable, the same request may enter run setup. + assert_eq!(harness.try_accept_provisioning(provision), Ok(())); +} + +// This proves NodeAvailable is not overloaded with graph or weight meaning. A +// booted node is eligible for orchestration, but it has not selected itself for +// any stage, edge, layer range, role, or weight assignment. +#[test] +fn node_available_contains_no_run_assignment_or_weight_claims() { + // Complete node boot through the same public readiness facts as production. + let mut harness = boot::BootHarness::launch(valid_boot_config()); + for fact in required_readiness_facts() { + harness.observe(fact); + } + + // Inspect every emitted boot command. The proof is negative: boot may emit + // lifecycle facts, but it must not emit graph ownership commands. + for command in harness.commands() { + assert!(!matches!(command, boot::BootCommand::LoadWeights { .. })); + assert!(!matches!(command, boot::BootCommand::ConfigureRole { .. })); + assert!(!matches!(command, boot::BootCommand::EstablishEdge { .. })); + assert!(!matches!(command, boot::BootCommand::AssignStage { .. })); + } +} + +// This proves every boot-blocking resource failure emits a typed boot fault and +// keeps the node out of the candidate run pool. +#[test] +fn boot_resource_failure_faults_without_node_availability() { + // Each failure case runs in a fresh boot attempt so outcomes cannot mask + // each other. + for (failed_resource, expected_kind) in boot_fault_cases() { + let mut harness = boot::BootHarness::launch(valid_boot_config()); + + // Deliver the failed public resource outcome. + harness.observe(failed_resource); + + // The emitted fault must carry the stable reason enum for operators and + // tests; logs are not part of the contract. + assert!(harness.events().contains(&boot::LifecycleEvent::NodeFaulted { + node_id: boot::NodeId(10), + kind: expected_kind, + })); + + // A faulted boot attempt must not also become available. + assert!(!harness.events().iter().any(|event| { + matches!(event, boot::LifecycleEvent::NodeAvailable { .. }) + })); + + // The orchestrator-facing eligibility check must agree with the + // lifecycle transcript. + assert!(!harness.is_candidate_eligible(boot::NodeId(10))); + } +} + +// This proves node boot authority is local-resource authority only. The node +// may report boot facts, but stage, edge, layer, and object-spec assignment +// remain absent until the orchestrator provisions a committed plan. +#[test] +fn boot_never_self_assigns_run_topology() { + // Complete a successful boot attempt. + let mut harness = boot::BootHarness::launch(valid_boot_config()); + for fact in required_readiness_facts() { + harness.observe(fact); + } + + // Walk every emitted command and require it to stay in the boot domain. + for command in harness.commands() { + match command { + boot::BootCommand::AdvertiseLifecycle { .. } + | boot::BootCommand::JoinMembership { .. } + | boot::BootCommand::OpenProvisioningInbox { .. } => {} + boot::BootCommand::AssignStage { .. } + | boot::BootCommand::AssignLayerRange { .. } + | boot::BootCommand::AssignEdge { .. } + | boot::BootCommand::AssignObjectSpec { .. } => { + panic!("boot emitted graph assignment command: {command:?}") + } + } + } +} diff --git a/tests/mvp_system/observability_surface_guarantees.rs b/tests/mvp_system/observability_surface_guarantees.rs new file mode 100644 index 0000000..cf0c27f --- /dev/null +++ b/tests/mvp_system/observability_surface_guarantees.rs @@ -0,0 +1,275 @@ +//! Black-box contract tests for the MVP observability surface. +//! +//! These tests intentionally know only the public event stream surface: +//! +//! - structured lifecycle and fault events emitted by components +//! - event identities, reason enums, and ordering observed by subscribers +//! +//! They assert the guarantees in +//! `specs/mvp_system/observability_surface_contract.md`. + +use mvp_system::observability_surface as obs; + +// The trace fixture contains one successful run from boot through teardown. +// Tests use structured events only; logs, transport, storage, and batching stay +// outside the contract. +fn successful_run_trace() -> Vec { + obs::TraceBuilder::new(obs::RunId(7)) + .node_started(obs::NodeId(10)) + .node_available(obs::NodeId(10)) + .pool_ready(vec![obs::NodeId(10)]) + .run_planned() + .stage_provision_started(obs::StageIndex(0), obs::NodeId(10)) + .weights_download_started(obs::StageIndex(0)) + .weights_downloaded(obs::StageIndex(0)) + .weights_loaded(obs::StageIndex(0)) + .edge_provision_started(obs::EdgeId(7000)) + .edge_ready(obs::EdgeId(7000)) + .stage_ready(obs::StageIndex(0)) + .readiness_barrier_passed() + .prompt_injected(obs::Sequence(0)) + .object_loaded(obs::EdgeId(7000), obs::ObjectId(9000), obs::Sequence(0)) + .execute_step_started(obs::StepId(77)) + .object_produced(obs::EdgeId(7001), obs::ObjectId(9001), obs::Sequence(0)) + .step_completed(obs::StepId(77)) + .token_received(obs::ObjectId(9002), obs::Sequence(0)) + .run_completed() + .stop_run_sent(obs::StageIndex(0)) + .stage_stopped(obs::StageIndex(0)) + .run_torn_down() + .finish() +} + +// A fault trace gives the tests one stable reason enum and detecting component +// without relying on diagnostic log text. +fn fault_trace() -> Vec { + obs::TraceBuilder::new(obs::RunId(7)) + .node_started(obs::NodeId(10)) + .node_available(obs::NodeId(10)) + .pool_ready(vec![obs::NodeId(10)]) + .run_planned() + .stage_provision_started(obs::StageIndex(0), obs::NodeId(10)) + .stage_faulted( + obs::StageIndex(0), + obs::FaultReason::WorkerCrashed, + obs::Component::StageController, + ) + .run_faulted(obs::FaultReason::WorkerCrashed, obs::Component::StageController) + .stop_run_sent(obs::StageIndex(0)) + .stage_stopped(obs::StageIndex(0)) + .run_torn_down() + .finish() +} + +// This helper returns the position of an event kind in a trace. Ordering tests +// use positions so they prove causal ordering without depending on exact event +// batching or adjacent placement. +fn position_of_kind(events: &[obs::Event], kind: obs::EventKind) -> usize { + events + .iter() + .position(|event| event.kind() == kind) + .expect("event kind missing from trace") +} + +// This helper checks structured identity fields directly. If callers have to +// scrape logs to recover an id, the event fails this contract test. +fn assert_required_identity(event: &obs::Event) { + match event { + obs::Event::RunScoped { run_id, .. } => assert_eq!(*run_id, obs::RunId(7)), + obs::Event::NodeScoped { node_id, .. } => assert_eq!(*node_id, obs::NodeId(10)), + obs::Event::StageScoped { + run_id, + stage_index, + .. + } => { + assert_eq!(*run_id, obs::RunId(7)); + assert_eq!(*stage_index, obs::StageIndex(0)); + } + obs::Event::EdgeScoped { edge_id, .. } => { + assert!([obs::EdgeId(7000), obs::EdgeId(7001)].contains(edge_id)); + } + obs::Event::RingScoped { ring_id, .. } => assert_eq!(*ring_id, obs::RingId(8000)), + obs::Event::ObjectScoped { + object_id, + sequence, + .. + } => { + assert!([obs::ObjectId(9000), obs::ObjectId(9001), obs::ObjectId(9002)].contains(object_id)); + assert_eq!(*sequence, obs::Sequence(0)); + } + obs::Event::StepScoped { step_id, .. } => assert_eq!(*step_id, obs::StepId(77)), + obs::Event::WorkerScoped { + worker_generation, + .. + } => assert_eq!(*worker_generation, obs::WorkerGeneration(1)), + } +} + +// This proves required event identity fields are structured on the event itself +// for run, node, stage, edge, ring, object, step, and worker scopes. +#[test] +fn required_event_identity_is_structured_not_log_derived() { + // Build one trace that includes all required identity scopes. + let mut events = successful_run_trace(); + events.push(obs::Event::RingScoped { + kind: obs::EventKind::RingReadable, + ring_id: obs::RingId(8000), + component: obs::Component::SharedRingHelper, + }); + events.push(obs::Event::WorkerScoped { + kind: obs::EventKind::WorkerReady, + worker_generation: obs::WorkerGeneration(1), + component: obs::Component::GpuWorkerCtl, + }); + + // Every event exposes its required identity directly. + for event in &events { + assert_required_identity(event); + } +} + +// This proves the lifecycle event stream covers the required successful-run +// milestones from node boot through run teardown. +#[test] +fn lifecycle_events_cover_successful_run_milestones() { + // Build the successful trace. + let events = successful_run_trace(); + let observed = events + .iter() + .map(|event| event.kind()) + .collect::>(); + + // The required lifecycle event kinds must all be present, regardless of + // batching or transport. + let required = [ + obs::EventKind::NodeStarted, + obs::EventKind::NodeAvailable, + obs::EventKind::PoolReady, + obs::EventKind::RunPlanned, + obs::EventKind::StageProvisionStarted, + obs::EventKind::WeightsDownloadStarted, + obs::EventKind::WeightsDownloaded, + obs::EventKind::WeightsLoaded, + obs::EventKind::EdgeProvisionStarted, + obs::EventKind::EdgeReady, + obs::EventKind::StageReady, + obs::EventKind::ReadinessBarrierPassed, + obs::EventKind::PromptInjected, + obs::EventKind::ObjectLoaded, + obs::EventKind::ExecuteStepStarted, + obs::EventKind::ObjectProduced, + obs::EventKind::StepCompleted, + obs::EventKind::TokenReceived, + obs::EventKind::RunCompleted, + obs::EventKind::StopRunSent, + obs::EventKind::StageStopped, + obs::EventKind::RunTornDown, + ]; + for kind in required { + assert!(observed.contains(&kind), "missing lifecycle event: {kind:?}"); + } +} + +// This proves fault events carry a stable reason enum and detecting component, +// and tests do not need free-form log text to determine lifecycle progress. +#[test] +fn fault_events_include_stable_reason_and_detecting_component() { + // Build a fault trace with a stage-detected worker crash. + let events = fault_trace(); + + // The structured stage fault carries the reason and detector. + assert!(events.iter().any(|event| { + matches!( + event, + obs::Event::StageScoped { + kind: obs::EventKind::StageFaulted, + reason: Some(obs::FaultReason::WorkerCrashed), + component: obs::Component::StageController, + .. + } + ) + })); + + // The run fault carries the same structured reason. + assert!(events.iter().any(|event| { + matches!( + event, + obs::Event::RunScoped { + kind: obs::EventKind::RunFaulted, + reason: Some(obs::FaultReason::WorkerCrashed), + component: obs::Component::StageController, + .. + } + ) + })); + + // Logs may exist, but they are not required to classify progress. + assert!(!obs::requires_log_scraping(&events)); +} + +// This proves observability ordering reflects component contracts: +// prompt_injected follows readiness_barrier_passed, stage_ready follows local +// readiness, run_torn_down follows teardown completion, and terminal run outcome +// is emitted exactly once. +#[test] +fn event_ordering_reflects_component_contracts_and_one_terminal_outcome() { + // Build the successful trace. + let events = successful_run_trace(); + + // Prompt injection cannot precede the global barrier. + assert!( + position_of_kind(&events, obs::EventKind::ReadinessBarrierPassed) + < position_of_kind(&events, obs::EventKind::PromptInjected) + ); + + // StageReady cannot precede required local readiness facts. + assert!( + position_of_kind(&events, obs::EventKind::WeightsLoaded) + < position_of_kind(&events, obs::EventKind::StageReady) + ); + assert!( + position_of_kind(&events, obs::EventKind::EdgeReady) + < position_of_kind(&events, obs::EventKind::StageReady) + ); + + // RunTornDown cannot precede teardown completion. + assert!( + position_of_kind(&events, obs::EventKind::StageStopped) + < position_of_kind(&events, obs::EventKind::RunTornDown) + ); + + // Exactly one terminal run outcome is emitted. + let terminal_count = events + .iter() + .filter(|event| { + matches!(event.kind(), obs::EventKind::RunCompleted | obs::EventKind::RunFaulted) + }) + .count(); + assert_eq!(terminal_count, 1); +} + +// This proves observability tests are independent of transport, storage, and +// batching policy by asserting the same event facts after batching is changed. +#[test] +fn event_contract_survives_transport_storage_and_batching_policy() { + // Build the same logical events under two batching policies. + let unbatched = obs::EventSubscriberHarness::collect(successful_run_trace(), obs::Batching::None); + let batched = obs::EventSubscriberHarness::collect(successful_run_trace(), obs::Batching::Fixed(8)); + + // Flattened public event facts must match as an ordered stream. + let unbatched_kinds = unbatched + .flattened_events() + .iter() + .map(|event| event.kind()) + .collect::>(); + let batched_kinds = batched + .flattened_events() + .iter() + .map(|event| event.kind()) + .collect::>(); + assert_eq!(batched_kinds, unbatched_kinds); + + // Neither subscriber depends on transport or storage implementation names. + assert!(!unbatched.used_transport_specific_assertions()); + assert!(!batched.used_storage_specific_assertions()); +} diff --git a/tests/mvp_system/orchestrator_run_fsm_guarantees.rs b/tests/mvp_system/orchestrator_run_fsm_guarantees.rs new file mode 100644 index 0000000..750aa2e --- /dev/null +++ b/tests/mvp_system/orchestrator_run_fsm_guarantees.rs @@ -0,0 +1,413 @@ +//! Black-box contract tests for the MVP orchestrator run FSM. +//! +//! These tests intentionally know only the public orchestrator surface: +//! +//! - pool, plan, stage, endpoint, token, fault, stop, and timeout events in +//! - commands, lifecycle events, and terminal outcome out +//! +//! They assert the guarantees in +//! `specs/mvp_system/orchestrator_run_fsm_contract.md`. + +use mvp_system::orchestrator_run_fsm as fsm; + +// A three-stage plan proves multi-stage provisioning and readiness without +// making tests depend on any placement heuristic. The plan is already valid; +// these tests are about how the orchestrator consumes it. +fn committed_plan() -> fsm::RunPlan { + fsm::RunPlan::test_linear( + fsm::RunId(7), + vec![ + fsm::StageRef { + stage_index: 0, + node_id: fsm::NodeId(10), + }, + fsm::StageRef { + stage_index: 1, + node_id: fsm::NodeId(11), + }, + fsm::StageRef { + stage_index: 2, + node_id: fsm::NodeId(12), + }, + ], + ) +} + +// The harness is the black-box public boundary for the run FSM. It accepts +// observable events and records emitted commands/events; tests never inspect an +// internal FSM enum or private readiness counter. +fn new_run() -> fsm::OrchestratorHarness { + fsm::OrchestratorHarness::new(fsm::RunConfig { + run_id: fsm::RunId(7), + max_tokens: 4, + prompt: vec![101, 102, 103], + }) +} + +// Stage readiness events are generated from the committed plan so the tests +// prove readiness by stage identity instead of relying on command ordering. +fn stage_ready_events(plan: &fsm::RunPlan) -> Vec { + plan.stages + .iter() + .map(|stage| fsm::RunEvent::StageReady { + run_id: plan.run_id, + stage_index: stage.stage_index, + }) + .collect() +} + +// Transcript positions turn ordering claims into proofs over observable output. +// If an event is missing, the test fails at the boundary where users and other +// components would also lose the guarantee. +fn position_of(events: &[fsm::LifecycleEvent], needle: &fsm::LifecycleEvent) -> usize { + events + .iter() + .position(|event| event == needle) + .expect("expected lifecycle event missing") +} + +// This proves planning and provisioning are gated by PoolReady, and that the +// orchestrator provisions exactly the committed stages and local token endpoints +// from a valid RunPlan. +#[test] +fn planning_and_provisioning_start_only_after_pool_ready() { + // Start the run and give it a valid plan, but no PoolReady event. + let plan = committed_plan(); + let mut harness = new_run(); + harness.observe(fsm::RunEvent::PlanAvailable(plan.clone())); + + // Without PoolReady, provisioning must not begin. + assert!(!harness.commands().iter().any(|command| { + matches!(command, fsm::RunCommand::ProvisionStage { .. }) + })); + + // Once PoolReady is observed, the committed plan may be provisioned. + harness.observe(fsm::RunEvent::PoolReady { + nodes: plan.stage_nodes(), + }); + + // Every planned stage gets exactly one provision command. + let provisioned = harness + .commands() + .iter() + .filter_map(|command| match command { + fsm::RunCommand::ProvisionStage { provision } => Some(provision.stage_index), + _ => None, + }) + .collect::>(); + let expected = plan + .stages + .iter() + .map(|stage| stage.stage_index) + .collect::>(); + assert_eq!(provisioned, expected); + + // Provisioning must not mention nodes outside the committed plan. + let plan_nodes = plan.stage_nodes().into_iter().collect::>(); + for command in harness.commands() { + if let fsm::RunCommand::ProvisionStage { provision } = command { + assert!(plan_nodes.contains(&provision.node_id)); + } + } + + // Token endpoints are created locally from the same committed plan. + assert!(harness.commands().iter().any(|command| { + matches!(command, fsm::RunCommand::CreateTokenInEndpoint { run_id } if *run_id == fsm::RunId(7)) + })); + assert!(harness.commands().iter().any(|command| { + matches!(command, fsm::RunCommand::CreateTokenOutEndpoint { run_id } if *run_id == fsm::RunId(7)) + })); +} + +// This proves prompt injection is blocked until every planned stage and both +// local token endpoints are ready. Duplicate readiness must not count as a +// missing stage, and foreign readiness must fault or reject. +#[test] +fn readiness_barrier_controls_prompt_injection() { + // Provision a valid plan after PoolReady. + let plan = committed_plan(); + let mut harness = new_run(); + harness.observe(fsm::RunEvent::PoolReady { + nodes: plan.stage_nodes(), + }); + harness.observe(fsm::RunEvent::PlanAvailable(plan.clone())); + + // A duplicate StageReady for stage 0 cannot satisfy stage 1 or 2. + harness.observe(fsm::RunEvent::StageReady { + run_id: fsm::RunId(7), + stage_index: 0, + }); + harness.observe(fsm::RunEvent::StageReady { + run_id: fsm::RunId(7), + stage_index: 0, + }); + harness.observe(fsm::RunEvent::TokenInEndpointReady); + harness.observe(fsm::RunEvent::TokenOutEndpointReady); + assert!(!harness.commands().iter().any(|command| { + matches!(command, fsm::RunCommand::InjectPrompt { .. }) + })); + + // Complete the remaining stage readiness facts. + for event in stage_ready_events(&plan).into_iter().skip(1) { + harness.observe(event); + } + + // Prompt injection is the public start signal after the full barrier. + assert!(harness.commands().iter().any(|command| { + matches!( + command, + fsm::RunCommand::InjectPrompt { + run_id: fsm::RunId(7), + sequence: 0, + .. + } + ) + })); + + // Unknown stage readiness must not silently advance another run. + let mut invalid = new_run(); + invalid.observe(fsm::RunEvent::PoolReady { + nodes: plan.stage_nodes(), + }); + invalid.observe(fsm::RunEvent::PlanAvailable(plan)); + invalid.observe(fsm::RunEvent::StageReady { + run_id: fsm::RunId(7), + stage_index: 99, + }); + assert!(invalid.events().iter().any(|event| { + matches!(event, fsm::LifecycleEvent::RunFaulted { .. }) + || matches!(event, fsm::LifecycleEvent::RunRejected { .. }) + })); +} + +// This proves execution has one start signal and advances by the token feedback +// rule: inject sequence 0 first, then inject k + 1 only after consuming k. +#[test] +fn execution_injects_next_sequence_only_after_consuming_previous_token() { + // Drive a run through the complete readiness barrier. + let plan = committed_plan(); + let mut harness = new_run(); + harness.observe(fsm::RunEvent::PoolReady { + nodes: plan.stage_nodes(), + }); + harness.observe(fsm::RunEvent::PlanAvailable(plan.clone())); + harness.observe(fsm::RunEvent::TokenInEndpointReady); + harness.observe(fsm::RunEvent::TokenOutEndpointReady); + for event in stage_ready_events(&plan) { + harness.observe(event); + } + + // No separate broadcast start command may exist alongside prompt injection. + assert!(!harness.commands().iter().any(|command| { + matches!(command, fsm::RunCommand::BroadcastStart { .. }) + })); + + // Sequence 0 must be injected first. + assert_eq!(harness.injected_sequences(), vec![0]); + + // Consuming token 0 permits injecting sequence 1. + harness.observe(fsm::RunEvent::TokenReceived { + sequence: 0, + token_id: 201, + eos: false, + }); + assert_eq!(harness.injected_sequences(), vec![0, 1]); + + // No additional injection may happen without consuming sequence 1. + harness.advance_time_ms(10); + assert_eq!(harness.injected_sequences(), vec![0, 1]); + + // EOS stops further injection after the consumed sequence. + harness.observe(fsm::RunEvent::TokenReceived { + sequence: 1, + token_id: 2, + eos: true, + }); + assert_eq!(harness.injected_sequences(), vec![0, 1]); +} + +// This proves every run-level fault source records one terminal fault, and the +// first failure reason is retained if later failures arrive. +#[test] +fn first_run_fault_reason_is_terminal_and_sticky() { + // Prepare an executing run so both setup and execution-time faults would be + // meaningful if observed. + let plan = committed_plan(); + let mut harness = new_run(); + harness.observe(fsm::RunEvent::PoolReady { + nodes: plan.stage_nodes(), + }); + harness.observe(fsm::RunEvent::PlanAvailable(plan.clone())); + harness.observe(fsm::RunEvent::TokenInEndpointReady); + harness.observe(fsm::RunEvent::TokenOutEndpointReady); + for event in stage_ready_events(&plan) { + harness.observe(event); + } + + // Inject the first failure source. + harness.observe(fsm::RunEvent::StageFault { + run_id: fsm::RunId(7), + stage_index: 1, + reason: fsm::StageFaultReason::WorkerCrashed, + }); + + // Inject later failures that must not replace the terminal reason. + harness.observe(fsm::RunEvent::EndpointFault { + run_id: fsm::RunId(7), + endpoint: fsm::EndpointKind::TokenOut, + }); + harness.observe(fsm::RunEvent::Timeout { + run_id: fsm::RunId(7), + kind: fsm::TimeoutKind::Execution, + }); + + // Exactly one terminal fault is recorded. + let faults = harness + .events() + .iter() + .filter_map(|event| match event { + fsm::LifecycleEvent::RunFaulted { reason, .. } => Some(reason), + _ => None, + }) + .collect::>(); + assert_eq!(faults.len(), 1); + assert_eq!( + *faults[0], + fsm::RunFaultReason::StageFault { + stage_index: 1, + reason: fsm::StageFaultReason::WorkerCrashed, + } + ); +} + +// This proves terminal outcomes are mutually exclusive, reject new work, and +// always lead into teardown for success, fault, and operator stop. +#[test] +fn terminal_outcome_is_single_and_requires_teardown() { + // Complete a run by reaching EOS. + let plan = committed_plan(); + let mut harness = new_run(); + harness.observe(fsm::RunEvent::PoolReady { + nodes: plan.stage_nodes(), + }); + harness.observe(fsm::RunEvent::PlanAvailable(plan.clone())); + harness.observe(fsm::RunEvent::TokenInEndpointReady); + harness.observe(fsm::RunEvent::TokenOutEndpointReady); + for event in stage_ready_events(&plan) { + harness.observe(event); + } + harness.observe(fsm::RunEvent::TokenReceived { + sequence: 0, + token_id: 2, + eos: true, + }); + + // Completed and Faulted are mutually exclusive public outcomes. + let completed = harness + .events() + .iter() + .filter(|event| matches!(event, fsm::LifecycleEvent::RunCompleted { .. })) + .count(); + let faulted = harness + .events() + .iter() + .filter(|event| matches!(event, fsm::LifecycleEvent::RunFaulted { .. })) + .count(); + assert_eq!(completed, 1); + assert_eq!(faulted, 0); + + // New token work after terminal outcome begins must be rejected. + let before = harness.injected_sequences(); + harness.observe(fsm::RunEvent::TokenReceived { + sequence: 99, + token_id: 333, + eos: false, + }); + assert_eq!(harness.injected_sequences(), before); + + // Teardown commands must be emitted for every provisioned stage and local + // endpoint after the terminal outcome. + let stopped_stages = harness + .commands() + .iter() + .filter_map(|command| match command { + fsm::RunCommand::StopRun { stage_index, .. } => Some(*stage_index), + _ => None, + }) + .collect::>(); + let expected_stages = plan + .stages + .iter() + .map(|stage| stage.stage_index) + .collect::>(); + assert_eq!(stopped_stages, expected_stages); + assert!(harness.commands().iter().any(|command| { + matches!(command, fsm::RunCommand::TearDownTokenEndpoints { .. }) + })); +} + +// This proves run_torn_down is emitted exactly once and only after teardown +// reaches a terminal state by stage stops or timeout. +#[test] +fn run_torn_down_is_emitted_once_after_teardown_terminal_state() { + // Fault a provisioned run so teardown is required. + let plan = committed_plan(); + let mut harness = new_run(); + harness.observe(fsm::RunEvent::PoolReady { + nodes: plan.stage_nodes(), + }); + harness.observe(fsm::RunEvent::PlanAvailable(plan.clone())); + harness.observe(fsm::RunEvent::StageFault { + run_id: fsm::RunId(7), + stage_index: 0, + reason: fsm::StageFaultReason::WorkerCrashed, + }); + + // StageStopped from only a prefix of stages is not enough to finish + // teardown. + harness.observe(fsm::RunEvent::StageStopped { + run_id: fsm::RunId(7), + stage_index: 0, + }); + assert!(!harness.events().iter().any(|event| { + matches!(event, fsm::LifecycleEvent::RunTornDown { .. }) + })); + + // Finish teardown through remaining stopped events and local endpoint stop. + harness.observe(fsm::RunEvent::StageStopped { + run_id: fsm::RunId(7), + stage_index: 1, + }); + harness.observe(fsm::RunEvent::StageStopped { + run_id: fsm::RunId(7), + stage_index: 2, + }); + harness.observe(fsm::RunEvent::TokenEndpointsStopped); + + // The final event may now appear, exactly once. + let torn_down_count = harness + .events() + .iter() + .filter(|event| matches!(event, fsm::LifecycleEvent::RunTornDown { .. })) + .count(); + assert_eq!(torn_down_count, 1); + + // Ordering is proven over the lifecycle transcript. + let fault_pos = position_of( + harness.events(), + &fsm::LifecycleEvent::RunFaulted { + run_id: fsm::RunId(7), + reason: fsm::RunFaultReason::StageFault { + stage_index: 0, + reason: fsm::StageFaultReason::WorkerCrashed, + }, + }, + ); + let torn_down_pos = position_of( + harness.events(), + &fsm::LifecycleEvent::RunTornDown { + run_id: fsm::RunId(7), + }, + ); + assert!(fault_pos < torn_down_pos); +} diff --git a/tests/mvp_system/orchestrator_token_endpoint_guarantees.rs b/tests/mvp_system/orchestrator_token_endpoint_guarantees.rs new file mode 100644 index 0000000..c4b555c --- /dev/null +++ b/tests/mvp_system/orchestrator_token_endpoint_guarantees.rs @@ -0,0 +1,248 @@ +//! Black-box contract tests for MVP orchestrator token endpoints. +//! +//! These tests intentionally know only the public token-endpoint surface: +//! +//! - committed token edges and endpoint formation in +//! - readiness, prompt, token, and fault events in +//! - token writes, lifecycle events, and run faults out +//! +//! They assert the guarantees in +//! `specs/mvp_system/orchestrator_token_endpoint_contract.md`. + +use mvp_system::orchestrator_token_endpoint as token; + +// The endpoint plan names the orchestrator and both token edges. Tests use the +// committed plan as input and do not assume how endpoints allocate local rings +// or actors. +fn token_plan() -> token::TokenEndpointPlan { + token::TokenEndpointPlan { + run_id: token::RunId(7), + orchestrator_node_id: token::NodeId(99), + token_in_edge: token::EdgePlan::token_in(token::EdgeId(7000), token::NodeId(99), token::NodeId(10)), + token_out_edge: token::EdgePlan::token_out(token::EdgeId(7003), token::NodeId(12), token::NodeId(99)), + token_spec: token::ObjectSpec::test_tokens(), + max_tokens: 4, + } +} + +// The harness exposes only endpoint-visible inputs and outputs. It does not +// reveal pump tasks, local actor addresses, or ring internals. +fn new_endpoint_harness() -> token::TokenEndpointHarness { + token::TokenEndpointHarness::new(token_plan()) +} + +// This helper drives the global readiness barrier as the orchestrator would see +// it. Prompt-injection tests use it to prove the endpoint does not start early. +fn pass_global_barrier(harness: &mut token::TokenEndpointHarness) { + harness.observe(token::EndpointEvent::TokenInReady { + edge_id: token::EdgeId(7000), + }); + harness.observe(token::EndpointEvent::TokenOutReady { + edge_id: token::EdgeId(7003), + }); + harness.observe(token::EndpointEvent::AllStagesReady); + harness.observe(token::EndpointEvent::ReadinessBarrierPassed); +} + +// This proves endpoint formation is derived from the committed plan, uses the +// same edge semantics as stage endpoints, and keeps a stable orchestrator node +// identity even when co-located with GPU nodes. +#[test] +fn token_endpoints_are_created_from_committed_plan_with_stable_node_id() { + // Create endpoints from the committed token plan. + let harness = new_endpoint_harness(); + + // Token-in and token-out endpoint creation must reference the plan edges. + assert!(harness.commands().iter().any(|command| { + matches!( + command, + token::EndpointCommand::CreateTokenInProducer { + edge_id: token::EdgeId(7000), + orchestrator_node_id: token::NodeId(99), + .. + } + ) + })); + assert!(harness.commands().iter().any(|command| { + matches!( + command, + token::EndpointCommand::CreateTokenOutConsumer { + edge_id: token::EdgeId(7003), + orchestrator_node_id: token::NodeId(99), + .. + } + ) + })); + + // Co-location must not change the edge shape or remove the orchestrator + // endpoint identity. + for endpoint in harness.local_endpoints() { + assert_eq!(endpoint.orchestrator_node_id, token::NodeId(99)); + assert!(matches!( + endpoint.edge_semantics, + token::EdgeSemantics::SingleProducerSingleConsumer + )); + } +} + +// This proves prompt injection waits for the global readiness barrier, writes +// token object sequence 0, conforms to the token ObjectSpec, and is the only +// start signal. +#[test] +fn prompt_injection_is_barrier_gated_sequence_zero_token_object() { + // Build endpoints but do not pass readiness. + let mut harness = new_endpoint_harness(); + harness.request_prompt_injection(vec![101, 102, 103]); + + // No prompt object may be written before the barrier. + assert!(!harness.commands().iter().any(|command| { + matches!(command, token::EndpointCommand::WriteTokenObject { .. }) + })); + + // Passing the global barrier permits the first prompt write. + pass_global_barrier(&mut harness); + + // The first token-in object must be sequence 0 and match the token spec. + let writes = harness + .commands() + .iter() + .filter_map(|command| match command { + token::EndpointCommand::WriteTokenObject(write) => Some(write), + _ => None, + }) + .collect::>(); + assert_eq!(writes.len(), 1); + assert_eq!(writes[0].sequence, 0); + assert_eq!(writes[0].object_spec, token::ObjectSpec::test_tokens()); + + // Prompt injection is the start signal; there must not be a separate + // broadcast start command. + assert!(!harness.commands().iter().any(|command| { + matches!(command, token::EndpointCommand::BroadcastStart { .. }) + })); +} + +// This proves token-out is consumed in sequence order and sequence k + 1 is +// injected only after the orchestrator consumes token sequence k. +#[test] +fn token_feedback_controls_next_injection() { + // Start with prompt sequence 0 injected. + let mut harness = new_endpoint_harness(); + harness.request_prompt_injection(vec![101, 102, 103]); + pass_global_barrier(&mut harness); + assert_eq!(harness.injected_sequences(), vec![0]); + + // Consuming token sequence 0 authorizes sequence 1. + harness.observe(token::EndpointEvent::TokenObjectReceived { + edge_id: token::EdgeId(7003), + object_id: token::ObjectId(9000), + sequence: 0, + token_id: 201, + eos: false, + }); + assert_eq!(harness.injected_sequences(), vec![0, 1]); + + // Time and readiness events alone do not authorize another decode step. + harness.advance_time_ms(100); + assert_eq!(harness.injected_sequences(), vec![0, 1]); + + // An out-of-order token faults the run rather than skipping ahead. + harness.observe(token::EndpointEvent::TokenObjectReceived { + edge_id: token::EdgeId(7003), + object_id: token::ObjectId(9002), + sequence: 3, + token_id: 203, + eos: false, + }); + assert!(harness.events().iter().any(|event| { + matches!( + event, + token::EndpointLifecycleEvent::RunFaulted { + reason: token::RunFaultReason::TokenSequenceViolation, + .. + } + ) + })); +} + +// This proves EOS and max_tokens both stop further token-in injection. +#[test] +fn eos_and_max_tokens_stop_injection() { + // EOS stops immediately after the consumed token. + let mut eos_harness = new_endpoint_harness(); + eos_harness.request_prompt_injection(vec![101]); + pass_global_barrier(&mut eos_harness); + eos_harness.observe(token::EndpointEvent::TokenObjectReceived { + edge_id: token::EdgeId(7003), + object_id: token::ObjectId(9000), + sequence: 0, + token_id: 2, + eos: true, + }); + assert_eq!(eos_harness.injected_sequences(), vec![0]); + + // max_tokens stops after the configured number of injections. + let mut max_harness = new_endpoint_harness(); + max_harness.request_prompt_injection(vec![101]); + pass_global_barrier(&mut max_harness); + for sequence in 0..4 { + max_harness.observe(token::EndpointEvent::TokenObjectReceived { + edge_id: token::EdgeId(7003), + object_id: token::ObjectId(9000 + sequence), + sequence, + token_id: 300 + sequence as u32, + eos: false, + }); + } + assert_eq!(max_harness.injected_sequences(), vec![0, 1, 2, 3]); +} + +// This proves token endpoint, malformed object, and sequence faults are +// surfaced as run faults, while teardown failure contributes to teardown +// failure rather than being lost in logs. +#[test] +fn token_endpoint_failures_fault_the_run_or_teardown() { + // Endpoint fault during active run faults the run. + let mut harness = new_endpoint_harness(); + harness.request_prompt_injection(vec![101]); + pass_global_barrier(&mut harness); + harness.observe(token::EndpointEvent::EndpointFault { + edge_id: token::EdgeId(7000), + direction: token::EndpointDirection::TokenIn, + }); + assert!(harness.events().iter().any(|event| { + matches!( + event, + token::EndpointLifecycleEvent::RunFaulted { + reason: token::RunFaultReason::TokenEndpointFault, + .. + } + ) + })); + + // Malformed token objects fault the run at the token boundary. + let mut malformed = new_endpoint_harness(); + malformed.request_prompt_injection(vec![101]); + pass_global_barrier(&mut malformed); + malformed.observe(token::EndpointEvent::MalformedTokenObject { + edge_id: token::EdgeId(7003), + object_id: token::ObjectId(9999), + }); + assert!(malformed.events().iter().any(|event| { + matches!( + event, + token::EndpointLifecycleEvent::RunFaulted { + reason: token::RunFaultReason::MalformedTokenObject, + .. + } + ) + })); + + // Teardown failure must be visible as teardown failure. + malformed.observe(token::EndpointEvent::TeardownFailed { + edge_id: token::EdgeId(7003), + }); + assert!(malformed.events().iter().any(|event| { + matches!(event, token::EndpointLifecycleEvent::TeardownFailed { .. }) + })); +} diff --git a/tests/mvp_system/resource_inventory_guarantees.rs b/tests/mvp_system/resource_inventory_guarantees.rs new file mode 100644 index 0000000..72bc72f --- /dev/null +++ b/tests/mvp_system/resource_inventory_guarantees.rs @@ -0,0 +1,246 @@ +//! Black-box contract tests for MVP resource inventory. +//! +//! These tests intentionally know only the public inventory/planner input +//! surface: +//! +//! - orchestrator-owned inventory entries +//! - candidate pool and placement input derived from that inventory +//! - planner acceptance or typed rejection +//! +//! They assert the guarantees in +//! `specs/mvp_system/resource_inventory_contract.md`. + +use mvp_system::resource_inventory as inventory; + +// The inventory fixture has more nodes than the placement needs. That proves +// the planner may choose among known inventory entries but may not invent hidden +// nodes or accept nodes that the orchestrator did not provide. +fn inventory_entries() -> Vec { + vec![ + inventory::InventoryEntry::ready(inventory::NodeId(10), inventory::GpuClass::TestSmall), + inventory::InventoryEntry::ready(inventory::NodeId(11), inventory::GpuClass::TestSmall), + inventory::InventoryEntry::ready(inventory::NodeId(12), inventory::GpuClass::TestSmall), + inventory::InventoryEntry::ready(inventory::NodeId(13), inventory::GpuClass::TestSmall), + ] +} + +// The fixed placement input names stage ownership without giving the planner +// permission to derive any other topology. The plan still owns the resulting +// stage records. +fn fixed_placement() -> inventory::PlacementInput { + inventory::PlacementInput::FixedLinear(vec![ + inventory::StagePlacement { + stage_index: 0, + node_id: inventory::NodeId(10), + }, + inventory::StagePlacement { + stage_index: 1, + node_id: inventory::NodeId(11), + }, + inventory::StagePlacement { + stage_index: 2, + node_id: inventory::NodeId(12), + }, + ]) +} + +// This helper builds the public planning request from inventory facts. Tests +// mutate only the fact under examination so a rejection can be attributed to a +// specific inventory contract violation. +fn planning_request() -> inventory::PlanningRequest { + inventory::PlanningRequest { + run_id: inventory::RunId(7), + stage_count: 3, + entries: inventory_entries(), + placement: fixed_placement(), + } +} + +// Node reports after boot are inventory inputs, not placement negotiations. +// This helper gives tests one post-boot report they can send and then prove did +// not rewrite the committed planning input. +fn post_boot_health_report(node_id: inventory::NodeId) -> inventory::NodeReport { + inventory::NodeReport::BootHealth { + node_id, + health: inventory::BootHealth::Ready, + } +} + +// This proves inventory formation is orchestrator-owned and available before +// planning: every entry is tied to a known node id, and node reports do not +// negotiate placement after boot. +#[test] +fn inventory_entries_are_known_before_planning_and_not_negotiated_by_nodes() { + // Create the orchestrator-owned inventory and snapshot the public entries. + let mut harness = inventory::InventoryHarness::new(inventory_entries()); + let before_report = harness.entries().to_vec(); + + // Let a node report boot health after the inventory already exists. + harness.observe_node_report(post_boot_health_report(inventory::NodeId(10))); + + // Boot health may update readiness metadata, but it must not rewrite the + // node set or add placement facts. + let after_report = harness.entries().to_vec(); + let before_nodes = before_report + .iter() + .map(|entry| entry.node_id) + .collect::>(); + let after_nodes = after_report + .iter() + .map(|entry| entry.node_id) + .collect::>(); + assert_eq!(after_nodes, before_nodes); + assert!(!harness.commands().iter().any(|command| { + matches!(command, inventory::InventoryCommand::AcceptPlacementNegotiation { .. }) + })); +} + +// This proves the planner may place stages only onto nodes present in the +// orchestrator-owned inventory, and the emitted RunPlan is the sole placement +// result. +#[test] +fn planned_stage_nodes_are_subset_of_inventory_nodes() { + // Build a valid public planning request from inventory entries. + let request = planning_request(); + let inventory_nodes = request + .entries + .iter() + .map(|entry| entry.node_id) + .collect::>(); + + // Plan through the public inventory/planner boundary. + let plan = inventory::plan_from_inventory(request).expect("valid inventory must plan"); + + // Prove every planned stage node came from the inventory snapshot. + for stage in &plan.stages { + assert!( + inventory_nodes.contains(&stage.node_id), + "stage used node outside inventory: {:?}", + stage.node_id + ); + } +} + +// This proves unknown nodes, duplicate stage assignments, and missing stage +// assignments reject as typed planning-input errors. +#[test] +fn invalid_inventory_placement_rejects_with_typed_errors() { + // Each case corrupts one public placement fact. + let cases = vec![ + ( + inventory::PlacementInput::FixedLinear(vec![ + inventory::StagePlacement { + stage_index: 0, + node_id: inventory::NodeId(10), + }, + inventory::StagePlacement { + stage_index: 1, + node_id: inventory::NodeId(99), + }, + inventory::StagePlacement { + stage_index: 2, + node_id: inventory::NodeId(12), + }, + ]), + inventory::InventoryRejectionKind::UnknownNode { + node_id: inventory::NodeId(99), + }, + ), + ( + inventory::PlacementInput::FixedLinear(vec![ + inventory::StagePlacement { + stage_index: 0, + node_id: inventory::NodeId(10), + }, + inventory::StagePlacement { + stage_index: 0, + node_id: inventory::NodeId(11), + }, + inventory::StagePlacement { + stage_index: 2, + node_id: inventory::NodeId(12), + }, + ]), + inventory::InventoryRejectionKind::DuplicateStage { stage_index: 0 }, + ), + ( + inventory::PlacementInput::FixedLinear(vec![ + inventory::StagePlacement { + stage_index: 0, + node_id: inventory::NodeId(10), + }, + inventory::StagePlacement { + stage_index: 2, + node_id: inventory::NodeId(12), + }, + ]), + inventory::InventoryRejectionKind::MissingStage { stage_index: 1 }, + ), + ]; + + for (placement, expected_kind) in cases { + // Replace only the placement fact in an otherwise valid request. + let mut request = planning_request(); + request.placement = placement; + + // The rejection must be typed and no RunPlan may be emitted. + let rejection = inventory::plan_from_inventory(request) + .expect_err("invalid inventory placement must reject"); + assert_eq!(rejection.kind, expected_kind); + } +} + +// This proves planning does not mutate the candidate pool and does not derive +// hidden nodes outside the inventory. +#[test] +fn planning_preserves_candidate_pool_and_emits_no_hidden_nodes() { + // Snapshot the public inventory before planning. + let request = planning_request(); + let original_entries = request.entries.clone(); + let original_nodes = original_entries + .iter() + .map(|entry| entry.node_id) + .collect::>(); + + // Plan using a cloned request so the caller-owned facts remain inspectable. + let plan = inventory::plan_from_inventory(request.clone()).expect("valid inventory must plan"); + + // The caller's candidate facts must be unchanged. + assert_eq!(request.entries, original_entries); + + // Every node mentioned by the plan must be explainable by the original + // inventory snapshot. + let planned_nodes = plan + .stages + .iter() + .map(|stage| stage.node_id) + .collect::>(); + assert!(planned_nodes.is_subset(&original_nodes)); +} + +// This proves inventory authority ends at planner input. After provisioning, +// stages consume their RunPlan-derived assignment and do not reinterpret +// inventory reports. +#[test] +fn provisioned_stages_do_not_reinterpret_inventory() { + // Commit a plan and provision its stages. + let plan = inventory::plan_from_inventory(planning_request()).expect("valid inventory must plan"); + let mut harness = inventory::InventoryHarness::new(inventory_entries()); + harness.commit_plan(plan.clone()); + harness.provision_stages(); + + // Send a later inventory report that would be dangerous if treated as a + // graph rewrite. + harness.observe_node_report(inventory::NodeReport::CapacityChanged { + node_id: inventory::NodeId(11), + gpu_class: inventory::GpuClass::TestLarge, + }); + + // No provisioned stage may be asked to reinterpret its assignment. + assert!(!harness.commands().iter().any(|command| { + matches!(command, inventory::InventoryCommand::RewriteStagePlacement { .. }) + })); + + // The committed plan remains the only graph-visible placement fact. + assert_eq!(harness.committed_plan(), Some(&plan)); +} diff --git a/tests/mvp_system/run_plan_guarantees.rs b/tests/mvp_system/run_plan_guarantees.rs new file mode 100644 index 0000000..992f7fe --- /dev/null +++ b/tests/mvp_system/run_plan_guarantees.rs @@ -0,0 +1,691 @@ +//! Black-box contract tests for MVP RunPlan formation. +//! +//! These tests intentionally know only the public planning surface: +//! +//! - `plan_run(input) -> Result` +//! - `derive_stage_provision(&plan, stage_index) -> Result` +//! +//! They assert the guarantees in `specs/mvp_system/run_plan_contract.md`. +//! The planner implementation, placement heuristic, helper APIs, internal graph +//! representation, and allocation strategy are not observable here. + +use mvp_system::run_plan as plan; + +// Local aliases keep the test prose readable while the file imports only the +// public planning module. The aliases do not grant access to planner internals. +type DTypeFamily = plan::DTypeFamily; +type EdgeEndpoint = plan::EdgeEndpoint; +type EdgeId = plan::EdgeId; +type EdgeKind = plan::EdgeKind; +type EdgePlan = plan::EdgePlan; +type InboundEdgeProvision = plan::InboundEdgeProvision; +type ModelFacts = plan::ModelFacts; +type NodeId = plan::NodeId; +type ObjectKind = plan::ObjectKind; +type OutboundEdgeProvision = plan::OutboundEdgeProvision; +type PlacementInput = plan::PlacementInput; +type PlanRejectionKind = plan::PlanRejectionKind; +type PlannerInput = plan::PlannerInput; +type RingSpec = plan::RingSpec; +type RunPlan = plan::RunPlan; +type RuntimeConfig = plan::RuntimeConfig; +type StagePlacement = plan::StagePlacement; + +// Keep test node ids small and readable. The concrete identity mechanism is +// outside this contract; these ids exist only so assertions can name topology +// facts without depending on any address or discovery machinery. +fn node(id: u64) -> NodeId { + NodeId(id) +} + +// The candidate pool is deliberately larger than some test placements. That +// lets the tests distinguish "known to the orchestrator" from "assigned to a +// stage", which is one of the planner authority boundaries. +fn valid_nodes() -> Vec { + vec![node(10), node(11), node(12), node(13)] +} + +// Fixed linear placement is the smallest placement input that still exercises +// the contract. It supplies stage-to-node intent, while the planner remains +// responsible for validating it and minting the full RunPlan topology. +fn linear_placement(stage_count: u32) -> PlacementInput { + PlacementInput::FixedLinear( + (0..stage_count) + .map(|stage_index| StagePlacement { + stage_index, + node_id: node(10 + u64::from(stage_index)), + }) + .collect(), + ) +} + +// This is the canonical valid fixture for RunPlan guarantees. Each test tweaks +// only the fact it is trying to prove, so a failure points at the violated +// contract instead of at accidental fixture drift. +fn valid_input(stage_count: u32, num_layers: u32) -> PlannerInput { + PlannerInput { + run_id: 7.into(), + orchestrator_node_id: node(99), + model: ModelFacts { + model_id: "test-gguf".into(), + num_layers, + hidden_dim: 4096, + dtype_family: DTypeFamily::BFloat, + dtype_width_bytes: 2, + max_seq_len: 2048, + eos_token_id: 2, + }, + runtime: RuntimeConfig::test_default(), + candidate_pool: valid_nodes(), + stage_count, + placement: linear_placement(stage_count), + activation_ring: RingSpec::test_default_activation(), + token_ring: RingSpec::test_default_token(), + } +} + +// Tests frequently need to compare a provisioned edge id back to the canonical +// edge record in the RunPlan. This helper makes that lookup explicit without +// giving tests access to any planner-private index. +fn plan_edges_by_id(plan: &RunPlan) -> std::collections::BTreeMap { + plan.edges + .iter() + .map(|edge| (edge.edge_id, edge)) + .collect::>() +} + +// Edge endpoints can be orchestrator or stage endpoints. Tests use this helper +// when they care only about stage adjacency and want orchestrator endpoints to +// remain visibly outside the stage index space. +fn edge_stage_index(endpoint: &EdgeEndpoint) -> Option { + match endpoint { + EdgeEndpoint::Orchestrator { .. } => None, + EdgeEndpoint::Stage { stage_index, .. } => Some(*stage_index), + } +} + +// Provisioning sends concrete node ids across the data-flow boundary. This +// helper extracts the observable node id from either endpoint shape so tests +// can compare projection output to plan topology. +fn edge_node_id(endpoint: &EdgeEndpoint) -> NodeId { + match endpoint { + EdgeEndpoint::Orchestrator { node_id } => *node_id, + EdgeEndpoint::Stage { node_id, .. } => *node_id, + } +} + +// This proves RunPlan formation is a total public boundary for valid input: +// the caller observes one complete plan, not hidden follow-up topology work or +// a partially initialized result. +#[test] +fn valid_input_emits_one_complete_plan() { + // Build one ordinary valid planning request. + let input = valid_input(3, 36); + + // Planning valid input must produce a usable plan, not a deferred partial. + let plan = plan::plan_run(input).expect("valid input must emit a plan"); + + // The plan-level identifiers and counts must be complete immediately. + assert_eq!(plan.run_id, 7.into()); + assert_eq!(plan.stages.len(), 3); + assert_eq!(plan.edges.len(), 4); + + // Every stage must be bound to this run and know the run's stage count. + for stage in &plan.stages { + assert_eq!(stage.run_id, plan.run_id); + assert_eq!(stage.stage_count, 3); + } + + // Every edge must also be bound to this run; no edge can be a loose fact. + for edge in &plan.edges { + assert_eq!(edge.run_id, plan.run_id); + } +} + +// This proves planning does not mutate the candidate pool supplied by the +// caller. The only topology facts the caller can use after planning are the +// facts emitted in the RunPlan itself. +#[test] +fn planner_does_not_mutate_candidate_pool() { + // Keep a copy of the caller-owned pool before the planner sees it. + let input = valid_input(3, 36); + let original_pool = input.candidate_pool.clone(); + + // Run planning through the public API only. + let _ = plan::plan_run(input.clone()).expect("valid input must emit a plan"); + + // The input pool remains the caller's fact; topology facts must be in the + // returned plan, not back-written into the input. + assert_eq!(input.candidate_pool, original_pool); +} + +// This proves layer assignment is a contiguous, non-overlapping partition of +// the intended GGUF block range, with one non-empty range per stage. +#[test] +fn stage_ranges_partition_the_model_layers() { + // Exercise several deterministic sizes so the check covers one-stage and + // multi-stage partitioning without relying on random generation. + for (stage_count, num_layers) in [(1, 12), (2, 24), (3, 36), (4, 40)] { + // Produce the plan from public inputs. + let plan = plan::plan_run(valid_input(stage_count, num_layers)).unwrap(); + + // Read only the public stage assignments and sort by stage index. + let mut ranges = plan + .stages + .iter() + .map(|stage| { + ( + stage.stage_index, + stage.layer_start, + stage.layer_end_exclusive, + ) + }) + .collect::>(); + ranges.sort_by_key(|(stage_index, _, _)| *stage_index); + + // Walk the sorted ranges as a proof of contiguity. The next start must + // equal the previous end, and every range must consume at least one + // layer inside the model range. + let mut expected_start = 0; + for (_, start, end) in ranges { + assert_eq!(start, expected_start, "range gap or overlap"); + assert!(end > start, "stage range must be non-empty"); + assert!(end <= num_layers, "stage range exceeds model layer range"); + expected_start = end; + } + + // The final end must cover the whole intended block range. + assert_eq!(expected_start, num_layers); + } +} + +// This proves stage indices are exactly the dense range required by the +// contract. Missing, duplicate, or out-of-range stage indices are observable in +// the returned RunPlan and fail this check. +#[test] +fn stage_indices_are_exactly_zero_to_stage_count_minus_one() { + // Check several stage counts so the dense-index guarantee is not tied to + // the canonical three-stage fixture. + for stage_count in 1..=4 { + // Produce a valid plan and observe only its public stage indices. + let plan = plan::plan_run(valid_input(stage_count, stage_count * 8)).unwrap(); + let observed = plan + .stages + .iter() + .map(|stage| stage.stage_index) + .collect::>(); + + // Compare against the contract's exact dense index set. + let expected = (0..stage_count).collect::>(); + + assert_eq!(observed, expected); + } +} + +// This proves every edge has one public producer and one public consumer, and +// that the returned edge graph is exactly the MVP linear pipeline. +#[test] +fn edge_graph_is_exactly_the_linear_pipeline() { + // Use four stages so the activation chain has multiple interior edges. + let stage_count = 4; + let plan = plan::plan_run(valid_input(stage_count, 40)).unwrap(); + + // Token-in must be unique and must enter stage 0 from the orchestrator. + let token_in = plan + .edges + .iter() + .filter(|edge| edge.kind == EdgeKind::TokenIn) + .collect::>(); + assert_eq!(token_in.len(), 1); + assert!(matches!( + token_in[0].producer, + EdgeEndpoint::Orchestrator { node_id } if node_id == node(99) + )); + assert_eq!( + token_in[0].consumer, + EdgeEndpoint::Stage { + node_id: node(10), + stage_index: 0, + } + ); + + // Activation edges must be the only stage-to-stage edges, one per adjacent + // stage pair. + let activation_edges = plan + .edges + .iter() + .filter(|edge| edge.kind == EdgeKind::Activation) + .collect::>(); + assert_eq!(activation_edges.len(), (stage_count - 1) as usize); + + // Each activation edge produced by stage i must be consumed by stage i+1. + for stage_index in 0..stage_count - 1 { + let edge = activation_edges + .iter() + .find(|edge| edge_stage_index(&edge.producer) == Some(stage_index)) + .expect("activation edge produced by stage"); + + assert_eq!( + edge.consumer, + EdgeEndpoint::Stage { + node_id: node(11 + u64::from(stage_index)), + stage_index: stage_index + 1, + } + ); + assert_ne!(edge.producer, edge.consumer, "self-edge is forbidden"); + } + + // Token-out must be unique and must leave the final stage for the + // orchestrator. + let token_out = plan + .edges + .iter() + .filter(|edge| edge.kind == EdgeKind::TokenOut) + .collect::>(); + assert_eq!(token_out.len(), 1); + assert_eq!(edge_stage_index(&token_out[0].producer), Some(stage_count - 1)); + assert!(matches!( + token_out[0].consumer, + EdgeEndpoint::Orchestrator { node_id } if node_id == node(99) + )); +} + +// This proves edge ids are run-unique and that stage plans refer only to edge +// ids present in the returned RunPlan, so stages receive assigned ids rather +// than deriving data-flow identity themselves. +#[test] +fn edge_ids_are_unique_and_stage_references_resolve_to_plan_edges() { + // Produce a plan with enough edges to make duplicate ids observable. + let plan = plan::plan_run(valid_input(4, 40)).unwrap(); + + // Insert every public edge id into a set; a duplicate shrinks the set. + let edge_ids = plan + .edges + .iter() + .map(|edge| edge.edge_id) + .collect::>(); + + assert_eq!(edge_ids.len(), plan.edges.len(), "duplicate edge id"); + + // Stage plans may reference only ids that the RunPlan itself assigned. + for stage in &plan.stages { + assert!( + edge_ids.contains(&stage.inbound_edge), + "stage inbound edge id must come from RunPlan edges" + ); + assert!( + edge_ids.contains(&stage.outbound_edge), + "stage outbound edge id must come from RunPlan edges" + ); + } +} + +// This proves deriving ProvisionStage is deterministic and stage-local: +// repeated projection returns the same value, and the projected layer range is +// exactly the range assigned to that stage in the RunPlan. +#[test] +fn provision_stage_projection_is_deterministic_and_stage_local() { + // Start from one committed plan; projection is a pure public view of it. + let plan = plan::plan_run(valid_input(3, 36)).unwrap(); + + // Check every stage projection, not only one representative stage. + for stage_index in 0..3 { + // Derive twice to prove projection does not depend on hidden mutable + // state or call order. + let first = plan::derive_stage_provision(&plan, stage_index).unwrap(); + let second = plan::derive_stage_provision(&plan, stage_index).unwrap(); + + // Find the corresponding public stage assignment in the plan. + let stage = plan + .stages + .iter() + .find(|stage| stage.stage_index == stage_index) + .unwrap(); + + // The projected message must be stable and expose only that stage's + // assigned run position and layer range. + assert_eq!(first, second); + assert_eq!(first.stage_index, stage_index); + assert_eq!(first.stage_count, 3); + assert_eq!(first.layer_start, stage.layer_start); + assert_eq!(first.layer_end_exclusive, stage.layer_end_exclusive); + } +} + +// This proves each stage receives exactly one inbound and one outbound edge +// provision, and that the provisioned ids are the ids assigned to that stage by +// the RunPlan. +#[test] +fn provision_stage_contains_exactly_the_assigned_inbound_and_outbound_edges() { + // Build a valid plan and test projection for every stage in it. + let plan = plan::plan_run(valid_input(3, 36)).unwrap(); + + for stage in &plan.stages { + // Derive the public provisioning message for this stage. + let provision = plan::derive_stage_provision(&plan, stage.stage_index).unwrap(); + + // The message exposes exactly the inbound and outbound ids assigned in + // that stage's StagePlan. + assert_eq!(provision.inbound.edge_id, stage.inbound_edge); + assert_eq!(provision.outbound.edge_id, stage.outbound_edge); + } +} + +// This proves the data-flow addressing contract of provisioning. The outbound +// side carries the consumer node id; exhaustive struct destructuring also makes +// remote actor-address fields a compile-time contract violation. +#[test] +fn provision_stage_uses_node_id_addressing_not_remote_actor_addresses() { + // Build one plan and an edge lookup using only public edge records. + let plan = plan::plan_run(valid_input(3, 36)).unwrap(); + let edges = plan_edges_by_id(&plan); + + for stage in &plan.stages { + // Project the stage-local provisioning message. + let provision = plan::derive_stage_provision(&plan, stage.stage_index).unwrap(); + + // Destructure the inbound provision exhaustively. If a remote actor + // address becomes part of this public type, this test must be updated + // consciously instead of silently accepting it. + let InboundEdgeProvision { + edge_id: inbound_edge_id, + kind: _, + object_spec: _, + ring_spec: _, + } = provision.inbound.clone(); + + // Destructure the outbound provision exhaustively. The only remote + // routing fact it may expose is the consumer node id. + let OutboundEdgeProvision { + edge_id: outbound_edge_id, + kind: _, + consumer_node_id, + object_spec: _, + ring_spec: _, + } = provision.outbound.clone(); + + assert_eq!(inbound_edge_id, stage.inbound_edge); + assert_eq!(outbound_edge_id, stage.outbound_edge); + + // The provisioned consumer node id must match the consumer endpoint of + // the canonical RunPlan edge. + let outbound_edge = edges.get(&outbound_edge_id).unwrap(); + assert_eq!(consumer_node_id, edge_node_id(&outbound_edge.consumer)); + } +} + +// This proves every edge carries object and ring specs, activation capacity is +// derived from model facts, and edge kind selects the correct object kind. +#[test] +fn object_and_ring_specs_are_present_and_match_edge_kind() { + // Use the canonical model facts so the expected activation capacity is + // known directly from the public input. + let plan = plan::plan_run(valid_input(3, 36)).unwrap(); + let expected_activation_extent = 2048 * 4096 * 2; + + // Every edge must carry complete movement specs; no worker or transport may + // invent these later. + for edge in &plan.edges { + assert!(edge.object_spec.max_extent > 0); + assert!(edge.ring_spec.data_capacity > 0); + + // Edge kind selects the object kind, and activation capacity is derived + // from model facts. + match edge.kind { + EdgeKind::Activation => { + assert_eq!(edge.object_spec.kind, ObjectKind::Activation); + assert_eq!(edge.object_spec.max_extent, expected_activation_extent); + } + EdgeKind::TokenIn | EdgeKind::TokenOut => { + assert_eq!(edge.object_spec.kind, ObjectKind::Token); + } + } + } +} + +// This proves object and ring specs are copied consistently into every stage +// projection. A stage provision may narrow visibility to its own edges, but it +// may not revise the specs for those edges. +#[test] +fn object_and_ring_specs_are_copied_consistently_into_stage_provisions() { + // Build a canonical plan and index its public edge records. + let plan = plan::plan_run(valid_input(3, 36)).unwrap(); + let edges = plan_edges_by_id(&plan); + + for stage in &plan.stages { + // Project a stage-local provisioning message. + let provision = plan::derive_stage_provision(&plan, stage.stage_index).unwrap(); + + // The inbound edge spec must be the plan's exact edge spec. + let inbound_edge = edges.get(&provision.inbound.edge_id).unwrap(); + assert_eq!(provision.inbound.kind, inbound_edge.kind); + assert_eq!(provision.inbound.object_spec, inbound_edge.object_spec); + assert_eq!(provision.inbound.ring_spec, inbound_edge.ring_spec); + + // The outbound edge spec must also be copied, not recomputed or revised. + let outbound_edge = edges.get(&provision.outbound.edge_id).unwrap(); + assert_eq!(provision.outbound.kind, outbound_edge.kind); + assert_eq!(provision.outbound.object_spec, outbound_edge.object_spec); + assert_eq!(provision.outbound.ring_spec, outbound_edge.ring_spec); + } +} + +// This proves typed rejection is the public behavior for invalid authority and +// topology inputs. No invalid case is allowed to emit a partial plan. +#[test] +fn invalid_authority_and_topology_inputs_reject_without_plan() { + // Each case changes one authority/topology fact from the valid fixture and + // names the typed rejection the planner must expose. + let cases = [ + ( + invalid_unknown_node(), + PlanRejectionKind::UnknownNode, + "unknown node id", + ), + ( + invalid_duplicate_stage_assignment(), + PlanRejectionKind::DuplicateStageAssignment, + "duplicate stage assignment", + ), + ( + invalid_missing_stage_assignment(), + PlanRejectionKind::MissingStage, + "missing stage", + ), + ( + invalid_zero_stage_count(), + PlanRejectionKind::InvalidStageCount, + "invalid stage count", + ), + ( + invalid_edge_endpoint_mismatch(), + PlanRejectionKind::EdgeEndpointMismatch, + "edge endpoint mismatch", + ), + ( + invalid_model_stage_layout(), + PlanRejectionKind::ModelStageLayoutMismatch, + "model/stage layout mismatch", + ), + ]; + + // Invalid inputs must not produce a partial plan. The observable result is + // a typed rejection kind. + for (input, expected, label) in cases { + let err = plan::plan_run(input).expect_err(label); + assert_eq!(err.kind(), expected, "{label}"); + } +} + +// This proves invalid object and ring spec facts reject before provisioning. +// The planner may choose the exact diagnostic payload, but the rejection kind +// must be typed and no RunPlan may be emitted. +#[test] +fn invalid_object_or_ring_specs_reject_without_plan() { + // Each case changes one object/ring fact from the valid fixture and names + // the typed rejection expected at the planning boundary. + let cases = [ + ( + invalid_zero_activation_extent(), + PlanRejectionKind::InvalidObjectSpec, + "zero activation extent", + ), + ( + invalid_dtype_width(), + PlanRejectionKind::InvalidObjectSpec, + "invalid dtype width", + ), + ( + invalid_unsupported_shape_or_layout(), + PlanRejectionKind::UnsupportedShapeOrLayout, + "unsupported shape/layout", + ), + ( + invalid_ring_alignment(), + PlanRejectionKind::InvalidRingSpec, + "invalid ring alignment", + ), + ]; + + // Rejection happens before provisioning: the only public output is the + // typed error, never a RunPlan with invalid specs. + for (input, expected, label) in cases { + let err = plan::plan_run(input).expect_err(label); + assert_eq!(err.kind(), expected, "{label}"); + } +} + +// Unknown-node rejection needs the placement to name a node outside the +// orchestrator's candidate pool. The rest of the input stays valid so the +// expected rejection is isolated to authority over node identity. +fn invalid_unknown_node() -> PlannerInput { + let mut input = valid_input(3, 36); + input.placement = PlacementInput::FixedLinear(vec![ + StagePlacement { + stage_index: 0, + node_id: node(10), + }, + StagePlacement { + stage_index: 1, + node_id: node(404), + }, + StagePlacement { + stage_index: 2, + node_id: node(12), + }, + ]); + input +} + +// Duplicate-stage rejection is observable when two placement entries claim the +// same stage index. This checks that the planner does not silently pick one and +// continue with ambiguous authority. +fn invalid_duplicate_stage_assignment() -> PlannerInput { + let mut input = valid_input(3, 36); + input.placement = PlacementInput::FixedLinear(vec![ + StagePlacement { + stage_index: 0, + node_id: node(10), + }, + StagePlacement { + stage_index: 1, + node_id: node(11), + }, + StagePlacement { + stage_index: 1, + node_id: node(12), + }, + ]); + input +} + +// Missing-stage rejection is observable when placement skips an index inside +// `0..stage_count`. This checks that the planner does not invent hidden stage +// ownership to patch an incomplete placement. +fn invalid_missing_stage_assignment() -> PlannerInput { + let mut input = valid_input(3, 36); + input.placement = PlacementInput::FixedLinear(vec![ + StagePlacement { + stage_index: 0, + node_id: node(10), + }, + StagePlacement { + stage_index: 2, + node_id: node(12), + }, + ]); + input +} + +// Zero stages cannot form the MVP pipeline. This fixture isolates the invalid +// stage-count path without adding any other contradictory facts. +fn invalid_zero_stage_count() -> PlannerInput { + valid_input(0, 36) +} + +// Endpoint mismatch rejection needs an input that tries to override the linear +// edge contract. The planner must reject a skipped-stage activation edge rather +// than accepting a non-MVP topology. +fn invalid_edge_endpoint_mismatch() -> PlannerInput { + let mut input = valid_input(3, 36); + input.placement = PlacementInput::FixedLinearWithEdgeOverride { + stages: vec![ + StagePlacement { + stage_index: 0, + node_id: node(10), + }, + StagePlacement { + stage_index: 1, + node_id: node(11), + }, + StagePlacement { + stage_index: 2, + node_id: node(12), + }, + ], + forced_activation_edges: vec![(0, 2)], + }; + input +} + +// The current contract requires one non-empty layer range per stage. Fewer +// layers than stages forces an empty range unless explicitly allowed, so this +// fixture should reject at the model/stage-layout boundary. +fn invalid_model_stage_layout() -> PlannerInput { + let mut input = valid_input(4, 3); + input.placement = linear_placement(4); + input +} + +// Zero sequence length makes activation capacity zero. The planner must reject +// before creating edges whose object specs cannot carry an activation. +fn invalid_zero_activation_extent() -> PlannerInput { + let mut input = valid_input(3, 36); + input.model.max_seq_len = 0; + input +} + +// Dtype width participates directly in activation extent and object layout. +// A zero width is not a valid dtype fact and must reject before planning. +fn invalid_dtype_width() -> PlannerInput { + let mut input = valid_input(3, 36); + input.model.dtype_width_bytes = 0; + input +} + +// Hidden dimension participates directly in activation shape. A zero hidden +// dimension represents an unsupported shape/layout fact for the MVP contract. +fn invalid_unsupported_shape_or_layout() -> PlannerInput { + let mut input = valid_input(3, 36); + input.model.hidden_dim = 0; + input +} + +// Ring alignment must be a usable alignment contract for shared memory and +// device copy boundaries. A non-power-of-two alignment makes the ring spec +// invalid before any edge can be provisioned. +fn invalid_ring_alignment() -> PlannerInput { + let mut input = valid_input(3, 36); + input.activation_ring.alignment = 3; + input +} diff --git a/tests/mvp_system/shared_ring_helper_abi_guarantees.rs b/tests/mvp_system/shared_ring_helper_abi_guarantees.rs new file mode 100644 index 0000000..01f62b4 --- /dev/null +++ b/tests/mvp_system/shared_ring_helper_abi_guarantees.rs @@ -0,0 +1,219 @@ +//! Black-box contract tests for the MVP shared ring helper ABI. +//! +//! These tests intentionally know only the public helper surface: +//! +//! - helper-created bounded SPSC rings +//! - reserve, write, publish, read, consume, and wake operations +//! - cursor snapshots and wake hints observed through the helper +//! +//! They assert the guarantees in +//! `specs/mvp_system/shared_ring_helper_abi_contract.md`. + +use mvp_system::shared_ring_helper_abi as ring; + +// A small ring forces wraparound and full/empty transitions quickly. The helper +// still owns the actual shared-memory atomics and process-local address math. +fn new_ring() -> ring::RingHelperHarness { + ring::RingHelperHarness::create(ring::RingConfig { + node_id: ring::NodeId(10), + ring_id: ring::RingId(7000), + capacity: 8, + producer: ring::EndpointId("producer".into()), + consumer: ring::EndpointId("consumer".into()), + }) +} + +// This helper reads all currently committed bytes through the consumer API. +// It proves visibility through acquire reads instead of peeking at backing +// memory directly. +fn drain_committed(helper: &mut ring::RingHelperHarness) -> Vec { + let readable = helper.consumer_readable(); + let bytes = helper.consumer_read(readable); + helper.consumer_consume(readable); + bytes +} + +// Wake hints must remain hints. This helper checks the public wake enum without +// depending on scheduler internals. +fn assert_wake_is_payload_free(wake: &ring::WakeHint) { + match wake { + ring::WakeHint::RingReadable { ring_id } + | ring::WakeHint::RingWritable { ring_id } => { + assert_eq!(*ring_id, ring::RingId(7000)); + } + } +} + +// This proves ring identity is bounded SPSC, has one producer and one consumer, +// uses unique RingId values, and does not allow stale wakes to alias a +// replacement ring. +#[test] +fn ring_identity_is_unique_bounded_spsc_and_stale_wakes_do_not_alias() { + // Create one ring and inspect its public identity. + let mut helper = new_ring(); + let identity = helper.identity(); + assert_eq!(identity.ring_id, ring::RingId(7000)); + assert_eq!(identity.capacity, 8); + assert_eq!(identity.producer_count, 1); + assert_eq!(identity.consumer_count, 1); + + // Retire it and create a replacement with the same numeric id but a new + // generation. Stale wake from the retired generation must not wake the new + // ring. + let stale_wake = helper.retire_and_capture_stale_wake(); + let mut replacement = ring::RingHelperHarness::create_replacement(identity.ring_id); + replacement.deliver_wake(stale_wake); + assert!(!replacement.wake_log().iter().any(|wake| wake.was_accepted)); +} + +// This proves commit and consume are monotonic logical byte positions, writes +// are invisible before commit, and physical wrap uses cursor modulo capacity. +#[test] +fn cursors_are_monotonic_and_wrap_by_modulo_capacity() { + // Reserve and write without publishing. + let mut helper = new_ring(); + let reservation = helper.producer_reserve(6).expect("space must exist"); + helper.producer_write(&reservation, b"abcdef"); + + // Producer-local write is not readable before commit. + assert_eq!(helper.consumer_readable(), 0); + + // Publishing makes the prefix readable and advances commit. + helper.producer_commit(reservation); + assert_eq!(helper.cursor_snapshot().commit, 6); + assert_eq!(drain_committed(&mut helper), b"abcdef"); + assert_eq!(helper.cursor_snapshot().consume, 6); + + // Wrap the physical index while logical cursors keep increasing. + let wrapped = helper.producer_reserve(5).expect("space must exist after consume"); + helper.producer_write(&wrapped, b"ghijk"); + helper.producer_commit(wrapped); + assert_eq!(helper.cursor_snapshot().commit, 11); + assert_eq!(helper.cursor_snapshot().commit % helper.identity().capacity, 3); + assert_eq!(drain_committed(&mut helper), b"ghijk"); + assert_eq!(helper.cursor_snapshot().consume, 11); +} + +// This proves the producer computes free space from acquired consume, never +// reserves beyond capacity, writes before publishing commit, and emits readable +// wake hints after publication. +#[test] +fn producer_respects_free_space_and_publishes_after_writing() { + // Reserve the full ring and publish it. + let mut helper = new_ring(); + let reservation = helper.producer_reserve(8).expect("full ring reservation fits"); + helper.producer_write(&reservation, b"12345678"); + helper.producer_commit(reservation); + + // With no consumed bytes, producer cannot reserve additional space. + assert!(matches!( + helper.producer_reserve(1), + Err(ring::ReserveError::InsufficientSpace) + )); + + // The readable wake must be a payload-free hint. + let wake = helper + .wake_hints() + .iter() + .find(|wake| matches!(wake, ring::WakeHint::RingReadable { .. })) + .expect("readable wake must be emitted"); + assert_wake_is_payload_free(wake); + + // Consumer sees the written bytes, proving commit was not published before + // the payload became valid. + assert_eq!(drain_committed(&mut helper), b"12345678"); +} + +// This proves the consumer computes readable bytes from acquired commit, never +// reads beyond committed data, advances consume only after release, and emits +// writable wake hints after freeing space. +#[test] +fn consumer_reads_only_committed_bytes_and_releases_after_safe_consume() { + // Publish three committed bytes. + let mut helper = new_ring(); + let reservation = helper.producer_reserve(3).expect("space must exist"); + helper.producer_write(&reservation, b"abc"); + helper.producer_commit(reservation); + + // The consumer cannot read beyond the committed prefix. + assert_eq!(helper.consumer_readable(), 3); + assert!(matches!( + helper.consumer_try_read(4), + Err(ring::ReadError::BeyondCommittedBytes) + )); + + // Reading alone does not release bytes. + assert_eq!(helper.consumer_read(3), b"abc"); + assert_eq!(helper.cursor_snapshot().consume, 0); + + // Consuming releases space and emits a writable hint. + helper.consumer_consume(3); + assert_eq!(helper.cursor_snapshot().consume, 3); + let wake = helper + .wake_hints() + .iter() + .find(|wake| matches!(wake, ring::WakeHint::RingWritable { .. })) + .expect("writable wake must be emitted"); + assert_wake_is_payload_free(wake); +} + +// This proves wake hints carry no byte ranges, counts, pointers, or credits, +// and coalescing cannot hide the only readable or writable transition. +#[test] +fn wake_hints_are_edge_hints_without_hiding_transitions() { + // Create an empty ring and publish one byte, causing empty-to-readable. + let mut helper = new_ring(); + let reservation = helper.producer_reserve(1).expect("space must exist"); + helper.producer_write(&reservation, b"x"); + helper.producer_commit(reservation); + + // The readable transition must be discoverable even if duplicate wakes are + // coalesced. + helper.coalesce_duplicate_wakes(); + assert!(helper.scheduler_state().readable_rings.contains(&ring::RingId(7000))); + + // Fill then release space to cause full-to-writable. + let _ = drain_committed(&mut helper); + helper.coalesce_duplicate_wakes(); + assert!(helper.scheduler_state().writable_rings.contains(&ring::RingId(7000))); + + // Every wake remains a payload-free hint. + for wake in helper.wake_hints() { + assert_wake_is_payload_free(wake); + } +} + +// This proves Python-facing helper operations own shared atomics and wrap math, +// while returned pointers are process-local addresses derived from arena base +// plus arena offsets. +#[test] +fn helper_abi_owns_atomics_wrap_math_and_process_local_pointers() { + // Ask the helper for a process-local view of a layout. + let helper = new_ring(); + let view = helper.map_process_local_view(ring::ArenaBase(0x1000)); + + // The helper returns process-local addresses derived from offsets. + assert_eq!( + view.data_pointer, + ring::ProcessLocalPointer::from_base_plus_offset( + ring::ArenaBase(0x1000), + view.layout.data_offset + ) + ); + + // Python operations use helper calls for cursor and wrap behavior instead + // of implementing atomics directly. + for operation in helper.python_visible_operations() { + match operation { + ring::PythonOperation::ReserveViaHelper { .. } + | ring::PythonOperation::CommitViaHelper { .. } + | ring::PythonOperation::ReadableViaHelper { .. } + | ring::PythonOperation::ConsumeViaHelper { .. } + | ring::PythonOperation::MapPointerViaHelper { .. } => {} + ring::PythonOperation::DirectAtomicAccess { .. } + | ring::PythonOperation::DirectWrapArithmetic { .. } => { + panic!("Python operation bypassed helper ABI: {operation:?}") + } + } + } +} diff --git a/tests/mvp_system/stage_controller_guarantees.rs b/tests/mvp_system/stage_controller_guarantees.rs new file mode 100644 index 0000000..0ce241b --- /dev/null +++ b/tests/mvp_system/stage_controller_guarantees.rs @@ -0,0 +1,360 @@ +//! Black-box contract tests for MVP StageController behavior. +//! +//! These tests intentionally know only the public stage-controller surface: +//! +//! - `ProvisionStage`, worker, edge, object, stop, and fault events in +//! - worker commands, lifecycle events, and teardown events out +//! +//! They assert the guarantees in +//! `specs/mvp_system/stage_controller_contract.md`. + +use mvp_system::stage_controller as stage; + +// This provision fixture represents a single middle stage. It has one inbound +// and one outbound edge so tests can prove the controller uses assigned edges +// without relying on endpoint internals. +fn valid_provision() -> stage::ProvisionStage { + stage::ProvisionStage { + run_id: stage::RunId(7), + authorized_orchestrator: stage::NodeId(99), + node_id: stage::NodeId(11), + stage_index: 1, + stage_count: 3, + layer_range: stage::LayerRange { + start: 12, + end_exclusive: 24, + }, + inbound: stage::EdgeProvision::inbound(stage::EdgeId(7001)), + outbound: stage::EdgeProvision::outbound(stage::EdgeId(7002)), + weight_source: stage::WeightSource::TestArtifact("model.gguf".into()), + } +} + +// The harness exposes only public messages. Tests intentionally do not inspect +// private controller states such as "Preparing" or "Executing"; they infer +// controller behavior from emitted commands and lifecycle events. +fn new_controller() -> stage::StageControllerHarness { + stage::StageControllerHarness::new(stage::NodeId(11)) +} + +// Preparation readiness has four independent prerequisites. Listing them as +// public observations lets tests prove StageReady is a barrier across worker, +// weights, inbound edge, and outbound edge readiness. +fn preparation_ready_events() -> Vec { + vec![ + stage::StageEvent::WorkerReady, + stage::StageEvent::WeightsReady, + stage::StageEvent::InboundEdgeReady { + edge_id: stage::EdgeId(7001), + }, + stage::StageEvent::OutboundEdgeReady { + edge_id: stage::EdgeId(7002), + }, + ] +} + +// This helper provisions and readies a stage through public events. Tests that +// focus on execution use it to avoid duplicating setup while still going through +// the same observable path as production. +fn ready_stage() -> stage::StageControllerHarness { + let mut harness = new_controller(); + harness.observe(stage::StageEvent::ProvisionStage { + from: stage::NodeId(99), + provision: valid_provision(), + }); + for event in preparation_ready_events() { + harness.observe(event); + } + harness +} + +// This proves provisioning is authorized, validated before setup, and does not +// allow a stage to rewire its assigned inbound or outbound edge. +#[test] +fn provisioning_validates_authority_and_assigned_shape_before_setup() { + // Send a valid provision from the authorized orchestrator. + let mut harness = new_controller(); + harness.observe(stage::StageEvent::ProvisionStage { + from: stage::NodeId(99), + provision: valid_provision(), + }); + + // Setup commands should be derived from the provided assignment. + assert!(harness.commands().iter().any(|command| { + matches!( + command, + stage::StageCommand::EstablishInboundEdge { + edge_id: stage::EdgeId(7001), + .. + } + ) + })); + assert!(harness.commands().iter().any(|command| { + matches!( + command, + stage::StageCommand::EstablishOutboundEdge { + edge_id: stage::EdgeId(7002), + .. + } + ) + })); + + // The controller must not emit any command that replaces the provisioned + // edge ids with a locally chosen edge. + assert!(!harness.commands().iter().any(|command| { + matches!(command, stage::StageCommand::RewireEdge { .. }) + })); + + // An unauthorized provision attempt must fault before setup can begin. + let mut unauthorized = new_controller(); + unauthorized.observe(stage::StageEvent::ProvisionStage { + from: stage::NodeId(123), + provision: valid_provision(), + }); + assert!(unauthorized.events().iter().any(|event| { + matches!( + event, + stage::StageLifecycleEvent::StageFault { + reason: stage::StageFaultReason::UnauthorizedProvision, + .. + } + ) + })); + assert!(!unauthorized.commands().iter().any(|command| { + matches!(command, stage::StageCommand::ConfigureWorkerRole { .. }) + })); +} + +// This proves StageReady is emitted only after worker readiness, weight +// readiness, inbound edge readiness, and outbound edge readiness are all +// observed. +#[test] +fn stage_ready_waits_for_worker_weights_and_both_edges() { + // Provision the stage so preparation can begin. + let mut harness = new_controller(); + harness.observe(stage::StageEvent::ProvisionStage { + from: stage::NodeId(99), + provision: valid_provision(), + }); + + // Feed every readiness event except the final one and prove no prefix is + // enough for StageReady. + let mut events = preparation_ready_events(); + let final_event = events.pop().expect("fixture has final setup event"); + for event in events { + harness.observe(event); + assert!(!harness.events().iter().any(|event| { + matches!(event, stage::StageLifecycleEvent::StageReady { .. }) + })); + } + + // The final prerequisite crosses the barrier. + harness.observe(final_event); + + // StageReady appears exactly once for the provisioned stage. + let ready_count = harness + .events() + .iter() + .filter(|event| { + matches!( + event, + stage::StageLifecycleEvent::StageReady { + run_id: stage::RunId(7), + stage_index: 1, + } + ) + }) + .count(); + assert_eq!(ready_count, 1); +} + +// This proves a ready stage admits work only from inbound ObjectLoaded, issues +// one ExecuteStep per accepted object, and binds output with the same sequence. +#[test] +fn accepted_inbound_object_creates_one_same_sequence_execute_step() { + // Bring the stage to ready state through public setup events. + let mut harness = ready_stage(); + + // Deliver the first inbound object, sequence 0. + harness.observe(stage::StageEvent::ObjectLoaded { + edge_id: stage::EdgeId(7001), + object_id: stage::ObjectId(9000), + sequence: 0, + handle: stage::DeviceHandle::new_current(42), + }); + + // Exactly one ExecuteStep command must result from that accepted object. + let execute_steps = harness + .commands() + .iter() + .filter_map(|command| match command { + stage::StageCommand::ExecuteStep(step) => Some(step), + _ => None, + }) + .collect::>(); + assert_eq!(execute_steps.len(), 1); + + // The output binding must preserve the input sequence. + assert_eq!(execute_steps[0].input.sequence, 0); + assert_eq!(execute_steps[0].outputs[0].sequence, 0); + + // A second object while the first step is active must not create another + // active ExecuteStep in the MVP. + harness.observe(stage::StageEvent::ObjectLoaded { + edge_id: stage::EdgeId(7001), + object_id: stage::ObjectId(9001), + sequence: 1, + handle: stage::DeviceHandle::new_current(43), + }); + let active_steps = harness + .commands() + .iter() + .filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_))) + .count(); + assert_eq!(active_steps, 1); +} + +// This proves the sequence contract: sequence 0 is accepted as prefill, decode +// sequences must strictly increase, and duplicate, skipped, or out-of-order +// inputs fault the stage. +#[test] +fn duplicate_skipped_and_out_of_order_sequences_fault() { + // Each invalid trace starts from a freshly readied stage. + let invalid_traces = vec![ + vec![0, 0], + vec![0, 2], + vec![0, 1, 0], + ]; + + for trace in invalid_traces { + // Accept the first object and complete its step when needed so the next + // object is admitted through the normal public path. + let mut harness = ready_stage(); + for (i, sequence) in trace.iter().enumerate() { + harness.observe(stage::StageEvent::ObjectLoaded { + edge_id: stage::EdgeId(7001), + object_id: stage::ObjectId(9000 + i as u64), + sequence: *sequence, + handle: stage::DeviceHandle::new_current(100 + i as u64), + }); + if i + 1 < trace.len() { + harness.observe(stage::StageEvent::StepCompleted { + step_id: stage::StepId(i as u64), + }); + } + } + + // The transcript must contain a sequence fault for the invalid trace. + assert!(harness.events().iter().any(|event| { + matches!( + event, + stage::StageLifecycleEvent::StageFault { + reason: stage::StageFaultReason::SequenceViolation, + .. + } + ) + })); + } +} + +// This proves compute completion is observed only after the worker reports +// StepCompleted, and completion returns the stage to ready-for-next-object. +#[test] +fn step_completed_releases_input_and_admits_next_object() { + // Start one accepted step. + let mut harness = ready_stage(); + harness.observe(stage::StageEvent::ObjectLoaded { + edge_id: stage::EdgeId(7001), + object_id: stage::ObjectId(9000), + sequence: 0, + handle: stage::DeviceHandle::new_current(42), + }); + + // Before worker completion, no compute-complete lifecycle event is allowed. + assert!(!harness.events().iter().any(|event| { + matches!(event, stage::StageLifecycleEvent::StepAccepted { sequence: 1, .. }) + })); + + // Worker StepCompleted is the public completion signal. + harness.observe(stage::StageEvent::StepCompleted { + step_id: stage::StepId(0), + }); + + // The controller releases per-step input according to policy. + assert!(harness.commands().iter().any(|command| { + matches!( + command, + stage::StageCommand::ReleaseInputHandle { + object_id: stage::ObjectId(9000), + .. + } + ) + })); + + // The next sequence is now admissible. + harness.observe(stage::StageEvent::ObjectLoaded { + edge_id: stage::EdgeId(7001), + object_id: stage::ObjectId(9001), + sequence: 1, + handle: stage::DeviceHandle::new_current(43), + }); + let execute_count = harness + .commands() + .iter() + .filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_))) + .count(); + assert_eq!(execute_count, 2); +} + +// This proves worker, object, output-edge, and step failures fault the stage, +// and after fault no new run work is accepted until StopRun. +#[test] +fn stage_fault_rejects_new_work_until_stopped() { + // Start from a ready stage and inject a worker crash. + let mut harness = ready_stage(); + harness.observe(stage::StageEvent::WorkerCrashed); + + // Fault must be visible at the stage boundary. + assert!(harness.events().iter().any(|event| { + matches!( + event, + stage::StageLifecycleEvent::StageFault { + reason: stage::StageFaultReason::WorkerCrashed, + .. + } + ) + })); + + // New work after fault must not produce ExecuteStep. + let before = harness + .commands() + .iter() + .filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_))) + .count(); + harness.observe(stage::StageEvent::ObjectLoaded { + edge_id: stage::EdgeId(7001), + object_id: stage::ObjectId(9999), + sequence: 0, + handle: stage::DeviceHandle::new_current(77), + }); + let after = harness + .commands() + .iter() + .filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_))) + .count(); + assert_eq!(after, before); + + // StopRun moves the stage through local teardown and emits StageStopped. + harness.observe(stage::StageEvent::StopRun { + run_id: stage::RunId(7), + }); + assert!(harness.commands().iter().any(|command| { + matches!(command, stage::StageCommand::StopLocalEdges { .. }) + })); + assert!(harness.commands().iter().any(|command| { + matches!(command, stage::StageCommand::ReleaseRunDeviceObjects { .. }) + })); + assert!(harness.events().iter().any(|event| { + matches!(event, stage::StageLifecycleEvent::StageStopped { .. }) + })); +} diff --git a/tests/mvp_system/tx_rx_edge_actor_guarantees.rs b/tests/mvp_system/tx_rx_edge_actor_guarantees.rs new file mode 100644 index 0000000..d64733b --- /dev/null +++ b/tests/mvp_system/tx_rx_edge_actor_guarantees.rs @@ -0,0 +1,247 @@ +//! Black-box contract tests for MVP Tx and Rx edge actors. +//! +//! These tests intentionally know only the public edge-actor surface: +//! +//! - lifecycle, object identity, stop, stream fault, and object fault events in +//! - role-facing lifecycle/object events out +//! +//! They assert the guarantees in +//! `specs/mvp_system/tx_rx_edge_actor_contract.md`. + +use mvp_system::tx_rx_edge_actor as edge_actor; + +// The edge id fixture gives both actors a shared identity while keeping Tx and +// Rx lifecycle tests independent from driver and ring internals. +fn edge_id() -> edge_actor::EdgeId { + edge_actor::EdgeId(7001) +} + +// Tx starts in provisioning and represents the producer side of one edge. The +// harness records only actor messages, not bytes or flow-control details. +fn new_tx() -> edge_actor::TxActorHarness { + edge_actor::TxActorHarness::new(edge_actor::TxConfig { + edge_id: edge_id(), + role_port: edge_actor::PortId("out".into()), + }) +} + +// Rx starts in provisioning and represents the consumer side of one edge. It +// exposes complete object identities and opaque handles to the role layer. +fn new_rx() -> edge_actor::RxActorHarness { + edge_actor::RxActorHarness::new(edge_actor::RxConfig { + edge_id: edge_id(), + role_port: edge_actor::PortId("in".into()), + }) +} + +// This helper is a compile-time and runtime guard for payload isolation. If the +// public actor message enum grows a payload-bearing variant, this exhaustive +// match has to be updated and the test discussion becomes explicit. +fn assert_actor_message_is_payload_free(message: &edge_actor::ActorMessage) { + match message { + edge_actor::ActorMessage::Lifecycle { .. } + | edge_actor::ActorMessage::ObjectIdentity { .. } + | edge_actor::ActorMessage::OpaqueHandle { .. } + | edge_actor::ActorMessage::CoarseFault { .. } => {} + edge_actor::ActorMessage::PayloadBytes { .. } + | edge_actor::ActorMessage::HostPointer { .. } + | edge_actor::ActorMessage::ByteRange { .. } + | edge_actor::ActorMessage::CreditCount { .. } => { + panic!("edge actor message carried payload or flow-control detail: {message:?}") + } + } +} + +// This proves Tx and Rx actors are tied to one edge id, receive lifecycle/object +// events only, and do not traffic payload bytes, pointers, ranges, credits, or +// free-space counts. +#[test] +fn edge_actor_messages_are_lifecycle_identity_and_handle_only() { + // Create one Tx and one Rx actor for the same edge id. + let mut tx = new_tx(); + let mut rx = new_rx(); + + // Drive typical lifecycle and object events. + tx.observe(edge_actor::TxEvent::EdgeReady { edge_id: edge_id() }); + tx.observe(edge_actor::TxEvent::ObjectProduced { + edge_id: edge_id(), + object_id: edge_actor::ObjectId(9000), + sequence: 0, + }); + rx.observe(edge_actor::RxEvent::EdgeReady { edge_id: edge_id() }); + rx.observe(edge_actor::RxEvent::ObjectLoaded { + edge_id: edge_id(), + object_id: edge_actor::ObjectId(9000), + sequence: 0, + handle: edge_actor::OpaqueHandle::new(42), + }); + + // Every emitted actor message must stay payload-free. + for message in tx.messages().iter().chain(rx.messages()) { + assert_actor_message_is_payload_free(message); + } + + // Both actors remain tied to exactly one edge id. + assert!(tx.messages().iter().all(|message| message.edge_id() == edge_id())); + assert!(rx.messages().iter().all(|message| message.edge_id() == edge_id())); +} + +// This proves Tx starts in provisioning, becomes ready only after EdgeReady, +// allows producing only after ready, reports produced object identity, and moves +// to faulted on stream or object faults. +#[test] +fn tx_lifecycle_gates_production_and_faults_on_stream_or_object_failure() { + // Before EdgeReady, production is rejected. + let mut tx = new_tx(); + tx.observe(edge_actor::TxEvent::ObjectProduced { + edge_id: edge_id(), + object_id: edge_actor::ObjectId(9000), + sequence: 0, + }); + assert!(!tx.messages().iter().any(|message| { + matches!(message, edge_actor::ActorMessage::ObjectIdentity { .. }) + })); + + // EdgeReady admits production. + tx.observe(edge_actor::TxEvent::EdgeReady { edge_id: edge_id() }); + tx.observe(edge_actor::TxEvent::ObjectProduced { + edge_id: edge_id(), + object_id: edge_actor::ObjectId(9000), + sequence: 0, + }); + assert!(tx.messages().iter().any(|message| { + matches!( + message, + edge_actor::ActorMessage::ObjectIdentity { + edge_id: edge_actor::EdgeId(7001), + object_id: edge_actor::ObjectId(9000), + sequence: 0, + .. + } + ) + })); + + // Stream fault moves Tx to faulted and suppresses later production. + tx.observe(edge_actor::TxEvent::StreamFault { edge_id: edge_id() }); + let produced_before = tx + .messages() + .iter() + .filter(|message| matches!(message, edge_actor::ActorMessage::ObjectIdentity { .. })) + .count(); + tx.observe(edge_actor::TxEvent::ObjectProduced { + edge_id: edge_id(), + object_id: edge_actor::ObjectId(9001), + sequence: 1, + }); + let produced_after = tx + .messages() + .iter() + .filter(|message| matches!(message, edge_actor::ActorMessage::ObjectIdentity { .. })) + .count(); + assert_eq!(produced_after, produced_before); +} + +// This proves Rx starts in provisioning, becomes ready only after EdgeReady, +// exposes loaded objects only after ObjectLoaded, and moves to faulted on stream +// or object faults. +#[test] +fn rx_lifecycle_gates_loaded_objects_and_faults_on_stream_or_object_failure() { + // Before EdgeReady, loaded objects are not exposed to the role layer. + let mut rx = new_rx(); + rx.observe(edge_actor::RxEvent::ObjectLoaded { + edge_id: edge_id(), + object_id: edge_actor::ObjectId(9000), + sequence: 0, + handle: edge_actor::OpaqueHandle::new(42), + }); + assert!(!rx.messages().iter().any(|message| { + matches!(message, edge_actor::ActorMessage::OpaqueHandle { .. }) + })); + + // EdgeReady admits ObjectLoaded exposure. + rx.observe(edge_actor::RxEvent::EdgeReady { edge_id: edge_id() }); + rx.observe(edge_actor::RxEvent::ObjectLoaded { + edge_id: edge_id(), + object_id: edge_actor::ObjectId(9000), + sequence: 0, + handle: edge_actor::OpaqueHandle::new(42), + }); + assert!(rx.messages().iter().any(|message| { + matches!( + message, + edge_actor::ActorMessage::OpaqueHandle { + edge_id: edge_actor::EdgeId(7001), + object_id: edge_actor::ObjectId(9000), + sequence: 0, + .. + } + ) + })); + + // Object failure faults Rx and suppresses later loaded objects. + rx.observe(edge_actor::RxEvent::ObjectFailed { edge_id: edge_id() }); + let loaded_before = rx + .messages() + .iter() + .filter(|message| matches!(message, edge_actor::ActorMessage::OpaqueHandle { .. })) + .count(); + rx.observe(edge_actor::RxEvent::ObjectLoaded { + edge_id: edge_id(), + object_id: edge_actor::ObjectId(9001), + sequence: 1, + handle: edge_actor::OpaqueHandle::new(43), + }); + let loaded_after = rx + .messages() + .iter() + .filter(|message| matches!(message, edge_actor::ActorMessage::OpaqueHandle { .. })) + .count(); + assert_eq!(loaded_after, loaded_before); +} + +// This proves StopEdge moves actors toward stopped, stale events after stop are +// ignored, and mismatched edge ids reject or fault according to policy. +#[test] +fn stop_and_mismatched_edge_events_do_not_create_run_work() { + // Ready Tx and Rx actors, then stop them. + let mut tx = new_tx(); + let mut rx = new_rx(); + tx.observe(edge_actor::TxEvent::EdgeReady { edge_id: edge_id() }); + rx.observe(edge_actor::RxEvent::EdgeReady { edge_id: edge_id() }); + tx.observe(edge_actor::TxEvent::StopEdge { edge_id: edge_id() }); + rx.observe(edge_actor::RxEvent::StopEdge { edge_id: edge_id() }); + + // Stale post-stop object events must be ignored. + tx.observe(edge_actor::TxEvent::ObjectProduced { + edge_id: edge_id(), + object_id: edge_actor::ObjectId(9000), + sequence: 0, + }); + rx.observe(edge_actor::RxEvent::ObjectLoaded { + edge_id: edge_id(), + object_id: edge_actor::ObjectId(9000), + sequence: 0, + handle: edge_actor::OpaqueHandle::new(42), + }); + assert!(!tx.messages().iter().any(|message| { + matches!(message, edge_actor::ActorMessage::ObjectIdentity { .. }) + })); + assert!(!rx.messages().iter().any(|message| { + matches!(message, edge_actor::ActorMessage::OpaqueHandle { .. }) + })); + + // A mismatched edge id must reject or fault, not create work on this actor. + let mut mismatched = new_tx(); + mismatched.observe(edge_actor::TxEvent::EdgeReady { + edge_id: edge_actor::EdgeId(9999), + }); + assert!(mismatched.messages().iter().any(|message| { + matches!( + message, + edge_actor::ActorMessage::CoarseFault { + reason: edge_actor::ActorFaultReason::MismatchedEdgeId, + .. + } + ) + })); +} diff --git a/tests/mvp_system/weight_lifecycle_guarantees.rs b/tests/mvp_system/weight_lifecycle_guarantees.rs new file mode 100644 index 0000000..b5b40b2 --- /dev/null +++ b/tests/mvp_system/weight_lifecycle_guarantees.rs @@ -0,0 +1,225 @@ +//! Black-box contract tests for MVP stage-local weight lifecycle. +//! +//! These tests intentionally know only the public weight-work surface: +//! +//! - `ProvisionStage` assignment in +//! - artifact, parse, allocation, binding, and cache outcomes in +//! - `WeightsReady`, `StageReady`, and `StageFault` out +//! +//! They assert the guarantees in +//! `specs/mvp_system/weight_lifecycle_contract.md`. + +use mvp_system::weight_lifecycle as weights; + +// A valid assignment gives the stage exactly one layer range and one source. +// Tests vary only source or failure outcome so the assignment contract remains +// visible. +fn valid_assignment() -> weights::WeightAssignment { + weights::WeightAssignment { + run_id: weights::RunId(7), + stage_index: 1, + plan_layer_range: weights::LayerRange { + start: 12, + end_exclusive: 24, + }, + assigned_layer_range: weights::LayerRange { + start: 12, + end_exclusive: 24, + }, + source: weights::WeightSource::WholeGguf { + uri: "test://model.gguf".into(), + }, + } +} + +// Weight loading is intentionally opaque. The harness accepts public loader and +// worker outcomes and records only stage-visible events and commands. +fn new_weight_harness() -> weights::WeightLifecycleHarness { + weights::WeightLifecycleHarness::new(weights::NodeId(11)) +} + +// The success facts represent the observable prerequisites for WeightsReady: +// artifact bytes exist, the assigned layer range is valid, and the worker has +// loaded or bound that range. +fn successful_load_events() -> Vec { + vec![ + weights::WeightEvent::ArtifactAvailable { + bytes: weights::ArtifactBytes::Local, + }, + weights::WeightEvent::LayerRangeValidated, + weights::WeightEvent::WorkerRangeBound, + ] +} + +// Each failure case maps one public loader or worker failure to the stable +// stage fault reason expected at the control boundary. +fn failure_cases() -> Vec<(weights::WeightEvent, weights::StageFaultReason)> { + vec![ + ( + weights::WeightEvent::DownloadFailed, + weights::StageFaultReason::WeightDownloadFailed, + ), + ( + weights::WeightEvent::ParseFailed, + weights::StageFaultReason::WeightParseFailed, + ), + ( + weights::WeightEvent::DeviceAllocationFailed, + weights::StageFaultReason::DeviceAllocationFailed, + ), + ( + weights::WeightEvent::BindingFailed, + weights::StageFaultReason::WeightBindingFailed, + ), + ( + weights::WeightEvent::InvalidLayerRange, + weights::StageFaultReason::InvalidLayerRange, + ), + ] +} + +// This proves a stage receives weight source and exactly one assigned layer +// range from provisioning, validates it against the plan, and does not claim +// graph-visible ownership outside that range. +#[test] +fn assignment_is_stage_local_and_range_limited() { + // Start weight work from the provisioned assignment. + let mut harness = new_weight_harness(); + harness.observe(weights::WeightEvent::Provisioned(valid_assignment())); + + // The load command may use the physical source, but its graph-visible layer + // range must be the assigned range. + for command in harness.commands() { + if let weights::WeightCommand::LoadOrBindRange { range, .. } = command { + assert_eq!( + *range, + weights::LayerRange { + start: 12, + end_exclusive: 24, + } + ); + } + } + + // There must be no command claiming ownership of neighboring layers. + assert!(!harness.commands().iter().any(|command| { + matches!( + command, + weights::WeightCommand::AdvertiseLoadedLayerRange { + range, + .. + } if range.start < 12 || range.end_exclusive > 24 + ) + })); +} + +// This proves whole GGUF download, shard download, and cache use are physical +// mechanisms with the same public outcome: WeightsReady or StageFault. +#[test] +fn supported_physical_sources_have_same_visible_success_contract() { + // Exercise every supported source without asserting how bytes are obtained. + let sources = vec![ + weights::WeightSource::WholeGguf { + uri: "test://model.gguf".into(), + }, + weights::WeightSource::ShardSet { + uris: vec!["test://model.layers.12-24.gguf".into()], + }, + weights::WeightSource::CachedArtifact { + cache_key: "model:layers:12-24".into(), + }, + ]; + + for source in sources { + // Install the source in an otherwise valid assignment. + let mut assignment = valid_assignment(); + assignment.source = source; + let mut harness = new_weight_harness(); + harness.observe(weights::WeightEvent::Provisioned(assignment)); + + // Drive the same public success facts for every source. + for event in successful_load_events() { + harness.observe(event); + } + + // The system-visible success outcome is WeightsReady. + assert!(harness.events().iter().any(|event| { + matches!( + event, + weights::WeightLifecycleEvent::WeightsReady { + run_id: weights::RunId(7), + stage_index: 1, + } + ) + })); + } +} + +// This proves WeightsReady requires artifact availability, layer validation, +// and worker bind/load completion, and that WeightsReady precedes StageReady. +#[test] +fn weights_ready_requires_all_weight_facts_and_precedes_stage_ready() { + // Start from a valid assignment. + let mut harness = new_weight_harness(); + harness.observe(weights::WeightEvent::Provisioned(valid_assignment())); + + // Feed every success fact except the final one and prove no prefix is + // sufficient for WeightsReady. + let mut events = successful_load_events(); + let final_event = events.pop().expect("fixture has final weight event"); + for event in events { + harness.observe(event); + assert!(!harness.events().iter().any(|event| { + matches!(event, weights::WeightLifecycleEvent::WeightsReady { .. }) + })); + } + + // The final weight prerequisite emits WeightsReady. + harness.observe(final_event); + let weights_ready_pos = harness + .events() + .iter() + .position(|event| matches!(event, weights::WeightLifecycleEvent::WeightsReady { .. })) + .expect("WeightsReady must be emitted"); + + // StageReady may occur only after the StageController observes WeightsReady + // and the other local setup prerequisites. + harness.observe(weights::WeightEvent::OtherStagePrerequisitesReady); + let stage_ready_pos = harness + .events() + .iter() + .position(|event| matches!(event, weights::WeightLifecycleEvent::StageReady { .. })) + .expect("StageReady must be emitted after prerequisites"); + assert!(weights_ready_pos < stage_ready_pos); +} + +// This proves every weight failure source faults the stage and suppresses both +// WeightsReady and StageReady. +#[test] +fn weight_failures_emit_stage_fault_without_readiness() { + // Each failure source gets an isolated attempt. + for (failure_event, expected_reason) in failure_cases() { + let mut harness = new_weight_harness(); + harness.observe(weights::WeightEvent::Provisioned(valid_assignment())); + + // Deliver the public failure outcome from weight work. + harness.observe(failure_event); + + // The stable fault reason must be observable. + assert!(harness.events().iter().any(|event| { + matches!( + event, + weights::WeightLifecycleEvent::StageFault { + reason, + .. + } if *reason == expected_reason + ) + })); + + // Readiness cannot also be emitted after a faulted weight attempt. + assert!(!harness.events().iter().any(|event| { + matches!(event, weights::WeightLifecycleEvent::WeightsReady { .. }) + || matches!(event, weights::WeightLifecycleEvent::StageReady { .. }) + })); + } +}