Zachery Aaron Shores-Chmielewski
8d1588aaf7
Reorganize crates/mvp-system from flat files into domain module trees (chat, node, node_data, observability, orchestration, prompt, staging, transport, worker) with documented mod.rs boundaries, and drop the stale inline spec docs.
- lib.rs: replace ~20 flat mod declarations with one pub-mod-per-domain (chat/node/node_data/observability/orchestration/prompt/staging/transport/worker)
- node/, chat/, observability/, orchestration/, staging/, prompt/, transport/, worker/: add mod.rs files with module-boundary doc comments and re-exports (e.g. chat re-exports run_from_args; orchestration re-exports RunConfig/RunId/GgufSource/TokenizerSource/ProviderKind)
- orchestration: group providers under provider_adapters/{docker_cluster,relay,vastai} and fold engine_builder/, config, run_fsm, run_plan, provisioning, resource_inventory, membership_readiness, and token_endpoint under orchestration/
- transport: consolidate codec registration into transport/codec_registry::register_mvp_actor_codecs (was crate::actors::register_mvp_actor_codecs) and rename actors/codec.rs to transport/json_codec.rs
- rename and relocate files into their domains (arena_manager->node_data/arena, actors/node_agent->node/actor, actors/orchestrator->orchestration/actor, stage_controller->staging/actor, telemetry/dashboard_view/etc->observability/, benchmark_observability->observability::benchmark, edge_establisher->node::edge_lifecycle, prompt_rpc->prompt::rpc) and update all crate:: imports accordingly
- remove the stale crates/mvp-system/specs/*.md (MVP_SYSTEM_MODULE_BOUNDARY_SPEC, mvp_chat, orchestrator) now that module boundaries live in mod.rs docs
Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
83 lines
2.7 KiB
Rust
83 lines
2.7 KiB
Rust
use std::fs::{self, File, OpenOptions};
|
|
use std::io::{BufRead, BufReader, Write};
|
|
use std::path::{Path, PathBuf};
|
|
|
|
use datastream::{Frame, StreamId};
|
|
use serde_json::json;
|
|
|
|
use crate::observability::benchmark;
|
|
|
|
/// JSONL archive for MVP-owned datastream frames.
|
|
///
|
|
/// The datastream crate owns frame transport; this helper owns the MVP archive
|
|
/// record shape used as benchmark and contract evidence.
|
|
pub struct FrameArchive {
|
|
file: File,
|
|
next_seq: u64,
|
|
path: PathBuf,
|
|
label: &'static str,
|
|
}
|
|
|
|
impl FrameArchive {
|
|
pub fn open(path: &Path) -> Result<Self, String> {
|
|
Self::open_with_label(path, "datastream frame log")
|
|
}
|
|
|
|
pub fn open_with_label(path: &Path, label: &'static str) -> Result<Self, String> {
|
|
if let Some(parent) = path.parent()
|
|
&& !parent.as_os_str().is_empty()
|
|
{
|
|
fs::create_dir_all(parent)
|
|
.map_err(|e| format!("create {label} dir {}: {e}", parent.display()))?;
|
|
}
|
|
let next_seq = match File::open(path) {
|
|
Ok(file) => BufReader::new(file).lines().count() as u64,
|
|
Err(error) if error.kind() == std::io::ErrorKind::NotFound => 0,
|
|
Err(error) => return Err(format!("read {label} {}: {error}", path.display())),
|
|
};
|
|
let file = OpenOptions::new()
|
|
.create(true)
|
|
.append(true)
|
|
.open(path)
|
|
.map_err(|e| format!("open {label} {}: {e}", path.display()))?;
|
|
Ok(Self {
|
|
file,
|
|
next_seq,
|
|
path: path.to_path_buf(),
|
|
label,
|
|
})
|
|
}
|
|
|
|
pub fn record(
|
|
&mut self,
|
|
source: &str,
|
|
stream: &StreamId,
|
|
channel: &str,
|
|
frame: &Frame,
|
|
) -> Result<(), String> {
|
|
let payload = match std::str::from_utf8(&frame.payload) {
|
|
Ok(text) => json!({"encoding": "utf8", "value": text}),
|
|
Err(_) => json!({"encoding": "bytes", "value": frame.payload}),
|
|
};
|
|
let record = json!({
|
|
"arrival_seq": self.next_seq,
|
|
"arrival_unix_ms": benchmark::unix_ms_now(),
|
|
"source": source,
|
|
"stream": stream.to_string(),
|
|
"channel": channel,
|
|
"channel_id": frame.channel.0,
|
|
"position": frame.position.0,
|
|
"payload": payload,
|
|
});
|
|
self.next_seq += 1;
|
|
let mut line =
|
|
serde_json::to_vec(&record).map_err(|e| format!("serialize {}: {e}", self.label))?;
|
|
line.push(b'\n');
|
|
self.file
|
|
.write_all(&line)
|
|
.map_err(|e| format!("write {} {}: {e}", self.label, self.path.display()))?;
|
|
self.file
|
|
.flush()
|
|
.map_err(|e| format!("flush {} {}: {e}", self.label, self.path.display()))
|
|
}
|
|
}
|