zacheryasc/swactor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

datastore #41

Merged
zacheryasc merged 3 commits from datastore into master 2026-02-15 17:03:32 +00:00
Conversation 0 Commits 3 Files changed 57 +10538 -14
57 changed files with 10538 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.cargo/config.toml Normal file
View file

@ -0,0 +1,2 @@
[alias]
xtask = "run --package xtask --"

91
Cargo.lock generated
View file

@ -11,6 +11,12 @@ dependencies = [
"gimli",
]
[[package]]
name = "adler2"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
[[package]]
name = "ahash"
version = "0.8.12"
@ -1323,6 +1329,16 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
[[package]]
name = "flate2"
version = "1.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
dependencies = [
"crc32fast",
"miniz_oxide",
]
[[package]]
name = "fnv"
version = "1.0.7"
@ -1831,7 +1847,7 @@ dependencies = [
"tokio",
"tokio-rustls",
"tower-service",
"webpki-roots",
"webpki-roots 1.0.6",
]
[[package]]
@ -2152,7 +2168,7 @@ dependencies = [
"tracing",
"url",
"wasm-bindgen-futures",
"webpki-roots",
"webpki-roots 1.0.6",
]
[[package]]
@ -2304,7 +2320,7 @@ dependencies = [
"tracing",
"url",
"vergen-gitcl",
"webpki-roots",
"webpki-roots 1.0.6",
"ws_stream_wasm",
"z32",
]
@ -2569,6 +2585,16 @@ version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "miniz_oxide"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
dependencies = [
"adler2",
"simd-adler32",
]
[[package]]
name = "mio"
version = "1.1.1"
@ -3723,7 +3749,7 @@ dependencies = [
"wasm-bindgen-futures",
"wasm-streams",
"web-sys",
"webpki-roots",
"webpki-roots 1.0.6",
]
[[package]]
@ -4120,6 +4146,12 @@ version = "3.0.0-rc.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f1880df446116126965eeec169136b2e0251dba37c6223bcc819569550edea3"
[[package]]
name = "simd-adler32"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2"
[[package]]
name = "simdutf8"
version = "0.1.5"
@ -4343,6 +4375,26 @@ dependencies = [
"wat",
]
[[package]]
name = "swactor-datastore"
version = "0.1.0"
dependencies = [
"blake3",
"clap",
"ctrlc",
"distribution",
"proptest",
"runtime-dashboard",
"serde",
"serde_json",
"swactor",
"swactor-std",
"tempfile",
"tiny_http",
"toml",
"ureq",
]
[[package]]
name = "swactor-std"
version = "0.1.0"
@ -4928,6 +4980,24 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "ureq"
version = "2.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d"
dependencies = [
"base64 0.22.1",
"flate2",
"log",
"once_cell",
"rustls",
"rustls-pki-types",
"serde",
"serde_json",
"url",
"webpki-roots 0.26.11",
]
[[package]]
name = "url"
version = "2.5.8"
@ -5557,6 +5627,15 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "webpki-roots"
version = "0.26.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9"
dependencies = [
"webpki-roots 1.0.6",
]
[[package]]
name = "webpki-roots"
version = "1.0.6"
@ -6153,6 +6232,10 @@ dependencies = [
"xml-rs",
]
[[package]]
name = "xtask"
version = "0.1.0"
[[package]]
name = "yoke"
version = "0.8.1"

2
Cargo.toml
View file

@ -1,5 +1,5 @@
[workspace]
members = [".", "crates/python", "crates/wasm", "crates/bin-runner", "crates/simulation", "crates/runtime-dashboard", "crates/distribution", "crates/std", "tests/docker"]
members = [".", "crates/python", "crates/wasm", "crates/bin-runner", "crates/simulation", "crates/runtime-dashboard", "crates/distribution", "crates/std", "crates/datastore", "tests/docker", "xtask"]
exclude = ["tools/depgraph"]
[package]

41
crates/datastore/Cargo.toml Normal file
View file

@ -0,0 +1,41 @@
[package]
name = "swactor-datastore"
version = "0.1.0"
edition = "2024"
[dependencies]
swactor = { path = "../..", features = ["serde", "transport"] }
distribution = { path = "../distribution" }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
blake3 = "1"
tiny_http = { version = "0.12", optional = true }
clap = { version = "4", features = ["derive"], optional = true }
ureq = { version = "2", features = ["json"], optional = true }
ctrlc = { version = "3", optional = true }
runtime-dashboard = { path = "../runtime-dashboard", optional = true }
toml = { version = "0.8", optional = true }
[dev-dependencies]
serde_json = "1"
proptest = "1"
tempfile = "3"
swactor = { path = "../.." }
swactor-std = { path = "../std" }
ureq = { version = "2", features = ["json"] }
tiny_http = "0.12"
runtime-dashboard = { path = "../runtime-dashboard" }
[features]
node = ["dep:tiny_http", "dep:clap", "dep:ctrlc", "dep:runtime-dashboard", "dep:toml"]
cli = ["dep:clap", "dep:ureq"]
[[bin]]
name = "swactor-store-node"
path = "src/bin/store_node.rs"
required-features = ["node"]
[[bin]]
name = "swactor-store"
path = "src/bin/store_cli.rs"
required-features = ["cli"]

141
crates/datastore/README.md Normal file
View file

@ -0,0 +1,141 @@
# swactor-datastore
Distributed content-addressed datastore built on [swactor](../../README.md). Objects are split into fixed-size chunks, identified by their blake3 hash, and replicated across a peer-to-peer network via epidemic gossip.
## Building
Node binary (HTTP server + actor runtime):
```sh
cargo build -p swactor-datastore --features node
```
CLI client:
```sh
cargo build -p swactor-datastore --features cli
```
Both at once:
```sh
cargo build -p swactor-datastore --features node,cli
```
## Node
Start a datastore node:
```sh
swactor-store-node
```
### Flags
| Flag | Default | Description |
|------|---------|-------------|
| `--port` | `9091` | HTTP API port |
| `--storage-path` | *(in-memory)* | Directory for persistent storage |
| `--dashboard-port` | *(disabled)* | Runtime dashboard port |
| `--chunk-size` | `1048576` | Chunk size in bytes (1 MB) |
| `--gc-interval` | `1000` | GC interval in ticks (~100s) |
| `--disseminate-interval` | `50` | Gossip interval in ticks (~5s) |
| `--config` | *(none)* | Path to a TOML config file |
Example with persistent storage and dashboard:
```sh
swactor-store-node --storage-path ./data --dashboard-port 9090
```
### Config file
Create a `store.toml` and pass it with `--config`:
```toml
port = 9091
storage_path = "./my-data"
dashboard_port = 9090
chunk_size = 1048576
gc_interval = 1000
disseminate_interval = 50
```
CLI flags override config file values. Omitted fields use built-in defaults.
```sh
swactor-store-node --config store.toml --port 8080
```
## CLI
The `swactor-store` command talks to a running node over HTTP.
### Status
```sh
swactor-store status
```
### Put
```sh
swactor-store put photo.jpg --name "vacation"
```
### Get (metadata)
```sh
swactor-store get <hash>
```
### Get (download)
```sh
swactor-store get <hash> --output photo.jpg
```
### Delete
```sh
swactor-store delete <hash>
```
### List (local)
```sh
swactor-store list
```
### List (swarm-wide)
```sh
swactor-store list --all
```
### Filter by name
```sh
swactor-store list --name vacation
```
Use `--url` to point at a different node:
```sh
swactor-store --url http://192.168.1.50:9091 list
```
## Web UI
Visit `http://<host>:<port>/` in a browser. The UI supports uploading, listing, downloading, inspecting, and deleting objects — works on desktop and mobile.
## HTTP API
| Method | Path | Description |
|--------|------|-------------|
| `GET` | `/api/status` | Node identity |
| `POST` | `/api/put?name=...` | Upload (body = raw bytes) |
| `GET` | `/api/list` | List objects (`?all=true` for swarm) |
| `GET` | `/api/get?hash=...` | Object metadata + manifest |
| `GET` | `/api/data?hash=...` | Download reassembled binary |
| `POST` | `/api/delete?hash=...` | Delete object |

178
crates/datastore/src/actors/blob_store.rs Normal file
View file

@ -0,0 +1,178 @@
//! BlobStoreActor — content-addressed chunk and manifest storage.
//!
//! Delegates all I/O through a `StorageBackend` trait, allowing pluggable
//! backends (filesystem for MVP, IndexedDB for browser, etc.).
use std::collections::HashSet;
use swactor::actor::{ActorInterface, Ctx};
use crate::messages::{BlobStoreMsg, DatastoreResponse};
use crate::storage::StorageBackend;
use crate::types::{ContentHash, ObjectManifest};
/// Manages chunk and manifest storage via a pluggable backend.
pub struct BlobStoreActor {
backend: Box<dyn StorageBackend>,
}
impl BlobStoreActor {
pub fn new(backend: Box<dyn StorageBackend>) -> Self {
Self { backend }
}
fn handle_write_chunk(
&mut self,
ctx: &Ctx,
hash: ContentHash,
data: Vec<u8>,
reply_to: swactor::actor::ActorAddress,
) {
match self.backend.write_chunk(&hash, &data) {
Ok(()) => {
let _ = ctx.send(reply_to, DatastoreResponse::ChunkStored { hash });
}
Err(e) => {
let _ = ctx.send(
reply_to,
DatastoreResponse::Error {
reason: format!("write chunk failed: {e}"),
},
);
}
}
}
fn handle_read_chunk(
&self,
ctx: &Ctx,
hash: ContentHash,
reply_to: swactor::actor::ActorAddress,
) {
match self.backend.read_chunk(&hash) {
Ok(Some(data)) => {
let _ = ctx.send(reply_to, DatastoreResponse::ChunkOk { hash, data });
}
Ok(None) => {
let _ = ctx.send(reply_to, DatastoreResponse::NotFound);
}
Err(e) => {
let _ = ctx.send(
reply_to,
DatastoreResponse::Error {
reason: format!("read chunk failed: {e}"),
},
);
}
}
}
fn handle_delete_chunk(&mut self, hash: ContentHash) {
let _ = self.backend.delete_chunk(&hash);
}
fn handle_has_chunk(
&self,
ctx: &Ctx,
hash: ContentHash,
reply_to: swactor::actor::ActorAddress,
) {
let exists = self.backend.has_chunk(&hash);
let _ = ctx.send(reply_to, DatastoreResponse::Bool(exists));
}
fn handle_list_chunks(&self, ctx: &Ctx, reply_to: swactor::actor::ActorAddress) {
let hashes = self.backend.list_chunks();
let _ = ctx.send(reply_to, DatastoreResponse::ChunkList { hashes });
}
fn handle_gc_unreferenced(&mut self, referenced: HashSet<ContentHash>) {
let all_chunks = self.backend.list_chunks();
for hash in all_chunks {
if !referenced.contains(&hash) {
let _ = self.backend.delete_chunk(&hash);
}
}
}
fn handle_write_manifest(
&mut self,
ctx: &Ctx,
manifest: ObjectManifest,
reply_to: swactor::actor::ActorAddress,
) {
let hash = manifest.content_hash;
match self.backend.write_manifest(&manifest) {
Ok(()) => {
let _ = ctx.send(reply_to, DatastoreResponse::ManifestStored { hash });
}
Err(e) => {
let _ = ctx.send(
reply_to,
DatastoreResponse::Error {
reason: format!("write manifest failed: {e}"),
},
);
}
}
}
fn handle_read_manifest(
&self,
ctx: &Ctx,
hash: ContentHash,
reply_to: swactor::actor::ActorAddress,
) {
match self.backend.read_manifest(&hash) {
Ok(Some(manifest)) => {
let _ = ctx.send(reply_to, DatastoreResponse::ManifestOk { manifest });
}
Ok(None) => {
let _ = ctx.send(reply_to, DatastoreResponse::NotFound);
}
Err(e) => {
let _ = ctx.send(
reply_to,
DatastoreResponse::Error {
reason: format!("read manifest failed: {e}"),
},
);
}
}
}
}
impl ActorInterface for BlobStoreActor {
type Incoming = BlobStoreMsg;
type Response = DatastoreResponse;
fn handle(&mut self, ctx: &Ctx, msg: BlobStoreMsg) {
match msg {
BlobStoreMsg::WriteChunk {
hash,
data,
reply_to,
} => self.handle_write_chunk(ctx, hash, data, reply_to),
BlobStoreMsg::ReadChunk { hash, reply_to } => {
self.handle_read_chunk(ctx, hash, reply_to)
}
BlobStoreMsg::DeleteChunk { hash } => self.handle_delete_chunk(hash),
BlobStoreMsg::HasChunk { hash, reply_to } => {
self.handle_has_chunk(ctx, hash, reply_to)
}
BlobStoreMsg::ListChunks { reply_to } => {
self.handle_list_chunks(ctx, reply_to)
}
BlobStoreMsg::GcUnreferenced { referenced } => {
self.handle_gc_unreferenced(referenced)
}
BlobStoreMsg::WriteManifest {
manifest,
reply_to,
} => self.handle_write_manifest(ctx, manifest, reply_to),
BlobStoreMsg::ReadManifest { hash, reply_to } => {
self.handle_read_manifest(ctx, hash, reply_to)
}
}
}
}

282
crates/datastore/src/actors/datastore_node.rs Normal file
View file

@ -0,0 +1,282 @@
//! DatastoreNode — coordinator/facade actor for the datastore stack.
//!
//! Encapsulates the internal actor topology (BlobStoreActor, MetadataActor)
//! behind a single address. Callers send high-level commands (Put, Get,
//! Delete, List, Status) and receive responses. Also routes incoming network
//! protocol messages to the appropriate internal actors.
use std::collections::BTreeMap;
use swactor::actor::{ActorAddress, ActorInterface, Ctx};
use distribution::types::NodeId;
use crate::chunking::chunk_blob;
use crate::messages::{
BlobStoreMsg, DatastoreNodeMsg, DatastoreResponse, MetadataMsg,
};
use crate::types::{ContentHash, DatastoreConfig, ObjectEntry};
/// Top-level coordinator actor for the datastore.
///
/// Pure router/facade — delegates all work to BlobStoreActor and MetadataActor.
/// Callers interact with a single address instead of knowing about internal actors.
pub struct DatastoreNode {
node_id: NodeId,
blob_store: ActorAddress,
metadata: ActorAddress,
config: DatastoreConfig,
}
impl DatastoreNode {
pub fn new(
node_id: NodeId,
blob_store: ActorAddress,
metadata: ActorAddress,
config: DatastoreConfig,
) -> Self {
Self {
node_id,
blob_store,
metadata,
config,
}
}
fn handle_put(
&self,
ctx: &Ctx,
data: Vec<u8>,
name: Option<String>,
tags: BTreeMap<String, String>,
reply_to: ActorAddress,
) {
let (content_hash, manifest, chunks) = chunk_blob(&data, self.config.chunk_size);
// Fire-and-forget chunk writes to BlobStoreActor.
// reply_to: self — ChunkStored responses are silently dropped (type mismatch).
for (hash, chunk_data) in chunks {
let _ = ctx.send(
self.blob_store,
BlobStoreMsg::WriteChunk {
hash,
data: chunk_data,
reply_to: ctx.self_addr(),
},
);
}
// Fire-and-forget manifest write to BlobStoreActor.
let _ = ctx.send(
self.blob_store,
BlobStoreMsg::WriteManifest {
manifest: manifest.clone(),
reply_to: ctx.self_addr(),
},
);
// Build ObjectEntry and send to MetadataActor with caller's reply_to.
let entry = ObjectEntry {
content_hash,
name,
node_id: self.node_id,
tags,
size_bytes: data.len() as u64,
created_at: 0,
};
let _ = ctx.send(
self.metadata,
MetadataMsg::PutObject {
entry,
manifest,
reply_to,
},
);
}
fn handle_get(&self, ctx: &Ctx, content_hash: ContentHash, reply_to: ActorAddress) {
let _ = ctx.send(
self.metadata,
MetadataMsg::GetObject {
content_hash,
reply_to,
},
);
}
fn handle_read_chunk(&self, ctx: &Ctx, hash: ContentHash, reply_to: ActorAddress) {
let _ = ctx.send(
self.blob_store,
BlobStoreMsg::ReadChunk { hash, reply_to },
);
}
fn handle_delete(&self, ctx: &Ctx, content_hash: ContentHash, reply_to: ActorAddress) {
let _ = ctx.send(
self.metadata,
MetadataMsg::DeleteObject {
content_hash,
reply_to,
},
);
}
fn handle_list(
&self,
ctx: &Ctx,
name_filter: Option<String>,
all: bool,
reply_to: ActorAddress,
) {
if all {
let _ = ctx.send(
self.metadata,
MetadataMsg::ListSwarm {
name_filter,
reply_to,
},
);
} else {
let _ = ctx.send(
self.metadata,
MetadataMsg::ListLocal {
name_filter,
reply_to,
},
);
}
}
fn handle_status(&self, ctx: &Ctx, reply_to: ActorAddress) {
let _ = ctx.send(
reply_to,
DatastoreResponse::NodeStatus {
node_id: self.node_id,
},
);
}
fn handle_incoming_get_chunk(
&self,
ctx: &Ctx,
request: crate::messages::GetChunkRequest,
reply_to: ActorAddress,
) {
let _ = ctx.send(
self.blob_store,
BlobStoreMsg::ReadChunk {
hash: request.hash,
reply_to,
},
);
}
fn handle_incoming_get_manifest(
&self,
ctx: &Ctx,
request: crate::messages::GetManifestRequest,
reply_to: ActorAddress,
) {
let _ = ctx.send(
self.blob_store,
BlobStoreMsg::ReadManifest {
hash: request.hash,
reply_to,
},
);
}
fn handle_incoming_store_object(
&self,
ctx: &Ctx,
request: crate::messages::StoreObjectRequest,
) {
let _ = ctx.send(
self.metadata,
MetadataMsg::HandleStoreObject {
entry: request.entry,
manifest: None,
},
);
}
fn handle_incoming_find_object(
&self,
ctx: &Ctx,
request: crate::messages::FindObjectRequest,
reply_to: ActorAddress,
) {
let _ = ctx.send(
self.metadata,
MetadataMsg::HandleFindObject {
from: request.from,
content_hash: request.content_hash,
reply_to,
},
);
}
fn handle_incoming_list_objects(
&self,
ctx: &Ctx,
request: crate::messages::ListObjectsRequest,
reply_to: ActorAddress,
) {
let _ = ctx.send(
self.metadata,
MetadataMsg::ListLocal {
name_filter: request.name_filter,
reply_to,
},
);
}
}
impl ActorInterface for DatastoreNode {
type Incoming = DatastoreNodeMsg;
type Response = DatastoreResponse;
fn handle(&mut self, ctx: &Ctx, msg: DatastoreNodeMsg) {
match msg {
DatastoreNodeMsg::Put {
data,
name,
tags,
reply_to,
} => self.handle_put(ctx, data, name, tags, reply_to),
DatastoreNodeMsg::Get {
content_hash,
reply_to,
} => self.handle_get(ctx, content_hash, reply_to),
DatastoreNodeMsg::Delete {
content_hash,
reply_to,
} => self.handle_delete(ctx, content_hash, reply_to),
DatastoreNodeMsg::List {
name_filter,
all,
reply_to,
} => self.handle_list(ctx, name_filter, all, reply_to),
DatastoreNodeMsg::Status { reply_to } => self.handle_status(ctx, reply_to),
DatastoreNodeMsg::ReadChunk { hash, reply_to } => {
self.handle_read_chunk(ctx, hash, reply_to)
}
DatastoreNodeMsg::IncomingGetChunk { request, reply_to } => {
self.handle_incoming_get_chunk(ctx, request, reply_to)
}
DatastoreNodeMsg::IncomingGetManifest { request, reply_to } => {
self.handle_incoming_get_manifest(ctx, request, reply_to)
}
DatastoreNodeMsg::IncomingStoreObject { request } => {
self.handle_incoming_store_object(ctx, request)
}
DatastoreNodeMsg::IncomingFindObject { request, reply_to } => {
self.handle_incoming_find_object(ctx, request, reply_to)
}
DatastoreNodeMsg::IncomingListObjects { request, reply_to } => {
self.handle_incoming_list_objects(ctx, request, reply_to)
}
}
}
}

339
crates/datastore/src/actors/metadata.rs Normal file
View file

@ -0,0 +1,339 @@
//! MetadataActor — object metadata index with DHT overlay.
//!
//! Owns:
//! - Local object index: `HashMap<ContentHash, ObjectEntry>` keyed by content hash
//! - Dissemination queue for DHT replication (reuses `ClusterRegistry` pattern)
//!
//! The metadata DHT is a separate Kademlia overlay from the actor directory.
//! Objects are keyed by `blake3(blob_bytes)` — the content hash of the entire blob.
use std::collections::{HashMap, HashSet};
use swactor::actor::{ActorAddress, ActorInterface, Ctx};
use distribution::types::NodeId;
use crate::messages::{BlobStoreMsg, DatastoreResponse, MetadataMsg};
use crate::types::{ContentHash, DatastoreConfig, ObjectEntry, ObjectManifest};
// ─── Dissemination entry (reuses registry.rs pattern) ───────────────────────
#[derive(Debug, Clone)]
struct DisseminationEntry {
entry: ObjectEntry,
manifest: Option<ObjectManifest>,
remaining: usize,
}
// ─── MetadataActor ──────────────────────────────────────────────────────────
/// Manages the object metadata index for a single node.
pub struct MetadataActor {
/// This node's identity.
node_id: NodeId,
/// Local object index: content_hash → ObjectEntry.
entries: HashMap<ContentHash, ObjectEntry>,
/// Local manifest cache: content_hash → ObjectManifest.
manifests: HashMap<ContentHash, ObjectManifest>,
/// Pending entries to disseminate to DHT peers.
dissemination: Vec<DisseminationEntry>,
/// Tick counter for periodic GC.
tick_count: u64,
/// Configuration.
gc_interval: u64,
/// Dissemination multiplier (Λ) — same role as in SWIM.
dissemination_lambda: usize,
/// Address of the local BlobStoreActor (for forwarding manifest writes).
blob_store_addr: Option<ActorAddress>,
/// Addresses of peer MetadataActors for epidemic dissemination.
peers: Vec<ActorAddress>,
}
impl MetadataActor {
pub fn new(node_id: NodeId, config: &DatastoreConfig) -> Self {
Self {
node_id,
entries: HashMap::new(),
manifests: HashMap::new(),
dissemination: Vec::new(),
tick_count: 0,
gc_interval: config.gc_interval,
dissemination_lambda: 3,
blob_store_addr: None,
peers: Vec::new(),
}
}
/// Set the address of the co-located BlobStoreActor.
pub fn set_blob_store(&mut self, addr: ActorAddress) {
self.blob_store_addr = Some(addr);
}
fn transmit_budget(&self, cluster_size: usize) -> usize {
let n = cluster_size.max(2) as f64;
let log_n = n.log2().ceil() as usize;
self.dissemination_lambda * log_n.max(1)
}
fn enqueue(&mut self, entry: ObjectEntry, manifest: Option<ObjectManifest>, cluster_size: usize) {
let budget = self.transmit_budget(cluster_size);
// Replace existing entry for same content hash if present.
if let Some(existing) = self
.dissemination
.iter_mut()
.find(|e| e.entry.content_hash == entry.content_hash)
{
existing.entry = entry;
if manifest.is_some() {
existing.manifest = manifest;
}
existing.remaining = budget;
return;
}
self.dissemination.push(DisseminationEntry {
entry,
manifest,
remaining: budget,
});
}
/// Take pending entries for dissemination, up to `max_count`.
/// Returns `(ObjectEntry, Option<ObjectManifest>)` pairs.
pub fn take_pending(&mut self, max_count: usize) -> Vec<(ObjectEntry, Option<ObjectManifest>)> {
let count = max_count.min(self.dissemination.len());
let mut result = Vec::with_capacity(count);
for entry in self.dissemination.iter_mut().take(count) {
result.push((entry.entry.clone(), entry.manifest.clone()));
entry.remaining = entry.remaining.saturating_sub(1);
}
// Evict exhausted entries.
self.dissemination.retain(|e| e.remaining > 0);
result
}
/// Periodic GC: build referenced chunk set from all manifests and send
/// `GcUnreferenced` to BlobStoreActor to delete orphaned chunks.
fn gc_tick(&mut self, ctx: &Ctx) {
self.tick_count += 1;
if self.tick_count % self.gc_interval != 0 {
return;
}
let blob_store_addr = match self.blob_store_addr {
Some(addr) => addr,
None => return,
};
let mut referenced = HashSet::new();
for manifest in self.manifests.values() {
for chunk_ref in &manifest.chunks {
referenced.insert(chunk_ref.hash);
}
}
let _ = ctx.send(blob_store_addr, BlobStoreMsg::GcUnreferenced { referenced });
}
// ─── Message handlers ───────────────────────────────────────────────
fn handle_put_object(
&mut self,
ctx: &Ctx,
entry: ObjectEntry,
manifest: ObjectManifest,
reply_to: ActorAddress,
) {
let content_hash = entry.content_hash;
// Store manifest locally.
self.manifests.insert(content_hash, manifest.clone());
// Insert entry keyed by content hash.
let mut entry = entry;
entry.node_id = self.node_id;
self.entries.insert(content_hash, entry.clone());
// Enqueue for DHT dissemination (include manifest for peer replication).
self.enqueue(entry, Some(manifest), 3);
let _ = ctx.send(
reply_to,
DatastoreResponse::PutOk { content_hash },
);
}
fn handle_get_object(&self, ctx: &Ctx, content_hash: ContentHash, reply_to: ActorAddress) {
match self.entries.get(&content_hash) {
Some(entry) => {
if let Some(manifest) = self.manifests.get(&content_hash) {
let _ = ctx.send(
reply_to,
DatastoreResponse::GetOk {
entry: entry.clone(),
manifest: manifest.clone(),
},
);
} else {
let _ = ctx.send(
reply_to,
DatastoreResponse::Error {
reason: format!("manifest not found for content hash: {content_hash}"),
},
);
}
}
None => {
let _ = ctx.send(reply_to, DatastoreResponse::NotFound);
}
}
}
fn handle_delete_object(&mut self, ctx: &Ctx, content_hash: ContentHash, reply_to: ActorAddress) {
if self.entries.remove(&content_hash).is_some() {
self.manifests.remove(&content_hash);
let _ = ctx.send(reply_to, DatastoreResponse::DeleteOk { content_hash });
} else {
let _ = ctx.send(reply_to, DatastoreResponse::NotFound);
}
}
fn handle_list_local(
&self,
ctx: &Ctx,
name_filter: Option<String>,
reply_to: ActorAddress,
) {
let entries: Vec<ObjectEntry> = self
.entries
.values()
.filter(|e| {
match (&name_filter, &e.name) {
(Some(filter), Some(name)) => name.contains(filter.as_str()),
(Some(_), None) => false,
(None, _) => true,
}
})
.cloned()
.collect();
let _ = ctx.send(reply_to, DatastoreResponse::ListOk { entries });
}
fn handle_list_swarm(
&self,
ctx: &Ctx,
name_filter: Option<String>,
reply_to: ActorAddress,
) {
// Delegates to local index. Swarm-wide fan-out to peer MetadataActors
// will be wired when networking integration is added.
self.handle_list_local(ctx, name_filter, reply_to);
}
fn handle_find_object(
&self,
ctx: &Ctx,
_from: NodeId,
content_hash: ContentHash,
reply_to: ActorAddress,
) {
match self.entries.get(&content_hash) {
Some(entry) => {
let _ = ctx.send(
reply_to,
DatastoreResponse::GetOk {
entry: entry.clone(),
manifest: self
.manifests
.get(&content_hash)
.cloned()
.unwrap_or_else(|| ObjectManifest {
content_hash,
chunks: vec![],
total_size: entry.size_bytes,
chunk_size: 0,
content_type: None,
}),
},
);
}
None => {
let _ = ctx.send(reply_to, DatastoreResponse::NotFound);
}
}
}
fn handle_set_peers(&mut self, peers: Vec<ActorAddress>) {
self.peers = peers;
}
fn handle_disseminate_tick(&mut self, ctx: &Ctx) {
if self.peers.is_empty() {
return;
}
let pending = self.take_pending(10);
for (entry, manifest) in pending {
for &peer in &self.peers {
let _ = ctx.send(peer, MetadataMsg::HandleStoreObject {
entry: entry.clone(),
manifest: manifest.clone(),
});
}
}
}
fn handle_store_object(&mut self, entry: ObjectEntry, manifest: Option<ObjectManifest>) {
// Insert if absent — content-addressed entries don't conflict.
let content_hash = entry.content_hash;
if !self.entries.contains_key(&content_hash) {
if let Some(ref m) = manifest {
self.manifests.insert(content_hash, m.clone());
}
self.entries.insert(content_hash, entry.clone());
self.enqueue(entry, manifest, 3);
}
}
}
impl ActorInterface for MetadataActor {
type Incoming = MetadataMsg;
type Response = DatastoreResponse;
fn handle(&mut self, ctx: &Ctx, msg: MetadataMsg) {
match msg {
MetadataMsg::PutObject {
entry,
manifest,
reply_to,
} => self.handle_put_object(ctx, entry, manifest, reply_to),
MetadataMsg::GetObject { content_hash, reply_to } => {
self.handle_get_object(ctx, content_hash, reply_to)
}
MetadataMsg::DeleteObject { content_hash, reply_to } => {
self.handle_delete_object(ctx, content_hash, reply_to)
}
MetadataMsg::ListLocal { name_filter, reply_to } => {
self.handle_list_local(ctx, name_filter, reply_to)
}
MetadataMsg::ListSwarm { name_filter, reply_to } => {
self.handle_list_swarm(ctx, name_filter, reply_to)
}
MetadataMsg::HandleFindObject {
from,
content_hash,
reply_to,
} => self.handle_find_object(ctx, from, content_hash, reply_to),
MetadataMsg::HandleStoreObject { entry, manifest } => {
self.handle_store_object(entry, manifest)
}
MetadataMsg::SetPeers { peers } => self.handle_set_peers(peers),
MetadataMsg::DisseminateTick => self.handle_disseminate_tick(ctx),
MetadataMsg::GcTick => self.gc_tick(ctx),
}
}
}

9
crates/datastore/src/actors/mod.rs Normal file
View file

@ -0,0 +1,9 @@
pub mod blob_store;
pub mod datastore_node;
pub mod metadata;
pub mod transfer;
pub use blob_store::BlobStoreActor;
pub use datastore_node::DatastoreNode;
pub use metadata::MetadataActor;
pub use transfer::TransferActor;

180
crates/datastore/src/actors/transfer.rs Normal file
View file

@ -0,0 +1,180 @@
//! TransferActor — ephemeral actor for downloading an object from a remote node.
//!
//! One TransferActor is spawned per download. It walks the manifest's chunk list,
//! requests each chunk from the source node's BlobStoreActor, forwards received
//! chunks to the local BlobStoreActor for persistence, and replies to the
//! original requester when all chunks are received (or on failure).
//!
//! Sequential chunk fetching for MVP (parallel fetching planned for later).
//! Self-terminates via `ctx.stop_self()` on completion, failure, or cancel.
use std::collections::HashSet;
use swactor::actor::{ActorAddress, ActorInterface, Ctx};
use distribution::types::NodeId;
use crate::messages::{DatastoreResponse, TransferMsg};
use crate::types::{ContentHash, ObjectManifest, TransferStatus};
/// Ephemeral actor that manages a single object download.
pub struct TransferActor {
/// The manifest describing which chunks to download.
manifest: Option<ObjectManifest>,
/// The remote node to fetch chunks from.
source_node: Option<NodeId>,
/// Address to send the final result to.
reply_to: Option<ActorAddress>,
/// Address of the local BlobStoreActor for storing received chunks.
blob_store_addr: ActorAddress,
/// Chunk hashes still pending download.
pending: HashSet<ContentHash>,
/// Chunk hashes successfully received and stored.
received: HashSet<ContentHash>,
/// Current status of the transfer.
status: TransferStatus,
/// Number of retry attempts per chunk.
max_retries: usize,
/// Tracks which chunks have been retried and how many times.
retry_counts: std::collections::HashMap<ContentHash, usize>,
}
impl TransferActor {
/// Create a new transfer actor.
///
/// `blob_store_addr` is the address of the local BlobStoreActor where
/// downloaded chunks will be persisted.
pub fn new(blob_store_addr: ActorAddress) -> Self {
Self {
manifest: None,
source_node: None,
reply_to: None,
blob_store_addr,
pending: HashSet::new(),
received: HashSet::new(),
status: TransferStatus::Downloading {
chunks_received: 0,
chunks_total: 0,
},
max_retries: 1,
retry_counts: std::collections::HashMap::new(),
}
}
fn handle_start_download(
&mut self,
_ctx: &Ctx,
manifest: ObjectManifest,
source_node: NodeId,
reply_to: ActorAddress,
) {
let total = manifest.chunks.len();
self.pending = manifest.chunks.iter().map(|c| c.hash).collect();
self.manifest = Some(manifest);
self.source_node = Some(source_node);
self.reply_to = Some(reply_to);
self.status = TransferStatus::Downloading {
chunks_received: 0,
chunks_total: total,
};
// Chunks are fed externally via ChunkReceived/ChunkFailed messages.
// In production, a network adapter (or DatastoreNode) reads chunks from
// the remote BlobStoreActor and forwards them here. In simulation, the
// test harness plays this role.
}
fn handle_chunk_received(
&mut self,
ctx: &Ctx,
hash: ContentHash,
data: Vec<u8>,
) {
if !self.pending.remove(&hash) {
return; // Duplicate or unexpected chunk.
}
// Forward chunk to local BlobStoreActor for persistence.
let _ = ctx.send(
self.blob_store_addr,
crate::messages::BlobStoreMsg::WriteChunk {
hash,
data,
reply_to: ctx.self_addr(),
},
);
self.received.insert(hash);
let total = self.received.len() + self.pending.len();
self.status = TransferStatus::Downloading {
chunks_received: self.received.len(),
chunks_total: total,
};
// Check if all chunks are received.
if self.pending.is_empty() {
self.status = TransferStatus::Complete;
if let (Some(manifest), Some(reply_to)) = (&self.manifest, self.reply_to) {
let _ = ctx.send(
reply_to,
DatastoreResponse::TransferComplete {
content_hash: manifest.content_hash,
},
);
}
ctx.stop_self();
}
}
fn handle_chunk_failed(
&mut self,
ctx: &Ctx,
hash: ContentHash,
reason: String,
) {
let retries = self.retry_counts.entry(hash).or_insert(0);
if *retries < self.max_retries {
*retries += 1;
// Retry is tracked; the external driver (network adapter or test
// harness) is expected to re-send the chunk on retry.
return;
}
// Exhausted retries — fail the whole transfer.
self.status = TransferStatus::Failed {
reason: reason.clone(),
};
if let Some(reply_to) = self.reply_to {
let _ = ctx.send(reply_to, DatastoreResponse::TransferFailed { reason });
}
ctx.stop_self();
}
fn handle_cancel(&mut self, ctx: &Ctx) {
self.status = TransferStatus::Cancelled;
ctx.stop_self();
}
}
impl ActorInterface for TransferActor {
type Incoming = TransferMsg;
type Response = DatastoreResponse;
fn handle(&mut self, ctx: &Ctx, msg: TransferMsg) {
match msg {
TransferMsg::StartDownload {
manifest,
source_node,
reply_to,
} => self.handle_start_download(ctx, manifest, source_node, reply_to),
TransferMsg::ChunkReceived { hash, data } => {
self.handle_chunk_received(ctx, hash, data)
}
TransferMsg::ChunkFailed { hash, reason } => {
self.handle_chunk_failed(ctx, hash, reason)
}
TransferMsg::Cancel => self.handle_cancel(ctx),
}
}
}

793
crates/datastore/src/api.rs Normal file
View file

@ -0,0 +1,793 @@
//! HTTP API for the datastore node.
//!
//! Bridges HTTP requests to actor messages using `Runtime::new_inbox()` +
//! `try_recv()` polling for synchronous request/response with actors.
use std::collections::BTreeMap;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::thread;
use std::time::{Duration, Instant};
use swactor::actor::ActorAddress;
use swactor::runtime::{Inbox, Runtime};
use crate::chunking::reassemble_blob;
use crate::messages::{BlobStoreMsg, DatastoreNodeMsg, DatastoreResponse, MetadataMsg};
use crate::metrics::DatastoreMetrics;
use crate::types::ContentHash;
/// Per-peer actor addresses needed for remote operations.
#[derive(Clone)]
pub struct PeerInfo {
pub metadata: ActorAddress,
pub blob_store: ActorAddress,
}
/// Shared state passed to HTTP handler threads.
struct ApiState {
runtime: Arc<Runtime>,
datastore_addr: ActorAddress,
metadata_addr: ActorAddress,
blob_store_addr: ActorAddress,
peers: Arc<Mutex<Vec<PeerInfo>>>,
metrics: Arc<DatastoreMetrics>,
}
const POLL_TIMEOUT: Duration = Duration::from_secs(5);
const POLL_INTERVAL: Duration = Duration::from_millis(1);
/// Poll an inbox for a response with timeout.
fn poll_response(inbox: &Inbox<DatastoreResponse>, timeout: Duration) -> Option<DatastoreResponse> {
let start = Instant::now();
loop {
if let Some(resp) = inbox.try_recv() {
return Some(resp);
}
if start.elapsed() > timeout {
return None;
}
thread::sleep(POLL_INTERVAL);
}
}
fn respond_json(request: tiny_http::Request, json: &str) {
let response = tiny_http::Response::from_string(json).with_header(
"Content-Type: application/json"
.parse::<tiny_http::Header>()
.unwrap(),
);
let _ = request.respond(response);
}
fn respond_bytes(request: tiny_http::Request, data: &[u8]) {
let response = tiny_http::Response::from_data(data.to_vec()).with_header(
"Content-Type: application/octet-stream"
.parse::<tiny_http::Header>()
.unwrap(),
);
let _ = request.respond(response);
}
fn respond_html(request: tiny_http::Request) {
let response =
tiny_http::Response::from_string(crate::ui_html::DATASTORE_UI_HTML).with_header(
"Content-Type: text/html; charset=utf-8"
.parse::<tiny_http::Header>()
.unwrap(),
);
let _ = request.respond(response);
}
fn respond_error(request: tiny_http::Request, status: u16, msg: &str) {
let json = serde_json::json!({ "error": msg }).to_string();
let response = tiny_http::Response::from_string(json)
.with_status_code(status)
.with_header(
"Content-Type: application/json"
.parse::<tiny_http::Header>()
.unwrap(),
);
let _ = request.respond(response);
}
fn parse_query_string(url: &str) -> BTreeMap<String, String> {
let mut params = BTreeMap::new();
if let Some(qs) = url.split('?').nth(1) {
for pair in qs.split('&') {
let mut kv = pair.splitn(2, '=');
if let (Some(k), Some(v)) = (kv.next(), kv.next()) {
params.insert(
url_decode(k),
url_decode(v),
);
}
}
}
params
}
fn url_decode(s: &str) -> String {
let mut result = String::with_capacity(s.len());
let mut chars = s.bytes();
while let Some(b) = chars.next() {
match b {
b'%' => {
let hi = chars.next().and_then(hex_val);
let lo = chars.next().and_then(hex_val);
if let (Some(h), Some(l)) = (hi, lo) {
result.push((h << 4 | l) as char);
}
}
b'+' => result.push(' '),
_ => result.push(b as char),
}
}
result
}
fn hex_val(b: u8) -> Option<u8> {
match b {
b'0'..=b'9' => Some(b - b'0'),
b'a'..=b'f' => Some(b - b'a' + 10),
b'A'..=b'F' => Some(b - b'A' + 10),
_ => None,
}
}
// ── JSON serialization helpers ───────────────────────────────────────────
//
// ContentHash/NodeId derive Serialize as byte arrays ([u8; 32]).
// The API should expose them as hex strings. These helpers convert
// domain types into JSON with human-readable hex fields.
fn entry_to_json(entry: &crate::types::ObjectEntry) -> serde_json::Value {
let node_hex: String = entry.node_id.0.iter().map(|b| format!("{b:02x}")).collect();
serde_json::json!({
"content_hash": entry.content_hash.to_hex(),
"name": entry.name,
"node_id": node_hex,
"tags": entry.tags,
"size_bytes": entry.size_bytes,
"created_at": entry.created_at,
})
}
fn manifest_to_json(manifest: &crate::types::ObjectManifest) -> serde_json::Value {
let chunks: Vec<serde_json::Value> = manifest
.chunks
.iter()
.map(|c| {
serde_json::json!({
"hash": c.hash.to_hex(),
"offset": c.offset,
"size": c.size,
})
})
.collect();
serde_json::json!({
"content_hash": manifest.content_hash.to_hex(),
"chunks": chunks,
"total_size": manifest.total_size,
"chunk_size": manifest.chunk_size,
"content_type": manifest.content_type,
})
}
fn entries_to_json(entries: &[crate::types::ObjectEntry]) -> Vec<serde_json::Value> {
entries.iter().map(entry_to_json).collect()
}
// ── PUT handler ─────────────────────────────────────────────────────────
fn handle_put(mut request: tiny_http::Request, url: &str, state: &ApiState) {
let params = parse_query_string(url);
let name = params.get("name").cloned();
// Collect tags from query params (skip "name")
let mut tags = BTreeMap::new();
for (k, v) in &params {
if k != "name" {
tags.insert(k.clone(), v.clone());
}
}
// Read body
let mut body = Vec::new();
if request.as_reader().read_to_end(&mut body).is_err() {
// Can't respond — request consumed
return;
}
let body_len = body.len();
let inbox = match state.runtime.new_inbox::<DatastoreResponse>() {
Ok(i) => i,
Err(_) => {
respond_error(request, 500, "failed to create inbox");
return;
}
};
let _ = state.runtime.send_to(
state.datastore_addr,
DatastoreNodeMsg::Put {
data: body,
name,
tags,
reply_to: *inbox.addr(),
},
);
match poll_response(&inbox, POLL_TIMEOUT) {
Some(DatastoreResponse::PutOk { content_hash }) => {
let hex = content_hash.to_hex();
state.metrics.record_put(
&hex,
params.get("name").map(|s| s.as_str()),
body_len as u64,
);
let json = serde_json::json!({ "content_hash": hex }).to_string();
respond_json(request, &json);
}
Some(DatastoreResponse::Error { reason }) => {
respond_error(request, 500, &reason);
}
_ => {
respond_error(request, 504, "timeout waiting for put response");
}
}
}
// ── GET handler (metadata) ──────────────────────────────────────────────
fn handle_get(request: tiny_http::Request, url: &str, state: &ApiState) {
let params = parse_query_string(url);
let hash_hex = match params.get("hash") {
Some(h) => h,
None => {
respond_error(request, 400, "missing ?hash= parameter");
return;
}
};
let content_hash = match ContentHash::from_hex(hash_hex) {
Some(h) => h,
None => {
respond_error(request, 400, "invalid content hash hex");
return;
}
};
let inbox = match state.runtime.new_inbox::<DatastoreResponse>() {
Ok(i) => i,
Err(_) => {
respond_error(request, 500, "failed to create inbox");
return;
}
};
let _ = state.runtime.send_to(
state.datastore_addr,
DatastoreNodeMsg::Get {
content_hash,
reply_to: *inbox.addr(),
},
);
match poll_response(&inbox, POLL_TIMEOUT) {
Some(DatastoreResponse::GetOk { entry, manifest }) => {
state.metrics.record_get(&content_hash.to_hex());
let json = serde_json::json!({
"entry": entry_to_json(&entry),
"manifest": manifest_to_json(&manifest),
})
.to_string();
respond_json(request, &json);
}
Some(DatastoreResponse::NotFound) => {
respond_error(request, 404, "not found");
}
Some(DatastoreResponse::Error { reason }) => {
respond_error(request, 500, &reason);
}
_ => {
respond_error(request, 504, "timeout");
}
}
}
// ── DATA handler (reassembled binary) ───────────────────────────────────
fn handle_data(request: tiny_http::Request, url: &str, state: &ApiState) {
let params = parse_query_string(url);
let hash_hex = match params.get("hash") {
Some(h) => h,
None => {
respond_error(request, 400, "missing ?hash= parameter");
return;
}
};
let content_hash = match ContentHash::from_hex(hash_hex) {
Some(h) => h,
None => {
respond_error(request, 400, "invalid content hash hex");
return;
}
};
// Step 1: Get entry + manifest (try local first)
let inbox = match state.runtime.new_inbox::<DatastoreResponse>() {
Ok(i) => i,
Err(_) => {
respond_error(request, 500, "failed to create inbox");
return;
}
};
let _ = state.runtime.send_to(
state.datastore_addr,
DatastoreNodeMsg::Get {
content_hash,
reply_to: *inbox.addr(),
},
);
state.metrics.record_get(&content_hash.to_hex());
let (entry, manifest) = match poll_response(&inbox, POLL_TIMEOUT) {
Some(DatastoreResponse::GetOk { entry, manifest }) => (entry, manifest),
Some(DatastoreResponse::NotFound) => {
// Try remote GET
match try_remote_get(content_hash, state) {
Some((e, m)) => (e, m),
None => {
respond_error(request, 404, "not found");
return;
}
}
}
Some(DatastoreResponse::Error { reason }) => {
respond_error(request, 500, &reason);
return;
}
_ => {
respond_error(request, 504, "timeout");
return;
}
};
// Step 2: Read all chunks
let _ = entry; // entry used for metadata context, manifest for chunks
let mut chunk_data = Vec::new();
for chunk_ref in &manifest.chunks {
let chunk_inbox = match state.runtime.new_inbox::<DatastoreResponse>() {
Ok(i) => i,
Err(_) => {
respond_error(request, 500, "failed to create inbox");
return;
}
};
let _ = state.runtime.send_to(
state.datastore_addr,
DatastoreNodeMsg::ReadChunk {
hash: chunk_ref.hash,
reply_to: *chunk_inbox.addr(),
},
);
match poll_response(&chunk_inbox, POLL_TIMEOUT) {
Some(DatastoreResponse::ChunkOk { hash, data }) => {
chunk_data.push((hash, data));
}
_ => {
respond_error(request, 500, "failed to read chunk");
return;
}
}
}
// Step 3: Reassemble
match reassemble_blob(&manifest, &chunk_data) {
Ok(data) => respond_bytes(request, &data),
Err(e) => respond_error(request, 500, &format!("reassembly failed: {e:?}")),
}
}
// ── DELETE handler ──────────────────────────────────────────────────────
fn handle_delete(request: tiny_http::Request, url: &str, state: &ApiState) {
let params = parse_query_string(url);
let hash_hex = match params.get("hash") {
Some(h) => h,
None => {
respond_error(request, 400, "missing ?hash= parameter");
return;
}
};
let content_hash = match ContentHash::from_hex(hash_hex) {
Some(h) => h,
None => {
respond_error(request, 400, "invalid content hash hex");
return;
}
};
let inbox = match state.runtime.new_inbox::<DatastoreResponse>() {
Ok(i) => i,
Err(_) => {
respond_error(request, 500, "failed to create inbox");
return;
}
};
let _ = state.runtime.send_to(
state.datastore_addr,
DatastoreNodeMsg::Delete {
content_hash,
reply_to: *inbox.addr(),
},
);
match poll_response(&inbox, POLL_TIMEOUT) {
Some(DatastoreResponse::DeleteOk { content_hash }) => {
let hex = content_hash.to_hex();
state.metrics.record_delete(&hex, 0);
let json = serde_json::json!({ "content_hash": hex }).to_string();
respond_json(request, &json);
}
Some(DatastoreResponse::NotFound) => {
respond_error(request, 404, "not found");
}
Some(DatastoreResponse::Error { reason }) => {
respond_error(request, 500, &reason);
}
_ => {
respond_error(request, 504, "timeout");
}
}
}
// ── LIST handler ────────────────────────────────────────────────────────
fn handle_list(request: tiny_http::Request, url: &str, state: &ApiState) {
let params = parse_query_string(url);
let name_filter = params.get("name").cloned();
let all = params.get("all").map_or(false, |v| v == "true" || v == "1");
if all {
handle_list_swarm(request, name_filter, state);
} else {
handle_list_local(request, name_filter, state);
}
}
fn handle_list_local(
request: tiny_http::Request,
name_filter: Option<String>,
state: &ApiState,
) {
let inbox = match state.runtime.new_inbox::<DatastoreResponse>() {
Ok(i) => i,
Err(_) => {
respond_error(request, 500, "failed to create inbox");
return;
}
};
let _ = state.runtime.send_to(
state.datastore_addr,
DatastoreNodeMsg::List {
name_filter,
all: false,
reply_to: *inbox.addr(),
},
);
match poll_response(&inbox, POLL_TIMEOUT) {
Some(DatastoreResponse::ListOk { entries }) => {
let json = serde_json::json!({ "entries": entries_to_json(&entries) }).to_string();
respond_json(request, &json);
}
Some(DatastoreResponse::Error { reason }) => {
respond_error(request, 500, &reason);
}
_ => {
respond_error(request, 504, "timeout");
}
}
}
/// ListSwarm fan-out: query local + all peers, merge and deduplicate.
fn handle_list_swarm(
request: tiny_http::Request,
name_filter: Option<String>,
state: &ApiState,
) {
let mut all_entries = Vec::new();
// Query local
let inbox = match state.runtime.new_inbox::<DatastoreResponse>() {
Ok(i) => i,
Err(_) => {
respond_error(request, 500, "failed to create inbox");
return;
}
};
let _ = state.runtime.send_to(
state.metadata_addr,
MetadataMsg::ListLocal {
name_filter: name_filter.clone(),
reply_to: *inbox.addr(),
},
);
if let Some(DatastoreResponse::ListOk { entries }) = poll_response(&inbox, POLL_TIMEOUT) {
all_entries.extend(entries);
}
// Query each peer
let peers = state.peers.lock().unwrap().clone();
for peer in &peers {
let peer_inbox = match state.runtime.new_inbox::<DatastoreResponse>() {
Ok(i) => i,
Err(_) => continue,
};
let _ = state.runtime.send_to(
peer.metadata,
MetadataMsg::ListLocal {
name_filter: name_filter.clone(),
reply_to: *peer_inbox.addr(),
},
);
if let Some(DatastoreResponse::ListOk { entries }) =
poll_response(&peer_inbox, Duration::from_secs(2))
{
all_entries.extend(entries);
}
}
// Deduplicate by content_hash
let mut seen = std::collections::HashSet::new();
all_entries.retain(|e| seen.insert(e.content_hash));
let json = serde_json::json!({ "entries": entries_to_json(&all_entries) }).to_string();
respond_json(request, &json);
}
// ── STATUS handler ──────────────────────────────────────────────────────
fn handle_status(request: tiny_http::Request, state: &ApiState) {
let inbox = match state.runtime.new_inbox::<DatastoreResponse>() {
Ok(i) => i,
Err(_) => {
respond_error(request, 500, "failed to create inbox");
return;
}
};
let _ = state.runtime.send_to(
state.datastore_addr,
DatastoreNodeMsg::Status {
reply_to: *inbox.addr(),
},
);
match poll_response(&inbox, POLL_TIMEOUT) {
Some(DatastoreResponse::NodeStatus { node_id }) => {
let hex: String = node_id.0.iter().map(|b| format!("{b:02x}")).collect();
let json = serde_json::json!({ "node_id": hex }).to_string();
respond_json(request, &json);
}
_ => {
respond_error(request, 504, "timeout");
}
}
}
// ── Remote GET orchestration ────────────────────────────────────────────
/// Try to fetch an object from peers when not found locally.
/// Returns (entry, manifest) on success, stores chunks locally as a side effect.
fn try_remote_get(
content_hash: ContentHash,
state: &ApiState,
) -> Option<(crate::types::ObjectEntry, crate::types::ObjectManifest)> {
let peers = state.peers.lock().unwrap().clone();
for peer in &peers {
// Ask peer's metadata actor for the object
let find_inbox = state.runtime.new_inbox::<DatastoreResponse>().ok()?;
let _ = state.runtime.send_to(
peer.metadata,
MetadataMsg::HandleFindObject {
from: distribution::types::NodeId([0; 32]), // placeholder
content_hash,
reply_to: *find_inbox.addr(),
},
);
let (entry, _) = match poll_response(&find_inbox, Duration::from_secs(2)) {
Some(DatastoreResponse::GetOk { entry, manifest }) => (entry, manifest),
_ => continue,
};
// Get manifest from peer's blob store
let manifest_inbox = state.runtime.new_inbox::<DatastoreResponse>().ok()?;
let _ = state.runtime.send_to(
peer.blob_store,
BlobStoreMsg::ReadManifest {
hash: content_hash,
reply_to: *manifest_inbox.addr(),
},
);
let manifest = match poll_response(&manifest_inbox, Duration::from_secs(2)) {
Some(DatastoreResponse::ManifestOk { manifest }) => manifest,
_ => continue,
};
// Fetch each chunk from peer and store locally
let hash_hex = content_hash.to_hex();
state.metrics.begin_transfer(&hash_hex, manifest.chunks.len());
let mut all_ok = true;
for chunk_ref in &manifest.chunks {
let chunk_inbox = match state.runtime.new_inbox::<DatastoreResponse>() {
Ok(i) => i,
Err(_) => {
all_ok = false;
break;
}
};
let _ = state.runtime.send_to(
peer.blob_store,
BlobStoreMsg::ReadChunk {
hash: chunk_ref.hash,
reply_to: *chunk_inbox.addr(),
},
);
match poll_response(&chunk_inbox, Duration::from_secs(2)) {
Some(DatastoreResponse::ChunkOk { hash, data }) => {
// Store locally
let store_inbox =
match state.runtime.new_inbox::<DatastoreResponse>() {
Ok(i) => i,
Err(_) => {
all_ok = false;
break;
}
};
let _ = state.runtime.send_to(
state.blob_store_addr,
BlobStoreMsg::WriteChunk {
hash,
data,
reply_to: *store_inbox.addr(),
},
);
// Wait for confirmation
let _ = poll_response(&store_inbox, Duration::from_secs(2));
state.metrics.advance_transfer(&hash_hex);
}
_ => {
all_ok = false;
break;
}
}
}
state.metrics.end_transfer(&hash_hex);
if !all_ok {
continue;
}
// Store manifest locally
let m_inbox = match state.runtime.new_inbox::<DatastoreResponse>() {
Ok(i) => i,
Err(_) => continue,
};
let _ = state.runtime.send_to(
state.blob_store_addr,
BlobStoreMsg::WriteManifest {
manifest: manifest.clone(),
reply_to: *m_inbox.addr(),
},
);
let _ = poll_response(&m_inbox, Duration::from_secs(2));
// Store entry+manifest in local metadata
let put_inbox = match state.runtime.new_inbox::<DatastoreResponse>() {
Ok(i) => i,
Err(_) => continue,
};
let _ = state.runtime.send_to(
state.metadata_addr,
MetadataMsg::PutObject {
entry: entry.clone(),
manifest: manifest.clone(),
reply_to: *put_inbox.addr(),
},
);
let _ = poll_response(&put_inbox, Duration::from_secs(2));
return Some((entry, manifest));
}
None
}
// ── Server startup ──────────────────────────────────────────────────────
/// Start the HTTP API server for the datastore.
///
/// Returns a shared shutdown flag (set to `true` to stop the server)
/// and a peer list that can be updated to enable remote operations.
pub fn start_api_server(
runtime: Arc<Runtime>,
datastore_addr: ActorAddress,
metadata_addr: ActorAddress,
blob_store_addr: ActorAddress,
port: u16,
metrics: Arc<DatastoreMetrics>,
) -> (Arc<AtomicBool>, Arc<Mutex<Vec<PeerInfo>>>) {
let shutdown = Arc::new(AtomicBool::new(false));
let peers: Arc<Mutex<Vec<PeerInfo>>> = Arc::new(Mutex::new(Vec::new()));
let state = Arc::new(ApiState {
runtime,
datastore_addr,
metadata_addr,
blob_store_addr,
peers: Arc::clone(&peers),
metrics,
});
let addr = format!("0.0.0.0:{port}");
let server = tiny_http::Server::http(&addr).expect("failed to bind datastore API server");
let server = Arc::new(server);
for _ in 0..4 {
let server = Arc::clone(&server);
let state = Arc::clone(&state);
let shutdown = Arc::clone(&shutdown);
thread::spawn(move || {
loop {
if shutdown.load(Ordering::Relaxed) {
break;
}
let request = match server.recv_timeout(Duration::from_millis(500)) {
Ok(Some(r)) => r,
Ok(None) => continue,
Err(_) => break,
};
let url = request.url().to_string();
let path = url.split('?').next().unwrap_or(&url);
let method = request.method().as_str();
match (method, path) {
("POST", "/api/put") => handle_put(request, &url, &state),
("GET", "/api/get") => handle_get(request, &url, &state),
("GET", "/api/data") => handle_data(request, &url, &state),
("POST", "/api/delete") => handle_delete(request, &url, &state),
("GET", "/api/list") => handle_list(request, &url, &state),
("GET", "/api/status") => handle_status(request, &state),
("GET", "/") => respond_html(request),
_ => {
respond_error(request, 404, "not found");
}
}
}
});
}
(shutdown, peers)
}

329
crates/datastore/src/bin/store_cli.rs Normal file
View file

@ -0,0 +1,329 @@
//! swactor-store — CLI client for the datastore node.
//!
//! Talks to a running `swactor-store-node` over its HTTP API.
use std::fs;
use std::io::Read;
use std::path::PathBuf;
use clap::{Parser, Subcommand};
#[derive(Parser)]
#[command(name = "swactor-store", about = "Swactor datastore CLI")]
struct Args {
/// Base URL of the datastore node
#[arg(long, default_value = "http://localhost:9091")]
url: String,
#[command(subcommand)]
command: Command,
}
#[derive(Subcommand)]
enum Command {
/// Store a file in the datastore
Put {
/// Path to the local file to store
path: PathBuf,
/// Optional name label
#[arg(long)]
name: Option<String>,
},
/// Retrieve object metadata (or download with --output)
Get {
/// Content hash (hex)
hash: String,
/// Download file to this path
#[arg(long)]
output: Option<PathBuf>,
},
/// Delete an object
Delete {
/// Content hash (hex)
hash: String,
},
/// List stored objects
List {
/// Filter by name substring
#[arg(long)]
name: Option<String>,
/// List from all nodes (swarm-wide)
#[arg(long)]
all: bool,
},
/// Query node status
Status,
}
fn main() {
let args = Args::parse();
let base = args.url.trim_end_matches('/');
match args.command {
Command::Put { path, name } => cmd_put(base, &path, name.as_deref()),
Command::Get { hash, output } => cmd_get(base, &hash, output.as_deref()),
Command::Delete { hash } => cmd_delete(base, &hash),
Command::List { name, all } => cmd_list(base, name.as_deref(), all),
Command::Status => cmd_status(base),
}
}
fn cmd_put(base: &str, path: &PathBuf, name: Option<&str>) {
let data = match fs::read(path) {
Ok(d) => d,
Err(e) => {
eprintln!("Error reading {}: {e}", path.display());
std::process::exit(1);
}
};
let label = name
.map(|n| n.to_string())
.or_else(|| {
path.file_name()
.and_then(|f| f.to_str())
.map(|s| s.to_string())
});
let mut url = format!("{base}/api/put");
if let Some(ref n) = label {
url.push_str(&format!("?name={}", url_encode(n)));
}
let resp = match ureq::post(&url).send_bytes(&data) {
Ok(r) => r,
Err(e) => {
eprintln!("Error: {e}");
std::process::exit(1);
}
};
let body: serde_json::Value = match resp.into_json() {
Ok(v) => v,
Err(e) => {
eprintln!("Error parsing response: {e}");
std::process::exit(1);
}
};
if let Some(hash) = body.get("content_hash").and_then(|v| v.as_str()) {
println!("{hash}");
} else if let Some(err) = body.get("error").and_then(|v| v.as_str()) {
eprintln!("Error: {err}");
std::process::exit(1);
}
}
fn cmd_get(base: &str, hash: &str, output: Option<&std::path::Path>) {
if let Some(out_path) = output {
// Download raw data
let url = format!("{base}/api/data?hash={hash}");
let resp = match ureq::get(&url).call() {
Ok(r) => r,
Err(e) => {
eprintln!("Error: {e}");
std::process::exit(1);
}
};
if resp.status() != 200 {
let body = resp.into_string().unwrap_or_default();
eprintln!("Error: {body}");
std::process::exit(1);
}
let mut data = Vec::new();
if let Err(e) = resp.into_reader().read_to_end(&mut data) {
eprintln!("Error reading response: {e}");
std::process::exit(1);
}
if let Err(e) = fs::write(out_path, &data) {
eprintln!("Error writing {}: {e}", out_path.display());
std::process::exit(1);
}
println!("Written {} bytes to {}", data.len(), out_path.display());
} else {
// Metadata only
let url = format!("{base}/api/get?hash={hash}");
let resp = match ureq::get(&url).call() {
Ok(r) => r,
Err(e) => {
eprintln!("Error: {e}");
std::process::exit(1);
}
};
let body: serde_json::Value = match resp.into_json() {
Ok(v) => v,
Err(e) => {
eprintln!("Error parsing response: {e}");
std::process::exit(1);
}
};
if let Some(err) = body.get("error").and_then(|v| v.as_str()) {
eprintln!("Error: {err}");
std::process::exit(1);
}
if let Some(entry) = body.get("entry") {
println!("Hash: {}", entry.get("content_hash").and_then(|v| v.as_str()).unwrap_or("?"));
println!(
"Name: {}",
entry.get("name").and_then(|v| v.as_str()).unwrap_or("(none)")
);
println!(
"Size: {} bytes",
entry.get("size_bytes").and_then(|v| v.as_u64()).unwrap_or(0)
);
println!(
"Node: {}",
entry.get("node_id").and_then(|v| v.as_str()).unwrap_or("?")
);
if let Some(tags) = entry.get("tags").and_then(|v| v.as_object()) {
if !tags.is_empty() {
println!("Tags:");
for (k, v) in tags {
println!(" {k}: {v}");
}
}
}
}
if let Some(manifest) = body.get("manifest") {
println!(
"Chunks: {}",
manifest
.get("chunks")
.and_then(|v| v.as_array())
.map(|a| a.len())
.unwrap_or(0)
);
}
}
}
fn cmd_delete(base: &str, hash: &str) {
let url = format!("{base}/api/delete?hash={hash}");
let resp = match ureq::post(&url).send_bytes(&[]) {
Ok(r) => r,
Err(e) => {
eprintln!("Error: {e}");
std::process::exit(1);
}
};
let body: serde_json::Value = match resp.into_json() {
Ok(v) => v,
Err(e) => {
eprintln!("Error parsing response: {e}");
std::process::exit(1);
}
};
if let Some(h) = body.get("content_hash").and_then(|v| v.as_str()) {
println!("Deleted {h}");
} else if let Some(err) = body.get("error").and_then(|v| v.as_str()) {
eprintln!("Error: {err}");
std::process::exit(1);
}
}
fn cmd_list(base: &str, name: Option<&str>, all: bool) {
let mut url = format!("{base}/api/list");
let mut sep = '?';
if let Some(n) = name {
url.push_str(&format!("{sep}name={}", url_encode(n)));
sep = '&';
}
if all {
url.push_str(&format!("{sep}all=true"));
}
let resp = match ureq::get(&url).call() {
Ok(r) => r,
Err(e) => {
eprintln!("Error: {e}");
std::process::exit(1);
}
};
let body: serde_json::Value = match resp.into_json() {
Ok(v) => v,
Err(e) => {
eprintln!("Error parsing response: {e}");
std::process::exit(1);
}
};
if let Some(err) = body.get("error").and_then(|v| v.as_str()) {
eprintln!("Error: {err}");
std::process::exit(1);
}
if let Some(entries) = body.get("entries").and_then(|v| v.as_array()) {
if entries.is_empty() {
println!("(no entries)");
return;
}
// Print header
println!("{:<64} {:>10} {}", "HASH", "SIZE", "NAME");
println!("{}", "-".repeat(90));
for entry in entries {
let hash = entry
.get("content_hash")
.and_then(|v| v.as_str())
.unwrap_or("?");
let size = entry
.get("size_bytes")
.and_then(|v| v.as_u64())
.unwrap_or(0);
let name = entry
.get("name")
.and_then(|v| v.as_str())
.unwrap_or("(none)");
println!("{hash:<64} {size:>10} {name}");
}
}
}
fn cmd_status(base: &str) {
let url = format!("{base}/api/status");
let resp = match ureq::get(&url).call() {
Ok(r) => r,
Err(e) => {
eprintln!("Error: {e}");
std::process::exit(1);
}
};
let body: serde_json::Value = match resp.into_json() {
Ok(v) => v,
Err(e) => {
eprintln!("Error parsing response: {e}");
std::process::exit(1);
}
};
if let Some(node_id) = body.get("node_id").and_then(|v| v.as_str()) {
println!("Node ID: {node_id}");
} else if let Some(err) = body.get("error").and_then(|v| v.as_str()) {
eprintln!("Error: {err}");
std::process::exit(1);
}
}
fn url_encode(s: &str) -> String {
let mut result = String::with_capacity(s.len());
for b in s.bytes() {
match b {
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
result.push(b as char);
}
_ => {
result.push_str(&format!("%{b:02X}"));
}
}
}
result
}

254
crates/datastore/src/bin/store_node.rs Normal file
View file

@ -0,0 +1,254 @@
//! swactor-store-node — standalone datastore node with HTTP API.
//!
//! Starts the actor runtime, spawns datastore actors, and serves
//! a REST API for external tools (the `swactor-store` CLI).
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::thread;
use std::time::Duration;
use clap::Parser;
use serde::Deserialize;
use swactor::config::RuntimeConfig;
use swactor::runtime::Runtime;
use swactor_datastore::actors::{BlobStoreActor, DatastoreNode, MetadataActor};
use swactor_datastore::api::start_api_server;
use swactor_datastore::messages::MetadataMsg;
use swactor_datastore::metrics::DatastoreMetrics;
use swactor_datastore::storage::{FilesystemBackend, InMemoryBackend};
use swactor_datastore::DatastoreConfig;
use distribution::types::NodeId;
#[derive(Parser)]
#[command(name = "swactor-store-node", about = "Swactor distributed datastore node")]
struct Args {
/// Path to a TOML config file
#[arg(long)]
config: Option<std::path::PathBuf>,
/// HTTP API port
#[arg(long)]
port: Option<u16>,
/// Storage directory (omit for in-memory)
#[arg(long)]
storage_path: Option<String>,
/// Dashboard HTTP port (omit to disable dashboard)
#[arg(long)]
dashboard_port: Option<u16>,
/// Chunk size in bytes
#[arg(long)]
chunk_size: Option<u32>,
/// GC interval in ticks (each tick is ~100ms)
#[arg(long)]
gc_interval: Option<u64>,
/// Dissemination interval in ticks
#[arg(long)]
disseminate_interval: Option<u64>,
}
#[derive(Deserialize, Default)]
struct NodeConfig {
port: Option<u16>,
storage_path: Option<String>,
dashboard_port: Option<u16>,
chunk_size: Option<u32>,
gc_interval: Option<u64>,
disseminate_interval: Option<u64>,
}
/// Resolved configuration with CLI > config file > defaults applied.
struct ResolvedConfig {
port: u16,
storage_path: Option<String>,
dashboard_port: Option<u16>,
chunk_size: u32,
gc_interval: u64,
disseminate_interval: u64,
}
fn resolve_config(args: &Args) -> ResolvedConfig {
let file_cfg = match &args.config {
Some(path) => {
let contents = std::fs::read_to_string(path)
.unwrap_or_else(|e| panic!("failed to read config file {}: {e}", path.display()));
toml::from_str::<NodeConfig>(&contents)
.unwrap_or_else(|e| panic!("failed to parse config file {}: {e}", path.display()))
}
None => NodeConfig::default(),
};
ResolvedConfig {
port: args.port.or(file_cfg.port).unwrap_or(9091),
storage_path: args.storage_path.clone().or(file_cfg.storage_path),
dashboard_port: args.dashboard_port.or(file_cfg.dashboard_port),
chunk_size: args.chunk_size.or(file_cfg.chunk_size).unwrap_or(1_048_576),
gc_interval: args.gc_interval.or(file_cfg.gc_interval).unwrap_or(1000),
disseminate_interval: args.disseminate_interval.or(file_cfg.disseminate_interval).unwrap_or(50),
}
}
fn main() {
let args = Args::parse();
let cfg = resolve_config(&args);
let stop = Arc::new(AtomicBool::new(false));
// Signal handler
{
let stop = Arc::clone(&stop);
ctrlc::set_handler(move || {
stop.store(true, Ordering::Relaxed);
})
.expect("failed to set signal handler");
}
// Optionally start dashboard
let dash = cfg.dashboard_port.map(|port| {
let d = runtime_dashboard::start_dashboard(runtime_dashboard::DashboardConfig {
port,
..Default::default()
});
d.install_tracing();
d
});
// Create runtime
let num_threads = 2;
let collector = runtime_dashboard::collector::StatsCollector::new(num_threads);
let mut rt = Runtime::new(RuntimeConfig {
num_threads,
max_actors: 1024,
channel_buffer_size: 2000,
..Default::default()
});
rt.set_stats_hook(collector.clone());
// Generate node ID from random bytes
let node_id = {
let mut bytes = [0u8; 32];
for (i, b) in std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
.to_le_bytes()
.iter()
.enumerate()
{
bytes[i % 32] ^= *b;
}
// Mix in process id for uniqueness
let pid = std::process::id();
for (i, b) in pid.to_le_bytes().iter().enumerate() {
bytes[i + 16] ^= *b;
}
NodeId(bytes)
};
// Datastore config
let config = DatastoreConfig {
chunk_size: cfg.chunk_size,
storage_path: cfg
.storage_path
.as_ref()
.map(|s| s.into())
.unwrap_or_else(|| "datastore".into()),
gc_interval: cfg.gc_interval,
..Default::default()
};
// Create storage backend
let backend: Box<dyn swactor_datastore::StorageBackend> = match &cfg.storage_path {
Some(path) => {
let p = std::path::PathBuf::from(path);
std::fs::create_dir_all(&p).expect("failed to create storage directory");
Box::new(FilesystemBackend::new(p))
}
None => Box::new(InMemoryBackend::new()),
};
// Spawn actors before starting runtime threads
let blob_store_addr = rt
.spawn(BlobStoreActor::new(backend))
.expect("failed to spawn BlobStoreActor");
let mut metadata = MetadataActor::new(node_id, &config);
metadata.set_blob_store(blob_store_addr);
let metadata_addr = rt
.spawn(metadata)
.expect("failed to spawn MetadataActor");
let datastore_node = DatastoreNode::new(node_id, blob_store_addr, metadata_addr, config);
let datastore_addr = rt
.spawn(datastore_node)
.expect("failed to spawn DatastoreNode");
// Start runtime
let handle = rt.run().expect("failed to start runtime");
// Create datastore metrics
let node_hex: String = node_id.0.iter().map(|b| format!("{b:02x}")).collect();
let metrics = Arc::new(DatastoreMetrics::new());
metrics.set_node_id(node_hex.clone());
if let Some(ref d) = dash {
d.set_runtime(handle.runtime.clone(), collector);
d.set_datastore(Arc::clone(&metrics) as Arc<dyn runtime_dashboard::datastore_collector::DatastoreStatsProvider>);
}
// Start HTTP API
let (api_shutdown, _peers) = start_api_server(
handle.runtime.clone(),
datastore_addr,
metadata_addr,
blob_store_addr,
cfg.port,
Arc::clone(&metrics),
);
eprintln!("Node {} started", &node_hex[..8]);
eprintln!("API at http://0.0.0.0:{}", cfg.port);
if let Some(port) = cfg.dashboard_port {
eprintln!("Dashboard at http://0.0.0.0:{port}");
}
if cfg.storage_path.is_some() {
eprintln!("Storage: {}", cfg.storage_path.as_ref().unwrap());
} else {
eprintln!("Storage: in-memory");
}
// Main loop
let mut round: u64 = 0;
while !stop.load(Ordering::Relaxed) {
round += 1;
if round % cfg.gc_interval == 0 {
let _ = handle
.runtime
.send_to(metadata_addr, MetadataMsg::GcTick);
}
if round % cfg.disseminate_interval == 0 {
let _ = handle
.runtime
.send_to(metadata_addr, MetadataMsg::DisseminateTick);
}
thread::sleep(Duration::from_millis(100));
}
eprintln!("\nShutting down...");
api_shutdown.store(true, Ordering::Relaxed);
handle.shutdown();
if let Some(d) = dash {
d.shutdown();
}
handle.join();
}

138
crates/datastore/src/chunking.rs Normal file
View file

@ -0,0 +1,138 @@
//! Chunking engine — pure functions for splitting blobs into content-addressed
//! chunks and reassembling them.
//!
//! No I/O. All functions are deterministic and side-effect-free.
use crate::types::{ChunkRef, ContentHash, ObjectManifest};
/// Errors that can occur during chunk reassembly.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ChunkingError {
/// A chunk referenced by the manifest was not provided.
MissingChunk { hash: ContentHash },
/// The reassembled data size does not match the manifest's `total_size`.
SizeMismatch { expected: u64, actual: u64 },
/// The reassembled data's content hash does not match the manifest's `content_hash`.
HashMismatch {
expected: ContentHash,
actual: ContentHash,
},
}
impl std::fmt::Display for ChunkingError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
ChunkingError::MissingChunk { hash } => write!(f, "missing chunk: {hash}"),
ChunkingError::SizeMismatch { expected, actual } => {
write!(f, "size mismatch: expected {expected}, got {actual}")
}
ChunkingError::HashMismatch { expected, actual } => {
write!(f, "hash mismatch: expected {expected}, got {actual}")
}
}
}
}
impl std::error::Error for ChunkingError {}
/// Split a blob into fixed-size chunks and produce a manifest.
///
/// Returns `(content_hash, manifest, chunks)` where:
/// - `content_hash` is `blake3(data)` — the whole-blob hash
/// - `manifest` describes the chunk layout
/// - `chunks` is a vec of `(chunk_hash, chunk_bytes)` pairs
///
/// `chunk_size` must be > 0.
pub fn chunk_blob(
data: &[u8],
chunk_size: u32,
) -> (ContentHash, ObjectManifest, Vec<(ContentHash, Vec<u8>)>) {
assert!(chunk_size > 0, "chunk_size must be > 0");
let content_hash = ContentHash::of(data);
let mut chunks = Vec::new();
let mut chunk_refs = Vec::new();
let mut offset: u64 = 0;
if data.is_empty() {
let manifest = ObjectManifest {
content_hash,
chunks: chunk_refs,
total_size: 0,
chunk_size,
content_type: None,
};
return (content_hash, manifest, chunks);
}
for chunk_data in data.chunks(chunk_size as usize) {
let hash = ContentHash::of(chunk_data);
chunk_refs.push(ChunkRef {
hash,
offset,
size: chunk_data.len() as u32,
});
chunks.push((hash, chunk_data.to_vec()));
offset += chunk_data.len() as u64;
}
let manifest = ObjectManifest {
content_hash,
chunks: chunk_refs,
total_size: data.len() as u64,
chunk_size,
content_type: None,
};
(content_hash, manifest, chunks)
}
/// Reassemble a blob from its manifest and chunk data.
///
/// Chunks are looked up by hash from the provided slice. The manifest's
/// `chunks` field determines the ordering. Verifies total size and
/// content hash after reassembly.
pub fn reassemble_blob(
manifest: &ObjectManifest,
chunks: &[(ContentHash, Vec<u8>)],
) -> Result<Vec<u8>, ChunkingError> {
let mut result = Vec::with_capacity(manifest.total_size as usize);
for chunk_ref in &manifest.chunks {
let chunk_data = chunks
.iter()
.find(|(h, _)| *h == chunk_ref.hash)
.map(|(_, d)| d);
match chunk_data {
Some(data) => result.extend_from_slice(data),
None => {
return Err(ChunkingError::MissingChunk {
hash: chunk_ref.hash,
})
}
}
}
if result.len() as u64 != manifest.total_size {
return Err(ChunkingError::SizeMismatch {
expected: manifest.total_size,
actual: result.len() as u64,
});
}
let actual_hash = ContentHash::of(&result);
if actual_hash != manifest.content_hash {
return Err(ChunkingError::HashMismatch {
expected: manifest.content_hash,
actual: actual_hash,
});
}
Ok(result)
}
/// Verify that `data` hashes to `expected`.
pub fn verify_integrity(data: &[u8], expected: &ContentHash) -> bool {
ContentHash::of(data) == *expected
}

72
crates/datastore/src/cli.rs Normal file
View file

@ -0,0 +1,72 @@
//! CLI command type definitions for `swactor-store`.
//!
//! Types only — no implementation. These define the CLI interface that will
//! be wired to the actor system in a future milestone.
use std::collections::BTreeMap;
use std::path::PathBuf;
use distribution::types::NodeId;
/// Top-level CLI commands for `swactor-store`.
#[derive(Debug, Clone)]
pub enum CliCommand {
/// Store a local file as a distributed object.
///
/// ```text
/// swactor-store put <local-path> [--name <label>] [--tag key=value...]
/// ```
Put {
/// Path to the local file to store.
local_path: PathBuf,
/// Optional human-readable name for the object.
name: Option<String>,
/// Key-value tags to attach to the object.
tags: BTreeMap<String, String>,
},
/// Retrieve an object from the datastore by content hash.
///
/// ```text
/// swactor-store get <content-hash>[@<node>] [--output <local-path>]
/// ```
Get {
/// Content hash (hex) of the object to retrieve.
content_hash: String,
/// Specific node to fetch from (optional).
node: Option<NodeId>,
/// Local path to write the object to.
output: Option<PathBuf>,
},
/// Delete an object from the datastore by content hash.
///
/// ```text
/// swactor-store delete <content-hash>
/// ```
Delete {
/// Content hash (hex) of the object to delete.
content_hash: String,
},
/// List objects in the datastore.
///
/// ```text
/// swactor-store list [--name <substring>] [--node <node-name>] [--all]
/// ```
List {
/// Filter by name substring.
name: Option<String>,
/// List objects from a specific node only.
node: Option<NodeId>,
/// If true, query all nodes (swarm-wide). Otherwise, local only.
all: bool,
},
/// Show node status: identity, chunk count, storage usage.
///
/// ```text
/// swactor-store status
/// ```
Status,
}

17
crates/datastore/src/lib.rs Normal file
View file

@ -0,0 +1,17 @@
pub mod types;
pub mod messages;
pub mod chunking;
pub mod storage;
pub mod actors;
pub mod cli;
pub mod metrics;
#[cfg(feature = "node")]
pub mod api;
#[cfg(feature = "node")]
pub mod ui_html;
pub use types::{ChunkRef, ContentHash, DatastoreConfig, ObjectEntry, ObjectManifest};
pub use messages::{BlobStoreMsg, DatastoreNodeMsg, DatastoreResponse, MetadataMsg, TransferMsg};
pub use chunking::{chunk_blob, reassemble_blob, verify_integrity, ChunkingError};
pub use storage::{StorageBackend, FilesystemBackend, InMemoryBackend};
pub use actors::{BlobStoreActor, DatastoreNode, MetadataActor, TransferActor};

372
crates/datastore/src/messages.rs Normal file
View file

@ -0,0 +1,372 @@
//! Protocol messages for the distributed datastore.
//!
//! Split into two categories:
//! - **Inter-node messages** — travel over the wire (iroh/QUIC) between nodes.
//! Each implements `NetworkMessage` with a stable `type_tag()`.
//! - **Intra-node messages** — actor-to-actor within a single node.
//! Plain enums routed through the local actor system.
use std::collections::BTreeMap;
use std::collections::HashSet;
use std::net::SocketAddr;
use serde::{Deserialize, Serialize};
use swactor::actor::ActorAddress;
use swactor::transport::NetworkMessage;
use distribution::types::NodeId;
use crate::types::{ContentHash, ObjectEntry, ObjectManifest};
// ═══════════════════════════════════════════════════════════════════════════
// Inter-node messages (wire protocol over iroh)
// ═══════════════════════════════════════════════════════════════════════════
// ─── Chunk transfer ─────────────────────────────────────────────────────────
/// Request a chunk by its content hash.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GetChunkRequest {
pub from: NodeId,
pub hash: ContentHash,
}
impl NetworkMessage for GetChunkRequest {
fn type_tag() -> &'static str {
"swactor_datastore::GetChunkRequest"
}
}
/// Response to a chunk request. `data` is `None` if the chunk is not found.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GetChunkResponse {
pub hash: ContentHash,
pub data: Option<Vec<u8>>,
}
impl NetworkMessage for GetChunkResponse {
fn type_tag() -> &'static str {
"swactor_datastore::GetChunkResponse"
}
}
// ─── Object metadata (DHT operations) ───────────────────────────────────────
/// Store object metadata in the DHT (Kademlia STORE).
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StoreObjectRequest {
pub entry: ObjectEntry,
}
impl NetworkMessage for StoreObjectRequest {
fn type_tag() -> &'static str {
"swactor_datastore::StoreObjectRequest"
}
}
/// Look up object metadata by content hash (Kademlia FIND_VALUE).
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FindObjectRequest {
pub from: NodeId,
pub content_hash: ContentHash,
}
impl NetworkMessage for FindObjectRequest {
fn type_tag() -> &'static str {
"swactor_datastore::FindObjectRequest"
}
}
/// Response to a FIND_VALUE for object metadata.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum FindObjectResponse {
/// Found the object — here's the metadata entry.
Found(ObjectEntry),
/// Don't have it — here are closer nodes to ask.
Closer(Vec<(NodeId, SocketAddr)>),
}
impl NetworkMessage for FindObjectResponse {
fn type_tag() -> &'static str {
"swactor_datastore::FindObjectResponse"
}
}
/// Request an object manifest by its content hash.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GetManifestRequest {
pub from: NodeId,
pub hash: ContentHash,
}
impl NetworkMessage for GetManifestRequest {
fn type_tag() -> &'static str {
"swactor_datastore::GetManifestRequest"
}
}
/// Response to a manifest request.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GetManifestResponse {
pub manifest: Option<ObjectManifest>,
}
impl NetworkMessage for GetManifestResponse {
fn type_tag() -> &'static str {
"swactor_datastore::GetManifestResponse"
}
}
// ─── Listing ────────────────────────────────────────────────────────────────
/// List objects stored on a specific node, optionally filtered by name substring.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ListObjectsRequest {
pub from: NodeId,
pub name_filter: Option<String>,
}
impl NetworkMessage for ListObjectsRequest {
fn type_tag() -> &'static str {
"swactor_datastore::ListObjectsRequest"
}
}
/// Response to a list request.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ListObjectsResponse {
pub entries: Vec<ObjectEntry>,
}
impl NetworkMessage for ListObjectsResponse {
fn type_tag() -> &'static str {
"swactor_datastore::ListObjectsResponse"
}
}
// ═══════════════════════════════════════════════════════════════════════════
// Intra-node messages (actor-to-actor)
// ═══════════════════════════════════════════════════════════════════════════
// ─── BlobStoreMsg ───────────────────────────────────────────────────────────
/// Messages handled by the `BlobStoreActor`.
#[derive(Debug, Clone)]
pub enum BlobStoreMsg {
/// Write a chunk to disk. The hash must match `ContentHash::of(data)`.
WriteChunk {
hash: ContentHash,
data: Vec<u8>,
reply_to: ActorAddress,
},
/// Read a chunk from disk by its content hash.
ReadChunk {
hash: ContentHash,
reply_to: ActorAddress,
},
/// Delete a chunk from disk.
DeleteChunk { hash: ContentHash },
/// Check whether a chunk exists locally.
HasChunk {
hash: ContentHash,
reply_to: ActorAddress,
},
/// List all chunk hashes stored locally.
ListChunks { reply_to: ActorAddress },
/// Garbage-collect chunks not in the referenced set.
GcUnreferenced { referenced: HashSet<ContentHash> },
/// Store a manifest to disk.
WriteManifest {
manifest: ObjectManifest,
reply_to: ActorAddress,
},
/// Read a manifest from disk by its content hash.
ReadManifest {
hash: ContentHash,
reply_to: ActorAddress,
},
}
// ─── MetadataMsg ────────────────────────────────────────────────────────────
/// Messages handled by the `MetadataActor`.
#[derive(Debug, Clone)]
pub enum MetadataMsg {
/// Store object metadata and manifest locally, then replicate to DHT.
PutObject {
entry: ObjectEntry,
manifest: ObjectManifest,
reply_to: ActorAddress,
},
/// Look up an object by content hash (local first, then DHT).
GetObject {
content_hash: ContentHash,
reply_to: ActorAddress,
},
/// Delete an object by content hash (remove from local index).
DeleteObject {
content_hash: ContentHash,
reply_to: ActorAddress,
},
/// List objects stored on this node, optionally filtered by name substring.
ListLocal {
name_filter: Option<String>,
reply_to: ActorAddress,
},
/// Fan-out list to all known alive nodes, merge results.
ListSwarm {
name_filter: Option<String>,
reply_to: ActorAddress,
},
/// Handle an incoming FIND_VALUE request from the DHT.
HandleFindObject {
from: NodeId,
content_hash: ContentHash,
reply_to: ActorAddress,
},
/// Handle an incoming STORE request from the DHT.
HandleStoreObject {
entry: ObjectEntry,
manifest: Option<ObjectManifest>,
},
/// Set the list of peer MetadataActor addresses for dissemination.
SetPeers { peers: Vec<ActorAddress> },
/// Trigger one round of epidemic dissemination to peers.
DisseminateTick,
/// Periodic garbage collection tick.
GcTick,
}
// ─── TransferMsg ────────────────────────────────────────────────────────────
/// Messages handled by the `TransferActor` (ephemeral, one per download).
#[derive(Debug, Clone)]
pub enum TransferMsg {
/// Start downloading an object from a remote node.
StartDownload {
manifest: ObjectManifest,
source_node: NodeId,
reply_to: ActorAddress,
},
/// A chunk has been received from the remote node.
ChunkReceived {
hash: ContentHash,
data: Vec<u8>,
},
/// A chunk fetch failed.
ChunkFailed {
hash: ContentHash,
reason: String,
},
/// Cancel this transfer.
Cancel,
}
// ─── DatastoreNodeMsg ───────────────────────────────────────────────────────
/// Messages handled by the `DatastoreNode` coordinator actor.
#[derive(Debug, Clone)]
pub enum DatastoreNodeMsg {
// ── User-facing commands ────────────────────────────────────────────
/// Store a blob with optional name and tags.
Put {
data: Vec<u8>,
name: Option<String>,
tags: BTreeMap<String, String>,
reply_to: ActorAddress,
},
/// Retrieve object metadata and manifest by content hash.
Get {
content_hash: ContentHash,
reply_to: ActorAddress,
},
/// Delete an object by content hash.
Delete {
content_hash: ContentHash,
reply_to: ActorAddress,
},
/// List stored objects with optional name filter.
List {
name_filter: Option<String>,
all: bool,
reply_to: ActorAddress,
},
/// Query node identity.
Status { reply_to: ActorAddress },
/// Read a single chunk by hash.
ReadChunk {
hash: ContentHash,
reply_to: ActorAddress,
},
// ── Incoming network protocol ───────────────────────────────────────
/// Route an incoming GetChunk request to BlobStoreActor.
IncomingGetChunk {
request: GetChunkRequest,
reply_to: ActorAddress,
},
/// Route an incoming GetManifest request to BlobStoreActor.
IncomingGetManifest {
request: GetManifestRequest,
reply_to: ActorAddress,
},
/// Route an incoming StoreObject request to MetadataActor.
IncomingStoreObject { request: StoreObjectRequest },
/// Route an incoming FindObject request to MetadataActor.
IncomingFindObject {
request: FindObjectRequest,
reply_to: ActorAddress,
},
/// Route an incoming ListObjects request to MetadataActor.
IncomingListObjects {
request: ListObjectsRequest,
reply_to: ActorAddress,
},
}
// ─── DatastoreResponse ──────────────────────────────────────────────────────
/// Response messages sent back to the requester by datastore actors.
#[derive(Debug, Clone)]
pub enum DatastoreResponse {
/// Object was stored successfully.
PutOk {
content_hash: ContentHash,
},
/// Object was found — here's the metadata and manifest.
GetOk {
entry: ObjectEntry,
manifest: ObjectManifest,
},
/// Object was deleted.
DeleteOk {
content_hash: ContentHash,
},
/// List of matching objects.
ListOk { entries: Vec<ObjectEntry> },
/// Chunk data retrieved.
ChunkOk {
hash: ContentHash,
data: Vec<u8>,
},
/// Chunk was stored successfully.
ChunkStored { hash: ContentHash },
/// Manifest stored successfully.
ManifestStored { hash: ContentHash },
/// Manifest retrieved.
ManifestOk { manifest: ObjectManifest },
/// Transfer completed — all chunks downloaded.
TransferComplete { content_hash: ContentHash },
/// Transfer failed.
TransferFailed { reason: String },
/// Node identity status response.
NodeStatus { node_id: NodeId },
/// Requested resource was not found.
NotFound,
/// An error occurred.
Error { reason: String },
/// Boolean response (e.g. HasChunk).
Bool(bool),
/// List of chunk hashes.
ChunkList { hashes: Vec<ContentHash> },
}

197
crates/datastore/src/metrics.rs Normal file
View file

@ -0,0 +1,197 @@
//! Thread-safe metrics for the datastore, consumed by the runtime dashboard.
//!
//! `DatastoreMetrics` accumulates counters and event history from any thread
//! (API handlers run on `tiny_http` worker threads). The dashboard polls
//! `snapshot()` every ~200ms via the `DatastoreStatsProvider` trait.
use std::collections::VecDeque;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Mutex;
use std::time::{SystemTime, UNIX_EPOCH};
use serde::{Deserialize, Serialize};
// ── Snapshot types (serializable, sent to the dashboard) ────────────────────
/// Point-in-time snapshot of the datastore's state and metrics.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DatastoreSnapshot {
pub node_id: String,
pub object_count: u64,
pub total_bytes: u64,
pub put_ops: u64,
pub get_ops: u64,
pub delete_ops: u64,
pub objects: Vec<ObjectSummary>,
pub recent_events: Vec<DatastoreEvent>,
pub active_transfers: Vec<TransferProgress>,
}
/// Summary of a single stored object (for the dashboard table).
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ObjectSummary {
pub hash: String,
pub name: Option<String>,
pub size_bytes: u64,
}
/// A recorded datastore operation.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DatastoreEvent {
pub timestamp_ms: u64,
pub kind: String,
pub hash: String,
pub name: Option<String>,
pub size_bytes: u64,
}
/// Progress of an in-flight chunk transfer.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TransferProgress {
pub hash: String,
pub chunks_received: usize,
pub chunks_total: usize,
}
// ── Live metrics (thread-safe, mutated from API handlers) ───────────────────
const MAX_EVENTS: usize = 200;
/// Thread-safe metrics accumulator for the datastore.
///
/// Atomic counters for the hot path (put/get/delete counts). A `Mutex`-guarded
/// ring buffer for the event timeline and a small vec for active transfers.
pub struct DatastoreMetrics {
node_id: Mutex<String>,
put_ops: AtomicU64,
get_ops: AtomicU64,
delete_ops: AtomicU64,
objects: Mutex<Vec<ObjectSummary>>,
events: Mutex<VecDeque<DatastoreEvent>>,
transfers: Mutex<Vec<TransferProgress>>,
}
impl DatastoreMetrics {
pub fn new() -> Self {
Self {
node_id: Mutex::new(String::new()),
put_ops: AtomicU64::new(0),
get_ops: AtomicU64::new(0),
delete_ops: AtomicU64::new(0),
objects: Mutex::new(Vec::new()),
events: Mutex::new(VecDeque::with_capacity(MAX_EVENTS + 1)),
transfers: Mutex::new(Vec::new()),
}
}
pub fn set_node_id(&self, hex: String) {
*self.node_id.lock().unwrap() = hex;
}
/// Record a successful PUT operation.
pub fn record_put(&self, hash: &str, name: Option<&str>, size_bytes: u64) {
self.put_ops.fetch_add(1, Ordering::Relaxed);
self.push_event("put", hash, name, size_bytes);
let mut objs = self.objects.lock().unwrap();
objs.push(ObjectSummary {
hash: hash.to_string(),
name: name.map(|s| s.to_string()),
size_bytes,
});
}
/// Record a successful GET operation.
pub fn record_get(&self, hash: &str) {
self.get_ops.fetch_add(1, Ordering::Relaxed);
self.push_event("get", hash, None, 0);
}
/// Record a successful DELETE operation.
pub fn record_delete(&self, hash: &str, size_bytes: u64) {
self.delete_ops.fetch_add(1, Ordering::Relaxed);
self.push_event("delete", hash, None, size_bytes);
let mut objs = self.objects.lock().unwrap();
objs.retain(|o| o.hash != hash);
}
/// Begin tracking a chunk transfer.
pub fn begin_transfer(&self, hash: &str, chunks_total: usize) {
let mut transfers = self.transfers.lock().unwrap();
transfers.push(TransferProgress {
hash: hash.to_string(),
chunks_received: 0,
chunks_total,
});
}
/// Advance a tracked transfer by one chunk.
pub fn advance_transfer(&self, hash: &str) {
let mut transfers = self.transfers.lock().unwrap();
if let Some(t) = transfers.iter_mut().find(|t| t.hash == hash) {
t.chunks_received += 1;
}
}
/// Remove a completed/failed transfer from tracking.
pub fn end_transfer(&self, hash: &str) {
let mut transfers = self.transfers.lock().unwrap();
transfers.retain(|t| t.hash != hash);
}
/// Seed the object list (e.g. from an initial LIST query at startup).
pub fn seed_objects(&self, objects: Vec<ObjectSummary>) {
*self.objects.lock().unwrap() = objects;
}
/// Capture a serializable snapshot of the current metrics.
pub fn snapshot(&self) -> DatastoreSnapshot {
let objs = self.objects.lock().unwrap();
let total_bytes: u64 = objs.iter().map(|o| o.size_bytes).sum();
let events = self.events.lock().unwrap();
let transfers = self.transfers.lock().unwrap();
DatastoreSnapshot {
node_id: self.node_id.lock().unwrap().clone(),
object_count: objs.len() as u64,
total_bytes,
put_ops: self.put_ops.load(Ordering::Relaxed),
get_ops: self.get_ops.load(Ordering::Relaxed),
delete_ops: self.delete_ops.load(Ordering::Relaxed),
objects: objs.clone(),
recent_events: events.iter().cloned().collect(),
active_transfers: transfers.clone(),
}
}
fn push_event(&self, kind: &str, hash: &str, name: Option<&str>, size_bytes: u64) {
let event = DatastoreEvent {
timestamp_ms: now_ms(),
kind: kind.to_string(),
hash: hash.to_string(),
name: name.map(|s| s.to_string()),
size_bytes,
};
let mut events = self.events.lock().unwrap();
if events.len() >= MAX_EVENTS {
events.pop_front();
}
events.push_back(event);
}
}
fn now_ms() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis() as u64
}
// ── Dashboard integration (only when runtime-dashboard is available) ────────
#[cfg(feature = "node")]
impl runtime_dashboard::datastore_collector::DatastoreStatsProvider for DatastoreMetrics {
fn snapshot_json(&self) -> Option<String> {
let snap = self.snapshot();
serde_json::to_string(&snap).ok()
}
}

72
crates/datastore/src/storage/in_memory.rs Normal file
View file

@ -0,0 +1,72 @@
//! In-memory storage backend — useful for tests and browser/WASM targets.
use std::collections::HashMap;
use crate::types::{ContentHash, ObjectManifest};
use super::StorageBackend;
/// A purely in-memory storage backend.
///
/// All data lives in `HashMap`s. No persistence across restarts.
/// Useful for unit tests and browser/WASM environments.
pub struct InMemoryBackend {
chunks: HashMap<ContentHash, Vec<u8>>,
manifests: HashMap<ContentHash, ObjectManifest>,
}
impl InMemoryBackend {
pub fn new() -> Self {
Self {
chunks: HashMap::new(),
manifests: HashMap::new(),
}
}
}
impl Default for InMemoryBackend {
fn default() -> Self {
Self::new()
}
}
impl StorageBackend for InMemoryBackend {
fn write_chunk(&mut self, hash: &ContentHash, data: &[u8]) -> Result<(), std::io::Error> {
self.chunks.insert(*hash, data.to_vec());
Ok(())
}
fn read_chunk(&self, hash: &ContentHash) -> Result<Option<Vec<u8>>, std::io::Error> {
Ok(self.chunks.get(hash).cloned())
}
fn delete_chunk(&mut self, hash: &ContentHash) -> Result<(), std::io::Error> {
self.chunks.remove(hash);
Ok(())
}
fn has_chunk(&self, hash: &ContentHash) -> bool {
self.chunks.contains_key(hash)
}
fn list_chunks(&self) -> Vec<ContentHash> {
self.chunks.keys().copied().collect()
}
fn write_manifest(&mut self, manifest: &ObjectManifest) -> Result<(), std::io::Error> {
self.manifests.insert(manifest.content_hash, manifest.clone());
Ok(())
}
fn read_manifest(
&self,
content_hash: &ContentHash,
) -> Result<Option<ObjectManifest>, std::io::Error> {
Ok(self.manifests.get(content_hash).cloned())
}
fn delete_manifest(&mut self, content_hash: &ContentHash) -> Result<(), std::io::Error> {
self.manifests.remove(content_hash);
Ok(())
}
}

175
crates/datastore/src/storage/mod.rs Normal file
View file

@ -0,0 +1,175 @@
//! StorageBackend trait and implementations.
//!
//! Abstracts chunk and manifest I/O so backends can be swapped
//! (filesystem for MVP, IndexedDB for browser, in-memory for tests/WASM).
pub mod in_memory;
use std::collections::HashSet;
use std::fs;
use std::io::Write;
use std::path::PathBuf;
use crate::types::{ContentHash, ObjectManifest};
pub use in_memory::InMemoryBackend;
/// Pluggable storage backend for chunks and manifests.
pub trait StorageBackend: Send {
fn write_chunk(&mut self, hash: &ContentHash, data: &[u8]) -> Result<(), std::io::Error>;
fn read_chunk(&self, hash: &ContentHash) -> Result<Option<Vec<u8>>, std::io::Error>;
fn delete_chunk(&mut self, hash: &ContentHash) -> Result<(), std::io::Error>;
fn has_chunk(&self, hash: &ContentHash) -> bool;
fn list_chunks(&self) -> Vec<ContentHash>;
fn write_manifest(&mut self, manifest: &ObjectManifest) -> Result<(), std::io::Error>;
fn read_manifest(&self, content_hash: &ContentHash) -> Result<Option<ObjectManifest>, std::io::Error>;
fn delete_manifest(&mut self, content_hash: &ContentHash) -> Result<(), std::io::Error>;
}
/// Filesystem-backed storage with 2-level directory sharding.
///
/// Layout:
/// ```text
/// {root}/
/// ├── chunks/{hex[0..2]}/{hex[2..4]}/{full_hex_hash}
/// └── manifests/{hex[0..2]}/{hex[2..4]}/{full_hex_hash}
/// ```
pub struct FilesystemBackend {
root: PathBuf,
chunk_index: HashSet<ContentHash>,
}
impl FilesystemBackend {
pub fn new(root: PathBuf) -> Self {
let mut backend = Self {
root,
chunk_index: HashSet::new(),
};
backend.scan_chunks();
backend
}
fn chunk_path(&self, hash: &ContentHash) -> PathBuf {
let hex = hash.to_hex();
self.root
.join("chunks")
.join(&hex[..2])
.join(&hex[2..4])
.join(&hex)
}
fn manifest_path(&self, hash: &ContentHash) -> PathBuf {
let hex = hash.to_hex();
self.root
.join("manifests")
.join(&hex[..2])
.join(&hex[2..4])
.join(&hex)
}
fn scan_chunks(&mut self) {
let chunks_dir = self.root.join("chunks");
if !chunks_dir.exists() {
return;
}
let Ok(level1) = fs::read_dir(&chunks_dir) else {
return;
};
for d1 in level1.flatten() {
let Ok(level2) = fs::read_dir(d1.path()) else {
continue;
};
for d2 in level2.flatten() {
let Ok(files) = fs::read_dir(d2.path()) else {
continue;
};
for file in files.flatten() {
if let Some(name) = file.file_name().to_str() {
if let Some(hash) = ContentHash::from_hex(name) {
self.chunk_index.insert(hash);
}
}
}
}
}
}
fn write_and_sync(path: &PathBuf, data: &[u8]) -> std::io::Result<()> {
if let Some(parent) = path.parent() {
fs::create_dir_all(parent)?;
}
let mut file = fs::File::create(path)?;
file.write_all(data)?;
file.sync_all()?;
Ok(())
}
}
impl StorageBackend for FilesystemBackend {
fn write_chunk(&mut self, hash: &ContentHash, data: &[u8]) -> Result<(), std::io::Error> {
let path = self.chunk_path(hash);
Self::write_and_sync(&path, data)?;
self.chunk_index.insert(*hash);
Ok(())
}
fn read_chunk(&self, hash: &ContentHash) -> Result<Option<Vec<u8>>, std::io::Error> {
if !self.chunk_index.contains(hash) {
return Ok(None);
}
let path = self.chunk_path(hash);
match fs::read(&path) {
Ok(data) => Ok(Some(data)),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(e) => Err(e),
}
}
fn delete_chunk(&mut self, hash: &ContentHash) -> Result<(), std::io::Error> {
self.chunk_index.remove(hash);
let path = self.chunk_path(hash);
match fs::remove_file(&path) {
Ok(()) => Ok(()),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(e) => Err(e),
}
}
fn has_chunk(&self, hash: &ContentHash) -> bool {
self.chunk_index.contains(hash)
}
fn list_chunks(&self) -> Vec<ContentHash> {
self.chunk_index.iter().copied().collect()
}
fn write_manifest(&mut self, manifest: &ObjectManifest) -> Result<(), std::io::Error> {
let path = self.manifest_path(&manifest.content_hash);
let data = serde_json::to_vec(manifest)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
Self::write_and_sync(&path, &data)
}
fn read_manifest(&self, content_hash: &ContentHash) -> Result<Option<ObjectManifest>, std::io::Error> {
let path = self.manifest_path(content_hash);
match fs::read(&path) {
Ok(data) => {
let manifest: ObjectManifest = serde_json::from_slice(&data)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
Ok(Some(manifest))
}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(e) => Err(e),
}
}
fn delete_manifest(&mut self, content_hash: &ContentHash) -> Result<(), std::io::Error> {
let path = self.manifest_path(content_hash);
match fs::remove_file(&path) {
Ok(()) => Ok(()),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(e) => Err(e),
}
}
}

211
crates/datastore/src/types.rs Normal file
View file

@ -0,0 +1,211 @@
//! Core data types for the distributed datastore protocol.
//!
//! Content-hash-first addressing: every object is identified by
//! `blake3(blob_bytes)`. Names are optional metadata, not keys.
use std::collections::BTreeMap;
use std::fmt;
use std::path::PathBuf;
use serde::{Deserialize, Serialize};
use distribution::types::NodeId;
// ─── ContentHash ────────────────────────────────────────────────────────────
/// A blake3 content hash (32 bytes).
///
/// The primary identifier for blobs and the DHT key. Mirrors the `NodeId`
/// pattern from `distribution::types` — XOR distance for DHT routing, compact
/// Debug/Display for logging.
#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct ContentHash(pub [u8; 32]);
impl ContentHash {
/// Compute the blake3 hash of the given data.
pub fn of(data: &[u8]) -> Self {
let hash = blake3::hash(data);
ContentHash(*hash.as_bytes())
}
/// XOR distance between two content hashes (Kademlia metric).
pub fn xor_distance(&self, other: &ContentHash) -> [u8; 32] {
let mut out = [0u8; 32];
for i in 0..32 {
out[i] = self.0[i] ^ other.0[i];
}
out
}
/// Number of leading zero bits in the XOR distance to `other`.
/// Returns 0..=256. Used to select the k-bucket index in the metadata DHT.
pub fn xor_leading_zeros(&self, other: &ContentHash) -> u32 {
let dist = self.xor_distance(other);
let mut zeros = 0u32;
for byte in dist {
if byte == 0 {
zeros += 8;
} else {
zeros += byte.leading_zeros();
break;
}
}
zeros
}
/// Parse a 64-character hex string into a ContentHash.
/// Returns `None` if the string is not exactly 64 hex characters.
pub fn from_hex(hex: &str) -> Option<Self> {
if hex.len() != 64 {
return None;
}
let mut bytes = [0u8; 32];
for (i, chunk) in hex.as_bytes().chunks(2).enumerate() {
let hi = hex_digit(chunk[0])?;
let lo = hex_digit(chunk[1])?;
bytes[i] = (hi << 4) | lo;
}
Some(ContentHash(bytes))
}
/// Encode as lowercase hex string.
pub fn to_hex(&self) -> String {
let mut s = String::with_capacity(64);
for b in &self.0 {
use fmt::Write;
write!(s, "{:02x}", b).unwrap();
}
s
}
/// The zero hash (all zeroes). Used as a sentinel.
pub const ZERO: ContentHash = ContentHash([0u8; 32]);
}
impl fmt::Debug for ContentHash {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "Hash(")?;
for b in &self.0[..4] {
write!(f, "{:02x}", b)?;
}
write!(f, "\u{2026})")
}
}
impl fmt::Display for ContentHash {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
for b in &self.0[..8] {
write!(f, "{:02x}", b)?;
}
write!(f, "\u{2026}")
}
}
fn hex_digit(b: u8) -> Option<u8> {
match b {
b'0'..=b'9' => Some(b - b'0'),
b'a'..=b'f' => Some(b - b'a' + 10),
b'A'..=b'F' => Some(b - b'A' + 10),
_ => None,
}
}
// ─── ObjectEntry ────────────────────────────────────────────────────────────
/// Metadata record for a stored object — content-addressed by `blake3(blob_bytes)`.
///
/// Names are optional flat strings, not hierarchical paths.
/// No LWW conflict resolution — content hashes are unique identifiers.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct ObjectEntry {
/// Primary identifier: `blake3(entire_blob)`.
pub content_hash: ContentHash,
/// Optional human-readable name (flat string, not a path).
pub name: Option<String>,
/// Node that stores (or last wrote) this object.
pub node_id: NodeId,
/// User-defined key-value tags for filtering and search.
pub tags: BTreeMap<String, String>,
/// Total object size in bytes.
pub size_bytes: u64,
/// Wall-clock creation time (informational).
pub created_at: u64,
}
// ─── ObjectManifest ─────────────────────────────────────────────────────────
/// Describes the chunked layout of a stored object.
///
/// Keyed by `content_hash = blake3(entire_blob)`, computed via a streaming
/// hasher alongside chunking. The manifest itself is stored under this key.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct ObjectManifest {
/// Content hash of the entire blob: `blake3(all_bytes)`.
/// This is the primary key for looking up the manifest.
pub content_hash: ContentHash,
/// Ordered list of chunk references.
pub chunks: Vec<ChunkRef>,
/// Total size of the original object in bytes.
pub total_size: u64,
/// Fixed chunk size used during chunking (e.g. 1MB).
pub chunk_size: u32,
/// MIME type of the object, if known.
pub content_type: Option<String>,
}
/// A reference to a single chunk within an object manifest.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct ChunkRef {
/// Content hash of this chunk's data.
pub hash: ContentHash,
/// Byte offset of this chunk within the original object.
pub offset: u64,
/// Actual size of this chunk in bytes (last chunk may be smaller).
pub size: u32,
}
// ─── DatastoreConfig ────────────────────────────────────────────────────────
/// Configuration for a datastore node.
#[derive(Debug, Clone)]
pub struct DatastoreConfig {
/// Fixed chunk size in bytes. Default: 1,048,576 (1 MB).
pub chunk_size: u32,
/// Root directory for on-disk storage (chunks and manifests).
pub storage_path: PathBuf,
/// Ticks between GC sweeps.
pub gc_interval: u64,
/// Maximum number of simultaneous transfers.
pub max_concurrent_transfers: usize,
}
impl Default for DatastoreConfig {
fn default() -> Self {
Self {
chunk_size: 1_048_576,
storage_path: PathBuf::from("datastore"),
gc_interval: 1000,
max_concurrent_transfers: 4,
}
}
}
// ─── Transfer state ─────────────────────────────────────────────────────────
/// Status of an in-progress transfer.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum TransferStatus {
/// Download is in progress.
Downloading {
/// Number of chunks received so far.
chunks_received: usize,
/// Total number of chunks in the manifest.
chunks_total: usize,
},
/// Transfer completed successfully.
Complete,
/// Transfer failed.
Failed { reason: String },
/// Transfer was cancelled.
Cancelled,
}

326
crates/datastore/src/ui_html.rs Normal file
View file

@ -0,0 +1,326 @@
pub const DATASTORE_UI_HTML: &str = r##"<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>swactor-store</title>
<style>
* { margin: 0; padding: 0; box-sizing: border-box; }
body { font-family: 'Menlo', 'Consolas', 'Monaco', monospace; background: #0f1117; color: #e0e0e0; font-size: 13px; }
.header {
display: flex; align-items: center; justify-content: space-between;
padding: 12px 20px; background: #161822; border-bottom: 1px solid #2a2d3e;
}
.header h1 { font-size: 16px; font-weight: 600; color: #fff; }
.header .node-id { font-size: 11px; color: #888; margin-left: 12px; }
.container { max-width: 960px; margin: 0 auto; padding: 16px; }
.panel {
background: #161822; border: 1px solid #2a2d3e; border-radius: 6px;
padding: 16px; margin-bottom: 16px;
}
.panel h2 {
font-size: 12px; color: #888; text-transform: uppercase;
letter-spacing: 1px; margin-bottom: 12px;
}
.upload-row {
display: flex; gap: 10px; align-items: center; flex-wrap: wrap;
}
input[type="file"] {
background: #1c1f2e; color: #e0e0e0; border: 1px solid #2a2d3e;
border-radius: 4px; padding: 8px; font-family: inherit; font-size: 13px;
min-height: 44px; cursor: pointer;
}
input[type="file"]::file-selector-button {
background: #1e2030; color: #e0e0e0; border: 1px solid #2a2d3e;
border-radius: 4px; padding: 6px 12px; font-family: inherit;
font-size: 12px; cursor: pointer; margin-right: 8px;
}
input[type="file"]::file-selector-button:hover { border-color: #6366f1; }
input[type="text"] {
background: #1c1f2e; color: #e0e0e0; border: 1px solid #2a2d3e;
border-radius: 4px; padding: 8px 12px; font-family: inherit;
font-size: 13px; min-height: 44px; width: 200px;
}
input[type="text"]:focus { outline: none; border-color: #6366f1; }
button {
background: #1e2030; color: #e0e0e0; border: 1px solid #2a2d3e;
border-radius: 4px; padding: 8px 16px; font-family: inherit;
font-size: 13px; cursor: pointer; min-height: 44px;
transition: border-color 0.15s;
}
button:hover { border-color: #6366f1; color: #fff; }
button:disabled { opacity: 0.4; cursor: default; }
button.danger:hover { border-color: #f44336; }
button.primary { background: #6366f1; border-color: #6366f1; color: #fff; font-weight: 600; }
button.primary:hover { background: #5558e6; }
.toast {
position: fixed; bottom: 20px; right: 20px; padding: 10px 16px;
border-radius: 4px; font-size: 12px; z-index: 100; opacity: 0;
transition: opacity 0.3s; pointer-events: none;
}
.toast.show { opacity: 1; }
.toast.success { background: #4caf50; color: #fff; }
.toast.error { background: #f44336; color: #fff; }
table { width: 100%; border-collapse: collapse; }
th {
text-align: left; font-size: 10px; color: #888;
text-transform: uppercase; letter-spacing: 1px;
padding: 6px 8px; border-bottom: 1px solid #2a2d3e;
}
td {
padding: 8px; border-bottom: 1px solid #1c1f2e;
font-size: 13px; vertical-align: middle;
}
tr:hover td { background: #1c1f2e; }
tr { cursor: pointer; }
.hash-cell { color: #6366f1; font-size: 12px; }
.size-cell { color: #888; white-space: nowrap; }
.actions-cell { white-space: nowrap; text-align: right; }
.actions-cell button { min-height: 32px; padding: 4px 10px; font-size: 11px; }
.empty-state {
text-align: center; color: #555; padding: 32px; font-size: 14px;
}
/* Modal overlay */
.modal-overlay {
display: none; position: fixed; inset: 0;
background: rgba(0,0,0,0.6); z-index: 50;
align-items: center; justify-content: center;
}
.modal-overlay.open { display: flex; }
.modal {
background: #161822; border: 1px solid #2a2d3e; border-radius: 6px;
padding: 20px; width: 90%; max-width: 560px; max-height: 80vh;
overflow-y: auto;
}
.modal h2 { font-size: 14px; color: #fff; margin-bottom: 16px; text-transform: none; letter-spacing: 0; }
.modal-close {
float: right; background: none; border: none; color: #888;
font-size: 18px; cursor: pointer; min-height: auto; padding: 0;
}
.modal-close:hover { color: #fff; border: none; }
.detail-row { display: flex; margin-bottom: 8px; }
.detail-label { color: #888; width: 110px; flex-shrink: 0; font-size: 11px; text-transform: uppercase; padding-top: 2px; }
.detail-value { color: #e0e0e0; word-break: break-all; font-size: 13px; }
.chunk-list { margin-top: 8px; }
.chunk-item { color: #888; font-size: 11px; padding: 2px 0; }
@media (max-width: 640px) {
.upload-row { flex-direction: column; align-items: stretch; }
input[type="text"] { width: 100%; }
.header { flex-direction: column; align-items: flex-start; gap: 4px; }
.header .node-id { margin-left: 0; }
.actions-cell { display: flex; gap: 4px; justify-content: flex-end; }
}
</style>
</head>
<body>
<div class="header">
<div style="display:flex;align-items:center;flex-wrap:wrap;">
<h1>swactor-store</h1>
<span class="node-id" id="nodeId">connecting...</span>
</div>
</div>
<div class="container">
<!-- Upload panel -->
<div class="panel">
<h2>Upload</h2>
<div class="upload-row">
<input type="file" id="fileInput" />
<input type="text" id="nameInput" placeholder="name (optional)" />
<button class="primary" id="uploadBtn" onclick="upload()">Upload</button>
</div>
</div>
<!-- Object table -->
<div class="panel">
<h2>Objects</h2>
<div id="tableWrap"></div>
</div>
</div>
<!-- Detail modal -->
<div class="modal-overlay" id="modal" onclick="if(event.target===this)closeModal()">
<div class="modal">
<button class="modal-close" onclick="closeModal()">&times;</button>
<h2>Object Detail</h2>
<div id="modalBody"></div>
</div>
</div>
<div class="toast" id="toast"></div>
<script>
const $ = id => document.getElementById(id);
function toast(msg, type) {
const t = $('toast');
t.textContent = msg;
t.className = 'toast show ' + type;
setTimeout(() => t.className = 'toast', 2500);
}
function fmtSize(b) {
if (b < 1024) return b + ' B';
if (b < 1048576) return (b / 1024).toFixed(1) + ' KB';
if (b < 1073741824) return (b / 1048576).toFixed(1) + ' MB';
return (b / 1073741824).toFixed(1) + ' GB';
}
async function fetchStatus() {
try {
const r = await fetch('/api/status');
const j = await r.json();
$('nodeId').textContent = j.node_id.substring(0, 16) + '...';
$('nodeId').title = j.node_id;
} catch(e) {
$('nodeId').textContent = 'offline';
}
}
async function refreshList() {
try {
const r = await fetch('/api/list');
const j = await r.json();
renderTable(j.entries || []);
} catch(e) {
$('tableWrap').innerHTML = '<div class="empty-state">failed to load</div>';
}
}
function renderTable(entries) {
if (entries.length === 0) {
$('tableWrap').innerHTML = '<div class="empty-state">no objects stored</div>';
return;
}
let html = '<table><thead><tr><th>Hash</th><th>Name</th><th>Size</th><th style="text-align:right">Actions</th></tr></thead><tbody>';
for (const e of entries) {
const h = e.content_hash;
const short = h.substring(0, 16);
const name = e.name || '—';
const size = fmtSize(e.size_bytes);
html += '<tr onclick="showDetail(\'' + h + '\')">';
html += '<td class="hash-cell" title="' + h + '">' + short + '</td>';
html += '<td>' + escHtml(name) + '</td>';
html += '<td class="size-cell">' + size + '</td>';
html += '<td class="actions-cell">';
html += '<button onclick="event.stopPropagation();download(\'' + h + '\',\'' + escAttr(e.name || h.substring(0,12)) + '\')">download</button> ';
html += '<button class="danger" onclick="event.stopPropagation();del(\'' + h + '\')">delete</button>';
html += '</td></tr>';
}
html += '</tbody></table>';
$('tableWrap').innerHTML = html;
}
function escHtml(s) { const d = document.createElement('div'); d.textContent = s; return d.innerHTML; }
function escAttr(s) { return s.replace(/'/g, "\\'").replace(/"/g, '&quot;'); }
async function upload() {
const file = $('fileInput').files[0];
if (!file) { toast('select a file first', 'error'); return; }
const name = $('nameInput').value.trim();
const btn = $('uploadBtn');
btn.disabled = true;
btn.textContent = 'uploading...';
try {
let url = '/api/put';
if (name) url += '?name=' + encodeURIComponent(name);
const r = await fetch(url, { method: 'POST', body: file });
if (!r.ok) throw new Error((await r.json()).error || r.statusText);
const j = await r.json();
toast('uploaded ' + j.content_hash.substring(0, 12), 'success');
$('fileInput').value = '';
$('nameInput').value = '';
refreshList();
} catch(e) {
toast('upload failed: ' + e.message, 'error');
} finally {
btn.disabled = false;
btn.textContent = 'Upload';
}
}
async function download(hash, filename) {
try {
const r = await fetch('/api/data?hash=' + hash);
if (!r.ok) throw new Error('not found');
const blob = await r.blob();
const a = document.createElement('a');
a.href = URL.createObjectURL(blob);
a.download = filename;
a.click();
URL.revokeObjectURL(a.href);
} catch(e) {
toast('download failed: ' + e.message, 'error');
}
}
async function del(hash) {
if (!confirm('Delete ' + hash.substring(0, 16) + '?')) return;
try {
const r = await fetch('/api/delete?hash=' + hash, { method: 'POST' });
if (!r.ok) throw new Error((await r.json()).error || r.statusText);
toast('deleted', 'success');
refreshList();
} catch(e) {
toast('delete failed: ' + e.message, 'error');
}
}
async function showDetail(hash) {
try {
const r = await fetch('/api/get?hash=' + hash);
if (!r.ok) throw new Error('not found');
const j = await r.json();
const e = j.entry;
const m = j.manifest;
let html = '';
html += row('Hash', e.content_hash);
html += row('Name', e.name || '—');
html += row('Size', fmtSize(e.size_bytes));
html += row('Node', e.node_id.substring(0, 16) + '...');
if (e.tags && Object.keys(e.tags).length > 0) {
html += row('Tags', Object.entries(e.tags).map(([k,v]) => k + '=' + v).join(', '));
}
html += row('Chunks', m.chunks.length + ' (' + fmtSize(m.chunk_size) + ' each)');
if (m.chunks.length > 0) {
html += '<div class="chunk-list">';
for (let i = 0; i < m.chunks.length; i++) {
const c = m.chunks[i];
html += '<div class="chunk-item">#' + i + ' ' + c.hash.substring(0, 16) + ' (' + fmtSize(c.size) + ')</div>';
}
html += '</div>';
}
$('modalBody').innerHTML = html;
$('modal').classList.add('open');
} catch(e) {
toast('failed to load detail', 'error');
}
}
function row(label, value) {
return '<div class="detail-row"><div class="detail-label">' + label + '</div><div class="detail-value">' + escHtml(String(value)) + '</div></div>';
}
function closeModal() { $('modal').classList.remove('open'); }
document.addEventListener('keydown', e => { if (e.key === 'Escape') closeModal(); });
// Init
fetchStatus();
refreshList();
</script>
</body>
</html>"##;

190
crates/datastore/tests/api_integration_test.rs Normal file
View file

@ -0,0 +1,190 @@
#![cfg(feature = "node")]
//! Integration test: spins up a real datastore node with HTTP API and exercises
//! the full CRUD lifecycle over HTTP.
use std::sync::Arc;
use std::thread;
use std::time::Duration;
use swactor::config::RuntimeConfig;
use swactor::runtime::Runtime;
use swactor_datastore::actors::{BlobStoreActor, DatastoreNode, MetadataActor};
use swactor_datastore::api::start_api_server;
use swactor_datastore::metrics::DatastoreMetrics;
use swactor_datastore::storage::InMemoryBackend;
use swactor_datastore::DatastoreConfig;
use distribution::types::NodeId;
fn find_free_port() -> u16 {
std::net::TcpListener::bind("127.0.0.1:0")
.unwrap()
.local_addr()
.unwrap()
.port()
}
/// Full CRUD lifecycle over HTTP:
/// status → put → list → get metadata → get data → delete → list (empty) → get (404)
#[test]
fn http_crud_lifecycle() {
let port = find_free_port();
let base = format!("http://127.0.0.1:{port}");
// Set up runtime with worker threads (needed for HTTP server)
let collector = runtime_dashboard::collector::StatsCollector::new(2);
let mut rt = Runtime::new(RuntimeConfig {
num_threads: 2,
max_actors: 1024,
channel_buffer_size: 2000,
..Default::default()
});
rt.set_stats_hook(collector);
let node_id = NodeId([0xAA; 32]);
let config = DatastoreConfig {
chunk_size: 1_048_576,
gc_interval: 1000,
..Default::default()
};
let backend: Box<dyn swactor_datastore::StorageBackend> = Box::new(InMemoryBackend::new());
let blob_store_addr = rt.spawn(BlobStoreActor::new(backend)).unwrap();
let mut metadata = MetadataActor::new(node_id, &config);
metadata.set_blob_store(blob_store_addr);
let metadata_addr = rt.spawn(metadata).unwrap();
let datastore_node = DatastoreNode::new(node_id, blob_store_addr, metadata_addr, config);
let datastore_addr = rt.spawn(datastore_node).unwrap();
let handle = rt.run().expect("failed to start runtime");
let metrics = Arc::new(DatastoreMetrics::new());
let (api_shutdown, _peers) = start_api_server(
handle.runtime.clone(),
datastore_addr,
metadata_addr,
blob_store_addr,
port,
Arc::clone(&metrics),
);
// Give the HTTP server time to bind
thread::sleep(Duration::from_millis(200));
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
run_crud_scenario(&base, &metrics);
}));
// Cleanup
api_shutdown.store(true, std::sync::atomic::Ordering::Relaxed);
handle.shutdown();
handle.join();
if let Err(e) = result {
std::panic::resume_unwind(e);
}
}
fn run_crud_scenario(base: &str, metrics: &Arc<DatastoreMetrics>) {
// 1. Status — should return a node_id
let status: serde_json::Value = ureq::get(&format!("{base}/api/status"))
.call()
.expect("status request failed")
.into_json()
.unwrap();
let node_id = status["node_id"].as_str().expect("node_id should be a string");
assert_eq!(node_id.len(), 64, "node_id should be 64 hex chars");
// 2. Put — upload some content
let content = b"hello from the integration test!";
let put_resp: serde_json::Value = ureq::post(&format!("{base}/api/put?name=greeting"))
.send_bytes(content)
.expect("put request failed")
.into_json()
.unwrap();
let hash = put_resp["content_hash"]
.as_str()
.expect("put should return content_hash");
assert_eq!(hash.len(), 64, "content_hash should be 64 hex chars");
// 3. List — should contain exactly one entry matching our upload
let list_resp: serde_json::Value = ureq::get(&format!("{base}/api/list"))
.call()
.expect("list request failed")
.into_json()
.unwrap();
let entries = list_resp["entries"].as_array().expect("entries should be an array");
assert_eq!(entries.len(), 1, "should have exactly 1 entry after put");
assert_eq!(entries[0]["content_hash"].as_str().unwrap(), hash);
assert_eq!(entries[0]["name"].as_str().unwrap(), "greeting");
// 4. Get metadata — entry + manifest for the uploaded object
let get_resp: serde_json::Value = ureq::get(&format!("{base}/api/get?hash={hash}"))
.call()
.expect("get request failed")
.into_json()
.unwrap();
let entry = &get_resp["entry"];
assert_eq!(entry["content_hash"].as_str().unwrap(), hash);
assert_eq!(entry["name"].as_str().unwrap(), "greeting");
assert_eq!(entry["size_bytes"].as_u64().unwrap(), content.len() as u64);
let manifest = &get_resp["manifest"];
let chunks = manifest["chunks"].as_array().expect("manifest should have chunks");
assert!(!chunks.is_empty(), "manifest should have at least one chunk");
// 5. Get data — download the raw bytes and verify content matches
let data_resp = ureq::get(&format!("{base}/api/data?hash={hash}"))
.call()
.expect("data request failed");
let mut downloaded = Vec::new();
data_resp
.into_reader()
.read_to_end(&mut downloaded)
.unwrap();
assert_eq!(downloaded, content, "downloaded bytes should match uploaded content");
// 6. Delete — remove the object
let del_resp: serde_json::Value = ureq::post(&format!("{base}/api/delete?hash={hash}"))
.call()
.expect("delete request failed")
.into_json()
.unwrap();
assert_eq!(del_resp["content_hash"].as_str().unwrap(), hash);
// 7. List after delete — should be empty
let list_resp2: serde_json::Value = ureq::get(&format!("{base}/api/list"))
.call()
.expect("list request failed")
.into_json()
.unwrap();
let entries2 = list_resp2["entries"]
.as_array()
.expect("entries should be an array");
assert!(entries2.is_empty(), "list should be empty after delete");
// 8. Get after delete — should 404
let get_err = ureq::get(&format!("{base}/api/get?hash={hash}")).call();
match get_err {
Err(ureq::Error::Status(404, _)) => {} // expected
Err(e) => panic!("expected 404, got error: {e}"),
Ok(_) => panic!("expected 404, got 200"),
}
// 9. Verify metrics snapshot reflects the full lifecycle
let snap = metrics.snapshot();
assert_eq!(snap.put_ops, 1, "one put recorded");
// handle_get + handle_data = 2 get operations
assert_eq!(snap.get_ops, 2, "metadata-get + data-get recorded");
assert_eq!(snap.delete_ops, 1, "one delete recorded");
assert!(snap.objects.is_empty(), "no objects after delete");
assert!(
snap.recent_events.len() >= 4,
"at least 4 events (put + get + get + delete), got {}",
snap.recent_events.len()
);
}

231
crates/datastore/tests/blob_store_tests.rs Normal file
View file

@ -0,0 +1,231 @@
//! Tests for BlobStoreActor — exercised as a black box through the swactor runtime.
mod common;
use std::collections::HashSet;
use swactor_datastore::chunking::chunk_blob;
use swactor_datastore::messages::{BlobStoreMsg, DatastoreResponse};
use swactor_datastore::types::{ChunkRef, ContentHash, ObjectManifest};
use common::{spawn_blob_store, test_runtime, tick_and_drain, tick_n, tick_until_recv};
// ═══════════════════════════════════════════════════════════════════════════
// Scenario: single chunk CRUD through actor
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn write_chunk_and_read_it_back_through_actor() {
let rt = test_runtime();
let blob = spawn_blob_store(&rt);
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
let reply = *inbox.addr();
let data = b"actor-chunk".to_vec();
let hash = ContentHash::of(&data);
// Write.
rt.send_to(blob, BlobStoreMsg::WriteChunk { hash, data: data.clone(), reply_to: reply }).unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
assert!(matches!(resp, DatastoreResponse::ChunkStored { hash: h } if h == hash));
// Read.
rt.send_to(blob, BlobStoreMsg::ReadChunk { hash, reply_to: reply }).unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
match resp {
DatastoreResponse::ChunkOk { hash: h, data: d } => {
assert_eq!(h, hash);
assert_eq!(d, data);
}
other => panic!("expected ChunkOk, got: {other:?}"),
}
}
#[test]
fn reading_absent_chunk_returns_not_found() {
let rt = test_runtime();
let blob = spawn_blob_store(&rt);
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
rt.send_to(blob, BlobStoreMsg::ReadChunk {
hash: ContentHash::of(b"ghost"),
reply_to: *inbox.addr(),
}).unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
assert!(matches!(resp, DatastoreResponse::NotFound));
}
#[test]
fn has_chunk_reports_presence_correctly() {
let rt = test_runtime();
let blob = spawn_blob_store(&rt);
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
let reply = *inbox.addr();
let data = b"check-me";
let hash = ContentHash::of(data);
// Before write.
rt.send_to(blob, BlobStoreMsg::HasChunk { hash, reply_to: reply }).unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
assert!(matches!(resp, DatastoreResponse::Bool(false)));
// After write.
rt.send_to(blob, BlobStoreMsg::WriteChunk { hash, data: data.to_vec(), reply_to: reply }).unwrap();
tick_until_recv(&rt, &inbox, 10); // drain ChunkStored
rt.send_to(blob, BlobStoreMsg::HasChunk { hash, reply_to: reply }).unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
assert!(matches!(resp, DatastoreResponse::Bool(true)));
}
#[test]
fn list_chunks_after_storing_several() {
let rt = test_runtime();
let blob = spawn_blob_store(&rt);
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
let reply = *inbox.addr();
let mut expected = HashSet::new();
for i in 0u8..3 {
let data = vec![i; 16];
let hash = ContentHash::of(&data);
expected.insert(hash);
rt.send_to(blob, BlobStoreMsg::WriteChunk { hash, data, reply_to: reply }).unwrap();
}
// Drain write responses.
tick_and_drain(&rt, &inbox, 5);
// List.
rt.send_to(blob, BlobStoreMsg::ListChunks { reply_to: reply }).unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
match resp {
DatastoreResponse::ChunkList { hashes } => {
let listed: HashSet<_> = hashes.into_iter().collect();
assert_eq!(listed, expected);
}
other => panic!("expected ChunkList, got: {other:?}"),
}
}
// ═══════════════════════════════════════════════════════════════════════════
// Scenario: GC unreferenced through actor
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn gc_unreferenced_removes_orphan_chunks() {
let rt = test_runtime();
let blob = spawn_blob_store(&rt);
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
let reply = *inbox.addr();
// Store 5 chunks.
let mut all_hashes = Vec::new();
for i in 0u8..5 {
let data = vec![i; 32];
let hash = ContentHash::of(&data);
all_hashes.push(hash);
rt.send_to(blob, BlobStoreMsg::WriteChunk { hash, data, reply_to: reply }).unwrap();
}
tick_and_drain(&rt, &inbox, 5);
// Mark only the first 2 as referenced.
let referenced: HashSet<_> = all_hashes[..2].iter().copied().collect();
rt.send_to(blob, BlobStoreMsg::GcUnreferenced { referenced: referenced.clone() }).unwrap();
tick_n(&rt, 3);
// List remaining.
rt.send_to(blob, BlobStoreMsg::ListChunks { reply_to: reply }).unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
match resp {
DatastoreResponse::ChunkList { hashes } => {
let remaining: HashSet<_> = hashes.into_iter().collect();
assert_eq!(remaining, referenced);
}
other => panic!("expected ChunkList, got: {other:?}"),
}
}
// ═══════════════════════════════════════════════════════════════════════════
// Scenario: manifest CRUD through actor
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn write_and_read_manifest_through_actor() {
let rt = test_runtime();
let blob = spawn_blob_store(&rt);
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
let reply = *inbox.addr();
let manifest = ObjectManifest {
content_hash: ContentHash::of(b"test-blob"),
chunks: vec![ChunkRef {
hash: ContentHash::of(b"chunk-0"),
offset: 0,
size: 256,
}],
total_size: 256,
chunk_size: 1024,
content_type: None,
};
// Write.
rt.send_to(blob, BlobStoreMsg::WriteManifest { manifest: manifest.clone(), reply_to: reply }).unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
assert!(matches!(resp, DatastoreResponse::ManifestStored { .. }));
// Read.
rt.send_to(blob, BlobStoreMsg::ReadManifest { hash: manifest.content_hash, reply_to: reply }).unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
match resp {
DatastoreResponse::ManifestOk { manifest: m } => assert_eq!(m, manifest),
other => panic!("expected ManifestOk, got: {other:?}"),
}
}
// ═══════════════════════════════════════════════════════════════════════════
// Scenario: full blob lifecycle through actor
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn full_blob_lifecycle_through_actor() {
let rt = test_runtime();
let blob = spawn_blob_store(&rt);
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
let reply = *inbox.addr();
let original: Vec<u8> = (0..300).map(|i| (i % 256) as u8).collect();
let (content_hash, manifest, chunks) = chunk_blob(&original, 128);
// Store all chunks.
for (hash, data) in &chunks {
rt.send_to(blob, BlobStoreMsg::WriteChunk { hash: *hash, data: data.clone(), reply_to: reply }).unwrap();
}
tick_and_drain(&rt, &inbox, 5);
// Store manifest.
rt.send_to(blob, BlobStoreMsg::WriteManifest { manifest: manifest.clone(), reply_to: reply }).unwrap();
tick_and_drain(&rt, &inbox, 3);
// Read manifest back.
rt.send_to(blob, BlobStoreMsg::ReadManifest { hash: content_hash, reply_to: reply }).unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
let read_manifest = match resp {
DatastoreResponse::ManifestOk { manifest: m } => m,
other => panic!("expected ManifestOk, got: {other:?}"),
};
// Read all chunks and reassemble.
let mut reassembled = Vec::new();
for chunk_ref in &read_manifest.chunks {
rt.send_to(blob, BlobStoreMsg::ReadChunk { hash: chunk_ref.hash, reply_to: reply }).unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
match resp {
DatastoreResponse::ChunkOk { data, .. } => reassembled.extend_from_slice(&data),
other => panic!("expected ChunkOk, got: {other:?}"),
}
}
assert_eq!(reassembled, original);
assert!(swactor_datastore::verify_integrity(&reassembled, &content_hash));
}

200
crates/datastore/tests/chunking_tests.rs Normal file
View file

@ -0,0 +1,200 @@
//! Tests for the chunking engine — pure function contracts.
use std::collections::HashSet;
use swactor_datastore::chunking::{chunk_blob, reassemble_blob, verify_integrity, ChunkingError};
use swactor_datastore::types::ContentHash;
// ═══════════════════════════════════════════════════════════════════════════
// Scenario: basic chunking behaviour
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn small_file_fits_in_single_chunk() {
let data = b"tiny";
let (content_hash, manifest, chunks) = chunk_blob(data, 1024);
assert_eq!(chunks.len(), 1);
assert_eq!(manifest.chunks.len(), 1);
assert_eq!(content_hash, ContentHash::of(data));
// Single chunk's hash == hash of that chunk's data (which is the whole blob).
assert_eq!(manifest.chunks[0].hash, ContentHash::of(data));
}
#[test]
fn multi_chunk_blob_reassembles_to_original() {
let data: Vec<u8> = (0..300).map(|i| (i % 256) as u8).collect();
let chunk_size = 64;
let (_, manifest, chunks) = chunk_blob(&data, chunk_size);
assert!(manifest.chunks.len() >= 3);
let reassembled = reassemble_blob(&manifest, &chunks).unwrap();
assert_eq!(reassembled, data);
}
#[test]
fn last_chunk_is_smaller_when_not_aligned() {
let data = vec![0xAB; 100];
let (_, manifest, chunks) = chunk_blob(&data, 64);
assert_eq!(chunks.len(), 2);
assert_eq!(manifest.chunks[0].size, 64);
assert_eq!(manifest.chunks[1].size, 36);
assert_eq!(manifest.chunks[0].offset, 0);
assert_eq!(manifest.chunks[1].offset, 64);
}
#[test]
fn empty_blob_produces_empty_manifest() {
let (content_hash, manifest, chunks) = chunk_blob(b"", 1024);
assert!(chunks.is_empty());
assert!(manifest.chunks.is_empty());
assert_eq!(manifest.total_size, 0);
assert_eq!(content_hash, ContentHash::of(b""));
// Reassembly of empty manifest yields empty data.
let reassembled = reassemble_blob(&manifest, &chunks).unwrap();
assert!(reassembled.is_empty());
}
// ═══════════════════════════════════════════════════════════════════════════
// Scenario: reassembly failure modes
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn reassembly_with_missing_chunk_fails() {
// Use non-repeating data so each chunk has a unique hash.
let data: Vec<u8> = (0..128).collect();
let (_, manifest, mut chunks) = chunk_blob(&data, 64);
// Remove the last chunk.
chunks.pop();
let err = reassemble_blob(&manifest, &chunks).unwrap_err();
match err {
ChunkingError::MissingChunk { hash } => {
assert_eq!(hash, manifest.chunks.last().unwrap().hash);
}
other => panic!("expected MissingChunk, got: {other:?}"),
}
}
#[test]
fn reassembly_detects_wrong_content_hash() {
let data = vec![42; 128];
let (_, mut manifest, chunks) = chunk_blob(&data, 64);
// Corrupt the manifest's content hash.
manifest.content_hash = ContentHash::of(b"wrong");
let err = reassemble_blob(&manifest, &chunks).unwrap_err();
assert!(matches!(err, ChunkingError::HashMismatch { .. }));
}
// ═══════════════════════════════════════════════════════════════════════════
// Scenario: determinism and deduplication
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn identical_blobs_produce_identical_manifests() {
let data = b"deterministic input";
let (h1, m1, c1) = chunk_blob(data, 8);
let (h2, m2, c2) = chunk_blob(data, 8);
assert_eq!(h1, h2);
assert_eq!(m1, m2);
assert_eq!(c1.len(), c2.len());
for (a, b) in c1.iter().zip(c2.iter()) {
assert_eq!(a.0, b.0);
assert_eq!(a.1, b.1);
}
}
#[test]
fn deduplication_across_objects() {
// Two blobs that share a common prefix produce the same chunk hash for that prefix.
let shared_prefix = vec![0xBE; 64];
let mut blob_a = shared_prefix.clone();
blob_a.extend_from_slice(&[0xAA; 64]);
let mut blob_b = shared_prefix.clone();
blob_b.extend_from_slice(&[0xBB; 64]);
let (_, _, chunks_a) = chunk_blob(&blob_a, 64);
let (_, _, chunks_b) = chunk_blob(&blob_b, 64);
// First chunk should be identical (shared prefix).
assert_eq!(chunks_a[0].0, chunks_b[0].0);
// Second chunk should differ.
assert_ne!(chunks_a[1].0, chunks_b[1].0);
// A HashSet of all chunk hashes should have 3 unique entries (shared + 2 distinct).
let all_hashes: HashSet<_> = chunks_a
.iter()
.chain(chunks_b.iter())
.map(|(h, _)| *h)
.collect();
assert_eq!(all_hashes.len(), 3);
}
#[test]
fn verify_integrity_passes_for_correct_data() {
let data = b"check me";
let hash = ContentHash::of(data);
assert!(verify_integrity(data, &hash));
}
#[test]
fn verify_integrity_fails_for_wrong_data() {
let hash = ContentHash::of(b"original");
assert!(!verify_integrity(b"tampered", &hash));
}
// ═══════════════════════════════════════════════════════════════════════════
// Property-based tests
// ═══════════════════════════════════════════════════════════════════════════
mod proptests {
use super::*;
use proptest::prelude::*;
proptest! {
#[test]
fn chunk_then_reassemble_is_identity(
data in proptest::collection::vec(any::<u8>(), 0..8192),
chunk_size in 1u32..=512,
) {
let (_, manifest, chunks) = chunk_blob(&data, chunk_size);
let reassembled = reassemble_blob(&manifest, &chunks).unwrap();
prop_assert_eq!(data, reassembled);
}
}
proptest! {
#[test]
fn content_hash_matches_blake3_of_whole_blob(
data in proptest::collection::vec(any::<u8>(), 0..4096),
) {
let (content_hash, manifest, _) = chunk_blob(&data, 256);
let expected = ContentHash::of(&data);
prop_assert_eq!(content_hash, expected);
prop_assert_eq!(manifest.content_hash, expected);
}
}
proptest! {
#[test]
fn chunk_offsets_are_contiguous(
data in proptest::collection::vec(any::<u8>(), 1..4096),
chunk_size in 1u32..=256,
) {
let (_, manifest, _) = chunk_blob(&data, chunk_size);
let mut expected_offset = 0u64;
for chunk_ref in &manifest.chunks {
prop_assert_eq!(chunk_ref.offset, expected_offset);
expected_offset += chunk_ref.size as u64;
}
prop_assert_eq!(expected_offset, manifest.total_size);
}
}
}

756
crates/datastore/tests/common/mod.rs Normal file
View file

@ -0,0 +1,756 @@
//! Shared test harness for datastore actor tests.
#![allow(dead_code)]
use std::collections::BTreeMap;
use std::sync::Arc;
use swactor::actor::{ActorAddress, Message};
use swactor::runtime::{Inbox, Runtime, RuntimeConfig};
use swactor_std::StdExtension;
use swactor_datastore::chunking::chunk_blob;
use swactor_datastore::messages::{BlobStoreMsg, DatastoreResponse, MetadataMsg};
use swactor_datastore::storage::InMemoryBackend;
use swactor_datastore::types::{ChunkRef, ContentHash, DatastoreConfig, ObjectEntry, ObjectManifest};
use swactor_datastore::{BlobStoreActor, DatastoreNode, MetadataActor, TransferActor};
use distribution::types::NodeId;
/// Create a single-threaded runtime with StdExtension.
pub fn test_runtime() -> Runtime {
Runtime::new(RuntimeConfig::default()).with_extension(Arc::new(StdExtension::new()))
}
/// Tick exactly `n` times.
pub fn tick_n(rt: &Runtime, n: usize) {
for _ in 0..n {
rt.tick();
}
}
/// Tick up to `max` times, returning as soon as `inbox` has a message.
pub fn tick_until_recv<M: Message>(rt: &Runtime, inbox: &Inbox<M>, max: usize) -> Option<M> {
for _ in 0..max {
rt.tick();
if let Some(msg) = inbox.try_recv() {
return Some(msg);
}
}
None
}
/// Tick `n` times, then drain all messages from the inbox.
pub fn tick_and_drain<M: Message>(rt: &Runtime, inbox: &Inbox<M>, ticks: usize) -> Vec<M> {
for _ in 0..ticks {
rt.tick();
}
std::iter::from_fn(|| inbox.try_recv()).collect()
}
/// Spawn a BlobStoreActor backed by InMemoryBackend.
pub fn spawn_blob_store(rt: &Runtime) -> ActorAddress {
rt.spawn(BlobStoreActor::new(Box::new(InMemoryBackend::new())))
.unwrap()
}
/// Spawn a TransferActor wired to the given BlobStoreActor.
pub fn spawn_transfer(rt: &Runtime, blob_store_addr: ActorAddress) -> ActorAddress {
rt.spawn(TransferActor::new(blob_store_addr)).unwrap()
}
/// Spawn a MetadataActor with the given node_id and default config.
pub fn spawn_metadata(rt: &Runtime, node_id: NodeId) -> ActorAddress {
let config = DatastoreConfig::default();
rt.spawn(MetadataActor::new(node_id, &config)).unwrap()
}
/// A test node ID.
pub fn test_node_id() -> NodeId {
NodeId([0x42; 32])
}
/// Create a simple ObjectEntry for testing.
pub fn make_entry(data: &[u8], name: Option<&str>) -> ObjectEntry {
ObjectEntry {
content_hash: ContentHash::of(data),
name: name.map(|s| s.to_string()),
node_id: test_node_id(),
tags: BTreeMap::new(),
size_bytes: data.len() as u64,
created_at: 0,
}
}
/// Create a single-chunk ObjectManifest for testing.
pub fn make_manifest(data: &[u8]) -> ObjectManifest {
let content_hash = ContentHash::of(data);
ObjectManifest {
content_hash,
chunks: vec![ChunkRef {
hash: content_hash,
offset: 0,
size: data.len() as u32,
}],
total_size: data.len() as u64,
chunk_size: data.len() as u32,
content_type: None,
}
}
/// Full lifecycle harness: spawns both BlobStore and Metadata actors.
pub struct DatastoreHarness {
pub rt: Runtime,
pub blob_store: ActorAddress,
pub metadata: ActorAddress,
pub inbox: Inbox<DatastoreResponse>,
pub node_id: NodeId,
}
impl DatastoreHarness {
pub fn new() -> Self {
let rt = test_runtime();
let blob_store = spawn_blob_store(&rt);
let node_id = test_node_id();
let metadata = spawn_metadata(&rt, node_id);
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
// Let actors initialize.
tick_n(&rt, 2);
Self {
rt,
blob_store,
metadata,
inbox,
node_id,
}
}
pub fn reply_addr(&self) -> ActorAddress {
*self.inbox.addr()
}
/// Store a blob: chunk it, store chunks via BlobStoreActor, store metadata via MetadataActor.
pub fn put_blob(
&self,
data: &[u8],
name: Option<&str>,
) -> ContentHash {
let (content_hash, manifest, chunks) = chunk_blob(data, 1_048_576);
// Store all chunks.
for (hash, chunk_data) in &chunks {
self.rt
.send_to(
self.blob_store,
BlobStoreMsg::WriteChunk {
hash: *hash,
data: chunk_data.clone(),
reply_to: self.reply_addr(),
},
)
.unwrap();
}
// Let chunk writes complete.
tick_n(&self.rt, 3);
// Drain chunk stored responses.
let _ = tick_and_drain(&self.rt, &self.inbox, 2);
// Store manifest via BlobStoreActor.
self.rt
.send_to(
self.blob_store,
BlobStoreMsg::WriteManifest {
manifest: manifest.clone(),
reply_to: self.reply_addr(),
},
)
.unwrap();
tick_n(&self.rt, 2);
let _ = tick_and_drain(&self.rt, &self.inbox, 1);
// Store metadata via MetadataActor.
let entry = ObjectEntry {
content_hash,
name: name.map(|s| s.to_string()),
node_id: self.node_id,
tags: BTreeMap::new(),
size_bytes: data.len() as u64,
created_at: 0,
};
self.rt
.send_to(
self.metadata,
MetadataMsg::PutObject {
entry,
manifest,
reply_to: self.reply_addr(),
},
)
.unwrap();
tick_n(&self.rt, 2);
// Drain PutOk.
let responses = tick_and_drain(&self.rt, &self.inbox, 1);
assert!(
responses.iter().any(|r| matches!(r, DatastoreResponse::PutOk { .. })),
"expected PutOk response"
);
content_hash
}
/// Read blob data back through the actor system.
pub fn get_blob(&self, content_hash: &ContentHash) -> Option<Vec<u8>> {
// Ask metadata actor for the manifest.
self.rt
.send_to(
self.metadata,
MetadataMsg::GetObject {
content_hash: *content_hash,
reply_to: self.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&self.rt, &self.inbox, 10)?;
let manifest = match resp {
DatastoreResponse::GetOk { manifest, .. } => manifest,
DatastoreResponse::NotFound => return None,
other => panic!("unexpected response: {other:?}"),
};
// Read each chunk.
let mut data = Vec::new();
for chunk_ref in &manifest.chunks {
self.rt
.send_to(
self.blob_store,
BlobStoreMsg::ReadChunk {
hash: chunk_ref.hash,
reply_to: self.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&self.rt, &self.inbox, 10).unwrap();
match resp {
DatastoreResponse::ChunkOk { data: chunk_data, .. } => {
data.extend_from_slice(&chunk_data);
}
other => panic!("unexpected chunk response: {other:?}"),
}
}
Some(data)
}
/// Delete an object by content hash.
pub fn delete_blob(&self, content_hash: &ContentHash) -> DatastoreResponse {
self.rt
.send_to(
self.metadata,
MetadataMsg::DeleteObject {
content_hash: *content_hash,
reply_to: self.reply_addr(),
},
)
.unwrap();
tick_until_recv(&self.rt, &self.inbox, 10).unwrap()
}
/// List local objects with optional name filter.
pub fn list_local(&self, name_filter: Option<&str>) -> Vec<ObjectEntry> {
self.rt
.send_to(
self.metadata,
MetadataMsg::ListLocal {
name_filter: name_filter.map(|s| s.to_string()),
reply_to: self.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&self.rt, &self.inbox, 10).unwrap();
match resp {
DatastoreResponse::ListOk { entries } => entries,
other => panic!("unexpected list response: {other:?}"),
}
}
}
/// Spawn a MetadataActor with blob_store_addr wired up before spawning.
pub fn spawn_metadata_with_config(
rt: &Runtime,
node_id: NodeId,
config: &DatastoreConfig,
blob_store: ActorAddress,
) -> ActorAddress {
let mut meta = MetadataActor::new(node_id, config);
meta.set_blob_store(blob_store);
rt.spawn(meta).unwrap()
}
// ─── GcHarness ──────────────────────────────────────────────────────────────
/// Test harness for garbage collection scenarios.
/// Uses gc_interval=3, chunk_size=64 so GC triggers every 3 ticks
/// and 200-byte blobs produce multiple chunks.
pub struct GcHarness {
pub rt: Runtime,
pub blob_store: ActorAddress,
pub metadata: ActorAddress,
pub inbox: Inbox<DatastoreResponse>,
pub node_id: NodeId,
pub chunk_size: u32,
}
impl GcHarness {
pub fn new() -> Self {
let rt = test_runtime();
let blob_store = spawn_blob_store(&rt);
let node_id = test_node_id();
let mut config = DatastoreConfig::default();
config.gc_interval = 3;
config.chunk_size = 64;
let metadata = spawn_metadata_with_config(&rt, node_id, &config, blob_store);
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
tick_n(&rt, 2);
Self {
rt,
blob_store,
metadata,
inbox,
node_id,
chunk_size: 64,
}
}
pub fn reply_addr(&self) -> ActorAddress {
*self.inbox.addr()
}
/// Chunk a blob, store all chunks in BlobStore, store entry+manifest in Metadata.
pub fn put_blob(&self, data: &[u8], name: Option<&str>) -> ContentHash {
let (content_hash, manifest, chunks) = chunk_blob(data, self.chunk_size);
// Store all chunks.
for (hash, chunk_data) in &chunks {
self.rt
.send_to(
self.blob_store,
BlobStoreMsg::WriteChunk {
hash: *hash,
data: chunk_data.clone(),
reply_to: self.reply_addr(),
},
)
.unwrap();
}
tick_n(&self.rt, 3);
let _ = tick_and_drain(&self.rt, &self.inbox, 2);
// Store manifest via BlobStoreActor.
self.rt
.send_to(
self.blob_store,
BlobStoreMsg::WriteManifest {
manifest: manifest.clone(),
reply_to: self.reply_addr(),
},
)
.unwrap();
tick_n(&self.rt, 2);
let _ = tick_and_drain(&self.rt, &self.inbox, 1);
// Store metadata via MetadataActor.
let entry = ObjectEntry {
content_hash,
name: name.map(|s| s.to_string()),
node_id: self.node_id,
tags: BTreeMap::new(),
size_bytes: data.len() as u64,
created_at: 0,
};
self.rt
.send_to(
self.metadata,
MetadataMsg::PutObject {
entry,
manifest,
reply_to: self.reply_addr(),
},
)
.unwrap();
tick_n(&self.rt, 2);
let responses = tick_and_drain(&self.rt, &self.inbox, 1);
assert!(
responses
.iter()
.any(|r| matches!(r, DatastoreResponse::PutOk { .. })),
"expected PutOk response"
);
content_hash
}
/// Delete an object by content hash.
pub fn delete_blob(&self, content_hash: &ContentHash) {
self.rt
.send_to(
self.metadata,
MetadataMsg::DeleteObject {
content_hash: *content_hash,
reply_to: self.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&self.rt, &self.inbox, 10).unwrap();
assert!(
matches!(resp, DatastoreResponse::DeleteOk { .. }),
"expected DeleteOk, got {resp:?}"
);
}
/// Send `n` GcTick messages, ticking the runtime between each to allow
/// the two-hop flow: GcTick → MetadataActor → GcUnreferenced → BlobStoreActor.
pub fn gc_ticks(&self, n: usize) {
for _ in 0..n {
self.rt
.send_to(self.metadata, MetadataMsg::GcTick)
.unwrap();
tick_n(&self.rt, 3);
}
}
/// Query BlobStore for all chunk hashes.
pub fn list_chunks(&self) -> Vec<ContentHash> {
self.rt
.send_to(
self.blob_store,
BlobStoreMsg::ListChunks {
reply_to: self.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&self.rt, &self.inbox, 10).unwrap();
match resp {
DatastoreResponse::ChunkList { hashes } => hashes,
other => panic!("expected ChunkList, got {other:?}"),
}
}
/// Check whether a specific chunk exists in BlobStore.
pub fn has_chunk(&self, hash: &ContentHash) -> bool {
self.rt
.send_to(
self.blob_store,
BlobStoreMsg::HasChunk {
hash: *hash,
reply_to: self.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&self.rt, &self.inbox, 10).unwrap();
match resp {
DatastoreResponse::Bool(b) => b,
other => panic!("expected Bool, got {other:?}"),
}
}
}
// ─── NodeHarness (DatastoreNode coordinator) ────────────────────────────────
/// Spawn a DatastoreNode wired to the given BlobStore and Metadata actors.
pub fn spawn_datastore_node(
rt: &Runtime,
node_id: NodeId,
blob_store: ActorAddress,
metadata: ActorAddress,
) -> ActorAddress {
let mut config = DatastoreConfig::default();
config.chunk_size = 64; // Small chunks for multi-chunk testing
rt.spawn(DatastoreNode::new(node_id, blob_store, metadata, config))
.unwrap()
}
/// Test harness that routes all commands through the DatastoreNode coordinator.
pub struct NodeHarness {
pub rt: Runtime,
pub node: ActorAddress,
pub blob_store: ActorAddress,
pub inbox: Inbox<DatastoreResponse>,
pub node_id: NodeId,
}
impl NodeHarness {
pub fn new() -> Self {
let rt = test_runtime();
let blob_store = spawn_blob_store(&rt);
let node_id = test_node_id();
let metadata = spawn_metadata(&rt, node_id);
let node = spawn_datastore_node(&rt, node_id, blob_store, metadata);
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
// Let actors initialize.
tick_n(&rt, 2);
Self {
rt,
node,
blob_store,
inbox,
node_id,
}
}
pub fn reply_addr(&self) -> ActorAddress {
*self.inbox.addr()
}
}
// ─── MultiNodeHarness (distributed simulation) ──────────────────────────────
/// A simulated node in a multi-node cluster.
pub struct SimNode {
pub blob_store: ActorAddress,
pub metadata: ActorAddress,
pub node_id: NodeId,
}
/// Multi-node simulation harness. All nodes share a single Runtime so actor
/// addresses are globally unique and cross-node messaging works via `ctx.send()`.
pub struct MultiNodeHarness {
pub rt: Runtime,
pub nodes: Vec<SimNode>,
pub inbox: Inbox<DatastoreResponse>,
pub chunk_size: u32,
}
impl MultiNodeHarness {
/// Create a cluster of `n` nodes, all wired as peers.
pub fn new(n: usize) -> Self {
let rt = test_runtime();
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
let chunk_size = 64u32;
let mut nodes = Vec::with_capacity(n);
for i in 0..n {
let node_id = NodeId([(i + 1) as u8; 32]);
let blob_store = spawn_blob_store(&rt);
let mut config = DatastoreConfig::default();
config.gc_interval = 3;
config.chunk_size = chunk_size;
let metadata = spawn_metadata_with_config(&rt, node_id, &config, blob_store);
nodes.push(SimNode {
blob_store,
metadata,
node_id,
});
}
// Wire all MetadataActors as peers of each other.
for i in 0..n {
let peers: Vec<ActorAddress> = (0..n)
.filter(|&j| j != i)
.map(|j| nodes[j].metadata)
.collect();
rt.send_to(nodes[i].metadata, MetadataMsg::SetPeers { peers })
.unwrap();
}
tick_n(&rt, 2);
Self {
rt,
nodes,
inbox,
chunk_size,
}
}
pub fn reply_addr(&self) -> ActorAddress {
*self.inbox.addr()
}
/// Store a blob on a specific node: chunk it, store chunks, store metadata.
pub fn put_on(&self, node_idx: usize, data: &[u8], name: Option<&str>) -> ContentHash {
let node = &self.nodes[node_idx];
let (content_hash, manifest, chunks) = chunk_blob(data, self.chunk_size);
for (hash, chunk_data) in &chunks {
self.rt
.send_to(
node.blob_store,
BlobStoreMsg::WriteChunk {
hash: *hash,
data: chunk_data.clone(),
reply_to: self.reply_addr(),
},
)
.unwrap();
}
tick_n(&self.rt, 3);
let _ = tick_and_drain(&self.rt, &self.inbox, 2);
self.rt
.send_to(
node.blob_store,
BlobStoreMsg::WriteManifest {
manifest: manifest.clone(),
reply_to: self.reply_addr(),
},
)
.unwrap();
tick_n(&self.rt, 2);
let _ = tick_and_drain(&self.rt, &self.inbox, 1);
let entry = ObjectEntry {
content_hash,
name: name.map(|s| s.to_string()),
node_id: node.node_id,
tags: BTreeMap::new(),
size_bytes: data.len() as u64,
created_at: 0,
};
self.rt
.send_to(
node.metadata,
MetadataMsg::PutObject {
entry,
manifest,
reply_to: self.reply_addr(),
},
)
.unwrap();
tick_n(&self.rt, 2);
let responses = tick_and_drain(&self.rt, &self.inbox, 1);
assert!(
responses
.iter()
.any(|r| matches!(r, DatastoreResponse::PutOk { .. })),
"expected PutOk response"
);
content_hash
}
/// Query metadata on a specific node via GetObject.
pub fn get_from(&self, node_idx: usize, content_hash: ContentHash) -> Option<(ObjectEntry, ObjectManifest)> {
let node = &self.nodes[node_idx];
self.rt
.send_to(
node.metadata,
MetadataMsg::GetObject {
content_hash,
reply_to: self.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&self.rt, &self.inbox, 10)?;
match resp {
DatastoreResponse::GetOk { entry, manifest } => Some((entry, manifest)),
DatastoreResponse::NotFound => None,
other => panic!("unexpected response: {other:?}"),
}
}
/// Send DisseminateTick to all MetadataActors and tick the runtime.
pub fn disseminate_all(&self) {
for node in &self.nodes {
self.rt
.send_to(node.metadata, MetadataMsg::DisseminateTick)
.unwrap();
}
// Tick enough for: DisseminateTick → MetadataActor → HandleStoreObject → peer MetadataActor
tick_n(&self.rt, 5);
}
/// List objects on a specific node with optional name filter.
pub fn list_on(&self, node_idx: usize, name_filter: Option<&str>) -> Vec<ObjectEntry> {
let node = &self.nodes[node_idx];
self.rt
.send_to(
node.metadata,
MetadataMsg::ListLocal {
name_filter: name_filter.map(|s| s.to_string()),
reply_to: self.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&self.rt, &self.inbox, 10).unwrap();
match resp {
DatastoreResponse::ListOk { entries } => entries,
other => panic!("unexpected list response: {other:?}"),
}
}
/// Delete an object on a specific node.
pub fn delete_on(&self, node_idx: usize, content_hash: &ContentHash) {
let node = &self.nodes[node_idx];
self.rt
.send_to(
node.metadata,
MetadataMsg::DeleteObject {
content_hash: *content_hash,
reply_to: self.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&self.rt, &self.inbox, 10).unwrap();
assert!(
matches!(resp, DatastoreResponse::DeleteOk { .. }),
"expected DeleteOk, got {resp:?}"
);
}
/// Query BlobStore for all chunk hashes on a specific node.
pub fn list_chunks_on(&self, node_idx: usize) -> Vec<ContentHash> {
let node = &self.nodes[node_idx];
self.rt
.send_to(
node.blob_store,
BlobStoreMsg::ListChunks {
reply_to: self.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&self.rt, &self.inbox, 10).unwrap();
match resp {
DatastoreResponse::ChunkList { hashes } => hashes,
other => panic!("expected ChunkList, got {other:?}"),
}
}
/// Read a chunk from a specific node's BlobStore.
pub fn read_chunk_from(&self, node_idx: usize, hash: ContentHash) -> Option<Vec<u8>> {
let node = &self.nodes[node_idx];
self.rt
.send_to(
node.blob_store,
BlobStoreMsg::ReadChunk {
hash,
reply_to: self.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&self.rt, &self.inbox, 10).unwrap();
match resp {
DatastoreResponse::ChunkOk { data, .. } => Some(data),
DatastoreResponse::NotFound => None,
other => panic!("unexpected chunk response: {other:?}"),
}
}
/// Send GcTick messages to a specific node's MetadataActor.
pub fn gc_ticks_on(&self, node_idx: usize, n: usize) {
let node = &self.nodes[node_idx];
for _ in 0..n {
self.rt
.send_to(node.metadata, MetadataMsg::GcTick)
.unwrap();
tick_n(&self.rt, 3);
}
}
}

205
crates/datastore/tests/dashboard_integration_test.rs Normal file
View file

@ -0,0 +1,205 @@
#![cfg(feature = "node")]
//! End-to-end integration test: spins up a datastore node with an HTTP API
//! **and** a runtime dashboard, performs CRUD over HTTP, then verifies:
//!
//! - `DatastoreMetrics::snapshot()` reflects the operations
//! - Dashboard `/datastore` serves HTML
//! - Dashboard `/api/datastore` returns a JSON snapshot matching the metrics
use std::sync::Arc;
use std::thread;
use std::time::Duration;
use swactor::config::RuntimeConfig;
use swactor::runtime::Runtime;
use swactor_datastore::actors::{BlobStoreActor, DatastoreNode, MetadataActor};
use swactor_datastore::api::start_api_server;
use swactor_datastore::metrics::DatastoreMetrics;
use swactor_datastore::storage::InMemoryBackend;
use swactor_datastore::DatastoreConfig;
use distribution::types::NodeId;
fn find_free_port() -> u16 {
std::net::TcpListener::bind("127.0.0.1:0")
.unwrap()
.local_addr()
.unwrap()
.port()
}
/// Scenario: datastore CRUD → dashboard reflects live operation metrics.
///
/// Story:
/// Two files are uploaded. One is fetched (metadata + data). The other
/// is deleted. Afterwards we check that the metrics snapshot, the
/// dashboard HTML page, and the dashboard JSON API all agree on what
/// happened.
#[test]
fn dashboard_reflects_datastore_operations() {
let api_port = find_free_port();
let dash_port = find_free_port();
let api_base = format!("http://127.0.0.1:{api_port}");
let dash_base = format!("http://127.0.0.1:{dash_port}");
// ── Infrastructure: runtime + actors + dashboard + API ──────────────
let dash = runtime_dashboard::start_dashboard(runtime_dashboard::DashboardConfig {
port: dash_port,
..Default::default()
});
let collector = runtime_dashboard::collector::StatsCollector::new(2);
let mut rt = Runtime::new(RuntimeConfig {
num_threads: 2,
max_actors: 1024,
channel_buffer_size: 2000,
..Default::default()
});
rt.set_stats_hook(collector.clone());
let node_id = NodeId([0xBB; 32]);
let config = DatastoreConfig {
chunk_size: 1_048_576,
gc_interval: 1000,
..Default::default()
};
let backend: Box<dyn swactor_datastore::StorageBackend> = Box::new(InMemoryBackend::new());
let blob_store_addr = rt.spawn(BlobStoreActor::new(backend)).unwrap();
let mut metadata = MetadataActor::new(node_id, &config);
metadata.set_blob_store(blob_store_addr);
let metadata_addr = rt.spawn(metadata).unwrap();
let datastore_node = DatastoreNode::new(node_id, blob_store_addr, metadata_addr, config);
let datastore_addr = rt.spawn(datastore_node).unwrap();
let handle = rt.run().expect("failed to start runtime");
let metrics = Arc::new(DatastoreMetrics::new());
let node_hex: String = node_id.0.iter().map(|b| format!("{b:02x}")).collect();
metrics.set_node_id(node_hex);
dash.set_runtime(handle.runtime.clone(), collector);
dash.set_datastore(
Arc::clone(&metrics)
as Arc<dyn runtime_dashboard::datastore_collector::DatastoreStatsProvider>,
);
let (api_shutdown, _peers) = start_api_server(
handle.runtime.clone(),
datastore_addr,
metadata_addr,
blob_store_addr,
api_port,
Arc::clone(&metrics),
);
thread::sleep(Duration::from_millis(300));
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
run_dashboard_scenario(&api_base, &dash_base, &metrics);
}));
// Cleanup
api_shutdown.store(true, std::sync::atomic::Ordering::Relaxed);
dash.shutdown();
handle.shutdown();
handle.join();
if let Err(e) = result {
std::panic::resume_unwind(e);
}
}
fn run_dashboard_scenario(api: &str, dash: &str, metrics: &Arc<DatastoreMetrics>) {
// ── 1. Upload two objects ───────────────────────────────────────────
let put_a: serde_json::Value = ureq::post(&format!("{api}/api/put?name=alpha"))
.send_bytes(b"payload-alpha")
.expect("put A failed")
.into_json()
.unwrap();
let hash_a = put_a["content_hash"].as_str().unwrap().to_string();
let _put_b: serde_json::Value = ureq::post(&format!("{api}/api/put?name=bravo"))
.send_bytes(b"payload-bravo")
.expect("put B failed")
.into_json()
.unwrap();
let hash_b = _put_b["content_hash"].as_str().unwrap().to_string();
// ── 2. GET object A (metadata + raw data → 2 get ops) ──────────────
let _: serde_json::Value = ureq::get(&format!("{api}/api/get?hash={hash_a}"))
.call()
.unwrap()
.into_json()
.unwrap();
let data_resp = ureq::get(&format!("{api}/api/data?hash={hash_a}"))
.call()
.unwrap();
let mut body = Vec::new();
data_resp.into_reader().read_to_end(&mut body).unwrap();
assert_eq!(body, b"payload-alpha", "downloaded data should match");
// ── 3. DELETE object B ──────────────────────────────────────────────
let _: serde_json::Value = ureq::post(&format!("{api}/api/delete?hash={hash_b}"))
.call()
.unwrap()
.into_json()
.unwrap();
// ── 4. Assert: in-process metrics snapshot ──────────────────────────
let snap = metrics.snapshot();
assert_eq!(snap.put_ops, 2, "two puts recorded");
assert_eq!(snap.get_ops, 2, "metadata-get + data-get recorded");
assert_eq!(snap.delete_ops, 1, "one delete recorded");
assert_eq!(snap.objects.len(), 1, "only alpha remains after deleting bravo");
assert_eq!(snap.objects[0].hash, hash_a);
assert!(
snap.recent_events.len() >= 5,
"at least 5 events (2 put + 2 get + 1 delete), got {}",
snap.recent_events.len()
);
// ── 5. Assert: dashboard /datastore serves HTML ─────────────────────
let page = ureq::get(&format!("{dash}/datastore")).call().unwrap();
assert_eq!(page.status(), 200);
assert!(
page.header("Content-Type")
.unwrap_or("")
.contains("text/html"),
);
let html = page.into_string().unwrap();
assert!(html.contains("Datastore"), "page should mention Datastore");
// ── 6. Assert: /api/datastore JSON matches metrics ──────────────────
let ds: serde_json::Value = ureq::get(&format!("{dash}/api/datastore"))
.call()
.unwrap()
.into_json()
.unwrap();
assert_eq!(ds["put_ops"].as_u64().unwrap(), 2);
assert_eq!(ds["get_ops"].as_u64().unwrap(), 2);
assert_eq!(ds["delete_ops"].as_u64().unwrap(), 1);
let objects = ds["objects"].as_array().expect("objects should be array");
assert_eq!(objects.len(), 1);
assert_eq!(objects[0]["hash"].as_str().unwrap(), hash_a);
let events = ds["recent_events"]
.as_array()
.expect("recent_events should be array");
assert!(events.len() >= 5);
}

564
crates/datastore/tests/datastore_node_tests.rs Normal file
View file

@ -0,0 +1,564 @@
//! Scenario tests for the DatastoreNode coordinator actor.
//!
//! All tests route commands through a single DatastoreNode address.
//! Black-box only — no direct access to BlobStoreActor or MetadataActor.
mod common;
use std::collections::BTreeMap;
use common::{tick_n, tick_until_recv, NodeHarness};
use swactor_datastore::messages::DatastoreNodeMsg;
use swactor_datastore::types::ContentHash;
use swactor_datastore::messages::{DatastoreResponse, GetChunkRequest};
use swactor_datastore::{reassemble_blob, verify_integrity};
use distribution::types::NodeId;
// ═══════════════════════════════════════════════════════════════════════════
// Put & Retrieve
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn put_returns_content_hash() {
let h = NodeHarness::new();
let data = b"hello datastore";
h.rt.send_to(
h.node,
DatastoreNodeMsg::Put {
data: data.to_vec(),
name: None,
tags: BTreeMap::new(),
reply_to: h.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&h.rt, &h.inbox, 20).unwrap();
match resp {
DatastoreResponse::PutOk { content_hash } => {
assert_eq!(content_hash, ContentHash::of(data));
}
other => panic!("expected PutOk, got {other:?}"),
}
}
#[test]
fn put_then_get_returns_entry_and_manifest() {
let h = NodeHarness::new();
let data = b"greeting.txt contents";
// Put
h.rt.send_to(
h.node,
DatastoreNodeMsg::Put {
data: data.to_vec(),
name: Some("greeting.txt".to_string()),
tags: BTreeMap::new(),
reply_to: h.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&h.rt, &h.inbox, 20).unwrap();
let content_hash = match resp {
DatastoreResponse::PutOk { content_hash } => content_hash,
other => panic!("expected PutOk, got {other:?}"),
};
// Get
h.rt.send_to(
h.node,
DatastoreNodeMsg::Get {
content_hash,
reply_to: h.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&h.rt, &h.inbox, 20).unwrap();
match resp {
DatastoreResponse::GetOk { entry, manifest } => {
assert_eq!(entry.name.as_deref(), Some("greeting.txt"));
assert_eq!(entry.content_hash, content_hash);
assert_eq!(manifest.total_size, data.len() as u64);
assert!(!manifest.chunks.is_empty());
}
other => panic!("expected GetOk, got {other:?}"),
}
}
#[test]
fn put_and_read_chunks_recovers_original_data() {
let h = NodeHarness::new();
// 200 bytes with chunk_size=64 → 4 chunks (64+64+64+8)
let data: Vec<u8> = (0..200).map(|i| (i % 251) as u8).collect();
// Put
h.rt.send_to(
h.node,
DatastoreNodeMsg::Put {
data: data.clone(),
name: Some("multi-chunk".to_string()),
tags: BTreeMap::new(),
reply_to: h.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&h.rt, &h.inbox, 20).unwrap();
let content_hash = match resp {
DatastoreResponse::PutOk { content_hash } => content_hash,
other => panic!("expected PutOk, got {other:?}"),
};
// Get manifest
h.rt.send_to(
h.node,
DatastoreNodeMsg::Get {
content_hash,
reply_to: h.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&h.rt, &h.inbox, 20).unwrap();
let manifest = match resp {
DatastoreResponse::GetOk { manifest, .. } => manifest,
other => panic!("expected GetOk, got {other:?}"),
};
assert!(manifest.chunks.len() > 1, "expected multi-chunk manifest");
// ReadChunk for each chunk
let mut chunks = Vec::new();
for chunk_ref in &manifest.chunks {
h.rt.send_to(
h.node,
DatastoreNodeMsg::ReadChunk {
hash: chunk_ref.hash,
reply_to: h.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&h.rt, &h.inbox, 20).unwrap();
match resp {
DatastoreResponse::ChunkOk { hash, data } => {
chunks.push((hash, data));
}
other => panic!("expected ChunkOk, got {other:?}"),
}
}
// Reassemble and verify
let recovered = reassemble_blob(&manifest, &chunks).unwrap();
assert!(verify_integrity(&recovered, &content_hash));
assert_eq!(recovered, data);
}
#[test]
fn put_with_tags_preserves_metadata() {
let h = NodeHarness::new();
let data = b"tagged blob";
let mut tags = BTreeMap::new();
tags.insert("album".to_string(), "vacation".to_string());
tags.insert("year".to_string(), "2024".to_string());
h.rt.send_to(
h.node,
DatastoreNodeMsg::Put {
data: data.to_vec(),
name: None,
tags: tags.clone(),
reply_to: h.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&h.rt, &h.inbox, 20).unwrap();
let content_hash = match resp {
DatastoreResponse::PutOk { content_hash } => content_hash,
other => panic!("expected PutOk, got {other:?}"),
};
h.rt.send_to(
h.node,
DatastoreNodeMsg::Get {
content_hash,
reply_to: h.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&h.rt, &h.inbox, 20).unwrap();
match resp {
DatastoreResponse::GetOk { entry, .. } => {
assert_eq!(entry.tags, tags);
}
other => panic!("expected GetOk, got {other:?}"),
}
}
#[test]
fn put_empty_blob_succeeds() {
let h = NodeHarness::new();
let data = b"";
h.rt.send_to(
h.node,
DatastoreNodeMsg::Put {
data: data.to_vec(),
name: None,
tags: BTreeMap::new(),
reply_to: h.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&h.rt, &h.inbox, 20).unwrap();
let content_hash = match resp {
DatastoreResponse::PutOk { content_hash } => content_hash,
other => panic!("expected PutOk, got {other:?}"),
};
h.rt.send_to(
h.node,
DatastoreNodeMsg::Get {
content_hash,
reply_to: h.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&h.rt, &h.inbox, 20).unwrap();
match resp {
DatastoreResponse::GetOk { manifest, .. } => {
assert_eq!(manifest.chunks.len(), 0);
assert_eq!(manifest.total_size, 0);
}
other => panic!("expected GetOk, got {other:?}"),
}
}
// ═══════════════════════════════════════════════════════════════════════════
// Get & Delete
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn get_nonexistent_returns_not_found() {
let h = NodeHarness::new();
let bogus_hash = ContentHash::of(b"never stored");
h.rt.send_to(
h.node,
DatastoreNodeMsg::Get {
content_hash: bogus_hash,
reply_to: h.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&h.rt, &h.inbox, 20).unwrap();
assert!(
matches!(resp, DatastoreResponse::NotFound),
"expected NotFound, got {resp:?}"
);
}
#[test]
fn delete_makes_object_unretrievable() {
let h = NodeHarness::new();
let data = b"ephemeral data";
// Put
h.rt.send_to(
h.node,
DatastoreNodeMsg::Put {
data: data.to_vec(),
name: None,
tags: BTreeMap::new(),
reply_to: h.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&h.rt, &h.inbox, 20).unwrap();
let content_hash = match resp {
DatastoreResponse::PutOk { content_hash } => content_hash,
other => panic!("expected PutOk, got {other:?}"),
};
// Delete
h.rt.send_to(
h.node,
DatastoreNodeMsg::Delete {
content_hash,
reply_to: h.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&h.rt, &h.inbox, 20).unwrap();
assert!(
matches!(resp, DatastoreResponse::DeleteOk { .. }),
"expected DeleteOk, got {resp:?}"
);
// Get should now return NotFound
h.rt.send_to(
h.node,
DatastoreNodeMsg::Get {
content_hash,
reply_to: h.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&h.rt, &h.inbox, 20).unwrap();
assert!(
matches!(resp, DatastoreResponse::NotFound),
"expected NotFound after delete, got {resp:?}"
);
}
#[test]
fn delete_nonexistent_returns_not_found() {
let h = NodeHarness::new();
let bogus_hash = ContentHash::of(b"never stored");
h.rt.send_to(
h.node,
DatastoreNodeMsg::Delete {
content_hash: bogus_hash,
reply_to: h.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&h.rt, &h.inbox, 20).unwrap();
assert!(
matches!(resp, DatastoreResponse::NotFound),
"expected NotFound, got {resp:?}"
);
}
// ═══════════════════════════════════════════════════════════════════════════
// List
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn list_returns_all_stored_objects() {
let h = NodeHarness::new();
let blobs: Vec<&[u8]> = vec![b"blob-one", b"blob-two", b"blob-three"];
let mut expected_hashes = Vec::new();
for blob in &blobs {
h.rt.send_to(
h.node,
DatastoreNodeMsg::Put {
data: blob.to_vec(),
name: None,
tags: BTreeMap::new(),
reply_to: h.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&h.rt, &h.inbox, 20).unwrap();
match resp {
DatastoreResponse::PutOk { content_hash } => {
expected_hashes.push(content_hash);
}
other => panic!("expected PutOk, got {other:?}"),
}
}
// List all
h.rt.send_to(
h.node,
DatastoreNodeMsg::List {
name_filter: None,
all: false,
reply_to: h.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&h.rt, &h.inbox, 20).unwrap();
match resp {
DatastoreResponse::ListOk { entries } => {
assert_eq!(entries.len(), 3);
let listed_hashes: Vec<_> = entries.iter().map(|e| e.content_hash).collect();
for hash in &expected_hashes {
assert!(
listed_hashes.contains(hash),
"expected hash {hash:?} in list"
);
}
}
other => panic!("expected ListOk, got {other:?}"),
}
}
#[test]
fn list_with_name_filter_matches_correctly() {
let h = NodeHarness::new();
let named_blobs = vec![
(b"alpha content" as &[u8], Some("alpha.txt")),
(b"alphabet content", Some("alphabet.txt")),
(b"unnamed content", None),
];
for (data, name) in &named_blobs {
h.rt.send_to(
h.node,
DatastoreNodeMsg::Put {
data: data.to_vec(),
name: name.map(|s| s.to_string()),
tags: BTreeMap::new(),
reply_to: h.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&h.rt, &h.inbox, 20).unwrap();
assert!(matches!(resp, DatastoreResponse::PutOk { .. }));
}
// Filter "alpha" → should match both alpha.txt and alphabet.txt
h.rt.send_to(
h.node,
DatastoreNodeMsg::List {
name_filter: Some("alpha".to_string()),
all: false,
reply_to: h.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&h.rt, &h.inbox, 20).unwrap();
match resp {
DatastoreResponse::ListOk { entries } => {
assert_eq!(entries.len(), 2, "expected 2 matches for 'alpha'");
}
other => panic!("expected ListOk, got {other:?}"),
}
// Filter "zzz" → should match nothing
h.rt.send_to(
h.node,
DatastoreNodeMsg::List {
name_filter: Some("zzz".to_string()),
all: false,
reply_to: h.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&h.rt, &h.inbox, 20).unwrap();
match resp {
DatastoreResponse::ListOk { entries } => {
assert_eq!(entries.len(), 0, "expected 0 matches for 'zzz'");
}
other => panic!("expected ListOk, got {other:?}"),
}
}
// ═══════════════════════════════════════════════════════════════════════════
// Status & Network Protocol
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn status_reports_node_identity() {
let h = NodeHarness::new();
h.rt.send_to(
h.node,
DatastoreNodeMsg::Status {
reply_to: h.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&h.rt, &h.inbox, 20).unwrap();
match resp {
DatastoreResponse::NodeStatus { node_id } => {
assert_eq!(node_id, h.node_id);
}
other => panic!("expected NodeStatus, got {other:?}"),
}
}
#[test]
fn incoming_chunk_request_serves_stored_data() {
let h = NodeHarness::new();
let data = b"network accessible blob";
// Put the blob so chunks are persisted
h.rt.send_to(
h.node,
DatastoreNodeMsg::Put {
data: data.to_vec(),
name: None,
tags: BTreeMap::new(),
reply_to: h.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&h.rt, &h.inbox, 20).unwrap();
let content_hash = match resp {
DatastoreResponse::PutOk { content_hash } => content_hash,
other => panic!("expected PutOk, got {other:?}"),
};
// Let chunk writes settle
tick_n(&h.rt, 5);
// Get the manifest to find chunk hashes
h.rt.send_to(
h.node,
DatastoreNodeMsg::Get {
content_hash,
reply_to: h.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&h.rt, &h.inbox, 20).unwrap();
let manifest = match resp {
DatastoreResponse::GetOk { manifest, .. } => manifest,
other => panic!("expected GetOk, got {other:?}"),
};
// Simulate an incoming network GetChunk request for the first chunk
let chunk_hash = manifest.chunks[0].hash;
let remote_node = NodeId([0xAA; 32]);
h.rt.send_to(
h.node,
DatastoreNodeMsg::IncomingGetChunk {
request: GetChunkRequest {
from: remote_node,
hash: chunk_hash,
},
reply_to: h.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&h.rt, &h.inbox, 20).unwrap();
match resp {
DatastoreResponse::ChunkOk { hash, data: chunk_data } => {
assert_eq!(hash, chunk_hash);
assert!(!chunk_data.is_empty());
}
other => panic!("expected ChunkOk, got {other:?}"),
}
}

310
crates/datastore/tests/datastore_tests.rs Normal file
View file

@ -0,0 +1,310 @@
//! Integration-style tests for the swactor-datastore crate.
//!
//! Focuses on scenario/story tests and property-based tests that exercise
//! the protocol through its public types — low coupling to internals.
use std::collections::HashSet;
use swactor_datastore::types::{
ChunkRef, ContentHash, DatastoreConfig, ObjectEntry, ObjectManifest,
};
use distribution::types::NodeId;
// ═══════════════════════════════════════════════════════════════════════════
// Scenario: Content-addressing round-trip
// ═══════════════════════════════════════════════════════════════════════════
/// Chunking a file, building a manifest, and reassembling produces the
/// original data — the core content-addressing contract.
#[test]
fn chunking_and_reassembly_preserves_data() {
let original_data = b"Hello, distributed world! This is a test blob.";
let chunk_size: u32 = 16; // Small chunks for testing.
// Chunk the data.
let mut chunks: Vec<(ContentHash, Vec<u8>)> = Vec::new();
let mut chunk_refs: Vec<ChunkRef> = Vec::new();
let mut offset: u64 = 0;
for chunk_data in original_data.chunks(chunk_size as usize) {
let hash = ContentHash::of(chunk_data);
chunk_refs.push(ChunkRef {
hash,
offset,
size: chunk_data.len() as u32,
});
chunks.push((hash, chunk_data.to_vec()));
offset += chunk_data.len() as u64;
}
// content_hash = blake3(entire_blob)
let content_hash = ContentHash::of(original_data);
let manifest = ObjectManifest {
content_hash,
chunks: chunk_refs,
total_size: original_data.len() as u64,
chunk_size,
content_type: Some("application/octet-stream".to_string()),
};
// Reassemble from chunks using manifest order.
let mut reassembled = Vec::new();
for chunk_ref in &manifest.chunks {
let (_, data) = chunks
.iter()
.find(|(h, _)| *h == chunk_ref.hash)
.expect("chunk not found");
reassembled.extend_from_slice(data);
}
assert_eq!(reassembled.as_slice(), original_data.as_slice());
assert_eq!(reassembled.len() as u64, manifest.total_size);
// Verify content hash matches the reassembled data.
assert_eq!(ContentHash::of(&reassembled), content_hash);
}
/// Identical data produces identical content hashes (deterministic).
#[test]
fn identical_data_produces_same_hash() {
let data = b"same content";
assert_eq!(ContentHash::of(data), ContentHash::of(data));
}
/// Different data produces different content hashes (collision resistance).
#[test]
fn different_data_produces_different_hashes() {
assert_ne!(ContentHash::of(b"alpha"), ContentHash::of(b"beta"));
}
// ═══════════════════════════════════════════════════════════════════════════
// Scenario: Manifest serialization round-trip
// ═══════════════════════════════════════════════════════════════════════════
/// A manifest can be serialized to JSON and deserialized back without data loss.
#[test]
fn manifest_survives_json_round_trip() {
let manifest = ObjectManifest {
content_hash: ContentHash::of(b"my-entire-blob"),
chunks: vec![
ChunkRef {
hash: ContentHash::of(b"chunk-0"),
offset: 0,
size: 1_048_576,
},
ChunkRef {
hash: ContentHash::of(b"chunk-1"),
offset: 1_048_576,
size: 524_288,
},
],
total_size: 1_572_864,
chunk_size: 1_048_576,
content_type: Some("image/jpeg".to_string()),
};
let json = serde_json::to_string(&manifest).unwrap();
let deserialized: ObjectManifest = serde_json::from_str(&json).unwrap();
assert_eq!(manifest, deserialized);
}
// ═══════════════════════════════════════════════════════════════════════════
// Scenario: ContentHash hex round-trip
// ═══════════════════════════════════════════════════════════════════════════
/// to_hex → from_hex round-trips for any content hash.
#[test]
fn content_hash_hex_round_trip() {
let data = b"round-trip through hex encoding";
let hash = ContentHash::of(data);
let hex = hash.to_hex();
let recovered = ContentHash::from_hex(&hex).expect("valid hex should parse");
assert_eq!(hash, recovered);
}
/// from_hex rejects strings that are not exactly 64 hex characters.
#[test]
fn from_hex_rejects_wrong_length() {
assert!(ContentHash::from_hex("abcd").is_none());
assert!(ContentHash::from_hex("").is_none());
// 63 chars
assert!(ContentHash::from_hex(
&"a".repeat(63)
).is_none());
// 65 chars
assert!(ContentHash::from_hex(
&"a".repeat(65)
).is_none());
}
/// from_hex rejects non-hex characters.
#[test]
fn from_hex_rejects_non_hex_chars() {
// 'g' is not valid hex
let bad = format!("{}g{}", "a".repeat(31), "a".repeat(32));
assert_eq!(bad.len(), 64);
assert!(ContentHash::from_hex(&bad).is_none());
}
/// from_hex accepts uppercase hex.
#[test]
fn from_hex_accepts_uppercase() {
let hash = ContentHash::of(b"uppercase test");
let hex_upper = hash.to_hex().to_uppercase();
let recovered = ContentHash::from_hex(&hex_upper).expect("uppercase hex should parse");
assert_eq!(hash, recovered);
}
// ═══════════════════════════════════════════════════════════════════════════
// Scenario: ContentHash DHT properties
// ═══════════════════════════════════════════════════════════════════════════
/// XOR distance is symmetric — required for Kademlia correctness.
#[test]
fn xor_distance_is_symmetric() {
let a = ContentHash::of(b"node-alpha");
let b = ContentHash::of(b"node-beta");
assert_eq!(a.xor_distance(&b), b.xor_distance(&a));
}
/// XOR distance to self is zero — a node is closest to itself.
#[test]
fn xor_distance_to_self_is_zero() {
let a = ContentHash::of(b"self");
assert_eq!(a.xor_distance(&a), [0u8; 32]);
assert_eq!(a.xor_leading_zeros(&a), 256);
}
// ═══════════════════════════════════════════════════════════════════════════
// Scenario: ObjectEntry serialization
// ═══════════════════════════════════════════════════════════════════════════
/// An ObjectEntry with tags survives JSON round-trip.
#[test]
fn object_entry_with_tags_survives_round_trip() {
let mut tags = std::collections::BTreeMap::new();
tags.insert("album".to_string(), "vacation-2024".to_string());
tags.insert("device".to_string(), "phone".to_string());
let entry = ObjectEntry {
content_hash: ContentHash::of(b"beach-photo-bytes"),
name: Some("beach.jpg".to_string()),
node_id: NodeId([0x42; 32]),
tags,
size_bytes: 4_500_000,
created_at: 1700000000,
};
let json = serde_json::to_string(&entry).unwrap();
let deserialized: ObjectEntry = serde_json::from_str(&json).unwrap();
assert_eq!(entry, deserialized);
}
/// An ObjectEntry without a name survives JSON round-trip.
#[test]
fn object_entry_without_name_survives_round_trip() {
let entry = ObjectEntry {
content_hash: ContentHash::of(b"anonymous-blob"),
name: None,
node_id: NodeId([0x01; 32]),
tags: std::collections::BTreeMap::new(),
size_bytes: 1024,
created_at: 0,
};
let json = serde_json::to_string(&entry).unwrap();
let deserialized: ObjectEntry = serde_json::from_str(&json).unwrap();
assert_eq!(entry, deserialized);
}
// ═══════════════════════════════════════════════════════════════════════════
// Scenario: DatastoreConfig defaults
// ═══════════════════════════════════════════════════════════════════════════
/// Default config uses 1MB chunks — the documented default for the protocol.
#[test]
fn default_config_uses_1mb_chunks() {
let config = DatastoreConfig::default();
assert_eq!(config.chunk_size, 1_048_576);
}
// ═══════════════════════════════════════════════════════════════════════════
// Property-based tests
// ═══════════════════════════════════════════════════════════════════════════
mod proptests {
use super::*;
use proptest::prelude::*;
// ContentHash::of is a pure function — same input always gives same output.
proptest! {
#[test]
fn content_hash_is_deterministic(data in proptest::collection::vec(any::<u8>(), 0..4096)) {
prop_assert_eq!(ContentHash::of(&data), ContentHash::of(&data));
}
}
// XOR distance is always symmetric.
proptest! {
#[test]
fn xor_distance_symmetry(
a_bytes in proptest::collection::vec(any::<u8>(), 32..=32),
b_bytes in proptest::collection::vec(any::<u8>(), 32..=32),
) {
let a = ContentHash(a_bytes.try_into().unwrap());
let b = ContentHash(b_bytes.try_into().unwrap());
prop_assert_eq!(a.xor_distance(&b), b.xor_distance(&a));
}
}
// ContentHash::from_hex is the inverse of to_hex.
proptest! {
#[test]
fn hex_round_trip(data in proptest::collection::vec(any::<u8>(), 1..1024)) {
let hash = ContentHash::of(&data);
let hex = hash.to_hex();
let recovered = ContentHash::from_hex(&hex).unwrap();
prop_assert_eq!(hash, recovered);
}
}
// Chunking any data and reassembling preserves the original.
proptest! {
#[test]
fn chunk_reassemble_identity(
data in proptest::collection::vec(any::<u8>(), 1..8192),
chunk_size in 1u32..=256,
) {
let mut chunks: Vec<(ContentHash, Vec<u8>)> = Vec::new();
for chunk_data in data.chunks(chunk_size as usize) {
chunks.push((ContentHash::of(chunk_data), chunk_data.to_vec()));
}
let reassembled: Vec<u8> = chunks.iter().flat_map(|(_, d)| d.iter().copied()).collect();
prop_assert_eq!(data, reassembled);
}
}
// Content deduplication: chunks with identical data share the same hash,
// so storing them once is correct.
proptest! {
#[test]
fn duplicate_chunks_deduplicate(data in proptest::collection::vec(any::<u8>(), 1..512)) {
let hash1 = ContentHash::of(&data);
let hash2 = ContentHash::of(&data);
prop_assert_eq!(hash1, hash2);
// Storing both in a HashSet yields one entry.
let mut set = HashSet::new();
set.insert(hash1);
set.insert(hash2);
prop_assert_eq!(set.len(), 1);
}
}
}

191
crates/datastore/tests/gc_tests.rs Normal file
View file

@ -0,0 +1,191 @@
//! Garbage collection scenario tests.
//!
//! Verifies the full GC flow: MetadataActor builds a referenced chunk set
//! from its manifests and sends GcUnreferenced to BlobStoreActor, which
//! deletes orphaned chunks.
mod common;
use common::GcHarness;
use swactor_datastore::chunking::chunk_blob;
#[test]
fn gc_cleans_up_chunks_after_object_deleted() {
let h = GcHarness::new();
// Store a 200-byte blob (produces 4 chunks at chunk_size=64).
let data = vec![0xAB; 200];
let hash = h.put_blob(&data, Some("doomed.bin"));
let chunks_before = h.list_chunks();
assert!(!chunks_before.is_empty(), "chunks should exist after put");
// Delete the object from the metadata index.
h.delete_blob(&hash);
// Run enough ticks for GC to fire (gc_interval=3).
h.gc_ticks(3);
// All chunks should be gone — nothing references them anymore.
let chunks_after = h.list_chunks();
assert!(
chunks_after.is_empty(),
"expected all chunks removed after GC, found {}",
chunks_after.len()
);
}
#[test]
fn gc_preserves_chunks_still_referenced() {
let h = GcHarness::new();
// Store two distinct blobs.
let data_a = vec![0x11; 200];
let data_b = vec![0x22; 150];
let hash_a = h.put_blob(&data_a, Some("keep.bin"));
let _hash_b = h.put_blob(&data_b, Some("also-keep.bin"));
let chunks_before = h.list_chunks();
// Delete only blob A.
h.delete_blob(&hash_a);
// GC should clean up A's orphaned chunks but preserve B's.
h.gc_ticks(3);
// Blob B's chunks should all survive.
let (_, manifest_b, _) = chunk_blob(&data_b, h.chunk_size);
for chunk_ref in &manifest_b.chunks {
assert!(
h.has_chunk(&chunk_ref.hash),
"blob B chunk {:?} should survive GC",
chunk_ref.hash
);
}
// Blob A's chunks should be gone (they don't overlap with B since data differs).
let (_, manifest_a, _) = chunk_blob(&data_a, h.chunk_size);
for chunk_ref in &manifest_a.chunks {
assert!(
!h.has_chunk(&chunk_ref.hash),
"blob A chunk {:?} should be removed by GC",
chunk_ref.hash
);
}
// Total chunk count should have decreased.
let chunks_after = h.list_chunks();
assert!(
chunks_after.len() < chunks_before.len(),
"chunk count should decrease after GC removes orphans"
);
}
#[test]
fn gc_handles_deduplication_correctly() {
let h = GcHarness::new();
// Two 128-byte blobs sharing the same 64-byte prefix (first chunk is identical).
let mut data_x = vec![0xCC; 128];
let mut data_y = vec![0xCC; 128];
// The first 64 bytes are identical → same first chunk hash.
// Differ in the second 64 bytes → different second chunk + different content hash.
data_x[64..].fill(0xAA);
data_y[64..].fill(0xBB);
let hash_x = h.put_blob(&data_x, Some("x.bin"));
let _hash_y = h.put_blob(&data_y, Some("y.bin"));
// Verify the shared chunk exists.
let (_, manifest_x, _) = chunk_blob(&data_x, h.chunk_size);
let (_, manifest_y, _) = chunk_blob(&data_y, h.chunk_size);
let shared_chunk = manifest_x.chunks[0].hash;
assert_eq!(
shared_chunk, manifest_y.chunks[0].hash,
"first chunk should be identical (shared prefix)"
);
// Delete only X.
h.delete_blob(&hash_x);
h.gc_ticks(3);
// Shared chunk should survive (Y still references it).
assert!(
h.has_chunk(&shared_chunk),
"shared chunk should survive — still referenced by Y"
);
// X's unique second chunk should be gone.
let x_unique = manifest_x.chunks[1].hash;
assert!(
!h.has_chunk(&x_unique),
"X's unique chunk should be removed by GC"
);
// Y's unique second chunk should survive.
let y_unique = manifest_y.chunks[1].hash;
assert!(
h.has_chunk(&y_unique),
"Y's unique chunk should survive GC"
);
}
#[test]
fn gc_is_no_op_when_nothing_deleted() {
let h = GcHarness::new();
let data = vec![0xFF; 200];
h.put_blob(&data, Some("survivor.bin"));
let chunks_before = h.list_chunks();
// GC fires but nothing was deleted — all chunks should survive.
h.gc_ticks(3);
let chunks_after = h.list_chunks();
assert_eq!(
chunks_before.len(),
chunks_after.len(),
"GC without any deletes should preserve all chunks"
);
}
#[test]
fn gc_runs_on_interval_not_every_tick() {
let h = GcHarness::new();
let data = vec![0xDD; 200];
let hash = h.put_blob(&data, Some("interval-test.bin"));
h.delete_blob(&hash);
// Send gc_interval - 1 = 2 ticks. GC should NOT have fired yet.
h.gc_ticks(2);
let chunks_mid = h.list_chunks();
assert!(
!chunks_mid.is_empty(),
"chunks should still exist before gc_interval is reached"
);
// One more tick reaches gc_interval=3. GC fires and cleans up.
h.gc_ticks(1);
let chunks_after = h.list_chunks();
assert!(
chunks_after.is_empty(),
"chunks should be cleaned after gc_interval reached"
);
}
#[test]
fn gc_with_empty_datastore_is_harmless() {
let h = GcHarness::new();
// GC on an empty store — should not panic.
h.gc_ticks(3);
let chunks = h.list_chunks();
assert!(chunks.is_empty(), "empty store should remain empty after GC");
}

372
crates/datastore/tests/metadata_tests.rs Normal file
View file

@ -0,0 +1,372 @@
//! Tests for MetadataActor — exercised as a black box through the swactor runtime.
mod common;
use std::collections::{BTreeMap, HashSet};
use swactor_datastore::messages::{DatastoreResponse, MetadataMsg};
use swactor_datastore::types::{ContentHash, ObjectEntry};
use distribution::types::NodeId;
use common::{
make_entry, make_manifest, spawn_metadata, test_node_id, test_runtime, tick_n, tick_until_recv,
};
// ═══════════════════════════════════════════════════════════════════════════
// Scenario: Store & Retrieve
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn store_and_retrieve_object_with_manifest() {
let rt = test_runtime();
let meta = spawn_metadata(&rt, test_node_id());
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
let reply = *inbox.addr();
let data = b"hello metadata";
let entry = make_entry(data, Some("greeting.txt"));
let manifest = make_manifest(data);
let content_hash = entry.content_hash;
// Put.
rt.send_to(meta, MetadataMsg::PutObject { entry, manifest: manifest.clone(), reply_to: reply })
.unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
assert!(matches!(resp, DatastoreResponse::PutOk { content_hash: h } if h == content_hash));
// Get.
rt.send_to(meta, MetadataMsg::GetObject { content_hash, reply_to: reply })
.unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
match resp {
DatastoreResponse::GetOk { entry: e, manifest: m } => {
assert_eq!(e.content_hash, content_hash);
assert_eq!(e.name, Some("greeting.txt".to_string()));
assert_eq!(m, manifest);
}
other => panic!("expected GetOk, got: {other:?}"),
}
}
#[test]
fn query_for_nonexistent_object_returns_not_found() {
let rt = test_runtime();
let meta = spawn_metadata(&rt, test_node_id());
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
rt.send_to(meta, MetadataMsg::GetObject {
content_hash: ContentHash::of(b"does-not-exist"),
reply_to: *inbox.addr(),
})
.unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
assert!(matches!(resp, DatastoreResponse::NotFound));
}
// ═══════════════════════════════════════════════════════════════════════════
// Scenario: Delete
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn delete_object_makes_it_unretrievable() {
let rt = test_runtime();
let meta = spawn_metadata(&rt, test_node_id());
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
let reply = *inbox.addr();
let data = b"ephemeral";
let entry = make_entry(data, Some("temp.txt"));
let manifest = make_manifest(data);
let content_hash = entry.content_hash;
// Put.
rt.send_to(meta, MetadataMsg::PutObject { entry, manifest, reply_to: reply }).unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
assert!(matches!(resp, DatastoreResponse::PutOk { .. }));
// Delete.
rt.send_to(meta, MetadataMsg::DeleteObject { content_hash, reply_to: reply }).unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
assert!(matches!(resp, DatastoreResponse::DeleteOk { content_hash: h } if h == content_hash));
// Get → NotFound.
rt.send_to(meta, MetadataMsg::GetObject { content_hash, reply_to: reply }).unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
assert!(matches!(resp, DatastoreResponse::NotFound));
}
#[test]
fn delete_nonexistent_object_returns_not_found() {
let rt = test_runtime();
let meta = spawn_metadata(&rt, test_node_id());
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
rt.send_to(meta, MetadataMsg::DeleteObject {
content_hash: ContentHash::of(b"ghost"),
reply_to: *inbox.addr(),
})
.unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
assert!(matches!(resp, DatastoreResponse::NotFound));
}
// ═══════════════════════════════════════════════════════════════════════════
// Scenario: List & Filter
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn list_local_returns_all_stored_objects() {
let rt = test_runtime();
let meta = spawn_metadata(&rt, test_node_id());
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
let reply = *inbox.addr();
let items: Vec<(&[u8], Option<&str>)> = vec![
(b"aaa", Some("first.txt")),
(b"bbb", Some("second.txt")),
(b"ccc", None),
];
let mut expected_hashes = HashSet::new();
for (data, name) in &items {
let entry = make_entry(*data, *name);
let manifest = make_manifest(*data);
expected_hashes.insert(entry.content_hash);
rt.send_to(meta, MetadataMsg::PutObject { entry, manifest, reply_to: reply }).unwrap();
tick_until_recv(&rt, &inbox, 10); // drain PutOk
}
// ListLocal with no filter.
rt.send_to(meta, MetadataMsg::ListLocal { name_filter: None, reply_to: reply }).unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
match resp {
DatastoreResponse::ListOk { entries } => {
let got: HashSet<ContentHash> = entries.iter().map(|e| e.content_hash).collect();
assert_eq!(got, expected_hashes);
}
other => panic!("expected ListOk, got: {other:?}"),
}
}
#[test]
fn list_local_with_name_filter_excludes_non_matching_and_unnamed() {
let rt = test_runtime();
let meta = spawn_metadata(&rt, test_node_id());
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
let reply = *inbox.addr();
let items: Vec<(&[u8], Option<&str>)> = vec![
(b"one", Some("alpha.txt")),
(b"two", Some("alphabet.txt")),
(b"three", None),
];
for (data, name) in &items {
let entry = make_entry(*data, *name);
let manifest = make_manifest(*data);
rt.send_to(meta, MetadataMsg::PutObject { entry, manifest, reply_to: reply }).unwrap();
tick_until_recv(&rt, &inbox, 10);
}
// Filter "alpha" → both named entries match.
rt.send_to(meta, MetadataMsg::ListLocal {
name_filter: Some("alpha".to_string()),
reply_to: reply,
})
.unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
match resp {
DatastoreResponse::ListOk { entries } => assert_eq!(entries.len(), 2),
other => panic!("expected ListOk, got: {other:?}"),
}
// Filter "zzz" → no matches.
rt.send_to(meta, MetadataMsg::ListLocal {
name_filter: Some("zzz".to_string()),
reply_to: reply,
})
.unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
match resp {
DatastoreResponse::ListOk { entries } => assert_eq!(entries.len(), 0),
other => panic!("expected ListOk, got: {other:?}"),
}
}
// ═══════════════════════════════════════════════════════════════════════════
// Scenario: DHT Protocol
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn dht_store_from_remote_makes_object_findable() {
let rt = test_runtime();
let node_id = test_node_id();
let meta = spawn_metadata(&rt, node_id);
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
let reply = *inbox.addr();
let remote_node = NodeId([0xAA; 32]);
let data = b"remote-object";
let entry = ObjectEntry {
content_hash: ContentHash::of(data),
name: Some("remote.dat".to_string()),
node_id: remote_node,
tags: BTreeMap::new(),
size_bytes: data.len() as u64,
created_at: 0,
};
let content_hash = entry.content_hash;
// HandleStoreObject — fire-and-forget.
rt.send_to(meta, MetadataMsg::HandleStoreObject { entry, manifest: None }).unwrap();
tick_n(&rt, 3);
// HandleFindObject → GetOk with synthetic empty manifest.
rt.send_to(meta, MetadataMsg::HandleFindObject {
from: remote_node,
content_hash,
reply_to: reply,
})
.unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
match resp {
DatastoreResponse::GetOk { entry: e, manifest: m } => {
assert_eq!(e.content_hash, content_hash);
assert!(m.chunks.is_empty());
assert_eq!(m.chunk_size, 0);
}
other => panic!("expected GetOk, got: {other:?}"),
}
}
#[test]
fn put_object_stamps_local_node_id_on_entry() {
let rt = test_runtime();
let local_node = test_node_id();
let meta = spawn_metadata(&rt, local_node);
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
let reply = *inbox.addr();
let foreign_node = NodeId([0xFF; 32]);
let data = b"stamped";
let entry = ObjectEntry {
content_hash: ContentHash::of(data),
name: Some("stamped.txt".to_string()),
node_id: foreign_node,
tags: BTreeMap::new(),
size_bytes: data.len() as u64,
created_at: 0,
};
let manifest = make_manifest(data);
let content_hash = entry.content_hash;
// Put with foreign node_id.
rt.send_to(meta, MetadataMsg::PutObject { entry, manifest, reply_to: reply }).unwrap();
tick_until_recv(&rt, &inbox, 10); // drain PutOk
// Get → entry should have local node_id stamped.
rt.send_to(meta, MetadataMsg::GetObject { content_hash, reply_to: reply }).unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
match resp {
DatastoreResponse::GetOk { entry: e, .. } => {
assert_eq!(e.node_id, local_node);
}
other => panic!("expected GetOk, got: {other:?}"),
}
}
#[test]
fn dht_store_is_idempotent_for_existing_entries() {
let rt = test_runtime();
let meta = spawn_metadata(&rt, test_node_id());
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
let reply = *inbox.addr();
let data = b"duplicate";
let entry = ObjectEntry {
content_hash: ContentHash::of(data),
name: Some("dup.dat".to_string()),
node_id: NodeId([0xBB; 32]),
tags: BTreeMap::new(),
size_bytes: data.len() as u64,
created_at: 0,
};
// Two HandleStoreObject with same content_hash.
rt.send_to(meta, MetadataMsg::HandleStoreObject { entry: entry.clone(), manifest: None }).unwrap();
tick_n(&rt, 3);
rt.send_to(meta, MetadataMsg::HandleStoreObject { entry, manifest: None }).unwrap();
tick_n(&rt, 3);
// ListLocal → exactly 1 entry.
rt.send_to(meta, MetadataMsg::ListLocal { name_filter: None, reply_to: reply }).unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
match resp {
DatastoreResponse::ListOk { entries } => assert_eq!(entries.len(), 1),
other => panic!("expected ListOk, got: {other:?}"),
}
}
// ═══════════════════════════════════════════════════════════════════════════
// Scenario: Full Lifecycle
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn full_lifecycle_put_list_get_delete_verify_empty() {
let rt = test_runtime();
let meta = spawn_metadata(&rt, test_node_id());
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
let reply = *inbox.addr();
let data = b"lifecycle-test";
let entry = make_entry(data, Some("lifecycle.bin"));
let manifest = make_manifest(data);
let content_hash = entry.content_hash;
// 1. Put.
rt.send_to(meta, MetadataMsg::PutObject { entry, manifest: manifest.clone(), reply_to: reply })
.unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
assert!(matches!(resp, DatastoreResponse::PutOk { .. }));
// 2. ListLocal → 1 entry.
rt.send_to(meta, MetadataMsg::ListLocal { name_filter: None, reply_to: reply }).unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
match &resp {
DatastoreResponse::ListOk { entries } => assert_eq!(entries.len(), 1),
other => panic!("expected ListOk, got: {other:?}"),
}
// 3. GetObject → verify entry + manifest.
rt.send_to(meta, MetadataMsg::GetObject { content_hash, reply_to: reply }).unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
match resp {
DatastoreResponse::GetOk { entry: e, manifest: m } => {
assert_eq!(e.content_hash, content_hash);
assert_eq!(e.name, Some("lifecycle.bin".to_string()));
assert_eq!(m, manifest);
}
other => panic!("expected GetOk, got: {other:?}"),
}
// 4. Delete → DeleteOk.
rt.send_to(meta, MetadataMsg::DeleteObject { content_hash, reply_to: reply }).unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
assert!(matches!(resp, DatastoreResponse::DeleteOk { content_hash: h } if h == content_hash));
// 5. ListLocal → 0 entries.
rt.send_to(meta, MetadataMsg::ListLocal { name_filter: None, reply_to: reply }).unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
match resp {
DatastoreResponse::ListOk { entries } => assert_eq!(entries.len(), 0),
other => panic!("expected ListOk, got: {other:?}"),
}
// 6. GetObject → NotFound.
rt.send_to(meta, MetadataMsg::GetObject { content_hash, reply_to: reply }).unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
assert!(matches!(resp, DatastoreResponse::NotFound));
}

329
crates/datastore/tests/multi_node_tests.rs Normal file
View file

@ -0,0 +1,329 @@
//! Multi-node simulation tests for metadata dissemination and cross-node operations.
//!
//! All nodes share a single Runtime — actor addresses are globally unique,
//! so cross-node messaging works via `ctx.send()` without a transport layer.
mod common;
use common::{tick_n, tick_until_recv, MultiNodeHarness};
use swactor_datastore::messages::{MetadataMsg, TransferMsg};
use swactor_datastore::types::ContentHash;
use swactor_datastore::TransferActor;
use distribution::types::NodeId;
// ═══════════════════════════════════════════════════════════════════════════
// Phase 3: Metadata dissemination tests
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn metadata_replicates_to_peer_after_dissemination() {
let h = MultiNodeHarness::new(2);
let data = b"hello distributed world";
let hash = h.put_on(0, data, Some("greeting.txt"));
// Before dissemination: node 1 doesn't have it.
assert!(h.get_from(1, hash).is_none());
// After dissemination: node 1 has the entry.
h.disseminate_all();
let (entry, _manifest) = h.get_from(1, hash).expect("node 1 should have the entry after dissemination");
assert_eq!(entry.content_hash, hash);
assert_eq!(entry.name.as_deref(), Some("greeting.txt"));
}
#[test]
fn metadata_replicates_to_all_peers_in_3_node_cluster() {
let h = MultiNodeHarness::new(3);
let data = b"replicate me everywhere";
let hash = h.put_on(0, data, Some("everywhere.bin"));
// Multiple rounds of dissemination to let entries propagate through all peers.
// Node 0 → nodes 1,2 on first round. Nodes 1,2 may re-disseminate to each other.
for _ in 0..3 {
h.disseminate_all();
}
for node_idx in 0..3 {
let result = h.get_from(node_idx, hash);
assert!(
result.is_some(),
"node {node_idx} should have the entry after dissemination"
);
}
}
#[test]
fn dissemination_budget_expires_after_enough_rounds() {
let h = MultiNodeHarness::new(2);
let data = b"budget test data";
let _hash = h.put_on(0, data, Some("budget.dat"));
// The dissemination budget is lambda * ceil(log2(cluster_size)).
// With lambda=3, cluster_size=3 (hardcoded in enqueue), budget = 3 * ceil(log2(3)) = 3*2 = 6.
// After 6+ rounds of dissemination, take_pending should return empty.
for _ in 0..10 {
h.disseminate_all();
}
// Put a new entry to verify dissemination still works for new entries
// while old ones have expired.
let data2 = b"fresh data after budget expired";
let hash2 = h.put_on(0, data2, Some("fresh.dat"));
h.disseminate_all();
let result = h.get_from(1, hash2);
assert!(result.is_some(), "fresh entry should disseminate normally");
}
#[test]
fn delete_on_origin_does_not_propagate_to_peers() {
let h = MultiNodeHarness::new(2);
let data = b"delete me locally";
let hash = h.put_on(0, data, Some("local-delete.dat"));
// Disseminate so node 1 has the entry.
h.disseminate_all();
assert!(h.get_from(1, hash).is_some());
// Delete on node 0.
h.delete_on(0, &hash);
// Node 0 no longer has it.
assert!(h.get_from(0, hash).is_none());
// Node 1 still has it — delete is local only.
let (entry, _) = h.get_from(1, hash).expect("peer should retain entry after origin deletes");
assert_eq!(entry.content_hash, hash);
}
#[test]
fn duplicate_put_via_dissemination_is_idempotent() {
let h = MultiNodeHarness::new(2);
let data = b"idempotent dissemination";
let hash = h.put_on(0, data, Some("idem.dat"));
// Disseminate multiple times.
for _ in 0..5 {
h.disseminate_all();
}
// Node 1 should have exactly 1 entry, not duplicates.
let entries = h.list_on(1, None);
let matching: Vec<_> = entries.iter().filter(|e| e.content_hash == hash).collect();
assert_eq!(matching.len(), 1, "should have exactly 1 entry, not duplicates");
}
// ═══════════════════════════════════════════════════════════════════════════
// Phase 4: Cross-node operation tests
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn find_object_on_peer_after_dissemination() {
let h = MultiNodeHarness::new(2);
let data = b"findable across nodes";
let hash = h.put_on(0, data, Some("findable.dat"));
h.disseminate_all();
// HandleFindObject on node 1 should find the entry.
let remote_node = NodeId([0xFF; 32]);
h.rt.send_to(
h.nodes[1].metadata,
MetadataMsg::HandleFindObject {
from: remote_node,
content_hash: hash,
reply_to: h.reply_addr(),
},
)
.unwrap();
let resp = tick_until_recv(&h.rt, &h.inbox, 10).unwrap();
match resp {
swactor_datastore::DatastoreResponse::GetOk { entry, .. } => {
assert_eq!(entry.content_hash, hash);
}
other => panic!("expected GetOk from HandleFindObject, got: {other:?}"),
}
}
#[test]
fn chunk_transfer_from_remote_blob_store() {
let h = MultiNodeHarness::new(2);
let data = vec![0xAB; 200]; // > chunk_size(64), so multiple chunks
let hash = h.put_on(0, &data, Some("transfer-test.bin"));
// Get the manifest from node 0.
let (_entry, manifest) = h.get_from(0, hash).expect("node 0 should have the entry");
assert!(manifest.chunks.len() > 1, "should have multiple chunks");
// Spawn a TransferActor wired to node 1's BlobStore.
let transfer_addr = h.rt.spawn(TransferActor::new(h.nodes[1].blob_store)).unwrap();
tick_n(&h.rt, 1);
// Start the download.
h.rt.send_to(
transfer_addr,
TransferMsg::StartDownload {
manifest: manifest.clone(),
source_node: h.nodes[0].node_id,
reply_to: h.reply_addr(),
},
)
.unwrap();
tick_n(&h.rt, 2);
// Feed chunks from node 0's BlobStore to the TransferActor.
for chunk_ref in &manifest.chunks {
let chunk_data = h.read_chunk_from(0, chunk_ref.hash)
.expect("chunk should exist on node 0");
h.rt.send_to(
transfer_addr,
TransferMsg::ChunkReceived {
hash: chunk_ref.hash,
data: chunk_data,
},
)
.unwrap();
tick_n(&h.rt, 3);
}
// Should get TransferComplete.
let resp = tick_until_recv(&h.rt, &h.inbox, 10).unwrap();
match resp {
swactor_datastore::DatastoreResponse::TransferComplete { content_hash } => {
assert_eq!(content_hash, hash);
}
other => panic!("expected TransferComplete, got: {other:?}"),
}
// Verify chunks are now on node 1's BlobStore.
for chunk_ref in &manifest.chunks {
let data_on_1 = h.read_chunk_from(1, chunk_ref.hash);
assert!(data_on_1.is_some(), "chunk should now exist on node 1");
}
}
#[test]
fn full_remote_get_scenario() {
let h = MultiNodeHarness::new(2);
let original_data = vec![0xCD; 200]; // Multiple chunks
let hash = h.put_on(0, &original_data, Some("full-remote.bin"));
// Disseminate metadata (including manifest) to node 1.
h.disseminate_all();
// Node 1 now has the entry and manifest via dissemination.
let (_entry, manifest) = h.get_from(1, hash)
.expect("node 1 should have entry+manifest via dissemination");
// Spawn TransferActor wired to node 1's BlobStore.
let transfer_addr = h.rt.spawn(TransferActor::new(h.nodes[1].blob_store)).unwrap();
tick_n(&h.rt, 1);
h.rt.send_to(
transfer_addr,
TransferMsg::StartDownload {
manifest: manifest.clone(),
source_node: h.nodes[0].node_id,
reply_to: h.reply_addr(),
},
)
.unwrap();
tick_n(&h.rt, 2);
// Transfer chunks from node 0 → node 1.
for chunk_ref in &manifest.chunks {
let chunk_data = h.read_chunk_from(0, chunk_ref.hash)
.expect("chunk should exist on node 0");
h.rt.send_to(
transfer_addr,
TransferMsg::ChunkReceived {
hash: chunk_ref.hash,
data: chunk_data,
},
)
.unwrap();
tick_n(&h.rt, 3);
}
let resp = tick_until_recv(&h.rt, &h.inbox, 10).unwrap();
assert!(
matches!(resp, swactor_datastore::DatastoreResponse::TransferComplete { .. }),
"expected TransferComplete"
);
// Reassemble from node 1's BlobStore and verify byte-for-byte match.
let mut reassembled = Vec::new();
for chunk_ref in &manifest.chunks {
let chunk_data = h.read_chunk_from(1, chunk_ref.hash)
.expect("chunk should be on node 1 after transfer");
reassembled.extend_from_slice(&chunk_data);
}
assert_eq!(reassembled, original_data, "reassembled data should match original");
}
#[test]
fn list_across_all_nodes_finds_objects_from_any_node() {
let h = MultiNodeHarness::new(3);
// Put distinct blobs on each node.
let hash0 = h.put_on(0, b"data from node zero", Some("zero.txt"));
let hash1 = h.put_on(1, b"data from node one", Some("one.txt"));
let hash2 = h.put_on(2, b"data from node two", Some("two.txt"));
// Query all nodes and merge results (simulating ListSwarm fan-out).
let mut all_entries = Vec::new();
for i in 0..3 {
all_entries.extend(h.list_on(i, None));
}
// Deduplicate by content hash (simulating the merge step).
let mut seen = std::collections::HashSet::new();
all_entries.retain(|e| seen.insert(e.content_hash));
assert_eq!(all_entries.len(), 3);
let hashes: std::collections::HashSet<ContentHash> = all_entries.iter().map(|e| e.content_hash).collect();
assert!(hashes.contains(&hash0));
assert!(hashes.contains(&hash1));
assert!(hashes.contains(&hash2));
}
#[test]
fn gc_on_one_node_does_not_affect_other_nodes() {
let h = MultiNodeHarness::new(2);
// Put the same data on both nodes (each stores its own chunks).
let data = vec![0xEE; 200];
let hash = h.put_on(0, &data, Some("gc-test.bin"));
let _hash1 = h.put_on(1, &data, Some("gc-test.bin"));
// Verify both nodes have chunks.
let chunks_0_before = h.list_chunks_on(0);
let chunks_1_before = h.list_chunks_on(1);
assert!(!chunks_0_before.is_empty());
assert!(!chunks_1_before.is_empty());
// Delete + GC on node 0.
h.delete_on(0, &hash);
h.gc_ticks_on(0, 5); // gc_interval=3, so 5 ticks guarantees at least 1 GC sweep.
// Node 0's chunks should be gone.
let chunks_0_after = h.list_chunks_on(0);
// Only manifest-related chunks referenced by remaining manifests survive.
// Since we deleted the only object, all chunks should be gone.
assert!(
chunks_0_after.is_empty(),
"node 0 chunks should be GC'd after delete, found {}",
chunks_0_after.len()
);
// Node 1's chunks should be untouched.
let chunks_1_after = h.list_chunks_on(1);
assert_eq!(
chunks_1_before.len(),
chunks_1_after.len(),
"node 1 chunks should be unaffected by node 0 GC"
);
}

240
crates/datastore/tests/storage_tests.rs Normal file
View file

@ -0,0 +1,240 @@
//! Tests for StorageBackend implementations — parameterized across backends.
use std::collections::HashSet;
use swactor_datastore::chunking::chunk_blob;
use swactor_datastore::storage::{FilesystemBackend, InMemoryBackend, StorageBackend};
use swactor_datastore::types::{ChunkRef, ContentHash, ObjectManifest};
// ═══════════════════════════════════════════════════════════════════════════
// Backend factory helpers
// ═══════════════════════════════════════════════════════════════════════════
fn run_with_both_backends(test: impl Fn(&mut dyn StorageBackend)) {
// In-memory
let mut mem = InMemoryBackend::new();
test(&mut mem);
// Filesystem
let dir = tempfile::tempdir().unwrap();
let mut fs = FilesystemBackend::new(dir.path().to_path_buf());
test(&mut fs);
}
// ═══════════════════════════════════════════════════════════════════════════
// Scenario: chunk storage contract
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn store_and_retrieve_a_single_chunk() {
run_with_both_backends(|backend| {
let data = b"hello chunk";
let hash = ContentHash::of(data);
backend.write_chunk(&hash, data).unwrap();
let read_back = backend.read_chunk(&hash).unwrap();
assert_eq!(read_back, Some(data.to_vec()));
});
}
#[test]
fn chunk_not_found_returns_none() {
run_with_both_backends(|backend| {
let hash = ContentHash::of(b"nonexistent");
assert_eq!(backend.read_chunk(&hash).unwrap(), None);
});
}
#[test]
fn delete_chunk_makes_it_unretrievable() {
run_with_both_backends(|backend| {
let data = b"ephemeral";
let hash = ContentHash::of(data);
backend.write_chunk(&hash, data).unwrap();
backend.delete_chunk(&hash).unwrap();
assert_eq!(backend.read_chunk(&hash).unwrap(), None);
});
}
#[test]
fn has_chunk_reflects_storage_state() {
run_with_both_backends(|backend| {
let data = b"existence check";
let hash = ContentHash::of(data);
assert!(!backend.has_chunk(&hash));
backend.write_chunk(&hash, data).unwrap();
assert!(backend.has_chunk(&hash));
backend.delete_chunk(&hash).unwrap();
assert!(!backend.has_chunk(&hash));
});
}
#[test]
fn list_chunks_returns_all_stored_hashes() {
run_with_both_backends(|backend| {
let mut expected = HashSet::new();
for i in 0u8..5 {
let data = vec![i; 32];
let hash = ContentHash::of(&data);
backend.write_chunk(&hash, &data).unwrap();
expected.insert(hash);
}
let listed: HashSet<ContentHash> = backend.list_chunks().into_iter().collect();
assert_eq!(listed, expected);
});
}
#[test]
fn store_and_retrieve_manifest() {
run_with_both_backends(|backend| {
let manifest = ObjectManifest {
content_hash: ContentHash::of(b"my-blob"),
chunks: vec![ChunkRef {
hash: ContentHash::of(b"chunk-0"),
offset: 0,
size: 1024,
}],
total_size: 1024,
chunk_size: 1024,
content_type: Some("text/plain".to_string()),
};
backend.write_manifest(&manifest).unwrap();
let read_back = backend.read_manifest(&manifest.content_hash).unwrap();
assert_eq!(read_back, Some(manifest));
});
}
#[test]
fn delete_manifest_removes_it() {
run_with_both_backends(|backend| {
let manifest = ObjectManifest {
content_hash: ContentHash::of(b"deletable"),
chunks: vec![],
total_size: 0,
chunk_size: 1024,
content_type: None,
};
backend.write_manifest(&manifest).unwrap();
backend.delete_manifest(&manifest.content_hash).unwrap();
assert_eq!(backend.read_manifest(&manifest.content_hash).unwrap(), None);
});
}
#[test]
fn overwriting_chunk_is_idempotent() {
run_with_both_backends(|backend| {
let data = b"idempotent write";
let hash = ContentHash::of(data);
backend.write_chunk(&hash, data).unwrap();
backend.write_chunk(&hash, data).unwrap();
assert_eq!(backend.read_chunk(&hash).unwrap(), Some(data.to_vec()));
assert_eq!(backend.list_chunks().len(), 1);
});
}
#[test]
fn chunked_blob_round_trips_through_storage() {
run_with_both_backends(|backend| {
let original: Vec<u8> = (0..500).map(|i| (i % 256) as u8).collect();
let (_, manifest, chunks) = chunk_blob(&original, 128);
// Store all chunks and manifest.
for (hash, data) in &chunks {
backend.write_chunk(hash, data).unwrap();
}
backend.write_manifest(&manifest).unwrap();
// Read back and reassemble.
let read_manifest = backend.read_manifest(&manifest.content_hash).unwrap().unwrap();
let mut reassembled = Vec::new();
for chunk_ref in &read_manifest.chunks {
let data = backend.read_chunk(&chunk_ref.hash).unwrap().unwrap();
reassembled.extend_from_slice(&data);
}
assert_eq!(reassembled, original);
});
}
// ═══════════════════════════════════════════════════════════════════════════
// Filesystem-only scenarios
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn backend_rescans_chunks_on_reopen() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().to_path_buf();
let data = b"persistent";
let hash = ContentHash::of(data);
// Write with first instance.
{
let mut backend = FilesystemBackend::new(path.clone());
backend.write_chunk(&hash, data).unwrap();
}
// Reopen — should discover existing chunks.
let backend = FilesystemBackend::new(path);
assert!(backend.has_chunk(&hash));
assert_eq!(backend.read_chunk(&hash).unwrap(), Some(data.to_vec()));
}
#[test]
fn sharding_creates_expected_directory_structure() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().to_path_buf();
let data = b"shard-check";
let hash = ContentHash::of(data);
let hex = hash.to_hex();
let mut backend = FilesystemBackend::new(path.clone());
backend.write_chunk(&hash, data).unwrap();
// Verify the 2-level sharded path exists.
let expected_path = path
.join("chunks")
.join(&hex[..2])
.join(&hex[2..4])
.join(&hex);
assert!(expected_path.exists(), "sharded chunk path should exist: {expected_path:?}");
}
// ═══════════════════════════════════════════════════════════════════════════
// Property-based tests
// ═══════════════════════════════════════════════════════════════════════════
mod proptests {
use super::*;
use proptest::prelude::*;
proptest! {
#[test]
fn any_chunk_survives_write_read_round_trip(
data in proptest::collection::vec(any::<u8>(), 1..4096),
) {
// In-memory
let mut mem = InMemoryBackend::new();
let hash = ContentHash::of(&data);
mem.write_chunk(&hash, &data).unwrap();
prop_assert_eq!(mem.read_chunk(&hash).unwrap(), Some(data.clone()));
// Filesystem
let dir = tempfile::tempdir().unwrap();
let mut fs = FilesystemBackend::new(dir.path().to_path_buf());
fs.write_chunk(&hash, &data).unwrap();
prop_assert_eq!(fs.read_chunk(&hash).unwrap(), Some(data));
}
}
}

668
crates/datastore/tests/transfer_tests.rs Normal file
View file

@ -0,0 +1,668 @@
//! Tests for TransferActor — exercised as a black box through the swactor runtime.
mod common;
use swactor_datastore::chunking::{chunk_blob, reassemble_blob, verify_integrity};
use swactor_datastore::messages::{BlobStoreMsg, DatastoreResponse, TransferMsg};
use swactor_datastore::types::{ChunkRef, ContentHash, ObjectManifest};
use distribution::types::NodeId;
use common::{
spawn_blob_store, spawn_transfer, test_runtime, tick_and_drain, tick_n, tick_until_recv,
};
// ─── Helpers ─────────────────────────────────────────────────────────────────
fn test_source_node() -> NodeId {
NodeId([0xAA; 32])
}
/// Build a manifest with `n` distinct 16-byte chunks.
/// Returns the manifest and a vec of (hash, data) pairs for each chunk.
fn make_multi_chunk_manifest(n: usize) -> (ObjectManifest, Vec<(ContentHash, Vec<u8>)>) {
let mut all_data = Vec::new();
let mut chunks_data = Vec::new();
let mut chunk_refs = Vec::new();
for i in 0..n {
let data = vec![i as u8; 16];
let hash = ContentHash::of(&data);
chunk_refs.push(ChunkRef {
hash,
offset: (i * 16) as u64,
size: 16,
});
chunks_data.push((hash, data.clone()));
all_data.extend_from_slice(&data);
}
let content_hash = ContentHash::of(&all_data);
let manifest = ObjectManifest {
content_hash,
chunks: chunk_refs,
total_size: all_data.len() as u64,
chunk_size: 16,
content_type: None,
};
(manifest, chunks_data)
}
// ═══════════════════════════════════════════════════════════════════════════
// Download Completion
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn single_chunk_download_completes() {
let rt = test_runtime();
let blob = spawn_blob_store(&rt);
let transfer = spawn_transfer(&rt, blob);
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
let reply = *inbox.addr();
tick_n(&rt, 2);
let (manifest, chunks) = make_multi_chunk_manifest(1);
let expected_content_hash = manifest.content_hash;
// Start the download.
rt.send_to(
transfer,
TransferMsg::StartDownload {
manifest,
source_node: test_source_node(),
reply_to: reply,
},
)
.unwrap();
tick_n(&rt, 2);
// Deliver the single chunk.
let (hash, data) = &chunks[0];
rt.send_to(
transfer,
TransferMsg::ChunkReceived {
hash: *hash,
data: data.clone(),
},
)
.unwrap();
let resp = tick_until_recv(&rt, &inbox, 20).unwrap();
match resp {
DatastoreResponse::TransferComplete { content_hash } => {
assert_eq!(content_hash, expected_content_hash);
}
other => panic!("expected TransferComplete, got: {other:?}"),
}
}
#[test]
fn multi_chunk_download_completes_after_all_chunks_arrive() {
let rt = test_runtime();
let blob = spawn_blob_store(&rt);
let transfer = spawn_transfer(&rt, blob);
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
let reply = *inbox.addr();
tick_n(&rt, 2);
let (manifest, chunks) = make_multi_chunk_manifest(3);
let expected_content_hash = manifest.content_hash;
rt.send_to(
transfer,
TransferMsg::StartDownload {
manifest,
source_node: test_source_node(),
reply_to: reply,
},
)
.unwrap();
tick_n(&rt, 2);
// Deliver first two chunks — no TransferComplete yet.
for (hash, data) in &chunks[..2] {
rt.send_to(
transfer,
TransferMsg::ChunkReceived {
hash: *hash,
data: data.clone(),
},
)
.unwrap();
}
let partial = tick_and_drain(&rt, &inbox, 10);
assert!(
partial.is_empty(),
"expected no response after partial delivery, got: {partial:?}"
);
// Deliver the final chunk.
let (hash, data) = &chunks[2];
rt.send_to(
transfer,
TransferMsg::ChunkReceived {
hash: *hash,
data: data.clone(),
},
)
.unwrap();
let resp = tick_until_recv(&rt, &inbox, 20).unwrap();
match resp {
DatastoreResponse::TransferComplete { content_hash } => {
assert_eq!(content_hash, expected_content_hash);
}
other => panic!("expected TransferComplete, got: {other:?}"),
}
}
// ═══════════════════════════════════════════════════════════════════════════
// Chunk Deduplication & Filtering
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn duplicate_chunk_is_silently_ignored() {
let rt = test_runtime();
let blob = spawn_blob_store(&rt);
let transfer = spawn_transfer(&rt, blob);
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
let reply = *inbox.addr();
tick_n(&rt, 2);
let (manifest, chunks) = make_multi_chunk_manifest(2);
let expected_content_hash = manifest.content_hash;
rt.send_to(
transfer,
TransferMsg::StartDownload {
manifest,
source_node: test_source_node(),
reply_to: reply,
},
)
.unwrap();
tick_n(&rt, 2);
let (hash_a, data_a) = &chunks[0];
let (hash_b, data_b) = &chunks[1];
// Send chunk A.
rt.send_to(
transfer,
TransferMsg::ChunkReceived {
hash: *hash_a,
data: data_a.clone(),
},
)
.unwrap();
tick_n(&rt, 3);
// Send chunk A again (duplicate).
rt.send_to(
transfer,
TransferMsg::ChunkReceived {
hash: *hash_a,
data: data_a.clone(),
},
)
.unwrap();
let after_dup = tick_and_drain(&rt, &inbox, 5);
assert!(
after_dup.is_empty(),
"duplicate chunk should not trigger early completion: {after_dup:?}"
);
// Send chunk B — now transfer completes.
rt.send_to(
transfer,
TransferMsg::ChunkReceived {
hash: *hash_b,
data: data_b.clone(),
},
)
.unwrap();
let resp = tick_until_recv(&rt, &inbox, 20).unwrap();
match resp {
DatastoreResponse::TransferComplete { content_hash } => {
assert_eq!(content_hash, expected_content_hash);
}
other => panic!("expected TransferComplete, got: {other:?}"),
}
}
#[test]
fn unexpected_chunk_hash_is_ignored() {
let rt = test_runtime();
let blob = spawn_blob_store(&rt);
let transfer = spawn_transfer(&rt, blob);
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
let reply = *inbox.addr();
tick_n(&rt, 2);
let (manifest, chunks) = make_multi_chunk_manifest(1);
let expected_content_hash = manifest.content_hash;
rt.send_to(
transfer,
TransferMsg::StartDownload {
manifest,
source_node: test_source_node(),
reply_to: reply,
},
)
.unwrap();
tick_n(&rt, 2);
// Send a chunk with a bogus hash (not in manifest).
let bogus_data = b"bogus-data-not-in-manifest".to_vec();
let bogus_hash = ContentHash::of(&bogus_data);
rt.send_to(
transfer,
TransferMsg::ChunkReceived {
hash: bogus_hash,
data: bogus_data,
},
)
.unwrap();
let after_bogus = tick_and_drain(&rt, &inbox, 5);
assert!(
after_bogus.is_empty(),
"unexpected chunk should have no effect: {after_bogus:?}"
);
// Now send the correct chunk.
let (hash, data) = &chunks[0];
rt.send_to(
transfer,
TransferMsg::ChunkReceived {
hash: *hash,
data: data.clone(),
},
)
.unwrap();
let resp = tick_until_recv(&rt, &inbox, 20).unwrap();
match resp {
DatastoreResponse::TransferComplete { content_hash } => {
assert_eq!(content_hash, expected_content_hash);
}
other => panic!("expected TransferComplete, got: {other:?}"),
}
}
// ═══════════════════════════════════════════════════════════════════════════
// Retry & Failure
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn first_chunk_failure_allows_retry_and_eventual_success() {
let rt = test_runtime();
let blob = spawn_blob_store(&rt);
let transfer = spawn_transfer(&rt, blob);
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
let reply = *inbox.addr();
tick_n(&rt, 2);
let (manifest, chunks) = make_multi_chunk_manifest(1);
let expected_content_hash = manifest.content_hash;
let (hash, data) = &chunks[0];
rt.send_to(
transfer,
TransferMsg::StartDownload {
manifest,
source_node: test_source_node(),
reply_to: reply,
},
)
.unwrap();
tick_n(&rt, 2);
// First failure — should be silently retried (no TransferFailed).
rt.send_to(
transfer,
TransferMsg::ChunkFailed {
hash: *hash,
reason: "timeout".into(),
},
)
.unwrap();
let after_fail = tick_and_drain(&rt, &inbox, 5);
assert!(
after_fail.is_empty(),
"first failure should not produce TransferFailed: {after_fail:?}"
);
// Retry succeeds.
rt.send_to(
transfer,
TransferMsg::ChunkReceived {
hash: *hash,
data: data.clone(),
},
)
.unwrap();
let resp = tick_until_recv(&rt, &inbox, 20).unwrap();
match resp {
DatastoreResponse::TransferComplete { content_hash } => {
assert_eq!(content_hash, expected_content_hash);
}
other => panic!("expected TransferComplete after retry, got: {other:?}"),
}
}
#[test]
fn second_chunk_failure_fails_entire_transfer() {
let rt = test_runtime();
let blob = spawn_blob_store(&rt);
let transfer = spawn_transfer(&rt, blob);
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
let reply = *inbox.addr();
tick_n(&rt, 2);
let (manifest, chunks) = make_multi_chunk_manifest(1);
let (hash, _) = &chunks[0];
rt.send_to(
transfer,
TransferMsg::StartDownload {
manifest,
source_node: test_source_node(),
reply_to: reply,
},
)
.unwrap();
tick_n(&rt, 2);
// First failure — silent retry.
rt.send_to(
transfer,
TransferMsg::ChunkFailed {
hash: *hash,
reason: "timeout".into(),
},
)
.unwrap();
tick_and_drain(&rt, &inbox, 5);
// Second failure — exhausts max_retries=1 → TransferFailed.
rt.send_to(
transfer,
TransferMsg::ChunkFailed {
hash: *hash,
reason: "connection lost".into(),
},
)
.unwrap();
let resp = tick_until_recv(&rt, &inbox, 20).unwrap();
match resp {
DatastoreResponse::TransferFailed { reason } => {
assert_eq!(reason, "connection lost");
}
other => panic!("expected TransferFailed, got: {other:?}"),
}
}
#[test]
fn partial_progress_lost_when_one_chunk_exhausts_retries() {
let rt = test_runtime();
let blob = spawn_blob_store(&rt);
let transfer = spawn_transfer(&rt, blob);
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
let reply = *inbox.addr();
tick_n(&rt, 2);
let (manifest, chunks) = make_multi_chunk_manifest(3);
let (hash_a, data_a) = &chunks[0];
let (hash_b, _) = &chunks[1];
rt.send_to(
transfer,
TransferMsg::StartDownload {
manifest,
source_node: test_source_node(),
reply_to: reply,
},
)
.unwrap();
tick_n(&rt, 2);
// Chunk A received successfully.
rt.send_to(
transfer,
TransferMsg::ChunkReceived {
hash: *hash_a,
data: data_a.clone(),
},
)
.unwrap();
tick_n(&rt, 3);
// Chunk B fails twice → transfer fails despite chunk A being received.
rt.send_to(
transfer,
TransferMsg::ChunkFailed {
hash: *hash_b,
reason: "fail-1".into(),
},
)
.unwrap();
tick_and_drain(&rt, &inbox, 5);
rt.send_to(
transfer,
TransferMsg::ChunkFailed {
hash: *hash_b,
reason: "fail-2".into(),
},
)
.unwrap();
let resp = tick_until_recv(&rt, &inbox, 20).unwrap();
match resp {
DatastoreResponse::TransferFailed { reason } => {
assert_eq!(reason, "fail-2");
}
other => panic!("expected TransferFailed, got: {other:?}"),
}
}
// ═══════════════════════════════════════════════════════════════════════════
// Cancellation
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn cancel_stops_transfer_with_no_response() {
let rt = test_runtime();
let blob = spawn_blob_store(&rt);
let transfer = spawn_transfer(&rt, blob);
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
let reply = *inbox.addr();
tick_n(&rt, 2);
let (manifest, chunks) = make_multi_chunk_manifest(2);
rt.send_to(
transfer,
TransferMsg::StartDownload {
manifest,
source_node: test_source_node(),
reply_to: reply,
},
)
.unwrap();
tick_n(&rt, 2);
// Deliver one chunk (partial progress).
let (hash_a, data_a) = &chunks[0];
rt.send_to(
transfer,
TransferMsg::ChunkReceived {
hash: *hash_a,
data: data_a.clone(),
},
)
.unwrap();
tick_n(&rt, 3);
// Cancel.
rt.send_to(transfer, TransferMsg::Cancel).unwrap();
// After cancel, no TransferComplete or TransferFailed should appear.
let after_cancel = tick_and_drain(&rt, &inbox, 10);
assert!(
after_cancel.is_empty(),
"cancel should produce no response: {after_cancel:?}"
);
}
// ═══════════════════════════════════════════════════════════════════════════
// Persistence to BlobStoreActor
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn received_chunks_are_forwarded_to_blob_store() {
let rt = test_runtime();
let blob = spawn_blob_store(&rt);
let transfer = spawn_transfer(&rt, blob);
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
let reply = *inbox.addr();
tick_n(&rt, 2);
let (manifest, chunks) = make_multi_chunk_manifest(2);
rt.send_to(
transfer,
TransferMsg::StartDownload {
manifest,
source_node: test_source_node(),
reply_to: reply,
},
)
.unwrap();
tick_n(&rt, 2);
// Deliver both chunks.
for (hash, data) in &chunks {
rt.send_to(
transfer,
TransferMsg::ChunkReceived {
hash: *hash,
data: data.clone(),
},
)
.unwrap();
}
// Wait for TransferComplete.
let resp = tick_until_recv(&rt, &inbox, 20).unwrap();
assert!(
matches!(resp, DatastoreResponse::TransferComplete { .. }),
"expected TransferComplete, got: {resp:?}"
);
// Give extra ticks for blob store writes to settle.
tick_n(&rt, 5);
// Verify both chunks are readable from BlobStoreActor.
for (hash, expected_data) in &chunks {
rt.send_to(
blob,
BlobStoreMsg::ReadChunk {
hash: *hash,
reply_to: reply,
},
)
.unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
match resp {
DatastoreResponse::ChunkOk { hash: h, data } => {
assert_eq!(h, *hash);
assert_eq!(data, *expected_data);
}
other => panic!("expected ChunkOk for {hash:?}, got: {other:?}"),
}
}
}
// ═══════════════════════════════════════════════════════════════════════════
// Full Lifecycle
// ═══════════════════════════════════════════════════════════════════════════
#[test]
fn download_and_reassemble_recovers_original_data() {
let rt = test_runtime();
let blob = spawn_blob_store(&rt);
let transfer = spawn_transfer(&rt, blob);
let inbox = rt.new_inbox::<DatastoreResponse>().unwrap();
let reply = *inbox.addr();
tick_n(&rt, 2);
// Use real chunking to produce a multi-chunk manifest.
let original: Vec<u8> = (0..200).map(|i| (i % 256) as u8).collect();
let (content_hash, manifest, chunks) = chunk_blob(&original, 64);
rt.send_to(
transfer,
TransferMsg::StartDownload {
manifest: manifest.clone(),
source_node: test_source_node(),
reply_to: reply,
},
)
.unwrap();
tick_n(&rt, 2);
// Deliver all chunks via TransferActor.
for (hash, data) in &chunks {
rt.send_to(
transfer,
TransferMsg::ChunkReceived {
hash: *hash,
data: data.clone(),
},
)
.unwrap();
}
let resp = tick_until_recv(&rt, &inbox, 20).unwrap();
match resp {
DatastoreResponse::TransferComplete {
content_hash: ch, ..
} => {
assert_eq!(ch, content_hash);
}
other => panic!("expected TransferComplete, got: {other:?}"),
}
// Give blob store time to persist.
tick_n(&rt, 5);
// Read all chunks from BlobStoreActor and reassemble.
let mut chunk_pairs = Vec::new();
for chunk_ref in &manifest.chunks {
rt.send_to(
blob,
BlobStoreMsg::ReadChunk {
hash: chunk_ref.hash,
reply_to: reply,
},
)
.unwrap();
let resp = tick_until_recv(&rt, &inbox, 10).unwrap();
match resp {
DatastoreResponse::ChunkOk { hash, data } => {
chunk_pairs.push((hash, data));
}
other => panic!("expected ChunkOk, got: {other:?}"),
}
}
let reassembled = reassemble_blob(&manifest, &chunk_pairs).expect("reassembly should succeed");
assert!(verify_integrity(&reassembled, &content_hash));
assert_eq!(reassembled, original);
}

3
crates/distribution/src/kademlia/routing_table.rs
View file

@ -96,7 +96,6 @@ impl KBucket {
pub struct RoutingTable {
self_id: NodeId,
buckets: Vec<KBucket>,
k: usize,
}
impl RoutingTable {
@ -109,7 +108,7 @@ impl RoutingTable {
for _ in 0..NUM_BUCKETS {
buckets.push(KBucket::new(k));
}
Self { self_id, buckets, k }
Self { self_id, buckets }
}
pub fn self_id(&self) -> NodeId {

1
crates/distribution/src/transport.rs
View file

@ -195,6 +195,7 @@ pub fn encode_wire_envelope(envelope: &WireEnvelope) -> Vec<u8> {
enum ReadError {
WouldBlock,
Disconnected,
#[allow(dead_code)]
Other(std::io::Error),
}

1
crates/distribution/tests/common/mod.rs
View file

@ -3,6 +3,7 @@
//! `TestCluster` makes sender-misattribution structurally impossible by
//! tagging every response with the responder's index, mirroring the
//! simulation crate's `deliver_actions_tagged_with_net`.
#![allow(dead_code)]
use distribution::node::{DistributedNode, DistributedNodeConfig};
use distribution::registry::RegistryConfig;

2
crates/distribution/tests/kademlia_directory.rs
View file

@ -3,7 +3,7 @@ use distribution::crypto::Keypair;
use distribution::kademlia::directory::{
actor_addr_as_node_id, resolve_quorum, DirectoryShard, QuorumResult,
};
use distribution::types::{DirectoryEntry, NodeId, Signature};
use distribution::types::{NodeId, Signature};
// ─── DirectoryShard ─────────────────────────────────────────────────────────

2
crates/runtime-dashboard/src/actor_detail_html.rs
View file

@ -123,6 +123,8 @@ pub const ACTOR_DETAIL_HTML: &str = r##"<!DOCTYPE html>
<nav class="nav-links">
<a href="/" class="nav-link">Overview</a>
<a href="/actors" class="nav-link">Actors</a>
<a href="/distribution" class="nav-link">Distribution</a>
<a href="/datastore" class="nav-link">Datastore</a>
</nav>
</div>
</div>

1
crates/runtime-dashboard/src/actors_html.rs
View file

@ -152,6 +152,7 @@ pub const ACTORS_HTML: &str = r##"<!DOCTYPE html>
<a href="/" class="nav-link">Overview</a>
<a href="/actors" class="nav-link active">Actors</a>
<a href="/distribution" class="nav-link">Distribution</a>
<a href="/datastore" class="nav-link">Datastore</a>
</nav>
</div>
<div class="header-right">

1
crates/runtime-dashboard/src/dashboard_html.rs
View file

@ -153,6 +153,7 @@ pub const DASHBOARD_HTML: &str = r##"<!DOCTYPE html>
<a href="/" class="nav-link active">Overview</a>
<a href="/actors" class="nav-link">Actors</a>
<a href="/distribution" class="nav-link">Distribution</a>
<a href="/datastore" class="nav-link">Datastore</a>
</nav>
</div>
<div class="header-right">

16
crates/runtime-dashboard/src/datastore_collector.rs Normal file
View file

@ -0,0 +1,16 @@
//! Datastore stats provider for the runtime dashboard.
//!
//! The trait returns a pre-serialized JSON string so that `runtime-dashboard`
//! has no compile-time dependency on `swactor-datastore` (which would create a
//! circular dependency since `swactor-datastore[node]` depends on us).
//!
//! The `swactor-datastore` crate implements this trait in its `node` feature.
/// Trait for providing datastore stats to the dashboard.
///
/// Implementations capture a point-in-time snapshot as serialized JSON.
/// The dashboard polls this every ~200ms via SSE.
pub trait DatastoreStatsProvider: Send + Sync {
/// Return a JSON-serialized datastore snapshot, or `None` if unavailable.
fn snapshot_json(&self) -> Option<String>;
}

286
crates/runtime-dashboard/src/datastore_html.rs Normal file
View file

@ -0,0 +1,286 @@
pub const DATASTORE_HTML: &str = r##"<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Swactor Runtime – Datastore</title>
<style>
* { margin: 0; padding: 0; box-sizing: border-box; }
body { font-family: 'Menlo', 'Consolas', 'Monaco', monospace; background: #0f1117; color: #e0e0e0; font-size: 13px; }
.header {
display: flex; align-items: center; justify-content: space-between;
padding: 12px 20px; background: #161822; border-bottom: 1px solid #2a2d3e;
}
.header-left { display: flex; align-items: center; }
.header h1 { font-size: 16px; font-weight: 600; color: #fff; }
.status-dot {
width: 10px; height: 10px; border-radius: 50%; background: #4caf50;
display: inline-block; margin-left: 8px; vertical-align: middle;
}
.status-dot.disconnected { background: #f44336; }
.status-dot.done { background: #ff9800; }
.nav-links { display: flex; gap: 4px; margin-left: 20px; }
.nav-link {
color: #888; text-decoration: none; font-size: 12px;
padding: 4px 10px; border-radius: 3px; transition: color 0.2s;
}
.nav-link:hover { color: #e0e0e0; }
.nav-link.active { color: #fff; background: #2a2d3e; }
.header-right { display: flex; align-items: center; gap: 12px; }
.grid {
display: grid;
grid-template-columns: 1fr 1fr;
gap: 12px; padding: 12px;
}
.panel {
background: #161822; border: 1px solid #2a2d3e; border-radius: 6px;
padding: 14px; overflow: hidden;
}
.panel h2 { font-size: 12px; color: #888; text-transform: uppercase; letter-spacing: 1px; margin-bottom: 10px; }
.full-width { grid-column: 1 / -1; }
.stats-cards {
display: grid; grid-template-columns: repeat(5, 1fr); gap: 10px;
}
.stat-card {
background: #1c1f2e; border-radius: 4px; padding: 10px; text-align: center;
}
.stat-card .value { font-size: 22px; font-weight: 700; color: #fff; }
.stat-card .label { font-size: 10px; color: #888; text-transform: uppercase; margin-top: 2px; }
.event-timeline {
max-height: 400px; overflow-y: auto;
}
.event-row {
display: flex; gap: 8px; padding: 4px 0; border-bottom: 1px solid #1c1f2e;
font-size: 11px; align-items: center;
}
.event-time { color: #555; min-width: 70px; }
.event-kind {
min-width: 48px; font-weight: 700; text-transform: uppercase; font-size: 10px;
padding: 1px 6px; border-radius: 3px; text-align: center;
}
.event-kind.put { background: #1b3a2a; color: #4caf50; }
.event-kind.get { background: #1a2a3e; color: #2196f3; }
.event-kind.delete { background: #3a1a1a; color: #f44336; }
.event-hash { color: #aaa; font-size: 10px; min-width: 120px; }
.event-name { color: #e0e0e0; flex: 1; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
.event-size { color: #888; min-width: 70px; text-align: right; }
.objects-table { max-height: 400px; overflow-y: auto; }
.objects-table table { width: 100%; border-collapse: collapse; }
.objects-table th, .objects-table td {
padding: 4px 8px; text-align: left; border-bottom: 1px solid #1c1f2e; font-size: 11px;
white-space: nowrap;
}
.objects-table th { color: #888; font-weight: 500; position: sticky; top: 0; background: #161822; }
.transfers-section { display: none; }
.transfers-section.visible { display: block; }
.transfer-row {
display: flex; align-items: center; gap: 10px; margin-bottom: 8px;
}
.transfer-hash { color: #aaa; font-size: 10px; min-width: 100px; }
.transfer-bar {
flex: 1; height: 16px; background: #1c1f2e; border-radius: 3px; overflow: hidden;
}
.transfer-fill {
height: 100%; background: #6366f1; border-radius: 3px;
transition: width 0.3s ease;
}
.transfer-label { color: #888; font-size: 11px; min-width: 80px; text-align: right; }
::-webkit-scrollbar { width: 6px; }
::-webkit-scrollbar-track { background: #0f1117; }
::-webkit-scrollbar-thumb { background: #2a2d3e; border-radius: 3px; }
</style>
</head>
<body>
<div class="header">
<div class="header-left">
<h1>
Swactor Runtime Dashboard
<span id="statusDot" class="status-dot disconnected"></span>
</h1>
<nav class="nav-links">
<a href="/" class="nav-link">Overview</a>
<a href="/actors" class="nav-link">Actors</a>
<a href="/distribution" class="nav-link">Distribution</a>
<a href="/datastore" class="nav-link active">Datastore</a>
</nav>
</div>
<div class="header-right">
<span id="nodeLabel" style="color:#888;font-size:12px;">Waiting for data...</span>
</div>
</div>
<div class="grid">
<!-- Stat cards -->
<div class="panel full-width">
<h2>Datastore Stats</h2>
<div class="stats-cards">
<div class="stat-card"><div class="value" id="statObjects">0</div><div class="label">Objects</div></div>
<div class="stat-card"><div class="value" id="statSize">0</div><div class="label">Total Size</div></div>
<div class="stat-card"><div class="value" id="statPuts">0</div><div class="label">Puts</div></div>
<div class="stat-card"><div class="value" id="statGets">0</div><div class="label">Gets</div></div>
<div class="stat-card"><div class="value" id="statDeletes">0</div><div class="label">Deletes</div></div>
</div>
</div>
<!-- Event timeline -->
<div class="panel">
<h2>Event Timeline <span id="eventCount" style="color:#555;font-weight:400;"></span></h2>
<div class="event-timeline" id="eventTimeline"></div>
</div>
<!-- Objects table -->
<div class="panel">
<h2>Objects <span id="objectCount" style="color:#555;font-weight:400;"></span></h2>
<div class="objects-table">
<table>
<thead><tr><th>Hash</th><th>Name</th><th>Size</th></tr></thead>
<tbody id="objectsBody"></tbody>
</table>
</div>
</div>
<!-- Transfers -->
<div class="panel full-width transfers-section" id="transfersPanel">
<h2>Active Transfers</h2>
<div id="transfersList"></div>
</div>
</div>
<script>
(function() {
var DASHBOARD_MODE = '__DASHBOARD_MODE__';
var dot = document.getElementById('statusDot');
function formatBytes(b) {
if (b === 0) return '0 B';
var units = ['B', 'KB', 'MB', 'GB', 'TB'];
var i = Math.floor(Math.log(b) / Math.log(1024));
if (i >= units.length) i = units.length - 1;
return (b / Math.pow(1024, i)).toFixed(i > 0 ? 1 : 0) + ' ' + units[i];
}
function formatTime(ms) {
var d = new Date(ms);
return ('0' + d.getHours()).slice(-2) + ':' +
('0' + d.getMinutes()).slice(-2) + ':' +
('0' + d.getSeconds()).slice(-2);
}
function escapeHtml(s) {
if (!s) return '';
return s.replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;');
}
function updateFromSnapshot(snap) {
// Node label
if (snap.node_id) {
document.getElementById('nodeLabel').textContent = 'Node: ' + snap.node_id.substring(0, 16) + '\u2026';
}
// Stat cards
document.getElementById('statObjects').textContent = snap.object_count;
document.getElementById('statSize').textContent = formatBytes(snap.total_bytes);
document.getElementById('statPuts').textContent = snap.put_ops;
document.getElementById('statGets').textContent = snap.get_ops;
document.getElementById('statDeletes').textContent = snap.delete_ops;
// Event timeline
var timeline = document.getElementById('eventTimeline');
var wasAtBottom = timeline.scrollTop + timeline.clientHeight >= timeline.scrollHeight - 20;
timeline.innerHTML = '';
document.getElementById('eventCount').textContent = '(' + snap.recent_events.length + ')';
for (var i = snap.recent_events.length - 1; i >= 0; i--) {
var ev = snap.recent_events[i];
var row = document.createElement('div');
row.className = 'event-row';
row.innerHTML =
'<span class="event-time">' + formatTime(ev.timestamp_ms) + '</span>' +
'<span class="event-kind ' + ev.kind + '">' + ev.kind + '</span>' +
'<span class="event-hash">' + ev.hash.substring(0, 16) + '\u2026</span>' +
'<span class="event-name">' + escapeHtml(ev.name || '') + '</span>' +
'<span class="event-size">' + (ev.size_bytes > 0 ? formatBytes(ev.size_bytes) : '') + '</span>';
timeline.appendChild(row);
}
if (wasAtBottom) {
timeline.scrollTop = timeline.scrollHeight;
}
// Objects table
var tbody = document.getElementById('objectsBody');
tbody.innerHTML = '';
document.getElementById('objectCount').textContent = '(' + snap.objects.length + ')';
for (var i = 0; i < snap.objects.length; i++) {
var obj = snap.objects[i];
var tr = document.createElement('tr');
tr.innerHTML =
'<td style="color:#aaa;font-size:10px;">' + obj.hash.substring(0, 16) + '\u2026</td>' +
'<td>' + escapeHtml(obj.name || '\u2014') + '</td>' +
'<td style="color:#888;">' + formatBytes(obj.size_bytes) + '</td>';
tbody.appendChild(tr);
}
// Transfers
var panel = document.getElementById('transfersPanel');
var list = document.getElementById('transfersList');
if (snap.active_transfers.length === 0) {
panel.className = 'panel full-width transfers-section';
} else {
panel.className = 'panel full-width transfers-section visible';
list.innerHTML = '';
for (var i = 0; i < snap.active_transfers.length; i++) {
var t = snap.active_transfers[i];
var pct = t.chunks_total > 0 ? Math.round((t.chunks_received / t.chunks_total) * 100) : 0;
var row = document.createElement('div');
row.className = 'transfer-row';
row.innerHTML =
'<span class="transfer-hash">' + t.hash.substring(0, 16) + '\u2026</span>' +
'<div class="transfer-bar"><div class="transfer-fill" style="width:' + pct + '%"></div></div>' +
'<span class="transfer-label">' + t.chunks_received + ' / ' + t.chunks_total + '</span>';
list.appendChild(row);
}
}
}
// SSE connection
var es = new EventSource('/events');
es.addEventListener('datastore', function(e) {
try {
var snap = JSON.parse(e.data);
updateFromSnapshot(snap);
} catch(err) { console.error('datastore parse error', err); }
});
es.addEventListener('done', function() {
dot.className = 'status-dot done';
es.close();
});
es.onerror = function() {
dot.className = 'status-dot disconnected';
};
es.onopen = function() {
dot.className = 'status-dot';
};
})();
</script>
</body>
</html>
"##;

1
crates/runtime-dashboard/src/distribution_html.rs
View file

@ -121,6 +121,7 @@ pub const DISTRIBUTION_HTML: &str = r##"<!DOCTYPE html>
<a href="/" class="nav-link">Overview</a>
<a href="/actors" class="nav-link">Actors</a>
<a href="/distribution" class="nav-link active">Distribution</a>
<a href="/datastore" class="nav-link">Datastore</a>
</nav>
</div>
<div class="header-right">

14
crates/runtime-dashboard/src/lib.rs
View file

@ -20,6 +20,9 @@ mod distribution_html;
#[cfg(feature = "distribution")]
pub mod distribution_collector;
mod datastore_html;
pub mod datastore_collector;
use std::io;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
@ -91,6 +94,7 @@ pub struct DashboardHandle {
recording: bool,
#[cfg(feature = "distribution")]
distribution: Arc<Mutex<Option<Arc<dyn distribution_collector::DistributionStatsProvider>>>>,
datastore: Arc<Mutex<Option<Arc<dyn datastore_collector::DatastoreStatsProvider>>>>,
}
impl DashboardHandle {
@ -123,6 +127,11 @@ impl DashboardHandle {
*self.distribution.lock().unwrap() = Some(provider);
}
/// Attach a datastore stats provider, enabling the `/datastore` page.
pub fn set_datastore(&self, provider: Arc<dyn datastore_collector::DatastoreStatsProvider>) {
*self.datastore.lock().unwrap() = Some(provider);
}
/// Access the time-series history store (for TUI sparklines, etc.).
pub fn history(&self) -> &Arc<DashboardHistory> {
&self.history
@ -178,6 +187,9 @@ pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle {
let distribution: Arc<Mutex<Option<Arc<dyn distribution_collector::DistributionStatsProvider>>>> =
Arc::new(Mutex::new(None));
let datastore: Arc<Mutex<Option<Arc<dyn datastore_collector::DatastoreStatsProvider>>>> =
Arc::new(Mutex::new(None));
server::spawn_http_server(
Arc::clone(&store),
Arc::clone(&runtime),
@ -187,6 +199,7 @@ pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle {
config.port,
#[cfg(feature = "distribution")]
Arc::clone(&distribution),
Arc::clone(&datastore),
);
// Start stats recorder thread when recording is enabled
@ -229,6 +242,7 @@ pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle {
recording: config.record,
#[cfg(feature = "distribution")]
distribution,
datastore,
}
}

46
crates/runtime-dashboard/src/server.rs
View file

@ -24,6 +24,9 @@ use crate::distribution_collector::DistributionStatsProvider;
#[cfg(feature = "distribution")]
use crate::distribution_html::DISTRIBUTION_HTML;
use crate::datastore_collector::DatastoreStatsProvider;
use crate::datastore_html::DATASTORE_HTML;
/// Format a server-sent event.
fn format_sse(event: &str, data: &str) -> Vec<u8> {
format!("event: {event}\ndata: {data}\n\n").into_bytes()
@ -128,6 +131,7 @@ pub(crate) fn spawn_http_server(
port: u16,
#[cfg(feature = "distribution")]
distribution: Arc<Mutex<Option<Arc<dyn DistributionStatsProvider>>>>,
datastore: Arc<Mutex<Option<Arc<dyn DatastoreStatsProvider>>>>,
) {
let addr = format!("0.0.0.0:{port}");
let server = tiny_http::Server::http(&addr).expect("failed to bind HTTP server");
@ -144,6 +148,7 @@ pub(crate) fn spawn_http_server(
let cmd_router = Arc::clone(&cmd_router);
#[cfg(feature = "distribution")]
let distribution = Arc::clone(&distribution);
let datastore = Arc::clone(&datastore);
thread::spawn(move || {
loop {
let request = match server.recv() {
@ -159,6 +164,7 @@ pub(crate) fn spawn_http_server(
"/topology" => respond_html(request, TOPOLOGY_HTML, "live"),
#[cfg(feature = "distribution")]
"/distribution" => respond_html(request, DISTRIBUTION_HTML, "live"),
"/datastore" => respond_html(request, DATASTORE_HTML, "live"),
"/events" => {
handle_live_sse(
request,
@ -169,6 +175,7 @@ pub(crate) fn spawn_http_server(
Arc::clone(&history),
#[cfg(feature = "distribution")]
Arc::clone(&distribution),
Arc::clone(&datastore),
);
}
"/api/stats" => {
@ -204,6 +211,12 @@ pub(crate) fn spawn_http_server(
Arc::clone(&distribution),
);
}
"/api/datastore" => {
handle_datastore_api(
request,
Arc::clone(&datastore),
);
}
"/api/logs" => {
handle_logs_api(request, &url, Arc::clone(&store));
}
@ -239,6 +252,7 @@ fn handle_live_sse(
history: Arc<DashboardHistory>,
#[cfg(feature = "distribution")]
distribution: Arc<Mutex<Option<Arc<dyn DistributionStatsProvider>>>>,
datastore: Arc<Mutex<Option<Arc<dyn DatastoreStatsProvider>>>>,
) {
let (tx, rx) = mpsc::channel::<Vec<u8>>();
let response = make_sse_response(rx);
@ -309,6 +323,18 @@ fn handle_live_sse(
}
}
// Send datastore snapshot if provider is attached
{
let maybe_ds = datastore.lock().unwrap().clone();
if let Some(provider) = maybe_ds {
if let Some(json) = provider.snapshot_json() {
if tx.send(format_sse("datastore", &json)).is_err() {
return;
}
}
}
}
// Send new activity events
let (batch, new_cursor) = store.read_from(cursor);
if !batch.is_empty() {
@ -423,6 +449,26 @@ fn handle_distribution_api(
let _ = request.respond(response);
}
fn handle_datastore_api(
request: tiny_http::Request,
datastore: Arc<Mutex<Option<Arc<dyn DatastoreStatsProvider>>>>,
) {
let json = match datastore.lock().unwrap().as_ref() {
Some(provider) => provider.snapshot_json().unwrap_or_else(|| "{}".into()),
None => serde_json::json!({
"error": "datastore provider not attached"
})
.to_string(),
};
let response = tiny_http::Response::from_string(json).with_header(
"Content-Type: application/json"
.parse::<tiny_http::Header>()
.unwrap(),
);
let _ = request.respond(response);
}
fn handle_topology_api(
request: tiny_http::Request,
runtime: Arc<Mutex<Option<Arc<Runtime>>>>,

1
crates/runtime-dashboard/src/topology_html.rs
View file

@ -47,6 +47,7 @@ pub const TOPOLOGY_HTML: &str = r##"<!DOCTYPE html>
<a href="/actors" class="nav-link">Actors</a>
<a href="/topology" class="nav-link active">Topology</a>
<a href="/distribution" class="nav-link">Distribution</a>
<a href="/datastore" class="nav-link">Datastore</a>
</nav>
</div>
</div>

2
crates/runtime-dashboard/tests/command_api.rs
View file

@ -8,7 +8,7 @@ use std::sync::Arc;
use swactor::actor::ActorInterface;
use swactor::runtime::{Ctx, Runtime, RuntimeConfig};
use runtime_dashboard::command::{
from_query_params, parse_line, CommandContext, CommandRequest, CommandResponse, CommandRouter,
from_query_params, parse_line, CommandContext, CommandResponse, CommandRouter,
};
// ── Test Helpers ─────────────────────────────────────────────────────────────

278
docs/datastore/actors.md Normal file
View file

@ -0,0 +1,278 @@
# Datastore Actor Reference
## Overview
The datastore is built from four actors within the swactor runtime. `DatastoreNode` is the public facade — all external requests (HTTP API, network protocol) enter through it and are routed to two long-lived worker actors: `BlobStoreActor` (content-addressed chunk/manifest I/O) and `MetadataActor` (object index, DHT replication, GC). A fourth actor, `TransferActor`, is spawned ephemerally for each remote download and self-terminates on completion or failure.
```
┌─────────────────────────┐
│ store_node (main) │
│ spawns all 3 long-lived │
│ actors, drives ticks │
└────┬──────┬──────┬───────┘
│ │ │
spawn │ │ │ spawn
┌──────────────┘ │ └──────────────┐
▼ │ spawn ▼
┌───────────────────┐ │ ┌───────────────────┐
│ BlobStoreActor │ │ │ MetadataActor │
│ (chunks, manifests)│ │ │ (index, DHT, GC) │
└─────────▲─────────┘ │ └──▲────────┬───────┘
│ ▼ │ │
│ ┌───────────────────┐ │ │
│ │ DatastoreNode │────┘ │
│ │ (facade/router) │─────────────┘
└────────────│ │
└────────┬──────────┘
│ spawns (per download)
▼
┌───────────────────┐
│ TransferActor │
│ (ephemeral) │
└───────────────────┘
Arrows: ──▶ sends messages to
```
## Actors
### DatastoreNode
| | |
|---|---|
| **Role** | Top-level coordinator/facade. Accepts user-facing commands and incoming network protocol messages, delegates all work to `BlobStoreActor` and `MetadataActor`. |
| **Source** | `crates/datastore/src/actors/datastore_node.rs` |
| **Spawned by** | `store_node` binary (`crates/datastore/src/bin/store_node.rs:188`) |
| **Lifecycle** | Long-lived — runs for the lifetime of the process |
**Inbound messages** (`DatastoreNodeMsg` — 11 variants):
User-facing commands:
- `Put { data, name, tags, reply_to }` — chunk a blob, write chunks + manifest to `BlobStoreActor`, register in `MetadataActor`
- `Get { content_hash, reply_to }` — retrieve object metadata + manifest via `MetadataActor`
- `Delete { content_hash, reply_to }` — remove object via `MetadataActor`
- `List { name_filter, all, reply_to }` — list objects (local or swarm-wide) via `MetadataActor`
- `Status { reply_to }` — return this node's `NodeId`
- `ReadChunk { hash, reply_to }` — read a single chunk via `BlobStoreActor`
Protocol routing (incoming network messages):
- `IncomingGetChunk` — forwards to `BlobStoreActor::ReadChunk`
- `IncomingGetManifest` — forwards to `BlobStoreActor::ReadManifest`
- `IncomingStoreObject` — forwards to `MetadataActor::HandleStoreObject`
- `IncomingFindObject` — forwards to `MetadataActor::HandleFindObject`
- `IncomingListObjects` — forwards to `MetadataActor::ListLocal`
**Key outbound messages:**
- `BlobStoreMsg::WriteChunk`, `WriteManifest`, `ReadChunk`, `ReadManifest` — to `BlobStoreActor`
- `MetadataMsg::PutObject`, `GetObject`, `DeleteObject`, `ListLocal`, `ListSwarm`, `HandleStoreObject`, `HandleFindObject` — to `MetadataActor`
- `DatastoreResponse::NodeStatus` — directly to caller for `Status`
---
### BlobStoreActor
| | |
|---|---|
| **Role** | Content-addressed storage for chunks and manifests. All I/O goes through a pluggable `StorageBackend` (filesystem or in-memory). |
| **Source** | `crates/datastore/src/actors/blob_store.rs` |
| **Spawned by** | `store_node` binary (`store_node.rs:178`) |
| **Lifecycle** | Long-lived — runs for the lifetime of the process |
**Inbound messages** (`BlobStoreMsg` — 8 variants):
Chunk operations:
- `WriteChunk { hash, data, reply_to }` — persist a chunk, reply `ChunkStored`
- `ReadChunk { hash, reply_to }` — read a chunk, reply `ChunkOk` or `NotFound`
- `DeleteChunk { hash }` — remove a chunk (fire-and-forget)
- `HasChunk { hash, reply_to }` — existence check, reply `Bool`
- `ListChunks { reply_to }` — list all chunk hashes, reply `ChunkList`
- `GcUnreferenced { referenced }` — delete chunks not in the referenced set (fire-and-forget)
Manifest operations:
- `WriteManifest { manifest, reply_to }` — persist a manifest, reply `ManifestStored`
- `ReadManifest { hash, reply_to }` — read a manifest, reply `ManifestOk` or `NotFound`
**Key outbound messages:**
- `DatastoreResponse` variants (`ChunkStored`, `ChunkOk`, `ManifestStored`, `ManifestOk`, `NotFound`, `Error`, `Bool`, `ChunkList`) — always back to the `reply_to` address
---
### MetadataActor
| | |
|---|---|
| **Role** | Object metadata index. Maintains a `HashMap<ContentHash, ObjectEntry>` and a manifest cache. Handles DHT-style find/store operations, epidemic dissemination of entries to peers, and periodic garbage collection. |
| **Source** | `crates/datastore/src/actors/metadata.rs` |
| **Spawned by** | `store_node` binary (`store_node.rs:185`) |
| **Lifecycle** | Long-lived — runs for the lifetime of the process |
**Inbound messages** (`MetadataMsg` — 11 variants):
Object operations:
- `PutObject { entry, manifest, reply_to }` — store metadata + manifest locally, enqueue for dissemination, reply `PutOk`
- `GetObject { content_hash, reply_to }` — local lookup, reply `GetOk` or `NotFound`
- `DeleteObject { content_hash, reply_to }` — remove from local index, reply `DeleteOk` or `NotFound`
- `ListLocal { name_filter, reply_to }` — list local entries with optional name filter, reply `ListOk`
- `ListSwarm { name_filter, reply_to }` — swarm-wide list (currently delegates to `ListLocal`)
DHT protocol:
- `HandleFindObject { from, content_hash, reply_to }` — answer an incoming FIND_VALUE from a peer
- `HandleStoreObject { entry, manifest }` — accept an incoming STORE from a peer (fire-and-forget)
Peer management:
- `SetPeers { peers }` — update the list of peer `MetadataActor` addresses for dissemination
Periodic ticks (driven by the `store_node` main loop):
- `DisseminateTick` — send pending entries to all known peers
- `GcTick` — collect referenced chunks from all manifests, send `BlobStoreMsg::GcUnreferenced` to `BlobStoreActor`
**Key outbound messages:**
- `DatastoreResponse` variants (`PutOk`, `GetOk`, `DeleteOk`, `ListOk`, `NotFound`, `Error`) — to caller
- `MetadataMsg::HandleStoreObject` — to peer `MetadataActor` addresses during dissemination
- `BlobStoreMsg::GcUnreferenced` — to local `BlobStoreActor` during GC
---
### TransferActor
| | |
|---|---|
| **Role** | Manages a single object download from a remote node. Tracks pending/received chunks, forwards received data to the local `BlobStoreActor`, and reports completion or failure to the original requester. |
| **Source** | `crates/datastore/src/actors/transfer.rs` |
| **Spawned by** | `DatastoreNode` (one per remote download) |
| **Lifecycle** | Ephemeral — self-terminates via `ctx.stop_self()` on completion, failure, or cancel |
**Inbound messages** (`TransferMsg` — 4 variants):
- `StartDownload { manifest, source_node, reply_to }` — initialize the download with a manifest and source
- `ChunkReceived { hash, data }` — a chunk arrived from the remote node
- `ChunkFailed { hash, reason }` — a chunk fetch failed (retries up to `max_retries`, then fails the whole transfer)
- `Cancel` — abort the transfer immediately
**Key outbound messages:**
- `BlobStoreMsg::WriteChunk` — to local `BlobStoreActor` for each received chunk
- `DatastoreResponse::TransferComplete` — to `reply_to` when all chunks received
- `DatastoreResponse::TransferFailed` — to `reply_to` when retries are exhausted
---
## Message Reference
All message types are defined in `crates/datastore/src/messages.rs`.
### Intra-node actor messages
| Enum | Variants | Handled by |
|------|----------|------------|
| `DatastoreNodeMsg` | 11 (6 user-facing + 5 protocol routing) | `DatastoreNode` |
| `BlobStoreMsg` | 8 (5 chunk ops + 1 GC + 2 manifest ops) | `BlobStoreActor` |
| `MetadataMsg` | 11 (5 object ops + 2 DHT + 1 peer mgmt + 2 ticks) | `MetadataActor` |
| `TransferMsg` | 4 (start + chunk received + chunk failed + cancel) | `TransferActor` |
### Shared response enum
`DatastoreResponse` — 15 variants used as the return type for all four actors:
| Variant | Meaning |
|---------|---------|
| `PutOk { content_hash }` | Object stored successfully |
| `GetOk { entry, manifest }` | Object found |
| `DeleteOk { content_hash }` | Object deleted |
| `ListOk { entries }` | List result |
| `ChunkOk { hash, data }` | Chunk data retrieved |
| `ChunkStored { hash }` | Chunk written to storage |
| `ManifestStored { hash }` | Manifest written to storage |
| `ManifestOk { manifest }` | Manifest retrieved |
| `TransferComplete { content_hash }` | All chunks downloaded |
| `TransferFailed { reason }` | Transfer failed |
| `NodeStatus { node_id }` | Node identity |
| `NotFound` | Resource not found |
| `Error { reason }` | Generic error |
| `Bool(bool)` | Boolean result (e.g. `HasChunk`) |
| `ChunkList { hashes }` | List of chunk hashes |
### Inter-node wire messages (NetworkMessage)
| Struct | Direction | Purpose |
|--------|-----------|---------|
| `GetChunkRequest` | requester → holder | Fetch a chunk by hash |
| `GetChunkResponse` | holder → requester | Return chunk data (or `None`) |
| `StoreObjectRequest` | origin → DHT peer | Kademlia STORE for object metadata |
| `FindObjectRequest` | requester → DHT peer | Kademlia FIND_VALUE for object metadata |
| `FindObjectResponse` | DHT peer → requester | Return `Found(entry)` or `Closer(nodes)` |
| `GetManifestRequest` | requester → holder | Fetch a manifest by content hash |
| `GetManifestResponse` | holder → requester | Return manifest (or `None`) |
| `ListObjectsRequest` | requester → peer | List objects with optional name filter |
| `ListObjectsResponse` | peer → requester | Return matching entries |
Wire messages are distinguished from intra-node messages by implementing the `NetworkMessage` trait with a stable `type_tag()` string. They are serialized with serde for transport over iroh/QUIC.
---
## Key Flows
### Put (store a blob)
```
Client → DatastoreNode::Put
→ chunk_blob() splits data into chunks
→ BlobStoreActor::WriteChunk (for each chunk, fire-and-forget)
→ BlobStoreActor::WriteManifest
→ MetadataActor::PutObject
→ stores entry + manifest locally
→ enqueues for dissemination
→ replies DatastoreResponse::PutOk
```
### Get (retrieve metadata)
```
Client → DatastoreNode::Get
→ MetadataActor::GetObject
→ local index lookup
→ replies DatastoreResponse::GetOk (or NotFound)
```
### Data (reassemble from chunks)
See [streaming.md](streaming.md) for the full transfer protocol. In summary:
```
API server → DatastoreNode::Get → MetadataActor (local miss)
→ iterate peers:
→ FindObjectRequest (wire) → peer MetadataActor
→ GetManifestRequest (wire) → peer BlobStoreActor
→ GetChunkRequest (wire) → peer BlobStoreActor (per chunk)
→ BlobStoreActor::WriteChunk (store locally)
→ BlobStoreActor::WriteManifest
→ MetadataActor::PutObject
→ reassemble_blob() → verify blake3 → respond
```
### Dissemination (epidemic replication)
```
store_node main loop (every disseminate_interval ticks)
→ MetadataActor::DisseminateTick
→ take_pending() selects entries with remaining budget
→ for each peer: MetadataActor::HandleStoreObject
→ peer inserts if absent, re-enqueues for further dissemination
```
Budget per entry = `Λ * ceil(log2(cluster_size))` (SWIM-style, Λ=3).
### GC (garbage collection)
```
store_node main loop (every gc_interval ticks)
→ MetadataActor::GcTick
→ scans all manifests → builds referenced chunk set
→ BlobStoreActor::GcUnreferenced { referenced }
→ deletes any chunk not in the referenced set
```
---
## Related
- [streaming.md](streaming.md) — chunking, transfer protocol, reassembly, and progress tracking

103
docs/datastore/streaming.md Normal file
View file

@ -0,0 +1,103 @@
# Datastore Streaming Architecture
## Overview
"Streaming" in the swactor datastore refers to **progressive chunk-based transfer**, not byte-level streaming. When an object is stored, it is split into fixed-size chunks, each content-addressed with blake3. When retrieved from a remote peer, chunks are fetched individually and reassembled — enabling progress tracking and partial recovery.
This design trades a small amount of per-chunk overhead for:
- **Progress visibility**: the dashboard shows `chunks_received / chunks_total` in real time
- **Resumability**: a failed transfer can (in principle) restart from the last chunk
- **Deduplication**: identical chunks across objects are stored once
## Content-Addressed Chunking
The `chunk_blob()` function (`chunking.rs`) splits raw bytes into fixed-size pieces:
1. Compute `ContentHash = blake3(entire_blob)` — this is the object's identity
2. Split the blob into `ceil(total_size / chunk_size)` pieces (default chunk size: 1 MB)
3. For each piece, compute `chunk_hash = blake3(piece_bytes)`
4. Build a `ChunkRef { hash, offset, size }` for each piece
5. Return an `ObjectManifest` containing the full list of `ChunkRef`s
```
Blob (5.2 MB, chunk_size=1MB)
├── Chunk 0: hash=abc1…, offset=0, size=1048576
├── Chunk 1: hash=def2…, offset=1048576, size=1048576
├── Chunk 2: hash=789a…, offset=2097152, size=1048576
├── Chunk 3: hash=bcd3…, offset=3145728, size=1048576
└── Chunk 4: hash=ef45…, offset=4194304, size=1048576 (last: 209920 bytes)
```
The object's identity (`ContentHash`) is the hash of the *entire* blob, not of the manifest. This means the same data always produces the same hash regardless of chunk size.
## Transfer Protocol
When a client requests an object via `GET /api/data?hash=...`, the API server:
1. Sends a `DatastoreNodeMsg::Get` to the local `DatastoreNode` actor
2. If found locally, reads all chunks from the local `BlobStore` and reassembles
3. If **not found locally**, enters `try_remote_get()`:
![Transfer Flow](../diagrams/datastore_transfer.svg)
### Remote GET step-by-step
1. **Iterate peers**: for each known peer node:
2. **FindObject**: send `MetadataMsg::HandleFindObject` to the peer's `MetadataActor`
3. **Read manifest**: send `BlobStoreMsg::ReadManifest` to the peer's `BlobStore`
4. **Fetch chunks**: for each `ChunkRef` in the manifest:
- Send `BlobStoreMsg::ReadChunk` to the peer
- Receive `DatastoreResponse::ChunkOk { hash, data }`
- Store locally via `BlobStoreMsg::WriteChunk`
- Update `DatastoreMetrics::advance_transfer()` for dashboard progress
5. **Store manifest locally**: `BlobStoreMsg::WriteManifest`
6. **Store metadata locally**: `MetadataMsg::PutObject`
7. **Reassemble and respond**: `reassemble_blob()` concatenates chunks and verifies integrity
If a peer doesn't have the object (or any step fails), the loop continues to the next peer.
## Reassembly
`reassemble_blob()` (`chunking.rs`) takes a manifest and a set of `(hash, data)` pairs:
1. For each `ChunkRef` in manifest order, find the matching `(hash, data)` pair
2. Concatenate all chunk data into a single buffer
3. Compute `blake3(result)` and verify it matches `manifest.content_hash`
4. Return the reassembled blob (or a `ChunkingError` on mismatch)
This integrity check ensures that even if individual chunks are corrupted or swapped, the final result is always verified against the original content hash.
## Progress Tracking
The `DatastoreMetrics` struct provides thread-safe transfer tracking:
```
begin_transfer(hash, chunks_total) // called when remote GET starts
advance_transfer(hash) // called after each chunk is stored locally
end_transfer(hash) // called on completion or failure
```
The dashboard SSE stream includes a `datastore` event every ~200ms with a `DatastoreSnapshot` containing `active_transfers: Vec<TransferProgress>`. The web UI renders these as animated progress bars.
```
TransferProgress {
hash: "abc123...",
chunks_received: 3,
chunks_total: 5,
}
```
## GC Integration
When an object is deleted, its `ObjectEntry` and `ObjectManifest` are removed from the `MetadataActor`. However, the underlying chunks are **not immediately deleted** — they may be referenced by other manifests (deduplication).
Instead, garbage collection runs periodically:
1. `MetadataMsg::GcTick` triggers a scan
2. The `MetadataActor` collects all chunk hashes referenced by any live manifest
3. Sends `BlobStoreMsg::GcUnreferenced` with the referenced set
4. The `BlobStore` deletes any chunks **not** in the referenced set
This two-phase approach prevents data loss when chunks are shared between objects.
![Chunk Lifecycle](../diagrams/datastore_chunk_lifecycle.svg)

471
docs/development_history/datastore/PROTOCOL.md Normal file
View file

@ -0,0 +1,471 @@
# Swactor Datastore Protocol Specification
**Version:** 0.2.0 (MVP)
**Status:** Draft
## 1. Overview
The Swactor Datastore is a distributed personal file/blob storage protocol for small trusted clusters (laptop, phone, browser). It provides content-hash-first addressing with immutable content-addressed objects, replicated via a Kademlia-based metadata DHT.
### Design Principles
- **Content-hash-first addressing** — every object is identified by `blake3(blob_bytes)`. This is the primary key for all operations.
- **Immutable content-addressed objects** — content hashes are unique identifiers. There are no write conflicts by construction.
- **Names are metadata** — optional flat strings attached to objects, not keys. Multiple objects can share a name; distinguished by content hash.
- **Separation of data and metadata** — chunks are large opaque blobs; metadata is small, gossiped, and queryable.
- **Crash-safe** — fsync before acknowledge on all writes.
- **Actor-based** — three actor types coordinate via message passing within the swactor runtime.
- **Transport-agnostic** — protocol messages defined as `NetworkMessage` types; MVP uses iroh (QUIC + NAT hole-punch + encryption).
- **Pluggable storage** — `StorageBackend` trait abstracts I/O for filesystem (MVP), IndexedDB (browser), etc.
## 2. Terminology
| Term | Definition |
|------|-----------|
| **Object** | Content-addressed blob identified by `blake3(blob_bytes)`. May carry an optional human-readable name as metadata. |
| **Blob** | The raw byte content of an object. |
| **Chunk** | A fixed-size (1 MB default) slice of a blob, identified by its blake3 content hash. |
| **Manifest** | An ordered list of `ChunkRef`s describing how to reassemble an object from chunks. Stored under the object's content hash. |
| **ContentHash** | 32-byte blake3 digest. Primary identifier for blobs and DHT key. |
| **ObjectEntry** | Metadata record: content hash, optional name, owner node, tags. |
| **DHT overlay** | A Kademlia distributed hash table for object metadata, separate from the actor directory DHT. Keys are `blake3(blob_bytes)`. |
| **StorageBackend** | Trait abstracting chunk and manifest I/O for pluggable backends (filesystem, IndexedDB, etc.). |
| **Node** | A device running the swactor runtime with a datastore actor set (BlobStoreActor + MetadataActor). |
## 3. Data Model
### 3.1 ContentHash
```
ContentHash = blake3(data)[0..32] // 32 bytes
```
- **Hashing algorithm:** blake3 — 2-3x faster than sha256, tree-hashable (parallel hashing of large chunks), same 32-byte output. Supports streaming hashing for large blobs via `blake3::Hasher`.
- **Display:** first 8 bytes as hex + ellipsis (e.g. `a1b2c3d4e5f6a7b8…`).
- **XOR distance:** bitwise XOR of the 32-byte arrays, used for Kademlia routing in the metadata DHT.
### 3.2 ObjectEntry
```
ObjectEntry {
content_hash: ContentHash, // blake3(entire_blob) — primary identifier
name: Option<String>, // Optional flat string, not a path
node_id: NodeId, // Node that stores the object
tags: BTreeMap<String, String>, // User-defined key-value tags
size_bytes: u64, // Total object size
created_at: u64, // Wall-clock creation time (informational)
}
```
No conflict resolution is needed — content hashes are unique identifiers. Storing the same blob twice is a no-op (same content hash). Different blobs always have different content hashes.
### 3.3 ObjectManifest
```
ObjectManifest {
content_hash: ContentHash, // blake3(entire_blob) — NOT the hash of this manifest
chunks: Vec<ChunkRef>, // Ordered list of chunks
total_size: u64, // Total object size in bytes
chunk_size: u32, // Fixed chunk size used (e.g. 1MB)
content_type: Option<String>, // MIME type
}
ChunkRef {
hash: ContentHash, // Content hash of chunk data
offset: u64, // Byte offset in original object
size: u32, // Actual size (last chunk may be smaller)
}
```
The `content_hash` field is `blake3(entire_blob)`, computed via a streaming hasher alongside chunking. The manifest is stored and looked up using this content hash as the key.
### 3.4 Storage Backend
The `StorageBackend` trait abstracts all chunk and manifest I/O:
```rust
pub trait StorageBackend: Send {
fn write_chunk(&mut self, hash: &ContentHash, data: &[u8]) -> Result<(), io::Error>;
fn read_chunk(&self, hash: &ContentHash) -> Result<Option<Vec<u8>>, io::Error>;
fn delete_chunk(&mut self, hash: &ContentHash) -> Result<(), io::Error>;
fn has_chunk(&self, hash: &ContentHash) -> bool;
fn list_chunks(&self) -> Vec<ContentHash>;
fn write_manifest(&mut self, manifest: &ObjectManifest) -> Result<(), io::Error>;
fn read_manifest(&self, content_hash: &ContentHash) -> Result<Option<ObjectManifest>, io::Error>;
fn delete_manifest(&mut self, content_hash: &ContentHash) -> Result<(), io::Error>;
}
```
#### MVP: FilesystemBackend
Two-level directory sharding to avoid huge directories:
```
{storage_path}/
├── chunks/
│ └── {hex[0..2]}/
│ └── {hex[2..4]}/
│ └── {full_hex_hash} # Raw chunk bytes
└── manifests/
└── {hex[0..2]}/
└── {hex[2..4]}/
└── {full_hex_hash} # JSON-serialized ObjectManifest
```
Example: chunk with hash `abcdef12...` is stored at `chunks/ab/cd/abcdef12...`.
All writes are fsynced before acknowledging.
## 4. Content Addressing
### 4.1 Chunking Algorithm
Fixed-size chunking (MVP):
1. Read the input file in `chunk_size` byte blocks (default: 1,048,576 = 1 MB).
2. For each block, compute `ContentHash::of(block)`.
3. Store each chunk via the `StorageBackend`.
4. Build a `Vec<ChunkRef>` with sequential offsets.
5. Compute `content_hash = blake3(entire_blob)` using a streaming hasher fed alongside chunking.
6. Create the `ObjectManifest` with this `content_hash` and store it via the `StorageBackend` keyed by `content_hash`.
The last chunk may be smaller than `chunk_size`.
### 4.2 Reassembly
1. Read the `ObjectManifest` (by its `content_hash`).
2. For each `ChunkRef` in order, read the chunk by `hash`.
3. Concatenate all chunk data to reconstruct the original blob.
4. Verify: `blake3(reassembled) == content_hash` (optional integrity check).
## 5. Metadata DHT
> **Status:** Types and routing table logic exist in the `distribution` crate. `MetadataActor` has a dissemination queue (`enqueue`/`take_pending`) and peer-to-peer replication via `SetPeers` + `DisseminateTick`. Verified in local multi-node simulation. Full Kademlia iterative lookup (FIND_VALUE with α-parallel queries) is not yet implemented — dissemination is epidemic/gossip-style.
### 5.1 Overlay Design
The metadata DHT is a **separate Kademlia overlay** from the actor directory. It stores `ObjectEntry` records keyed by `blake3(blob_bytes)` — the content hash of the entire blob.
This separation ensures:
- Object metadata routing doesn't interfere with actor discovery.
- Different replication factors can be used (objects may be stored on fewer nodes).
- The DHT can be independently tuned for the metadata workload.
### 5.2 Key Mapping
```
DHT key = blake3(blob_bytes) = entry.content_hash
```
### 5.3 Store Flow
When storing object metadata:
1. Use `key = entry.content_hash`.
2. Find the `k` closest nodes to `key` in the metadata DHT routing table.
3. Send `StoreObjectRequest { entry }` to each of the `k` closest nodes.
### 5.4 Lookup Flow
When looking up object metadata:
1. Send `FindObjectRequest { content_hash }` to the `α` closest known nodes.
2. Each node responds with either `Found(ObjectEntry)` or `Closer(Vec<(NodeId, SocketAddr)>)`.
3. Continue querying closer nodes until convergence.
No merge step is needed — content hashes are unique identifiers.
## 6. Protocol Flows
### 6.1 PUT — Store an Object
```
User MetadataActor BlobStoreActor
│ │ │
│─── PutObject ────────────>│ │
│ │ │
│ │ (chunk the file, │
│ │ stream blake3 hash) │
│ │ │
│ │─── WriteChunk ────────>│
│ │<── ChunkStored ────────│ (repeat for each chunk)
│ │ │
│ │─── WriteManifest ─────>│
│ │<── ManifestStored ─────│
│ │ │
│ │ (create ObjectEntry, │
│ │ store in local index,│
│ │ enqueue for DHT │
│ │ dissemination) │
│ │ │
│<── PutOk {content_hash} ─│ │
```
### 6.2 GET — Retrieve an Object (Local)
```
User MetadataActor BlobStoreActor
│ │ │
│─── GetObject ────────────>│ │
│ {content_hash} │ │
│ │ (lookup content_hash │
│ │ in local index) │
│ │ │
│<── GetOk { entry, │ │
│ manifest } ──────│ │
│ │
│ (for each chunk in manifest) │
│───────────── ReadChunk ───────────────────────────>│
│<────────────── ChunkOk ───────────────────────────│
│ │
│ (reassemble chunks into original file) │
```
### 6.3 GET — Retrieve an Object (Remote)
> **Status:** `TransferActor` state machine is functional and stores received chunks to the local `BlobStoreActor`. Chunks must be fed externally (via `ChunkReceived` messages). Automatic chunk pulling from remote nodes is not yet implemented — the test harness or a future network adapter plays the "pull" role. Verified in multi-node simulation.
```
User MetadataActor TransferActor Remote BlobStore
│ │ │ │
│─ GetObject ──>│ │ │
│ {content_hash}│ │ │
│ │ (not in local │ │
│ │ index; DHT │ │
│ │ lookup) │ │
│ │ │ │
│ │─ StartDownload ───>│ │
│ │ │ │
│ │ │── GetChunkRequest ─>│
│ │ │<─ GetChunkResponse ─│
│ │ │ │
│ │ │ (repeat for each │
│ │ │ chunk) │
│ │ │ │
│ │<─ TransferComplete │ │
│ │ │ │
│<── GetOk ────│ │ (stops self) │
```
### 6.4 DELETE — Remove an Object
```
User MetadataActor
│ │
│─── DeleteObject ─────────>│
│ {content_hash} │
│ │
│ │ (remove from local )
│ │ (index, best-effort )
│ │ (notify DHT peers )
│ │
│<── DeleteOk │
│ {content_hash} ────────│
```
Chunk data is **not** immediately deleted. Unreferenced chunks are cleaned up during GC sweeps (see Section 10).
### 6.5 LIST — List Objects (Local)
```
User MetadataActor
│ │
│─── ListLocal ────────────>│
│ {name_filter} │
│ │
│ │ (filter local index )
│ │ (by name substring )
│ │
│<── ListOk { entries } ───│
```
### 6.6 LIST — List Objects (Swarm-Wide)
> **Status:** `ListSwarm` currently delegates to `ListLocal` (returns local entries only). Fan-out to peer MetadataActors is not yet wired. Swarm-wide listing is verified in simulation by querying each node and merging results in the test harness.
```
User MetadataActor Remote MetadataActors
│ │ │
│─ ListSwarm ──>│ │
│ {name_filter} │ │
│ │── ListObjectsRequest ─>│ (fan-out to all known
│ │<─ ListObjectsResponse ─│ alive nodes)
│ │ │
│ │ (merge all results, │
│ │ deduplicate by │
│ │ content hash) │
│ │ │
│<── ListOk ───│ │
```
## 7. Actor Architecture
> **Status:** All three actor types are fully implemented and tested. `DatastoreNode` coordinator routes commands to internal actors. 83+ tests across 8 test files verify single-node operations. Multi-node dissemination and cross-node transfers verified in simulation.
### 7.1 BlobStoreActor
**Responsibility:** Chunk and manifest I/O via `StorageBackend` trait.
- **State:** `Box<dyn StorageBackend>`
- **Lifecycle:** Long-lived, one per node.
- **Guarantees:** Delegates to backend; filesystem backend fsyncs before acknowledging.
**Message types:** `BlobStoreMsg` (see `messages.rs`)
### 7.2 MetadataActor
**Responsibility:** Object metadata index, DHT routing.
- **State:** Local object index (`HashMap<ContentHash, ObjectEntry>`), manifest cache, dissemination queue.
- **Lifecycle:** Long-lived, one per node.
- **Coordinates with:** BlobStoreActor (for manifest storage), remote MetadataActors (DHT operations).
**Message types:** `MetadataMsg` (see `messages.rs`)
### 7.3 TransferActor
**Responsibility:** Downloading an object (all its chunks) from a remote node.
- **State:** Manifest, pending/received chunk sets, retry counts.
- **Lifecycle:** Ephemeral — spawned per download, self-terminates on completion/failure/cancel.
- **Coordinates with:** Remote BlobStoreActor (chunk requests), local BlobStoreActor (chunk storage).
**Message types:** `TransferMsg` (see `messages.rs`)
## 8. Wire Protocol
> **Status:** All message types are defined with `NetworkMessage` implementations and stable type tags. Serialization is JSON (serde). No transport integration yet — messages are passed directly via actor addresses in simulation.
### 8.1 Message Types
All inter-node messages implement `NetworkMessage` with a stable `type_tag()`:
| Message | type_tag | Direction |
|---------|----------|-----------|
| `GetChunkRequest` | `swactor_datastore::GetChunkRequest` | requester → holder |
| `GetChunkResponse` | `swactor_datastore::GetChunkResponse` | holder → requester |
| `StoreObjectRequest` | `swactor_datastore::StoreObjectRequest` | writer → DHT nodes |
| `FindObjectRequest` | `swactor_datastore::FindObjectRequest` | reader → DHT nodes |
| `FindObjectResponse` | `swactor_datastore::FindObjectResponse` | DHT node → reader |
| `GetManifestRequest` | `swactor_datastore::GetManifestRequest` | requester → holder |
| `GetManifestResponse` | `swactor_datastore::GetManifestResponse` | holder → requester |
| `ListObjectsRequest` | `swactor_datastore::ListObjectsRequest` | requester → remote node |
| `ListObjectsResponse` | `swactor_datastore::ListObjectsResponse` | remote node → requester |
`FindObjectRequest` contains a `content_hash` field (the `blake3(blob_bytes)` key).
### 8.2 Serialization
MVP: serde JSON for all messages. Binary format (bincode or msgpack) planned for later to reduce overhead, especially for `GetChunkResponse` which carries large payloads.
### 8.3 Framing
Messages are framed over iroh QUIC streams:
- Each request/response pair uses a single bidirectional stream.
- Message format: `[4-byte length (big-endian)][JSON payload]`.
## 9. Naming
Names are **optional flat strings** — human-readable labels attached to objects as metadata.
- Names are not keys. The content hash is the only primary identifier.
- Multiple objects can share the same name. They are distinguished by content hash.
- Names are simple strings (e.g. `"vacation.jpg"`, `"backup-2024-01"`). No path hierarchy, no separators enforced.
- No conflict resolution is needed — different content always produces different content hashes.
## 10. Garbage Collection
> **Status:** Fully implemented. `MetadataActor::gc_tick()` builds a referenced chunk set from all local manifests and sends `GcUnreferenced` to `BlobStoreActor`. Verified with 6 GC-specific tests including deduplication safety, interval gating, and empty-store edge case.
### 10.1 Entry Removal
Deleting an object:
1. Remove the `ObjectEntry` from the local index.
2. Best-effort notify DHT peers to remove their replicas.
3. Remove the local manifest.
### 10.2 Chunk Reference Counting
Unreferenced chunk cleanup:
1. Build a referenced set: union of all chunk hashes from all local manifest entries.
2. Send `BlobStoreMsg::GcUnreferenced { referenced }` to the BlobStoreActor.
3. BlobStoreActor diffs its chunk list against the referenced set and deletes unreferenced chunks.
**Safety:** A chunk may be referenced by multiple objects (deduplication). Only delete when zero references remain.
### 10.3 GC Schedule
- `MetadataActor` runs `gc_tick()` every tick. Actual GC sweep happens every `gc_interval` ticks (default: 1000).
- Chunk GC is triggered less frequently (order of minutes) to avoid overhead.
## 11. Failure Modes
### 11.1 Node Offline
- **Metadata persists** in the DHT (replicated to k-closest nodes). Lookups succeed as long as any replica is alive.
- **Chunk fetches fail** if the only copy is on the offline node. The TransferActor retries once, then reports failure.
- **Recovery:** When the node comes back, its metadata is re-disseminated (anti-entropy).
### 11.2 Transfer Interrupted
- **Partial state:** Some chunks may be written to the local BlobStoreActor before the transfer fails.
- **Cleanup:** Partially downloaded chunks are not harmful — they're content-addressed and may be useful for future downloads. Unreferenced chunks are cleaned up by GC.
- **Retry:** The user can retry the GET, and only missing chunks need to be fetched (future optimization).
### 11.3 DHT Inconsistency
- **Stale metadata:** A node may serve an outdated ObjectEntry. Anti-entropy dissemination ensures replicas converge.
- Content addressing eliminates write conflicts — storing the same content hash twice is idempotent.
### 11.4 Disk Full
- `StorageBackend::write_chunk` fails with an I/O error, which is propagated back to the requester as `DatastoreResponse::Error`.
- No partial writes — fsync ensures atomicity (filesystem backend).
## 12. CLI Interface
> **Status:** Command types defined in `src/cli.rs`. Parser, dispatcher, and `[[bin]]` target not yet implemented. Planned for a follow-up session.
```
swactor-store put <local-path> [--name <label>] [--tag key=value...]
Store a local file as a distributed object.
Returns the content hash of the stored object.
--name sets an optional human-readable label.
swactor-store get <content-hash>[@<node>] [--output <local-path>]
Retrieve an object by content hash. Fetches from the specified node or discovers via DHT.
--output defaults to the object's name (if set) in the current directory.
swactor-store delete <content-hash>
Remove an object from the local index and notify DHT peers.
swactor-store list [--name <substring>] [--node <node-name>] [--all]
List objects. --name filters by name substring. --all queries all nodes (swarm-wide). Default is local.
swactor-store status
Show node info: identity, chunk count, storage usage.
```
## 13. Browser API
> **Status:** Not started. Separate milestone.
WASM-exposed functions for browser integration:
```
list_objects(name_filter: Option<String>) -> Vec<ObjectEntry>
List objects visible to this node, optionally filtered by name.
get_object(content_hash: ContentHash) -> Result<Vec<u8>, Error>
Download and reassemble an object by content hash.
put_object(data: Vec<u8>, name: Option<String>) -> Result<ContentHash, Error>
Chunk, store, and register an object. Returns the content hash.
delete_object(content_hash: ContentHash) -> Result<(), Error>
Remove an object from the local index.
get_node_status() -> NodeStatus
Node identity, chunk count, connected peers.
```
These map directly to the MetadataActor message types. The WASM runtime handles serialization across the JS/Rust boundary.

317
docs/development_history/datastore/SUMMARY.md Normal file
View file

@ -0,0 +1,317 @@
# swactor-datastore: Development History & Status
## Overview
`swactor-datastore` is a distributed personal file/blob storage protocol for small trusted clusters (laptop, phone, browser). It provides content-hash-first addressing with immutable content-addressed objects, replicated via epidemic metadata dissemination across peers.
The implementation is organized as a single Rust crate (`crates/datastore/`) built on the `swactor` actor runtime. It was developed in 7 ordered modules (local single-node operations) followed by a multi-node simulation phase.
**Current state: 93 tests across 9 test files, all passing. Zero warnings.**
---
## Architecture
```
┌─────────────────────────────────────────────────────┐
│ DatastoreNode │ Coordinator/facade
│ (single entry point for callers) │
├────────────────────┬────────────────────────────────┤
│ MetadataActor │ BlobStoreActor │ Long-lived, one per node
│ (object index, │ (chunk & manifest I/O │
│ dissemination, │ via StorageBackend) │
│ GC orchestration)│ │
├────────────────────┴────────────────────────────────┤
│ TransferActor (ephemeral) │ One per download
│ (chunk tracking, retry, self-termination) │
├─────────────────────────────────────────────────────┤
│ StorageBackend trait │ Pluggable I/O
│ FilesystemBackend │ InMemoryBackend │
├─────────────────────────────────────────────────────┤
│ Chunking Engine (pure functions) │ No I/O, deterministic
│ chunk_blob · reassemble_blob · verify_integrity │
└─────────────────────────────────────────────────────┘
```
### Core Design Principles
- **Content-hash-first addressing** -- every object identified by `blake3(blob_bytes)`.
- **Immutable content-addressed objects** -- no write conflicts by construction.
- **Names are metadata** -- optional flat strings, not keys.
- **Separation of data and metadata** -- chunks are large opaque blobs; metadata is small and gossiped.
- **Actor-based** -- three actor types coordinate via message passing.
- **Transport-agnostic** -- protocol messages defined as `NetworkMessage` types.
- **Pluggable storage** -- `StorageBackend` trait abstracts I/O.
---
## Module-by-Module Development History
### Module 1: Chunking Engine
**What:** Pure functions for content-addressed blob chunking and reassembly. `chunk_blob()`, `reassemble_blob()`, `verify_integrity()`, plus `ContentHash`, `ObjectManifest`, `ChunkRef` types.
**Key decisions:**
- Fixed-size chunking over content-defined chunking (simpler, deterministic; CDC dedup unnecessary for small clusters)
- BLAKE3 for all hashing (3-7 GB/s, 32-byte output matching `NodeId`)
- Whole-blob hash as content hash rather than Merkle root of chunk hashes
- Empty blob produces a valid zero-chunk manifest
**Tests:** 13 (10 scenario + 3 proptest). Round-trips, edge cases, integrity verification, determinism.
**Files:** `src/chunking.rs`, `src/types.rs`
---
### Module 2: Storage Backend
**What:** `StorageBackend` trait with two implementations: `FilesystemBackend` (2-level hex-sharded dirs, fsync-on-write) and `InMemoryBackend` (HashMap-based for tests/WASM).
**Key decisions:**
- 2-level hex sharding (65,536 possible directories) to avoid hot directories
- In-memory chunk index for O(1) `has_chunk` lookups, populated via scan-on-init
- JSON manifest serialization for debuggability
- `Send` but not `Sync` on the trait (single-actor ownership)
- Idempotent writes and deletes
**Tests:** 12 (9 parameterized across both backends + 2 FS-only + 1 proptest).
**Files:** `src/storage/mod.rs`, `src/storage/in_memory.rs`
---
### Module 3: BlobStoreActor
**What:** Message-driven actor wrapping `Box<dyn StorageBackend>` for chunk/manifest CRUD plus garbage collection. Also introduced the shared test harness (`tests/common/mod.rs`).
**Key decisions:**
- Thin delegation -- actor adds no logic beyond message dispatch
- Explicit `reply_to` pattern (tell, not ask) for response routing
- Fire-and-forget deletes and GC (no reply needed)
- `Box<dyn StorageBackend>` for dynamic dispatch (one backend per instance)
- Single-threaded tick-based testing for determinism
**Tests:** 7 scenario tests through the swactor runtime.
**Established patterns:** `reply_to` pattern, shared test harness with `test_runtime()`, `tick_n()`, `tick_until_recv()`, `DatastoreHarness`.
**Files:** `src/actors/blob_store.rs`, `tests/blob_store_tests.rs`, `tests/common/mod.rs`
---
### Module 4: MetadataActor
**What:** Object metadata index (`HashMap<ContentHash, ObjectEntry>`), manifest cache, SWIM-inspired gossip dissemination queue. Handles local CRUD, DHT protocol messages (`HandleFindObject`, `HandleStoreObject`), and GC tick orchestration.
**Key decisions:**
- Node ID stamping on `PutObject` (prevents spoofing; remote entries retain original owner)
- Idempotent DHT store (insert-if-absent semantics)
- Synthetic empty manifest for `HandleFindObject` when manifest is missing
- Separate `GetObject` (local, error on missing manifest) vs `HandleFindObject` (DHT, synthesizes empty manifest)
- SWIM-style dissemination with budget `Lambda * ceil(log2(n))`, Lambda=3
**Tests:** 10 scenario tests covering CRUD, DHT operations, idempotency, lifecycle.
**Files:** `src/actors/metadata.rs`, `tests/metadata_tests.rs`
---
### Module 5: TransferActor
**What:** Ephemeral per-download actor. Tracks pending/received chunks, forwards received chunks to BlobStoreActor, implements per-chunk retry, self-terminates on completion/failure/cancel.
**Key decisions:**
- Ephemeral actor pattern (one per download, isolates transfer state)
- Passive design -- chunks driven externally via `ChunkReceived`/`ChunkFailed` (decoupled from networking)
- Whole-transfer failure on any chunk exhausting retries
- `max_retries` defaults to 1 (first failure allows retry, second aborts)
- Fire-and-forget chunk persistence (ChunkStored reply silently dropped)
**Tests:** 10 scenario tests covering state machine transitions, retry logic, cancellation, data recovery.
**Files:** `src/actors/transfer.rs`, `tests/transfer_tests.rs`
---
### Module 6: DatastoreNode Coordinator
**What:** Facade actor encapsulating BlobStoreActor + MetadataActor behind a single address. Routes 6 user-facing commands and 5 network protocol variants.
**Key decisions:**
- Pass-through `reply_to` pattern (responses go directly to caller, coordinator never intercepts)
- Inline chunking in `handle_put` (synchronous, no async coordination)
- Fire-and-forget chunk writes (same pattern as TransferActor)
- Immutable state after construction
- `Put` uses `data: Vec<u8>` not `PathBuf` (testable, WASM-compatible)
**Tests:** 12 scenario tests through `NodeHarness`.
**Files:** `src/actors/datastore_node.rs`, `tests/datastore_node_tests.rs`
---
### Module 7: Garbage Collection
**What:** Completed `MetadataActor::gc_tick()` to build a referenced chunk set from all manifests and send `GcUnreferenced` to BlobStoreActor for orphan cleanup.
**Key decisions:**
- `blob_store_addr = None` guard for backward compatibility (GC no-ops when not wired)
- Fire-and-forget `GcUnreferenced` (no reply needed)
- `spawn_metadata_with_config()` helper to wire blob_store_addr before spawning
- Mark-and-sweep: union of all chunk hashes from all manifests = referenced set
**Tests:** 6 scenario tests covering cleanup, preservation, deduplication safety, interval gating, empty-store edge case.
**Files:** `src/actors/metadata.rs` (delta), `tests/gc_tests.rs`, `tests/common/mod.rs` (GcHarness)
---
### Multi-Node Simulation (Phases 3-4)
**What:** Wired up MetadataActor for peer-to-peer metadata dissemination. Added `SetPeers` and `DisseminateTick` messages. Created `MultiNodeHarness` for simulating clusters on a single Runtime. Extended `HandleStoreObject` to carry manifests alongside entries for full metadata replication.
**Key decisions:**
- Single Runtime for simulation -- all nodes' actors share one Runtime; actor addresses are globally unique so cross-node messaging "just works" via `ctx.send()`
- MetadataActor owns peer relationships (simpler than routing through DatastoreNode)
- Manifest dissemination alongside entry dissemination (peers receive both)
- TransferActor stays passive -- tests feed chunks from remote BlobStoreActor (test harness plays the "network adapter" role)
- No automatic remote GET orchestration yet -- DatastoreNode remains a stateless router
**New messages:**
- `MetadataMsg::SetPeers { peers: Vec<ActorAddress> }`
- `MetadataMsg::DisseminateTick`
- `MetadataMsg::HandleStoreObject` extended with `manifest: Option<ObjectManifest>`
**Tests:** 10 new scenario tests in `tests/multi_node_tests.rs`:
| # | Test | Verifies |
|---|------|----------|
| 1 | `metadata_replicates_to_peer_after_dissemination` | Put on 0, disseminate, node 1 finds it |
| 2 | `metadata_replicates_to_all_peers_in_3_node_cluster` | Full cluster replication |
| 3 | `dissemination_budget_expires_after_enough_rounds` | Budget exhaustion, fresh entries still work |
| 4 | `delete_on_origin_does_not_propagate_to_peers` | Delete is local only |
| 5 | `duplicate_put_via_dissemination_is_idempotent` | No duplicate entries on peer |
| 6 | `find_object_on_peer_after_dissemination` | HandleFindObject succeeds on peer |
| 7 | `chunk_transfer_from_remote_blob_store` | TransferActor pulls chunks cross-node |
| 8 | `full_remote_get_scenario` | End-to-end: put on 0, disseminate, transfer 0->1, reassemble matches |
| 9 | `list_across_all_nodes_finds_objects_from_any_node` | Simulated ListSwarm fan-out |
| 10 | `gc_on_one_node_does_not_affect_other_nodes` | GC isolation between nodes |
**Files:** `src/messages.rs`, `src/actors/metadata.rs`, `tests/common/mod.rs` (MultiNodeHarness), `tests/multi_node_tests.rs`
---
## Code Quality Pass
Alongside the multi-node work, a cleanup pass was performed:
- **PROTOCOL.md** -- Added honest `> Status:` annotations to sections 5 (Metadata DHT), 6.3 (Remote GET), 6.6 (ListSwarm), 7 (Actor Architecture), 8 (Wire Protocol), 10 (GC), 12 (CLI), and 13 (Browser API)
- **metadata.rs** -- Updated stale `ListSwarm` "MVP" comment
- **transfer.rs** -- Updated architecture comments describing simulation-ready passive design
- **tests/common/mod.rs** -- Removed 3 unused imports (`ActorInterface`, `Ctx`, `DatastoreNodeMsg`)
---
## Test Summary
| Test File | Count | What |
|-----------|-------|------|
| `chunking_tests.rs` | 13 | Pure function round-trips, edge cases, proptests |
| `storage_tests.rs` | 12 | Backend CRUD, parameterized across FS + InMemory, proptests |
| `blob_store_tests.rs` | 7 | Actor-level chunk/manifest CRUD, GC |
| `metadata_tests.rs` | 10 | Object index, DHT protocol, lifecycle |
| `transfer_tests.rs` | 10 | Download state machine, retry, cancel |
| `datastore_node_tests.rs` | 12 | Coordinator routing, network protocol |
| `datastore_tests.rs` | 13 | Content-addressing properties, proptests |
| `gc_tests.rs` | 6 | Mark-and-sweep GC, dedup safety |
| `multi_node_tests.rs` | 10 | Dissemination, cross-node transfer, GC isolation |
| **Total** | **93** | |
**Testing philosophy:** Scenario/story tests first, property-based tests for invariants, contract tests for serialization. No white-box/structural tests. Low coupling to internals -- tests should survive a refactor.
---
## Future Work
### Near-Term (Next Sessions)
**CLI Implementation (Phase 2)**
- Parser and dispatcher using `clap`
- `[[bin]]` target in Cargo.toml
- `ContentHash::from_hex()` for CLI input
- Commands: `put <path>`, `fetch <hash>` (metadata only), `get <hash> --output <path>` (full download), `delete <hash>`, `list`, `status`
- Single-node only (no networking); spawns its own actor set
- Follow `crates/node/src/main.rs` pattern
**ListSwarm Fan-Out**
- Currently delegates to `ListLocal`. Wire MetadataActor to query all peers and merge/deduplicate results by content hash.
**Automatic Remote GET Orchestration**
- Currently, remote GET requires manual orchestration (test harness or external driver reads chunks from remote BlobStore and feeds them to TransferActor).
- DatastoreNode needs to become stateful: detect local miss, query peers via `HandleFindObject`, spawn TransferActor, coordinate chunk pulling from the remote BlobStoreActor.
- This is the largest remaining architectural change for local functionality.
### Medium-Term
**Transport Integration (iroh/QUIC)**
- Wire `NetworkMessage` types to actual network transport.
- DatastoreNode gains peer management (`AddPeer`/`RemovePeer`) at the node level.
- Replace simulation-only direct actor addressing with network-routed messages.
- Framing: `[4-byte length (big-endian)][JSON payload]` over QUIC streams.
**Anti-Entropy / Repair**
- When a node comes back online, re-disseminate its metadata to peers.
- Periodic full-index comparison between peers to detect and repair drift.
**Active Chunk Pulling in TransferActor**
- `StartDownload` sends `GetChunkRequest` to the source node for each chunk.
- Currently passive (chunks fed externally); make it drive its own downloads.
**Parallel Chunk Fetching**
- TransferActor currently fetches sequentially. Add configurable concurrency (`max_concurrent_transfers` in config already exists).
### Longer-Term
**Browser API (WASM)**
- Expose `list_objects`, `get_object`, `put_object`, `delete_object`, `get_node_status` via WASM bindings.
- Use `InMemoryBackend` (or IndexedDB backend) in the browser.
- Coordinate with `crates/wasm/` for the in-browser swactor runtime.
**Binary Wire Format**
- Replace JSON serialization with bincode or msgpack for `GetChunkResponse` and other payload-heavy messages.
**Content-Defined Chunking (CDC)**
- Replace fixed-size chunking with FastCDC or similar for better cross-object deduplication.
- Transparent to the rest of the system -- only `chunk_blob()` changes; manifest format is the same.
**Streaming / Large File Support**
- Current `Put` takes `data: Vec<u8>` (entire blob in memory). For large files, add streaming chunking that reads from a `Read` source.
**Delete Propagation**
- Currently, delete is local only (by design). Add optional "tombstone dissemination" to remove entries from peers.
**Replication Factor Control**
- Currently, dissemination is epidemic (all peers get everything). Add configurable k-closest replication for the metadata DHT.
**IndexedDB Backend**
- Implement `StorageBackend` for browser IndexedDB for persistent storage in web contexts.
---
## Key Files
| File | Purpose |
|------|---------|
| `src/types.rs` | Core types: `ContentHash`, `ObjectEntry`, `ObjectManifest`, `ChunkRef`, `DatastoreConfig` |
| `src/messages.rs` | All inter-node and intra-node message types |
| `src/chunking.rs` | Pure chunking/reassembly functions |
| `src/storage/mod.rs` | `StorageBackend` trait + `FilesystemBackend` |
| `src/storage/in_memory.rs` | `InMemoryBackend` |
| `src/actors/blob_store.rs` | Chunk/manifest I/O actor |
| `src/actors/metadata.rs` | Object index, dissemination, GC orchestration |
| `src/actors/transfer.rs` | Ephemeral download actor |
| `src/actors/datastore_node.rs` | Coordinator/facade |
| `src/cli.rs` | CLI command type definitions (types only, no implementation) |
| `PROTOCOL.md` | Protocol specification with status annotations |
| `PROTOCOL_IMPLEMENTATION_PLAN.md` | Original 7-module implementation plan |
| `tests/common/mod.rs` | Shared test harness: DatastoreHarness, GcHarness, MultiNodeHarness, NodeHarness |

170
docs/diagrams/datastore_chunk_lifecycle.svg Normal file
View file

@ -0,0 +1,170 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1000 600" font-family="system-ui, sans-serif">
<defs>
<filter id="shadow" x="-4%" y="-4%" width="108%" height="108%">
<feDropShadow dx="1" dy="2" stdDeviation="3" flood-color="#000" flood-opacity="0.10"/>
</filter>
<marker id="arrow-gray" viewBox="0 0 10 7" refX="10" refY="3.5"
markerWidth="10" markerHeight="7" orient="auto-start-reverse" fill="#5f6368">
<polygon points="0 0, 10 3.5, 0 7"/>
</marker>
<marker id="arrow-green" viewBox="0 0 10 7" refX="10" refY="3.5"
markerWidth="10" markerHeight="7" orient="auto-start-reverse" fill="#34a853">
<polygon points="0 0, 10 3.5, 0 7"/>
</marker>
<marker id="arrow-red" viewBox="0 0 10 7" refX="10" refY="3.5"
markerWidth="10" markerHeight="7" orient="auto-start-reverse" fill="#ea4335">
<polygon points="0 0, 10 3.5, 0 7"/>
</marker>
</defs>
<!-- Background -->
<rect width="1000" height="600" rx="8" fill="#f8f9fb" stroke="#e0e3e8" stroke-width="1.5"/>
<!-- Title -->
<text x="500" y="42" text-anchor="middle" font-size="20" font-weight="bold" fill="#202124">Chunk Lifecycle</text>
<text x="500" y="62" text-anchor="middle" font-size="12" fill="#5f6368">chunking.rs &#xB7; storage.rs &#xB7; actors/blob_store.rs</text>
<!-- ═══════════════════════════════════════════════════════════ -->
<!-- Row 1: Creation -->
<!-- ═══════════════════════════════════════════════════════════ -->
<!-- Blob Data -->
<rect x="40" y="100" width="120" height="50" rx="12" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.8" filter="url(#shadow)"/>
<text x="100" y="125" text-anchor="middle" font-size="13" font-weight="600" fill="#1a73e8">Blob Data</text>
<text x="100" y="140" text-anchor="middle" font-size="10" fill="#5f6368">raw bytes</text>
<!-- Arrow -->
<line x1="160" y1="125" x2="210" y2="125" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
<!-- blake3 hash -->
<rect x="222" y="100" width="130" height="50" rx="12" fill="#fef7e0" stroke="#f9ab00" stroke-width="1.8" filter="url(#shadow)"/>
<text x="287" y="125" text-anchor="middle" font-size="13" font-weight="600" fill="#e37400">blake3 hash</text>
<text x="287" y="140" text-anchor="middle" font-size="10" fill="#5f6368">ContentHash</text>
<!-- Arrow -->
<line x1="352" y1="125" x2="405" y2="125" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
<!-- chunk_blob() -->
<rect x="417" y="100" width="140" height="50" rx="12" fill="#e6f4ea" stroke="#34a853" stroke-width="1.8" filter="url(#shadow)"/>
<text x="487" y="125" text-anchor="middle" font-size="13" font-weight="600" fill="#137333">chunk_blob()</text>
<text x="487" y="140" text-anchor="middle" font-size="10" fill="#5f6368">split by chunk_size</text>
<!-- Arrow fan-out -->
<line x1="557" y1="115" x2="620" y2="105" stroke="#34a853" stroke-width="1.5" marker-end="url(#arrow-green)"/>
<line x1="557" y1="125" x2="620" y2="125" stroke="#34a853" stroke-width="1.5" marker-end="url(#arrow-green)"/>
<line x1="557" y1="135" x2="620" y2="145" stroke="#34a853" stroke-width="1.5" marker-end="url(#arrow-green)"/>
<!-- ChunkRef boxes -->
<rect x="630" y="88" width="110" height="30" rx="6" fill="#e6f4ea" stroke="#34a853" stroke-width="1.2"/>
<text x="685" y="108" text-anchor="middle" font-size="11" font-weight="600" fill="#137333">ChunkRef[0]</text>
<rect x="630" y="122" width="110" height="30" rx="6" fill="#e6f4ea" stroke="#34a853" stroke-width="1.2"/>
<text x="685" y="142" text-anchor="middle" font-size="11" font-weight="600" fill="#137333">ChunkRef[1]</text>
<rect x="630" y="156" width="110" height="30" rx="6" fill="#e6f4ea" stroke="#34a853" stroke-width="1.2"/>
<text x="685" y="176" text-anchor="middle" font-size="11" font-weight="600" fill="#137333">ChunkRef[N]</text>
<!-- Brace → ObjectManifest -->
<line x1="740" y1="103" x2="770" y2="137" stroke="#5f6368" stroke-width="1"/>
<line x1="740" y1="137" x2="770" y2="137" stroke="#5f6368" stroke-width="1"/>
<line x1="740" y1="171" x2="770" y2="137" stroke="#5f6368" stroke-width="1"/>
<line x1="770" y1="137" x2="800" y2="137" stroke="#5f6368" stroke-width="1.5" marker-end="url(#arrow-gray)"/>
<rect x="812" y="112" width="155" height="50" rx="12" fill="#fef7e0" stroke="#f9ab00" stroke-width="1.8" filter="url(#shadow)"/>
<text x="889" y="137" text-anchor="middle" font-size="13" font-weight="600" fill="#e37400">ObjectManifest</text>
<text x="889" y="152" text-anchor="middle" font-size="10" fill="#5f6368">chunks + total_size</text>
<!-- ═══════════════════════════════════════════════════════════ -->
<!-- Row 2: Storage -->
<!-- ═══════════════════════════════════════════════════════════ -->
<text x="500" y="220" text-anchor="middle" font-size="14" font-weight="600" fill="#202124">Storage</text>
<!-- WriteChunk -->
<rect x="100" y="240" width="200" height="50" rx="12" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.8" filter="url(#shadow)"/>
<text x="200" y="260" text-anchor="middle" font-size="13" font-weight="600" fill="#1a73e8">BlobStoreMsg</text>
<text x="200" y="278" text-anchor="middle" font-size="10" fill="#5f6368">::WriteChunk { hash, data }</text>
<line x1="300" y1="265" x2="380" y2="265" stroke="#34a853" stroke-width="1.8" marker-end="url(#arrow-green)"/>
<!-- StorageBackend -->
<rect x="392" y="240" width="220" height="50" rx="12" fill="#e6f4ea" stroke="#34a853" stroke-width="1.8" filter="url(#shadow)"/>
<text x="502" y="260" text-anchor="middle" font-size="13" font-weight="600" fill="#137333">StorageBackend</text>
<text x="502" y="278" text-anchor="middle" font-size="10" fill="#5f6368">::write_chunk(hash, data)</text>
<line x1="612" y1="265" x2="690" y2="265" stroke="#34a853" stroke-width="1.8" marker-end="url(#arrow-green)"/>
<!-- Disk/Memory -->
<rect x="702" y="240" width="180" height="50" rx="12" fill="#f1f3f4" stroke="#9aa0a6" stroke-width="1.8" filter="url(#shadow)"/>
<text x="792" y="260" text-anchor="middle" font-size="13" font-weight="600" fill="#5f6368">Filesystem /</text>
<text x="792" y="278" text-anchor="middle" font-size="10" fill="#5f6368">InMemoryBackend</text>
<!-- ═══════════════════════════════════════════════════════════ -->
<!-- Row 3: Reference tracking -->
<!-- ═══════════════════════════════════════════════════════════ -->
<text x="500" y="340" text-anchor="middle" font-size="14" font-weight="600" fill="#202124">Reference Tracking</text>
<rect x="100" y="360" width="200" height="50" rx="12" fill="#fef7e0" stroke="#f9ab00" stroke-width="1.8" filter="url(#shadow)"/>
<text x="200" y="380" text-anchor="middle" font-size="13" font-weight="600" fill="#e37400">ObjectManifest</text>
<text x="200" y="398" text-anchor="middle" font-size="10" fill="#5f6368">references chunk hashes</text>
<line x1="300" y1="385" x2="380" y2="385" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
<text x="340" y="377" text-anchor="middle" font-size="10" fill="#5f6368">owns</text>
<rect x="392" y="360" width="220" height="50" rx="12" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.8" filter="url(#shadow)"/>
<text x="502" y="385" text-anchor="middle" font-size="13" font-weight="600" fill="#1a73e8">MetadataActor</text>
<text x="502" y="398" text-anchor="middle" font-size="10" fill="#5f6368">tracks all manifests</text>
<!-- ═══════════════════════════════════════════════════════════ -->
<!-- Row 4: GC -->
<!-- ═══════════════════════════════════════════════════════════ -->
<text x="500" y="460" text-anchor="middle" font-size="14" font-weight="600" fill="#202124">Garbage Collection</text>
<rect x="60" y="480" width="150" height="50" rx="12" fill="#fce8e6" stroke="#ea4335" stroke-width="1.8" filter="url(#shadow)"/>
<text x="135" y="505" text-anchor="middle" font-size="13" font-weight="600" fill="#c5221f">GcTick</text>
<text x="135" y="520" text-anchor="middle" font-size="10" fill="#5f6368">periodic trigger</text>
<line x1="210" y1="505" x2="280" y2="505" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
<rect x="292" y="480" width="200" height="50" rx="12" fill="#fef7e0" stroke="#f9ab00" stroke-width="1.8" filter="url(#shadow)"/>
<text x="392" y="500" text-anchor="middle" font-size="12" font-weight="600" fill="#e37400">Collect referenced</text>
<text x="392" y="518" text-anchor="middle" font-size="10" fill="#5f6368">hashes from manifests</text>
<line x1="492" y1="505" x2="560" y2="505" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
<rect x="572" y="480" width="200" height="50" rx="12" fill="#fce8e6" stroke="#ea4335" stroke-width="1.8" filter="url(#shadow)"/>
<text x="672" y="500" text-anchor="middle" font-size="12" font-weight="600" fill="#c5221f">GcUnreferenced</text>
<text x="672" y="518" text-anchor="middle" font-size="10" fill="#5f6368">delete orphan chunks</text>
<line x1="772" y1="505" x2="830" y2="505" stroke="#ea4335" stroke-width="1.8" marker-end="url(#arrow-red)"/>
<!-- Deleted -->
<rect x="842" y="488" width="100" height="34" rx="8" fill="#f1f3f4" stroke="#9aa0a6" stroke-width="1.5"/>
<text x="892" y="510" text-anchor="middle" font-size="12" font-weight="600" fill="#5f6368">Deleted</text>
<!-- ============================================ -->
<!-- Legend -->
<rect x="50" y="555" width="900" height="30" rx="6" fill="none" stroke="#e0e3e8" stroke-width="1"/>
<text x="70" y="575" font-size="11" font-weight="600" fill="#202124">Legend</text>
<line x1="130" y1="572" x2="160" y2="572" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
<text x="168" y="576" font-size="10" fill="#5f6368">Transition</text>
<line x1="260" y1="572" x2="290" y2="572" stroke="#34a853" stroke-width="1.8" marker-end="url(#arrow-green)"/>
<text x="298" y="576" font-size="10" fill="#34a853">Data flow</text>
<line x1="380" y1="572" x2="410" y2="572" stroke="#ea4335" stroke-width="1.8" marker-end="url(#arrow-red)"/>
<text x="418" y="576" font-size="10" fill="#ea4335">Deletion</text>
<rect x="510" y="564" width="40" height="16" rx="4" fill="#e8f0fe" stroke="#4285f4" stroke-width="1"/>
<text x="560" y="576" font-size="10" fill="#4285f4">Actor / Component</text>
<rect x="680" y="564" width="40" height="16" rx="4" fill="#fef7e0" stroke="#f9ab00" stroke-width="1"/>
<text x="730" y="576" font-size="10" fill="#e37400">Data type</text>
<rect x="810" y="564" width="40" height="16" rx="4" fill="#e6f4ea" stroke="#34a853" stroke-width="1"/>
<text x="860" y="576" font-size="10" fill="#34a853">Success state</text>
</svg>
Side by side

After

Width:  |  Height:  |  Size: 11 KiB

153
docs/diagrams/datastore_transfer.svg Normal file
View file

@ -0,0 +1,153 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1000 700" font-family="system-ui, sans-serif">
<defs>
<filter id="shadow" x="-4%" y="-4%" width="108%" height="108%">
<feDropShadow dx="1" dy="2" stdDeviation="3" flood-color="#000" flood-opacity="0.10"/>
</filter>
<marker id="arrow-gray" viewBox="0 0 10 7" refX="10" refY="3.5"
markerWidth="10" markerHeight="7" orient="auto-start-reverse" fill="#5f6368">
<polygon points="0 0, 10 3.5, 0 7"/>
</marker>
<marker id="arrow-green" viewBox="0 0 10 7" refX="10" refY="3.5"
markerWidth="10" markerHeight="7" orient="auto-start-reverse" fill="#34a853">
<polygon points="0 0, 10 3.5, 0 7"/>
</marker>
<marker id="arrow-red" viewBox="0 0 10 7" refX="10" refY="3.5"
markerWidth="10" markerHeight="7" orient="auto-start-reverse" fill="#ea4335">
<polygon points="0 0, 10 3.5, 0 7"/>
</marker>
<marker id="arrow-blue" viewBox="0 0 10 7" refX="10" refY="3.5"
markerWidth="10" markerHeight="7" orient="auto-start-reverse" fill="#4285f4">
<polygon points="0 0, 10 3.5, 0 7"/>
</marker>
</defs>
<!-- Background -->
<rect width="1000" height="700" rx="8" fill="#f8f9fb" stroke="#e0e3e8" stroke-width="1.5"/>
<!-- Title -->
<text x="500" y="42" text-anchor="middle" font-size="20" font-weight="bold" fill="#202124">Datastore Remote GET Transfer Flow</text>
<text x="500" y="62" text-anchor="middle" font-size="12" fill="#5f6368">api.rs:583-706 &#xB7; types.rs</text>
<!-- ── Client ──────────────────────────────────────────── -->
<rect x="40" y="100" width="130" height="50" rx="12" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.8" filter="url(#shadow)"/>
<text x="105" y="130" text-anchor="middle" font-size="14" font-weight="600" fill="#1a73e8">Client</text>
<!-- Arrow: Client → API Server -->
<line x1="170" y1="125" x2="240" y2="125" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
<text x="205" y="117" text-anchor="middle" font-size="10" fill="#5f6368">GET /api/data</text>
<!-- ── API Server ──────────────────────────────────────── -->
<rect x="252" y="100" width="150" height="50" rx="12" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.8" filter="url(#shadow)"/>
<text x="327" y="130" text-anchor="middle" font-size="14" font-weight="600" fill="#1a73e8">API Server</text>
<!-- Arrow: API Server → DatastoreNode -->
<line x1="402" y1="125" x2="470" y2="125" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
<text x="436" y="117" text-anchor="middle" font-size="10" fill="#5f6368">Get msg</text>
<!-- ── DatastoreNode ───────────────────────────────────── -->
<rect x="482" y="100" width="160" height="50" rx="12" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.8" filter="url(#shadow)"/>
<text x="562" y="130" text-anchor="middle" font-size="14" font-weight="600" fill="#1a73e8">DatastoreNode</text>
<!-- Arrow: DatastoreNode → MetadataActor -->
<line x1="562" y1="150" x2="562" y2="200" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
<text x="580" y="180" font-size="10" fill="#5f6368">lookup</text>
<!-- ── MetadataActor (local) ───────────────────────────── -->
<rect x="482" y="210" width="160" height="50" rx="12" fill="#fef7e0" stroke="#f9ab00" stroke-width="1.8" filter="url(#shadow)"/>
<text x="562" y="235" text-anchor="middle" font-size="13" font-weight="600" fill="#e37400">MetadataActor</text>
<text x="562" y="250" text-anchor="middle" font-size="10" fill="#5f6368">(local)</text>
<!-- Arrow: MetadataActor → Not Found (red, left) -->
<line x1="482" y1="235" x2="420" y2="235" stroke="#ea4335" stroke-width="1.8" marker-end="url(#arrow-red)"/>
<text x="450" y="227" text-anchor="middle" font-size="10" fill="#ea4335">Not Found</text>
<!-- ── try_remote_get box ──────────────────────────────── -->
<rect x="210" y="210" width="200" height="50" rx="12" fill="#fce8e6" stroke="#ea4335" stroke-width="1.5" filter="url(#shadow)" stroke-dasharray="5,3"/>
<text x="310" y="235" text-anchor="middle" font-size="12" font-weight="600" fill="#c5221f">try_remote_get()</text>
<text x="310" y="250" text-anchor="middle" font-size="10" fill="#5f6368">iterate peers</text>
<!-- Arrow: try_remote_get → Peer Metadata -->
<line x1="310" y1="260" x2="310" y2="320" stroke="#34a853" stroke-width="1.8" marker-end="url(#arrow-green)"/>
<text x="340" y="295" font-size="10" fill="#34a853">FindObject</text>
<!-- ── Peer MetadataActor ──────────────────────────────── -->
<rect x="210" y="330" width="200" height="50" rx="12" fill="#e6f4ea" stroke="#34a853" stroke-width="1.8" filter="url(#shadow)"/>
<text x="310" y="355" text-anchor="middle" font-size="13" font-weight="600" fill="#137333">Peer MetadataActor</text>
<text x="310" y="370" text-anchor="middle" font-size="10" fill="#5f6368">(remote node)</text>
<!-- Arrow: Peer Metadata → Peer BlobStore -->
<line x1="310" y1="380" x2="310" y2="430" stroke="#34a853" stroke-width="1.8" marker-end="url(#arrow-green)"/>
<text x="340" y="410" font-size="10" fill="#34a853">Found!</text>
<!-- ── Peer BlobStore ──────────────────────────────────── -->
<rect x="210" y="440" width="200" height="50" rx="12" fill="#e6f4ea" stroke="#34a853" stroke-width="1.8" filter="url(#shadow)"/>
<text x="310" y="465" text-anchor="middle" font-size="13" font-weight="600" fill="#137333">Peer BlobStore</text>
<text x="310" y="480" text-anchor="middle" font-size="10" fill="#5f6368">ReadChunk &#xD7; N</text>
<!-- Arrow fan-out: Peer BlobStore → Chunks -->
<line x1="230" y1="490" x2="150" y2="540" stroke="#4285f4" stroke-width="1.5" marker-end="url(#arrow-blue)"/>
<line x1="310" y1="490" x2="310" y2="540" stroke="#4285f4" stroke-width="1.5" marker-end="url(#arrow-blue)"/>
<line x1="390" y1="490" x2="470" y2="540" stroke="#4285f4" stroke-width="1.5" marker-end="url(#arrow-blue)"/>
<!-- Chunk boxes -->
<rect x="100" y="545" width="100" height="36" rx="8" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.2" filter="url(#shadow)"/>
<text x="150" y="568" text-anchor="middle" font-size="12" font-weight="600" fill="#1a73e8">Chunk 1</text>
<rect x="260" y="545" width="100" height="36" rx="8" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.2" filter="url(#shadow)"/>
<text x="310" y="568" text-anchor="middle" font-size="12" font-weight="600" fill="#1a73e8">Chunk 2</text>
<rect x="420" y="545" width="100" height="36" rx="8" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.2" filter="url(#shadow)"/>
<text x="470" y="568" text-anchor="middle" font-size="12" font-weight="600" fill="#1a73e8">Chunk N</text>
<!-- Arrows: Chunks → Local BlobStore -->
<line x1="150" y1="581" x2="560" y2="610" stroke="#4285f4" stroke-width="1.2" marker-end="url(#arrow-blue)"/>
<line x1="310" y1="581" x2="560" y2="610" stroke="#4285f4" stroke-width="1.2" marker-end="url(#arrow-blue)"/>
<line x1="470" y1="581" x2="560" y2="610" stroke="#4285f4" stroke-width="1.2" marker-end="url(#arrow-blue)"/>
<!-- ── Local BlobStore ─────────────────────────────────── -->
<rect x="510" y="600" width="180" height="50" rx="12" fill="#e6f4ea" stroke="#34a853" stroke-width="1.8" filter="url(#shadow)"/>
<text x="600" y="625" text-anchor="middle" font-size="13" font-weight="600" fill="#137333">Local BlobStore</text>
<text x="600" y="640" text-anchor="middle" font-size="10" fill="#5f6368">WriteChunk &#xD7; N</text>
<!-- Arrow: Local BlobStore → Reassemble -->
<line x1="690" y1="625" x2="740" y2="625" stroke="#34a853" stroke-width="1.8" marker-end="url(#arrow-green)"/>
<!-- ── Reassemble ──────────────────────────────────────── -->
<rect x="752" y="600" width="150" height="50" rx="12" fill="#e6f4ea" stroke="#34a853" stroke-width="1.8" filter="url(#shadow)"/>
<text x="827" y="625" text-anchor="middle" font-size="13" font-weight="600" fill="#137333">reassemble_blob</text>
<text x="827" y="640" text-anchor="middle" font-size="10" fill="#5f6368">verify integrity</text>
<!-- Arrow: Reassemble → Client (curved up) -->
<path d="M 902 620 Q 950 600 960 400 Q 960 200 180 130"
fill="none" stroke="#34a853" stroke-width="1.8" marker-end="url(#arrow-green)"/>
<text x="960" y="370" font-size="11" fill="#34a853" font-weight="600">200 OK</text>
<text x="960" y="385" font-size="10" fill="#34a853">blob data</text>
<!-- ── Dashboard integration ────────────────────────────── -->
<rect x="700" y="330" width="180" height="50" rx="12" fill="#f3e8fd" stroke="#9c27b0" stroke-width="1.5" filter="url(#shadow)"/>
<text x="790" y="350" text-anchor="middle" font-size="12" font-weight="600" fill="#7b1fa2">DatastoreMetrics</text>
<text x="790" y="367" text-anchor="middle" font-size="10" fill="#5f6368">begin/advance/end</text>
<line x1="410" y1="460" x2="700" y2="355" stroke="#9c27b0" stroke-width="1.2" stroke-dasharray="4,3" marker-end="url(#arrow-gray)"/>
<text x="570" y="395" font-size="9" fill="#9c27b0">transfer progress</text>
<!-- ============================================ -->
<!-- Legend -->
<rect x="50" y="660" width="900" height="30" rx="6" fill="none" stroke="#e0e3e8" stroke-width="1"/>
<text x="70" y="680" font-size="11" font-weight="600" fill="#202124">Legend</text>
<line x1="130" y1="677" x2="160" y2="677" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
<text x="168" y="681" font-size="10" fill="#5f6368">Transition</text>
<line x1="260" y1="677" x2="290" y2="677" stroke="#34a853" stroke-width="1.8" marker-end="url(#arrow-green)"/>
<text x="298" y="681" font-size="10" fill="#34a853">Success path</text>
<line x1="400" y1="677" x2="430" y2="677" stroke="#ea4335" stroke-width="1.8" marker-end="url(#arrow-red)"/>
<text x="438" y="681" font-size="10" fill="#ea4335">Not found / fallback</text>
<line x1="580" y1="677" x2="610" y2="677" stroke="#4285f4" stroke-width="1.5" marker-end="url(#arrow-blue)"/>
<text x="618" y="681" font-size="10" fill="#4285f4">Chunk transfer</text>
<line x1="720" y1="677" x2="750" y2="677" stroke="#9c27b0" stroke-width="1.2" stroke-dasharray="4,3"/>
<text x="758" y="681" font-size="10" fill="#9c27b0">Metrics tracking</text>
</svg>
Side by side

After

Width:  |  Height:  |  Size: 11 KiB

5
src/runtime.rs
View file

@ -265,11 +265,6 @@ impl Runtime {
/// Creates a temporary inbox, calls `msg_builder` with the inbox's address
/// (so you can embed it as `reply_to`), sends the message, and returns an
/// [`Ask`] handle for receiving the response.
///
/// ```ignore
/// let ask = rt.ask(actor, |reply_to| GetValue { reply_to })?;
/// let value = ask.recv_ticking(&rt, 10)?;
/// ```
pub fn ask<Req: Message, Resp: Message>(
&self,
addr: ActorAddress,

4
xtask/Cargo.toml Normal file
View file

@ -0,0 +1,4 @@
[package]
name = "xtask"
version = "0.1.0"
edition = "2024"

197
xtask/src/main.rs Normal file
View file

@ -0,0 +1,197 @@
use std::process::Command;
use std::time::Instant;
struct TestStep {
label: &'static str,
args: &'static [&'static str],
}
struct Group {
name: &'static str,
description: &'static str,
steps: &'static [TestStep],
}
const CORE: Group = Group {
name: "core",
description: "Actor runtime, message delivery, property tests",
steps: &[TestStep {
label: "actor runtime",
args: &["test", "-p", "swactor", "--features", "transport"],
}],
};
const DISTRIBUTION: Group = Group {
name: "distribution",
description: "Distribution protocol + datastore",
steps: &[
TestStep {
label: "distribution protocol",
args: &["test", "-p", "distribution"],
},
TestStep {
label: "datastore",
args: &["test", "-p", "swactor-datastore"],
},
],
};
const CLUSTER_SIMS: Group = Group {
name: "cluster-sims",
description: "Deterministic cluster simulations",
steps: &[TestStep {
label: "cluster simulations",
args: &["test", "-p", "simulation"],
}],
};
const INTEGRATED: Group = Group {
name: "integrated",
description: "HTTP API + dashboard end-to-end tests",
steps: &[
TestStep {
label: "datastore integration (node features)",
args: &[
"test",
"-p",
"swactor-datastore",
"--features",
"node",
"--test",
"api_integration_test",
"--test",
"dashboard_integration_test",
],
},
TestStep {
label: "runtime dashboard",
args: &["test", "-p", "runtime-dashboard"],
},
],
};
fn groups_for(name: &str) -> Option<Vec<&'static Group>> {
match name {
"core" => Some(vec![&CORE]),
"distribution" => Some(vec![&DISTRIBUTION]),
"cluster-sims" => Some(vec![&CLUSTER_SIMS]),
"integrated" => Some(vec![&INTEGRATED]),
"essential" => Some(vec![&CORE, &DISTRIBUTION, &INTEGRATED]),
"all" => Some(vec![&CORE, &DISTRIBUTION, &CLUSTER_SIMS, &INTEGRATED]),
_ => None,
}
}
fn run_step(group_name: &str, step: &TestStep) -> bool {
println!("\n=== {group_name}: {} ===", step.label);
println!(" cargo {}", step.args.join(" "));
println!();
let status = Command::new("cargo")
.args(step.args)
.status();
match status {
Ok(s) => s.success(),
Err(e) => {
eprintln!("Failed to execute cargo: {e}");
false
}
}
}
fn print_usage() {
println!(
"\
USAGE: cargo xtask test <GROUP>
GROUPS:
core Actor runtime, message delivery, property tests
distribution Distribution protocol + datastore
cluster-sims Deterministic cluster simulations
integrated HTTP API + dashboard end-to-end tests
essential core + distribution + integrated (merge gate)
all Every test group
FLAGS:
--list Show all groups and the cargo commands they run"
);
}
fn print_list() {
let all_groups: &[(&[&str], &Group)] = &[
(&[], &CORE),
(&[], &DISTRIBUTION),
(&[], &CLUSTER_SIMS),
(&[], &INTEGRATED),
];
println!("Available test groups:\n");
for &(_, group) in all_groups {
println!(" {:<14}{}", group.name, group.description);
for step in group.steps {
println!(" → cargo {}", step.args.join(" "));
}
println!();
}
println!(" {:<14}core + distribution + integrated (merge gate)", "essential");
println!(" {:<14}Every test group", "all");
}
fn main() {
let args: Vec<String> = std::env::args().collect();
if args.len() < 2 || args[1] != "test" {
print_usage();
std::process::exit(if args.len() < 2 { 1 } else { 1 });
}
if args.len() < 3 {
print_usage();
std::process::exit(1);
}
let target = &args[2];
if target == "--list" {
print_list();
return;
}
let groups = match groups_for(target) {
Some(g) => g,
None => {
eprintln!("Unknown test group: {target}\n");
print_usage();
std::process::exit(1);
}
};
let start = Instant::now();
let mut passed = 0usize;
let mut failed = 0usize;
for group in &groups {
for step in group.steps {
if run_step(group.name, step) {
passed += 1;
} else {
failed += 1;
let elapsed = start.elapsed();
println!(
"\n--- FAILED after {:.1}s ({passed} passed, {failed} failed) ---",
elapsed.as_secs_f64()
);
std::process::exit(1);
}
}
}
let elapsed = start.elapsed();
println!(
"\n--- All {passed} step(s) passed in {:.1}s ---",
elapsed.as_secs_f64()
);
}