//! Crash-consistent durable state for the virtual blob namespace. use std::collections::BTreeMap; use std::fmt; use std::fs::{self, File, OpenOptions}; use std::io::{Read, Write}; use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; use swactor::actor::ActorAddress; use crate::path::DataPath; const SCHEMA_VERSION: u32 = 1; #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct OperationId([u8; 16]); impl OperationId { pub const fn from_u128(value: u128) -> Self { Self(value.to_be_bytes()) } pub const fn bytes(self) -> [u8; 16] { self.0 } } impl Serialize for OperationId { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, { serializer.serialize_str(&format!("{:032x}", u128::from_be_bytes(self.0))) } } impl<'de> Deserialize<'de> for OperationId { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { let encoded = String::deserialize(deserializer)?; if encoded.len() != 32 { return Err(serde::de::Error::custom( "namespace operation ID must contain 32 hexadecimal digits", )); } u128::from_str_radix(&encoded, 16) .map(Self::from_u128) .map_err(serde::de::Error::custom) } } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum SourceRecovery { File { path: PathBuf }, Actor { actor: ActorAddress }, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct PersistedBinding { pub length: u64, pub revision: u64, pub recovery: SourceRecovery, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum MutationRequest { Register { path: DataPath, length: u64, recovery: SourceRecovery, }, Unregister { path: DataPath, }, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct MutationReceipt { pub revision: u64, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum MutationRejection { PathNotFound(DataPath), } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum PersistedMutationResult { Committed(MutationReceipt), Rejected(MutationRejection), } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct PersistedOperation { pub request: MutationRequest, pub result: PersistedMutationResult, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct NamespaceSnapshot { schema_version: u32, pub authority_epoch: u64, pub next_revision: u64, pub bindings: BTreeMap, pub operations: BTreeMap, #[serde(default)] pub retirements: Vec, } impl Default for NamespaceSnapshot { fn default() -> Self { Self { schema_version: SCHEMA_VERSION, authority_epoch: 0, next_revision: 1, bindings: BTreeMap::new(), operations: BTreeMap::new(), retirements: Vec::new(), } } } impl NamespaceSnapshot { fn validate(&self) -> Result<(), NamespaceStoreError> { if self.schema_version != SCHEMA_VERSION { return Err(NamespaceStoreError::UnsupportedSchema { found: self.schema_version, supported: SCHEMA_VERSION, }); } if self.next_revision == 0 { return Err(NamespaceStoreError::Corrupt( "next namespace revision is zero".to_owned(), )); } if self .bindings .values() .any(|binding| binding.revision == 0 || binding.revision >= self.next_revision) { return Err(NamespaceStoreError::Corrupt( "binding revision is outside the committed revision range".to_owned(), )); } if self.operations.values().any(|operation| { matches!( operation.result, PersistedMutationResult::Committed(receipt) if receipt.revision == 0 || receipt.revision >= self.next_revision ) }) { return Err(NamespaceStoreError::Corrupt( "operation revision is outside the committed revision range".to_owned(), )); } Ok(()) } } #[derive(Clone, Debug, PartialEq, Eq)] pub enum NamespaceStoreError { Io(String), Corrupt(String), UnsupportedSchema { found: u32, supported: u32 }, RevisionExhausted, AuthorityEpochExhausted, } impl fmt::Display for NamespaceStoreError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Io(reason) => write!(f, "namespace storage I/O failed: {reason}"), Self::Corrupt(reason) => write!(f, "namespace storage is corrupt: {reason}"), Self::UnsupportedSchema { found, supported } => write!( f, "unsupported namespace schema {found}; supported schema is {supported}" ), Self::RevisionExhausted => f.write_str("namespace revision exhausted"), Self::AuthorityEpochExhausted => f.write_str("namespace authority epoch exhausted"), } } } impl std::error::Error for NamespaceStoreError {} impl From for NamespaceStoreError { fn from(error: std::io::Error) -> Self { Self::Io(error.to_string()) } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum CommitFailpoint { None, BeforeRename, AfterRename, } pub struct NamespaceStore { path: PathBuf, snapshot: NamespaceSnapshot, } impl NamespaceStore { pub fn open(path: impl AsRef) -> Result { let path = path.as_ref().to_path_buf(); let snapshot = match File::open(&path) { Ok(mut file) => { let mut bytes = Vec::new(); file.read_to_end(&mut bytes)?; let snapshot: NamespaceSnapshot = serde_json::from_slice(&bytes) .map_err(|error| NamespaceStoreError::Corrupt(error.to_string()))?; snapshot.validate()?; snapshot } Err(error) if error.kind() == std::io::ErrorKind::NotFound => { NamespaceSnapshot::default() } Err(error) => return Err(error.into()), }; Ok(Self { path, snapshot }) } pub fn snapshot(&self) -> &NamespaceSnapshot { &self.snapshot } pub fn advance_authority_epoch(&mut self) -> Result { let mut next = self.snapshot.clone(); next.authority_epoch = next .authority_epoch .checked_add(1) .filter(|epoch| *epoch != 0) .ok_or(NamespaceStoreError::AuthorityEpochExhausted)?; self.commit(next)?; Ok(self.snapshot.authority_epoch) } pub fn commit(&mut self, next: NamespaceSnapshot) -> Result<(), NamespaceStoreError> { next.validate()?; self.persist(&next)?; self.snapshot = next; Ok(()) } fn persist(&self, snapshot: &NamespaceSnapshot) -> Result<(), NamespaceStoreError> { self.persist_with_failpoint(snapshot, CommitFailpoint::None) } fn persist_with_failpoint( &self, snapshot: &NamespaceSnapshot, failpoint: CommitFailpoint, ) -> Result<(), NamespaceStoreError> { let parent = self.path.parent().ok_or_else(|| { NamespaceStoreError::Io("namespace state path has no parent".to_owned()) })?; fs::create_dir_all(parent)?; let file_name = self.path.file_name().ok_or_else(|| { NamespaceStoreError::Io("namespace state path has no file name".to_owned()) })?; let temporary = parent.join(format!(".{}.tmp", file_name.to_string_lossy())); let bytes = serde_json::to_vec(snapshot) .map_err(|error| NamespaceStoreError::Corrupt(error.to_string()))?; let write_result = (|| -> Result<(), NamespaceStoreError> { let mut file = OpenOptions::new() .create(true) .truncate(true) .write(true) .open(&temporary)?; file.write_all(&bytes)?; file.sync_all()?; if failpoint == CommitFailpoint::BeforeRename { return Err(NamespaceStoreError::Io( "injected failure before durable replacement".to_owned(), )); } fs::rename(&temporary, &self.path)?; File::open(parent)?.sync_all()?; if failpoint == CommitFailpoint::AfterRename { return Err(NamespaceStoreError::Io( "injected crash after durable replacement".to_owned(), )); } Ok(()) })(); if write_result.is_err() { let _ = fs::remove_file(&temporary); } write_result } #[cfg(test)] fn commit_with_failpoint( &mut self, next: NamespaceSnapshot, failpoint: CommitFailpoint, ) -> Result<(), NamespaceStoreError> { next.validate()?; self.persist_with_failpoint(&next, failpoint)?; self.snapshot = next; Ok(()) } } #[cfg(test)] mod tests { use std::sync::atomic::{AtomicU64, Ordering}; use super::*; static NEXT_TEMP: AtomicU64 = AtomicU64::new(1); struct TestState { root: PathBuf, file: PathBuf, } impl TestState { fn new() -> Self { let sequence = NEXT_TEMP.fetch_add(1, Ordering::Relaxed); let root = std::env::temp_dir().join(format!( "swactor-namespace-store-{}-{sequence}", std::process::id() )); fs::create_dir_all(&root).expect("create test state directory"); let file = root.join("namespace.json"); Self { root, file } } } impl Drop for TestState { fn drop(&mut self) { let _ = fs::remove_dir_all(&self.root); } } fn state_with_binding(store: &NamespaceStore) -> NamespaceSnapshot { let mut next = store.snapshot().clone(); next.bindings.insert( DataPath::parse("/models/a").unwrap(), PersistedBinding { length: 8, revision: 1, recovery: SourceRecovery::Actor { actor: ActorAddress([7; 32]), }, }, ); next.next_revision = 2; next } #[test] fn failure_before_rename_preserves_previous_snapshot() { let state = TestState::new(); let mut store = NamespaceStore::open(&state.file).unwrap(); store.advance_authority_epoch().unwrap(); let next = state_with_binding(&store); assert!( store .commit_with_failpoint(next, CommitFailpoint::BeforeRename) .is_err() ); let reopened = NamespaceStore::open(&state.file).unwrap(); assert!(reopened.snapshot().bindings.is_empty()); } #[test] fn crash_after_rename_recovers_new_snapshot() { let state = TestState::new(); let mut store = NamespaceStore::open(&state.file).unwrap(); store.advance_authority_epoch().unwrap(); let next = state_with_binding(&store); assert!( store .commit_with_failpoint(next, CommitFailpoint::AfterRename) .is_err() ); let reopened = NamespaceStore::open(&state.file).unwrap(); assert_eq!( reopened .snapshot() .bindings .get(&DataPath::parse("/models/a").unwrap()) .unwrap() .length, 8 ); } #[test] fn malformed_snapshot_is_rejected() { let state = TestState::new(); fs::write(&state.file, b"{not-json").unwrap(); assert!(matches!( NamespaceStore::open(&state.file), Err(NamespaceStoreError::Corrupt(_)) )); } }