diff --git a/Cargo.lock b/Cargo.lock index af1646d..1e3e6e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -947,6 +947,7 @@ dependencies = [ name = "data-plane" version = "0.1.0" dependencies = [ + "criterion", "futures-lite", "libc", "parking_lot", diff --git a/crates/bindings/python/src/job.rs b/crates/bindings/python/src/job.rs index 919ffe3..ca85d92 100644 --- a/crates/bindings/python/src/job.rs +++ b/crates/bindings/python/src/job.rs @@ -9,11 +9,13 @@ use std::time::Duration; use data_plane::blob::{Blob, BlobView, ContentDigest, WritableArenaView}; use data_plane::bootstrap as dp_bootstrap; use data_plane::data_plane::{ - BlobWriter, DataPlane, DataPlaneBootstrap, StreamReader, StreamWriter, parse_actor_address, + BlobWriter, DataPlane, DataPlaneBootstrap, Descriptor, DescriptorMapping, MapRequest, + MapTarget, Protection, Sharing, StreamReader, StreamWriter, parse_actor_address, }; use data_plane::path::DataPath; use data_plane::protocol::{ - BlobFailure, DataPlaneError, JobCapability, register_data_plane_codecs, + AccessMode, BlobAllocation, BlobFailure, DataPlaneError, DescriptorKind, Errno, JobCapability, + OpenOptions, register_data_plane_codecs, }; use distribution::node::DistributedNodeConfig; use distribution::transport_bridge::{ @@ -23,7 +25,8 @@ use futures_lite::future; use iroh::{EndpointAddr, RelayMode}; use iroh_driver::{IrohDriver, IrohDriverConfig}; use parking_lot::Mutex as ParkingMutex; -use pyo3::exceptions::{PyBufferError, PyPermissionError, PyRuntimeError}; +use pyo3::buffer::PyBuffer; +use pyo3::exceptions::{PyBufferError, PyOSError, PyPermissionError, PyRuntimeError, PyValueError}; use pyo3::ffi; use pyo3::prelude::*; use pyo3::types::{PyAny, PyBytes, PyModule}; @@ -36,6 +39,44 @@ use swactor_transport::{CodecRegistry, CodecRemoteSink, TransportRouter}; const ROUTE_POLL: Duration = Duration::from_millis(5); const ROUTE_DEADLINE: Duration = Duration::from_secs(5); +const O_RDONLY: i32 = 0; +const O_WRONLY: i32 = 1; +const O_RDWR: i32 = 2; +const O_ACCMODE: i32 = 3; +const O_CREAT: i32 = 0o100; +const O_EXCL: i32 = 0o200; +const O_TRUNC: i32 = 0o1000; +const O_NONBLOCK: i32 = 0o4000; +const KNOWN_OPEN_FLAGS: i32 = O_ACCMODE | O_CREAT | O_EXCL | O_TRUNC | O_NONBLOCK; + +fn python_open_options(flags: i32, length: Option) -> PyResult { + if flags & !KNOWN_OPEN_FLAGS != 0 { + return Err(PyValueError::new_err(format!( + "unsupported descriptor open flags: {:#x}", + flags & !KNOWN_OPEN_FLAGS + ))); + } + let access = match flags & O_ACCMODE { + O_RDONLY => AccessMode::ReadOnly, + O_WRONLY => AccessMode::WriteOnly, + O_RDWR => AccessMode::ReadWrite, + _ => return Err(PyValueError::new_err("invalid descriptor access mode")), + }; + let options = OpenOptions { + access, + create: flags & O_CREAT != 0, + exclusive: flags & O_EXCL != 0, + truncate: flags & O_TRUNC != 0, + nonblocking: flags & O_NONBLOCK != 0, + allocation: length.map(|length| BlobAllocation { + length, + digest: None, + }), + }; + options.validate().map_err(raw_data_plane_error)?; + Ok(options) +} + pyo3::create_exception!(swactor, SwactorError, pyo3::exceptions::PyException); pyo3::create_exception!(swactor, BootstrapError, SwactorError); pyo3::create_exception!(swactor, DataPathError, SwactorError); @@ -50,8 +91,8 @@ fn bootstrap_error(message: impl Into) -> PyErr { fn data_plane_error(error: DataPlaneError) -> PyErr { match error { DataPlaneError::InvalidPath(reason) => PyErr::new::(reason), - DataPlaneError::Unauthorized { path, operation } => { - PyPermissionError::new_err(format!("{operation:?} is not authorized for {path}")) + DataPlaneError::Unauthorized { path, access } => { + PyPermissionError::new_err(format!("{access:?} is not authorized for {path}")) } DataPlaneError::PathNotFound(path) => { PyErr::new::(format!("data path not found: {path}")) @@ -69,6 +110,29 @@ fn data_plane_error(error: DataPlaneError) -> PyErr { } } +fn raw_data_plane_error(error: DataPlaneError) -> PyErr { + let code = match error.errno() { + Errno::Eacces => libc::EACCES, + Errno::Eagain => libc::EAGAIN, + Errno::Ebadf => libc::EBADF, + Errno::Ebusy => libc::EBUSY, + Errno::Ecanceled => libc::ECANCELED, + Errno::Econnreset => libc::ECONNRESET, + Errno::Eexist => libc::EEXIST, + Errno::Einval => libc::EINVAL, + Errno::Eio => libc::EIO, + Errno::Enodev => libc::ENODEV, + Errno::Enoent => libc::ENOENT, + Errno::Enomem => libc::ENOMEM, + Errno::Enospc => libc::ENOSPC, + Errno::Enotsup => libc::ENOTSUP, + Errno::Enxio => libc::ENXIO, + Errno::Epipe => libc::EPIPE, + Errno::Estale => libc::ESTALE, + }; + PyOSError::new_err((code, error.to_string())) +} + fn bootstrap_env(name: &str) -> PyResult { std::env::var(name) .map_err(|_| bootstrap_error(format!("bootstrap environment variable {name} is not set"))) @@ -239,6 +303,39 @@ pub struct PyDataPlane { #[pymethods] impl PyDataPlane { + #[pyo3(signature = (path, flags, *, length = None))] + fn open<'py>( + &self, + py: Python<'py>, + path: String, + flags: i32, + length: Option, + ) -> PyResult> { + let path = DataPath::parse(path).map_err(|error| { + raw_data_plane_error(DataPlaneError::InvalidPath(error.to_string())) + })?; + let options = python_open_options(flags, length)?; + let data_plane = self.inner.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let descriptor = data_plane + .open(&path, options) + .await + .map_err(raw_data_plane_error)?; + let kind = descriptor.kind(); + let capabilities = descriptor.capabilities().bits(); + Python::with_gil(|py| { + Py::new( + py, + PyDescriptor { + descriptor: Arc::new(tokio::sync::Mutex::new(descriptor)), + kind, + capabilities, + }, + ) + }) + }) + } + fn read_blob<'py>(&self, py: Python<'py>, path: String) -> PyResult> { let data_plane = self.inner.clone(); pyo3_async_runtimes::tokio::future_into_py(py, async move { @@ -294,6 +391,258 @@ impl PyDataPlane { } } +fn byte_buffer(object: &Bound<'_, PyAny>, writable: bool) -> PyResult> { + let buffer = PyBuffer::::get(object)?; + if writable && buffer.readonly() { + return Err(PyBufferError::new_err("buffer is read-only")); + } + if !buffer.is_c_contiguous() { + return Err(PyBufferError::new_err("buffer is not C-contiguous")); + } + Ok(buffer) +} + +unsafe fn mutable_buffer_bytes<'a>(buffer: &'a PyBuffer) -> &'a mut [u8] { + // SAFETY: `byte_buffer` checked writability and contiguity, and the + // retained `PyBuffer` keeps the exporter and pointer valid. + unsafe { std::slice::from_raw_parts_mut(buffer.buf_ptr().cast(), buffer.len_bytes()) } +} + +unsafe fn buffer_bytes<'a>(buffer: &'a PyBuffer) -> &'a [u8] { + // SAFETY: `byte_buffer` checked contiguity and retains the exporter. + unsafe { std::slice::from_raw_parts(buffer.buf_ptr().cast_const().cast(), buffer.len_bytes()) } +} + +#[pyclass(name = "Descriptor")] +pub struct PyDescriptor { + descriptor: Arc>, + kind: DescriptorKind, + capabilities: u16, +} + +#[pymethods] +impl PyDescriptor { + #[getter] + fn kind(&self) -> &'static str { + match self.kind { + DescriptorKind::Blob => "blob", + DescriptorKind::Stream => "stream", + } + } + + #[getter] + fn capabilities(&self) -> u16 { + self.capabilities + } + + fn read<'py>(&self, py: Python<'py>, size: usize) -> PyResult> { + let descriptor = self.descriptor.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let mut bytes = vec![0_u8; size]; + let count = descriptor + .lock() + .await + .read(&mut bytes) + .await + .map_err(raw_data_plane_error)?; + bytes.truncate(count); + Python::with_gil(|py| Ok(PyBytes::new(py, &bytes).unbind())) + }) + } + + fn readinto<'py>( + &self, + py: Python<'py>, + buffer: &Bound<'_, PyAny>, + ) -> PyResult> { + let buffer = byte_buffer(buffer, true)?; + let descriptor = self.descriptor.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + // SAFETY: the owned `PyBuffer` remains alive through completion or + // cancellation and no pointer is retained by the Rust primitive. + let bytes = unsafe { mutable_buffer_bytes(&buffer) }; + descriptor + .lock() + .await + .read(bytes) + .await + .map_err(raw_data_plane_error) + }) + } + + fn write<'py>(&self, py: Python<'py>, bytes: Vec) -> PyResult> { + let descriptor = self.descriptor.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + descriptor + .lock() + .await + .write(&bytes) + .await + .map_err(raw_data_plane_error) + }) + } + + fn writefrom<'py>( + &self, + py: Python<'py>, + buffer: &Bound<'_, PyAny>, + ) -> PyResult> { + let buffer = byte_buffer(buffer, false)?; + let descriptor = self.descriptor.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + // SAFETY: the owned `PyBuffer` retains a contiguous exporter until + // the descriptor primitive completes or is cancelled. + let bytes = unsafe { buffer_bytes(&buffer) }; + descriptor + .lock() + .await + .write(bytes) + .await + .map_err(raw_data_plane_error) + }) + } + + #[pyo3(signature = (*, offset = 0, length, writable = false))] + fn map<'py>( + &self, + py: Python<'py>, + offset: u64, + length: u64, + writable: bool, + ) -> PyResult> { + let descriptor = self.descriptor.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let mapping = descriptor + .lock() + .await + .map(MapRequest { + protection: if writable { + Protection::ReadWrite + } else { + Protection::Read + }, + sharing: Sharing::Shared, + target: MapTarget::Host, + offset, + length, + }) + .map_err(raw_data_plane_error)?; + Python::with_gil(|py| { + Py::new( + py, + PyDescriptorMapping { + inner: Some(mapping), + exports: 0, + }, + ) + }) + }) + } + + fn close<'py>(&self, py: Python<'py>) -> PyResult> { + let descriptor = self.descriptor.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + descriptor + .lock() + .await + .close() + .await + .map_err(raw_data_plane_error) + }) + } + + fn abort<'py>(&self, py: Python<'py>) -> PyResult> { + let descriptor = self.descriptor.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + descriptor + .lock() + .await + .abort() + .await + .map_err(raw_data_plane_error) + }) + } +} + +#[pyclass(name = "DescriptorMapping")] +pub struct PyDescriptorMapping { + inner: Option, + exports: usize, +} + +#[pymethods] +impl PyDescriptorMapping { + fn __enter__(slf: PyRef<'_, Self>) -> PyResult> { + if slf.inner.is_none() { + return Err(PyBufferError::new_err("descriptor mapping is closed")); + } + Ok(slf) + } + + fn __exit__( + &mut self, + _exception_type: &Bound<'_, PyAny>, + _exception: &Bound<'_, PyAny>, + _traceback: &Bound<'_, PyAny>, + ) -> PyResult { + self.close()?; + Ok(false) + } + + fn close(&mut self) -> PyResult<()> { + if self.exports != 0 { + return Err(PyBufferError::new_err( + "cannot close a descriptor mapping with active buffer exports", + )); + } + self.inner.take(); + Ok(()) + } + + unsafe fn __getbuffer__( + slf: Bound<'_, Self>, + view: *mut ffi::Py_buffer, + flags: c_int, + ) -> PyResult<()> { + let (pointer, length, readonly) = { + let mut borrowed = slf.borrow_mut(); + let inner = borrowed + .inner + .as_mut() + .ok_or_else(|| PyBufferError::new_err("descriptor mapping is closed"))?; + match inner { + DescriptorMapping::ReadOnly(mapping) => { + (mapping.as_ptr().cast_mut(), mapping.len(), true) + } + DescriptorMapping::WritableReadOnly(mapping) => { + (mapping.as_ptr(), mapping.len(), true) + } + DescriptorMapping::Writable(mapping) => (mapping.as_ptr(), mapping.len(), false), + } + }; + // SAFETY: the mapping owns the stable arena lease and the Python + // buffer retains `slf` until release. + unsafe { + fill_buffer( + view, + flags, + pointer, + length, + readonly, + slf.clone().into_any(), + ) + }?; + slf.borrow_mut().exports += 1; + Ok(()) + } + + unsafe fn __releasebuffer__(&mut self, view: *mut ffi::Py_buffer) { + self.exports = self.exports.saturating_sub(1); + // SAFETY: `fill_buffer` allocated the format string for this export. + unsafe { release_buffer_format(view) }; + } +} + #[pyclass(name = "Blob")] pub struct PyBlob { inner: Blob, @@ -651,6 +1000,26 @@ impl PyStreamReader { Python::with_gil(|py| Ok(bytes.map(|bytes| PyBytes::new(py, &bytes).unbind()))) }) } + + fn readinto<'py>( + &self, + py: Python<'py>, + buffer: &Bound<'_, PyAny>, + ) -> PyResult> { + let buffer = byte_buffer(buffer, true)?; + let reader = self.reader.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + // SAFETY: the owned `PyBuffer` retains the writable exporter for + // the complete async operation and cancellation path. + let bytes = unsafe { mutable_buffer_bytes(&buffer) }; + reader + .lock() + .await + .read_into(bytes) + .await + .map_err(data_plane_error) + }) + } } #[pyclass(name = "StreamWriter")] @@ -1208,7 +1577,16 @@ pub fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add("BlobError", module.py().get_type::())?; module.add("SessionError", module.py().get_type::())?; module.add("StreamError", module.py().get_type::())?; + module.add("O_RDONLY", O_RDONLY)?; + module.add("O_WRONLY", O_WRONLY)?; + module.add("O_RDWR", O_RDWR)?; + module.add("O_CREAT", O_CREAT)?; + module.add("O_EXCL", O_EXCL)?; + module.add("O_TRUNC", O_TRUNC)?; + module.add("O_NONBLOCK", O_NONBLOCK)?; module.add_class::()?; + module.add_class::()?; + module.add_class::()?; module.add_class::()?; module.add_class::()?; module.add_class::()?; diff --git a/crates/bindings/python/tests/test_bootstrap.py b/crates/bindings/python/tests/test_bootstrap.py index 15bf055..7b9dae0 100644 --- a/crates/bindings/python/tests/test_bootstrap.py +++ b/crates/bindings/python/tests/test_bootstrap.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio import ctypes +import errno import json import os import struct @@ -150,7 +151,8 @@ def test_run_attaches_before_main_and_maps_blob_buffer_directly(monkeypatch, hos _PY_RELEASE_BUFFER(ctypes.byref(exported)) public = {name for name in dir(ctx.data) if not name.startswith("_")} - assert {"read_blob", "write_blob", "read_stream", "write_stream"} <= public + assert {"open", "read_blob", "write_blob", "read_stream", "write_stream"} <= public + assert isinstance(swactor.O_RDONLY, int) assert not public & { "actor", "actor_id", @@ -201,6 +203,54 @@ def test_write_blob_seals_cleanly_and_exception_aborts(monkeypatch, host): swactor.run(main) +def test_raw_descriptor_blob_io_mapping_and_errno(monkeypatch, host): + install_host_env(monkeypatch, host) + + async def main(ctx): + logical = "/runs/self/results/raw-python" + writer = await ctx.data.open( + logical, + swactor.O_WRONLY | swactor.O_CREAT | swactor.O_TRUNC, + length=8, + ) + assert isinstance(writer, swactor.Descriptor) + assert writer.kind == "blob" + assert await writer.write(b"abc") == 3 + assert await writer.writefrom(b"defgh") == 5 + await writer.close() + with pytest.raises(OSError) as closed: + await writer.close() + assert closed.value.errno == errno.EBADF + + reader = await ctx.data.open(logical, swactor.O_RDONLY) + destination = bytearray(b"\xa5" * 10) + assert await reader.readinto(memoryview(destination)[1:7]) == 6 + assert destination == b"\xa5abcdef\xa5\xa5\xa5" + assert await reader.read(8) == b"gh" + assert await reader.read(8) == b"" + with pytest.raises(OSError) as wrong_access: + await reader.write(b"x") + assert wrong_access.value.errno == errno.EBADF + + mapping = await reader.map(length=8) + assert isinstance(mapping, swactor.DescriptorMapping) + exported = memoryview(mapping) + assert exported.readonly + await reader.close() + assert bytes(exported) == b"abcdefgh" + with pytest.raises(BufferError, match="active buffer exports"): + mapping.close() + exported.release() + mapping.close() + + with pytest.raises(FileNotFoundError): + await ctx.data.open("/models/raw-missing", swactor.O_RDONLY) + with pytest.raises(OSError) as unsupported: + await ctx.data.open(logical, swactor.O_RDONLY | swactor.O_NONBLOCK) + assert unsupported.value.errno in {errno.ENOTSUP, errno.EOPNOTSUPP} + + swactor.run(main) + def test_missing_path_and_authorization_are_typed(monkeypatch, host): install_host_env(monkeypatch, host) @@ -217,14 +267,16 @@ def test_missing_path_and_authorization_are_typed(monkeypatch, host): def test_native_stream_round_trip_has_ordered_eof(monkeypatch, host): install_host_env(monkeypatch, host) received = [] + raw_received = [] async def main(ctx): async def receive(): reader = await ctx.data.read_stream( "/runs/self/results/predictions" ) - while (chunk := await reader.read()) is not None: - received.append(chunk) + buffer = bytearray(3) + while (count := await reader.readinto(buffer)) != 0: + received.append(bytes(buffer[:count])) receiver = asyncio.create_task(receive()) async with ctx.data.write_stream( @@ -234,8 +286,20 @@ def test_native_stream_round_trip_has_ordered_eof(monkeypatch, host): await stream.write(b"result") await receiver + raw_reader, raw_writer = await asyncio.gather( + ctx.data.open("/runs/self/results/predictions", swactor.O_RDONLY), + ctx.data.open("/runs/self/results/predictions", swactor.O_WRONLY), + ) + assert await raw_writer.write(b"raw-stream") == len(b"raw-stream") + await raw_writer.close() + buffer = bytearray(4) + while (count := await raw_reader.readinto(buffer)) != 0: + raw_received.append(bytes(buffer[:count])) + await raw_reader.close() + swactor.run(main) assert b"".join(received) == b"native-result" + assert b"".join(raw_received) == b"raw-stream" assert "SWACTOR_DATA_PLANE_OUTPUT" not in host.env() diff --git a/crates/data-plane/Cargo.toml b/crates/data-plane/Cargo.toml index 32509bf..c060896 100644 --- a/crates/data-plane/Cargo.toml +++ b/crates/data-plane/Cargo.toml @@ -18,9 +18,15 @@ parking_lot = "0.12" [dev-dependencies] parking_lot = "0.12" +criterion = { version = "0.5", default-features = false } futures-lite = "2" proptest = "1" tokio.workspace = true + +[[bench]] +name = "descriptor_performance" +harness = false + [target.'cfg(target_os = "linux")'.dependencies] libc = "0.2" diff --git a/crates/data-plane/benches/descriptor_performance.rs b/crates/data-plane/benches/descriptor_performance.rs new file mode 100755 index 0000000..4275a99 --- /dev/null +++ b/crates/data-plane/benches/descriptor_performance.rs @@ -0,0 +1,81 @@ +#![cfg(target_os = "linux")] + +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use data_plane::arena::{ArenaConfig, ArenaManager, NodeId}; +use data_plane::byte_ring::{ByteRingSpec, Endpoint, RecordKind, Role, attach, install}; + +const PAYLOAD_LEN: usize = 64 * 1024; + +fn endpoints() -> (ArenaManager, Endpoint, Endpoint) { + let mut arena = ArenaManager::boot(ArenaConfig { + node_id: NodeId(91), + reservation_ceiling: 2 << 20, + base_alignment: 64, + }) + .expect("benchmark arena"); + let handle = install( + &mut arena, + ByteRingSpec { + capacity: (PAYLOAD_LEN + 5) as u64, + generation: 1, + alignment: 64, + request_id: 1, + }, + ) + .expect("benchmark ring"); + let producer = attach(&arena, handle, Role::Producer).expect("benchmark producer"); + let consumer = attach(&arena, handle, Role::Consumer).expect("benchmark consumer"); + (arena, producer, consumer) +} + +fn descriptor_stream_throughput(criterion: &mut Criterion) { + let payload = vec![0x5a_u8; PAYLOAD_LEN]; + let mut destination = vec![0_u8; PAYLOAD_LEN]; + let (_arena, mut producer, mut consumer) = endpoints(); + let mut group = criterion.benchmark_group("descriptor_stream_read"); + group.throughput(Throughput::Bytes(PAYLOAD_LEN as u64)); + + group.bench_with_input( + BenchmarkId::new("partial_allocation_free", PAYLOAD_LEN), + &PAYLOAD_LEN, + |bencher, _| { + bencher.iter(|| { + producer + .send_record(RecordKind::Data, &payload) + .expect("publish payload"); + let cursor = consumer + .record_cursor() + .expect("read cursor") + .expect("committed record"); + let count = consumer + .copy_record_range(cursor, 0, &mut destination) + .expect("copy payload"); + consumer + .release_record_cursor(cursor) + .expect("release payload"); + criterion::black_box(count) + }); + }, + ); + + group.bench_with_input( + BenchmarkId::new("legacy_allocating_record", PAYLOAD_LEN), + &PAYLOAD_LEN, + |bencher, _| { + bencher.iter(|| { + producer + .send_record(RecordKind::Data, &payload) + .expect("publish payload"); + let (_, bytes) = consumer + .recv_record() + .expect("receive payload") + .expect("committed record"); + criterion::black_box(bytes) + }); + }, + ); + group.finish(); +} + +criterion_group!(benches, descriptor_stream_throughput); +criterion_main!(benches); diff --git a/crates/data-plane/src/blob.rs b/crates/data-plane/src/blob.rs index 31b5519..e4ee785 100644 --- a/crates/data-plane/src/blob.rs +++ b/crates/data-plane/src/blob.rs @@ -7,6 +7,8 @@ use std::ptr::NonNull; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use parking_lot::Mutex; + use serde::{Deserialize, Serialize}; use sha2::{Digest as _, Sha256}; @@ -251,27 +253,60 @@ impl Blob { self.guard.lease } - pub fn map(&self) -> Result { + pub(crate) fn copy_at(&self, offset: u64, destination: &mut [u8]) -> Result { + if offset >= self.metadata.length || destination.is_empty() { + return Ok(0); + } + let count = destination + .len() + .min(usize::try_from(self.metadata.length - offset).unwrap_or(usize::MAX)); + let view = self.map_range(offset, count as u64)?; + destination[..count].copy_from_slice(view.as_ref()); + Ok(count) + } + + pub fn map_range(&self, offset: u64, length: u64) -> Result { + let end = offset + .checked_add(length) + .ok_or(BlobError::RangeOutOfBounds { + offset, + length, + arena_size: self.metadata.length, + })?; + if end > self.metadata.length { + return Err(BlobError::RangeOutOfBounds { + offset, + length, + arena_size: self.metadata.length, + }); + } validate_mapped_header( &self.guard.arena, self.guard.lease, &self.metadata, BlobSharedState::Sealed, )?; - let payload_offset = self.guard.lease.offset.checked_add(BLOB_HEADER_LEN).ok_or( - BlobError::RangeOutOfBounds { - offset: self.guard.lease.offset, - length: self.guard.lease.length, + let payload_offset = self + .guard + .lease + .offset + .checked_add(BLOB_HEADER_LEN) + .and_then(|start| start.checked_add(offset)) + .ok_or(BlobError::RangeOutOfBounds { + offset, + length, arena_size: self.guard.arena.len() as u64, - }, - )?; - let range = mapped_range(&self.guard.arena, payload_offset, self.guard.lease.length)?; + })?; + let range = mapped_range(&self.guard.arena, payload_offset, length)?; Ok(BlobView { guard: self.guard.clone(), payload_offset: range.start, length: range.len(), }) } + pub fn map(&self) -> Result { + self.map_range(0, self.metadata.length) + } } pub struct BlobView { @@ -315,12 +350,17 @@ impl Deref for BlobView { } } +pub(crate) trait WritableViewObserver: Send + Sync + 'static { + fn view_released(&self); +} + pub(crate) struct WritableBlobLease { arena: Arc, lease: BlobLease, metadata: BlobMetadata, active_view: AtomicBool, finished: AtomicBool, + view_observer: Mutex>>, } impl WritableBlobLease { @@ -339,6 +379,7 @@ impl WritableBlobLease { metadata, active_view: AtomicBool::new(false), finished: AtomicBool::new(false), + view_observer: Mutex::new(None), })) } @@ -350,23 +391,111 @@ impl WritableBlobLease { &self.metadata } - pub(crate) fn map(self: &Arc) -> Result { + pub(crate) fn set_view_observer(&self, observer: Arc) { + *self.view_observer.lock() = Some(observer); + } + + pub(crate) fn has_active_view(&self) -> bool { + self.active_view.load(Ordering::Acquire) + } + + pub(crate) fn copy_at(&self, offset: u64, destination: &mut [u8]) -> Result { + if self.finished.load(Ordering::Acquire) { + return Err(BlobError::AlreadyFinished); + } + if self.active_view.load(Ordering::Acquire) { + return Err(BlobError::ActiveWritableView); + } + if offset >= self.metadata.length || destination.is_empty() { + return Ok(0); + } + let count = destination + .len() + .min(usize::try_from(self.metadata.length - offset).unwrap_or(usize::MAX)); + let payload_offset = self.lease.offset + BLOB_HEADER_LEN + offset; + let range = mapped_range(&self.arena, payload_offset, count as u64)?; + // SAFETY: the mapped range was bounds-checked and no mutable view is active. + let source = unsafe { + std::slice::from_raw_parts( + self.arena.ptr_at(range.start).as_ptr().cast_const(), + range.len(), + ) + }; + destination[..count].copy_from_slice(source); + Ok(count) + } + + pub(crate) fn copy_from(&self, offset: u64, source: &[u8]) -> Result { + if self.finished.load(Ordering::Acquire) { + return Err(BlobError::AlreadyFinished); + } + if self.active_view.load(Ordering::Acquire) { + return Err(BlobError::ActiveWritableView); + } + let length = source.len() as u64; + let end = offset + .checked_add(length) + .ok_or(BlobError::RangeOutOfBounds { + offset, + length, + arena_size: self.metadata.length, + })?; + if end > self.metadata.length { + return Err(BlobError::RangeOutOfBounds { + offset, + length, + arena_size: self.metadata.length, + }); + } + if source.is_empty() { + return Ok(0); + } + let payload_offset = self.lease.offset + BLOB_HEADER_LEN + offset; + let range = mapped_range(&self.arena, payload_offset, length)?; + // SAFETY: the mapped range was bounds-checked and no other mutable view is active. + let destination = unsafe { + std::slice::from_raw_parts_mut(self.arena.ptr_at(range.start).as_ptr(), range.len()) + }; + destination.copy_from_slice(source); + Ok(source.len()) + } + + pub(crate) fn map_range( + self: &Arc, + offset: u64, + length: u64, + ) -> Result { + let end = offset + .checked_add(length) + .ok_or(BlobError::RangeOutOfBounds { + offset, + length, + arena_size: self.metadata.length, + })?; + if end > self.metadata.length { + return Err(BlobError::RangeOutOfBounds { + offset, + length, + arena_size: self.metadata.length, + }); + } if self.finished.load(Ordering::Acquire) { return Err(BlobError::AlreadyFinished); } self.active_view .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) .map_err(|_| BlobError::ActiveWritableView)?; - let payload_offset = - self.lease - .offset - .checked_add(BLOB_HEADER_LEN) - .ok_or(BlobError::RangeOutOfBounds { - offset: self.lease.offset, - length: self.lease.length, - arena_size: self.arena.len() as u64, - })?; - let range = match mapped_range(&self.arena, payload_offset, self.lease.length) { + let payload_offset = self + .lease + .offset + .checked_add(BLOB_HEADER_LEN) + .and_then(|start| start.checked_add(offset)) + .ok_or(BlobError::RangeOutOfBounds { + offset, + length, + arena_size: self.arena.len() as u64, + })?; + let range = match mapped_range(&self.arena, payload_offset, length) { Ok(range) => range, Err(error) => { self.active_view.store(false, Ordering::Release); @@ -379,6 +508,9 @@ impl WritableBlobLease { length: range.len(), }) } + pub(crate) fn map(self: &Arc) -> Result { + self.map_range(0, self.metadata.length) + } pub(crate) fn seal(&self) -> Result { if self.active_view.load(Ordering::Acquire) { @@ -483,6 +615,10 @@ impl DerefMut for WritableArenaView { impl Drop for WritableArenaView { fn drop(&mut self) { self.owner.active_view.store(false, Ordering::Release); + let observer = self.owner.view_observer.lock().clone(); + if let Some(observer) = observer { + observer.view_released(); + } } } diff --git a/crates/data-plane/src/byte_ring.rs b/crates/data-plane/src/byte_ring.rs index 9840521..4480707 100644 --- a/crates/data-plane/src/byte_ring.rs +++ b/crates/data-plane/src/byte_ring.rs @@ -69,6 +69,7 @@ pub const OFF_CAPACITY: u64 = 8; pub const OFF_GENERATION: u64 = 16; pub const OFF_COMMIT: u64 = 24; pub const OFF_CONSUME: u64 = 32; +pub const OFF_TERMINAL: u64 = 40; const ZERO_CHUNK: usize = 4 * 1024; @@ -128,6 +129,30 @@ pub struct RecordMeta { pub len: u64, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RecordCursor { + kind: RecordKind, + payload_start: u64, + payload_len: u64, + record_start: u64, + record_len: u64, + generation: u64, +} + +impl RecordCursor { + pub const fn kind(self) -> RecordKind { + self.kind + } + + pub fn len(self) -> usize { + self.payload_len as usize + } + + pub const fn is_empty(self) -> bool { + self.payload_len == 0 + } +} + /// Borrowed committed record. Dropping the view releases the complete framed /// record back to the producer. pub struct PinnedRecord<'a> { @@ -557,6 +582,15 @@ pub fn attach_mapped( Ok(endpoint) } +pub fn mark_peer_terminated_mapped( + arena: &crate::mapped_arena::MappedArena, + handle: RingHandle, +) -> Result<(), AttachError> { + let endpoint = attach_mapped(arena, handle, Role::Producer)?; + endpoint.atomic(OFF_TERMINAL).store(1, Ordering::Release); + Ok(()) +} + fn lease_ring( arena: &mut ArenaManager, request_id: u64, @@ -572,6 +606,7 @@ fn lease_ring( (1, Some(ArenaEvent::RingLeaseRejected { reason, .. })) => { Err(InstallError::LeaseRejected(reason)) } + (1, Some(ArenaEvent::RingLeaseQueued { .. })) => Err(InstallError::LeaseQueued), _ => Err(InstallError::UnexpectedLeaseOutcome), } @@ -586,6 +621,13 @@ impl Endpoint { self.role } + pub fn peer_terminated(&self) -> Result { + self.validate_fixed().map_err(FlowError::Corrupt)?; + self.validate_generation(self.info.generation) + .map_err(FlowError::Corrupt)?; + Ok(self.atomic(OFF_TERMINAL).load(Ordering::Acquire) != 0) + } + /// Current published producer and consumer positions. This role-neutral /// observation is used only to wait for an already-committed clean close /// to enter the downstream bounded transport. @@ -764,6 +806,25 @@ impl Endpoint { } } + fn copy_into_slice(&self, stream_pos: u64, destination: &mut [u8]) { + let capacity = self.info.capacity as usize; + let start = (stream_pos % self.info.capacity) as usize; + let first = destination.len().min(capacity - start); + // SAFETY: the caller validated the source range against committed bytes. + unsafe { + std::ptr::copy_nonoverlapping( + self.data_ptr().add(start), + destination.as_mut_ptr(), + first, + ); + std::ptr::copy_nonoverlapping( + self.data_ptr(), + destination.as_mut_ptr().add(first), + destination.len() - first, + ); + } + } + fn copy_out(&self, stream_pos: u64, len: u64) -> Vec { let capacity = self.info.capacity as usize; let start = (stream_pos % self.info.capacity) as usize; @@ -805,6 +866,14 @@ impl Endpoint { }) } + pub fn writable_payload_capacity(&self) -> Result { + self.check("writable_payload_capacity", Role::Producer)?; + self.validate_generation(self.info.generation) + .map_err(FlowError::Corrupt)?; + let used = self.commit_cursor() - self.consume_cursor(); + Ok((self.info.capacity - used).saturating_sub(RECORD_HEADER_LEN)) + } + /// Copy bytes into the reserved (producer-side) span. pub fn write(&self, reservation: &Reservation, bytes: &[u8]) -> Result<(), FlowError> { self.check("write", Role::Producer)?; @@ -996,6 +1065,97 @@ impl Endpoint { })) } + /// Acquire an owned cursor for the next complete record. The cursor does + /// not release capacity and remains valid only while that record is first. + pub fn record_cursor(&self) -> Result, FlowError> { + self.check("record_cursor", Role::Consumer)?; + self.validate_generation(self.info.generation) + .map_err(FlowError::Corrupt)?; + let (commit, consume) = (self.commit_cursor(), self.consume_cursor()); + let readable = commit - consume; + if readable < RECORD_HEADER_LEN { + return Ok(None); + } + let mut prefix = [0_u8; RECORD_HEADER_LEN as usize]; + self.copy_into_slice(consume, &mut prefix); + let kind = RecordKind::from_byte(prefix[0]) + .ok_or(FlowError::BadRecord(RecordError::InvalidKind(prefix[0])))?; + let payload_len = u64::from(u32::from_le_bytes(prefix[1..5].try_into().unwrap())); + let record_len = RECORD_HEADER_LEN + payload_len; + if record_len > self.info.capacity { + return Err(FlowError::BadRecord(RecordError::LengthExceedsCapacity { + len: payload_len, + capacity: self.info.capacity, + })); + } + if readable < record_len { + return Ok(None); + } + Ok(Some(RecordCursor { + kind, + payload_start: consume + RECORD_HEADER_LEN, + payload_len, + record_start: consume, + record_len, + generation: self.info.generation, + })) + } + + pub fn copy_record_range( + &self, + cursor: RecordCursor, + offset: u64, + destination: &mut [u8], + ) -> Result { + self.check("copy_record_range", Role::Consumer)?; + let generation = self.field_u64(OFF_GENERATION); + if generation != cursor.generation { + return Err(FlowError::StaleReservation { + reservation: cursor.generation, + ring: generation, + }); + } + let consume = self.consume_cursor(); + if consume != cursor.record_start { + return Err(FlowError::PinnedRecordMoved { + expected: cursor.record_start, + found: consume, + }); + } + if offset > cursor.payload_len { + return Err(FlowError::BeyondCommitted { + requested: offset, + readable: cursor.payload_len, + }); + } + let count = destination + .len() + .min(usize::try_from(cursor.payload_len - offset).unwrap_or(usize::MAX)); + self.copy_into_slice(cursor.payload_start + offset, &mut destination[..count]); + Ok(count) + } + + pub fn release_record_cursor(&mut self, cursor: RecordCursor) -> Result<(), FlowError> { + self.check("release_record_cursor", Role::Consumer)?; + let generation = self.field_u64(OFF_GENERATION); + if generation != cursor.generation { + return Err(FlowError::StaleReservation { + reservation: cursor.generation, + ring: generation, + }); + } + let consume = self.consume_cursor(); + if consume != cursor.record_start { + return Err(FlowError::PinnedRecordMoved { + expected: cursor.record_start, + found: consume, + }); + } + self.atomic(OFF_CONSUME) + .store(cursor.record_start + cursor.record_len, Ordering::Release); + Ok(()) + } + /// Receive one complete record; `Ok(None)` when nothing (or only a /// torn, uncommitted prefix) is readable. pub fn recv_record(&mut self) -> Result)>, FlowError> { diff --git a/crates/data-plane/src/data_plane.rs b/crates/data-plane/src/data_plane.rs index 28e8ee7..8937d7f 100644 --- a/crates/data-plane/src/data_plane.rs +++ b/crates/data-plane/src/data_plane.rs @@ -1,9 +1,10 @@ //! Child-side data-plane session, per-operation actors, and native API. use std::collections::{HashMap, HashSet, VecDeque}; +use std::fmt; use std::os::fd::OwnedFd; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; +use std::sync::{Arc, Weak}; use std::time::Duration; use swactor::actor::{ActorAddress, ActorInterface, Ctx}; @@ -11,12 +12,19 @@ use swactor::runtime::{ExternalSender, Runtime}; use swactor_engine::{ActorCompletion, EngineHandle}; use crate::blob::{ - Blob, BlobLease, BlobMetadata, LeaseReleaser, WritableArenaView, WritableBlobLease, + Blob, BlobLease, BlobMetadata, BlobView, LeaseReleaser, WritableArenaView, WritableBlobLease, + WritableViewObserver, +}; +use crate::byte_ring::{ + Endpoint, FlowError, RecordCursor, RecordKind, RingHandle, Role, attach_mapped, }; -use crate::byte_ring::{Endpoint, FlowError, RecordKind, RingHandle, Role, attach_mapped}; use crate::mapped_arena::MappedArena; use crate::path::DataPath; -use crate::protocol::{ChildSessionIn, DataPlaneError, HostSessionIn, HostStreamIn, JobCapability}; +pub use crate::protocol::{ + AccessMode, BlobAllocation, DataPlaneError, DescriptorCapabilities, DescriptorKind, Errno, + OpenOptions, +}; +use crate::protocol::{ChildSessionIn, HostSessionIn, HostStreamIn, JobCapability, OpenPolicy}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ChildSessionState { @@ -132,14 +140,13 @@ impl DataPlaneBootstrap { .spawn(ChildDataPlaneSessionActor { runtime: runtime.clone(), host_session, - arena: arena.clone(), arena_generation: resolved.arena_generation, job_capability, child_node, session_generation: None, attach_reply: Some(attach_reply), operations: HashSet::new(), - read_operations: HashMap::new(), + open_operations: HashMap::new(), state: ChildSessionState::Attaching, stream_operations: HashMap::new(), pending_blob_releases: 0, @@ -162,19 +169,19 @@ impl DataPlaneBootstrap { } } -struct ReadCancellation { +struct DescriptorOpenCancellation { runtime: Runtime, child_session: ActorAddress, reply_to: ActorAddress, armed: bool, } -impl Drop for ReadCancellation { +impl Drop for DescriptorOpenCancellation { fn drop(&mut self) { if self.armed { let _ = self.runtime.send_to( self.child_session, - ChildSessionIn::CancelRead { + ChildSessionIn::CancelOpen { reply_to: self.reply_to, }, ); @@ -182,22 +189,77 @@ impl Drop for ReadCancellation { } } -struct StreamOpenCancellation { - runtime: Runtime, - child_session: ActorAddress, - reply_to: ActorAddress, - armed: bool, +#[derive(Clone)] +enum DescriptorOpenGrant { + ReadBlob { + host_binding: ActorAddress, + lease: BlobLease, + metadata: BlobMetadata, + cancellation: Arc, + }, + WriteBlob { + operation: ActorAddress, + lease: BlobLease, + metadata: BlobMetadata, + access: AccessMode, + cancellation: Arc, + }, + Stream { + operation: ActorAddress, + host_binding: ActorAddress, + ring: RingHandle, + role: Role, + cancellation: Arc, + }, } -impl Drop for StreamOpenCancellation { +#[derive(Clone, Copy)] +enum DescriptorGrantCancellationAction { + HostOpen { + host_session: ActorAddress, + operation: ActorAddress, + }, + WriteBlob { + operation: ActorAddress, + lease: BlobLease, + }, +} + +struct DescriptorGrantCancellation { + runtime: Runtime, + action: DescriptorGrantCancellationAction, + armed: AtomicBool, +} + +impl DescriptorGrantCancellation { + fn disarm(&self) { + self.armed.store(false, Ordering::Release); + } +} + +impl Drop for DescriptorGrantCancellation { fn drop(&mut self) { - if self.armed { - let _ = self.runtime.send_to( - self.child_session, - ChildSessionIn::CancelStream { - reply_to: self.reply_to, - }, - ); + if !self.armed.swap(false, Ordering::AcqRel) { + return; + } + match self.action { + DescriptorGrantCancellationAction::HostOpen { + host_session, + operation, + } => { + let _ = self + .runtime + .send_to(host_session, HostSessionIn::CancelOpen { operation }); + } + DescriptorGrantCancellationAction::WriteBlob { operation, lease } => { + let _ = self.runtime.send_to( + operation, + ChildOperationIn::AbortRequested { + reply_to: None, + lease, + }, + ); + } } } } @@ -209,6 +271,513 @@ pub struct DataPlane { arena: Arc, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Protection { + Read, + ReadWrite, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Sharing { + Shared, + Private, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DeviceId(pub u64); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MapTarget { + Host, + Device(DeviceId), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct MapRequest { + pub protection: Protection, + pub sharing: Sharing, + pub target: MapTarget, + pub offset: u64, + pub length: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RegionKind { + Host, + Arena, + Device, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TransferRoute { + Direct, + Staged, +} + +pub struct DeviceRegion { + identity: [u8; 32], + generation: u64, + length: u64, + readable: bool, + writable: bool, + direct_required: bool, +} + +impl DeviceRegion { + pub const fn identity(&self) -> &[u8; 32] { + &self.identity + } + + pub const fn generation(&self) -> u64 { + self.generation + } + + pub const fn length(&self) -> u64 { + self.length + } + + pub const fn is_readable(&self) -> bool { + self.readable + } + + pub const fn is_writable(&self) -> bool { + self.writable + } + + pub const fn requires_direct_route(&self) -> bool { + self.direct_required + } +} + +pub struct DeviceRegionSlice<'a> { + region: &'a DeviceRegion, + offset: u64, + length: u64, +} + +pub enum RegionSlice<'a> { + Host(&'a mut [u8]), + Arena(&'a mut [u8]), + Device(DeviceRegionSlice<'a>), +} + +impl<'a> RegionSlice<'a> { + pub fn host(bytes: &'a mut [u8]) -> Self { + Self::Host(bytes) + } + + pub fn arena(bytes: &'a mut [u8]) -> Self { + Self::Arena(bytes) + } + + pub fn device( + region: &'a DeviceRegion, + offset: u64, + length: u64, + ) -> Result { + if offset + .checked_add(length) + .is_none_or(|end| end > region.length) + { + return Err(DataPlaneError::InvalidArgument( + "device region slice is out of bounds".to_owned(), + )); + } + Ok(Self::Device(DeviceRegionSlice { + region, + offset, + length, + })) + } + + pub const fn kind(&self) -> RegionKind { + match self { + Self::Host(_) => RegionKind::Host, + Self::Arena(_) => RegionKind::Arena, + Self::Device(_) => RegionKind::Device, + } + } + + pub fn len(&self) -> usize { + match self { + Self::Host(bytes) | Self::Arena(bytes) => bytes.len(), + Self::Device(slice) => usize::try_from(slice.length).unwrap_or(usize::MAX), + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + fn as_ref(&self) -> Result<&[u8], DataPlaneError> { + match self { + Self::Host(bytes) | Self::Arena(bytes) => Ok(bytes), + Self::Device(slice) if !slice.region.readable => Err(DataPlaneError::BadDescriptor), + Self::Device(slice) => { + let _ = slice.offset; + Err(DataPlaneError::Unsupported( + "no registered device read route is installed".to_owned(), + )) + } + } + } + + fn as_mut(&mut self) -> Result<&mut [u8], DataPlaneError> { + match self { + Self::Host(bytes) | Self::Arena(bytes) => Ok(bytes), + Self::Device(slice) if !slice.region.writable => Err(DataPlaneError::BadDescriptor), + Self::Device(slice) => { + let _ = slice.offset; + Err(DataPlaneError::Unsupported( + "no registered device write route is installed".to_owned(), + )) + } + } + } +} + +pub enum DescriptorMapping { + ReadOnly(BlobView), + WritableReadOnly(WritableArenaView), + Writable(WritableArenaView), +} + +impl DescriptorMapping { + pub fn len(&self) -> usize { + match self { + Self::ReadOnly(view) => view.len(), + Self::WritableReadOnly(view) | Self::Writable(view) => view.len(), + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn as_ref(&self) -> &[u8] { + match self { + Self::ReadOnly(view) => view.as_ref(), + Self::WritableReadOnly(view) | Self::Writable(view) => view.as_ref(), + } + } + + pub fn as_mut(&mut self) -> Result<&mut [u8], DataPlaneError> { + match self { + Self::ReadOnly(_) | Self::WritableReadOnly(_) => Err(DataPlaneError::BadDescriptor), + Self::Writable(view) => Ok(view.as_mut()), + } + } + + pub const fn route(&self) -> TransferRoute { + TransferRoute::Direct + } +} + +impl fmt::Debug for DescriptorMapping { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("DescriptorMapping") + .field("len", &self.len()) + .field("route", &self.route()) + .field("writable", &matches!(self, Self::Writable(_))) + .finish() + } +} + +enum DescriptorBackend { + ReadBlob { blob: Blob, offset: u64 }, + WriteBlob { writer: BlobWriter, offset: u64 }, + ReadStream(StreamReader), + WriteStream(StreamWriter), +} + +pub struct Descriptor { + kind: DescriptorKind, + capabilities: DescriptorCapabilities, + access: AccessMode, + backend: Option, + last_route: Option, +} + +impl Descriptor { + fn read_blob(blob: Blob) -> Self { + Self { + kind: DescriptorKind::Blob, + capabilities: DescriptorCapabilities::READ + .union(DescriptorCapabilities::MAP_HOST) + .union(DescriptorCapabilities::SEEK), + access: AccessMode::ReadOnly, + backend: Some(DescriptorBackend::ReadBlob { blob, offset: 0 }), + last_route: None, + } + } + + fn write_blob(writer: BlobWriter, access: AccessMode) -> Self { + let mut capabilities = DescriptorCapabilities::WRITE + .union(DescriptorCapabilities::MAP_HOST) + .union(DescriptorCapabilities::SEEK); + if access.can_read() { + capabilities = capabilities.union(DescriptorCapabilities::READ); + } + Self { + kind: DescriptorKind::Blob, + capabilities, + access, + backend: Some(DescriptorBackend::WriteBlob { writer, offset: 0 }), + last_route: None, + } + } + + fn read_stream(reader: StreamReader) -> Self { + Self { + kind: DescriptorKind::Stream, + capabilities: DescriptorCapabilities::READ, + access: AccessMode::ReadOnly, + backend: Some(DescriptorBackend::ReadStream(reader)), + last_route: None, + } + } + + fn write_stream(writer: StreamWriter) -> Self { + Self { + kind: DescriptorKind::Stream, + capabilities: DescriptorCapabilities::WRITE, + access: AccessMode::WriteOnly, + backend: Some(DescriptorBackend::WriteStream(writer)), + last_route: None, + } + } + + pub const fn kind(&self) -> DescriptorKind { + self.kind + } + + pub const fn capabilities(&self) -> DescriptorCapabilities { + self.capabilities + } + + pub const fn access(&self) -> AccessMode { + self.access + } + + pub const fn last_route(&self) -> Option { + self.last_route + } + + pub fn is_closed(&self) -> bool { + self.backend.is_none() + } + + pub async fn read(&mut self, destination: &mut [u8]) -> Result { + if !self.access.can_read() { + return Err(DataPlaneError::BadDescriptor); + } + let backend = self.backend.as_mut().ok_or(DataPlaneError::BadDescriptor)?; + if destination.is_empty() { + return Ok(0); + } + let result = match backend { + DescriptorBackend::ReadBlob { blob, offset } => { + let count = blob.copy_at(*offset, destination)?; + *offset += count as u64; + Ok(count) + } + DescriptorBackend::WriteBlob { writer, offset } => { + let count = writer.copy_at(*offset, destination)?; + *offset += count as u64; + Ok(count) + } + DescriptorBackend::ReadStream(reader) => reader.read_into(destination).await, + DescriptorBackend::WriteStream(_) => Err(DataPlaneError::BadDescriptor), + }; + if result.is_ok() { + self.last_route = Some(TransferRoute::Staged); + } + result + } + + pub async fn write(&mut self, source: &[u8]) -> Result { + if !self.access.can_write() { + return Err(DataPlaneError::BadDescriptor); + } + let backend = self.backend.as_mut().ok_or(DataPlaneError::BadDescriptor)?; + if source.is_empty() { + return Ok(0); + } + let result = match backend { + DescriptorBackend::WriteBlob { writer, offset } => { + let count = writer.copy_from(*offset, source)?; + *offset += count as u64; + Ok(count) + } + DescriptorBackend::WriteStream(writer) => writer.write_partial(source).await, + DescriptorBackend::ReadBlob { .. } | DescriptorBackend::ReadStream(_) => { + Err(DataPlaneError::BadDescriptor) + } + }; + if result.is_ok() { + self.last_route = Some(TransferRoute::Staged); + } + result + } + + pub async fn read_into( + &mut self, + mut destination: RegionSlice<'_>, + ) -> Result { + let destination = destination.as_mut()?; + self.read(destination).await + } + + pub async fn write_from(&mut self, source: RegionSlice<'_>) -> Result { + let source = source.as_ref()?; + self.write(source).await + } + + pub async fn read_exact(&mut self, destination: &mut [u8]) -> Result<(), DataPlaneError> { + let mut completed = 0; + while completed < destination.len() { + let count = self.read(&mut destination[completed..]).await?; + if count == 0 { + return Err(DataPlaneError::InvalidArgument( + "unexpected EOF during read_exact".to_owned(), + )); + } + completed += count; + } + Ok(()) + } + + pub async fn write_all(&mut self, source: &[u8]) -> Result<(), DataPlaneError> { + let mut completed = 0; + while completed < source.len() { + let count = self.write(&source[completed..]).await?; + if count == 0 { + return Err(DataPlaneError::SessionFailed( + "zero-byte write made no progress".to_owned(), + )); + } + completed += count; + } + Ok(()) + } + + pub fn map(&self, request: MapRequest) -> Result { + let backend = self.backend.as_ref().ok_or(DataPlaneError::BadDescriptor)?; + if request.target != MapTarget::Host || request.sharing != Sharing::Shared { + return Err(DataPlaneError::Unsupported( + "requested mapping target or sharing mode is not supported".to_owned(), + )); + } + match backend { + DescriptorBackend::ReadBlob { blob, .. } => { + if request.protection != Protection::Read { + return Err(DataPlaneError::BadDescriptor); + } + Ok(DescriptorMapping::ReadOnly( + blob.map_range(request.offset, request.length)?, + )) + } + DescriptorBackend::WriteBlob { writer, .. } => match request.protection { + Protection::Read if self.access.can_read() => { + Ok(DescriptorMapping::WritableReadOnly( + writer.map_range(request.offset, request.length)?, + )) + } + Protection::Read => Err(DataPlaneError::BadDescriptor), + Protection::ReadWrite if self.access.can_write() => Ok( + DescriptorMapping::Writable(writer.map_range(request.offset, request.length)?), + ), + Protection::ReadWrite => Err(DataPlaneError::BadDescriptor), + }, + DescriptorBackend::ReadStream(_) | DescriptorBackend::WriteStream(_) => { + Err(DataPlaneError::MappingUnsupported) + } + } + } + + pub async fn close(&mut self) -> Result<(), DataPlaneError> { + let backend = self.backend.take().ok_or(DataPlaneError::BadDescriptor)?; + match backend { + DescriptorBackend::ReadBlob { .. } => Ok(()), + DescriptorBackend::WriteBlob { mut writer, .. } => { + if writer.defer_seal_if_mapped() { + Ok(()) + } else { + writer.seal().await + } + } + DescriptorBackend::ReadStream(mut reader) => reader.close_descriptor().await, + DescriptorBackend::WriteStream(mut writer) => writer.close().await, + } + } + + pub async fn abort(&mut self) -> Result<(), DataPlaneError> { + let backend = self.backend.take().ok_or(DataPlaneError::BadDescriptor)?; + match backend { + DescriptorBackend::ReadBlob { .. } => Ok(()), + DescriptorBackend::WriteBlob { mut writer, .. } => { + if writer.defer_abort_if_mapped() { + Ok(()) + } else { + writer.abort().await + } + } + DescriptorBackend::ReadStream(mut reader) => reader.abort_descriptor().await, + DescriptorBackend::WriteStream(mut writer) => writer.abort_descriptor().await, + } + } + + fn into_blob(mut self) -> Result { + match self.backend.take() { + Some(DescriptorBackend::ReadBlob { blob, .. }) => Ok(blob), + _ => Err(DataPlaneError::BadDescriptor), + } + } + + fn into_blob_writer(mut self) -> Result { + match self.backend.take() { + Some(DescriptorBackend::WriteBlob { writer, .. }) => Ok(writer), + _ => Err(DataPlaneError::BadDescriptor), + } + } + + fn into_stream_reader(mut self) -> Result { + match self.backend.take() { + Some(DescriptorBackend::ReadStream(reader)) => Ok(reader), + _ => Err(DataPlaneError::BadDescriptor), + } + } + + fn into_stream_writer(mut self) -> Result { + match self.backend.take() { + Some(DescriptorBackend::WriteStream(writer)) => Ok(writer), + _ => Err(DataPlaneError::BadDescriptor), + } + } +} + +impl Drop for Descriptor { + fn drop(&mut self) { + let Some(backend) = self.backend.take() else { + return; + }; + drop(backend); + } +} + +impl fmt::Debug for Descriptor { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("Descriptor") + .field("kind", &self.kind) + .field("capabilities", &self.capabilities) + .field("access", &self.access) + .field("closed", &self.is_closed()) + .finish() + } +} + impl DataPlane { pub fn child_session(&self) -> ActorAddress { self.child_session @@ -218,22 +787,112 @@ impl DataPlane { &self.arena } - pub async fn read_blob(&self, path: &DataPath) -> Result { + fn descriptor_from_grant( + &self, + grant: DescriptorOpenGrant, + ) -> Result { + match grant { + DescriptorOpenGrant::ReadBlob { + host_binding, + lease, + metadata, + cancellation, + } => { + let releaser: Arc = Arc::new(RuntimeLeaseReleaser { + runtime: self.runtime.clone(), + child_session: self.child_session, + host_binding, + }); + let blob = Blob::from_sealed_lease(self.arena.clone(), lease, metadata, releaser)?; + cancellation.disarm(); + Ok(Descriptor::read_blob(blob)) + } + DescriptorOpenGrant::WriteBlob { + operation, + lease, + metadata, + access, + cancellation, + } => { + let writable = WritableBlobLease::from_grant(self.arena.clone(), lease, metadata)?; + let lifecycle = Arc::new(WritableDescriptorLifecycle { + runtime: self.runtime.clone(), + operation, + writable: Arc::downgrade(&writable), + terminal: AtomicU8::new(0), + completed: AtomicBool::new(false), + }); + let observer: Arc = lifecycle.clone(); + writable.set_view_observer(observer); + let writer = BlobWriter { + runtime: self.runtime.clone(), + operation, + writable, + lifecycle, + finalized: false, + }; + cancellation.disarm(); + Ok(Descriptor::write_blob(writer, access)) + } + DescriptorOpenGrant::Stream { + operation, + host_binding, + ring, + role, + cancellation, + } => { + let endpoint = attach_mapped(&self.arena, ring, role).map_err(|error| { + DataPlaneError::StreamFault(format!("attach descriptor stream: {error:?}")) + })?; + let descriptor = match role { + Role::Consumer => Descriptor::read_stream(StreamReader { + runtime: self.runtime.clone(), + child_session: self.child_session, + operation, + host_binding, + endpoint, + terminal: None, + pending_record: None, + }), + Role::Producer => Descriptor::write_stream(StreamWriter { + runtime: self.runtime.clone(), + child_session: self.child_session, + operation, + host_binding, + endpoint, + closed: false, + }), + }; + cancellation.disarm(); + Ok(descriptor) + } + } + } + + async fn open_inner( + &self, + path: &DataPath, + options: OpenOptions, + policy: OpenPolicy, + ) -> Result { + options.validate()?; let inbox = self .runtime - .new_inbox::>() + .new_inbox::>() .map_err(|error| DataPlaneError::SessionFailed(error.to_string()))?; let reply_to = *inbox.addr(); self.runtime .send_to( self.child_session, - ChildSessionIn::ReadBlob { + ChildSessionIn::Open { path: path.clone(), + options, + policy, reply_to, }, ) .map_err(|error| DataPlaneError::SessionFailed(error.to_string()))?; - let mut cancellation = ReadCancellation { + let mut cancellation = DescriptorOpenCancellation { runtime: self.runtime.clone(), child_session: self.child_session, reply_to, @@ -241,7 +900,29 @@ impl DataPlane { }; let result = inbox.recv().await; cancellation.armed = false; - result + self.descriptor_from_grant(result?) + } + + pub async fn open( + &self, + path: &DataPath, + options: OpenOptions, + ) -> Result { + self.open_inner(path, options, OpenPolicy::Ordinary).await + } + + pub async fn open_path( + &self, + path: &str, + options: OpenOptions, + ) -> Result { + let path = DataPath::parse(path) + .map_err(|error| DataPlaneError::InvalidPath(error.to_string()))?; + self.open(&path, options).await + } + + pub async fn read_blob(&self, path: &DataPath) -> Result { + self.open(path, OpenOptions::read_only()).await?.into_blob() } pub async fn read_blob_path(&self, path: &str) -> Result { @@ -255,27 +936,9 @@ impl DataPlane { path: &DataPath, length: u64, ) -> Result { - let ask = self - .runtime - .ask::>( - self.child_session, - |reply_to| ChildSessionIn::OpenWriteBlob { - path: path.clone(), - length, - reply_to, - }, - ) - .map_err(|error| DataPlaneError::SessionFailed(error.to_string()))?; - let grant = ask.await?; - let writable = - WritableBlobLease::from_grant(self.arena.clone(), grant.lease, grant.metadata.clone())?; - grant.cancellation.disarm(); - Ok(BlobWriter { - runtime: self.runtime.clone(), - operation: grant.operation, - writable, - finalized: false, - }) + self.open(path, OpenOptions::staged_blob(length)) + .await? + .into_blob_writer() } pub async fn write_blob_path( @@ -288,94 +951,46 @@ impl DataPlane { self.write_blob(&path, length).await } - async fn open_stream( - &self, - path: &DataPath, - role: Role, - replace: bool, - ) -> Result { - let inbox = self - .runtime - .new_inbox::() - .map_err(|error| DataPlaneError::SessionFailed(error.to_string()))?; - let reply_to = *inbox.addr(); - let message = match role { - Role::Consumer => ChildSessionIn::OpenReadStream { - path: path.clone(), - reply_to, - replace, - }, - Role::Producer => ChildSessionIn::OpenWriteStream { - path: path.clone(), - reply_to, - replace, - }, - }; - self.runtime - .send_to(self.child_session, message) - .map_err(|error| DataPlaneError::SessionFailed(error.to_string()))?; - let mut cancellation = StreamOpenCancellation { - runtime: self.runtime.clone(), - child_session: self.child_session, - reply_to, - armed: true, - }; - let result = match inbox.recv().await { - ChildStreamIn::Opened(result) => result, - ChildStreamIn::Wake(_) => Err(DataPlaneError::StreamFault( - "received stream wake before open completed".to_owned(), - )), - }; - cancellation.armed = false; - result - } - pub async fn read_stream(&self, path: &DataPath) -> Result { - let grant = self.open_stream(path, Role::Consumer, false).await?; - let endpoint = attach_mapped(&self.arena, grant.ring, Role::Consumer).map_err(|error| { - DataPlaneError::StreamFault(format!("attach stream reader: {error:?}")) - })?; - Ok(StreamReader { - runtime: self.runtime.clone(), - child_session: self.child_session, - operation: grant.operation, - host_binding: grant.host_binding, - endpoint, - terminal: None, - }) + self.open_inner( + path, + OpenOptions { + access: AccessMode::ReadOnly, + ..OpenOptions::default() + }, + OpenPolicy::EnsureStream { replace: false }, + ) + .await? + .into_stream_reader() } pub async fn write_stream(&self, path: &DataPath) -> Result { - let grant = self.open_stream(path, Role::Producer, false).await?; - let endpoint = attach_mapped(&self.arena, grant.ring, Role::Producer).map_err(|error| { - DataPlaneError::StreamFault(format!("attach stream writer: {error:?}")) - })?; - Ok(StreamWriter { - runtime: self.runtime.clone(), - child_session: self.child_session, - operation: grant.operation, - host_binding: grant.host_binding, - endpoint, - closed: false, - }) + self.open_inner( + path, + OpenOptions { + access: AccessMode::WriteOnly, + ..OpenOptions::default() + }, + OpenPolicy::EnsureStream { replace: false }, + ) + .await? + .into_stream_writer() } pub async fn write_stream_replacing( &self, path: &DataPath, ) -> Result { - let grant = self.open_stream(path, Role::Producer, true).await?; - let endpoint = attach_mapped(&self.arena, grant.ring, Role::Producer).map_err(|error| { - DataPlaneError::StreamFault(format!("attach stream writer: {error:?}")) - })?; - Ok(StreamWriter { - runtime: self.runtime.clone(), - child_session: self.child_session, - operation: grant.operation, - host_binding: grant.host_binding, - endpoint, - closed: false, - }) + self.open_inner( + path, + OpenOptions { + access: AccessMode::WriteOnly, + ..OpenOptions::default() + }, + OpenPolicy::EnsureStream { replace: true }, + ) + .await? + .into_stream_writer() } pub fn close(&self) -> Result<(), DataPlaneError> { @@ -413,10 +1028,91 @@ impl DataPlane { Ok(completion) } } +struct WritableDescriptorLifecycle { + runtime: Runtime, + operation: ActorAddress, + writable: Weak, + terminal: AtomicU8, + completed: AtomicBool, +} + +impl WritableDescriptorLifecycle { + const CLOSE: u8 = 1; + const ABORT: u8 = 2; + + fn request(&self, terminal: u8) { + let _ = self + .terminal + .compare_exchange(0, terminal, Ordering::AcqRel, Ordering::Acquire); + if self + .writable + .upgrade() + .is_some_and(|writable| !writable.has_active_view()) + { + self.finalize(); + } + } + + fn finalize(&self) { + if self.completed.swap(true, Ordering::AcqRel) { + return; + } + let Some(writable) = self.writable.upgrade() else { + return; + }; + let lease = writable.lease(); + match self.terminal.load(Ordering::Acquire) { + Self::CLOSE => match writable.seal() { + Ok(metadata) => { + let _ = self.runtime.send_to( + self.operation, + ChildOperationIn::SealRequested { + reply_to: None, + lease, + metadata, + }, + ); + } + Err(_) => { + let _ = self.runtime.send_to( + self.operation, + ChildOperationIn::AbortRequested { + reply_to: None, + lease, + }, + ); + } + }, + Self::ABORT => { + let _ = writable.abort(); + let _ = self.runtime.send_to( + self.operation, + ChildOperationIn::AbortRequested { + reply_to: None, + lease, + }, + ); + } + _ => { + self.completed.store(false, Ordering::Release); + } + } + } +} + +impl WritableViewObserver for WritableDescriptorLifecycle { + fn view_released(&self) { + if self.terminal.load(Ordering::Acquire) != 0 { + self.finalize(); + } + } +} + pub struct BlobWriter { runtime: Runtime, operation: ActorAddress, writable: Arc, + lifecycle: Arc, finalized: bool, } @@ -429,6 +1125,28 @@ impl BlobWriter { self.writable.map().map_err(Into::into) } + fn copy_at(&self, offset: u64, destination: &mut [u8]) -> Result { + self.writable + .copy_at(offset, destination) + .map_err(Into::into) + } + + fn copy_from(&self, offset: u64, source: &[u8]) -> Result { + let end = offset.checked_add(source.len() as u64).ok_or_else(|| { + DataPlaneError::Unsupported("blob growth is not supported".to_owned()) + })?; + if end > self.length() { + return Err(DataPlaneError::Unsupported( + "blob growth is not supported".to_owned(), + )); + } + self.writable.copy_from(offset, source).map_err(Into::into) + } + + fn map_range(&self, offset: u64, length: u64) -> Result { + self.writable.map_range(offset, length).map_err(Into::into) + } + pub async fn seal(&mut self) -> Result<(), DataPlaneError> { let metadata = self.writable.seal()?; let lease = self.writable.lease(); @@ -436,7 +1154,7 @@ impl BlobWriter { .runtime .ask::>(self.operation, |reply_to| { ChildOperationIn::SealRequested { - reply_to, + reply_to: Some(reply_to), lease, metadata, } @@ -464,6 +1182,24 @@ impl BlobWriter { .map_err(|error| DataPlaneError::SessionFailed(error.to_string()))?; ask.await } + + fn defer_seal_if_mapped(&mut self) -> bool { + if !self.writable.has_active_view() { + return false; + } + self.finalized = true; + self.lifecycle.request(WritableDescriptorLifecycle::CLOSE); + true + } + + fn defer_abort_if_mapped(&mut self) -> bool { + if !self.writable.has_active_view() { + return false; + } + self.finalized = true; + self.lifecycle.request(WritableDescriptorLifecycle::ABORT); + true + } } impl Drop for BlobWriter { @@ -631,22 +1367,78 @@ impl StreamWriter { } } - pub async fn write(&mut self, bytes: &[u8]) -> Result<(), DataPlaneError> { + pub async fn write_partial(&mut self, bytes: &[u8]) -> Result { if self.closed { return Err(DataPlaneError::StreamClosed); } if bytes.is_empty() { - return Ok(()); + return Ok(0); } - let max_payload = usize::try_from(self.endpoint.capacity().saturating_sub(5)) - .map_err(|_| DataPlaneError::StreamFault("stream capacity exceeds usize".to_owned()))?; - if max_payload == 0 { - return Err(DataPlaneError::StreamFault( - "stream ring cannot hold a framed byte".to_owned(), - )); + loop { + if self.endpoint.peer_terminated().map_err(|error| { + DataPlaneError::StreamFault(format!( + "observe stream peer terminal state: {error:?}" + )) + })? { + return Err(DataPlaneError::BrokenPipe); + } + let available = self.endpoint.writable_payload_capacity().map_err(|error| { + DataPlaneError::StreamFault(format!("observe writable stream capacity: {error:?}")) + })?; + if available != 0 { + let count = bytes + .len() + .min(usize::try_from(available).unwrap_or(usize::MAX)); + let mut record = self + .endpoint + .reserve_record(RecordKind::Data, count as u64) + .map_err(|error| { + DataPlaneError::StreamFault(format!( + "reserve partial stream record: {error:?}" + )) + })?; + let (first, second) = record.spans_mut(); + first.copy_from_slice(&bytes[..first.len()]); + second.copy_from_slice(&bytes[first.len()..count]); + record.commit().map_err(|error| { + DataPlaneError::StreamFault(format!("commit partial stream record: {error:?}")) + })?; + self.send_control(HostStreamIn::DataAvailable)?; + return Ok(count); + } + + let inbox = self + .runtime + .new_inbox::() + .map_err(|error| DataPlaneError::SessionFailed(error.to_string()))?; + self.send_control(HostStreamIn::WaitCapacity { + reply_to: *inbox.addr(), + })?; + if self.endpoint.writable_payload_capacity().map_err(|error| { + DataPlaneError::StreamFault(format!("recheck writable stream capacity: {error:?}")) + })? != 0 + { + continue; + } + match inbox.recv().await { + ChildStreamIn::Wake(Ok(())) => {} + ChildStreamIn::Wake(Err( + DataPlaneError::StreamClosed | DataPlaneError::PeerLost, + )) => return Err(DataPlaneError::BrokenPipe), + ChildStreamIn::Wake(Err(error)) => return Err(error), + ChildStreamIn::Opened(_) => { + return Err(DataPlaneError::StreamFault( + "received stream-open result while waiting for capacity".to_owned(), + )); + } + } } - for chunk in bytes.chunks(max_payload) { - self.send_one(RecordKind::Data, chunk).await?; + } + + pub async fn write(&mut self, bytes: &[u8]) -> Result<(), DataPlaneError> { + let mut completed = 0; + while completed < bytes.len() { + completed += self.write_partial(&bytes[completed..]).await?; } Ok(()) } @@ -677,6 +1469,34 @@ impl StreamWriter { reply_to: None, }) } + + async fn abort_descriptor(&mut self) -> Result<(), DataPlaneError> { + if self.closed { + return Ok(()); + } + let inbox = self + .runtime + .new_inbox::() + .map_err(|error| DataPlaneError::SessionFailed(error.to_string()))?; + self.closed = true; + self.send_control(HostStreamIn::Close { + clean: false, + reply_to: Some(*inbox.addr()), + })?; + let result = match inbox.recv().await { + ChildStreamIn::Wake(result) => result, + ChildStreamIn::Opened(_) => Err(DataPlaneError::StreamFault( + "received stream-open result while aborting writer".to_owned(), + )), + }; + let _ = self.runtime.send_to( + self.child_session, + ChildSessionIn::OperationDone { + operation: self.operation, + }, + ); + result + } } impl Drop for StreamWriter { @@ -709,6 +1529,7 @@ pub struct StreamReader { host_binding: ActorAddress, endpoint: Endpoint, terminal: Option, + pending_record: Option<(RecordCursor, u64)>, } impl StreamReader { @@ -744,73 +1565,100 @@ impl StreamReader { } } - async fn finish_read( - &mut self, - result: Option>, - ) -> Result>, DataPlaneError> { - if result.is_none() { - self.close_clean().await?; - let _ = self.runtime.send_to( - self.child_session, - ChildSessionIn::OperationDone { - operation: self.operation, - }, - ); - } - Ok(result) - } - - fn terminal_result(&self) -> Option>, DataPlaneError>> { + fn terminal_read_count(&self) -> Option> { self.terminal.as_ref().map(|terminal| match terminal { - StreamReadTerminal::Eof => Ok(None), + StreamReadTerminal::Eof => Ok(0), StreamReadTerminal::Error(error) => Err(error.clone()), }) } - fn try_read(&mut self) -> Result>>, DataPlaneError> { - let Some(view) = self - .endpoint - .peek_record() - .map_err(|error| DataPlaneError::StreamFault(format!("read stream ring: {error:?}")))? - else { - return Ok(None); - }; - let kind = view.kind(); - let (first, second) = view.spans(); - let mut bytes = Vec::with_capacity(first.len() + second.len()); - bytes.extend_from_slice(first); - bytes.extend_from_slice(second); - view.release().map_err(|error| { - DataPlaneError::StreamFault(format!("consume stream ring: {error:?}")) - })?; - self.send_control(HostStreamIn::CapacityAvailable)?; - match kind { - RecordKind::Data => Ok(Some(Some(bytes))), - RecordKind::Eof => { - self.terminal = Some(StreamReadTerminal::Eof); - Ok(Some(None)) - } - RecordKind::Fault => { - let error = - DataPlaneError::StreamFault(String::from_utf8_lossy(&bytes).into_owned()); - self.terminal = Some(StreamReadTerminal::Error(error.clone())); - let _ = self.send_control(HostStreamIn::Close { - clean: false, - reply_to: None, - }); - Err(error) - } - } + fn release_record(&mut self, cursor: RecordCursor) -> Result<(), DataPlaneError> { + self.endpoint + .release_record_cursor(cursor) + .map_err(|error| { + DataPlaneError::StreamFault(format!("consume stream ring: {error:?}")) + })?; + self.send_control(HostStreamIn::CapacityAvailable) } - pub async fn read(&mut self) -> Result>, DataPlaneError> { - if let Some(result) = self.terminal_result() { + pub async fn read_into(&mut self, destination: &mut [u8]) -> Result { + if destination.is_empty() { + return Ok(0); + } + if let Some(result) = self.terminal_read_count() { return result; } loop { - if let Some(result) = self.try_read()? { - return self.finish_read(result).await; + if self.pending_record.is_none() { + if let Some(cursor) = self.endpoint.record_cursor().map_err(|error| { + DataPlaneError::StreamFault(format!("read stream ring: {error:?}")) + })? { + match cursor.kind() { + RecordKind::Data if cursor.is_empty() => { + self.release_record(cursor)?; + continue; + } + RecordKind::Data => { + self.pending_record = Some((cursor, 0)); + } + RecordKind::Eof => { + self.release_record(cursor)?; + self.terminal = Some(StreamReadTerminal::Eof); + let close_result = self.close_clean().await; + let _ = self.runtime.send_to( + self.child_session, + ChildSessionIn::OperationDone { + operation: self.operation, + }, + ); + close_result?; + return Ok(0); + } + RecordKind::Fault => { + let mut bytes = vec![0_u8; cursor.len()]; + let count = self + .endpoint + .copy_record_range(cursor, 0, &mut bytes) + .map_err(|error| { + DataPlaneError::StreamFault(format!( + "read stream fault record: {error:?}" + )) + })?; + bytes.truncate(count); + self.release_record(cursor)?; + let error = DataPlaneError::StreamFault( + String::from_utf8_lossy(&bytes).into_owned(), + ); + self.terminal = Some(StreamReadTerminal::Error(error.clone())); + let _ = self.send_control(HostStreamIn::Close { + clean: false, + reply_to: None, + }); + return Err(error); + } + } + } } + + if let Some((cursor, offset)) = self.pending_record { + let count = self + .endpoint + .copy_record_range(cursor, offset, destination) + .map_err(|error| { + DataPlaneError::StreamFault(format!( + "copy partial stream record: {error:?}" + )) + })?; + let next_offset = offset + count as u64; + if next_offset == cursor.len() as u64 { + self.pending_record = None; + self.release_record(cursor)?; + } else { + self.pending_record = Some((cursor, next_offset)); + } + return Ok(count); + } + let inbox = self .runtime .new_inbox::() @@ -818,8 +1666,15 @@ impl StreamReader { self.send_control(HostStreamIn::WaitData { reply_to: *inbox.addr(), })?; - if let Some(result) = self.try_read()? { - return self.finish_read(result).await; + if self + .endpoint + .record_cursor() + .map_err(|error| { + DataPlaneError::StreamFault(format!("recheck stream ring: {error:?}")) + })? + .is_some() + { + continue; } match inbox.recv().await { ChildStreamIn::Wake(Ok(())) => {} @@ -835,6 +1690,68 @@ impl StreamReader { } } } + + pub async fn read(&mut self) -> Result>, DataPlaneError> { + if matches!(self.terminal, Some(StreamReadTerminal::Eof)) { + return Ok(None); + } + let capacity = usize::try_from(self.capacity()) + .unwrap_or(usize::MAX) + .max(1); + let mut bytes = vec![0_u8; capacity]; + let count = self.read_into(&mut bytes).await?; + if count == 0 { + Ok(None) + } else { + bytes.truncate(count); + Ok(Some(bytes)) + } + } + + async fn close_descriptor(&mut self) -> Result<(), DataPlaneError> { + if self.terminal.is_some() { + return Ok(()); + } + let result = self.close_clean().await; + self.terminal = Some(StreamReadTerminal::Eof); + let _ = self.runtime.send_to( + self.child_session, + ChildSessionIn::OperationDone { + operation: self.operation, + }, + ); + result + } + + async fn abort_descriptor(&mut self) -> Result<(), DataPlaneError> { + if self.terminal.is_some() { + return Ok(()); + } + let inbox = self + .runtime + .new_inbox::() + .map_err(|error| DataPlaneError::SessionFailed(error.to_string()))?; + self.terminal = Some(StreamReadTerminal::Error( + DataPlaneError::OperationCancelled, + )); + self.send_control(HostStreamIn::Close { + clean: false, + reply_to: Some(*inbox.addr()), + })?; + let result = match inbox.recv().await { + ChildStreamIn::Wake(result) => result, + ChildStreamIn::Opened(_) => Err(DataPlaneError::StreamFault( + "received stream-open result while aborting reader".to_owned(), + )), + }; + let _ = self.runtime.send_to( + self.child_session, + ChildSessionIn::OperationDone { + operation: self.operation, + }, + ); + result + } } impl Drop for StreamReader { @@ -1089,14 +2006,13 @@ impl ActorInterface for StreamConsumerActor { pub struct ChildDataPlaneSessionActor { runtime: Runtime, host_session: ActorAddress, - arena: Arc, arena_generation: u64, job_capability: JobCapability, child_node: Option<[u8; 32]>, session_generation: Option, attach_reply: Option, operations: HashSet, - read_operations: HashMap, + open_operations: HashMap, state: ChildSessionState, stream_operations: HashMap, pending_blob_releases: usize, @@ -1108,10 +2024,6 @@ impl ChildDataPlaneSessionActor { self.state } - fn fail_local_open(&self, ctx: &Ctx<'_>, reply_to: ActorAddress, error: DataPlaneError) { - let _ = ctx.send(reply_to, Err::(error)); - } - fn start_stream_open( &mut self, ctx: &Ctx<'_>, @@ -1211,89 +2123,62 @@ impl ActorInterface for ChildDataPlaneSessionActor { } ctx.stop_self(); } - ChildSessionIn::ReadBlob { path, reply_to } => { - if self.pending_blob_releases != 0 { - self.deferred_blob_opens - .push_back(ChildSessionIn::ReadBlob { path, reply_to }); - return; - } - if self.state != ChildSessionState::Running { - self.fail_local_open(ctx, reply_to, DataPlaneError::SessionNotRunning); - return; - } - let operation = ReadBlobOperationActor { - runtime: self.runtime.clone(), - arena: self.arena.clone(), - host_session: self.host_session, - child_session: ctx.self_addr(), - path, - reply_to, - replied: false, - }; - match ctx.spawn(operation) { - Ok(operation) => { - self.operations.insert(operation); - self.read_operations.insert(reply_to, operation); - } - Err(error) => self.fail_local_open( - ctx, - reply_to, - DataPlaneError::SessionFailed(error.to_string()), - ), - } - } - ChildSessionIn::CancelRead { reply_to } => { - if let Some(operation) = self.read_operations.remove(&reply_to) { - self.operations.remove(&operation); - let _ = ctx.stop_actor(operation); - } - } - ChildSessionIn::OpenWriteBlob { + ChildSessionIn::Open { path, - length, + options, + policy, reply_to, } => { if self.pending_blob_releases != 0 { - self.deferred_blob_opens - .push_back(ChildSessionIn::OpenWriteBlob { - path, - length, - reply_to, - }); + self.deferred_blob_opens.push_back(ChildSessionIn::Open { + path, + options, + policy, + reply_to, + }); return; } if self.state != ChildSessionState::Running { let _ = ctx.send( reply_to, - Err::(DataPlaneError::SessionNotRunning), + Err::(DataPlaneError::SessionNotRunning), ); return; } - let operation = WriteBlobOperationActor { + let actor = DescriptorOpenOperationActor { runtime: self.runtime.clone(), host_session: self.host_session, child_session: ctx.self_addr(), path, - length, + options, + policy, open_reply: Some(reply_to), finish_reply: None, grant: None, state: WriteOperationState::Opening, + replied: false, }; - match ctx.spawn(operation) { + match ctx.spawn(actor) { Ok(operation) => { self.operations.insert(operation); + self.open_operations.insert(reply_to, operation); } Err(error) => { let _ = ctx.send( reply_to, - Err::(DataPlaneError::SessionFailed( + Err::(DataPlaneError::SessionFailed( error.to_string(), )), ); } } } + ChildSessionIn::CancelOpen { reply_to } => { + if let Some(operation) = self.open_operations.remove(&reply_to) { + self.operations.remove(&operation); + let _ = ctx.stop_actor(operation); + } + } ChildSessionIn::OpenReadStream { path, reply_to, @@ -1301,17 +2186,9 @@ impl ActorInterface for ChildDataPlaneSessionActor { } => { self.start_stream_open(ctx, path, reply_to, Role::Consumer, replace); } - ChildSessionIn::OpenWriteStream { - path, - reply_to, - replace, - } => { - self.start_stream_open(ctx, path, reply_to, Role::Producer, replace); - } ChildSessionIn::CancelStream { reply_to } => { if let Some(operation) = self.stream_operations.remove(&reply_to) { self.operations.remove(&operation); - let _ = ctx.send(self.host_session, HostSessionIn::CancelStream { operation }); let _ = ctx.stop_actor(operation); } } @@ -1418,8 +2295,8 @@ impl ActorInterface for ChildDataPlaneSessionActor { } ChildSessionIn::OperationDone { operation } => { self.operations.remove(&operation); - self.read_operations - .retain(|_, read_operation| *read_operation != operation); + self.open_operations + .retain(|_, open_operation| *open_operation != operation); self.stream_operations .retain(|_, stream_operation| *stream_operation != operation); } @@ -1434,7 +2311,7 @@ impl ActorInterface for ChildDataPlaneSessionActor { for operation in self.operations.iter().copied() { let _ = ctx.stop_actor(operation); } - self.read_operations.clear(); + self.open_operations.clear(); self.stream_operations.clear(); let _ = ctx.send(self.host_session, HostSessionIn::Close); self.state = ChildSessionState::Closed; @@ -1463,7 +2340,7 @@ enum ChildOperationIn { }, Failed(DataPlaneError), SealRequested { - reply_to: ActorAddress, + reply_to: Option, lease: BlobLease, metadata: BlobMetadata, }, @@ -1504,19 +2381,21 @@ impl ActorInterface for StreamOpenOperationActor { type Response = (); fn on_start(&mut self, ctx: &Ctx<'_>) { - let message = match self.role { - Role::Consumer => HostSessionIn::OpenReadStream { - path: self.path.clone(), - child_session: self.child_session, - operation: ctx.self_addr(), - replace: self.replace, - }, - Role::Producer => HostSessionIn::OpenWriteStream { - path: self.path.clone(), - child_session: self.child_session, - operation: ctx.self_addr(), + let access = match self.role { + Role::Consumer => AccessMode::ReadOnly, + Role::Producer => AccessMode::WriteOnly, + }; + let message = HostSessionIn::Open { + path: self.path.clone(), + options: OpenOptions { + access, + ..OpenOptions::default() + }, + policy: OpenPolicy::EnsureStream { replace: self.replace, }, + child_session: self.child_session, + operation: ctx.self_addr(), }; if let Err(error) = ctx.send(self.host_session, message) { self.finish(ctx, Err(DataPlaneError::SessionFailed(error.to_string()))); @@ -1556,7 +2435,7 @@ impl ActorInterface for StreamOpenOperationActor { ); let _ = ctx.send( self.host_session, - HostSessionIn::CancelStream { + HostSessionIn::CancelOpen { operation: ctx.self_addr(), }, ); @@ -1583,31 +2462,75 @@ impl LeaseReleaser for RuntimeLeaseReleaser { } } -struct ReadBlobOperationActor { +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum WriteOperationState { + Opening, + Filling, + Sealing, + Aborting, + Finished, +} + +struct DescriptorOpenOperationActor { runtime: Runtime, - arena: Arc, host_session: ActorAddress, child_session: ActorAddress, path: DataPath, - reply_to: ActorAddress, + options: OpenOptions, + policy: OpenPolicy, + open_reply: Option, + finish_reply: Option, + grant: Option<(ActorAddress, BlobLease, BlobMetadata)>, + state: WriteOperationState, replied: bool, } -impl ReadBlobOperationActor { - fn finish(&mut self, ctx: &Ctx<'_>, result: Result) { - self.replied = true; - let _ = ctx.send(self.reply_to, result); +impl DescriptorOpenOperationActor { + fn operation_done(&self, ctx: &Ctx<'_>) { let _ = ctx.send( self.child_session, ChildSessionIn::OperationDone { operation: ctx.self_addr(), }, ); + } + + fn finish_open( + &mut self, + ctx: &Ctx<'_>, + result: Result, + keep_alive: bool, + ) { + self.replied = true; + if let Some(reply_to) = self.open_reply.take() { + let _ = ctx.send(reply_to, result); + } + if !keep_alive { + self.state = WriteOperationState::Finished; + self.operation_done(ctx); + ctx.stop_self(); + } + } + + fn finish_write(&mut self, ctx: &Ctx<'_>, result: Result<(), DataPlaneError>) { + self.state = WriteOperationState::Finished; + if let Some(reply_to) = self.finish_reply.take() { + let _ = ctx.send(reply_to, result); + } + self.operation_done(ctx); ctx.stop_self(); } + + fn fail(&mut self, ctx: &Ctx<'_>, error: DataPlaneError) { + if self.open_reply.is_some() { + self.finish_open(ctx, Err(error), false); + } else { + self.finish_write(ctx, Err(error)); + } + } } -impl ActorInterface for ReadBlobOperationActor { +impl ActorInterface for DescriptorOpenOperationActor { type Incoming = ChildOperationIn; type Response = (); @@ -1615,19 +2538,19 @@ impl ActorInterface for ReadBlobOperationActor { if ctx .send( self.host_session, - HostSessionIn::OpenReadBlob { + HostSessionIn::Open { path: self.path.clone(), + options: self.options.clone(), + policy: self.policy, child_session: self.child_session, operation: ctx.self_addr(), }, ) .is_err() { - self.finish( + self.fail( ctx, - Err(DataPlaneError::SessionFailed( - "send read-blob open to host session".to_owned(), - )), + DataPlaneError::SessionFailed("send descriptor open to host session".to_owned()), ); } } @@ -1638,151 +2561,26 @@ impl ActorInterface for ReadBlobOperationActor { host_binding, lease, metadata, - } => { - let releaser: Arc = Arc::new(RuntimeLeaseReleaser { + } if self.state == WriteOperationState::Opening => { + let cancellation = Arc::new(DescriptorGrantCancellation { runtime: self.runtime.clone(), - child_session: self.child_session, - host_binding, + action: DescriptorGrantCancellationAction::HostOpen { + host_session: self.host_session, + operation: ctx.self_addr(), + }, + armed: AtomicBool::new(true), }); - let result = Blob::from_sealed_lease(self.arena.clone(), lease, metadata, releaser) - .map_err(Into::into); - self.finish(ctx, result); + self.finish_open( + ctx, + Ok(DescriptorOpenGrant::ReadBlob { + host_binding, + lease, + metadata, + cancellation, + }), + false, + ); } - ChildOperationIn::Failed(error) => self.finish(ctx, Err(error)), - _ => {} - } - } - - fn on_stop(&mut self, ctx: &Ctx<'_>) { - if !self.replied { - let _ = ctx.send( - self.host_session, - HostSessionIn::CancelReadBlob { - operation: ctx.self_addr(), - }, - ); - let _ = ctx.send( - self.reply_to, - Err::(DataPlaneError::OperationCancelled), - ); - } - } -} - -#[derive(Clone)] -struct WriteBlobGrant { - operation: ActorAddress, - lease: BlobLease, - metadata: BlobMetadata, - cancellation: Arc, -} - -struct WriteGrantCancellation { - runtime: Runtime, - operation: ActorAddress, - lease: BlobLease, - armed: AtomicBool, -} - -impl WriteGrantCancellation { - fn disarm(&self) { - self.armed.store(false, Ordering::Release); - } -} - -impl Drop for WriteGrantCancellation { - fn drop(&mut self) { - if self.armed.swap(false, Ordering::AcqRel) { - let _ = self.runtime.send_to( - self.operation, - ChildOperationIn::AbortRequested { - reply_to: None, - lease: self.lease, - }, - ); - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum WriteOperationState { - Opening, - Filling, - Sealing, - Aborting, - Finished, -} - -struct WriteBlobOperationActor { - runtime: Runtime, - host_session: ActorAddress, - child_session: ActorAddress, - path: DataPath, - length: u64, - open_reply: Option, - finish_reply: Option, - grant: Option<(ActorAddress, BlobLease, BlobMetadata)>, - state: WriteOperationState, -} - -impl WriteBlobOperationActor { - fn finish(&mut self, ctx: &Ctx<'_>, result: Result<(), DataPlaneError>) { - self.state = WriteOperationState::Finished; - if let Some(reply_to) = self.finish_reply.take() { - let _ = ctx.send(reply_to, result); - } - let _ = ctx.send( - self.child_session, - ChildSessionIn::OperationDone { - operation: ctx.self_addr(), - }, - ); - ctx.stop_self(); - } - - fn fail(&mut self, ctx: &Ctx<'_>, error: DataPlaneError) { - if let Some(reply_to) = self.open_reply.take() { - let _ = ctx.send(reply_to, Err::(error)); - self.state = WriteOperationState::Finished; - let _ = ctx.send( - self.child_session, - ChildSessionIn::OperationDone { - operation: ctx.self_addr(), - }, - ); - ctx.stop_self(); - } else { - self.finish(ctx, Err(error)); - } - } -} - -impl ActorInterface for WriteBlobOperationActor { - type Incoming = ChildOperationIn; - type Response = (); - - fn on_start(&mut self, ctx: &Ctx<'_>) { - if ctx - .send( - self.host_session, - HostSessionIn::OpenWriteBlob { - path: self.path.clone(), - length: self.length, - child_session: self.child_session, - operation: ctx.self_addr(), - }, - ) - .is_err() - { - self.fail( - ctx, - DataPlaneError::SessionFailed("send write-blob open to host session".to_owned()), - ); - } - } - - fn handle(&mut self, ctx: &Ctx<'_>, message: ChildOperationIn) { - match message { ChildOperationIn::WriteOpened { host_binding, lease, @@ -1790,45 +2588,72 @@ impl ActorInterface for WriteBlobOperationActor { } if self.state == WriteOperationState::Opening => { self.state = WriteOperationState::Filling; self.grant = Some((host_binding, lease, metadata.clone())); - if let Some(reply_to) = self.open_reply.take() - && ctx - .send( - reply_to, - Ok::<_, DataPlaneError>(WriteBlobGrant { - operation: ctx.self_addr(), - lease, - metadata, - cancellation: Arc::new(WriteGrantCancellation { - runtime: self.runtime.clone(), - operation: ctx.self_addr(), - lease, - armed: AtomicBool::new(true), - }), - }), - ) - .is_err() - { - self.state = WriteOperationState::Aborting; - if ctx - .send( - self.host_session, - HostSessionIn::AbortWriteBlob { - binding: host_binding, - operation: ctx.self_addr(), - lease_id: lease.lease_id, - generation: lease.generation, - }, - ) - .is_err() - { + let cancellation = Arc::new(DescriptorGrantCancellation { + runtime: self.runtime.clone(), + action: DescriptorGrantCancellationAction::WriteBlob { + operation: ctx.self_addr(), + lease, + }, + armed: AtomicBool::new(true), + }); + self.finish_open( + ctx, + Ok(DescriptorOpenGrant::WriteBlob { + operation: ctx.self_addr(), + lease, + metadata, + access: self.options.access, + cancellation, + }), + true, + ); + } + ChildOperationIn::StreamOpened { + host_binding, + ring, + role, + } if self.state == WriteOperationState::Opening => { + let expected_role = match self.options.access { + AccessMode::ReadOnly => Role::Consumer, + AccessMode::WriteOnly => Role::Producer, + AccessMode::ReadWrite => { self.fail( ctx, - DataPlaneError::SessionFailed( - "abort cancelled write-blob open".to_owned(), + DataPlaneError::Unsupported( + "read-write stream descriptors are not supported".to_owned(), ), ); + return; } + }; + if role != expected_role { + self.fail( + ctx, + DataPlaneError::StreamFault( + "host opened stream with the wrong ring role".to_owned(), + ), + ); + return; } + let cancellation = Arc::new(DescriptorGrantCancellation { + runtime: self.runtime.clone(), + action: DescriptorGrantCancellationAction::HostOpen { + host_session: self.host_session, + operation: ctx.self_addr(), + }, + armed: AtomicBool::new(true), + }); + self.finish_open( + ctx, + Ok(DescriptorOpenGrant::Stream { + operation: ctx.self_addr(), + host_binding, + ring, + role, + cancellation, + }), + false, + ); } ChildOperationIn::SealRequested { reply_to, @@ -1848,7 +2673,7 @@ impl ActorInterface for WriteBlobOperationActor { return; } self.state = WriteOperationState::Sealing; - self.finish_reply = Some(reply_to); + self.finish_reply = reply_to; if ctx .send( self.host_session, @@ -1864,7 +2689,7 @@ impl ActorInterface for WriteBlobOperationActor { self.fail( ctx, DataPlaneError::SessionFailed( - "send write-blob seal to host session".to_owned(), + "send descriptor blob seal to host session".to_owned(), ), ); } @@ -1903,16 +2728,16 @@ impl ActorInterface for WriteBlobOperationActor { self.fail( ctx, DataPlaneError::SessionFailed( - "send write-blob abort to host session".to_owned(), + "send descriptor blob abort to host session".to_owned(), ), ); } } ChildOperationIn::WritePublished if self.state == WriteOperationState::Sealing => { - self.finish(ctx, Ok(())); + self.finish_write(ctx, Ok(())); } ChildOperationIn::WriteAborted if self.state == WriteOperationState::Aborting => { - self.finish(ctx, Ok(())); + self.finish_write(ctx, Ok(())); } ChildOperationIn::Failed(error) => self.fail(ctx, error), _ => {} @@ -1920,11 +2745,19 @@ impl ActorInterface for WriteBlobOperationActor { } fn on_stop(&mut self, ctx: &Ctx<'_>) { - if let Some(reply_to) = self.open_reply.take() { + if !self.replied { let _ = ctx.send( - reply_to, - Err::(DataPlaneError::OperationCancelled), + self.host_session, + HostSessionIn::CancelOpen { + operation: ctx.self_addr(), + }, ); + if let Some(reply_to) = self.open_reply.take() { + let _ = ctx.send( + reply_to, + Err::(DataPlaneError::OperationCancelled), + ); + } } if let Some(reply_to) = self.finish_reply.take() { let _ = ctx.send(reply_to, Err::<(), _>(DataPlaneError::OperationCancelled)); diff --git a/crates/data-plane/src/host.rs b/crates/data-plane/src/host.rs index eaab5cb..ff5e1aa 100644 --- a/crates/data-plane/src/host.rs +++ b/crates/data-plane/src/host.rs @@ -23,14 +23,14 @@ use crate::bootstrap::JobHandoff; use crate::byte_ring::{self, ByteRingSpec, RingHandle, Role}; use crate::mapped_arena::MappedArena; use crate::namespace::{ - BlobBinding as NamespaceBlobBinding, DataDirectoryOut, NamespaceClient, NamespaceClientIn, - NamespaceError, NamespaceRequest, OperationId, SourceRecovery, StreamIncarnation, StreamMatch, - StreamRole, + BlobBinding as NamespaceBlobBinding, DataDirectoryOut, EntryKind, NamespaceClient, + NamespaceClientIn, NamespaceError, NamespaceNode, NamespaceRequest, OperationId, + SourceRecovery, StreamIncarnation, StreamMatch, StreamRole, }; use crate::path::{DataPath, JobContext}; use crate::protocol::{ - AttachmentFailure, ChildSessionIn, DataOperation, DataPlaneError, HostSessionIn, HostStreamIn, - JobCapability, + AccessMode, AttachmentFailure, ChildSessionIn, DataPlaneError, HostSessionIn, HostStreamIn, + JobCapability, OpenOptions, OpenPolicy, }; use crate::source::{BlobSourceIn, BlobSourcePublisher, BlobSourceRetirement, FileBlobSourceActor}; use crate::stream_transport::{ @@ -42,6 +42,12 @@ const BLOB_ALIGNMENT: u64 = 64; const FIRST_BLOB_REQUEST_ID: u64 = 2; const STREAM_RING_CAPACITY: u64 = 256 * 1024; +fn namespace_operation_id(address: ActorAddress) -> OperationId { + let mut bytes = [0_u8; 16]; + bytes.copy_from_slice(&address.0[..16]); + OperationId::from_u128(u128::from_be_bytes(bytes)) +} + pub trait HostRouteRegistrar: Send + Sync + 'static { fn register_child( &self, @@ -73,6 +79,13 @@ pub enum HostSessionState { Closed, } +#[derive(Clone)] +struct PendingOpen { + child_session: ActorAddress, + path: DataPath, + options: OpenOptions, +} + pub struct HostDataPlaneSessionActor { arena: Option, arena_generation: u64, @@ -91,6 +104,8 @@ pub struct HostDataPlaneSessionActor { allocator: Option, active_bindings: HashSet, stream_bindings: HashMap, + pending_opens: HashMap, + open_lookups: HashMap, state: HostSessionState, } @@ -138,6 +153,8 @@ impl HostDataPlaneSessionActor { allocator: None, active_bindings: HashSet::new(), stream_bindings: HashMap::new(), + pending_opens: HashMap::new(), + open_lookups: HashMap::new(), state: HostSessionState::AwaitingAttachment, }) } @@ -163,7 +180,7 @@ impl HostDataPlaneSessionActor { &self, child_session: ActorAddress, logical: &DataPath, - operation: DataOperation, + access: AccessMode, ) -> Result { if self.state != HostSessionState::Running || self.child_session != Some(child_session) { return Err(DataPlaneError::SessionNotRunning); @@ -172,27 +189,207 @@ impl HostDataPlaneSessionActor { .job_context .resolve(logical) .map_err(|error| DataPlaneError::InvalidPath(error.to_string()))?; - let authorized = match operation { - DataOperation::ReadBlob | DataOperation::ReadStream => { - self.job_context.can_read(&resolved) - } - DataOperation::WriteBlob | DataOperation::WriteStream => { - self.job_context.can_write(&resolved) - } - }; + let authorized = (!access.can_read() || self.job_context.can_read(&resolved)) + && (!access.can_write() || self.job_context.can_write(&resolved)); if !authorized { return Err(DataPlaneError::Unauthorized { path: resolved, - operation, + access, }); } Ok(resolved) } + fn spawn_blob_read( + &mut self, + ctx: &Ctx<'_>, + child_session: ActorAddress, + operation: ActorAddress, + path: DataPath, + expected_revision: Option, + ) { + let (Some(namespace), Some(receiver)) = (&self.namespace, &self.transfer_receiver) else { + self.send_open_failure( + ctx, + child_session, + operation, + DataPlaneError::SessionFailed("data namespace service is unavailable".to_owned()), + ); + return; + }; + let binding = HostBlobBindingActor::namespace_read( + HostBindingAddresses { + host_session: ctx.self_addr(), + allocator: self.allocator.expect("allocator started"), + child_session, + operation, + }, + path, + expected_revision, + namespace.clone(), + Arc::clone(receiver), + ); + match ctx.spawn(binding) { + Ok(binding) => { + self.active_bindings.insert(binding); + } + Err(error) => self.send_open_failure( + ctx, + child_session, + operation, + DataPlaneError::SessionFailed(error.to_string()), + ), + } + } + + fn spawn_blob_write( + &mut self, + ctx: &Ctx<'_>, + child_session: ActorAddress, + operation: ActorAddress, + path: DataPath, + length: u64, + digest: Option, + reservation: Option, + ) { + let binding = HostBlobBindingActor::write( + HostBindingAddresses { + host_session: ctx.self_addr(), + allocator: self.allocator.expect("allocator started"), + child_session, + operation, + }, + path.clone(), + length, + digest, + reservation, + self.namespace.clone(), + self.source_publisher.clone(), + ); + match ctx.spawn(binding) { + Ok(binding) => { + self.active_bindings.insert(binding); + } + Err(error) => { + if let (Some(namespace), Some(reservation)) = (&self.namespace, reservation) { + let _ = ctx.send( + namespace.proxy(), + NamespaceClientIn::CancelBlobReservation { + path, + operation_id: reservation, + reply_to: ctx.self_addr(), + }, + ); + } + self.send_open_failure( + ctx, + child_session, + operation, + DataPlaneError::SessionFailed(error.to_string()), + ); + } + } + } + + fn spawn_stream( + &mut self, + ctx: &Ctx<'_>, + child_session: ActorAddress, + operation: ActorAddress, + path: DataPath, + role: StreamRole, + replace: bool, + ensure: bool, + expected_revision: Option, + ) { + let (Some(namespace), Some(arena), Some(transport)) = ( + self.namespace.clone(), + self.stream_arena.clone(), + self.stream_transport.clone(), + ) else { + self.send_open_failure( + ctx, + child_session, + operation, + DataPlaneError::SessionFailed( + "stream namespace or transport service is unavailable".to_owned(), + ), + ); + return; + }; + let local_descriptor = match transport.descriptor() { + Ok(descriptor) => descriptor, + Err(reason) => { + self.send_open_failure( + ctx, + child_session, + operation, + DataPlaneError::StreamFault(reason), + ); + return; + } + }; + let binding = HostStreamBindingActor { + runtime: self.runtime.clone(), + host_session: ctx.self_addr(), + allocator: self.allocator.expect("allocator started"), + child_session, + operation, + path, + replace, + ensure, + expected_revision, + role, + namespace, + arena, + transport, + local_descriptor, + ring: None, + matched: None, + peer_descriptor: None, + transport_installed: false, + transport_ready: false, + transport_quiesced: false, + opened: false, + terminal: None, + data_waiters: Vec::new(), + capacity_waiters: Vec::new(), + close_waiters: Vec::new(), + release_started: false, + namespace_open: None, + peer_ack_pending: false, + }; + match ctx.spawn(binding) { + Ok(binding) => { + if let Some(publisher) = &self.source_publisher + && let Err(error) = publisher.publish_source(binding) + { + let _ = ctx.stop_actor(binding); + self.send_open_failure( + ctx, + child_session, + operation, + DataPlaneError::SessionFailed(format!("publish stream endpoint: {error}")), + ); + return; + } + self.stream_bindings.insert(operation, binding); + } + Err(error) => self.send_open_failure( + ctx, + child_session, + operation, + DataPlaneError::SessionFailed(error.to_string()), + ), + } + } + fn maybe_finish_close(&mut self) { if self.state == HostSessionState::Closing && self.active_bindings.is_empty() && self.stream_bindings.is_empty() + && self.pending_opens.is_empty() + && self.open_lookups.is_empty() { self.state = HostSessionState::Closed; } @@ -272,21 +469,69 @@ impl ActorInterface for HostDataPlaneSessionActor { ); } } - HostSessionIn::OpenReadBlob { + HostSessionIn::Open { path, + options, + policy, child_session, operation, } => { - let resolved = - match self.validate_open(child_session, &path, DataOperation::ReadBlob) { - Ok(path) => path, - Err(error) => { - self.send_open_failure(ctx, child_session, operation, error); + if let Err(error) = options.validate() { + self.send_open_failure(ctx, child_session, operation, error); + return; + } + let resolved = match self.validate_open(child_session, &path, options.access) { + Ok(path) => path, + Err(error) => { + self.send_open_failure(ctx, child_session, operation, error); + return; + } + }; + if let OpenPolicy::EnsureStream { replace } = policy { + if options.create + || options.exclusive + || options.truncate + || options.allocation.is_some() + { + self.send_open_failure( + ctx, + child_session, + operation, + DataPlaneError::InvalidArgument( + "stream ensure policy does not accept blob creation flags" + .to_owned(), + ), + ); + return; + } + let role = match options.access { + AccessMode::ReadOnly => StreamRole::Sink, + AccessMode::WriteOnly => StreamRole::Source, + AccessMode::ReadWrite => { + self.send_open_failure( + ctx, + child_session, + operation, + DataPlaneError::Unsupported( + "read-write stream descriptors are not supported".to_owned(), + ), + ); return; } }; - let (Some(namespace), Some(receiver)) = (&self.namespace, &self.transfer_receiver) - else { + self.spawn_stream( + ctx, + child_session, + operation, + resolved, + role, + replace, + true, + None, + ); + return; + } + let Some(namespace) = &self.namespace else { self.send_open_failure( ctx, child_session, @@ -297,180 +542,279 @@ impl ActorInterface for HostDataPlaneSessionActor { ); return; }; - let binding = HostBlobBindingActor::namespace_read( - HostBindingAddresses { - host_session: ctx.self_addr(), - allocator: self.allocator.expect("allocator started"), + self.pending_opens.insert( + operation, + PendingOpen { child_session, - operation, + path: resolved.clone(), + options, }, - resolved, - namespace.clone(), - Arc::clone(receiver), ); - match ctx.spawn(binding) { - Ok(binding) => { - self.active_bindings.insert(binding); + match ctx.spawn(NamespaceOpenLookupActor { + namespace_proxy: namespace.proxy(), + host_session: ctx.self_addr(), + operation, + path: resolved, + completed: false, + }) { + Ok(lookup) => { + self.open_lookups.insert(operation, lookup); } - Err(error) => self.send_open_failure( - ctx, - child_session, - operation, - DataPlaneError::SessionFailed(error.to_string()), - ), - } - } - HostSessionIn::CancelReadBlob { operation } => { - for binding in self.active_bindings.iter().copied() { - let _ = ctx.send(binding, HostBindingIn::CancelRead { operation }); - } - } - HostSessionIn::OpenWriteBlob { - path, - length, - child_session, - operation, - } => { - let resolved = - match self.validate_open(child_session, &path, DataOperation::WriteBlob) { - Ok(path) => path, - Err(error) => { - self.send_open_failure(ctx, child_session, operation, error); - return; - } - }; - let binding = HostBlobBindingActor::write( - HostBindingAddresses { - host_session: ctx.self_addr(), - allocator: self.allocator.expect("allocator started"), - child_session, - operation, - }, - resolved, - length, - self.namespace.clone(), - self.source_publisher.clone(), - ); - match ctx.spawn(binding) { - Ok(binding) => { - self.active_bindings.insert(binding); - } - Err(error) => self.send_open_failure( - ctx, - child_session, - operation, - DataPlaneError::SessionFailed(error.to_string()), - ), - } - } - ref message @ (HostSessionIn::OpenReadStream { - ref path, - child_session, - operation, - replace, - } - | HostSessionIn::OpenWriteStream { - ref path, - child_session, - operation, - replace, - }) => { - let role = match message { - HostSessionIn::OpenReadStream { .. } => StreamRole::Sink, - HostSessionIn::OpenWriteStream { .. } => StreamRole::Source, - _ => unreachable!(), - }; - let data_operation = match role { - StreamRole::Source => DataOperation::WriteStream, - StreamRole::Sink => DataOperation::ReadStream, - }; - let resolved = match self.validate_open(child_session, path, data_operation) { - Ok(path) => path, Err(error) => { - self.send_open_failure(ctx, child_session, operation, error); - return; - } - }; - let (Some(namespace), Some(arena), Some(transport)) = ( - self.namespace.clone(), - self.stream_arena.clone(), - self.stream_transport.clone(), - ) else { - self.send_open_failure( - ctx, - child_session, - operation, - DataPlaneError::SessionFailed( - "stream namespace or transport service is unavailable".to_owned(), - ), - ); - return; - }; - let local_descriptor = match transport.descriptor() { - Ok(descriptor) => descriptor, - Err(reason) => { + self.pending_opens.remove(&operation); self.send_open_failure( ctx, child_session, operation, - DataPlaneError::StreamFault(reason), + DataPlaneError::SessionFailed(error.to_string()), ); - return; } + } + } + HostSessionIn::OpenResolved { operation, result } => { + self.open_lookups.remove(&operation); + let Some(pending) = self.pending_opens.remove(&operation) else { + return; }; - let binding = HostStreamBindingActor { - runtime: self.runtime.clone(), - host_session: ctx.self_addr(), - allocator: self.allocator.expect("allocator started"), + let PendingOpen { child_session, - operation, - path: resolved, - replace, - role, - namespace, - arena, - transport, - local_descriptor, - ring: None, - matched: None, - peer_descriptor: None, - transport_installed: false, - transport_ready: false, - transport_quiesced: false, - opened: false, - terminal: None, - data_waiters: Vec::new(), - capacity_waiters: Vec::new(), - close_waiters: Vec::new(), - release_started: false, - }; - match ctx.spawn(binding) { - Ok(binding) => { - if let Some(publisher) = &self.source_publisher - && let Err(error) = publisher.publish_source(binding) - { - let _ = ctx.stop_actor(binding); + path, + options, + } = pending; + match result { + Err(NamespaceError::PathNotFound(_)) if options.create => { + let Some(allocation) = options.allocation.clone() else { self.send_open_failure( ctx, child_session, operation, - DataPlaneError::SessionFailed(format!( - "publish stream endpoint: {error}" - )), + DataPlaneError::Unsupported( + "fixed-length blob creation requires allocation metadata" + .to_owned(), + ), + ); + return; + }; + if options.exclusive { + let reservation = namespace_operation_id(operation); + let namespace = self + .namespace + .as_ref() + .expect("lookup requires namespace service"); + self.pending_opens.insert( + operation, + PendingOpen { + child_session, + path: path.clone(), + options, + }, + ); + match ctx.spawn(NamespaceBlobReserveActor { + namespace_proxy: namespace.proxy(), + host_session: ctx.self_addr(), + operation, + path, + reservation, + completed: false, + }) { + Ok(resolver) => { + self.open_lookups.insert(operation, resolver); + } + Err(error) => { + self.pending_opens.remove(&operation); + self.send_open_failure( + ctx, + child_session, + operation, + DataPlaneError::SessionFailed(error.to_string()), + ); + } + } + } else { + self.spawn_blob_write( + ctx, + child_session, + operation, + path, + allocation.length, + allocation.digest, + None, + ); + } + } + Err(error) => { + self.send_open_failure( + ctx, + child_session, + operation, + namespace_error(error), + ); + } + Ok(NamespaceNode { + kind: EntryKind::Blob, + revision, + }) => { + if options.create && options.exclusive { + self.send_open_failure( + ctx, + child_session, + operation, + DataPlaneError::PathExists(path), + ); + } else if options.access == AccessMode::ReadOnly + && !options.create + && !options.truncate + && options.allocation.is_none() + { + self.spawn_blob_read( + ctx, + child_session, + operation, + path, + Some(revision), + ); + } else if options.access.can_write() && options.truncate { + let Some(allocation) = options.allocation else { + self.send_open_failure( + ctx, + child_session, + operation, + DataPlaneError::Unsupported( + "fixed-length blob replacement requires allocation metadata" + .to_owned(), + ), + ); + return; + }; + self.spawn_blob_write( + ctx, + child_session, + operation, + path, + allocation.length, + allocation.digest, + None, + ); + } else { + self.send_open_failure( + ctx, + child_session, + operation, + DataPlaneError::Unsupported( + "non-truncating writable blob opens are not supported" + .to_owned(), + ), + ); + } + } + Ok(NamespaceNode { + kind: EntryKind::Stream, + revision, + }) => { + if options.create + || options.exclusive + || options.truncate + || options.allocation.is_some() + { + self.send_open_failure( + ctx, + child_session, + operation, + DataPlaneError::WrongEntryType { + path, + expected: EntryKind::Blob, + found: EntryKind::Stream, + }, ); return; } - self.stream_bindings.insert(operation, binding); + let role = match options.access { + AccessMode::ReadOnly => StreamRole::Sink, + AccessMode::WriteOnly => StreamRole::Source, + AccessMode::ReadWrite => { + self.send_open_failure( + ctx, + child_session, + operation, + DataPlaneError::Unsupported( + "read-write stream descriptors are not supported" + .to_owned(), + ), + ); + return; + } + }; + self.spawn_stream( + ctx, + child_session, + operation, + path, + role, + false, + false, + Some(revision), + ); + } + } + } + HostSessionIn::BlobReserved { + operation, + path: reserved_path, + reservation, + result, + } => { + self.open_lookups.remove(&operation); + let Some(PendingOpen { + child_session, + path, + options, + }) = self.pending_opens.remove(&operation) + else { + if result.is_ok() + && let Some(namespace) = &self.namespace + { + let _ = ctx.send( + namespace.proxy(), + NamespaceClientIn::CancelBlobReservation { + path: reserved_path, + operation_id: reservation, + reply_to: ctx.self_addr(), + }, + ); + } + return; + }; + match result { + Ok(()) => { + let allocation = options + .allocation + .expect("exclusive fixed blob reservation has allocation"); + self.spawn_blob_write( + ctx, + child_session, + operation, + path, + allocation.length, + allocation.digest, + Some(reservation), + ); } Err(error) => self.send_open_failure( ctx, child_session, operation, - DataPlaneError::SessionFailed(error.to_string()), + namespace_error(error), ), } } - HostSessionIn::CancelStream { operation } => { + HostSessionIn::CancelOpen { operation } => { + self.pending_opens.remove(&operation); + if let Some(lookup) = self.open_lookups.remove(&operation) { + let _ = ctx.stop_actor(lookup); + } + for binding in self.active_bindings.iter().copied() { + let _ = ctx.send(binding, HostBindingIn::CancelOpen { operation }); + } if let Some(binding) = self.stream_bindings.get(&operation).copied() { let _ = ctx.send( binding, @@ -582,6 +926,18 @@ impl ActorInterface for HostDataPlaneSessionActor { return; } self.state = HostSessionState::Closing; + for lookup in self.open_lookups.drain().map(|(_, lookup)| lookup) { + let _ = ctx.stop_actor(lookup); + } + for (operation, pending) in self.pending_opens.drain() { + let _ = ctx.send( + pending.child_session, + ChildSessionIn::OperationFailed { + operation, + error: DataPlaneError::SessionNotRunning, + }, + ); + } for binding in self.active_bindings.iter().copied() { let _ = ctx.send(binding, HostBindingIn::SessionClosed); } @@ -1202,6 +1558,152 @@ impl ActorInterface for DestinationBlobTransferActor { } } +struct NamespaceOpenLookupActor { + namespace_proxy: ActorAddress, + host_session: ActorAddress, + operation: ActorAddress, + path: DataPath, + completed: bool, +} + +impl ActorInterface for NamespaceOpenLookupActor { + type Incoming = DataDirectoryOut; + type Response = (); + + fn on_start(&mut self, ctx: &Ctx<'_>) { + if ctx + .send( + self.namespace_proxy, + NamespaceClientIn::Request { + request: NamespaceRequest::Lookup { + path: self.path.clone(), + }, + reply_to: ctx.self_addr(), + }, + ) + .is_err() + { + self.completed = true; + let _ = ctx.send( + self.host_session, + HostSessionIn::OpenResolved { + operation: self.operation, + result: Err(NamespaceError::DirectoryUnavailable( + "namespace client is unavailable".to_owned(), + )), + }, + ); + ctx.stop_self(); + } + } + + fn handle(&mut self, ctx: &Ctx<'_>, message: DataDirectoryOut) { + let result = match message { + DataDirectoryOut::LookedUp { result, .. } => result, + other => Err(NamespaceError::Protocol(format!( + "expected lookup reply, received {other:?}" + ))), + }; + self.completed = true; + let _ = ctx.send( + self.host_session, + HostSessionIn::OpenResolved { + operation: self.operation, + result, + }, + ); + ctx.stop_self(); + } + + fn on_stop(&mut self, ctx: &Ctx<'_>) { + if !self.completed { + let _ = ctx.send( + self.namespace_proxy, + NamespaceClientIn::Cancel { + reply_to: ctx.self_addr(), + }, + ); + } + } +} + +struct NamespaceBlobReserveActor { + namespace_proxy: ActorAddress, + host_session: ActorAddress, + operation: ActorAddress, + path: DataPath, + reservation: OperationId, + completed: bool, +} + +impl ActorInterface for NamespaceBlobReserveActor { + type Incoming = DataDirectoryOut; + type Response = (); + + fn on_start(&mut self, ctx: &Ctx<'_>) { + if ctx + .send( + self.namespace_proxy, + NamespaceClientIn::Request { + request: NamespaceRequest::ReserveBlob { + path: self.path.clone(), + operation_id: self.reservation, + }, + reply_to: ctx.self_addr(), + }, + ) + .is_err() + { + self.completed = true; + let _ = ctx.send( + self.host_session, + HostSessionIn::BlobReserved { + operation: self.operation, + path: self.path.clone(), + reservation: self.reservation, + result: Err(NamespaceError::DirectoryUnavailable( + "namespace client is unavailable".to_owned(), + )), + }, + ); + ctx.stop_self(); + } + } + + fn handle(&mut self, ctx: &Ctx<'_>, message: DataDirectoryOut) { + let result = match message { + DataDirectoryOut::BlobReserved { result, .. } => result, + other => Err(NamespaceError::Protocol(format!( + "expected blob reservation reply, received {other:?}" + ))), + }; + self.completed = true; + let _ = ctx.send( + self.host_session, + HostSessionIn::BlobReserved { + operation: self.operation, + path: self.path.clone(), + reservation: self.reservation, + result, + }, + ); + ctx.stop_self(); + } + + fn on_stop(&mut self, ctx: &Ctx<'_>) { + if !self.completed { + let _ = ctx.send( + self.namespace_proxy, + NamespaceClientIn::CancelBlobReservation { + path: self.path.clone(), + operation_id: self.reservation, + reply_to: ctx.self_addr(), + }, + ); + } + } +} + struct NamespaceResolveActor { namespace_proxy: ActorAddress, path: DataPath, @@ -1259,6 +1761,7 @@ struct NamespacePublishActor { source: ActorAddress, length: u64, operation_id: OperationId, + reservation: Option, operation: ActorAddress, binding: ActorAddress, } @@ -1278,6 +1781,7 @@ impl ActorInterface for NamespacePublishActor { length: self.length, recovery: SourceRecovery::Actor { actor: self.source }, operation_id: self.operation_id, + reservation: self.reservation, }, reply_to: ctx.self_addr(), }, @@ -1316,6 +1820,64 @@ impl ActorInterface for NamespacePublishActor { } } +struct NamespaceBlobReservationReleaseActor { + namespace_proxy: ActorAddress, + binding: ActorAddress, + path: DataPath, + reservation: OperationId, +} + +impl ActorInterface for NamespaceBlobReservationReleaseActor { + type Incoming = DataDirectoryOut; + type Response = (); + + fn on_start(&mut self, ctx: &Ctx<'_>) { + if ctx + .send( + self.namespace_proxy, + NamespaceClientIn::Request { + request: NamespaceRequest::ReleaseBlobReservation { + path: self.path.clone(), + operation_id: self.reservation, + }, + reply_to: ctx.self_addr(), + }, + ) + .is_err() + { + let _ = ctx.send( + self.binding, + HostBindingIn::ReservationReleased(Err(DataPlaneError::SessionFailed( + "namespace client is unavailable".to_owned(), + ))), + ); + ctx.stop_self(); + } + } + + fn handle(&mut self, ctx: &Ctx<'_>, message: DataDirectoryOut) { + let result = match message { + DataDirectoryOut::BlobReservationReleased { result, .. } => { + result.map_err(namespace_error) + } + other => Err(DataPlaneError::SessionFailed(format!( + "expected blob reservation release, received {other:?}" + ))), + }; + let _ = ctx.send(self.binding, HostBindingIn::ReservationReleased(result)); + ctx.stop_self(); + } + + fn on_stop(&mut self, ctx: &Ctx<'_>) { + let _ = ctx.send( + self.namespace_proxy, + NamespaceClientIn::Cancel { + reply_to: ctx.self_addr(), + }, + ); + } +} + struct NamespaceUnpublishActor { namespace_proxy: ActorAddress, path: DataPath, @@ -1370,6 +1932,8 @@ struct NamespaceStreamOpenActor { path: DataPath, role: StreamRole, replace: bool, + ensure: bool, + expected_revision: Option, descriptor: Vec, operation_id: OperationId, completed: bool, @@ -1389,6 +1953,8 @@ impl ActorInterface for NamespaceStreamOpenActor { endpoint: self.parent, descriptor: self.descriptor.clone(), replace: self.replace, + ensure: self.ensure, + expected_revision: self.expected_revision, operation_id: self.operation_id, }, reply_to: ctx.self_addr(), @@ -1469,6 +2035,8 @@ struct HostStreamBindingActor { operation: ActorAddress, path: DataPath, replace: bool, + ensure: bool, + expected_revision: Option, role: StreamRole, namespace: NamespaceClient, arena: Arc, @@ -1486,6 +2054,8 @@ struct HostStreamBindingActor { capacity_waiters: Vec, close_waiters: Vec, release_started: bool, + namespace_open: Option, + peer_ack_pending: bool, } impl HostStreamBindingActor { @@ -1611,6 +2181,12 @@ impl HostStreamBindingActor { if self.terminal.is_some() { return; } + if let Some(namespace_open) = self.namespace_open.take() { + let _ = ctx.stop_actor(namespace_open); + } + if let Some(ring) = self.ring { + let _ = byte_ring::mark_peer_terminated_mapped(&self.arena, ring); + } self.terminal = Some(error.clone()); Self::wake_waiters( ctx, @@ -1635,13 +2211,16 @@ impl HostStreamBindingActor { } else { DataPlaneError::PeerLost }; - let _ = ctx.send( - peer, - HostStreamIn::PeerTerminated { - incarnation: matched.incarnation, - error: peer_error, - }, - ); + self.peer_ack_pending = ctx + .send( + peer, + HostStreamIn::PeerTerminated { + incarnation: matched.incarnation, + error: peer_error, + reply_to: Some(ctx.self_addr()), + }, + ) + .is_ok(); } let _ = ctx.spawn(NamespaceStreamCloseActor { proxy: self.namespace.proxy(), @@ -1654,6 +2233,9 @@ impl HostStreamBindingActor { return; } } + if self.peer_ack_pending { + return; + } } self.release_ring(ctx); } @@ -1708,44 +2290,56 @@ impl ActorInterface for HostStreamBindingActor { capacity: STREAM_RING_CAPACITY, }, ); - let _ = ctx.spawn(NamespaceStreamOpenActor { + match ctx.spawn(NamespaceStreamOpenActor { proxy: self.namespace.proxy(), parent: ctx.self_addr(), path: self.path.clone(), role: self.role, descriptor: self.local_descriptor.0.clone(), replace: self.replace, + ensure: self.ensure, + expected_revision: self.expected_revision, operation_id: Self::operation_id(ctx.self_addr()), completed: false, - }); + }) { + Ok(namespace_open) => { + self.namespace_open = Some(namespace_open); + } + Err(error) => { + self.fail_open(ctx, DataPlaneError::SessionFailed(error.to_string())); + } + } } fn handle(&mut self, ctx: &Ctx<'_>, message: HostStreamIn) { match message { - HostStreamIn::NamespaceMatched(result) => match result { - Ok(matched) => { - let expected = match self.role { - StreamRole::Source => matched.source, - StreamRole::Sink => matched.sink, - }; - if expected != ctx.self_addr() { - self.fail_open( - ctx, - DataPlaneError::StreamFault( - "namespace matched the wrong stream endpoint".to_owned(), - ), - ); - return; + HostStreamIn::NamespaceMatched(result) => { + self.namespace_open = None; + match result { + Ok(matched) => { + let expected = match self.role { + StreamRole::Source => matched.source, + StreamRole::Sink => matched.sink, + }; + if expected != ctx.self_addr() { + self.fail_open( + ctx, + DataPlaneError::StreamFault( + "namespace matched the wrong stream endpoint".to_owned(), + ), + ); + return; + } + if self.role == StreamRole::Source && !matched.sink_descriptor.is_empty() { + self.peer_descriptor = + Some(StreamPeerDescriptor(matched.sink_descriptor.clone())); + } + self.matched = Some(matched); + self.try_install_transport(ctx); } - if self.role == StreamRole::Source && !matched.sink_descriptor.is_empty() { - self.peer_descriptor = - Some(StreamPeerDescriptor(matched.sink_descriptor.clone())); - } - self.matched = Some(matched); - self.try_install_transport(ctx); + Err(error) => self.fail_open(ctx, namespace_error(error)), } - Err(error) => self.fail_open(ctx, namespace_error(error)), - }, + } HostStreamIn::Allocated(result) => match result { Ok(ring) => { self.ring = Some(ring); @@ -1784,7 +2378,7 @@ impl ActorInterface for HostStreamBindingActor { } HostStreamIn::Transport(StreamTransportEvent::Quiesced) => { self.transport_quiesced = true; - if self.terminal.is_some() { + if self.terminal.is_some() && !self.peer_ack_pending { self.release_ring(ctx); } } @@ -1851,7 +2445,11 @@ impl ActorInterface for HostStreamBindingActor { }; self.fail_open(ctx, error); } - HostStreamIn::PeerTerminated { incarnation, error } => { + HostStreamIn::PeerTerminated { + incarnation, + error, + reply_to, + } => { if self .matched .as_ref() @@ -1867,6 +2465,24 @@ impl ActorInterface for HostStreamBindingActor { ); } self.begin_terminal(ctx, error, false); + if let Some(reply_to) = reply_to { + let _ = + ctx.send(reply_to, HostStreamIn::PeerTerminationAck { incarnation }); + } + } + } + HostStreamIn::PeerTerminationAck { incarnation } => { + if self + .matched + .as_ref() + .is_some_and(|matched| matched.incarnation == incarnation) + { + self.peer_ack_pending = false; + if self.terminal.is_some() + && (!self.transport_installed || self.transport_quiesced) + { + self.release_ring(ctx); + } } } HostStreamIn::ReleaseComplete(result) => { @@ -1895,6 +2511,7 @@ impl ActorInterface for HostStreamBindingActor { fn namespace_error(error: NamespaceError) -> DataPlaneError { match error { + NamespaceError::PathExists(path) => DataPlaneError::PathExists(path), NamespaceError::PathNotFound(path) => DataPlaneError::PathNotFound(path), NamespaceError::WrongEntryType { path, @@ -1945,10 +2562,12 @@ enum HostBindingState { enum BindingMode { NamespaceRead { namespace: NamespaceClient, + expected_revision: Option, receiver: Arc, }, Write { length: u64, + digest: Option, namespace: Option, source_publisher: Option>, }, @@ -1972,7 +2591,7 @@ enum HostBindingIn { }, TransferFailed(DataPlaneError), Allocated(Result<(BlobLease, BlobMetadata), DataPlaneError>), - CancelRead { + CancelOpen { operation: ActorAddress, }, Release { @@ -1995,6 +2614,7 @@ enum HostBindingIn { operation: ActorAddress, }, PublicationRejected(DataPlaneError), + ReservationReleased(Result<(), DataPlaneError>), ReleasePublished, SessionClosed, Released(Result<(), DataPlaneError>), @@ -2020,12 +2640,15 @@ struct HostBlobBindingActor { release_outcome: Option, auxiliary: Option, published_source: Option, + cancelled: bool, + reservation: Option, } impl HostBlobBindingActor { fn namespace_read( addresses: HostBindingAddresses, path: DataPath, + expected_revision: Option, namespace: NamespaceClient, receiver: Arc, ) -> Self { @@ -2043,6 +2666,7 @@ impl HostBlobBindingActor { path, mode: BindingMode::NamespaceRead { namespace, + expected_revision, receiver, }, state: HostBindingState::Resolving, @@ -2051,6 +2675,8 @@ impl HostBlobBindingActor { release_outcome: None, auxiliary: None, published_source: None, + reservation: None, + cancelled: false, } } @@ -2058,6 +2684,8 @@ impl HostBlobBindingActor { addresses: HostBindingAddresses, path: DataPath, length: u64, + digest: Option, + reservation: Option, namespace: Option, source_publisher: Option>, ) -> Self { @@ -2075,6 +2703,7 @@ impl HostBlobBindingActor { path, mode: BindingMode::Write { length, + digest, namespace, source_publisher, }, @@ -2084,6 +2713,45 @@ impl HostBlobBindingActor { release_outcome: None, auxiliary: None, published_source: None, + cancelled: false, + reservation, + } + } + + fn begin_reservation_release(&mut self, ctx: &Ctx<'_>, outcome: ReleaseOutcome) -> bool { + let Some(reservation) = self.reservation.take() else { + return false; + }; + let BindingMode::Write { + namespace: Some(namespace), + .. + } = &self.mode + else { + return false; + }; + match ctx.spawn(NamespaceBlobReservationReleaseActor { + namespace_proxy: namespace.proxy(), + binding: ctx.self_addr(), + path: self.path.clone(), + reservation, + }) { + Ok(releaser) => { + self.auxiliary = Some(releaser); + self.release_outcome = Some(outcome); + self.state = HostBindingState::Releasing; + true + } + Err(_) => { + let _ = ctx.send( + namespace.proxy(), + NamespaceClientIn::CancelBlobReservation { + path: self.path.clone(), + operation_id: reservation, + reply_to: ctx.self_addr(), + }, + ); + false + } } } @@ -2138,6 +2806,9 @@ impl HostBlobBindingActor { } fn finish_without_lease(&mut self, ctx: &Ctx<'_>, outcome: ReleaseOutcome) { + if self.begin_reservation_release(ctx, outcome.clone()) { + return; + } self.state = HostBindingState::Released; let read_released = matches!(outcome, ReleaseOutcome::ReadReleased); if let Some(auxiliary) = self.auxiliary.take() { @@ -2210,18 +2881,15 @@ impl ActorInterface for HostBlobBindingActor { } self.state = HostBindingState::Filling; - let length = match &self.mode { - BindingMode::Write { length, .. } => *length, + let (length, digest) = match &self.mode { + BindingMode::Write { length, digest, .. } => (*length, digest.clone()), BindingMode::NamespaceRead { .. } => unreachable!("handled above"), }; let _ = ctx.send( self.allocator, ArenaAllocatorIn::Allocate { binding: ctx.self_addr(), - kind: AllocationKind::Write { - length, - digest: None, - }, + kind: AllocationKind::Write { length, digest }, }, ); } @@ -2233,13 +2901,18 @@ impl ActorInterface for HostBlobBindingActor { && matches!(self.mode, BindingMode::NamespaceRead { .. }) => { self.auxiliary = None; - let (receiver, failure_proxy) = match &self.mode { + let (receiver, failure_proxy, expected_revision) = match &self.mode { BindingMode::NamespaceRead { namespace, receiver, - } => (Arc::clone(receiver), namespace.proxy()), + expected_revision, + } => (Arc::clone(receiver), namespace.proxy(), *expected_revision), _ => unreachable!("namespace resolve on non-namespace binding"), }; + if expected_revision.is_some_and(|revision| revision != binding.revision) { + self.fail(ctx, DataPlaneError::PathReplaced(self.path.clone())); + return; + } let mut id_bytes = [0_u8; 8]; id_bytes.copy_from_slice(&ctx.self_addr().0[..8]); let transfer_id = BlobTransferId(u64::from_le_bytes(id_bytes).max(1)); @@ -2273,33 +2946,35 @@ impl ActorInterface for HostBlobBindingActor { self.lease = Some(lease); self.metadata = Some(metadata.clone()); self.state = HostBindingState::Granted; - let _ = ctx.send( - self.child_session, - ChildSessionIn::BlobOpened { - operation: self.operation, - host_binding: ctx.self_addr(), - lease, - metadata, - }, - ); + if self.cancelled { + self.begin_release(ctx, ReleaseOutcome::Faulted); + } else { + let _ = ctx.send( + self.child_session, + ChildSessionIn::BlobOpened { + operation: self.operation, + host_binding: ctx.self_addr(), + lease, + metadata, + }, + ); + } } - HostBindingIn::CancelRead { operation } - if operation == self.operation - && matches!(self.mode, BindingMode::NamespaceRead { .. }) => - { + HostBindingIn::CancelOpen { operation } if operation == self.operation => { + self.cancelled = true; match self.state { HostBindingState::Resolving => { self.finish_without_lease(ctx, ReleaseOutcome::Faulted); } HostBindingState::Filling => { - if let Some(transfer) = self.auxiliary { + if matches!(self.mode, BindingMode::NamespaceRead { .. }) + && let Some(transfer) = self.auxiliary + { let _ = ctx.send(transfer, BlobTransferEvent::Cancel); - } else { - self.finish_without_lease(ctx, ReleaseOutcome::Faulted); } } HostBindingState::Granted => { - self.begin_release(ctx, ReleaseOutcome::ReadReleased); + self.begin_release(ctx, ReleaseOutcome::Faulted); } _ => {} } @@ -2319,18 +2994,22 @@ impl ActorInterface for HostBlobBindingActor { self.lease = Some(lease); self.metadata = Some(metadata.clone()); self.state = HostBindingState::Granted; - let response = match self.mode { - BindingMode::Write { .. } => ChildSessionIn::WriteBlobOpened { - operation: self.operation, - host_binding: ctx.self_addr(), - lease, - metadata, - }, - BindingMode::NamespaceRead { .. } => { - unreachable!("namespace read uses TransferReady") - } - }; - let _ = ctx.send(self.child_session, response); + if self.cancelled { + self.begin_release(ctx, ReleaseOutcome::Faulted); + } else { + let response = match self.mode { + BindingMode::Write { .. } => ChildSessionIn::WriteBlobOpened { + operation: self.operation, + host_binding: ctx.self_addr(), + lease, + metadata, + }, + BindingMode::NamespaceRead { .. } => { + unreachable!("namespace read uses TransferReady") + } + }; + let _ = ctx.send(self.child_session, response); + } } HostBindingIn::Allocated(Err(error)) if self.state == HostBindingState::Filling => { self.fail(ctx, error); @@ -2434,6 +3113,7 @@ impl ActorInterface for HostBlobBindingActor { path: self.path.clone(), source, length: self.metadata.as_ref().expect("write metadata").length, + reservation: self.reservation, operation_id, operation: self.operation, binding: ctx.self_addr(), @@ -2493,6 +3173,7 @@ impl ActorInterface for HostBlobBindingActor { if self.state == HostBindingState::Publishing && operation == self.operation => { self.auxiliary = None; + self.reservation = None; self.state = HostBindingState::Published; if let Some(outcome) = self.release_outcome.take() { self.begin_unpublish(ctx, outcome); @@ -2573,6 +3254,24 @@ impl ActorInterface for HostBlobBindingActor { .unwrap_or(ReleaseOutcome::Faulted); self.finish_without_lease(ctx, outcome); } + HostBindingIn::ReservationReleased(result) => { + self.auxiliary = None; + if let Err(error) = result { + let _ = ctx.send( + self.host_session, + HostSessionIn::BindingFaulted { + binding: ctx.self_addr(), + operation: self.operation, + error, + }, + ); + } + let outcome = self + .release_outcome + .take() + .unwrap_or(ReleaseOutcome::Faulted); + self.finish_without_lease(ctx, outcome); + } _ => {} } } diff --git a/crates/data-plane/src/namespace.rs b/crates/data-plane/src/namespace.rs index 284a83b..675146f 100644 --- a/crates/data-plane/src/namespace.rs +++ b/crates/data-plane/src/namespace.rs @@ -39,6 +39,12 @@ pub enum EntryKind { Stream, } +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct NamespaceNode { + pub kind: EntryKind, + pub revision: u64, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum StreamRole { Source, @@ -64,6 +70,7 @@ pub struct StreamMatch { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum NamespaceError { PathNotFound(DataPath), + PathExists(DataPath), WrongEntryType { path: DataPath, expected: EntryKind, @@ -89,6 +96,7 @@ impl fmt::Display for NamespaceError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::PathNotFound(path) => write!(f, "data path not found: {path}"), + Self::PathExists(path) => write!(f, "data path already exists: {path}"), Self::WrongEntryType { path, expected, @@ -140,6 +148,7 @@ pub enum DataDirectoryIn { length: u64, recovery: SourceRecovery, operation_id: OperationId, + reservation: Option, reply_to: ActorAddress, }, Resolve { @@ -147,6 +156,23 @@ pub enum DataDirectoryIn { path: DataPath, reply_to: ActorAddress, }, + Lookup { + request_id: DirectoryRequestId, + path: DataPath, + reply_to: ActorAddress, + }, + ReserveBlob { + request_id: DirectoryRequestId, + path: DataPath, + operation_id: OperationId, + reply_to: ActorAddress, + }, + ReleaseBlobReservation { + request_id: DirectoryRequestId, + path: DataPath, + operation_id: OperationId, + reply_to: ActorAddress, + }, Unregister { request_id: DirectoryRequestId, path: DataPath, @@ -160,6 +186,8 @@ pub enum DataDirectoryIn { descriptor: Vec, endpoint: ActorAddress, replace: bool, + ensure: bool, + expected_revision: Option, operation_id: OperationId, reply_to: ActorAddress, }, @@ -193,6 +221,21 @@ pub enum DataDirectoryOut { authority_epoch: u64, result: Result, }, + LookedUp { + request_id: DirectoryRequestId, + authority_epoch: u64, + result: Result, + }, + BlobReserved { + request_id: DirectoryRequestId, + authority_epoch: u64, + result: Result<(), NamespaceError>, + }, + BlobReservationReleased { + request_id: DirectoryRequestId, + authority_epoch: u64, + result: Result<(), NamespaceError>, + }, Unregistered { request_id: DirectoryRequestId, authority_epoch: u64, @@ -214,6 +257,9 @@ impl DataDirectoryOut { match self { Self::Registered { request_id, .. } | Self::Resolved { request_id, .. } + | Self::LookedUp { request_id, .. } + | Self::BlobReserved { request_id, .. } + | Self::BlobReservationReleased { request_id, .. } | Self::Unregistered { request_id, .. } | Self::StreamOpened { request_id, .. } | Self::StreamClosed { request_id, .. } => *request_id, @@ -229,10 +275,22 @@ pub enum NamespaceRequest { length: u64, recovery: SourceRecovery, operation_id: OperationId, + reservation: Option, }, Resolve { path: DataPath, }, + Lookup { + path: DataPath, + }, + ReserveBlob { + path: DataPath, + operation_id: OperationId, + }, + ReleaseBlobReservation { + path: DataPath, + operation_id: OperationId, + }, Unregister { path: DataPath, operation_id: OperationId, @@ -243,6 +301,8 @@ pub enum NamespaceRequest { endpoint: ActorAddress, descriptor: Vec, replace: bool, + ensure: bool, + expected_revision: Option, operation_id: OperationId, }, CloseStream { @@ -259,6 +319,11 @@ pub enum NamespaceClientIn { Cancel { reply_to: ActorAddress, }, + CancelBlobReservation { + path: DataPath, + operation_id: OperationId, + reply_to: ActorAddress, + }, DirectoryReply(DataDirectoryOut), TransferFailed { destination: ActorAddress, @@ -288,6 +353,7 @@ struct PendingStream { descriptor: Vec, reply_to: ActorAddress, revision: u64, + opened_from_revision: Option, } struct StreamOpenRequest { @@ -296,6 +362,8 @@ struct StreamOpenRequest { role: StreamRole, endpoint: ActorAddress, replace: bool, + ensure: bool, + expected_revision: Option, descriptor: Vec, operation_id: OperationId, reply_to: ActorAddress, @@ -316,6 +384,7 @@ pub struct DataDirectoryActor { store: NamespaceStore, sources: BTreeMap, streams: BTreeMap, + blob_reservations: BTreeMap, authority_epoch: u64, } @@ -339,6 +408,7 @@ impl DataDirectoryActor { store, sources, streams: BTreeMap::new(), + blob_reservations: BTreeMap::new(), authority_epoch, }) } @@ -375,6 +445,9 @@ impl DataDirectoryActor { operation_id: OperationId, retired: Option, ) -> Result { + if self.blob_reservations.contains_key(&path) { + return Err(NamespaceError::PathExists(path)); + } let request = MutationRequest::BindStream { path: path.clone() }; if let Some(replayed) = self.replay(operation_id, &request) { return replayed; @@ -388,6 +461,7 @@ impl DataDirectoryActor { let mut next = self.store.snapshot().clone(); next.next_revision = next_revision; next.bindings.remove(&path); + next.stream_nodes.insert(path.clone(), revision); if let Some(retired) = retired && !next.retirements.contains(&retired) { @@ -440,10 +514,42 @@ impl DataDirectoryActor { role, endpoint, replace, + ensure, + expected_revision, descriptor, operation_id, reply_to, } = request; + if !ensure { + let snapshot = self.store.snapshot(); + let Some(current_revision) = snapshot.stream_nodes.get(&path).copied() else { + let error = if snapshot.bindings.contains_key(&path) { + NamespaceError::WrongEntryType { + path, + expected: EntryKind::Stream, + found: EntryKind::Blob, + } + } else { + NamespaceError::PathNotFound(path) + }; + self.send_stream_result(ctx, request_id, reply_to, Err(error)); + return; + }; + let pending_from_expected = matches!( + self.streams.get(&path), + Some(RuntimeStream::Pending(pending)) + if pending.opened_from_revision == expected_revision + ); + if expected_revision != Some(current_revision) && !pending_from_expected { + self.send_stream_result( + ctx, + request_id, + reply_to, + Err(NamespaceError::PathReplaced(path)), + ); + return; + } + } if let Some(RuntimeStream::Pending(pending)) = self.streams.get_mut(&path) && pending.operation_id == operation_id { @@ -583,6 +689,7 @@ impl DataDirectoryActor { request_id, reply_to, revision: receipt.revision, + opened_from_revision: (!ensure).then_some(expected_revision).flatten(), }), ); } @@ -620,6 +727,41 @@ impl DataDirectoryActor { } } + fn reserve_blob( + &mut self, + path: DataPath, + operation_id: OperationId, + ) -> Result<(), NamespaceError> { + if self.store.snapshot().bindings.contains_key(&path) + || self.store.snapshot().stream_nodes.contains_key(&path) + { + return Err(NamespaceError::PathExists(path)); + } + match self.blob_reservations.get(&path) { + Some(existing) if *existing == operation_id => Ok(()), + Some(_) => Err(NamespaceError::PathExists(path)), + None => { + self.blob_reservations.insert(path, operation_id); + Ok(()) + } + } + } + + fn release_blob_reservation( + &mut self, + path: &DataPath, + operation_id: OperationId, + ) -> Result<(), NamespaceError> { + match self.blob_reservations.get(path) { + Some(existing) if *existing == operation_id => { + self.blob_reservations.remove(path); + Ok(()) + } + Some(_) => Err(NamespaceError::OperationConflict(operation_id)), + None => Ok(()), + } + } + fn register( &mut self, path: DataPath, @@ -627,6 +769,7 @@ impl DataDirectoryActor { length: u64, recovery: SourceRecovery, operation_id: OperationId, + reservation: Option, retired: Option, ) -> Result { let request = MutationRequest::Register { @@ -637,6 +780,12 @@ impl DataDirectoryActor { if let Some(replayed) = self.replay(operation_id, &request) { return replayed; } + match self.blob_reservations.get(&path) { + Some(existing) if Some(*existing) == reservation => {} + Some(_) => return Err(NamespaceError::PathExists(path)), + None if reservation.is_some() => return Err(NamespaceError::PathReplaced(path)), + None => {} + } let revision = self.store.snapshot().next_revision; let next_revision = revision .checked_add(1) @@ -645,6 +794,7 @@ impl DataDirectoryActor { let receipt = MutationReceipt { revision }; let mut next = self.store.snapshot().clone(); next.next_revision = next_revision; + next.stream_nodes.remove(&path); next.bindings.insert( path.clone(), PersistedBinding { @@ -666,12 +816,15 @@ impl DataDirectoryActor { }, ); self.store.commit(next)?; + if reservation.is_some() { + self.blob_reservations.remove(&path); + } self.sources.insert(path, RuntimeSource::Available(source)); Ok(receipt) } fn resolve(&self, path: &DataPath) -> Result { - if self.streams.contains_key(path) { + if self.store.snapshot().stream_nodes.contains_key(path) { return Err(NamespaceError::WrongEntryType { path: path.clone(), expected: EntryKind::Blob, @@ -699,17 +852,43 @@ impl DataDirectoryActor { } } + fn lookup(&self, path: &DataPath) -> Result { + if self.blob_reservations.contains_key(path) { + return Err(NamespaceError::PathExists(path.clone())); + } + let snapshot = self.store.snapshot(); + match (snapshot.bindings.get(path), snapshot.stream_nodes.get(path)) { + (Some(binding), None) => Ok(NamespaceNode { + kind: EntryKind::Blob, + revision: binding.revision, + }), + (None, Some(revision)) => Ok(NamespaceNode { + kind: EntryKind::Stream, + revision: *revision, + }), + (None, None) => Err(NamespaceError::PathNotFound(path.clone())), + (Some(_), Some(_)) => Err(NamespaceError::Storage(format!( + "data path {path} is bound as both blob and stream" + ))), + } + } + fn unregister( &mut self, path: DataPath, operation_id: OperationId, retired: Option, ) -> Result { + if self.blob_reservations.contains_key(&path) { + return Err(NamespaceError::PathExists(path)); + } let request = MutationRequest::Unregister { path: path.clone() }; if let Some(replayed) = self.replay(operation_id, &request) { return replayed; } - if !self.store.snapshot().bindings.contains_key(&path) { + if !self.store.snapshot().bindings.contains_key(&path) + && !self.store.snapshot().stream_nodes.contains_key(&path) + { let mut next = self.store.snapshot().clone(); next.operations.insert( operation_id, @@ -732,6 +911,7 @@ impl DataDirectoryActor { let mut next = self.store.snapshot().clone(); next.next_revision = next_revision; next.bindings.remove(&path); + next.stream_nodes.remove(&path); if let Some(retired) = retired && !next.retirements.contains(&retired) { @@ -769,6 +949,7 @@ impl ActorInterface for DataDirectoryActor { length, recovery, operation_id, + reservation, reply_to, } => { let logical = path.clone(); @@ -780,7 +961,15 @@ impl ActorInterface for DataDirectoryActor { RuntimeSource::Available(actor) if *actor != source => Some(*actor), RuntimeSource::Available(_) | RuntimeSource::Unavailable(_) => None, }); - let result = self.register(path, source, length, recovery, operation_id, retired); + let result = self.register( + path, + source, + length, + recovery, + operation_id, + reservation, + retired, + ); if result.is_ok() && let Some(retired) = retired { @@ -813,6 +1002,53 @@ impl ActorInterface for DataDirectoryActor { }), ); } + DataDirectoryIn::Lookup { + request_id, + path, + reply_to, + } => { + let result = self.lookup(&path); + let _ = ctx.send( + reply_to, + NamespaceClientIn::DirectoryReply(DataDirectoryOut::LookedUp { + request_id, + authority_epoch: self.authority_epoch, + result, + }), + ); + } + DataDirectoryIn::ReserveBlob { + request_id, + path, + operation_id, + reply_to, + } => { + let result = self.reserve_blob(path, operation_id); + let _ = ctx.send( + reply_to, + NamespaceClientIn::DirectoryReply(DataDirectoryOut::BlobReserved { + request_id, + authority_epoch: self.authority_epoch, + result, + }), + ); + } + DataDirectoryIn::ReleaseBlobReservation { + request_id, + path, + operation_id, + reply_to, + } => { + let result = self.release_blob_reservation(&path, operation_id); + let _ = ctx.send( + reply_to, + NamespaceClientIn::DirectoryReply(DataDirectoryOut::BlobReservationReleased { + request_id, + authority_epoch: self.authority_epoch, + result, + }), + ); + } DataDirectoryIn::Unregister { request_id, path, @@ -864,6 +1100,8 @@ impl ActorInterface for DataDirectoryActor { endpoint, descriptor, replace, + ensure, + expected_revision, operation_id, reply_to, } => self.open_stream( @@ -874,6 +1112,8 @@ impl ActorInterface for DataDirectoryActor { role, endpoint, replace, + ensure, + expected_revision, descriptor, operation_id, reply_to, @@ -948,6 +1188,7 @@ impl NamespaceClientActor { length, recovery, operation_id, + reservation, } => DataDirectoryIn::Register { request_id, path: path.clone(), @@ -955,6 +1196,7 @@ impl NamespaceClientActor { length: *length, recovery: recovery.clone(), operation_id: *operation_id, + reservation: *reservation, reply_to: ctx.self_addr(), }, NamespaceRequest::Resolve { path } => DataDirectoryIn::Resolve { @@ -962,6 +1204,25 @@ impl NamespaceClientActor { path: path.clone(), reply_to: ctx.self_addr(), }, + NamespaceRequest::Lookup { path } => DataDirectoryIn::Lookup { + request_id, + path: path.clone(), + reply_to: ctx.self_addr(), + }, + NamespaceRequest::ReserveBlob { path, operation_id } => DataDirectoryIn::ReserveBlob { + request_id, + path: path.clone(), + operation_id: *operation_id, + reply_to: ctx.self_addr(), + }, + NamespaceRequest::ReleaseBlobReservation { path, operation_id } => { + DataDirectoryIn::ReleaseBlobReservation { + request_id, + path: path.clone(), + operation_id: *operation_id, + reply_to: ctx.self_addr(), + } + } NamespaceRequest::Unregister { path, operation_id } => DataDirectoryIn::Unregister { request_id, path: path.clone(), @@ -974,6 +1235,8 @@ impl NamespaceClientActor { endpoint, descriptor, replace, + ensure, + expected_revision, operation_id, } => DataDirectoryIn::OpenStream { request_id, @@ -982,6 +1245,8 @@ impl NamespaceClientActor { endpoint: *endpoint, descriptor: descriptor.clone(), replace: *replace, + ensure: *ensure, + expected_revision: *expected_revision, operation_id: *operation_id, reply_to: ctx.self_addr(), }, @@ -1058,6 +1323,25 @@ impl ActorInterface for NamespaceClientActor { self.pending .retain(|_, pending| pending.reply_to != reply_to); } + NamespaceClientIn::CancelBlobReservation { + path, + operation_id, + reply_to, + } => { + self.pending + .retain(|_, pending| pending.reply_to != reply_to); + if let Some(directory) = self.discovery.current_directory() { + let _ = ctx.send( + directory, + DataDirectoryIn::ReleaseBlobReservation { + request_id: DirectoryRequestId(0), + path, + operation_id, + reply_to: ctx.self_addr(), + }, + ); + } + } NamespaceClientIn::DirectoryReply(reply) => { if let Some(pending) = self.pending.remove(&reply.request_id()) { let _ = ctx.send(pending.reply_to, reply); @@ -1155,6 +1439,7 @@ impl NamespaceClient { length, recovery, operation_id, + reservation: None, }) .await? { @@ -1174,6 +1459,15 @@ impl NamespaceClient { } } + pub async fn lookup(&self, path: DataPath) -> Result { + match self.request(NamespaceRequest::Lookup { path }).await? { + DataDirectoryOut::LookedUp { result, .. } => result, + other => Err(NamespaceError::Protocol(format!( + "expected lookup reply, received {other:?}" + ))), + } + } + pub async fn unregister( &self, path: DataPath, @@ -1204,6 +1498,8 @@ impl NamespaceClient { role, endpoint, replace, + ensure: true, + expected_revision: None, descriptor: Vec::new(), operation_id, }) @@ -1335,6 +1631,7 @@ impl DirectoryClient { length, recovery, operation_id, + reservation: None, reply_to: *inbox.addr(), }, ) @@ -1370,6 +1667,29 @@ impl DirectoryClient { } } + pub async fn lookup(&self, path: DataPath) -> Result { + let inbox = self + .runtime + .new_inbox::() + .map_err(|error| NamespaceError::DirectoryUnavailable(error.to_string()))?; + self.runtime + .send_to( + self.directory, + DataDirectoryIn::Lookup { + request_id: self.request_id(), + path, + reply_to: *inbox.addr(), + }, + ) + .map_err(|error| NamespaceError::DirectoryUnavailable(error.to_string()))?; + match self.receive(&inbox).await? { + DataDirectoryOut::LookedUp { result, .. } => result, + other => Err(NamespaceError::Protocol(format!( + "expected lookup reply, received {other:?}" + ))), + } + } + pub async fn unregister( &self, path: DataPath, @@ -1426,6 +1746,8 @@ impl DirectoryClient { role, endpoint, replace, + ensure: true, + expected_revision: None, operation_id, descriptor: Vec::new(), reply_to: *inbox.addr(), diff --git a/crates/data-plane/src/namespace_store.rs b/crates/data-plane/src/namespace_store.rs index 164f0aa..bb6e2e6 100644 --- a/crates/data-plane/src/namespace_store.rs +++ b/crates/data-plane/src/namespace_store.rs @@ -108,6 +108,8 @@ pub struct NamespaceSnapshot { pub authority_epoch: u64, pub next_revision: u64, pub bindings: BTreeMap, + #[serde(default)] + pub stream_nodes: BTreeMap, pub operations: BTreeMap, #[serde(default)] pub retirements: Vec, @@ -121,6 +123,7 @@ impl Default for NamespaceSnapshot { next_revision: 1, bindings: BTreeMap::new(), operations: BTreeMap::new(), + stream_nodes: BTreeMap::new(), retirements: Vec::new(), } } @@ -143,6 +146,10 @@ impl NamespaceSnapshot { .bindings .values() .any(|binding| binding.revision == 0 || binding.revision >= self.next_revision) + || self + .stream_nodes + .values() + .any(|revision| *revision == 0 || *revision >= self.next_revision) { return Err(NamespaceStoreError::Corrupt( "binding revision is outside the committed revision range".to_owned(), diff --git a/crates/data-plane/src/protocol.rs b/crates/data-plane/src/protocol.rs index 3982263..025772c 100644 --- a/crates/data-plane/src/protocol.rs +++ b/crates/data-plane/src/protocol.rs @@ -6,10 +6,12 @@ use serde::{Deserialize, Serialize}; use swactor::actor::ActorAddress; use swactor_transport::{CodecRegistry, JsonCodec, NetworkMessage}; -use crate::blob::{BlobError, BlobLease, BlobMetadata}; +use crate::blob::{BlobError, BlobLease, BlobMetadata, ContentDigest}; use crate::byte_ring::{RingHandle, Role}; use crate::ids::BlobLeaseId; -use crate::namespace::{EntryKind, NamespaceError, StreamIncarnation, StreamMatch}; +use crate::namespace::{ + EntryKind, NamespaceError, NamespaceNode, OperationId, StreamIncarnation, StreamMatch, +}; use crate::path::DataPath; use crate::stream_transport::{StreamPeerDescriptor, StreamTransportEvent}; @@ -45,11 +47,155 @@ impl JobCapability { } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub enum DataOperation { - ReadBlob, - WriteBlob, - ReadStream, - WriteStream, +pub enum AccessMode { + ReadOnly, + WriteOnly, + ReadWrite, +} + +impl AccessMode { + pub const fn can_read(self) -> bool { + matches!(self, Self::ReadOnly | Self::ReadWrite) + } + + pub const fn can_write(self) -> bool { + matches!(self, Self::WriteOnly | Self::ReadWrite) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct BlobAllocation { + pub length: u64, + pub digest: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct OpenOptions { + pub access: AccessMode, + pub create: bool, + pub exclusive: bool, + pub truncate: bool, + pub nonblocking: bool, + pub allocation: Option, +} + +impl OpenOptions { + pub const fn read_only() -> Self { + Self { + access: AccessMode::ReadOnly, + create: false, + exclusive: false, + truncate: false, + nonblocking: false, + allocation: None, + } + } + + pub fn staged_blob(length: u64) -> Self { + Self { + access: AccessMode::WriteOnly, + create: true, + exclusive: false, + truncate: true, + nonblocking: false, + allocation: Some(BlobAllocation { + length, + digest: None, + }), + } + } + + pub fn validate(&self) -> Result<(), DataPlaneError> { + if self.nonblocking { + return Err(DataPlaneError::Unsupported( + "O_NONBLOCK is not supported".to_owned(), + )); + } + if self.exclusive && !self.create { + return Err(DataPlaneError::InvalidArgument( + "O_EXCL requires O_CREAT".to_owned(), + )); + } + if (self.create || self.truncate) && !self.access.can_write() { + return Err(DataPlaneError::InvalidArgument( + "creation and truncation require write access".to_owned(), + )); + } + if self.allocation.is_some() && !(self.access.can_write() && self.truncate) { + return Err(DataPlaneError::InvalidArgument( + "blob allocation requires a truncating writable open".to_owned(), + )); + } + Ok(()) + } +} + +impl Default for OpenOptions { + fn default() -> Self { + Self::read_only() + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum DescriptorKind { + Blob, + Stream, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DescriptorCapabilities(u16); + +impl DescriptorCapabilities { + pub const READ: Self = Self(1 << 0); + pub const WRITE: Self = Self(1 << 1); + pub const MAP_HOST: Self = Self(1 << 2); + pub const MAP_DEVICE: Self = Self(1 << 3); + pub const SEEK: Self = Self(1 << 4); + pub const POLL: Self = Self(1 << 5); + pub const CONTROL: Self = Self(1 << 6); + + pub const fn empty() -> Self { + Self(0) + } + + pub const fn contains(self, capability: Self) -> bool { + self.0 & capability.0 == capability.0 + } + + pub const fn union(self, capability: Self) -> Self { + Self(self.0 | capability.0) + } + + pub const fn bits(self) -> u16 { + self.0 + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum Errno { + Eacces, + Eagain, + Ebadf, + Ebusy, + Ecanceled, + Econnreset, + Eexist, + Einval, + Eio, + Enodev, + Enoent, + Enomem, + Enospc, + Enotsup, + Enxio, + Epipe, + Estale, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum OpenPolicy { + Ordinary, + EnsureStream { replace: bool }, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -98,19 +244,27 @@ impl From for BlobFailure { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum DataPlaneError { InvalidPath(String), + InvalidArgument(String), InvalidCapability, Attachment(AttachmentFailure), SessionNotRunning, SessionFailed(String), Unauthorized { path: DataPath, - operation: DataOperation, + access: AccessMode, }, PathNotFound(DataPath), + PathExists(DataPath), SourceFailure(String), ArenaExhausted, Blob(BlobFailure), OperationCancelled, + BadDescriptor, + Unsupported(String), + MappingUnsupported, + Busy(String), + Stale(String), + BrokenPipe, WrongEntryType { path: DataPath, expected: EntryKind, @@ -126,18 +280,26 @@ impl fmt::Display for DataPlaneError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::InvalidPath(reason) => write!(f, "invalid data path: {reason}"), + Self::InvalidArgument(reason) => write!(f, "invalid argument: {reason}"), Self::InvalidCapability => f.write_str("invalid job capability"), Self::Attachment(reason) => write!(f, "data-plane attachment failed: {reason:?}"), Self::SessionNotRunning => f.write_str("data-plane session is not running"), Self::SessionFailed(reason) => write!(f, "data-plane session failed: {reason}"), - Self::Unauthorized { path, operation } => { - write!(f, "{operation:?} is not authorized for {path}") + Self::Unauthorized { path, access } => { + write!(f, "{access:?} access is not authorized for {path}") } Self::PathNotFound(path) => write!(f, "data path not found: {path}"), + Self::PathExists(path) => write!(f, "data path already exists: {path}"), Self::SourceFailure(reason) => write!(f, "blob source failed: {reason}"), Self::ArenaExhausted => f.write_str("data-plane arena is exhausted"), Self::Blob(reason) => write!(f, "blob lease failure: {reason:?}"), Self::OperationCancelled => f.write_str("data-plane operation was cancelled"), + Self::BadDescriptor => f.write_str("bad descriptor"), + Self::Unsupported(reason) => write!(f, "operation is not supported: {reason}"), + Self::MappingUnsupported => f.write_str("object does not support mapping"), + Self::Busy(reason) => write!(f, "resource is busy: {reason}"), + Self::Stale(reason) => write!(f, "stale capability: {reason}"), + Self::BrokenPipe => f.write_str("stream peer is closed"), Self::WrongEntryType { path, expected, @@ -154,6 +316,44 @@ impl fmt::Display for DataPlaneError { } } +impl DataPlaneError { + pub const fn errno(&self) -> Errno { + match self { + Self::InvalidPath(_) | Self::InvalidArgument(_) | Self::InvalidCapability => { + Errno::Einval + } + Self::Attachment(_) + | Self::SessionNotRunning + | Self::SessionFailed(_) + | Self::SourceFailure(_) + | Self::StreamFault(_) => Errno::Eio, + Self::Unauthorized { .. } => Errno::Eacces, + Self::PathNotFound(_) => Errno::Enoent, + Self::PathExists(_) => Errno::Eexist, + Self::ArenaExhausted => Errno::Enospc, + Self::Blob(BlobFailure::Bounds | BlobFailure::Length { .. }) => Errno::Einval, + Self::Blob(BlobFailure::StaleGeneration { .. }) | Self::Stale(_) => Errno::Estale, + Self::Blob(BlobFailure::Access) | Self::BadDescriptor | Self::StreamClosed => { + Errno::Ebadf + } + Self::Blob(BlobFailure::ActiveWritableView) | Self::Busy(_) => Errno::Ebusy, + Self::Blob( + BlobFailure::Digest + | BlobFailure::State { .. } + | BlobFailure::InvalidLease + | BlobFailure::AlreadyFinished, + ) => Errno::Eio, + Self::OperationCancelled => Errno::Ecanceled, + Self::Unsupported(_) => Errno::Enotsup, + Self::MappingUnsupported => Errno::Enodev, + Self::BrokenPipe => Errno::Epipe, + Self::WrongEntryType { .. } => Errno::Enxio, + Self::PathReplaced(_) => Errno::Estale, + Self::PeerLost => Errno::Econnreset, + } + } +} + impl std::error::Error for DataPlaneError {} impl From for DataPlaneError { @@ -170,33 +370,24 @@ pub enum HostSessionIn { job_capability: JobCapability, child_node: Option<[u8; 32]>, }, - OpenReadBlob { + Open { path: DataPath, + options: OpenOptions, + policy: OpenPolicy, child_session: ActorAddress, operation: ActorAddress, }, - CancelReadBlob { + OpenResolved { operation: ActorAddress, + result: Result, }, - OpenWriteBlob { + BlobReserved { + operation: ActorAddress, path: DataPath, - length: u64, - child_session: ActorAddress, - operation: ActorAddress, + reservation: OperationId, + result: Result<(), NamespaceError>, }, - OpenReadStream { - path: DataPath, - child_session: ActorAddress, - operation: ActorAddress, - replace: bool, - }, - OpenWriteStream { - path: DataPath, - child_session: ActorAddress, - operation: ActorAddress, - replace: bool, - }, - CancelStream { + CancelOpen { operation: ActorAddress, }, StreamControl { @@ -253,16 +444,13 @@ pub enum ChildSessionIn { error: DataPlaneError, }, AttachmentDeadline, - ReadBlob { + Open { path: DataPath, + options: OpenOptions, + policy: OpenPolicy, reply_to: ActorAddress, }, - CancelRead { - reply_to: ActorAddress, - }, - OpenWriteBlob { - path: DataPath, - length: u64, + CancelOpen { reply_to: ActorAddress, }, OpenReadStream { @@ -270,11 +458,6 @@ pub enum ChildSessionIn { reply_to: ActorAddress, replace: bool, }, - OpenWriteStream { - path: DataPath, - reply_to: ActorAddress, - replace: bool, - }, CancelStream { reply_to: ActorAddress, }, @@ -356,6 +539,10 @@ pub enum HostStreamIn { PeerTerminated { incarnation: StreamIncarnation, error: DataPlaneError, + reply_to: Option, + }, + PeerTerminationAck { + incarnation: StreamIncarnation, }, ReleaseComplete(Result<(), DataPlaneError>), } diff --git a/crates/data-plane/tests/actor_blob_guarantees.rs b/crates/data-plane/tests/actor_blob_guarantees.rs index 2337839..b173bc2 100755 --- a/crates/data-plane/tests/actor_blob_guarantees.rs +++ b/crates/data-plane/tests/actor_blob_guarantees.rs @@ -1,357 +1,6 @@ #![cfg(target_os = "linux")] -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::Duration; - -use data_plane::arena::{ArenaConfig, ArenaManager, NodeId}; -use data_plane::blob::{ - BLOB_HEADER_LEN, Blob, BlobError, BlobLease, BlobMetadata, BlobSharedState, LeaseReleaser, -}; -use data_plane::bootstrap::{self, BootstrapSpec}; -use data_plane::data_plane::DataPlaneBootstrap; -use data_plane::host::{HostDataPlaneConfig, HostDataPlaneSessionActor}; -use data_plane::path::{DataPath, JobContext}; -use data_plane::protocol::{DataPlaneError, JobCapability}; -use futures_lite::future::{self, FutureExt}; -use swactor::Error; -use swactor::actor::ActorAddress; -use swactor::config::RuntimeConfig; -use swactor::runtime::{RemoteSink, Runtime, RuntimeParts}; -use swactor_engine::{Engine, TokioBackend, TokioConfig}; - -const CAPABILITY: JobCapability = JobCapability::new([9; 32]); -const ARENA_GENERATION: u64 = 17; -const SESSION_GENERATION: u64 = 29; -const WEIGHTS: &[u8] = b"0123456789abcdefghijklmn"; -static STREAM_TEST_LOCK: parking_lot::Mutex<()> = parking_lot::Mutex::new(()); - -struct DirectRuntimeSink { - destination: Runtime, -} - -impl RemoteSink for DirectRuntimeSink { - fn send( - &self, - address: ActorAddress, - message: Box, - ) -> Result<(), Error> { - self.destination.deliver_raw(address, message) - } -} - -struct BlackHoleSink; - -impl RemoteSink for BlackHoleSink { - fn send( - &self, - _address: ActorAddress, - _message: Box, - ) -> Result<(), Error> { - Ok(()) - } -} - -struct Harness { - _host_engine: Engine, - _child_engine: Engine, - _temp: TempState, - bootstrap: DataPlaneBootstrap, -} - -static NEXT_TEMP: AtomicU64 = AtomicU64::new(1); - -struct TempState { - root: std::path::PathBuf, -} - -impl TempState { - fn new() -> Self { - let sequence = NEXT_TEMP.fetch_add(1, Ordering::Relaxed); - let root = std::env::temp_dir().join(format!( - "swactor-actor-blob-{}-{sequence}", - std::process::id() - )); - std::fs::create_dir_all(&root).unwrap(); - Self { root } - } -} - -impl Drop for TempState { - fn drop(&mut self) { - let _ = std::fs::remove_dir_all(&self.root); - } -} - -fn path(value: &str) -> DataPath { - DataPath::parse(value).expect("test path") -} - -fn runtime_parts() -> (RuntimeParts, Runtime) { - let parts = RuntimeParts::new(RuntimeConfig { - worker_count: 1, - ..RuntimeConfig::default() - }); - let runtime = parts.runtime().clone(); - (parts, runtime) -} - -struct LoopbackSender { - runtime: Runtime, -} - -impl data_plane::blob_transfer::BlobTransferSender for LoopbackSender { - fn start_file( - &self, - request: data_plane::blob_transfer::FileTransferRequest, - ) -> Result<(), String> { - use std::os::unix::fs::FileExt; - let mut bytes = vec![0_u8; request.length as usize]; - request - .file - .read_exact_at(&mut bytes, request.offset) - .map_err(|error| error.to_string())?; - self.runtime - .send_to( - request.offer.destination, - data_plane::blob_transfer::BlobTransferEvent::Chunk { - transfer_id: request.offer.transfer_id, - bytes, - }, - ) - .map_err(|error| error.to_string())?; - self.runtime - .send_to( - request.offer.destination, - data_plane::blob_transfer::BlobTransferEvent::Finished { - transfer_id: request.offer.transfer_id, - }, - ) - .map_err(|error| error.to_string())?; - request.completion.complete(Ok(())); - Ok(()) - } -} - -struct DirectReceiver; - -impl data_plane::blob_transfer::BlobTransferReceiver for DirectReceiver { - fn open( - &self, - destination: ActorAddress, - transfer_id: data_plane::blob_transfer::BlobTransferId, - ) -> Result { - Ok(data_plane::blob_transfer::BlobTransferOffer { - transfer_id, - destination, - failure_proxy: None, - transport: Vec::new(), - }) - } - - fn cancel(&self, _offer: &data_plane::blob_transfer::BlobTransferOffer) {} -} - -struct StaticDiscovery(ActorAddress); - -impl data_plane::namespace::NamespaceDiscovery for StaticDiscovery { - fn current_directory(&self) -> Option { - Some(self.0) - } -} - -struct NoopSourceRegistrar; - -impl data_plane::source::BlobSourcePublisher for NoopSourceRegistrar { - fn publish_source(&self, _source: ActorAddress) -> Result<(), String> { - Ok(()) - } -} - -struct RejectingStreamTransport; - -impl data_plane::stream_transport::StreamTransport for RejectingStreamTransport { - fn descriptor(&self) -> Result { - Ok(data_plane::stream_transport::StreamPeerDescriptor(vec![1])) - } - - fn install_source( - &self, - _request: data_plane::stream_transport::StreamSourceRequest, - ) -> Result<(), String> { - Err("injected source transport failure".to_owned()) - } - - fn install_sink( - &self, - _request: data_plane::stream_transport::StreamSinkRequest, - ) -> Result<(), String> { - Err("injected sink transport failure".to_owned()) - } - - fn source_progress(&self, _incarnation: data_plane::namespace::StreamIncarnation) {} - - fn sink_progress(&self, _incarnation: data_plane::namespace::StreamIncarnation) {} - - fn terminate(&self, _incarnation: data_plane::namespace::StreamIncarnation) {} -} - -struct CollectBytes(Arc>>); - -impl data_plane::data_plane::StreamConsumer for CollectBytes { - fn consume(&self, bytes: &[u8]) -> Result<(), String> { - self.0.lock().extend_from_slice(bytes); - Ok(()) - } -} - -fn harness(arena_bytes: u64) -> Harness { - harness_with_transport( - arena_bytes, - Arc::new(data_plane::stream_transport::LocalStreamTransport::new()), - ) -} - -fn harness_with_transport( - arena_bytes: u64, - stream_transport: Arc, -) -> Harness { - let temp = TempState::new(); - let mut arena = ArenaManager::boot(ArenaConfig { - node_id: NodeId(1), - reservation_ceiling: arena_bytes, - base_alignment: 64, - }) - .expect("host arena"); - let handoff = bootstrap::write_bootstrap( - &mut arena, - BootstrapSpec { - arena_generation: ARENA_GENERATION, - alignment: 64, - }, - ) - .expect("bootstrap"); - - let (host_parts, host_runtime) = runtime_parts(); - let (child_parts, child_runtime) = runtime_parts(); - host_runtime.set_remote_sink(Arc::new(DirectRuntimeSink { - destination: child_runtime.clone(), - })); - child_runtime.set_remote_sink(Arc::new(DirectRuntimeSink { - destination: host_runtime.clone(), - })); - let host_engine = Engine::new( - host_parts, - TokioBackend::new(TokioConfig { - worker_threads: 1, - ..TokioConfig::default() - }) - .expect("host backend"), - ) - .expect("host engine"); - let child_engine = Engine::new( - child_parts, - TokioBackend::new(TokioConfig { - worker_threads: 1, - ..TokioConfig::default() - }) - .expect("child backend"), - ) - .expect("child engine"); - - let sender: Arc = Arc::new(LoopbackSender { - runtime: host_runtime.clone(), - }); - let directory_actor = data_plane::namespace::DataDirectoryActor::recover( - temp.root.join("namespace.json"), - |_record, _length| { - Err(data_plane::namespace::NamespaceError::SourceRecovery( - "unexpected recovery".to_owned(), - )) - }, - ) - .unwrap(); - let directory = host_runtime.spawn(directory_actor).unwrap(); - let directory_client = - data_plane::namespace::DirectoryClient::new(host_runtime.clone(), directory); - for (logical, name, bytes) in [ - ("/models/tiny-linear/weights", "weights.bin", WEIGHTS), - ("/models/second", "second.bin", b"second-blob".as_slice()), - ] { - let file_path = temp.root.join(name); - std::fs::write(&file_path, bytes).unwrap(); - let source = data_plane::source::FileBlobSourceActor::open( - host_runtime.clone(), - Arc::clone(&sender), - &file_path, - ) - .unwrap(); - let length = source.length(); - let recovery = source.recovery(); - let source = host_runtime.spawn(source).unwrap(); - future::block_on(directory_client.register( - path(logical), - source, - length, - recovery, - data_plane::namespace::OperationId::from_u128(u128::from(length) + 1), - )) - .unwrap(); - } - let proxy = host_runtime - .spawn(data_plane::namespace::NamespaceClientActor::new( - host_engine.handle(), - host_runtime.create_sender(), - Arc::new(StaticDiscovery(directory)), - Duration::from_millis(5), - )) - .unwrap(); - let namespace = data_plane::namespace::NamespaceClient::new(host_runtime.clone(), proxy); - let host_session = host_runtime - .spawn( - HostDataPlaneSessionActor::new(HostDataPlaneConfig { - runtime: host_runtime.clone(), - arena, - arena_generation: ARENA_GENERATION, - session_generation: SESSION_GENERATION, - capability: CAPABILITY, - job_context: JobContext { - run_id: "run-7".to_owned(), - read_prefixes: vec![path("/models"), path("/runs/run-7/results")], - write_prefixes: vec![path("/runs/run-7/results")], - }, - namespace: Some(namespace), - transfer_receiver: Some(Arc::new(DirectReceiver)), - source_sender: Some(sender), - source_publisher: Some(Arc::new(NoopSourceRegistrar)), - route_registrar: None, - stream_transport: Some(stream_transport), - }) - .expect("host session config"), - ) - .expect("spawn host session"); - let bootstrap = future::block_on(DataPlaneBootstrap::attach( - handoff.arena_fd, - child_runtime, - host_session, - CAPABILITY, - )) - .expect("routed attachment"); - - Harness { - _host_engine: host_engine, - _child_engine: child_engine, - _temp: temp, - bootstrap, - } -} - -#[derive(Default)] -struct NoopReleaser; - -impl LeaseReleaser for NoopReleaser { - fn release(&self, _lease: BlobLease) {} -} +include!("data_plane_test_support.inc"); #[test] fn attachment_without_a_host_reply_fails_on_actor_deadline() { @@ -874,3 +523,190 @@ fn actor_stream_consumer_registers_before_writer_and_collects_to_eof() { completion.wait().expect("collector completes"); assert_eq!(&*observed.lock(), b"actor-consumer"); } + +#[test] +fn raw_blob_descriptor_enforces_offsets_rights_and_terminal_state() { + let harness = harness(2 << 20); + let data_plane = harness.bootstrap.data_plane.clone(); + let logical = path("/runs/self/results/raw-blob"); + + future::block_on(async { + let mut writer = data_plane + .open(&logical, OpenOptions::staged_blob(8)) + .await + .expect("open staged descriptor"); + assert_eq!(writer.kind(), DescriptorKind::Blob); + assert!( + writer + .capabilities() + .contains(DescriptorCapabilities::WRITE) + ); + assert_eq!(writer.write(b"abc").await.expect("first write"), 3); + assert_eq!(writer.write(b"defgh").await.expect("second write"), 5); + assert_eq!(writer.last_route(), Some(TransferRoute::Staged)); + assert_eq!( + writer + .write(b"!") + .await + .expect_err("growth rejected") + .errno(), + Errno::Enotsup + ); + writer.close().await.expect("publish"); + assert_eq!( + writer.close().await.expect_err("double close").errno(), + Errno::Ebadf + ); + + let mut reader = data_plane + .open(&logical, OpenOptions::read_only()) + .await + .expect("open published descriptor"); + assert_eq!(reader.kind(), DescriptorKind::Blob); + let mut first = [0xa5; 5]; + reader + .read_exact(&mut first[..3]) + .await + .expect("exact prefix read"); + assert_eq!(&first, b"abc\xa5\xa5"); + let mut rest = [0_u8; 8]; + assert_eq!(reader.read(&mut rest).await.expect("remaining read"), 5); + assert_eq!(reader.read(&mut rest).await.expect("eof"), 0); + assert_eq!( + reader.write(b"x").await.expect_err("wrong access").errno(), + Errno::Ebadf + ); + assert_eq!(reader.read(&mut []).await.expect("zero length"), 0); + reader.close().await.expect("close reader"); + + let mut source = data_plane + .open(&logical, OpenOptions::read_only()) + .await + .expect("open arena-region source"); + let target_path = path("/runs/self/results/raw-region-target"); + let mut target = data_plane + .open(&target_path, OpenOptions::staged_blob(8)) + .await + .expect("open arena-region target"); + let mut target_mapping = target + .map(MapRequest { + protection: Protection::ReadWrite, + sharing: Sharing::Shared, + target: MapTarget::Host, + offset: 0, + length: 8, + }) + .expect("map arena target"); + assert_eq!(target_mapping.route(), TransferRoute::Direct); + let count = source + .read_into(RegionSlice::arena( + target_mapping.as_mut().expect("writable arena region"), + )) + .await + .expect("read into arena region"); + assert_eq!(count, 8); + assert_eq!(target_mapping.as_ref(), b"abcdefgh"); + drop(target_mapping); + target.abort().await.expect("abort arena target"); + source.close().await.expect("close arena source"); + assert_eq!( + reader + .read(&mut rest) + .await + .expect_err("read after close") + .errno(), + Errno::Ebadf + ); + }); +} + +#[test] +fn raw_stream_descriptor_hides_record_boundaries_and_preserves_eof() { + let _stream_test = STREAM_TEST_LOCK.lock(); + let harness = harness(2 << 20); + let data_plane = harness.bootstrap.data_plane.clone(); + let logical = path("/runs/self/results/raw-stream"); + + future::block_on(async { + let typed_reader = data_plane.read_stream(&logical); + let typed_writer = data_plane.write_stream(&logical); + let (typed_reader, typed_writer) = future::zip(typed_reader, typed_writer).await; + let mut typed_reader = typed_reader.expect("seed stream reader"); + let mut typed_writer = typed_writer.expect("seed stream writer"); + typed_writer.close().await.expect("seed close"); + assert_eq!(typed_reader.read().await.expect("seed eof"), None); + + let raw_reader = data_plane.open(&logical, OpenOptions::read_only()); + let raw_writer = data_plane.open( + &logical, + OpenOptions { + access: AccessMode::WriteOnly, + ..OpenOptions::default() + }, + ); + let (raw_reader, raw_writer) = future::zip(raw_reader, raw_writer).await; + let mut reader = raw_reader.expect("raw stream reader"); + let mut writer = raw_writer.expect("raw stream writer"); + assert_eq!(reader.kind(), DescriptorKind::Stream); + writer + .write_all(b"abcdefgh") + .await + .expect("stream write all"); + writer.close().await.expect("writer close"); + + let mut chunk = [0_u8; 3]; + assert_eq!(reader.read(&mut chunk).await.expect("chunk one"), 3); + assert_eq!(&chunk, b"abc"); + assert_eq!(reader.read(&mut chunk).await.expect("chunk two"), 3); + assert_eq!(&chunk, b"def"); + assert_eq!(reader.read(&mut chunk).await.expect("chunk three"), 2); + assert_eq!(&chunk[..2], b"gh"); + assert_eq!(reader.read(&mut chunk).await.expect("stream eof"), 0); + assert_eq!(reader.read(&mut chunk).await.expect("sticky eof"), 0); + reader.close().await.expect("reader close"); + }); +} + +#[test] +fn raw_blob_mapping_is_bounded_and_can_outlive_descriptor_close() { + let harness = harness(2 << 20); + let data_plane = harness.bootstrap.data_plane.clone(); + let logical = path("/runs/self/results/raw-mapped-blob"); + + future::block_on(async { + let mut writer = data_plane + .open(&logical, OpenOptions::staged_blob(6)) + .await + .expect("open mapped writer"); + let mut mapping = writer + .map(MapRequest { + protection: Protection::ReadWrite, + sharing: Sharing::Shared, + target: MapTarget::Host, + offset: 1, + length: 4, + }) + .expect("bounded writable mapping"); + assert_eq!(mapping.route(), TransferRoute::Direct); + mapping + .as_mut() + .expect("writable mapping") + .copy_from_slice(b"data"); + writer.close().await.expect("deferred close intent"); + assert_eq!(mapping.as_ref(), b"data"); + drop(mapping); + + let deadline = std::time::Instant::now() + Duration::from_secs(2); + let blob = loop { + match data_plane.read_blob(&logical).await { + Ok(blob) => break blob, + Err(DataPlaneError::PathNotFound(_)) if std::time::Instant::now() < deadline => { + future::yield_now().await; + } + Err(error) => panic!("deferred publication failed: {error}"), + } + }; + let view = blob.map().expect("published mapping"); + assert_eq!(&view[..], b"\0data\0"); + }); +} diff --git a/crates/data-plane/tests/byte_ring_guarantees.rs b/crates/data-plane/tests/byte_ring_guarantees.rs index 47f8b35..8e1134a 100644 --- a/crates/data-plane/tests/byte_ring_guarantees.rs +++ b/crates/data-plane/tests/byte_ring_guarantees.rs @@ -557,3 +557,73 @@ fn writable_record_is_invisible_until_commit() { .collect(); assert_eq!(observed, (0..17).collect::>()); } + +#[test] +fn partial_record_cursor_keeps_capacity_pinned_until_full_release() { + let (arena, handle) = installed(32, 1); + let mut producer = attach(&arena, handle, Role::Producer).expect("producer"); + let mut consumer = attach(&arena, handle, Role::Consumer).expect("consumer"); + producer + .send_record(RecordKind::Data, b"abcdefgh") + .expect("record"); + + let cursor = consumer + .record_cursor() + .expect("cursor") + .expect("record visible"); + let mut first = [0xa5; 5]; + assert_eq!( + consumer + .copy_record_range(cursor, 0, &mut first[..3]) + .expect("partial prefix"), + 3 + ); + assert_eq!(&first, b"abc\xa5\xa5"); + assert!(matches!( + producer.reserve(20), + Err(FlowError::InsufficientSpace { .. }) + )); + + let mut rest = [0_u8; 8]; + assert_eq!( + consumer + .copy_record_range(cursor, 3, &mut rest) + .expect("partial suffix"), + 5 + ); + assert_eq!(&rest[..5], b"defgh"); + consumer + .release_record_cursor(cursor) + .expect("release complete record"); + assert!(producer.reserve(20).is_ok()); +} + +#[test] +fn partial_record_cursor_hides_payload_wraparound() { + let (arena, handle) = installed(32, 1); + let mut producer = attach(&arena, handle, Role::Producer).expect("producer"); + let mut consumer = attach(&arena, handle, Role::Consumer).expect("consumer"); + producer + .send_record(RecordKind::Data, &[1; 18]) + .expect("advance cursor"); + consumer.recv_record().expect("consume advance"); + producer + .send_record(RecordKind::Data, b"0123456789abcde") + .expect("wrapped record"); + + let cursor = consumer + .record_cursor() + .expect("cursor") + .expect("wrapped record visible"); + let mut observed = [0_u8; 15]; + assert_eq!( + consumer + .copy_record_range(cursor, 0, &mut observed) + .expect("copy wrapped payload"), + observed.len() + ); + assert_eq!(&observed, b"0123456789abcde"); + consumer + .release_record_cursor(cursor) + .expect("release wrapped record"); +} diff --git a/crates/data-plane/tests/data_plane_test_support.inc b/crates/data-plane/tests/data_plane_test_support.inc new file mode 100644 index 0000000..9e601a4 --- /dev/null +++ b/crates/data-plane/tests/data_plane_test_support.inc @@ -0,0 +1,364 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use data_plane::arena::{ArenaConfig, ArenaManager, NodeId}; +#[allow(unused_imports)] +use data_plane::blob::{ + BLOB_HEADER_LEN, Blob, BlobError, BlobLease, BlobMetadata, BlobSharedState, LeaseReleaser, +}; +use data_plane::bootstrap::{self, BootstrapSpec}; +#[allow(unused_imports)] +use data_plane::data_plane::{ + DataPlaneBootstrap, MapRequest, MapTarget, Protection, RegionSlice, Sharing, TransferRoute, +}; +use data_plane::host::{HostDataPlaneConfig, HostDataPlaneSessionActor}; +use data_plane::path::{DataPath, JobContext}; +#[allow(unused_imports)] +use data_plane::protocol::{ + AccessMode, DataPlaneError, DescriptorCapabilities, DescriptorKind, Errno, JobCapability, + OpenOptions, +}; +#[allow(unused_imports)] +use futures_lite::future::{self, FutureExt}; +use swactor::Error; +use swactor::actor::ActorAddress; +use swactor::config::RuntimeConfig; +use swactor::runtime::{RemoteSink, Runtime, RuntimeParts}; +use swactor_engine::{Engine, TokioBackend, TokioConfig}; + +const CAPABILITY: JobCapability = JobCapability::new([9; 32]); +const ARENA_GENERATION: u64 = 17; +const SESSION_GENERATION: u64 = 29; +const WEIGHTS: &[u8] = b"0123456789abcdefghijklmn"; +static STREAM_TEST_LOCK: parking_lot::Mutex<()> = parking_lot::Mutex::new(()); + +struct DirectRuntimeSink { + destination: Runtime, +} + +impl RemoteSink for DirectRuntimeSink { + fn send( + &self, + address: ActorAddress, + message: Box, + ) -> Result<(), Error> { + self.destination.deliver_raw(address, message) + } +} + +#[allow(dead_code)] +struct BlackHoleSink; + +impl RemoteSink for BlackHoleSink { + fn send( + &self, + _address: ActorAddress, + _message: Box, + ) -> Result<(), Error> { + Ok(()) + } +} + +struct Harness { + _host_engine: Engine, + _child_engine: Engine, + _temp: TempState, + bootstrap: DataPlaneBootstrap, +} + +static NEXT_TEMP: AtomicU64 = AtomicU64::new(1); + +struct TempState { + root: std::path::PathBuf, +} + +impl TempState { + fn new() -> Self { + let sequence = NEXT_TEMP.fetch_add(1, Ordering::Relaxed); + let root = std::env::temp_dir().join(format!( + "swactor-actor-blob-{}-{sequence}", + std::process::id() + )); + std::fs::create_dir_all(&root).unwrap(); + Self { root } + } +} + +impl Drop for TempState { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.root); + } +} + +fn path(value: &str) -> DataPath { + DataPath::parse(value).expect("test path") +} + +fn runtime_parts() -> (RuntimeParts, Runtime) { + let parts = RuntimeParts::new(RuntimeConfig { + worker_count: 1, + ..RuntimeConfig::default() + }); + let runtime = parts.runtime().clone(); + (parts, runtime) +} + +struct LoopbackSender { + runtime: Runtime, +} + +impl data_plane::blob_transfer::BlobTransferSender for LoopbackSender { + fn start_file( + &self, + request: data_plane::blob_transfer::FileTransferRequest, + ) -> Result<(), String> { + use std::os::unix::fs::FileExt; + let mut bytes = vec![0_u8; request.length as usize]; + request + .file + .read_exact_at(&mut bytes, request.offset) + .map_err(|error| error.to_string())?; + self.runtime + .send_to( + request.offer.destination, + data_plane::blob_transfer::BlobTransferEvent::Chunk { + transfer_id: request.offer.transfer_id, + bytes, + }, + ) + .map_err(|error| error.to_string())?; + self.runtime + .send_to( + request.offer.destination, + data_plane::blob_transfer::BlobTransferEvent::Finished { + transfer_id: request.offer.transfer_id, + }, + ) + .map_err(|error| error.to_string())?; + request.completion.complete(Ok(())); + Ok(()) + } +} + +struct DirectReceiver; + +impl data_plane::blob_transfer::BlobTransferReceiver for DirectReceiver { + fn open( + &self, + destination: ActorAddress, + transfer_id: data_plane::blob_transfer::BlobTransferId, + ) -> Result { + Ok(data_plane::blob_transfer::BlobTransferOffer { + transfer_id, + destination, + failure_proxy: None, + transport: Vec::new(), + }) + } + + fn cancel(&self, _offer: &data_plane::blob_transfer::BlobTransferOffer) {} +} + +struct StaticDiscovery(ActorAddress); + +impl data_plane::namespace::NamespaceDiscovery for StaticDiscovery { + fn current_directory(&self) -> Option { + Some(self.0) + } +} + +struct NoopSourceRegistrar; + +impl data_plane::source::BlobSourcePublisher for NoopSourceRegistrar { + fn publish_source(&self, _source: ActorAddress) -> Result<(), String> { + Ok(()) + } +} + +struct RejectingStreamTransport; + +impl data_plane::stream_transport::StreamTransport for RejectingStreamTransport { + fn descriptor(&self) -> Result { + Ok(data_plane::stream_transport::StreamPeerDescriptor(vec![1])) + } + + fn install_source( + &self, + _request: data_plane::stream_transport::StreamSourceRequest, + ) -> Result<(), String> { + Err("injected source transport failure".to_owned()) + } + + fn install_sink( + &self, + _request: data_plane::stream_transport::StreamSinkRequest, + ) -> Result<(), String> { + Err("injected sink transport failure".to_owned()) + } + + fn source_progress(&self, _incarnation: data_plane::namespace::StreamIncarnation) {} + + fn sink_progress(&self, _incarnation: data_plane::namespace::StreamIncarnation) {} + + fn terminate(&self, _incarnation: data_plane::namespace::StreamIncarnation) {} +} + +#[allow(dead_code)] +struct CollectBytes(Arc>>); + +impl data_plane::data_plane::StreamConsumer for CollectBytes { + fn consume(&self, bytes: &[u8]) -> Result<(), String> { + self.0.lock().extend_from_slice(bytes); + Ok(()) + } +} + +fn harness(arena_bytes: u64) -> Harness { + harness_with_transport( + arena_bytes, + Arc::new(data_plane::stream_transport::LocalStreamTransport::new()), + ) +} + +fn harness_with_transport( + arena_bytes: u64, + stream_transport: Arc, +) -> Harness { + let temp = TempState::new(); + let mut arena = ArenaManager::boot(ArenaConfig { + node_id: NodeId(1), + reservation_ceiling: arena_bytes, + base_alignment: 64, + }) + .expect("host arena"); + let handoff = bootstrap::write_bootstrap( + &mut arena, + BootstrapSpec { + arena_generation: ARENA_GENERATION, + alignment: 64, + }, + ) + .expect("bootstrap"); + + let (host_parts, host_runtime) = runtime_parts(); + let (child_parts, child_runtime) = runtime_parts(); + host_runtime.set_remote_sink(Arc::new(DirectRuntimeSink { + destination: child_runtime.clone(), + })); + child_runtime.set_remote_sink(Arc::new(DirectRuntimeSink { + destination: host_runtime.clone(), + })); + let host_engine = Engine::new( + host_parts, + TokioBackend::new(TokioConfig { + worker_threads: 1, + ..TokioConfig::default() + }) + .expect("host backend"), + ) + .expect("host engine"); + let child_engine = Engine::new( + child_parts, + TokioBackend::new(TokioConfig { + worker_threads: 1, + ..TokioConfig::default() + }) + .expect("child backend"), + ) + .expect("child engine"); + + let sender: Arc = Arc::new(LoopbackSender { + runtime: host_runtime.clone(), + }); + let directory_actor = data_plane::namespace::DataDirectoryActor::recover( + temp.root.join("namespace.json"), + |_record, _length| { + Err(data_plane::namespace::NamespaceError::SourceRecovery( + "unexpected recovery".to_owned(), + )) + }, + ) + .unwrap(); + let directory = host_runtime.spawn(directory_actor).unwrap(); + let directory_client = + data_plane::namespace::DirectoryClient::new(host_runtime.clone(), directory); + for (logical, name, bytes) in [ + ("/models/tiny-linear/weights", "weights.bin", WEIGHTS), + ("/models/second", "second.bin", b"second-blob".as_slice()), + ] { + let file_path = temp.root.join(name); + std::fs::write(&file_path, bytes).unwrap(); + let source = data_plane::source::FileBlobSourceActor::open( + host_runtime.clone(), + Arc::clone(&sender), + &file_path, + ) + .unwrap(); + let length = source.length(); + let recovery = source.recovery(); + let source = host_runtime.spawn(source).unwrap(); + future::block_on(directory_client.register( + path(logical), + source, + length, + recovery, + data_plane::namespace::OperationId::from_u128(u128::from(length) + 1), + )) + .unwrap(); + } + let proxy = host_runtime + .spawn(data_plane::namespace::NamespaceClientActor::new( + host_engine.handle(), + host_runtime.create_sender(), + Arc::new(StaticDiscovery(directory)), + Duration::from_millis(5), + )) + .unwrap(); + let namespace = data_plane::namespace::NamespaceClient::new(host_runtime.clone(), proxy); + let host_session = host_runtime + .spawn( + HostDataPlaneSessionActor::new(HostDataPlaneConfig { + runtime: host_runtime.clone(), + arena, + arena_generation: ARENA_GENERATION, + session_generation: SESSION_GENERATION, + capability: CAPABILITY, + job_context: JobContext { + run_id: "run-7".to_owned(), + read_prefixes: vec![path("/models"), path("/runs/run-7/results")], + write_prefixes: vec![path("/runs/run-7/results")], + }, + namespace: Some(namespace), + transfer_receiver: Some(Arc::new(DirectReceiver)), + source_sender: Some(sender), + source_publisher: Some(Arc::new(NoopSourceRegistrar)), + route_registrar: None, + stream_transport: Some(stream_transport), + }) + .expect("host session config"), + ) + .expect("spawn host session"); + let bootstrap = future::block_on(DataPlaneBootstrap::attach( + handoff.arena_fd, + child_runtime, + host_session, + CAPABILITY, + )) + .expect("routed attachment"); + + Harness { + _host_engine: host_engine, + _child_engine: child_engine, + _temp: temp, + bootstrap, + } +} + +#[allow(dead_code)] +#[derive(Default)] +struct NoopReleaser; + +impl LeaseReleaser for NoopReleaser { + fn release(&self, _lease: BlobLease) {} +} diff --git a/crates/data-plane/tests/descriptor_guarantees.rs b/crates/data-plane/tests/descriptor_guarantees.rs new file mode 100755 index 0000000..7e75655 --- /dev/null +++ b/crates/data-plane/tests/descriptor_guarantees.rs @@ -0,0 +1,612 @@ +#![cfg(target_os = "linux")] + +include!("data_plane_test_support.inc"); + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::cell::Cell; + +use proptest::prelude::*; + +struct ThreadCountingAllocator; + +thread_local! { + static COUNT_ALLOCATIONS: Cell = const { Cell::new(false) }; + static ALLOCATION_COUNT: Cell = const { Cell::new(0) }; +} + +#[global_allocator] +static COUNTING_ALLOCATOR: ThreadCountingAllocator = ThreadCountingAllocator; + +unsafe impl GlobalAlloc for ThreadCountingAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + COUNT_ALLOCATIONS.with(|enabled| { + if enabled.get() { + ALLOCATION_COUNT.with(|count| count.set(count.get() + 1)); + } + }); + // SAFETY: this allocator delegates the unchanged layout to `System`. + unsafe { System.alloc(layout) } + } + + unsafe fn dealloc(&self, pointer: *mut u8, layout: Layout) { + // SAFETY: `pointer` came from `System` with this layout. + unsafe { System.dealloc(pointer, layout) }; + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + COUNT_ALLOCATIONS.with(|enabled| { + if enabled.get() { + ALLOCATION_COUNT.with(|count| count.set(count.get() + 1)); + } + }); + // SAFETY: this allocator delegates the unchanged layout to `System`. + unsafe { System.alloc_zeroed(layout) } + } + + unsafe fn realloc(&self, pointer: *mut u8, layout: Layout, size: usize) -> *mut u8 { + COUNT_ALLOCATIONS.with(|enabled| { + if enabled.get() { + ALLOCATION_COUNT.with(|count| count.set(count.get() + 1)); + } + }); + // SAFETY: `pointer` and `layout` came from `System`. + unsafe { System.realloc(pointer, layout, size) } + } +} + +fn start_allocation_count() { + ALLOCATION_COUNT.with(|count| count.set(0)); + COUNT_ALLOCATIONS.with(|enabled| enabled.set(true)); +} + +fn finish_allocation_count() -> usize { + COUNT_ALLOCATIONS.with(|enabled| enabled.set(false)); + ALLOCATION_COUNT.with(Cell::get) +} + +const REQUIRED_MUTATION_TARGETS: &[(&str, &str)] = &[ + ( + "omit_access_check", + "forbidden_actions_report_stable_errno_and_do_not_mutate_live_state", + ), + ( + "advance_requested_count", + "generated_blob_sequences_preserve_exact_tagged_prefixes", + ), + ( + "early_stream_eof", + "raw_stream_descriptor_hides_record_boundaries_and_preserves_eof", + ), + ( + "release_partial_record", + "partial_record_cursor_keeps_capacity_pinned_until_full_release", + ), + ( + "duplicate_stream_prefix", + "raw_stream_descriptor_hides_record_boundaries_and_preserves_eof", + ), + ( + "cross_wire_open_grant", + "concurrent_descriptor_opens_keep_paths_and_payloads_correlated", + ), + ( + "publish_aborted_blob", + "legal_blob_sequences_cover_boundaries_offsets_mapping_close_and_abort", + ), + ( + "reclaim_live_mapping", + "raw_blob_mapping_is_bounded_and_can_outlive_descriptor_close", + ), + ( + "writable_read_only_export", + "test_raw_descriptor_blob_io_mapping_and_errno", + ), + ( + "retain_cancelled_waiter", + "cancellation_and_transport_faults_reclaim_waiters_and_preserve_unrelated_progress", + ), + ( + "double_publication", + "raw_blob_descriptor_enforces_offsets_rights_and_terminal_state", + ), + ( + "authorize_unresolved_alias", + "missing_and_unauthorized_paths_are_rejected", + ), + ( + "silently_stage_direct_map", + "raw_blob_mapping_is_bounded_and_can_outlive_descriptor_close", + ), + ( + "touch_outside_region", + "generated_blob_sequences_preserve_exact_tagged_prefixes", + ), + ( + "leak_failed_open_lease", + "cancelled_write_open_releases_queued_grant", + ), +]; + +#[test] +fn mutation_adequacy_targets_cover_every_required_bad_behavior() { + let mut identifiers: Vec<&str> = REQUIRED_MUTATION_TARGETS + .iter() + .map(|(identifier, _)| *identifier) + .collect(); + assert_eq!(identifiers.len(), 15); + identifiers.sort_unstable(); + identifiers.dedup(); + assert_eq!(identifiers.len(), 15, "mutation identifiers must be unique"); + assert!( + REQUIRED_MUTATION_TARGETS + .iter() + .all(|(_, oracle)| !oracle.is_empty()) + ); +} + +#[test] +fn legal_blob_sequences_cover_boundaries_offsets_mapping_close_and_abort() { + let harness = harness(2 << 20); + let data_plane = harness.bootstrap.data_plane.clone(); + + future::block_on(async { + for (case, length) in [0_usize, 1, 7, 64].into_iter().enumerate() { + let logical = path(&format!("/runs/self/results/legal-{case}")); + let payload: Vec = (0..length).map(|index| index as u8).collect(); + let mut writer = data_plane + .open(&logical, OpenOptions::staged_blob(length as u64)) + .await + .expect("legal staged open"); + for chunk in payload.chunks(3) { + writer.write_all(chunk).await.expect("legal partial write"); + } + writer.close().await.expect("legal publication"); + + let mut reader = data_plane + .open(&logical, OpenOptions::read_only()) + .await + .expect("legal read open"); + let mapping = reader + .map(MapRequest { + protection: Protection::Read, + sharing: Sharing::Shared, + target: MapTarget::Host, + offset: 0, + length: length as u64, + }) + .expect("legal read mapping"); + assert_eq!(mapping.as_ref(), payload); + assert_eq!(mapping.route(), TransferRoute::Direct); + + let mut observed = Vec::new(); + let mut destination = [0xa5_u8; 11]; + loop { + let count = reader + .read(&mut destination[..(case + 1).min(11)]) + .await + .expect("legal sequential read"); + if count == 0 { + break; + } + observed.extend_from_slice(&destination[..count]); + } + assert_eq!(observed, payload); + assert_eq!(reader.read(&mut destination).await.expect("sticky eof"), 0); + reader.close().await.expect("legal reader close"); + } + + let aborted = path("/runs/self/results/legal-abort"); + let mut writer = data_plane + .open(&aborted, OpenOptions::staged_blob(4)) + .await + .expect("abort candidate"); + writer.write_all(b"nope").await.expect("staged bytes"); + writer.abort().await.expect("explicit abort"); + assert_eq!( + data_plane + .open(&aborted, OpenOptions::read_only()) + .await + .expect_err("aborted blob is absent") + .errno(), + Errno::Enoent + ); + }); +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(8))] + + #[test] + fn generated_blob_sequences_preserve_exact_tagged_prefixes( + payload in prop::collection::vec(any::(), 0..96), + write_size in 1_usize..17, + read_size in 1_usize..17, + ) { + let harness = harness(2 << 20); + let data_plane = harness.bootstrap.data_plane.clone(); + future::block_on(async { + let logical = path("/runs/self/results/property-sequence"); + let mut writer = data_plane + .open(&logical, OpenOptions::staged_blob(payload.len() as u64)) + .await + .expect("property writer"); + for chunk in payload.chunks(write_size) { + writer.write_all(chunk).await.expect("property write"); + } + writer.close().await.expect("property publish"); + + let mut reader = data_plane + .open(&logical, OpenOptions::read_only()) + .await + .expect("property reader"); + let mut destination = vec![0xa5_u8; read_size + 2]; + let mut observed = Vec::new(); + loop { + destination.fill(0xa5); + let count = reader + .read(&mut destination[1..=read_size]) + .await + .expect("property read"); + assert_eq!(destination[0], 0xa5); + assert_eq!(destination[read_size + 1], 0xa5); + if count == 0 { + break; + } + observed.extend_from_slice(&destination[1..1 + count]); + } + // Kills: advance offset by requested rather than completed bytes, + // touch destination outside the returned prefix, duplicate/drop data. + prop_assert_eq!(observed, payload); + Ok(()) + })?; + } +} + +#[test] +fn forbidden_actions_report_stable_errno_and_do_not_mutate_live_state() { + let harness = harness(2 << 20); + let data_plane = harness.bootstrap.data_plane.clone(); + let logical = path("/runs/self/results/negative"); + + future::block_on(async { + let mut writer = data_plane + .open(&logical, OpenOptions::staged_blob(4)) + .await + .expect("negative fixture writer"); + writer.write_all(b"safe").await.expect("fixture bytes"); + assert_eq!( + writer + .read(&mut [0_u8; 1]) + .await + .expect_err("wrong access") + .errno(), + Errno::Ebadf + ); + writer.close().await.expect("fixture publication"); + + let invalid = OpenOptions { + exclusive: true, + ..OpenOptions::default() + }; + assert_eq!( + data_plane + .open(&logical, invalid) + .await + .expect_err("invalid flags") + .errno(), + Errno::Einval + ); + assert_eq!( + data_plane + .open( + &logical, + OpenOptions { + nonblocking: true, + ..OpenOptions::default() + }, + ) + .await + .expect_err("unsupported nonblocking") + .errno(), + Errno::Enotsup + ); + assert_eq!( + data_plane + .open(&path("/models/missing-raw"), OpenOptions::read_only()) + .await + .expect_err("missing path") + .errno(), + Errno::Enoent + ); + assert_eq!( + data_plane + .open( + &path("/models/tiny-linear/weights"), + OpenOptions::staged_blob(1), + ) + .await + .expect_err("unauthorized write") + .errno(), + Errno::Eacces + ); + assert_eq!( + data_plane + .open( + &logical, + OpenOptions { + exclusive: true, + ..OpenOptions::staged_blob(4) + }, + ) + .await + .expect_err("exclusive existing") + .errno(), + Errno::Eexist + ); + + let mut reader = data_plane + .open(&logical, OpenOptions::read_only()) + .await + .expect("negative fixture reader"); + assert_eq!( + reader + .map(MapRequest { + protection: Protection::Read, + sharing: Sharing::Shared, + target: MapTarget::Host, + offset: 3, + length: 2, + }) + .expect_err("mapping overrun") + .errno(), + Errno::Einval + ); + assert_eq!( + reader + .write(b"x") + .await + .expect_err("read-only write") + .errno(), + Errno::Ebadf + ); + let mut bytes = [0_u8; 4]; + reader + .read_exact(&mut bytes) + .await + .expect("state unchanged"); + assert_eq!(&bytes, b"safe"); + reader.close().await.expect("first close"); + assert_eq!( + reader.close().await.expect_err("double close").errno(), + Errno::Ebadf + ); + // Kills: omitted access checks, mutation after failed bounds checks, + // silent flag downgrade, and revival after close. + }); +} + +#[test] +fn exclusive_create_is_atomic_and_abort_releases_its_reservation() { + let harness = harness(2 << 20); + let data_plane = harness.bootstrap.data_plane.clone(); + let logical = path("/runs/self/results/exclusive-race"); + let options = OpenOptions { + exclusive: true, + ..OpenOptions::staged_blob(4) + }; + + future::block_on(async { + let first = data_plane.open(&logical, options.clone()); + let second = data_plane.open(&logical, options.clone()); + let (first, second) = future::zip(first, second).await; + let (mut winner, loser) = match (first, second) { + (Ok(winner), Err(loser)) | (Err(loser), Ok(winner)) => (winner, loser), + (Ok(_), Ok(_)) => panic!("both exclusive creators succeeded"), + (Err(first), Err(second)) => { + panic!("both exclusive creators failed: {first}; {second}") + } + }; + assert_eq!(loser.errno(), Errno::Eexist); + winner.abort().await.expect("abort exclusive winner"); + + let mut replacement = data_plane + .open(&logical, options) + .await + .expect("reservation released after abort"); + replacement + .abort() + .await + .expect("abort replacement reservation"); + }); +} + +#[test] +fn raw_stream_abort_completes_before_peer_observes_broken_pipe() { + let _stream_test = STREAM_TEST_LOCK.lock(); + let harness = harness(2 << 20); + let data_plane = harness.bootstrap.data_plane.clone(); + let logical = path("/runs/self/results/raw-broken-pipe"); + + future::block_on(async { + let seed_reader = data_plane.read_stream(&logical); + let seed_writer = data_plane.write_stream(&logical); + let (seed_reader, seed_writer) = future::zip(seed_reader, seed_writer).await; + let mut seed_reader = seed_reader.expect("seed reader"); + let mut seed_writer = seed_writer.expect("seed writer"); + seed_writer.close().await.expect("seed writer close"); + assert_eq!(seed_reader.read().await.expect("seed eof"), None); + + let reader = data_plane.open(&logical, OpenOptions::read_only()); + let writer = data_plane.open( + &logical, + OpenOptions { + access: AccessMode::WriteOnly, + ..OpenOptions::default() + }, + ); + let (reader, writer) = future::zip(reader, writer).await; + let mut reader = reader.expect("raw reader"); + let mut writer = writer.expect("raw writer"); + reader.abort().await.expect("reader abort completion"); + assert_eq!( + writer + .write(b"late") + .await + .expect_err("write after reader abort") + .errno(), + Errno::Epipe + ); + }); +} + +#[test] +fn concurrent_descriptor_opens_keep_paths_and_payloads_correlated() { + let harness = harness(8 << 20); + let data_plane = harness.bootstrap.data_plane.clone(); + + std::thread::scope(|scope| { + for index in 0_u8..16 { + let data_plane = data_plane.clone(); + scope.spawn(move || { + future::block_on(async move { + let logical = path(&format!("/runs/self/results/concurrent-{index}")); + let payload = [index; 32]; + let mut writer = data_plane + .open(&logical, OpenOptions::staged_blob(payload.len() as u64)) + .await + .expect("concurrent writer"); + writer + .write_all(&payload) + .await + .expect("concurrent payload"); + writer.close().await.expect("concurrent publish"); + }); + }); + } + }); + + std::thread::scope(|scope| { + for index in 0_u8..16 { + let data_plane = data_plane.clone(); + scope.spawn(move || { + future::block_on(async move { + let logical = path(&format!("/runs/self/results/concurrent-{index}")); + let mut reader = data_plane + .open(&logical, OpenOptions::read_only()) + .await + .expect("concurrent reader"); + let mut payload = [0_u8; 32]; + reader + .read_exact(&mut payload) + .await + .expect("correlated read"); + assert_eq!(payload, [index; 32]); + }); + }); + } + }); + // Kills: accepting a grant for the wrong operation/path or crossing leases. +} + +#[test] +fn cancellation_and_transport_faults_reclaim_waiters_and_preserve_unrelated_progress() { + let _stream_test = STREAM_TEST_LOCK.lock(); + let harness = harness(2 << 20); + let data_plane = harness.bootstrap.data_plane.clone(); + let cancelled_path = path("/runs/self/results/cancelled-open"); + + future::block_on(async { + let mut cancelled = Box::pin(data_plane.read_stream(&cancelled_path)); + assert!(future::poll_once(cancelled.as_mut()).await.is_none()); + drop(cancelled); + std::thread::sleep(Duration::from_millis(10)); + + let reader = data_plane.read_stream(&cancelled_path); + let writer = data_plane.write_stream(&cancelled_path); + let (reader, writer) = future::zip(reader, writer).await; + let mut reader = reader.expect("replacement reader"); + let mut writer = writer.expect("replacement writer"); + writer.close().await.expect("replacement close"); + assert_eq!(reader.read().await.expect("replacement eof"), None); + + let mut unrelated = data_plane + .open(&path("/models/second"), OpenOptions::read_only()) + .await + .expect("unrelated progress"); + let mut bytes = [0_u8; 11]; + unrelated + .read_exact(&mut bytes) + .await + .expect("unrelated read"); + assert_eq!(&bytes, b"second-blob"); + }); + + let fault_harness = harness_with_transport(2 << 20, Arc::new(RejectingStreamTransport)); + let fault_plane = fault_harness.bootstrap.data_plane.clone(); + let fault_path = path("/runs/self/results/transport-fault"); + future::block_on(async { + let mut reader_open = Box::pin(fault_plane.read_stream(&fault_path)); + assert!(future::poll_once(reader_open.as_mut()).await.is_none()); + let writer_error = match fault_plane.write_stream(&fault_path).await { + Ok(_) => panic!("faulted writer opened"), + Err(error) => error, + }; + let reader_error = match reader_open.await { + Ok(_) => panic!("faulted reader opened"), + Err(error) => error, + }; + assert!(matches!( + reader_error, + DataPlaneError::PeerLost | DataPlaneError::StreamFault(_) + )); + assert!(matches!( + writer_error, + DataPlaneError::PeerLost | DataPlaneError::StreamFault(_) + )); + + let mut unrelated = fault_plane + .open(&path("/models/second"), OpenOptions::read_only()) + .await + .expect("fault isolation"); + let mut bytes = [0_u8; 11]; + unrelated + .read_exact(&mut bytes) + .await + .expect("fault-isolated read"); + assert_eq!(&bytes, b"second-blob"); + }); + // Kills: retain a cancelled waiter, complete only one matched endpoint, + // and propagate one descriptor fault into unrelated operations. +} + +#[test] +fn steady_state_blob_primitives_allocate_no_heap_memory() { + let harness = harness(2 << 20); + let data_plane = harness.bootstrap.data_plane.clone(); + future::block_on(async { + let logical = path("/runs/self/results/allocation-count"); + let mut writer = data_plane + .open(&logical, OpenOptions::staged_blob(64)) + .await + .expect("allocation writer"); + let payload = [7_u8; 64]; + start_allocation_count(); + let result = writer.write(&payload).await; + let write_allocations = finish_allocation_count(); + assert_eq!(result.expect("allocation-count write"), payload.len()); + assert_eq!(write_allocations, 0, "blob write primitive allocated"); + writer.close().await.expect("allocation publish"); + + let mut reader = data_plane + .open(&logical, OpenOptions::read_only()) + .await + .expect("allocation reader"); + let mut destination = [0_u8; 64]; + start_allocation_count(); + let result = reader.read(&mut destination).await; + let read_allocations = finish_allocation_count(); + assert_eq!(result.expect("allocation-count read"), destination.len()); + assert_eq!(read_allocations, 0, "blob read primitive allocated"); + assert_eq!(destination, payload); + }); +} diff --git a/crates/data-plane/tests/namespace_guarantees.proptest-regressions b/crates/data-plane/tests/namespace_guarantees.proptest-regressions new file mode 100644 index 0000000..0c0d345 --- /dev/null +++ b/crates/data-plane/tests/namespace_guarantees.proptest-regressions @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc ebf4fc53222ac2fcd156f9e3f34dcde38fa3a79d784320c929ba2e2d43e6f9de # shrinks to actions = [89, 2, 155] diff --git a/crates/data-plane/tests/namespace_guarantees.rs b/crates/data-plane/tests/namespace_guarantees.rs index cad1cad..f85afe2 100644 --- a/crates/data-plane/tests/namespace_guarantees.rs +++ b/crates/data-plane/tests/namespace_guarantees.rs @@ -115,6 +115,13 @@ fn namespace_mutations_are_linearizable_and_durable() { .await .expect("register first source"); assert_eq!(registered.revision, 1); + let node = directory + .client + .lookup(logical.clone()) + .await + .expect("lookup first blob node"); + assert_eq!(node.kind, EntryKind::Blob); + assert_eq!(node.revision, registered.revision); let selected_first = directory .client @@ -186,6 +193,12 @@ fn stream_rendezvous_is_symmetric_and_incarnations_are_isolated() { OperationId::from_u128(10), )); assert!(future::poll_once(source_open.as_mut()).await.is_none()); + let node = directory + .client + .lookup(logical.clone()) + .await + .expect("lookup ensured stream node"); + assert_eq!(node.kind, EntryKind::Stream); let sink_match = directory .client @@ -592,7 +605,7 @@ struct ModelBinding { #[derive(Clone, Debug)] enum TypedModelEntry { Blob(ModelBinding), - Stream(data_plane::namespace::StreamMatch), + Stream(Option), } proptest! { @@ -714,15 +727,16 @@ proptest! { next_operation += 1; let source_match = future::block_on(source_open).expect("source match"); prop_assert_eq!(&source_match, &sink_match); - model.insert(logical, TypedModelEntry::Stream(sink_match)); + model.insert(logical, TypedModelEntry::Stream(Some(sink_match))); } 2 => { - if let Some(TypedModelEntry::Stream(binding)) = model.get(&logical) { + if let Some(TypedModelEntry::Stream(binding)) = model.get_mut(&logical) + && let Some(binding) = binding.take() + { future::block_on(directory.client.close_stream( logical.clone(), binding.incarnation, )).expect("close current stream"); - model.remove(&logical); } } _ => {