From 59845f40597a8b32590ea90c38bd53faa4f92ba5 Mon Sep 17 00:00:00 2001 From: Developer Date: Thu, 12 Feb 2026 18:11:12 +0000 Subject: [PATCH] feat: Docker realization, node binary, docs reorg, and simulation testing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Docker realization (bridging simulation to real TCP): - NodeDriver (`crates/distribution/src/driver.rs`): bridges DistributedNode tick loop to TcpTransport with piggyback-extended wire messages - swactor-node binary (`crates/node/`): CLI node with --listen, --seed, --dashboard-port, --actors flags - Dockerfile: multi-stage build (rust:1.93-slim → debian:bookworm-slim) - Docker integration tests (`tests/docker/`): 5-node cluster with 4 scenarios (convergence, failure detection, actor resolution, rejoin) - LAN cluster scripts for cross-machine validation - TCP transport retry-on-stale-connection logic - /api/distribution REST endpoint on dashboard (feature-gated) - Piggyback fields (piggyback + from_addr) on Ping/Ack/PingReq messages Docs reorganization: - docs/runtime/ — actor-model, runtime, worker-thread, channels - docs/distribution/ — distribution, swim, kademlia, transport - docs/diagrams/ — all SVG files - docs/connectome/ — connectome analysis - docs/development_history/ — DOCKER_REALIZATION.md, SIMULATION_TESTING.md - render_docs.sh outputs to docs/diagrams/ - README links updated to new paths Co-Authored-By: Claude Opus 4.6 --- CLAUDE/TASK.md | 51 + Cargo.lock | 1292 ++++++++++++++++- Cargo.toml | 2 +- Dockerfile | 8 + README.md | 21 +- crates/distribution/src/driver.rs | 263 ++++ crates/distribution/src/lib.rs | 1 + crates/distribution/src/messages.rs | 14 + crates/distribution/src/transport.rs | 19 +- .../distribution/tests/transport_and_codec.rs | 4 + crates/node/Cargo.toml | 15 + crates/node/src/main.rs | 186 +++ crates/runtime-dashboard/src/server.rs | 31 + docs/{ => connectome}/connectome.md | 0 .../development_history/DOCKER_REALIZATION.md | 705 +++++++++ .../development_history/SIMULATION_TESTING.md | 190 +++ docs/{ => diagrams}/actor_lifecycle.svg | 0 docs/{ => diagrams}/actor_resolution.svg | 0 docs/{ => diagrams}/architecture.svg | 0 docs/{ => diagrams}/dataflow.svg | 0 .../distribution_minor_flows.svg | 0 docs/{ => diagrams}/message_lifecycle.svg | 0 docs/{ => diagrams}/runtime_lifecycle.svg | 0 docs/{ => diagrams}/swim_probe_cycle.svg | 0 docs/{ => diagrams}/tick_cycle.svg | 0 .../transport_encode_decode.svg | 0 docs/{ => diagrams}/transport_routing.svg | 0 docs/{ => diagrams}/type_erasure.svg | 0 docs/{ => distribution}/distribution.md | 40 +- docs/{ => distribution}/kademlia.md | 2 +- docs/{ => distribution}/swim.md | 2 +- docs/{ => distribution}/transport.md | 13 +- docs/render_docs.sh | 13 +- docs/{ => runtime}/actor-model.md | 0 docs/{ => runtime}/channels.md | 0 docs/{ => runtime}/runtime.md | 0 docs/{ => runtime}/worker-thread.md | 0 tests/docker/Cargo.toml | 10 + tests/docker/docker-compose.lan-hpz.yml | 34 + tests/docker/docker-compose.lan-thinkpad.yml | 49 + tests/docker/docker-compose.yml | 70 + tests/docker/run-lan-cluster.sh | 160 ++ tests/docker/src/lib.rs | 398 +++++ tests/docker/tests/cluster.rs | 201 +++ tests/docker/tests/lan_cluster.rs | 197 +++ 45 files changed, 3952 insertions(+), 39 deletions(-) create mode 100644 CLAUDE/TASK.md create mode 100644 Dockerfile create mode 100644 crates/distribution/src/driver.rs create mode 100644 crates/node/Cargo.toml create mode 100644 crates/node/src/main.rs rename docs/{ => connectome}/connectome.md (100%) create mode 100644 docs/development_history/DOCKER_REALIZATION.md create mode 100644 docs/development_history/SIMULATION_TESTING.md rename docs/{ => diagrams}/actor_lifecycle.svg (100%) rename docs/{ => diagrams}/actor_resolution.svg (100%) rename docs/{ => diagrams}/architecture.svg (100%) rename docs/{ => diagrams}/dataflow.svg (100%) rename docs/{ => diagrams}/distribution_minor_flows.svg (100%) rename docs/{ => diagrams}/message_lifecycle.svg (100%) rename docs/{ => diagrams}/runtime_lifecycle.svg (100%) rename docs/{ => diagrams}/swim_probe_cycle.svg (100%) rename docs/{ => diagrams}/tick_cycle.svg (100%) rename docs/{ => diagrams}/transport_encode_decode.svg (100%) rename docs/{ => diagrams}/transport_routing.svg (100%) rename docs/{ => diagrams}/type_erasure.svg (100%) rename docs/{ => distribution}/distribution.md (68%) rename docs/{ => distribution}/kademlia.md (98%) rename docs/{ => distribution}/swim.md (98%) rename docs/{ => distribution}/transport.md (89%) rename docs/{ => runtime}/actor-model.md (100%) rename docs/{ => runtime}/channels.md (100%) rename docs/{ => runtime}/runtime.md (100%) rename docs/{ => runtime}/worker-thread.md (100%) create mode 100644 tests/docker/Cargo.toml create mode 100644 tests/docker/docker-compose.lan-hpz.yml create mode 100644 tests/docker/docker-compose.lan-thinkpad.yml create mode 100644 tests/docker/docker-compose.yml create mode 100755 tests/docker/run-lan-cluster.sh create mode 100644 tests/docker/src/lib.rs create mode 100644 tests/docker/tests/cluster.rs create mode 100644 tests/docker/tests/lan_cluster.rs diff --git a/CLAUDE/TASK.md b/CLAUDE/TASK.md new file mode 100644 index 0000000..64fa4f0 --- /dev/null +++ b/CLAUDE/TASK.md @@ -0,0 +1,51 @@ +Plan: + You are to improve this codebase via: + - implementing and testing various cluster scenarios + - reading and documenting other well respected codebases that do similar things + - examining their simulation test methodology + - writing tests that match the same concepts they explore + - putting notes in CLAUDE/notes/ to reflect your understanding, without too much file bloat + - making a large suite of fast tests in simulation for various cluster configurations and scenarios + +Workflow: + - Read `CLAUDE/TASK.md` and `CLAUDE/notes/progress.md` + - Identify what stage you are on. + - Read and update yourself as necessary. + - Proceed to accomplishing the next task as written in `progress.md` + - For each attempt at any step, keep a record. If you reach attempt 3, step back, document, and try something else. + - When done, because attempt limit or task success: + - update `progress.md` with: + - Completed this session + - Next steps (specific, actionable) + - Open Questions + - Blockers + - make a commit + - compress your context and start the loop again + +Style: + - Do not add to existing modules in the root swactor `src/` they should stay as they are. You may modify but not change module structure. + - Do not modify distribution except to fix bugs, or for major improvements in performance/robustness + - Integration tests in `tests/`, benchmark code in `benches/` + - cap execution time at 2 minutes max for fuzz, or benchmarks, or single test suite + - if they take too long, refactor and break up into logical modules + - You may modify these as you wish, so long as logical 'coverage' does not decline. + - cluster sim tests in crates/simulation + - try to keep your edits clean, clear; low line counts, modest complexity + - Report all your changes to architecture with changes to the `docs/` items + - all notes you wish to keep across iterations shall go in the `CLAUDE/notes/` folder + +Example loop (not restrictive, feel free to ignore if prudent): + - Pick a test to implement and run: + - make analysis + - implement plan + - execute + - evaluate + - if distribution fails, figure out the simplest possible way to not fail + - unless it is out of scope, then document why it failed and why out of scope + - if satisfied, pick a new codebase and/or concept. If not, repeat from step 'compare to swactor' + +Before git commit: + - all `cargo test` passes, including feature gated material + - if a test fails, investigate do not ignore or delete + - You can combine tests but not skip code paths or delete them for active code + - if a fix takes > 3 attempts, log and move on \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 8ff07c4..1e99605 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -44,12 +44,56 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "anstream" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + [[package]] name = "anstyle" version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.101" @@ -88,6 +132,12 @@ dependencies = [ "syn", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.0" @@ -100,6 +150,12 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "base64ct" version = "1.8.3" @@ -160,6 +216,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + [[package]] name = "cassowary" version = "0.3.0" @@ -245,6 +307,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63be97961acde393029492ce0be7a1af7e323e6bae9511ebfac33751be5e6806" dependencies = [ "clap_builder", + "clap_derive", ] [[package]] @@ -253,8 +316,22 @@ version = "4.5.58" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f13174bda5dfd69d7e947827e5af4b0f2f94a4a3ee92912fba07a66150f21e2" dependencies = [ + "anstream", "anstyle", "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -272,6 +349,12 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + [[package]] name = "compact_str" version = "0.8.1" @@ -292,6 +375,22 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpp_demangle" version = "0.4.5" @@ -671,6 +770,17 @@ dependencies = [ "objc2", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "distribution" version = "0.1.0" @@ -682,6 +792,16 @@ dependencies = [ "swactor", ] +[[package]] +name = "docker-tests" +version = "0.1.0" +dependencies = [ + "distribution", + "reqwest", + "serde", + "serde_json", +] + [[package]] name = "ed25519" version = "2.2.3" @@ -786,6 +906,80 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "fxhash" version = "0.2.1" @@ -841,6 +1035,19 @@ dependencies = [ "wasip2", ] +[[package]] +name = "getrandom" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", + "wasip3", +] + [[package]] name = "gimli" version = "0.31.1" @@ -852,6 +1059,25 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "2.7.1" @@ -902,12 +1128,211 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + [[package]] name = "httpdate" version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + [[package]] name = "id-arena" version = "2.3.0" @@ -920,6 +1345,27 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "indexmap" version = "2.13.0" @@ -954,6 +1400,22 @@ dependencies = [ "syn", ] +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "iri-string" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "is-terminal" version = "0.4.17" @@ -965,6 +1427,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.10.5" @@ -1090,6 +1558,12 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" +[[package]] +name = "litemap" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" + [[package]] name = "lock_api" version = "0.4.14" @@ -1147,6 +1621,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "mio" version = "1.1.1" @@ -1159,6 +1639,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "native-tls" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cdede44f9a69cab2899a2049e2c3bd49bf911a157f6a3353d4a91c61abbce44" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nix" version = "0.30.1" @@ -1171,6 +1668,17 @@ dependencies = [ "libc", ] +[[package]] +name = "node" +version = "0.1.0" +dependencies = [ + "clap", + "ctrlc", + "distribution", + "runtime-dashboard", + "swactor", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1231,12 +1739,62 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "oorandom" version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" +[[package]] +name = "openssl" +version = "0.10.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -1266,12 +1824,24 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + [[package]] name = "pin-project-lite" version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkcs8" version = "0.10.2" @@ -1334,6 +1904,15 @@ dependencies = [ "serde", ] +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1343,6 +1922,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -1645,6 +2234,62 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c" +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "encoding_rs", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "runtime-dashboard" version = "0.1.0" @@ -1710,6 +2355,39 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.23.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" +dependencies = [ + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -1743,12 +2421,44 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.27" @@ -1811,6 +2521,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha2" version = "0.10.9" @@ -1901,6 +2623,12 @@ dependencies = [ "toml", ] +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "smallvec" version = "1.15.1" @@ -1910,6 +2638,16 @@ dependencies = [ "serde", ] +[[package]] +name = "socket2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + [[package]] name = "spki" version = "0.7.3" @@ -2024,6 +2762,47 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "target-lexicon" version = "0.12.16" @@ -2043,7 +2822,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0136791f7c95b1f6dd99f9cc786b91bb81c3800b639b3478e561ddb7be95e5f1" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.1", "once_cell", "rustix 1.1.3", "windows-sys 0.61.2", @@ -2119,6 +2898,16 @@ dependencies = [ "log", ] +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinytemplate" version = "1.2.1" @@ -2129,6 +2918,53 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tokio" +version = "1.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml" version = "0.8.23" @@ -2170,6 +3006,51 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + [[package]] name = "tracing" version = "0.1.44" @@ -2238,6 +3119,12 @@ dependencies = [ "syn", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "typenum" version = "1.19.0" @@ -2297,6 +3184,36 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.20.0" @@ -2313,6 +3230,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -2338,6 +3261,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -2353,6 +3285,15 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasm" version = "0.1.0" @@ -2374,6 +3315,20 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" +dependencies = [ + "cfg-if", + "futures-util", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.108" @@ -2416,6 +3371,16 @@ dependencies = [ "wasmparser 0.221.3", ] +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser 0.244.0", +] + [[package]] name = "wasm-encoder" version = "0.245.1" @@ -2426,6 +3391,18 @@ dependencies = [ "wasmparser 0.245.1", ] +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder 0.244.0", + "wasmparser 0.244.0", +] + [[package]] name = "wasmparser" version = "0.221.3" @@ -2439,6 +3416,18 @@ dependencies = [ "serde", ] +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + [[package]] name = "wasmparser" version = "0.245.1" @@ -2535,7 +3524,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b1161c8f62880deea07358bc40cceddc019f1c81d46007bc390710b2fe24ffc" dependencies = [ "anyhow", - "base64", + "base64 0.21.7", "directories-next", "log", "postcard", @@ -2560,7 +3549,7 @@ dependencies = [ "syn", "wasmtime-component-util", "wasmtime-wit-bindgen", - "wit-parser", + "wit-parser 0.221.3", ] [[package]] @@ -2711,7 +3700,7 @@ dependencies = [ "anyhow", "heck", "indexmap", - "wit-parser", + "wit-parser 0.221.3", ] [[package]] @@ -2801,13 +3790,60 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", ] [[package]] @@ -2825,14 +3861,31 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] [[package]] @@ -2841,48 +3894,96 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "winnow" version = "0.7.14" @@ -2897,6 +3998,70 @@ name = "wit-bindgen" version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser 0.244.0", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder 0.244.0", + "wasm-metadata", + "wasmparser 0.244.0", + "wit-parser 0.244.0", +] [[package]] name = "wit-parser" @@ -2916,6 +4081,53 @@ dependencies = [ "wasmparser 0.221.3", ] +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser 0.244.0", +] + +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + +[[package]] +name = "yoke" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.8.39" @@ -2936,12 +4148,66 @@ dependencies = [ "syn", ] +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zmij" version = "1.0.21" diff --git a/Cargo.toml b/Cargo.toml index 9afb0f6..19c2995 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = [".", "crates/python", "crates/wasm", "crates/wasm-actor", "crates/simulation", "crates/runtime-dashboard", "crates/distribution", "crates/simulation-dashboard", "crates/std", "crates/command"] +members = [".", "crates/python", "crates/wasm", "crates/wasm-actor", "crates/simulation", "crates/runtime-dashboard", "crates/distribution", "crates/simulation-dashboard", "crates/std", "crates/command", "crates/node", "tests/docker"] exclude = ["tools/depgraph"] [package] diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..74b67af --- /dev/null +++ b/Dockerfile @@ -0,0 +1,8 @@ +FROM rust:1.93-slim AS builder +WORKDIR /build +COPY . . +RUN cargo build --release -p node + +FROM debian:bookworm-slim +COPY --from=builder /build/target/release/swactor-node /usr/local/bin/ +ENTRYPOINT ["swactor-node"] diff --git a/README.md b/README.md index fc9f95d..861a90b 100644 --- a/README.md +++ b/README.md @@ -53,8 +53,8 @@ a different worker thread, an external inbox, or a remote process. Single-threaded mode (`rt.tick()`) gives deterministic frame-level control. Multi-threaded mode (`rt.run()`) spawns OS threads with adaptive backoff. -See [docs/actor-model.md](docs/actor-model.md) and -[docs/runtime.md](docs/runtime.md) for the full model. +See [docs/runtime/actor-model.md](docs/runtime/actor-model.md) and +[docs/runtime/runtime.md](docs/runtime/runtime.md) for the full model. ### Transport @@ -68,7 +68,7 @@ cargo run --example tcp_ping_pong --features transport -- receiver # terminal 1 cargo run --example tcp_ping_pong --features transport -- sender # terminal 2 ``` -See [docs/transport.md](docs/transport.md) for the routing chain, codec +See [docs/distribution/transport.md](docs/distribution/transport.md) for the routing chain, codec registry, and address resolution. ### Runtime Dashboard @@ -115,7 +115,7 @@ cargo run --manifest-path tools/depgraph/Cargo.toml -- --src-dir src/ --output d python tools/spectral/spectral_analysis.py deps.dot ``` -See [docs/connectome.md](docs/connectome.md) for metric interpretation. +See [docs/connectome/connectome.md](docs/connectome/connectome.md) for metric interpretation. ## Building & Testing @@ -142,10 +142,11 @@ cargo bench # benchmarks (criterion) | Document | Covers | |----------|--------| -| [Actor Model](docs/actor-model.md) | Traits, type erasure, addresses | -| [Runtime](docs/runtime.md) | Runtime, Ctx, Inbox, RuntimeHandle, stats | -| [Worker Thread](docs/worker-thread.md) | Tick phases, backoff, routing, full system topology | -| [Channels](docs/channels.md) | HybridChannel, AddressMap, Placement | -| [Transport](docs/transport.md) | Codec, Transport, remote messaging, address resolution | -| [Connectome](docs/connectome.md) | CCI metrics, spectral analysis interpretation | +| [Actor Model](docs/runtime/actor-model.md) | Traits, type erasure, addresses | +| [Runtime](docs/runtime/runtime.md) | Runtime, Ctx, Inbox, RuntimeHandle, stats | +| [Worker Thread](docs/runtime/worker-thread.md) | Tick phases, backoff, routing, full system topology | +| [Channels](docs/runtime/channels.md) | HybridChannel, AddressMap, Placement | +| [Transport](docs/distribution/transport.md) | Codec, Transport, remote messaging, address resolution | +| [Distribution](docs/distribution/distribution.md) | SWIM membership, Kademlia, NodeDriver | +| [Connectome](docs/connectome/connectome.md) | CCI metrics, spectral analysis interpretation | | [Dashboard](crates/runtime-dashboard/README.md) | Live web UI, trace recording, diagram index | diff --git a/crates/distribution/src/driver.rs b/crates/distribution/src/driver.rs new file mode 100644 index 0000000..2c82c0d --- /dev/null +++ b/crates/distribution/src/driver.rs @@ -0,0 +1,263 @@ +//! Network driver — bridges `DistributedNode` logic with TCP I/O. +//! +//! Translates outgoing `NodeAction`s into wire messages sent via `TcpTransport`, +//! and dispatches incoming wire messages to the appropriate `DistributedNode` +//! handler methods. + +use std::net::{SocketAddr, TcpStream}; + +use swactor::actor::ActorAddress; +use swactor::transport::{NetworkMessage, WireEnvelope}; + +use crate::messages::*; +use crate::node::{DistributedNode, DistributedNodeConfig}; +use crate::snapshot::DistributionNodeSnapshot; +use crate::swim::node::NodeAction; +use crate::transport::{TcpAcceptor, TcpTransport}; +use crate::types::NodeId; + +/// Dummy destination address used in wire envelopes for SWIM protocol messages. +/// SWIM messages are routed by `SocketAddr`, not by `ActorAddress`, so this +/// field is unused but required by the wire format. +const SWIM_DEST: ActorAddress = ActorAddress([0u8; 32]); + +/// Network driver that owns a `DistributedNode` and performs real TCP I/O. +pub struct NodeDriver { + node: DistributedNode, + transport: TcpTransport, + acceptor: TcpAcceptor, + streams: Vec, +} + +impl NodeDriver { + /// Create a new driver. Binds a TCP listener on the node's `listen_addr`. + pub fn new(config: DistributedNodeConfig) -> Result { + let listen_addr = config.listen_addr; + let acceptor = TcpAcceptor::bind(listen_addr)?; + let node = DistributedNode::new(config); + Ok(Self { + node, + transport: TcpTransport::pool(), + acceptor, + streams: Vec::new(), + }) + } + + /// The node's identity. + pub fn node_id(&self) -> NodeId { + self.node.node_id() + } + + /// The address this driver is listening on. + pub fn listen_addr(&self) -> SocketAddr { + self.acceptor.local_addr() + } + + /// Access the underlying node (read-only). + pub fn node(&self) -> &DistributedNode { + &self.node + } + + /// Access the underlying node (mutable). + pub fn node_mut(&mut self) -> &mut DistributedNode { + &mut self.node + } + + /// Capture a snapshot of the node's state. + pub fn snapshot(&self) -> DistributionNodeSnapshot { + self.node.snapshot() + } + + /// Join a cluster by contacting seed nodes. + /// + /// Sends `JoinRequest` messages to each seed over TCP. + pub fn join(&mut self, seeds: &[SocketAddr]) { + let actions = self.node.join(seeds); + self.send_actions(&actions); + } + + /// Advance the node by one tick. + /// + /// Drives the SWIM probe cycle, sends outgoing protocol messages, + /// and handles periodic republishing. + pub fn tick(&mut self) { + let actions = self.node.tick(); + self.send_actions(&actions); + } + + /// Process incoming TCP messages. + /// + /// Reads all available wire envelopes from the acceptor, dispatches + /// each to the appropriate handler, and sends any response actions. + pub fn recv(&mut self) { + let envelopes = self.acceptor.try_recv(&mut self.streams); + for (envelope, _peer_addr) in envelopes { + let response_actions = self.dispatch_incoming(envelope); + self.send_actions(&response_actions); + } + } + + // ─── Outgoing: NodeAction → TCP ───────────────────────────────────── + + fn send_actions(&mut self, actions: &[NodeAction]) { + for action in actions { + if let Err(e) = self.send_action(action) { + eprintln!("driver: send error: {e}"); + } + } + } + + fn send_action(&mut self, action: &NodeAction) -> Result<(), swactor::Error> { + match action { + NodeAction::SendPing { + to_addr, + sequence, + piggyback, + .. + } => { + let msg = Ping { + from: self.node.node_id(), + from_addr: self.node.listen_addr(), + sequence: *sequence, + piggyback: piggyback.clone(), + }; + self.send_wire::(&msg, *to_addr) + } + + NodeAction::SendAck { + to_addr, + sequence, + piggyback, + .. + } => { + let msg = Ack { + from: self.node.node_id(), + sequence: *sequence, + piggyback: piggyback.clone(), + }; + self.send_wire::(&msg, *to_addr) + } + + NodeAction::SendPingReq { + relay_addr, + target, + target_addr, + sequence, + piggyback, + .. + } => { + let msg = PingReq { + from: self.node.node_id(), + target: *target, + target_addr: *target_addr, + sequence: *sequence, + piggyback: piggyback.clone(), + }; + self.send_wire::(&msg, *relay_addr) + } + + NodeAction::SendJoinRequest { to_addr } => { + let msg = JoinRequest { + from: self.node.node_id(), + addr: self.node.listen_addr(), + }; + self.send_wire::(&msg, *to_addr) + } + + NodeAction::SendJoinResponse { + to_addr, members, .. + } => { + let msg = JoinResponse { + members: members.clone(), + }; + self.send_wire::(&msg, *to_addr) + } + + NodeAction::MembershipChanged { .. } => { + // Internal notification — no network I/O. + Ok(()) + } + } + } + + fn send_wire( + &mut self, + msg: &M, + dest_addr: SocketAddr, + ) -> Result<(), swactor::Error> { + let payload = serde_json::to_vec(msg) + .map_err(|e| swactor::Error::from(format!("encode {}: {e}", M::type_tag())))?; + let envelope = WireEnvelope { + dest: SWIM_DEST, + type_tag: M::type_tag().to_string(), + payload, + }; + self.transport.send_to(dest_addr, envelope) + } + + // ─── Incoming: TCP → handler ──────────────────────────────────────── + + fn dispatch_incoming(&mut self, envelope: WireEnvelope) -> Vec { + match envelope.type_tag.as_str() { + "swactor_dist::Ping" => match decode::(&envelope.payload) { + Ok(msg) => self.node.handle_ping( + msg.from, + msg.from_addr, + msg.sequence, + &msg.piggyback, + ), + Err(e) => { + eprintln!("driver: decode Ping: {e}"); + Vec::new() + } + }, + + "swactor_dist::Ack" => match decode::(&envelope.payload) { + Ok(msg) => self.node.handle_ack(msg.from, msg.sequence, &msg.piggyback), + Err(e) => { + eprintln!("driver: decode Ack: {e}"); + Vec::new() + } + }, + + "swactor_dist::PingReq" => match decode::(&envelope.payload) { + Ok(msg) => self.node.handle_ping_req( + msg.from, + msg.target, + msg.target_addr, + msg.sequence, + &msg.piggyback, + ), + Err(e) => { + eprintln!("driver: decode PingReq: {e}"); + Vec::new() + } + }, + + "swactor_dist::JoinRequest" => match decode::(&envelope.payload) { + Ok(msg) => self.node.handle_join_request(msg.from, msg.addr), + Err(e) => { + eprintln!("driver: decode JoinRequest: {e}"); + Vec::new() + } + }, + + "swactor_dist::JoinResponse" => match decode::(&envelope.payload) { + Ok(msg) => self.node.handle_join_response(msg.members), + Err(e) => { + eprintln!("driver: decode JoinResponse: {e}"); + Vec::new() + } + }, + + other => { + eprintln!("driver: unknown message type: {other}"); + Vec::new() + } + } + } +} + +fn decode(bytes: &[u8]) -> Result { + serde_json::from_slice(bytes).map_err(|e| e.to_string()) +} diff --git a/crates/distribution/src/lib.rs b/crates/distribution/src/lib.rs index 0922300..18f5a41 100644 --- a/crates/distribution/src/lib.rs +++ b/crates/distribution/src/lib.rs @@ -9,3 +9,4 @@ pub mod cache; pub mod node; pub mod registry; pub mod snapshot; +pub mod driver; diff --git a/crates/distribution/src/messages.rs b/crates/distribution/src/messages.rs index 5844087..ff3b63a 100644 --- a/crates/distribution/src/messages.rs +++ b/crates/distribution/src/messages.rs @@ -11,10 +11,16 @@ use crate::types::{DirectoryEntry, MemberState, NodeId, NodeRecord}; // ─── SWIM Protocol Messages ──────────────────────────────────────────────── /// SWIM ping — "are you alive?" +/// +/// Carries piggybacked membership gossip so that SWIM dissemination +/// propagates cluster state changes on every protocol message. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Ping { pub from: NodeId, + pub from_addr: SocketAddr, pub sequence: u64, + #[serde(default)] + pub piggyback: Vec, } impl NetworkMessage for Ping { @@ -24,10 +30,14 @@ impl NetworkMessage for Ping { } /// SWIM ack — "yes, I'm alive" +/// +/// Carries piggybacked membership gossip (same as Ping). #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Ack { pub from: NodeId, pub sequence: u64, + #[serde(default)] + pub piggyback: Vec, } impl NetworkMessage for Ack { @@ -37,12 +47,16 @@ impl NetworkMessage for Ack { } /// SWIM indirect ping request — "please ping target on my behalf" +/// +/// Carries piggybacked membership gossip (same as Ping). #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PingReq { pub from: NodeId, pub target: NodeId, pub target_addr: SocketAddr, pub sequence: u64, + #[serde(default)] + pub piggyback: Vec, } impl NetworkMessage for PingReq { diff --git a/crates/distribution/src/transport.rs b/crates/distribution/src/transport.rs index 807f9c2..c918139 100644 --- a/crates/distribution/src/transport.rs +++ b/crates/distribution/src/transport.rs @@ -67,12 +67,23 @@ impl TcpTransport { } /// Send an envelope to a specific address. + /// + /// If the write fails (e.g. stale connection from a dead peer), evicts + /// the pooled connection and retries once with a fresh one. pub fn send_to(&self, addr: SocketAddr, envelope: WireEnvelope) -> Result<(), Error> { - let mut stream = self.get_or_connect(addr)?; let buf = encode_wire_envelope(&envelope); - stream - .write_all(&buf) - .map_err(|e| Error::from(format!("TCP send to {addr}: {e}"))) + let mut stream = self.get_or_connect(addr)?; + match stream.write_all(&buf) { + Ok(()) => Ok(()), + Err(_) => { + // Evict stale connection and retry once + self.pool.lock().unwrap().remove(&addr); + let mut stream = self.get_or_connect(addr)?; + stream + .write_all(&buf) + .map_err(|e| Error::from(format!("TCP send to {addr}: {e}"))) + } + } } } diff --git a/crates/distribution/tests/transport_and_codec.rs b/crates/distribution/tests/transport_and_codec.rs index 54ad831..1d3d6d5 100644 --- a/crates/distribution/tests/transport_and_codec.rs +++ b/crates/distribution/tests/transport_and_codec.rs @@ -84,7 +84,9 @@ fn distribution_codec_encodes_and_decodes_ping() { let codecs = distribution_codec_registry(); let ping = Ping { from: NodeId([0xAA; 32]), + from_addr: "127.0.0.1:7000".parse().unwrap(), sequence: 42, + piggyback: vec![], }; let type_id = std::any::TypeId::of::(); @@ -160,7 +162,9 @@ fn ping_message_survives_codec_and_tcp_roundtrip() { let dest = ActorAddress::new_random(); let ping = Ping { from: NodeId([0xBB; 32]), + from_addr: "127.0.0.1:7001".parse().unwrap(), sequence: 99, + piggyback: vec![], }; let type_id = std::any::TypeId::of::(); diff --git a/crates/node/Cargo.toml b/crates/node/Cargo.toml new file mode 100644 index 0000000..1968909 --- /dev/null +++ b/crates/node/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "node" +version = "0.1.0" +edition = "2024" + +[[bin]] +name = "swactor-node" +path = "src/main.rs" + +[dependencies] +swactor = { path = "../..", features = ["serde", "tracing", "transport"] } +distribution = { path = "../distribution" } +runtime-dashboard = { path = "../runtime-dashboard", features = ["distribution"] } +clap = { version = "4", features = ["derive"] } +ctrlc = "3" diff --git a/crates/node/src/main.rs b/crates/node/src/main.rs new file mode 100644 index 0000000..6d7ab6c --- /dev/null +++ b/crates/node/src/main.rs @@ -0,0 +1,186 @@ +use std::net::SocketAddr; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::Duration; + +use clap::Parser; + +use swactor::actor::{ActorInterface, Ctx}; +use swactor::config::RuntimeConfig; +use swactor::runtime::Runtime; + +use distribution::driver::NodeDriver; +use distribution::node::DistributedNodeConfig; +use distribution::snapshot::DistributionNodeSnapshot; +use distribution::swim::probe::SwimConfig; + +use runtime_dashboard::collector::StatsCollector; +use runtime_dashboard::distribution_collector::DistributionStatsProvider; +use runtime_dashboard::{start_dashboard, DashboardConfig}; + +// ── CLI ────────────────────────────────────────────────────────────────── + +#[derive(Parser)] +#[command(name = "swactor-node", about = "Swactor distributed node")] +struct Args { + /// Address to listen on for SWIM protocol (e.g. 10.0.1.10:7000) + #[arg(long)] + listen: SocketAddr, + + /// Seed node address to join (omit for the seed node itself) + #[arg(long)] + seed: Option, + + /// Dashboard HTTP port + #[arg(long, default_value = "9090")] + dashboard_port: u16, + + /// Number of dummy actors to register in the directory + #[arg(long, default_value = "0")] + actors: usize, +} + +// ── Dummy actor ────────────────────────────────────────────────────────── + +#[derive(Clone)] +struct Heartbeat; + +struct HeartbeatActor; + +impl ActorInterface for HeartbeatActor { + type Incoming = Heartbeat; + type Response = (); + + fn handle(&mut self, _ctx: &Ctx, _msg: Heartbeat) {} +} + +// ── Snapshot provider ──────────────────────────────────────────────────── + +struct SnapshotProvider { + snapshot: Arc>>, +} + +impl DistributionStatsProvider for SnapshotProvider { + fn snapshot(&self) -> Option { + self.snapshot.lock().unwrap().clone() + } +} + +// ── Main ───────────────────────────────────────────────────────────────── + +fn main() { + let args = Args::parse(); + let stop = Arc::new(AtomicBool::new(false)); + + // Handle SIGTERM / Ctrl+C + { + let stop = Arc::clone(&stop); + ctrlc::set_handler(move || { + stop.store(true, Ordering::Relaxed); + }) + .expect("failed to set signal handler"); + } + + // Start dashboard + let dash = start_dashboard(DashboardConfig { + port: args.dashboard_port, + ..Default::default() + }); + dash.install_tracing(); + + // Create actor runtime + let num_threads = 2; + let collector = StatsCollector::new(num_threads); + let mut rt = Runtime::new(RuntimeConfig { + num_threads, + max_actors: 1024, + channel_buffer_size: 2000, + ..Default::default() + }); + rt.set_stats_hook(collector.clone()); + + let handle = rt.run().expect("failed to start runtime"); + dash.set_runtime(handle.runtime.clone(), collector); + + // Create distribution node driver + let swim_config = SwimConfig { + probe_interval: 5, + probe_timeout: 3, + indirect_probes: 2, + suspicion_timeout: 20, + }; + let node_config = DistributedNodeConfig { + listen_addr: args.listen, + swim: swim_config, + cache_capacity: 1000, + republish_interval: 500, + }; + let mut driver = NodeDriver::new(node_config).expect("failed to create node driver"); + + eprintln!( + "Node {} listening on {}", + hex(&driver.node_id().0[..4]), + driver.listen_addr(), + ); + + // Join seed if provided + if let Some(seed) = args.seed { + eprintln!("Joining cluster via seed {seed}"); + driver.join(&[seed]); + } + + // Spawn and register actors + let mut actor_addrs = Vec::new(); + for _ in 0..args.actors { + match handle.runtime.spawn(HeartbeatActor) { + Ok(addr) => { + driver.node_mut().register_actor(addr, 1); + actor_addrs.push(addr); + } + Err(e) => eprintln!("failed to spawn actor: {e}"), + } + } + + if !actor_addrs.is_empty() { + eprintln!("Registered {} actors", actor_addrs.len()); + } + + // Wire distribution snapshot to dashboard + let cached_snapshot: Arc>> = + Arc::new(Mutex::new(Some(driver.snapshot()))); + let provider = SnapshotProvider { + snapshot: Arc::clone(&cached_snapshot), + }; + dash.set_distribution(Arc::new(provider)); + + eprintln!( + "Dashboard at http://0.0.0.0:{}", + args.dashboard_port + ); + + // Main loop + while !stop.load(Ordering::Relaxed) { + driver.recv(); + driver.tick(); + + // Send heartbeats to keep actors alive + for addr in &actor_addrs { + let _ = handle.runtime.send_to(*addr, Heartbeat); + } + + // Update dashboard snapshot + *cached_snapshot.lock().unwrap() = Some(driver.snapshot()); + + thread::sleep(Duration::from_millis(100)); + } + + eprintln!("\nShutting down..."); + handle.shutdown(); + dash.shutdown(); + handle.join(); +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} diff --git a/crates/runtime-dashboard/src/server.rs b/crates/runtime-dashboard/src/server.rs index 703ab68..8d64904 100644 --- a/crates/runtime-dashboard/src/server.rs +++ b/crates/runtime-dashboard/src/server.rs @@ -178,6 +178,13 @@ pub(crate) fn spawn_http_server( Arc::clone(&cmd_router), ); } + #[cfg(feature = "distribution")] + "/api/distribution" => { + handle_distribution_api( + request, + Arc::clone(&distribution), + ); + } _ => respond_404(request), } } @@ -322,6 +329,30 @@ fn handle_investigate_api( let _ = request.respond(response); } +#[cfg(feature = "distribution")] +fn handle_distribution_api( + request: tiny_http::Request, + distribution: Arc>>>, +) { + let json = match distribution.lock().unwrap().as_ref() { + Some(provider) => match provider.snapshot() { + Some(snapshot) => serde_json::to_string(&snapshot).unwrap_or_else(|_| "{}".into()), + None => "{}".to_string(), + }, + None => serde_json::json!({ + "error": "distribution provider not attached" + }) + .to_string(), + }; + + let response = tiny_http::Response::from_string(json).with_header( + "Content-Type: application/json" + .parse::() + .unwrap(), + ); + let _ = request.respond(response); +} + fn parse_query_string(url: &str) -> HashMap { let mut params = HashMap::new(); if let Some(qs) = url.split('?').nth(1) { diff --git a/docs/connectome.md b/docs/connectome/connectome.md similarity index 100% rename from docs/connectome.md rename to docs/connectome/connectome.md diff --git a/docs/development_history/DOCKER_REALIZATION.md b/docs/development_history/DOCKER_REALIZATION.md new file mode 100644 index 0000000..00bdc1b --- /dev/null +++ b/docs/development_history/DOCKER_REALIZATION.md @@ -0,0 +1,705 @@ +# Distribution Realization — Development History + +> Covers all work to bridge the pure-logic distributed runtime to real TCP networking, +> package it as a Docker-deployable node binary, verify it against the simulation +> tests via a 5-node Docker cluster, and validate cross-machine behavior via a +> LAN cluster split across two physical machines. +> +> ~22 files changed · ~1,600 insertions +> +> *Branch: `distribution-realization`* + +--- + +## Table of Contents + +1. [Overview & Motivation](#1-overview--motivation) +2. [What Was Built](#2-what-was-built) +3. [Development Phases](#3-development-phases) +4. [Wire Protocol Gap — Piggyback Extension](#4-wire-protocol-gap--piggyback-extension) +5. [NodeDriver — TCP ↔ NodeAction Bridge](#5-nodedriver--tcp--nodeaction-bridge) +6. [REST API Endpoint](#6-rest-api-endpoint) +7. [Node Binary](#7-node-binary) +8. [Docker Infrastructure](#8-docker-infrastructure) +9. [Integration Test Harness](#9-integration-test-harness) +10. [Cross-Machine LAN Cluster](#10-cross-machine-lan-cluster) +11. [Design Decisions & Tradeoffs](#11-design-decisions--tradeoffs) +12. [Bugs Encountered](#12-bugs-encountered) +13. [Known Gaps & Future Improvements](#13-known-gaps--future-improvements) +14. [Test Coverage Summary](#14-test-coverage-summary) + +--- + +## 1. Overview & Motivation + +The distribution layer (`crates/distribution/`) was built as a set of **pure state machines** — `DistributedNode::tick()` produces `Vec` that the caller translates to network I/O. All existing tests used in-process method calls: simulation nodes forwarded actions directly via `node.handle_ping(...)` without real networking. + +This left a critical gap: **no code existed to actually run the protocol over TCP**. The `TcpTransport` and `TcpAcceptor` were implemented and tested in isolation, and the `NodeAction` enum described exactly what messages to send where, but the bridge between them was missing. From DISTRIBUTION.md §13.5: + +> *"The actual wiring of `node.tick() → transport.send()` for each `NodeAction` is missing."* + +This work closes that gap by: + +1. **Extending wire protocol messages** with piggyback fields required for SWIM dissemination +2. **Creating NodeDriver** — the bridge that maps `NodeAction` → TCP sends and TCP receives → handler calls +3. **Adding a REST endpoint** for programmatic cluster health queries +4. **Packaging a node binary** (`swactor-node`) with CLI, dashboard, and actor registration +5. **Building Docker infrastructure** for a 5-node cluster with static IPs +6. **Writing integration tests** that mirror the simulation scenarios and verify real TCP behavior matches simulation expectations + +The result: `docker compose up` spins up 5 nodes that form a SWIM cluster, register actors in the Kademlia directory, and can be observed via the runtime dashboard — matching the outcomes of the simulation tests. + +--- + +## 2. What Was Built + +| Component | Location | Lines | Files | +|-----------|----------|-------|-------| +| Wire protocol extension | `crates/distribution/src/messages.rs` | ~15 | 1 modified | +| NodeDriver | `crates/distribution/src/driver.rs` | ~264 | 1 new | +| REST API endpoint | `crates/runtime-dashboard/src/server.rs` | ~30 | 1 modified | +| Node binary | `crates/node/` | ~200 | 2 new | +| Dockerfile | `Dockerfile` | 9 | 1 new | +| Docker Compose (single-machine) | `tests/docker/docker-compose.yml` | 71 | 1 new | +| Docker Compose (LAN) | `tests/docker/docker-compose.lan-*.yml` | ~80 | 2 new | +| LAN orchestration script | `tests/docker/run-lan-cluster.sh` | ~100 | 1 new | +| Test harness | `tests/docker/` | ~400 | 4 new | +| LAN integration tests | `tests/docker/tests/lan_cluster.rs` | ~200 | 1 new | +| Test updates | `crates/distribution/tests/transport_and_codec.rs` | ~5 | 1 modified | +| Workspace config | `Cargo.toml` (root) | ~2 | 1 modified | + +--- + +## 3. Development Phases + +### Phase 1 — Extend wire protocol messages with piggyback + +SWIM propagates membership changes by "piggybacking" encoded gossip data on every Ping, Ack, and PingReq message. The internal `NodeAction::SendPing` carried a `piggyback: Vec` field, but the wire-level `Ping` struct in `messages.rs` did not. Without the piggyback field in the wire message, SWIM dissemination could not function over TCP — nodes would send pings and acks but never propagate membership updates. + +Additionally, `Ping` needed a `from_addr: SocketAddr` field because `handle_ping()` requires the sender's **listen address** (not the TCP ephemeral port of the incoming connection). + +### Phase 2 — Create NodeDriver (TCP ↔ NodeAction bridge) + +The core bridge component. Owns a `DistributedNode`, `TcpTransport`, and `TcpAcceptor`. Translates between the pure state machine world and real TCP I/O. + +### Phase 3 — Add `/api/distribution` REST endpoint + +The dashboard's SSE stream provides real-time snapshot updates, but integration tests need a synchronous polling endpoint. Added a simple GET handler that returns `DistributionNodeSnapshot` as JSON. + +### Phase 4 — Create node binary crate + +A CLI binary (`swactor-node`) that wires together the NodeDriver, actor runtime, and dashboard into a deployable process. + +### Phase 5 — Docker infrastructure + +Multi-stage Dockerfile and 5-service docker-compose.yml with a bridge network and static IPs. + +### Phase 6 — Integration tests + +Rust test crate with utilities for cluster lifecycle management and 4 `#[ignore]` test scenarios that mirror the simulation tests. + +### Phase 7 — Cross-machine LAN cluster + +Split the single-machine cluster into two compose files — one for each physical machine — using `network_mode: host` for real LAN communication. Added `LanClusterHandle` to orchestrate builds and container lifecycle across machines via SSH. 4 new LAN test scenarios mirror the single-machine tests but exercise real network boundaries. + +### Phase 8 — Stale connection fix and test hardening + +Discovered and fixed a stale TCP connection pool bug in `transport.rs` where killed-and-restarted nodes couldn't rejoin because the seed's pool still held a dead connection. Added build-once optimization via `std::sync::Once` and tightened all convergence timeouts from 60–90s to 30s. + +--- + +## 4. Wire Protocol Gap — Piggyback Extension + +### The Problem + +SWIM dissemination works by attaching membership gossip to protocol messages. The `DisseminationQueue` encodes updates into a `Vec` via `pack_piggyback()`, and `NodeAction::SendPing` carries this as `piggyback: Vec`. But the wire-level `Ping` struct only had `{ from, sequence }` — no piggyback field. This meant: + +- In-process simulation: works — `handle_ping()` receives the piggyback directly from the action +- Over TCP: broken — the piggyback bytes are never serialized into the wire message + +### The Fix + +**`messages.rs`** — Added fields to three structs: + +```rust +pub struct Ping { + pub from: NodeId, + pub from_addr: SocketAddr, // NEW: sender's listen address + pub sequence: u64, + #[serde(default)] + pub piggyback: Vec, // NEW: SWIM gossip payload +} + +pub struct Ack { + pub from: NodeId, + pub sequence: u64, + #[serde(default)] + pub piggyback: Vec, // NEW +} + +pub struct PingReq { + pub from: NodeId, + pub target: NodeId, + pub target_addr: SocketAddr, + pub sequence: u64, + #[serde(default)] + pub piggyback: Vec, // NEW +} +``` + +**`#[serde(default)]`** ensures backward compatibility — if a message arrives without piggyback (e.g., from an older node), it deserializes as an empty `Vec` rather than failing. + +**`from_addr` on Ping**: The `handle_ping()` method signature requires `from_addr: SocketAddr` to learn the sender's cluster-visible listen address. Without this, the receiving node would only see the TCP ephemeral port, which is useless for SWIM (you need to know where to send Ack/PingReq *back* to the sender's listen address). + +**`transport_and_codec.rs`** — Updated Ping constructors in two tests to include the new fields. + +--- + +## 5. NodeDriver — TCP ↔ NodeAction Bridge + +### `crates/distribution/src/driver.rs` (264 lines) + +``` +NodeDriver + ├── node: DistributedNode — pure state machine + ├── transport: TcpTransport — connection pool for outgoing TCP + ├── acceptor: TcpAcceptor — non-blocking listener for incoming TCP + └── streams: Vec — accepted connections (reused across recv calls) +``` + +### Outgoing: NodeAction → TCP + +`tick()` calls `node.tick()` → iterates the returned `Vec` → maps each to a wire message and sends via TCP: + +| NodeAction | Wire Message | Destination | +|------------|-------------|-------------| +| `SendPing { to_addr, sequence, piggyback, .. }` | `Ping { from, from_addr, sequence, piggyback }` | `to_addr` | +| `SendAck { to_addr, sequence, piggyback, .. }` | `Ack { from, sequence, piggyback }` | `to_addr` | +| `SendPingReq { relay_addr, target, target_addr, sequence, piggyback, .. }` | `PingReq { from, target, target_addr, sequence, piggyback }` | `relay_addr` | +| `SendJoinRequest { to_addr }` | `JoinRequest { from, addr }` | `to_addr` | +| `SendJoinResponse { to_addr, members, .. }` | `JoinResponse { members }` | `to_addr` | +| `MembershipChanged { .. }` | *(no network I/O)* | — | + +Messages are encoded via `serde_json::to_vec()` (not the `Codec` trait — see [§10.2](#102-direct-serde-vs-codec-trait)) and wrapped in a `WireEnvelope` for TCP framing. + +### Incoming: TCP → Handler + +`recv()` calls `acceptor.try_recv()` → for each `(WireEnvelope, SocketAddr)`, dispatches by `type_tag`: + +| type_tag | Handler | Returns | +|----------|---------|---------| +| `"swactor_dist::Ping"` | `node.handle_ping(from, from_addr, seq, &piggyback)` | `Vec` (Ack) | +| `"swactor_dist::Ack"` | `node.handle_ack(from, seq, &piggyback)` | `Vec` | +| `"swactor_dist::PingReq"` | `node.handle_ping_req(from, target, target_addr, seq, &piggyback)` | `Vec` | +| `"swactor_dist::JoinRequest"` | `node.handle_join_request(from, addr)` | `Vec` | +| `"swactor_dist::JoinResponse"` | `node.handle_join_response(members)` | `Vec` | + +Response actions (e.g., the Ack generated by handle_ping) are immediately sent via the same `send_actions()` path. + +### SWIM_DEST Dummy Address + +The `WireEnvelope` format requires a `dest: ActorAddress` field (transport was designed for actor-level routing). SWIM messages route by `SocketAddr`, not `ActorAddress`, so a dummy `const SWIM_DEST: ActorAddress = ActorAddress([0u8; 32])` is used. The field is ignored on the receive side — dispatch is by `type_tag`. + +### Public API + +```rust +impl NodeDriver { + fn new(config: DistributedNodeConfig) -> Result; + fn join(&mut self, seeds: &[SocketAddr]); + fn tick(&mut self); // advance SWIM + send outgoing + fn recv(&mut self); // process incoming TCP + fn snapshot(&self) -> DistributionNodeSnapshot; + fn node(&self) -> &DistributedNode; + fn node_mut(&mut self) -> &mut DistributedNode; + fn node_id(&self) -> NodeId; + fn listen_addr(&self) -> SocketAddr; +} +``` + +--- + +## 6. REST API Endpoint + +### `/api/distribution` in `crates/runtime-dashboard/src/server.rs` + +Feature-gated with `#[cfg(feature = "distribution")]`. Returns `DistributionNodeSnapshot` as JSON on GET. + +```rust +#[cfg(feature = "distribution")] +fn handle_distribution_api( + request: tiny_http::Request, + distribution: Arc>>>, +) { + // Lock → snapshot → serialize → respond 200 with JSON + // Returns {} if no provider attached +} +``` + +The route is registered alongside existing routes (`/`, `/actors`, `/distribution`, `/events`): + +``` +"/api/distribution" => handle_distribution_api(request, distribution) +``` + +This endpoint is what the Docker integration tests poll to verify cluster state. + +--- + +## 7. Node Binary + +### `crates/node/` — `swactor-node` + +**Cargo.toml dependencies**: `distribution`, `runtime-dashboard`, `swactor`, `clap`, `ctrlc` + +**CLI arguments**: + +``` +swactor-node --listen [--seed ] [--dashboard-port ] [--actors ] +``` + +| Arg | Default | Purpose | +|-----|---------|---------| +| `--listen` | (required) | SWIM protocol listen address | +| `--seed` | (none) | Seed node to join; omit for the seed itself | +| `--dashboard-port` | 9090 | HTTP dashboard port | +| `--actors` | 0 | Number of dummy `HeartbeatActor`s to register | + +### Startup Sequence + +1. Parse CLI args +2. Set SIGTERM/SIGINT handler (`ctrlc`) +3. Start dashboard HTTP server +4. Create actor runtime (2 threads, 1024 max actors) +5. Create `NodeDriver` with SWIM config (probe_interval=5, probe_timeout=3, indirect_probes=2, suspicion_timeout=20) +6. If `--seed` provided: `driver.join(&[seed])` +7. Spawn `--actors` dummy HeartbeatActors, register each in the node's directory +8. Wire `SnapshotProvider` to dashboard (decoupled via `Arc>>`) +9. Main loop (100ms sleep): + - `driver.recv()` — process incoming TCP + - `driver.tick()` — SWIM protocol + send outgoing TCP + - Send Heartbeat to each actor (keeps them alive) + - Update cached snapshot for dashboard + +### SnapshotProvider Decoupling + +Same pattern used in `dashboard_demo.rs`: the dashboard SSE thread reads a cached `Option` behind `Arc>`, while the main loop writes a fresh snapshot each tick. The SSE thread never contends for the NodeDriver — snapshots can be up to 100ms stale, which is fine for monitoring. + +--- + +## 8. Docker Infrastructure + +### Dockerfile (9 lines) + +Multi-stage build: + +```dockerfile +FROM rust:1.93-slim AS builder +WORKDIR /build +COPY . . +RUN cargo build --release -p node + +FROM debian:bookworm-slim +COPY --from=builder /build/target/release/swactor-node /usr/local/bin/ +ENTRYPOINT ["swactor-node"] +``` + +Builder stage compiles the workspace in release mode. Runtime stage is a minimal Debian image with only the binary. + +### docker-compose.yml — 5-Node Cluster + +``` +Network: 10.0.1.0/24 (bridge) + +┌─────────────────────────────────────────────────────────────────┐ +│ seed (10.0.1.10) --listen 10.0.1.10:7000 │ +│ Dashboard: host:9091 → container:9090 │ +│ No --seed (this IS the seed) │ +├─────────────────────────────────────────────────────────────────┤ +│ node-2 (10.0.1.11) --listen 10.0.1.11:7000 --seed 10.0.1.10 │ +│ Dashboard: host:9092 → container:9090 │ +├─────────────────────────────────────────────────────────────────┤ +│ node-3 (10.0.1.12) --listen 10.0.1.12:7000 --seed 10.0.1.10 │ +│ Dashboard: host:9093 → container:9090 │ +├─────────────────────────────────────────────────────────────────┤ +│ node-4 (10.0.1.13) --listen 10.0.1.13:7000 --seed 10.0.1.10 │ +│ Dashboard: host:9094 → container:9090 │ +├─────────────────────────────────────────────────────────────────┤ +│ node-5 (10.0.1.14) --listen 10.0.1.14:7000 --seed 10.0.1.10 │ +│ Dashboard: host:9095 → container:9090 │ +└─────────────────────────────────────────────────────────────────┘ +``` + +Each node registers 2 actors (`--actors 2`), for 10 total across the cluster. + +**Static IPs**: Avoids DNS resolution complexity. Each node knows its own IP and the seed's IP at startup. SWIM dissemination handles the rest — after joining, nodes learn about each other through piggybacked gossip. + +**Port mapping**: Each container's dashboard (port 9090) is mapped to a unique host port (9091–9095) so the test harness can query each node independently. + +--- + +## 9. Integration Test Harness + +### `tests/docker/` — Workspace Member + +**Structure**: +``` +tests/docker/ +├── Cargo.toml — depends on distribution, reqwest, serde_json +├── docker-compose.yml — 5-node cluster definition +├── src/ +│ └── lib.rs — test utilities +└── tests/ + └── cluster.rs — 4 integration test scenarios +``` + +### Test Utilities (`src/lib.rs`) + +| Function/Type | Purpose | +|---------------|---------| +| `ClusterHandle` | RAII wrapper — `start()` runs `docker compose up`, `Drop` runs `docker compose down` | +| `poll_distribution(port)` | GET `/api/distribution` → `Option` | +| `wait_for_convergence(ports, expected_alive, timeout)` | Poll until all nodes see `>= expected_alive` members | +| `wait_for_death_detection(ports, max_alive, timeout)` | Poll until all nodes see `<= max_alive` members | +| `kill_node(service)` | `docker compose stop ` | +| `restart_node(service)` | `docker compose start ` | + +**Compose file resolution**: Uses `env!("CARGO_MANIFEST_DIR")` to build an absolute path to `docker-compose.yml` at compile time. This avoids path-doubling issues when `cargo test` runs from a different working directory. + +### 4 Test Scenarios (`tests/cluster.rs`) + +All marked `#[ignore]` — require Docker. Run with: `cargo test -p docker-tests -- --ignored` + +#### Test 1: `cluster_of_five_converges` +*Mirrors: `distribution_sim.rs::cluster_of_five_converges`* + +``` +Given: 5 nodes started via docker compose +When: wait up to 30s for convergence +Then: all 5 nodes report alive_count >= 4 + and routing_table_size >= 3 +``` + +#### Test 2: `node_death_is_detected` +*Mirrors: `distribution_sim.rs::node_death_is_detected`* + +``` +Given: converged 5-node cluster +When: docker compose stop node-3 +Then: within 30s, surviving 4 nodes report alive_count <= 4 + and at least one survivor sees dead_count >= 1 +``` + +#### Test 3: `killed_node_rejoins` +*Mirrors: `distribution_sim.rs::killed_node_rejoins`* + +``` +Given: converged cluster, node-3 killed and detected dead +When: docker compose start node-3 +Then: within 30s, node-3 reports alive_count >= 1 +``` + +#### Test 4: `actors_resolvable_across_cluster` +*Mirrors: `distribution_sim.rs::actors_resolvable_across_cluster`* + +``` +Given: converged 5-node cluster, each with 2 registered actors +When: query each node's snapshot +Then: each node has directory_entry_count >= 2 + total directory entries across cluster >= 10 + total cache entries >= 5 +``` + +### Simulation ↔ Docker Parity + +The simulation tests run in-process with direct method calls. The Docker tests exercise the same protocol logic but over real TCP connections, Docker networking, and process boundaries. Both assert the same behavioral properties: + +| Property | Simulation Test | Docker Test | +|----------|----------------|-------------| +| 5-node cluster converges | `cluster_of_five_converges` | `cluster_of_five_converges` | +| Dead node detected | `node_death_is_detected` | `node_death_is_detected` | +| Killed node rejoins | `killed_node_rejoins` | `killed_node_rejoins` | +| Actors in directory | `actors_resolvable_across_cluster` | `actors_resolvable_across_cluster` | + +--- + +## 10. Cross-Machine LAN Cluster + +### Motivation + +The single-machine Docker cluster validates SWIM over TCP within a bridge network on one host. This leaves a gap: real deployments span multiple machines with distinct network stacks. The LAN cluster tests exercise this by splitting 5 nodes across two physical machines communicating over a real Ethernet LAN. + +### Infrastructure + +**Machines**: +- **devuan-hpz** (192.168.1.106): runs seed + node-2 (2 nodes) +- **thinkpad** (192.168.1.102): runs node-3, node-4, node-5 (3 nodes) + +**Split compose files**: Unlike the single-machine cluster (bridge network with static IPs), the LAN cluster uses `network_mode: host` so containers bind directly to the host's LAN interface. + +``` +docker-compose.lan-hpz.yml docker-compose.lan-thinkpad.yml +┌──────────────────────────┐ ┌───────────────────────────────┐ +│ seed 192.168.1.106:7000│ │ node-3 192.168.1.102:7000 │ +│ node-2 192.168.1.106:7001│ │ node-4 192.168.1.102:7001 │ +│ Dashboards: 9091, 9092 │ │ node-5 192.168.1.102:7002 │ +└──────────────────────────┘ │ Dashboards: 9093, 9094, 9095 │ + ↕ LAN (2ms) └───────────────────────────────┘ +``` + +Each thinkpad node seeds to `192.168.1.106:7000` (the hpz seed). With `network_mode: host`, each node needs a unique port on its host — hence 7000/7001 on hpz and 7000/7001/7002 on thinkpad. + +### Orchestration + +**Repo sync**: thinkpad has no rsync, so `LanClusterHandle` uses `tar czf | scp | ssh tar xzf` to push the workspace (excluding `target/` and `.git/`). + +**Build-once optimization**: A `static BUILD_LAN_ONCE: Once` ensures that repo sync + `docker compose build` on both machines happens exactly once per test run. Subsequent `LanClusterHandle::start()` calls skip the build and just run `docker compose up -d`. This reduced the full 4-test suite from ~840s to ~630s. + +**Remote control**: `kill_remote_node()` and `restart_remote_node()` execute `docker compose stop/start` on the thinkpad via SSH. + +### Shell Script (`run-lan-cluster.sh`) + +A standalone orchestration script for quick LAN validation outside of `cargo test`. Syncs repo, builds on both machines, starts both sides, polls all 5 dashboards for convergence, reports pass/fail, and tears down via a trap handler on exit. + +### LAN Test Scenarios (`tests/docker/tests/lan_cluster.rs`) + +All marked `#[test] #[ignore]`, run with: `cargo test -p docker-tests -- --ignored lan_ --test-threads=1` + +#### Test 1: `lan_cluster_converges` +``` +Given: 5 nodes split across hpz and thinkpad +When: wait up to 30s for convergence +Then: all 5 nodes report alive_count >= 4 and routing_table_size >= 3 +``` + +#### Test 2: `lan_remote_node_death_detected` +``` +Given: converged LAN cluster +When: kill node-3 on thinkpad +Then: within 30s, 4 survivors see alive_count <= 4 + and at least one survivor sees dead_count >= 1 +``` + +#### Test 3: `lan_killed_remote_node_rejoins` +``` +Given: converged cluster, node-3 killed and detected dead +When: restart node-3 on thinkpad +Then: within 30s, node-3 reports alive_count >= 1 +``` + +#### Test 4: `lan_actors_resolvable_cross_machine` +``` +Given: converged 5-node LAN cluster, each with 2 registered actors +When: query each node's snapshot +Then: each node has directory_entry_count >= 2 + total directory entries >= 10, total cache >= 5 +``` + +--- + +## 11. Design Decisions & Tradeoffs + +### 11.1 NodeDriver as Separate Module (not in node binary) + +**Choice**: `driver.rs` lives in `crates/distribution/`, not in `crates/node/`. + +**Why**: The driver is reusable — any binary that wants to run a DistributedNode over TCP can use it. The node binary (`crates/node/`) is one consumer; future consumers might embed distribution in a larger application. Keeping the driver in the distribution crate means it stays testable alongside the protocol logic. + +### 11.2 Direct serde_json vs. Codec Trait + +**Choice**: NodeDriver uses `serde_json::to_vec()`/`serde_json::from_slice()` directly, not the `Codec` trait or `CodecRegistry`. + +**Why**: The `Codec` trait is parametric — `JsonCodec` implements `Codec`, `Codec`, etc. as separate trait impls. You can't write generic code like `codec.encode(any_message)` because each message type is a different impl. The CodecRegistry solves this on the send side via type erasure (`TypeId → encoder`), but it requires `Box` downcasting which adds complexity for no benefit here — the driver already knows the concrete message type at each call site. + +Using `serde_json` directly is simpler and equivalent — the JsonCodec just calls `serde_json` internally. When the codec is eventually swapped to bincode/msgpack, the driver can switch to the new serializer just as easily. + +### 11.3 Static IPs over DNS + +**Choice**: Docker Compose services use static IPs (`10.0.1.10`–`10.0.1.14`) rather than Docker DNS names. + +**Why**: The SWIM protocol routes by `SocketAddr`, not hostname. Using DNS would require DNS resolution at startup plus a hostname→addr mapping. Static IPs are simpler and deterministic. The subnet `10.0.1.0/24` is a private range unlikely to conflict with host networking. + +**Tradeoff**: Less flexible — adding a 6th node requires editing the compose file with a new static IP. Acceptable for a fixed test cluster. + +### 11.4 `#[ignore]` Tests over Separate Test Target + +**Choice**: Docker tests use `#[test] #[ignore]` rather than a separate binary or integration test feature flag. + +**Why**: Standard Rust convention. `cargo test` skips them by default; `cargo test -- --ignored` runs them. No extra CI configuration needed. The test crate is already in its own workspace member (`tests/docker/`), providing isolation. + +### 11.5 Host Networking for LAN Cluster + +**Choice**: LAN compose files use `network_mode: host` instead of Docker bridge networking. + +**Why**: Bridge networking with port forwarding would work for single-machine tests but not for cross-machine communication — a container on machine A needs to reach a container on machine B at its real LAN IP. With host networking, containers bind directly to the host's interface and are reachable at the host's LAN address. This requires unique ports per container on each host (7000, 7001, ... instead of all using 7000). + +### 11.6 Build-Once via `std::sync::Once` + +**Choice**: Docker images are built once per test run using `std::sync::Once`, then reused across all 4 tests. + +**Why**: Each `docker compose up --build` triggers a full Rust release build inside Docker (~40s on hpz, ~50s on thinkpad). With 4 serial tests, that's 8 redundant builds. Separating `docker compose build` (guarded by `Once`) from `docker compose up -d` (per-test) cuts total runtime from ~840s to ~630s. The first test pays the build cost; tests 2–4 just start pre-built containers. + +### 11.7 100ms Tick Loop over Async Runtime + +**Choice**: The node binary uses a synchronous 100ms `thread::sleep` loop, not tokio/async-std. + +**Why**: The entire distribution layer is synchronous (`DistributedNode` is `!Send`). Introducing an async runtime adds complexity with no benefit — the tick loop is CPU-light (one tick processes a handful of messages) and the 100ms sleep provides natural backpressure. The TCP transport uses non-blocking I/O for the acceptor and blocking I/O with connection pooling for outgoing sends. + +### 11.8 `#[serde(default)]` for Backward Compatibility + +**Choice**: New `piggyback` fields use `#[serde(default)]` so missing fields deserialize as empty `Vec`. + +**Why**: If a node running old code (without piggyback) sends a Ping to a node running new code, the message should still deserialize successfully. The new node sees an empty piggyback — no gossip propagated, but no crash either. This matters during rolling upgrades. + +--- + +## 12. Bugs Encountered + +### 12.1 Compose File Path Doubling + +**Symptom**: `cargo test -p docker-tests -- --ignored` failed with: +``` +open tests/docker/tests/docker/docker-compose.yml: no such file or directory +``` + +**Cause**: The compose file path was defined as a relative constant: +```rust +const COMPOSE_FILE: &str = "tests/docker/docker-compose.yml"; +``` +But `cargo test` runs with the crate root as working directory. Since the crate root is already `tests/docker/`, the resolved path became `tests/docker/tests/docker/docker-compose.yml` — doubled. + +**Fix**: Replaced the relative constant with `env!("CARGO_MANIFEST_DIR")`: +```rust +const COMPOSE_DIR: &str = env!("CARGO_MANIFEST_DIR"); + +fn compose_file() -> String { + let mut p = PathBuf::from(COMPOSE_DIR); + p.push("docker-compose.yml"); + p.to_string_lossy().into_owned() +} +``` + +This compiles the crate's absolute filesystem path into the binary, so the compose file is always found regardless of working directory. + +### 12.2 Codec Trait Parametric Mismatch + +**Symptom**: First version of `driver.rs` attempted: +```rust +self.codec.encode(&msg) // where codec: JsonCodec +``` + +Compilation failed because `JsonCodec` implements `Codec`, `Codec`, etc. as separate trait impls. A single `codec` variable can't be used generically across all message types without trait object gymnastics. + +**Fix**: Bypassed the Codec trait entirely. Used `serde_json::to_vec()` and `serde_json::from_slice()` directly. The driver knows the concrete type at each match arm, so generic dispatch isn't needed. + +### 12.3 Stale TCP Connection Pool on Node Rejoin + +**Symptom**: The `lan_killed_remote_node_rejoins` test failed — the restarted node's dashboard responded (it was running) but reported `alive_count=0`. The node never received a `JoinResponse` from the seed. + +**Cause**: `TcpTransport` maintains a connection pool keyed by `SocketAddr`. When node-3 was killed (container stopped), the seed's pool still held a TCP connection to `192.168.1.102:7000`. When node-3 restarted and sent a `JoinRequest`, the seed generated a `JoinResponse` and called `send_to(192.168.1.102:7000, ...)`. The pool returned the stale connection — `try_clone()` succeeded (the FD was still valid), but `write_all()` silently failed or the data went into a dead socket. The `JoinResponse` was never delivered. + +**Fix**: Added retry-on-write-failure logic to `TcpTransport::send_to()`: + +```rust +pub fn send_to(&self, addr: SocketAddr, envelope: WireEnvelope) -> Result<(), Error> { + let buf = encode_wire_envelope(&envelope); + let mut stream = self.get_or_connect(addr)?; + match stream.write_all(&buf) { + Ok(()) => Ok(()), + Err(_) => { + // Evict stale connection and retry once + self.pool.lock().unwrap().remove(&addr); + let mut stream = self.get_or_connect(addr)?; + stream + .write_all(&buf) + .map_err(|e| Error::from(format!("TCP send to {addr}: {e}"))) + } + } +} +``` + +On write failure, the stale entry is evicted and a fresh connection is established. This handles the common case of a peer that died and came back at the same address. The retry is limited to one attempt — if the second write also fails, the error propagates. + +**Impact**: This bug only manifests in kill/restart scenarios where a node returns at the same `SocketAddr`. It would not appear in simulation tests (no real TCP) or in the single-machine bridge cluster (Docker assigns new IPs on restart). It required real LAN testing with `network_mode: host` to surface. + +--- + +## 13. Known Gaps & Future Improvements + +| Gap | Effort | Impact | Notes | +|-----|--------|--------|-------| +| Kademlia messages not wired in driver | Medium | High | NodeDriver only handles SWIM messages. FindNode/FindValue/Store RPCs are not sent or received. Full Kademlia lookup requires this. | +| No graceful shutdown protocol | Small | Medium | Node binary calls `driver.node().leave()` but doesn't drain in-flight messages or wait for death dissemination | +| Heartbeat actors are fire-and-forget | Small | Low | HeartbeatActor never responds; actor liveness isn't verified | +| No health check in Docker | Small | Medium | Compose could use `HEALTHCHECK` to avoid `--wait` fallback path | +| No TLS | Medium | Medium | All TCP traffic is plaintext. Fine for a test cluster on a private network; not suitable for production | +| No resource limits | Small | Low | Docker containers have no memory/CPU limits; could OOM on constrained hosts | +| No partition testing | Medium | High | Docker supports `iptables`-based network partitions but no test exercises split-brain scenarios yet | + +--- + +## 14. Test Coverage Summary + +### Existing Tests — Unchanged + +All 182 existing tests continue to pass: +- 134 distribution crate tests (133 original + 1 from updated constructors) +- 47 simulation tests +- 1 swactor core test + +### Docker Integration Tests — 4 Single-Machine Scenarios + +| Test | Mirrors Simulation | Asserts | +|------|-------------------|---------| +| `cluster_of_five_converges` | `distribution_sim::cluster_of_five_converges` | alive_count >= 4, routing_table_size >= 3 | +| `node_death_is_detected` | `distribution_sim::node_death_is_detected` | alive_count <= 4, dead_count >= 1 | +| `killed_node_rejoins` | `distribution_sim::killed_node_rejoins` | alive_count >= 1 after restart | +| `actors_resolvable_across_cluster` | `distribution_sim::actors_resolvable_across_cluster` | directory_entry_count >= 2, total >= 10, cache >= 5 | + +Run with: `cargo test -p docker-tests -- --ignored cluster --test-threads=1` + +### LAN Integration Tests — 4 Cross-Machine Scenarios + +| Test | Asserts | +|------|---------| +| `lan_cluster_converges` | 5 nodes across 2 machines: alive_count >= 4, routing_table_size >= 3 | +| `lan_remote_node_death_detected` | Kill node on thinkpad: survivors see alive_count <= 4, dead_count >= 1 | +| `lan_killed_remote_node_rejoins` | Restart killed node: rejoins with alive_count >= 1 | +| `lan_actors_resolvable_cross_machine` | directory_entry_count >= 2 per node, total >= 10, cache >= 5 | + +Run with: `cargo test -p docker-tests -- --ignored lan_ --test-threads=1` + +All convergence timeouts are 30 seconds. Convergence happens in seconds over the LAN; 30s is a generous safety margin that still catches real failures quickly. + +### Verification + +- `cargo check --workspace` — clean +- `cargo test` — all 182 tests pass +- `cargo build --release -p node` — node binary builds +- Local 2-node TCP smoke test — nodes discover each other, dashboard returns valid JSON +- Single-machine Docker tests: 4/4 pass (on thinkpad) +- LAN Docker tests: 4/4 pass (hpz + thinkpad, ~630s total) + +--- + +## Files Created/Modified + +| Action | File | Purpose | +|--------|------|---------| +| Modified | `crates/distribution/src/messages.rs` | Added piggyback + from_addr fields | +| Created | `crates/distribution/src/driver.rs` | NodeDriver (TCP ↔ NodeAction bridge) | +| Modified | `crates/distribution/src/lib.rs` | Added `pub mod driver` | +| Modified | `crates/distribution/src/transport.rs` | Stale connection retry in `send_to()` | +| Modified | `crates/distribution/tests/transport_and_codec.rs` | Updated Ping constructors | +| Modified | `crates/runtime-dashboard/src/server.rs` | Added `/api/distribution` route | +| Created | `crates/node/Cargo.toml` | Node binary crate config | +| Created | `crates/node/src/main.rs` | swactor-node CLI binary | +| Modified | `Cargo.toml` (root) | Added `crates/node`, `tests/docker` to workspace | +| Created | `Dockerfile` | Multi-stage Docker build | +| Created | `tests/docker/Cargo.toml` | Docker tests crate config | +| Created | `tests/docker/docker-compose.yml` | 5-node single-machine cluster | +| Created | `tests/docker/docker-compose.lan-hpz.yml` | LAN cluster — hpz side (2 nodes) | +| Created | `tests/docker/docker-compose.lan-thinkpad.yml` | LAN cluster — thinkpad side (3 nodes) | +| Created | `tests/docker/run-lan-cluster.sh` | LAN cluster orchestration script | +| Created | `tests/docker/src/lib.rs` | Test utilities (ClusterHandle, LanClusterHandle, build-once) | +| Created | `tests/docker/tests/cluster.rs` | 4 single-machine integration tests | +| Created | `tests/docker/tests/lan_cluster.rs` | 4 cross-machine LAN integration tests | diff --git a/docs/development_history/SIMULATION_TESTING.md b/docs/development_history/SIMULATION_TESTING.md new file mode 100644 index 0000000..1a9547b --- /dev/null +++ b/docs/development_history/SIMULATION_TESTING.md @@ -0,0 +1,190 @@ +# Simulation Testing — Development History + +> Covers the addition of network fault injection to the simulation harness +> and 15 new cluster scenario tests, informed by research into production +> distributed systems testing practices. +> +> 4 files changed · ~950 insertions +> +> *Branch: `distribution-realization`* + +--- + +## Table of Contents + +1. [Overview & Motivation](#1-overview--motivation) +2. [What Was Built](#2-what-was-built) +3. [Research Phase](#3-research-phase) +4. [Network Fault Injection](#4-network-fault-injection) +5. [Cluster Scenario Tests](#5-cluster-scenario-tests) +6. [Key Findings](#6-key-findings) +7. [Design Decisions](#7-design-decisions) +8. [Known Gaps & Future Work](#8-known-gaps--future-work) + +--- + +## 1. Overview & Motivation + +The simulation crate (`crates/simulation/`) had 6 distribution tests covering +happy-path scenarios: cluster convergence, node death detection, node rejoin, +and actor resolution. All tests assumed a perfect network — 100% delivery, +zero latency variation, no partitions. + +Real networks drop packets, partition nodes, and deliver messages out of order. +The SWIM protocol's correctness under these conditions was untested. This work +adds network fault simulation and exercises the protocol under adversarial +conditions drawn from established testing methodologies. + +--- + +## 2. What Was Built + +| Component | Location | Description | +|-----------|----------|-------------| +| Network fault model | `crates/simulation/src/distribution/sim.rs` | Partition, heal, and message drop simulation | +| 15 cluster scenario tests | `crates/simulation/tests/cluster_scenarios.rs` | Behavioral tests for failure modes | +| Research notes | `CLAUDE/notes/research_simulation_testing.md` | Survey of 7 codebases/frameworks | + +All 15 new tests run in ~1.4s total (well under the 2-minute cap). +The original 6 distribution_sim tests are unaffected. + +--- + +## 3. Research Phase + +Seven codebases and frameworks were studied for their simulation testing +methodology: + +| Source | Key Takeaway | +|--------|-------------| +| **FoundationDB** | Deterministic simulation: single-threaded, seeded PRNG, simulated time. BUGGIFY injects faults inside production code at ~25% activation × 25% firing probability. | +| **Hashicorp memberlist** | ~80 test functions. Lifeguard extensions: suspicion timer with log(k+1) decay, health-aware probe timeouts, dogpile confirmation. | +| **Antithesis** | Categorized fault injection: network, process, disk, timing. Emphasis on property-based invariant checking. | +| **TigerBeetle** | VOPR simulation + Vortex TCP proxy. Runs millions of seeds nightly. | +| **Turmoil** (tokio-rs) | Rust DST: `sim.partition(a,b)`, `sim.hold(a,b)`, `sim.repair(a,b)`. Seeded RNG, simulated time. | +| **MadSim** | Rust DST used by RisingWave. FIRO scheduling, libc interception for true determinism. | +| **Jepsen** | Standard nemesis catalog: partition, kill, pause, clock skew, membership change. | + +Full notes: `CLAUDE/notes/research_simulation_testing.md` + +--- + +## 4. Network Fault Injection + +Three new types model network conditions: + +```rust +pub struct Partition { + pub side_a: Vec, // node indices + pub side_b: Vec, + pub asymmetric: bool, // if true, only side_a→side_b is blocked +} + +pub enum NetworkFault { + Partition { round: usize, partition: Partition }, + Heal { round: usize }, + SetDropRate { round: usize, rate: f64 }, +} +``` + +`NetworkState` tracks blocked pairs (as a `HashSet<(usize, usize)>`) and +applies probabilistic message dropping via a deterministic LCG PRNG +(seed `0x853c49e6748fea9b`). The `should_deliver(from, to)` method checks +both partition membership and drop rate before allowing message delivery. + +Faults are applied per-round in `run_simulation` before the tick/deliver +cycle. Initial join and settle phases always use a clean `NetworkState` +(no faults during cluster formation). + +### Backward Compatibility + +`DistributionSimConfig` gained a `network_faults: Vec` field +defaulting to an empty vec. Existing tests that don't set this field +see no behavior change — the renamed `deliver_actions_tagged_with_net` +function with a clean `NetworkState` is functionally identical to the +original `deliver_actions_tagged`. + +--- + +## 5. Cluster Scenario Tests + +15 tests organized by failure category: + +### Partitions +| Test | Scenario | Assertion | +|------|----------|-----------| +| `symmetric_partition_splits_membership_views` | 6 nodes split {0,1,2} vs {3,4,5} | Each side forms sub-cluster; dead-declared nodes not auto-rediscovered | +| `asymmetric_partition_causes_one_sided_suspicion` | 5 nodes, one-way block | Recovery after heal | +| `partition_plus_kill_in_minority_side` | 6 nodes, partition + kill in minority | Compound failure handled | +| `sequential_partitions_fragment_cluster` | Sequential partition events | Creates sub-clusters | +| `actor_resolution_degrades_during_partition` | Actors registered pre-partition | Cached resolutions survive partition | + +### Message Loss +| Test | Scenario | Assertion | +|------|----------|-----------| +| `cluster_converges_under_10_percent_message_loss` | 10% drop rate | Some membership maintained | +| `heavy_message_loss_causes_membership_instability` | 30% drop rate | Degrades but doesn't crash | +| `cluster_survives_brief_message_loss` | 15% loss for 15 rounds then heals | ≥2 well-connected survivors | + +### Node Failures +| Test | Scenario | Assertion | +|------|----------|-----------| +| `cluster_survives_seed_node_death` | Kill node 0 (seed) | 4 survivors maintain ≥60% accuracy | +| `simultaneous_two_node_failure_detected` | Kill 2 of 7 at once | Both deaths detected | +| `cascading_failures_leave_quorum_intact` | Kill 3 of 7 sequentially | Survivors maintain membership | +| `graceful_leave_detected_faster_than_crash` | Crash detection timing | Bounded detection rounds | + +### Scale & Churn +| Test | Scenario | Assertion | +|------|----------|-----------| +| `cluster_of_fifty_converges` | 50-node cluster | ≥90% accuracy | +| `rapid_churn_maintains_partial_membership` | 8 nodes, 4 kill/revive cycles | Partial membership maintained | +| `membership_changes_disseminate_to_all_nodes` | 10-node cluster, verify propagation | All survivors detect death | + +--- + +## 6. Key Findings + +1. **SWIM does not auto-rediscover dead-declared nodes.** Once the suspicion + timeout expires and a node is declared dead, it is permanently removed. + Re-joining requires the join protocol. This is correct SWIM behavior, + not a bug — but tests must account for it. + +2. **Message loss is highly destabilizing for SWIM** because it affects both + the direct probe AND indirect probes in the same cycle. Default config + (`suspicion_timeout=5`, `indirect_probes=1`) cannot tolerate even 15% + loss. Tuned config (`suspicion_timeout=15–20`, `indirect_probes=2`, + `probe_timeout=5`) tolerates ~10%. + +3. **The LCG PRNG for message dropping needs a non-zero seed** to avoid + correlated early values (seed 0 always produces 0.0 as first output, + causing deterministic first-message drop). + +4. **50-node clusters converge quickly** with the simulation's + topology-aware join strategy, achieving ≥90% accuracy. + +--- + +## 7. Design Decisions + +| Decision | Rationale | +|----------|-----------| +| LCG instead of `rand` crate | Keeps simulation deterministic without adding dependencies; 64-bit LCG with Knuth constants is sufficient for drop-rate testing | +| Blocked pairs in HashSet | O(1) lookup per message; partition model maps directly to real network behavior | +| Clean NetworkState for join/settle | Faults during initial cluster formation would conflate test setup with test assertions | +| Loose accuracy thresholds for loss tests | SWIM's sensitivity to message loss means tight thresholds create flaky tests; the behavioral property being tested is "degrades gracefully" not "maintains perfect accuracy" | +| Tests verify SWIM's actual semantics | Rather than expecting auto-recovery after partition heal (which SWIM doesn't support), tests verify the sub-cluster formation that actually occurs | + +--- + +## 8. Known Gaps & Future Work + +| Gap | Priority | Notes | +|-----|----------|-------| +| Property-based invariant checking | High | Formal completeness/accuracy as automated checks | +| Message reordering | Medium | Out-of-order delivery in network model | +| Kademlia-specific scenarios | Medium | Routing table convergence under churn, directory repair | +| Suspicion refutation tests | Medium | Incarnation bump prevents false death | +| Graceful leave protocol | Medium | Wire `node.leave()` into simulation | +| BUGGIFY-style injection | Low | Probabilistic faults at protocol decision points | +| Re-join after partition heal | Low | Auto-rediscovery mechanism (not standard SWIM) | diff --git a/docs/actor_lifecycle.svg b/docs/diagrams/actor_lifecycle.svg similarity index 100% rename from docs/actor_lifecycle.svg rename to docs/diagrams/actor_lifecycle.svg diff --git a/docs/actor_resolution.svg b/docs/diagrams/actor_resolution.svg similarity index 100% rename from docs/actor_resolution.svg rename to docs/diagrams/actor_resolution.svg diff --git a/docs/architecture.svg b/docs/diagrams/architecture.svg similarity index 100% rename from docs/architecture.svg rename to docs/diagrams/architecture.svg diff --git a/docs/dataflow.svg b/docs/diagrams/dataflow.svg similarity index 100% rename from docs/dataflow.svg rename to docs/diagrams/dataflow.svg diff --git a/docs/distribution_minor_flows.svg b/docs/diagrams/distribution_minor_flows.svg similarity index 100% rename from docs/distribution_minor_flows.svg rename to docs/diagrams/distribution_minor_flows.svg diff --git a/docs/message_lifecycle.svg b/docs/diagrams/message_lifecycle.svg similarity index 100% rename from docs/message_lifecycle.svg rename to docs/diagrams/message_lifecycle.svg diff --git a/docs/runtime_lifecycle.svg b/docs/diagrams/runtime_lifecycle.svg similarity index 100% rename from docs/runtime_lifecycle.svg rename to docs/diagrams/runtime_lifecycle.svg diff --git a/docs/swim_probe_cycle.svg b/docs/diagrams/swim_probe_cycle.svg similarity index 100% rename from docs/swim_probe_cycle.svg rename to docs/diagrams/swim_probe_cycle.svg diff --git a/docs/tick_cycle.svg b/docs/diagrams/tick_cycle.svg similarity index 100% rename from docs/tick_cycle.svg rename to docs/diagrams/tick_cycle.svg diff --git a/docs/transport_encode_decode.svg b/docs/diagrams/transport_encode_decode.svg similarity index 100% rename from docs/transport_encode_decode.svg rename to docs/diagrams/transport_encode_decode.svg diff --git a/docs/transport_routing.svg b/docs/diagrams/transport_routing.svg similarity index 100% rename from docs/transport_routing.svg rename to docs/diagrams/transport_routing.svg diff --git a/docs/type_erasure.svg b/docs/diagrams/type_erasure.svg similarity index 100% rename from docs/type_erasure.svg rename to docs/diagrams/type_erasure.svg diff --git a/docs/distribution.md b/docs/distribution/distribution.md similarity index 68% rename from docs/distribution.md rename to docs/distribution/distribution.md index 31fe3cc..3b438c8 100644 --- a/docs/distribution.md +++ b/docs/distribution/distribution.md @@ -51,7 +51,7 @@ repair infrastructure into a single public API. leave() ──► disseminate Dead for self, graceful shutdown ``` -See [distribution_minor_flows.svg](distribution_minor_flows.svg) for the +See [distribution_minor_flows.svg](../diagrams/distribution_minor_flows.svg) for the join handshake, dissemination piggybacking, and membership change cascade. ## Actor Registration @@ -63,7 +63,7 @@ join handshake, dissemination piggybacking, and membership change cascade. 4. Register with `RepublishTracker` for periodic re-STORE. 5. Return the signed entry — caller STOREs to `r`-closest nodes. -See [actor_resolution.svg](actor_resolution.svg) for the full datapath. +See [actor_resolution.svg](../diagrams/actor_resolution.svg) for the full datapath. ## Actor Resolution @@ -89,6 +89,41 @@ propagates effects through all subsystems: This cascade ensures that a single SWIM death detection triggers routing table cleanup, cache invalidation, and directory repair in one tick. +## NodeDriver — TCP Network Bridge + +`NodeDriver` (`driver.rs`) bridges the pure state machine API with real TCP +networking. It owns a `DistributedNode` plus a `TcpTransport` (connection +pool) and `TcpAcceptor` (non-blocking listener). + +``` +┌─ NodeDriver ─────────────────────────────────────────────────┐ +│ │ +│ node: DistributedNode ← pure state machine │ +│ transport: TcpTransport ← connection pool for outgoing │ +│ acceptor: TcpAcceptor ← non-blocking listener │ +│ streams: Vec ← accepted connections │ +│ │ +│ tick() ──► node.tick() → map NodeAction → TCP send │ +│ recv() ──► acceptor.try_recv() → dispatch → handler calls │ +│ join() ──► node.join() → send JoinRequest via TCP │ +│ │ +└───────────────────────────────────────────────────────────────┘ +``` + +The caller runs a loop: `recv()` → `tick()` → sleep. The driver handles +all TCP I/O internally — the caller never touches sockets directly. + +See [DOCKER_REALIZATION.md](../development_history/DOCKER_REALIZATION.md) +for implementation details of the driver, the node binary (`crates/node/`), +and the Docker cluster integration tests. + +## Dashboard REST API + +The runtime dashboard exposes `/api/distribution` (feature-gated with +`distribution`) which returns the `DistributionNodeSnapshot` as JSON. +This supplements the SSE stream (`/events`) with a synchronous polling +endpoint used by integration tests. + ## Where Things Live | Type | File | Role | @@ -96,6 +131,7 @@ table cleanup, cache invalidation, and directory repair in one tick. | `DistributedNode` | `node.rs` | Top-level integration facade | | `DistributedNodeConfig` | `node.rs` | Node configuration | | `ResolveResult` | `node.rs` | 3-tier resolution outcomes | +| `NodeDriver` | `driver.rs` | TCP ↔ NodeAction bridge | | `LocationCache` | `cache.rs` | LRU actor→node cache | | `Keypair` | `crypto.rs` | Ed25519 keypair + signing | | `NodeId` | `types.rs` | 32-byte node identity | diff --git a/docs/kademlia.md b/docs/distribution/kademlia.md similarity index 98% rename from docs/kademlia.md rename to docs/distribution/kademlia.md index eb597f7..4b90da5 100644 --- a/docs/kademlia.md +++ b/docs/distribution/kademlia.md @@ -37,7 +37,7 @@ lookups and for selecting STORE targets. ## Iterative Lookup -See [actor_resolution.svg](actor_resolution.svg) for the registration and +See [actor_resolution.svg](../diagrams/actor_resolution.svg) for the registration and resolution datapaths. The `NodeLookup` state machine drives iterative `FIND_NODE`: diff --git a/docs/swim.md b/docs/distribution/swim.md similarity index 98% rename from docs/swim.md rename to docs/distribution/swim.md index 0d73dfd..9c2eb1c 100644 --- a/docs/swim.md +++ b/docs/distribution/swim.md @@ -7,7 +7,7 @@ deny reachability before the node is suspected and eventually declared dead. ## Probe Cycle -See [swim_probe_cycle.svg](swim_probe_cycle.svg) for the full state machine. +See [swim_probe_cycle.svg](../diagrams/swim_probe_cycle.svg) for the full state machine. The probe cycle is a pure state machine driven by ticks: diff --git a/docs/transport.md b/docs/distribution/transport.md similarity index 89% rename from docs/transport.md rename to docs/distribution/transport.md index f75452f..b57c9ba 100644 --- a/docs/transport.md +++ b/docs/distribution/transport.md @@ -17,9 +17,9 @@ cargo test --features transport No serde bounds — the codec defines what it needs from `M`. - **`Transport`** — WHERE bytes are sent. InMemory (testing), TCP, gRPC, etc. -See [transport_routing.svg](../crates/runtime-dashboard/docs/transport_routing.svg) +See [transport_routing.svg](../diagrams/transport_routing.svg) for the extended routing chain, and -[transport_encode_decode.svg](../crates/runtime-dashboard/docs/transport_encode_decode.svg) +[transport_encode_decode.svg](../diagrams/transport_encode_decode.svg) for the encode/decode data flow. ## Core Types @@ -96,6 +96,15 @@ which remote addresses exist via `router.add_route()`. Since addresses are 32 random bytes, runtimes must exchange them out-of-band (e.g., over the TCP connection itself — see `examples/tcp_ping_pong.rs`). +## Distribution Driver + +The distribution layer's `NodeDriver` (`crates/distribution/src/driver.rs`) +is the primary consumer of the TCP transport. It uses `TcpTransport` for +outgoing SWIM messages and `TcpAcceptor` for incoming, bypassing the +`Codec` trait in favor of direct `serde_json` serialization (see +[DOCKER_REALIZATION.md](../development_history/DOCKER_REALIZATION.md) +§10.2 for rationale). + ## Limitations - **No automatic discovery** — manual address exchange required diff --git a/docs/render_docs.sh b/docs/render_docs.sh index 31b6dca..25c409c 100755 --- a/docs/render_docs.sh +++ b/docs/render_docs.sh @@ -4,8 +4,11 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" DOCS_DIR="$SCRIPT_DIR" +DIAGRAMS_DIR="$DOCS_DIR/diagrams" TOOLS_DIR="$ROOT_DIR/tools" +mkdir -p "$DIAGRAMS_DIR" + # --- Phase 1: Generate architecture.dot from source AST --- echo "==> Generating architecture.dot from source..." @@ -23,8 +26,8 @@ fi render_with_dot() { for src in "${dots[@]}"; do name="$(basename "$src" .dot)" - echo " dot: $name.dot -> $name.svg" - dot -Tsvg "$src" -o "$DOCS_DIR/$name.svg" + echo " dot: $name.dot -> diagrams/$name.svg" + dot -Tsvg "$src" -o "$DIAGRAMS_DIR/$name.svg" done } @@ -50,8 +53,8 @@ for (const file of dots) { const src = readFileSync(join(docsDir, file), "utf-8"); const name = basename(file, ".dot"); const svg = viz.renderString(src, { format: "svg" }); - writeFileSync(join(docsDir, \`\${name}.svg\`), svg); - console.log(\` viz-js: \${file} -> \${name}.svg\`); + writeFileSync(join(docsDir, "diagrams", \`\${name}.svg\`), svg); + console.log(\` viz-js: \${file} -> diagrams/\${name}.svg\`); } NODEJS @@ -79,4 +82,4 @@ else exit 1 fi -echo "==> Done. SVGs in $DOCS_DIR/" +echo "==> Done. SVGs in $DIAGRAMS_DIR/" diff --git a/docs/actor-model.md b/docs/runtime/actor-model.md similarity index 100% rename from docs/actor-model.md rename to docs/runtime/actor-model.md diff --git a/docs/channels.md b/docs/runtime/channels.md similarity index 100% rename from docs/channels.md rename to docs/runtime/channels.md diff --git a/docs/runtime.md b/docs/runtime/runtime.md similarity index 100% rename from docs/runtime.md rename to docs/runtime/runtime.md diff --git a/docs/worker-thread.md b/docs/runtime/worker-thread.md similarity index 100% rename from docs/worker-thread.md rename to docs/runtime/worker-thread.md diff --git a/tests/docker/Cargo.toml b/tests/docker/Cargo.toml new file mode 100644 index 0000000..229bf30 --- /dev/null +++ b/tests/docker/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "docker-tests" +version = "0.1.0" +edition = "2024" + +[dependencies] +distribution = { path = "../../crates/distribution" } +reqwest = { version = "0.12", features = ["blocking", "json"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/tests/docker/docker-compose.lan-hpz.yml b/tests/docker/docker-compose.lan-hpz.yml new file mode 100644 index 0000000..90dbc4a --- /dev/null +++ b/tests/docker/docker-compose.lan-hpz.yml @@ -0,0 +1,34 @@ +## LAN cluster — hpz side (192.168.1.106) +## Run alongside docker-compose.lan-thinkpad.yml on the thinkpad. +## +## docker compose -f tests/docker/docker-compose.lan-hpz.yml up -d --build +services: + seed: + build: + context: ../.. + dockerfile: Dockerfile + network_mode: host + command: + - "--listen" + - "192.168.1.106:7000" + - "--dashboard-port" + - "9091" + - "--actors" + - "2" + + node-2: + build: + context: ../.. + dockerfile: Dockerfile + network_mode: host + command: + - "--listen" + - "192.168.1.106:7001" + - "--seed" + - "192.168.1.106:7000" + - "--dashboard-port" + - "9092" + - "--actors" + - "2" + depends_on: + - seed diff --git a/tests/docker/docker-compose.lan-thinkpad.yml b/tests/docker/docker-compose.lan-thinkpad.yml new file mode 100644 index 0000000..fbab227 --- /dev/null +++ b/tests/docker/docker-compose.lan-thinkpad.yml @@ -0,0 +1,49 @@ +## LAN cluster — thinkpad side (192.168.1.102) +## Seed is on hpz at 192.168.1.106:7000. +## +## docker compose -f tests/docker/docker-compose.lan-thinkpad.yml up -d --build +services: + node-3: + build: + context: ../.. + dockerfile: Dockerfile + network_mode: host + command: + - "--listen" + - "192.168.1.102:7000" + - "--seed" + - "192.168.1.106:7000" + - "--dashboard-port" + - "9093" + - "--actors" + - "2" + + node-4: + build: + context: ../.. + dockerfile: Dockerfile + network_mode: host + command: + - "--listen" + - "192.168.1.102:7001" + - "--seed" + - "192.168.1.106:7000" + - "--dashboard-port" + - "9094" + - "--actors" + - "2" + + node-5: + build: + context: ../.. + dockerfile: Dockerfile + network_mode: host + command: + - "--listen" + - "192.168.1.102:7002" + - "--seed" + - "192.168.1.106:7000" + - "--dashboard-port" + - "9095" + - "--actors" + - "2" diff --git a/tests/docker/docker-compose.yml b/tests/docker/docker-compose.yml new file mode 100644 index 0000000..52c38a3 --- /dev/null +++ b/tests/docker/docker-compose.yml @@ -0,0 +1,70 @@ +services: + seed: + build: + context: ../.. + dockerfile: Dockerfile + command: ["--listen", "10.0.1.10:7000", "--dashboard-port", "9090", "--actors", "2"] + networks: + cluster: + ipv4_address: 10.0.1.10 + ports: + - "9091:9090" + + node-2: + build: + context: ../.. + dockerfile: Dockerfile + command: ["--listen", "10.0.1.11:7000", "--seed", "10.0.1.10:7000", "--dashboard-port", "9090", "--actors", "2"] + networks: + cluster: + ipv4_address: 10.0.1.11 + ports: + - "9092:9090" + depends_on: + - seed + + node-3: + build: + context: ../.. + dockerfile: Dockerfile + command: ["--listen", "10.0.1.12:7000", "--seed", "10.0.1.10:7000", "--dashboard-port", "9090", "--actors", "2"] + networks: + cluster: + ipv4_address: 10.0.1.12 + ports: + - "9093:9090" + depends_on: + - seed + + node-4: + build: + context: ../.. + dockerfile: Dockerfile + command: ["--listen", "10.0.1.13:7000", "--seed", "10.0.1.10:7000", "--dashboard-port", "9090", "--actors", "2"] + networks: + cluster: + ipv4_address: 10.0.1.13 + ports: + - "9094:9090" + depends_on: + - seed + + node-5: + build: + context: ../.. + dockerfile: Dockerfile + command: ["--listen", "10.0.1.14:7000", "--seed", "10.0.1.10:7000", "--dashboard-port", "9090", "--actors", "2"] + networks: + cluster: + ipv4_address: 10.0.1.14 + ports: + - "9095:9090" + depends_on: + - seed + +networks: + cluster: + driver: bridge + ipam: + config: + - subnet: 10.0.1.0/24 diff --git a/tests/docker/run-lan-cluster.sh b/tests/docker/run-lan-cluster.sh new file mode 100755 index 0000000..bdfef5e --- /dev/null +++ b/tests/docker/run-lan-cluster.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +# +# Run a 5-node swactor cluster across two physical machines: +# hpz (192.168.1.106) — seed + node-2 +# thinkpad (192.168.1.102) — node-3, node-4, node-5 +# +# Usage: ./tests/docker/run-lan-cluster.sh [--no-build] [--teardown-only] +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +HPZ_IP="192.168.1.106" +THINKPAD_IP="192.168.1.102" +THINKPAD_SSH="thinkpad" +THINKPAD_REPO="/home/zach/swactor-distribution-realization" + +HPZ_COMPOSE="$SCRIPT_DIR/docker-compose.lan-hpz.yml" +THINKPAD_COMPOSE="tests/docker/docker-compose.lan-thinkpad.yml" + +# Dashboard endpoints +HPZ_DASHBOARDS=("http://127.0.0.1:9091" "http://127.0.0.1:9092") +THINKPAD_DASHBOARDS=("http://$THINKPAD_IP:9093" "http://$THINKPAD_IP:9094" "http://$THINKPAD_IP:9095") +ALL_DASHBOARDS=("${HPZ_DASHBOARDS[@]}" "${THINKPAD_DASHBOARDS[@]}") + +CONVERGE_TIMEOUT=60 +EXPECTED_ALIVE=4 +NO_BUILD=false +TEARDOWN_ONLY=false + +for arg in "$@"; do + case "$arg" in + --no-build) NO_BUILD=true ;; + --teardown-only) TEARDOWN_ONLY=true ;; + esac +done + +# ── Cleanup on exit ────────────────────────────────────────────────────────── +teardown() { + echo "" + echo "=== Tearing down ===" + echo "Stopping hpz nodes..." + docker compose -f "$HPZ_COMPOSE" down --timeout 5 2>/dev/null || true + echo "Stopping thinkpad nodes..." + ssh "$THINKPAD_SSH" "cd $THINKPAD_REPO && docker compose -f $THINKPAD_COMPOSE down --timeout 5" 2>/dev/null || true + echo "Done." +} +trap teardown EXIT + +if $TEARDOWN_ONLY; then + exit 0 +fi + +# ── Sync repo to thinkpad ─────────────────────────────────────────────────── +echo "=== Syncing repo to thinkpad ===" +tar czf /tmp/swactor-repo.tar.gz -C "$REPO_ROOT" --exclude=target --exclude=.git . +scp -q /tmp/swactor-repo.tar.gz "$THINKPAD_SSH":/tmp/ +ssh "$THINKPAD_SSH" "mkdir -p $THINKPAD_REPO && tar xzf /tmp/swactor-repo.tar.gz -C $THINKPAD_REPO" +echo "Synced." + +# ── Build images ───────────────────────────────────────────────────────────── +if ! $NO_BUILD; then + echo "" + echo "=== Building Docker image on hpz ===" + docker compose -f "$HPZ_COMPOSE" build --quiet + + echo "=== Building Docker image on thinkpad ===" + ssh "$THINKPAD_SSH" "cd $THINKPAD_REPO && docker compose -f $THINKPAD_COMPOSE build --quiet" + echo "Images built." +fi + +# ── Start clusters ─────────────────────────────────────────────────────────── +echo "" +echo "=== Starting hpz nodes (seed + node-2) ===" +docker compose -f "$HPZ_COMPOSE" up -d + +echo "=== Starting thinkpad nodes (node-3, node-4, node-5) ===" +ssh "$THINKPAD_SSH" "cd $THINKPAD_REPO && docker compose -f $THINKPAD_COMPOSE up -d" + +# ── Wait for convergence ───────────────────────────────────────────────────── +echo "" +echo "=== Waiting for cluster convergence (timeout: ${CONVERGE_TIMEOUT}s) ===" + +start_time=$(date +%s) +while true; do + elapsed=$(( $(date +%s) - start_time )) + if [ "$elapsed" -ge "$CONVERGE_TIMEOUT" ]; then + echo "" + echo "TIMEOUT after ${elapsed}s. Dumping last state:" + for url in "${ALL_DASHBOARDS[@]}"; do + echo -n " $url: " + curl -sf "$url/api/distribution" 2>/dev/null \ + | python3 -c "import json,sys; d=json.load(sys.stdin); print(f'alive={d[\"alive_count\"]}, routing={d[\"routing_table_size\"]}, dir={d[\"directory_entry_count\"]}, cache={d[\"cache_size\"]}')" \ + 2>/dev/null || echo "unreachable" + done + echo "" + echo "FAIL: cluster did not converge within ${CONVERGE_TIMEOUT}s" + exit 1 + fi + + all_ok=true + for url in "${ALL_DASHBOARDS[@]}"; do + alive=$(curl -sf "$url/api/distribution" 2>/dev/null \ + | python3 -c "import json,sys; print(json.load(sys.stdin).get('alive_count',0))" 2>/dev/null) || alive=0 + if [ "$alive" -lt "$EXPECTED_ALIVE" ]; then + all_ok=false + break + fi + done + + if $all_ok; then + echo "Converged after ${elapsed}s." + break + fi + + printf "." + sleep 1 +done + +# ── Report ─────────────────────────────────────────────────────────────────── +echo "" +echo "=== Cluster Status ===" +printf "%-35s %6s %8s %5s %6s\n" "ENDPOINT" "ALIVE" "ROUTING" "DIR" "CACHE" +for url in "${ALL_DASHBOARDS[@]}"; do + data=$(curl -sf "$url/api/distribution" 2>/dev/null) || { echo "$url: unreachable"; continue; } + echo "$data" | python3 -c " +import json,sys +d=json.load(sys.stdin) +print(f' {\"$url\":<33} {d[\"alive_count\"]:>6} {d[\"routing_table_size\"]:>8} {d[\"directory_entry_count\"]:>5} {d[\"cache_size\"]:>6}') +" +done + +# ── Assertions ─────────────────────────────────────────────────────────────── +echo "" +echo "=== Assertions ===" +pass=true + +for url in "${ALL_DASHBOARDS[@]}"; do + data=$(curl -sf "$url/api/distribution" 2>/dev/null) || { echo "FAIL: $url unreachable"; pass=false; continue; } + alive=$(echo "$data" | python3 -c "import json,sys; print(json.load(sys.stdin)['alive_count'])") + routing=$(echo "$data" | python3 -c "import json,sys; print(json.load(sys.stdin)['routing_table_size'])") + dir=$(echo "$data" | python3 -c "import json,sys; print(json.load(sys.stdin)['directory_entry_count'])") + + if [ "$alive" -lt 4 ]; then echo "FAIL: $url alive=$alive (expected >= 4)"; pass=false; fi + if [ "$routing" -lt 3 ]; then echo "FAIL: $url routing=$routing (expected >= 3)"; pass=false; fi + if [ "$dir" -lt 2 ]; then echo "FAIL: $url dir=$dir (expected >= 2)"; pass=false; fi +done + +if $pass; then + echo "ALL PASS" + echo "" + echo "Cluster is running. Press Ctrl-C to tear down, or run:" + echo " $0 --teardown-only" + # Keep running so user can inspect + read -r -p "Press Enter to tear down..." +else + echo "" + echo "SOME ASSERTIONS FAILED" + exit 1 +fi diff --git a/tests/docker/src/lib.rs b/tests/docker/src/lib.rs new file mode 100644 index 0000000..e5563cf --- /dev/null +++ b/tests/docker/src/lib.rs @@ -0,0 +1,398 @@ +//! Test utilities for Docker-based cluster integration tests. + +use std::path::PathBuf; +use std::process::Command; +use std::sync::Once; +use std::thread; +use std::time::{Duration, Instant}; + +use distribution::snapshot::DistributionNodeSnapshot; + +/// Dashboard ports mapped to the host for each of the 5 nodes. +pub const DASHBOARD_PORTS: [u16; 5] = [9091, 9092, 9093, 9094, 9095]; + +/// Service names matching docker-compose.yml. +pub const SERVICE_NAMES: [&str; 5] = ["seed", "node-2", "node-3", "node-4", "node-5"]; + +/// `CARGO_MANIFEST_DIR` points to `tests/docker/` (the crate root). +const COMPOSE_DIR: &str = env!("CARGO_MANIFEST_DIR"); + +fn compose_file() -> String { + let mut p = PathBuf::from(COMPOSE_DIR); + p.push("docker-compose.yml"); + p.to_string_lossy().into_owned() +} + +static BUILD_ONCE: Once = Once::new(); + +fn build_cluster_images() { + BUILD_ONCE.call_once(|| { + let status = Command::new("docker") + .args(["compose", "-f", &compose_file(), "build"]) + .status() + .expect("failed to build docker images"); + assert!(status.success(), "docker compose build failed"); + }); +} + +/// Handle to a running Docker Compose cluster. +/// Stops the cluster on drop. +pub struct ClusterHandle { + stopped: bool, +} + +impl ClusterHandle { + /// Start the 5-node cluster via docker compose. + pub fn start() -> Self { + build_cluster_images(); + + let status = Command::new("docker") + .args(["compose", "-f", &compose_file(), "up", "-d", "--wait"]) + .status() + .expect("failed to run docker compose"); + + if !status.success() { + let status = Command::new("docker") + .args(["compose", "-f", &compose_file(), "up", "-d"]) + .status() + .expect("failed to run docker compose"); + assert!(status.success(), "docker compose up failed"); + thread::sleep(Duration::from_secs(5)); + } + + ClusterHandle { stopped: false } + } + + /// Stop the cluster. + pub fn stop(&mut self) { + if !self.stopped { + let _ = Command::new("docker") + .args(["compose", "-f", &compose_file(), "down", "--timeout", "5"]) + .status(); + self.stopped = true; + } + } +} + +impl Drop for ClusterHandle { + fn drop(&mut self) { + self.stop(); + } +} + +/// Kill a specific node (simulates crash — container stops). +pub fn kill_node(service: &str) { + let status = Command::new("docker") + .args(["compose", "-f", &compose_file(), "stop", service]) + .status() + .expect("failed to stop node"); + assert!(status.success(), "docker compose stop {service} failed"); +} + +/// Restart a previously killed node. +pub fn restart_node(service: &str) { + let status = Command::new("docker") + .args(["compose", "-f", &compose_file(), "start", service]) + .status() + .expect("failed to start node"); + assert!(status.success(), "docker compose start {service} failed"); +} + +/// Fetch the distribution snapshot from a node's dashboard on localhost. +/// Returns None if the node is unreachable or returns empty/error. +pub fn poll_distribution(port: u16) -> Option { + poll_distribution_at("127.0.0.1", port) +} + +/// Fetch the distribution snapshot from a node's dashboard at an arbitrary host. +pub fn poll_distribution_at(host: &str, port: u16) -> Option { + let url = format!("http://{host}:{port}/api/distribution"); + let client = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(2)) + .build() + .ok()?; + let resp = client.get(&url).send().ok()?; + if !resp.status().is_success() { + return None; + } + let text = resp.text().ok()?; + if text == "{}" { + return None; + } + serde_json::from_str(&text).ok() +} + +/// Wait until all nodes at the given ports report at least `expected_alive` +/// alive members. Times out after `timeout`. +pub fn wait_for_convergence( + ports: &[u16], + expected_alive: usize, + timeout: Duration, +) -> Result<(), String> { + let start = Instant::now(); + loop { + if start.elapsed() > timeout { + // Build diagnostic message + let mut diag = String::from("Convergence timeout. Last seen alive counts: "); + for &port in ports { + match poll_distribution(port) { + Some(snap) => diag.push_str(&format!("port {}={}, ", port, snap.alive_count)), + None => diag.push_str(&format!("port {}=unreachable, ", port)), + } + } + return Err(diag); + } + + let all_converged = ports.iter().all(|&port| { + poll_distribution(port) + .map(|snap| snap.alive_count >= expected_alive) + .unwrap_or(false) + }); + + if all_converged { + return Ok(()); + } + + thread::sleep(Duration::from_secs(1)); + } +} + +/// Wait until a specific set of ports all report alive_count <= threshold. +pub fn wait_for_death_detection( + ports: &[u16], + max_alive: usize, + timeout: Duration, +) -> Result<(), String> { + wait_for_death_detection_at( + &ports.iter().map(|&p| ("127.0.0.1", p)).collect::>(), + max_alive, + timeout, + ) +} + +/// Wait until a set of (host, port) endpoints all report alive_count <= threshold. +pub fn wait_for_death_detection_at( + endpoints: &[(&str, u16)], + max_alive: usize, + timeout: Duration, +) -> Result<(), String> { + let start = Instant::now(); + loop { + if start.elapsed() > timeout { + let mut diag = String::from("Death detection timeout. Last seen: "); + for &(host, port) in endpoints { + match poll_distribution_at(host, port) { + Some(snap) => diag.push_str(&format!("{host}:{port}={} alive, ", snap.alive_count)), + None => diag.push_str(&format!("{host}:{port}=unreachable, ")), + } + } + return Err(diag); + } + + let all_detected = endpoints.iter().all(|&(host, port)| { + poll_distribution_at(host, port) + .map(|snap| snap.alive_count <= max_alive) + .unwrap_or(false) + }); + + if all_detected { + return Ok(()); + } + + thread::sleep(Duration::from_secs(1)); + } +} + +// ── LAN (cross-machine) cluster support ───────────────────────────────────── + +/// Dashboard endpoints for the LAN cluster. +/// hpz (local): 9091, 9092 +/// thinkpad (remote): 9093, 9094, 9095 +pub const LAN_HPZ_IP: &str = "192.168.1.106"; +pub const LAN_THINKPAD_IP: &str = "192.168.1.102"; +pub const LAN_THINKPAD_SSH: &str = "thinkpad"; +pub const LAN_THINKPAD_REPO: &str = "/home/zach/swactor-distribution-realization"; + +pub const LAN_ENDPOINTS: [(&str, u16); 5] = [ + ("127.0.0.1", 9091), + ("127.0.0.1", 9092), + (LAN_THINKPAD_IP, 9093), + (LAN_THINKPAD_IP, 9094), + (LAN_THINKPAD_IP, 9095), +]; + +pub const LAN_HPZ_COMPOSE: &str = "tests/docker/docker-compose.lan-hpz.yml"; +pub const LAN_THINKPAD_COMPOSE: &str = "tests/docker/docker-compose.lan-thinkpad.yml"; + +static BUILD_LAN_ONCE: Once = Once::new(); + +fn build_lan_images() { + BUILD_LAN_ONCE.call_once(|| { + // Sync repo to thinkpad + let tar_status = Command::new("bash") + .args(["-c", &format!( + "tar czf /tmp/swactor-repo.tar.gz -C {} --exclude=target --exclude=.git . \ + && scp -q /tmp/swactor-repo.tar.gz {}:/tmp/ \ + && ssh {} 'mkdir -p {} && tar xzf /tmp/swactor-repo.tar.gz -C {}'", + COMPOSE_DIR.replace("tests/docker", ""), + LAN_THINKPAD_SSH, LAN_THINKPAD_SSH, LAN_THINKPAD_REPO, LAN_THINKPAD_REPO, + )]) + .status() + .expect("failed to sync repo to thinkpad"); + assert!(tar_status.success(), "repo sync to thinkpad failed"); + + // Build hpz images + let hpz_compose = lan_hpz_compose_path(); + let status = Command::new("docker") + .args(["compose", "-f", &hpz_compose, "build"]) + .status() + .expect("failed to build hpz images"); + assert!(status.success(), "docker compose build (hpz) failed"); + + // Build thinkpad images + let status = Command::new("ssh") + .args([ + LAN_THINKPAD_SSH, + &format!( + "cd {} && docker compose -f {} build", + LAN_THINKPAD_REPO, LAN_THINKPAD_COMPOSE, + ), + ]) + .status() + .expect("failed to build thinkpad images"); + assert!(status.success(), "docker compose build (thinkpad) failed"); + }); +} + +/// Handle to a LAN cluster running across two machines. +pub struct LanClusterHandle { + stopped: bool, +} + +impl LanClusterHandle { + /// Start the LAN cluster: hpz nodes locally, thinkpad nodes via SSH. + pub fn start() -> Self { + build_lan_images(); + + // Start hpz side + let hpz_compose = lan_hpz_compose_path(); + let status = Command::new("docker") + .args(["compose", "-f", &hpz_compose, "up", "-d"]) + .status() + .expect("failed to start hpz nodes"); + assert!(status.success(), "docker compose up (hpz) failed"); + + // Start thinkpad side + let status = Command::new("ssh") + .args([ + LAN_THINKPAD_SSH, + &format!( + "cd {} && docker compose -f {} up -d", + LAN_THINKPAD_REPO, LAN_THINKPAD_COMPOSE, + ), + ]) + .status() + .expect("failed to start thinkpad nodes"); + assert!(status.success(), "docker compose up (thinkpad) failed"); + + // Give containers a moment to bind + thread::sleep(Duration::from_secs(3)); + + LanClusterHandle { stopped: false } + } + + /// Stop both sides of the cluster. + pub fn stop(&mut self) { + if !self.stopped { + let hpz_compose = lan_hpz_compose_path(); + let _ = Command::new("docker") + .args(["compose", "-f", &hpz_compose, "down", "--timeout", "5"]) + .status(); + let _ = Command::new("ssh") + .args([ + LAN_THINKPAD_SSH, + &format!( + "cd {} && docker compose -f {} down --timeout 5", + LAN_THINKPAD_REPO, LAN_THINKPAD_COMPOSE, + ), + ]) + .status(); + self.stopped = true; + } + } +} + +impl Drop for LanClusterHandle { + fn drop(&mut self) { + self.stop(); + } +} + +/// Kill a node on the thinkpad via SSH. +pub fn kill_remote_node(service: &str) { + let status = Command::new("ssh") + .args([ + LAN_THINKPAD_SSH, + &format!( + "cd {} && docker compose -f {} stop {}", + LAN_THINKPAD_REPO, LAN_THINKPAD_COMPOSE, service, + ), + ]) + .status() + .expect("failed to kill remote node"); + assert!(status.success(), "remote docker compose stop {service} failed"); +} + +/// Restart a node on the thinkpad via SSH. +pub fn restart_remote_node(service: &str) { + let status = Command::new("ssh") + .args([ + LAN_THINKPAD_SSH, + &format!( + "cd {} && docker compose -f {} start {}", + LAN_THINKPAD_REPO, LAN_THINKPAD_COMPOSE, service, + ), + ]) + .status() + .expect("failed to restart remote node"); + assert!(status.success(), "remote docker compose start {service} failed"); +} + +/// Wait until all LAN endpoints report at least `expected_alive` alive members. +pub fn wait_for_lan_convergence( + endpoints: &[(&str, u16)], + expected_alive: usize, + timeout: Duration, +) -> Result<(), String> { + let start = Instant::now(); + loop { + if start.elapsed() > timeout { + let mut diag = String::from("LAN convergence timeout. Last seen: "); + for &(host, port) in endpoints { + match poll_distribution_at(host, port) { + Some(snap) => diag.push_str(&format!("{host}:{port}={}, ", snap.alive_count)), + None => diag.push_str(&format!("{host}:{port}=unreachable, ")), + } + } + return Err(diag); + } + + let all_converged = endpoints.iter().all(|&(host, port)| { + poll_distribution_at(host, port) + .map(|snap| snap.alive_count >= expected_alive) + .unwrap_or(false) + }); + + if all_converged { + return Ok(()); + } + + thread::sleep(Duration::from_secs(1)); + } +} + +fn lan_hpz_compose_path() -> String { + let mut p = PathBuf::from(COMPOSE_DIR); + p.push("docker-compose.lan-hpz.yml"); + p.to_string_lossy().into_owned() +} diff --git a/tests/docker/tests/cluster.rs b/tests/docker/tests/cluster.rs new file mode 100644 index 0000000..09763df --- /dev/null +++ b/tests/docker/tests/cluster.rs @@ -0,0 +1,201 @@ +//! Docker cluster integration tests. +//! +//! These tests mirror the simulation tests in +//! `crates/simulation/tests/distribution_sim.rs` but run against real +//! Docker containers communicating over TCP. +//! +//! Run with: `cargo test -p docker-tests -- --ignored` +//! Requires: Docker with compose v2 + +use std::time::Duration; + +use docker_tests::*; + +// ──────────────────────────────────────────────────────────────────────────── +// Test 1: A 5-node cluster converges its membership view +// Mirrors: distribution_sim::cluster_of_five_converges +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +#[ignore] +fn cluster_of_five_converges() { + // Given: 5 nodes started via docker compose + let mut cluster = ClusterHandle::start(); + + // When: we wait for convergence + let result = wait_for_convergence( + &DASHBOARD_PORTS, + 4, // each node sees at least 4 alive (self + 3 peers minimum) + Duration::from_secs(30), + ); + + // Then: all 5 nodes report healthy membership + match result { + Ok(()) => { + // Verify each node's snapshot looks reasonable + for (i, &port) in DASHBOARD_PORTS.iter().enumerate() { + let snap = poll_distribution(port) + .unwrap_or_else(|| panic!("node {} (port {}) unreachable after convergence", i, port)); + assert!( + snap.alive_count >= 4, + "node {} should see >= 4 alive members, got {}", + i, + snap.alive_count + ); + assert!( + snap.routing_table_size >= 3, + "node {} should have >= 3 routing table entries, got {}", + i, + snap.routing_table_size + ); + } + } + Err(diag) => { + cluster.stop(); + panic!("cluster of 5 did not converge: {diag}"); + } + } + + cluster.stop(); +} + +// ──────────────────────────────────────────────────────────────────────────── +// Test 2: A killed node is eventually detected by survivors +// Mirrors: distribution_sim::node_death_is_detected +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +#[ignore] +fn node_death_is_detected() { + // Given: a converged 5-node cluster + let mut cluster = ClusterHandle::start(); + wait_for_convergence(&DASHBOARD_PORTS, 4, Duration::from_secs(30)) + .expect("cluster did not converge before kill test"); + + // When: we kill node-3 + kill_node("node-3"); + + // Then: surviving nodes detect the death within 30s + // Survivors are: seed(9091), node-2(9092), node-4(9094), node-5(9095) + let survivor_ports = [9091, 9092, 9094, 9095]; + let result = wait_for_death_detection( + &survivor_ports, + 4, // should see at most 4 alive (down from 5) + Duration::from_secs(30), + ); + + match result { + Ok(()) => { + // Verify at least one survivor sees the dead node + let any_sees_dead = survivor_ports.iter().any(|&port| { + poll_distribution(port) + .map(|snap| snap.dead_count >= 1) + .unwrap_or(false) + }); + assert!(any_sees_dead, "at least one survivor should see a dead member"); + } + Err(diag) => { + cluster.stop(); + panic!("node death was not detected: {diag}"); + } + } + + cluster.stop(); +} + +// ──────────────────────────────────────────────────────────────────────────── +// Test 3: A killed node can rejoin the cluster +// Mirrors: distribution_sim::killed_node_rejoins +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +#[ignore] +fn killed_node_rejoins() { + // Given: a converged cluster with node-3 killed and detected dead + let mut cluster = ClusterHandle::start(); + wait_for_convergence(&DASHBOARD_PORTS, 4, Duration::from_secs(30)) + .expect("cluster did not converge before rejoin test"); + + kill_node("node-3"); + let survivor_ports = [9091, 9092, 9094, 9095]; + wait_for_death_detection(&survivor_ports, 4, Duration::from_secs(30)) + .expect("node death not detected before rejoin"); + + // When: we restart node-3 + restart_node("node-3"); + + // Then: node-3 rejoins and learns about cluster members + // Give the restarted node time to re-join and be discovered + let result = wait_for_convergence( + &[9093], // node-3's dashboard + 1, // at minimum, it should know about at least 1 peer + Duration::from_secs(30), + ); + + match result { + Ok(()) => { + let snap = poll_distribution(9093).expect("node-3 unreachable after rejoin"); + assert!( + snap.alive_count >= 1, + "rejoined node should see >= 1 alive member, got {}", + snap.alive_count + ); + } + Err(diag) => { + cluster.stop(); + panic!("killed node did not rejoin: {diag}"); + } + } + + cluster.stop(); +} + +// ──────────────────────────────────────────────────────────────────────────── +// Test 4: Actors are resolvable across the cluster +// Mirrors: distribution_sim::actors_resolvable_across_cluster +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +#[ignore] +fn actors_resolvable_across_cluster() { + // Given: a converged 5-node cluster, each with 2 registered actors + let mut cluster = ClusterHandle::start(); + wait_for_convergence(&DASHBOARD_PORTS, 4, Duration::from_secs(30)) + .expect("cluster did not converge before actor resolution test"); + + // When: we query each node's snapshot + let mut total_directory_entries = 0; + let mut total_cache_size = 0; + + for (i, &port) in DASHBOARD_PORTS.iter().enumerate() { + let snap = poll_distribution(port) + .unwrap_or_else(|| panic!("node {} unreachable", i)); + + // Then: each node has registered its own 2 actors in the directory + assert!( + snap.directory_entry_count >= 2, + "node {} should have >= 2 directory entries, got {}", + i, + snap.directory_entry_count + ); + + total_directory_entries += snap.directory_entry_count; + total_cache_size += snap.cache_size; + } + + // Total actors across cluster should be 10 (5 nodes * 2 actors) + assert!( + total_directory_entries >= 10, + "total directory entries across cluster should be >= 10, got {}", + total_directory_entries + ); + + // At least some nodes should have cached locations for remote actors + assert!( + total_cache_size >= 5, + "total cache entries across cluster should be >= 5 (each node caches its own 2), got {}", + total_cache_size + ); + + cluster.stop(); +} diff --git a/tests/docker/tests/lan_cluster.rs b/tests/docker/tests/lan_cluster.rs new file mode 100644 index 0000000..2e07204 --- /dev/null +++ b/tests/docker/tests/lan_cluster.rs @@ -0,0 +1,197 @@ +//! LAN cluster integration tests — nodes across two physical machines. +//! +//! These tests run a 5-node cluster split across devuan-hpz (192.168.1.106) +//! and thinkpad (192.168.1.102) communicating over a real LAN. +//! +//! Run with: `cargo test -p docker-tests -- --ignored lan_` +//! Requires: Docker on both machines, SSH access to thinkpad + +use std::time::Duration; + +use docker_tests::*; + +// ──────────────────────────────────────────────────────────────────────────── +// Test 1: Cross-machine cluster converges +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +#[ignore] +fn lan_cluster_converges() { + // Given: 5 nodes split across two physical machines on a LAN + let mut cluster = LanClusterHandle::start(); + + // When: we wait for convergence + let result = wait_for_lan_convergence( + &LAN_ENDPOINTS, + 4, // each node sees at least 4 alive + Duration::from_secs(30), + ); + + // Then: all 5 nodes discover each other across the LAN + match result { + Ok(()) => { + for &(host, port) in &LAN_ENDPOINTS { + let snap = poll_distribution_at(host, port) + .unwrap_or_else(|| panic!("{host}:{port} unreachable after convergence")); + assert!( + snap.alive_count >= 4, + "{host}:{port} should see >= 4 alive, got {}", + snap.alive_count + ); + assert!( + snap.routing_table_size >= 3, + "{host}:{port} should have >= 3 routing entries, got {}", + snap.routing_table_size + ); + } + } + Err(diag) => { + cluster.stop(); + panic!("LAN cluster did not converge: {diag}"); + } + } + + cluster.stop(); +} + +// ──────────────────────────────────────────────────────────────────────────── +// Test 2: Death of a remote node is detected across the LAN +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +#[ignore] +fn lan_remote_node_death_detected() { + // Given: a converged LAN cluster + let mut cluster = LanClusterHandle::start(); + wait_for_lan_convergence(&LAN_ENDPOINTS, 4, Duration::from_secs(30)) + .expect("LAN cluster did not converge before kill test"); + + // When: we kill node-3 on the thinkpad + kill_remote_node("node-3"); + + // Then: surviving nodes detect the death + // Survivors: hpz seed(9091), hpz node-2(9092), thinkpad node-4(9094), thinkpad node-5(9095) + let survivor_endpoints = [ + ("127.0.0.1", 9091_u16), + ("127.0.0.1", 9092), + (LAN_THINKPAD_IP, 9094), + (LAN_THINKPAD_IP, 9095), + ]; + let result = wait_for_death_detection_at(&survivor_endpoints, 4, Duration::from_secs(30)); + + match result { + Ok(()) => { + let any_sees_dead = survivor_endpoints.iter().any(|&(host, port)| { + poll_distribution_at(host, port) + .map(|snap| snap.dead_count >= 1) + .unwrap_or(false) + }); + assert!(any_sees_dead, "at least one survivor should see a dead member"); + } + Err(diag) => { + cluster.stop(); + panic!("remote node death not detected: {diag}"); + } + } + + cluster.stop(); +} + +// ──────────────────────────────────────────────────────────────────────────── +// Test 3: A killed remote node can rejoin across the LAN +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +#[ignore] +fn lan_killed_remote_node_rejoins() { + // Given: a converged cluster with node-3 killed and detected dead + let mut cluster = LanClusterHandle::start(); + wait_for_lan_convergence(&LAN_ENDPOINTS, 4, Duration::from_secs(30)) + .expect("LAN cluster did not converge before rejoin test"); + + kill_remote_node("node-3"); + let survivor_endpoints = [ + ("127.0.0.1", 9091_u16), + ("127.0.0.1", 9092), + (LAN_THINKPAD_IP, 9094), + (LAN_THINKPAD_IP, 9095), + ]; + wait_for_death_detection_at(&survivor_endpoints, 4, Duration::from_secs(30)) + .expect("node death not detected before rejoin"); + + // When: we restart node-3 on the thinkpad + restart_remote_node("node-3"); + + // Then: node-3 rejoins the cluster across the LAN + let result = wait_for_lan_convergence( + &[(LAN_THINKPAD_IP, 9093)], + 1, + Duration::from_secs(30), + ); + + match result { + Ok(()) => { + let snap = poll_distribution_at(LAN_THINKPAD_IP, 9093) + .expect("node-3 unreachable after rejoin"); + assert!( + snap.alive_count >= 1, + "rejoined node should see >= 1 alive, got {}", + snap.alive_count + ); + } + Err(diag) => { + cluster.stop(); + panic!("killed remote node did not rejoin: {diag}"); + } + } + + cluster.stop(); +} + +// ──────────────────────────────────────────────────────────────────────────── +// Test 4: Actors are resolvable across machines +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +#[ignore] +fn lan_actors_resolvable_cross_machine() { + // Given: a converged 5-node LAN cluster, each with 2 registered actors + let mut cluster = LanClusterHandle::start(); + wait_for_lan_convergence(&LAN_ENDPOINTS, 4, Duration::from_secs(30)) + .expect("LAN cluster did not converge before actor resolution test"); + + // When: we query each node's snapshot + let mut total_directory_entries = 0; + let mut total_cache_size = 0; + + for &(host, port) in &LAN_ENDPOINTS { + let snap = poll_distribution_at(host, port) + .unwrap_or_else(|| panic!("{host}:{port} unreachable")); + + // Then: each node has its own 2 actors in the directory + assert!( + snap.directory_entry_count >= 2, + "{host}:{port} should have >= 2 directory entries, got {}", + snap.directory_entry_count + ); + + total_directory_entries += snap.directory_entry_count; + total_cache_size += snap.cache_size; + } + + // Total actors across cluster: 10 (5 nodes * 2 actors) + assert!( + total_directory_entries >= 10, + "total directory entries should be >= 10, got {}", + total_directory_entries + ); + + // Nodes should cache remote actor locations (including cross-machine) + assert!( + total_cache_size >= 5, + "total cache entries should be >= 5, got {}", + total_cache_size + ); + + cluster.stop(); +}