feat: view simulation test traces in dashboard
This commit is contained in:
Claude
parent
6197361b22
commit
693d66fc9d
9 changed files with 530 additions and 139 deletions
6
.gitignore
vendored
6
.gitignore
vendored
|
|
@ -1,4 +1,3 @@
|
|||
CLAUDE/
|
||||
**/target
|
||||
**/node_modules/
|
||||
.vscode/
|
||||
|
|
@ -13,6 +12,5 @@ corpus
|
|||
docs/architecture.dot
|
||||
docs/architecture.html
|
||||
|
||||
# Claude session files
|
||||
CLAUDE/
|
||||
.claude/
|
||||
# Simulation traces
|
||||
crates/simulation/traces
|
||||
|
|
@ -45,6 +45,11 @@ pub const DASHBOARD_HTML: &str = r##"<!DOCTYPE html>
|
|||
td { padding: 3px 8px; border-bottom: 1px solid #1a1d2e; white-space: nowrap; }
|
||||
tr.highlight-push td { background: rgba(59,130,246,0.1); }
|
||||
tr.highlight-set td { background: rgba(34,197,94,0.1); }
|
||||
tr.highlight-kill td { background: rgba(239,68,68,0.15); }
|
||||
tr.highlight-revive td { background: rgba(34,197,94,0.15); }
|
||||
tr.highlight-membership td { background: rgba(245,158,11,0.1); }
|
||||
tr.highlight-registry td { background: rgba(168,85,247,0.1); }
|
||||
tr.highlight-ping td { background: rgba(59,130,246,0.08); }
|
||||
.replay-controls { display: flex; align-items: center; gap: 8px; padding: 8px 16px; background: #161822; border-bottom: 1px solid #2a2d3a; }
|
||||
.replay-controls button { background: #2a2d3a; color: #e0e0e0; border: none; border-radius: 4px; padding: 4px 10px; cursor: pointer; font-size: 13px; }
|
||||
.replay-controls button:hover { background: #3b3f52; }
|
||||
|
|
@ -217,6 +222,17 @@ let cumulPushRecv = [];
|
|||
// round number for each event index: eventRoundIdx[evtIdx] = roundIdx into snapRounds
|
||||
let eventRoundMap = []; // eventRoundMap[evtIdx] = tick
|
||||
|
||||
// ── Distribution-specific state ────────────────────────────────
|
||||
let isDist = false;
|
||||
let snapMembers = []; // snapMembers[roundIdx] = Int32Array(N) — member_count
|
||||
let snapRegistry = []; // snapRegistry[roundIdx] = Int32Array(N) — registry_size
|
||||
let snapCache = []; // snapCache[roundIdx] = Int32Array(N) — cache_size
|
||||
let snapAlive = []; // snapAlive[roundIdx] = Uint8Array(N) — is_alive
|
||||
let snapRouting = []; // snapRouting[roundIdx] = Int32Array(N) — routing_table_size
|
||||
let snapRepair = []; // snapRepair[roundIdx] = Int32Array(N) — repair_queue_size
|
||||
let snapDirectory = []; // snapDirectory[roundIdx] = Int32Array(N) — directory_entry_count
|
||||
let snapTombstone = []; // snapTombstone[roundIdx] = Int32Array(N) — registry_tombstone_count
|
||||
|
||||
// ── Community detection ──────────────────────────────────────────
|
||||
let community = new Int32Array(0); // community[nodeIdx] = community id
|
||||
let numCommunities = 0;
|
||||
|
|
@ -492,11 +508,41 @@ function showNodeDetail(ni) {
|
|||
}
|
||||
|
||||
let html = '';
|
||||
html += '<div class="nd-row"><span class="nd-key">Community</span><span class="nd-val">' + (community[ni] !== undefined ? community[ni] : '-') + '</span></div>';
|
||||
html += '<div class="nd-row"><span class="nd-key">Pushes Sent</span><span class="nd-val">' + (metricPushesSent[ni] || 0) + '</span></div>';
|
||||
html += '<div class="nd-row"><span class="nd-key">Pushes Recv</span><span class="nd-val">' + (metricPushesRecv[ni] || 0) + '</span></div>';
|
||||
html += '<div class="nd-row"><span class="nd-key">Keys</span><span class="nd-val">' + keyCount + '/' + totalKeys + '</span></div>';
|
||||
html += '<div class="nd-row"><span class="nd-key">Peers</span><span class="nd-val">' + peerCount + '</span></div>';
|
||||
if (isDist) {
|
||||
// Find latest distribution snapshot for this node
|
||||
let snap = null;
|
||||
for (let r = snapRounds.length - 1; r >= 0; r--) {
|
||||
if (snapRounds[r] <= curRound) {
|
||||
snap = {
|
||||
members: snapMembers[r][ni] || 0,
|
||||
registry: snapRegistry[r][ni] || 0,
|
||||
cache: snapCache[r][ni] || 0,
|
||||
alive: snapAlive[r][ni],
|
||||
routing: snapRouting[r][ni] || 0,
|
||||
repair: snapRepair[r][ni] || 0,
|
||||
directory: snapDirectory[r][ni] || 0,
|
||||
tombstones: snapTombstone[r][ni] || 0,
|
||||
};
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (snap) {
|
||||
html += '<div class="nd-row"><span class="nd-key">Status</span><span class="nd-val" style="color:' + (snap.alive ? '#22c55e' : '#ef4444') + '">' + (snap.alive ? 'Alive' : 'Dead') + '</span></div>';
|
||||
html += '<div class="nd-row"><span class="nd-key">Members</span><span class="nd-val">' + snap.members + '</span></div>';
|
||||
html += '<div class="nd-row"><span class="nd-key">Routing Table</span><span class="nd-val">' + snap.routing + '</span></div>';
|
||||
html += '<div class="nd-row"><span class="nd-key">Directory</span><span class="nd-val">' + snap.directory + '</span></div>';
|
||||
html += '<div class="nd-row"><span class="nd-key">Cache</span><span class="nd-val">' + snap.cache + '</span></div>';
|
||||
html += '<div class="nd-row"><span class="nd-key">Registry</span><span class="nd-val">' + snap.registry + '</span></div>';
|
||||
html += '<div class="nd-row"><span class="nd-key">Tombstones</span><span class="nd-val">' + snap.tombstones + '</span></div>';
|
||||
html += '<div class="nd-row"><span class="nd-key">Repair Queue</span><span class="nd-val">' + snap.repair + '</span></div>';
|
||||
}
|
||||
} else {
|
||||
html += '<div class="nd-row"><span class="nd-key">Community</span><span class="nd-val">' + (community[ni] !== undefined ? community[ni] : '-') + '</span></div>';
|
||||
html += '<div class="nd-row"><span class="nd-key">Pushes Sent</span><span class="nd-val">' + (metricPushesSent[ni] || 0) + '</span></div>';
|
||||
html += '<div class="nd-row"><span class="nd-key">Pushes Recv</span><span class="nd-val">' + (metricPushesRecv[ni] || 0) + '</span></div>';
|
||||
html += '<div class="nd-row"><span class="nd-key">Keys</span><span class="nd-val">' + keyCount + '/' + totalKeys + '</span></div>';
|
||||
html += '<div class="nd-row"><span class="nd-key">Peers</span><span class="nd-val">' + peerCount + '</span></div>';
|
||||
}
|
||||
document.getElementById('ndStats').innerHTML = html;
|
||||
|
||||
// Mini event log: last 20 events for this node up to cursor
|
||||
|
|
@ -584,6 +630,61 @@ function drawAllCharts() {
|
|||
|
||||
function drawBadges() {
|
||||
const grid = document.getElementById('badgeGrid');
|
||||
|
||||
if (isDist) {
|
||||
const hasSnaps = snapRounds.length > 0;
|
||||
const lastRound = hasSnaps ? snapRounds.length - 1 : -1;
|
||||
|
||||
// Alive count
|
||||
let aliveCount = N;
|
||||
if (lastRound >= 0) {
|
||||
aliveCount = 0;
|
||||
for (let i = 0; i < N; i++) if (snapAlive[lastRound][i]) aliveCount++;
|
||||
}
|
||||
|
||||
// Membership accuracy: fraction of alive nodes with correct member_count
|
||||
let memAccuracy = 1.0;
|
||||
if (lastRound >= 0) {
|
||||
let correct = 0, alive = 0;
|
||||
for (let i = 0; i < N; i++) {
|
||||
if (!snapAlive[lastRound][i]) continue;
|
||||
alive++;
|
||||
if (snapMembers[lastRound][i] >= aliveCount - 1) correct++;
|
||||
}
|
||||
memAccuracy = alive > 0 ? correct / alive : 1;
|
||||
}
|
||||
|
||||
// Registry max
|
||||
let maxReg = 0;
|
||||
if (lastRound >= 0) {
|
||||
for (let i = 0; i < N; i++) if (snapRegistry[lastRound][i] > maxReg) maxReg = snapRegistry[lastRound][i];
|
||||
}
|
||||
|
||||
// Cache total
|
||||
let totalCache = 0;
|
||||
if (lastRound >= 0) {
|
||||
for (let i = 0; i < N; i++) if (snapAlive[lastRound][i]) totalCache += snapCache[lastRound][i];
|
||||
}
|
||||
|
||||
// Repair queue total
|
||||
let totalRepair = 0;
|
||||
if (lastRound >= 0) {
|
||||
for (let i = 0; i < N; i++) if (snapAlive[lastRound][i]) totalRepair += snapRepair[lastRound][i];
|
||||
}
|
||||
|
||||
const memColor = memAccuracy >= 0.99 ? 'green' : memAccuracy >= 0.8 ? 'yellow' : 'red';
|
||||
const aliveColor = aliveCount === N ? 'green' : aliveCount >= N * 0.8 ? 'yellow' : 'red';
|
||||
|
||||
grid.innerHTML =
|
||||
badge(aliveColor, aliveCount + '/' + N, 'Alive Nodes') +
|
||||
badge(memColor, (memAccuracy * 100).toFixed(0) + '%', 'Membership') +
|
||||
badge('', maxReg, 'Registry Size') +
|
||||
badge('', totalCache, 'Cache Total') +
|
||||
badge(totalRepair > 0 ? 'yellow' : 'green', totalRepair, 'Repair Queue') +
|
||||
badge('', allEvents.length, 'Events');
|
||||
return;
|
||||
}
|
||||
|
||||
const hasSnaps = snapRounds.length > 0;
|
||||
|
||||
// Delivery ratio
|
||||
|
|
@ -676,10 +777,26 @@ function drawConvergenceChart() {
|
|||
|
||||
// Compute data points
|
||||
const pts = [];
|
||||
for (let r = 0; r < snapRounds.length; r++) {
|
||||
let full = 0;
|
||||
for (let i = 0; i < N; i++) if (snapEntries[r][i] >= totalKeys) full++;
|
||||
pts.push({ round: snapRounds[r], pct: N > 0 ? full / N * 100 : 0 });
|
||||
if (isDist) {
|
||||
// Membership convergence: fraction of alive nodes with correct member count per round
|
||||
for (let r = 0; r < snapRounds.length; r++) {
|
||||
let alive = 0, correct = 0;
|
||||
let aliveCount = 0;
|
||||
for (let i = 0; i < N; i++) if (snapAlive[r][i]) aliveCount++;
|
||||
for (let i = 0; i < N; i++) {
|
||||
if (!snapAlive[r][i]) continue;
|
||||
alive++;
|
||||
if (snapMembers[r][i] >= aliveCount - 1) correct++;
|
||||
}
|
||||
pts.push({ round: snapRounds[r], pct: alive > 0 ? correct / alive * 100 : 0 });
|
||||
}
|
||||
} else {
|
||||
// existing gossip convergence
|
||||
for (let r = 0; r < snapRounds.length; r++) {
|
||||
let full = 0;
|
||||
for (let i = 0; i < N; i++) if (snapEntries[r][i] >= totalKeys) full++;
|
||||
pts.push({ round: snapRounds[r], pct: N > 0 ? full / N * 100 : 0 });
|
||||
}
|
||||
}
|
||||
|
||||
// Draw fill
|
||||
|
|
@ -803,7 +920,12 @@ function drawHeatmap() {
|
|||
const ni = sortedIdx[row];
|
||||
const entries = snapEntries[col][ni] || 0;
|
||||
const pct = totalKeys > 0 ? Math.round(entries / totalKeys * 100) : 0;
|
||||
tooltip.textContent = nodeNames[ni] + ' at round ' + snapRounds[col] + ': ' + entries + '/' + totalKeys + ' keys (' + pct + '%)';
|
||||
// Tooltip text depends on trace type
|
||||
if (isDist) {
|
||||
tooltip.textContent = nodeNames[ni] + ' at round ' + snapRounds[col] + ': registry ' + entries;
|
||||
} else {
|
||||
tooltip.textContent = nodeNames[ni] + ' at round ' + snapRounds[col] + ': ' + entries + '/' + totalKeys + ' keys (' + pct + '%)';
|
||||
}
|
||||
tooltip.style.display = 'block';
|
||||
tooltip.style.left = (ex + 12) + 'px'; tooltip.style.top = (ey - 20) + 'px';
|
||||
} else { tooltip.style.display = 'none'; }
|
||||
|
|
@ -831,16 +953,28 @@ function drawLoadHistogram() {
|
|||
let curRound = 0;
|
||||
if (replayCursor > 0 && replayCursor <= allEvents.length) curRound = allEvents[replayCursor - 1].tick;
|
||||
|
||||
// Find the closest round in cumulPushRecv
|
||||
let bestR = -1;
|
||||
for (let r = 0; r < snapRounds.length; r++) {
|
||||
if (snapRounds[r] <= curRound) bestR = r;
|
||||
}
|
||||
|
||||
if (bestR >= 0 && cumulPushRecv.length > bestR) {
|
||||
data = cumulPushRecv[bestR];
|
||||
if (isDist) {
|
||||
// Show cache size per node at current round
|
||||
data = new Int32Array(N);
|
||||
let bestR = -1;
|
||||
for (let r = 0; r < snapRounds.length; r++) {
|
||||
if (snapRounds[r] <= curRound) bestR = r;
|
||||
}
|
||||
if (bestR >= 0) {
|
||||
for (let i = 0; i < N; i++) data[i] = snapCache[bestR][i];
|
||||
}
|
||||
} else {
|
||||
data = metricPushesRecv; // fallback: total
|
||||
// Find the closest round in cumulPushRecv
|
||||
let bestR = -1;
|
||||
for (let r = 0; r < snapRounds.length; r++) {
|
||||
if (snapRounds[r] <= curRound) bestR = r;
|
||||
}
|
||||
|
||||
if (bestR >= 0 && cumulPushRecv.length > bestR) {
|
||||
data = cumulPushRecv[bestR];
|
||||
} else {
|
||||
data = metricPushesRecv; // fallback: total
|
||||
}
|
||||
}
|
||||
|
||||
// Compute stats
|
||||
|
|
@ -998,37 +1132,76 @@ function drawGraph() {
|
|||
}
|
||||
}
|
||||
|
||||
// ── Nodes (community-colored) ──
|
||||
// ── Nodes ──
|
||||
if (showNodes) {
|
||||
const useCommunityColor = numCommunities > 1;
|
||||
let flashNodeI = -1;
|
||||
|
||||
if (useCommunityColor) {
|
||||
// Batch by community color
|
||||
for (let c = 0; c < numCommunities; c++) {
|
||||
ctx2d.beginPath();
|
||||
for (let i = 0; i < N; i++) {
|
||||
if (community[i] !== c) continue;
|
||||
const px = posX[i], py = posY[i];
|
||||
if (px+baseR < v0x || px-baseR > v1x || py+baseR < v0y || py-baseR > v1y) continue;
|
||||
if (hasNodeFlash && i === flashNode) { flashNodeI = i; continue; }
|
||||
ctx2d.moveTo(px+baseR, py); ctx2d.arc(px, py, baseR, 0, 2*Math.PI);
|
||||
}
|
||||
ctx2d.fillStyle = communityColors[c % communityColors.length]; ctx2d.fill();
|
||||
if (showStroke) { ctx2d.strokeStyle = '#1a1d2e'; ctx2d.lineWidth = 1/vs; ctx2d.stroke(); }
|
||||
if (isDist) {
|
||||
// Distribution: color by alive/dead
|
||||
let curRound = 0;
|
||||
if (replayCursor > 0 && replayCursor <= allEvents.length) curRound = allEvents[replayCursor - 1].tick;
|
||||
let curAlive = null;
|
||||
for (let r = snapRounds.length - 1; r >= 0; r--) {
|
||||
if (snapRounds[r] <= curRound) { curAlive = snapAlive[r]; break; }
|
||||
}
|
||||
} else {
|
||||
|
||||
// Draw alive nodes
|
||||
ctx2d.beginPath();
|
||||
for (let i = 0; i < N; i++) {
|
||||
const alive = curAlive ? curAlive[i] : 1;
|
||||
if (!alive) continue;
|
||||
const px = posX[i], py = posY[i];
|
||||
if (px+baseR < v0x || px-baseR > v1x || py+baseR < v0y || py-baseR > v1y) continue;
|
||||
if (hasNodeFlash && i === flashNode) { flashNodeI = i; continue; }
|
||||
ctx2d.moveTo(px+baseR, py); ctx2d.arc(px, py, baseR, 0, 2*Math.PI);
|
||||
}
|
||||
ctx2d.fillStyle = '#6366f1'; ctx2d.fill();
|
||||
if (showStroke) { ctx2d.strokeStyle = '#4f46e5'; ctx2d.lineWidth = 1.5/vs; ctx2d.stroke(); }
|
||||
ctx2d.fillStyle = '#22c55e'; ctx2d.fill();
|
||||
if (showStroke) { ctx2d.strokeStyle = '#16a34a'; ctx2d.lineWidth = 1.5/vs; ctx2d.stroke(); }
|
||||
|
||||
// Draw dead nodes
|
||||
ctx2d.beginPath();
|
||||
for (let i = 0; i < N; i++) {
|
||||
const alive = curAlive ? curAlive[i] : 1;
|
||||
if (alive) continue;
|
||||
const px = posX[i], py = posY[i];
|
||||
if (px+baseR < v0x || px-baseR > v1x || py+baseR < v0y || py-baseR > v1y) continue;
|
||||
if (hasNodeFlash && i === flashNode) { flashNodeI = i; continue; }
|
||||
ctx2d.moveTo(px+baseR, py); ctx2d.arc(px, py, baseR, 0, 2*Math.PI);
|
||||
}
|
||||
ctx2d.fillStyle = '#ef4444'; ctx2d.fill();
|
||||
if (showStroke) { ctx2d.strokeStyle = '#dc2626'; ctx2d.lineWidth = 1.5/vs; ctx2d.stroke(); }
|
||||
} else {
|
||||
// Gossip: existing community/default coloring
|
||||
const useCommunityColor = numCommunities > 1;
|
||||
|
||||
if (useCommunityColor) {
|
||||
// Batch by community color
|
||||
for (let c = 0; c < numCommunities; c++) {
|
||||
ctx2d.beginPath();
|
||||
for (let i = 0; i < N; i++) {
|
||||
if (community[i] !== c) continue;
|
||||
const px = posX[i], py = posY[i];
|
||||
if (px+baseR < v0x || px-baseR > v1x || py+baseR < v0y || py-baseR > v1y) continue;
|
||||
if (hasNodeFlash && i === flashNode) { flashNodeI = i; continue; }
|
||||
ctx2d.moveTo(px+baseR, py); ctx2d.arc(px, py, baseR, 0, 2*Math.PI);
|
||||
}
|
||||
ctx2d.fillStyle = communityColors[c % communityColors.length]; ctx2d.fill();
|
||||
if (showStroke) { ctx2d.strokeStyle = '#1a1d2e'; ctx2d.lineWidth = 1/vs; ctx2d.stroke(); }
|
||||
}
|
||||
} else {
|
||||
ctx2d.beginPath();
|
||||
for (let i = 0; i < N; i++) {
|
||||
const px = posX[i], py = posY[i];
|
||||
if (px+baseR < v0x || px-baseR > v1x || py+baseR < v0y || py-baseR > v1y) continue;
|
||||
if (hasNodeFlash && i === flashNode) { flashNodeI = i; continue; }
|
||||
ctx2d.moveTo(px+baseR, py); ctx2d.arc(px, py, baseR, 0, 2*Math.PI);
|
||||
}
|
||||
ctx2d.fillStyle = '#6366f1'; ctx2d.fill();
|
||||
if (showStroke) { ctx2d.strokeStyle = '#4f46e5'; ctx2d.lineWidth = 1.5/vs; ctx2d.stroke(); }
|
||||
}
|
||||
}
|
||||
|
||||
// Flash node (shared)
|
||||
if (flashNodeI >= 0) {
|
||||
const t = 1-(now-flashNodeT)/400, rad = baseR+6*t;
|
||||
ctx2d.beginPath();
|
||||
|
|
@ -1081,6 +1254,19 @@ function formatDetail(kind, detail) {
|
|||
case 'PeerRemoved': return '- ' + (detail.peer_name || detail.peer);
|
||||
case 'QueryReceived': return 'key=' + detail.key;
|
||||
case 'StateSnapshot': return detail.entries + ' entries, ' + detail.peer_count + ' peers';
|
||||
case 'Joined': return 'seed: ' + detail.seed_addr;
|
||||
case 'MembershipChanged': return detail.target + ' → ' + detail.new_state;
|
||||
case 'PingSent': return '→ ' + detail.target;
|
||||
case 'AckReceived': return '← ' + detail.from;
|
||||
case 'ActorRegistered': return 'actor: ' + detail.actor_id;
|
||||
case 'ActorStored': return detail.actor_id + ' on ' + detail.on_node;
|
||||
case 'ActorResolved': return detail.actor_id + ' → ' + detail.found_on;
|
||||
case 'ActorResolveFailed': return detail.actor_id + ': ' + detail.reason;
|
||||
case 'NameRegistered': return '"' + detail.name + '" on node ' + detail.node_idx;
|
||||
case 'NameUnregistered': return '"' + detail.name + '" on node ' + detail.node_idx;
|
||||
case 'NameResolved': return '"' + detail.name + '" → ' + detail.result;
|
||||
case 'NodeKilled': return '';
|
||||
case 'NodeRevived': return '';
|
||||
default: return JSON.stringify(detail);
|
||||
}
|
||||
}
|
||||
|
|
@ -1089,6 +1275,11 @@ function makeRow(ev) {
|
|||
const tr = document.createElement('tr');
|
||||
if (ev.kind === 'GossipRoundStarted') tr.className = 'highlight-push';
|
||||
else if (ev.kind === 'LocalSet') tr.className = 'highlight-set';
|
||||
else if (ev.kind === 'NodeKilled') tr.className = 'highlight-kill';
|
||||
else if (ev.kind === 'NodeRevived') tr.className = 'highlight-revive';
|
||||
else if (ev.kind === 'MembershipChanged') tr.className = 'highlight-membership';
|
||||
else if (ev.kind === 'NameRegistered' || ev.kind === 'NameUnregistered') tr.className = 'highlight-registry';
|
||||
else if (ev.kind === 'PingSent' || ev.kind === 'AckReceived') tr.className = 'highlight-ping';
|
||||
tr.innerHTML = '<td>'+ev.seq+'</td><td>'+ev.tick+'</td><td>'+(ev.thread||'-')+'</td><td>'+ev.node+'</td><td>'+ev.kind+'</td><td>'+formatDetail(ev.kind, ev.detail)+'</td>';
|
||||
return tr;
|
||||
}
|
||||
|
|
@ -1200,6 +1391,14 @@ function replayToImpl(pos) {
|
|||
const f = ev.detail.from_name || ev.detail.from;
|
||||
if (f) { flashSrc = nodeIdx.get(f) ?? -1; flashDst = nodeIdx.get(ev.node) ?? -1; flashEdgeT = performance.now(); }
|
||||
}
|
||||
if (ev.kind === 'PingSent' && ev.detail) {
|
||||
const t = ev.detail.target;
|
||||
if (t) { flashSrc = nodeIdx.get(ev.node) ?? -1; flashDst = nodeIdx.get(t) ?? -1; flashEdgeT = performance.now(); }
|
||||
}
|
||||
if (ev.kind === 'AckReceived' && ev.detail) {
|
||||
const f = ev.detail.from;
|
||||
if (f) { flashSrc = nodeIdx.get(f) ?? -1; flashDst = nodeIdx.get(ev.node) ?? -1; flashEdgeT = performance.now(); }
|
||||
}
|
||||
}
|
||||
|
||||
document.getElementById('replaySlider').value = pos;
|
||||
|
|
@ -1229,6 +1428,9 @@ function resetState() {
|
|||
metricPushesSent = new Int32Array(0); metricPushesRecv = new Int32Array(0);
|
||||
metricRedundant = 0; metricTotalPushes = 0; metricNoPeers = 0;
|
||||
cumulPushRecv = []; eventRoundMap = [];
|
||||
isDist = false;
|
||||
snapMembers = []; snapRegistry = []; snapCache = []; snapAlive = [];
|
||||
snapRouting = []; snapRepair = []; snapDirectory = []; snapTombstone = [];
|
||||
community = new Int32Array(0); numCommunities = 0; communityHulls = [];
|
||||
document.getElementById('eventTableBody').textContent = '';
|
||||
document.getElementById('workerColumns').innerHTML = '';
|
||||
|
|
@ -1247,6 +1449,7 @@ async function loadTrace(file) {
|
|||
const resp = await fetch('/trace.json?file=' + encodeURIComponent(file));
|
||||
if (!resp.ok) { dot.className = 'status-dot error'; statusText.textContent = 'Failed to load trace'; return; }
|
||||
const trace = await resp.json();
|
||||
isDist = (trace.trace_type === 'distribution');
|
||||
|
||||
// Build node index
|
||||
nodeNames = trace.node_names;
|
||||
|
|
@ -1271,85 +1474,138 @@ async function loadTrace(file) {
|
|||
allEvents = [];
|
||||
numRounds = trace.num_rounds || 0;
|
||||
|
||||
// First pass: collect snapshots grouped by tick
|
||||
const snapByTick = new Map(); // tick -> Map(nodeIdx -> {entries, peer_count})
|
||||
for (const ev of trace.events) {
|
||||
const isObj = typeof ev.kind === 'object';
|
||||
if (isObj && 'StateSnapshot' in ev.kind) {
|
||||
const snap = ev.kind.StateSnapshot.snapshot || ev.kind.StateSnapshot;
|
||||
const ni = nodeIdx.get(ev.node_name);
|
||||
if (ni === undefined) continue;
|
||||
const entriesCount = snap.entries ? Object.keys(snap.entries).length : 0;
|
||||
const peerCount = snap.peer_count || 0;
|
||||
if (!snapByTick.has(ev.tick)) snapByTick.set(ev.tick, new Map());
|
||||
snapByTick.get(ev.tick).set(ni, { entries: entriesCount, peer_count: peerCount });
|
||||
if (entriesCount > totalKeys) totalKeys = entriesCount;
|
||||
continue;
|
||||
if (isDist) {
|
||||
// Distribution: snapshots come from trace.snapshots_per_round directly
|
||||
snapRounds = [];
|
||||
for (let r = 0; r < (trace.snapshots_per_round || []).length; r++) {
|
||||
const roundSnaps = trace.snapshots_per_round[r];
|
||||
snapRounds.push(r + 1); // 1-indexed round
|
||||
const memArr = new Int32Array(N);
|
||||
const regArr = new Int32Array(N);
|
||||
const cacheArr = new Int32Array(N);
|
||||
const aliveArr = new Uint8Array(N);
|
||||
const routingArr = new Int32Array(N);
|
||||
const repairArr = new Int32Array(N);
|
||||
const dirArr = new Int32Array(N);
|
||||
const tombArr = new Int32Array(N);
|
||||
for (const [nodeName, snap] of roundSnaps) {
|
||||
const ni = nodeIdx.get(nodeName);
|
||||
if (ni === undefined) continue;
|
||||
memArr[ni] = snap.member_count || 0;
|
||||
regArr[ni] = snap.registry_size || 0;
|
||||
cacheArr[ni] = snap.cache_size || 0;
|
||||
aliveArr[ni] = snap.is_alive ? 1 : 0;
|
||||
routingArr[ni] = snap.routing_table_size || 0;
|
||||
repairArr[ni] = snap.repair_queue_size || 0;
|
||||
dirArr[ni] = snap.directory_entry_count || 0;
|
||||
tombArr[ni] = snap.registry_tombstone_count || 0;
|
||||
}
|
||||
snapMembers.push(memArr);
|
||||
snapRegistry.push(regArr);
|
||||
snapCache.push(cacheArr);
|
||||
snapAlive.push(aliveArr);
|
||||
snapRouting.push(routingArr);
|
||||
snapRepair.push(repairArr);
|
||||
snapDirectory.push(dirArr);
|
||||
snapTombstone.push(tombArr);
|
||||
// For convergence chart compatibility, use registry_size as "entries"
|
||||
snapEntries.push(regArr);
|
||||
snapPeerCount.push(memArr);
|
||||
// Track max for heatmap scaling
|
||||
for (let i = 0; i < N; i++) {
|
||||
if (regArr[i] > totalKeys) totalKeys = regArr[i];
|
||||
}
|
||||
}
|
||||
if (!isObj && ev.kind === 'StateSnapshot') continue;
|
||||
let kind, detail;
|
||||
if (isObj) { for (kind in ev.kind) break; detail = ev.kind[kind] || null; }
|
||||
else { kind = ev.kind; detail = null; }
|
||||
allEvents.push({ seq: seq++, tick: ev.tick, node: ev.node_name, thread: ev.thread_name, kind, detail });
|
||||
}
|
||||
|
||||
// Build snapshot arrays sorted by round
|
||||
snapRounds = Array.from(snapByTick.keys()).sort((a, b) => a - b);
|
||||
snapEntries = []; snapPeerCount = [];
|
||||
for (const tick of snapRounds) {
|
||||
const eArr = new Int32Array(N);
|
||||
const pArr = new Int32Array(N);
|
||||
const m = snapByTick.get(tick);
|
||||
for (const [ni, d] of m) { eArr[ni] = d.entries; pArr[ni] = d.peer_count; }
|
||||
snapEntries.push(eArr);
|
||||
snapPeerCount.push(pArr);
|
||||
}
|
||||
|
||||
// Compute per-node metrics from events
|
||||
metricPushesSent = new Int32Array(N);
|
||||
metricPushesRecv = new Int32Array(N);
|
||||
metricRedundant = 0; metricTotalPushes = 0; metricNoPeers = 0;
|
||||
|
||||
// Also build cumulative push-recv per round
|
||||
const roundSet = new Set(snapRounds);
|
||||
cumulPushRecv = [];
|
||||
let runningRecv = new Int32Array(N);
|
||||
|
||||
for (let i = 0; i < allEvents.length; i++) {
|
||||
const ev = allEvents[i];
|
||||
const ni = nodeIdx.get(ev.node);
|
||||
if (ni === undefined) continue;
|
||||
if (ev.kind === 'GossipRoundStarted') {
|
||||
metricPushesSent[ni]++;
|
||||
metricTotalPushes++;
|
||||
// Distribution events (no StateSnapshot filtering needed)
|
||||
for (const ev of trace.events) {
|
||||
const isObj = typeof ev.kind === 'object';
|
||||
let kind, detail;
|
||||
if (isObj) { for (kind in ev.kind) break; detail = ev.kind[kind] || null; }
|
||||
else { kind = ev.kind; detail = null; }
|
||||
allEvents.push({ seq: seq++, tick: ev.tick, node: ev.node_name, thread: null, kind, detail });
|
||||
}
|
||||
if (ev.kind === 'PushReceived') {
|
||||
metricPushesRecv[ni]++;
|
||||
runningRecv[ni]++;
|
||||
if (ev.detail && ev.detail.keys_updated === 0) metricRedundant++;
|
||||
} else {
|
||||
// First pass: collect snapshots grouped by tick
|
||||
const snapByTick = new Map(); // tick -> Map(nodeIdx -> {entries, peer_count})
|
||||
for (const ev of trace.events) {
|
||||
const isObj = typeof ev.kind === 'object';
|
||||
if (isObj && 'StateSnapshot' in ev.kind) {
|
||||
const snap = ev.kind.StateSnapshot.snapshot || ev.kind.StateSnapshot;
|
||||
const ni = nodeIdx.get(ev.node_name);
|
||||
if (ni === undefined) continue;
|
||||
const entriesCount = snap.entries ? Object.keys(snap.entries).length : 0;
|
||||
const peerCount = snap.peer_count || 0;
|
||||
if (!snapByTick.has(ev.tick)) snapByTick.set(ev.tick, new Map());
|
||||
snapByTick.get(ev.tick).set(ni, { entries: entriesCount, peer_count: peerCount });
|
||||
if (entriesCount > totalKeys) totalKeys = entriesCount;
|
||||
continue;
|
||||
}
|
||||
if (!isObj && ev.kind === 'StateSnapshot') continue;
|
||||
let kind, detail;
|
||||
if (isObj) { for (kind in ev.kind) break; detail = ev.kind[kind] || null; }
|
||||
else { kind = ev.kind; detail = null; }
|
||||
allEvents.push({ seq: seq++, tick: ev.tick, node: ev.node_name, thread: ev.thread_name, kind, detail });
|
||||
}
|
||||
if (ev.kind === 'GossipRoundNoPeers') { metricNoPeers++; }
|
||||
}
|
||||
|
||||
// Build cumulative recv snapshots aligned to snap rounds
|
||||
// Re-scan to build per-round cumulative
|
||||
if (snapRounds.length > 0) {
|
||||
const cumRecv = new Int32Array(N);
|
||||
let sri = 0;
|
||||
for (let i = 0; i < allEvents.length && sri < snapRounds.length; i++) {
|
||||
// Build snapshot arrays sorted by round
|
||||
snapRounds = Array.from(snapByTick.keys()).sort((a, b) => a - b);
|
||||
snapEntries = []; snapPeerCount = [];
|
||||
for (const tick of snapRounds) {
|
||||
const eArr = new Int32Array(N);
|
||||
const pArr = new Int32Array(N);
|
||||
const m = snapByTick.get(tick);
|
||||
for (const [ni, d] of m) { eArr[ni] = d.entries; pArr[ni] = d.peer_count; }
|
||||
snapEntries.push(eArr);
|
||||
snapPeerCount.push(pArr);
|
||||
}
|
||||
|
||||
// Compute per-node metrics from events
|
||||
metricPushesSent = new Int32Array(N);
|
||||
metricPushesRecv = new Int32Array(N);
|
||||
metricRedundant = 0; metricTotalPushes = 0; metricNoPeers = 0;
|
||||
|
||||
// Also build cumulative push-recv per round
|
||||
const roundSet = new Set(snapRounds);
|
||||
cumulPushRecv = [];
|
||||
let runningRecv = new Int32Array(N);
|
||||
|
||||
for (let i = 0; i < allEvents.length; i++) {
|
||||
const ev = allEvents[i];
|
||||
const ni = nodeIdx.get(ev.node);
|
||||
if (ni === undefined) continue;
|
||||
if (ev.kind === 'GossipRoundStarted') {
|
||||
metricPushesSent[ni]++;
|
||||
metricTotalPushes++;
|
||||
}
|
||||
if (ev.kind === 'PushReceived') {
|
||||
const ni = nodeIdx.get(ev.node);
|
||||
if (ni !== undefined) cumRecv[ni]++;
|
||||
}
|
||||
// When we pass a snapshot round boundary, save
|
||||
while (sri < snapRounds.length && ev.tick >= snapRounds[sri]) {
|
||||
cumulPushRecv.push(new Int32Array(cumRecv));
|
||||
sri++;
|
||||
metricPushesRecv[ni]++;
|
||||
runningRecv[ni]++;
|
||||
if (ev.detail && ev.detail.keys_updated === 0) metricRedundant++;
|
||||
}
|
||||
if (ev.kind === 'GossipRoundNoPeers') { metricNoPeers++; }
|
||||
}
|
||||
|
||||
// Build cumulative recv snapshots aligned to snap rounds
|
||||
// Re-scan to build per-round cumulative
|
||||
if (snapRounds.length > 0) {
|
||||
const cumRecv = new Int32Array(N);
|
||||
let sri = 0;
|
||||
for (let i = 0; i < allEvents.length && sri < snapRounds.length; i++) {
|
||||
const ev = allEvents[i];
|
||||
if (ev.kind === 'PushReceived') {
|
||||
const ni = nodeIdx.get(ev.node);
|
||||
if (ni !== undefined) cumRecv[ni]++;
|
||||
}
|
||||
// When we pass a snapshot round boundary, save
|
||||
while (sri < snapRounds.length && ev.tick >= snapRounds[sri]) {
|
||||
cumulPushRecv.push(new Int32Array(cumRecv));
|
||||
sri++;
|
||||
}
|
||||
}
|
||||
// Fill remaining
|
||||
while (sri < snapRounds.length) { cumulPushRecv.push(new Int32Array(cumRecv)); sri++; }
|
||||
}
|
||||
// Fill remaining
|
||||
while (sri < snapRounds.length) { cumulPushRecv.push(new Int32Array(cumRecv)); sri++; }
|
||||
}
|
||||
|
||||
// Index PeerAdded edges
|
||||
|
|
@ -1395,10 +1651,33 @@ async function loadTrace(file) {
|
|||
computeCommunityHulls();
|
||||
|
||||
dot.className = 'status-dot ready';
|
||||
statusText.textContent = trace.name + ' (' + allEvents.length + ' events)';
|
||||
statusText.textContent = trace.name + (isDist ? ' [distribution]' : ' [gossip]') + ' (' + allEvents.length + ' events)';
|
||||
drawGraph();
|
||||
drawAllCharts();
|
||||
|
||||
// Update panel labels
|
||||
document.querySelector('.worker-panel h2').textContent = isDist ? 'Node Status' : 'Worker Logs';
|
||||
document.querySelector('#heatSection h3').textContent = isDist ? 'Registry Propagation' : 'Propagation Heatmap';
|
||||
document.querySelector('#loadSection h3').textContent = isDist ? 'Cache Utilization' : 'Load Distribution';
|
||||
document.querySelector('#convSection h3').textContent = isDist ? 'Membership Convergence' : 'Convergence Curve';
|
||||
|
||||
// Update stat labels
|
||||
if (isDist) {
|
||||
document.querySelectorAll('.stat-label')[1].textContent = 'Alive';
|
||||
} else {
|
||||
document.querySelectorAll('.stat-label')[1].textContent = 'Edges';
|
||||
}
|
||||
|
||||
// Update stats with distribution-specific values
|
||||
if (isDist && snapAlive.length > 0) {
|
||||
let alive = 0;
|
||||
const lastSnap = snapAlive[snapAlive.length - 1];
|
||||
for (let i = 0; i < N; i++) if (lastSnap[i]) alive++;
|
||||
updateStats(N, alive, allEvents.length, numRounds, numRounds);
|
||||
} else {
|
||||
updateStats(N, totalEdges, allEvents.length, numRounds, numRounds);
|
||||
}
|
||||
|
||||
// Replay controls
|
||||
document.getElementById('replayControls').style.display = 'flex';
|
||||
const slider = document.getElementById('replaySlider');
|
||||
|
|
@ -1426,7 +1705,7 @@ resizeCanvas();
|
|||
catch { dot.className='status-dot error'; statusText.textContent='Failed to fetch trace list'; select.innerHTML='<option value="">Error</option>'; return; }
|
||||
if (!traces.length) { dot.className='status-dot error'; statusText.textContent='No traces found'; select.innerHTML='<option value="">No traces found</option>'; return; }
|
||||
select.innerHTML = '';
|
||||
traces.forEach(t => { const o = document.createElement('option'); o.value = t.file; o.textContent = t.name+' ('+t.nodes+' nodes, '+t.events+' events)'; select.appendChild(o); });
|
||||
traces.forEach(t => { const o = document.createElement('option'); o.value = t.file; o.textContent = '[' + (t.trace_type || 'gossip') + '] ' + t.name+' ('+t.nodes+' nodes, '+t.events+' events)'; select.appendChild(o); });
|
||||
select.disabled = false;
|
||||
select.onchange = () => { if (select.value) loadTrace(select.value); };
|
||||
loadTrace(traces[0].file);
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ struct TraceEntry {
|
|||
name: String,
|
||||
nodes: usize,
|
||||
events: usize,
|
||||
trace_type: String,
|
||||
}
|
||||
|
||||
fn scan_traces(dir: &Path) -> Vec<TraceEntry> {
|
||||
|
|
@ -51,11 +52,17 @@ fn scan_traces(dir: &Path) -> Vec<TraceEntry> {
|
|||
.and_then(|v| v.as_array())
|
||||
.map(|a| a.len())
|
||||
.unwrap_or(0);
|
||||
let trace_type = val
|
||||
.get("trace_type")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("gossip")
|
||||
.to_string();
|
||||
entries.push(TraceEntry {
|
||||
file: fname,
|
||||
name,
|
||||
nodes,
|
||||
events,
|
||||
trace_type,
|
||||
});
|
||||
}
|
||||
entries.sort_by(|a, b| a.file.cmp(&b.file));
|
||||
|
|
|
|||
|
|
@ -155,7 +155,7 @@ impl NetworkState {
|
|||
}
|
||||
}
|
||||
|
||||
type DistTrace = SimulationTrace<DistributionEventKind, DistributionSnapshot>;
|
||||
pub type DistTrace = SimulationTrace<DistributionEventKind, DistributionSnapshot>;
|
||||
|
||||
/// Run a distribution simulation, returning both the trace and the final node states.
|
||||
///
|
||||
|
|
@ -550,6 +550,7 @@ fn run_simulation_inner(config: DistributionSimConfig) -> (DistTrace, Vec<Option
|
|||
|
||||
let trace = SimulationTrace {
|
||||
name: config.name,
|
||||
trace_type: "distribution".into(),
|
||||
node_names,
|
||||
topology_edges,
|
||||
events,
|
||||
|
|
|
|||
|
|
@ -1,11 +1,17 @@
|
|||
use std::sync::atomic::AtomicU64;
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Shared tick counter — the simulation harness increments this.
|
||||
pub type TickCounter = Arc<AtomicU64>;
|
||||
|
||||
/// A single simulation event, generic over the event kind `K`.
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(bound(
|
||||
serialize = "K: Serialize",
|
||||
deserialize = "K: serde::de::DeserializeOwned"
|
||||
))]
|
||||
pub struct Event<K> {
|
||||
pub tick: u64,
|
||||
pub node_name: String,
|
||||
|
|
@ -13,9 +19,16 @@ pub struct Event<K> {
|
|||
}
|
||||
|
||||
/// Complete output of a simulation run, generic over event kind `K` and snapshot type `S`.
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(bound(
|
||||
serialize = "K: Serialize, S: Serialize",
|
||||
deserialize = "K: serde::de::DeserializeOwned, S: serde::de::DeserializeOwned"
|
||||
))]
|
||||
pub struct SimulationTrace<K, S> {
|
||||
pub name: String,
|
||||
/// Discriminator for dashboard rendering ("gossip" or "distribution").
|
||||
#[serde(default)]
|
||||
pub trace_type: String,
|
||||
pub node_names: Vec<String>,
|
||||
pub topology_edges: Vec<(String, String)>,
|
||||
pub events: Vec<Event<K>>,
|
||||
|
|
|
|||
|
|
@ -10,9 +10,22 @@ use simulation::distribution::properties::{
|
|||
check_repair_queue_populated, check_routing_table_bounded,
|
||||
};
|
||||
use simulation::distribution::sim::{
|
||||
run_simulation_with_nodes, DistributionSimConfig, NetworkFault, Partition, SimAction,
|
||||
run_simulation_with_nodes, DistributionSimConfig, DistTrace, NetworkFault, Partition, SimAction,
|
||||
};
|
||||
|
||||
fn maybe_save_trace(trace: &DistTrace) {
|
||||
if let Ok(dir) = std::env::var("SWACTOR_TRACE_DIR") {
|
||||
std::fs::create_dir_all(&dir).ok();
|
||||
let filename = format!(
|
||||
"{}/{}.trace.json",
|
||||
dir,
|
||||
trace.name.to_lowercase().replace(' ', "_").replace(['(', ')'], "")
|
||||
);
|
||||
let json = serde_json::to_string_pretty(trace).expect("trace serialization failed");
|
||||
std::fs::write(&filename, json).expect("trace write failed");
|
||||
}
|
||||
}
|
||||
|
||||
fn default_config() -> DistributionSimConfig {
|
||||
DistributionSimConfig::default()
|
||||
}
|
||||
|
|
@ -35,6 +48,7 @@ fn dead_node_triggers_repair_queue_and_cache_invalidation() {
|
|||
};
|
||||
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
// Then: at least one survivor should have a non-empty repair queue
|
||||
let result = check_repair_queue_populated(&trace, 10);
|
||||
|
|
@ -89,7 +103,8 @@ fn revived_node_has_empty_directory() {
|
|||
..default_config()
|
||||
};
|
||||
|
||||
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
// Then: the revived node should have an empty directory
|
||||
// (it's a fresh DistributedNode, not carrying over old state)
|
||||
|
|
@ -126,6 +141,7 @@ fn cache_shrinks_after_node_death() {
|
|||
};
|
||||
|
||||
let (trace, _nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
// Check that cache_size decreased for at least some survivors after death
|
||||
// Before death (round 9), survivors should have cache entries
|
||||
|
|
@ -194,7 +210,8 @@ fn routing_table_recovers_after_partition_heals() {
|
|||
..default_config()
|
||||
};
|
||||
|
||||
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
// Then: after healing, all nodes should have recovered routing tables
|
||||
// Each node should see at least 4 of 5 other nodes in their routing table
|
||||
|
|
@ -227,6 +244,7 @@ fn routing_table_bounded_by_alive_count() {
|
|||
};
|
||||
|
||||
let (trace, _) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
let result = check_routing_table_bounded(&trace);
|
||||
assert!(
|
||||
|
|
@ -281,7 +299,8 @@ fn partition_then_death_during_partition_then_heal() {
|
|||
..default_config()
|
||||
};
|
||||
|
||||
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
let survivors: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect();
|
||||
assert_eq!(survivors.len(), 5, "5 of 6 should survive");
|
||||
|
|
@ -347,6 +366,7 @@ fn registry_tombstones_gc_after_ttl() {
|
|||
};
|
||||
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
// Shortly after unregister (round 12), tombstones should exist
|
||||
let mid_tombstones: usize = trace.snapshots_per_round
|
||||
|
|
@ -439,7 +459,8 @@ fn asymmetric_one_way_block_does_not_kill_node() {
|
|||
..default_config()
|
||||
};
|
||||
|
||||
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
// Then: all 6 nodes should still be alive (asymmetric block doesn't kill either side)
|
||||
let alive_count = nodes.iter().filter(|n| n.is_some()).count();
|
||||
|
|
@ -500,7 +521,8 @@ fn names_registered_during_partition_propagate_after_heal() {
|
|||
..default_config()
|
||||
};
|
||||
|
||||
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
let alive: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect();
|
||||
assert_eq!(alive.len(), 6, "all 6 nodes should survive");
|
||||
|
|
@ -562,7 +584,8 @@ fn bidirectional_suspicion_both_nodes_recover() {
|
|||
..default_config()
|
||||
};
|
||||
|
||||
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
// All 6 nodes alive
|
||||
let alive_count = nodes.iter().filter(|n| n.is_some()).count();
|
||||
|
|
|
|||
|
|
@ -8,9 +8,22 @@ use simulation::distribution::properties::{
|
|||
check_routing_table_bounded,
|
||||
};
|
||||
use simulation::distribution::sim::{
|
||||
run_simulation_with_nodes, DistributionSimConfig, NetworkFault, Partition, SimAction,
|
||||
run_simulation_with_nodes, DistributionSimConfig, DistTrace, NetworkFault, Partition, SimAction,
|
||||
};
|
||||
|
||||
fn maybe_save_trace(trace: &DistTrace) {
|
||||
if let Ok(dir) = std::env::var("SWACTOR_TRACE_DIR") {
|
||||
std::fs::create_dir_all(&dir).ok();
|
||||
let filename = format!(
|
||||
"{}/{}.trace.json",
|
||||
dir,
|
||||
trace.name.to_lowercase().replace(' ', "_").replace(['(', ')'], "")
|
||||
);
|
||||
let json = serde_json::to_string_pretty(trace).expect("trace serialization failed");
|
||||
std::fs::write(&filename, json).expect("trace write failed");
|
||||
}
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// 1. Routing table size ≤ alive membership at every round
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
|
|
@ -49,6 +62,7 @@ fn routing_table_bounded_across_configs() {
|
|||
for config in configs {
|
||||
let name = config.name.clone();
|
||||
let (trace, _) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
let result = check_routing_table_bounded(&trace);
|
||||
assert!(
|
||||
result.passed,
|
||||
|
|
@ -90,6 +104,7 @@ fn cache_bounded_across_configs() {
|
|||
for config in configs {
|
||||
let name = config.name.clone();
|
||||
let (trace, _) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
let result = check_cache_bounded(&trace, capacity);
|
||||
assert!(
|
||||
result.passed,
|
||||
|
|
@ -117,6 +132,7 @@ fn repair_queue_populates_on_death_with_directory_entries() {
|
|||
};
|
||||
|
||||
let (trace, _) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
// Repair queue should be populated within 2-3 rounds of death detection
|
||||
let result = check_repair_queue_populated(&trace, 15);
|
||||
|
|
@ -208,6 +224,7 @@ fn registry_eventually_consistent_across_configs() {
|
|||
for (config, min_size) in configs {
|
||||
let name = config.name.clone();
|
||||
let (trace, _) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
let result = check_registry_propagation(&trace, min_size);
|
||||
assert!(
|
||||
result.passed,
|
||||
|
|
@ -236,6 +253,7 @@ fn cascading_deaths_maintain_invariants() {
|
|||
};
|
||||
|
||||
let (trace, _) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
let rt_result = check_routing_table_bounded(&trace);
|
||||
assert!(
|
||||
|
|
@ -299,7 +317,8 @@ fn asymmetric_partition_registry_converges_after_heal() {
|
|||
..DistributionSimConfig::default()
|
||||
};
|
||||
|
||||
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
// After healing, all nodes should resolve both names
|
||||
let alive_nodes: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect();
|
||||
|
|
@ -353,7 +372,8 @@ fn revived_node_re_registration_overwrites_tombstone() {
|
|||
..DistributionSimConfig::default()
|
||||
};
|
||||
|
||||
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
// Then: all alive nodes should resolve "svc" to the new registration
|
||||
let alive_nodes: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect();
|
||||
|
|
@ -393,6 +413,7 @@ fn registry_convergence_is_monotonic_in_stable_cluster() {
|
|||
};
|
||||
|
||||
let (trace, _) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
// Measure "divergence" = number of alive nodes with registry_size < 4
|
||||
// Once it reaches 0, it should never increase again
|
||||
|
|
@ -449,6 +470,7 @@ fn large_cluster_registry_converges() {
|
|||
};
|
||||
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
let survivors: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect();
|
||||
assert_eq!(survivors.len(), 13, "13 of 15 nodes should survive");
|
||||
|
|
@ -537,7 +559,8 @@ fn three_way_partition_heals_and_converges() {
|
|||
..DistributionSimConfig::default()
|
||||
};
|
||||
|
||||
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
let alive_count = nodes.iter().filter(|n| n.is_some()).count();
|
||||
assert_eq!(alive_count, 9, "all 9 nodes should survive the three-way partition");
|
||||
|
|
|
|||
|
|
@ -7,10 +7,23 @@ use simulation::distribution::properties::{
|
|||
check_registry_propagation, check_registry_tombstones,
|
||||
};
|
||||
use simulation::distribution::sim::{
|
||||
run_simulation_with_nodes, DistributionSimConfig, NetworkFault, Partition,
|
||||
run_simulation_with_nodes, DistributionSimConfig, DistTrace, NetworkFault, Partition,
|
||||
SimAction,
|
||||
};
|
||||
|
||||
fn maybe_save_trace(trace: &DistTrace) {
|
||||
if let Ok(dir) = std::env::var("SWACTOR_TRACE_DIR") {
|
||||
std::fs::create_dir_all(&dir).ok();
|
||||
let filename = format!(
|
||||
"{}/{}.trace.json",
|
||||
dir,
|
||||
trace.name.to_lowercase().replace(' ', "_").replace(['(', ')'], "")
|
||||
);
|
||||
let json = serde_json::to_string_pretty(trace).expect("trace serialization failed");
|
||||
std::fs::write(&filename, json).expect("trace write failed");
|
||||
}
|
||||
}
|
||||
|
||||
fn default_config() -> DistributionSimConfig {
|
||||
DistributionSimConfig {
|
||||
actors_per_node: 0, // Registry tests don't need actors
|
||||
|
|
@ -38,6 +51,7 @@ fn registry_name_converges_across_cluster() {
|
|||
|
||||
// When: we run the simulation
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
// Then: all alive nodes should have the registry entry
|
||||
let result = check_registry_propagation(&trace, 1);
|
||||
|
|
@ -110,7 +124,8 @@ fn split_brain_naming_converges_after_partition_heals() {
|
|||
};
|
||||
|
||||
// When: we run the simulation
|
||||
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
// Then: all alive nodes should resolve "leader" to the same value (LWW winner)
|
||||
let resolutions: Vec<_> = nodes
|
||||
|
|
@ -160,6 +175,7 @@ fn tombstone_propagates_when_name_owner_dies() {
|
|||
|
||||
// When: we run the simulation
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
// Then: surviving nodes should have tombstoned "svc"
|
||||
let result = check_registry_tombstones(&trace, 1);
|
||||
|
|
@ -207,7 +223,8 @@ fn rapid_re_registration_converges_to_latest() {
|
|||
};
|
||||
|
||||
// When: we run the simulation
|
||||
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
// Then: all nodes should resolve "svc" to actor_c (the latest registration)
|
||||
let resolutions: Vec<_> = nodes
|
||||
|
|
@ -251,7 +268,8 @@ fn simultaneous_registration_converges_deterministically() {
|
|||
};
|
||||
|
||||
// When: we run the simulation
|
||||
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
// Then: all nodes should agree on one winner
|
||||
let resolutions: Vec<_> = nodes
|
||||
|
|
@ -297,6 +315,7 @@ fn multiple_names_from_different_nodes_all_propagate() {
|
|||
|
||||
// When: we run the simulation
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
// Then: all 5 nodes should have all 5 registry entries
|
||||
let result = check_registry_propagation(&trace, 5);
|
||||
|
|
@ -337,7 +356,8 @@ fn explicit_unregister_propagates_to_all_nodes() {
|
|||
..default_config()
|
||||
};
|
||||
|
||||
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
// Then: all nodes should resolve "svc" to None (tombstoned)
|
||||
for (i, node) in nodes.iter().filter_map(|n| n.as_ref()).enumerate() {
|
||||
|
|
@ -373,7 +393,8 @@ fn re_registration_after_tombstone_succeeds() {
|
|||
..default_config()
|
||||
};
|
||||
|
||||
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
// Then: all survivors should resolve "svc" to the new actor from node 1
|
||||
let resolutions: Vec<_> = nodes
|
||||
|
|
@ -414,7 +435,8 @@ fn all_names_tombstoned_when_owner_dies() {
|
|||
..default_config()
|
||||
};
|
||||
|
||||
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
// Then: all survivors should resolve all 3 names to None
|
||||
for node in nodes.iter().filter_map(|n| n.as_ref()) {
|
||||
|
|
@ -447,7 +469,8 @@ fn graceful_leave_tombstones_registry_names() {
|
|||
..default_config()
|
||||
};
|
||||
|
||||
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
// Then: all survivors should resolve "svc" to None (tombstoned via death notification)
|
||||
let resolutions: Vec<_> = nodes
|
||||
|
|
@ -486,6 +509,7 @@ fn piggyback_contention_both_propagate() {
|
|||
};
|
||||
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
// Then: all survivors should have all 3 registry names
|
||||
let result = check_registry_propagation(&trace, 3);
|
||||
|
|
@ -543,7 +567,8 @@ fn registry_converges_despite_message_loss() {
|
|||
..default_config()
|
||||
};
|
||||
|
||||
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
// Then: all surviving nodes should resolve all 3 names despite packet loss
|
||||
let alive_nodes: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect();
|
||||
|
|
@ -589,7 +614,8 @@ fn simultaneous_kill_of_multiple_name_owners() {
|
|||
..default_config()
|
||||
};
|
||||
|
||||
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
// Then: all 4 survivors should resolve all 3 names to None (tombstoned)
|
||||
let survivors: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect();
|
||||
|
|
@ -645,7 +671,8 @@ fn suspected_name_owner_recovers_and_registry_survives() {
|
|||
..default_config()
|
||||
};
|
||||
|
||||
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
// Then: all nodes should still resolve "svc" (node 0 never died, only suspected)
|
||||
let resolutions: Vec<_> = nodes
|
||||
|
|
@ -685,7 +712,8 @@ fn registry_correct_under_rapid_churn() {
|
|||
..default_config()
|
||||
};
|
||||
|
||||
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||
maybe_save_trace(&trace);
|
||||
|
||||
let survivors: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect();
|
||||
assert_eq!(survivors.len(), 5, "5 of 8 nodes should survive");
|
||||
|
|
|
|||
19
scripts/sim-dashboard.sh
Executable file
19
scripts/sim-dashboard.sh
Executable file
|
|
@ -0,0 +1,19 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
DIR="$(cd "$(dirname "${1:-.}")" && pwd)/$(basename "${1:-traces}")"
|
||||
PORT="${2:-8080}"
|
||||
|
||||
rm -rf "$DIR"
|
||||
mkdir -p "$DIR"
|
||||
|
||||
echo "Running distribution sim tests with trace export..."
|
||||
SWACTOR_TRACE_DIR="$DIR" cargo test -p simulation \
|
||||
--test distribution_registry \
|
||||
--test distribution_lifecycle \
|
||||
--test distribution_properties || echo "WARNING: some tests failed (traces from passing tests are still available)"
|
||||
|
||||
COUNT=$(find "$DIR" -name '*.trace.json' 2>/dev/null | wc -l)
|
||||
echo "$COUNT traces in $DIR/"
|
||||
echo "Dashboard at http://localhost:$PORT"
|
||||
cargo run -p simulation-dashboard --example replay -- "$DIR" "$PORT"
|
||||
Loading…
Reference in a new issue