Compare commits
10 commits
29b4f53d69
...
0f5c739337
| Author | SHA1 | Date | |
|---|---|---|---|
|
0f5c739337 | ||
|
|
982d82776d | ||
|
|
c7a0bd2523 | ||
|
|
991b212191 | ||
|
|
70555c2daf | ||
|
|
6cfafa04b7 | ||
|
|
41930ef8f4 | ||
|
|
5e3a779cba | ||
|
|
144b4ca5ef | ||
|
|
a1f5f581fe |
18 changed files with 2596 additions and 117 deletions
403
crates/runtime-dashboard/src/actor_detail_html.rs
Normal file
403
crates/runtime-dashboard/src/actor_detail_html.rs
Normal file
|
|
@ -0,0 +1,403 @@
|
|||
pub const ACTOR_DETAIL_HTML: &str = r##"<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Actor Detail — Swactor Dashboard</title>
|
||||
<style>
|
||||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||
body { font-family: 'Menlo', 'Consolas', 'Monaco', monospace; background: #0f1117; color: #e0e0e0; font-size: 13px; }
|
||||
|
||||
.header {
|
||||
display: flex; align-items: center; justify-content: space-between;
|
||||
padding: 12px 20px; background: #161822; border-bottom: 1px solid #2a2d3e;
|
||||
}
|
||||
.header h1 { font-size: 16px; font-weight: 600; color: #fff; }
|
||||
.status-dot {
|
||||
width: 10px; height: 10px; border-radius: 50%; background: #4caf50;
|
||||
display: inline-block; margin-left: 8px; vertical-align: middle;
|
||||
}
|
||||
.status-dot.disconnected { background: #f44336; }
|
||||
|
||||
.header-left { display: flex; align-items: center; }
|
||||
.nav-links { display: flex; gap: 4px; margin-left: 20px; }
|
||||
.nav-link {
|
||||
color: #888; text-decoration: none; font-size: 12px;
|
||||
padding: 4px 10px; border-radius: 3px; transition: color 0.2s;
|
||||
}
|
||||
.nav-link:hover { color: #e0e0e0; }
|
||||
.nav-link.active { color: #fff; background: #2a2d3e; }
|
||||
|
||||
.content { padding: 16px 20px; max-width: 900px; }
|
||||
|
||||
.breadcrumb { color: #555; font-size: 12px; margin-bottom: 12px; }
|
||||
.breadcrumb a { color: #888; text-decoration: none; }
|
||||
.breadcrumb a:hover { color: #e0e0e0; }
|
||||
|
||||
.info-card {
|
||||
background: #161822; border: 1px solid #2a2d3e; border-radius: 6px;
|
||||
padding: 16px; margin-bottom: 12px;
|
||||
}
|
||||
.info-row { display: flex; gap: 24px; margin-bottom: 6px; flex-wrap: wrap; }
|
||||
.info-label { color: #888; font-size: 11px; text-transform: uppercase; }
|
||||
.info-value { color: #fff; font-weight: 700; font-size: 15px; }
|
||||
.info-value.healthy { color: #4caf50; }
|
||||
.info-value.poisoned { color: #f44336; }
|
||||
|
||||
.stats-cards {
|
||||
display: grid; grid-template-columns: repeat(4, 1fr); gap: 10px;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
.stat-card {
|
||||
background: #161822; border: 1px solid #2a2d3e; border-radius: 4px;
|
||||
padding: 12px; text-align: center;
|
||||
}
|
||||
.stat-card .value { font-size: 22px; font-weight: 700; color: #fff; }
|
||||
.stat-card .label { font-size: 10px; color: #888; text-transform: uppercase; margin-top: 2px; }
|
||||
|
||||
.sparkline-panel {
|
||||
background: #161822; border: 1px solid #2a2d3e; border-radius: 6px;
|
||||
padding: 14px; margin-bottom: 12px;
|
||||
}
|
||||
.sparkline-panel h3 { font-size: 11px; color: #888; text-transform: uppercase; letter-spacing: 1px; margin-bottom: 8px; }
|
||||
.sparkline-panel svg { width: 100%; height: 50px; }
|
||||
|
||||
.type-breakdown {
|
||||
background: #161822; border: 1px solid #2a2d3e; border-radius: 6px;
|
||||
padding: 14px; margin-bottom: 12px;
|
||||
}
|
||||
.type-breakdown h3 { font-size: 11px; color: #888; text-transform: uppercase; letter-spacing: 1px; margin-bottom: 8px; }
|
||||
.type-row { display: flex; align-items: center; gap: 8px; margin-bottom: 4px; font-size: 11px; }
|
||||
.type-name { color: #e0e0e0; min-width: 180px; text-overflow: ellipsis; overflow: hidden; white-space: nowrap; }
|
||||
.type-bar-bg { flex: 1; background: #1e2030; height: 14px; border-radius: 2px; overflow: hidden; }
|
||||
.type-bar-fill { height: 100%; border-radius: 2px; }
|
||||
.type-count { color: #888; min-width: 60px; text-align: right; }
|
||||
.type-pct { color: #555; min-width: 40px; text-align: right; }
|
||||
|
||||
.logs-panel {
|
||||
background: #161822; border: 1px solid #2a2d3e; border-radius: 6px;
|
||||
padding: 14px; margin-bottom: 12px;
|
||||
}
|
||||
.logs-panel h3 {
|
||||
font-size: 11px; color: #888; text-transform: uppercase; letter-spacing: 1px;
|
||||
margin-bottom: 8px; display: flex; align-items: center; gap: 12px;
|
||||
}
|
||||
.level-filter { display: flex; gap: 4px; }
|
||||
.level-btn {
|
||||
background: #1e2030; border: 1px solid #2a2d3e; border-radius: 3px;
|
||||
color: #888; font-size: 10px; padding: 1px 6px; cursor: pointer;
|
||||
font-family: inherit;
|
||||
}
|
||||
.level-btn.active { border-color: #555; color: #fff; }
|
||||
.level-btn.error { color: #f44336; }
|
||||
.level-btn.warn { color: #ff9800; }
|
||||
.level-btn.info { color: #2196f3; }
|
||||
.level-btn.debug { color: #888; }
|
||||
.level-btn.trace { color: #555; }
|
||||
|
||||
.log-list {
|
||||
max-height: 400px; overflow-y: auto; font-size: 11px; line-height: 1.6;
|
||||
}
|
||||
.log-entry { display: flex; gap: 8px; padding: 1px 0; border-bottom: 1px solid #1a1c2e; }
|
||||
.log-time { color: #555; white-space: nowrap; min-width: 80px; }
|
||||
.log-level { font-weight: 700; min-width: 50px; }
|
||||
.log-level.ERROR { color: #f44336; }
|
||||
.log-level.WARN { color: #ff9800; }
|
||||
.log-level.INFO { color: #2196f3; }
|
||||
.log-level.DEBUG { color: #888; }
|
||||
.log-level.TRACE { color: #555; }
|
||||
.log-msg { color: #e0e0e0; word-break: break-all; }
|
||||
|
||||
::-webkit-scrollbar { width: 6px; }
|
||||
::-webkit-scrollbar-track { background: #0f1117; }
|
||||
::-webkit-scrollbar-thumb { background: #2a2d3e; border-radius: 3px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="header">
|
||||
<div class="header-left">
|
||||
<h1>
|
||||
Swactor Runtime Dashboard
|
||||
<span id="statusDot" class="status-dot"></span>
|
||||
</h1>
|
||||
<nav class="nav-links">
|
||||
<a href="/" class="nav-link">Overview</a>
|
||||
<a href="/actors" class="nav-link">Actors</a>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="content">
|
||||
<div class="breadcrumb">
|
||||
<a href="/">Overview</a> / <a href="/actors">Actors</a> / <span id="addrBreadcrumb">—</span>
|
||||
</div>
|
||||
|
||||
<div class="info-card">
|
||||
<div class="info-row">
|
||||
<div><div class="info-label">Address</div><div class="info-value" id="addrFull">—</div></div>
|
||||
<div><div class="info-label">Worker</div><div class="info-value" id="addrWorker">—</div></div>
|
||||
<div><div class="info-label">Status</div><div class="info-value" id="addrStatus">—</div></div>
|
||||
</div>
|
||||
<div class="info-row">
|
||||
<div><div class="info-label">Last Message Type</div><div class="info-value" id="addrLastMsg" style="color:#4caf50;font-size:13px;">—</div></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="stats-cards">
|
||||
<div class="stat-card"><div class="value" id="addrMsgs">0</div><div class="label">Messages</div></div>
|
||||
<div class="stat-card"><div class="value" id="addrMailbox">0</div><div class="label">Mailbox</div></div>
|
||||
<div class="stat-card"><div class="value" id="addrRate">0</div><div class="label">Msg/s</div></div>
|
||||
<div class="stat-card"><div class="value" id="addrWorkerLoad">—</div><div class="label">Worker Load</div></div>
|
||||
</div>
|
||||
|
||||
<div class="sparkline-panel">
|
||||
<h3>Message Rate</h3>
|
||||
<svg id="rateSpark" viewBox="0 0 400 50" preserveAspectRatio="none"></svg>
|
||||
</div>
|
||||
|
||||
<div class="sparkline-panel">
|
||||
<h3>Mailbox Depth</h3>
|
||||
<svg id="mboxSpark" viewBox="0 0 400 50" preserveAspectRatio="none"></svg>
|
||||
</div>
|
||||
|
||||
<div class="type-breakdown" id="typeBreakdown" style="display:none;">
|
||||
<h3>Message Types</h3>
|
||||
<div id="typeRows"></div>
|
||||
</div>
|
||||
|
||||
<div class="logs-panel">
|
||||
<h3>
|
||||
<span>Logs</span>
|
||||
<span id="logCount" style="color:#555;">(0)</span>
|
||||
<div class="level-filter">
|
||||
<button class="level-btn error active" data-level="ERROR" onclick="toggleLevel(this)">ERR</button>
|
||||
<button class="level-btn warn active" data-level="WARN" onclick="toggleLevel(this)">WARN</button>
|
||||
<button class="level-btn info active" data-level="INFO" onclick="toggleLevel(this)">INFO</button>
|
||||
<button class="level-btn debug active" data-level="DEBUG" onclick="toggleLevel(this)">DBG</button>
|
||||
<button class="level-btn trace active" data-level="TRACE" onclick="toggleLevel(this)">TRC</button>
|
||||
</div>
|
||||
</h3>
|
||||
<div class="log-list" id="logList"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(function() {
|
||||
var targetAddr = '__ACTOR_ADDR__';
|
||||
var dot = document.getElementById('statusDot');
|
||||
|
||||
var history = { rates: [], mailbox: [], prev_msgs: 0 };
|
||||
|
||||
function formatAddr(addr) {
|
||||
if (!addr) return '';
|
||||
var bytes = Array.isArray(addr) ? addr : Object.values(addr);
|
||||
var hex = '';
|
||||
for (var i = 0; i < bytes.length; i++) {
|
||||
hex += ('0' + bytes[i].toString(16)).slice(-2);
|
||||
}
|
||||
return hex;
|
||||
}
|
||||
|
||||
function shortAddr(hex) {
|
||||
return hex.length > 16 ? hex.substring(0, 16) + '\u2026' : hex;
|
||||
}
|
||||
|
||||
function shortTypeName(full) {
|
||||
if (!full) return '\u2014';
|
||||
var parts = full.split('::');
|
||||
return parts[parts.length - 1];
|
||||
}
|
||||
|
||||
function updateSparklineSvg(svgEl, data, color) {
|
||||
if (!data || data.length < 2) { svgEl.innerHTML = ''; return; }
|
||||
var max = Math.max.apply(null, data);
|
||||
if (max === 0) max = 1;
|
||||
var w = 400, h = 50;
|
||||
var step = w / (data.length - 1);
|
||||
var points = data.map(function(v, i) {
|
||||
return (i * step).toFixed(1) + ',' + (h - (v / max) * (h - 4) - 2).toFixed(1);
|
||||
}).join(' ');
|
||||
svgEl.innerHTML = '<polyline fill="none" stroke="' + color + '" stroke-width="2" points="' + points + '"/>';
|
||||
}
|
||||
|
||||
function findActor(stats) {
|
||||
if (!stats.actor_details) return null;
|
||||
for (var i = 0; i < stats.actor_details.length; i++) {
|
||||
var a = stats.actor_details[i];
|
||||
var hex = formatAddr(a.address);
|
||||
if (hex === targetAddr || hex.indexOf(targetAddr) === 0) return a;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function updateDetail(stats) {
|
||||
var actor = findActor(stats);
|
||||
if (!actor) return;
|
||||
|
||||
var hex = formatAddr(actor.address);
|
||||
document.getElementById('addrBreadcrumb').textContent = shortAddr(hex);
|
||||
document.getElementById('addrFull').textContent = hex;
|
||||
document.getElementById('addrWorker').textContent = 'W' + actor.worker_id;
|
||||
|
||||
var statusEl = document.getElementById('addrStatus');
|
||||
if (actor.poisoned) {
|
||||
statusEl.textContent = 'POISONED';
|
||||
statusEl.className = 'info-value poisoned';
|
||||
} else {
|
||||
statusEl.textContent = 'Healthy';
|
||||
statusEl.className = 'info-value healthy';
|
||||
}
|
||||
|
||||
document.getElementById('addrLastMsg').textContent = shortTypeName(actor.last_msg_type);
|
||||
document.getElementById('addrMsgs').textContent = actor.messages_processed.toLocaleString();
|
||||
document.getElementById('addrMailbox').textContent = actor.mailbox_depth;
|
||||
|
||||
// Compute rate
|
||||
var rate = actor.messages_processed - history.prev_msgs;
|
||||
if (rate < 0) rate = 0;
|
||||
history.prev_msgs = actor.messages_processed;
|
||||
history.rates.push(rate);
|
||||
history.mailbox.push(actor.mailbox_depth);
|
||||
if (history.rates.length > 300) { history.rates.shift(); history.mailbox.shift(); }
|
||||
|
||||
// Rate per second (SSE interval is ~200ms, so multiply by 5)
|
||||
document.getElementById('addrRate').textContent = (rate * 5).toLocaleString();
|
||||
|
||||
// Worker load
|
||||
if (stats.workers) {
|
||||
var w = stats.workers.find(function(w) { return w.id === actor.worker_id; });
|
||||
if (w) {
|
||||
document.getElementById('addrWorkerLoad').textContent =
|
||||
w.num_actors + ' actors, mbox ' + w.mailbox_depth;
|
||||
}
|
||||
}
|
||||
|
||||
updateSparklineSvg(document.getElementById('rateSpark'), history.rates, '#4caf50');
|
||||
updateSparklineSvg(document.getElementById('mboxSpark'), history.mailbox, '#2196f3');
|
||||
|
||||
// Update message type breakdown
|
||||
updateTypeBreakdown(actor.message_type_counts);
|
||||
}
|
||||
|
||||
var typeColors = ['#4caf50','#2196f3','#ff9800','#9c27b0','#00bcd4','#f44336','#ffeb3b','#e91e63'];
|
||||
|
||||
function updateTypeBreakdown(types) {
|
||||
var panel = document.getElementById('typeBreakdown');
|
||||
var container = document.getElementById('typeRows');
|
||||
if (!types || types.length === 0) { panel.style.display = 'none'; return; }
|
||||
panel.style.display = '';
|
||||
var total = 0;
|
||||
for (var i = 0; i < types.length; i++) total += types[i][1];
|
||||
if (total === 0) { panel.style.display = 'none'; return; }
|
||||
|
||||
var html = '';
|
||||
for (var i = 0; i < types.length; i++) {
|
||||
var name = types[i][0];
|
||||
var count = types[i][1];
|
||||
var pct = (count / total * 100).toFixed(1);
|
||||
var barPct = (count / types[0][1] * 100).toFixed(1);
|
||||
var color = typeColors[i % typeColors.length];
|
||||
var shortName = name.split('::').pop();
|
||||
html += '<div class="type-row">' +
|
||||
'<span class="type-name" title="' + escapeHtml(name) + '">' + escapeHtml(shortName) + '</span>' +
|
||||
'<div class="type-bar-bg"><div class="type-bar-fill" style="width:' + barPct + '%;background:' + color + ';"></div></div>' +
|
||||
'<span class="type-count">' + count.toLocaleString() + '</span>' +
|
||||
'<span class="type-pct">' + pct + '%</span>' +
|
||||
'</div>';
|
||||
}
|
||||
container.innerHTML = html;
|
||||
}
|
||||
|
||||
// ─── Logging ─────────────────────────────────────────────────
|
||||
var activeLevels = { ERROR: true, WARN: true, INFO: true, DEBUG: true, TRACE: true };
|
||||
var allLogs = [];
|
||||
var logList = document.getElementById('logList');
|
||||
var logCount = document.getElementById('logCount');
|
||||
var autoScroll = true;
|
||||
|
||||
window.toggleLevel = function(btn) {
|
||||
var lvl = btn.getAttribute('data-level');
|
||||
activeLevels[lvl] = !activeLevels[lvl];
|
||||
btn.classList.toggle('active');
|
||||
renderLogs();
|
||||
};
|
||||
|
||||
function formatLogTime(ms) {
|
||||
var d = new Date(ms);
|
||||
return ('0' + d.getHours()).slice(-2) + ':' +
|
||||
('0' + d.getMinutes()).slice(-2) + ':' +
|
||||
('0' + d.getSeconds()).slice(-2) + '.' +
|
||||
('00' + d.getMilliseconds()).slice(-3);
|
||||
}
|
||||
|
||||
function renderLogs() {
|
||||
var visible = allLogs.filter(function(e) { return activeLevels[e.level]; });
|
||||
logCount.textContent = '(' + visible.length + ')';
|
||||
var html = '';
|
||||
for (var i = 0; i < visible.length; i++) {
|
||||
var e = visible[i];
|
||||
html += '<div class="log-entry">' +
|
||||
'<span class="log-time">' + formatLogTime(e.timestamp_ms) + '</span>' +
|
||||
'<span class="log-level ' + e.level + '">' + e.level + '</span>' +
|
||||
'<span class="log-msg">' + escapeHtml(e.message) + '</span>' +
|
||||
'</div>';
|
||||
}
|
||||
logList.innerHTML = html;
|
||||
if (autoScroll) {
|
||||
logList.scrollTop = logList.scrollHeight;
|
||||
}
|
||||
}
|
||||
|
||||
function escapeHtml(s) {
|
||||
if (!s) return '';
|
||||
return s.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
|
||||
}
|
||||
|
||||
function addLogEntries(events) {
|
||||
for (var i = 0; i < events.length; i++) {
|
||||
var e = events[i];
|
||||
if (!e.actor_addr) continue;
|
||||
// Match if event's actor_addr starts with or contains target
|
||||
var a = e.actor_addr.toLowerCase();
|
||||
if (a.indexOf(targetAddr.toLowerCase()) === 0 || a === targetAddr.toLowerCase()) {
|
||||
allLogs.push(e);
|
||||
}
|
||||
}
|
||||
// Keep bounded
|
||||
while (allLogs.length > 500) allLogs.shift();
|
||||
renderLogs();
|
||||
}
|
||||
|
||||
// Fetch initial logs
|
||||
fetch('/api/logs?actor=' + targetAddr + '&limit=200')
|
||||
.then(function(r) { return r.json(); })
|
||||
.then(function(data) { if (Array.isArray(data)) addLogEntries(data); })
|
||||
.catch(function() {});
|
||||
|
||||
logList.addEventListener('scroll', function() {
|
||||
autoScroll = (logList.scrollTop + logList.clientHeight >= logList.scrollHeight - 20);
|
||||
});
|
||||
|
||||
// ─── SSE ────────────────────────────────────────────────────
|
||||
var es = new EventSource('/events');
|
||||
|
||||
es.addEventListener('stats', function(e) {
|
||||
try { updateDetail(JSON.parse(e.data)); } catch(err) { console.error(err); }
|
||||
});
|
||||
|
||||
es.addEventListener('activity', function(e) {
|
||||
try { addLogEntries(JSON.parse(e.data)); } catch(err) { console.error(err); }
|
||||
});
|
||||
|
||||
es.addEventListener('done', function() {
|
||||
dot.className = 'status-dot disconnected';
|
||||
es.close();
|
||||
});
|
||||
|
||||
es.onerror = function() { dot.className = 'status-dot disconnected'; };
|
||||
es.onopen = function() { dot.className = 'status-dot'; };
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
"##;
|
||||
|
|
@ -60,7 +60,7 @@ pub const ACTORS_HTML: &str = r##"<!DOCTYPE html>
|
|||
|
||||
.panel {
|
||||
background: #161822; border: 1px solid #2a2d3e; border-radius: 6px;
|
||||
padding: 14px; overflow: hidden;
|
||||
padding: 14px; overflow: visible;
|
||||
}
|
||||
.panel h2 { font-size: 12px; color: #888; text-transform: uppercase; letter-spacing: 1px; margin-bottom: 10px; }
|
||||
|
||||
|
|
@ -75,7 +75,7 @@ pub const ACTORS_HTML: &str = r##"<!DOCTYPE html>
|
|||
.stat-card .value { font-size: 22px; font-weight: 700; color: #fff; }
|
||||
.stat-card .label { font-size: 10px; color: #888; text-transform: uppercase; margin-top: 2px; }
|
||||
|
||||
canvas { width: 100%; height: 200px; }
|
||||
canvas { width: 100%; height: 220px; }
|
||||
|
||||
.search-wrap { margin-bottom: 10px; display: flex; align-items: center; gap: 12px; }
|
||||
.search-input {
|
||||
|
|
@ -109,7 +109,12 @@ pub const ACTORS_HTML: &str = r##"<!DOCTYPE html>
|
|||
tr.clickable { cursor: pointer; }
|
||||
tr.clickable:hover { background: #1a1d2c; }
|
||||
|
||||
.detail-panel { display: none; }
|
||||
.detail-panel {
|
||||
display: none; position: fixed; bottom: 12px; right: 12px;
|
||||
width: 420px; max-height: 320px; overflow-y: auto;
|
||||
background: #161822; border: 1px solid #2a2d3e; border-radius: 6px;
|
||||
padding: 14px; z-index: 100; box-shadow: 0 4px 24px rgba(0,0,0,0.5);
|
||||
}
|
||||
.detail-panel.visible { display: block; }
|
||||
.detail-header { display: flex; justify-content: space-between; align-items: center; margin-bottom: 10px; }
|
||||
.detail-close {
|
||||
|
|
@ -118,17 +123,17 @@ pub const ACTORS_HTML: &str = r##"<!DOCTYPE html>
|
|||
}
|
||||
.detail-close:hover { color: #e0e0e0; border-color: #555; }
|
||||
.detail-grid {
|
||||
display: grid; grid-template-columns: repeat(3, 1fr); gap: 10px; margin-bottom: 12px;
|
||||
display: grid; grid-template-columns: repeat(2, 1fr); gap: 8px; margin-bottom: 10px;
|
||||
}
|
||||
.detail-item { background: #1c1f2e; border-radius: 4px; padding: 8px 10px; }
|
||||
.detail-item { background: #1c1f2e; border-radius: 4px; padding: 6px 8px; }
|
||||
.detail-item .d-label { font-size: 10px; color: #888; text-transform: uppercase; }
|
||||
.detail-item .d-value { font-size: 14px; font-weight: 700; color: #fff; margin-top: 2px; word-break: break-all; }
|
||||
.detail-item .d-value { font-size: 13px; font-weight: 700; color: #fff; margin-top: 2px; word-break: break-all; }
|
||||
.poisoned-badge {
|
||||
background: #f44336; color: #fff; font-size: 10px; font-weight: 700;
|
||||
padding: 2px 6px; border-radius: 3px; letter-spacing: 0.5px;
|
||||
}
|
||||
.healthy-badge { color: #4caf50; font-size: 12px; }
|
||||
canvas.sparkline { width: 100%; height: 60px; }
|
||||
canvas.sparkline { width: 100%; height: 50px; }
|
||||
|
||||
::-webkit-scrollbar { width: 6px; }
|
||||
::-webkit-scrollbar-track { background: #0f1117; }
|
||||
|
|
@ -187,7 +192,16 @@ pub const ACTORS_HTML: &str = r##"<!DOCTYPE html>
|
|||
<div class="panel full-width">
|
||||
<h2>All Actors <span id="actorCount" style="color:#555;font-weight:400;"></span></h2>
|
||||
<div class="search-wrap">
|
||||
<input type="text" id="actorSearch" class="search-input" placeholder="Filter by address or worker..." />
|
||||
<input type="text" id="actorSearch" class="search-input" placeholder="Filter by address, type, or worker..." />
|
||||
<select id="workerFilter" class="search-input" style="width:120px;">
|
||||
<option value="">All Workers</option>
|
||||
</select>
|
||||
<select id="statusFilter" class="search-input" style="width:120px;">
|
||||
<option value="">All Status</option>
|
||||
<option value="healthy">Healthy</option>
|
||||
<option value="poisoned">Poisoned</option>
|
||||
</select>
|
||||
<input type="number" id="minDepth" class="search-input" style="width:100px;" placeholder="Min depth" min="0" />
|
||||
<span id="searchInfo" class="search-info"></span>
|
||||
</div>
|
||||
<div class="actor-list-wrap">
|
||||
|
|
@ -207,23 +221,25 @@ pub const ACTORS_HTML: &str = r##"<!DOCTYPE html>
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Actor detail panel -->
|
||||
<div id="detailPanel" class="panel full-width detail-panel">
|
||||
<div class="detail-header">
|
||||
<h2>Actor Detail <span id="detailAddr" style="color:#aaa;font-weight:400;"></span></h2>
|
||||
<button class="detail-close" id="detailClose">Close</button>
|
||||
</div>
|
||||
<div class="detail-grid">
|
||||
<div class="detail-item"><div class="d-label">Full Address</div><div class="d-value" id="detailFullAddr" style="font-size:11px;"></div></div>
|
||||
<div class="detail-item"><div class="d-label">Worker</div><div class="d-value" id="detailWorker"></div></div>
|
||||
<div class="detail-item"><div class="d-label">Mailbox Depth</div><div class="d-value" id="detailMailbox"></div></div>
|
||||
<div class="detail-item"><div class="d-label">Messages Processed</div><div class="d-value" id="detailMsgCount"></div></div>
|
||||
<div class="detail-item"><div class="d-label">Last Message Type</div><div class="d-value" id="detailLastMsg"></div></div>
|
||||
<div class="detail-item"><div class="d-label">Status</div><div class="d-value" id="detailStatus"></div></div>
|
||||
</div>
|
||||
<h2>Mailbox Depth History</h2>
|
||||
<canvas id="sparkline" class="sparkline"></canvas>
|
||||
|
||||
</div>
|
||||
|
||||
<!-- Actor detail overlay (fixed position, doesn't affect grid layout) -->
|
||||
<div id="detailPanel" class="detail-panel">
|
||||
<div class="detail-header">
|
||||
<h2 style="font-size:12px;color:#888;text-transform:uppercase;letter-spacing:1px;">Actor <span id="detailAddr" style="color:#aaa;font-weight:400;"></span></h2>
|
||||
<button class="detail-close" id="detailClose">×</button>
|
||||
</div>
|
||||
<div class="detail-grid">
|
||||
<div class="detail-item"><div class="d-label">Address</div><div class="d-value" id="detailFullAddr" style="font-size:10px;"></div></div>
|
||||
<div class="detail-item"><div class="d-label">Worker</div><div class="d-value" id="detailWorker"></div></div>
|
||||
<div class="detail-item"><div class="d-label">Mailbox</div><div class="d-value" id="detailMailbox"></div></div>
|
||||
<div class="detail-item"><div class="d-label">Messages</div><div class="d-value" id="detailMsgCount"></div></div>
|
||||
<div class="detail-item"><div class="d-label">Last Type</div><div class="d-value" id="detailLastMsg"></div></div>
|
||||
<div class="detail-item"><div class="d-label">Status</div><div class="d-value" id="detailStatus"></div></div>
|
||||
</div>
|
||||
<div style="font-size:10px;color:#888;text-transform:uppercase;letter-spacing:1px;margin-bottom:4px;">Mailbox History</div>
|
||||
<canvas id="sparkline" class="sparkline"></canvas>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
|
|
@ -374,11 +390,12 @@ pub const ACTORS_HTML: &str = r##"<!DOCTYPE html>
|
|||
|
||||
var maxCount = Math.max(1, Math.max.apply(null, counts));
|
||||
var barW = Math.max(12, Math.floor((W - 40) / buckets.length) - 8);
|
||||
var topPad = 20;
|
||||
var chartH = H - 35;
|
||||
|
||||
for (var i = 0; i < buckets.length; i++) {
|
||||
var x = 20 + i * (barW + 8);
|
||||
var h = (counts[i] / maxCount) * (chartH - 10);
|
||||
var h = (counts[i] / maxCount) * (chartH - topPad);
|
||||
depthCtx.fillStyle = bucketColors[i];
|
||||
depthCtx.globalAlpha = 0.85;
|
||||
depthCtx.fillRect(x, chartH - h, barW, h);
|
||||
|
|
@ -424,11 +441,12 @@ pub const ACTORS_HTML: &str = r##"<!DOCTYPE html>
|
|||
|
||||
var maxCount = Math.max(1, Math.max.apply(null, entries.map(function(e) { return e.count; })));
|
||||
var barW = Math.max(12, Math.floor((W - 40) / entries.length) - 8);
|
||||
var topPad = 20;
|
||||
var chartH = H - 35;
|
||||
|
||||
for (var i = 0; i < entries.length; i++) {
|
||||
var x = 20 + i * (barW + 8);
|
||||
var h = (entries[i].count / maxCount) * (chartH - 10);
|
||||
var h = (entries[i].count / maxCount) * (chartH - topPad);
|
||||
workerCtx.fillStyle = colors[entries[i].id % colors.length];
|
||||
workerCtx.globalAlpha = 0.85;
|
||||
workerCtx.fillRect(x, chartH - h, barW, h);
|
||||
|
|
@ -449,13 +467,29 @@ pub const ACTORS_HTML: &str = r##"<!DOCTYPE html>
|
|||
|
||||
function renderActorTable() {
|
||||
var filter = document.getElementById('actorSearch').value.toLowerCase();
|
||||
var filtered = currentActors;
|
||||
if (filter) {
|
||||
filtered = currentActors.filter(function(a) {
|
||||
var workerFilter = document.getElementById('workerFilter').value;
|
||||
var statusFilter = document.getElementById('statusFilter').value;
|
||||
var minDepthVal = document.getElementById('minDepth').value;
|
||||
var minDepth = minDepthVal ? parseInt(minDepthVal, 10) : 0;
|
||||
|
||||
var filtered = currentActors.filter(function(a) {
|
||||
// Text search
|
||||
if (filter) {
|
||||
var hex = addrToHex(a.address).toLowerCase();
|
||||
return hex.indexOf(filter) >= 0 || ('w' + a.worker_id).indexOf(filter) >= 0;
|
||||
});
|
||||
}
|
||||
var msgType = (a.last_msg_type || '').toLowerCase();
|
||||
if (hex.indexOf(filter) < 0 && ('w' + a.worker_id).indexOf(filter) < 0 && msgType.indexOf(filter) < 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Worker filter
|
||||
if (workerFilter && a.worker_id !== parseInt(workerFilter, 10)) return false;
|
||||
// Status filter
|
||||
if (statusFilter === 'healthy' && a.poisoned) return false;
|
||||
if (statusFilter === 'poisoned' && !a.poisoned) return false;
|
||||
// Min depth
|
||||
if (minDepth > 0 && a.mailbox_depth < minDepth) return false;
|
||||
return true;
|
||||
});
|
||||
|
||||
// Sort
|
||||
filtered.sort(function(a, b) {
|
||||
|
|
@ -531,14 +565,14 @@ pub const ACTORS_HTML: &str = r##"<!DOCTYPE html>
|
|||
// ── Focus / detail panel ───────────────────────────────
|
||||
function focusActor(fullHex) {
|
||||
focusedAddrHex = fullHex;
|
||||
document.getElementById('detailPanel').className = 'panel full-width detail-panel visible';
|
||||
document.getElementById('detailPanel').className = 'detail-panel visible';
|
||||
updateDetailPanel();
|
||||
renderActorTable();
|
||||
}
|
||||
|
||||
function clearFocus() {
|
||||
focusedAddrHex = null;
|
||||
document.getElementById('detailPanel').className = 'panel full-width detail-panel';
|
||||
document.getElementById('detailPanel').className = 'detail-panel';
|
||||
renderActorTable();
|
||||
}
|
||||
|
||||
|
|
@ -558,8 +592,8 @@ pub const ACTORS_HTML: &str = r##"<!DOCTYPE html>
|
|||
return;
|
||||
}
|
||||
|
||||
document.getElementById('detailAddr').textContent = addrToHex(actor.address);
|
||||
document.getElementById('detailFullAddr').textContent = focusedAddrHex;
|
||||
document.getElementById('detailAddr').innerHTML = '<a href="/actor/' + focusedAddrHex + '" style="color:#aaa;text-decoration:none;">' + escapeHtml(addrToHex(actor.address)) + '</a>';
|
||||
document.getElementById('detailFullAddr').innerHTML = '<a href="/actor/' + focusedAddrHex + '" style="color:#fff;text-decoration:none;">' + escapeHtml(focusedAddrHex) + '</a>';
|
||||
document.getElementById('detailWorker').textContent = 'W' + actor.worker_id;
|
||||
document.getElementById('detailMailbox').textContent = actor.mailbox_depth;
|
||||
document.getElementById('detailMsgCount').textContent = (actor.messages_processed || 0).toLocaleString();
|
||||
|
|
@ -647,11 +681,17 @@ pub const ACTORS_HTML: &str = r##"<!DOCTYPE html>
|
|||
});
|
||||
}
|
||||
|
||||
// ── Search handler ─────────────────────────────────────
|
||||
// ── Search/filter handlers ─────────────────────────────
|
||||
document.getElementById('actorSearch').addEventListener('keyup', function() {
|
||||
clearTimeout(searchTimer);
|
||||
searchTimer = setTimeout(renderActorTable, 150);
|
||||
});
|
||||
document.getElementById('workerFilter').addEventListener('change', renderActorTable);
|
||||
document.getElementById('statusFilter').addEventListener('change', renderActorTable);
|
||||
document.getElementById('minDepth').addEventListener('input', function() {
|
||||
clearTimeout(searchTimer);
|
||||
searchTimer = setTimeout(renderActorTable, 150);
|
||||
});
|
||||
|
||||
// ── Status helpers ─────────────────────────────────────
|
||||
function setStatus(s) {
|
||||
|
|
@ -697,6 +737,21 @@ pub const ACTORS_HTML: &str = r##"<!DOCTYPE html>
|
|||
if (!liveAddrs[key]) delete depthHistory[key];
|
||||
}
|
||||
|
||||
// Update worker filter dropdown
|
||||
var wSelect = document.getElementById('workerFilter');
|
||||
var curVal = wSelect.value;
|
||||
var workerIds = {};
|
||||
for (var i = 0; i < currentActors.length; i++) workerIds[currentActors[i].worker_id] = true;
|
||||
var wids = Object.keys(workerIds).sort(function(a,b) { return +a - +b; });
|
||||
wSelect.innerHTML = '<option value="">All Workers</option>';
|
||||
wids.forEach(function(wid) {
|
||||
var opt = document.createElement('option');
|
||||
opt.value = wid;
|
||||
opt.textContent = 'W' + wid;
|
||||
wSelect.appendChild(opt);
|
||||
});
|
||||
wSelect.value = curVal;
|
||||
|
||||
renderActorTable();
|
||||
if (focusedAddrHex) updateDetailPanel();
|
||||
} catch(err) { console.error('stats parse error', err); }
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ impl StatsHook for StatsCollector {
|
|||
last_msg_type: s.last_msg_type.map(|t| t.to_string()),
|
||||
messages_processed: s.messages_processed,
|
||||
poisoned: s.poisoned,
|
||||
message_type_counts: s.message_type_counts.iter().map(|(k, v)| (k.to_string(), *v)).collect(),
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -77,6 +77,17 @@ pub const DASHBOARD_HTML: &str = r##"<!DOCTYPE html>
|
|||
.chart-panel { grid-row: span 2; }
|
||||
canvas#workerChart { width: 100%; height: 200px; }
|
||||
|
||||
.worker-cards { display: flex; flex-direction: column; gap: 6px; }
|
||||
.worker-card {
|
||||
display: flex; align-items: center; gap: 10px;
|
||||
background: #1c1f2e; border-radius: 4px; padding: 6px 10px;
|
||||
}
|
||||
.worker-card .wc-id { font-weight: 700; min-width: 32px; }
|
||||
.worker-card .wc-bar-wrap { flex: 1; height: 14px; background: #0f1117; border-radius: 2px; overflow: hidden; display: flex; }
|
||||
.worker-card .wc-bar-seg { height: 100%; }
|
||||
.worker-card .wc-stats { font-size: 11px; color: #888; min-width: 200px; text-align: right; }
|
||||
.worker-card .wc-spark { display: inline-flex; gap: 4px; margin-left: 6px; }
|
||||
|
||||
.actor-table-wrap { max-height: 200px; overflow-y: auto; }
|
||||
.actor-table-wrap table { width: 100%; border-collapse: collapse; }
|
||||
.actor-table-wrap th, .actor-table-wrap td {
|
||||
|
|
@ -113,6 +124,8 @@ pub const DASHBOARD_HTML: &str = r##"<!DOCTYPE html>
|
|||
.worker-group-header .wid { font-weight: 700; }
|
||||
.worker-group-header .summary { color: #888; font-size: 11px; }
|
||||
.worker-group-header .toggle { color: #555; font-size: 14px; }
|
||||
.worker-group-header .sparkline-wrap { display: inline-flex; gap: 8px; margin-left: 12px; }
|
||||
.worker-group-header .sparkline-wrap svg { vertical-align: middle; }
|
||||
.worker-group-body { display: none; }
|
||||
.worker-group.open .worker-group-body { display: block; }
|
||||
.worker-group-body table { width: 100%; border-collapse: collapse; }
|
||||
|
|
@ -152,11 +165,17 @@ pub const DASHBOARD_HTML: &str = r##"<!DOCTYPE html>
|
|||
<div id="progressFill" class="progress-fill"></div>
|
||||
</div>
|
||||
|
||||
<div id="warningBanner" style="display:none;padding:8px 20px;background:#1c1f2e;border-bottom:1px solid #2a2d3e;font-size:12px;"></div>
|
||||
<div class="grid">
|
||||
<div class="panel chart-panel">
|
||||
<h2>Worker Distribution</h2>
|
||||
<canvas id="workerChart"></canvas>
|
||||
<div id="workerLegend" style="margin-top:8px;font-size:11px;color:#888;"></div>
|
||||
<h2>Worker Utilization</h2>
|
||||
<div id="workerCards" class="worker-cards"></div>
|
||||
<div style="margin-top:6px;font-size:10px;color:#555;">
|
||||
<span style="color:#4caf50;">\u25A0</span> processing
|
||||
<span style="color:#2196f3;">\u25A0</span> delivery
|
||||
<span style="color:#00bcd4;">\u25A0</span> spawns
|
||||
<span style="color:#f44336;">\u25A0</span> overhead
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="panel">
|
||||
|
|
@ -203,6 +222,7 @@ pub const DASHBOARD_HTML: &str = r##"<!DOCTYPE html>
|
|||
var isReplay = (DASHBOARD_MODE === 'replay');
|
||||
var lastUptimeMs = null;
|
||||
var lastStatsTime = null;
|
||||
var workerHistory = {}; // { id: { message_rates: [], mailbox_depths: [] } }
|
||||
|
||||
var dot = document.getElementById('statusDot');
|
||||
var uptimeLabel = document.getElementById('uptimeLabel');
|
||||
|
|
@ -240,48 +260,83 @@ pub const DASHBOARD_HTML: &str = r##"<!DOCTYPE html>
|
|||
}
|
||||
setInterval(updateUptime, 1000);
|
||||
|
||||
var canvas = document.getElementById('workerChart');
|
||||
var ctx = canvas.getContext('2d');
|
||||
var colors = ['#4caf50','#2196f3','#ff9800','#f44336','#9c27b0','#00bcd4','#ffeb3b','#e91e63'];
|
||||
|
||||
function drawWorkerChart(workers) {
|
||||
var dpr = window.devicePixelRatio || 1;
|
||||
var rect = canvas.getBoundingClientRect();
|
||||
canvas.width = rect.width * dpr;
|
||||
canvas.height = rect.height * dpr;
|
||||
ctx.scale(dpr, dpr);
|
||||
var W = rect.width, H = rect.height;
|
||||
ctx.clearRect(0, 0, W, H);
|
||||
var phaseColors = ['#4caf50', '#2196f3', '#00bcd4', '#f44336'];
|
||||
// Group tick phases: processing=2, delivery=1+4, spawns=0+3, overhead=5
|
||||
|
||||
if (!workers || workers.length === 0) return;
|
||||
function computePhases(timings) {
|
||||
if (!timings || timings.length === 0) return [0.25, 0.25, 0.25, 0.25];
|
||||
var sums = [0,0,0,0,0,0];
|
||||
var active = 0;
|
||||
for (var i = 0; i < timings.length; i++) {
|
||||
var t = timings[i];
|
||||
if (t.did_work) active++;
|
||||
for (var p = 0; p < 6 && p < t.phase_us.length; p++) sums[p] += t.phase_us[p];
|
||||
}
|
||||
var total = sums.reduce(function(a,b) { return a+b; }, 0);
|
||||
if (total === 0) return [0.25, 0.25, 0.25, 0.25];
|
||||
var processing = sums[2] / total;
|
||||
var delivery = (sums[1] + sums[4]) / total;
|
||||
var spawns = (sums[0] + sums[3]) / total;
|
||||
var overhead = sums[5] / total;
|
||||
var load = timings.length > 0 ? active / timings.length : 0;
|
||||
return { fracs: [processing, delivery, spawns, overhead], load: load };
|
||||
}
|
||||
|
||||
var maxActors = Math.max(1, Math.max.apply(null, workers.map(function(w) { return w.num_actors; })));
|
||||
var barW = Math.max(8, Math.floor((W - 40) / workers.length) - 6);
|
||||
var chartH = H - 30;
|
||||
function renderWorkerCards(data) {
|
||||
var container = document.getElementById('workerCards');
|
||||
if (!data.workers) return;
|
||||
container.innerHTML = '';
|
||||
|
||||
workers.forEach(function(w, i) {
|
||||
var x = 20 + i * (barW + 6);
|
||||
var h = (w.num_actors / maxActors) * (chartH * 0.45);
|
||||
ctx.fillStyle = colors[i % colors.length];
|
||||
ctx.globalAlpha = 0.8;
|
||||
ctx.fillRect(x, chartH * 0.5 - h, barW, h);
|
||||
data.workers.forEach(function(w, idx) {
|
||||
var timings = data.tick_timings ? data.tick_timings[idx] : null;
|
||||
var phases = computePhases(timings);
|
||||
var load = phases.load || 0;
|
||||
var fracs = phases.fracs || [0.25, 0.25, 0.25, 0.25];
|
||||
|
||||
var mh = Math.min(w.mailbox_depth * 2, chartH * 0.4);
|
||||
ctx.globalAlpha = 0.4;
|
||||
ctx.fillRect(x, chartH * 0.55, barW, mh);
|
||||
var card = document.createElement('div');
|
||||
card.className = 'worker-card';
|
||||
|
||||
ctx.globalAlpha = 1;
|
||||
ctx.fillStyle = '#888';
|
||||
ctx.font = '10px monospace';
|
||||
ctx.textAlign = 'center';
|
||||
ctx.fillText('W' + w.id, x + barW / 2, H - 2);
|
||||
// ID
|
||||
var idSpan = document.createElement('span');
|
||||
idSpan.className = 'wc-id';
|
||||
idSpan.style.color = colors[w.id % colors.length];
|
||||
idSpan.textContent = 'W' + w.id;
|
||||
card.appendChild(idSpan);
|
||||
|
||||
// Phase bar
|
||||
var barWrap = document.createElement('span');
|
||||
barWrap.className = 'wc-bar-wrap';
|
||||
var filledPct = Math.round(load * 100);
|
||||
for (var p = 0; p < 4; p++) {
|
||||
var seg = document.createElement('span');
|
||||
seg.className = 'wc-bar-seg';
|
||||
seg.style.width = (fracs[p] * filledPct) + '%';
|
||||
seg.style.background = phaseColors[p];
|
||||
barWrap.appendChild(seg);
|
||||
}
|
||||
card.appendChild(barWrap);
|
||||
|
||||
// Sparklines
|
||||
var sparkWrap = document.createElement('span');
|
||||
sparkWrap.className = 'wc-spark';
|
||||
var wh = workerHistory[w.id];
|
||||
if (wh) {
|
||||
sparkWrap.innerHTML = renderSparklineSvg(wh.message_rates, 60, 14, '#4caf50');
|
||||
}
|
||||
card.appendChild(sparkWrap);
|
||||
|
||||
// Stats
|
||||
var statsSpan = document.createElement('span');
|
||||
statsSpan.className = 'wc-stats';
|
||||
statsSpan.textContent = w.num_actors + ' actors ' +
|
||||
w.messages_processed.toLocaleString() + ' msgs mbox ' + w.mailbox_depth +
|
||||
' ' + Math.round(load * 100) + '%';
|
||||
card.appendChild(statsSpan);
|
||||
|
||||
container.appendChild(card);
|
||||
});
|
||||
|
||||
var legend = document.getElementById('workerLegend');
|
||||
legend.innerHTML = workers.map(function(w, i) {
|
||||
return '<span style="color:' + colors[i % colors.length] + '">W' + w.id +
|
||||
': ' + w.num_actors + ' actors, ' + w.messages_processed + ' msgs, mbox ' + w.mailbox_depth + '</span>';
|
||||
}).join(' | ');
|
||||
}
|
||||
|
||||
function updateStats(data) {
|
||||
|
|
@ -298,7 +353,7 @@ pub const DASHBOARD_HTML: &str = r##"<!DOCTYPE html>
|
|||
document.getElementById('statWorkers').textContent = data.num_workers || 0;
|
||||
document.getElementById('statMailbox').textContent = totalMailbox;
|
||||
|
||||
drawWorkerChart(data.workers);
|
||||
renderWorkerCards(data);
|
||||
|
||||
var tbody = document.getElementById('actorTableBody');
|
||||
tbody.innerHTML = '';
|
||||
|
|
@ -316,7 +371,7 @@ pub const DASHBOARD_HTML: &str = r##"<!DOCTYPE html>
|
|||
hex += '\u2026';
|
||||
}
|
||||
var tr = document.createElement('tr');
|
||||
tr.innerHTML = '<td style="color:#aaa;font-size:11px;">' + hex + '</td><td>W' + wid + '</td>';
|
||||
tr.innerHTML = '<td style="color:#aaa;font-size:11px;"><a href="/actor/' + hex + '" style="color:#aaa;text-decoration:none;">' + hex + '</a></td><td>W' + wid + '</td>';
|
||||
tbody.appendChild(tr);
|
||||
});
|
||||
if (data.actors.length > 200) {
|
||||
|
|
@ -347,6 +402,34 @@ pub const DASHBOARD_HTML: &str = r##"<!DOCTYPE html>
|
|||
return parts[parts.length - 1];
|
||||
}
|
||||
|
||||
function renderSparklineSvg(data, w, h, color) {
|
||||
if (!data || data.length < 2) return '';
|
||||
var max = Math.max.apply(null, data);
|
||||
if (max === 0) max = 1;
|
||||
var step = w / (data.length - 1);
|
||||
var points = data.map(function(v, i) {
|
||||
return (i * step).toFixed(1) + ',' + (h - (v / max) * (h - 2) - 1).toFixed(1);
|
||||
}).join(' ');
|
||||
return '<svg width="' + w + '" height="' + h + '" style="vertical-align:middle">' +
|
||||
'<polyline fill="none" stroke="' + color + '" stroke-width="1.5" points="' + points + '"/></svg>';
|
||||
}
|
||||
|
||||
function pushHistorySample(stats) {
|
||||
if (!stats.workers) return;
|
||||
stats.workers.forEach(function(w) {
|
||||
if (!workerHistory[w.id]) {
|
||||
workerHistory[w.id] = { message_rates: [], mailbox_depths: [], prev_msgs: w.messages_processed };
|
||||
}
|
||||
var wh = workerHistory[w.id];
|
||||
var rate = w.messages_processed - wh.prev_msgs;
|
||||
if (rate < 0) rate = 0;
|
||||
wh.prev_msgs = w.messages_processed;
|
||||
wh.message_rates.push(rate);
|
||||
wh.mailbox_depths.push(w.mailbox_depth);
|
||||
if (wh.message_rates.length > 300) { wh.message_rates.shift(); wh.mailbox_depths.shift(); }
|
||||
});
|
||||
}
|
||||
|
||||
function updateWorkerDetails(data) {
|
||||
var container = document.getElementById('workerDetailContainer');
|
||||
if (!data.workers) return;
|
||||
|
|
@ -377,8 +460,17 @@ pub const DASHBOARD_HTML: &str = r##"<!DOCTYPE html>
|
|||
var hdr = document.createElement('div');
|
||||
hdr.className = 'worker-group-header';
|
||||
var panicHtml = g.info.panics > 0 ? ', <span style="color:#f44336">' + g.info.panics + ' panics</span>' : '';
|
||||
var wh = workerHistory[wid];
|
||||
var sparkHtml = '';
|
||||
if (wh) {
|
||||
sparkHtml = '<span class="sparkline-wrap">' +
|
||||
renderSparklineSvg(wh.message_rates, 80, 16, '#4caf50') +
|
||||
renderSparklineSvg(wh.mailbox_depths, 80, 16, '#2196f3') +
|
||||
'</span>';
|
||||
}
|
||||
hdr.innerHTML =
|
||||
'<span class="wid" style="color:' + colors[wid % colors.length] + '">W' + wid + '</span>' +
|
||||
sparkHtml +
|
||||
'<span class="summary">' + g.actors.length + ' actors, ' +
|
||||
g.info.messages_processed.toLocaleString() + ' msgs, mbox ' + g.info.mailbox_depth + panicHtml + '</span>' +
|
||||
'<span class="toggle">' + (isOpen ? '\u25BC' : '\u25B6') + '</span>';
|
||||
|
|
@ -400,7 +492,7 @@ pub const DASHBOARD_HTML: &str = r##"<!DOCTYPE html>
|
|||
var msgShort = hasMsg ? shortTypeName(a.last_msg_type) : 'none';
|
||||
var msgClass = hasMsg ? 'msg-type' : 'msg-type none';
|
||||
var title = hasMsg ? ' title="' + escapeHtml(a.last_msg_type) + '"' : '';
|
||||
rows += '<tr><td style="color:#aaa;">' + hex + '</td>' +
|
||||
rows += '<tr><td style="color:#aaa;"><a href="/actor/' + hex + '" style="color:#aaa;text-decoration:none;">' + hex + '</a></td>' +
|
||||
'<td>' + a.mailbox_depth + '</td>' +
|
||||
'<td class="' + msgClass + '"' + title + '>' + escapeHtml(msgShort) + '</td></tr>';
|
||||
});
|
||||
|
|
@ -467,7 +559,44 @@ pub const DASHBOARD_HTML: &str = r##"<!DOCTYPE html>
|
|||
var es = new EventSource('/events');
|
||||
|
||||
es.addEventListener('stats', function(e) {
|
||||
try { updateStats(JSON.parse(e.data)); } catch(err) { console.error('stats parse error', err); }
|
||||
try {
|
||||
var data = JSON.parse(e.data);
|
||||
pushHistorySample(data);
|
||||
updateStats(data);
|
||||
} catch(err) { console.error('stats parse error', err); }
|
||||
});
|
||||
|
||||
es.addEventListener('history', function(e) {
|
||||
try {
|
||||
var data = JSON.parse(e.data);
|
||||
if (data.workers) {
|
||||
data.workers.forEach(function(w) {
|
||||
workerHistory[w.id] = {
|
||||
message_rates: w.message_rates || [],
|
||||
mailbox_depths: w.mailbox_depths || [],
|
||||
prev_msgs: 0
|
||||
};
|
||||
});
|
||||
}
|
||||
} catch(err) { console.error('history parse error', err); }
|
||||
});
|
||||
|
||||
es.addEventListener('warnings', function(e) {
|
||||
try {
|
||||
var warnings = JSON.parse(e.data);
|
||||
var banner = document.getElementById('warningBanner');
|
||||
if (warnings.length === 0) {
|
||||
banner.style.display = 'none';
|
||||
return;
|
||||
}
|
||||
banner.style.display = 'block';
|
||||
var sevColors = {critical:'#f44336',high:'#ff5722',medium:'#ff9800',low:'#888'};
|
||||
var html = warnings.map(function(w) {
|
||||
var c = sevColors[w.severity] || '#888';
|
||||
return '<span style="color:' + c + ';">\u26A0 ' + w.description + '</span>';
|
||||
}).join(' ');
|
||||
banner.innerHTML = '<span style="color:#ff9800;font-weight:700;">WARNINGS (' + warnings.length + ')</span> ' + html;
|
||||
} catch(err) { console.error('warnings parse error', err); }
|
||||
});
|
||||
|
||||
es.addEventListener('activity', function(e) {
|
||||
|
|
|
|||
|
|
@ -40,9 +40,9 @@ pub const DISTRIBUTION_HTML: &str = r##"<!DOCTYPE html>
|
|||
|
||||
.graph-panel {
|
||||
grid-row: 1 / 3; border-right: 1px solid #2a2d3e; position: relative;
|
||||
min-height: 0;
|
||||
min-height: 0; overflow: hidden;
|
||||
}
|
||||
.graph-panel canvas { width: 100%; height: 100%; display: block; }
|
||||
.graph-panel canvas { position: absolute; top: 0; left: 0; width: 100%; height: 100%; display: block; }
|
||||
|
||||
.side-panel { display: flex; flex-direction: column; overflow: hidden; min-height: 0; }
|
||||
|
||||
|
|
@ -176,8 +176,17 @@ pub const DISTRIBUTION_HTML: &str = r##"<!DOCTYPE html>
|
|||
</div>
|
||||
</div>
|
||||
<div class="bottom-section">
|
||||
<h2>Recent Probes</h2>
|
||||
<div class="scroll-wrap" id="probesWrap"></div>
|
||||
<h2>Recent Probes <span id="probeCount" style="color:#555;font-weight:400;"></span></h2>
|
||||
<div class="scroll-wrap">
|
||||
<table style="width:100%;border-collapse:collapse;">
|
||||
<thead><tr>
|
||||
<th style="color:#888;font-weight:500;font-size:11px;">State</th>
|
||||
<th style="color:#888;font-weight:500;font-size:11px;">Node</th>
|
||||
<th style="color:#888;font-weight:500;font-size:11px;">Address</th>
|
||||
</tr></thead>
|
||||
<tbody id="probesBody"></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
<div class="bottom-section">
|
||||
<h2>Routing Buckets</h2>
|
||||
|
|
@ -653,14 +662,26 @@ pub const DISTRIBUTION_HTML: &str = r##"<!DOCTYPE html>
|
|||
cacheBody.appendChild(tr);
|
||||
}
|
||||
|
||||
// Recent probes
|
||||
var probesWrap = document.getElementById('probesWrap');
|
||||
probesWrap.innerHTML = '';
|
||||
// Recent probes — cross-reference with members for state + address
|
||||
var memberMap = {};
|
||||
for (var i = 0; i < d.members.length; i++) {
|
||||
memberMap[d.members[i].node_id] = d.members[i];
|
||||
}
|
||||
var probesBody = document.getElementById('probesBody');
|
||||
probesBody.innerHTML = '';
|
||||
document.getElementById('probeCount').textContent = '(' + d.recent_probe_targets.length + ')';
|
||||
for (var i = d.recent_probe_targets.length - 1; i >= 0; i--) {
|
||||
var div = document.createElement('div');
|
||||
div.style.cssText = 'padding:2px 0;color:#aaa;font-size:11px;border-bottom:1px solid #1c1f2e;';
|
||||
div.textContent = d.recent_probe_targets[i].substring(0, 16) + '\u2026';
|
||||
probesWrap.appendChild(div);
|
||||
var pid = d.recent_probe_targets[i];
|
||||
var mem = memberMap[pid];
|
||||
var state = mem ? mem.state : 'unknown';
|
||||
var addr = mem ? mem.addr : '\u2014';
|
||||
var cls = 'state-' + state;
|
||||
var tr = document.createElement('tr');
|
||||
tr.innerHTML =
|
||||
'<td class="' + cls + '" style="font-size:10px;">' + state + '</td>' +
|
||||
'<td style="color:#aaa;font-size:10px;">' + pid.substring(0, 12) + '\u2026</td>' +
|
||||
'<td style="font-size:10px;">' + addr + '</td>';
|
||||
probesBody.appendChild(tr);
|
||||
}
|
||||
|
||||
// Routing bucket histogram
|
||||
|
|
|
|||
344
crates/runtime-dashboard/src/history.rs
Normal file
344
crates/runtime-dashboard/src/history.rs
Normal file
|
|
@ -0,0 +1,344 @@
|
|||
//! In-process time-series history for dashboard sparklines and trend detection.
|
||||
//!
|
||||
//! Stores bounded ring buffers of per-worker and per-actor stats, sampled at
|
||||
//! a configurable interval. All data is kept in memory with automatic eviction
|
||||
//! of the oldest samples when capacity is reached.
|
||||
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::sync::RwLock;
|
||||
|
||||
use swactor::actor::ActorAddress;
|
||||
use swactor::stats::{ActorInfo, RuntimeStats};
|
||||
|
||||
/// Configuration for history collection.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HistoryConfig {
|
||||
/// Maximum samples per worker (default: 300 = 5 min at 1/sec).
|
||||
pub max_worker_samples: usize,
|
||||
/// Maximum samples per actor (default: 300).
|
||||
pub max_actor_samples: usize,
|
||||
/// Maximum number of actors tracked (LRU eviction). Default: 1000.
|
||||
pub max_tracked_actors: usize,
|
||||
}
|
||||
|
||||
impl Default for HistoryConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_worker_samples: 300,
|
||||
max_actor_samples: 300,
|
||||
max_tracked_actors: 1000,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Time-series data for a single worker.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct WorkerHistory {
|
||||
pub message_rates: VecDeque<f64>,
|
||||
pub mailbox_depths: VecDeque<u64>,
|
||||
pub actor_counts: VecDeque<u32>,
|
||||
prev_messages: u64,
|
||||
}
|
||||
|
||||
impl WorkerHistory {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
message_rates: VecDeque::new(),
|
||||
mailbox_depths: VecDeque::new(),
|
||||
actor_counts: VecDeque::new(),
|
||||
prev_messages: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn push(&mut self, messages_processed: u64, mailbox_depth: usize, num_actors: usize, cap: usize) {
|
||||
let rate = messages_processed.saturating_sub(self.prev_messages) as f64;
|
||||
self.prev_messages = messages_processed;
|
||||
|
||||
push_bounded(&mut self.message_rates, rate, cap);
|
||||
push_bounded(&mut self.mailbox_depths, mailbox_depth as u64, cap);
|
||||
push_bounded(&mut self.actor_counts, num_actors as u32, cap);
|
||||
}
|
||||
}
|
||||
|
||||
/// Time-series data for a single actor.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ActorHistory {
|
||||
pub mailbox_depths: VecDeque<u64>,
|
||||
pub message_rates: VecDeque<f64>,
|
||||
prev_messages: u64,
|
||||
last_seen_sample: u64,
|
||||
}
|
||||
|
||||
impl ActorHistory {
|
||||
fn new(sample_counter: u64) -> Self {
|
||||
Self {
|
||||
mailbox_depths: VecDeque::new(),
|
||||
message_rates: VecDeque::new(),
|
||||
prev_messages: 0,
|
||||
last_seen_sample: sample_counter,
|
||||
}
|
||||
}
|
||||
|
||||
fn push(&mut self, info: &ActorInfo, cap: usize, sample_counter: u64) {
|
||||
let rate = info.messages_processed.saturating_sub(self.prev_messages) as f64;
|
||||
self.prev_messages = info.messages_processed;
|
||||
self.last_seen_sample = sample_counter;
|
||||
|
||||
push_bounded(&mut self.mailbox_depths, info.mailbox_depth as u64, cap);
|
||||
push_bounded(&mut self.message_rates, rate, cap);
|
||||
}
|
||||
}
|
||||
|
||||
fn push_bounded<T>(buf: &mut VecDeque<T>, val: T, cap: usize) {
|
||||
if buf.len() >= cap {
|
||||
buf.pop_front();
|
||||
}
|
||||
buf.push_back(val);
|
||||
}
|
||||
|
||||
/// Thread-safe history store. Written by the sampler, read by SSE/TUI.
|
||||
pub struct DashboardHistory {
|
||||
inner: RwLock<HistoryInner>,
|
||||
config: HistoryConfig,
|
||||
}
|
||||
|
||||
struct HistoryInner {
|
||||
workers: Vec<WorkerHistory>,
|
||||
actors: HashMap<ActorAddress, ActorHistory>,
|
||||
sample_counter: u64,
|
||||
}
|
||||
|
||||
impl DashboardHistory {
|
||||
pub fn new(config: HistoryConfig) -> Self {
|
||||
Self {
|
||||
inner: RwLock::new(HistoryInner {
|
||||
workers: Vec::new(),
|
||||
actors: HashMap::new(),
|
||||
sample_counter: 0,
|
||||
}),
|
||||
config,
|
||||
}
|
||||
}
|
||||
|
||||
/// Record a stats snapshot. Called by the sampler thread.
|
||||
pub fn record(&self, stats: &RuntimeStats) {
|
||||
let mut inner = self.inner.write().unwrap();
|
||||
inner.sample_counter += 1;
|
||||
let counter = inner.sample_counter;
|
||||
|
||||
// Resize workers vec if needed
|
||||
while inner.workers.len() < stats.workers.len() {
|
||||
inner.workers.push(WorkerHistory::new());
|
||||
}
|
||||
|
||||
// Record per-worker data
|
||||
for w in &stats.workers {
|
||||
if let Some(wh) = inner.workers.get_mut(w.id) {
|
||||
wh.push(
|
||||
w.messages_processed,
|
||||
w.mailbox_depth,
|
||||
w.num_actors,
|
||||
self.config.max_worker_samples,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Record per-actor data
|
||||
for a in &stats.actor_details {
|
||||
let ah = inner.actors.entry(a.address).or_insert_with(|| ActorHistory::new(counter));
|
||||
ah.push(a, self.config.max_actor_samples, counter);
|
||||
}
|
||||
|
||||
// LRU eviction: remove actors not seen recently if over capacity
|
||||
if inner.actors.len() > self.config.max_tracked_actors {
|
||||
let mut entries: Vec<(ActorAddress, u64)> = inner
|
||||
.actors
|
||||
.iter()
|
||||
.map(|(addr, ah)| (*addr, ah.last_seen_sample))
|
||||
.collect();
|
||||
entries.sort_by_key(|&(_, seen)| seen);
|
||||
let to_remove = inner.actors.len() - self.config.max_tracked_actors;
|
||||
for (addr, _) in entries.into_iter().take(to_remove) {
|
||||
inner.actors.remove(&addr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Get a snapshot of worker history for rendering sparklines.
|
||||
/// Returns Vec indexed by worker_id, each containing recent message rates.
|
||||
pub fn worker_sparklines(&self) -> Vec<Vec<u64>> {
|
||||
let inner = self.inner.read().unwrap();
|
||||
inner
|
||||
.workers
|
||||
.iter()
|
||||
.map(|wh| wh.message_rates.iter().map(|r| *r as u64).collect())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Get worker mailbox depth history.
|
||||
pub fn worker_mailbox_sparklines(&self) -> Vec<Vec<u64>> {
|
||||
let inner = self.inner.read().unwrap();
|
||||
inner
|
||||
.workers
|
||||
.iter()
|
||||
.map(|wh| wh.mailbox_depths.iter().copied().collect())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Get sparkline data for a specific actor.
|
||||
pub fn actor_sparkline(&self, addr: &ActorAddress) -> Option<(Vec<u64>, Vec<u64>)> {
|
||||
let inner = self.inner.read().unwrap();
|
||||
inner.actors.get(addr).map(|ah| {
|
||||
let mailbox: Vec<u64> = ah.mailbox_depths.iter().copied().collect();
|
||||
let rates: Vec<u64> = ah.message_rates.iter().map(|r| *r as u64).collect();
|
||||
(mailbox, rates)
|
||||
})
|
||||
}
|
||||
|
||||
/// Get total sample count (useful for knowing if history is available).
|
||||
pub fn sample_count(&self) -> u64 {
|
||||
self.inner.read().unwrap().sample_counter
|
||||
}
|
||||
|
||||
/// Serialize worker history as JSON for the SSE initial payload.
|
||||
pub fn worker_history_json(&self) -> String {
|
||||
let sparklines = self.worker_sparklines();
|
||||
let mailbox = self.worker_mailbox_sparklines();
|
||||
serde_json::json!({
|
||||
"workers": sparklines.iter().enumerate().map(|(i, rates)| {
|
||||
serde_json::json!({
|
||||
"id": i,
|
||||
"message_rates": rates,
|
||||
"mailbox_depths": mailbox.get(i).unwrap_or(&Vec::new()),
|
||||
})
|
||||
}).collect::<Vec<_>>(),
|
||||
})
|
||||
.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use swactor::stats::{ActorInfo, WorkerInfo};
|
||||
|
||||
fn make_stats(workers: Vec<(u64, usize, usize)>, actors: Vec<ActorInfo>) -> RuntimeStats {
|
||||
RuntimeStats {
|
||||
num_workers: workers.len(),
|
||||
uptime_ms: 0,
|
||||
actors: actors.iter().map(|a| (a.address, a.worker_id)).collect(),
|
||||
workers: workers
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(id, (msgs, depth, n_actors))| WorkerInfo {
|
||||
id,
|
||||
num_actors: n_actors,
|
||||
mailbox_depth: depth,
|
||||
messages_processed: msgs,
|
||||
local_sends: 0,
|
||||
cross_sends: 0,
|
||||
inbox_sends: 0,
|
||||
type_mismatches: 0,
|
||||
panics: 0,
|
||||
messages_dropped: 0,
|
||||
restarts: 0,
|
||||
stops: 0,
|
||||
})
|
||||
.collect(),
|
||||
actor_details: actors,
|
||||
tick_timings: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn make_actor(id: u8, worker: usize, depth: usize, msgs: u64) -> ActorInfo {
|
||||
ActorInfo {
|
||||
address: ActorAddress([id; 32]),
|
||||
worker_id: worker,
|
||||
mailbox_depth: depth,
|
||||
last_msg_type: None,
|
||||
messages_processed: msgs,
|
||||
poisoned: false,
|
||||
message_type_counts: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_rates_accumulate_over_samples() {
|
||||
let history = DashboardHistory::new(HistoryConfig::default());
|
||||
|
||||
// First sample: establishes baseline (rate will be the raw value since prev=0)
|
||||
let stats1 = make_stats(vec![(100, 5, 2)], vec![]);
|
||||
history.record(&stats1);
|
||||
|
||||
// Second sample: delta = 150 - 100 = 50
|
||||
let stats2 = make_stats(vec![(150, 3, 2)], vec![]);
|
||||
history.record(&stats2);
|
||||
|
||||
let sparklines = history.worker_sparklines();
|
||||
assert_eq!(sparklines.len(), 1);
|
||||
assert_eq!(sparklines[0].len(), 2);
|
||||
assert_eq!(sparklines[0][0], 100); // first sample: 100 - 0
|
||||
assert_eq!(sparklines[0][1], 50); // second sample: 150 - 100
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounded_eviction_drops_oldest() {
|
||||
let config = HistoryConfig {
|
||||
max_worker_samples: 3,
|
||||
..Default::default()
|
||||
};
|
||||
let history = DashboardHistory::new(config);
|
||||
|
||||
for i in 0..5u64 {
|
||||
let stats = make_stats(vec![(i * 10, 0, 0)], vec![]);
|
||||
history.record(&stats);
|
||||
}
|
||||
|
||||
let sparklines = history.worker_sparklines();
|
||||
assert_eq!(sparklines[0].len(), 3); // capped at 3
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn actor_lru_eviction_keeps_most_recent() {
|
||||
let config = HistoryConfig {
|
||||
max_tracked_actors: 2,
|
||||
..Default::default()
|
||||
};
|
||||
let history = DashboardHistory::new(config);
|
||||
|
||||
// Sample 1: actors A and B
|
||||
let stats1 = make_stats(
|
||||
vec![(0, 0, 2)],
|
||||
vec![make_actor(1, 0, 0, 0), make_actor(2, 0, 0, 0)],
|
||||
);
|
||||
history.record(&stats1);
|
||||
|
||||
// Sample 2: actors B and C (A not seen)
|
||||
let stats2 = make_stats(
|
||||
vec![(0, 0, 2)],
|
||||
vec![make_actor(2, 0, 0, 0), make_actor(3, 0, 0, 0)],
|
||||
);
|
||||
history.record(&stats2);
|
||||
|
||||
// A should be evicted (LRU), B and C kept
|
||||
assert!(history.actor_sparkline(&ActorAddress([1; 32])).is_none());
|
||||
assert!(history.actor_sparkline(&ActorAddress([2; 32])).is_some());
|
||||
assert!(history.actor_sparkline(&ActorAddress([3; 32])).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn actor_rates_track_deltas() {
|
||||
let history = DashboardHistory::new(HistoryConfig::default());
|
||||
|
||||
let stats1 = make_stats(vec![(0, 0, 1)], vec![make_actor(1, 0, 5, 100)]);
|
||||
history.record(&stats1);
|
||||
|
||||
let stats2 = make_stats(vec![(0, 0, 1)], vec![make_actor(1, 0, 3, 175)]);
|
||||
history.record(&stats2);
|
||||
|
||||
let (mailbox, rates) = history.actor_sparkline(&ActorAddress([1; 32])).unwrap();
|
||||
assert_eq!(mailbox, vec![5, 3]);
|
||||
assert_eq!(rates[0], 100); // first: 100 - 0
|
||||
assert_eq!(rates[1], 75); // second: 175 - 100
|
||||
}
|
||||
}
|
||||
|
|
@ -20,6 +20,7 @@ pub struct DashboardEvent {
|
|||
pub level: String,
|
||||
pub message: String,
|
||||
pub worker_id: Option<usize>,
|
||||
pub actor_addr: Option<String>,
|
||||
pub fields: serde_json::Map<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
|
|
@ -83,6 +84,28 @@ impl EventStore {
|
|||
(batch, new_cursor)
|
||||
}
|
||||
|
||||
/// Read recent events for a specific actor address (hex prefix match).
|
||||
/// Returns up to `limit` most recent matching events.
|
||||
pub fn read_for_actor(&self, actor_hex: &str, limit: usize) -> Vec<DashboardEvent> {
|
||||
let events = self.events.lock().unwrap();
|
||||
let lower = actor_hex.to_lowercase();
|
||||
events
|
||||
.iter()
|
||||
.rev()
|
||||
.filter(|e| {
|
||||
e.actor_addr
|
||||
.as_ref()
|
||||
.map(|a| a.to_lowercase().starts_with(&lower) || a.to_lowercase().contains(&lower))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.take(limit)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>()
|
||||
.into_iter()
|
||||
.rev()
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Drains the full recording log. Only available when recording is enabled.
|
||||
/// This is destructive — events are consumed. Intended for `save_trace()`.
|
||||
pub fn all_events(&self) -> Option<Vec<DashboardEvent>> {
|
||||
|
|
@ -183,24 +206,39 @@ where
|
|||
let mut visitor = FieldVisitor::new();
|
||||
event.record(&mut visitor);
|
||||
|
||||
// Walk span context to find worker_id
|
||||
// Walk span context to find worker_id and actor_addr
|
||||
let mut worker_id = None;
|
||||
let mut actor_addr = None;
|
||||
if let Some(scope) = ctx.event_scope(event) {
|
||||
for span in scope {
|
||||
let exts = span.extensions();
|
||||
if let Some(wid) = exts.get::<WorkerIdField>() {
|
||||
worker_id = Some(wid.0);
|
||||
if worker_id.is_none() {
|
||||
if let Some(wid) = exts.get::<WorkerIdField>() {
|
||||
worker_id = Some(wid.0);
|
||||
}
|
||||
}
|
||||
if actor_addr.is_none() {
|
||||
if let Some(aa) = exts.get::<ActorAddrField>() {
|
||||
actor_addr = Some(aa.0.clone());
|
||||
}
|
||||
}
|
||||
if worker_id.is_some() && actor_addr.is_some() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Also check if worker_id was a field on the event itself
|
||||
// Also check if worker_id or actor_addr was a field on the event itself
|
||||
if worker_id.is_none() {
|
||||
if let Some(serde_json::Value::Number(n)) = visitor.fields.get("worker_id") {
|
||||
worker_id = n.as_u64().map(|v| v as usize);
|
||||
}
|
||||
}
|
||||
if actor_addr.is_none() {
|
||||
if let Some(serde_json::Value::String(s)) = visitor.fields.get("actor_addr") {
|
||||
actor_addr = Some(s.clone());
|
||||
}
|
||||
}
|
||||
|
||||
let dashboard_event = DashboardEvent {
|
||||
seq: 0, // filled by push()
|
||||
|
|
@ -208,6 +246,7 @@ where
|
|||
level: event.metadata().level().to_string(),
|
||||
message: visitor.message,
|
||||
worker_id,
|
||||
actor_addr,
|
||||
fields: visitor.fields,
|
||||
};
|
||||
|
||||
|
|
@ -215,19 +254,25 @@ where
|
|||
}
|
||||
|
||||
fn on_new_span(&self, attrs: &span::Attributes<'_>, id: &span::Id, ctx: Context<'_, S>) {
|
||||
// Extract worker_id from span fields and store in extensions
|
||||
// Extract worker_id and actor_addr from span fields and store in extensions
|
||||
let mut visitor = FieldVisitor::new();
|
||||
attrs.record(&mut visitor);
|
||||
|
||||
if let Some(serde_json::Value::Number(n)) = visitor.fields.get("worker_id") {
|
||||
if let Some(wid) = n.as_u64() {
|
||||
if let Some(span) = ctx.span(id) {
|
||||
if let Some(span) = ctx.span(id) {
|
||||
if let Some(serde_json::Value::Number(n)) = visitor.fields.get("worker_id") {
|
||||
if let Some(wid) = n.as_u64() {
|
||||
span.extensions_mut().insert(WorkerIdField(wid as usize));
|
||||
}
|
||||
}
|
||||
if let Some(serde_json::Value::String(s)) = visitor.fields.get("actor_addr") {
|
||||
span.extensions_mut().insert(ActorAddrField(s.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Stored in span extensions to propagate worker_id to child events.
|
||||
struct WorkerIdField(usize);
|
||||
|
||||
/// Stored in span extensions to propagate actor_addr to child events.
|
||||
struct ActorAddrField(String);
|
||||
|
|
|
|||
|
|
@ -1,10 +1,15 @@
|
|||
pub mod collector;
|
||||
pub mod history;
|
||||
pub mod investigate;
|
||||
pub mod layer;
|
||||
pub mod trace;
|
||||
pub mod warnings;
|
||||
mod actor_detail_html;
|
||||
mod actors_html;
|
||||
mod dashboard_html;
|
||||
mod server;
|
||||
pub mod topology;
|
||||
mod topology_html;
|
||||
|
||||
#[cfg(feature = "tui")]
|
||||
pub mod tui;
|
||||
|
|
@ -27,6 +32,7 @@ use tracing_subscriber::layer::SubscriberExt;
|
|||
use tracing_subscriber::util::SubscriberInitExt;
|
||||
|
||||
use crate::collector::StatsCollector;
|
||||
use crate::history::{DashboardHistory, HistoryConfig};
|
||||
use crate::layer::{now_ms, DashboardLayer, EventStore};
|
||||
use crate::trace::{RuntimeTrace, TimestampedStats};
|
||||
|
||||
|
|
@ -80,6 +86,7 @@ pub struct DashboardHandle {
|
|||
collector: Arc<Mutex<Option<Arc<StatsCollector>>>>,
|
||||
shutdown: Arc<AtomicBool>,
|
||||
stats_timeline: Arc<ArrayQueue<TimestampedStats>>,
|
||||
history: Arc<DashboardHistory>,
|
||||
recording: bool,
|
||||
#[cfg(feature = "distribution")]
|
||||
distribution: Arc<Mutex<Option<Arc<dyn distribution_collector::DistributionStatsProvider>>>>,
|
||||
|
|
@ -115,6 +122,11 @@ impl DashboardHandle {
|
|||
*self.distribution.lock().unwrap() = Some(provider);
|
||||
}
|
||||
|
||||
/// Access the time-series history store (for TUI sparklines, etc.).
|
||||
pub fn history(&self) -> &Arc<DashboardHistory> {
|
||||
&self.history
|
||||
}
|
||||
|
||||
/// Signal the dashboard to shut down (SSE clients receive "done").
|
||||
pub fn shutdown(&self) {
|
||||
self.shutdown.store(true, Ordering::Release);
|
||||
|
|
@ -159,6 +171,7 @@ pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle {
|
|||
let collector: Arc<Mutex<Option<Arc<StatsCollector>>>> = Arc::new(Mutex::new(None));
|
||||
let shutdown = Arc::new(AtomicBool::new(false));
|
||||
let stats_timeline = Arc::new(ArrayQueue::new(config.record_stats_capacity.max(1)));
|
||||
let history = Arc::new(DashboardHistory::new(HistoryConfig::default()));
|
||||
|
||||
#[cfg(feature = "distribution")]
|
||||
let distribution: Arc<Mutex<Option<Arc<dyn distribution_collector::DistributionStatsProvider>>>> =
|
||||
|
|
@ -169,6 +182,7 @@ pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle {
|
|||
Arc::clone(&runtime),
|
||||
Arc::clone(&collector),
|
||||
Arc::clone(&shutdown),
|
||||
Arc::clone(&history),
|
||||
config.port,
|
||||
#[cfg(feature = "distribution")]
|
||||
Arc::clone(&distribution),
|
||||
|
|
@ -210,6 +224,7 @@ pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle {
|
|||
collector,
|
||||
shutdown,
|
||||
stats_timeline,
|
||||
history,
|
||||
recording: config.record,
|
||||
#[cfg(feature = "distribution")]
|
||||
distribution,
|
||||
|
|
|
|||
|
|
@ -8,11 +8,16 @@ use std::collections::HashMap;
|
|||
|
||||
use swactor::runtime::Runtime;
|
||||
|
||||
use crate::actor_detail_html::ACTOR_DETAIL_HTML;
|
||||
use crate::actors_html::ACTORS_HTML;
|
||||
use crate::collector::StatsCollector;
|
||||
use crate::dashboard_html::DASHBOARD_HTML;
|
||||
use crate::history::DashboardHistory;
|
||||
use crate::layer::EventStore;
|
||||
use crate::topology;
|
||||
use crate::topology_html::TOPOLOGY_HTML;
|
||||
use crate::trace::RuntimeTrace;
|
||||
use crate::warnings::{WarningConfig, WarningDetector};
|
||||
|
||||
#[cfg(feature = "distribution")]
|
||||
use crate::distribution_collector::DistributionStatsProvider;
|
||||
|
|
@ -119,6 +124,7 @@ pub(crate) fn spawn_http_server(
|
|||
runtime: Arc<Mutex<Option<Arc<Runtime>>>>,
|
||||
collector: Arc<Mutex<Option<Arc<StatsCollector>>>>,
|
||||
shutdown: Arc<AtomicBool>,
|
||||
history: Arc<DashboardHistory>,
|
||||
port: u16,
|
||||
#[cfg(feature = "distribution")]
|
||||
distribution: Arc<Mutex<Option<Arc<dyn DistributionStatsProvider>>>>,
|
||||
|
|
@ -134,6 +140,7 @@ pub(crate) fn spawn_http_server(
|
|||
let runtime = Arc::clone(&runtime);
|
||||
let collector = Arc::clone(&collector);
|
||||
let shutdown = Arc::clone(&shutdown);
|
||||
let history = Arc::clone(&history);
|
||||
let cmd_router = Arc::clone(&cmd_router);
|
||||
#[cfg(feature = "distribution")]
|
||||
let distribution = Arc::clone(&distribution);
|
||||
|
|
@ -149,6 +156,7 @@ pub(crate) fn spawn_http_server(
|
|||
match path {
|
||||
"/" => respond_html(request, DASHBOARD_HTML, "live"),
|
||||
"/actors" => respond_html(request, ACTORS_HTML, "live"),
|
||||
"/topology" => respond_html(request, TOPOLOGY_HTML, "live"),
|
||||
#[cfg(feature = "distribution")]
|
||||
"/distribution" => respond_html(request, DISTRIBUTION_HTML, "live"),
|
||||
"/events" => {
|
||||
|
|
@ -158,6 +166,7 @@ pub(crate) fn spawn_http_server(
|
|||
Arc::clone(&runtime),
|
||||
Arc::clone(&collector),
|
||||
Arc::clone(&shutdown),
|
||||
Arc::clone(&history),
|
||||
#[cfg(feature = "distribution")]
|
||||
Arc::clone(&distribution),
|
||||
);
|
||||
|
|
@ -169,6 +178,16 @@ pub(crate) fn spawn_http_server(
|
|||
Arc::clone(&collector),
|
||||
);
|
||||
}
|
||||
"/api/history" => {
|
||||
handle_history_api(request, Arc::clone(&history));
|
||||
}
|
||||
"/api/topology" => {
|
||||
handle_topology_api(
|
||||
request,
|
||||
Arc::clone(&runtime),
|
||||
Arc::clone(&collector),
|
||||
);
|
||||
}
|
||||
"/api/investigate" => {
|
||||
handle_investigate_api(
|
||||
request,
|
||||
|
|
@ -185,6 +204,13 @@ pub(crate) fn spawn_http_server(
|
|||
Arc::clone(&distribution),
|
||||
);
|
||||
}
|
||||
"/api/logs" => {
|
||||
handle_logs_api(request, &url, Arc::clone(&store));
|
||||
}
|
||||
_ if path.starts_with("/actor/") => {
|
||||
let hex = &path[7..]; // strip "/actor/"
|
||||
respond_actor_detail(request, hex);
|
||||
}
|
||||
_ => respond_404(request),
|
||||
}
|
||||
}
|
||||
|
|
@ -192,12 +218,25 @@ pub(crate) fn spawn_http_server(
|
|||
}
|
||||
}
|
||||
|
||||
fn respond_actor_detail(request: tiny_http::Request, hex_addr: &str) {
|
||||
let html = ACTOR_DETAIL_HTML
|
||||
.replace("__DASHBOARD_MODE__", "live")
|
||||
.replace("__ACTOR_ADDR__", hex_addr);
|
||||
let response = tiny_http::Response::from_string(html).with_header(
|
||||
"Content-Type: text/html; charset=utf-8"
|
||||
.parse::<tiny_http::Header>()
|
||||
.unwrap(),
|
||||
);
|
||||
let _ = request.respond(response);
|
||||
}
|
||||
|
||||
fn handle_live_sse(
|
||||
request: tiny_http::Request,
|
||||
store: Arc<EventStore>,
|
||||
runtime: Arc<Mutex<Option<Arc<Runtime>>>>,
|
||||
collector: Arc<Mutex<Option<Arc<StatsCollector>>>>,
|
||||
shutdown: Arc<AtomicBool>,
|
||||
history: Arc<DashboardHistory>,
|
||||
#[cfg(feature = "distribution")]
|
||||
distribution: Arc<Mutex<Option<Arc<dyn DistributionStatsProvider>>>>,
|
||||
) {
|
||||
|
|
@ -207,6 +246,14 @@ fn handle_live_sse(
|
|||
// Spawn producer thread
|
||||
thread::spawn(move || {
|
||||
let mut cursor: u64 = 0;
|
||||
let mut warning_detector = WarningDetector::new(WarningConfig::default());
|
||||
let mut tick_count: u64 = 0;
|
||||
|
||||
// Send initial history snapshot so sparklines render immediately
|
||||
if history.sample_count() > 0 {
|
||||
let json = history.worker_history_json();
|
||||
let _ = tx.send(format_sse("history", &json));
|
||||
}
|
||||
|
||||
loop {
|
||||
// Send stats if runtime is available
|
||||
|
|
@ -217,10 +264,33 @@ fn handle_live_sse(
|
|||
if let Some(col) = collector.lock().unwrap().as_ref() {
|
||||
col.enrich(&mut stats);
|
||||
}
|
||||
history.record(&stats);
|
||||
|
||||
// Run warning detection
|
||||
let warnings = warning_detector.check(&stats);
|
||||
if !warnings.is_empty() {
|
||||
if let Ok(wjson) = serde_json::to_string(&warnings) {
|
||||
if tx.send(format_sse("warnings", &wjson)).is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let json = serde_json::to_string(&stats).unwrap();
|
||||
if tx.send(format_sse("stats", &json)).is_err() {
|
||||
return;
|
||||
}
|
||||
|
||||
// Send topology every 5th tick (~1/sec)
|
||||
tick_count += 1;
|
||||
if tick_count % 5 == 0 {
|
||||
let topo = topology::worker_topology(&stats);
|
||||
if let Ok(tjson) = serde_json::to_string(&topo) {
|
||||
if tx.send(format_sse("topology", &tjson)).is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -353,6 +423,67 @@ fn handle_distribution_api(
|
|||
let _ = request.respond(response);
|
||||
}
|
||||
|
||||
fn handle_topology_api(
|
||||
request: tiny_http::Request,
|
||||
runtime: Arc<Mutex<Option<Arc<Runtime>>>>,
|
||||
collector: Arc<Mutex<Option<Arc<StatsCollector>>>>,
|
||||
) {
|
||||
let maybe_rt = runtime.lock().unwrap().clone();
|
||||
let json = match maybe_rt {
|
||||
Some(rt) => {
|
||||
let mut stats = rt.stats();
|
||||
if let Some(col) = collector.lock().unwrap().as_ref() {
|
||||
col.enrich(&mut stats);
|
||||
}
|
||||
let topo = topology::worker_topology(&stats);
|
||||
serde_json::to_string(&topo).unwrap_or_else(|_| "{}".into())
|
||||
}
|
||||
None => "{}".to_string(),
|
||||
};
|
||||
let response = tiny_http::Response::from_string(json).with_header(
|
||||
"Content-Type: application/json"
|
||||
.parse::<tiny_http::Header>()
|
||||
.unwrap(),
|
||||
);
|
||||
let _ = request.respond(response);
|
||||
}
|
||||
|
||||
fn handle_logs_api(request: tiny_http::Request, url: &str, store: Arc<EventStore>) {
|
||||
let params = parse_query_string(url);
|
||||
let actor = params.get("actor").cloned().unwrap_or_default();
|
||||
let limit: usize = params
|
||||
.get("limit")
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(200);
|
||||
let level = params.get("level").cloned();
|
||||
|
||||
let mut events = store.read_for_actor(&actor, limit);
|
||||
|
||||
// Filter by level if specified
|
||||
if let Some(ref lvl) = level {
|
||||
let lvl_upper = lvl.to_uppercase();
|
||||
events.retain(|e| e.level == lvl_upper);
|
||||
}
|
||||
|
||||
let json = serde_json::to_string(&events).unwrap_or_else(|_| "[]".into());
|
||||
let response = tiny_http::Response::from_string(json).with_header(
|
||||
"Content-Type: application/json"
|
||||
.parse::<tiny_http::Header>()
|
||||
.unwrap(),
|
||||
);
|
||||
let _ = request.respond(response);
|
||||
}
|
||||
|
||||
fn handle_history_api(request: tiny_http::Request, history: Arc<DashboardHistory>) {
|
||||
let json = history.worker_history_json();
|
||||
let response = tiny_http::Response::from_string(json).with_header(
|
||||
"Content-Type: application/json"
|
||||
.parse::<tiny_http::Header>()
|
||||
.unwrap(),
|
||||
);
|
||||
let _ = request.respond(response);
|
||||
}
|
||||
|
||||
fn parse_query_string(url: &str) -> HashMap<String, String> {
|
||||
let mut params = HashMap::new();
|
||||
if let Some(qs) = url.split('?').nth(1) {
|
||||
|
|
|
|||
82
crates/runtime-dashboard/src/topology.rs
Normal file
82
crates/runtime-dashboard/src/topology.rs
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
//! Actor-to-actor (and worker-to-worker) message flow topology.
|
||||
//!
|
||||
//! Currently derives topology from per-worker cross_sends/local_sends stats.
|
||||
//! Future: sample-based per-actor source→destination tracking with core instrumentation.
|
||||
|
||||
use swactor::stats::RuntimeStats;
|
||||
|
||||
/// An edge in the topology graph.
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct TopologyEdge {
|
||||
pub source: String,
|
||||
pub target: String,
|
||||
pub weight: u64,
|
||||
pub label: String,
|
||||
}
|
||||
|
||||
/// A node in the topology graph.
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct TopologyNode {
|
||||
pub id: String,
|
||||
pub label: String,
|
||||
pub actor_count: usize,
|
||||
pub group: usize,
|
||||
}
|
||||
|
||||
/// A snapshot of the current topology.
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct TopologySnapshot {
|
||||
pub nodes: Vec<TopologyNode>,
|
||||
pub edges: Vec<TopologyEdge>,
|
||||
}
|
||||
|
||||
/// Build a worker-level topology from RuntimeStats.
|
||||
///
|
||||
/// Workers are nodes, edges represent message flow:
|
||||
/// - Self-loops for local_sends
|
||||
/// - Cross-edges distributed proportionally (until per-destination tracking exists)
|
||||
pub fn worker_topology(stats: &RuntimeStats) -> TopologySnapshot {
|
||||
let mut nodes = Vec::new();
|
||||
let mut edges = Vec::new();
|
||||
|
||||
for w in &stats.workers {
|
||||
nodes.push(TopologyNode {
|
||||
id: format!("w{}", w.id),
|
||||
label: format!("W{}", w.id),
|
||||
actor_count: w.num_actors,
|
||||
group: w.id,
|
||||
});
|
||||
|
||||
// Local sends = self-loop
|
||||
if w.local_sends > 0 {
|
||||
edges.push(TopologyEdge {
|
||||
source: format!("w{}", w.id),
|
||||
target: format!("w{}", w.id),
|
||||
weight: w.local_sends,
|
||||
label: format!("{} local", w.local_sends),
|
||||
});
|
||||
}
|
||||
|
||||
// Cross sends — without per-destination data, distribute evenly to other workers
|
||||
if w.cross_sends > 0 && stats.workers.len() > 1 {
|
||||
let others: Vec<&swactor::stats::WorkerInfo> =
|
||||
stats.workers.iter().filter(|o| o.id != w.id).collect();
|
||||
let per_worker = w.cross_sends / others.len() as u64;
|
||||
let remainder = w.cross_sends % others.len() as u64;
|
||||
|
||||
for (i, other) in others.iter().enumerate() {
|
||||
let count = per_worker + if (i as u64) < remainder { 1 } else { 0 };
|
||||
if count > 0 {
|
||||
edges.push(TopologyEdge {
|
||||
source: format!("w{}", w.id),
|
||||
target: format!("w{}", other.id),
|
||||
weight: count,
|
||||
label: format!("{} cross", count),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TopologySnapshot { nodes, edges }
|
||||
}
|
||||
280
crates/runtime-dashboard/src/topology_html.rs
Normal file
280
crates/runtime-dashboard/src/topology_html.rs
Normal file
|
|
@ -0,0 +1,280 @@
|
|||
pub const TOPOLOGY_HTML: &str = r##"<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Topology — Swactor Dashboard</title>
|
||||
<style>
|
||||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||
body { font-family: 'Menlo', 'Consolas', 'Monaco', monospace; background: #0f1117; color: #e0e0e0; font-size: 13px; }
|
||||
|
||||
.header {
|
||||
display: flex; align-items: center; justify-content: space-between;
|
||||
padding: 12px 20px; background: #161822; border-bottom: 1px solid #2a2d3e;
|
||||
}
|
||||
.header h1 { font-size: 16px; font-weight: 600; color: #fff; }
|
||||
.status-dot {
|
||||
width: 10px; height: 10px; border-radius: 50%; background: #4caf50;
|
||||
display: inline-block; margin-left: 8px; vertical-align: middle;
|
||||
}
|
||||
.status-dot.disconnected { background: #f44336; }
|
||||
|
||||
.header-left { display: flex; align-items: center; }
|
||||
.nav-links { display: flex; gap: 4px; margin-left: 20px; }
|
||||
.nav-link {
|
||||
color: #888; text-decoration: none; font-size: 12px;
|
||||
padding: 4px 10px; border-radius: 3px;
|
||||
}
|
||||
.nav-link:hover { color: #e0e0e0; }
|
||||
.nav-link.active { color: #fff; background: #2a2d3e; }
|
||||
|
||||
.content { padding: 0; display: flex; flex-direction: column; height: calc(100vh - 49px); }
|
||||
canvas#topoCanvas { flex: 1; width: 100%; cursor: grab; }
|
||||
canvas#topoCanvas:active { cursor: grabbing; }
|
||||
|
||||
.legend {
|
||||
padding: 8px 20px; background: #161822; border-top: 1px solid #2a2d3e;
|
||||
font-size: 11px; color: #888;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="header">
|
||||
<div class="header-left">
|
||||
<h1>Swactor Runtime Dashboard <span id="statusDot" class="status-dot"></span></h1>
|
||||
<nav class="nav-links">
|
||||
<a href="/" class="nav-link">Overview</a>
|
||||
<a href="/actors" class="nav-link">Actors</a>
|
||||
<a href="/topology" class="nav-link active">Topology</a>
|
||||
<a href="/distribution" class="nav-link">Distribution</a>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="content">
|
||||
<canvas id="topoCanvas"></canvas>
|
||||
<div class="legend">
|
||||
Node size = actor count. Edge thickness = message volume. Green = local sends. Blue = cross-worker sends.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(function() {
|
||||
var canvas = document.getElementById('topoCanvas');
|
||||
var ctx = canvas.getContext('2d');
|
||||
var dot = document.getElementById('statusDot');
|
||||
|
||||
var colors = ['#4caf50','#2196f3','#ff9800','#f44336','#9c27b0','#00bcd4','#ffeb3b','#e91e63'];
|
||||
var nodes = [];
|
||||
var edges = [];
|
||||
var positions = {};
|
||||
|
||||
function resize() {
|
||||
var dpr = window.devicePixelRatio || 1;
|
||||
var rect = canvas.getBoundingClientRect();
|
||||
canvas.width = rect.width * dpr;
|
||||
canvas.height = rect.height * dpr;
|
||||
ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
|
||||
}
|
||||
window.addEventListener('resize', resize);
|
||||
resize();
|
||||
|
||||
function initPositions() {
|
||||
var W = canvas.getBoundingClientRect().width;
|
||||
var H = canvas.getBoundingClientRect().height;
|
||||
var cx = W / 2, cy = H / 2;
|
||||
var r = Math.min(W, H) * 0.3;
|
||||
|
||||
nodes.forEach(function(n, i) {
|
||||
if (!positions[n.id]) {
|
||||
var angle = (2 * Math.PI * i) / Math.max(1, nodes.length);
|
||||
positions[n.id] = {
|
||||
x: cx + r * Math.cos(angle),
|
||||
y: cy + r * Math.sin(angle),
|
||||
vx: 0, vy: 0
|
||||
};
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function simulate() {
|
||||
var W = canvas.getBoundingClientRect().width;
|
||||
var H = canvas.getBoundingClientRect().height;
|
||||
var cx = W / 2, cy = H / 2;
|
||||
|
||||
// Repulsion between nodes
|
||||
for (var i = 0; i < nodes.length; i++) {
|
||||
var pi = positions[nodes[i].id];
|
||||
if (!pi) continue;
|
||||
for (var j = i + 1; j < nodes.length; j++) {
|
||||
var pj = positions[nodes[j].id];
|
||||
if (!pj) continue;
|
||||
var dx = pi.x - pj.x;
|
||||
var dy = pi.y - pj.y;
|
||||
var dist = Math.sqrt(dx * dx + dy * dy) || 1;
|
||||
var force = 8000 / (dist * dist);
|
||||
pi.vx += dx / dist * force;
|
||||
pi.vy += dy / dist * force;
|
||||
pj.vx -= dx / dist * force;
|
||||
pj.vy -= dy / dist * force;
|
||||
}
|
||||
}
|
||||
|
||||
// Attraction along edges
|
||||
edges.forEach(function(e) {
|
||||
if (e.source === e.target) return;
|
||||
var ps = positions[e.source];
|
||||
var pt = positions[e.target];
|
||||
if (!ps || !pt) return;
|
||||
var dx = pt.x - ps.x;
|
||||
var dy = pt.y - ps.y;
|
||||
var dist = Math.sqrt(dx * dx + dy * dy) || 1;
|
||||
var force = (dist - 150) * 0.01;
|
||||
ps.vx += dx / dist * force;
|
||||
ps.vy += dy / dist * force;
|
||||
pt.vx -= dx / dist * force;
|
||||
pt.vy -= dy / dist * force;
|
||||
});
|
||||
|
||||
// Gravity toward center
|
||||
for (var i = 0; i < nodes.length; i++) {
|
||||
var p = positions[nodes[i].id];
|
||||
if (!p) continue;
|
||||
p.vx += (cx - p.x) * 0.002;
|
||||
p.vy += (cy - p.y) * 0.002;
|
||||
}
|
||||
|
||||
// Apply velocity with damping
|
||||
for (var i = 0; i < nodes.length; i++) {
|
||||
var p = positions[nodes[i].id];
|
||||
if (!p) continue;
|
||||
p.vx *= 0.85;
|
||||
p.vy *= 0.85;
|
||||
p.x += p.vx;
|
||||
p.y += p.vy;
|
||||
p.x = Math.max(30, Math.min(W - 30, p.x));
|
||||
p.y = Math.max(30, Math.min(H - 30, p.y));
|
||||
}
|
||||
}
|
||||
|
||||
function draw() {
|
||||
var W = canvas.getBoundingClientRect().width;
|
||||
var H = canvas.getBoundingClientRect().height;
|
||||
ctx.clearRect(0, 0, W, H);
|
||||
|
||||
if (nodes.length === 0) {
|
||||
ctx.fillStyle = '#555';
|
||||
ctx.font = '14px monospace';
|
||||
ctx.textAlign = 'center';
|
||||
ctx.fillText('Waiting for topology data...', W / 2, H / 2);
|
||||
return;
|
||||
}
|
||||
|
||||
// Draw edges
|
||||
var maxWeight = Math.max(1, Math.max.apply(null, edges.map(function(e) { return e.weight; })));
|
||||
|
||||
edges.forEach(function(e) {
|
||||
var ps = positions[e.source];
|
||||
var pt = positions[e.target];
|
||||
if (!ps || !pt) return;
|
||||
|
||||
var isSelf = e.source === e.target;
|
||||
var thickness = Math.max(1, (e.weight / maxWeight) * 6);
|
||||
var color = isSelf ? 'rgba(76, 175, 80, 0.5)' : 'rgba(33, 150, 243, 0.5)';
|
||||
|
||||
if (isSelf) {
|
||||
// Self-loop: small arc above the node
|
||||
ctx.beginPath();
|
||||
ctx.arc(ps.x, ps.y - 25, 15, 0.3, Math.PI - 0.3);
|
||||
ctx.strokeStyle = color;
|
||||
ctx.lineWidth = thickness;
|
||||
ctx.stroke();
|
||||
} else {
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(ps.x, ps.y);
|
||||
ctx.lineTo(pt.x, pt.y);
|
||||
ctx.strokeStyle = color;
|
||||
ctx.lineWidth = thickness;
|
||||
ctx.stroke();
|
||||
|
||||
// Arrow
|
||||
var angle = Math.atan2(pt.y - ps.y, pt.x - ps.x);
|
||||
var headLen = 8;
|
||||
var mx = (ps.x + pt.x) / 2;
|
||||
var my = (ps.y + pt.y) / 2;
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(mx, my);
|
||||
ctx.lineTo(mx - headLen * Math.cos(angle - 0.3), my - headLen * Math.sin(angle - 0.3));
|
||||
ctx.moveTo(mx, my);
|
||||
ctx.lineTo(mx - headLen * Math.cos(angle + 0.3), my - headLen * Math.sin(angle + 0.3));
|
||||
ctx.strokeStyle = color;
|
||||
ctx.lineWidth = 1.5;
|
||||
ctx.stroke();
|
||||
|
||||
// Edge label
|
||||
ctx.fillStyle = '#666';
|
||||
ctx.font = '9px monospace';
|
||||
ctx.textAlign = 'center';
|
||||
ctx.fillText(e.label, mx, my - 6);
|
||||
}
|
||||
});
|
||||
|
||||
// Draw nodes
|
||||
nodes.forEach(function(n) {
|
||||
var p = positions[n.id];
|
||||
if (!p) return;
|
||||
var r = Math.max(12, 8 + n.actor_count * 2);
|
||||
var color = colors[n.group % colors.length];
|
||||
|
||||
ctx.beginPath();
|
||||
ctx.arc(p.x, p.y, r, 0, 2 * Math.PI);
|
||||
ctx.fillStyle = color;
|
||||
ctx.globalAlpha = 0.7;
|
||||
ctx.fill();
|
||||
ctx.globalAlpha = 1;
|
||||
ctx.strokeStyle = '#fff';
|
||||
ctx.lineWidth = 1.5;
|
||||
ctx.stroke();
|
||||
|
||||
ctx.fillStyle = '#fff';
|
||||
ctx.font = 'bold 11px monospace';
|
||||
ctx.textAlign = 'center';
|
||||
ctx.textBaseline = 'middle';
|
||||
ctx.fillText(n.label, p.x, p.y);
|
||||
|
||||
ctx.fillStyle = '#888';
|
||||
ctx.font = '9px monospace';
|
||||
ctx.fillText(n.actor_count + ' actors', p.x, p.y + r + 12);
|
||||
});
|
||||
}
|
||||
|
||||
function updateTopology(data) {
|
||||
nodes = data.nodes || [];
|
||||
edges = data.edges || [];
|
||||
initPositions();
|
||||
}
|
||||
|
||||
function tick() {
|
||||
simulate();
|
||||
draw();
|
||||
requestAnimationFrame(tick);
|
||||
}
|
||||
tick();
|
||||
|
||||
var es = new EventSource('/events');
|
||||
|
||||
es.addEventListener('topology', function(e) {
|
||||
try { updateTopology(JSON.parse(e.data)); } catch(err) { console.error(err); }
|
||||
});
|
||||
|
||||
es.addEventListener('done', function() {
|
||||
dot.className = 'status-dot disconnected';
|
||||
es.close();
|
||||
});
|
||||
es.onerror = function() { dot.className = 'status-dot disconnected'; };
|
||||
es.onopen = function() { dot.className = 'status-dot'; };
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
"##;
|
||||
|
|
@ -1,9 +1,14 @@
|
|||
use std::collections::{HashMap, VecDeque};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use crossterm::event::{KeyCode, KeyEvent, KeyModifiers};
|
||||
use swactor::actor::ActorAddress;
|
||||
use swactor::stats::{RuntimeStats, TickTiming};
|
||||
|
||||
use crate::layer::{DashboardEvent, EventStore};
|
||||
use crate::warnings::{Warning, WarningConfig, WarningDetector};
|
||||
|
||||
/// Per-worker data prepared for rendering.
|
||||
pub struct WorkerView {
|
||||
pub id: usize,
|
||||
|
|
@ -15,6 +20,10 @@ pub struct WorkerView {
|
|||
/// Fraction of bar for each phase group: [processing, delivery, spawns, overhead]
|
||||
pub phase_fractions: [f64; 4],
|
||||
pub panics: u64,
|
||||
/// Recent message rates for sparkline rendering.
|
||||
pub sparkline_rates: Vec<u64>,
|
||||
/// Recent mailbox depths for sparkline rendering.
|
||||
pub sparkline_mailbox: Vec<u64>,
|
||||
}
|
||||
|
||||
/// One row in the actor table.
|
||||
|
|
@ -25,6 +34,12 @@ pub struct ActorRow {
|
|||
pub last_msg_type: Option<String>,
|
||||
pub messages_processed: u64,
|
||||
pub poisoned: bool,
|
||||
/// Per-actor mailbox sparkline (from local ring buffer).
|
||||
pub sparkline_mailbox: Vec<u64>,
|
||||
/// Per-actor message rate sparkline.
|
||||
pub sparkline_rates: Vec<u64>,
|
||||
/// Per-message-type counts, sorted descending.
|
||||
pub message_type_counts: Vec<(String, u64)>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
|
|
@ -62,6 +77,7 @@ impl SortColumn {
|
|||
pub enum ViewMode {
|
||||
Overview,
|
||||
WorkerDetail,
|
||||
ActorDetail,
|
||||
#[cfg(feature = "distribution")]
|
||||
Distribution,
|
||||
}
|
||||
|
|
@ -79,7 +95,13 @@ pub struct App {
|
|||
pub total_panics: u64,
|
||||
pub num_workers: usize,
|
||||
pub view_mode: ViewMode,
|
||||
pub prev_view_mode: ViewMode,
|
||||
pub focused_worker: usize,
|
||||
pub focused_actor: Option<ActorAddress>,
|
||||
pub search_active: bool,
|
||||
pub search_query: String,
|
||||
pub search_locked: bool,
|
||||
pub warnings: Vec<Warning>,
|
||||
|
||||
#[cfg(feature = "distribution")]
|
||||
pub distribution: Option<distribution::snapshot::DistributionNodeSnapshot>,
|
||||
|
|
@ -90,6 +112,21 @@ pub struct App {
|
|||
prev_time: Instant,
|
||||
/// Rolling msg rates (smoothed)
|
||||
msg_rates: Vec<f64>,
|
||||
/// Per-worker sparkline history (message rate deltas).
|
||||
sparkline_rates: Vec<VecDeque<u64>>,
|
||||
/// Per-worker sparkline history (mailbox depths).
|
||||
sparkline_mailbox: Vec<VecDeque<u64>>,
|
||||
/// Per-actor sparkline history: address → (prev_msgs, rates, mailbox_depths).
|
||||
actor_sparklines: HashMap<ActorAddress, (u64, VecDeque<u64>, VecDeque<u64>)>,
|
||||
warning_detector: WarningDetector,
|
||||
/// Event store for fetching per-actor logs.
|
||||
event_store: Option<Arc<EventStore>>,
|
||||
/// Cached log entries for the focused actor.
|
||||
pub actor_logs: Vec<DashboardEvent>,
|
||||
/// Scroll offset for actor log view.
|
||||
pub log_scroll: usize,
|
||||
/// Active log level filter (all enabled by default).
|
||||
pub log_levels: [bool; 5], // ERROR, WARN, INFO, DEBUG, TRACE
|
||||
}
|
||||
|
||||
impl App {
|
||||
|
|
@ -107,7 +144,13 @@ impl App {
|
|||
total_panics: 0,
|
||||
num_workers: 0,
|
||||
view_mode: ViewMode::Overview,
|
||||
prev_view_mode: ViewMode::Overview,
|
||||
focused_worker: 0,
|
||||
focused_actor: None,
|
||||
search_active: false,
|
||||
search_query: String::new(),
|
||||
search_locked: false,
|
||||
warnings: Vec::new(),
|
||||
#[cfg(feature = "distribution")]
|
||||
distribution: None,
|
||||
#[cfg(feature = "distribution")]
|
||||
|
|
@ -115,9 +158,22 @@ impl App {
|
|||
prev_messages: Vec::new(),
|
||||
prev_time: Instant::now(),
|
||||
msg_rates: Vec::new(),
|
||||
sparkline_rates: Vec::new(),
|
||||
sparkline_mailbox: Vec::new(),
|
||||
actor_sparklines: HashMap::new(),
|
||||
warning_detector: WarningDetector::new(WarningConfig::default()),
|
||||
event_store: None,
|
||||
actor_logs: Vec::new(),
|
||||
log_scroll: 0,
|
||||
log_levels: [true; 5],
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the event store for per-actor log retrieval.
|
||||
pub fn set_event_store(&mut self, store: Arc<EventStore>) {
|
||||
self.event_store = Some(store);
|
||||
}
|
||||
|
||||
#[cfg(feature = "distribution")]
|
||||
pub fn update_distribution(&mut self, snapshot: distribution::snapshot::DistributionNodeSnapshot) {
|
||||
let max = if snapshot.members.is_empty() { 0 } else { snapshot.members.len() - 1 };
|
||||
|
|
@ -125,6 +181,62 @@ impl App {
|
|||
self.distribution = Some(snapshot);
|
||||
}
|
||||
|
||||
/// Get visible actor rows (filtered by search query if active).
|
||||
pub fn visible_actor_rows(&self) -> Vec<&ActorRow> {
|
||||
if self.search_query.is_empty() {
|
||||
self.actor_rows.iter().collect()
|
||||
} else {
|
||||
let q = self.search_query.to_lowercase();
|
||||
self.actor_rows.iter().filter(|r| {
|
||||
let addr = format!("{}", r.address).to_lowercase();
|
||||
let msg = r.last_msg_type.as_deref().unwrap_or("").to_lowercase();
|
||||
let worker = format!("w{}", r.worker_id);
|
||||
addr.contains(&q) || msg.contains(&q) || worker.contains(&q)
|
||||
}).collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the focused actor's data (for actor detail view).
|
||||
pub fn focused_actor_row(&self) -> Option<&ActorRow> {
|
||||
self.focused_actor.as_ref().and_then(|addr| {
|
||||
self.actor_rows.iter().find(|r| r.address == *addr)
|
||||
})
|
||||
}
|
||||
|
||||
/// Refresh the actor log cache from the event store.
|
||||
pub fn refresh_actor_logs(&mut self) {
|
||||
if let (Some(addr), Some(store)) = (&self.focused_actor, &self.event_store) {
|
||||
let hex = format!("{}", addr);
|
||||
self.actor_logs = store.read_for_actor(&hex, 200);
|
||||
} else {
|
||||
self.actor_logs.clear();
|
||||
}
|
||||
}
|
||||
|
||||
/// Get visible log entries (filtered by level).
|
||||
pub fn visible_logs(&self) -> Vec<&DashboardEvent> {
|
||||
self.actor_logs
|
||||
.iter()
|
||||
.filter(|e| {
|
||||
match e.level.as_str() {
|
||||
"ERROR" => self.log_levels[0],
|
||||
"WARN" => self.log_levels[1],
|
||||
"INFO" => self.log_levels[2],
|
||||
"DEBUG" => self.log_levels[3],
|
||||
"TRACE" => self.log_levels[4],
|
||||
_ => true,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Toggle a log level filter (0=ERROR, 1=WARN, 2=INFO, 3=DEBUG, 4=TRACE).
|
||||
pub fn toggle_log_level(&mut self, idx: usize) {
|
||||
if idx < 5 {
|
||||
self.log_levels[idx] = !self.log_levels[idx];
|
||||
}
|
||||
}
|
||||
|
||||
/// Actor rows filtered to the focused worker (for worker detail view).
|
||||
pub fn focused_actor_rows(&self) -> Vec<&ActorRow> {
|
||||
self.actor_rows
|
||||
|
|
@ -144,6 +256,8 @@ impl App {
|
|||
if self.prev_messages.len() != stats.workers.len() {
|
||||
self.prev_messages = stats.workers.iter().map(|w| w.messages_processed).collect();
|
||||
self.msg_rates = vec![0.0; stats.workers.len()];
|
||||
self.sparkline_rates.resize_with(stats.workers.len(), VecDeque::new);
|
||||
self.sparkline_mailbox.resize_with(stats.workers.len(), VecDeque::new);
|
||||
}
|
||||
|
||||
// Compute per-worker views
|
||||
|
|
@ -163,8 +277,19 @@ impl App {
|
|||
self.msg_rates[i] * 0.6 + rate * 0.4
|
||||
};
|
||||
self.msg_rates[i] = smoothed;
|
||||
|
||||
// Track sparkline history
|
||||
let delta = w.messages_processed.saturating_sub(self.prev_messages[i]);
|
||||
self.prev_messages[i] = w.messages_processed;
|
||||
|
||||
let spark_rates = &mut self.sparkline_rates[i];
|
||||
if spark_rates.len() >= 60 { spark_rates.pop_front(); }
|
||||
spark_rates.push_back(delta);
|
||||
|
||||
let spark_mbox = &mut self.sparkline_mailbox[i];
|
||||
if spark_mbox.len() >= 60 { spark_mbox.pop_front(); }
|
||||
spark_mbox.push_back(w.mailbox_depth as u64);
|
||||
|
||||
// Load % and phase fractions from tick timings
|
||||
let timings = stats.tick_timings.get(i).map(|v| v.as_slice()).unwrap_or(&[]);
|
||||
let (load_pct, phase_fractions) = compute_load_and_phases(timings);
|
||||
|
|
@ -178,6 +303,8 @@ impl App {
|
|||
load_pct,
|
||||
phase_fractions,
|
||||
panics: w.panics,
|
||||
sparkline_rates: spark_rates.iter().copied().collect(),
|
||||
sparkline_mailbox: spark_mbox.iter().copied().collect(),
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -187,9 +314,20 @@ impl App {
|
|||
self.total_mailbox = stats.workers.iter().map(|w| w.mailbox_depth).sum();
|
||||
self.total_panics = stats.workers.iter().map(|w| w.panics).sum();
|
||||
|
||||
// Build actor table
|
||||
// Build actor table with sparkline history
|
||||
self.actor_rows.clear();
|
||||
for a in &stats.actor_details {
|
||||
let (prev, rates_buf, mbox_buf) = self.actor_sparklines
|
||||
.entry(a.address)
|
||||
.or_insert_with(|| (0, VecDeque::new(), VecDeque::new()));
|
||||
|
||||
let rate_delta = a.messages_processed.saturating_sub(*prev);
|
||||
*prev = a.messages_processed;
|
||||
if rates_buf.len() >= 60 { rates_buf.pop_front(); }
|
||||
rates_buf.push_back(rate_delta);
|
||||
if mbox_buf.len() >= 60 { mbox_buf.pop_front(); }
|
||||
mbox_buf.push_back(a.mailbox_depth as u64);
|
||||
|
||||
self.actor_rows.push(ActorRow {
|
||||
address: a.address,
|
||||
worker_id: a.worker_id,
|
||||
|
|
@ -197,10 +335,21 @@ impl App {
|
|||
last_msg_type: a.last_msg_type.clone(),
|
||||
messages_processed: a.messages_processed,
|
||||
poisoned: a.poisoned,
|
||||
sparkline_rates: rates_buf.iter().copied().collect(),
|
||||
sparkline_mailbox: mbox_buf.iter().copied().collect(),
|
||||
message_type_counts: a.message_type_counts.clone(),
|
||||
});
|
||||
}
|
||||
self.sort_actors();
|
||||
|
||||
// Run warning detection
|
||||
self.warnings = self.warning_detector.check(&stats);
|
||||
|
||||
// Refresh actor logs if in detail view
|
||||
if self.view_mode == ViewMode::ActorDetail {
|
||||
self.refresh_actor_logs();
|
||||
}
|
||||
|
||||
// Clamp selection
|
||||
if !self.actor_rows.is_empty() {
|
||||
self.selected = self.selected.min(self.actor_rows.len() - 1);
|
||||
|
|
@ -246,12 +395,45 @@ impl App {
|
|||
}
|
||||
|
||||
pub fn handle_key(&mut self, key: KeyEvent) {
|
||||
// Search mode input handling
|
||||
if self.search_active {
|
||||
match key.code {
|
||||
KeyCode::Esc => {
|
||||
self.search_active = false;
|
||||
if !self.search_locked {
|
||||
self.search_query.clear();
|
||||
}
|
||||
return;
|
||||
}
|
||||
KeyCode::Enter => {
|
||||
self.search_active = false;
|
||||
self.search_locked = !self.search_query.is_empty();
|
||||
return;
|
||||
}
|
||||
KeyCode::Backspace => {
|
||||
self.search_query.pop();
|
||||
return;
|
||||
}
|
||||
KeyCode::Char(c) => {
|
||||
self.search_query.push(c);
|
||||
return;
|
||||
}
|
||||
_ => return,
|
||||
}
|
||||
}
|
||||
|
||||
// Global keys
|
||||
match key.code {
|
||||
KeyCode::Char('q') => { self.should_quit = true; return; }
|
||||
KeyCode::Char('c') if key.modifiers.contains(KeyModifiers::CONTROL) => {
|
||||
self.should_quit = true; return;
|
||||
}
|
||||
KeyCode::Char('/') => {
|
||||
self.search_active = true;
|
||||
self.search_locked = false;
|
||||
self.search_query.clear();
|
||||
return;
|
||||
}
|
||||
KeyCode::Char('s') => {
|
||||
self.sort_column = self.sort_column.next();
|
||||
self.sort_actors();
|
||||
|
|
@ -265,6 +447,7 @@ impl App {
|
|||
KeyCode::Tab => {
|
||||
self.view_mode = match self.view_mode {
|
||||
ViewMode::Overview => ViewMode::WorkerDetail,
|
||||
ViewMode::ActorDetail => ViewMode::Overview,
|
||||
#[cfg(feature = "distribution")]
|
||||
ViewMode::WorkerDetail => ViewMode::Distribution,
|
||||
#[cfg(not(feature = "distribution"))]
|
||||
|
|
@ -280,13 +463,15 @@ impl App {
|
|||
match self.view_mode {
|
||||
ViewMode::Overview => self.handle_key_overview(key),
|
||||
ViewMode::WorkerDetail => self.handle_key_worker_detail(key),
|
||||
ViewMode::ActorDetail => self.handle_key_actor_detail(key),
|
||||
#[cfg(feature = "distribution")]
|
||||
ViewMode::Distribution => self.handle_key_distribution(key),
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_key_overview(&mut self, key: KeyEvent) {
|
||||
let max = if self.actor_rows.is_empty() { 0 } else { self.actor_rows.len() - 1 };
|
||||
let visible = self.visible_actor_rows();
|
||||
let max = if visible.is_empty() { 0 } else { visible.len() - 1 };
|
||||
match key.code {
|
||||
KeyCode::Up | KeyCode::Char('k') => {
|
||||
self.selected = self.selected.saturating_sub(1);
|
||||
|
|
@ -303,11 +488,14 @@ impl App {
|
|||
KeyCode::Home => { self.selected = 0; }
|
||||
KeyCode::End => { self.selected = max; }
|
||||
KeyCode::Enter | KeyCode::Char('l') | KeyCode::Right => {
|
||||
// Enter worker detail for the selected actor's worker
|
||||
if let Some(row) = self.actor_rows.get(self.selected) {
|
||||
self.focused_worker = row.worker_id;
|
||||
// Enter actor detail for the selected visible actor
|
||||
if let Some(&row) = visible.get(self.selected) {
|
||||
self.focused_actor = Some(row.address);
|
||||
self.prev_view_mode = ViewMode::Overview;
|
||||
self.view_mode = ViewMode::ActorDetail;
|
||||
self.log_scroll = 0;
|
||||
self.refresh_actor_logs();
|
||||
}
|
||||
self.view_mode = ViewMode::WorkerDetail;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
|
@ -337,6 +525,38 @@ impl App {
|
|||
}
|
||||
}
|
||||
|
||||
fn handle_key_actor_detail(&mut self, key: KeyEvent) {
|
||||
let max_scroll = self.visible_logs().len().saturating_sub(1);
|
||||
match key.code {
|
||||
KeyCode::Esc | KeyCode::Char('h') | KeyCode::Left => {
|
||||
self.view_mode = self.prev_view_mode;
|
||||
self.focused_actor = None;
|
||||
self.actor_logs.clear();
|
||||
self.log_scroll = 0;
|
||||
}
|
||||
KeyCode::Down | KeyCode::Char('j') => {
|
||||
self.log_scroll = (self.log_scroll + 1).min(max_scroll);
|
||||
}
|
||||
KeyCode::Up | KeyCode::Char('k') => {
|
||||
self.log_scroll = self.log_scroll.saturating_sub(1);
|
||||
}
|
||||
KeyCode::PageDown => {
|
||||
self.log_scroll = (self.log_scroll + 20).min(max_scroll);
|
||||
}
|
||||
KeyCode::PageUp => {
|
||||
self.log_scroll = self.log_scroll.saturating_sub(20);
|
||||
}
|
||||
KeyCode::Home => { self.log_scroll = 0; }
|
||||
KeyCode::End => { self.log_scroll = max_scroll; }
|
||||
KeyCode::Char('1') => self.toggle_log_level(0),
|
||||
KeyCode::Char('2') => self.toggle_log_level(1),
|
||||
KeyCode::Char('3') => self.toggle_log_level(2),
|
||||
KeyCode::Char('4') => self.toggle_log_level(3),
|
||||
KeyCode::Char('5') => self.toggle_log_level(4),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "distribution")]
|
||||
fn handle_key_distribution(&mut self, key: KeyEvent) {
|
||||
let max = self
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ use swactor::runtime::Runtime;
|
|||
use crate::collector::StatsCollector;
|
||||
#[cfg(feature = "distribution")]
|
||||
use crate::distribution_collector::DistributionStatsProvider;
|
||||
use crate::layer::EventStore;
|
||||
use self::app::App;
|
||||
use self::event::{AppEvent, EventLoop};
|
||||
use self::types::RuntimeEndpoint;
|
||||
|
|
@ -47,6 +48,7 @@ pub fn start_tui(
|
|||
#[cfg(feature = "distribution")]
|
||||
distribution: Option<Arc<dyn DistributionStatsProvider>>,
|
||||
config: TuiConfig,
|
||||
event_store: Option<Arc<EventStore>>,
|
||||
) -> io::Result<()> {
|
||||
// Set up terminal
|
||||
crossterm::terminal::enable_raw_mode()?;
|
||||
|
|
@ -72,6 +74,7 @@ pub fn start_tui(
|
|||
#[cfg(feature = "distribution")]
|
||||
distribution,
|
||||
config,
|
||||
event_store,
|
||||
);
|
||||
|
||||
// Restore terminal
|
||||
|
|
@ -92,8 +95,12 @@ fn run_loop(
|
|||
#[cfg(feature = "distribution")]
|
||||
distribution: Option<Arc<dyn DistributionStatsProvider>>,
|
||||
config: TuiConfig,
|
||||
event_store: Option<Arc<EventStore>>,
|
||||
) -> io::Result<()> {
|
||||
let mut app = App::new();
|
||||
if let Some(store) = event_store {
|
||||
app.set_event_store(store);
|
||||
}
|
||||
let mut table_state = TableState::default();
|
||||
let events = EventLoop::new(config.poll_interval_ms);
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ use ratatui::Frame;
|
|||
use ratatui::layout::{Constraint, Layout, Rect};
|
||||
use ratatui::style::{Color, Modifier, Style};
|
||||
use ratatui::text::{Line, Span};
|
||||
use ratatui::widgets::{Block, Borders, Cell, Paragraph, Row, Table, TableState};
|
||||
use ratatui::widgets::{Block, Borders, Cell, Paragraph, Row, Sparkline, Table, TableState};
|
||||
|
||||
use super::app::{App, SortColumn, ViewMode};
|
||||
|
||||
|
|
@ -11,6 +11,7 @@ pub fn draw(f: &mut Frame, app: &App, table_state: &mut TableState) {
|
|||
match app.view_mode {
|
||||
ViewMode::Overview => draw_overview(f, app, table_state),
|
||||
ViewMode::WorkerDetail => draw_worker_detail(f, app, table_state),
|
||||
ViewMode::ActorDetail => draw_actor_detail(f, app),
|
||||
#[cfg(feature = "distribution")]
|
||||
ViewMode::Distribution => draw_distribution(f, app, table_state),
|
||||
}
|
||||
|
|
@ -20,17 +21,23 @@ pub fn draw(f: &mut Frame, app: &App, table_state: &mut TableState) {
|
|||
|
||||
fn draw_overview(f: &mut Frame, app: &App, table_state: &mut TableState) {
|
||||
let num_workers = app.workers.len().max(1);
|
||||
let has_sparkline_data = app.workers.iter().any(|w| w.sparkline_rates.len() > 1);
|
||||
let sparkline_height = if has_sparkline_data { 4u16 } else { 0u16 };
|
||||
|
||||
let chunks = Layout::vertical([
|
||||
Constraint::Length(num_workers as u16),
|
||||
Constraint::Length(sparkline_height),
|
||||
Constraint::Length(1),
|
||||
Constraint::Fill(1),
|
||||
])
|
||||
.split(f.area());
|
||||
|
||||
draw_worker_bars(f, app, chunks[0]);
|
||||
draw_summary(f, app, chunks[1]);
|
||||
draw_actor_table(f, app, table_state, chunks[2]);
|
||||
if has_sparkline_data {
|
||||
draw_worker_sparklines(f, app, chunks[1]);
|
||||
}
|
||||
draw_summary(f, app, chunks[2]);
|
||||
draw_actor_table(f, app, table_state, chunks[3]);
|
||||
}
|
||||
|
||||
/// Render htop-style worker bars.
|
||||
|
|
@ -45,6 +52,36 @@ fn draw_worker_bars(f: &mut Frame, app: &App, area: Rect) {
|
|||
f.render_widget(paragraph, area);
|
||||
}
|
||||
|
||||
/// Render per-worker sparklines showing message rate trends.
|
||||
fn draw_worker_sparklines(f: &mut Frame, app: &App, area: Rect) {
|
||||
if app.workers.is_empty() {
|
||||
return;
|
||||
}
|
||||
// Split area horizontally: one sparkline per worker
|
||||
let constraints: Vec<Constraint> = app
|
||||
.workers
|
||||
.iter()
|
||||
.map(|_| Constraint::Ratio(1, app.workers.len() as u32))
|
||||
.collect();
|
||||
let cols = Layout::horizontal(constraints).split(area);
|
||||
|
||||
for (i, w) in app.workers.iter().enumerate() {
|
||||
if let Some(&col_area) = cols.get(i) {
|
||||
let block = Block::default()
|
||||
.borders(Borders::NONE)
|
||||
.title(Span::styled(
|
||||
format!(" W{} ", w.id),
|
||||
Style::default().fg(Color::DarkGray),
|
||||
));
|
||||
let sparkline = Sparkline::default()
|
||||
.block(block)
|
||||
.data(&w.sparkline_rates)
|
||||
.style(Style::default().fg(Color::Green));
|
||||
f.render_widget(sparkline, col_area);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn build_worker_line(w: &super::app::WorkerView, total_width: usize) -> Line<'static> {
|
||||
let id_str = format!("{:>3}", w.id);
|
||||
let suffix = format!(
|
||||
|
|
@ -129,7 +166,7 @@ fn draw_summary(f: &mut Frame, app: &App, area: Rect) {
|
|||
Style::default().fg(Color::DarkGray)
|
||||
};
|
||||
|
||||
let line = Line::from(vec![
|
||||
let mut spans = vec![
|
||||
Span::styled(" Workers: ", Style::default().fg(Color::DarkGray)),
|
||||
Span::styled(
|
||||
format!("{}", app.num_workers),
|
||||
|
|
@ -152,13 +189,47 @@ fn draw_summary(f: &mut Frame, app: &App, area: Rect) {
|
|||
),
|
||||
Span::styled(" Panics: ", Style::default().fg(Color::DarkGray)),
|
||||
Span::styled(format!("{}", app.total_panics), panics_style),
|
||||
]);
|
||||
];
|
||||
|
||||
if !app.warnings.is_empty() {
|
||||
spans.push(Span::styled(" Warnings: ", Style::default().fg(Color::DarkGray)));
|
||||
spans.push(Span::styled(
|
||||
format!("{}", app.warnings.len()),
|
||||
Style::default().fg(Color::Yellow).add_modifier(Modifier::BOLD),
|
||||
));
|
||||
}
|
||||
|
||||
let line = Line::from(spans);
|
||||
f.render_widget(Paragraph::new(line), area);
|
||||
}
|
||||
|
||||
/// Render the actor table with scrolling and selection.
|
||||
/// Render the actor table with scrolling, selection, and search.
|
||||
fn draw_actor_table(f: &mut Frame, app: &App, table_state: &mut TableState, area: Rect) {
|
||||
// Split area: optional search bar + table
|
||||
let has_search = app.search_active || !app.search_query.is_empty();
|
||||
let search_height = if has_search { 1u16 } else { 0 };
|
||||
let chunks = Layout::vertical([
|
||||
Constraint::Length(search_height),
|
||||
Constraint::Fill(1),
|
||||
])
|
||||
.split(area);
|
||||
|
||||
// Draw search bar
|
||||
if has_search {
|
||||
let search_style = if app.search_active {
|
||||
Style::default().fg(Color::Yellow)
|
||||
} else {
|
||||
Style::default().fg(Color::DarkGray)
|
||||
};
|
||||
let cursor = if app.search_active { "\u{2588}" } else { "" };
|
||||
let prefix = if app.search_locked { " [locked] /" } else { " /" };
|
||||
let line = Line::from(vec![
|
||||
Span::styled(prefix, Style::default().fg(Color::DarkGray)),
|
||||
Span::styled(format!("{}{}", app.search_query, cursor), search_style),
|
||||
]);
|
||||
f.render_widget(Paragraph::new(line), chunks[0]);
|
||||
}
|
||||
|
||||
let sort_arrow = if app.sort_desc { " \u{25bc}" } else { " \u{25b2}" };
|
||||
|
||||
let columns = [
|
||||
|
|
@ -181,8 +252,8 @@ fn draw_actor_table(f: &mut Frame, app: &App, table_state: &mut TableState, area
|
|||
});
|
||||
let header = Row::new(header_cells).height(1);
|
||||
|
||||
let rows: Vec<Row> = app
|
||||
.actor_rows
|
||||
let visible = app.visible_actor_rows();
|
||||
let rows: Vec<Row> = visible
|
||||
.iter()
|
||||
.map(|a| actor_row_cells(a))
|
||||
.collect();
|
||||
|
|
@ -190,7 +261,22 @@ fn draw_actor_table(f: &mut Frame, app: &App, table_state: &mut TableState, area
|
|||
table_state.select(Some(app.selected));
|
||||
|
||||
let help_text =
|
||||
" q: quit \u{2191}\u{2193}: scroll s: sort column r: reverse Tab: worker view Enter: drill in";
|
||||
" q: quit /: search \u{2191}\u{2193}: scroll s: sort r: reverse Tab: worker view Enter: detail";
|
||||
|
||||
let title = if !app.search_query.is_empty() {
|
||||
format!(
|
||||
" Actors ({} of {} matching \"{}\") ",
|
||||
visible.len(),
|
||||
app.actor_rows.len(),
|
||||
app.search_query,
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
" Actors (sorted by {}{}) ",
|
||||
app.sort_column.label(),
|
||||
sort_arrow
|
||||
)
|
||||
};
|
||||
|
||||
let table = Table::new(
|
||||
rows,
|
||||
|
|
@ -206,11 +292,7 @@ fn draw_actor_table(f: &mut Frame, app: &App, table_state: &mut TableState, area
|
|||
.block(
|
||||
Block::default()
|
||||
.borders(Borders::ALL)
|
||||
.title(format!(
|
||||
" Actors (sorted by {}{}) ",
|
||||
app.sort_column.label(),
|
||||
sort_arrow
|
||||
))
|
||||
.title(title)
|
||||
.title_bottom(Line::from(help_text).centered()),
|
||||
)
|
||||
.row_highlight_style(
|
||||
|
|
@ -443,6 +525,231 @@ fn draw_focused_actor_table(f: &mut Frame, app: &App, table_state: &mut TableSta
|
|||
f.render_stateful_widget(table, area, table_state);
|
||||
}
|
||||
|
||||
// ─── Actor Detail View ──────────────────────────────────────────────────────
|
||||
|
||||
fn draw_actor_detail(f: &mut Frame, app: &App) {
|
||||
let actor = match app.focused_actor_row() {
|
||||
Some(a) => a,
|
||||
None => {
|
||||
let msg = Paragraph::new(" No actor selected. Press Esc to go back.")
|
||||
.style(Style::default().fg(Color::DarkGray));
|
||||
f.render_widget(msg, f.area());
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let has_types = !actor.message_type_counts.is_empty();
|
||||
let type_height = if has_types {
|
||||
(actor.message_type_counts.len() as u16 + 2).min(10)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
let chunks = Layout::vertical([
|
||||
Constraint::Length(5), // Info card
|
||||
Constraint::Length(5), // Rate sparkline
|
||||
Constraint::Length(5), // Mailbox sparkline
|
||||
Constraint::Length(type_height), // Type breakdown
|
||||
Constraint::Length(1), // Help bar
|
||||
Constraint::Fill(1), // Logs panel
|
||||
])
|
||||
.split(f.area());
|
||||
|
||||
// Info card
|
||||
let addr_str = format!("{}", actor.address);
|
||||
let status = if actor.poisoned { "POISONED" } else { "Healthy" };
|
||||
let status_color = if actor.poisoned { Color::Red } else { Color::Green };
|
||||
let msg_type = actor.last_msg_type.as_deref()
|
||||
.map(|s| short_type_name(Some(s)))
|
||||
.unwrap_or_else(|| "\u{2014}".to_string());
|
||||
|
||||
let info_lines = vec![
|
||||
Line::from(vec![
|
||||
Span::styled(" Address: ", Style::default().fg(Color::DarkGray)),
|
||||
Span::styled(addr_str, Style::default().fg(Color::White).add_modifier(Modifier::BOLD)),
|
||||
]),
|
||||
Line::from(vec![
|
||||
Span::styled(" Worker: ", Style::default().fg(Color::DarkGray)),
|
||||
Span::styled(format!("W{}", actor.worker_id), Style::default().fg(Color::Cyan)),
|
||||
Span::styled(" Status: ", Style::default().fg(Color::DarkGray)),
|
||||
Span::styled(status, Style::default().fg(status_color).add_modifier(Modifier::BOLD)),
|
||||
]),
|
||||
Line::from(vec![
|
||||
Span::styled(" Messages: ", Style::default().fg(Color::DarkGray)),
|
||||
Span::styled(
|
||||
format_num(actor.messages_processed),
|
||||
Style::default().fg(Color::Cyan).add_modifier(Modifier::BOLD),
|
||||
),
|
||||
Span::styled(" Mailbox: ", Style::default().fg(Color::DarkGray)),
|
||||
Span::styled(
|
||||
format!("{}", actor.mailbox_depth),
|
||||
Style::default().fg(Color::Blue).add_modifier(Modifier::BOLD),
|
||||
),
|
||||
Span::styled(" Last Msg: ", Style::default().fg(Color::DarkGray)),
|
||||
Span::styled(msg_type, Style::default().fg(Color::Green)),
|
||||
]),
|
||||
];
|
||||
|
||||
let info_block = Block::default()
|
||||
.borders(Borders::ALL)
|
||||
.title(" Actor Detail ");
|
||||
let info = Paragraph::new(info_lines).block(info_block);
|
||||
f.render_widget(info, chunks[0]);
|
||||
|
||||
// Rate sparkline
|
||||
let rate_block = Block::default()
|
||||
.borders(Borders::ALL)
|
||||
.title(Span::styled(" Msg Rate ", Style::default().fg(Color::Green)));
|
||||
let rate_sparkline = Sparkline::default()
|
||||
.block(rate_block)
|
||||
.data(&actor.sparkline_rates)
|
||||
.style(Style::default().fg(Color::Green));
|
||||
f.render_widget(rate_sparkline, chunks[1]);
|
||||
|
||||
// Mailbox sparkline
|
||||
let mbox_block = Block::default()
|
||||
.borders(Borders::ALL)
|
||||
.title(Span::styled(" Mailbox Depth ", Style::default().fg(Color::Blue)));
|
||||
let mbox_sparkline = Sparkline::default()
|
||||
.block(mbox_block)
|
||||
.data(&actor.sparkline_mailbox)
|
||||
.style(Style::default().fg(Color::Blue));
|
||||
f.render_widget(mbox_sparkline, chunks[2]);
|
||||
|
||||
// Message type breakdown
|
||||
if has_types {
|
||||
draw_type_breakdown(f, &actor.message_type_counts, chunks[3]);
|
||||
}
|
||||
|
||||
// Help bar
|
||||
let level_names = ["ERR", "WARN", "INFO", "DBG", "TRC"];
|
||||
let level_colors = [Color::Red, Color::Yellow, Color::Blue, Color::DarkGray, Color::DarkGray];
|
||||
let mut help_spans: Vec<Span> = vec![
|
||||
Span::styled(
|
||||
" Esc: back \u{2191}\u{2193}: scroll logs ",
|
||||
Style::default().fg(Color::DarkGray),
|
||||
),
|
||||
];
|
||||
for (i, &name) in level_names.iter().enumerate() {
|
||||
let active = app.log_levels[i];
|
||||
let color = if active { level_colors[i] } else { Color::DarkGray };
|
||||
let style = if active {
|
||||
Style::default().fg(color).add_modifier(Modifier::BOLD)
|
||||
} else {
|
||||
Style::default().fg(color)
|
||||
};
|
||||
help_spans.push(Span::styled(format!("{}:{} ", i + 1, name), style));
|
||||
}
|
||||
f.render_widget(Paragraph::new(Line::from(help_spans)), chunks[4]);
|
||||
|
||||
// Logs panel
|
||||
draw_actor_logs(f, app, chunks[5]);
|
||||
}
|
||||
|
||||
fn draw_type_breakdown(f: &mut Frame, types: &[(String, u64)], area: Rect) {
|
||||
let total: u64 = types.iter().map(|(_, c)| *c).sum();
|
||||
let max_count = types.first().map(|(_, c)| *c).unwrap_or(1).max(1);
|
||||
let inner_height = area.height.saturating_sub(2) as usize;
|
||||
|
||||
let bar_colors = [Color::Green, Color::Blue, Color::Yellow, Color::Magenta, Color::Cyan, Color::Red];
|
||||
|
||||
let lines: Vec<Line> = types
|
||||
.iter()
|
||||
.take(inner_height)
|
||||
.enumerate()
|
||||
.map(|(i, (name, count))| {
|
||||
let short = name.rsplit("::").next().unwrap_or(name);
|
||||
let pct = if total > 0 { *count as f64 / total as f64 * 100.0 } else { 0.0 };
|
||||
let bar_width = 20usize;
|
||||
let filled = ((*count as f64 / max_count as f64) * bar_width as f64).round() as usize;
|
||||
let color = bar_colors[i % bar_colors.len()];
|
||||
|
||||
Line::from(vec![
|
||||
Span::styled(
|
||||
format!(" {:>16} ", short),
|
||||
Style::default().fg(Color::White),
|
||||
),
|
||||
Span::styled(
|
||||
"\u{2588}".repeat(filled),
|
||||
Style::default().fg(color),
|
||||
),
|
||||
Span::styled(
|
||||
" ".repeat(bar_width.saturating_sub(filled)),
|
||||
Style::default(),
|
||||
),
|
||||
Span::styled(
|
||||
format!(" {:>8} ", count),
|
||||
Style::default().fg(Color::Cyan).add_modifier(Modifier::BOLD),
|
||||
),
|
||||
Span::styled(
|
||||
format!("{:>5.1}%", pct),
|
||||
Style::default().fg(Color::DarkGray),
|
||||
),
|
||||
])
|
||||
})
|
||||
.collect();
|
||||
|
||||
let block = Block::default()
|
||||
.borders(Borders::ALL)
|
||||
.title(format!(" Message Types ({}) ", types.len()));
|
||||
let paragraph = Paragraph::new(lines).block(block);
|
||||
f.render_widget(paragraph, area);
|
||||
}
|
||||
|
||||
fn draw_actor_logs(f: &mut Frame, app: &App, area: Rect) {
|
||||
let visible = app.visible_logs();
|
||||
let log_count = visible.len();
|
||||
let inner_height = area.height.saturating_sub(2) as usize; // borders
|
||||
|
||||
// Compute scroll window
|
||||
let scroll = app.log_scroll.min(log_count.saturating_sub(inner_height));
|
||||
|
||||
let lines: Vec<Line> = visible
|
||||
.iter()
|
||||
.skip(scroll)
|
||||
.take(inner_height)
|
||||
.map(|e| {
|
||||
let level_color = match e.level.as_str() {
|
||||
"ERROR" => Color::Red,
|
||||
"WARN" => Color::Yellow,
|
||||
"INFO" => Color::Blue,
|
||||
"DEBUG" => Color::DarkGray,
|
||||
"TRACE" => Color::DarkGray,
|
||||
_ => Color::White,
|
||||
};
|
||||
let ts = {
|
||||
let secs = e.timestamp_ms / 1000;
|
||||
let ms = e.timestamp_ms % 1000;
|
||||
let h = (secs / 3600) % 24;
|
||||
let m = (secs / 60) % 60;
|
||||
let s = secs % 60;
|
||||
format!("{:02}:{:02}:{:02}.{:03}", h, m, s, ms)
|
||||
};
|
||||
Line::from(vec![
|
||||
Span::styled(
|
||||
format!(" {} ", ts),
|
||||
Style::default().fg(Color::DarkGray),
|
||||
),
|
||||
Span::styled(
|
||||
format!("{:<5} ", e.level),
|
||||
Style::default().fg(level_color).add_modifier(Modifier::BOLD),
|
||||
),
|
||||
Span::styled(
|
||||
e.message.clone(),
|
||||
Style::default().fg(Color::White),
|
||||
),
|
||||
])
|
||||
})
|
||||
.collect();
|
||||
|
||||
let title = format!(" Logs ({}) ", log_count);
|
||||
let block = Block::default()
|
||||
.borders(Borders::ALL)
|
||||
.title(title);
|
||||
let paragraph = Paragraph::new(lines).block(block);
|
||||
f.render_widget(paragraph, area);
|
||||
}
|
||||
|
||||
// ─── Distribution View ──────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(feature = "distribution")]
|
||||
|
|
|
|||
392
crates/runtime-dashboard/src/warnings.rs
Normal file
392
crates/runtime-dashboard/src/warnings.rs
Normal file
|
|
@ -0,0 +1,392 @@
|
|||
//! Automated anomaly detection for the runtime dashboard.
|
||||
//!
|
||||
//! Runs on each stats sample, comparing consecutive snapshots to detect
|
||||
//! growing mailboxes, stalled actors, worker imbalance, and other conditions.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use swactor::actor::ActorAddress;
|
||||
use swactor::stats::RuntimeStats;
|
||||
|
||||
/// Types of warnings the detector can produce.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum WarningType {
|
||||
GrowingMailbox,
|
||||
StalledActor,
|
||||
PoisonedActor,
|
||||
WorkerImbalance,
|
||||
EmptyWorker,
|
||||
MailboxOverflow,
|
||||
}
|
||||
|
||||
/// Severity levels for warnings.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum Severity {
|
||||
Low,
|
||||
Medium,
|
||||
High,
|
||||
Critical,
|
||||
}
|
||||
|
||||
/// An active warning.
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct Warning {
|
||||
pub warning_type: WarningType,
|
||||
pub severity: Severity,
|
||||
pub entity: String,
|
||||
pub description: String,
|
||||
}
|
||||
|
||||
/// Configuration for warning thresholds.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct WarningConfig {
|
||||
/// Consecutive samples with increasing mailbox depth before warning.
|
||||
pub growing_mailbox_threshold: usize,
|
||||
/// Consecutive ticks with no message processing while mailbox > 0.
|
||||
pub stalled_actor_threshold: usize,
|
||||
/// A worker is "imbalanced" if it has > this ratio times the average load.
|
||||
pub worker_imbalance_ratio: f64,
|
||||
}
|
||||
|
||||
impl Default for WarningConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
growing_mailbox_threshold: 5,
|
||||
stalled_actor_threshold: 10,
|
||||
worker_imbalance_ratio: 2.0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-actor tracking state.
|
||||
struct ActorState {
|
||||
prev_mailbox: usize,
|
||||
prev_messages: u64,
|
||||
growing_streak: usize,
|
||||
stalled_streak: usize,
|
||||
}
|
||||
|
||||
/// Warning detection engine. Call `check()` on each stats sample.
|
||||
pub struct WarningDetector {
|
||||
config: WarningConfig,
|
||||
actors: HashMap<ActorAddress, ActorState>,
|
||||
}
|
||||
|
||||
impl WarningDetector {
|
||||
pub fn new(config: WarningConfig) -> Self {
|
||||
Self {
|
||||
config,
|
||||
actors: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Analyze a stats snapshot and return all active warnings.
|
||||
pub fn check(&mut self, stats: &RuntimeStats) -> Vec<Warning> {
|
||||
let mut warnings = Vec::new();
|
||||
|
||||
// Track which actors are still alive
|
||||
let mut live_addrs: std::collections::HashSet<ActorAddress> =
|
||||
std::collections::HashSet::new();
|
||||
|
||||
for actor in &stats.actor_details {
|
||||
live_addrs.insert(actor.address);
|
||||
|
||||
// Poisoned actor — immediate critical warning
|
||||
if actor.poisoned {
|
||||
warnings.push(Warning {
|
||||
warning_type: WarningType::PoisonedActor,
|
||||
severity: Severity::Critical,
|
||||
entity: format!("{}", actor.address),
|
||||
description: "Actor is poisoned (panicked)".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let state = self.actors.entry(actor.address).or_insert(ActorState {
|
||||
prev_mailbox: actor.mailbox_depth,
|
||||
prev_messages: actor.messages_processed,
|
||||
growing_streak: 0,
|
||||
stalled_streak: 0,
|
||||
});
|
||||
|
||||
// Growing mailbox detection
|
||||
if actor.mailbox_depth > state.prev_mailbox && actor.mailbox_depth > 0 {
|
||||
state.growing_streak += 1;
|
||||
} else {
|
||||
state.growing_streak = 0;
|
||||
}
|
||||
|
||||
if state.growing_streak >= self.config.growing_mailbox_threshold {
|
||||
warnings.push(Warning {
|
||||
warning_type: WarningType::GrowingMailbox,
|
||||
severity: Severity::Medium,
|
||||
entity: format!("{}", actor.address),
|
||||
description: format!(
|
||||
"Mailbox growing for {} consecutive samples (depth: {})",
|
||||
state.growing_streak, actor.mailbox_depth,
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
// Stalled actor detection
|
||||
if actor.messages_processed == state.prev_messages && actor.mailbox_depth > 0 {
|
||||
state.stalled_streak += 1;
|
||||
} else {
|
||||
state.stalled_streak = 0;
|
||||
}
|
||||
|
||||
if state.stalled_streak >= self.config.stalled_actor_threshold {
|
||||
warnings.push(Warning {
|
||||
warning_type: WarningType::StalledActor,
|
||||
severity: Severity::High,
|
||||
entity: format!("{}", actor.address),
|
||||
description: format!(
|
||||
"No messages processed for {} ticks with {} pending",
|
||||
state.stalled_streak, actor.mailbox_depth,
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
state.prev_mailbox = actor.mailbox_depth;
|
||||
state.prev_messages = actor.messages_processed;
|
||||
}
|
||||
|
||||
// Clean up dead actors
|
||||
self.actors.retain(|addr, _| live_addrs.contains(addr));
|
||||
|
||||
// Mailbox overflow detection
|
||||
for w in &stats.workers {
|
||||
if w.messages_dropped > 0 {
|
||||
warnings.push(Warning {
|
||||
warning_type: WarningType::MailboxOverflow,
|
||||
severity: Severity::Medium,
|
||||
entity: format!("Worker {}", w.id),
|
||||
description: format!("{} messages dropped", w.messages_dropped),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Worker imbalance and empty worker detection
|
||||
if stats.workers.len() > 1 {
|
||||
let total_actors: usize = stats.workers.iter().map(|w| w.num_actors).sum();
|
||||
let avg = total_actors as f64 / stats.workers.len() as f64;
|
||||
|
||||
for w in &stats.workers {
|
||||
if avg > 0.0 && w.num_actors as f64 > avg * self.config.worker_imbalance_ratio {
|
||||
warnings.push(Warning {
|
||||
warning_type: WarningType::WorkerImbalance,
|
||||
severity: Severity::Low,
|
||||
entity: format!("Worker {}", w.id),
|
||||
description: format!(
|
||||
"{} actors vs {:.0} average ({:.1}x)",
|
||||
w.num_actors, avg, w.num_actors as f64 / avg,
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
if w.num_actors == 0 && total_actors > 0 {
|
||||
warnings.push(Warning {
|
||||
warning_type: WarningType::EmptyWorker,
|
||||
severity: Severity::Low,
|
||||
entity: format!("Worker {}", w.id),
|
||||
description: "Worker has no actors while others do".to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by severity (critical first)
|
||||
warnings.sort_by(|a, b| b.severity.cmp(&a.severity));
|
||||
warnings
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use swactor::stats::{ActorInfo, WorkerInfo};
|
||||
|
||||
fn make_worker(id: usize, actors: usize, dropped: u64) -> WorkerInfo {
|
||||
WorkerInfo {
|
||||
id,
|
||||
num_actors: actors,
|
||||
mailbox_depth: 0,
|
||||
messages_processed: 0,
|
||||
local_sends: 0,
|
||||
cross_sends: 0,
|
||||
inbox_sends: 0,
|
||||
type_mismatches: 0,
|
||||
panics: 0,
|
||||
messages_dropped: dropped,
|
||||
restarts: 0,
|
||||
stops: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn make_actor(id: u8, depth: usize, msgs: u64, poisoned: bool) -> ActorInfo {
|
||||
ActorInfo {
|
||||
address: ActorAddress([id; 32]),
|
||||
worker_id: 0,
|
||||
mailbox_depth: depth,
|
||||
last_msg_type: None,
|
||||
messages_processed: msgs,
|
||||
poisoned,
|
||||
message_type_counts: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn make_stats(workers: Vec<WorkerInfo>, actors: Vec<ActorInfo>) -> RuntimeStats {
|
||||
RuntimeStats {
|
||||
num_workers: workers.len(),
|
||||
uptime_ms: 0,
|
||||
actors: actors.iter().map(|a| (a.address, a.worker_id)).collect(),
|
||||
workers,
|
||||
actor_details: actors,
|
||||
tick_timings: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn poisoned_actor_triggers_critical_warning() {
|
||||
let mut detector = WarningDetector::new(WarningConfig::default());
|
||||
let stats = make_stats(
|
||||
vec![make_worker(0, 1, 0)],
|
||||
vec![make_actor(1, 0, 10, true)],
|
||||
);
|
||||
let warnings = detector.check(&stats);
|
||||
assert!(warnings.iter().any(|w| w.warning_type == WarningType::PoisonedActor));
|
||||
assert!(warnings.iter().any(|w| w.severity == Severity::Critical));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn growing_mailbox_triggers_after_threshold() {
|
||||
let config = WarningConfig {
|
||||
growing_mailbox_threshold: 3,
|
||||
..Default::default()
|
||||
};
|
||||
let mut detector = WarningDetector::new(config);
|
||||
|
||||
// 4 samples with increasing mailbox: should trigger at sample 4
|
||||
for depth in 1..=4 {
|
||||
let stats = make_stats(
|
||||
vec![make_worker(0, 1, 0)],
|
||||
vec![make_actor(1, depth, 0, false)],
|
||||
);
|
||||
let warnings = detector.check(&stats);
|
||||
if depth < 4 {
|
||||
assert!(!warnings.iter().any(|w| w.warning_type == WarningType::GrowingMailbox),
|
||||
"should not trigger at depth {}", depth);
|
||||
} else {
|
||||
assert!(warnings.iter().any(|w| w.warning_type == WarningType::GrowingMailbox),
|
||||
"should trigger at depth {}", depth);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn growing_mailbox_resets_on_decrease() {
|
||||
let config = WarningConfig {
|
||||
growing_mailbox_threshold: 3,
|
||||
..Default::default()
|
||||
};
|
||||
let mut detector = WarningDetector::new(config);
|
||||
|
||||
// Grow for 2 samples, then decrease, then grow again
|
||||
for depth in [1, 2, 1, 2, 3, 4] {
|
||||
let stats = make_stats(
|
||||
vec![make_worker(0, 1, 0)],
|
||||
vec![make_actor(1, depth, 0, false)],
|
||||
);
|
||||
detector.check(&stats);
|
||||
}
|
||||
// After 1,2 → streak=2; then 1 → streak=0; then 2,3,4 → streak=3 → triggers
|
||||
let stats = make_stats(
|
||||
vec![make_worker(0, 1, 0)],
|
||||
vec![make_actor(1, 5, 0, false)],
|
||||
);
|
||||
let warnings = detector.check(&stats);
|
||||
assert!(warnings.iter().any(|w| w.warning_type == WarningType::GrowingMailbox));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stalled_actor_triggers_when_not_processing() {
|
||||
let config = WarningConfig {
|
||||
stalled_actor_threshold: 3,
|
||||
..Default::default()
|
||||
};
|
||||
let mut detector = WarningDetector::new(config);
|
||||
|
||||
// Same messages_processed, nonzero mailbox for 4 ticks
|
||||
for _ in 0..4 {
|
||||
let stats = make_stats(
|
||||
vec![make_worker(0, 1, 0)],
|
||||
vec![make_actor(1, 5, 100, false)],
|
||||
);
|
||||
let warnings = detector.check(&stats);
|
||||
// Last one should trigger
|
||||
if warnings.iter().any(|w| w.warning_type == WarningType::StalledActor) {
|
||||
return; // test passed
|
||||
}
|
||||
}
|
||||
panic!("expected StalledActor warning");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_imbalance_detected() {
|
||||
let mut detector = WarningDetector::new(WarningConfig::default());
|
||||
// Worker 0: 10 actors, Worker 1: 1 actor. Avg=5.5, ratio=10/5.5=1.8
|
||||
// With ratio threshold 2.0, this should NOT trigger
|
||||
let stats = make_stats(
|
||||
vec![make_worker(0, 10, 0), make_worker(1, 1, 0)],
|
||||
vec![],
|
||||
);
|
||||
let warnings = detector.check(&stats);
|
||||
assert!(!warnings.iter().any(|w| w.warning_type == WarningType::WorkerImbalance));
|
||||
|
||||
// Worker 0: 20 actors, Worker 1: 1 actor. Avg=10.5, ratio=20/10.5=1.9 — still no
|
||||
// Worker 0: 30 actors, Worker 1: 1 actor. Avg=15.5, ratio=30/15.5=1.9 — still no
|
||||
// Worker 0: 100 actors, Worker 1: 1 actor. Avg=50.5, ratio=100/50.5=1.98 — almost
|
||||
// Worker 0: 100 actors, Worker 1: 0 actor. Avg=50, ratio=100/50=2.0 — at threshold
|
||||
|
||||
let stats2 = make_stats(
|
||||
vec![make_worker(0, 100, 0), make_worker(1, 1, 0)],
|
||||
vec![],
|
||||
);
|
||||
let warnings2 = detector.check(&stats2);
|
||||
// 100 / 50.5 = 1.98 — not > 2.0
|
||||
assert!(!warnings2.iter().any(|w| w.warning_type == WarningType::WorkerImbalance));
|
||||
|
||||
// Now 200 vs 1: 200/100.5 = ~1.99 — still not. Let's do 300 vs 1: 300/150.5 = ~2.0
|
||||
// Actually need > 2x. Let's do 50 vs 1: avg=25.5, ratio=50/25.5=1.96. Nope.
|
||||
// 10 vs 1 vs 1: avg=4, ratio=10/4=2.5 — triggers!
|
||||
let stats3 = make_stats(
|
||||
vec![make_worker(0, 10, 0), make_worker(1, 1, 0), make_worker(2, 1, 0)],
|
||||
vec![],
|
||||
);
|
||||
let warnings3 = detector.check(&stats3);
|
||||
assert!(warnings3.iter().any(|w| w.warning_type == WarningType::WorkerImbalance));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_worker_detected() {
|
||||
let mut detector = WarningDetector::new(WarningConfig::default());
|
||||
let stats = make_stats(
|
||||
vec![make_worker(0, 5, 0), make_worker(1, 0, 0)],
|
||||
vec![],
|
||||
);
|
||||
let warnings = detector.check(&stats);
|
||||
assert!(warnings.iter().any(|w| w.warning_type == WarningType::EmptyWorker));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mailbox_overflow_detected() {
|
||||
let mut detector = WarningDetector::new(WarningConfig::default());
|
||||
let stats = make_stats(
|
||||
vec![make_worker(0, 1, 42)],
|
||||
vec![],
|
||||
);
|
||||
let warnings = detector.check(&stats);
|
||||
assert!(warnings.iter().any(|w| w.warning_type == WarningType::MailboxOverflow));
|
||||
}
|
||||
}
|
||||
28
docs/development_history/dashboard-improvements-research.md
Normal file
28
docs/development_history/dashboard-improvements-research.md
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
# Dashboard Improvements — Research Phase
|
||||
|
||||
## Summary
|
||||
Researched 10 comparable monitoring/dashboard systems to inform swactor's dashboard improvement plan.
|
||||
|
||||
## Systems Analyzed
|
||||
- **Actor runtimes**: Erlang Observer (GUI/CLI/Web), Akka Insights, Ray Dashboard, Orleans Dashboard
|
||||
- **Async/runtime tools**: tokio-console, Lunatic
|
||||
- **Message/infrastructure**: RabbitMQ Management, Consul UI, Nomad UI
|
||||
- **Web frameworks**: Phoenix LiveDashboard
|
||||
|
||||
## Key Findings
|
||||
1. **Time-series history** is table-stakes — every system provides it
|
||||
2. **Actor detail drill-down** is universal (Observer has 6-tab process info, Orleans has grain state inspection)
|
||||
3. **Search/filter** exists in every system
|
||||
4. **Warning/anomaly detection** (tokio-console's lint system) is a high-value differentiator
|
||||
5. **Topology visualization** (Consul golden metrics, Observer supervision tree) is rare but powerful
|
||||
|
||||
## Implementation Plan
|
||||
8 feature stages defined (see `CLAUDE/notes/feature-stages/`):
|
||||
1. Time-Series History Infrastructure
|
||||
2. Actor Detail Drill-Down
|
||||
3. Search and Filter
|
||||
4. Per-Worker Utilization Visualization
|
||||
5. Warning/Anomaly Detection
|
||||
6. Actor-to-Actor Message Flow Topology
|
||||
7. Per-Actor Logging
|
||||
8. Per-Message-Type Breakdown
|
||||
|
|
@ -120,6 +120,8 @@ pub struct ActorSnapshot {
|
|||
pub last_msg_type: Option<&'static str>,
|
||||
pub messages_processed: u64,
|
||||
pub poisoned: bool,
|
||||
/// Per-message-type counts, sorted descending by count.
|
||||
pub message_type_counts: Vec<(&'static str, u64)>,
|
||||
}
|
||||
|
||||
/// Observer hook called by workers after productive ticks.
|
||||
|
|
@ -150,6 +152,9 @@ pub struct ActorInfo {
|
|||
/// Whether the actor has panicked and is no longer processing messages.
|
||||
#[cfg_attr(feature = "serde", serde(default))]
|
||||
pub poisoned: bool,
|
||||
/// Per-message-type counts, sorted descending by count. Top 32 types.
|
||||
#[cfg_attr(feature = "serde", serde(default))]
|
||||
pub message_type_counts: Vec<(String, u64)>,
|
||||
}
|
||||
|
||||
/// Snapshot of overall runtime state.
|
||||
|
|
|
|||
|
|
@ -610,6 +610,8 @@ struct ActorSlot {
|
|||
started: bool,
|
||||
last_msg_type: Option<&'static str>,
|
||||
messages_processed: u64,
|
||||
/// Per-message-type counters (bounded to 32 entries).
|
||||
msg_type_counts: HashMap<&'static str, u64>,
|
||||
/// Per-actor mailbox capacity. 0 = unbounded.
|
||||
mailbox_capacity: usize,
|
||||
overflow_policy: MailboxOverflow,
|
||||
|
|
@ -645,6 +647,7 @@ impl ActorPool {
|
|||
started: false,
|
||||
last_msg_type: None,
|
||||
messages_processed: 0,
|
||||
msg_type_counts: HashMap::new(),
|
||||
mailbox_capacity: self.default_mailbox_capacity,
|
||||
overflow_policy: self.default_overflow_policy,
|
||||
});
|
||||
|
|
@ -698,6 +701,9 @@ impl ActorPool {
|
|||
continue;
|
||||
}
|
||||
|
||||
#[cfg(feature = "tracing")]
|
||||
let _actor_span = tracing::trace_span!("actor.tick", actor_addr = %addr).entered();
|
||||
|
||||
let ctx = Ctx::new(inner, addr);
|
||||
|
||||
// Call on_start once, before first message
|
||||
|
|
@ -762,6 +768,10 @@ impl ActorPool {
|
|||
Ok(Some(type_name)) => {
|
||||
slot.last_msg_type = Some(type_name);
|
||||
slot.messages_processed += 1;
|
||||
// Track per-type counts (bounded to 32 distinct types)
|
||||
if slot.msg_type_counts.len() < 32 || slot.msg_type_counts.contains_key(type_name) {
|
||||
*slot.msg_type_counts.entry(type_name).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
count += 1;
|
||||
|
|
@ -834,12 +844,16 @@ impl ActorPool {
|
|||
pub fn mailbox_depths_into(&self, out: &mut Vec<ActorSnapshot>) {
|
||||
out.clear();
|
||||
out.extend(self.actors.iter().map(|(&addr, slot)| {
|
||||
let mut type_counts: Vec<(&'static str, u64)> =
|
||||
slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect();
|
||||
type_counts.sort_by(|a, b| b.1.cmp(&a.1));
|
||||
ActorSnapshot {
|
||||
address: addr,
|
||||
mailbox_depth: slot.mailbox.len(),
|
||||
last_msg_type: slot.last_msg_type,
|
||||
messages_processed: slot.messages_processed,
|
||||
poisoned: slot.poisoned,
|
||||
message_type_counts: type_counts,
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue