2026-02-11 15:23:26 +00:00
|
|
|
pub mod collector;
|
2026-02-13 08:49:19 +00:00
|
|
|
pub mod history;
|
2026-02-12 09:13:50 +00:00
|
|
|
pub mod investigate;
|
2026-02-09 09:04:57 +00:00
|
|
|
pub mod layer;
|
|
|
|
|
pub mod trace;
|
2026-02-11 15:23:26 +00:00
|
|
|
mod actors_html;
|
2026-02-09 09:04:57 +00:00
|
|
|
mod dashboard_html;
|
|
|
|
|
mod server;
|
|
|
|
|
|
2026-02-11 15:23:26 +00:00
|
|
|
#[cfg(feature = "tui")]
|
|
|
|
|
pub mod tui;
|
|
|
|
|
|
2026-02-12 09:13:50 +00:00
|
|
|
#[cfg(feature = "distribution")]
|
|
|
|
|
mod distribution_html;
|
|
|
|
|
#[cfg(feature = "distribution")]
|
|
|
|
|
pub mod distribution_collector;
|
2026-02-11 15:23:26 +00:00
|
|
|
|
2026-02-09 09:04:57 +00:00
|
|
|
use std::io;
|
|
|
|
|
use std::sync::atomic::{AtomicBool, Ordering};
|
|
|
|
|
use std::sync::{Arc, Mutex};
|
|
|
|
|
use std::thread;
|
|
|
|
|
use std::time::Duration;
|
|
|
|
|
|
2026-02-11 15:23:26 +00:00
|
|
|
use crossbeam_queue::ArrayQueue;
|
2026-02-09 09:04:57 +00:00
|
|
|
use swactor::config::RuntimeConfig;
|
|
|
|
|
use swactor::runtime::{Runtime, RuntimeHandle};
|
|
|
|
|
use tracing_subscriber::layer::SubscriberExt;
|
|
|
|
|
use tracing_subscriber::util::SubscriberInitExt;
|
|
|
|
|
|
2026-02-11 15:23:26 +00:00
|
|
|
use crate::collector::StatsCollector;
|
2026-02-13 08:49:19 +00:00
|
|
|
use crate::history::{DashboardHistory, HistoryConfig};
|
2026-02-09 09:04:57 +00:00
|
|
|
use crate::layer::{now_ms, DashboardLayer, EventStore};
|
|
|
|
|
use crate::trace::{RuntimeTrace, TimestampedStats};
|
|
|
|
|
|
|
|
|
|
/// Configuration for the runtime dashboard.
|
|
|
|
|
#[derive(Debug, Clone)]
|
|
|
|
|
pub struct DashboardConfig {
|
|
|
|
|
pub port: u16,
|
|
|
|
|
pub event_capacity: usize,
|
2026-02-11 15:23:26 +00:00
|
|
|
/// Enable trace recording for `save_trace()`. When true, events and stats
|
|
|
|
|
/// are kept in lock-free bounded ring buffers and stats are periodically sampled.
|
2026-02-09 09:04:57 +00:00
|
|
|
pub record: bool,
|
2026-02-11 15:23:26 +00:00
|
|
|
/// Maximum events retained in the recording log. Only used when `record = true`.
|
|
|
|
|
pub record_event_capacity: usize,
|
|
|
|
|
/// Maximum stats snapshots retained in the timeline. Only used when `record = true`.
|
|
|
|
|
pub record_stats_capacity: usize,
|
2026-02-09 09:04:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
impl Default for DashboardConfig {
|
|
|
|
|
fn default() -> Self {
|
|
|
|
|
Self {
|
|
|
|
|
port: 9090,
|
|
|
|
|
event_capacity: 10_000,
|
|
|
|
|
record: false,
|
2026-02-11 15:23:26 +00:00
|
|
|
record_event_capacity: 100_000,
|
|
|
|
|
record_stats_capacity: 18_000,
|
2026-02-09 09:04:57 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Configuration for replaying a recorded trace.
|
|
|
|
|
#[derive(Debug, Clone)]
|
|
|
|
|
pub struct ReplayConfig {
|
|
|
|
|
pub port: u16,
|
|
|
|
|
/// Playback speed multiplier (1.0 = real-time, 2.0 = double speed).
|
|
|
|
|
pub speed: f64,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
impl Default for ReplayConfig {
|
|
|
|
|
fn default() -> Self {
|
|
|
|
|
Self {
|
|
|
|
|
port: 9090,
|
|
|
|
|
speed: 1.0,
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Handle to a running dashboard. Allows attaching a runtime after creation.
|
|
|
|
|
pub struct DashboardHandle {
|
|
|
|
|
store: Arc<EventStore>,
|
|
|
|
|
runtime: Arc<Mutex<Option<Arc<Runtime>>>>,
|
2026-02-11 15:23:26 +00:00
|
|
|
collector: Arc<Mutex<Option<Arc<StatsCollector>>>>,
|
2026-02-09 09:04:57 +00:00
|
|
|
shutdown: Arc<AtomicBool>,
|
2026-02-11 15:23:26 +00:00
|
|
|
stats_timeline: Arc<ArrayQueue<TimestampedStats>>,
|
2026-02-13 08:49:19 +00:00
|
|
|
history: Arc<DashboardHistory>,
|
2026-02-09 09:04:57 +00:00
|
|
|
recording: bool,
|
2026-02-12 09:13:50 +00:00
|
|
|
#[cfg(feature = "distribution")]
|
|
|
|
|
distribution: Arc<Mutex<Option<Arc<dyn distribution_collector::DistributionStatsProvider>>>>,
|
2026-02-09 09:04:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
impl DashboardHandle {
|
|
|
|
|
/// Install a global tracing subscriber with the dashboard layer.
|
|
|
|
|
pub fn install_tracing(&self) {
|
|
|
|
|
tracing_subscriber::registry()
|
|
|
|
|
.with(DashboardLayer::new(Arc::clone(&self.store)))
|
|
|
|
|
.init();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Return the raw `DashboardLayer` for users who want to compose their own subscriber.
|
|
|
|
|
pub fn layer(&self) -> DashboardLayer {
|
|
|
|
|
DashboardLayer::new(Arc::clone(&self.store))
|
|
|
|
|
}
|
|
|
|
|
|
2026-02-11 15:23:26 +00:00
|
|
|
/// Attach a runtime and its stats collector, enabling stats polling.
|
|
|
|
|
pub fn set_runtime(&self, runtime: Arc<Runtime>, collector: Arc<StatsCollector>) {
|
2026-02-09 09:04:57 +00:00
|
|
|
*self.runtime.lock().unwrap() = Some(runtime);
|
2026-02-11 15:23:26 +00:00
|
|
|
*self.collector.lock().unwrap() = Some(collector);
|
2026-02-09 09:04:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Whether trace recording is enabled.
|
|
|
|
|
pub fn is_recording(&self) -> bool {
|
|
|
|
|
self.recording
|
|
|
|
|
}
|
|
|
|
|
|
2026-02-12 09:13:50 +00:00
|
|
|
/// Attach a distribution stats provider, enabling the `/distribution` page.
|
|
|
|
|
#[cfg(feature = "distribution")]
|
|
|
|
|
pub fn set_distribution(&self, provider: Arc<dyn distribution_collector::DistributionStatsProvider>) {
|
|
|
|
|
*self.distribution.lock().unwrap() = Some(provider);
|
|
|
|
|
}
|
|
|
|
|
|
2026-02-13 08:49:19 +00:00
|
|
|
/// Access the time-series history store (for TUI sparklines, etc.).
|
|
|
|
|
pub fn history(&self) -> &Arc<DashboardHistory> {
|
|
|
|
|
&self.history
|
|
|
|
|
}
|
|
|
|
|
|
2026-02-09 09:04:57 +00:00
|
|
|
/// Signal the dashboard to shut down (SSE clients receive "done").
|
|
|
|
|
pub fn shutdown(&self) {
|
|
|
|
|
self.shutdown.store(true, Ordering::Release);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Save the recorded trace to a JSON file.
|
|
|
|
|
///
|
|
|
|
|
/// Only works when `DashboardConfig::record` was set to `true`.
|
2026-02-11 15:23:26 +00:00
|
|
|
/// This drains the recording buffers — each call consumes the buffered data.
|
2026-02-09 09:04:57 +00:00
|
|
|
pub fn save_trace(&self, path: &str) -> io::Result<()> {
|
|
|
|
|
let events = self.store.all_events().ok_or_else(|| {
|
|
|
|
|
io::Error::new(
|
|
|
|
|
io::ErrorKind::Other,
|
|
|
|
|
"recording not enabled (set DashboardConfig::record = true)",
|
|
|
|
|
)
|
|
|
|
|
})?;
|
2026-02-11 15:23:26 +00:00
|
|
|
let mut stats_timeline = Vec::new();
|
|
|
|
|
while let Some(ts) = self.stats_timeline.pop() {
|
|
|
|
|
stats_timeline.push(ts);
|
|
|
|
|
}
|
2026-02-09 09:04:57 +00:00
|
|
|
let trace = RuntimeTrace {
|
|
|
|
|
events,
|
|
|
|
|
stats_timeline,
|
|
|
|
|
};
|
|
|
|
|
let json = serde_json::to_string(&trace)
|
|
|
|
|
.map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
|
|
|
|
|
std::fs::write(path, json)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Start a dashboard server and return a handle.
|
|
|
|
|
///
|
|
|
|
|
/// The dashboard starts serving immediately. Call `install_tracing()` to set up
|
|
|
|
|
/// the global subscriber, and `set_runtime()` to enable stats polling.
|
|
|
|
|
pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle {
|
2026-02-11 15:23:26 +00:00
|
|
|
let store = Arc::new(EventStore::new(
|
|
|
|
|
config.event_capacity,
|
|
|
|
|
config.record,
|
|
|
|
|
config.record_event_capacity,
|
|
|
|
|
));
|
2026-02-09 09:04:57 +00:00
|
|
|
let runtime: Arc<Mutex<Option<Arc<Runtime>>>> = Arc::new(Mutex::new(None));
|
2026-02-11 15:23:26 +00:00
|
|
|
let collector: Arc<Mutex<Option<Arc<StatsCollector>>>> = Arc::new(Mutex::new(None));
|
2026-02-09 09:04:57 +00:00
|
|
|
let shutdown = Arc::new(AtomicBool::new(false));
|
2026-02-11 15:23:26 +00:00
|
|
|
let stats_timeline = Arc::new(ArrayQueue::new(config.record_stats_capacity.max(1)));
|
2026-02-13 08:49:19 +00:00
|
|
|
let history = Arc::new(DashboardHistory::new(HistoryConfig::default()));
|
2026-02-09 09:04:57 +00:00
|
|
|
|
2026-02-12 09:13:50 +00:00
|
|
|
#[cfg(feature = "distribution")]
|
|
|
|
|
let distribution: Arc<Mutex<Option<Arc<dyn distribution_collector::DistributionStatsProvider>>>> =
|
|
|
|
|
Arc::new(Mutex::new(None));
|
|
|
|
|
|
2026-02-09 09:04:57 +00:00
|
|
|
server::spawn_http_server(
|
|
|
|
|
Arc::clone(&store),
|
|
|
|
|
Arc::clone(&runtime),
|
2026-02-11 15:23:26 +00:00
|
|
|
Arc::clone(&collector),
|
2026-02-09 09:04:57 +00:00
|
|
|
Arc::clone(&shutdown),
|
2026-02-13 08:49:19 +00:00
|
|
|
Arc::clone(&history),
|
2026-02-09 09:04:57 +00:00
|
|
|
config.port,
|
2026-02-12 09:13:50 +00:00
|
|
|
#[cfg(feature = "distribution")]
|
|
|
|
|
Arc::clone(&distribution),
|
2026-02-09 09:04:57 +00:00
|
|
|
);
|
|
|
|
|
|
|
|
|
|
// Start stats recorder thread when recording is enabled
|
|
|
|
|
if config.record {
|
|
|
|
|
let rt_ref = Arc::clone(&runtime);
|
2026-02-11 15:23:26 +00:00
|
|
|
let col_ref = Arc::clone(&collector);
|
2026-02-09 09:04:57 +00:00
|
|
|
let timeline = Arc::clone(&stats_timeline);
|
|
|
|
|
let stop = Arc::clone(&shutdown);
|
|
|
|
|
thread::spawn(move || {
|
|
|
|
|
loop {
|
|
|
|
|
if stop.load(Ordering::Relaxed) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
let maybe_rt = rt_ref.lock().unwrap().clone();
|
|
|
|
|
if let Some(rt) = maybe_rt {
|
2026-02-11 15:23:26 +00:00
|
|
|
let mut stats = rt.stats();
|
|
|
|
|
if let Some(col) = col_ref.lock().unwrap().as_ref() {
|
|
|
|
|
col.enrich(&mut stats);
|
|
|
|
|
}
|
2026-02-09 09:04:57 +00:00
|
|
|
let ts = TimestampedStats {
|
|
|
|
|
timestamp_ms: now_ms(),
|
|
|
|
|
stats,
|
|
|
|
|
};
|
2026-02-11 15:23:26 +00:00
|
|
|
let _ = timeline.force_push(ts);
|
2026-02-09 09:04:57 +00:00
|
|
|
}
|
|
|
|
|
thread::sleep(Duration::from_millis(200));
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
eprintln!("Runtime dashboard at http://localhost:{}", config.port);
|
|
|
|
|
|
|
|
|
|
DashboardHandle {
|
|
|
|
|
store,
|
|
|
|
|
runtime,
|
2026-02-11 15:23:26 +00:00
|
|
|
collector,
|
2026-02-09 09:04:57 +00:00
|
|
|
shutdown,
|
|
|
|
|
stats_timeline,
|
2026-02-13 08:49:19 +00:00
|
|
|
history,
|
2026-02-09 09:04:57 +00:00
|
|
|
recording: config.record,
|
2026-02-12 09:13:50 +00:00
|
|
|
#[cfg(feature = "distribution")]
|
|
|
|
|
distribution,
|
2026-02-09 09:04:57 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Convenience: create a runtime, start a dashboard, install tracing, and run.
|
|
|
|
|
///
|
|
|
|
|
/// Returns the runtime handle and dashboard handle.
|
|
|
|
|
pub fn run_with_dashboard(
|
|
|
|
|
rt_config: RuntimeConfig,
|
|
|
|
|
dash_config: DashboardConfig,
|
|
|
|
|
) -> (RuntimeHandle, DashboardHandle) {
|
|
|
|
|
let dash = start_dashboard(dash_config);
|
|
|
|
|
dash.install_tracing();
|
|
|
|
|
|
2026-02-11 15:23:26 +00:00
|
|
|
let num_workers = if rt_config.num_threads < 2 { 1 } else { rt_config.num_threads };
|
|
|
|
|
let collector = StatsCollector::new(num_workers);
|
|
|
|
|
|
|
|
|
|
let mut rt = Runtime::new(rt_config);
|
|
|
|
|
rt.set_stats_hook(collector.clone());
|
2026-02-09 09:04:57 +00:00
|
|
|
let handle = rt.run().expect("failed to start runtime");
|
2026-02-11 15:23:26 +00:00
|
|
|
dash.set_runtime(Arc::clone(&handle.runtime), collector);
|
2026-02-09 09:04:57 +00:00
|
|
|
|
|
|
|
|
(handle, dash)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Load a trace file and serve a replay dashboard. Blocks indefinitely.
|
|
|
|
|
pub fn serve_replay(path: &str, config: ReplayConfig) -> io::Result<()> {
|
|
|
|
|
let data = std::fs::read_to_string(path)?;
|
|
|
|
|
let trace: RuntimeTrace = serde_json::from_str(&data)
|
|
|
|
|
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
|
|
|
|
|
|
|
|
|
|
server::spawn_replay_server(Arc::new(trace), config.port, config.speed);
|
|
|
|
|
|
|
|
|
|
eprintln!("Replay dashboard at http://localhost:{}", config.port);
|
|
|
|
|
eprintln!("Press Ctrl+C to stop");
|
|
|
|
|
|
|
|
|
|
loop {
|
|
|
|
|
thread::sleep(Duration::from_secs(3600));
|
|
|
|
|
}
|
|
|
|
|
}
|