300 lines
8.8 KiB
Rust
300 lines
8.8 KiB
Rust
|
|
//! Saturation stress tests - find where the runtime breaks.
|
||
|
|
//!
|
||
|
|
//! These tests intentionally push past limits to document failure modes.
|
||
|
|
|
||
|
|
use super::{BlackHole, Counter, Msg, Stress, StressResult};
|
||
|
|
use std::time::Duration;
|
||
|
|
use swactor::runtime::{Runtime, RuntimeConfig};
|
||
|
|
|
||
|
|
/// FIXME: does this even make sense to test?
|
||
|
|
/// Blast the router inbox until it overflows.
|
||
|
|
/// Documents: What happens when router can't keep up?
|
||
|
|
#[test]
|
||
|
|
#[cfg(feature = "stress")]
|
||
|
|
fn router_inbox_overflow() {
|
||
|
|
println!("\n>>> STRESS: Router Inbox Overflow");
|
||
|
|
|
||
|
|
let config = RuntimeConfig {
|
||
|
|
max_actors: 10,
|
||
|
|
router_max_messages: 100, // Tiny buffer
|
||
|
|
actor_max_messages: 1000,
|
||
|
|
num_threads: 1,
|
||
|
|
};
|
||
|
|
let runtime = Runtime::new(config);
|
||
|
|
let sink = runtime.spawn(BlackHole).unwrap();
|
||
|
|
|
||
|
|
// Blast messages without processing
|
||
|
|
let mut result = StressResult::new("router_inbox_overflow");
|
||
|
|
let start = std::time::Instant::now();
|
||
|
|
|
||
|
|
for _ in 0..10_000 {
|
||
|
|
result.operations += 1;
|
||
|
|
if runtime.send_to::<Msg>(sink, Msg).is_ok() {
|
||
|
|
result.successes += 1;
|
||
|
|
} else {
|
||
|
|
result.failures += 1;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
result.duration = start.elapsed();
|
||
|
|
result.note(format!("Router buffer: 100, Messages sent: 10,000"));
|
||
|
|
result.note(format!(
|
||
|
|
"Expected: ~99% failure rate (buffer fills immediately)"
|
||
|
|
));
|
||
|
|
result.print();
|
||
|
|
|
||
|
|
// Verify we actually saw failures
|
||
|
|
assert!(result.failures > 0, "Expected router to reject messages");
|
||
|
|
println!(">>> PASS: Router correctly rejects messages when full\n");
|
||
|
|
}
|
||
|
|
|
||
|
|
/// FIXME: This test makes no sense until we make sure panics happen when
|
||
|
|
/// actor inbox buffers are full.
|
||
|
|
/// Blast a single actor's inbox until it overflows.
|
||
|
|
/// Documents: What happens when actor can't keep up?
|
||
|
|
#[test]
|
||
|
|
#[cfg(feature = "stress")]
|
||
|
|
fn actor_inbox_overflow() {
|
||
|
|
println!("\n>>> STRESS: Actor Inbox Overflow");
|
||
|
|
|
||
|
|
let config = RuntimeConfig {
|
||
|
|
max_actors: 10,
|
||
|
|
router_max_messages: 100_000, // Large router buffer
|
||
|
|
actor_max_messages: 100, // Tiny actor inbox
|
||
|
|
num_threads: 1,
|
||
|
|
};
|
||
|
|
let runtime = Runtime::new(config);
|
||
|
|
let sink = runtime.spawn(BlackHole).unwrap();
|
||
|
|
|
||
|
|
// Process router registration
|
||
|
|
runtime.tick();
|
||
|
|
|
||
|
|
// Now blast messages - router will accept them but actor inbox will fill
|
||
|
|
let mut sent = 0u64;
|
||
|
|
for _ in 0..10_000 {
|
||
|
|
if runtime.send_to::<Msg>(sink, Msg).is_ok() {
|
||
|
|
sent += 1;
|
||
|
|
}
|
||
|
|
// Tick occasionally to let router deliver
|
||
|
|
if sent % 100 == 0 {
|
||
|
|
runtime.tick();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// The router accepted messages, but many were dropped at actor inbox
|
||
|
|
// We can't easily count these drops from outside, but we can document the behavior
|
||
|
|
println!(" Router accepted {} messages", sent);
|
||
|
|
println!(" Actor inbox capacity: 100");
|
||
|
|
println!(" Note: Messages beyond inbox capacity are silently dropped");
|
||
|
|
println!(">>> This is a known limitation - bounded queues drop overflow\n");
|
||
|
|
}
|
||
|
|
|
||
|
|
/// FIXME: Does it make sense to have this as a test? Yes the runtime
|
||
|
|
/// fails if you try and spawn actors when the queue is full.
|
||
|
|
/// Spawn actors until the queue rejects.
|
||
|
|
/// Documents: What happens when actor queue fills?
|
||
|
|
#[test]
|
||
|
|
#[cfg(feature = "stress")]
|
||
|
|
fn actor_queue_overflow() {
|
||
|
|
println!("\n>>> STRESS: Actor Queue Overflow");
|
||
|
|
|
||
|
|
let config = RuntimeConfig {
|
||
|
|
max_actors: 100, // Small actor queue
|
||
|
|
router_max_messages: 10_000,
|
||
|
|
actor_max_messages: 100,
|
||
|
|
num_threads: 1,
|
||
|
|
};
|
||
|
|
let runtime = Runtime::new(config);
|
||
|
|
|
||
|
|
let mut result = StressResult::new("actor_queue_overflow");
|
||
|
|
let start = std::time::Instant::now();
|
||
|
|
|
||
|
|
// Try to spawn 500 actors into 100-slot queue
|
||
|
|
for _ in 0..500 {
|
||
|
|
result.operations += 1;
|
||
|
|
match runtime.spawn(BlackHole) {
|
||
|
|
Ok(_) => result.successes += 1,
|
||
|
|
Err(_) => result.failures += 1,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
result.duration = start.elapsed();
|
||
|
|
result.note(format!("Queue capacity: 100, Spawn attempts: 500"));
|
||
|
|
result.note(format!(
|
||
|
|
"Expected: ~80% failure rate (queue fills after ~100)"
|
||
|
|
));
|
||
|
|
result.print();
|
||
|
|
|
||
|
|
// Note: Router also takes a slot, so we expect ~99 actors max
|
||
|
|
assert!(
|
||
|
|
result.successes <= 100,
|
||
|
|
"Spawned more actors than queue capacity"
|
||
|
|
);
|
||
|
|
assert!(result.failures > 0, "Expected spawn failures");
|
||
|
|
println!(">>> PASS: Actor queue correctly rejects when full\n");
|
||
|
|
}
|
||
|
|
|
||
|
|
/// FIXME: IS this actually testing what it should be?
|
||
|
|
/// Sustained overload - run at 2x capacity for extended period.
|
||
|
|
/// Documents: Does the system degrade gracefully or crash?
|
||
|
|
#[test]
|
||
|
|
#[cfg(feature = "stress")]
|
||
|
|
fn sustained_overload() {
|
||
|
|
println!("\n>>> STRESS: Sustained Overload");
|
||
|
|
|
||
|
|
let config = RuntimeConfig {
|
||
|
|
max_actors: 100,
|
||
|
|
router_max_messages: 1000,
|
||
|
|
actor_max_messages: 100,
|
||
|
|
num_threads: 1,
|
||
|
|
};
|
||
|
|
let runtime = Runtime::new(config);
|
||
|
|
|
||
|
|
// Spawn some actors
|
||
|
|
let mut actors = Vec::new();
|
||
|
|
for _ in 0..50 {
|
||
|
|
if let Ok(addr) = runtime.spawn(Counter::new()) {
|
||
|
|
actors.push(addr);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Process registrations
|
||
|
|
for _ in 0..200 {
|
||
|
|
runtime.tick();
|
||
|
|
}
|
||
|
|
|
||
|
|
let result = Stress::new("sustained_overload")
|
||
|
|
.for_duration(Duration::from_secs(2))
|
||
|
|
.run(|| {
|
||
|
|
// Send to random actor
|
||
|
|
let idx = (std::time::Instant::now().elapsed().as_nanos() as usize) % actors.len();
|
||
|
|
let success = runtime.send_to::<Msg>(actors[idx], Msg).is_ok();
|
||
|
|
|
||
|
|
// Process some (but not all) - simulating overload
|
||
|
|
runtime.tick();
|
||
|
|
|
||
|
|
success
|
||
|
|
});
|
||
|
|
|
||
|
|
result.print();
|
||
|
|
println!(">>> System survived sustained overload without panic\n");
|
||
|
|
}
|
||
|
|
|
||
|
|
/// FIXME: The logic for timing recovery does not make sense
|
||
|
|
/// Burst traffic - idle to 100x normal, back to idle.
|
||
|
|
/// Documents: Recovery behavior after traffic spikes.
|
||
|
|
#[test]
|
||
|
|
#[cfg(feature = "stress")]
|
||
|
|
fn burst_traffic() {
|
||
|
|
println!("\n>>> STRESS: Burst Traffic");
|
||
|
|
|
||
|
|
let config = RuntimeConfig {
|
||
|
|
max_actors: 100,
|
||
|
|
router_max_messages: 10_000,
|
||
|
|
actor_max_messages: 1000,
|
||
|
|
num_threads: 1,
|
||
|
|
};
|
||
|
|
let runtime = Runtime::new(config);
|
||
|
|
let sink = runtime.spawn(Counter::new()).unwrap();
|
||
|
|
|
||
|
|
// Process registration
|
||
|
|
for _ in 0..10 {
|
||
|
|
runtime.tick();
|
||
|
|
}
|
||
|
|
|
||
|
|
let mut total_sent = 0u64;
|
||
|
|
let mut total_failed = 0u64;
|
||
|
|
|
||
|
|
// 5 burst cycles
|
||
|
|
for cycle in 0..5 {
|
||
|
|
// Burst: send 1000 messages as fast as possible
|
||
|
|
let mut burst_sent = 0;
|
||
|
|
let mut burst_failed = 0;
|
||
|
|
for _ in 0..1000 {
|
||
|
|
if runtime.send_to::<Msg>(sink, Msg).is_ok() {
|
||
|
|
burst_sent += 1;
|
||
|
|
} else {
|
||
|
|
burst_failed += 1;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
total_sent += burst_sent;
|
||
|
|
total_failed += burst_failed;
|
||
|
|
|
||
|
|
// Recovery: process until queue is drained
|
||
|
|
let recovery_start = std::time::Instant::now();
|
||
|
|
for _ in 0..5000 {
|
||
|
|
runtime.tick();
|
||
|
|
}
|
||
|
|
let recovery_time = recovery_start.elapsed();
|
||
|
|
|
||
|
|
println!(
|
||
|
|
" Cycle {}: sent={}, failed={}, recovery={:?}",
|
||
|
|
cycle + 1,
|
||
|
|
burst_sent,
|
||
|
|
burst_failed,
|
||
|
|
recovery_time
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
println!(
|
||
|
|
"\n Total sent: {}, Total failed: {}",
|
||
|
|
total_sent, total_failed
|
||
|
|
);
|
||
|
|
println!(">>> Burst traffic test complete\n");
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Find the message drop cliff - at what load factor do drops spike?
|
||
|
|
#[test]
|
||
|
|
#[cfg(feature = "stress")]
|
||
|
|
fn message_drop_curve() {
|
||
|
|
println!("\n>>> STRESS: Message Drop Curve");
|
||
|
|
println!(" Testing drop rate at various load factors...\n");
|
||
|
|
|
||
|
|
// Test at different load factors (messages per tick)
|
||
|
|
for msgs_per_tick in [1, 5, 10, 20, 50, 100] {
|
||
|
|
let config = RuntimeConfig {
|
||
|
|
max_actors: 10,
|
||
|
|
router_max_messages: 1000,
|
||
|
|
actor_max_messages: 500,
|
||
|
|
num_threads: 1,
|
||
|
|
};
|
||
|
|
let runtime = Runtime::new(config);
|
||
|
|
let sink = runtime.spawn(BlackHole).unwrap();
|
||
|
|
|
||
|
|
// Warmup
|
||
|
|
for _ in 0..10 {
|
||
|
|
runtime.tick();
|
||
|
|
}
|
||
|
|
|
||
|
|
let mut sent = 0u64;
|
||
|
|
let mut failed = 0u64;
|
||
|
|
|
||
|
|
// Run for fixed iterations
|
||
|
|
for _ in 0..100 {
|
||
|
|
// Send burst
|
||
|
|
for _ in 0..msgs_per_tick {
|
||
|
|
if runtime.send_to::<Msg>(sink, Msg).is_ok() {
|
||
|
|
sent += 1;
|
||
|
|
} else {
|
||
|
|
failed += 1;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Process one tick
|
||
|
|
runtime.tick();
|
||
|
|
}
|
||
|
|
|
||
|
|
let drop_rate = if sent + failed > 0 {
|
||
|
|
(failed as f64 / (sent + failed) as f64) * 100.0
|
||
|
|
} else {
|
||
|
|
0.0
|
||
|
|
};
|
||
|
|
|
||
|
|
println!(
|
||
|
|
" msgs/tick={:3} sent={:5} failed={:5} drop_rate={:.1}%",
|
||
|
|
msgs_per_tick, sent, failed, drop_rate
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
println!("\n>>> Message drop curve test complete\n");
|
||
|
|
}
|