chore(test): enforce audited timeout policy
Route cargo lint through repository timeout validation and strict Clippy, reject unregistered timing primitives with compiler-resolved call budgets, bound Nextest concurrency, and replace polling waits in test infrastructure.
This commit is contained in:
parent
7ac9925ac3
commit
4729577621
18 changed files with 1041 additions and 74 deletions
|
|
@ -3,4 +3,4 @@ rustc-workspace-wrapper = "tools/actor-control-flow-lint/rustc-wrapper.py"
|
||||||
|
|
||||||
[alias]
|
[alias]
|
||||||
xtask = "run --package xtask --"
|
xtask = "run --package xtask --"
|
||||||
lint = "clippy --workspace --all-targets --all-features"
|
lint = "run --package xtask -- lint"
|
||||||
|
|
|
||||||
|
|
@ -1,2 +1,12 @@
|
||||||
|
# Workspace-wide test-runner configuration for cargo-nextest.
|
||||||
|
#
|
||||||
|
# The suite mixes deadline-sensitive process and network tests (real children,
|
||||||
|
# real QUIC engines) with compute-heavy property tests. At full 12-thread
|
||||||
|
# parallelism OS scheduling delays blow through those deadlines — different
|
||||||
|
# timing tests fail on every loaded run — and the namespace reference-model
|
||||||
|
# guarantee test (~25s unloaded) stretches past the default 60s slow-timeout.
|
||||||
|
# Bounded threads plus a 180s slow period trades wall time for determinism
|
||||||
|
# without masking true hangs.
|
||||||
[profile.default]
|
[profile.default]
|
||||||
slow-timeout = { period = "60s", terminate-after = 1 }
|
test-threads = 4
|
||||||
|
slow-timeout = { period = "180s" }
|
||||||
|
|
|
||||||
|
|
@ -26,23 +26,22 @@ const EXECUTION_OWNERS: &[&str] = &[
|
||||||
"swactor",
|
"swactor",
|
||||||
"swactor-engine",
|
"swactor-engine",
|
||||||
"swactor-process",
|
"swactor-process",
|
||||||
|
"swactor-process-context",
|
||||||
"swactor-transport",
|
"swactor-transport",
|
||||||
"swactor-vastai",
|
"swactor-vastai",
|
||||||
"telemetry",
|
"telemetry",
|
||||||
];
|
];
|
||||||
|
|
||||||
/// Test and benchmark harness packages that must not be workspace dependencies.
|
/// Test and benchmark harness packages that must not be workspace dependencies.
|
||||||
const TEST_SUPPORT_OWNERS: &[&str] =
|
const TEST_SUPPORT_OWNERS: &[&str] = &[
|
||||||
&["actor-control-flow-lint-tests", "swactor-benchmarks"];
|
"actor-control-flow-lint-tests",
|
||||||
|
"myelin-e2e-fuzz",
|
||||||
|
"swactor-benchmarks",
|
||||||
|
];
|
||||||
|
|
||||||
/// Policy-bearing crates that must never enter an execution owner's dependency
|
/// Policy-bearing crates that must never enter an execution owner's dependency
|
||||||
/// closure.
|
/// closure.
|
||||||
const DOMAIN_CONTROL_CRATES: &[&str] = &[
|
const DOMAIN_CONTROL_CRATES: &[&str] = &["myelin", "provisioning", "xtask"];
|
||||||
"myelin",
|
|
||||||
"provisioning",
|
|
||||||
"swactor-job-runner",
|
|
||||||
"xtask",
|
|
||||||
];
|
|
||||||
|
|
||||||
#[derive(Clone, Copy)]
|
#[derive(Clone, Copy)]
|
||||||
struct Capability {
|
struct Capability {
|
||||||
|
|
@ -54,8 +53,7 @@ struct Capability {
|
||||||
|
|
||||||
const MOVE_TO_OWNER: &str =
|
const MOVE_TO_OWNER: &str =
|
||||||
"move stream mechanics into an approved execution owner or move the decision into an actor";
|
"move stream mechanics into an approved execution owner or move the decision into an actor";
|
||||||
const USE_ACTOR_TIMER: &str =
|
const USE_ACTOR_TIMER: &str = "schedule a typed actor message through the engine; the receiving actor owns the deadline decision";
|
||||||
"schedule a typed actor message through the engine; the receiving actor owns the deadline decision";
|
|
||||||
|
|
||||||
/// Stable resolved item paths. These are deliberately compiler identities, not
|
/// Stable resolved item paths. These are deliberately compiler identities, not
|
||||||
/// spellings found in source, so re-exports, renamed imports, and local wrappers
|
/// spellings found in source, so re-exports, renamed imports, and local wrappers
|
||||||
|
|
@ -179,6 +177,8 @@ const CAPABILITIES: &[Capability] = &[
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
|
include!("timing_policy.rs");
|
||||||
|
|
||||||
struct ActorControlFlowCallbacks {
|
struct ActorControlFlowCallbacks {
|
||||||
package: String,
|
package: String,
|
||||||
test_build: bool,
|
test_build: bool,
|
||||||
|
|
@ -186,17 +186,11 @@ struct ActorControlFlowCallbacks {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Callbacks for ActorControlFlowCallbacks {
|
impl Callbacks for ActorControlFlowCallbacks {
|
||||||
fn after_analysis<'tcx>(
|
fn after_analysis<'tcx>(&mut self, _compiler: &Compiler, tcx: TyCtxt<'tcx>) -> Compilation {
|
||||||
&mut self,
|
let role_exempt = TEST_SUPPORT_OWNERS.contains(&self.package.as_str())
|
||||||
_compiler: &Compiler,
|
|| EXECUTION_OWNERS.contains(&self.package.as_str());
|
||||||
tcx: TyCtxt<'tcx>,
|
|
||||||
) -> Compilation {
|
|
||||||
if TEST_SUPPORT_OWNERS.contains(&self.package.as_str()) {
|
|
||||||
return Compilation::Continue;
|
|
||||||
}
|
|
||||||
if EXECUTION_OWNERS.contains(&self.package.as_str()) {
|
if EXECUTION_OWNERS.contains(&self.package.as_str()) {
|
||||||
check_owner_dependencies(tcx, &self.package);
|
check_owner_dependencies(tcx, &self.package);
|
||||||
return Compilation::Continue;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for owner in tcx.hir_body_owners() {
|
for owner in tcx.hir_body_owners() {
|
||||||
|
|
@ -206,8 +200,11 @@ impl Callbacks for ActorControlFlowCallbacks {
|
||||||
tcx,
|
tcx,
|
||||||
typeck,
|
typeck,
|
||||||
package: &self.package,
|
package: &self.package,
|
||||||
|
caller: tcx.def_path_str(owner.to_def_id()),
|
||||||
|
role_exempt,
|
||||||
test_build: self.test_build,
|
test_build: self.test_build,
|
||||||
trace: self.trace,
|
trace: self.trace,
|
||||||
|
timing_counts: Vec::new(),
|
||||||
};
|
};
|
||||||
visitor.visit_body(body);
|
visitor.visit_body(body);
|
||||||
}
|
}
|
||||||
|
|
@ -232,8 +229,11 @@ struct CapabilityVisitor<'a, 'tcx> {
|
||||||
tcx: TyCtxt<'tcx>,
|
tcx: TyCtxt<'tcx>,
|
||||||
typeck: &'tcx TypeckResults<'tcx>,
|
typeck: &'tcx TypeckResults<'tcx>,
|
||||||
package: &'a str,
|
package: &'a str,
|
||||||
|
caller: String,
|
||||||
|
role_exempt: bool,
|
||||||
test_build: bool,
|
test_build: bool,
|
||||||
trace: bool,
|
trace: bool,
|
||||||
|
timing_counts: Vec<(String, usize)>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<'tcx> Visitor<'tcx> for CapabilityVisitor<'_, 'tcx> {
|
impl<'tcx> Visitor<'tcx> for CapabilityVisitor<'_, 'tcx> {
|
||||||
|
|
@ -256,11 +256,46 @@ impl<'tcx> Visitor<'tcx> for CapabilityVisitor<'_, 'tcx> {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CapabilityVisitor<'_, '_> {
|
impl CapabilityVisitor<'_, '_> {
|
||||||
fn check(&self, def_id: rustc_hir::def_id::DefId, span: Span) {
|
fn check(&mut self, def_id: rustc_hir::def_id::DefId, span: Span) {
|
||||||
let path = self.tcx.def_path_str(def_id);
|
let path = self.tcx.def_path_str(def_id);
|
||||||
let normalized_path = normalize_def_path(&path);
|
let normalized_path = normalize_def_path(&path);
|
||||||
if self.trace && is_candidate_name(self.tcx.item_name(def_id)) {
|
let item_name = self.tcx.item_name(def_id);
|
||||||
eprintln!("actor-control-flow trace: {path}");
|
if self.trace && is_candidate_name(item_name) {
|
||||||
|
eprintln!(
|
||||||
|
"actor-control-flow trace: package=`{}` caller=`{}` callee=`{path}`",
|
||||||
|
self.package, self.caller
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if GUARDED_TIMING_ITEMS.contains(&item_name.as_str()) {
|
||||||
|
let occurrence = self.next_timing_occurrence(&normalized_path);
|
||||||
|
let allowance = TIMING_ALLOWANCES.iter().find(|allowance| {
|
||||||
|
allowance.package == self.package
|
||||||
|
&& allowance.caller == self.caller
|
||||||
|
&& allowance.callee == normalized_path
|
||||||
|
});
|
||||||
|
match allowance {
|
||||||
|
Some(allowance) if occurrence <= allowance.calls => return,
|
||||||
|
Some(allowance) => {
|
||||||
|
self.tcx.dcx().span_err(
|
||||||
|
span,
|
||||||
|
format!(
|
||||||
|
"unapproved timing primitive: `{normalized_path}` occurrence {occurrence} exceeds the {} audited call(s) in `{}` (`{}`); remove the timer or update the central timing registry after architectural review",
|
||||||
|
allowance.calls, self.caller, allowance.purpose
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
self.tcx.dcx().span_err(
|
||||||
|
span,
|
||||||
|
format!(
|
||||||
|
"unapproved timing primitive: `{normalized_path}` is forbidden in workspace package `{}` at `{}`; use typed engine scheduling or register an exact external-resource boundary",
|
||||||
|
self.package, self.caller
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
let Some(capability) = CAPABILITIES
|
let Some(capability) = CAPABILITIES
|
||||||
|
|
@ -270,8 +305,7 @@ impl CapabilityVisitor<'_, '_> {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
if self.role_exempt || (self.test_build && capability.test_wait) {
|
||||||
if self.test_build && capability.test_wait {
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -283,6 +317,19 @@ impl CapabilityVisitor<'_, '_> {
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn next_timing_occurrence(&mut self, callee: &str) -> usize {
|
||||||
|
if let Some((_, count)) = self
|
||||||
|
.timing_counts
|
||||||
|
.iter_mut()
|
||||||
|
.find(|(known, _)| known == callee)
|
||||||
|
{
|
||||||
|
*count += 1;
|
||||||
|
return *count;
|
||||||
|
}
|
||||||
|
self.timing_counts.push((callee.to_owned(), 1));
|
||||||
|
1
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn normalize_def_path(path: &str) -> String {
|
fn normalize_def_path(path: &str) -> String {
|
||||||
|
|
@ -314,29 +361,24 @@ fn normalize_def_path(path: &str) -> String {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn is_candidate_name(name: Symbol) -> bool {
|
fn is_candidate_name(name: Symbol) -> bool {
|
||||||
matches!(
|
GUARDED_TIMING_ITEMS.contains(&name.as_str())
|
||||||
name.as_str(),
|
|| matches!(
|
||||||
"block_on"
|
name.as_str(),
|
||||||
| "blocking_recv"
|
"block_on"
|
||||||
| "has_work"
|
| "blocking_recv"
|
||||||
| "interval"
|
| "has_work"
|
||||||
| "interval_at"
|
| "interval"
|
||||||
| "new_current_thread"
|
| "interval_at"
|
||||||
| "new_multi_thread"
|
| "new_current_thread"
|
||||||
| "recv"
|
| "new_multi_thread"
|
||||||
| "recv_deadline"
|
| "recv"
|
||||||
| "recv_timeout"
|
| "spawn"
|
||||||
| "sleep"
|
| "spawn_blocking"
|
||||||
| "sleep_until"
|
| "spawn_local"
|
||||||
| "spawn"
|
| "tick"
|
||||||
| "spawn_blocking"
|
| "timer"
|
||||||
| "spawn_local"
|
| "try_tick"
|
||||||
| "tick"
|
)
|
||||||
| "timeout"
|
|
||||||
| "timeout_at"
|
|
||||||
| "timer"
|
|
||||||
| "try_tick"
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn main() -> ExitCode {
|
fn main() -> ExitCode {
|
||||||
|
|
|
||||||
|
|
@ -24,6 +24,7 @@ def main() -> None:
|
||||||
rustc_args = sys.argv[2:]
|
rustc_args = sys.argv[2:]
|
||||||
root = Path(__file__).resolve().parents[2]
|
root = Path(__file__).resolve().parents[2]
|
||||||
source = Path(__file__).with_name("driver.rs")
|
source = Path(__file__).with_name("driver.rs")
|
||||||
|
policy = Path(__file__).with_name("timing_policy.rs")
|
||||||
target_root = Path(os.environ.get("CARGO_TARGET_DIR", root / "target"))
|
target_root = Path(os.environ.get("CARGO_TARGET_DIR", root / "target"))
|
||||||
if not target_root.is_absolute():
|
if not target_root.is_absolute():
|
||||||
target_root = root / target_root
|
target_root = root / target_root
|
||||||
|
|
@ -50,7 +51,9 @@ def main() -> None:
|
||||||
except (OSError, subprocess.CalledProcessError) as error:
|
except (OSError, subprocess.CalledProcessError) as error:
|
||||||
fail(f"cannot inspect pinned rustc: {error}")
|
fail(f"cannot inspect pinned rustc: {error}")
|
||||||
|
|
||||||
digest = hashlib.sha256(source.read_bytes() + version.encode()).hexdigest()[:20]
|
digest = hashlib.sha256(
|
||||||
|
source.read_bytes() + b"\0" + policy.read_bytes() + b"\0" + version.encode()
|
||||||
|
).hexdigest()[:20]
|
||||||
driver = cache / f"driver-{digest}"
|
driver = cache / f"driver-{digest}"
|
||||||
lock_path = cache / "build.lock"
|
lock_path = cache / "build.lock"
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,18 @@ fn repository_root() -> PathBuf {
|
||||||
.to_path_buf()
|
.to_path_buf()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn driver_revision(root: &Path) -> String {
|
||||||
|
let mut hash = 0xcbf29ce484222325_u64;
|
||||||
|
for path in ["driver.rs", "timing_policy.rs"] {
|
||||||
|
let source = std::fs::read(root.join("tools/actor-control-flow-lint").join(path))
|
||||||
|
.expect("read compiler policy source");
|
||||||
|
for byte in source {
|
||||||
|
hash = (hash ^ u64::from(byte)).wrapping_mul(0x100000001b3);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
format!("--cfg=actor_lint_driver_{hash:016x}")
|
||||||
|
}
|
||||||
|
|
||||||
fn cargo_check(fixture: &str, extra_args: &[&str]) -> Output {
|
fn cargo_check(fixture: &str, extra_args: &[&str]) -> Output {
|
||||||
let root = repository_root();
|
let root = repository_root();
|
||||||
let fixture_dir = Path::new(env!("CARGO_MANIFEST_DIR"))
|
let fixture_dir = Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||||
|
|
@ -28,6 +40,7 @@ fn cargo_check(fixture: &str, extra_args: &[&str]) -> Output {
|
||||||
.env("CARGO_TARGET_DIR", target_dir)
|
.env("CARGO_TARGET_DIR", target_dir)
|
||||||
.env("CARGO_TERM_COLOR", "never")
|
.env("CARGO_TERM_COLOR", "never")
|
||||||
.env("RUSTC_WORKSPACE_WRAPPER", wrapper)
|
.env("RUSTC_WORKSPACE_WRAPPER", wrapper)
|
||||||
|
.env("RUSTFLAGS", driver_revision(&root))
|
||||||
.env_remove("CARGO_MAKEFLAGS")
|
.env_remove("CARGO_MAKEFLAGS")
|
||||||
.env_remove("MAKEFLAGS");
|
.env_remove("MAKEFLAGS");
|
||||||
command.output().expect("run fixture cargo check")
|
command.output().expect("run fixture cargo check")
|
||||||
|
|
@ -50,7 +63,7 @@ fn compiler_policy_contracts() {
|
||||||
"blocking task spawning",
|
"blocking task spawning",
|
||||||
"engine task scheduling",
|
"engine task scheduling",
|
||||||
"OS thread creation",
|
"OS thread creation",
|
||||||
"thread sleeping",
|
"unapproved timing primitive",
|
||||||
"direct timer driving",
|
"direct timer driving",
|
||||||
"runtime construction or driving",
|
"runtime construction or driving",
|
||||||
"blocking receive used as a controller",
|
"blocking receive used as a controller",
|
||||||
|
|
@ -74,6 +87,30 @@ fn compiler_policy_contracts() {
|
||||||
"missing owner dependency diagnostic:\n{dependency_stderr}"
|
"missing owner dependency diagnostic:\n{dependency_stderr}"
|
||||||
);
|
);
|
||||||
|
|
||||||
|
let owner_timing = cargo_check("fail-owner-timing", &[]);
|
||||||
|
assert!(
|
||||||
|
!owner_timing.status.success(),
|
||||||
|
"unapproved execution-owner timer compiled"
|
||||||
|
);
|
||||||
|
let owner_timing_stderr = stderr(&owner_timing);
|
||||||
|
assert!(
|
||||||
|
owner_timing_stderr.contains(
|
||||||
|
"unapproved timing primitive: `tokio::time::sleep` is forbidden in workspace package `iroh-driver`"
|
||||||
|
),
|
||||||
|
"missing resolved owner-timer diagnostic:\n{owner_timing_stderr}"
|
||||||
|
);
|
||||||
|
|
||||||
|
let timing_overage = cargo_check("fail-timing-overage", &[]);
|
||||||
|
assert!(
|
||||||
|
!timing_overage.status.success(),
|
||||||
|
"timing allowance overage compiled"
|
||||||
|
);
|
||||||
|
let timing_overage_stderr = stderr(&timing_overage);
|
||||||
|
assert!(
|
||||||
|
timing_overage_stderr.contains("occurrence 2 exceeds the 1 audited call(s)"),
|
||||||
|
"missing timing-overage diagnostic:\n{timing_overage_stderr}"
|
||||||
|
);
|
||||||
|
|
||||||
for fixture in ["pass-actor-domain", "pass-execution-owner"] {
|
for fixture in ["pass-actor-domain", "pass-execution-owner"] {
|
||||||
let output = cargo_check(fixture, &[]);
|
let output = cargo_check(fixture, &[]);
|
||||||
assert!(
|
assert!(
|
||||||
|
|
|
||||||
25
tools/actor-control-flow-lint/tests/fixtures/fail-owner-timing/Cargo.lock
generated
vendored
Normal file
25
tools/actor-control-flow-lint/tests/fixtures/fail-owner-timing/Cargo.lock
generated
vendored
Normal file
|
|
@ -0,0 +1,25 @@
|
||||||
|
# This file is automatically @generated by Cargo.
|
||||||
|
# It is not intended for manual editing.
|
||||||
|
version = 4
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "iroh-driver"
|
||||||
|
version = "0.0.0"
|
||||||
|
dependencies = [
|
||||||
|
"tokio",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "pin-project-lite"
|
||||||
|
version = "0.2.17"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tokio"
|
||||||
|
version = "1.53.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"
|
||||||
|
dependencies = [
|
||||||
|
"pin-project-lite",
|
||||||
|
]
|
||||||
10
tools/actor-control-flow-lint/tests/fixtures/fail-owner-timing/Cargo.toml
vendored
Normal file
10
tools/actor-control-flow-lint/tests/fixtures/fail-owner-timing/Cargo.toml
vendored
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
[package]
|
||||||
|
name = "iroh-driver"
|
||||||
|
version = "0.0.0"
|
||||||
|
edition = "2024"
|
||||||
|
publish = false
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
tokio = { version = "1", features = ["time"] }
|
||||||
|
|
||||||
|
[workspace]
|
||||||
6
tools/actor-control-flow-lint/tests/fixtures/fail-owner-timing/src/lib.rs
vendored
Normal file
6
tools/actor-control-flow-lint/tests/fixtures/fail-owner-timing/src/lib.rs
vendored
Normal file
|
|
@ -0,0 +1,6 @@
|
||||||
|
use std::time::Duration;
|
||||||
|
use tokio::time::sleep as pause;
|
||||||
|
|
||||||
|
pub async fn unapproved_owner_wait() {
|
||||||
|
pause(Duration::from_millis(1)).await;
|
||||||
|
}
|
||||||
7
tools/actor-control-flow-lint/tests/fixtures/fail-timing-overage/Cargo.lock
generated
vendored
Normal file
7
tools/actor-control-flow-lint/tests/fixtures/fail-timing-overage/Cargo.lock
generated
vendored
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
# This file is automatically @generated by Cargo.
|
||||||
|
# It is not intended for manual editing.
|
||||||
|
version = 4
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "actor-lint-timing-overage"
|
||||||
|
version = "0.0.0"
|
||||||
7
tools/actor-control-flow-lint/tests/fixtures/fail-timing-overage/Cargo.toml
vendored
Normal file
7
tools/actor-control-flow-lint/tests/fixtures/fail-timing-overage/Cargo.toml
vendored
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
[package]
|
||||||
|
name = "actor-lint-timing-overage"
|
||||||
|
version = "0.0.0"
|
||||||
|
edition = "2024"
|
||||||
|
publish = false
|
||||||
|
|
||||||
|
[workspace]
|
||||||
7
tools/actor-control-flow-lint/tests/fixtures/fail-timing-overage/src/lib.rs
vendored
Normal file
7
tools/actor-control-flow-lint/tests/fixtures/fail-timing-overage/src/lib.rs
vendored
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
pub fn exceeds_registered_timing_budget() {
|
||||||
|
let (_sender, receiver) = std::sync::mpsc::channel::<()>();
|
||||||
|
let _ = receiver.recv_timeout(Duration::from_millis(1));
|
||||||
|
let _ = receiver.recv_timeout(Duration::from_millis(1));
|
||||||
|
}
|
||||||
|
|
@ -1,8 +1,5 @@
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
pub fn start_transport_pump() {
|
pub fn start_transport_pump() {
|
||||||
tokio::spawn(async {
|
tokio::spawn(async {
|
||||||
let _io_type: Option<tokio::net::TcpStream> = None;
|
let _io_type: Option<tokio::net::TcpStream> = None;
|
||||||
tokio::time::sleep(Duration::from_millis(1)).await;
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -4,9 +4,5 @@ use std::time::Duration;
|
||||||
fn bounded_observation_wait_is_allowed() {
|
fn bounded_observation_wait_is_allowed() {
|
||||||
let (sender, receiver) = std::sync::mpsc::channel();
|
let (sender, receiver) = std::sync::mpsc::channel();
|
||||||
sender.send(7_u8).expect("send observation");
|
sender.send(7_u8).expect("send observation");
|
||||||
assert_eq!(
|
assert_eq!(receiver.recv_timeout(Duration::from_millis(10)), Ok(7));
|
||||||
receiver.recv_timeout(Duration::from_millis(10)),
|
|
||||||
Ok(7)
|
|
||||||
);
|
|
||||||
std::thread::sleep(Duration::from_millis(1));
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
602
tools/actor-control-flow-lint/timing_policy.rs
Normal file
602
tools/actor-control-flow-lint/timing_policy.rs
Normal file
|
|
@ -0,0 +1,602 @@
|
||||||
|
/// Calls with these resolved item names can create wall-clock waits or
|
||||||
|
/// failure-producing deadlines. They are denied in every workspace role unless
|
||||||
|
/// the exact compiler-resolved call site is registered below.
|
||||||
|
const GUARDED_TIMING_ITEMS: &[&str] = &[
|
||||||
|
"park_timeout",
|
||||||
|
"recv_deadline",
|
||||||
|
"set_read_timeout",
|
||||||
|
"recv_timeout",
|
||||||
|
"sleep",
|
||||||
|
"sleep_until",
|
||||||
|
"timeout",
|
||||||
|
"timeout_at",
|
||||||
|
"wait_timeout",
|
||||||
|
"wait_timeout_while",
|
||||||
|
];
|
||||||
|
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
struct TimingAllowance {
|
||||||
|
package: &'static str,
|
||||||
|
caller: &'static str,
|
||||||
|
callee: &'static str,
|
||||||
|
calls: usize,
|
||||||
|
purpose: &'static str,
|
||||||
|
}
|
||||||
|
|
||||||
|
const fn timing(
|
||||||
|
package: &'static str,
|
||||||
|
caller: &'static str,
|
||||||
|
callee: &'static str,
|
||||||
|
calls: usize,
|
||||||
|
purpose: &'static str,
|
||||||
|
) -> TimingAllowance {
|
||||||
|
TimingAllowance {
|
||||||
|
package,
|
||||||
|
caller,
|
||||||
|
callee,
|
||||||
|
calls,
|
||||||
|
purpose,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Exact audited boundaries. A rename, a different API, or one additional call
|
||||||
|
/// exceeds this registry and fails compilation. Never register a generic
|
||||||
|
/// timeout wrapper: callers of a timing API must remain visible to this guard.
|
||||||
|
const TIMING_ALLOWANCES: &[TimingAllowance] = &[
|
||||||
|
// Runtime and external-resource boundaries.
|
||||||
|
timing(
|
||||||
|
"swactor-engine",
|
||||||
|
"<tokio::LazySleep as std::future::Future>::poll",
|
||||||
|
"tokio::time::sleep",
|
||||||
|
1,
|
||||||
|
"Tokio execution-backend adapter",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-process",
|
||||||
|
"<operations::FollowProcessFile as std::io::Read>::read",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"external process-file polling",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-process",
|
||||||
|
"operations::find_process_identities_with_retry",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"external process discovery retry",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-process",
|
||||||
|
"operations::spawn_identity_exit_wait::{closure#0}",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"external process exit polling",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-process",
|
||||||
|
"operations::spawn_shared_child_wait::{closure#0}",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"external child exit polling",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-process",
|
||||||
|
"operations::terminate_process_group",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"process termination grace period",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-process",
|
||||||
|
"operations::wait_for_path",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"external path publication polling",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-process",
|
||||||
|
"operations::wait_shared_child_or_kill",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"child termination grace period",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"iroh-driver",
|
||||||
|
"edge_transport::spawn_edge_send_pump",
|
||||||
|
"std::sync::mpsc::Receiver::recv_timeout",
|
||||||
|
1,
|
||||||
|
"edge sender startup handoff",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"iroh-driver",
|
||||||
|
"edge_transport::spawn_edge_send_pump::{closure#0}::{closure#0}",
|
||||||
|
"swactor_engine::EngineHandle::timeout",
|
||||||
|
2,
|
||||||
|
"edge stream write and finish deadlines",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"iroh-driver",
|
||||||
|
"iroh_driver::AdapterPump::spawn_connect::{closure#0}",
|
||||||
|
"swactor_engine::EngineHandle::timeout",
|
||||||
|
1,
|
||||||
|
"peer connection deadline",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"iroh-driver",
|
||||||
|
"iroh_driver::IrohDriver::send_tagged_gossip::{closure#0}",
|
||||||
|
"swactor_engine::EngineHandle::timeout",
|
||||||
|
1,
|
||||||
|
"gossip connection deadline",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"iroh-driver",
|
||||||
|
"iroh_driver::IrohDriver::connect_peer::{closure#0}",
|
||||||
|
"swactor_engine::EngineHandle::timeout",
|
||||||
|
1,
|
||||||
|
"peer connection deadline",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"iroh-driver",
|
||||||
|
"iroh_driver::IrohDriver::spawn_join_request::{closure#0}",
|
||||||
|
"swactor_engine::EngineHandle::timeout",
|
||||||
|
1,
|
||||||
|
"seed connection deadline",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-vastai",
|
||||||
|
"blocking::BlockingVastClient::list_by_label_with_retry",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"provider API retry backoff",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-vastai",
|
||||||
|
"client::VastClient::with_base_url",
|
||||||
|
"reqwest::ClientBuilder::timeout",
|
||||||
|
1,
|
||||||
|
"provider HTTP request deadline",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-vastai",
|
||||||
|
"lease::provision_fleet::{closure#0}",
|
||||||
|
"tokio::time::sleep",
|
||||||
|
1,
|
||||||
|
"provider lease pacing",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-vastai",
|
||||||
|
"lease::provision_one::{closure#0}",
|
||||||
|
"tokio::time::sleep",
|
||||||
|
1,
|
||||||
|
"provider retry backoff",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-vastai",
|
||||||
|
"monitor::wait_for_running_with_policy::{closure#0}",
|
||||||
|
"tokio::time::sleep",
|
||||||
|
2,
|
||||||
|
"provider lifecycle polling",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-vastai",
|
||||||
|
"monitor::wait_for_ssh_endpoint_with_policy::{closure#0}",
|
||||||
|
"tokio::time::sleep",
|
||||||
|
2,
|
||||||
|
"provider endpoint polling",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-vastai",
|
||||||
|
"teardown::destroy_instance_with_retry_policy::{closure#0}",
|
||||||
|
"tokio::time::sleep",
|
||||||
|
1,
|
||||||
|
"provider teardown backoff",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"telemetry",
|
||||||
|
"endpoint::TelemetrySubscription::recv_timeout",
|
||||||
|
"crossbeam_channel::Receiver::recv_timeout",
|
||||||
|
1,
|
||||||
|
"bounded external telemetry observation",
|
||||||
|
),
|
||||||
|
// Test and harness safety fuses. These bound real external concurrency;
|
||||||
|
// virtual-time behavior continues to use the stepping backend.
|
||||||
|
timing(
|
||||||
|
"dashboard",
|
||||||
|
"control::properties::generated_concurrent_bridge_commands_forward_once_and_shutdown::{closure#1}",
|
||||||
|
"std::sync::mpsc::Receiver::recv_timeout",
|
||||||
|
1,
|
||||||
|
"test completion fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"dashboard",
|
||||||
|
"server::tests::send_control_request::{closure#0}",
|
||||||
|
"tokio::time::timeout",
|
||||||
|
1,
|
||||||
|
"test HTTP completion fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"data-plane",
|
||||||
|
"cancellation_and_transport_faults_reclaim_waiters_and_preserve_unrelated_progress::{closure#0}",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"test cleanup observation poll",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"data-plane",
|
||||||
|
"cancelled_write_open_releases_queued_grant",
|
||||||
|
"std::thread::sleep",
|
||||||
|
2,
|
||||||
|
"test cancellation observation poll",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"data-plane",
|
||||||
|
"host_read_resolves_file_source_and_seals_final_arena_lease",
|
||||||
|
"std::thread::sleep",
|
||||||
|
4,
|
||||||
|
"test lease reclamation observation poll",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"data-plane",
|
||||||
|
"live_view_prevents_reclaim_until_last_guard_drops",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"test lease reclamation observation poll",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"distribution",
|
||||||
|
"snapshot_and_swim_telemetry::swim_telemetry_records_probe_events_and_timeout_state_without_fabricating_rtt",
|
||||||
|
"std::thread::sleep",
|
||||||
|
2,
|
||||||
|
"wall-clock telemetry contract",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"iroh-driver",
|
||||||
|
"common::iroh::IrohTestCluster::pump_until",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"live network convergence fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"iroh-driver",
|
||||||
|
"common::iroh::pump_until",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"live network convergence fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"iroh-driver",
|
||||||
|
"common::iroh::pump_until_pair",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"live network convergence fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"iroh-driver",
|
||||||
|
"endpoint_addr_includes_home_relay",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"live relay convergence fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"iroh-driver",
|
||||||
|
"pull_collector_cancellation_interrupts_inflight_io",
|
||||||
|
"std::sync::mpsc::Receiver::recv_timeout",
|
||||||
|
1,
|
||||||
|
"test setup completion fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"iroh-driver",
|
||||||
|
"pull_collector_cancellation_interrupts_inflight_io",
|
||||||
|
"std::thread::sleep",
|
||||||
|
2,
|
||||||
|
"live network cancellation observation",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"iroh-driver",
|
||||||
|
"real_iroh_transfer_delivers_exact_file_bytes",
|
||||||
|
"std::sync::mpsc::Receiver::recv_timeout",
|
||||||
|
2,
|
||||||
|
"live transfer completion fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"iroh-driver",
|
||||||
|
"real_iroh_transfer_delivers_exact_file_bytes",
|
||||||
|
"std::thread::sleep",
|
||||||
|
2,
|
||||||
|
"live transfer progress poll",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"iroh-driver",
|
||||||
|
"same_runtime_transfer_bypasses_iroh_self_connection",
|
||||||
|
"std::sync::mpsc::Receiver::recv_timeout",
|
||||||
|
4,
|
||||||
|
"local transfer completion fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"iroh-driver",
|
||||||
|
"edge_transport::EdgeSendHandle::finish",
|
||||||
|
"std::sync::mpsc::Receiver::recv_timeout",
|
||||||
|
1,
|
||||||
|
"edge transport completion fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"iroh-driver",
|
||||||
|
"recv_until",
|
||||||
|
"std::sync::mpsc::Receiver::recv_timeout",
|
||||||
|
1,
|
||||||
|
"live transport observation fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"myelin",
|
||||||
|
"node::worker_node_runtime::control_flow_properties::agent_cpu_sampler_emits_decodable_host_stats",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"hardware sampler observation",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"myelin",
|
||||||
|
"orchestration::app::hardware_telemetry_tests::orchestrator_emits_all_host_hardware_channels",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"hardware sampler observation",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"myelin",
|
||||||
|
"provisioning::tests::local_process_stop_releases_all_observer_actors",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"process shutdown observation",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"myelin",
|
||||||
|
"provisioning::tests::recv_provider_event",
|
||||||
|
"std::sync::mpsc::Receiver::recv_timeout",
|
||||||
|
1,
|
||||||
|
"provider observation fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"myelin",
|
||||||
|
"tests::data_namespace_guarantees::wait_until",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"live namespace convergence fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"myelin",
|
||||||
|
"tests::engine_composition::poll_connect",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"live server connection fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"myelin",
|
||||||
|
"tests::engine_composition::recv_within",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"engine observation fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"myelin-e2e-fuzz",
|
||||||
|
"harness::control::<impl harness::ClusterHarness>::kill_node",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"external node termination polling",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"myelin-e2e-fuzz",
|
||||||
|
"harness::convergence::<impl harness::ClusterHarness>::provision",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"external provisioning polling",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"myelin-e2e-fuzz",
|
||||||
|
"harness::ClusterHarness::teardown",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"external process teardown polling",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"myelin-e2e-fuzz",
|
||||||
|
"harness::convergence::<impl harness::ClusterHarness>::wait_contextual_control",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"external control-plane polling",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"myelin-e2e-fuzz",
|
||||||
|
"harness::control::<impl harness::ClusterHarness>::wait_execution",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"external execution polling",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"myelin-e2e-fuzz",
|
||||||
|
"harness::convergence::<impl harness::ClusterHarness>::wait_for_dashboard",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"external dashboard readiness polling",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"myelin-e2e-fuzz",
|
||||||
|
"harness::control::<impl harness::ClusterHarness>::stop_orchestrator_injection",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"external orchestrator termination polling",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"myelin-e2e-fuzz",
|
||||||
|
"harness::control::<impl harness::ClusterHarness>::wait_for_no_workload_processes",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"external native-process census polling",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"myelin-e2e-fuzz",
|
||||||
|
"harness::ClusterHarness::assert_healthy",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"external resource cleanup polling",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"myelin-e2e-fuzz",
|
||||||
|
"resources::http_json",
|
||||||
|
"ureq::Request::timeout",
|
||||||
|
2,
|
||||||
|
"external HTTP request fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-engine",
|
||||||
|
"common::wait_for",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"engine test completion fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-engine",
|
||||||
|
"engine_interval_recurs",
|
||||||
|
"std::sync::mpsc::Receiver::recv_timeout",
|
||||||
|
1,
|
||||||
|
"engine test completion fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-engine",
|
||||||
|
"engine_runs_without_an_ambient_tokio_runtime",
|
||||||
|
"std::sync::mpsc::Receiver::recv_timeout",
|
||||||
|
1,
|
||||||
|
"engine test completion fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-engine",
|
||||||
|
"engine_timer_can_be_created_off_runtime",
|
||||||
|
"std::sync::mpsc::Receiver::recv_timeout",
|
||||||
|
1,
|
||||||
|
"engine test completion fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-engine",
|
||||||
|
"engine_timer_fires",
|
||||||
|
"std::sync::mpsc::Receiver::recv_timeout",
|
||||||
|
1,
|
||||||
|
"engine test completion fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-engine",
|
||||||
|
"idle_core_observes_late_external_work_within_idle_interval",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"real idle-driver timing contract",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-engine",
|
||||||
|
"spawned_supporting_work_runs",
|
||||||
|
"std::sync::mpsc::Receiver::recv_timeout",
|
||||||
|
1,
|
||||||
|
"engine test completion fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-engine",
|
||||||
|
"stepping_blocking_work_runs_isolated",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"blocking worker completion poll",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-engine",
|
||||||
|
"stepping_blocking_work_runs_isolated::{closure#0}",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"blocking worker timing probe",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-process",
|
||||||
|
"drive_once",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"process integration driver pacing",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-process",
|
||||||
|
"supervisor::tests::collect_until_finished",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"process supervisor completion fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-process",
|
||||||
|
"supervisor::tests::join_finished",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"process supervisor completion fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-process",
|
||||||
|
"supervisor::tests::supervisor_stop_escalates_to_kill_after_deadline",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"process escalation timing contract",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-vastai",
|
||||||
|
"monitor::tests::loading_state_remains_slow_progress_before_terminal_evidence::{closure#0}",
|
||||||
|
"tokio::time::timeout",
|
||||||
|
1,
|
||||||
|
"provider test non-completion fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"telemetry",
|
||||||
|
"hardware::tests::blocking_sampler_runs_sequentially_with_monotonic_sequences",
|
||||||
|
"std::sync::mpsc::Receiver::recv_timeout",
|
||||||
|
2,
|
||||||
|
"sampler test completion fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"telemetry",
|
||||||
|
"submit_text_owned_queues_owned_string",
|
||||||
|
"telemetry::TelemetrySubscription::recv_timeout",
|
||||||
|
1,
|
||||||
|
"telemetry test observation fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"xtask",
|
||||||
|
"demo::feed::properties::generated_supervisor_transitions_are_once_only_nonblocking_and_clean::{closure#1}",
|
||||||
|
"std::sync::mpsc::Receiver::recv_timeout",
|
||||||
|
1,
|
||||||
|
"blocking-provider test completion fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"actor-lint-test-wait-pass",
|
||||||
|
"bounded_observation_wait_is_allowed",
|
||||||
|
"std::sync::mpsc::Receiver::recv_timeout",
|
||||||
|
1,
|
||||||
|
"compiler policy pass fixture",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"actor-lint-timing-overage",
|
||||||
|
"exceeds_registered_timing_budget",
|
||||||
|
"std::sync::mpsc::Receiver::recv_timeout",
|
||||||
|
1,
|
||||||
|
"compiler policy overage fixture",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"swactor-vastai",
|
||||||
|
"test_http::serve",
|
||||||
|
"std::net::TcpStream::set_read_timeout",
|
||||||
|
1,
|
||||||
|
"test HTTP request completion fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"telemetry",
|
||||||
|
"carry_over_real_socket",
|
||||||
|
"std::net::UdpSocket::set_read_timeout",
|
||||||
|
1,
|
||||||
|
"live telemetry socket completion fuse",
|
||||||
|
),
|
||||||
|
timing(
|
||||||
|
"python",
|
||||||
|
"context::PyTestDataPlaneHost::run_contextual_process",
|
||||||
|
"std::thread::sleep",
|
||||||
|
1,
|
||||||
|
"debug test-host contextual output poll interval",
|
||||||
|
),
|
||||||
|
];
|
||||||
198
tools/timeout-policy.py
Executable file
198
tools/timeout-policy.py
Executable file
|
|
@ -0,0 +1,198 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Reject unregistered Python timeout and sleep call sites."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import ast
|
||||||
|
import sys
|
||||||
|
from collections import defaultdict
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
SOURCE_ROOTS = ("apps", "crates", "src", "tests", "tools", "xtask")
|
||||||
|
SKIPPED_PARTS = {".venv", "__pycache__", "target"}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Allowance:
|
||||||
|
path: str
|
||||||
|
caller: str
|
||||||
|
callee: str
|
||||||
|
calls: int
|
||||||
|
purpose: str
|
||||||
|
|
||||||
|
|
||||||
|
ALLOWANCES = (
|
||||||
|
Allowance(
|
||||||
|
"apps/myelin/node-image/tinygrad_worker.py",
|
||||||
|
"fetch_whole",
|
||||||
|
"urllib.request.urlopen",
|
||||||
|
1,
|
||||||
|
"model download request deadline",
|
||||||
|
),
|
||||||
|
Allowance(
|
||||||
|
"apps/myelin/node-image/tinygrad_worker.py",
|
||||||
|
"CpuLineSampler._run",
|
||||||
|
"self._stop.wait",
|
||||||
|
1,
|
||||||
|
"cpu sampler poll interval",
|
||||||
|
),
|
||||||
|
Allowance(
|
||||||
|
"crates/bindings/python/tests/test_bootstrap.py",
|
||||||
|
"test_real_exec_attachment_and_blob_mapping",
|
||||||
|
"subprocess.run",
|
||||||
|
1,
|
||||||
|
"child-process test completion fuse",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TimingVisitor(ast.NodeVisitor):
|
||||||
|
def __init__(self, path: str) -> None:
|
||||||
|
self.path = path
|
||||||
|
self.aliases: dict[str, str] = {}
|
||||||
|
self.scopes: list[str] = []
|
||||||
|
self.occurrences: defaultdict[tuple[str, str], int] = defaultdict(int)
|
||||||
|
self.violations: list[str] = []
|
||||||
|
|
||||||
|
def visit_Import(self, node: ast.Import) -> None:
|
||||||
|
for name in node.names:
|
||||||
|
if name.asname:
|
||||||
|
self.aliases[name.asname] = name.name
|
||||||
|
else:
|
||||||
|
root = name.name.split(".", 1)[0]
|
||||||
|
self.aliases[root] = root
|
||||||
|
|
||||||
|
def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
|
||||||
|
if node.module is not None:
|
||||||
|
for name in node.names:
|
||||||
|
self.aliases[name.asname or name.name] = f"{node.module}.{name.name}"
|
||||||
|
|
||||||
|
def visit_ClassDef(self, node: ast.ClassDef) -> None:
|
||||||
|
self.scopes.append(node.name)
|
||||||
|
self.generic_visit(node)
|
||||||
|
self.scopes.pop()
|
||||||
|
|
||||||
|
def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
|
||||||
|
self.scopes.append(node.name)
|
||||||
|
self.generic_visit(node)
|
||||||
|
self.scopes.pop()
|
||||||
|
|
||||||
|
def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None:
|
||||||
|
self.scopes.append(node.name)
|
||||||
|
self.generic_visit(node)
|
||||||
|
self.scopes.pop()
|
||||||
|
|
||||||
|
def visit_Call(self, node: ast.Call) -> None:
|
||||||
|
callee = self._callee(node.func)
|
||||||
|
if callee is not None and self._is_guarded(node, callee):
|
||||||
|
caller = ".".join(self.scopes) or "<module>"
|
||||||
|
key = (caller, callee)
|
||||||
|
self.occurrences[key] += 1
|
||||||
|
occurrence = self.occurrences[key]
|
||||||
|
allowance = next(
|
||||||
|
(
|
||||||
|
item
|
||||||
|
for item in ALLOWANCES
|
||||||
|
if item.path == self.path
|
||||||
|
and item.caller == caller
|
||||||
|
and item.callee == callee
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if allowance is None:
|
||||||
|
self.violations.append(
|
||||||
|
f"{self.path}:{node.lineno}: unapproved timing primitive "
|
||||||
|
f"`{callee}` in `{caller}`"
|
||||||
|
)
|
||||||
|
elif occurrence > allowance.calls:
|
||||||
|
self.violations.append(
|
||||||
|
f"{self.path}:{node.lineno}: timing occurrence {occurrence} exceeds "
|
||||||
|
f"the {allowance.calls} audited call(s) in `{caller}` "
|
||||||
|
f"(`{allowance.purpose}`)"
|
||||||
|
)
|
||||||
|
self.generic_visit(node)
|
||||||
|
|
||||||
|
def _callee(self, node: ast.expr) -> str | None:
|
||||||
|
parts: list[str] = []
|
||||||
|
while isinstance(node, ast.Attribute):
|
||||||
|
parts.append(node.attr)
|
||||||
|
node = node.value
|
||||||
|
if not isinstance(node, ast.Name):
|
||||||
|
return None
|
||||||
|
root = self.aliases.get(node.id, node.id)
|
||||||
|
return ".".join((root, *reversed(parts)))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _is_guarded(node: ast.Call, callee: str) -> bool:
|
||||||
|
if any(keyword.arg == "timeout" for keyword in node.keywords):
|
||||||
|
return True
|
||||||
|
return callee in {"asyncio.sleep", "asyncio.wait_for", "time.sleep"} or callee.endswith(
|
||||||
|
(".set_read_timeout", ".set_write_timeout", ".settimeout", ".wait")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def python_sources() -> list[Path]:
|
||||||
|
sources: list[Path] = []
|
||||||
|
for source_root in SOURCE_ROOTS:
|
||||||
|
for path in (ROOT / source_root).rglob("*.py"):
|
||||||
|
if not SKIPPED_PARTS.intersection(path.parts):
|
||||||
|
sources.append(path)
|
||||||
|
return sorted(sources)
|
||||||
|
|
||||||
|
|
||||||
|
def check_source(path: Path) -> list[str]:
|
||||||
|
relative = path.relative_to(ROOT).as_posix()
|
||||||
|
source = path.read_text(encoding="utf-8")
|
||||||
|
syntax = ast.parse(source, filename=relative)
|
||||||
|
visitor = TimingVisitor(relative)
|
||||||
|
visitor.visit(syntax)
|
||||||
|
return visitor.violations
|
||||||
|
|
||||||
|
|
||||||
|
def self_test() -> None:
|
||||||
|
visitor = TimingVisitor("probe.py")
|
||||||
|
visitor.visit(ast.parse("import time as clock\ndef work():\n clock.sleep(1)\n"))
|
||||||
|
assert visitor.violations == [
|
||||||
|
"probe.py:3: unapproved timing primitive `time.sleep` in `work`"
|
||||||
|
]
|
||||||
|
|
||||||
|
overage = TimingVisitor("crates/bindings/python/tests/test_bootstrap.py")
|
||||||
|
overage.visit(
|
||||||
|
ast.parse(
|
||||||
|
"import subprocess\n"
|
||||||
|
"def test_real_exec_attachment_and_blob_mapping():\n"
|
||||||
|
" subprocess.run([], timeout=1)\n"
|
||||||
|
" subprocess.run([], timeout=1)\n"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assert overage.violations == [
|
||||||
|
"crates/bindings/python/tests/test_bootstrap.py:4: timing occurrence 2 "
|
||||||
|
"exceeds the 1 audited call(s) in "
|
||||||
|
"`test_real_exec_attachment_and_blob_mapping` "
|
||||||
|
"(`child-process test completion fuse`)"
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
if sys.argv[1:] == ["--self-test"]:
|
||||||
|
self_test()
|
||||||
|
return 0
|
||||||
|
if sys.argv[1:]:
|
||||||
|
print("usage: timeout-policy.py [--self-test]", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
self_test()
|
||||||
|
|
||||||
|
violations = [violation for path in python_sources() for violation in check_source(path)]
|
||||||
|
if violations:
|
||||||
|
print("Python timeout policy violations:", file=sys.stderr)
|
||||||
|
print("\n".join(violations), file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
print("python-timeout-policy: OK")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
|
|
@ -1,5 +1,3 @@
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
pub async fn request_logs(
|
pub async fn request_logs(
|
||||||
client: &reqwest::Client,
|
client: &reqwest::Client,
|
||||||
base_url: &str,
|
base_url: &str,
|
||||||
|
|
@ -24,7 +22,6 @@ pub async fn request_logs(
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn fetch_logs(client: &reqwest::Client, log_url: &str) -> Result<String, String> {
|
pub async fn fetch_logs(client: &reqwest::Client, log_url: &str) -> Result<String, String> {
|
||||||
tokio::time::sleep(Duration::from_secs(5)).await;
|
|
||||||
let resp = client
|
let resp = client
|
||||||
.get(log_url)
|
.get(log_url)
|
||||||
.send()
|
.send()
|
||||||
|
|
|
||||||
|
|
@ -55,9 +55,6 @@ impl TestHttpServer {
|
||||||
pub fn start(routes: Vec<TestHttpRoute>) -> Result<Self, String> {
|
pub fn start(routes: Vec<TestHttpRoute>) -> Result<Self, String> {
|
||||||
let listener = TcpListener::bind(("127.0.0.1", 0))
|
let listener = TcpListener::bind(("127.0.0.1", 0))
|
||||||
.map_err(|error| format!("bind test HTTP server: {error}"))?;
|
.map_err(|error| format!("bind test HTTP server: {error}"))?;
|
||||||
listener
|
|
||||||
.set_nonblocking(true)
|
|
||||||
.map_err(|error| format!("configure test HTTP server: {error}"))?;
|
|
||||||
let address = listener
|
let address = listener
|
||||||
.local_addr()
|
.local_addr()
|
||||||
.map_err(|error| format!("read test HTTP address: {error}"))?;
|
.map_err(|error| format!("read test HTTP address: {error}"))?;
|
||||||
|
|
@ -66,14 +63,11 @@ impl TestHttpServer {
|
||||||
let thread_requests = Arc::clone(&requests);
|
let thread_requests = Arc::clone(&requests);
|
||||||
let thread_stop = Arc::clone(&stop);
|
let thread_stop = Arc::clone(&stop);
|
||||||
let join = std::thread::spawn(move || {
|
let join = std::thread::spawn(move || {
|
||||||
while !thread_stop.load(Ordering::Acquire) {
|
while let Ok((stream, _)) = listener.accept() {
|
||||||
match listener.accept() {
|
if thread_stop.load(Ordering::Acquire) {
|
||||||
Ok((stream, _)) => serve(stream, &routes, &thread_requests),
|
break;
|
||||||
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
|
|
||||||
std::thread::sleep(Duration::from_millis(5));
|
|
||||||
}
|
|
||||||
Err(_) => break,
|
|
||||||
}
|
}
|
||||||
|
serve(stream, &routes, &thread_requests);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
|
|
@ -96,6 +90,7 @@ impl TestHttpServer {
|
||||||
impl Drop for TestHttpServer {
|
impl Drop for TestHttpServer {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
self.stop.store(true, Ordering::Release);
|
self.stop.store(true, Ordering::Release);
|
||||||
|
let _ = TcpStream::connect(self.address);
|
||||||
if let Some(join) = self.join.take() {
|
if let Some(join) = self.join.take() {
|
||||||
let _ = join.join();
|
let _ = join.join();
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -13,7 +13,7 @@ const TEST_STEPS: &[TestStep] = &[
|
||||||
args: &["lint"],
|
args: &["lint"],
|
||||||
},
|
},
|
||||||
TestStep {
|
TestStep {
|
||||||
label: "all Rust tests (60s per-test timeout)",
|
label: "all Rust tests (180s per-test timeout)",
|
||||||
args: &["nextest", "run", "--workspace", "--all-features"],
|
args: &["nextest", "run", "--workspace", "--all-features"],
|
||||||
},
|
},
|
||||||
TestStep {
|
TestStep {
|
||||||
|
|
@ -82,6 +82,7 @@ COMMANDS:
|
||||||
Run the visual provisioning-reconciler demo.
|
Run the visual provisioning-reconciler demo.
|
||||||
check-telemetry-isolation
|
check-telemetry-isolation
|
||||||
Verify no frame types appear in control-plane modules.
|
Verify no frame types appear in control-plane modules.
|
||||||
|
lint Run the Python timeout policy and strict workspace Clippy.
|
||||||
test Run strict lint plus every Rust and Python test."
|
test Run strict lint plus every Rust and Python test."
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
@ -138,6 +139,31 @@ fn run_tests() -> ExitCode {
|
||||||
ExitCode::SUCCESS
|
ExitCode::SUCCESS
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn run_lint() -> ExitCode {
|
||||||
|
let policy = workspace_root().join("tools/timeout-policy.py");
|
||||||
|
let policy_status = swactor_process::command_status(Command::new("python3").arg(policy));
|
||||||
|
if !policy_status.is_ok_and(|status| status.success()) {
|
||||||
|
return ExitCode::from(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut clippy = cargo_command();
|
||||||
|
clippy.args(["clippy", "--workspace", "--all-targets", "--all-features"]);
|
||||||
|
// Pin pyo3 to the same interpreter the Python test flow uses: a newer
|
||||||
|
// system python exceeds the bundled pyo3's supported maximum and would
|
||||||
|
// fail the workspace build here even though tests build fine.
|
||||||
|
if let Some(tools) = PythonTestTools::discover() {
|
||||||
|
clippy.env("PYO3_PYTHON", tools.python);
|
||||||
|
}
|
||||||
|
match swactor_process::command_status(&mut clippy) {
|
||||||
|
Ok(status) if status.success() => ExitCode::SUCCESS,
|
||||||
|
Ok(_) => ExitCode::from(1),
|
||||||
|
Err(error) => {
|
||||||
|
eprintln!("Failed to execute cargo clippy: {error}");
|
||||||
|
ExitCode::from(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn nextest_available() -> bool {
|
fn nextest_available() -> bool {
|
||||||
let available = swactor_process::command_status(
|
let available = swactor_process::command_status(
|
||||||
cargo_command()
|
cargo_command()
|
||||||
|
|
@ -185,7 +211,8 @@ impl PythonTestTools {
|
||||||
build_command
|
build_command
|
||||||
.current_dir(&self.directory)
|
.current_dir(&self.directory)
|
||||||
.arg("develop")
|
.arg("develop")
|
||||||
.env("PYO3_PYTHON", &self.python),
|
.arg("--features")
|
||||||
|
.arg("test-host"),
|
||||||
)
|
)
|
||||||
.is_ok_and(|status| status.success());
|
.is_ok_and(|status| status.success());
|
||||||
if !built {
|
if !built {
|
||||||
|
|
@ -298,6 +325,7 @@ fn main() -> ExitCode {
|
||||||
ExitCode::from(1)
|
ExitCode::from(1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Some("lint") if args.next().is_none() => run_lint(),
|
||||||
Some("test") if args.next().is_none() => run_tests(),
|
Some("test") if args.next().is_none() => run_tests(),
|
||||||
Some("demo") => demo::run(&args.collect::<Vec<_>>()),
|
Some("demo") => demo::run(&args.collect::<Vec<_>>()),
|
||||||
Some("help" | "--help" | "-h") | None => {
|
Some("help" | "--help" | "-h") | None => {
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue