From 7d11dfa6884f6656b7910133bc571af89f348b4e Mon Sep 17 00:00:00 2001 From: Zachery Aaron Shores-Chmielewski Date: Thu, 5 Feb 2026 23:01:05 +0700 Subject: [PATCH] feat: Scheduler with basic metrics, docker Implementation of a mock scheduler for jobs/tasks that will evolve into a job manager for running training jobs over vast.ai instances. --- .dockerignore | 12 + .python-version | 1 + docker/docker-compose.cpu.yml | 27 ++ docker/docker-compose.yml | 27 ++ docker/scheduler.Dockerfile | 15 + docker/smoke-test.sh | 18 + docker/worker-cpu.Dockerfile | 22 + docker/worker-entrypoint.sh | 8 + docker/worker.Dockerfile | 20 + jobs/__init__.py | 0 jobs/__pycache__/mnist.cpython-313.pyc | Bin 0 -> 2877 bytes jobs/mnist.py | 54 +++ main.py | 6 + pyproject.toml | 14 + report.html | 222 +++++++++ reports/log.txt | 31 ++ reports/report.html | 140 ++++++ sched/__init__.py | 0 sched/__main__.py | 122 +++++ sched/__pycache__/__init__.cpython-313.pyc | Bin 0 -> 139 bytes sched/__pycache__/__main__.cpython-313.pyc | Bin 0 -> 6458 bytes sched/__pycache__/aggregator.cpython-313.pyc | Bin 0 -> 969 bytes sched/__pycache__/job.cpython-313.pyc | Bin 0 -> 1058 bytes sched/__pycache__/report.cpython-313.pyc | Bin 0 -> 19100 bytes sched/__pycache__/scheduler.cpython-313.pyc | Bin 0 -> 16950 bytes sched/__pycache__/trace.cpython-313.pyc | Bin 0 -> 21349 bytes sched/__pycache__/transport.cpython-313.pyc | Bin 0 -> 5496 bytes sched/__pycache__/vastai.cpython-313.pyc | Bin 0 -> 2556 bytes sched/aggregator.py | 18 + sched/job.py | 17 + sched/report.py | 403 +++++++++++++++++ sched/scheduler.py | 300 +++++++++++++ sched/trace.py | 424 +++++++++++++++++ sched/transport.py | 84 ++++ sched/vastai.py | 47 ++ tests/__init__.py | 0 tests/__pycache__/__init__.cpython-313.pyc | Bin 0 -> 139 bytes .../mock_transport.cpython-313.pyc | Bin 0 -> 5399 bytes ...st_aggregator.cpython-313-pytest-9.0.2.pyc | Bin 0 -> 6740 bytes ...est_scheduler.cpython-313-pytest-9.0.2.pyc | Bin 0 -> 23154 bytes .../test_trace.cpython-313-pytest-9.0.2.pyc | Bin 0 -> 35130 bytes tests/mock_transport.py | 85 ++++ tests/test_aggregator.py | 57 +++ tests/test_scheduler.py | 261 +++++++++++ tests/test_trace.py | 265 +++++++++++ uv.lock | 425 ++++++++++++++++++ worker/__init__.py | 0 worker/worker.py | 62 +++ 48 files changed, 3187 insertions(+) create mode 100644 .dockerignore create mode 100644 .python-version create mode 100644 docker/docker-compose.cpu.yml create mode 100644 docker/docker-compose.yml create mode 100644 docker/scheduler.Dockerfile create mode 100755 docker/smoke-test.sh create mode 100644 docker/worker-cpu.Dockerfile create mode 100644 docker/worker-entrypoint.sh create mode 100644 docker/worker.Dockerfile create mode 100644 jobs/__init__.py create mode 100644 jobs/__pycache__/mnist.cpython-313.pyc create mode 100644 jobs/mnist.py create mode 100644 main.py create mode 100644 pyproject.toml create mode 100644 report.html create mode 100644 reports/log.txt create mode 100644 reports/report.html create mode 100644 sched/__init__.py create mode 100644 sched/__main__.py create mode 100644 sched/__pycache__/__init__.cpython-313.pyc create mode 100644 sched/__pycache__/__main__.cpython-313.pyc create mode 100644 sched/__pycache__/aggregator.cpython-313.pyc create mode 100644 sched/__pycache__/job.cpython-313.pyc create mode 100644 sched/__pycache__/report.cpython-313.pyc create mode 100644 sched/__pycache__/scheduler.cpython-313.pyc create mode 100644 sched/__pycache__/trace.cpython-313.pyc create mode 100644 sched/__pycache__/transport.cpython-313.pyc create mode 100644 sched/__pycache__/vastai.cpython-313.pyc create mode 100644 sched/aggregator.py create mode 100644 sched/job.py create mode 100644 sched/report.py create mode 100644 sched/scheduler.py create mode 100644 sched/trace.py create mode 100644 sched/transport.py create mode 100644 sched/vastai.py create mode 100644 tests/__init__.py create mode 100644 tests/__pycache__/__init__.cpython-313.pyc create mode 100644 tests/__pycache__/mock_transport.cpython-313.pyc create mode 100644 tests/__pycache__/test_aggregator.cpython-313-pytest-9.0.2.pyc create mode 100644 tests/__pycache__/test_scheduler.cpython-313-pytest-9.0.2.pyc create mode 100644 tests/__pycache__/test_trace.cpython-313-pytest-9.0.2.pyc create mode 100644 tests/mock_transport.py create mode 100644 tests/test_aggregator.py create mode 100644 tests/test_scheduler.py create mode 100644 tests/test_trace.py create mode 100644 uv.lock create mode 100644 worker/__init__.py create mode 100644 worker/worker.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..fbe08b2 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +.git +.venv +.pytest_cache +__pycache__ +*.pyc +.python-version +uv.lock +report.html +dtrain +DESIGN.md +README.md +.claude diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..24ee5b1 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.13 diff --git a/docker/docker-compose.cpu.yml b/docker/docker-compose.cpu.yml new file mode 100644 index 0000000..d169f78 --- /dev/null +++ b/docker/docker-compose.cpu.yml @@ -0,0 +1,27 @@ +services: + scheduler: + build: + context: .. + dockerfile: docker/scheduler.Dockerfile + depends_on: + - worker-0 + - worker-1 + networks: + - fedavg + + worker-0: + build: + context: .. + dockerfile: docker/worker-cpu.Dockerfile + networks: + - fedavg + + worker-1: + build: + context: .. + dockerfile: docker/worker-cpu.Dockerfile + networks: + - fedavg + +networks: + fedavg: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..74978a3 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,27 @@ +services: + scheduler: + build: + context: .. + dockerfile: docker/scheduler.Dockerfile + depends_on: + - worker-0 + - worker-1 + networks: + - fedavg + + worker-0: + build: + context: .. + dockerfile: docker/worker.Dockerfile + networks: + - fedavg + + worker-1: + build: + context: .. + dockerfile: docker/worker.Dockerfile + networks: + - fedavg + +networks: + fedavg: diff --git a/docker/scheduler.Dockerfile b/docker/scheduler.Dockerfile new file mode 100644 index 0000000..6db536e --- /dev/null +++ b/docker/scheduler.Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.13-slim + +RUN apt-get update && apt-get install -y openssh-client && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY pyproject.toml ./ +COPY sched/ ./sched/ +COPY worker/ ./worker/ +COPY jobs/ ./jobs/ +COPY tests/ ./tests/ + +RUN pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cpu + +CMD ["python", "-m", "sched", "run", "--script", "jobs/mnist.py", "--compose", "--trace"] diff --git a/docker/smoke-test.sh b/docker/smoke-test.sh new file mode 100755 index 0000000..5f4e242 --- /dev/null +++ b/docker/smoke-test.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +ROUNDS="${1:-1}" +NODES="${2:-2}" + +cd "$(dirname "$0")" + +mkdir -p ../reports + +docker compose -f docker-compose.cpu.yml run \ + -v "$(pwd)/../reports:/app/reports" \ + scheduler \ + bash -c "python -m sched run --script jobs/mnist.py --compose --trace --rounds $ROUNDS --nodes $NODES --report reports/report.html 2>&1 | tee reports/log.txt" + +docker compose -f docker-compose.cpu.yml down + +echo "Done. See reports/report.html and reports/log.txt" diff --git a/docker/worker-cpu.Dockerfile b/docker/worker-cpu.Dockerfile new file mode 100644 index 0000000..f504d58 --- /dev/null +++ b/docker/worker-cpu.Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.13-slim + +RUN apt-get update && apt-get install -y openssh-server && rm -rf /var/lib/apt/lists/* +RUN mkdir -p /run/sshd + +# Allow root login with no password +RUN sed -i 's/#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config \ + && sed -i 's/#PermitEmptyPasswords.*/PermitEmptyPasswords yes/' /etc/ssh/sshd_config \ + && passwd -d root + +RUN pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cpu + +WORKDIR /workspace + +COPY worker/worker.py /workspace/worker.py +COPY jobs/ /workspace/jobs/ + +COPY docker/worker-entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +EXPOSE 22 +CMD ["/entrypoint.sh"] diff --git a/docker/worker-entrypoint.sh b/docker/worker-entrypoint.sh new file mode 100644 index 0000000..98c1182 --- /dev/null +++ b/docker/worker-entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + +# Start SSH daemon +/usr/sbin/sshd + +# Keep container alive +exec tail -f /dev/null diff --git a/docker/worker.Dockerfile b/docker/worker.Dockerfile new file mode 100644 index 0000000..08966c4 --- /dev/null +++ b/docker/worker.Dockerfile @@ -0,0 +1,20 @@ +FROM pytorch/pytorch:2.5.0-cuda12.4-cudnn9-runtime + +RUN apt-get update && apt-get install -y openssh-server && rm -rf /var/lib/apt/lists/* +RUN mkdir -p /run/sshd + +# Allow root login with no password (vast.ai injects keys at runtime) +RUN sed -i 's/#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config \ + && sed -i 's/#PermitEmptyPasswords.*/PermitEmptyPasswords yes/' /etc/ssh/sshd_config \ + && passwd -d root + +WORKDIR /workspace + +COPY worker/worker.py /workspace/worker.py +COPY jobs/ /workspace/jobs/ + +COPY docker/worker-entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +EXPOSE 22 +CMD ["/entrypoint.sh"] diff --git a/jobs/__init__.py b/jobs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/jobs/__pycache__/mnist.cpython-313.pyc b/jobs/__pycache__/mnist.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fc41ac8dceddee5cdf6311382e56898370d4bbf3 GIT binary patch literal 2877 zcmahKOKcm*b%wi3E|;V!iV#J(q4-8)@ys}Q3eZC7`U*3^rD*zDXq~<``#=mu|m;2 zf^Xlv_h#nJ`|f5k1_AiZFMr~-H9|heMGs(&KzBC)KPDF8#G+PgmX=~XW+JW3DqN9Y zwWM*a%Zrz^0OX|v*Cj`tO_mI9lu|rZO7pZyPLMOi(ng3C|3E8`;2r-BlS;JaN35ik z7$G-Ro;^)0<2332St(0L*-gp^ZYYKH-S)M)_pEvIW^=(@YSvr6TQrO-E6YtD2Ii7; z%e8^)`sWL#!|D;E?u3rtbm}e#ZllS~aM3k`+M zn`W@eKr!@cCLUmV4Pf%x;%zVRn#}Y9lQl!rV*woQyY-^6jn64ik#ZPFoDj5vxq>2$ zZ^8EW<^3XVwH5+56q$LK1x=0;Sl0R-6N?sfw+O`dAtE0TP9Z8Wi*ltxCLlK851p%) zGD5g!sXSgGZ&RLljTAJYOYh9N;dU0b*J$6#D$w9^y8AH@)ATW^?hEH>g?tYZVY16H zNfosvl;b_>;0u8TGbX#HR*1i^{iXY2meRMcn|y@aAO&^K8vy&Z*WjLQ%-2F(XhCb) zo-Kwwrjo<1hyTZZRVO78((ZLx#Nz* zn{52H1HtmX(h5C47>5K3#+R5EgvI4megxFT(Vp{)z551$9lQbPL>oJ zk37o_J(>D=s%>qY?c~PR)t~AIoHL!=xpnnt`Y!AuwXTbhCiqwxfZqb~m{e%FKRl=b z09?>H3>Ka2>K?eA(Jym$g_M(rPr@5|XZ5}e!e37PKTXIjWsc)xw*L(Pkx_4Z?j3Nu zch?nkMyTBNTl^>}@IgR@7|+2+(3L0-)A{p!^uV0E6_7d@I+1ln4tkK&QO?kbDtSw6T90igs#; z0FvfPr5ech0sSl!Wf%yk)BQB*)hC35&cU5{$FE6rigd7X&;fY?gs0#iU@bxVrg9&> zwW$tC?OK11;VJUEMY!lObYI#3Nr^J06l1DMDoQy4I)}hgEY(sg>i^2Io}rdDr82FG zQISwR*`q3?ipEK(XuV8?-0WJ`-3SFzS5l(D=I!5LDCTM0|I^vP~hV~M^U7{@~(n_;pNpV+sZ z=~aw|gzGE&Hfwn@_hEy)t>j6A7E)0?LGRLWzQ7m=gb{Lw1yB-8!QJe5xU?KKC>s^# zVU%bf_ZVzw-eEY=L~?(FN8mBrm?_(@0OI&~96k$9SX;DZnI?k8R-@s&B3b9nWg8l0 zSQrqgzNyA9;MUgx2~Ae0(C3@cs1m97mT<^k;k?O3^2$ohUB(dyHsMqh8dzRuB7q-| zg&e1bPbg3n}Os4!b1`n7)2pTF9}}_Sul=Iz~{&Pvd!XRU|w( zzvc!a9>`G|VBaziAZj#OSqHd18vOh!Xko?#zXP;JcH&30vFG~4i@}j6#Ye@h^s8Hg z!&^fqw}#EX3>Tgb7dpcic9JAFxn(S=_y8RBkv7mGO2nB<287N*tBt0ytZ{t!%;;2a2 zcD-4PI%J~-2SSJBScv>BKth)xE;7ckmx)|&S!}de4U@p}g>gW0Oh7RZX~|(*#7M}? zq6Co15s&PR8YMO>86$XDKPA(9@gzO9&L6DaTiqi-?wz1?c*h`Lo7_>3Q;TjHnJ#habeDL2JR)9b L*?(xLBhB~^>~(*w literal 0 HcmV?d00001 diff --git a/jobs/mnist.py b/jobs/mnist.py new file mode 100644 index 0000000..7842e83 --- /dev/null +++ b/jobs/mnist.py @@ -0,0 +1,54 @@ +"""MNIST job module. + +Exports make_model() and make_dataloader() for the scheduler + worker. +Falls back to synthetic data if torchvision is not installed. +""" + +from __future__ import annotations + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.utils.data import DataLoader, Subset, TensorDataset + + +class MNISTNet(nn.Module): + def __init__(self): + super().__init__() + self.fc1 = nn.Linear(784, 128) + self.fc2 = nn.Linear(128, 10) + + def forward(self, x): + x = x.view(x.size(0), -1) + x = F.relu(self.fc1(x)) + return self.fc2(x) + + +def make_model() -> nn.Module: + return MNISTNet() + + +def make_dataloader( + rank: int, world_size: int, batch_size: int = 64 +) -> DataLoader: + try: + from torchvision import datasets, transforms + + dataset = datasets.MNIST( + "/tmp/mnist_data", + train=True, + download=True, + transform=transforms.ToTensor(), + ) + except (ImportError, Exception): + # synthetic fallback — same shape as MNIST + n = 4096 + x = torch.randn(n, 1, 28, 28) + y = torch.randint(0, 10, (n,)) + dataset = TensorDataset(x, y) + + # shard by rank: interleaved assignment + indices = list(range(rank, len(dataset), world_size)) + subset = Subset(dataset, indices) + + return DataLoader(subset, batch_size=batch_size, shuffle=True, drop_last=True) diff --git a/main.py b/main.py new file mode 100644 index 0000000..61bab30 --- /dev/null +++ b/main.py @@ -0,0 +1,6 @@ +def main(): + print("Hello from vastai-utils!") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..de029ae --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,14 @@ +[project] +name = "vastai-utils" +version = "0.1.0" +description = "Add your description here" +requires-python = ">=3.13" +dependencies = [ + "torch>=2.10.0", +] + +[dependency-groups] +dev = [ + "pytest>=9.0.2", + "pytest-asyncio>=1.3.0", +] diff --git a/report.html b/report.html new file mode 100644 index 0000000..e3f5ca8 --- /dev/null +++ b/report.html @@ -0,0 +1,222 @@ + + + + +Training Run Report + + + +

Training Run Report

+ +
+
+
3
+
Rounds
+
+
+
2
+
Workers
+
+
+
15.6s
+
Total Time
+
+
+
4.7MB
+
Data Transferred
+
+
+
2.3MB
+
Params Pushed
+
+
+
2.3MB
+
Weights Pulled
+
+
+
8.7
+
Final |W|
+
+
+ +
+
Push params
+
Train
+
Pull weights
+
Aggregate
+
Dead / Error
+
+ +
+ + +round 0 [2/2] 4.59s + +0.0s + +0.9s + +1.8s + +2.8s + +3.7s + +4.6s +scheduler + +push 399.7KB → node 0 (0.000s) +push 399.7KB → node 1 (0.000s) +pull 399.7KB ← node 0 (0.000s) +pull 399.7KB ← node 1 (0.000s) +aggregate: 2 workers, |W|=8.75, Δ=0.0891 +worker 0 + +push 399.7KB → node 0 (0.000s) +train node 0: 4.59s exit=0 +cpu = _conversion_method_template(device=torch.device("cpu")) +pull 399.7KB ← node 0 (0.000s) +worker 1 + +push 399.7KB → node 1 (0.000s) +train node 1: 4.31s exit=0 +cpu = _conversion_method_template(device=torch.device("cpu")) +pull 399.7KB ← node 1 (0.000s) +round 1 [2/2] 5.53s + +0.0s + +1.1s + +2.2s + +3.3s + +4.4s + +5.5s +scheduler + +push 399.6KB → node 0 (0.000s) +push 399.6KB → node 1 (0.002s) +pull 399.7KB ← node 0 (0.000s) +pull 399.7KB ← node 1 (0.000s) +aggregate: 2 workers, |W|=8.74, Δ=0.0876 +worker 0 + +push 399.6KB → node 0 (0.000s) +train node 0: 5.35s exit=0 +cpu = _conversion_method_template(device=torch.device("cpu")) +pull 399.7KB ← node 0 (0.000s) +worker 1 + +push 399.6KB → node 1 (0.002s) +train node 1: 5.53s exit=0 +cpu = _conversion_method_template(device=torch.device("cpu")) +pull 399.7KB ← node 1 (0.000s) +round 2 [2/2] 5.50s + +0.0s + +1.1s + +2.2s + +3.3s + +4.4s + +5.5s +scheduler + +push 399.6KB → node 0 (0.000s) +push 399.6KB → node 1 (0.000s) +pull 399.7KB ← node 0 (0.000s) +pull 399.7KB ← node 1 (0.000s) +aggregate: 2 workers, |W|=8.74, Δ=0.0883 +worker 0 + +push 399.6KB → node 0 (0.000s) +train node 0: 5.50s exit=0 +cpu = _conversion_method_template(device=torch.device("cpu")) +pull 399.7KB ← node 0 (0.000s) +worker 1 + +push 399.6KB → node 1 (0.000s) +train node 1: 5.38s exit=0 +cpu = _conversion_method_template(device=torch.device("cpu")) +pull 399.7KB ← node 1 (0.000s) + +
+ +
+
[   0.00s] push  node=0  worker.py               1.8KB  0.000s
+[   0.00s] push  node=0  job_module.py           1.4KB  0.000s
+[   0.00s] push  node=1  worker.py               1.8KB  0.000s
+[   0.00s] push  node=1  job_module.py           1.4KB  0.000s
+[   0.00s] ──── round 0 start (2 workers) ────────────────
+[   0.00s] push  node=0  params.pt             399.7KB  0.000s
+[   0.00s] push  node=1  params.pt             399.7KB  0.000s
+[   4.32s] exec  node=1  exit=0    4.313s  | cpu = _conversion_method_template(device=torch.device("cpu"))
+[   4.59s] exec  node=0  exit=0    4.587s  | cpu = _conversion_method_template(device=torch.device("cpu"))
+[   4.59s] pull  node=0  weights.pt            399.7KB  0.000s
+[   4.59s] pull  node=1  weights.pt            399.7KB  0.000s
+[   4.59s] agg   2 workers  |W|=8.75  Δ=0.0891  0.001s
+[   4.59s] ──── round 0 end (2/2 survived, 4.59s) ────────
+[   4.59s] ──── round 1 start (2 workers) ────────────────
+[   4.59s] push  node=0  params.pt             399.6KB  0.000s
+[   4.60s] push  node=1  params.pt             399.6KB  0.002s
+[   9.95s] exec  node=0  exit=0    5.350s  | cpu = _conversion_method_template(device=torch.device("cpu"))
+[  10.12s] exec  node=1  exit=0    5.525s  | cpu = _conversion_method_template(device=torch.device("cpu"))
+[  10.12s] pull  node=0  weights.pt            399.7KB  0.000s
+[  10.12s] pull  node=1  weights.pt            399.7KB  0.000s
+[  10.13s] agg   2 workers  |W|=8.74  Δ=0.0876  0.001s
+[  10.13s] ──── round 1 end (2/2 survived, 5.53s) ────────
+[  10.13s] ──── round 2 start (2 workers) ────────────────
+[  10.13s] push  node=0  params.pt             399.6KB  0.000s
+[  10.13s] push  node=1  params.pt             399.6KB  0.000s
+[  15.51s] exec  node=1  exit=0    5.378s  | cpu = _conversion_method_template(device=torch.device("cpu"))
+[  15.63s] exec  node=0  exit=0    5.498s  | cpu = _conversion_method_template(device=torch.device("cpu"))
+[  15.63s] pull  node=0  weights.pt            399.7KB  0.000s
+[  15.63s] pull  node=1  weights.pt            399.7KB  0.000s
+[  15.63s] agg   2 workers  |W|=8.74  Δ=0.0883  0.001s
+[  15.63s] ──── round 2 end (2/2 survived, 5.50s) ────────
+
+ + + \ No newline at end of file diff --git a/reports/log.txt b/reports/log.txt new file mode 100644 index 0000000..d0403cc --- /dev/null +++ b/reports/log.txt @@ -0,0 +1,31 @@ +/usr/local/lib/python3.13/site-packages/torch/_subclasses/functional_tensor.py:283: UserWarning: Failed to initialize NumPy: No module named 'numpy' (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:84.) + cpu = _conversion_method_template(device=torch.device("cpu")) +[ 0.15s] exec node=1 exit=0 0.142s | Warning: Permanently added 'worker-1' (ED25519) to the list of known hosts. +[ 0.15s] exec node=0 exit=0 0.144s | Warning: Permanently added 'worker-0' (ED25519) to the list of known hosts. +[ 0.30s] push node=1 job_module.py 1.4KB 0.146s +[ 0.30s] push node=0 job_module.py 1.4KB 0.146s +[ 0.30s] ──── round 0 start (2 workers) ──────────────────── +[ 0.45s] push node=0 params.pt 399.7KB 0.145s +[ 0.45s] push node=1 params.pt 399.7KB 0.146s +[ 9.86s] exec node=1 exit=0 9.416s | cpu = _conversion_method_template(device=torch.device("cpu")) +[ 9.87s] exec node=0 exit=0 9.419s | cpu = _conversion_method_template(device=torch.device("cpu")) +[ 10.04s] pull node=0 weights.pt 399.7KB 0.174s +[ 10.21s] pull node=1 weights.pt 399.7KB 0.165s +[ 10.21s] agg 2 workers |W|=8.72 Δ=0.3177 0.002s +[ 10.21s] ──── round 0 end (2/2 survived, 9.91s) ──────────── +2026-02-05 15:59:23,615 INFO sched.scheduler: Round 0: 2/2 workers + + ▓ push █ train ▒ pull ● agg ✗ dead ░ wait + + round 0 [2/2] 9.91s + scheduler ▓▓░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░▒▒● + worker 0 █████████████████████████████████████████████████████████▒▒─ + worker 1 ██████████████████████████████████████████████████████████▒▒ + + +Round Workers Params↑ Weights↓ Train(max) Agg Total Δ norm +──────────────────────────────────────────────────────────────────────────── +0 2/2 799.4KB 799.4KB 9.42 0.002 9.91 0.3177 +──────────────────────────────────────────────────────────────────────────── +total 799.4KB 799.4KB 9.91 +Report: /app/reports/report.html diff --git a/reports/report.html b/reports/report.html new file mode 100644 index 0000000..e871630 --- /dev/null +++ b/reports/report.html @@ -0,0 +1,140 @@ + + + + +Training Run Report + + + +

Training Run Report

+ +
+
+
1
+
Rounds
+
+
+
2
+
Workers
+
+
+
9.9s
+
Total Time
+
+
+
1.6MB
+
Data Transferred
+
+
+
799.4KB
+
Params Pushed
+
+
+
799.4KB
+
Weights Pulled
+
+
+
8.7
+
Final |W|
+
+
+ +
+
Push params
+
Train
+
Pull weights
+
Aggregate
+
Dead / Error
+
+ +
+ + +round 0 [2/2] 9.91s + +0.0s + +2.0s + +4.0s + +5.9s + +7.9s + +9.9s +scheduler + +push 399.7KB → node 0 (0.145s) +push 399.7KB → node 1 (0.146s) +pull 399.7KB ← node 0 (0.174s) +pull 399.7KB ← node 1 (0.165s) +aggregate: 2 workers, |W|=8.72, Δ=0.3177 +worker 0 + +push 399.7KB → node 0 (0.145s) +train node 0: 9.42s exit=0 +cpu = _conversion_method_template(device=torch.device("cpu")) +pull 399.7KB ← node 0 (0.174s) +worker 1 + +push 399.7KB → node 1 (0.146s) +train node 1: 9.42s exit=0 +cpu = _conversion_method_template(device=torch.device("cpu")) +pull 399.7KB ← node 1 (0.165s) + +
+ +
+
[   0.15s] exec  node=1  exit=0    0.142s  | Warning: Permanently added 'worker-1' (ED25519) to the list of known hosts.
+[   0.15s] exec  node=0  exit=0    0.144s  | Warning: Permanently added 'worker-0' (ED25519) to the list of known hosts.
+[   0.30s] push  node=1  job_module.py           1.4KB  0.146s
+[   0.30s] push  node=0  job_module.py           1.4KB  0.146s
+[   0.30s] ──── round 0 start (2 workers) ────────────────
+[   0.45s] push  node=0  params.pt             399.7KB  0.145s
+[   0.45s] push  node=1  params.pt             399.7KB  0.146s
+[   9.86s] exec  node=1  exit=0    9.416s  | cpu = _conversion_method_template(device=torch.device("cpu"))
+[   9.87s] exec  node=0  exit=0    9.419s  | cpu = _conversion_method_template(device=torch.device("cpu"))
+[  10.04s] pull  node=0  weights.pt            399.7KB  0.174s
+[  10.21s] pull  node=1  weights.pt            399.7KB  0.165s
+[  10.21s] agg   2 workers  |W|=8.72  Δ=0.3177  0.002s
+[  10.21s] ──── round 0 end (2/2 survived, 9.91s) ────────
+
+ + + \ No newline at end of file diff --git a/sched/__init__.py b/sched/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sched/__main__.py b/sched/__main__.py new file mode 100644 index 0000000..a20fd8d --- /dev/null +++ b/sched/__main__.py @@ -0,0 +1,122 @@ +"""Entry point: python -m sched run --script jobs/mnist.py --trace --report report.html""" + +from __future__ import annotations + +import argparse +import asyncio +import logging +import sys + +from .job import Job +from .scheduler import Scheduler +from .transport import SSHTransport + + +async def _run_mock(sched: Scheduler, job: Job): + """Run with mock transport: create local nodes, skip provisioning.""" + from .scheduler import Node + + for rank in range(job.num_nodes): + conn = await sched.transport.connect(f"mock{rank}", 22) + sched.nodes.append( + Node(rank=rank, host=f"mock{rank}", port=22, conn=conn, status="ready") + ) + sched._wrap_connections() + await sched.deploy() + + job_mod = sched._load_job_module() + state = job_mod.make_model().state_dict() + for r in range(job.rounds): + state = await sched.run_round(state, r) + + +async def _run_compose(sched: Scheduler, job: Job): + """Run against docker-compose workers: worker-0:22, worker-1:22, etc.""" + from .scheduler import Node + + for rank in range(job.num_nodes): + host = f"worker-{rank}" + conn = await sched.transport.connect(host, 22) + sched.nodes.append( + Node(rank=rank, host=host, port=22, conn=conn, status="ready") + ) + sched._wrap_connections() + await sched.deploy() + + job_mod = sched._load_job_module() + state = job_mod.make_model().state_dict() + for r in range(job.rounds): + state = await sched.run_round(state, r) + + +def main(): + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + + parser = argparse.ArgumentParser(prog="sched") + sub = parser.add_subparsers(dest="cmd") + + p = sub.add_parser("run", help="Run a training job") + p.add_argument("--script", required=True, help="Path to job module") + p.add_argument("--nodes", type=int, default=2) + p.add_argument("--vram", type=int, default=8) + p.add_argument("--rounds", type=int, default=10) + p.add_argument("--local-steps", type=int, default=100) + p.add_argument("--lr", type=float, default=0.01) + p.add_argument("--batch-size", type=int, default=64) + p.add_argument("--trace", action="store_true", help="Enable live tracing to stderr") + p.add_argument("--report", metavar="PATH", help="Write HTML report to PATH") + p.add_argument("--mock", action="store_true", help="Use local mock transport (no vast.ai)") + p.add_argument("--compose", action="store_true", help="Connect to docker-compose workers (worker-0:22, worker-1:22, ...)") + p.add_argument("--docker-image", metavar="IMAGE", help="Override docker_image in job config") + + args = parser.parse_args() + if not args.cmd: + parser.print_help() + return 1 + + tracer = None + if args.trace or args.report: + from .trace import Tracer + tracer = Tracer(file=sys.stderr) + + job = Job( + script=args.script, + num_nodes=args.nodes, + min_vram=args.vram, + rounds=args.rounds, + local_steps=args.local_steps, + lr=args.lr, + batch_size=args.batch_size, + ) + + if args.docker_image: + job.docker_image = args.docker_image + + if args.mock: + from tests.mock_transport import MockTransport + transport = MockTransport() + sched = Scheduler(job, transport, tracer=tracer) + asyncio.run(_run_mock(sched, job)) + elif args.compose: + sched = Scheduler(job, SSHTransport(), tracer=tracer) + asyncio.run(_run_compose(sched, job)) + else: + sched = Scheduler(job, SSHTransport(), tracer=tracer) + asyncio.run(sched.run()) + + if tracer: + tracer.gantt() + tracer.summary() + if args.report: + from .report import generate_html + path = generate_html(tracer, args.report) + print(f"Report: {path}", file=sys.stderr) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/sched/__pycache__/__init__.cpython-313.pyc b/sched/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5b17d9edbafcc7f95a4985116d190daa3d0c99c9 GIT binary patch literal 139 zcmey&%ge<81euFlGC}lX5CH>>P{wB#AY&>+I)f&o-%5reCLr%KNa~iBenx(7s(xZ( zQGT9&Sz>WXVy13sNoG#5esOX}YKne*d}dx|NqoFsLFFwDo80`A(wtPgB37VQkkQ2; O#z$sGM#ds$APWHSd>@Md literal 0 HcmV?d00001 diff --git a/sched/__pycache__/__main__.cpython-313.pyc b/sched/__pycache__/__main__.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a9a6d1b41c9394e4004b82efe4481940a4ae4cc0 GIT binary patch literal 6458 zcmc&2TWlN0@s4*Sk8i!Lhi#oD*%oC{a{SW6iEUZ3ElaWN^2ElC+^}czOfe>p%-%_f z=(JGN0G5&fl8~ly(Et?+f5NebxCj)MaI>&%fS$ugxT zZP6~sz1`W_+1c5d*_nOfa@i>;4}Sl4H07bFFR^13HrJq^d<=~bD1joBKntdRy3a&R zJhqvBhA{exP`1xvrJ&ra-m+uHBwa}p=CodQ=)y~+|7 zv}}6IE!e9m;<->HI6BW$lvqho7cBldgQ?YT9d*G|M;*)q4$`>>Peqsr<8`G5Psl16 zu^%Rd1M(I+0v<%W6G8?!7+HW!1t6yWkrBv@ZijtWC<>yvMjg}u* zSyu!mz__XXCV@Se=jSvfn1>mP8l^vA2Fy-MW`UY>-8AU>K||J;fT@lOFsn2JaMFUc z!))-!Q)m1X<$(PG&Eoa5sg?87$4fUJU(chCU%oNEYz~ zppQ8P>f}R25><(hB|>71mlI)0Y37y7(IlTFiP5M6`WTfXEd)*X2CcCJfT6KSWVr%B z1DYA;hsSQ1H5O=I*4W{MqG~KEr^bd7vaDGYQ0o!phDWpMmV$ar&_}>jQI1HOO&*B{ z^_4Z-3S*iTCrcq!Gv|gZVlpYoVXZj$IuVn>{3x1>W(iBlSYk{o3dRy*I0(iUj3>}u zG<#gUEMZ5AX?Em83WlRV(?Sv>a#+!9V6{QL?d3GH&Ys4=e4Rgn%1KI)Q)`|k;2;r= zCJNdHpD$Wd>u@42wTdE1$gQIyAVqy6YBZ*_>NHz}!MGTegF)~Hnk|Tnq6CQ-Xy77B z7OG`xocfxr{i>*ZIy4)fiGNYlFu`Wat7n5V!AyB`y1aQ_UFcajH2>pIk1v%U&3KN& zFz1?bOgb`Lb(*Wb=Pa5I&mEX&=SJtYE%-kT|G6#g6#mYbZ1%@ys(9xkJ7ueb5f=&nU$G%!GAe2gm)V z4sHM)oZ(`0=ky(29XO+not*jmIl1!(1U6uP#<_v>{XaN2OThAPJ2&h1bZ*=R&h3p2 zotyP3=k|wn&h0olH!%Vhrl@=v3`-)uP$Hg8C=&mAfrlzd9DL#Vt)c|opP25`)7Jg^i4C}{k-L-rOO4KTczL(*jr^_|0eqOK5zz>+xzX(-9v1J9KS(7@2B!vtZ`N+?t)+^M4k zMqoQ=8H0^Np%2(EjM5NNp`xrSf>q!ITZc(D1x$rP9llX4GKA`%E@dNZ7v9HDG1$L5W4SKsNn0c$`+u0;Mp;QIT0mY5F=oTtwv!hkv@x?FA8r4gRRT7Dm$A<3*Wh(^-vM@I zhhaE1*5AL1ZM2M8a0;ydi~(K!t{LClE`Fwf?=~GlGsVu`eQYjW-Byy$f<5Pc4y?r$0^!YA|YG~M} zL{rl8F~oocf$H!;m-vTNV)nnviom zBOm&sipHMn8thF~y+opr&h`$Tc`^UYf^%^QW?JKtDvpXIHx>4SSy*ak0OG`4{`+)t z%s0vj9<#X?G3rg(d`3{2>gvwrrbxPQyvaA_qsRjBW=l(pH|6&E@-w4xIC;%{`b^iU z6RG;Mqp-Z5#A)(;Mn-b>Mlo{PgFTLw{f_$I_&io$tXA=TEOp9sEEXj~jJ zPCPJz@ulX_cNo;qM>EBUX4g+6h^ImH1E)3D8IT81&Akoa68iDr60V3OO9T^We6>n( zeqHFDyw|H)V~I!v4oP!d6qRTQWCKiU?CF7%XEjbFk)%izNpp9R$Vgn0)pOV-np+Hq zgUZN7eUvDg9s9YC=EOG4)bTY2-rF?uCCKen&8}nMm_qQv004pEAnLnj*13h`UgwRV z^U)Y(3?2w_xq1mixfOa+@$T2MnkPxB%o zy*LFUNX;rLV{#~(Ab4ZxnLR#X=t&5%B4`g{nIodCs+v_9iN{4UM*4K1z6Y;Uv*h>B zfLt^>sj>P#(OK)8$z>h-lJJH}d&avTa*iqYqy~wqXcV$`g)q$T6On1(+i~CQF52V>%1JrSS{{oGW zeFmicdj0%s2zkzcbYAaXC_%^p19J4E(uEL0It|F7>*jg&DV!r8F$;c#v>P}ba~Brd zUqaA%1JpS;x_IC$g3cMB_PG}q+g?FX&;T8tdvme#rwIC49<*3mKmXF=;h!K(eCR07 zIJTx8TbCT$$9u94&s6(VdzN!&xT-W)b%(3TdWs&pia}ad$JNx;)b!x&(Cp9yZpXdi zEguTMefiffr;8gWEMM7OSx@QImB}mFl8V`;nWk)U`E1opl>scP%#<~x%NlTjw#l|^ z>6Y1}Ge;kLIEQ7t`%xLiIWyKu_+@KqGd0cWn&t`S4p;dQ)*QP!_JBKduWZNrXD4j; zi>j{GFI1$9+9%BSInR{i9mg+EUz3)~ymz^#tj#sGb!w}Q^WmGn`PssybkPf|crVVC zFO@al<$PZuk!KO6Pt2a4IsJbB9ghE-)deD$a!xwmb7KQk0=Plm4^JI_;Pl-q<>x9s zsku?JRC;it=f1Oey7$`PwUdh_^&i!L(sZNgbLYW_4(Ah4N^n-j3z?VAsWsYEPT$dJ zy(OA87zCpX1~q#ycxgl(f$Jhj7{r3d9E!YxRXP&WY3#Q+Ag!M3XU3 literal 0 HcmV?d00001 diff --git a/sched/__pycache__/aggregator.cpython-313.pyc b/sched/__pycache__/aggregator.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7dfc44842e3a498fdedc160ee6ce7781c9944a89 GIT binary patch literal 969 zcmZ8g&1(}u6o0d`n}mE-8%tBay0iri*rr8#&=#R;i#AkiLP{Y>SdvMycC%Y%Hx2E< z_TZsXPeQRri=cS(T5tXbDLE)h1)(bRrUnm!2jAqQNPmZ&_w{@4?RztgP{Z>RE zUjlF_ln$hpKpG<`Y(olCBmuL86IIAkLYk$VHf5TSxtvl`@>zH!bEQaQo+Cky5jUWj zrsZgkZke`F@AEw4EPq?iIrnl{@fbdoC;(}kLF+c8;d0xR4*+D2BU5C&Ei;nA_!iR8 zv8JT!ivXGem?1^+fFa0;qlxkMX_AgN0(b5Lm{P{mc0p2fasmUGEdmSFmf~Pb$ms#F zuBIiCIEnq8M5oTYG}j9d02Ns#?ev%oos0+UH6>poi`|#dHJ4~{(qssEEp7o~%G&7E z3gcRlsbwqA4AsWDGBuB*wABK)%4)e{IQp`|)T+go7`I2(dhS_jmFdM2hB%h0wfIM3 zgk0i8Ip%f!3z|`3)0|t}r3N#(jBZzPES@X53YN+(xk|yXG{=?8OfzG$8*F9H zHal>swo-KI5?ggi)fQwG;a-WCtTKyh8n?{&iiUag;fkXhcHGXDSU#>5i<}iTEHkpa z>UOmo%(i>x5zHg@VL!&t27C|n9)zQ{zR`MktmYd#@cK6=>)zMaBL7gGCxFH`x=Wi1xf(Dk&a&cSTK+ZA@mTSI;=7_m?fX)AW3k?*3cZ)dRB<{Fkup7J& dRk1Hy%;EEcne{{z^?-G~4H literal 0 HcmV?d00001 diff --git a/sched/__pycache__/job.cpython-313.pyc b/sched/__pycache__/job.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..68d59e814f5e1c2991388f8003791cc75a8aa2a8 GIT binary patch literal 1058 zcmYk5&ube;6vt<^D_PbL%W)({QA(@26^)>_l7K@}N?IDITRHf$$EpnD-Icw~?yT}= z)?lTFg$AOdFZl~v3cZB26dhMH$6Ed*hd7pW2X5YM-Syrnhf-U~> zS@@Gh$ln@XxBSYmmJiT)L0sZm2V`J@)zNnDAUCkV9ys81$k1+)ZQ|N(;yQb|)x5n` zM<9Qk?-drBdX_3P&J)RGm~hd{sdb-8=0!{hl^=yc6IZ!nXbZ~5 zchyiGDsBUPO-;kBko!#5Jw|2bLXSdPR2Iu#7AmuWQ z16A=8?{NS$jM;Ip&?l)1kfhuf3tvMyN<0=(A%n3vep-LI|JUT%XISVcVN&HHSbVOd zN=Hn3Csc%!;BZpHlsn&d4ig@z4euoI9*+~unff8%>DByT#L`I85%Vz7v&n`Zdh#2b z^Kl2|TV)6Q^cT{*qKcGq76+875~XqCr;)ZRls-vWbm^&4ItoF^DC7Z8(2WCcT@Om3 ziuplJ%chnsEp;tTEiEnAwdkv=9Gwf?&}M_;rty+_lt~HUQ7Qw0%e{lXC4NM8`zJ{p z^cjPM_fHuHhdZeZBheSQX}|wnagEfpbqK{LIP@a=NE43dE=8b0d^;FE9=Dgm> zzM6V7r!jBsXOGT5m^u6Nt$Oz5^x@2@&#$$!gY&C1r#;`SXJ1V7daT*X2Iuz7Y0cm7 zW{2m4nbZBf{oX6**Y;laaQg0V&K`XD`p1X8g0A2wl__8a!A6HFzj-x*0R2cz0an@S zJ+7vAb!hR^Rq-FdO+L=EB}`(S$8i8%F;&sSEE_we>tQZ AcK`qY literal 0 HcmV?d00001 diff --git a/sched/__pycache__/report.cpython-313.pyc b/sched/__pycache__/report.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1ff36ad842b8a214695c1912061ac70cac269633 GIT binary patch literal 19100 zcmeHvZBQFYmS9!s6A2-N`2H-!r)<6j#^%dy8w36TY>G0r>9$%R0WzXQmju{yf3P!q zJ9xWyExO%fd3Itf&mYSZ6Vd9ujTkm!;*Qz7h-EHrH^gXet*K-C`eys~!aptoui5SE zy^FYenWd5hhHiVJ<1XUjs+-iS%$F};zRY};`QA%E*Xc9_Jj0>icYbn&ApQ=$NRLtk z$mc--J|-A~Cm52U&XH#+o}v*fJ1661_*c%$;a7f6aaPGIY2tNdZJde?s4xmfc}{g! z&8wl7icvFZjE2!N=?w}#jnVNMCWF^9dOn@Wz}oR*o3d_(;X&cWNf4%$Y09ImPQNjK-UIXQ=&9vB@wPxFp>H}9pF zoZcDQ#?j2Be%kAtbGQJ5p5onew3oL{I(XW#=-|AbhNq|pW>V1FIL__0d7W;~GX(Ft z(JL2DTTcy~K6UQQQ2$d>#8Qsp%05+vW#VYd)DZw$S|;1t8~{umYqvC`oTC$Mleow+ z)e4VUA;{+!JTrpaali5w-$F{_50gq$D*@}!d>GKqhdH8f8N)VY%aHF)wdk{8K&5Eryi z+c@K%b2Qp)yqjxWw0U4G>KD9Dm#5J)IpeT5isR8Re^by{r{=uYiJM-BhtGz(W$@=Y z1<5iIQ>j;4zk4H=W%L{UCsvkLGEh$Eci%%f_SM>e#a|lWBYIOrZHi^+{hgZ`WkFd) zR~bAR(bh!dHP3+zu}dyS$iQ;>G0gpIHP>pdnXer}@)=2$9;yaUoA^U;B$Pv~y-%W? zVo7Pv+kwtBtdu!3BjaR@oRc#O4%PstWK@v*{zMc=ePYM$4*U)YDllq&f^| zU|4TSliHw(1L|g`;2Xx(5?P>>Wl<$-Ge%A~4l_u6SaL)s!O34qVp1(5(JvpE1Sw=( zE=Aqu^sJuAM$p_LfN~Jjos=!iYS1FNbG1;Ghw(E3%Eyp_Im_0twXB&v)GB8RKsTtb z%egG(9LjB_n8J9D@qk_yu{x$0$r1WhGM)hde&DT?&49Fw&4iR@4Um?P)-onoClwZ* z)DxPx{G=Xw%;}g)E_>V{)n~KCah33l!~h3aIcyG_jaCc`Rg#1GJJVDp(qEQ6F8G&oNXS{Nf%#}ypVN)cPYG{q@4@1odZQKWFY|2(#gnxXAF zpn=x{z`D^INHbVD(^?N}8eqeCZVI1myJVnej-c5Bqwudl5`}ExNWO%d{2eJu!V<)^ zGe_HyR)vQvf_lXwoSHJnI^oq0ScA<3FfS3N8Z92`&8RR{I{ z+nQaM$~{yg;h|uj!CHCP}@hBI;H48c{fCeV=N zKoijTo#*QKVX4VinC-7gV<-J`mHW?F=0pcx=$WjkQ}-^;3gq;~MO5#BYATd~hl*!~G>R zAk9VCznmFCs&&u9rb`%=pf_mvJa)YFB2dY;;W@>BGseu_R(Phr1KJVhV5r|7Hb{5gD`c(ibyB5C{D zUXnPy=yWWdbl>PUHPKD9K+?W66P%{(-kENbPh+BI9M0((FTj+k$EWM!9h2y=B^Ikq z^pw-(5)pYIan#h<^E4fr7Zh+X;Jt#{z5r)8IAl%#&&SVxH8^qMxRB;q;1`{XZr&p( zyl@0_?b^nxp)zdc$>-+*=#hU+jK??1u#G0i6I*qLVu?`&n#Q9twAqqAr_wJ+j>=hb zl#(`3aKw=ffeI3zIIw-(ObN0%+YLyZoLMQ5UO_$uCq6+nIRo5VmjoH`FQ{C$35N^b zC~uPoou6rX86F-TIRwQG>%7||6j&z~oG!Z+2CAXU<({;;JUtD#=n`Og(7gEX%fyy3 z*YEk^sh})q3z;5Hg=G(~MT=Xb#|6Kq(LD$jFPCwk`$oV)X-BTG&9Lpp&yw*+IcfT{2@p@s0jKJ@5zLXhfeO>6I^r^ zRe|rA)RVUY<$=ljHNpPP!oyKxT_n8@bM4FQ@;YvKX_ybqEhhSAx9KU`bo&}%w!qoF z-sAK+pvV$`owCh2U4S&_=G>lnIIAZnRZ$#cDLw@!+%CJRhj%Y2ZN<^vT|! zX*jO7O!?HFu13^w&r>8SpQa1j0B$tH^i%>H@!uOQ@ptqK^V{R)-Lt@tuU>-db zDcbJurP=(AG_y=S_#D3Aa`16+0IPV_!gxGI%M0}yEmjM3Lr{u#5kVs@RI7(an_;tz zhc7h2g8_lyVVM#5^(mVDzEZJ1jIA5S%jpH7G&LLpHRW}BU5*}~9va3_7gz%(y&x-7 z&yt&;1uKkC6=zCNIC$R8^N1lK&2hu&wNAS24j$$(;Vam$ftEJJua;Ywvx>w#zN-E3 zED1)WHj6uG&JBhl&pa4=9DA8lirQ5mHD)@e%`^^1^KBHBfWys1I8)`yv;gp zdIb6C>C2;n0+6S{F4RAArdLo5vZJSa1?pr!7>aCOn;@HZfL5Ay0tw}*;q${Ij37q_ zER#f`%+Rp@@Rj}*FoYz5dCt`Yw(B)Rg?TC@NWYFM8xv}ETR7a7BHRhiLPQ( zR}s}!E%(LrIZyOuoBFb-zI=J$Gp!+}KfbLb^jXXOvCORHGqLoH<gF*vzR~9@^4p`H#GRYkA;vb!PC|+R~@*ZC3Sf zX7_KHH?x^d^=M3!{!B)u>9%ErB7Iv!57GD-tCLZ{BhMws9N$5(N2D*}F*i@TV z)F4Y^_KIrDkn3Lx8dh)pjiF-YM9i4&Us~;i&$S^x&>F0{-?M2jubhmf8}6LHeLj{| z92g1o29?1Z!Ryd!b6za3z;6#!2lLk)f9~F_8`>-%iqOMQbb&0|Dk=+9N9aLl-7t(= z*JVKKc|`$c_4-PGf@D$PWS}*$5WKuz^^3;MrjgC6kqA4sSv3}^ybOe{kgt(jubp07 ziuApK_7E0-ec%QnY5pff@tKjxs@$dtL&1t7rq)R^S_ty@eE}ELKMXQDvDO|rIl3() zb(eq-nJ9(0JjDT31RWvULuYV2oVHf8RyjkKscFM_f)H@d)ifP;C!e&npp>O)En+B-Cl|mw*Lox`&NGG1I>L+u;o$v z!}g!{g^xTQispCv6~DWDcJHSb$(SWGv{9V zYI-cUEJy{s!M0FN*zmX@qzxZi8(HgJ*KD*!81|PP8+E|v7+Df4F$MYotSsFsH9eU9 z!EC4|TG|pRv4oF<%yN235Sk(DnVv9~ZRf*G2=uQGZ>lR+)H_N7+O7po>YilR!1!ku zubfVdkqH`ZdDsy|tu)5RW8E3KFuJiAVP6NS7_K3~!T_rYU6wdiiUR$CW5Ls*s&M&Z zbEpXB;q=;-_3M#~mm*iLZeF|^xiAjYUMCB-3W@^qz*$74RwSY!DyxGtKkka=o&0fE zz!q-)*^y6=tc`x!^MHKVwK5RNI~l8}S?T*Cx71I?vUBhCulC}A3Ld#X85P4$J2D5`#!-_TPQ%&e?lvlo0UK3J2ERE1jk)|^+Z3@nz z)ti(1PZ>mZaZnDOUwpV2np?XVtvMd0kEcvgRp4ew5pslWkDVbjy2namrB#6|!Rydw z)~AKxbja&2*(w1}?q7=(qv=#%6{|7_`yz+>wpFs60cc5K$lA^zbU8uQef?&7&HDq- zW=SHacAF!y>MD6({p>9nVW{5rf>y_B*`NPwd0|YezjN&NF+UsC7DePmU;TrXB=S#@ z9&}v$_l?(ds84|?}NQ4Xw$go;V70GL3{B%5BL-* z>4?I~4#1H~r)Eyh%F*{9JvKP~vLx6Slz8ylk< zM?JyM0qv=iJV`ABwX_4<(XcW`3s&-Uiwd`;OSF~oJGP~N>9#U)TVRqtpf3g{D*95EK0-$@RZF`z>^FPx~3r>+34}%g$Kq6pAvTcc1QOD*r16Hy6?o9^QU|1-!9*! z@lnR7Ku%g((4ga}=%rjh=N$UCfAAr$Nc$+d7EYFyDIeuA(CXmPZ*Wec)uYB5shZvBLBjgM^ ze$J_*-yVCrThNNL58-$a>RJtx`8&jcniyqA|#aj75sS?!4?BsSFP?&`|bAS-k4f* zM|E4Zd?KdN-l@7>TWK4D%=*)AB(7u z#Z=mr#qYiwGvwa&1m2RNb!%>O!7qp(Rq#3i`Xv z2z$nRZL4j8k`Ng+9tv3^S&gBE@Wr38pR#LR8-{4xxs6ki)(dFwG(v9U?U$O+8&@v* zTl_XZv(gih7l`JdR2_>>=U6lUJFo+Oqo12fn$B2C(r=tB>5*p0LZnR))jf~o*)=A- z=85`ph5`>b#VNn3<^b5CITfo&X-@|2sTet$$1hUGwpRtKOu<)tBffGFSVca{r2b`; zFW)PMVpPW{PR&w@uMK!SQvBXuD;-HDj2iWbNki!{eCN2daio9n!zO)s8b&jsm-3Up zjRd2;K#pfhIT^$Sl1Yc06yGo8=pYB#f$@WJG9V|#Hwrj<$U(;8Bu?fFwG1!R%6g%e z5poJ8I-&<3x0rK*WMvr21qfmgpv^ph;ng9bx2 zfB~~ae3Ijf%W5wqqdc_Zw1hQclAz@8NL3P+pp+O9GNpF4RG$1b5h>;SQAY;fJx`K) z(!+=R+cZQPFo&4K$0%4w$Y6mV*h|$(I{`|JML`3)=V4P~5}11QpHzp2uXQC+PSg?7 z+CtGZQ_w~kv?!K#%gTr;(%j%XKE^x0DcsBJrr(^md1u~iphvKk&O>`=95l*m*tLNL z6-j{dA)li!T?I6SR3$iXVLRwHFa#RvMdD)xO$=&9)iN}w8= z1d{h41swl03Ta)_5Seykp6?N81fdu;E3(8T?f(gk{|5CuXbnGw?`I^t>nFp(xBR~E zzwwjdubzYYCca}Zl6dE`i+98v=KU`KNa39YWlBOA4&?5A7y@m~aG(M>5C8{pyyw|} zOD-)kX@G1{bFlrDXi$ty8t}MGVdsGq^0XYwR3Vy_(dLYQ7bS?fp~td4`<2HO|2`^R zgd|ns{2!s*KZfKP{5?Mh2@>ZvAuH4f&D|j|DrDYSyuBEJqh8r_8KJ9<)tWz1K30CB zd8`SyKdJ4+nJXvm^xf{ebN2RG|J%WfPtt25={0{^{MB=$j=z(AqKy1ODI`CyfQ0XW zy#L3C1f7fUemP}G_*InoAtd|D{~uBDM<}@q$u0r??(3}rNQvu>|3d)xZ}}@!l2}N& zoj*Aqv<33M@B7K|ulPGq29b&?yJO0>;PO8GU-0nXg-rhUAo2ZKS9$O7DHP^OqgC3Y z=|VrW3ykmGCI@Fi8DJTPpE-xuMk5cP#{rH{cC@elC@K|w8W0ic)=?UB9>@+r~cYrgtlg5_q4%+q_Nf4b-j-66MTDw7Z`?cn1LAep3&xhQz( zSe%~S?FDk!Z7Pk57&aii?Gd>J(?k4uW_Twzi&qAaO9XC9%cmZT0gq5>F$$YVw%62D z)fo>{fK`H~DUvz%Yu3BmfCS~OuI;3=v;%n~NSFb_7M$=_?F4B{rMj%QBuV+w( zG`%+$XTF#%E)@tJx!%?BU>>+L}$&Jh-hf?2v|#fjC!5 z;ME$#&U&9aN$B7vM^Us3Jqpg-WMhfuU#(Rie01WBAKYFq_>XBASI#p9Gs0m{R7nD_ z*y>w}ng_Rf!N%L>JoE*yu6-L#LKMR{c61B}h@dvX75^=C^G7eyW}lOT!34ux0yhrC ze|9Y!QBS_GZc@}S1TCbDMN+S#2o%^;bemH2m1w`TqPQ?58YF4>fFY?Td3li-Qu?pd ziW=>ju1Kw5{qU{QM>|j7n-YCAcC`5A{C2ycLCw4&HFH9`hx2WA#n&N`?}Z&C-AJEC zu`Tfy;y=6QgY=<;8R4r2wTx%@E{udW2b3<*^F6-Ou155m#BPi$??$|x708BB*!`-( zzU=W@-LnF@G-S@>PeJeb(~t-<4@6xk@y^AAgTVy33s7-n0)GXe4WZ;LO8QZPBJlV@ zl%SI)HXsWID-Syn2=~DJS>f|4P!RnlM+a{~Wt)Hx695oiGV8EAA;M1uDQAOd6}cOt zUH+PQH!Ei;Rbxn^?ur_^me0lvSyGabeP{0W+#kCkgh8Ekck&ql=h9fSB|Pw_#m{8` zKPO~)8puu4tY}ccg7Ir{{!sHwOXy2?vIt}DW^QNb$S2*8yMKB-n%lX4aWkub`P^qc zy>}}ilH#4!cRqSAnpG2`HnR?g27i^+u~EK!?h8$}|58*_ynO0Q`5R=q_BnylZ9<)? z-^SohCPZVb*tZQhKa(&Piy-8eln0JT`5Kcxo?l61X0J?bS7H20GABm_cdCipLVw?O z4bHEY71fI1P90%x2rg~c4SVR~i)rU)@munU!{?yXikPNwD-C{9E%IizwFFUYZhhoiU?eS>I zYmwp;+e;$SA}K#fmc-w7-W9Q;C6-8W>-Kvh%Dbeztt9@k(??-Cag2Tn_r8xp81a`O zz%Yl5k>dL8VX=6KQa1FF@wc6;6kH`3BsVVof{UDHx5q`Cs}y;O1SIiw=k?u4Z-_{* z$C1R>ool<1tRm92IFk6f^JW}re6#OL2j$VsswbIsA!FG4D|y#enr_AJKlMRb;6zNF zhWuB4`3Jf$WrU{SbFF^)+?NV~J}ATRKV#?kMtjy7qwJSP%^8zwm#qVZX@b3je0~kw zCLa@UNe^8ogv`AUT=kF*fKdZG#9!GW2bT?zBYhutHzNb*t=u957dbV8KG%Dm39xB_ zLww&daGO%#pipwC9DO7AJl$kB$*8cK7tW=_Y4D~_yiqhUEYJj)h>?!P4^Vm8p20<7 z$xbL)9kC1PW$SB`^*L4YLf=TjZ6fhR5F^GUEJ3fpX*#2Ofm1arOV~)kI*Y;$;5x*< zGGz2HJ{ibu&cIbm#sF_wNyp7d!eJwEMi6khN+!t!Y8fpWF0G#$zVeE#WUxvm8(PZ* z7aMl)ThcL|HWPF;lBk+XW3^I$`bpa#K#$VkGAUfdwNTiBiFkY)B#tkG z*}z`?N)j(mkzD#KNZgcCPbbbB23_K*WYZz7!q-fz8Ms<$NyAb+3>?h01OFG%Dtz64yy>Ong7uwEKqlpC2QkdKZ_8~!2FjAipLV02%e z)0PxYb+1s*lA_r%U!h)WNFIM?B~PxOYjkdMixGs2mf0T z^hrURWhIRa*0nrDWI!wwEnfL@*z4kPP%;m+hN2S=A}n4xL19p$2j}+@9Tg2RQZQJ_ z$S9BnL`LCYC>jMr@vuw@{xwL<8Q6K35(UMeB5+8?!=F(61Rq2R8ec&@Ve>f9NtFlh zD#2euNgqnk$=IBQ53GU$8D~9W3>Bz*LcA^_N5U2;k57!IBH=;?!h^dBll=D()(0p- zPAeY1x^NJObFM%wad6py!@jJ<`7{BD!B-DZ6_g-nx#tROHQ{(F-UNW7<{!=BSou5C zx2NyS-kuHAL^DhfFAx7(2HyGxY?h9_MtX(eyu!8cSUo$Ayzgw|K9ZK^u5{D z*#o_>sddZRjdgybVqd)*YR zIUOzP^Y@Bf#1ukf;gK~e+`rbcHo0B_0o}Gp(SX1AKjjt0@{5C&L2qy>sN2l1|AG87 zLuqPkwl>5>vzz^7YFoR<4Y706AzO$EtJe%`vbB!&nf0rYybB)! zvEtN<`w>gzrHo{K*ci=i^UL;?)(ed)!*7Krp<&Bf)!MBM%Z6ve9APd+#;!#!StEIG zuFAJ|bAiI(vMs=Virh>-onAMsGwb;q zSr9jOf!R0~8D%4RmsTNoFH4dXj&2WJiR9Jz<)4e!9{i(E)P>K+NJ3w{eVHWEGgG6X zaeN(I@z+$c7`VdEUO%V7&0 zw##7)9ad!MdXX5ciOgA8Y2eTm+%M7WrS6(@+u>SZ55EB@piv0Vbx0uWkR*RYjQ$tG z^ml~zKN4wwOVEEW%lun;2?{dLjiu+H{|$wZKGP~S)Ut9rlThlHshBErSuQ4O!?FTH iJ4yvvyOVFF$l{$|g^?`WY4wq0=0Eymjbt(A<$nVueD`+% literal 0 HcmV?d00001 diff --git a/sched/__pycache__/scheduler.cpython-313.pyc b/sched/__pycache__/scheduler.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3e8aca5ad79a1ed0fb6090c18a0d9af7179ac659 GIT binary patch literal 16950 zcmb_^dvH@%n&-WGzb(n~L$+*RKQI;s8!#lnJPkJZflVS=gv9A+g)JKeS#sYi19mdq zY|=GjcQS*Wp0rImwKg-E0eY(|q-J*8sokpP(W%h8J2g_oM#>G{OK;Lun}4ZMtlAz?+If>j7#C94v?YE~_LHLOPXYFVxD)v-E!m3{hN18WfKs``w*Cf1Z` z$IP0&YOkix(p$t9Ax-Vo_E~#vtW5~(`s}^MY;muHbqH~NUrBE%TiRR3mNC=|YR_hA zxY=(V^@4LdEymf37fj_;9pyDtP+sG9g_ycMQ<$w3@=VAxOL?qoOyRbCNlLhB-WZ5P zqOm|M6pe5LBz}Y)39`YF&d_iSQT<3D78ni(IPOc*AySG0r-E!?Ea*QS42_+LaY$13 zMvuD{yk?LM3M!uoQ*nwyy5*#KiBh!$L@p=@9a1&8BHk=u!@KK5p`u->aZ&5E$ zEbXPR2Q;f(hy8d#;$5Vmhw@c}5{-V)b4#ViqA* z#2Us_ZY!@Eh>irmEJee-G8BpNDva&qz_bBj<`rfj8VT~+QvojK4~YVG=g^ z{okDogflrse*b8Qx52ENv-g|GAK;OncY>_4y7 zs~1Weyf$(~P|xT!qLj&NV$C=Rc3$r#6DCvrk_>HNSOs5lBn-Nw@1H>6E$WyxEagzg zm~iF{Qq-}{;mpaSs8-5Hg)_8FQA0YZf=A#)QcV2f+|pjf4jMHyQqCCVRt|tx$f*1{ zU786tI2t6GC?W}vl zk}h?gj zF=#zPH^3J|qm)}Q06wJ}2?j>sMs5Fx9uHAw)%2XxTiOYfmaezFvArCO?l% zu-OtGV>T>>uyrOWoc($TtH^WGRV7_O5&9cOXv=E=Q6DX}5>a5I3E#_e9>V^>S;nqW!*7VKvOCzS2-AADNA$ESnORO_xTJmbMvXT4y?EoHeF&H3?nKvhtR$ zAzfsjF+6;TWqK0y*Hgd4{yo_7*OmWwjEe zNi76O3s0g2{o`9ey9JnZ;rVSR`v&Kuv#h^AB(jz<+O3%0^DG;fUo@8y$Z5IgWz0l>*isKi~51dy!KkB>ImvlWnr~0w}o|-DDd6cNYFzHtu{^c|sbROv!Y3fmwvDb&xnLhR}hB@1k=SGhyc14MZOn|X?UOF5u6 zQ4z(ES!zAYK)!vO?O;v;9L*uZwgu-^fBV`N%Bv~VKhm{rYKff%=^>d(`V?N(w{KrX zNp0Zpy+BcupqPnmSn*CqH9`l@s0Ma}+R4FAK z+a~t!)q4#)l+w5vs{cjtmorC6g=d4lpGV&YpUzX8+p8G%89a3lr%L0F7=1>W zUA1A&jfL8P6qlK&To!BKGkVR!J{iLD#fUUZCQaU1Z;{UsmUpfH7%qNmMBfi8AanES zPBJoRl9i`AqVh0Hty` zgI!udoam>d>YklaDEsvm=Wqp+R0~|efxZjz#RA+8rAx9rgrUY%I_i*OU>ZI>kQDqs z&fsC~D5awYeuux=CzYeLyL9@1CkTmHx_>x^;2eesjxB)@6Nrp3rvsrFGa6+XueZB} zc{(~V84f_|#AOy9Z4oY~7M2Y3GDI3t;Dm6a4u{4=v4LszKrlGMOxv;@Y-FO5@Dwwx zWul{_L6+kcp%LCR!A4Kv7K}z9R0B&9CMXKQMC$!=7ra4U$8jh8f^;r~2)^UhEMWcA z!7s^?m@XNJ3imR{91jjd2xo-MY17kE;}{B0w@z!sQtoP=VvnFAZ{~slcKC!}9Fd6s zbmO6j{}dY-=k<}v@r(?h3nZ;5<(h{$7-ZT;& zJ{e^Fp>e2HgzKKy1SW{)K#;jH_CRC|wZ?+kN*u35G++J~M1O67n~DsFqP&_52ZIxW zu*YjNC_q#Z^kFjW6J$opaCD4Ug(9O-w^=}@;zTu>8Itwo)k1`){Xol^F>^c};?>8e zpcLWtfpFL_tRUM-Bj-_nJW;C~rmYGwp?_6{& zm7gz9IhzyC=A^Otw!J!SFI!YC>CWp?D;?!pw|q3daUX!qiOVNaE&CEJ`{L_5KB?S4+XpOUFHhMU6ZXdC$xDM5UcS;v zB)8PIp2W7Eq`fy~>W!Ov(^nB{xnsl`N)VbU-7n^@}G4ST|mz^_RaYtLa z+%?k!Z^`hEVNR7Ub1h8GPc1txx-PhqWi4~6-xQZEb}n_F?_PdB<=UKZZB8={%g-m6 z&2yc9-m@lL=br2Qao-(g-J3mkizsLP@2!;8`Kh(!w%xg~>CH{cbkbgzGSz)-s=Hf7 z*(+D;s1j!$7EISQUTnG0lB##XM*W*22ev0(^F-4AWZd-RZJqI)ZdNz9>s{q?*IybIU%9m9(%^f? zE+0#6+MC$47khG56Sp1upStdQeKci%j=pE1%zgAd1LfTQfOoXrEvL-&cMVk45&8i) z0ZMwcL(ylYuc_$XJ*sQ!{cc2Vw9tLD=0>3%CB zA2rf_Ce26d8xg)~YlHtQ72Rji{7T)?j>vzfY2qUZzE(qg3kk20%yk%m0CXDxFkjTX zs1?C3Sa#+bN51T&>w=+%K>VUBq|)%na*$LCSIXmS$Z}6k4f%qQE|Z)#3P|<@pl?@+ z^~be2Wix5nmMO|3L*H!bpjPVVUs4+@X3e%Gg$n5bq0GYD z$M@z_N&p35sXGJIyRwkiryi=7#weAPK?bOy-vz2*q!ONbDU|(swLVoLxCEG>^QkRV zn@UM7Rbv3511P ziV2~3K?!lwgMV(UOP zq-u6NEe2jAj(1uwe$nk3PC$UikBG?Ijg!AeA@c!gS#rJ}@QQrrmGxhsC@PntCb8EO zb{|e>`)LTC>jb!*Umfv6BK07iR3zHTZx;eUeU4VJ00$AoPMM2jy9E8H+j7J;(0Mjf z4iy}a(yMv&c*Y1MoxqW;xokOZEP?$$wvIaGsD3N;c0SkV7jzZSP5n{ z0KM!8Ilh8!?H?8G4BLjDrYjyQKv;wzc|b`9mL${{eiv*xO9bk#D zB8n;mptI0kP;Q-ww!cpj$OXe|L2{Gf7$*{m5;69VNapw8aXNUQ3b=jyM8eRB6u?{a zX(B?(Zv&HgK(>F?l#9H+C@EWbcK+Fg7v^8MZL*v@G<)dWk=Y~fbSF*g-eoUNU6{Jk zw9*G0TxysxKu26!E|m7q`Q)nRcoTEHCeUg%9Ecsch2_Ablxs#h>rKj6SVsUIAuIcTGA)Y}dkbT_scNO%u@ut;@d z?-L0Baz{rQKEt_88W)@h^PUGge35>!RAlduf99x@fki|)B!j4evU~yN3#M9t_2Qby z0KtfANCt6*9%bsNLE5L1N(&kcS)+*AWSj(jpifh;xz1%M08p5LvO%KcmjRa4LROm- zXb`TZ0wp|U+0s(TtMn;`WH9EFHJ@J94kZ9`g;dT1n3L}S%hUjC02i4zL;$7es0V(& zE2LwC)>AF53qTzsgE~Ald$k!*r^~&9BibPuULAW@s+$AbW`_t}3C^b@woRqbbeUN<~aL)HWp15UA~)jF9jxsVxAV>d@mU z&GAS|EBOXW>&98vdrJwL@s8i@lzOj%3suVWM@ z54pYboKKgn)Bhhb!pvG7ZI;7TNM(aGR^RM1VXglJG@<)9nf#KgmiaK`eS=T*$r*7`ia3F%XU8+qIfu@9QZt=h!Eq7 zf&VPLhWxme%HGymDu5d$V*Li5$vpBb8+_Ks-Oe>Qr4eg~^^l@}3+66_N9j>&x3Y#B zr3G=fdAi>VSt1-{4hBa$PK_~abTTs1E(oU#WR<`;$86e@k!PW!U<08DNx)M~n90~{ zl6c01vt?Q@R%)4m4I>^!GU^mj9OZbU&^$3DWMeRm(u+|%WALV2!=^RzwcH^4@3Dgr z+bW+X;}VCM5#7imks!OVucC}7yw#%nOutanz?zB)jK-k;5!w@KPfrhzIto`6{~)Ph48^2852;Gf14)GuRX^ zh)rV&Uq?GoN5vT7W$eqFB*orAHcL=`;gwAgc45aS8r%A(m1M6Ird`hp4T%(yza7&KvI$~OuvrO*VY3n(S);}xQV)b}`aYSAq{PB~ z2qXwAy^JJ)EZIq%vyf6t&Lh`(o|E!UQE!3Q^#fHBY&1rZb8wj+xUA&WqcDep1#TqD zj^oh@_KujlqGdBrpIT#{r{6Uej0De2uzUE%tgy^ZM*IRjXLTrYx&MQ)aBg@+f{s^t zKHmGxO6}6v`LUmbmLlgP$?BF=bz7pkEmgfeUcDW;4+J~PSD$KMtNmA@31P8*ia=K9 zCkh4;dJvXuWaf`swn61Lmm%!^U%9+s)!F4`-^QTWpTKjg*d=^;`XnAVgQo?P;vf$z zXuM*=s&qIe%v;u0u@^|izkv58{N1Rc~WrL402Z)e?H2Lx_=<>HokSISnOu+=AR z>n<6tY`JVsHSJ6^?Myc9PS|#*tOpX-14(OF%G#T-_NJ_d)&`@FZSe@pSCxBvyteR!ngo&GLF~ zl06kV6~#ldF$RP9u=tY8{zqg^YdnO{GVC!D(MgBXy`EQ*v(4+rqp+dJoAfX%E0x&S zP?jZT3%oiO#iKFoZwcEaB8;E`5{$c`FvRmW`yG-+WyauinuKPENAL(cOS};A zen>nbzX_J!qFM?thT@F^gTf|(LGg+(%Nqp7Hw}-E_=#CEu?ygJf>k6b@iQSavuy%Z@dj}!Lc8>7o(|$gd~D2*F16%4 zGRd-+$#{NBs%GvblK5AMb41={{+*yL%kwh2GrfHHbKjrk#5vN)cX4l#;pJ@?cVF0jr8!x;XU2G2XFg~6p<%9l@#R~(<~z0$nZRc> zzbV3-Dqc9ZChcu;Q`;S5(Yc-Hc9I9;zS+K%$(1m<($=bUMg8q6X1VkyL#tZKTy>XH z;UU^WRn#t3zg_)l_dwEo2hECo0m8e&TF-wDxSrO2aSQAGlX4ukOFfT@A)vhi_T?bF^oz(4N^#3ruBy(taRr zI0j*Pphy z76-*Dn-f)=~kb_ti=ZSVxV(I&Ag@&Af&v{uU3XT&)RL z>y>AIe)RpLaaU{Hwd-nd&X}|w{?u6meWJy(s?<~%F#ekQyGDe*rZi+#R;^S?*^K_S z)3x~U(j;WX;$m>Q^J341p5>jFLMzHElX2JXTh=|9l}XzibI;6|E^hfzWkwH~Q+)J_ zcg6Pp(JS36`&YQDTRu2-b>u3y^2$mi?&`l~9mq6s$LdU3nS_;Dc6?%OylbN>T7U1r z^SeMNZAtsKxM|y}hHBn4WBSZd1?8u+a;fZm*^=wLYkB12_=WK+XRfL~(qGfBynM4h zzI`C+JTlX}s?eC%vpZYlq~H@S$pEvp4$$*-oM$qIK862*_m)WGe@Vbu9USt zVXa^8AcKJRzM2feB@AMF(*8u;^aQlPQ1@5AS)H`i!n(pyzkE39*n$N_YRj7~FcDZj zny@#|XfsrOyP_&lDrWbtxs33OIOyd8inJa!EwOmITr@r97sAEmd_*{tuy+~%g!^yg~c}%hU8dCC=J|ds{(o_mm*8drwfdUiuGM zHAmHbN?~3nE2`U9w*6wq2Rl}puX=9MSGl;W@0RtjKn)AV9~uAOsv3p=i6fZ#2L}{` z&GZe;I&ZP=Mzev0x6|Ho)r}|CBmB$S11fwzTDQ|%sk_OL?3){CFQdBI+)3*FkCqy5 zqvG$(w6|XKca~-n-qFP%BUz~-em#w(WTRoQNuAuhaj;%}%cVv7t$H<5ZZ#JaQWnTJam3y{kQzjTd9@;KY!c&zkErp~x zLWMyPo!~r#&hi}If-~b6R;wm28&RpH0;{WRAg{&$34crY8^xbVd@-ILJ4mGAiO@ul z{H_kdvt;#0BxCVajf6zuS%5l%P(2(D1|pLaurlxnkCUPaz-TBEfEj}LlgJS!i;PrHB*8E%ATa?75_~2AKwcxZ&JrNV{w?u1;{7S{hKctp;%$J(YXS4d1MC!R0>vr0 z)#7hbg!gE4!K;OSU_VH3K2NjKqqOvcCncZ~Z(buysj*X>8J_UyXrPMAQug%;`}(Ba zJ)=!q%W@39($*63?n_$hX4GGZ_Fid=bFpEm>3kFU$x7V0;gXyD;NZ;c8S&{ZZmC;t zl2gbtUfj|UZ+L>R&8NOns=%2|6e{2+ElEo?EY0G&>a@j9fYR&ZDP3h;SNVmlbS}1V zX8z2=_vgR=iETZM?(j5@XYorV$%?Hh$JQTq-Gy}01QriumJ85o5~iA4rrOL%=ZFa# zX*)a{em#=Xx#Buky2vqOx?4u+Yd+Og0=Ja3-Nn0W)dQ{v3;ycZPGM9()4uGDaYN&p!+8U*rYI@y)aFko~ zE1ivC1TbQa!gNTocomJ8*#V4j0I&DY;Van6=KMZKJxtmWjB&anx z<=iW?udFJRHfOqmNmXn~RBXA@@biuDZ%kI~O*!^{?AUt`ps(h)*7byD_tAg2UB=v} zXq%IFl&>oB{U@#)y}sI^+V7&TY3Ti0%{6T?3D;EbH!7~}ruXYL*Y>CozOK+9{kone zVWVNcQ+>TeNm9yb5_Xa>_zc;6alr^H5`TwEWMo0*03#Y9UngggD@|Zp(`NFEDX0U* zUt~kkqY+I+rVU{ja}5oEF@6X21=dK^E`p@Wl53mgiAU041dKoqgZkw3BRaj84{``UcKazB<>k_e@y*ep;otlRZ6Rm z()TnPb@4r`M*R$ZuiUNfq3<1~8TIbfPMV|D*rj!=N+GayyBN5mcYLKLDPO&$tWcNy z!&64JQ|yg& z;_E-7iaw)^zozPcP1Sx*nSMvD|25_KU(`nYcSmDRY05s3#0h|I=41V literal 0 HcmV?d00001 diff --git a/sched/__pycache__/trace.cpython-313.pyc b/sched/__pycache__/trace.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dbaa914794873c9c2846fab8b9e01fb2c419b112 GIT binary patch literal 21349 zcmeHvdvH|OndiOTx8L28T575F=IQ}bBP8AkkN}I9co?vC)kd+1HI38~GOZT5-9knt z4rON&kvK!kK-Actp-HBeR!z2$YilRUZfZkf*DSTEN=a#t)xKdJ#*^Az*X|!+n0S)R z*6i;)w_g$sMxIA)ZS4V_d+s^kd41P%CTMQ;-Gr)X6$Lg@gJ72H9}HC>urYn4y@f<|JJy zC+Rn8)HXLJ`jAbMRWg)vFAB1mK^XyMNPc=n?yU!=qyGggiVlZ~{-S7&sFMdWV|K=FStc z*DDT>1W%0w#i8LoZ(x<^@%hAlZ=cvN4-cu(o*}R3J?-@eMc-f`xYaC*yIkVagQvY> z(EAcHhX+Ja{P^4@k%r`zn|e;kgZ^N>OT?(Wo}qwRIUsrh)ZQ8Kgy+;LuOH2{xx`Md zJT&O{_{4UPKNwut&H!bh8QJUa^UA0%y8VN_7^N5-9P;`G{a*2uR~9iZe;-=g=@R#K z9(o$!r-o%}D$$zy-ZbF#du311+jAl~KP1*tZI`y;!iZCLNp~%kkMyTU}R{>BcB1>CqzarUa@%m{^6h}I5_MN z9Cm3G6Rmo$&l3nJy8c10udg>jZ|otcFZ}%jNL=QQaWW6u)=0dpUCv3GV}hiuLy06H zrDG{QQU*#L(^YfF^pbH4FB>Eia?PyVA{z&+E}Np;LqzgPIUv!oZ`c!51X?LY3qnwY zlY{=g-URqApEiiaYEwDaM{2 zKd7*$M=|&GfKGv1q^&(YUm5ZE5+%7kJ^h1nAV_rKA4V}UG7!_#Bh!wS^C+^@5DR;H z0)(~R9#1eR4;~)@2>>b|W%u#NDRrw)3=er%dpz>6fAwkXG0$M*NN~^>Sj|>zHCv;m zQ)grYpjkvy4^$!gCU?i2``W>{+4kE0xY-dC93R??BEdQPs&l)?Yd;WH$tB4;y*yIM zKnz05as2&#NL=RJU^81WAPov8Mt7W++%9Z#=3oTbJm`t)Xm7}K|m*Mar zhh?BlEjQD@b`}+UPzLXMbd&n@?fwQFiU}$CEj$BjaVuer(VjD^eknN zaudwdDCJ3Z3CxtinxuTm#30O)S+)#lTm_20lkNXc)}X_~y$QBVW8hLN@G4#|bKNYH zqK-_cL~b>gWaVzomnvV4Rq-X}gcahq@%0J#K2V>?bsH0}TnGF7B;V90>gtm@1KbpU z*kzCjOfd}&W1A0y1uEK};2K5Gcya){>r6n=2ZG>{vZBK#@Oi-m1Y%{1KyLvWnV48v zY^6c}V378Qii4(P##%rHXhWalZWlNwx4*GHR!}o8yl#xwEXTtVFD#iatb1F!`uvsW z$A#~h)pbnG6BTJ6fEM}te~AQem!japPmVMeqx5 zt|NuPG*kzAUL4%Q#>5yztVxG$aXH$}gb=V-G>2XI8Us+aQj|jxu?wsM+vcoj1?3W| zWLjB8&pv2DqGlqqVb*%5DTL3M`AGq@6cdSu_P9PUN!+B@H8SUf-V2H zy>PPfjmnt4Dzf|fzH9ryKjQX6_5VGSF2_8sxohQ2whNZ?mU(08oUt^#?tNor+-Rko z(9YL$6le$u=H9+fx39KA{{}v{789cb+Vxt$2IZOc6M)BaiSuMH z%i-2*j_{JsuSI#5Rqr5lE(#El66=Lk=CiHxHeNDp?E*)PvYnJ#^Ck3QOp92l-gHZ- zZ%)_u>$_=@S?C~ASrj^mau$Up^aPHV6#P;nHMj+$ISL+d6SoFm1)E!oIL8eRi1o^I z3&6d>Z*12b`I{`|cDGK-2V9O@FBRbFSZ`*qKL>6L|37XQ;kM|JxLwb1yRiIGxUB$g zoea0%BD!4y4J@FxCDbdHN|LnWH)YTa=#`DTl+CM*(TWc1&mZ}DMt-^5BvmY+5u@L@ zgx)LDl$V_X?zSRcA=P1&CN{!WjIf@K@ch5h2wfQ?v@Ky~D>LflETP^)N+Z6KiIE1k z4LPgaIfxs)+N+ipTU81mr!p*M!mcx z)O#2+8yS6Ua_2HUG`sVp%?ox5IO>s_N_D5%(UVc-Dw(mg1vTsm?vfQ}Di`lANY?5) z*WpN}(@$xu)Urh@as!;J^{~Qsj|t69>-xtufiI8g{7{y+jp>=hAu2iuh{I6CPK^Xk zDCWT6m^UHe)0bT__l-b3h7Q>i00`7#g&Q-71S^Wa*NEbn(7J9-U`!CJsziks$F$ow z2cSu;?^k%LFyiwua99sSMex4l?NvU6beL)XP zWBRCp^&ZPfP8!HOdUo50DtZ3m(l!iZ1GK+QfzBz7+=zjYu8vi9SyZt%CGjfS0dG(h zVnqVe3nLBR3$|WmkpX^#s8~d#CkLw4{@00%M96)TP!INI{o( z5tv?Gb%mGZW%Mo(vJ@S|rJ@^UlhgIlBS3xOHF>y~Sl=+iS*F|jpPxGkiK7ZpzqObkq(oH!X- zHOUYGPPlbe!3X0;D%feE`5EUY) zqaD-2bZ7MCncA7ajCZzd_Zz$KWGSth*@^PaH`m?ly;XDbOSe3;WzW2^`~UjUWkXTC zyz*-MmG-L#uN;h8rlnZj$w_m%Ew#f&Ca(GuGGYiC8gxI9uDIUP9>$(<`+{ig7dy)y17nrxqF zpFB8mFk*>HF~`~vpP834t&KT0Ag{D6To7&x4}`xI8HjqK(zJ2LKBJx4dh^81=VzVI zyn)8bGb^SVQx|{o<%yRgEojWSAtc0$ORpNP7_M5bSR#j}*T;&RL%OV3d8kwUba1+V zrftSMbM)rH*=MBL&ZD=^%yxCp9_yR!_Rc!{CxqJz$6<>GVvhBgz=A&4#)>zh&y2jM zr>kPc8?uk%j@$j*zJ-Q*fVp=o3?uX6qG*7!`ovk6^H%8ld-__D9rO^#B z;~G^(O&JWZe@V4tDrz6%%iJ2y&r3WMQ7)UJTEMmQ``&~wsda0!DXNl&i6@63o@lcv zqmq{RBU;yHGeBg|h2opFtqz#w76(a}w*E_cNxvFuGyWNxOkMe8M>ynRBg>*2)oYtL8aH&!!bnY`qXoaB$JX-enSC%tiEo>hC{@X~m356v9^O8%OA$jZQOf3+=j7VHo4|7aCfa;+FB`-;` z$($$v1GKv!Dc96pZ~0zJEN}1kTEZPatb3>KqBE2~>Ad8;)-tY}&D;A?MfK}}(8h0% z%^F=%9?|bU#W%m550Fj#LSCCm_&!Aoj#Fq8GW3Rw9QOkMg5!n47m65~u|XDxi`$6U+UbyNWRO^IN{$1qHn<(`LN~ubE9K%i%=n@(R0M-g5ppzAo|@w~b%Pw$ zAlW9!%GZL&Er6gAHV= z=DPl(BY!ajO04WK$dav0D%wb!8A0R@VHZE57u5{=#d|5v=;eS5Iv?~nw(^qn+=n*1 zKFG?4Ytt0NTu9JT#!eR>+UeQ(&2qO*T7mCN-6Qx;PIAwH?_?aXw`&eT0B8C}_I5LP zRJ-=jpQ*4?eOBI65F*Tfpu!>TiU51XCXhI zcGi&iZY_XV=oBOkL1hT2e5a5p6(p%O`Gg*+RRz^gAi3d1mhpg~I_Oul&?upObyJ1_ zuf<-uhmsmNv*<<#`+_GDD%ZJl=Tym69~>s7ODRu?t;`*+sns{!>+uD)H>st6g*F1D z*!?|Z*y57#&MW#z&BQAq;kL6hy!P$h$nt2_HP;WMX~&H2o$^@Y_E;s^UuW~4Qni7E z(0~Rwj^r$crXL`&U2~c1G~lzqC&}3_u=f<1La5ur*fu^JX#~3A3`!r>P9vT`UJ^Ba zT^g}ndsLWY88&+lKn`Y})Fms5^okrTGOpw6y2uS!4KD1qH+1 zEKnkaOU=bVG5DF1*t^YMVYo<;^hsQtOM%HJTvn?)vm7s~f z0wj8he?)j(7@m6SDRs~5*d|x3nUpq7HU9)l^#|Bg?1l68<#YDsFm#OTKgunP7r5?_ zzuE-zS2Nxzs~FeCjkXKc^VU$u`^Mr#dbY6ZedCuh3w{IQ+;p|&O3Pc@<{hr_-Eq5f zq8t_)dqJpuq6~?AN9g&9rulqVWY74nctK(4ol+_Bm(JbnJ@@aD+w z$fjsPv@^PQx@vmQbn{HjOz+K_nJ?Y+%$DqX-@5-ZL1SyYiC2o*|NRyvSlITZ%V5}JF}ccfCsRHGj?{;Jnw9nb2fZf<9uav)crxl`Y<0a15>=R zDdPUq9r5KW!@DBuE1W-X zEx*^U;R;vYJ)q(892d@ygThB3&CN4he-!!ploKJOp}hJRXv|^sA7OX zs*~W|$DU@%!k*T|(;7=zN zz)vu3HsYVqY{c)Yg+yTJa*|L2?<=Fkj&R`3Wt0k8o;gEwFDfO!Dh6AS@=8w0EtU+( z;>SW-)2%#bODXpfMqqQ>vYC6Nw5bCt4jB6GEDG8DAf?je=#pxSM+0T1c}Z=QN%=_$ z3T6d-gPB<&XMy>m{IT=Rb?2n|$EP>#ZR1WUG4Nu|_vZmG6%6Zkq$*h|zw03LU=zBI zv8T?hciUM?{|LG=sMP8`RTboa#y% zrceTdG>`SRBoua0T?fewi$dB?4)w1jS;*8s$kB^~R4=&}sed-8e=EmUq|`rl_|tg& zOyaOBamz*wdXD?NP4XyOWU8H_>!9mKW17}Y@(yGwhNG%)5puSSkl^=>N5-9}kH0Z7 zrfprjW(4v-ca%BnAnzjOtsR572vxA2Z1;+`ZD0U(H!HeMav>TaKJkl7B00Ut-$q&$ zaR(`NkRp=G7O+{O#Py`v{S1j3CmGL#?{0f(owswYxP=AF_gnx-{O z0$p&w!PH6@yH#u8gJ~K+I!=`)Pd>I`BghuW{98EPv|Gj6XtMh)IOirFq_w)lq~<< zeX@VQ>S$ZWzgdas$2EwU%JTpCvXSPsc-iFdQ|6x|%3LJ*-%###MD=?ES5W_0k_s46_f^L>{>_?gym}7afgIK9qX$bN&q7`6W_V+=$GUzQUu-C*TNKUO zwFual%z|_7ZEA? z6Hx!WvSL=9AyqfiWE#eUL_S>!lrK@O%M_(_O~p(#RF_mr`+N!a)NfMba3=Nk_P}u< zpcvrdNGZh>BtA-~o2ns6#~dN$Hvu2OQ3JP7!&Ki?CDV}L2};gF9$}`1`cGKXN9dW0 zb<4YYF3%Z~$9>_d@P4>DZHKE<%XDCNWy{RknPazVW?K)F1EMu7&E{6Ym1<)&Ke95~ z79E&A9z6zMvMnIx5n~TL;|z94bipm{?tUU>H0zcJZIIQZ<3u5}Ex=Eu;>TfGlm53y>w#R+S-$ za57AroZLjw28z~Els1IDMX&#nB06Q1X^zY;$hNdxO|KNK!Czn{qHk~yG+JH#Z_OrM z{)5WRy0s6^@-@1m2M2hKZsl+DI(ePrv!G@-9%?gGnYd9Zz~DrIzu$>?(Y;g+yOBzF zQw7{c#l2MAM71VwrQ(LFA$2d+n7Ws0!o5^>D^ z{h%96Xzcq(WzQ*GN%x--(?ud2vOFWf;YKjNUfg~Mzg$g>ay4_U&veZ4zyqB;`~$e< zzX$gUQHt^$;dM{H9S|617ydI(KzcoXb_!x1J>dmJ!v7qL z?(J(b_X-R@Y6PaKs{14xUgo;;e5t}N(T6Fip=(Y0qU-?22yjY^o0Ef>K3!jBZYqWz zHbEIQvFVke(Wutx1-MLIZaqeforFi65fqiN%dD@PObeTaa|tZqqd?@*uo3#vjV6gVftjTa9~ zd0$GLP6~7cdwv-yZfy}WI+UwGkLFT`vWxU{V? zneyXeEnOs;7uU>*Yo-fk#Wl0y-ucqK=MRqW!r4G>!3F=f{IQ(MNd8<-&A66satpT| z<>BY99=mevtrzAiHqBLRidAg>z_A5~5&$1NKX&2lx6j6%Ws_qQV;9dxc11d7o%JE% z!@RN!*&RT6W{k(&-?CGZREIEhG&imsc?U@~pA+UISOw z&j0=G+~WJ3!IXa+POMFDG{y3(BOP=3b&$xcd2~cDZ>@mv70)XQchZjozsvsFUWv*no zpE=t&Jl=Kk$a=TLynCJY-I_M@?ndoH^~+} z%U~2tb&`#BW@C%#c3?c@{o2R>KA|@e&Sep|#>u zP7>K-R>rJda|E7+iQ!i#nCM_;E)PBITBb$z4h`dXjM$67nq322e-DTQ$t>8ID z+V_fdczLf(i{gJ8&q?pQrHWZe88gdH$CT8NjXR}#Oo&;!Zc?0B$abALn_!hTwlCm{ zk8dYsluz~s(JJLQ1FooHJm*P86!HKk$zW0FD#DLhqj%@pyjF@;H8Q`-wOX{i4n1bB&UJDXu~M z6T(U*e}#C)4~b>W5XboOPT>hS22hm>B(c1fG$|y_*d3|jbRyfqcT&N5m84a?0mbtxDu01z1L`B-yF7%Ti=$QUv-1IRM&Llzq}O$7V>6Flgb?)pdQCmk_3 zq8fE$yC_BcK(VBLywl|D-+&i1Npg#1RJG<-M!60ar zKJ{q#%(>^@xj*Ng@0@!lABhABkoD`IXC8p`H5Q8Gbp)E91mOlzh(i0wG0Lcnvd||m z0iI%?cuZo_F%R>|w=&Qm2`noAsJr621#m%(%GzGrLBySHT#o-?H^!X@dUOP|#> zwJ;O@D9n!L4?*z;IYSs#2&{=RQSmDQMW`q5OYc)ADM3Xuw5>`B1@+n^C zAN3_8cCg#XW;GQjvgCs<)xrRLFkH?@gwT8#geD@xfTQG1afD9O{%MjFZOPR15nEKp zhixBD+Q?ZZb|A~L{d)VDF|M_z)6B@WpHG`sI@6l7GP>Dr!qSG@Etbxj69%)|CN9{L z_G3-8V{4OXLq!0(GI3T*JPT`LZC<7n>zRH(?=QNs-|a?4WL_X+1TOw#Mub z)5Z-;Gl2HVp_#*}+t5+!)U3D2G$2ALY_|p(d6}jbxZ<>6xsSHBu=S z$Ifbz)gyZim|c@ffm>P`HI=q3mN}cVG!vSlz4P!V+*%CxoHGJbQFGH zN!wUkv4ILgiEwLxXdZ`hA=TsH6i_NM\+JTcrI4q$LzN}wO7aUv~GfWE{7+_WD zSw7XlxxJw@P+LK@0mdus`kh4it|z^07t}gChY`^hVOO2aKx@FpC9%E*+3U!5BHN9u z6&ZJ3ZXa9~`&N-*$@BobLjEcpaV&_H6&A!u0z~u2fV(VMh;%&8g`7wSQO^Srt{Ceg zrC1kLAa+Ci^D90U81*M(c0^H*t~vh_y4+J1htX+%6Ce3hT!*Jar)0eZz)`|s2JXqN zga%?Az$#4-?wZMFEO61z=_7E zLFLe3!QF~wd_q#Aif|gu%xQ=W7Sdx;gU`;wauiVkcT_y^_bNVU>o0HR$WhWE4AHVt zF(|rd5Le=&mw!X??}_w6${vGlI8Seg{hlz%im#Efo)C#HY9vGAAo{b5aA%N)1SPOX zaQB-7+fDSqn;2eQbgBpaco4V9Ma4dM3C+Y$`Ul{@ZA%g(Y+dM~>LO zw0R+`W(>PlWm+22rI|ZB!3bfe6oEZ znwUItv-f)M)x#s;3 zo4aNcU9;h?l?Lc+r4CyD^O2v#Uzv|J&BdDXLME2nr~B--w>g; zBKqJi@aS{z8;d?eRPj73?kL_N0iusohW~Fa_W1S$+%Y060%kF5xu}R)cl%OIL3ebK z;@|rLA!*2xry&|Rqy(O`@^#~OlqkU=&+{S^*vngt?i!rmAVO*IKfV@GRE#DAtQ+RQ zjv(s+X4AdwZ7g6)xtK832^r77@<=+PYs0c-$WMg5@^pw{T4sI78j+zXoJ9Ctl-@;l z9N7S{@6U~w=T_w zTU{`A6<1<@M75uzx{7RUn6Gb{t8aN&zjMBN=e599x)6$ea`4*0Tj__PhQ+Ea3sqaE zhJTa!G&6O2hR!s9k^DS4v+?#@x6l9K((f+){%5mY$7kaMc~MA2mx&OGE{h~Auhc;I zD^;YX^>KbHtmX3Ij_w9}x07~nl}1X+Pzu27d;$>!u`z@`2QNB&jlz&fDQ@& zhbZhGr~ewF?EBCUJB6$gqwEK$TSIA$gHl~DqI85A<1+gpw!-}=0*zygqYZ(^-b1!- z7R5`_(@@j(F3`U))YejG312Brg3KbhGwbL5NO&wFR~-9CG# z_RGz8H{WlVJ#=C=KKN{?u@o#wEgXSCBm|=#Pjr~^%0GInU8&h-s4BZqJGaS1<(Y%u zsQa-Y9;7icE%?KQ%N1I&KPkB`SL{2WW-_uZ$S?}9?Z|cjO9t1yYaPUTOfs&^?Yj4` zeb^2ex`4S6*cI|f5InE`!=LcTk6NOh=pz}}s!Tl*=T?TTRrr?(?F00J_u4QRrF7$5 z8iW@sLY4#yZfqHZ!kwnH8KAJ3ZkS0}y!a7^W^=zte0?pgfSX;+tzSG?gW*cNTX z^FwUYt&!TpYhgUm>dk z^&}npVNEiKUn)j&@O1>fR4@_Wz~pIrtqT)yZf7_R=0S$HYUi5F2jrbs4ru%LJKsFI z*g0s7)-ulkTNWv$e literal 0 HcmV?d00001 diff --git a/sched/__pycache__/vastai.cpython-313.pyc b/sched/__pycache__/vastai.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7335ca298108f846b952d0f8ff97ce450f79ef7a GIT binary patch literal 2556 zcmb7F%}*Og6rb^q*I(F#v@h;|sjU%&p0Yu`aa9FxehkW8Bp`|eEO;*9_dL`SXF7L*qny(dN`!zw z+Rzn3y+Tk330|gO!IT#ogs{NZMm`}T_#7+05EcB678vJ}d&=(&nlri_pQci=s8Ta7 zQKO_Qace@2Pf4bgmbCb}i^J*oIZ4;G%hIG)R+ab|H41UXqLQYk*VykSu^j>f!;&n` z(9N7&jiD-e=x!i{H`5rQP85frM&0!hkE^;$2LzeHdGtQ+=eJn`P&?ZQ^Sz_}{~dxS z?Kz2u0qO|=e+u8_a^41{<2)V->~IdlY;9_%eF-8dcETPfwXg0IxPfVeB#3_;9Kj!QjC)2^eUy4L6`K|K8dAJGwc?bs*cblwihMxjvBH5tt9AXTjr)n!9bZC+7XpCmSs=(uSU z=7=(ssSkw7auA1RcV{Myf|`+lFM6i>hpCdKO_~`~o=}zARJu506Oop5+6WsNnC2Xa zH8hJ>WBXQOnV(~sihs{KM;iG`Q|kune~-4TL=QiS9$txdJ&AU$zB>HV2PS_b(!kHY zyBxqsCUIC-K=P@~Ed9^q-nANO?OyL$K*OWAgW)RGBOQ zPa{b19Orb;DCD==1Wq7%kKh$}!8Zz=s5>P@b#%b=cGCFei{J2gZvfkxGR#r_0f}rmL2y7}NTh)5rSyfJ3T8SxT z%bt{B2bAK3Xh9KAmVK$zq*lL zz!ckJdp|9ybOt<^tYXP>DMMormL0C1D^`!ChhVN5KJx<*j24wh>w>Vf}>}1=n*&2E+IVJQyNN3D9IWC$Q#bRe?e6e%i_;T)o>ei0ViSTpkN# z+lr^Yl4XSgZD(ek+z=->Ua0D7t=fJ~hw9d4l|il*q;a*(r&5Y$e)7=cU{3deuOw&t z9mGUUvAvF&O_-6%0)cw0flF2066+r6QMLx3c>+YW=9(FtJ@@3r?r#eK=X`dd`@y*- za(@^AWp%|h@au^4@*7NhVK%_VxubpfAx@JYy?t1QvD>_$U1!xW1MFxQJb_Cb|twgc&~v@+t>uA2ca`rV(Vy) zLq%+D>xC7DnE5&g2d$>o1%5HK^x<+-b|svh9r`U4+m&WM6ggSLDs~Fr#{J|@;uMx$ z4uL<$2?wi-pzMa#HYT}o{ug?2ctzla@H$-La<)%Vp)QRX+5ygKCkXZ)*CPYUY>#?R z&Df2cW?(}aK65_^C-7k8=FCFR19_=;x$ER|uz!X0FO&XSh;0dc4|kC7_yjyplogjl zQJb$mBQJySFY(XgIYU>Ic#S=VNuIVtXLFivQFaJD&O{%Gm%&NRv_=(8wmt&uy)c#Z z(}ZhtglVh-XqP+3_R(V?k{-tgW%Pz$VKHoZwu$oxLE9(7wI_?B9Tdf}61;S(C{o5t z%C2DQFoeBM&Ryix3}sYsV0nl(K}FT-^1oFul+vVnhQ0}NEW2h8h;Mx?@D#TI literal 0 HcmV?d00001 diff --git a/sched/aggregator.py b/sched/aggregator.py new file mode 100644 index 0000000..e560325 --- /dev/null +++ b/sched/aggregator.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from collections import OrderedDict + +import torch + + +def average_weights(weight_dicts: list[OrderedDict]) -> OrderedDict: + """Average model state dicts from multiple workers.""" + if not weight_dicts: + raise ValueError("No weights to average") + if len(weight_dicts) == 1: + return weight_dicts[0] + + avg = OrderedDict() + for key in weight_dicts[0]: + avg[key] = torch.stack([w[key].float() for w in weight_dicts]).mean(dim=0) + return avg diff --git a/sched/job.py b/sched/job.py new file mode 100644 index 0000000..e26a248 --- /dev/null +++ b/sched/job.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass +class Job: + script: str + num_nodes: int = 2 + min_vram: int = 8 + docker_image: str = "pytorch/pytorch:2.1.0-cuda12.1-cudnn8-runtime" + rounds: int = 10 + local_steps: int = 100 + lr: float = 0.01 + batch_size: int = 64 + checkpoint_dir: str | None = None + env: dict[str, str] = field(default_factory=dict) diff --git a/sched/report.py b/sched/report.py new file mode 100644 index 0000000..29e7e4e --- /dev/null +++ b/sched/report.py @@ -0,0 +1,403 @@ +"""Generate a self-contained HTML report with an SVG timeline from tracer events.""" + +from __future__ import annotations + +import html +import os +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .trace import Tracer + +# ------------------------------------------------------------------ +# colours +# ------------------------------------------------------------------ + +COLORS = { + "push": "#3b82f6", + "exec": "#22c55e", + "pull": "#f97316", + "aggregate": "#8b5cf6", + "dead": "#ef4444", +} + +BG = "#0f172a" +CARD = "#1e293b" +TEXT = "#e2e8f0" +MUTED = "#94a3b8" +GRID = "#334155" + + +def _fmt_bytes(n: int) -> str: + if n < 1024: + return f"{n}B" + if n < 1024 * 1024: + return f"{n / 1024:.1f}KB" + return f"{n / (1024 * 1024):.1f}MB" + + +# ------------------------------------------------------------------ +# SVG builder +# ------------------------------------------------------------------ + +def _build_svg(tracer: Tracer) -> str: + rounds = tracer._collect_rounds() + max_rank = tracer._max_rank() + if not rounds: + return '' + + # layout constants + label_w = 100 + right_pad = 20 + row_h = 28 + row_gap = 4 + round_gap = 24 + round_header_h = 22 + chart_w = 700 + total_w = label_w + chart_w + right_pad + + # compute total height + rows_per_round = 1 + max_rank + 1 # scheduler + workers + n_rounds = len(rounds) + total_h = ( + n_rounds * (round_header_h + rows_per_round * (row_h + row_gap) + round_gap) + + 40 # bottom time axis + ) + + parts: list[str] = [] + parts.append( + f'' + ) + parts.append( + f'' + ) + + y_cursor = 12 + + for rnum in sorted(rounds.keys()): + rd = rounds[rnum] + t0 = rd["start"] + dur = rd["duration"] or 0.001 + surv = rd["survivors"] + total_nodes = rd["total"] + + def x_pos(t: float) -> float: + frac = max(0.0, min(1.0, (t - t0) / dur)) + return label_w + frac * chart_w + + def bar_w(d: float) -> float: + return max(3, d / dur * chart_w) # min 3px so it's visible + + # round header + parts.append( + f'' + f"round {rnum} [{surv}/{total_nodes}] {dur:.2f}s" + ) + y_cursor += round_header_h + + # time gridlines + n_ticks = 5 + for i in range(n_ticks + 1): + frac = i / n_ticks + gx = label_w + frac * chart_w + parts.append( + f'' + ) + t_label = frac * dur + parts.append( + f'' + f"{t_label:.1f}s" + ) + + # --- scheduler row --- + row_y = y_cursor + parts.append( + f'scheduler' + ) + # background bar + parts.append( + f'' + ) + for ev in rd["events"]: + d = ev.data.get("duration_s", 0) + if ev.kind in ("push", "pull", "aggregate"): + color = COLORS.get(ev.kind, COLORS["push"]) + bx = x_pos(ev.time - d) + bw = bar_w(d) + tooltip = _tooltip(ev) + parts.append( + f'' + f"{html.escape(tooltip)}" + ) + y_cursor += row_h + row_gap + + # --- worker rows --- + for rank in range(max_rank + 1): + row_y = y_cursor + parts.append( + f'worker {rank}' + ) + parts.append( + f'' + ) + + rank_evs = [e for e in rd["events"] if e.rank == rank] + dead_x: float | None = None + + for ev in rank_evs: + d = ev.data.get("duration_s", 0) + is_err = bool(ev.data.get("error")) + is_bad_exit = ev.kind == "exec" and ev.data.get("exit_code", 0) != 0 + + if is_err or is_bad_exit: + color = COLORS["dead"] + dead_x = x_pos(ev.time) + else: + color = COLORS.get(ev.kind, COLORS["push"]) + + bx = x_pos(ev.time - d) + bw = bar_w(d) + tooltip = _tooltip(ev) + parts.append( + f'' + f"{html.escape(tooltip)}" + ) + + # dead zone: hatched red from failure to end + if dead_x is not None: + dw = label_w + chart_w - dead_x + if dw > 0: + parts.append( + f'' + ) + + # no events at all — full dead bar + if not rank_evs and rank < rd.get("num_workers", 0): + parts.append( + f'' + f"worker {rank}: no response" + ) + + y_cursor += row_h + row_gap + + y_cursor += round_gap + + parts.append("") + return "\n".join(parts) + + +def _tooltip(ev) -> str: + kind = ev.kind + d = ev.data + dur = d.get("duration_s", 0) + err = d.get("error") + if err: + return f"{kind} node={ev.rank}: FAILED — {err}" + if kind == "push": + return f"push {_fmt_bytes(d.get('size_bytes', 0))} → node {ev.rank} ({dur:.3f}s)" + if kind == "pull": + return f"pull {_fmt_bytes(d.get('size_bytes', 0))} ← node {ev.rank} ({dur:.3f}s)" + if kind == "exec": + tail = d.get("output_tail", "") + return f"train node {ev.rank}: {dur:.2f}s exit={d.get('exit_code', '?')}\n{tail}" + if kind == "aggregate": + return ( + f"aggregate: {d.get('num_workers', '?')} workers, " + f"|W|={d.get('weight_norm', 0):.2f}, Δ={d.get('delta_norm', 0):.4f}" + ) + return f"{kind}: {d}" + + +# ------------------------------------------------------------------ +# Full HTML page +# ------------------------------------------------------------------ + +def generate_html(tracer: Tracer, path: str) -> str: + """Write a self-contained HTML report to `path`. Returns the path.""" + rounds = tracer._collect_rounds() + + # compute summary stats + n_rounds = len(rounds) + total_time = sum(r["duration"] for r in rounds.values()) + total_push = 0 + total_pull = 0 + for rd in rounds.values(): + total_push += sum( + e.data.get("size_bytes", 0) for e in rd["events"] + if e.kind == "push" and not e.data.get("error") + ) + total_pull += sum( + e.data.get("size_bytes", 0) for e in rd["events"] + if e.kind == "pull" and not e.data.get("error") + ) + max_workers = max((r["total"] for r in rounds.values()), default=0) + final_wnorm = list(rounds.values())[-1]["weight_norm"] if rounds else 0 + + svg = _build_svg(tracer) + + page = f""" + + + +Training Run Report + + + +

Training Run Report

+ +
+
+
{n_rounds}
+
Rounds
+
+
+
{max_workers}
+
Workers
+
+
+
{total_time:.1f}s
+
Total Time
+
+
+
{_fmt_bytes(total_push + total_pull)}
+
Data Transferred
+
+
+
{_fmt_bytes(total_push)}
+
Params Pushed
+
+
+
{_fmt_bytes(total_pull)}
+
Weights Pulled
+
+
+
{final_wnorm:.1f}
+
Final |W|
+
+
+ +
+
Push params
+
Train
+
Pull weights
+
Aggregate
+
Dead / Error
+
+ +
+{svg} +
+ +
+
{_render_event_log(tracer)}
+
+ + +""" + + path = os.path.abspath(path) + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + with open(path, "w") as f: + f.write(page) + return path + + +def _render_event_log(tracer: Tracer) -> str: + """Render the text event log as HTML-escaped pre-formatted text.""" + lines = [] + for ev in tracer.events: + err = ev.data.get("error") + t = f"[{ev.time:7.2f}s]" + node = f"node={ev.rank}" if ev.rank is not None else " " + + match ev.kind: + case "push" | "pull": + sz = _fmt_bytes(ev.data.get("size_bytes", 0)) + dur = ev.data.get("duration_s", 0) + p = os.path.basename(ev.data.get("path", "")) + if err: + line = f'{t} {ev.kind:5s} {node} {p:<20s} !! {html.escape(err)}' + else: + line = f"{t} {ev.kind:5s} {node} {p:<20s} {sz:>8s} {dur:.3f}s" + case "exec": + dur = ev.data.get("duration_s", 0) + ec = ev.data.get("exit_code", "?") + if err: + line = f'{t} exec {node} !! {html.escape(err)}' + else: + tail = html.escape(ev.data.get("output_tail", "")) + line = f"{t} exec {node} exit={ec:<3} {dur:.3f}s" + if tail: + line += f" | {tail}" + case "round_start": + r = ev.data.get("round_num", "?") + n = ev.data.get("active_nodes", "?") + line = f"{t} {'─'*4} round {r} start ({n} workers) {'─'*16}" + case "round_end": + r = ev.data.get("round_num", "?") + s = ev.data.get("survivors", "?") + tot = ev.data.get("total_nodes", "?") + dur = ev.data.get("duration_s", 0) + line = f"{t} {'─'*4} round {r} end ({s}/{tot} survived, {dur:.2f}s) {'─'*8}" + case "aggregate": + n = ev.data.get("num_workers", "?") + wn = ev.data.get("weight_norm", 0) + dn = ev.data.get("delta_norm", 0) + dur = ev.data.get("duration_s", 0) + line = f"{t} agg {n} workers |W|={wn:.2f} Δ={dn:.4f} {dur:.3f}s" + case _: + line = f"{t} {ev.kind}" + + lines.append(line) + return "\n".join(lines) diff --git a/sched/scheduler.py b/sched/scheduler.py new file mode 100644 index 0000000..38e0e8f --- /dev/null +++ b/sched/scheduler.py @@ -0,0 +1,300 @@ +from __future__ import annotations + +import asyncio +import importlib.util +import logging +import os +import tempfile +import time +from collections import OrderedDict +from dataclasses import dataclass + +import torch + +from .aggregator import average_weights +from .job import Job +from .trace import Tracer, TracingConnection +from .transport import Connection, Transport + +log = logging.getLogger(__name__) + + +@dataclass +class Node: + rank: int + vast_id: int | None = None + host: str | None = None + port: int | None = None + conn: Connection | None = None + status: str = "pending" # pending | ready | running | dead | done + + +class Scheduler: + def __init__(self, job: Job, transport: Transport, tracer: Tracer | None = None): + self.job = job + self.transport = transport + self.tracer = tracer + self.nodes: list[Node] = [] + self.work_dir = tempfile.mkdtemp(prefix="sched_") + + # ------------------------------------------------------------------ + # helpers + # ------------------------------------------------------------------ + + def _active_nodes(self) -> list[Node]: + return [n for n in self.nodes if n.conn and n.status not in ("dead", "done")] + + def _load_job_module(self): + spec = importlib.util.spec_from_file_location( + "job_module", os.path.abspath(self.job.script) + ) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + def _wrap_connections(self): + """Wrap node connections with TracingConnection if tracer is set.""" + if not self.tracer: + return + for node in self.nodes: + if node.conn and not isinstance(node.conn, TracingConnection): + node.conn = TracingConnection(node.conn, node.rank, self.tracer) + + # ------------------------------------------------------------------ + # lifecycle phases + # ------------------------------------------------------------------ + + async def provision(self): + """Rent instances from vast.ai and wait for SSH. Populates self.nodes.""" + from . import vastai + + offers = await vastai.search_offers(self.job.min_vram, limit=self.job.num_nodes) + if len(offers) < self.job.num_nodes: + raise RuntimeError( + f"Need {self.job.num_nodes} instances, only {len(offers)} offers" + ) + + for rank, offer in enumerate(offers[: self.job.num_nodes]): + await vastai.rent_instance(offer["id"], self.job.docker_image) + self.nodes.append(Node(rank=rank, vast_id=offer["id"], status="provisioning")) + + # poll until SSH ready (up to 5 min) + for _ in range(60): + instances = await vastai.get_instances() + by_id = {i["id"]: i for i in instances} + all_ready = True + for node in self.nodes: + inst = by_id.get(node.vast_id) + if inst and inst.get("ssh_host"): + node.host = inst["ssh_host"] + node.port = inst["ssh_port"] + node.status = "ready" + else: + all_ready = False + if all_ready: + break + await asyncio.sleep(5) + + # open connections + for node in self.nodes: + if node.host: + node.conn = await self.transport.connect(node.host, node.port) + + self._wrap_connections() + + ready = self._active_nodes() + if not ready: + raise RuntimeError("No nodes became ready") + log.info(f"Provisioned {len(ready)}/{self.job.num_nodes} nodes") + + async def deploy(self): + """Push worker script and job module to all active nodes. + + Skips pushing worker.py if it already exists on the remote + (e.g. baked into a Docker image). Always pushes the job module. + """ + self._wrap_connections() + + worker_src = os.path.join( + os.path.dirname(__file__), os.pardir, "worker", "worker.py" + ) + worker_src = os.path.abspath(worker_src) + job_src = os.path.abspath(self.job.script) + + async def _deploy_one(node: Node): + rc, _ = await node.conn.exec("test -f /workspace/worker.py") + if rc != 0: + await node.conn.push(worker_src, "/workspace/worker.py") + else: + log.debug(f"Node {node.rank}: worker.py already present, skipping push") + await node.conn.push(job_src, "/workspace/job_module.py") + + results = await asyncio.gather( + *[_deploy_one(n) for n in self._active_nodes()], + return_exceptions=True, + ) + for node, res in zip(self._active_nodes(), results): + if isinstance(res, Exception): + log.warning(f"Deploy failed on node {node.rank}: {res}") + node.status = "dead" + + if not self._active_nodes(): + raise RuntimeError("Deploy failed on all nodes") + + async def run_round( + self, state_dict: OrderedDict, round_num: int + ) -> OrderedDict: + """Single FedAvg round: push params -> workers train -> pull weights -> average.""" + round_t0 = time.monotonic() + + params_path = os.path.join(self.work_dir, "params.pt") + torch.save(state_dict, params_path) + + active = self._active_nodes() + if not active: + raise RuntimeError("No active workers") + + if self.tracer: + self.tracer.emit( + "round_start", + round_num=round_num, + active_nodes=len(active), + ) + + # --- push params --- + push_res = await asyncio.gather( + *[n.conn.push(params_path, "/workspace/params.pt") for n in active], + return_exceptions=True, + ) + for node, res in zip(active, push_res): + if isinstance(res, Exception): + log.warning(f"Param push failed for node {node.rank}: {res}") + node.status = "dead" + + active = self._active_nodes() + if not active: + raise RuntimeError("No workers alive after param push") + + # --- run training --- + cmd_template = ( + "cd /workspace && python worker.py" + " --params /workspace/params.pt" + " --output /workspace/weights.pt" + " --rank {rank}" + f" --world-size {self.job.num_nodes}" + f" --local-steps {self.job.local_steps}" + f" --lr {self.job.lr}" + f" --batch-size {self.job.batch_size}" + ) + exec_res = await asyncio.gather( + *[ + n.conn.exec(cmd_template.format(rank=n.rank), timeout=300.0) + for n in active + ], + return_exceptions=True, + ) + + survivors = [] + for node, res in zip(active, exec_res): + if isinstance(res, Exception): + log.warning(f"Node {node.rank} raised: {res}") + node.status = "dead" + elif res[0] != 0: + log.warning(f"Node {node.rank} exit {res[0]}: {res[1][:200]}") + node.status = "dead" + else: + survivors.append(node) + + if not survivors: + raise RuntimeError("All workers failed") + + # --- pull weights --- + weight_dicts: list[OrderedDict] = [] + for node in survivors: + local_path = os.path.join(self.work_dir, f"weights_{node.rank}.pt") + try: + await node.conn.pull("/workspace/weights.pt", local_path) + w = torch.load(local_path, map_location="cpu", weights_only=True) + weight_dicts.append(w) + except Exception as e: + log.warning(f"Weight pull failed for node {node.rank}: {e}") + node.status = "dead" + + if not weight_dicts: + raise RuntimeError("No weights collected") + + # --- aggregate --- + agg_t0 = time.monotonic() + new_state = average_weights(weight_dicts) + agg_dur = time.monotonic() - agg_t0 + + if self.tracer: + weight_norm = sum(v.float().norm().item() for v in new_state.values()) + delta_norm = sum( + (new_state[k].float() - state_dict[k].float()).norm().item() + for k in new_state + ) + self.tracer.emit( + "aggregate", + num_workers=len(weight_dicts), + weight_norm=weight_norm, + delta_norm=delta_norm, + duration_s=agg_dur, + ) + + round_dur = time.monotonic() - round_t0 + + if self.tracer: + self.tracer.emit( + "round_end", + round_num=round_num, + survivors=len(weight_dicts), + total_nodes=len(self.nodes), + duration_s=round_dur, + ) + + log.info( + f"Round {round_num}: {len(weight_dicts)}/{len(self.nodes)} workers" + ) + return new_state + + async def run(self) -> OrderedDict: + """Full pipeline: provision -> deploy -> train rounds -> cleanup. + + Returns the final averaged state_dict. + """ + job_mod = self._load_job_module() + model = job_mod.make_model() + state = model.state_dict() + + log.info( + f"Job start: {self.job.rounds} rounds, " + f"{self.job.num_nodes} nodes, " + f"{self.job.local_steps} local steps/round" + ) + + await self.provision() + await self.deploy() + + for r in range(self.job.rounds): + state = await self.run_round(state, r) + + if self.tracer: + self.tracer.summary() + + log.info("Job complete") + await self.cleanup() + return state + + async def cleanup(self): + """Close connections, destroy instances.""" + from . import vastai + + for node in self.nodes: + if node.conn: + await node.conn.close() + if node.vast_id: + try: + await vastai.destroy_instance(node.vast_id) + except Exception: + pass diff --git a/sched/trace.py b/sched/trace.py new file mode 100644 index 0000000..51ddfdc --- /dev/null +++ b/sched/trace.py @@ -0,0 +1,424 @@ +"""Tracing infrastructure for observing data flow through the system. + +Three output modes, all fed from the same event list: + C) Live text log — Tracer._print() streams events as they happen + A) Terminal Gantt — Tracer.gantt() renders a Unicode timeline per round + B) HTML report — report.generate_html(tracer, path) writes a self-contained SVG timeline + +Usage: + tracer = Tracer() + sched = Scheduler(job, transport, tracer=tracer) + await sched.run() + tracer.gantt() + tracer.summary() +""" + +from __future__ import annotations + +import os +import sys +import time +from dataclasses import dataclass, field + + +# ------------------------------------------------------------------ +# Events +# ------------------------------------------------------------------ + +@dataclass +class Event: + time: float # seconds since tracer start + kind: str # push | pull | exec | round_start | round_end | aggregate + rank: int | None # None for scheduler-level events + data: dict = field(default_factory=dict) + + +def _fmt_bytes(n: int) -> str: + if n < 1024: + return f"{n}B" + if n < 1024 * 1024: + return f"{n / 1024:.1f}KB" + return f"{n / (1024 * 1024):.1f}MB" + + +# ------------------------------------------------------------------ +# Tracer +# ------------------------------------------------------------------ + +class Tracer: + def __init__(self, file=None): + self._t0 = time.monotonic() + self.events: list[Event] = [] + self._file = file or sys.stderr + + def emit(self, kind: str, rank: int | None = None, **data) -> Event: + ev = Event( + time=time.monotonic() - self._t0, + kind=kind, + rank=rank, + data=data, + ) + self.events.append(ev) + self._print(ev) + return ev + + # ------------------------------------------------------------------ + # C) Live text log + # ------------------------------------------------------------------ + + def _print(self, ev: Event): + t = f"[{ev.time:7.2f}s]" + node = f"node={ev.rank}" if ev.rank is not None else " " + err = ev.data.get("error") + + match ev.kind: + case "push": + sz = _fmt_bytes(ev.data.get("size_bytes", 0)) + dur = ev.data.get("duration_s", 0) + path = os.path.basename(ev.data.get("path", "")) + if err: + line = f"{t} push {node} {path:<20s} !! {err}" + else: + line = f"{t} push {node} {path:<20s} {sz:>8s} {dur:.3f}s" + case "pull": + sz = _fmt_bytes(ev.data.get("size_bytes", 0)) + dur = ev.data.get("duration_s", 0) + path = os.path.basename(ev.data.get("path", "")) + if err: + line = f"{t} pull {node} {path:<20s} !! {err}" + else: + line = f"{t} pull {node} {path:<20s} {sz:>8s} {dur:.3f}s" + case "exec": + dur = ev.data.get("duration_s", 0) + exit_code = ev.data.get("exit_code", "?") + if err: + line = f"{t} exec {node} !! {err}" + else: + line = f"{t} exec {node} exit={exit_code:<3} {dur:.3f}s" + output = ev.data.get("output_tail", "") + if output: + line += f" | {output}" + case "round_start": + r = ev.data.get("round_num", "?") + n = ev.data.get("active_nodes", "?") + line = f"{t} {'─'*4} round {r} start ({n} workers) {'─'*20}" + case "round_end": + r = ev.data.get("round_num", "?") + surv = ev.data.get("survivors", "?") + total = ev.data.get("total_nodes", "?") + dur = ev.data.get("duration_s", 0) + line = f"{t} {'─'*4} round {r} end ({surv}/{total} survived, {dur:.2f}s) {'─'*12}" + case "aggregate": + n = ev.data.get("num_workers", "?") + wnorm = ev.data.get("weight_norm", 0) + dnorm = ev.data.get("delta_norm", 0) + dur = ev.data.get("duration_s", 0) + line = ( + f"{t} agg {n} workers" + f" |W|={wnorm:.2f}" + f" Δ={dnorm:.4f}" + f" {dur:.3f}s" + ) + case _: + line = f"{t} {ev.kind:6s} {node} {ev.data}" + + print(line, file=self._file, flush=True) + + # ------------------------------------------------------------------ + # helpers + # ------------------------------------------------------------------ + + def _collect_rounds(self) -> dict[int, dict]: + """Group events into per-round buckets with timing metadata.""" + rounds: dict[int, dict] = {} + current_round: int | None = None + + for ev in self.events: + if ev.kind == "round_start": + current_round = ev.data["round_num"] + rounds[current_round] = { + "start": ev.time, + "end": ev.time, + "duration": 0.0, + "num_workers": ev.data.get("active_nodes", 0), + "survivors": 0, + "total": 0, + "events": [], + "weight_norm": 0.0, + "delta_norm": 0.0, + } + elif ev.kind == "round_end": + r = ev.data["round_num"] + if r in rounds: + rounds[r]["end"] = ev.time + rounds[r]["duration"] = ev.data.get("duration_s", 0) + rounds[r]["survivors"] = ev.data.get("survivors", 0) + rounds[r]["total"] = ev.data.get("total_nodes", 0) + elif ev.kind == "aggregate": + if current_round is not None and current_round in rounds: + rounds[current_round]["weight_norm"] = ev.data.get("weight_norm", 0) + rounds[current_round]["delta_norm"] = ev.data.get("delta_norm", 0) + rounds[current_round]["events"].append(ev) + else: + if current_round is not None and current_round in rounds: + rounds[current_round]["events"].append(ev) + + return rounds + + def _max_rank(self) -> int: + ranks = [ev.rank for ev in self.events if ev.rank is not None] + return max(ranks) if ranks else 0 + + # ------------------------------------------------------------------ + # A) Terminal Gantt chart + # ------------------------------------------------------------------ + + def gantt(self, width: int = 60) -> str: + """Render a Unicode Gantt chart showing data flow per round.""" + rounds = self._collect_rounds() + if not rounds: + return "(no rounds recorded)" + + max_rank = self._max_rank() + + lines: list[str] = [] + lines.append("") + lines.append(" ▓ push █ train ▒ pull ● agg ✗ dead ░ wait") + lines.append("") + + for rnum in sorted(rounds.keys()): + rd = rounds[rnum] + t0 = rd["start"] + dur = rd["duration"] or 0.001 + surv = rd["survivors"] + total = rd["total"] + + lines.append(f" round {rnum} [{surv}/{total}] {dur:.2f}s") + + def to_col(t: float) -> int: + frac = (t - t0) / dur + return max(0, min(width - 1, int(frac * width))) + + def fill(row: list[str], t_start: float, t_end: float, char: str): + s = to_col(t_start) + e = to_col(t_end) + if s == e: + e = min(s + 1, width - 1) + for i in range(s, e + 1): + if 0 <= i < width: + row[i] = char + + # --- scheduler row --- + sched = list("░" * width) + for ev in rd["events"]: + d = ev.data.get("duration_s", 0) + if ev.kind == "push": + fill(sched, ev.time - d, ev.time, "▓") + elif ev.kind == "pull": + fill(sched, ev.time - d, ev.time, "▒") + elif ev.kind == "aggregate": + fill(sched, ev.time - d, ev.time, "●") + lines.append(f" scheduler {''.join(sched)}") + + # --- worker rows --- + for rank in range(max_rank + 1): + row = list("─" * width) + rank_evs = [e for e in rd["events"] if e.rank == rank] + dead_col = None + + for ev in rank_evs: + d = ev.data.get("duration_s", 0) + if ev.data.get("error"): + dead_col = to_col(ev.time) + fill(row, ev.time - d, ev.time, "✗") + elif ev.kind == "exec" and ev.data.get("exit_code", 0) != 0: + dead_col = to_col(ev.time) + fill(row, ev.time - d, ev.time, "✗") + elif ev.kind == "exec": + fill(row, ev.time - d, ev.time, "█") + elif ev.kind == "push": + fill(row, ev.time - d, ev.time, "▓") + elif ev.kind == "pull": + fill(row, ev.time - d, ev.time, "▒") + + # fill dead from failure point onward + if dead_col is not None: + for i in range(dead_col, width): + if row[i] == "─": + row[i] = "✗" + + # worker was expected but produced nothing + if not rank_evs and rank < rd.get("num_workers", 0): + row = list("✗" * width) + + lines.append(f" worker {rank:<3} {''.join(row)}") + + lines.append("") + + text = "\n".join(lines) + print(text, file=self._file, flush=True) + return text + + # ------------------------------------------------------------------ + # Summary table + # ------------------------------------------------------------------ + + def summary(self) -> str: + """Render a round-by-round summary table.""" + rounds = self._collect_rounds() + if not rounds: + return "(no rounds recorded)" + + lines: list[str] = [] + lines.append("") + + header = ( + f"{'Round':<6} {'Workers':<9} {'Params↑':<10} {'Weights↓':<11}" + f" {'Train(max)':<11} {'Agg':<8} {'Total':<8} {'Δ norm'}" + ) + lines.append(header) + lines.append("─" * len(header)) + + total_push = 0 + total_pull = 0 + total_time = 0.0 + + for rnum in sorted(rounds.keys()): + rd = rounds[rnum] + surv = rd["survivors"] + total = rd["total"] + + push_bytes = sum( + e.data.get("size_bytes", 0) + for e in rd["events"] + if e.kind == "push" and not e.data.get("error") + ) + pull_bytes = sum( + e.data.get("size_bytes", 0) + for e in rd["events"] + if e.kind == "pull" and not e.data.get("error") + ) + exec_durs = [ + e.data.get("duration_s", 0) + for e in rd["events"] + if e.kind == "exec" and not e.data.get("error") + ] + agg_dur = sum( + e.data.get("duration_s", 0) + for e in rd["events"] + if e.kind == "aggregate" + ) + train_max = max(exec_durs) if exec_durs else 0 + dur = rd["duration"] + delta = rd["delta_norm"] + + total_push += push_bytes + total_pull += pull_bytes + total_time += dur + + lines.append( + f"{rnum:<6} {surv}/{total:<6} {_fmt_bytes(push_bytes):<10}" + f" {_fmt_bytes(pull_bytes):<11} {train_max:<11.2f}" + f" {agg_dur:<8.3f} {dur:<8.2f} {delta:.4f}" + ) + + lines.append("─" * len(header)) + lines.append( + f"{'total':<6} {'':9} {_fmt_bytes(total_push):<10}" + f" {_fmt_bytes(total_pull):<11} {'':11}" + f" {'':8} {total_time:<8.2f}" + ) + + text = "\n".join(lines) + print(text, file=self._file, flush=True) + return text + + +# ------------------------------------------------------------------ +# TracingConnection wrapper +# ------------------------------------------------------------------ + +class TracingConnection: + """Wraps any Connection to auto-trace push/pull/exec with timing and sizes. + + Emits events on both success and failure so the Gantt chart can show + where workers died. + """ + + def __init__(self, inner, rank: int, tracer: Tracer): + self._inner = inner + self._rank = rank + self._tracer = tracer + + # proxy unknown attributes to inner (e.g. MockConnection.kill, .root_dir) + def __getattr__(self, name): + return getattr(self._inner, name) + + async def exec(self, cmd: str, timeout: float = 30.0) -> tuple[int, str]: + t0 = time.monotonic() + try: + ret, output = await self._inner.exec(cmd, timeout=timeout) + except Exception as exc: + dur = time.monotonic() - t0 + self._tracer.emit( + "exec", rank=self._rank, + duration_s=dur, exit_code=-1, output_tail="", + error=str(exc), + ) + raise + + dur = time.monotonic() - t0 + tail = "" + for line in reversed(output.strip().splitlines()): + if line.strip(): + tail = line.strip()[:80] + break + + self._tracer.emit( + "exec", rank=self._rank, + duration_s=dur, exit_code=ret, output_tail=tail, + ) + return ret, output + + async def push(self, local_path: str, remote_path: str) -> None: + size = os.path.getsize(local_path) if os.path.exists(local_path) else 0 + t0 = time.monotonic() + try: + await self._inner.push(local_path, remote_path) + except Exception as exc: + dur = time.monotonic() - t0 + self._tracer.emit( + "push", rank=self._rank, + path=remote_path, size_bytes=0, duration_s=dur, + error=str(exc), + ) + raise + + dur = time.monotonic() - t0 + self._tracer.emit( + "push", rank=self._rank, + path=remote_path, size_bytes=size, duration_s=dur, + ) + + async def pull(self, remote_path: str, local_path: str) -> None: + t0 = time.monotonic() + try: + await self._inner.pull(remote_path, local_path) + except Exception as exc: + dur = time.monotonic() - t0 + self._tracer.emit( + "pull", rank=self._rank, + path=remote_path, size_bytes=0, duration_s=dur, + error=str(exc), + ) + raise + + dur = time.monotonic() - t0 + size = os.path.getsize(local_path) if os.path.exists(local_path) else 0 + self._tracer.emit( + "pull", rank=self._rank, + path=remote_path, size_bytes=size, duration_s=dur, + ) + + async def close(self) -> None: + await self._inner.close() diff --git a/sched/transport.py b/sched/transport.py new file mode 100644 index 0000000..2e8a3eb --- /dev/null +++ b/sched/transport.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import asyncio +from typing import Protocol, runtime_checkable + + +@runtime_checkable +class Connection(Protocol): + async def exec(self, cmd: str, timeout: float = 30.0) -> tuple[int, str]: ... + async def push(self, local_path: str, remote_path: str) -> None: ... + async def pull(self, remote_path: str, local_path: str) -> None: ... + async def close(self) -> None: ... + + +@runtime_checkable +class Transport(Protocol): + async def connect(self, host: str, port: int) -> Connection: ... + + +class SSHConnection: + def __init__(self, host: str, port: int): + self.host = host + self.port = port + + async def exec(self, cmd: str, timeout: float = 30.0) -> tuple[int, str]: + proc = await asyncio.create_subprocess_exec( + "ssh", + "-p", str(self.port), + "-o", "StrictHostKeyChecking=no", + "-o", "ConnectTimeout=5", + "-o", "BatchMode=yes", + f"root@{self.host}", + cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, stderr = await asyncio.wait_for( + proc.communicate(), timeout=timeout + ) + return proc.returncode, (stdout + stderr).decode() + except asyncio.TimeoutError: + proc.kill() + return 1, "timeout" + + async def push(self, local_path: str, remote_path: str) -> None: + proc = await asyncio.create_subprocess_exec( + "scp", + "-P", str(self.port), + "-o", "StrictHostKeyChecking=no", + local_path, + f"root@{self.host}:{remote_path}", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + _, stderr = await proc.communicate() + if proc.returncode != 0: + raise ConnectionError( + f"scp push failed to {self.host}:{self.port}: {stderr.decode()}" + ) + + async def pull(self, remote_path: str, local_path: str) -> None: + proc = await asyncio.create_subprocess_exec( + "scp", + "-P", str(self.port), + "-o", "StrictHostKeyChecking=no", + f"root@{self.host}:{remote_path}", + local_path, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + _, stderr = await proc.communicate() + if proc.returncode != 0: + raise ConnectionError( + f"scp pull failed from {self.host}:{self.port}: {stderr.decode()}" + ) + + async def close(self) -> None: + pass + + +class SSHTransport: + async def connect(self, host: str, port: int) -> SSHConnection: + return SSHConnection(host, port) diff --git a/sched/vastai.py b/sched/vastai.py new file mode 100644 index 0000000..bfccb42 --- /dev/null +++ b/sched/vastai.py @@ -0,0 +1,47 @@ +"""Async wrappers around the vast.ai CLI. Cannibalized from dtrain.""" + +from __future__ import annotations + +import asyncio +import json + + +async def _run(*args: str) -> str | None: + proc = await asyncio.create_subprocess_exec( + "vastai", *args, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await proc.communicate() + if proc.returncode != 0: + return None + return stdout.decode() + + +async def search_offers(min_vram: int, limit: int = 10) -> list[dict]: + query = f"gpu_ram>={min_vram} inet_down>=100" + output = await _run( + "search", "offers", query, "-o", "dph_total", + "--limit", str(limit), "--raw", + ) + if not output: + return [] + return json.loads(output) + + +async def rent_instance(offer_id: int, image: str, disk: int = 20) -> str | None: + return await _run( + "create", "instance", str(offer_id), + "--image", image, "--disk", str(disk), + ) + + +async def get_instances() -> list[dict]: + output = await _run("show", "instances", "--raw") + if not output: + return [] + return json.loads(output) + + +async def destroy_instance(instance_id: int) -> None: + await _run("destroy", "instance", str(instance_id)) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/__pycache__/__init__.cpython-313.pyc b/tests/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4657d92f25fd777f32659483f418186719e0a9ac GIT binary patch literal 139 zcmey&%ge<81X+t)GC}lX5CH>>P{wB#AY&>+I)f&o-%5reCLr%KNa~iBenx(7s(xZ( zQGT9&Sz>WXVy13sNoG#5eo1O^NwI!>d}dx|NqoFsLFFwDo80`A(wtPgB37VQkkQ2; O#z$sGM#ds$APWHbLLayQ literal 0 HcmV?d00001 diff --git a/tests/__pycache__/mock_transport.cpython-313.pyc b/tests/__pycache__/mock_transport.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0112d1395df79467243b82ab7cc2528d5c346795 GIT binary patch literal 5399 zcmbtY-ES1v6~FVfJ6`XH@viwWB#b|xSxVLclT@K#J`5y4*-(!iMI~rD%#Q8BvoqU! zXK^4>8qq2;Q6d*92p;-C)fae*^AsNX5Rv==Hcqt8#1E0u7v2Dq(nNjeIWxQK_1a0* z*;t&bdp8M2tZC$KU^K{S`{zV5OGCu)r<74uxw(C6uU~%Ae#;@RV1u zESwam5ac2igIuCgkVj}_Bk3lp*hy6Bc|M$pMuWBpv_w zl1n|s8`nLBmK~-{x^%*%OtD=?3EN2X79CfMEhbys?ZkT{$R zbZ?v~)SRT2=b6Bn3$vPa24OR5xrC>IE1$=`0JEdIWtXYR_9}QBz@6jdCM?^2CX3^l zRV?GtD7xaf1ydARrA)mzV&Je81hUG!y5 zvmDFQG=IZNrgOC)X`DX^ z$WZWWC3&V=EwmDHraptpJzw3n$&GMB(#kPdQ+b##vbw#dW)61-4LF=FL2j{yc8#n@HAXDE*r^z9}jfbVoJf(3*X0IkEVKhiHc49`^Ha-Ec$89?`cZ0@Y{0odMf|AFE;@$g`Wp;t z0(NN2prx^K)3*Hv81u!`$4?*i!f%y424+<5(Ql#zK9^}r<*hp8}YiB59B`TVJbcS z=JcD3?Oiu>^XDjFp}_YhpAi1(7*?9fH)dM$Q{hLs%I|Mvqbn7Wg*@+ef-Mt_fCG9-ugg-hX1gaVSASkCype!yD4r^5%0E% zSnk+9*h}vInj7pE@9yh?@~2yQX#cdE!*Xx$kyha|8CyP!aae8@pu7T^5K97N+6_YU zGm(k^NysELX!)AR)bM{T1WLgUn$n0+!^{0#9Z_K{=;E3frwWJ^3OQ3pnIW`GO{tr_ zmagHcLdHm-t5TlgAi567@NLeQ%{MIOY3@We7N{OY>7G4AoSn> z_)#c3dJzPb~i&}~_zCfK+W z`mwV>)>e~+RA(jCJ(udfxo;-(@lzi?HJ>^#B`(Ub>DG5!KRA5-rK>Mpf92{cH;>F` zvX#ueV@-+du(|1Qwm%ZED-Z}2X z2{@Xjv@Az;DFS=+Fd`&LKVOqJv(d`dWodcR?naBThODn;%|`08H^?&BFmS*{ zRZ=5?+}3T`(E8=oY9)}{-K*p_+90==HiS|Y__*LTdf*I~ox)zl=W@dox1No!CCqdO z{GupVek2fk=C3;jZgSrBj^P9W-qT-VC$y_HholckRkGcS8DJ{H zRfwq}THNOHx5;fTP-})Gs!A&vw#!UE(`4L&)_u^8$v}e5%JJ`lW4g?XtJehYiCaS8 zf?y3@WhjMdjL!;}GHNX{2;%}U^y9f>d0+=!1bP5S15dOY>Z?v4O?S`{hrCU`UcVXr zU5ZZNsuTEeZgwB07|R5#oYNujdjyhD}XngBjp)$veq>zA9W zHs@hdb_fUp-(oydiEp_V-*WT(OlBrBqu&u{F5LO;Z0~`&_`xY|L5@$yuEZ+xra5`j zjjqq-EsLqAZs_wVWmZP0}*+eybRtO#Lx8`S?jbheujrH z9MPL)Cj(f(1YVKUr zuua`5msXuo*sNlId?PFYCJ)g&)dnAzdEul(|K)*a7 zkOp}OQNM?hF93XnsDnD2ZK1!x|EG}P70_3J_)$$OxJC_K&S5*kBt4C!2{-UvgCapK zW$2o`L%tOEKN2~q=W#@ky1tF^(yovv>;_AohYT@pv<$%`U>t>OA#@Xx9Y_!}C_Y1m zpcJgIa#dg67Xst}MIA=t$`D{52pq>f OY%am=dPIN(Z2cEyL0H28 literal 0 HcmV?d00001 diff --git a/tests/__pycache__/test_aggregator.cpython-313-pytest-9.0.2.pyc b/tests/__pycache__/test_aggregator.cpython-313-pytest-9.0.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..38f5468a9b4391d256fd8f84f9968e7d5d1a3473 GIT binary patch literal 6740 zcmeHLU2GG{9iRQMy|!Z<2nEUkG3Mjg!o_jEz}NULh;m0k&8gWMQGAuV*qg-1#%s;4 zf#iA!Ri}zWCy}Z>)z?-kQeP7Fh3B@?RefzA61h35B~l+wRmEG5RH{T(ssEYX*cqEul+fFe5m5>6M(>C?p+t5g zyhV|~CY31IUL^*0s}cvhP3ZyKH^a$^2V^RfGH1u%!W9i`m2Xtb7G#s^6%DI1TIs4* zotd>vTeza?b{K0`1J}x&`v4@0z+Zs><~v|MLPf8x$U^}QQ|y6R0mNx|=mHvbeY*Pc zzON$W_=89mCuRNt8TbL|UFNVvBHj)=Gq%M`Rn?RQOkC`cW#ICxElxLd9bDmsAd^fq zWoI+9#+;T>RczEUS5(tdtFJVys%~a1&9od_QfFqcHlso(1NC`3UYb)cYb9{CG@OK9 zqNKZ{U~ZuG_>T4D_S*}$7Mh3e9B3sEuP3(MzIf|mQ*4%7iS*|op(nh7giv@B+)CP?31hZ~=}Ay&oJ{;#DUN&@J?!xVjp{^z>K!D zGCxkELWH$uu?9Sm=V}7{#XMI(|E>QZ6h|pEgXGZUKb*E3uRl^$)6}q)I&`Eye?m56 z@4AUpo)!(UB1pJJgR(iG>Uvo>OfB}_5#m20o5!g8m~6_i_b^%L<&Z7Z=Ld-yB4&h` zF=9?E$0-~z*@4v8mRN^&&qRH`bg5C*t!mA*!@5ycb<>VPDZyDu#rCd}IcqfZN(pOq zT$<61OR8R~sdJhgEt%>xkW(GoTTar)sv5P^7#rB`D@_}CPPIzfwYsj>R10dBC7ZiS zxXNtkmMvV?0J3CsXGOBX%y2L>8q6GbGEVO~lfWS5226Igy~CNeOOw@#RRyuJj{bnVk4m9Cx=qa7vel2`OO$ScZPv)CoC5Rg|G$m?2wydsRe z5Kx&DCh#^$U_zFZPB5WJ4Ce%rV>YMS+$DSnGPVTZ*UFYw!H0>%%fyhqfMsI7MhrQf zO>%-U5T{a|U|gwA6QZeEcnExVl186^X@hY`K(8lvH-}rv{u^iRCwB&*opYd_yL)Om zzar%vXy+icmdxFg+J382jzHTj3*^>FJEx*0>H_(4Aw(f>x$%zx^4C1bfBZiQdGCv%D?Gmh zkh}Kjf!2e3>^UGGV^C+Y3weNhEdr4LpbheP0MO;aU7BzcyCyn9K8`csj0eHkT$v&t zXGzJ^K_3SfAM`Hjx3wMK)-2XEOg?2(@HhVi<_i$tj{Q3)o8N6EUcT|hi;DhdyJ-|q z#=n95|E2-E1EHzThqcFhT~YR;-!xpTZtsXyd;pfH!#8uB!p^MLW^@f72On}af5C^c z(n{=qK|YkpJ)7R#e+UVS<3T4OY6=aH=_Fhy9m7cb zN=8j6CRkfvIjD(>pokgX_)a18>KV{aJf-AKt^zx`=&2o@nL!B^!Xum#zHoYk9~GVc zl}M3%-_kJE@%I0Oy5f(rgSGTy*jb(u!d8Rr6soTHt#|pYSgKQNP*);~^jx!isckJf z&a&XWD{pq-7g)iLv-4ZQtfa2q-nV8~PoJ~9tWUw`>~5akrA>(rgdNRCi7D|yWRRmd zsR--jguwjvurfae1<1RCv3KfzQSkv4NCWk!XYv673rL``>2lT;Zf%LAi4=+_pc0=X z9^4~7pO?9XOGM*2&XgcX_(~9!e@0mqB;2ZDRs;#RC=0p~Xe=gZMxYVFFO_#RrYtB* zwy0N4i+-%hVf+;sohU_FVupzsCFZy-LfPO#GN4&yL71Y&LvoU8xQ?JC8a3du(1L~y zlhT+ilCkkA@^Je|_ghP#!st{)DjRck6>FHRE8a^CD3l1Nh}lmJp*A^+50W&!Nd|PH zp`aQx63mPRGr3^qgd=k~qER;6mK!z+C3#<>4)69+-mQaFrQs`@S)$KU0KYN#o1cTB zAELi({A%In3rqW#zTN82{Ql4Csb3-fl}wETpf`{%XlN&jy@50y5tK)~@2J1@fR*WJlaG zXDh_ZhOUD!0M4efhjOr@&PM7r%d=W#z*qL2-4vSE4b^gFOrqpb9e`d9==Tae6)|~1 zc$^sj1eM0;j7me-Uc=vm3J(6}n_xBsj^qA@(EDf&9rz3-?xUW+qhlLkkqa*tHV{P3 icmGMEhg(E$_~Di?_v*uK5ir|nmgn|t#sn@;2l_AYxpI{N literal 0 HcmV?d00001 diff --git a/tests/__pycache__/test_scheduler.cpython-313-pytest-9.0.2.pyc b/tests/__pycache__/test_scheduler.cpython-313-pytest-9.0.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ec061c271570ee0c093632ddf482a65baac5ce2c GIT binary patch literal 23154 zcmeHveQ*@lonZHT_KfCBGx|hAG(rLf#OMQI5C&l(5FmoUGmOox?92|DmNZ7{QMyNf z^+OCt2S^(Gae1S&#DOo!=SlVD=eZ$4}mEaYwNwTTedYZsV&ibHUa6sO=M?-HSeyj_Be zyxoEu-j>6jUa#OKFxJDq-cq5I_-%*%y#XQ6TPBo|u>Ej(Z-r3NTPaks)HBX-i_%E# z9p0+>72gs$w<+n``XDM9a_BNi!j?@6z~yNFH!1lIbH8Ak<+N9t0RtWCK*;~ ziLfS87IChkp0fyS9Th3Bqm;i$MWh7dpQDAf&$z<-^mWSqu>Zlhh-($4bX{2Y2ykv4 zT*D*4d311%j{xV@!L5G;IG+w~!y~|z>fkm$0-Rq57kmV`fDUfckU3a6`JWE)X>Le} zr;{n3O><&eWCv3Mn;zy^abTEBjE`^vJ1!>qA@)dW;M8#;&WmFyAsuo$j}LPqmxmF? z0UBW91L^Vj2zw?aoZ^Jg*aRCFS#kX2n2;LaM3EEOB+n+r)JQzdB>;4Ej7=m3vB`NR znI2{Z4nWf>VPKd|3-Kh6l|rk!k3~alUy3~)7t^755{eMhaURNJd5%kPiIDRiPPrg0 zmBe{Il_q@=`ylL%pXP-45EnbcC5ML7B1D;cQzwHa$v*of~Khc{KBHh1{N_#ED>wO z)?wr6C>^%yi;(?MN?C#|)o)kQBlZ?5O7Ei|qZ_Co({~RGy@#8^y)tlyo-g|k2mWsI zJA3Z25DMBPn>ZjO$I_C6A0LhJ&`(ja38`^DAxb49se$-NOiXiQqGTEoB63KzIkZuX)-3n~3trzsS;az$cOg)I-);BV&K+FzQC9bBM_)dA;m|Eh-Q5z;^qvb< z(|hJEt3Cx-YCs)z2*7utUd3-y4_{>+#TI-YWmDo!{rOFnqE48U?T|P3tKTR}D_dZ<_n8E{eBC&0$8#uf4d0a2C`&=fUIjvXE4?gk2}_-5Yd1Ae4Au7y_;+kK57I%~ zzxdA%Z+9v{=D zq|genn=$7FaPGl8{EVSW>%Q+jcj&IG{K7!i)p&0IXEv&|;zII{kDanCl-K3T*JsMt z&y~$3=AN2;;p)k3c}LFM0ZEMee93f4j#-sqR^4^^E+l5#=Pa|QXIEbhUrXHFo^eHf zV>UUQ_pMav+C>jl;yd3o-Sm1WXQ`UER4tVF=1XekEj9POlyB#K7iF#ftym9@dS%yM zFa6i_-V)o7yDZ?nVW#)GEjKI-`rS0R1CUF7a428J`QHw6uz3ChA!Le}J4`&K&3|nV zg85?6&)jiXS7ZJ<*o%zzE@hVSW|)NrD??4UL@i|WEY|MkiVRVl)0;XVi2TL6_k@f6QQWLKMHo@(lJYif` zMNd5igj^o;M4Ve7HBu6B^;ap`!mAXY_O4gSvd1AZQBzO{E|8+9o{FkR=tV6z}f31>H2ai*{=mGc=rDw{UqhTJJ!Z2RNkMh-7YRJM+2k5?M zkwFm6yp6XG;$C-m?TV0t)Nb<{YLF&7qI+`pQJ$0cHI{6IZFvGm46#qklm#mer^ZJT z?7%SYS?r*Y8U;EfnNG$>Lig~a+(UNfib+7~SO|dAC&2ecqQyZr%orf+p5v2^N_mN7 z0=C+;Qeb=#5U@%WP5PXjaUiFJG~2jwY@#J7N@jS$R!OQ)1Oo({lFdx=#YR%`1ftm3 zXbR~$$w_c71h^5&i5o8$BWT~61d2?QtOFw{9*85@%wwc&l9do;l9l@w5OlNduuK*$BlUN>wp=gkl0WCd7tDQYV29;p3xF zPfUytau_Xnx=1NNeeD;7lpqC^(z&x^BXOP(cF@TYF3yjS1zkc22NO9mp&1;>p6{b% zmHS4YlIYWt6TbtHJkU)biMTVeS3oL5GN0lmq!Mhf+>KVz1<^8s(4&bO(qE*4M>u&* zVgsqsu@Mg9g)o40!@t-DADW;H8&wjRSv%*?IGfIO-(B4_7s;+}%~iH$EM@2RFIv3T z+68}Q&c89^-er$ zQS+g<)3e{Y5Pz%ka^;^?zcG?&J~ZXbl=ptTYRy~Cmz(DrbDMT%Hto!A+I5qjX`Wxz zn=9*`axB`YK-HA}7lGAN-FE|3mmYuP@tMiXFI{!~(31=7$p-c;tg5?U|GBUHzMU$o z`wc@uW0*bgo3BE*=r_;onL4A!WeP%aG|BNXQ<`I{Gfeeursk7%R8`w=KXp-}P}jB@zZ)Gx(vz}US~-BU&X3B7-Z{q$@u!?-N?0|BBor%~Z_H7n9 zmbXVOuwC1gHu6lAF+x_nBf_MvM4dYiQdH`%6jYLUA5h5(2rJ@yP4t9xzG|l(rExr$HH`DHRIPT#2qH#<^8gx?)bJ(wc}X zWG2Yd6>&#AQ3aK{`&C4!D{H;dAMFi2b1P*jZw{nkyJ_CBn;v@~P-;-it@xDDQ&H+` z1hK9sh;-Tf1&Gu;dGH8ui%Hlw6)u5&iW8E9z&n8KU?i0q1BN2Mt+H`G0TO}qxWJ3R z`i}rUBzrgr3Gzn3RXb0yC{X5g{-9dVC3*ySQ zpy+%)?_qZ<=rbe^$HzG53+!$=Y8ztI)Ob2*7PdnXh+W&z**59X;oa>r-rXK7*U+ud zf%Wc02j`lAqa!2zPVh-yf{QhrE9}8o2%Mm&2qg-=m;}rr)FER)T9D|o!oR>6g7@U^wCV%x`oLB_dVEYD z*dcnL6g{qFS(&LSBiM2@|EXnsDGEu*-4Flb_OA#{`pR=YHsfQnzWOQauYe;HFHK&Y zyi-~~)wQ5t#O$iAxw5U_-GARq`PVHnl*5x_R%e*iGu$nv;a9HG^M}8Cc!r*7ojEyM z54_zJ{c}&?Lf7lVQ|1M)|NO-C#5}WZ<^XXr-gP-9|NDqpH#MP#WY2Ag6V}&%JZHIpbu=Ev}(`>##-7Dg@LMUuNYb?(uFS)AR%dV42*+2GT)E%+n4ChOurC|fw z7Uq}r!TFW=&O^_xfPQwZI6|dL8&}Nve>0Y4c{=~AAInnMMPN69@w6Dx`g-mUxB--f z7d;A|%5wvjJ{q_IbKhTpYwGh7o?uz10-m6JaxV(kWQ8Hn_Mp`}$$>Bi)el7Y6k=5+ z9WVzx2$jYm4-vzO68!>5W^E0cNEIn`Y1ssD!i{+M}!cF#xv$k93*~T1dSD zkF}++HmLlu-|5+w9WvXp18Qknq898eYHGCJK@C^TTB$-BiVjvR&G!&)$6YXD!U(nr zxf|gR(OI6K!6`&^Hj?F3a%4o-;XI3xgwyda$>|7j$fRds+61mBCeO#nAD}~@cZV## z%u54X3qo53v8bGFdVXlTK2T6-i-+DZb@DMAcy6|*l03AroI4M7ln0- zZ-A`$VI+$cD@Ya#SSAn*R@PohT};h#b5CX~nsdHp5IzQ~a)I@k!1{S6G#^->4QvF@ z_r&?Y#;i98-nl11=f3@R}K&A-7a)Z1G86?VQO*=n_<{nOub=`KgK~>m+`I3`WmOKcfI}# z+g_iXXX<94r{GM#lw<1TUzVwxnp8uw=QdM^{mkdc6olmHJ4_w+Ov|B!08}XsN}hr4 z^!q}>g@R!re6ThEKQt@cWsWz|2~4`7v#__%qe@bS}b^HX*Ij* zxy`I4klX?zTH5l72b>4E(IYT7U#UK}6S&c1wZM&n@0~8|u}13pM*7$q z%k^N-CJ5XN(8ua6H_HxehroP^A9BsFq0wJY+(!Sg?dJLIW{59fP(eo|GpPRv0)u}J zn#<+1mS7J+hlxoWv|9IokQRu4U|p?7Y^aeQQbVstfC)q~t&YGjB#t!#Q`S2KBLX0NhYYm!MY7Ml4 zrTW&%nlwTdEL&OuS-^}V8`CwSm=cL#a1c+h|Lu?7V8i1;1dMV-Lllxlen=I{n19mQ zmkMo?H&f89SwS^5Ff41@+)E=X71sz8P?BVm^;=*y6HkK8lH}4?DIjK(agfar2gR`n z3$aze87y!6@>X9?wtAH7$t@IP*el_CknIil7q5e(-{wm*&do*&BHve6VOAli~2U0JGv%=>|xl+iyvhl1)CPnQ(n>BEZelew;VTtoy|W*J;zK(KuC)ifcNnp-mBS5L{&za zFOArEf3!5}-wETNQU#TXg@^bWZX%}iAhb@b;_a_j+8S<9eA-(RoT`FK?YkJT-Rd6~^yucLG3~Ap-jl*8>bkg#+ecTkLmFc0q zmBj(X4?>YAqMBfGvSE&?Z_r2Jg1^1psueGx5`t#QJTRJ&>_|6&R!?#w zR4bTp@Vtn{v+Ow;K|2!@VB-SH=0G(?@T)R^RonoegFEt@Ezd--;GUf3v{h9 z>E~x!r(Xn~Kj&@8cpH%Uw-HA#0r&_d01Yx1ev1iBy{Kc@^U=t#+w^?8><3pw z&2Kl~s^5Am(Ed^V*2@huYro%>soy%!bYueUD0^tndbdI78Zf>c^WJUO=!~}`$K-z> z<>S-{*>jueK*l_uDWioT0ZNopvIJK3Ujr6dD_Hj5VLA#};?2ks*Mn~Mx4#Uuf1ywM zQp6ebYIW}#`Umu(PRpO#!1r#~_FgA-qlWGU>!sRm1_D2A@bp@2KW*!N90LDkiyw0R zjHbc=Gm8yfr@yz>{Igm!#1BA$>Z2-bcfrnKO!G!89p*w?Ua;`BcA(v=?W?!sU7jC) zL^CE-?y727wa?Uv{W{9Epbxx7HDjW~s`5jFEK>0Xl$3{5@oKaHuJxh^C|Nt9otOd@ zuR~{iv#JOLg3D4)hP0(-ex*&fN>1ZX5@C!yN zUR3WEs(4qjZMMl&A8}tJ^j@P@i{jJBr~>JVXR@M)OLUzclJzKCRdZ%yR;6iE!2Ai> zNzG_ENRx;@LZ=9)FpXeoVuL%1!Ed1>fTOoTUfw85IIEV^g}y&TrwH#C(FcSKa4$&) z2q&=!g6#>SC#d|}8217?FQW4jIHDVz0t7B&Zy&+l>0yD(D|>{00~xO)#It~=H4)8| zv){UUGUM#X3u&%(Wmb3QDm(LbzK(~))DLC5WTVQezuuC0J+AzulB(JRLS-~@0;?=d zzVxNI6NdSf>Vvf)J~~K)_y~OObgepQr>^g&4_a*3JNLIi;6JzcA>pQl2LDaF|6sNG zX0;h&3NR*0P=N5GKm8Tp&SH$ItPMTBRO4vvI`?)6JaJJiICFI;+7DZuR%)! z)B(#uHZR9EkU<+62lKXm6?29SIPcUMqZ27SvjWuRumKL@ogk5VAI{gDQpPxS6C^Y0 zPMi801v7!U$rU_f14c#aP9KJz6+Z^JTHVc!r6O9p4RN&wZ7d^E$4Y@lT&@kwo zYU|?v!7kwX{VpJXDI)_Z_;SDB1t{P(9%bkM{VqU-zk*%hfaS=#{GlT{yhYnh6rVzo zs77Vq1J1_M*9)A@3i%5-8~0>a{zOUsfVzAk#mEGRvEgb9S?x;hr2rweb|eyG;HC+p zw9Q+SvCuLBnYum!H)sJ`Pfa@10wzDun{R=2lx#!R3ckFQ)j;qH0#~EMCu8ySu=9lw zcNVUe0Ku+o-=%`LYJyu3yvp$4$=70F*amiNp?s0grINf-AXtty5lJr^o5`kYk|_md zXF@8K4mxGCu|L4BUP7lIooCRQL1zpdLlgUFF1Py zA_Q(ipA8+H0kU-$gVlrGlOW7O9Fz-#A=%`yz)Lrhyp9(n{Na9t?LkKq7$1UQ&?_^9 z&6q?uQpDM$7U6aDy^T(hfEO7;S-_h=36r<^%7?ZVq-`q@1#cHoURA(8DmGnJgn^W& zAgCF}(5h=NIRb}8|2=?Y;UBKw&@qy)Zvqa|x4Q6}fklhQT6?#wdS>mvItVKK>iS&u zrcCvwx#L$Gu2#(LzE+>D-kl5VM(sYFW0^jjbJb;Bb$5MbGYzxpxqWkw%|3h0|C5HB zXEMI0;i3Vz=f0IHU%lv}K(0G7J@PuAV`}G_T4E&dCK?F@Dlc_h?8pTgGl9mbeRl)p zm$qNrKJVIlsqzY985h3#(P*08R-3-0TJwZnjCyqr%Ol)i{7tLI8KQ zUxkAfYyJls4M|YHIu1SZIP@sv&?ApS&(+wxt4AJ({ouoK*q;sbfcH+IC+8~oWn4Wt z4*57;=r#btIAB5+%TV*ko?9-cR*eHVB?NFd4tY2nhl2V54tuWEr-T6RYD|TL7Hj?o z8Z9Tm@!)cH(Wz(FKdGk5IzNG^)nCB<6$M=UUYO&ru#pz}o%N9#>birDl-sU5yDSj6 z;h`f{mK)w<+aQp<QL#IgS1*;@0>hTg=%lW{5A~=5bpg+d1*-nvM(iYC2^{nN8Sv#lia0tyPJ3(cb`bULZ0&X1X zD^*r*#M^_c-5K?O%}dFNKh>-hbTBG4r?nj@U6ntCJ1|%!#prO10jICE$B^sO`mT(j z(jQ{apg|k9>_KSY0$nTiN9AHy%&GB%hisWGjo+Pf&F zrhu%-orY`0M)GHviERYjBL;UPrHA8bR=#VD9gPpcy%XbLogiPzA;OJmeQ=>ioCOVa z3hcW@Jk9_G4RLUn8M#J{-6&4*a1uJ59AM$btvK2>;K|_BXt0SrIi6-AcXAMD2`3(x zR zWWh^G;My@$sD)_gMWCC2=#mBXj$JT%a5;$3DQzMomu^VmQnMj21klVg)V%)-N)R`I zgM$3|iUaR^uC`t}bMeggCoa8s@x^TAhFoP+rm`tl8Je#Qz3-7NHD1}VQ?}H&_rKv8 z`tr*CkB*!?wg<12i=Tv><$h;ci-*kn(dGh?i5BLkxq#6*2(ot-61m|q(qoe||-d zB{HpW*n||YRUzYBp|<5UHvD2!y)FCQ3HBC&l(1-B{p0bO;HLBtkI> zUuvk+IgB~u;7Bfcb-@uFVN{gWJU_(5SJC0nc^;hysQMP74Jsht+X_PMA|lZuVqTeM zl}pSm;Re9`5BL{pn3Y5=1lM>iclC>JIh7Ecz9=}L@o{DKlnrt~IlkJFZ5c+~DxLi}KaZbWOjsNWq={>gHW`pgQRsh3FPeT@7E%JH&e z@qt*gj~s<3L*2cpA^Q%1zPZf|F|f(m^KtSe74|qFtbt@oX5G!=gh;k1`BDoXxqB}Z9~u(4 zp*Y+iC0MXpha3%p1JWh9fe-KC3=tPE8s@Mq|I z4|CcXhI zD#)oTVv|pdvxw@KuD*j7w7tJ(H-#1xk%dcG& zz3v|<>O0h{o?lRoUr^3pQtn?+uK!Lo{S8(9x76mpr8++@ai8Blz5V>I>0L9Xx9peg zXrnd9WUE@Qw*5uN4?C_6W=r;;V-~!n=g&-^Isel1OEbb-XD*+4>)V&VJ$LZxbJ;ar zS?}I+uFrf78{CEtJGBREk0J1>QPg{w^(14x_q0f^RV|# J;3J>w{{T+77WV)E literal 0 HcmV?d00001 diff --git a/tests/__pycache__/test_trace.cpython-313-pytest-9.0.2.pyc b/tests/__pycache__/test_trace.cpython-313-pytest-9.0.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..23a80b60c558e26acd2d999016842a1d2a0b9cfb GIT binary patch literal 35130 zcmeHw32+?OnO@J`a{x0KT)Z_TzyksZ61*T%q-04v1j2%BG)JNZ$x}mM0FF4Efx8DF z30*X$ia=WlDa*EoEAI+cvL(vdIC8QTE4EXSwtUIarZ5;#fZJ4-@yeCrRwV(Y-K?bA zs^t4$cfal#4grA?S=t1R!JGe`{raEo_y6y|`ooF}pMc}%KYS+XUnvNGO&Q~GYk=*; zPaJ~q9YGRgL9$8qPTPLFY>yD^=yb?V_U+p5R#3LzBa3>CSN5_RpX_7be%a5y19AZ0 z&d!Scm2xF(>*@^d56K}0yF07)hvo48YPp)_J)JfCYvo!7i=B1*>*e~0@UX9YqcKVI zyL+<<8_+dcZZ+!7?}wLewP`u@xy>v$Aj!KGSfE2*evMIgY4z6h8ZFDsbh+f)YS;Q2 zGm&puAxQNr1mjOyB>9&Jk2)W<$&rWs-FI8hq`}?0uP&1UO9bPB<%W6mtFZKIoJYS( zOTX3g=ohr~TQiS-Axpot^XOM)>9=kk{lb=hP4nniZRvOGJo?pG`Zdp^U#+EI%RKtk zS^BLXa5OC*`R4}`N>+*VXXHrsa3YeGl8dXF5C<7s6uBWE!V;NFDXLZhc6!wEUr zpG+vecnX~l%ZWrJGn5@1%0`Z6`VvZHP|oxX^(G>{8Cg#BW+Q#^Y`oQXneO~1o9d6J z)0r%rkkW;`V}IrlzA6S1>4Y55CSr%PM^jC9)!Bu?RIk)~IMFwhO2}yA)}}$NizcW# zAI%(6M#Q7(q>^nNJkjg2JY{~sPYZ(W!&mV1HV6@{ls&w3VeoOACuTt1-mpsN=)FDXXVXS<^w(mH)Tmd2Nk9xbqW)B1E zt~DU@yQkiOqds%ws9*9*zU?k!MiJq@PC@Vka~OHC(Zl>UuCa6`to>56odFr`(}8H9 zd%00+ejD}Ox0q!HWYncAEVvVlnF4oT2JR}OjzgOSA?iG|K@bkUFZtJ_`;LQx5MPVZ zve=0_=!fo(80Rs+yBkc{fQ(tND_m`qn%@Wi%Qb(Uiu)uDmj}Na6}LDV1yShw9sYCp z54Ah`ZB2oZ+wV=J1{3m*h&+_WGm6KvH_;ctEgp%d`|!wShvYPS&K2X?z!S<^$*r&1 zRA+xOl^A%e=7;&P>AUXEa1O>Wh#^vcP>tm;h<9gU^wIaS?qW+>gKsQy%@H=c?q z*~Flt+EcRXI~33M9*!x=kpw+;+R&==C>Djwbr`Nobz$M?JJIA)omeD}sLsO~C967V zu~418nRFVb!y`VVTqc^`RIS>R8C8_BGFF+6`(=7?WG50ea9>x4bRZ_()7|l<1G1OO z>G{I=@$^7K_0U*YLR6PN&kWjVah2Y7K zi-Fp+z2_gu2ezEt^M0uIZ1UYuWYjrbyJD(#O}=)``I_^6=N}k*;`Kv?+U--p?Whtf z&iGIJr^F?BammF%=xpEEmh;ZB;j!hfcfZm1>-Ky=`d<#a*LTGwRIQu|2>#HSmeVcY zY@Kp08h0+bR9QV$*_f|vEL5&JxoJs_{vjv zHSY=9-db}`J>(FCL0j6s0>q>oVX77$a47Z-f?~%)S1AaY z&E516l^kEP--o-Hmg}@L?T9+N^=*Q+S=uGJGKp0})NL%CX^&*fJR22v+>3F~t&8q=50k{+j;cNo3CmgN87w(4 zQ>QUgV{UP*M3!Dreie3+!3l zeUeY|M|~S@Ea&f`WrrP>v1mOmIdCO@V=Uu4fi2xGTiS(xH~u}lY=gOL-W!31d!rbo z3-^YG53Cy%3-`vty>ShiQhC+A5o9#w&*n{2LX4(VEzp$WdyR5fi~cdvl<-yeM)d-} zVNhoaJulD4{J+g_)LeCMXzRuTFN){=|C4*eKAUJMA`Cc1VV}F}1yW)t_Ozp4SSdv3 z=+iIka@->H+n89Tc4X@TCd7nTGL?c%L17{kIXQ4R8-YyZP(o(nl_N=rL?W5~NMbmV z&MK{!NkVv;q(iUSO+i7I>Sw|O2u{P#wMk;7xI^^?}>HPWJ{CMi_-v+ohdEtm;&`I> zGLy{URH;-`1rtxGZVizncfo{Q?3z{I7Y0v|bR!m*)v6ezPpO!k7?gW6M+f6_LJgog zc4#P>$|lp0Q!)9JqArRlhciQ|KGrlgkjfl_)G8f63Mp|+iT5XH1hw)mh7*Xp_sDWa zR;y!>iXV+nogr6BCF1F!!KQ%R281wqj!Y^m)t(rZ?;yZWQ_%3+X3cE2 zX0}>0w^=jo+BInBZrfOt+oaIu*eV5*NusDOV;IRMj%Oj!8i;9lizSXGv)M$SdFoy$({IlFzv9&jz2ZfJUS*DJdUp|)3c<+@+2 z|91WNca6HnLv7Q+nlq1`er#N9&Fw-m`dDsPA=m`y{Fhmh54KK;`e#CH9S=6~tOi^_ zry*XJH*-|+m{w7s(K*P+K=Ju6^R8Ba&#EPN`?%PGS>0X;t_5_yjV1YD%ao{pCd8KU z;98#5fD7n^S?NvkW{xT_E2=1n7@dQB3>2Sl<6W)5=W|vwq7bUpe#5o7ZR6rvU~5|; zxC+o%oF#cekN%ku*Nz8Q@vH`1Kqp{JZ<052Q~_*JMM1>q9OPr5cr4DlS^lR3^xt2h(n`_MYj0EVfbaAL5&g%o6y{{4a83zQIC^Q6^J){(1hbaPt7{ZxJ zMBO`{!8wl^BkZ&FLD<^Eg_PY~1Zj-G#Fb*y197GI-~$$MrLTtzDZ9&x9}SHU6IVhn zWjRY(hTUY^O0P4%q^Ng|IX44ZWnuY@-Vk3x6gaP0l2FyeMcG-J7&UOngsNt*va>UL zVOD<2Im)*-g{pIn%Qrbvpw-Dlo}yGCRYpZ!0O~dMm6o$EGVo@8V~gxHFlT&207`;S z2tWsiC0H^Q|4f-wpL-N0{SCx>tStcg$lS&I~e;aiiJaJnu*C3$Sp*CiKYA!wd2ON}`lJK*@uYz)(PtWlFw+1lyS6EdoP~d>B<$et(nvNJ+^S zty7eC+j#@DVm z>}%YAFW4h)V~5~&VTZ7~`PD~Wd8AO^`N|_X`Nso48u;;%A02sf?bNpYFWbNW$Y@}^ zzVl)*e0FCc7#SBE$H0gq$p;&!MEz3`8^?nYp4EU0V&iiPFUuF@wBmQg#?Ybk`=$Hn&N6x>LKAb}Tx?NegWCr|af%#ynxwxgR?QviHb z&k3=;h(D_W=05&F6c&Ba_7RxxA`Xq7y;Gr%4YqIDI>OEuJb+%R>{uea*3ePwd@JOp zv~FKBO5V1GQTcYQi{K^Ujuyw;Ee@2FXVf0T)2%aVID>YS(+Hb-)U;qRRf6d!OqFcV zB9~*mn3)n8Ai;b^=;rK|aUQ`Wi+0JK*&MYK)05e4kfy#rEn+OnGg+Rf6HJzPaFfYo zS;!4%w#;*aWIGH(A$fZ^6INbRTf~kPGdkso9yhNtuA+p|DL*6L3xpLWQLT1H1YMF} z3PfExQ{tWnN#qQUait)M6<|oVNR}wk~#D!B!;ar-Rv?w4va@&d;_SZE^mVnQ4Pdp;_zX~r0UZ&2Nu1tnyRkJ9i*kQDI%TQweFYuAW( zlg`*EAWdWq%RZVe2Mrkwg`!WkL8UCT|HAg@MQZVk`ni*mdejXqXLKzeF# z)86K5Ed5VnmsEa_K8L{&F@zF*6uL0&5u+}) zSH8!xjGKMvmMb~1jB`UB$!A&e%aVz!mT`0J@(VI25yn-pW!&slenz|(mKW@z^<~^G z1*D3oTVKXKJ!AyPjvjNo^1Eg(yRaQggJpaH&%ogg$8&H2&vdKaa6E?=@O)G7 zT(yAbn}X-?0-kRQo~sw|d^7M2Nh-1IC5ORP=|&T-u3Ba)F>A{r`Dx9_x_$9E_oLBp`iO~+G3*4pU6Hyo6Zsbs#ypKk3!TACED8FW2;2NN*e%hT56Y0GsmWuSw z(oCMEWQdYcN=_qDh@sT?Ho7ow3ESMTlg19km1HkWK;)rbl7GtQ&^k-q-htedOXA9? z==~7D1#8bde){pTRilrO2Uo+HBDiQ=jO2FbNv5WMCdA0-2`x8ig1j1Kx!pXMFUs-a zf*84>LoXQ@SLOQhq-54V6XGh;B2#YC1bH>ea(z6PFUs-ag1BlrxOiM#nLCgVuACC} z&xE*gbVSQdnjo)6S?&PO<%@E>xFD{)fkR&k){cwIb2X=*oD!F7p9yjK=&%M(njo)6 zS+0iX@PMRREMp+IbRK0P&D94Kn;))`1 z=VDSve>AhGSkSo^clD{t1G}J#KCl|9Xh7$r+YhW0CTqfgC)b4!>~Ku(Z~&&AJwHe` zfXCeSVGi5V&w5)Mb##+mR>@&3fAE%Za&H+dercx!sU&TJnW`Bsn%Lf&dFV}|PN<&Y z63JB0uC&v1^)x+0(r(F{8IF2({264TqqN%50o5}wWGa2H;VA>BPHdfYF7V52;*S&~H7JP()U8uMZ{%v>$Z zXIEF$tL*H|-vP6#u$-fOYc6k{vnAisOO?9n*=y-jR;DZkqYiS#HLxs&NJA)9MZLO6 z!Do>So7gC?uI5TvV>C#H$zKO5>%nJ1=7G`D!?@3NGH>hpV_jkD)NLbY?NdbX!$Y@X z^anL{shl~cz($gheA6}j5L9|cl*qc@e|NO0m8lCaLr^8i8r9!K;6Fv8x;3gzu51F@ zYO2)^WS}kG_$jrenoS|prA%d~2DBk#A2FkXOs*Yyb|(>r`4A>SPq zH6qlf@==;WrKT5r5is0B2UE$cHsk;bEDS`6l$0c=uu;@RpSnfhQDwlQD`aXye>t8) zwWZ`gq0t#zK^j7Nn37|Z97m#%r?|mj;9C=*LNE=}NR7vU$>TAF{5_eAeoVVe`4x2g zC)9tvRp$pzL(Z?>KXuD~L)B+@m*F%T*1me@_wSr+*ixvEPBd)E#orL$yzAdrykz_J z>T&77+qb@K%QtKx4Pf+rt^w>CyK7uTk`Jz)61%idLF{50J3XrvUJ$!T@urvMi*j1= zyJ8pV{LB_c4qf?p7<=q4-Uz22Z`${3=bOXhYm3gNwLoFerN?|W3`xJYfj|% z-+M7~G)^WoMf*Hztf{~H0CmaFQJ&WRz4G%&56Is}s;&Rud(pb_qs5kO{6?1W<-3pU zbo7%z9kwGfr!l zZSudMe$P_EmhDl3*lw`wVy#(iTNi7(sF+5yKJoA z|HM?NL;FmGI?fJh;G_xiYLtyRc`jd+L)YPUKWuibPDVCYI z^Awd~CDlZ*jB1YKo3k!5?r!hh++PLY3r4zE$C!`5wyPVLGVB9_~o z{G$~6-AIX|)mfyPk=1Z9SzQZ^p9*%g+Mc&{)H%Q90rW@rHg_x*-gbonzr8rz(d>A; z*#Wp0zY9Nzp=YG@Iaphi-?UuA<|LeUtPpxQ8I6M2t_O2uEItsOalzcdMk5v$qN$l| zc$v*aroWunEOQKx9Tygvn-hrKXmbOyJ+CbU3DEe)p}cXSIZNYu_*P;`QSY*1p{sji zOX0DcvurOj5YL<~O;OeSgY3+4?T3Im7aTvT}YM zhM?@Y#fyW_X*OR23b2#;Y1Bts<;*+AH8aa7a~`zcDLJt1^+yBP_683A#Io%*`;`BU zO@E5{i&M?9%Wg!?&7*M@Y};%0DnBE>3n)eCsODR{0~!1GPPbL|42Zwj947Vvyi@La!u z=bM3NsHIlYd5Vt`)wHK;8N?oQv@B9cFB)mP_rQH$M66>Lf864pLlKaOunPKyR~r69!-(2jD&W>_mx7orxkL$N%S>U zX*P2Xn5@aPLzn**5<~{VH6cz9yCjPyg{fGg)QqWExH?63W)OuLp`W4W(0pl`ekG`i z*`tFo8mv?o{^vBv4=L%P#PodCewA%j@7UYq*Qrf8{Y#nX$(q7kGs+`uQfweZp^h2R zLfVo;>{|zDA$S^%sJx0liBa@uE*&T>YHP%PhGI=m&c(YFwZpcpD9v6j#GP$rua$ zj2!yGMv&L=*6)Yw&JMrOG8JyfhZ_pv)uWzi^44u0+m{bEPl@_xLTqMgb;?d!VBP?Q zWBYh6UzFp;1+f_>i)5l$UuvQV<2Eu_Bojp)U44Ga+smCx2tgPFi5z0EOp0JeM!Z@#2EGVLAvS#nrh1X1k!HaRT^}?m?b|i?k)ji{a^fDOgXY zmF=abmB%$BNiwa}L2flkrj_kHN2Zl#ju$i2O4yn<b%SY8%7 zivh>g@=(v*^3aH2F)(x*5|ddSvR#sx>5saYyo$%Y`BVoE0R~4y~_Ll@m`RlnCi*TJW9;qzy=Dw9fV1Y zhU=eZju`Xo!<{09GVka$@1@;Qw^0X~G%N<@@_LlKQ9opdRR@RW@mrWu-_Pf75jIR_ zq-u$qzhwTVoZ(`WO9^4p;x};D2W8XTn0v4Sn5%sd?)US+9gjrwIlYhTble4{dTCKK zsLNeLQD0f?g^ZN~>xcQg(5o#<4l*f{9ESE~qrid}R7s1aCDAJV9AS%W*Bq<7Y&UG2 z*Z78Pmm(_BzYqTiDWCR3wf~pk^r2-bJ&YBC$M^XYwkMdiWzn8s>Bt)Eo`8&olnApQ z(oAibJ3W|Ot4~U)trCp3g%M9fqJIkxlke6z&SHhA>fU=-#~1hPRvloSVZ6$m_9?=T z?zB%lzn%H%GuD2@L&mMxP{{up=Ro@!=H!nkpGJo^idlo#8X?ZlMwV_G5m_7TJy?EA zaXe3q#fsaTLYm#`zs0~B*Z%Ja`Y|Q5TS0M-U0#jGYNcVsdMJ}gWd^nIHr4VQG*BBQ zwUlh6gfZ;b;=?s2ei-enaRRn$F)lTZzO7CE3DvVeo|@@w+vcL&)}q|DqTFptH9A|& zVhvE^_E$0evz6P|jAX5LSHHmkllWI&D5Hrk8x%*pn*A4R&-R1yxd=8p`@*d+Zg_q} zA>1(PxkRoyH->%st+;LU__(-DXZyB|C6GCPEWZY^81>HuYNF*PO^{ckjJQ6%alRE$rwCiH{TpEm2I!^z z&i$K(-`o-g{F}|;PM709xg3DYGipD>cCI|5_E{>euZu}@Uni61VN9BMolM$iA2o-w zTLzK-R8ezyFED9l)EtpT&H1cC`BAuq0oW{GA2YmwXRM6H2bhIxsMZGuK+u;Kx2Bwh zpi^vaTi2g6f<9}@B7$Byvd&7-lj+{n5WPN-y}^(o^RNKO6raap>!dlH$c&JCS%dXx zFgq!i&tSTS_s9PY6=h;B-~0)tLcEnh!we^nt7TT~awzn({0`dx0sqPhEA=WPv+{gu zqIE-sAUBL27SWL^ZO0!fx8vWp3_JdPPV(niW;?fc$v%(p%gtfHzw(6lEq45Bu>)`~ zP|6R!y}E0<7jF~h zY?jHkAu75p^QTcCdxay9U9Cq&hqH`!TqDw3NS8@IWXjc~@VsaVS=O3oDJ#G20cNzr zkRch@$z0RT-lEaY{6>RT*dtp}9bGGK2q&Zu1 z8%?Q3cTEpSwNhO)pv&+o<`KEB!oa)nt-G0%tH@8_!Z2%$ZT1QtXV=rZo4KgFna|p% zh~cjvX_SVJLU4B?l8qliSk1$6@-)|863(NjSmk#+@cSd?HdAa{c>J>{E(swc{~MGI z5l{D@za2^AjV}ljgSU*QG*88yDF5^h==wW&xC6S6Vn|!u&*c=IpShH4p{7-jqFvXg zmdCw}5$LtRcqOyyNh*EGfutUwn(Z+vOXYFwil3#^4pA~n3ArT7-=}1pl5*bV^c0%= z4uUiZT9`@D^hw9qk0Fz^>?8au--BKKYhojD?)~`E@U^XE#O*nJnLx)gNIlX+v%MYG=@TISeeeK-C zQ_I`(%iE@wZ~l9#%Ga-(aR}b(83A4KqUccd*&W|}3X;q*sBDo~wZYKjjv~&CsU8M= zL2Mw;ep3Z($!W#!iVbIXSX&r5bTx*<$Yap|7LAN7ii`Go%h_8|J_uo|AK}%}MNzjw z2jRiry%9UU?rQA#0G;dH1R2FU{>9yO&Ue-|BK^ydo6@@7EvT5VhtX)F&PDLjaQAx0 z#CiuxV5URvO7csT+=is9tI5s6{eJ?7;B4gaNLL0<%jDE8UqC|UJ@oeVk+s>O!BnD0 zGG4&mLS{C-3ArV5AL3*%lN&Nad8DaT^~7R*nci4T^~GZSLlh?}7L#e~sEWoLu9(e@ zx;Uo26RTAdp(DOltDK;)#CX3IFH${vnYPrB`b0D0aciAv*^_$@>w&ad@dk4JMe8iE z76zngF)e}`)QWh-Et$M?wAZiuwW=(;CL+da4XqeTE4>1qf>CI%Jl9^>PPzis`DiAY zRy}=5nZ1xp#I+)DGOpT<7kZ&o75CqNw-npm(XD#ohZJTcM3!TkL6VcSIjB%hqpX3C-^bE8Y`YFCi*% z`|0*GJ5TS-*BrlDwEd0#f`89RaauDDI`h=&r*iVk$Icyl z`SEj)pTGC@M+>*yRS4dFGJvr0N!wA|jANy(aeCVqwC|n-`@ZCA`8yZNXS!{IU3_-w z*OyM&YVx+4EB2sm86IhS?F>OPjU}}{UJGbuNlC3}TdDU2G!rSQt>k?H!Gfaryk1@l z2!WuAYs0)2(9EqRwSHa;Xl6}GZ8fh2G}BU2Tfu7qT{Y+8>n+7bNbh^a4oLp*___ZN D_9yDj literal 0 HcmV?d00001 diff --git a/tests/mock_transport.py b/tests/mock_transport.py new file mode 100644 index 0000000..dca8d55 --- /dev/null +++ b/tests/mock_transport.py @@ -0,0 +1,85 @@ +"""Mock transport that runs workers locally in isolated temp directories. + +Each MockConnection gets its own temp dir standing in for /workspace/. +push/pull copy files into/out of that dir. exec runs commands as local +subprocesses with /workspace paths rewritten to the temp dir. + +Supports injecting failures: call conn.kill() to simulate a node dying. +""" + +from __future__ import annotations + +import asyncio +import os +import shutil +import tempfile + + +class MockConnection: + def __init__(self, root_dir: str): + self.root_dir = root_dir + self._alive = True + + async def exec(self, cmd: str, timeout: float = 30.0) -> tuple[int, str]: + if not self._alive: + raise ConnectionError("node is dead") + + cmd = cmd.replace("/workspace", self.root_dir) + + proc = await asyncio.create_subprocess_shell( + cmd, + cwd=self.root_dir, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, stderr = await asyncio.wait_for( + proc.communicate(), timeout=timeout + ) + return proc.returncode, (stdout + stderr).decode() + except asyncio.TimeoutError: + proc.kill() + return 1, "timeout" + + async def push(self, local_path: str, remote_path: str) -> None: + if not self._alive: + raise ConnectionError("node is dead") + dest = remote_path.replace("/workspace", self.root_dir) + os.makedirs(os.path.dirname(dest) or self.root_dir, exist_ok=True) + shutil.copy2(local_path, dest) + + async def pull(self, remote_path: str, local_path: str) -> None: + if not self._alive: + raise ConnectionError("node is dead") + src = remote_path.replace("/workspace", self.root_dir) + if not os.path.exists(src): + raise FileNotFoundError(f"remote file not found: {src}") + os.makedirs(os.path.dirname(local_path) or ".", exist_ok=True) + shutil.copy2(src, local_path) + + async def close(self) -> None: + pass + + def kill(self): + """Simulate node death.""" + self._alive = False + + def revive(self): + """Bring node back (for testing reconnect scenarios).""" + self._alive = True + + +class MockTransport: + """Creates MockConnections backed by temp directories.""" + + def __init__(self): + self._dirs: list[str] = [] + + async def connect(self, host: str, port: int) -> MockConnection: + d = tempfile.mkdtemp(prefix=f"mock_node_{host}_{port}_") + self._dirs.append(d) + return MockConnection(d) + + def cleanup(self): + for d in self._dirs: + shutil.rmtree(d, ignore_errors=True) diff --git a/tests/test_aggregator.py b/tests/test_aggregator.py new file mode 100644 index 0000000..a149662 --- /dev/null +++ b/tests/test_aggregator.py @@ -0,0 +1,57 @@ +from collections import OrderedDict + +import pytest +import torch + +from sched.aggregator import average_weights + + +def _make_state(val: float) -> OrderedDict: + return OrderedDict( + weight=torch.full((3, 4), val), + bias=torch.full((3,), val), + ) + + +def test_average_identical(): + w = _make_state(1.0) + result = average_weights([w, w]) + for key in w: + assert torch.allclose(result[key], w[key]) + + +def test_average_different(): + a = _make_state(0.0) + b = _make_state(2.0) + result = average_weights([a, b]) + expected = _make_state(1.0) + for key in expected: + assert torch.allclose(result[key], expected[key]) + + +def test_average_three(): + a = _make_state(0.0) + b = _make_state(3.0) + c = _make_state(6.0) + result = average_weights([a, b, c]) + expected = _make_state(3.0) + for key in expected: + assert torch.allclose(result[key], expected[key]) + + +def test_single(): + w = _make_state(5.0) + result = average_weights([w]) + for key in w: + assert torch.allclose(result[key], w[key]) + + +def test_empty_raises(): + with pytest.raises(ValueError): + average_weights([]) + + +def test_preserves_keys(): + w = OrderedDict(fc1_weight=torch.ones(2, 2), fc1_bias=torch.zeros(2)) + result = average_weights([w, w]) + assert list(result.keys()) == ["fc1_weight", "fc1_bias"] diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py new file mode 100644 index 0000000..b2a99d7 --- /dev/null +++ b/tests/test_scheduler.py @@ -0,0 +1,261 @@ +"""Integration tests for the scheduler using MockTransport. + +These tests run the actual worker.py as subprocesses in isolated temp dirs, +with real torch training on CPU. No vast.ai instances needed. +""" + +from __future__ import annotations + +import asyncio +import os +import sys + +import pytest +import torch + +from sched.aggregator import average_weights +from sched.job import Job +from sched.scheduler import Node, Scheduler +from tests.mock_transport import MockConnection, MockTransport + +JOBS_DIR = os.path.join(os.path.dirname(__file__), os.pardir, "jobs") +MNIST_SCRIPT = os.path.abspath(os.path.join(JOBS_DIR, "mnist.py")) + + +# ------------------------------------------------------------------ +# helpers +# ------------------------------------------------------------------ + + +def make_job(**overrides) -> Job: + defaults = dict( + script=MNIST_SCRIPT, + num_nodes=2, + rounds=3, + local_steps=20, + lr=0.01, + batch_size=32, + ) + defaults.update(overrides) + return Job(**defaults) + + +async def setup_scheduler( + job: Job, num_nodes: int = 2 +) -> tuple[Scheduler, MockTransport]: + """Create a scheduler with mock nodes, deploy code. Ready to run rounds.""" + transport = MockTransport() + sched = Scheduler(job, transport) + + # manually create nodes (skip provision) + for rank in range(num_nodes): + conn = await transport.connect(f"mock{rank}", 22) + sched.nodes.append( + Node(rank=rank, host=f"mock{rank}", port=22, conn=conn, status="ready") + ) + + # deploy worker + job module + await sched.deploy() + return sched, transport + + +# ------------------------------------------------------------------ +# tests +# ------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_single_round_completes(): + """One round with 2 workers. Weights should change from initial.""" + job = make_job(rounds=1, local_steps=10) + sched, transport = await setup_scheduler(job) + + try: + job_mod = sched._load_job_module() + model = job_mod.make_model() + init_state = {k: v.clone() for k, v in model.state_dict().items()} + + new_state = await sched.run_round(model.state_dict(), round_num=0) + + # weights should have changed + changed = False + for key in init_state: + if not torch.equal(init_state[key], new_state[key]): + changed = True + break + assert changed, "Weights did not change after training" + finally: + transport.cleanup() + + +@pytest.mark.asyncio +async def test_multiple_rounds(): + """Multiple rounds. Verify the loop completes and returns valid state dict.""" + job = make_job(rounds=3, local_steps=15) + sched, transport = await setup_scheduler(job) + + try: + job_mod = sched._load_job_module() + model = job_mod.make_model() + state = model.state_dict() + + for r in range(job.rounds): + state = await sched.run_round(state, r) + + # final state should be loadable + model.load_state_dict(state) + # quick forward pass shouldn't crash + x = torch.randn(4, 1, 28, 28) + out = model(x) + assert out.shape == (4, 10) + finally: + transport.cleanup() + + +@pytest.mark.asyncio +async def test_worker_dies_midround(): + """One worker dies before a round. Scheduler continues with survivor.""" + job = make_job(rounds=2, local_steps=10) + sched, transport = await setup_scheduler(job) + + try: + job_mod = sched._load_job_module() + model = job_mod.make_model() + state = model.state_dict() + + # round 0 with both workers + state = await sched.run_round(state, 0) + assert len(sched._active_nodes()) == 2 + + # kill worker 1 + sched.nodes[1].conn.kill() + + # round 1 should still work with 1 survivor + state = await sched.run_round(state, 1) + + alive = sched._active_nodes() + assert len(alive) == 1 + assert alive[0].rank == 0 + + # state should be valid + model.load_state_dict(state) + finally: + transport.cleanup() + + +@pytest.mark.asyncio +async def test_all_workers_dead(): + """All workers dead → RuntimeError.""" + job = make_job(rounds=1, local_steps=5) + sched, transport = await setup_scheduler(job) + + try: + job_mod = sched._load_job_module() + state = job_mod.make_model().state_dict() + + for node in sched.nodes: + node.conn.kill() + + with pytest.raises(RuntimeError, match="No.*worker"): + await sched.run_round(state, 0) + finally: + transport.cleanup() + + +@pytest.mark.asyncio +async def test_worker_dies_after_push_before_exec(): + """Worker dies between param push and exec. Round completes with survivor.""" + job = make_job(rounds=1, local_steps=10) + sched, transport = await setup_scheduler(job) + + try: + job_mod = sched._load_job_module() + state = job_mod.make_model().state_dict() + + # wrap node 1's exec to kill it when called + node1_conn = sched.nodes[1].conn + original_exec = node1_conn.exec + + async def dying_exec(cmd, timeout=30.0): + node1_conn.kill() + raise ConnectionError("node is dead") + + node1_conn.exec = dying_exec + + # should still complete with node 0 + new_state = await sched.run_round(state, 0) + assert sched.nodes[1].status == "dead" + assert len([k for k, v in new_state.items()]) > 0 + finally: + transport.cleanup() + + +@pytest.mark.asyncio +async def test_three_nodes(): + """FedAvg with 3 workers.""" + job = make_job(num_nodes=3, rounds=2, local_steps=10) + sched, transport = await setup_scheduler(job, num_nodes=3) + + try: + job_mod = sched._load_job_module() + state = job_mod.make_model().state_dict() + + for r in range(job.rounds): + state = await sched.run_round(state, r) + + assert len(sched._active_nodes()) == 3 + finally: + transport.cleanup() + + +@pytest.mark.asyncio +async def test_deploy_copies_files(): + """Verify deploy actually puts worker.py and job_module.py on nodes.""" + job = make_job() + transport = MockTransport() + sched = Scheduler(job, transport) + + try: + for rank in range(2): + conn = await transport.connect(f"mock{rank}", 22) + sched.nodes.append( + Node(rank=rank, host=f"mock{rank}", port=22, conn=conn, status="ready") + ) + + await sched.deploy() + + for node in sched.nodes: + root = node.conn.root_dir + assert os.path.exists(os.path.join(root, "worker.py")) + assert os.path.exists(os.path.join(root, "job_module.py")) + finally: + transport.cleanup() + + +@pytest.mark.asyncio +async def test_convergence_signal(): + """Run several rounds and check that weight magnitude changes. + + Not a rigorous convergence test (synthetic data is random), but verifies + the training loop is actually doing gradient updates. + """ + job = make_job(rounds=5, local_steps=30) + sched, transport = await setup_scheduler(job) + + try: + job_mod = sched._load_job_module() + model = job_mod.make_model() + state = model.state_dict() + + weight_norms = [] + for r in range(job.rounds): + state = await sched.run_round(state, r) + norm = sum(v.float().norm().item() for v in state.values()) + weight_norms.append(norm) + + # weights should be changing across rounds (not frozen) + assert weight_norms[0] != pytest.approx(weight_norms[-1], abs=1e-3), ( + f"Weights did not change across rounds: {weight_norms}" + ) + finally: + transport.cleanup() diff --git a/tests/test_trace.py b/tests/test_trace.py new file mode 100644 index 0000000..0d734a3 --- /dev/null +++ b/tests/test_trace.py @@ -0,0 +1,265 @@ +"""Tests for the tracing + visualization pipeline. + +Runs a real training loop with MockTransport + Tracer, then verifies +all three output modes produce correct data. +""" + +from __future__ import annotations + +import io +import os + +import pytest +import torch + +from sched.job import Job +from sched.report import generate_html +from sched.scheduler import Node, Scheduler +from sched.trace import Tracer +from tests.mock_transport import MockTransport + +JOBS_DIR = os.path.join(os.path.dirname(__file__), os.pardir, "jobs") +MNIST_SCRIPT = os.path.abspath(os.path.join(JOBS_DIR, "mnist.py")) + + +async def _run_traced( + rounds=3, local_steps=10, num_nodes=2, kill_node: int | None = None, kill_after_round: int | None = None, +) -> tuple[Scheduler, Tracer, MockTransport]: + """Helper: run a traced job and return the scheduler + tracer.""" + buf = io.StringIO() + tracer = Tracer(file=buf) + job = Job( + script=MNIST_SCRIPT, + num_nodes=num_nodes, + rounds=rounds, + local_steps=local_steps, + lr=0.01, + batch_size=32, + ) + transport = MockTransport() + sched = Scheduler(job, transport, tracer=tracer) + + for rank in range(num_nodes): + conn = await transport.connect(f"mock{rank}", 22) + sched.nodes.append( + Node(rank=rank, host=f"mock{rank}", port=22, conn=conn, status="ready") + ) + + await sched.deploy() + + job_mod = sched._load_job_module() + state = job_mod.make_model().state_dict() + + for r in range(rounds): + if kill_node is not None and kill_after_round is not None and r == kill_after_round + 1: + sched.nodes[kill_node].conn.kill() + state = await sched.run_round(state, r) + + return sched, tracer, transport + + +# ------------------------------------------------------------------ +# C) text log tests +# ------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_text_log_events_emitted(): + """Tracer collects the right number and kinds of events.""" + sched, tracer, transport = await _run_traced(rounds=2, local_steps=5) + try: + kinds = [ev.kind for ev in tracer.events] + assert kinds.count("round_start") == 2 + assert kinds.count("round_end") == 2 + assert kinds.count("aggregate") == 2 + # 2 workers × (push + exec + pull) × 2 rounds = 12 + # plus deploy pushes (4: worker.py + job_module.py per node) + # plus deploy exec (2: "test -f" check per node) + assert kinds.count("push") >= 4 # at least deploy pushes + assert kinds.count("exec") == 6 # 2 deploy checks + 2 workers × 2 rounds + assert kinds.count("pull") == 4 + finally: + transport.cleanup() + + +@pytest.mark.asyncio +async def test_text_log_captures_errors(): + """Dead worker produces events with error field.""" + sched, tracer, transport = await _run_traced( + rounds=2, local_steps=5, kill_node=1, kill_after_round=0 + ) + try: + error_events = [ev for ev in tracer.events if ev.data.get("error")] + assert len(error_events) > 0 + assert any(ev.rank == 1 for ev in error_events) + finally: + transport.cleanup() + + +@pytest.mark.asyncio +async def test_text_log_output_to_buffer(): + """Live text log is written to the provided file object.""" + buf = io.StringIO() + tracer = Tracer(file=buf) + tracer.emit("round_start", round_num=0, active_nodes=2) + + output = buf.getvalue() + assert "round 0 start" in output + assert "2 workers" in output + + +# ------------------------------------------------------------------ +# A) gantt tests +# ------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_gantt_basic_structure(): + """Gantt output contains expected sections.""" + sched, tracer, transport = await _run_traced(rounds=2, local_steps=5) + try: + text = tracer.gantt(width=40) + assert "round 0" in text + assert "round 1" in text + assert "scheduler" in text + assert "worker 0" in text + assert "worker 1" in text + # legend + assert "push" in text + assert "train" in text + finally: + transport.cleanup() + + +@pytest.mark.asyncio +async def test_gantt_shows_training_blocks(): + """Worker rows contain training blocks (█).""" + sched, tracer, transport = await _run_traced(rounds=1, local_steps=10) + try: + text = tracer.gantt(width=50) + worker_lines = [l for l in text.splitlines() if "worker" in l and "█" in l] + # both workers should have training blocks + assert len(worker_lines) >= 2, f"Expected training blocks in:\n{text}" + finally: + transport.cleanup() + + +@pytest.mark.asyncio +async def test_gantt_shows_dead_worker(): + """Dead worker shows ✗ markers.""" + sched, tracer, transport = await _run_traced( + rounds=2, local_steps=5, kill_node=1, kill_after_round=0 + ) + try: + text = tracer.gantt(width=40) + # In round 1, worker 1 should have dead markers + lines = text.splitlines() + # find lines for round 1 + in_round_1 = False + for line in lines: + if "round 1" in line: + in_round_1 = True + if in_round_1 and "worker 1" in line: + assert "✗" in line, f"Expected dead markers in: {line}" + break + finally: + transport.cleanup() + + +@pytest.mark.asyncio +async def test_gantt_three_workers(): + """Gantt handles 3 workers.""" + sched, tracer, transport = await _run_traced( + rounds=1, local_steps=5, num_nodes=3 + ) + try: + text = tracer.gantt(width=40) + assert "worker 0" in text + assert "worker 1" in text + assert "worker 2" in text + finally: + transport.cleanup() + + +# ------------------------------------------------------------------ +# B) HTML report tests +# ------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_html_report_generated(tmp_path): + """HTML report is written and contains key elements.""" + sched, tracer, transport = await _run_traced(rounds=2, local_steps=5) + try: + path = str(tmp_path / "report.html") + result = generate_html(tracer, path) + assert os.path.exists(result) + + content = open(result).read() + assert "" in content + assert "= args.local_steps: + break + x, y = x.to(device), y.to(device) + opt.zero_grad() + loss = F.cross_entropy(model(x), y) + loss.backward() + opt.step() + step += 1 + + torch.save(model.state_dict(), args.output) + print(f"rank={args.rank} done steps={args.local_steps}") + + +if __name__ == "__main__": + main()