stash: right after fist mildly aggressive deploy test of single model
This commit is contained in:
parent
bdbbc2458e
commit
1da467d1d6
18 changed files with 2272 additions and 59 deletions
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -3,6 +3,9 @@ data/
|
|||
outputs/
|
||||
artifacts/
|
||||
.airfrans_resume/
|
||||
.env
|
||||
WANDB_API_KEY
|
||||
HF_TOKEN
|
||||
|
||||
# Python
|
||||
.venv/
|
||||
|
|
|
|||
|
|
@ -2,5 +2,8 @@
|
|||
/data/raw
|
||||
/.venv
|
||||
/notebooks
|
||||
.env
|
||||
HF_TOKEN
|
||||
WANDB_API_KEY
|
||||
__pycache__
|
||||
*.pyc
|
||||
|
|
|
|||
|
|
@ -44,3 +44,9 @@ interval_seconds = 1800
|
|||
|
||||
[stability]
|
||||
max_grad_norm = 1.0
|
||||
|
||||
[observability]
|
||||
backend = "wandb"
|
||||
entity = "zacheryasc-personal"
|
||||
project = "airfRANS-model-sweep"
|
||||
tags = ["airfrans", "remote", "aggressive-smoke"]
|
||||
|
|
|
|||
77
configs/remote_hf_smoke.toml
Normal file
77
configs/remote_hf_smoke.toml
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
[run]
|
||||
name = "airfrans-hf-smoke"
|
||||
timeout_minutes = 30
|
||||
local_artifact_dir = "artifacts/remote_runs"
|
||||
max_attempts = 1
|
||||
artifact_sync_interval_seconds = 300
|
||||
|
||||
[provider]
|
||||
kind = "vastai"
|
||||
disk_gb = 32
|
||||
max_price_per_hour = 0.08
|
||||
image = "vastai/base:0.0.2"
|
||||
|
||||
[provider.gpu]
|
||||
name = "RTX 3060"
|
||||
count = 1
|
||||
min_vram_gb = 12
|
||||
|
||||
[selection]
|
||||
min_reliability = 0.90
|
||||
min_down_mbps = 50
|
||||
min_up_mbps = 5
|
||||
require_verified = true
|
||||
blocked_geos = ["CN"]
|
||||
blacklist_hosts = [59017, 187467, 155385]
|
||||
drop_cheap_frac = 0.0
|
||||
image_size_gb = 5.0
|
||||
base_url = "https://cloud.vast.ai"
|
||||
|
||||
[workspace]
|
||||
workdir = "."
|
||||
exclude = [
|
||||
"/artifacts",
|
||||
"/data/raw",
|
||||
"/.venv",
|
||||
"/notebooks",
|
||||
"HF_TOKEN",
|
||||
"__pycache__",
|
||||
"*.pyc",
|
||||
]
|
||||
|
||||
[bootstrap]
|
||||
command = """
|
||||
uv sync --no-dev
|
||||
uv run --no-dev python -c "import torch; print('torch_cuda_available=' + str(torch.cuda.is_available())); print(torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'no_cuda_device')"
|
||||
uv run --no-dev python -c "import huggingface_hub; print('huggingface_hub_ok')"
|
||||
"""
|
||||
|
||||
[job]
|
||||
command = """
|
||||
uv run --no-dev remote-run hf-smoke --artifact-dir artifacts/current_run --run-id "$AIRFRANS_REMOTE_RUN_ID" --repo-id airfrans-hf-smoke
|
||||
"""
|
||||
artifact_dir = "artifacts/current_run"
|
||||
heartbeat_file = "artifacts/current_run/heartbeat.json"
|
||||
metrics_file = "artifacts/current_run/metrics.jsonl"
|
||||
|
||||
[artifacts]
|
||||
mode = "object_store_upload"
|
||||
required = [
|
||||
"config.toml",
|
||||
"metrics.jsonl",
|
||||
"latest_metrics.json",
|
||||
"heartbeat.json",
|
||||
"checkpoint_latest.pt",
|
||||
"checkpoint_best.pt",
|
||||
"checkpoint_final.pt",
|
||||
"final_metrics.json",
|
||||
"run_manifest.json",
|
||||
"environment_manifest.json",
|
||||
"hf_upload_manifest.json",
|
||||
"artifact_manifest.json",
|
||||
"checksums.txt",
|
||||
]
|
||||
|
||||
[cleanup]
|
||||
on_success = "sky_down"
|
||||
on_failure = "collect_then_keep"
|
||||
|
|
@ -30,3 +30,9 @@ benchmark_kernels = true
|
|||
|
||||
[loss]
|
||||
type = "normalized_mse"
|
||||
|
||||
[observability]
|
||||
backend = "wandb"
|
||||
entity = "zacheryasc-personal"
|
||||
project = "airfRANS-model-sweep"
|
||||
tags = ["airfrans", "remote", "tiny"]
|
||||
|
|
|
|||
78
configs/remote_wandb_smoke.toml
Normal file
78
configs/remote_wandb_smoke.toml
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
[run]
|
||||
name = "airfrans-wandb-smoke"
|
||||
timeout_minutes = 30
|
||||
local_artifact_dir = "artifacts/remote_runs"
|
||||
max_attempts = 1
|
||||
artifact_sync_interval_seconds = 60
|
||||
|
||||
[provider]
|
||||
kind = "vastai"
|
||||
disk_gb = 32
|
||||
max_price_per_hour = 0.09
|
||||
image = "vastai/base:0.0.2"
|
||||
|
||||
[provider.gpu]
|
||||
name = "RTX 3060 Ti"
|
||||
count = 1
|
||||
min_vram_gb = 8
|
||||
|
||||
[selection]
|
||||
min_reliability = 0.90
|
||||
min_down_mbps = 50
|
||||
min_up_mbps = 5
|
||||
require_verified = true
|
||||
blocked_geos = ["CN"]
|
||||
blacklist_hosts = [59017, 187467, 155385, 466789]
|
||||
drop_cheap_frac = 0.0
|
||||
image_size_gb = 5.0
|
||||
base_url = "https://cloud.vast.ai"
|
||||
|
||||
[workspace]
|
||||
workdir = "."
|
||||
exclude = [
|
||||
"/artifacts",
|
||||
"/data/raw",
|
||||
"/.venv",
|
||||
"/notebooks",
|
||||
".env",
|
||||
"HF_TOKEN",
|
||||
"WANDB_API_KEY",
|
||||
"__pycache__",
|
||||
"*.pyc",
|
||||
]
|
||||
|
||||
[bootstrap]
|
||||
command = """
|
||||
uv sync --no-dev
|
||||
uv run --no-dev python -c "import wandb; print('wandb_ok')"
|
||||
"""
|
||||
|
||||
[job]
|
||||
command = """
|
||||
uv run --no-dev remote-run wandb-smoke --artifact-dir artifacts/current_run --run-id "$AIRFRANS_REMOTE_RUN_ID" --entity zacheryasc-personal --project airfRANS-model-sweep --hf-repo-url https://huggingface.co/zacheryasc/airfrans-hf-smoke
|
||||
"""
|
||||
artifact_dir = "artifacts/current_run"
|
||||
heartbeat_file = "artifacts/current_run/heartbeat.json"
|
||||
metrics_file = "artifacts/current_run/metrics.jsonl"
|
||||
|
||||
[artifacts]
|
||||
mode = "rsync"
|
||||
required = [
|
||||
"config.toml",
|
||||
"metrics.jsonl",
|
||||
"latest_metrics.json",
|
||||
"heartbeat.json",
|
||||
"checkpoint_latest.pt",
|
||||
"checkpoint_best.pt",
|
||||
"checkpoint_final.pt",
|
||||
"final_metrics.json",
|
||||
"run_manifest.json",
|
||||
"environment_manifest.json",
|
||||
"wandb_smoke_manifest.json",
|
||||
"artifact_manifest.json",
|
||||
"checksums.txt",
|
||||
]
|
||||
|
||||
[cleanup]
|
||||
on_success = "sky_down"
|
||||
on_failure = "collect_then_keep"
|
||||
1219
notebooks/airfrans_aggressive_mechinterp.ipynb
Normal file
1219
notebooks/airfrans_aggressive_mechinterp.ipynb
Normal file
File diff suppressed because one or more lines are too long
|
|
@ -4,8 +4,10 @@ version = "0.1.0"
|
|||
description = "Utilities for inspecting local AirfRANS scaling-frontier data."
|
||||
requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
"huggingface-hub>=0.36.0",
|
||||
"numpy>=2.4.0",
|
||||
"torch>=2.8.0",
|
||||
"wandb>=0.23.0",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
|
|
|
|||
|
|
@ -69,7 +69,11 @@ def verify_artifacts(
|
|||
if path.is_file():
|
||||
_validate_checkpoint_metadata(path)
|
||||
|
||||
files = sorted(path for path in root.rglob("*") if path.is_file() and path.name not in {"artifact_manifest.json", "checksums.txt"})
|
||||
files = sorted(
|
||||
path
|
||||
for path in root.rglob("*")
|
||||
if not path.is_symlink() and path.is_file() and path.name not in {"artifact_manifest.json", "checksums.txt"}
|
||||
)
|
||||
manifest = {
|
||||
"artifact_dir": str(root),
|
||||
"file_count": len(files),
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ from __future__ import annotations
|
|||
import argparse
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
|
@ -14,7 +15,7 @@ from airfrans_frontier.remote.artifacts import verify_artifacts
|
|||
from airfrans_frontier.remote.config import RemoteRunConfig, load_remote_run_config
|
||||
from airfrans_frontier.remote.skypilot import render_skypilot_yaml, write_skyignore
|
||||
from airfrans_frontier.remote.skypilot_patch import apply_patch, patch_status, require_patch
|
||||
from airfrans_frontier.remote.smoke import run_smoke_training
|
||||
from airfrans_frontier.remote.smoke import run_hf_upload_smoke, run_smoke_training, run_wandb_smoke
|
||||
from airfrans_frontier.remote.vast import SelectionResult, select_offer
|
||||
|
||||
|
||||
|
|
@ -42,6 +43,22 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
verify.add_argument("artifact_dir")
|
||||
verify.set_defaults(command="verify-artifacts")
|
||||
|
||||
hf_smoke = subparsers.add_parser("hf-smoke", help="upload tiny smoke artifacts to Hugging Face")
|
||||
hf_smoke.add_argument("--artifact-dir", required=True)
|
||||
hf_smoke.add_argument("--run-id", required=True)
|
||||
hf_smoke.add_argument("--repo-id", help="HF repo id or repo slug; default is <user>/airfrans-hf-smoke")
|
||||
hf_smoke.set_defaults(command="hf-smoke")
|
||||
|
||||
wandb_smoke = subparsers.add_parser("wandb-smoke", help="log tiny smoke metrics to Weights & Biases")
|
||||
wandb_smoke.add_argument("--artifact-dir", required=True)
|
||||
wandb_smoke.add_argument("--run-id", required=True)
|
||||
wandb_smoke.add_argument("--entity", default="zacheryasc-personal")
|
||||
wandb_smoke.add_argument("--project", default="airfRANS-model-sweep")
|
||||
wandb_smoke.add_argument("--hf-repo-url")
|
||||
wandb_smoke.set_defaults(command="wandb-smoke")
|
||||
|
||||
|
||||
|
||||
smoke = subparsers.add_parser("smoke-train", help="run the configured smoke training job and flatten artifacts")
|
||||
smoke.add_argument("training_config")
|
||||
smoke.add_argument("--artifact-dir", required=True)
|
||||
|
|
@ -82,6 +99,24 @@ def main(argv: list[str] | None = None) -> int:
|
|||
manifest = verify_artifacts(args.artifact_dir)
|
||||
print(json.dumps({"status": "ok", "file_count": manifest["file_count"]}, sort_keys=True))
|
||||
return 0
|
||||
if args.command == "hf-smoke":
|
||||
output = run_hf_upload_smoke(
|
||||
artifact_dir=args.artifact_dir,
|
||||
run_id=args.run_id,
|
||||
repo_id=args.repo_id,
|
||||
)
|
||||
print(f"artifact_dir: {output}")
|
||||
return 0
|
||||
if args.command == "wandb-smoke":
|
||||
output = run_wandb_smoke(
|
||||
artifact_dir=args.artifact_dir,
|
||||
run_id=args.run_id,
|
||||
entity=args.entity,
|
||||
project=args.project,
|
||||
hf_repo_url=args.hf_repo_url,
|
||||
)
|
||||
print(f"artifact_dir: {output}")
|
||||
return 0
|
||||
if args.command == "smoke-train":
|
||||
output = run_smoke_training(
|
||||
args.training_config,
|
||||
|
|
@ -196,12 +231,22 @@ def _run(config_path: str | Path, *, dry_run: bool, skip_down: bool) -> int:
|
|||
local_run_dir=local_run_dir,
|
||||
env=env,
|
||||
)
|
||||
state("COLLECTING", selected_offer_id=selection.selected_offer_id, attempt=attempt, return_code=return_code)
|
||||
_collect_best_effort(cluster=run_id, remote_dir=config.job.artifact_dir, local_dir=local_run_dir, env=env)
|
||||
state("REMOTE_FINISHED", selected_offer_id=selection.selected_offer_id, attempt=attempt, return_code=return_code)
|
||||
_collect_terminal_best_effort(cluster=run_id, remote_dir=config.job.artifact_dir, local_dir=local_run_dir, required=config.artifacts.required, env=env)
|
||||
status = _classify_artifacts(local_run_dir)
|
||||
if status == "success":
|
||||
try:
|
||||
state("COLLECTING_REQUIRED", selected_offer_id=selection.selected_offer_id, attempt=attempt)
|
||||
_collect_required_artifacts(cluster=run_id, remote_dir=config.job.artifact_dir, local_dir=local_run_dir, required=config.artifacts.required, env=env)
|
||||
state("VERIFYING_ARTIFACTS", selected_offer_id=selection.selected_offer_id, attempt=attempt)
|
||||
verify_artifacts(local_run_dir, required=config.artifacts.required)
|
||||
except Exception as exc:
|
||||
_cleanup_partial_artifacts(local_run_dir)
|
||||
last_error = f"remote job succeeded but artifact collection failed: {exc}"
|
||||
state("FAILED_COLLECTION", selected_offer_id=selection.selected_offer_id, attempt=attempt, return_code=return_code, error=last_error)
|
||||
if config.cleanup.on_failure == "sky_down" and not skip_down:
|
||||
_run_best_effort(["sky", "down", run_id, "-y"], env=env)
|
||||
raise RuntimeError(last_error) from exc
|
||||
if config.cleanup.on_success == "sky_down" and not skip_down:
|
||||
state("CLEANING_UP", selected_offer_id=selection.selected_offer_id, attempt=attempt)
|
||||
_run_checked(["sky", "down", run_id, "-y"], env=env, timeout=300)
|
||||
|
|
@ -211,16 +256,33 @@ def _run(config_path: str | Path, *, dry_run: bool, skip_down: bool) -> int:
|
|||
return 0
|
||||
if status == "failure_report":
|
||||
last_error = "training wrote failure_report.json"
|
||||
state("FAILED_TRAINING", selected_offer_id=selection.selected_offer_id, attempt=attempt, error=last_error)
|
||||
state("FAILED_TRAINING", selected_offer_id=selection.selected_offer_id, attempt=attempt, return_code=return_code, error=last_error)
|
||||
if config.cleanup.on_failure == "sky_down" and not skip_down:
|
||||
_run_best_effort(["sky", "down", run_id, "-y"], env=env)
|
||||
raise RuntimeError(last_error)
|
||||
last_error = f"SkyPilot job ended without terminal artifacts, return_code={return_code}"
|
||||
state("RETRYING", selected_offer_id=selection.selected_offer_id, attempt=attempt, error=last_error)
|
||||
if return_code == 0:
|
||||
last_error = "remote job succeeded but terminal artifacts were not collected"
|
||||
state("FAILED_COLLECTION", selected_offer_id=selection.selected_offer_id, attempt=attempt, return_code=return_code, error=last_error)
|
||||
if config.cleanup.on_failure == "sky_down" and not skip_down:
|
||||
_run_best_effort(["sky", "down", run_id, "-y"], env=env)
|
||||
raise RuntimeError(last_error)
|
||||
|
||||
_collect_restart_best_effort(cluster=run_id, remote_dir=config.job.artifact_dir, local_dir=local_run_dir, env=env)
|
||||
status = _classify_artifacts(local_run_dir)
|
||||
if status == "failure_report":
|
||||
last_error = "training wrote failure_report.json"
|
||||
state("FAILED_TRAINING", selected_offer_id=selection.selected_offer_id, attempt=attempt, return_code=return_code, error=last_error)
|
||||
if config.cleanup.on_failure == "sky_down" and not skip_down:
|
||||
_run_best_effort(["sky", "down", run_id, "-y"], env=env)
|
||||
raise RuntimeError(last_error)
|
||||
last_error = f"remote job failed without terminal success, return_code={return_code}, local_artifacts={status}"
|
||||
if attempt < config.run.max_attempts:
|
||||
state("RETRYING", selected_offer_id=selection.selected_offer_id, attempt=attempt, return_code=return_code, error=last_error)
|
||||
if config.cleanup.on_failure == "sky_down" and not skip_down:
|
||||
_run_best_effort(["sky", "down", run_id, "-y"], env=env)
|
||||
else:
|
||||
state("FAILED", selected_offer_id=selection.selected_offer_id, attempt=attempt, return_code=return_code, error=last_error)
|
||||
|
||||
state("FAILED", selected_offer_id=selection.selected_offer_id, error=last_error or "max attempts exhausted")
|
||||
raise RuntimeError(last_error or "max attempts exhausted")
|
||||
|
||||
|
||||
|
|
@ -265,7 +327,13 @@ def _run_sky_with_periodic_collection(
|
|||
local_run_dir: Path,
|
||||
env: dict[str, str],
|
||||
) -> int:
|
||||
process = subprocess.Popen(["sky", "launch", "-c", cluster, str(sky_yaml_path), "-y"], env=env)
|
||||
argv = ["sky", "launch", "-c", cluster, str(sky_yaml_path), "-y"]
|
||||
if config.artifacts.mode == "object_store_upload":
|
||||
_ensure_hf_secret_env(env)
|
||||
argv.extend(["--secret", "HF_TOKEN"])
|
||||
if _load_secret_env(env, "WANDB_API_KEY", required=False, purpose="W&B remote runs"):
|
||||
argv.extend(["--secret", "WANDB_API_KEY"])
|
||||
process = subprocess.Popen(argv, env=env, start_new_session=True)
|
||||
deadline = time.monotonic() + config.run.timeout_minutes * 60
|
||||
next_collect = time.monotonic() + config.run.artifact_sync_interval_seconds
|
||||
while True:
|
||||
|
|
@ -274,37 +342,147 @@ def _run_sky_with_periodic_collection(
|
|||
return int(return_code)
|
||||
now = time.monotonic()
|
||||
if now >= deadline:
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=30)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
return int(process.returncode or 124)
|
||||
_terminate_process_group(process)
|
||||
return 124
|
||||
if config.run.artifact_sync_interval_seconds == 0 or now >= next_collect:
|
||||
_collect_best_effort(cluster=cluster, remote_dir=config.job.artifact_dir, local_dir=local_run_dir, env=env)
|
||||
if _classify_artifacts(local_run_dir) in {"success", "failure_report"}:
|
||||
return_code = process.poll()
|
||||
if return_code is not None:
|
||||
return int(return_code)
|
||||
_collect_terminal_best_effort(cluster=cluster, remote_dir=config.job.artifact_dir, local_dir=local_run_dir, required=config.artifacts.required, env=env)
|
||||
next_collect = now + max(1, config.run.artifact_sync_interval_seconds)
|
||||
time.sleep(min(5.0, max(0.1, next_collect - now)))
|
||||
|
||||
|
||||
def _collect_with_rsync(*, cluster: str, remote_dir: Path, local_dir: Path, env: dict[str, str]) -> None:
|
||||
def _collect_paths_with_rsync(
|
||||
*,
|
||||
cluster: str,
|
||||
remote_dir: Path,
|
||||
local_dir: Path,
|
||||
paths: tuple[str, ...],
|
||||
env: dict[str, str],
|
||||
timeout: int,
|
||||
) -> None:
|
||||
local_dir.mkdir(parents=True, exist_ok=True)
|
||||
source = f"{cluster}:~/sky_workdir/{remote_dir}/"
|
||||
_run_checked(["rsync", "-Pavz", source, f"{local_dir}/"], env=env, timeout=600)
|
||||
for relative_path in paths:
|
||||
source = f"{cluster}:~/sky_workdir/{remote_dir}/./{relative_path}"
|
||||
_run_checked(
|
||||
[
|
||||
"rsync",
|
||||
"-Pavz",
|
||||
"--relative",
|
||||
"--ignore-missing-args",
|
||||
"--delay-updates",
|
||||
"--timeout=30",
|
||||
source,
|
||||
f"{local_dir}/",
|
||||
],
|
||||
env=env,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
|
||||
def _collect_best_effort(*, cluster: str, remote_dir: Path, local_dir: Path, env: dict[str, str]) -> None:
|
||||
def _collect_required_artifacts(*, cluster: str, remote_dir: Path, local_dir: Path, required: tuple[str, ...], env: dict[str, str]) -> None:
|
||||
_collect_paths_with_rsync(
|
||||
cluster=cluster,
|
||||
remote_dir=remote_dir,
|
||||
local_dir=local_dir,
|
||||
paths=_large_artifact_names(required),
|
||||
env=env,
|
||||
timeout=3600,
|
||||
)
|
||||
|
||||
|
||||
def _collect_terminal_best_effort(*, cluster: str, remote_dir: Path, local_dir: Path, required: tuple[str, ...], env: dict[str, str]) -> None:
|
||||
try:
|
||||
_collect_with_rsync(cluster=cluster, remote_dir=remote_dir, local_dir=local_dir, env=env)
|
||||
_collect_paths_with_rsync(
|
||||
cluster=cluster,
|
||||
remote_dir=remote_dir,
|
||||
local_dir=local_dir,
|
||||
paths=_terminal_artifact_names(required),
|
||||
env=env,
|
||||
timeout=120,
|
||||
)
|
||||
except Exception:
|
||||
_cleanup_partial_artifacts(local_dir)
|
||||
|
||||
|
||||
def _collect_restart_best_effort(*, cluster: str, remote_dir: Path, local_dir: Path, env: dict[str, str]) -> None:
|
||||
try:
|
||||
_collect_paths_with_rsync(
|
||||
cluster=cluster,
|
||||
remote_dir=remote_dir,
|
||||
local_dir=local_dir,
|
||||
paths=("checkpoint_latest.pt",),
|
||||
env=env,
|
||||
timeout=3600,
|
||||
)
|
||||
except Exception:
|
||||
_cleanup_partial_artifacts(local_dir)
|
||||
|
||||
|
||||
_LARGE_ARTIFACT_SUFFIXES = (".pt", ".pth", ".ckpt", ".safetensors")
|
||||
_TERMINAL_ARTIFACT_NAMES = (
|
||||
"artifact_manifest.json",
|
||||
"checksums.txt",
|
||||
"config.toml",
|
||||
"data_manifest.json",
|
||||
"environment_manifest.json",
|
||||
"failure_report.json",
|
||||
"final_metrics.json",
|
||||
"heartbeat.json",
|
||||
"hf_upload_manifest.json",
|
||||
"latest_metrics.json",
|
||||
"metrics.jsonl",
|
||||
"normalization.json",
|
||||
"run_manifest.json",
|
||||
"split_manifest.json",
|
||||
"wandb_smoke_manifest.json",
|
||||
)
|
||||
|
||||
|
||||
def _terminal_artifact_names(required: tuple[str, ...]) -> tuple[str, ...]:
|
||||
names = set(_TERMINAL_ARTIFACT_NAMES)
|
||||
names.update(name for name in required if not _is_large_artifact(name))
|
||||
return tuple(sorted(names))
|
||||
|
||||
|
||||
def _large_artifact_names(required: tuple[str, ...]) -> tuple[str, ...]:
|
||||
return tuple(name for name in required if _is_large_artifact(name))
|
||||
|
||||
|
||||
def _is_large_artifact(name: str) -> bool:
|
||||
return name.endswith(_LARGE_ARTIFACT_SUFFIXES)
|
||||
|
||||
|
||||
def _cleanup_partial_artifacts(local_run_dir: Path) -> None:
|
||||
for partial_dir in (".rsync-partial", ".~tmp~"):
|
||||
for path in local_run_dir.rglob(partial_dir):
|
||||
if path.is_dir() and not path.is_symlink():
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
elif path.exists():
|
||||
try:
|
||||
path.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _terminate_process_group(process: subprocess.Popen[Any]) -> None:
|
||||
if process.poll() is not None:
|
||||
return
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
return
|
||||
try:
|
||||
process.wait(timeout=30)
|
||||
return
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
return
|
||||
process.wait()
|
||||
|
||||
def _classify_artifacts(local_run_dir: Path) -> str:
|
||||
if (local_run_dir / "final_metrics.json").is_file() and (local_run_dir / "checkpoint_final.pt").is_file():
|
||||
if (local_run_dir / "final_metrics.json").is_file():
|
||||
return "success"
|
||||
if (local_run_dir / "failure_report.json").is_file():
|
||||
return "failure_report"
|
||||
|
|
@ -314,7 +492,14 @@ def _classify_artifacts(local_run_dir: Path) -> str:
|
|||
|
||||
|
||||
def _run_checked(argv: list[str], *, env: dict[str, str], timeout: int) -> None:
|
||||
subprocess.run(argv, check=True, env=env, timeout=timeout)
|
||||
process = subprocess.Popen(argv, env=env, start_new_session=True)
|
||||
try:
|
||||
return_code = process.wait(timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
_terminate_process_group(process)
|
||||
raise
|
||||
if return_code != 0:
|
||||
raise subprocess.CalledProcessError(return_code, argv)
|
||||
|
||||
|
||||
def _run_best_effort(argv: list[str], *, env: dict[str, str]) -> None:
|
||||
|
|
@ -329,6 +514,36 @@ def _subprocess_env() -> dict[str, str]:
|
|||
env.pop("PYTHONPATH", None)
|
||||
return env
|
||||
|
||||
def _ensure_hf_secret_env(env: dict[str, str]) -> None:
|
||||
_load_secret_env(env, "HF_TOKEN", required=True, purpose="object_store_upload runs")
|
||||
|
||||
|
||||
def _load_secret_env(env: dict[str, str], name: str, *, required: bool, purpose: str) -> bool:
|
||||
if env.get(name):
|
||||
return True
|
||||
for path in (Path(name), Path(".env") / name):
|
||||
if path.is_file():
|
||||
value = path.read_text().strip()
|
||||
if value:
|
||||
env[name] = value
|
||||
return True
|
||||
env_file = Path(".env")
|
||||
if env_file.is_file():
|
||||
for raw_line in env_file.read_text().splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
if key.strip() == name:
|
||||
value = value.strip().strip("\"'")
|
||||
if value:
|
||||
env[name] = value
|
||||
return True
|
||||
if required:
|
||||
raise RuntimeError(f"{name} env var or local secret file is required for {purpose}")
|
||||
return False
|
||||
|
||||
|
||||
|
||||
def _emit_json(data: dict[str, Any], out: str | None) -> None:
|
||||
text = json.dumps(data, indent=2, sort_keys=True) + "\n"
|
||||
|
|
|
|||
|
|
@ -24,8 +24,6 @@ def render_skypilot_yaml(
|
|||
f" accelerators: {accelerator}",
|
||||
f" disk_size: {config.provider.disk_gb}",
|
||||
]
|
||||
if config.provider.max_price_per_hour is not None:
|
||||
lines.append(f" max_hourly_cost: {config.provider.max_price_per_hour}")
|
||||
if config.provider.image:
|
||||
image = config.provider.image
|
||||
if not image.startswith("docker:"):
|
||||
|
|
@ -62,6 +60,9 @@ def render_skypilot_yaml(
|
|||
def write_skyignore(config: RemoteRunConfig, path: str | Path = ".skyignore") -> Path:
|
||||
skyignore_path = Path(path)
|
||||
entries = list(config.workspace.exclude)
|
||||
for secret_path in ("HF_TOKEN", "WANDB_API_KEY", ".env"):
|
||||
if secret_path not in entries:
|
||||
entries.append(secret_path)
|
||||
content = "\n".join(entries).rstrip() + "\n"
|
||||
skyignore_path.write_text(content)
|
||||
return skyignore_path
|
||||
|
|
@ -96,11 +97,10 @@ def _compose_run(config: RemoteRunConfig, *, run_id: str) -> str:
|
|||
|
||||
def _accelerator(config: RemoteRunConfig) -> str:
|
||||
name = config.provider.gpu.name or "T4"
|
||||
# SkyPilot accelerator names omit spaces for common Vast names.
|
||||
sky_name = name.replace(" ", "")
|
||||
aliases = {"Tesla T4": "T4", "RTX 3060 Ti": "RTX3060"}
|
||||
sky_name = aliases.get(name, name.replace(" ", ""))
|
||||
return f"{sky_name}:{config.provider.gpu.count}"
|
||||
|
||||
|
||||
def _indent_block(text: str) -> list[str]:
|
||||
return [f" {line}" if line else "" for line in text.splitlines()]
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
|
|
@ -73,7 +74,6 @@ def run_smoke_training(
|
|||
"checkpoint_latest.pt",
|
||||
"checkpoint_best.pt",
|
||||
"checkpoint_final.pt",
|
||||
"checkpoint.pt",
|
||||
"config.toml",
|
||||
"normalization.json",
|
||||
"split_manifest.json",
|
||||
|
|
@ -116,6 +116,302 @@ def run_smoke_training(
|
|||
verify_artifacts(output_dir)
|
||||
return output_dir
|
||||
|
||||
def run_hf_upload_smoke(
|
||||
*,
|
||||
artifact_dir: str | Path,
|
||||
run_id: str,
|
||||
repo_id: str | None = None,
|
||||
) -> Path:
|
||||
output_dir = Path(artifact_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
started = time.time()
|
||||
heartbeat_path = output_dir / "heartbeat.json"
|
||||
_write_json(
|
||||
heartbeat_path,
|
||||
{
|
||||
"run_id": run_id,
|
||||
"phase": "starting",
|
||||
"timestamp": started,
|
||||
},
|
||||
)
|
||||
_write_json(output_dir / "environment_manifest.json", environment_manifest())
|
||||
(output_dir / "config.toml").write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"[run]",
|
||||
f'name = "{run_id}"',
|
||||
"kind = \"hf_upload_smoke\"",
|
||||
"",
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
checkpoint = {
|
||||
"schema_version": 1,
|
||||
"step": 0,
|
||||
"model_state_dict": {},
|
||||
"optimizer_state_dict": {},
|
||||
"normalization": {},
|
||||
}
|
||||
import torch
|
||||
|
||||
for name in ("checkpoint_latest.pt", "checkpoint_best.pt", "checkpoint_final.pt"):
|
||||
torch.save(checkpoint, output_dir / name)
|
||||
|
||||
metric = {
|
||||
"run_id": run_id,
|
||||
"epoch": 0,
|
||||
"step": 0,
|
||||
"loss": 0.0,
|
||||
"hf_smoke": True,
|
||||
"timestamp": time.time(),
|
||||
}
|
||||
(output_dir / "metrics.jsonl").write_text(json.dumps(metric, sort_keys=True) + "\n")
|
||||
_write_json(output_dir / "latest_metrics.json", metric)
|
||||
_write_json(output_dir / "final_metrics.json", {"hf_smoke": True, "loss": 0.0, "step": 0})
|
||||
|
||||
payload_dir = output_dir / "hf_payload"
|
||||
final_dir = payload_dir / "final"
|
||||
final_dir.mkdir(parents=True, exist_ok=True)
|
||||
model_bytes = f"airfrans HF upload smoke\nrun_id={run_id}\n".encode()
|
||||
model_path = final_dir / "smoke_model.bin"
|
||||
model_path.write_bytes(model_bytes)
|
||||
payload_manifest = {
|
||||
"run_id": run_id,
|
||||
"created_at": time.time(),
|
||||
"files": [
|
||||
{
|
||||
"path": "final/smoke_model.bin",
|
||||
"bytes": len(model_bytes),
|
||||
"sha256": hashlib.sha256(model_bytes).hexdigest(),
|
||||
}
|
||||
],
|
||||
}
|
||||
_write_json(payload_dir / "hf_smoke_manifest.json", payload_manifest)
|
||||
|
||||
token = _resolve_hf_token()
|
||||
try:
|
||||
from huggingface_hub import HfApi
|
||||
except ModuleNotFoundError as exc:
|
||||
raise RuntimeError("huggingface_hub is required for HF upload smoke") from exc
|
||||
|
||||
api = HfApi(token=token)
|
||||
resolved_repo_id = _resolve_hf_repo_id(api, repo_id or os.environ.get("AIRFRANS_HF_REPO_ID") or "airfrans-hf-smoke")
|
||||
api.create_repo(repo_id=resolved_repo_id, repo_type="model", private=False, exist_ok=True)
|
||||
path_in_repo = f"smoke/{run_id}"
|
||||
_write_json(
|
||||
heartbeat_path,
|
||||
{
|
||||
"run_id": run_id,
|
||||
"phase": "uploading",
|
||||
"repo_id": resolved_repo_id,
|
||||
"path_in_repo": path_in_repo,
|
||||
"timestamp": time.time(),
|
||||
},
|
||||
)
|
||||
commit = api.upload_folder(
|
||||
repo_id=resolved_repo_id,
|
||||
repo_type="model",
|
||||
folder_path=str(payload_dir),
|
||||
path_in_repo=path_in_repo,
|
||||
commit_message=f"Add AirfRANS HF smoke artifact {run_id}",
|
||||
)
|
||||
repo_files = set(api.list_repo_files(repo_id=resolved_repo_id, repo_type="model"))
|
||||
expected_paths = [
|
||||
f"{path_in_repo}/final/smoke_model.bin",
|
||||
f"{path_in_repo}/hf_smoke_manifest.json",
|
||||
]
|
||||
missing = [path for path in expected_paths if path not in repo_files]
|
||||
if missing:
|
||||
raise RuntimeError(f"HF upload completed but repo listing is missing: {', '.join(missing)}")
|
||||
|
||||
finished = time.time()
|
||||
hf_manifest = {
|
||||
"repo_id": resolved_repo_id,
|
||||
"repo_url": f"https://huggingface.co/{resolved_repo_id}",
|
||||
"path_in_repo": path_in_repo,
|
||||
"uploaded_paths": expected_paths,
|
||||
"commit": _commit_payload(commit),
|
||||
"started_at": started,
|
||||
"finished_at": finished,
|
||||
"elapsed_seconds": finished - started,
|
||||
}
|
||||
_write_json(output_dir / "hf_upload_manifest.json", hf_manifest)
|
||||
_write_json(
|
||||
output_dir / "run_manifest.json",
|
||||
{
|
||||
"run_id": run_id,
|
||||
"command": "remote-run hf-smoke",
|
||||
"started_at": started,
|
||||
"finished_at": finished,
|
||||
"elapsed_seconds": finished - started,
|
||||
"exit_code": 0,
|
||||
"artifact_dir": str(output_dir),
|
||||
"hf_repo_url": hf_manifest["repo_url"],
|
||||
"hf_path_in_repo": path_in_repo,
|
||||
},
|
||||
)
|
||||
_write_json(
|
||||
heartbeat_path,
|
||||
{
|
||||
"run_id": run_id,
|
||||
"phase": "completed",
|
||||
"latest_metrics": metric,
|
||||
"hf_repo_url": hf_manifest["repo_url"],
|
||||
"hf_path_in_repo": path_in_repo,
|
||||
"started_at": started,
|
||||
"finished_at": finished,
|
||||
"updated_at": time.time(),
|
||||
"timestamp": time.time(),
|
||||
},
|
||||
)
|
||||
verify_artifacts(output_dir)
|
||||
return output_dir
|
||||
|
||||
def run_wandb_smoke(
|
||||
*,
|
||||
artifact_dir: str | Path,
|
||||
run_id: str,
|
||||
entity: str,
|
||||
project: str,
|
||||
hf_repo_url: str | None = None,
|
||||
) -> Path:
|
||||
output_dir = Path(artifact_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
started = time.time()
|
||||
heartbeat_path = output_dir / "heartbeat.json"
|
||||
_write_json(
|
||||
heartbeat_path,
|
||||
{
|
||||
"run_id": run_id,
|
||||
"phase": "starting",
|
||||
"timestamp": started,
|
||||
},
|
||||
)
|
||||
_write_json(output_dir / "environment_manifest.json", environment_manifest())
|
||||
(output_dir / "config.toml").write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"[run]",
|
||||
f'name = "{run_id}"',
|
||||
"kind = \"wandb_smoke\"",
|
||||
"",
|
||||
"[wandb]",
|
||||
f'entity = "{entity}"',
|
||||
f'project = "{project}"',
|
||||
"",
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
os.environ.setdefault("WANDB_API_KEY", _resolve_secret("WANDB_API_KEY", "W&B smoke"))
|
||||
try:
|
||||
import wandb
|
||||
except ModuleNotFoundError as exc:
|
||||
raise RuntimeError("wandb is required for W&B smoke") from exc
|
||||
|
||||
wandb_dir = output_dir.parent / ".wandb"
|
||||
run = wandb.init(
|
||||
entity=entity,
|
||||
project=project,
|
||||
name=run_id,
|
||||
tags=["airfrans", "smoke", "remote-observability"],
|
||||
config={
|
||||
"run_id": run_id,
|
||||
"hf_repo_url": hf_repo_url,
|
||||
"artifact_dir": str(output_dir),
|
||||
},
|
||||
dir=str(wandb_dir),
|
||||
)
|
||||
if run is not None:
|
||||
run.define_metric("step")
|
||||
run.define_metric("*", step_metric="step")
|
||||
metrics_lines: list[str] = []
|
||||
latest_metric: dict[str, Any] = {}
|
||||
for step in range(20):
|
||||
loss = 1.0 / float(step + 1)
|
||||
latest_metric = {
|
||||
"run_id": run_id,
|
||||
"event": "wandb_smoke",
|
||||
"step": step,
|
||||
"loss": loss,
|
||||
"lr": 1.0e-3,
|
||||
"timestamp": time.time(),
|
||||
}
|
||||
wandb.log(latest_metric, step=step)
|
||||
metrics_lines.append(json.dumps(latest_metric, sort_keys=True))
|
||||
if hf_repo_url:
|
||||
run.summary["hf_repo_url"] = hf_repo_url
|
||||
wandb_run_url = run.get_url() if run is not None else None
|
||||
wandb.finish(exit_code=0)
|
||||
|
||||
checkpoint = {
|
||||
"schema_version": 1,
|
||||
"step": 19,
|
||||
"model_state_dict": {},
|
||||
"optimizer_state_dict": {},
|
||||
"normalization": {},
|
||||
}
|
||||
import torch
|
||||
|
||||
for name in ("checkpoint_latest.pt", "checkpoint_best.pt", "checkpoint_final.pt"):
|
||||
torch.save(checkpoint, output_dir / name)
|
||||
|
||||
(output_dir / "metrics.jsonl").write_text("\n".join(metrics_lines) + "\n")
|
||||
_write_json(output_dir / "latest_metrics.json", latest_metric)
|
||||
final_metrics = {
|
||||
"wandb_smoke": True,
|
||||
"loss": latest_metric["loss"],
|
||||
"step": latest_metric["step"],
|
||||
"wandb_run_url": wandb_run_url,
|
||||
"hf_repo_url": hf_repo_url,
|
||||
}
|
||||
_write_json(output_dir / "final_metrics.json", final_metrics)
|
||||
finished = time.time()
|
||||
_write_json(
|
||||
output_dir / "wandb_smoke_manifest.json",
|
||||
{
|
||||
"entity": entity,
|
||||
"project": project,
|
||||
"run_id": run_id,
|
||||
"run_url": wandb_run_url,
|
||||
"logged_steps": 20,
|
||||
"started_at": started,
|
||||
"finished_at": finished,
|
||||
"elapsed_seconds": finished - started,
|
||||
},
|
||||
)
|
||||
_write_json(
|
||||
output_dir / "run_manifest.json",
|
||||
{
|
||||
"run_id": run_id,
|
||||
"command": "remote-run wandb-smoke",
|
||||
"started_at": started,
|
||||
"finished_at": finished,
|
||||
"elapsed_seconds": finished - started,
|
||||
"exit_code": 0,
|
||||
"artifact_dir": str(output_dir),
|
||||
"wandb_run_url": wandb_run_url,
|
||||
},
|
||||
)
|
||||
_write_json(
|
||||
heartbeat_path,
|
||||
{
|
||||
"run_id": run_id,
|
||||
"phase": "completed",
|
||||
"latest_metrics": latest_metric,
|
||||
"wandb_run_url": wandb_run_url,
|
||||
"started_at": started,
|
||||
"finished_at": finished,
|
||||
"updated_at": time.time(),
|
||||
"timestamp": time.time(),
|
||||
},
|
||||
)
|
||||
verify_artifacts(output_dir)
|
||||
return output_dir
|
||||
|
||||
|
||||
|
||||
|
||||
def environment_manifest() -> dict[str, Any]:
|
||||
manifest: dict[str, Any] = {
|
||||
|
|
@ -165,3 +461,48 @@ def _read_json(path: Path) -> dict[str, Any]:
|
|||
|
||||
def _write_json(path: Path, data: Any) -> None:
|
||||
path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def _resolve_hf_token() -> str:
|
||||
return _resolve_secret("HF_TOKEN", "HF upload smoke")
|
||||
|
||||
|
||||
def _resolve_secret(name: str, purpose: str) -> str:
|
||||
value = os.environ.get(name)
|
||||
if value and value.strip():
|
||||
return value.strip()
|
||||
for path in (Path(name), Path(".env") / name):
|
||||
if path.is_file():
|
||||
value = path.read_text().strip()
|
||||
if value:
|
||||
return value
|
||||
env_file = Path(".env")
|
||||
if env_file.is_file():
|
||||
for raw_line in env_file.read_text().splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
if key.strip() == name:
|
||||
value = value.strip().strip("\"'")
|
||||
if value:
|
||||
return value
|
||||
raise RuntimeError(f"{name} env var or local secret file is required for {purpose}")
|
||||
|
||||
|
||||
def _resolve_hf_repo_id(api: Any, repo_id: str) -> str:
|
||||
if "/" in repo_id:
|
||||
return repo_id
|
||||
whoami = api.whoami()
|
||||
namespace = whoami.get("name") if isinstance(whoami, dict) else None
|
||||
if not isinstance(namespace, str) or not namespace:
|
||||
raise RuntimeError("Could not resolve Hugging Face username for unqualified repo id")
|
||||
return f"{namespace}/{repo_id}"
|
||||
|
||||
|
||||
def _commit_payload(commit: Any) -> dict[str, Any]:
|
||||
return {
|
||||
"commit_url": getattr(commit, "commit_url", None),
|
||||
"commit_hash": getattr(commit, "oid", None) or getattr(commit, "commit_hash", None),
|
||||
"pr_url": getattr(commit, "pr_url", None),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -71,6 +71,15 @@ class LossConfig:
|
|||
type: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ObservabilityConfig:
|
||||
backend: str
|
||||
project: str
|
||||
entity: str | None
|
||||
mode: str
|
||||
tags: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TrainingConfig:
|
||||
path: Path
|
||||
|
|
@ -84,7 +93,7 @@ class TrainingConfig:
|
|||
checkpoint: CheckpointConfig
|
||||
stability: StabilityConfig
|
||||
precision: PrecisionConfig
|
||||
|
||||
observability: ObservabilityConfig
|
||||
|
||||
_REQUIRED_SECTIONS = ("run", "data", "model", "optim", "device", "loss")
|
||||
|
||||
|
|
@ -127,6 +136,12 @@ def load_training_config(path: str | Path) -> TrainingConfig:
|
|||
precision_raw = {}
|
||||
if not isinstance(precision_raw, dict):
|
||||
raise ValueError("Training config [precision] section must be a table")
|
||||
observability_raw = raw.get("observability", {})
|
||||
if observability_raw is None:
|
||||
observability_raw = {}
|
||||
if not isinstance(observability_raw, dict):
|
||||
raise ValueError("Training config [observability] section must be a table")
|
||||
|
||||
|
||||
run = RunConfig(
|
||||
name=_string(run_raw, "name"),
|
||||
|
|
@ -173,6 +188,14 @@ def load_training_config(path: str | Path) -> TrainingConfig:
|
|||
precision = PrecisionConfig(
|
||||
dtype=_choice(_string(precision_raw, "dtype", default="float32").lower(), {"float32", "bf16"}, "precision.dtype"),
|
||||
)
|
||||
observability = ObservabilityConfig(
|
||||
backend=_choice(_string(observability_raw, "backend", default="none").lower(), {"none", "wandb"}, "observability.backend"),
|
||||
project=_string(observability_raw, "project", default="airfrans"),
|
||||
entity=_optional_string(observability_raw, "entity"),
|
||||
mode=_choice(_string(observability_raw, "mode", default="online").lower(), {"online", "offline", "disabled"}, "observability.mode"),
|
||||
tags=_string_tuple(observability_raw, "tags", default=()),
|
||||
)
|
||||
|
||||
|
||||
requested_cases = data.train_cases + data.val_cases + data.test_cases
|
||||
if requested_cases <= 0:
|
||||
|
|
@ -190,6 +213,7 @@ def load_training_config(path: str | Path) -> TrainingConfig:
|
|||
checkpoint=checkpoint,
|
||||
stability=stability,
|
||||
precision=precision,
|
||||
observability=observability,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -212,6 +236,12 @@ def _string(section: dict[str, Any], key: str, *, default: str | None = None) ->
|
|||
return value
|
||||
|
||||
|
||||
def _optional_string(section: dict[str, Any], key: str) -> str | None:
|
||||
if key not in section:
|
||||
return None
|
||||
return _string(section, key)
|
||||
|
||||
|
||||
def _integer(section: dict[str, Any], key: str, *, minimum: int | None = None, default: int | None = None) -> int:
|
||||
if key not in section:
|
||||
if default is not None:
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from torch.nn import functional as F
|
|||
from airfrans_frontier.models import FourierFiLMMLP, PointwiseMLP
|
||||
from airfrans_frontier.training.artifacts import ArtifactWriter
|
||||
from airfrans_frontier.training.config import TrainingConfig, load_training_config
|
||||
from airfrans_frontier.training.observability import start_observer
|
||||
from airfrans_frontier.training.data import DatasetBundle, build_dataset_bundle, load_processed_dataset
|
||||
from airfrans_frontier.training.metrics import count_parameters, device_metrics, overall_mse, per_channel_mse
|
||||
from airfrans_frontier.training.normalize import (
|
||||
|
|
@ -28,7 +29,6 @@ CHECKPOINT_SCHEMA_VERSION = 1
|
|||
LATEST_CHECKPOINT = "checkpoint_latest.pt"
|
||||
BEST_CHECKPOINT = "checkpoint_best.pt"
|
||||
FINAL_CHECKPOINT = "checkpoint_final.pt"
|
||||
LEGACY_CHECKPOINT = "checkpoint.pt"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
|
@ -48,6 +48,12 @@ def train(config: TrainingConfig, *, resume_path: str | Path | None = None) -> T
|
|||
resume = Path(resume_path).expanduser() if resume_path else None
|
||||
writer = ArtifactWriter.resume_or_create(config.run.artifact_dir, config.run.name, resume)
|
||||
writer.write_config(config.config_text)
|
||||
observer = start_observer(config, run_dir=writer.run_dir)
|
||||
|
||||
def record_metrics(metrics: dict[str, Any]) -> None:
|
||||
writer.append_metrics(metrics)
|
||||
observer.log(metrics)
|
||||
|
||||
|
||||
samples = load_processed_dataset(config.data.root)
|
||||
bundle = build_dataset_bundle(
|
||||
|
|
@ -125,7 +131,7 @@ def train(config: TrainingConfig, *, resume_path: str | Path | None = None) -> T
|
|||
latest_checkpoint=str(resume),
|
||||
)
|
||||
raise
|
||||
writer.append_metrics(
|
||||
record_metrics(
|
||||
_log_metrics(
|
||||
event="resume",
|
||||
step=start_step,
|
||||
|
|
@ -165,7 +171,7 @@ def train(config: TrainingConfig, *, resume_path: str | Path | None = None) -> T
|
|||
if best_val_loss is None and initial_val is not None:
|
||||
best_val_loss = initial_val["loss"]
|
||||
|
||||
writer.append_metrics(
|
||||
record_metrics(
|
||||
_log_metrics(
|
||||
event="initial_eval" if start_step == 0 else "resume_eval",
|
||||
step=start_step,
|
||||
|
|
@ -323,7 +329,7 @@ def train(config: TrainingConfig, *, resume_path: str | Path | None = None) -> T
|
|||
elapsed = time.perf_counter() - started
|
||||
interval_elapsed = max(time.perf_counter() - last_log_at, 1e-9)
|
||||
points_per_sec = (step - last_log_step) * config.data.batch_size / interval_elapsed
|
||||
writer.append_metrics(
|
||||
record_metrics(
|
||||
_log_metrics(
|
||||
event="train_eval",
|
||||
step=step,
|
||||
|
|
@ -399,6 +405,9 @@ def train(config: TrainingConfig, *, resume_path: str | Path | None = None) -> T
|
|||
"resumed_from": str(resume) if resume is not None else None,
|
||||
**device_metrics(device),
|
||||
}
|
||||
if observer.url is not None:
|
||||
final_metrics["wandb_run_url"] = observer.url
|
||||
observer.update_summary(final_metrics)
|
||||
writer.write_final_metrics(final_metrics)
|
||||
_save_training_checkpoint(
|
||||
writer,
|
||||
|
|
@ -414,21 +423,7 @@ def train(config: TrainingConfig, *, resume_path: str | Path | None = None) -> T
|
|||
initial_train_loss=initial_train_loss,
|
||||
final_metrics=final_metrics,
|
||||
)
|
||||
_save_training_checkpoint(
|
||||
writer,
|
||||
LEGACY_CHECKPOINT,
|
||||
config=config,
|
||||
bundle=bundle,
|
||||
stats=stats,
|
||||
model=model,
|
||||
optimizer=optimizer,
|
||||
rng=rng,
|
||||
step=config.optim.steps,
|
||||
best_val_loss=best_val_loss,
|
||||
initial_train_loss=initial_train_loss,
|
||||
final_metrics=final_metrics,
|
||||
)
|
||||
writer.append_metrics(
|
||||
record_metrics(
|
||||
_log_metrics(
|
||||
event="completed",
|
||||
phase="completed",
|
||||
|
|
@ -444,6 +439,7 @@ def train(config: TrainingConfig, *, resume_path: str | Path | None = None) -> T
|
|||
)
|
||||
)
|
||||
writer.write_artifact_manifest()
|
||||
observer.finish(exit_code=0)
|
||||
return TrainingResult(run_dir=writer.run_dir, final_metrics=final_metrics)
|
||||
|
||||
def _autocast_context(config: TrainingConfig, device: torch.device):
|
||||
|
|
|
|||
92
src/airfrans_frontier/training/observability.py
Normal file
92
src/airfrans_frontier/training/observability.py
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, is_dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
|
||||
from airfrans_frontier.training.config import TrainingConfig
|
||||
|
||||
|
||||
class TrainingObserver:
|
||||
@property
|
||||
def url(self) -> str | None:
|
||||
return None
|
||||
|
||||
def log(self, metrics: Mapping[str, Any]) -> None:
|
||||
return None
|
||||
|
||||
def update_summary(self, metrics: Mapping[str, Any]) -> None:
|
||||
return None
|
||||
|
||||
def finish(self, *, exit_code: int = 0) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class WandbObserver(TrainingObserver):
|
||||
def __init__(self, run: Any, wandb_module: Any) -> None:
|
||||
self._run = run
|
||||
self._wandb = wandb_module
|
||||
|
||||
@property
|
||||
def url(self) -> str | None:
|
||||
get_url = getattr(self._run, "get_url", None)
|
||||
if callable(get_url):
|
||||
return get_url()
|
||||
url = getattr(self._run, "url", None)
|
||||
return str(url) if url else None
|
||||
|
||||
def log(self, metrics: Mapping[str, Any]) -> None:
|
||||
payload = _json_safe(dict(metrics))
|
||||
step = payload.get("step")
|
||||
if isinstance(step, int):
|
||||
self._wandb.log(payload, step=step)
|
||||
else:
|
||||
self._wandb.log(payload)
|
||||
|
||||
def update_summary(self, metrics: Mapping[str, Any]) -> None:
|
||||
for key, value in _json_safe(dict(metrics)).items():
|
||||
self._run.summary[key] = value
|
||||
|
||||
def finish(self, *, exit_code: int = 0) -> None:
|
||||
self._wandb.finish(exit_code=exit_code)
|
||||
|
||||
|
||||
def start_observer(config: TrainingConfig, *, run_dir: Path) -> TrainingObserver:
|
||||
observability = config.observability
|
||||
if observability.backend == "none" or observability.mode == "disabled":
|
||||
return TrainingObserver()
|
||||
if observability.backend != "wandb":
|
||||
raise ValueError(f"Unsupported observability backend: {observability.backend}")
|
||||
try:
|
||||
import wandb
|
||||
except ModuleNotFoundError as exc:
|
||||
raise RuntimeError("wandb is required when [observability].backend = 'wandb'") from exc
|
||||
|
||||
wandb_dir = run_dir.parent / ".wandb"
|
||||
run = wandb.init(
|
||||
entity=observability.entity,
|
||||
project=observability.project,
|
||||
name=config.run.name,
|
||||
tags=list(observability.tags),
|
||||
mode=observability.mode,
|
||||
config=_json_safe(asdict(config)),
|
||||
dir=str(wandb_dir),
|
||||
)
|
||||
if run is not None:
|
||||
run.define_metric("step")
|
||||
run.define_metric("*", step_metric="step")
|
||||
return WandbObserver(run, wandb)
|
||||
|
||||
|
||||
def _json_safe(value: Any) -> Any:
|
||||
if is_dataclass(value):
|
||||
return _json_safe(asdict(value))
|
||||
if isinstance(value, Path):
|
||||
return str(value)
|
||||
if isinstance(value, Mapping):
|
||||
return {str(key): _json_safe(item) for key, item in value.items()}
|
||||
if isinstance(value, tuple):
|
||||
return [_json_safe(item) for item in value]
|
||||
if isinstance(value, list):
|
||||
return [_json_safe(item) for item in value]
|
||||
return value
|
||||
|
|
@ -24,6 +24,26 @@ class RemoteRunConfigTests(unittest.TestCase):
|
|||
self.assertEqual(config.job.artifact_dir.as_posix(), "artifacts/current_run")
|
||||
self.assertIn("checkpoint_latest.pt", config.artifacts.required)
|
||||
|
||||
def test_loads_remote_hf_smoke_config_for_cheap_upload(self) -> None:
|
||||
config = load_remote_run_config("configs/remote_hf_smoke.toml")
|
||||
|
||||
self.assertEqual(config.provider.kind, "vastai")
|
||||
self.assertEqual(config.provider.gpu.name, "RTX 3060")
|
||||
self.assertLessEqual(config.provider.max_price_per_hour or 999.0, 0.08)
|
||||
self.assertEqual(config.artifacts.mode, "object_store_upload")
|
||||
self.assertIn("remote-run hf-smoke", config.job.command)
|
||||
self.assertIn("hf_upload_manifest.json", config.artifacts.required)
|
||||
|
||||
def test_loads_remote_wandb_smoke_config_for_observability(self) -> None:
|
||||
config = load_remote_run_config("configs/remote_wandb_smoke.toml")
|
||||
|
||||
self.assertEqual(config.provider.kind, "vastai")
|
||||
self.assertEqual(config.provider.gpu.name, "RTX 3060 Ti")
|
||||
self.assertLessEqual(config.provider.max_price_per_hour or 999.0, 0.09)
|
||||
self.assertEqual(config.artifacts.mode, "rsync")
|
||||
self.assertIn("remote-run wandb-smoke", config.job.command)
|
||||
self.assertIn("wandb_smoke_manifest.json", config.artifacts.required)
|
||||
|
||||
|
||||
class VastSelectionTests(unittest.TestCase):
|
||||
def test_selection_filters_bad_hosts_and_drops_suspiciously_cheap_tail(self) -> None:
|
||||
|
|
@ -92,6 +112,13 @@ class ArtifactVerificationTests(unittest.TestCase):
|
|||
|
||||
self.assertGreaterEqual(manifest["file_count"], 7)
|
||||
|
||||
def test_classifies_remote_success_before_checkpoint_collection(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
(root / "final_metrics.json").write_text(json.dumps({"loss": 1.0}) + "\n")
|
||||
|
||||
self.assertEqual(_classify_artifacts(root), "success")
|
||||
|
||||
def test_classifies_and_stages_restart_checkpoint(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
|
|
|
|||
|
|
@ -159,7 +159,7 @@ class TrainingLoopTests(unittest.TestCase):
|
|||
self.assertTrue(np.isfinite(final_metrics["train_loss"]))
|
||||
self.assertLess(final_metrics["train_loss"], final_metrics["initial_train_loss"] * 0.1)
|
||||
self.assertLess(final_metrics["train_loss"], 1e-2)
|
||||
self.assertTrue((run_dir / "checkpoint.pt").exists())
|
||||
self.assertFalse((run_dir / "checkpoint.pt").exists())
|
||||
self.assertTrue((run_dir / "checkpoint_latest.pt").exists())
|
||||
self.assertTrue((run_dir / "checkpoint_best.pt").exists())
|
||||
self.assertTrue((run_dir / "checkpoint_final.pt").exists())
|
||||
|
|
|
|||
114
uv.lock
114
uv.lock
|
|
@ -176,9 +176,11 @@ name = "airfrans-frontier"
|
|||
version = "0.1.0"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "huggingface-hub" },
|
||||
{ name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" },
|
||||
{ name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
|
||||
{ name = "torch" },
|
||||
{ name = "wandb" },
|
||||
]
|
||||
|
||||
[package.dev-dependencies]
|
||||
|
|
@ -195,8 +197,10 @@ dev = [
|
|||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "huggingface-hub", specifier = ">=0.36.0" },
|
||||
{ name = "numpy", specifier = ">=2.4.0" },
|
||||
{ name = "torch", specifier = ">=2.8.0" },
|
||||
{ name = "wandb", specifier = ">=0.23.0" },
|
||||
]
|
||||
|
||||
[package.metadata.requires-dev]
|
||||
|
|
@ -1244,6 +1248,30 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hf-xet"
|
||||
version = "1.5.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/63/39/67be8d71f900d9a55761b6022821d6679fb56c64f1b6063d5af2c2606727/hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47", size = 903674, upload-time = "2026-07-16T17:29:56.833Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/29/be/525eabac5d1736b679c39e342ecd4292534012546a2d18f0043c8e3b6021/hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b", size = 4064284, upload-time = "2026-07-16T17:29:29.907Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c5/3f/699749dd78442480eda4e4fca494284b0e3542e4063cc37654d5fdc929e6/hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576", size = 3828537, upload-time = "2026-07-16T17:29:31.549Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/22/d7/2658ac0a5b9f4664ca27ce31bd015044fe9dea50ed455fb5197aba819c11/hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4", size = 4417133, upload-time = "2026-07-16T17:29:33.391Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d9/58/8343f3cb63c8fa058d576136df3871550f7d5214a8f048a7ea2eab6ac906/hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4", size = 4212613, upload-time = "2026-07-16T17:29:34.989Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0c/33/a968f4e4535037b36941ec00714625fb60e026302407e7e26ca9f3e65f4e/hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380", size = 4412710, upload-time = "2026-07-16T17:29:36.646Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7b/d9/9e33981173dbaf194ba0015202b02d467b624d44d4eba89e1bf06c0d2995/hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577", size = 4628455, upload-time = "2026-07-16T17:29:38.352Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e9/4b/cc682832de4264a03880a2d1b5ec3e1fab3bf307f508817250baafdb9996/hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e", size = 3979044, upload-time = "2026-07-16T17:29:40.329Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ea/09/b2cdf2a0fb39a08af3222b96092a36bd3b40c54123eef07de4422e870971/hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e", size = 3808037, upload-time = "2026-07-16T17:29:42.357Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/de/ba/2b70603c7552db82baeb2623e2336898304a17328845151be4fe1f48d420/hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799", size = 4033760, upload-time = "2026-07-16T17:29:43.884Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/60/ac/b097a86a1e4a6098f3a79382643ab09d5733d87ccc864877ad1e12b49b70/hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed", size = 3841438, upload-time = "2026-07-16T17:29:45.539Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d3/35/db860aa3a0780660324a506ad4b3d322ddc6ecbba4b9340aed0942cbf21c/hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c", size = 4428006, upload-time = "2026-07-16T17:29:46.996Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/af/6b/832dd980af4b0c3ae0660e309285f2ffcdff2faa38129390dbb47aa4a3f9/hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025", size = 4221099, upload-time = "2026-07-16T17:29:48.525Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9e/05/ae50f0d34e3254e6c3e208beb2519f6b8673016fc4b3643badaf6450d186/hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f", size = 4420766, upload-time = "2026-07-16T17:29:50.092Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/07/a9/c050bc2743a2bcd68928bfee157b08681667a164a24ec95fbfcfcd717e08/hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097", size = 4636716, upload-time = "2026-07-16T17:29:51.62Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e9/f8/68b01c5c2edb56ac9a67b3d076ffddcb90867abaee923923eb34e7a14e76/hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65", size = 3988373, upload-time = "2026-07-16T17:29:53.395Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/39/c6/988383e9dc17294d536fcbcd6fd16eed882e411ad16c954984a53e47b09c/hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d", size = 3816957, upload-time = "2026-07-16T17:29:54.991Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "httpcore"
|
||||
version = "1.0.9"
|
||||
|
|
@ -1315,6 +1343,26 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "huggingface-hub"
|
||||
version = "1.16.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "filelock" },
|
||||
{ name = "fsspec" },
|
||||
{ name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" },
|
||||
{ name = "httpx" },
|
||||
{ name = "packaging" },
|
||||
{ name = "pyyaml" },
|
||||
{ name = "tqdm" },
|
||||
{ name = "typer" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/48/0f/ed994dbade67a54407c28cab96ef845e0e6d25500be56aca6394f8bfc9dd/huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832", size = 792534, upload-time = "2026-05-21T18:40:00.908Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/49/79/621a7dbb80c70974f73a597275351ebe03ce5bc65cb5f8f4acb5859252bc/huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22", size = 668176, upload-time = "2026-05-21T18:39:58.596Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "3.18"
|
||||
|
|
@ -3598,6 +3646,19 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/10/85/48f0abdcef5cce4e034c7a5b0ceeceba0b01bf0d942824f4bb720afe2dec/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:8e65860d238379ed982fd9ba690579b5e95af2f4840f99c772816dbe573cb826", size = 586486, upload-time = "2026-06-30T07:17:51.141Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sentry-sdk"
|
||||
version = "2.66.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "certifi" },
|
||||
{ name = "urllib3" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/7f/6f/d59cad0889d15fde85254cf58e701484de3f3f0406003b3197746910b19b/sentry_sdk-2.66.1.tar.gz", hash = "sha256:f882fb08710c5f8bfc603aafa3e901b384009a19cc3f76a572b863392ee81cdc", size = 940543, upload-time = "2026-07-22T12:26:54.553Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/89/d3/726bd88f0eece09ddf431bea4c9191c18e7a8d070b854eb0014d447712ee/sentry_sdk-2.66.1-py3-none-any.whl", hash = "sha256:86002793161d9a95ef04bdd8d442e9bfece5d989b755f05d6360215094a7aff6", size = 505555, upload-time = "2026-07-22T12:26:52.71Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "setproctitle"
|
||||
version = "1.3.7"
|
||||
|
|
@ -3678,6 +3739,15 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shellingham"
|
||||
version = "1.5.4"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "simpleeval"
|
||||
version = "1.0.7"
|
||||
|
|
@ -3987,6 +4057,21 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/f0/ac/229b7d4589d2e5937310e72c6d46e89599d16a4a12b479ffa1499fee8eb8/triton-3.7.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10ba85fa2cca4a2fbdeb36bf1cb082f2c252bda55bf9fccd74f65ec5bc647e68", size = 197824404, upload-time = "2026-06-17T19:53:42.772Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typer"
|
||||
version = "0.27.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "annotated-doc" },
|
||||
{ name = "colorama", marker = "sys_platform == 'win32'" },
|
||||
{ name = "rich" },
|
||||
{ name = "shellingham" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/37/78/fda3361b56efc27944f24225f6ecd13d96d6fcfe37bd0eb34e2f4c63f9fc/typer-0.27.0.tar.gz", hash = "sha256:629bd12ea5d13a17148125d9a264f949eb171fb3f120f9b04d85873cab054fa5", size = 203430, upload-time = "2026-07-15T19:21:07.007Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/40/03/26a383c9e58c213199d1aad1c3d353cfc22d4444ec6d2c0bf8ad02523843/typer-0.27.0-py3-none-any.whl", hash = "sha256:6f4b27631e47f077871b7dc30e933ec0131c1390fbe0e387ea5574b5bac9ccf1", size = 122716, upload-time = "2026-07-15T19:21:05.553Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "types-paramiko"
|
||||
version = "5.0.0.20260617"
|
||||
|
|
@ -4121,6 +4206,35 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/de/4d/a801a53239c08aa81fcd144a1b0b58909febf503cb817c6b4da1cb23d29f/vastai_sdk-0.2.5-py3-none-any.whl", hash = "sha256:c8c4568554848cf6e439d754591fa83ec30cc4c380463bbdb2bf9d8549391e85", size = 72783, upload-time = "2025-11-19T01:14:44.331Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wandb"
|
||||
version = "0.26.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "click" },
|
||||
{ name = "gitpython" },
|
||||
{ name = "packaging" },
|
||||
{ name = "platformdirs" },
|
||||
{ name = "protobuf" },
|
||||
{ name = "pydantic" },
|
||||
{ name = "pyyaml" },
|
||||
{ name = "requests" },
|
||||
{ name = "sentry-sdk" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/6a/a4/72a6640e1f566e81f184a426e3e45298d4c6672664de41adb7eb6f64370a/wandb-0.26.1.tar.gz", hash = "sha256:eef2dbaea06f0b1c0cdc5d76f544ae4c2b8848fc512442a00bd59f0502fc8aa1", size = 42159814, upload-time = "2026-04-23T16:27:34.033Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/8c/09/3296235f3906e904f06f2df29eed4d672fb23c0932c9486e2af64f2f2a66/wandb-0.26.1-py3-none-macosx_12_0_arm64.whl", hash = "sha256:2955fe190c005fb83ee6d73f066c8a33f09f3212a1f2eb53faa6581440e456be", size = 24857204, upload-time = "2026-04-23T16:26:58.576Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a1/ad/e39ca3086534129e42208ba00ed2c6247ce425f890219eeec33b4f162864/wandb-0.26.1-py3-none-macosx_12_0_x86_64.whl", hash = "sha256:55d91cabde98162d7116a5e19ddd052bd9848556243f1da4cbb9ffb7ad435bfc", size = 26014649, upload-time = "2026-04-23T16:27:02.559Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/56/af/400d84a3bdce0b062b4baa70acb6becd2c8018697f4fbf5af9a9e1e406e5/wandb-0.26.1-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:7c78bc2454cfe1ffa1c3a256060a387356eed8a4488e024d9d2eba8f2b5bd51d", size = 25421317, upload-time = "2026-04-23T16:27:06.411Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7b/e9/b4bf8f3509dcea1cec52233a38991459654635b5a8e6a494eb912e1b9cfb/wandb-0.26.1-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:a2c8eeec8706dcd2872e69c3b4d20ec523082fdb4440295491556e219ad2aa67", size = 27192831, upload-time = "2026-04-23T16:27:10.308Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/62/cf/4a6dce0c782223ef0eeea7139daee73418a7322befcf083512c31cebaa18/wandb-0.26.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:2fa768ee0636a569afb7541cf996e56309c47070566a38916823f94e02afe586", size = 25593326, upload-time = "2026-04-23T16:27:14.259Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/df/99/58c3d8c36ae8e2b7d70bf6493eb5daa1cca0231a04b025717b4cd1a78f1e/wandb-0.26.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:5854928725cfeff1f284d5c043cd353f810e5da02eead2c120ef5056ad026fea", size = 27535542, upload-time = "2026-04-23T16:27:18.473Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7c/d0/4e846ffc1d0cc435518dfa581ce73ac82cfd0ebbf35f3853c9277f632e5f/wandb-0.26.1-py3-none-win32.whl", hash = "sha256:5c2bd44e575ae9944e2764d1aaa031461178276bf2636d5558399c2816ef5cfe", size = 24968151, upload-time = "2026-04-23T16:27:22.086Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e3/9b/487413eaccefdb58799a226726e24b486e9192d2671c75a4550c160aba23/wandb-0.26.1-py3-none-win_amd64.whl", hash = "sha256:5817785467d3f1676f1812ec19a89f77f6e56dfe67d9f47080075af95f705d3e", size = 24968155, upload-time = "2026-04-23T16:27:25.731Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/04/dc/5baf3e99b3eeb709d6f75124b5bec8cb73d4b38d2b10df7fdcfde4966200/wandb-0.26.1-py3-none-win_arm64.whl", hash = "sha256:f848b7744f896bc04cabbb28360a2814d1551a91fa2c456243e06435729c8a2e", size = 22912416, upload-time = "2026-04-23T16:27:29.456Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "watchfiles"
|
||||
version = "1.2.0"
|
||||
|
|
|
|||
Loading…
Reference in a new issue