feat(*): add staged curriculum training with an automated stage-by-stage orchestrator

training/curriculum.py

@@ -0,0 +1,258 @@
+"""Orchestrate the staged curriculum (see TRAINING.md's "Curriculum training"
+section): run each stage, evaluate the result against a reference bot, and
+either advance to the next stage or retry the same one.
+
+State is persisted to curriculum_state.json (committed to git) so the script
+is safe to Ctrl-C and re-run — it picks up exactly where it left off. Each
+attempt reuses run_training.sh (pull, train, export, commit+push) so every
+attempt's checkpoint, log, and exported policy is versioned like any other
+run; this script additionally evaluates the result and commits the updated
+eval_history.json + curriculum_state.json.
+
+The gate is deliberately lenient ("block only on a clear regression," not
+"require improvement") — see TRAINING.md. A 40-episode eval can call a real
+improvement a regression on sample noise alone (this happened with run11:
+it was the first model to deliberately score, but lost its head-to-head
+evals). A strict improvement-required gate would have retried that stage
+forever for the wrong reason. When a stage does fail MAX_RETRIES times in a
+row, the script stops and asks for a human look rather than retrying
+indefinitely or silently advancing past a bad stage.
+
+Usage:
+    .venv/bin/python curriculum.py                  # run/resume the curriculum
+    .venv/bin/python curriculum.py --seed-checkpoint checkpoints/run11/final.zip
+    .venv/bin/python curriculum.py --force-retry     # after fixing something, retry the blocked stage
+    .venv/bin/python curriculum.py --skip-to-next-stage  # human judgment call: good enough, move on anyway
+
+Typically started via curriculum.sh, which runs this in a detached tmux
+session the way start_training.sh does for a single run.
+"""
+
+import argparse
+import json
+import pathlib
+import subprocess
+import sys
+
+TRAINING_DIR = pathlib.Path(__file__).resolve().parent
+STATE_PATH = TRAINING_DIR / "curriculum_state.json"
+EVAL_HISTORY_PATH = TRAINING_DIR / "eval_history.json"
+ROOKIE_REFERENCE = TRAINING_DIR.parent / "Game" / "bots" / "rookie.json"
+
+MAX_RETRIES = 2
+EVAL_EPISODES = 100
+# "Clear regression" = the reference beats the candidate by at least this
+# many percentage points of win rate. Below this, noise in a 100-episode
+# sample is a more likely explanation than the stage actually failing (see
+# module docstring) — advance rather than retry.
+REGRESSION_MARGIN = 0.15
+
+# Standing flags applied to every attempt, mirroring next_run.sh: reset-std
+# reopens exploration every attempt (harmless on fresh starts — train.py
+# only applies it on --resume), ent-coef keeps it from re-collapsing.
+STANDING_ARGS = ["--reset-std", "0.3", "--ent-coef", "0.001"]
+
+STAGES = [
+    {
+        "name": "score",
+        "flags": [
+            "--opponent-mode", "inert",
+            "--attack-goal-bias", "1.0",
+            "--no-allow-vertical", "--no-allow-pitch-roll",
+        ],
+    },
+    {
+        "name": "defend",
+        "flags": [
+            "--opponent-mode", "self_play",
+            "--no-allow-vertical", "--no-allow-pitch-roll",
+        ],
+    },
+    {
+        "name": "no_draws",
+        "flags": ["--draw-penalty", "5"],
+    },
+    {
+        "name": "mechanics",
+        "flags": [],
+    },
+]
+
+
+def load_state() -> dict:
+    if STATE_PATH.exists():
+        return json.loads(STATE_PATH.read_text())
+    return {"stage_index": 0, "attempt": 0, "status": "in_progress", "log": []}
+
+
+def save_state(state: dict) -> None:
+    STATE_PATH.write_text(json.dumps(state, indent=2) + "\n")
+
+
+def experiment_name(stage_index: int, attempt: int) -> str:
+    name = f"curric-s{stage_index + 1}-{STAGES[stage_index]['name']}"
+    return name if attempt == 0 else f"{name}-retry{attempt}"
+
+
+def resume_checkpoint(stage_index: int, attempt: int, seed_checkpoint: str | None) -> str | None:
+    if attempt > 0:
+        # Retry: keep training the same stage's own last attempt.
+        prev = experiment_name(stage_index, attempt - 1)
+        return str(TRAINING_DIR / "checkpoints" / prev / "final.zip")
+    if stage_index == 0:
+        # Deliberately fresh by default: the curriculum exists because
+        # resuming self-play across a regime change (run10, run11) didn't
+        # work, so stage 1 starts from a random policy under its own
+        # regime unless --seed-checkpoint says otherwise.
+        return seed_checkpoint
+    prev_stage = STAGES[stage_index - 1]["name"]
+    prev_experiment = _passing_experiment_for_stage(stage_index - 1)
+    return str(TRAINING_DIR / "checkpoints" / prev_experiment / "final.zip")
+
+
+def reference_bot(stage_index: int) -> str:
+    if stage_index == 0:
+        return str(ROOKIE_REFERENCE)
+    prev_experiment = _passing_experiment_for_stage(stage_index - 1)
+    return str(TRAINING_DIR.parent / "Game" / "bots" / f"{prev_experiment}.json")
+
+
+def _passing_experiment_for_stage(stage_index: int) -> str:
+    state = load_state()
+    for entry in state["log"]:
+        if entry["stage_index"] == stage_index and entry["decision"] == "pass":
+            return entry["experiment"]
+    raise RuntimeError(f"No passing attempt recorded for stage {stage_index} ({STAGES[stage_index]['name']})")
+
+
+def run_stage_attempt(stage_index: int, attempt: int, args) -> str:
+    exp = experiment_name(stage_index, attempt)
+    resume = resume_checkpoint(stage_index, attempt, args.seed_checkpoint)
+    cmd = [
+        "./run_training.sh", exp,
+        "--timesteps", str(args.timesteps),
+        "--n-parallel", str(args.n_parallel),
+        "--speedup", str(args.speedup),
+        *STANDING_ARGS,
+        *STAGES[stage_index]["flags"],
+    ]
+    if resume:
+        cmd += ["--resume", resume]
+    print(f"\n=== Stage {stage_index + 1}/{len(STAGES)} ({STAGES[stage_index]['name']}), "
+          f"attempt {attempt + 1}/{MAX_RETRIES + 1}: {exp} ===")
+    print(" ".join(cmd))
+    subprocess.run(cmd, cwd=TRAINING_DIR, check=True)
+    return exp
+
+
+def evaluate_attempt(experiment: str, reference: str, episodes: int) -> dict:
+    candidate = TRAINING_DIR.parent / "Game" / "bots" / f"{experiment}.json"
+    cmd = [".venv/bin/python", "evaluate.py", str(candidate), reference, "--episodes", str(episodes)]
+    print(" ".join(cmd))
+    subprocess.run(cmd, cwd=TRAINING_DIR, check=True)
+    history = json.loads(EVAL_HISTORY_PATH.read_text())
+    return history[-1]
+
+
+def decide(record: dict) -> str:
+    win_rate_candidate = record["wins_a"] / record["episodes"]
+    win_rate_reference = record["wins_b"] / record["episodes"]
+    if win_rate_reference - win_rate_candidate >= REGRESSION_MARGIN:
+        return "fail"
+    return "pass"
+
+
+def commit_progress(experiment: str) -> None:
+    subprocess.run(["git", "add", "curriculum_state.json", "eval_history.json"], cwd=TRAINING_DIR, check=True)
+    result = subprocess.run(["git", "diff", "--cached", "--quiet"], cwd=TRAINING_DIR)
+    if result.returncode == 0:
+        return
+    subprocess.run(
+        ["git", "commit", "-m", f"chore(training): curriculum progress after {experiment}"],
+        cwd=TRAINING_DIR, check=True,
+    )
+    subprocess.run(["git", "push"], cwd=TRAINING_DIR, check=True)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--timesteps", type=int, default=20_000_000)
+    parser.add_argument("--n-parallel", type=int, default=14)
+    parser.add_argument("--speedup", type=int, default=16)
+    parser.add_argument("--seed-checkpoint", default=None, help="Resume stage 1 from this checkpoint instead of a fresh policy")
+    parser.add_argument("--force-retry", action="store_true", help="Retry a blocked stage after human review")
+    parser.add_argument("--skip-to-next-stage", action="store_true", help="Human judgment call: treat the blocked stage as good enough, advance anyway")
+    args = parser.parse_args()
+
+    state = load_state()
+
+    if state["status"] == "blocked":
+        if args.skip_to_next_stage:
+            print(f"Human override: advancing past stage {state['stage_index'] + 1} "
+                  f"({STAGES[state['stage_index']]['name']}) despite exhausted retries.")
+            state["log"].append({
+                "stage_index": state["stage_index"], "experiment": experiment_name(state["stage_index"], state["attempt"]),
+                "attempt": state["attempt"], "decision": "pass", "override": "skip_to_next_stage",
+            })
+            state["stage_index"] += 1
+            state["attempt"] = 0
+            state["status"] = "in_progress"
+            save_state(state)
+        elif args.force_retry:
+            print(f"Human override: retrying stage {state['stage_index'] + 1} "
+                  f"({STAGES[state['stage_index']]['name']}) after review.")
+            state["attempt"] += 1
+            state["status"] = "in_progress"
+            save_state(state)
+        else:
+            print(f"BLOCKED at stage {state['stage_index'] + 1} ({STAGES[state['stage_index']]['name']}) "
+                  f"after {MAX_RETRIES + 1} attempts — see curriculum_state.json's log for eval results.")
+            print("Re-run with --force-retry (after adjusting flags/timesteps) or "
+                  "--skip-to-next-stage (advance anyway) once you've looked at why.")
+            sys.exit(1)
+
+    while state["stage_index"] < len(STAGES):
+        stage_index = state["stage_index"]
+        attempt = state["attempt"]
+
+        experiment = run_stage_attempt(stage_index, attempt, args)
+        reference = reference_bot(stage_index)
+        record = evaluate_attempt(experiment, reference, EVAL_EPISODES)
+        decision = decide(record)
+
+        print(f"{experiment}: candidate {record['wins_a']}-{record['wins_b']} reference "
+              f"({record['draws']} draws) over {record['episodes']} episodes -> {decision}")
+
+        state["log"].append({
+            "stage_index": stage_index, "experiment": experiment, "attempt": attempt,
+            "eval": record, "decision": decision,
+        })
+
+        if decision == "pass":
+            state["stage_index"] += 1
+            state["attempt"] = 0
+            state["status"] = "in_progress"
+            save_state(state)
+            commit_progress(experiment)
+            continue
+
+        if attempt >= MAX_RETRIES:
+            state["status"] = "blocked"
+            save_state(state)
+            commit_progress(experiment)
+            print(f"\nBLOCKED: stage {stage_index + 1} ({STAGES[stage_index]['name']}) failed "
+                  f"{MAX_RETRIES + 1} attempts in a row. Stopping for human review — see "
+                  f"curriculum_state.json. Re-run with --force-retry or --skip-to-next-stage.")
+            sys.exit(1)
+
+        state["attempt"] += 1
+        save_state(state)
+        commit_progress(experiment)
+
+    print("\nCurriculum complete — all stages passed.")
+    state["status"] = "done"
+    save_state(state)
+
+
+if __name__ == "__main__":
+    main()

training/curriculum.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Runs the staged curriculum (see TRAINING.md, curriculum.py) detached from
+# your SSH session, the same way start_training.sh does for a single run —
+# safe to disconnect any time. Uses its own tmux session ("cosmic-curriculum")
+# so it doesn't collide with a manual next_run.sh/start_training.sh run.
+#
+# Usage: ./curriculum.sh [curriculum.py args...]
+# e.g.:  ./curriculum.sh
+#        ./curriculum.sh --seed-checkpoint checkpoints/run11/final.zip
+#        ./curriculum.sh --force-retry
+set -euo pipefail
+cd "$(dirname "$0")"
+
+SESSION="cosmic-curriculum"
+TB_PORT=6006
+
+command -v tmux >/dev/null 2>&1 || { echo "tmux is required: sudo apt install tmux" >&2; exit 1; }
+
+if tmux has-session -t "$SESSION" 2>/dev/null; then
+    echo "Session '$SESSION' already running — attaching (detach with Ctrl-B then D)."
+    exec tmux attach -t "$SESSION"
+fi
+
+tmux new-session -d -s "$SESSION" -n curriculum \
+    ".venv/bin/python curriculum.py $*; echo; echo '=== curriculum.py exited — press Enter to close ==='; read"
+
+if ! (exec 3<>"/dev/tcp/127.0.0.1/$TB_PORT") 2>/dev/null; then
+    tmux new-window -d -t "$SESSION" -n dashboard \
+        ".venv/bin/tensorboard --logdir logs --host 0.0.0.0 --port $TB_PORT"
+fi
+
+IP=$(hostname -I 2>/dev/null | awk '{print $1}')
+echo "Curriculum started in tmux session '$SESSION' — SSH disconnects won't touch it."
+echo "  watch it:   tmux attach -t $SESSION   (detach again with Ctrl-B then D)"
+echo "  dashboard:  http://${IP:-<this-box>}:$TB_PORT"
+echo "  progress:   cat curriculum_state.json"

training/train.py

@@ -51,9 +51,60 @@ def parse_args():
     parser.add_argument("--checkpoint-every", type=int, default=100_000, help="Timesteps between checkpoints")
     parser.add_argument("--viz", action="store_true", help="Show game windows (debugging; slow)")
     parser.add_argument("--wandb", action="store_true", help="Also log to Weights & Biases")
+
+    curriculum = parser.add_argument_group(
+        "curriculum", "Stage the training run — see TRAINING.md's Curriculum training section"
+    )
+    curriculum.add_argument(
+        "--opponent-mode",
+        choices=["self_play", "inert", "frozen"],
+        default=None,
+        help="self_play (default): both ships are live trainees. inert: team 1 is a "
+        "do-nothing placeholder (isolated scoring practice). frozen: team 1 runs a "
+        "fixed exported policy (--opponent-model)",
+    )
+    curriculum.add_argument("--opponent-model", default=None, help="Exported policy .json for --opponent-mode=frozen")
+    curriculum.add_argument(
+        "--draw-penalty", type=float, default=None, help="One-time penalty when an episode times out with no goal"
+    )
+    curriculum.add_argument(
+        "--attack-goal-bias",
+        type=float,
+        default=None,
+        help="0.5 = uniform between both goals (default); 1.0 = near-goal resets always target the goal team 0 attacks",
+    )
+    curriculum.add_argument("--kickoff-chance", type=float, default=None, help="Overrides kickoff_state_chance")
+    curriculum.add_argument("--near-goal-chance", type=float, default=None, help="Overrides ball_near_goal_chance")
+    curriculum.add_argument(
+        "--allow-vertical", action=argparse.BooleanOptionalAction, default=None, help="Allow vertical thrust (default true)"
+    )
+    curriculum.add_argument(
+        "--allow-pitch-roll",
+        action=argparse.BooleanOptionalAction,
+        default=None,
+        help="Allow pitch/roll rotation (default true)",
+    )
+
     return parser.parse_args()
 
 
+def _curriculum_kwargs(args) -> dict:
+    """Maps train.py's curriculum flags to the --key=value args training_mode.gd's
+    _parse_curriculum_args() reads, omitting anything not explicitly passed so
+    unset flags leave Godot's own @export defaults in place."""
+    mapping = {
+        "opponent_mode": args.opponent_mode,
+        "opponent_model": args.opponent_model,
+        "draw_penalty": args.draw_penalty,
+        "attack_goal_bias": args.attack_goal_bias,
+        "kickoff_state_chance": args.kickoff_chance,
+        "ball_near_goal_chance": args.near_goal_chance,
+        "ai_allow_vertical": args.allow_vertical,
+        "ai_allow_pitch_roll": args.allow_pitch_roll,
+    }
+    return {key: value for key, value in mapping.items() if value is not None}
+
+
 def main():
     args = parse_args()
     log_dir = TRAINING_DIR / "logs"
@@ -72,6 +123,7 @@ def main():
         port=args.port,
         show_window=args.viz,
         speedup=args.speedup,
+        **_curriculum_kwargs(args),
     )
     env = VecMonitor(env)