From 390bd18be79111e368efab2462ef809c6ed39cc8 Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:00:38 +0100 Subject: [PATCH] feat(*): Start curriculum generation 2, seeded from curric-s5-aggression --- TODO.md | 3 +- TRAINING.md | 93 ++++++++++-- training/curriculum.py | 216 +++++++++++++++++----------- training/curriculum_state.json | 139 +----------------- training/curriculum_state_gen1.json | 137 ++++++++++++++++++ training/train.py | 10 ++ 6 files changed, 367 insertions(+), 231 deletions(-) create mode 100644 training/curriculum_state_gen1.json diff --git a/TODO.md b/TODO.md index 79c601ba..556e5858 100644 --- a/TODO.md +++ b/TODO.md @@ -6,7 +6,8 @@ Deferred work, in rough priority order. The current architecture (ShipAction/Shi The training pipeline is built — see `TRAINING.md` (self-play PPO via the vendored godot_rl_agents bridge, JSON policy export, in-game GDScript inference, eval ladder). Remaining: -- [ ] Long training runs on the Linux/3090 box to produce actually-good bots; promote checkpoints into `Game/bots/` as `easy`/`medium`/`hard` tiers. +- [x] Promote a first tier: `curric-s6-unmask` copied into `Game/bots/promoted/easy.json` as the shipped "easy" bot (see TRAINING.md's "Promoted bots" section) — `match.tscn`/`spectate.tscn` now default there instead of `run05.json`. +- [ ] Long training runs on the Linux/3090 box to produce actually-good bots; promote further checkpoints into `Game/bots/promoted/` as `medium`/`hard` tiers once they clear `easy.json` in `evaluate.py`. - [x] Staged curriculum (score → defend → avoid draws → full mechanics) via `train.py`'s `--opponent-mode`/`--draw-penalty`/`--attack-goal-bias`/`--allow-vertical`/`--allow-pitch-roll` flags — see TRAINING.md's "Curriculum training" section. `--opponent-mode=frozen` is a single-fixed-model slice of the league idea below, not the full sampled pool. - [ ] Frozen-opponent league: train the live policy against a *pool* of past exported checkpoints, sampled per-episode (today's `--opponent-mode=frozen` only supports one fixed model per run) to prevent self-play strategy collapse on long runs. - [ ] Richer state setter / curriculum: aerial states, wall plays, rebound scenarios as skill grows (beyond the score/defend/draw staging already in place). diff --git a/TRAINING.md b/TRAINING.md index ee129a30..a30136a3 100644 --- a/TRAINING.md +++ b/TRAINING.md @@ -178,6 +178,14 @@ movement. Each stage is a normal chained run — a new `--experiment` resumed via `--resume checkpoints//final.zip`, same as any other run — just with different curriculum flags. +`curriculum.py` has run through two generations so far. Generation 1 (below) +ran stages 1-6 to completion/block and is archived; generation 2 (the one +`curriculum.py` actually runs today) starts a fresh stage 1 seeded from +generation 1's last clean pass instead of continuing to retry a stage that +kept getting worse — see "Generation 2" below. + +### Generation 1 (archived — see `curriculum_state_gen1.json`) + | Stage | Flags | What it teaches | |---|---|---| | 1 — score | `--opponent-mode inert --attack-goal-bias 1.0 --no-allow-vertical --no-allow-pitch-roll` | Team 1 is a do-nothing placeholder ship parked at its spawn (an effectively empty net); near-goal resets always target the goal the trainee attacks; the ship can't fly or pitch/roll, only drive and yaw. Isolated finishing practice. | @@ -185,7 +193,7 @@ just with different curriculum flags. | 3 — no draws | `--draw-penalty 5 --reset-std 0.3` | Training episodes are golden-goal (end at the *first* goal), so there's no in-episode goal-margin to penalize — `draw_penalty` is the closest available signal: a one-time penalty when an episode times out with no goal at all, on top of the existing per-tick `time_penalty`. Also lifts the locomotion mask (full 3D controls) by omitting `--allow-vertical`/`--allow-pitch-roll`; pair that with `--reset-std` since the policy never got a reward gradient on those axes before now, so expect a brief re-exploration wobble. | | 4 — mechanics/refinement | *(no curriculum flags — plain `next_run.sh`)* | Stock self-play, full controls, default reward/start-state mix. This is what all runs before this feature already did. | | 5 — aggression | `--opponent-mode self_play --no-allow-vertical --no-allow-pitch-roll --velocity-to-ball-weight 0.05 --ball-distance-penalty 0.006 --ball-touch-reward 0.5` | **Resumes from stage 2 (`curric-s2-defend`), not stage 4** — see the regression note below. Retunes ball-pursuit reward weights (up from 0.02/0.002/0.4) for much more aggressive, constantly-chasing floor play, deliberately keeping the locomotion mask on so it can't reopen the stage-3 regression. Passed 2026-07-22 (41-47 vs grounded stage 2 — close, not yet a clear win). | -| 6 — unmask | `--opponent-mode self_play --velocity-to-ball-weight 0.05 --ball-distance-penalty 0.006 --ball-touch-reward 0.5 --airborne-penalty 0.003` | Re-opens full 3D controls on top of the aggression retune — this is the same grounded-checkpoint-to-full-3D transition that regressed stage 3, but this time paired with `airborne_penalty` (dense, scaled by height above the floor — see `ship_ai_controller.gd`) so the policy learns to *prefer* staying grounded through incentives instead of a hard mask, and can still pick up genuinely useful aerial/wall plays instead of never touching those axes. Runs much longer (~240M timesteps / ~24h vs every prior stage's ~20M/~2h) to actually re-converge through the regime shift instead of stalling mid-way like stage 3 did in a fifth of the time. | +| 6 — unmask | `--opponent-mode self_play --velocity-to-ball-weight 0.05 --ball-distance-penalty 0.006 --ball-touch-reward 0.5 --airborne-penalty 0.003` | Re-opens full 3D controls on top of the aggression retune — this is the same grounded-checkpoint-to-full-3D transition that regressed stage 3, but this time paired with `airborne_penalty` (dense, scaled by height above the floor — see `ship_ai_controller.gd`) so the policy learns to *prefer* staying grounded through incentives instead of a hard mask, and can still pick up genuinely useful aerial/wall plays instead of never touching those axes. **Failed 3 attempts in a row** (25% → 20% → 15% win rate vs `curric-s5-aggression`) and blocked — see "Generation 2" below for what replaced it. | > **Stages 3-4 regressed; stage 6 deliberately reopens the same transition > with a mitigation.** The locomotion-mask inference bugfix (`8c15c46`) @@ -199,7 +207,62 @@ just with different curriculum flags. > overrides) instead of chaining through stages 3-4. Stage 6 is where full 3D > flight comes back — not masked away this time, but discouraged via > `airborne_penalty` and given ~12x the training time to settle. See -> `curriculum_state.json`'s log for the full eval numbers. +> `curriculum_state_gen1.json`'s log for the full eval numbers. + +Stage 6 (`unmask`, `retry1`, `retry2`) all used identical flags — +`curriculum.py` always reuses `STAGES[stage_index]["flags"]` on retry, only +the resume checkpoint changes — so continued training just drifted the same +policy further rather than converging differently (25% → 20% → 15% win rate +vs `curric-s5-aggression`). After 3 failed attempts the script blocked for +human review; rather than pile up `retry4`, `retry5`, ... on a lineage that +kept getting worse, generation 2 (below) replaces it with a fresh stage 1. + +### Generation 2 (current) + +`curriculum.py`'s live `STAGES` list now contains a single stage, `unmask` +(displays as stage 1 — `curric-s1-unmask`), which picks up exactly where +generation 1's regression analysis left off. It resumes directly from +`FOUNDATION_EXPERIMENT` (`curric-s5-aggression`'s own checkpoint — the last +stage that passed cleanly) via `resume_from_experiment`/`reference_experiment` +overrides, rather than re-running stages 1-5 or continuing generation 1's +drifted `retry2`: + +`--opponent-mode self_play --velocity-to-ball-weight 0.08 --ball-distance-penalty 0.01 --ball-touch-reward 0.7 --airborne-penalty 0.003 --ball-velocity-to-goal-weight 0.06 --goal-reward 80 --draw-penalty 5` + +Compared to generation 1's stage 6: +- `velocity_to_ball_weight` (0.05→0.08) and `ball_distance_penalty` + (0.006→0.01) — the actual ball-chasing terms, unchanged since stage 5 + despite three failed attempts — plus `ball_touch_reward` (0.5→0.7). +- Two scoring-specific terms newly exposed via `train.py` (they already + existed as `ship_ai_controller.gd`/`training_mode.gd` `@export`s, just not + as CLI flags): `ball_velocity_to_goal_weight` (0.004 default → 0.06) + rewards the ball actually moving toward the goal, not just being + chased/touched; `goal_reward` (40 default → 80) is the terminal reward for + scoring itself. +- `draw_penalty 5` (proven effective in stage 3 against passivity), which + generation 1's stage 6 had never set — previously all carrot for scoring, + no stick for never scoring. +- `reset_retry_checkpoint: True` on the stage dict, so if this stage itself + fails and retries, `resume_checkpoint()` resets to `FOUNDATION_EXPERIMENT` + again instead of drifting a failed attempt further — the specific bug that + made generation 1's 3 retries monotonically worse instead of converging. + +Deliberately not added: a cooldown/cap on `ball_velocity_to_goal_weight` to +guard against a bot farming near-misses (bouncing the ball toward goal +repeatedly without finishing) instead of actually scoring. Unlike the +touch-farming bug (see `ship_ai_controller.gd`'s `ball_touch_reward` +comments) this term is already direction-scaled by construction (it's a +velocity-toward-goal quantity, not an undirected contact count), so the risk +is theoretical rather than demonstrated. If this stage's eval shows high +`ball_velocity_to_goal_weight` accrual without a matching rise in actual +goals scored, that's the signal to add one. + +Every experiment name `curriculum.py` generates is now timestamped +(`YYYYMMDD-HHMM-`, e.g. `20260727-0930-curric-s1-unmask`), applied once +in `run_stage_attempt` — this keeps generation 2's names from colliding with +generation 1's plain ones (both checkpoint directories and TensorBoard run +names come straight from `--experiment`) and makes run order obvious in +TensorBoard without cross-referencing `curriculum_state.json`. All curriculum flags default to leaving Godot's own `@export` defaults alone (`train.py` only forwards a flag when you pass it), so ordinary runs @@ -208,7 +271,7 @@ are unaffected. Full flag list: `--opponent-mode {self_play,inert,frozen}`, `--attack-goal-bias`, `--kickoff-chance`, `--near-goal-chance`, `--allow-vertical`/`--no-allow-vertical`, `--allow-pitch-roll`/`--no-allow-pitch-roll`, `--velocity-to-ball-weight`, `--ball-distance-penalty`, `--ball-touch-reward`, -`--airborne-penalty`. +`--airborne-penalty`, `--ball-velocity-to-goal-weight`, `--goal-reward`. ### Running it automatically @@ -216,15 +279,17 @@ are unaffected. Full flag list: `--opponent-mode {self_play,inert,frozen}`, pattern as `start_training.sh`) drives all stages end to end: for each stage it runs `run_training.sh` (pull, train, export, commit+push) with that stage's flags, then evaluates the resulting checkpoint against a reference -bot over 100 episodes — the fixed `rookie.json` baseline for stage 1, the -previous stage's promoted checkpoint by default for stages 2+, or an -explicit `resume_from_experiment`/`reference_experiment` override in that -stage's dict when it deliberately skips a since-regressed branch (stage 5). +bot over 100 episodes — the fixed `rookie.json` baseline for a from-scratch +stage 1 (no `resume_from_experiment`/`reference_experiment` override on +`STAGES[0]`), the previous stage's promoted checkpoint by default for +stages 2+, or an explicit override in that stage's dict when it deliberately +skips a since-regressed branch (generation 1's stage 5) or seeds from a +fixed foundation checkpoint (generation 2's stage 1 — see above). ```bash cd training ./curriculum.sh # start/resume the curriculum -./curriculum.sh --seed-checkpoint checkpoints/run11/final.zip # seed stage 1 instead of a fresh policy +./curriculum.sh --seed-checkpoint checkpoints/run11/final.zip # override stage 1's resume source for this run ``` The gate is deliberately lenient: it blocks a stage only on a **clear @@ -237,10 +302,14 @@ still catches a genuinely broken stage. Progress and every attempt's eval result are logged to `curriculum_state.json` (committed alongside `eval_history.json` after each attempt). -A stage gets up to 2 retries (3 attempts total, each resuming from that -stage's own previous attempt with a fresh `--reset-std`) before the script -stops and asks for a human look — it will not retry indefinitely or advance -past a stage that keeps failing on its own. Once you've looked at why (more +A stage gets up to 2 retries (3 attempts total) before the script stops and +asks for a human look — it will not retry indefinitely or advance past a +stage that keeps failing on its own. By default a retry resumes from that +stage's own previous attempt with a fresh `--reset-std`; a stage can instead +set `reset_retry_checkpoint: True` (generation 2's stage 1 does) to always +reset to its normal resume source instead — see the generation 1 → 2 +postmortem above for why blind same-checkpoint retries can make things +monotonically worse. Once you've looked at why a block happened (more timesteps? a flag needs adjusting? the eval itself was misleading?), re-run with `--force-retry` to try again or `--skip-to-next-stage` if you judge the result good enough despite the gate. diff --git a/training/curriculum.py b/training/curriculum.py index 48772582..65f84466 100644 --- a/training/curriculum.py +++ b/training/curriculum.py @@ -18,6 +18,26 @@ forever for the wrong reason. When a stage does fail MAX_RETRIES times in a row, the script stops and asks for a human look rather than retrying indefinitely or silently advancing past a bad stage. +This is generation 2 of the curriculum. Generation 1 (6 stages: score, +defend, no_draws, mechanics, aggression, unmask) ran 2026-07-21 through +2026-07-26 and is archived in curriculum_state_gen1.json — its final stage +("unmask", full 3D flight on top of the aggression retune) failed 3 straight +attempts, monotonically worsening (25% -> 20% -> 15% win rate vs +curric-s5-aggression) because every retry resumed the same drifting +checkpoint under identical flags instead of actually changing anything. +Rather than let generation 1's stage numbering grow indefinitely +(unmask-retry4, retry5, ...), generation 2 starts a fresh stage 1 seeded +directly from curric-s5-aggression's own checkpoint (FOUNDATION_EXPERIMENT +below) — the last stage that actually passed cleanly — carrying over its +trained progress without re-running stages 1-5. See TRAINING.md for the +full generation 1 history and generation 2's design. + +Every experiment name this script generates is timestamped +(YYYYMMDD-HHMM-, applied once in run_stage_attempt) so runs stay +unique across restarts/generations and sort chronologically in TensorBoard +and checkpoints/ — plain names like "curric-s1-score" from generation 1 +would otherwise collide with generation 2's own stage 1. + Usage: .venv/bin/python curriculum.py # run/resume the curriculum .venv/bin/python curriculum.py --seed-checkpoint checkpoints/run11/final.zip @@ -33,12 +53,26 @@ import json import pathlib import subprocess import sys +from datetime import datetime TRAINING_DIR = pathlib.Path(__file__).resolve().parent STATE_PATH = TRAINING_DIR / "curriculum_state.json" EVAL_HISTORY_PATH = TRAINING_DIR / "eval_history.json" ROOKIE_REFERENCE = TRAINING_DIR.parent / "Game" / "bots" / "rookie.json" +# Generation 1's last cleanly-passing checkpoint (see curriculum_state_gen1.json) +# — generation 2's stage 1 builds on this directly instead of re-running +# stages 1-5. +FOUNDATION_EXPERIMENT = "curric-s5-aggression" + +# Groundedness (locomotion-mask state) for experiments that predate this +# generation's own log, so _grounded_for_experiment can still answer for +# them — see that function. +LEGACY_GROUNDED = { + "rookie": False, + FOUNDATION_EXPERIMENT: True, +} + MAX_RETRIES = 2 EVAL_EPISODES = 100 # "Clear regression" = the reference beats the candidate by at least this @@ -53,82 +87,58 @@ REGRESSION_MARGIN = 0.15 STANDING_ARGS = ["--reset-std", "0.3", "--ent-coef", "0.001"] STAGES = [ - { - "name": "score", - "flags": [ - "--opponent-mode", "inert", - "--attack-goal-bias", "1.0", - "--no-allow-vertical", "--no-allow-pitch-roll", - ], - "grounded": True, - }, - { - "name": "defend", - "flags": [ - "--opponent-mode", "self_play", - "--no-allow-vertical", "--no-allow-pitch-roll", - ], - "grounded": True, - }, - { - "name": "no_draws", - "flags": ["--draw-penalty", "5"], - "grounded": False, - }, - { - "name": "mechanics", - "flags": [], - "grounded": False, - }, - { - "name": "aggression", - # Deliberately resumes from stage 2 (curric-s2-defend), not stage 4 - # (see resume_from_experiment/reference_experiment below) — the - # locomotion-mask inference bugfix (8c15c46) revealed that stage 3's - # full-3D unmask was a clear regression, not an improvement: fairly - # evaluated, curric-s2-defend beats both curric-s3-no_draws (26-60) - # and curric-s4-mechanics (24-57). Rather than compound that - # regression, this stage keeps the locomotion mask ON (matching - # stage 2's own regime) and just retunes ball-pursuit reward weights, - # so it can't reopen the same grounded-to-3D transition that caused - # the earlier failure. Full 3D flight is parked as a separate, - # later initiative. - "flags": [ - "--opponent-mode", "self_play", - "--no-allow-vertical", "--no-allow-pitch-roll", - "--velocity-to-ball-weight", "0.05", # up from 0.02 - "--ball-distance-penalty", "0.006", # up from 0.002 - "--ball-touch-reward", "0.5", # up from 0.4 - ], - "grounded": True, - "resume_from_experiment": "curric-s2-defend", - "reference_experiment": "curric-s2-defend", - }, { "name": "unmask", # Re-opens full 3D controls (no more --no-allow-vertical/ # --no-allow-pitch-roll) on top of the aggression retune, instead of # keeping locomotion masked indefinitely. The mask blocked *thrust*- - # driven flight outright; the new airborne_penalty (dense, scaled by - # height above the floor — see ship_ai_controller.gd) is meant to - # teach the policy to prefer staying grounded through incentives - # rather than a hard constraint, so it can start learning when the - # other axes are actually useful (aerial saves, wall recoveries) - # instead of never touching them. This resumes the exact regime - # shift (grounded checkpoint -> full 3D) that regressed stage 3 — - # the mitigation this time is airborne_penalty plus a much longer - # run (24h / ~240M steps vs stage 3's 20M) to actually re-converge - # instead of stalling mid-shift like stage 3 did in a fifth of the - # time. + # driven flight outright; airborne_penalty (dense, scaled by height + # above the floor — see ship_ai_controller.gd) is meant to teach the + # policy to prefer staying grounded through incentives rather than a + # hard constraint, so it can start learning when the other axes are + # actually useful (aerial saves, wall recoveries) instead of never + # touching them. + # + # Generation 1 ran this exact transition 3 times (unmask, retry1, + # retry2) with identical flags and got monotonically worse each time + # (25% -> 20% -> 15% win rate vs curric-s5-aggression) — a blind + # retry just continues training the same drifting policy for + # longer, it was never going to converge differently. An adversarial + # review of a first patch (two modest new flags, still resuming the + # drifted retry2 checkpoint) found that insufficient too: the resume + # target was the worst of the three already-degraded checkpoints, + # and the new weights were too small to compete with the unchanged + # ball-pursuit terms. Generation 2's stage 1 instead: + # - resumes from FOUNDATION_EXPERIMENT (curric-s5-aggression) + # directly (resume_from_experiment below, plus + # reset_retry_checkpoint so this stage's own retries reset here + # too instead of drifting a failed attempt forward). + # - raises velocity_to_ball_weight and ball_distance_penalty + # further (the actual ball-chasing terms, unchanged since stage + # 5 despite three failed attempts) and ball_touch_reward + # alongside them. + # - raises ball_velocity_to_goal_weight (reward for moving the + # ball toward the goal, not just touching it) and goal_reward + # (the terminal reward for scoring) — both newly exposed via + # train.py, previously only reachable as raw Godot cmdline args. + # - adds draw_penalty (proven effective in generation 1's stage 3 + # against passivity), which this transition had never set: + # previously all carrot for scoring, no stick for never scoring. "flags": [ "--opponent-mode", "self_play", - "--velocity-to-ball-weight", "0.05", - "--ball-distance-penalty", "0.006", - "--ball-touch-reward", "0.5", + "--velocity-to-ball-weight", "0.08", # up from 0.05 + "--ball-distance-penalty", "0.01", # up from 0.006 + "--ball-touch-reward", "0.7", # up from 0.5 "--airborne-penalty", "0.003", + "--ball-velocity-to-goal-weight", "0.06", # up from 0.02 (0.004 default) + "--goal-reward", "80", # up from 60 (40 default) + "--draw-penalty", "5", ], "grounded": False, "timesteps": 240_000_000, # ~24h at the standing n-parallel/speedup (20M took ~2h) + "resume_from_experiment": FOUNDATION_EXPERIMENT, + "reference_experiment": FOUNDATION_EXPERIMENT, + "reset_retry_checkpoint": True, }, ] @@ -148,23 +158,43 @@ def experiment_name(stage_index: int, attempt: int) -> str: return name if attempt == 0 else f"{name}-retry{attempt}" +def _logged_experiment_name(stage_index: int, attempt: int) -> str: + """The actual (timestamped) experiment name recorded when this attempt + ran — needed anywhere a *past* attempt's real name matters, since + experiment_name() alone no longer identifies a run on disk (see + run_stage_attempt's timestamp prefix).""" + state = load_state() + for entry in state["log"]: + if entry["stage_index"] == stage_index and entry["attempt"] == attempt: + return entry["experiment"] + raise RuntimeError(f"No logged experiment for stage {stage_index} attempt {attempt}") + + def resume_checkpoint(stage_index: int, attempt: int, seed_checkpoint: str | None) -> str | None: - if attempt > 0: + if attempt > 0 and not STAGES[stage_index].get("reset_retry_checkpoint"): # Retry: keep training the same stage's own last attempt. - prev = experiment_name(stage_index, attempt - 1) + prev = _logged_experiment_name(stage_index, attempt - 1) return str(TRAINING_DIR / "checkpoints" / prev / "final.zip") - if stage_index == 0: + if stage_index == 0 and seed_checkpoint: + return seed_checkpoint + if stage_index == 0 and not STAGES[0].get("resume_from_experiment"): # Deliberately fresh by default: the curriculum exists because # resuming self-play across a regime change (run10, run11) didn't - # work, so stage 1 starts from a random policy under its own - # regime unless --seed-checkpoint says otherwise. - return seed_checkpoint + # work, so a from-scratch stage 1 starts from a random policy under + # its own regime unless --seed-checkpoint or resume_from_experiment + # says otherwise. + return None + # Either a later stage chaining off its predecessor, or + # reset_retry_checkpoint: this stage's own retries have been drifting + # rather than converging (see the "unmask" stage's comment) — resume + # from the stage's normal resume source instead of compounding the last + # failed attempt's drift. prev_experiment = _resume_source_experiment(stage_index) return str(TRAINING_DIR / "checkpoints" / prev_experiment / "final.zip") def reference_bot(stage_index: int) -> str: - if stage_index == 0: + if stage_index == 0 and not STAGES[0].get("reference_experiment"): return str(ROOKIE_REFERENCE) prev_experiment = _reference_source_experiment(stage_index) return str(TRAINING_DIR.parent / "Game" / "bots" / f"{prev_experiment}.json") @@ -172,8 +202,8 @@ def reference_bot(stage_index: int) -> str: # A stage normally chains off "whatever passed at the previous index," but a # stage can instead name an explicit resume_from_experiment/reference_experiment -# to skip a since-regressed branch (see the "aggression" stage) without -# rewriting history for the stages it's skipping past. +# to skip a since-regressed branch, or (stage 0) to seed from a fixed +# foundation checkpoint instead of a from-scratch policy. def _resume_source_experiment(stage_index: int) -> str: override = STAGES[stage_index].get("resume_from_experiment") return override if override else _passing_experiment_for_stage(stage_index - 1) @@ -193,16 +223,19 @@ def _passing_experiment_for_stage(stage_index: int) -> str: def _grounded_for_experiment(experiment: str) -> bool: - if experiment == "rookie": - return False - for index, stage in enumerate(STAGES): - if experiment_name(index, 0) == experiment: - return stage["grounded"] + if experiment in LEGACY_GROUNDED: + return LEGACY_GROUNDED[experiment] + state = load_state() + for entry in state["log"]: + if entry["experiment"] == experiment: + return STAGES[entry["stage_index"]]["grounded"] raise ValueError(f"Unknown experiment for groundedness lookup: {experiment}") def run_stage_attempt(stage_index: int, attempt: int, args) -> str: - exp = experiment_name(stage_index, attempt) + # Timestamped so names stay unique across restarts/generations and sort + # chronologically in TensorBoard/checkpoints — see module docstring. + exp = f"{datetime.now().strftime('%Y%m%d-%H%M')}-{experiment_name(stage_index, attempt)}" resume = resume_checkpoint(stage_index, attempt, args.seed_checkpoint) # A stage can override the run's timesteps budget (see "floor-lock", # which deliberately runs much longer than the ~20M/~2h every stage so @@ -226,7 +259,7 @@ def run_stage_attempt(stage_index: int, attempt: int, args) -> str: def reference_grounded(stage_index: int) -> bool: - if stage_index == 0: + if stage_index == 0 and not STAGES[0].get("reference_experiment"): # rookie.json predates the locomotion mask entirely — always full 3D. return False return _grounded_for_experiment(_reference_source_experiment(stage_index)) @@ -272,7 +305,11 @@ def main(): parser.add_argument("--timesteps", type=int, default=20_000_000) parser.add_argument("--n-parallel", type=int, default=14) parser.add_argument("--speedup", type=int, default=16) - parser.add_argument("--seed-checkpoint", default=None, help="Resume stage 1 from this checkpoint instead of a fresh policy") + parser.add_argument( + "--seed-checkpoint", default=None, + help="Resume stage 1 from this checkpoint instead of its default resume source " + "(FOUNDATION_EXPERIMENT's checkpoint)", + ) parser.add_argument("--force-retry", action="store_true", help="Retry a blocked stage after human review") parser.add_argument("--skip-to-next-stage", action="store_true", help="Human judgment call: treat the blocked stage as good enough, advance anyway") args = parser.parse_args() @@ -283,14 +320,20 @@ def main(): if args.skip_to_next_stage: print(f"Human override: advancing past stage {state['stage_index'] + 1} " f"({STAGES[state['stage_index']]['name']}) despite exhausted retries.") + skipped_experiment = _logged_experiment_name(state["stage_index"], state["attempt"]) state["log"].append({ - "stage_index": state["stage_index"], "experiment": experiment_name(state["stage_index"], state["attempt"]), + "stage_index": state["stage_index"], + "experiment": skipped_experiment, "attempt": state["attempt"], "decision": "pass", "override": "skip_to_next_stage", }) state["stage_index"] += 1 state["attempt"] = 0 state["status"] = "in_progress" save_state(state) + # If this was the last stage, the while loop below never runs + # (stage_index now == len(STAGES)), so this override's state + # change would otherwise never get committed/pushed. + commit_progress(skipped_experiment) elif args.force_retry: print(f"Human override: retrying stage {state['stage_index'] + 1} " f"({STAGES[state['stage_index']]['name']}) after review.") @@ -304,11 +347,13 @@ def main(): "--skip-to-next-stage (advance anyway) once you've looked at why.") sys.exit(1) + last_experiment = None while state["stage_index"] < len(STAGES): stage_index = state["stage_index"] attempt = state["attempt"] experiment = run_stage_attempt(stage_index, attempt, args) + last_experiment = experiment reference = reference_bot(stage_index) record = evaluate_attempt(experiment, reference, EVAL_EPISODES, stage_index) decision = decide(record) @@ -345,6 +390,11 @@ def main(): print("\nCurriculum complete — all stages passed.") state["status"] = "done" save_state(state) + if last_experiment is not None: + # None only if the loop above never ran at all (e.g. re-invoking + # after the curriculum was already "done") — nothing new to commit + # in that case. + commit_progress(last_experiment) if __name__ == "__main__": diff --git a/training/curriculum_state.json b/training/curriculum_state.json index 0cc009ae..30c376dc 100644 --- a/training/curriculum_state.json +++ b/training/curriculum_state.json @@ -1,137 +1,6 @@ { - "stage_index": 5, - "attempt": 2, - "status": "blocked", - "log": [ - { - "stage_index": 0, - "experiment": "curric-s1-score", - "attempt": 0, - "eval": { - "timestamp": "2026-07-21T15:52:20+00:00", - "model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s1-score.json", - "model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/rookie.json", - "episodes": 100, - "wins_a": 22, - "wins_b": 16, - "draws": 62, - "win_rate_a": 0.22 - }, - "decision": "pass" - }, - { - "stage_index": 1, - "experiment": "curric-s2-defend", - "attempt": 0, - "eval": { - "timestamp": "2026-07-21T18:19:44+00:00", - "model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s2-defend.json", - "model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s1-score.json", - "episodes": 100, - "wins_a": 23, - "wins_b": 16, - "draws": 61, - "win_rate_a": 0.23 - }, - "decision": "pass" - }, - { - "stage_index": 2, - "experiment": "curric-s3-no_draws", - "attempt": 0, - "eval": { - "timestamp": "2026-07-22T11:36:42+00:00", - "model_a": "/Users/jcreek/Documents/repos/GitHub/CosmicClash/Game/bots/curric-s3-no_draws.json", - "model_b": "/Users/jcreek/Documents/repos/GitHub/CosmicClash/Game/bots/curric-s2-defend.json", - "episodes": 100, - "wins_a": 26, - "wins_b": 60, - "draws": 14, - "win_rate_a": 0.26 - }, - "decision": "fail", - "note": "Original eval (44-19, recorded 2026-07-21T20:46:34) predates the locomotion-mask inference bugfix (8c15c46) and ran with the grounded stage-2 reference unfairly unmasked. Re-run post-fix with --grounded-b reverses the verdict: stage 3's full-3D unmask is a clear regression from stage 2, not an improvement. Not retried via the normal flag-retry mechanism \u2014 see stage_index 4 (aggression), which redirects around this branch by resuming from curric-s2-defend directly instead." - }, - { - "stage_index": 3, - "experiment": "curric-s4-mechanics", - "attempt": 0, - "eval": { - "timestamp": "2026-07-22T11:38:57+00:00", - "model_a": "/Users/jcreek/Documents/repos/GitHub/CosmicClash/Game/bots/curric-s4-mechanics.json", - "model_b": "/Users/jcreek/Documents/repos/GitHub/CosmicClash/Game/bots/curric-s3-no_draws.json", - "episodes": 100, - "wins_a": 26, - "wins_b": 33, - "draws": 41, - "win_rate_a": 0.26 - }, - "decision": "pass", - "note": "Passes only against its own (already-regressed) predecessor, curric-s3-no_draws. Evaluated directly against grounded curric-s2-defend (2026-07-22T11:40:07), curric-s4-mechanics also loses clearly: 24-57-19. Do not treat this stage's 'pass' as evidence curric-s4-mechanics is the strongest available model overall \u2014 see stage 2's note and stage_index 4 (aggression)." - }, - { - "stage_index": 4, - "experiment": "curric-s5-aggression", - "attempt": 0, - "eval": { - "timestamp": "2026-07-22T19:33:07+00:00", - "model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s5-aggression.json", - "model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s2-defend.json", - "episodes": 100, - "wins_a": 41, - "wins_b": 47, - "draws": 12, - "win_rate_a": 0.41 - }, - "decision": "pass" - }, - { - "stage_index": 5, - "experiment": "curric-s6-unmask", - "attempt": 0, - "eval": { - "timestamp": "2026-07-24T01:25:37+00:00", - "model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s6-unmask.json", - "model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s5-aggression.json", - "episodes": 100, - "wins_a": 25, - "wins_b": 50, - "draws": 25, - "win_rate_a": 0.25 - }, - "decision": "fail" - }, - { - "stage_index": 5, - "experiment": "curric-s6-unmask-retry1", - "attempt": 1, - "eval": { - "timestamp": "2026-07-25T06:41:43+00:00", - "model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s6-unmask-retry1.json", - "model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s5-aggression.json", - "episodes": 100, - "wins_a": 20, - "wins_b": 61, - "draws": 19, - "win_rate_a": 0.2 - }, - "decision": "fail" - }, - { - "stage_index": 5, - "experiment": "curric-s6-unmask-retry2", - "attempt": 2, - "eval": { - "timestamp": "2026-07-26T12:08:35+00:00", - "model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s6-unmask-retry2.json", - "model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s5-aggression.json", - "episodes": 100, - "wins_a": 15, - "wins_b": 68, - "draws": 17, - "win_rate_a": 0.15 - }, - "decision": "fail" - } - ] + "stage_index": 0, + "attempt": 0, + "status": "in_progress", + "log": [] } diff --git a/training/curriculum_state_gen1.json b/training/curriculum_state_gen1.json new file mode 100644 index 00000000..0cc009ae --- /dev/null +++ b/training/curriculum_state_gen1.json @@ -0,0 +1,137 @@ +{ + "stage_index": 5, + "attempt": 2, + "status": "blocked", + "log": [ + { + "stage_index": 0, + "experiment": "curric-s1-score", + "attempt": 0, + "eval": { + "timestamp": "2026-07-21T15:52:20+00:00", + "model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s1-score.json", + "model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/rookie.json", + "episodes": 100, + "wins_a": 22, + "wins_b": 16, + "draws": 62, + "win_rate_a": 0.22 + }, + "decision": "pass" + }, + { + "stage_index": 1, + "experiment": "curric-s2-defend", + "attempt": 0, + "eval": { + "timestamp": "2026-07-21T18:19:44+00:00", + "model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s2-defend.json", + "model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s1-score.json", + "episodes": 100, + "wins_a": 23, + "wins_b": 16, + "draws": 61, + "win_rate_a": 0.23 + }, + "decision": "pass" + }, + { + "stage_index": 2, + "experiment": "curric-s3-no_draws", + "attempt": 0, + "eval": { + "timestamp": "2026-07-22T11:36:42+00:00", + "model_a": "/Users/jcreek/Documents/repos/GitHub/CosmicClash/Game/bots/curric-s3-no_draws.json", + "model_b": "/Users/jcreek/Documents/repos/GitHub/CosmicClash/Game/bots/curric-s2-defend.json", + "episodes": 100, + "wins_a": 26, + "wins_b": 60, + "draws": 14, + "win_rate_a": 0.26 + }, + "decision": "fail", + "note": "Original eval (44-19, recorded 2026-07-21T20:46:34) predates the locomotion-mask inference bugfix (8c15c46) and ran with the grounded stage-2 reference unfairly unmasked. Re-run post-fix with --grounded-b reverses the verdict: stage 3's full-3D unmask is a clear regression from stage 2, not an improvement. Not retried via the normal flag-retry mechanism \u2014 see stage_index 4 (aggression), which redirects around this branch by resuming from curric-s2-defend directly instead." + }, + { + "stage_index": 3, + "experiment": "curric-s4-mechanics", + "attempt": 0, + "eval": { + "timestamp": "2026-07-22T11:38:57+00:00", + "model_a": "/Users/jcreek/Documents/repos/GitHub/CosmicClash/Game/bots/curric-s4-mechanics.json", + "model_b": "/Users/jcreek/Documents/repos/GitHub/CosmicClash/Game/bots/curric-s3-no_draws.json", + "episodes": 100, + "wins_a": 26, + "wins_b": 33, + "draws": 41, + "win_rate_a": 0.26 + }, + "decision": "pass", + "note": "Passes only against its own (already-regressed) predecessor, curric-s3-no_draws. Evaluated directly against grounded curric-s2-defend (2026-07-22T11:40:07), curric-s4-mechanics also loses clearly: 24-57-19. Do not treat this stage's 'pass' as evidence curric-s4-mechanics is the strongest available model overall \u2014 see stage 2's note and stage_index 4 (aggression)." + }, + { + "stage_index": 4, + "experiment": "curric-s5-aggression", + "attempt": 0, + "eval": { + "timestamp": "2026-07-22T19:33:07+00:00", + "model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s5-aggression.json", + "model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s2-defend.json", + "episodes": 100, + "wins_a": 41, + "wins_b": 47, + "draws": 12, + "win_rate_a": 0.41 + }, + "decision": "pass" + }, + { + "stage_index": 5, + "experiment": "curric-s6-unmask", + "attempt": 0, + "eval": { + "timestamp": "2026-07-24T01:25:37+00:00", + "model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s6-unmask.json", + "model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s5-aggression.json", + "episodes": 100, + "wins_a": 25, + "wins_b": 50, + "draws": 25, + "win_rate_a": 0.25 + }, + "decision": "fail" + }, + { + "stage_index": 5, + "experiment": "curric-s6-unmask-retry1", + "attempt": 1, + "eval": { + "timestamp": "2026-07-25T06:41:43+00:00", + "model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s6-unmask-retry1.json", + "model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s5-aggression.json", + "episodes": 100, + "wins_a": 20, + "wins_b": 61, + "draws": 19, + "win_rate_a": 0.2 + }, + "decision": "fail" + }, + { + "stage_index": 5, + "experiment": "curric-s6-unmask-retry2", + "attempt": 2, + "eval": { + "timestamp": "2026-07-26T12:08:35+00:00", + "model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s6-unmask-retry2.json", + "model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s5-aggression.json", + "episodes": 100, + "wins_a": 15, + "wins_b": 68, + "draws": 17, + "win_rate_a": 0.15 + }, + "decision": "fail" + } + ] +} diff --git a/training/train.py b/training/train.py index 42b4d6e9..82bf99a9 100644 --- a/training/train.py +++ b/training/train.py @@ -100,6 +100,14 @@ def parse_args(): "--airborne-penalty", type=float, default=None, help="Overrides ShipAIController.airborne_penalty (dense per-tick cost scaled by height above the floor)", ) + curriculum.add_argument( + "--ball-velocity-to-goal-weight", type=float, default=None, + help="Overrides ShipAIController.ball_velocity_to_goal_weight (dense reward for the ball's velocity toward the attack goal)", + ) + curriculum.add_argument( + "--goal-reward", type=float, default=None, + help="Overrides TrainingMode.goal_reward (terminal reward for actually scoring)", + ) return parser.parse_args() @@ -121,6 +129,8 @@ def _curriculum_kwargs(args) -> dict: "ai_ball_distance_penalty": args.ball_distance_penalty, "ai_ball_touch_reward": args.ball_touch_reward, "ai_airborne_penalty": args.airborne_penalty, + "ai_ball_velocity_to_goal_weight": args.ball_velocity_to_goal_weight, + "goal_reward": args.goal_reward, } return {key: value for key, value in mapping.items() if value is not None}