From fca6a46200d684c016ee9903a77aad23daa9a082 Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Wed, 22 Jul 2026 12:48:45 +0100 Subject: [PATCH] fix(training): correct stage-3 eval (locomotion-mask bugfix) and add grounded aggression stage Re-ran stage-3 (curric-s3-no_draws vs curric-s2-defend) and the missing stage-4 gate now that the locomotion-mask inference bugfix is in. Both reverse or contradict the pre-fix bookkeeping: curric-s2-defend (grounded) beats curric-s3-no_draws 60-26 and curric-s4-mechanics 57-24 when fairly evaluated, so lifting the locomotion mask in stage 3 was a real regression in floor play, not the improvement the buggy eval reported. Adds a stage-5 "aggression" curriculum entry that resumes from stage 2 directly (via new resume_from_experiment/reference_experiment stage-dict overrides in curriculum.py) instead of compounding the regression through stages 3-4, keeps the locomotion mask on, and retunes ball-pursuit reward weights for much more aggressive floor play. Extends train.py with the three new --velocity-to-ball-weight/--ball-distance-penalty/--ball-touch-reward flags needed to forward that retune to Godot's existing SHIP_AI_OVERRIDES. curriculum_state.json and TRAINING.md are corrected/annotated in place rather than silently rewritten, so the regression stays visible in history. --- TRAINING.md | 30 ++++++++++++++---- training/curriculum.py | 58 +++++++++++++++++++++++++++++++--- training/curriculum_state.json | 36 +++++++++++++++------ training/eval_history.json | 30 ++++++++++++++++++ training/train.py | 15 +++++++++ 5 files changed, 149 insertions(+), 20 deletions(-) diff --git a/TRAINING.md b/TRAINING.md index 4bb55372..526e56d1 100644 --- a/TRAINING.md +++ b/TRAINING.md @@ -122,8 +122,8 @@ appends to `training/eval_history.json` — the long-term progress record. Evaluate each new candidate against the previous promoted bot and a fixed early reference to see absolute progress over time. -If a model was trained with the locomotion mask on (curriculum stages 1-2 — -see below), pass `--grounded-a`/`--grounded-b` for whichever side it's on. +If a model was trained with the locomotion mask on (curriculum stages 1, 2, +and 5 — see below), pass `--grounded-a`/`--grounded-b` for whichever side it's on. The eval otherwise runs `AIShipController` fully unmasked regardless of how a model was trained, so a grounded model's untrained vertical/pitch-roll output reaches the ship as noise it never had to contend with during training — @@ -159,22 +159,40 @@ just with different curriculum flags. | 2 — defend too | `--opponent-mode self_play --no-allow-vertical --no-allow-pitch-roll` | Reintroduces a live opponent (self-play) and the default episode-start mix — the same near-goal state is now simultaneously a finishing chance for one side and a defensive save for the other. Locomotion stays grounded. | | 3 — no draws | `--draw-penalty 5 --reset-std 0.3` | Training episodes are golden-goal (end at the *first* goal), so there's no in-episode goal-margin to penalize — `draw_penalty` is the closest available signal: a one-time penalty when an episode times out with no goal at all, on top of the existing per-tick `time_penalty`. Also lifts the locomotion mask (full 3D controls) by omitting `--allow-vertical`/`--allow-pitch-roll`; pair that with `--reset-std` since the policy never got a reward gradient on those axes before now, so expect a brief re-exploration wobble. | | 4 — mechanics/refinement | *(no curriculum flags — plain `next_run.sh`)* | Stock self-play, full controls, default reward/start-state mix. This is what all runs before this feature already did. | +| 5 — aggression | `--opponent-mode self_play --no-allow-vertical --no-allow-pitch-roll --velocity-to-ball-weight 0.05 --ball-distance-penalty 0.006 --ball-touch-reward 0.5` | **Resumes from stage 2 (`curric-s2-defend`), not stage 4** — see the regression note below. Retunes ball-pursuit reward weights (up from 0.02/0.002/0.4) for much more aggressive, constantly-chasing floor play, deliberately keeping the locomotion mask on so it can't reopen the stage-3 regression. | + +> **Stages 3-4 regressed and are parked.** The locomotion-mask inference bugfix +> (`8c15c46`) revealed that stage 3's evals up to that point had been running +> with an unfairly unmasked grounded reference. Re-evaluated fairly, +> `curric-s2-defend` (grounded) beats both `curric-s3-no_draws` (26-60) and +> `curric-s4-mechanics` (24-57) — lifting the locomotion mask to full 3D in +> stage 3 was a clear regression in floor play that self-play never earned +> back. Stage 5 sidesteps this by resuming and evaluating against stage 2 +> directly (`curriculum.py`'s `resume_from_experiment`/`reference_experiment` +> stage-dict overrides) instead of chaining through stages 3-4. Full 3D +> flight is parked as a separate initiative — see TODO.md — that will need a +> redesigned unmasking approach (more timesteps and/or reward rebalancing) so +> it doesn't cost floor fundamentals again. See `curriculum_state.json`'s +> stage-2/stage-3 log entries for the full eval numbers. All curriculum flags default to leaving Godot's own `@export` defaults alone (`train.py` only forwards a flag when you pass it), so ordinary runs are unaffected. Full flag list: `--opponent-mode {self_play,inert,frozen}`, `--opponent-model ` (for `frozen`), `--draw-penalty`, `--attack-goal-bias`, `--kickoff-chance`, `--near-goal-chance`, -`--allow-vertical`/`--no-allow-vertical`, `--allow-pitch-roll`/`--no-allow-pitch-roll`. +`--allow-vertical`/`--no-allow-vertical`, `--allow-pitch-roll`/`--no-allow-pitch-roll`, +`--velocity-to-ball-weight`, `--ball-distance-penalty`, `--ball-touch-reward`. ### Running it automatically `training/curriculum.py` (started via `curriculum.sh`, same detached-tmux -pattern as `start_training.sh`) drives all four stages end to end: for each +pattern as `start_training.sh`) drives all stages end to end: for each stage it runs `run_training.sh` (pull, train, export, commit+push) with that stage's flags, then evaluates the resulting checkpoint against a reference -bot — the fixed `rookie.json` baseline for stage 1, or the previous stage's -promoted checkpoint for stages 2-4 — over 100 episodes. +bot over 100 episodes — the fixed `rookie.json` baseline for stage 1, the +previous stage's promoted checkpoint by default for stages 2+, or an +explicit `resume_from_experiment`/`reference_experiment` override in that +stage's dict when it deliberately skips a since-regressed branch (stage 5). ```bash cd training diff --git a/training/curriculum.py b/training/curriculum.py index 62df69ea..088ec193 100644 --- a/training/curriculum.py +++ b/training/curriculum.py @@ -80,6 +80,30 @@ STAGES = [ "flags": [], "grounded": False, }, + { + "name": "aggression", + # Deliberately resumes from stage 2 (curric-s2-defend), not stage 4 + # (see resume_from_experiment/reference_experiment below) — the + # locomotion-mask inference bugfix (8c15c46) revealed that stage 3's + # full-3D unmask was a clear regression, not an improvement: fairly + # evaluated, curric-s2-defend beats both curric-s3-no_draws (26-60) + # and curric-s4-mechanics (24-57). Rather than compound that + # regression, this stage keeps the locomotion mask ON (matching + # stage 2's own regime) and just retunes ball-pursuit reward weights, + # so it can't reopen the same grounded-to-3D transition that caused + # the earlier failure. Full 3D flight is parked as a separate, + # later initiative. + "flags": [ + "--opponent-mode", "self_play", + "--no-allow-vertical", "--no-allow-pitch-roll", + "--velocity-to-ball-weight", "0.05", # up from 0.02 + "--ball-distance-penalty", "0.006", # up from 0.002 + "--ball-touch-reward", "0.5", # up from 0.4 + ], + "grounded": True, + "resume_from_experiment": "curric-s2-defend", + "reference_experiment": "curric-s2-defend", + }, ] @@ -109,18 +133,31 @@ def resume_checkpoint(stage_index: int, attempt: int, seed_checkpoint: str | Non # work, so stage 1 starts from a random policy under its own # regime unless --seed-checkpoint says otherwise. return seed_checkpoint - prev_stage = STAGES[stage_index - 1]["name"] - prev_experiment = _passing_experiment_for_stage(stage_index - 1) + prev_experiment = _resume_source_experiment(stage_index) return str(TRAINING_DIR / "checkpoints" / prev_experiment / "final.zip") def reference_bot(stage_index: int) -> str: if stage_index == 0: return str(ROOKIE_REFERENCE) - prev_experiment = _passing_experiment_for_stage(stage_index - 1) + prev_experiment = _reference_source_experiment(stage_index) return str(TRAINING_DIR.parent / "Game" / "bots" / f"{prev_experiment}.json") +# A stage normally chains off "whatever passed at the previous index," but a +# stage can instead name an explicit resume_from_experiment/reference_experiment +# to skip a since-regressed branch (see the "aggression" stage) without +# rewriting history for the stages it's skipping past. +def _resume_source_experiment(stage_index: int) -> str: + override = STAGES[stage_index].get("resume_from_experiment") + return override if override else _passing_experiment_for_stage(stage_index - 1) + + +def _reference_source_experiment(stage_index: int) -> str: + override = STAGES[stage_index].get("reference_experiment") + return override if override else _passing_experiment_for_stage(stage_index - 1) + + def _passing_experiment_for_stage(stage_index: int) -> str: state = load_state() for entry in state["log"]: @@ -129,6 +166,15 @@ def _passing_experiment_for_stage(stage_index: int) -> str: raise RuntimeError(f"No passing attempt recorded for stage {stage_index} ({STAGES[stage_index]['name']})") +def _grounded_for_experiment(experiment: str) -> bool: + if experiment == "rookie": + return False + for index, stage in enumerate(STAGES): + if experiment_name(index, 0) == experiment: + return stage["grounded"] + raise ValueError(f"Unknown experiment for groundedness lookup: {experiment}") + + def run_stage_attempt(stage_index: int, attempt: int, args) -> str: exp = experiment_name(stage_index, attempt) resume = resume_checkpoint(stage_index, attempt, args.seed_checkpoint) @@ -150,8 +196,10 @@ def run_stage_attempt(stage_index: int, attempt: int, args) -> str: def reference_grounded(stage_index: int) -> bool: - # rookie.json predates the locomotion mask entirely — always full 3D. - return False if stage_index == 0 else STAGES[stage_index - 1]["grounded"] + if stage_index == 0: + # rookie.json predates the locomotion mask entirely — always full 3D. + return False + return _grounded_for_experiment(_reference_source_experiment(stage_index)) def evaluate_attempt(experiment: str, reference: str, episodes: int, stage_index: int) -> dict: diff --git a/training/curriculum_state.json b/training/curriculum_state.json index 0a657760..3e6f70f3 100644 --- a/training/curriculum_state.json +++ b/training/curriculum_state.json @@ -1,5 +1,5 @@ { - "stage_index": 3, + "stage_index": 4, "attempt": 0, "status": "in_progress", "log": [ @@ -40,16 +40,34 @@ "experiment": "curric-s3-no_draws", "attempt": 0, "eval": { - "timestamp": "2026-07-21T20:46:34+00:00", - "model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s3-no_draws.json", - "model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s2-defend.json", + "timestamp": "2026-07-22T11:36:42+00:00", + "model_a": "/Users/jcreek/Documents/repos/GitHub/CosmicClash/Game/bots/curric-s3-no_draws.json", + "model_b": "/Users/jcreek/Documents/repos/GitHub/CosmicClash/Game/bots/curric-s2-defend.json", "episodes": 100, - "wins_a": 44, - "wins_b": 19, - "draws": 37, - "win_rate_a": 0.44 + "wins_a": 26, + "wins_b": 60, + "draws": 14, + "win_rate_a": 0.26 }, - "decision": "pass" + "decision": "fail", + "note": "Original eval (44-19, recorded 2026-07-21T20:46:34) predates the locomotion-mask inference bugfix (8c15c46) and ran with the grounded stage-2 reference unfairly unmasked. Re-run post-fix with --grounded-b reverses the verdict: stage 3's full-3D unmask is a clear regression from stage 2, not an improvement. Not retried via the normal flag-retry mechanism — see stage_index 4 (aggression), which redirects around this branch by resuming from curric-s2-defend directly instead." + }, + { + "stage_index": 3, + "experiment": "curric-s4-mechanics", + "attempt": 0, + "eval": { + "timestamp": "2026-07-22T11:38:57+00:00", + "model_a": "/Users/jcreek/Documents/repos/GitHub/CosmicClash/Game/bots/curric-s4-mechanics.json", + "model_b": "/Users/jcreek/Documents/repos/GitHub/CosmicClash/Game/bots/curric-s3-no_draws.json", + "episodes": 100, + "wins_a": 26, + "wins_b": 33, + "draws": 41, + "win_rate_a": 0.26 + }, + "decision": "pass", + "note": "Passes only against its own (already-regressed) predecessor, curric-s3-no_draws. Evaluated directly against grounded curric-s2-defend (2026-07-22T11:40:07), curric-s4-mechanics also loses clearly: 24-57-19. Do not treat this stage's 'pass' as evidence curric-s4-mechanics is the strongest available model overall — see stage 2's note and stage_index 4 (aggression)." } ] } diff --git a/training/eval_history.json b/training/eval_history.json index 4ca470ca..f1ebda75 100644 --- a/training/eval_history.json +++ b/training/eval_history.json @@ -58,5 +58,35 @@ "wins_b": 19, "draws": 37, "win_rate_a": 0.44 + }, + { + "timestamp": "2026-07-22T11:36:42+00:00", + "model_a": "/Users/jcreek/Documents/repos/GitHub/CosmicClash/Game/bots/curric-s3-no_draws.json", + "model_b": "/Users/jcreek/Documents/repos/GitHub/CosmicClash/Game/bots/curric-s2-defend.json", + "episodes": 100, + "wins_a": 26, + "wins_b": 60, + "draws": 14, + "win_rate_a": 0.26 + }, + { + "timestamp": "2026-07-22T11:38:57+00:00", + "model_a": "/Users/jcreek/Documents/repos/GitHub/CosmicClash/Game/bots/curric-s4-mechanics.json", + "model_b": "/Users/jcreek/Documents/repos/GitHub/CosmicClash/Game/bots/curric-s3-no_draws.json", + "episodes": 100, + "wins_a": 26, + "wins_b": 33, + "draws": 41, + "win_rate_a": 0.26 + }, + { + "timestamp": "2026-07-22T11:40:07+00:00", + "model_a": "/Users/jcreek/Documents/repos/GitHub/CosmicClash/Game/bots/curric-s4-mechanics.json", + "model_b": "/Users/jcreek/Documents/repos/GitHub/CosmicClash/Game/bots/curric-s2-defend.json", + "episodes": 100, + "wins_a": 24, + "wins_b": 57, + "draws": 19, + "win_rate_a": 0.24 } ] diff --git a/training/train.py b/training/train.py index c84e7208..61ff55fc 100644 --- a/training/train.py +++ b/training/train.py @@ -84,6 +84,18 @@ def parse_args(): default=None, help="Allow pitch/roll rotation (default true)", ) + curriculum.add_argument( + "--velocity-to-ball-weight", type=float, default=None, + help="Overrides ShipAIController.velocity_to_ball_weight (dense reward for closing speed toward the ball)", + ) + curriculum.add_argument( + "--ball-distance-penalty", type=float, default=None, + help="Overrides ShipAIController.ball_distance_penalty (dense per-tick cost scaled by distance to the ball)", + ) + curriculum.add_argument( + "--ball-touch-reward", type=float, default=None, + help="Overrides ShipAIController.ball_touch_reward (event reward on ball contact, cooldown-gated)", + ) return parser.parse_args() @@ -101,6 +113,9 @@ def _curriculum_kwargs(args) -> dict: "ball_near_goal_chance": args.near_goal_chance, "ai_allow_vertical": args.allow_vertical, "ai_allow_pitch_roll": args.allow_pitch_roll, + "ai_velocity_to_ball_weight": args.velocity_to_ball_weight, + "ai_ball_distance_penalty": args.ball_distance_penalty, + "ai_ball_touch_reward": args.ball_touch_reward, } return {key: value for key, value in mapping.items() if value is not None}