diff --git a/TODO.md b/TODO.md index 1f5af620..01efa8a7 100644 --- a/TODO.md +++ b/TODO.md @@ -6,7 +6,7 @@ Deferred work, in rough priority order. The current architecture (ShipAction/Shi The training pipeline is built — see `TRAINING.md` (self-play PPO via the vendored godot_rl_agents bridge, JSON policy export, in-game GDScript inference, eval ladder). Remaining: -- [ ] Run the generation-5 handling/intercepts/league/teamplay curriculum described in `TRAINING.md`; promote later checkpoints as `medium`/`hard` only after they clear the match and behaviour gates. +- [ ] Run the generation-5 handling/intercepts/league/teamplay curriculum described in `TRAINING.md`; promote later checkpoints as `medium`/`hard` only after they clear the match and behaviour gates. The orchestrator now requires three independent paired evaluation seeds for each promotion decision; the current Stage 6 league run remains blocked on its recorded regression/telemetry results. - [ ] Extend generation 5's moving aerial-intercept states with wall plays and rebound scenarios after Stage 5 establishes a productive-air-touch baseline. - [ ] Design team-credit rewards and paired 2v2 evaluation before enabling the deferred teamplay stage. diff --git a/TRAINING.md b/TRAINING.md index db528877..44ba5148 100644 --- a/TRAINING.md +++ b/TRAINING.md @@ -855,7 +855,11 @@ real tail, the same way this one now has been. Stage 6's `league` opponent mode samples a historical exported policy at each episode reset. Each later stage preserves the preceding shaping and adds one -new difficulty. +new difficulty. The generation-5 orchestrator evaluates every candidate +against every reference on three independent paired seeds (`1, 19, 43`) before +advancing; pass `--evaluation-seeds` only when deliberately running a +different, recorded experiment. This avoids promoting a policy from a single +side-biased starting-state sequence. The physical-side gate is separate from the model-vs-model score. A paired side swap can make an identical policy appear perfectly balanced overall even diff --git a/training/generation5.py b/training/generation5.py index c959df18..64b5effb 100644 --- a/training/generation5.py +++ b/training/generation5.py @@ -38,6 +38,10 @@ PROMOTED_EASY = REPO_ROOT / "Game" / "bots" / "promoted" / "easy.json" MAX_RETRIES = 4 EVAL_EPISODES = 100 REGRESSION_MARGIN = 0.15 +# A single paired seed can produce a large physical-side swing even for a +# policy playing itself. Keep the first historical seed for continuity, but +# require two independent deterministic sequences before a stage can pass. +DEFAULT_EVALUATION_SEEDS = (1, 19, 43) # --min-head-entropy-frac / --ent-coef-max added 2026-08-24. The aggregate # entropy target is a SUM and read healthy (21% of h_max, on target) through # all nine Stage-5 attempts while thrust_y alone sat at 14% of its own ceiling @@ -582,11 +586,12 @@ def run_training(state: dict, stage_index: int, attempt: int, args) -> str: return experiment -def evaluate(experiment: str, reference: pathlib.Path, args) -> dict: +def evaluate(experiment: str, reference: pathlib.Path, args, seed: int) -> dict: candidate = REPO_ROOT / "Game" / "bots" / f"{experiment}.json" cmd = [ ".venv/bin/python", "evaluate.py", str(candidate), str(reference), "--episodes", str(EVAL_EPISODES), "--speedup", str(args.speedup), + "--seed", str(seed), ] if args.godot_bin: cmd += ["--godot_bin", args.godot_bin] @@ -628,11 +633,22 @@ def main() -> None: parser.add_argument("--n-parallel", type=int, default=14) parser.add_argument("--speedup", type=int, default=16) parser.add_argument("--godot-bin", default=None, help="Godot binary for post-stage evaluation") + parser.add_argument( + "--evaluation-seeds", + default=",".join(str(seed) for seed in DEFAULT_EVALUATION_SEEDS), + help="Comma-separated independent paired seeds required for every reference evaluation", + ) parser.add_argument("--foundation-checkpoint", default=str(FOUNDATION_CHECKPOINT)) parser.add_argument("--force-retry", action="store_true") parser.add_argument("--skip-to-next-stage", action="store_true") parser.add_argument("--dry-run", action="store_true", help="Print the next run command without executing it") args = parser.parse_args() + try: + evaluation_seeds = tuple(dict.fromkeys(int(value) for value in args.evaluation_seeds.split(",") if value.strip())) + except ValueError as error: + parser.error(f"--evaluation-seeds must be comma-separated integers: {error}") + if not evaluation_seeds: + parser.error("--evaluation-seeds requires at least one seed") state = load_state() if state["status"] == "done": @@ -670,7 +686,11 @@ def main() -> None: # Preserve order while avoiding a duplicate Stage-5 evaluation in # the league stage (its predecessor is also in the pool). references = list(dict.fromkeys(references)) - records = [evaluate(experiment, reference, args) for reference in references] + records = [ + evaluate(experiment, reference, args, seed) + for reference in references + for seed in evaluation_seeds + ] match_ok = all(match_passes(record) for record in records) evaluation_goal_floor = stage.get("evaluation_goal_rate_floor", 0.0) evaluation_goal_failures = [ diff --git a/training/test_generation5.py b/training/test_generation5.py index b3304bbb..d86f281c 100644 --- a/training/test_generation5.py +++ b/training/test_generation5.py @@ -11,6 +11,10 @@ def flag_value(flags: list[str], name: str) -> str: class Generation5ConfigTests(unittest.TestCase): + def test_default_evaluation_seeds_are_multiple_and_unique(self) -> None: + self.assertEqual(len(generation5.DEFAULT_EVALUATION_SEEDS), 3) + self.assertEqual(len(set(generation5.DEFAULT_EVALUATION_SEEDS)), 3) + def test_stage_sequence_and_lineage(self) -> None: self.assertEqual([stage["number"] for stage in generation5.STAGES], [4, 5, 6]) state = generation5.fresh_state()