diff --git a/training/evaluate.py b/training/evaluate.py index af2dd6c9..a8147fe5 100644 --- a/training/evaluate.py +++ b/training/evaluate.py @@ -2,8 +2,9 @@ Uses the same in-Godot inference path that ships in the game (AIShipController + PolicyNetwork), so eval strength = in-game strength. -Episodes are golden-goal: first goal wins, timeout is a draw. Half the -episodes are played with sides swapped for fairness. Results are appended to +Episodes are golden-goal: first goal wins, timeout is a draw. Every randomized +starting state is played twice with sides swapped, so physical-team or arena +asymmetry cannot be mistaken for model strength. Results are appended to eval_history.json — the bot-progress-over-time record. Example: @@ -47,9 +48,9 @@ def run_half( f"--env_seed={seed}", ] # Must match how each model was actually trained (see AIShipController's - # allow_vertical/allow_pitch_roll) — a curriculum stage 1/2 model never - # got a reward gradient on these axes, so leaving them unmasked here adds - # untrained aerial noise the model's own training never had to contend with. + # allow_vertical/allow_pitch_roll) — a grounded pre-generation-4 model + # never got a reward gradient on these axes, so leaving them unmasked here + # adds untrained aerial noise its training never had to contend with. if grounded_a: cmd += ["--eval_allow_vertical_a=false", "--eval_allow_pitch_roll_a=false"] if grounded_b: @@ -62,48 +63,95 @@ def run_half( raise RuntimeError(f"No EVAL_RESULT in godot output:\n{result.stdout[-2000:]}\n{result.stderr[-2000:]}") +def evaluate_pair( + godot_bin: str, + model_a: str, + model_b: str, + episodes: int, + speedup: int, + seed: int, + grounded_a: bool = False, + grounded_b: bool = False, +) -> dict: + """Replay one seeded state sequence with the models on opposite sides.""" + if episodes < 2 or episodes % 2 != 0: + raise ValueError("--episodes must be an even number of at least 2 for paired side swaps") + + episodes_per_side = episodes // 2 + first = run_half( + godot_bin, model_a, model_b, episodes_per_side, speedup, seed, + grounded_a=grounded_a, grounded_b=grounded_b, + ) + second = run_half( + godot_bin, model_b, model_a, episodes_per_side, speedup, seed, + grounded_a=grounded_b, grounded_b=grounded_a, + ) + + a_team_0 = { + "wins_a": first["goals_a"], + "wins_b": first["goals_b"], + "draws": first["draws"], + } + a_team_1 = { + "wins_a": second["goals_b"], + "wins_b": second["goals_a"], + "draws": second["draws"], + } + record = { + "timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"), + "model_a": model_a, + "model_b": model_b, + "seed": seed, + "episodes": first["episodes"] + second["episodes"], + "wins_a": a_team_0["wins_a"] + a_team_1["wins_a"], + "wins_b": a_team_0["wins_b"] + a_team_1["wins_b"], + "draws": a_team_0["draws"] + a_team_1["draws"], + "side_results": { + "a_team_0": a_team_0, + "a_team_1": a_team_1, + }, + "physical_team_wins": { + "team_0": first["goals_a"] + second["goals_a"], + "team_1": first["goals_b"] + second["goals_b"], + }, + } + record["win_rate_a"] = round(record["wins_a"] / record["episodes"], 3) + return record + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("model_a", help="Path to first exported policy .json") parser.add_argument("model_b", help="Path to second exported policy .json") - parser.add_argument("--episodes", type=int, default=20, help="Total episodes (split across side swap)") + parser.add_argument( + "--episodes", type=int, default=20, + help="Total episodes; must be even so every seeded state is replayed with sides swapped", + ) parser.add_argument( "--godot_bin", default=os.environ.get("GODOT_BIN", DEFAULT_GODOT_MACOS), help="Path to the Godot binary (or set GODOT_BIN)", ) parser.add_argument("--speedup", type=int, default=16) + parser.add_argument("--seed", type=int, default=1, help="Seed for the paired starting-state sequence") parser.add_argument("--history", default=str(TRAINING_DIR / "eval_history.json")) parser.add_argument( - "--grounded-a", action="store_true", help="model_a was trained with locomotion masked (curriculum stages 1-2)" + "--grounded-a", action="store_true", help="model_a was trained with locomotion masked (pre-generation-4 models)" ) parser.add_argument( - "--grounded-b", action="store_true", help="model_b was trained with locomotion masked (curriculum stages 1-2)" + "--grounded-b", action="store_true", help="model_b was trained with locomotion masked (pre-generation-4 models)" ) args = parser.parse_args() model_a = str(pathlib.Path(args.model_a).resolve()) model_b = str(pathlib.Path(args.model_b).resolve()) - half = max(args.episodes // 2, 1) - - # Half the episodes on each side to cancel any residual side asymmetry; - # different seeds so the halves see different randomized episode states. - # Groundedness is per physical model, so it swaps sides along with it. - first = run_half(args.godot_bin, model_a, model_b, half, args.speedup, seed=1, - grounded_a=args.grounded_a, grounded_b=args.grounded_b) - second = run_half(args.godot_bin, model_b, model_a, half, args.speedup, seed=2, - grounded_a=args.grounded_b, grounded_b=args.grounded_a) - - record = { - "timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"), - "model_a": model_a, - "model_b": model_b, - "episodes": first["episodes"] + second["episodes"], - "wins_a": first["goals_a"] + second["goals_b"], - "wins_b": first["goals_b"] + second["goals_a"], - "draws": first["draws"] + second["draws"], - } - record["win_rate_a"] = round(record["wins_a"] / record["episodes"], 3) + try: + record = evaluate_pair( + args.godot_bin, model_a, model_b, args.episodes, args.speedup, args.seed, + grounded_a=args.grounded_a, grounded_b=args.grounded_b, + ) + except ValueError as error: + parser.error(str(error)) history_path = pathlib.Path(args.history) history = json.loads(history_path.read_text()) if history_path.exists() else [] @@ -115,6 +163,15 @@ def main(): f"{record['wins_a']}-{record['wins_b']} ({record['draws']} draws), " f"win rate A = {record['win_rate_a']:.0%}" ) + a_team_0 = record["side_results"]["a_team_0"] + a_team_1 = record["side_results"]["a_team_1"] + physical = record["physical_team_wins"] + print( + f"Paired seed {record['seed']} side split: " + f"A as team 0 {a_team_0['wins_a']}-{a_team_0['wins_b']} ({a_team_0['draws']} draws); " + f"A as team 1 {a_team_1['wins_a']}-{a_team_1['wins_b']} ({a_team_1['draws']} draws); " + f"physical teams 0-1 = {physical['team_0']}-{physical['team_1']}" + ) print(f"Appended to {history_path}") diff --git a/training/test_evaluate.py b/training/test_evaluate.py new file mode 100644 index 00000000..fbff3f22 --- /dev/null +++ b/training/test_evaluate.py @@ -0,0 +1,62 @@ +"""Offline regression checks for evaluate.py's paired side-swap logic.""" + +import unittest +from unittest.mock import patch + +import evaluate + + +class EvaluatePairTests(unittest.TestCase): + @patch("evaluate.run_half") + def test_replays_same_seed_with_models_and_masks_swapped(self, run_half) -> None: + run_half.side_effect = [ + {"episodes": 4, "goals_a": 3, "goals_b": 1, "draws": 0}, + {"episodes": 4, "goals_a": 2, "goals_b": 1, "draws": 1}, + ] + + record = evaluate.evaluate_pair( + "godot", "candidate.json", "reference.json", 8, 16, 42, + grounded_a=False, grounded_b=True, + ) + + self.assertEqual(run_half.call_args_list[0].args, ("godot", "candidate.json", "reference.json", 4, 16, 42)) + self.assertEqual(run_half.call_args_list[1].args, ("godot", "reference.json", "candidate.json", 4, 16, 42)) + self.assertEqual( + run_half.call_args_list[0].kwargs, + {"grounded_a": False, "grounded_b": True}, + ) + self.assertEqual( + run_half.call_args_list[1].kwargs, + {"grounded_a": True, "grounded_b": False}, + ) + self.assertEqual(record["wins_a"], 4) + self.assertEqual(record["wins_b"], 3) + self.assertEqual(record["draws"], 1) + self.assertEqual(record["physical_team_wins"], {"team_0": 5, "team_1": 2}) + self.assertEqual(record["side_results"]["a_team_0"]["wins_a"], 3) + self.assertEqual(record["side_results"]["a_team_1"]["wins_a"], 1) + + def test_rejects_unpaired_episode_counts(self) -> None: + for episodes in (0, 1, 3, 99): + with self.subTest(episodes=episodes): + with self.assertRaisesRegex(ValueError, "even number"): + evaluate.evaluate_pair("godot", "a", "b", episodes, 16, 1) + + @patch("evaluate.run_half") + def test_identical_policy_results_cancel_physical_side_bias(self, run_half) -> None: + # Replaying the same deterministic matchup must produce the same + # physical-team result. Model A receives opposite sides in the two + # halves, so even a large team-0 advantage cancels exactly. + physical_result = {"episodes": 10, "goals_a": 8, "goals_b": 1, "draws": 1} + run_half.side_effect = [physical_result, physical_result] + + record = evaluate.evaluate_pair("godot", "same.json", "same.json", 20, 16, 7) + + self.assertEqual(record["wins_a"], 9) + self.assertEqual(record["wins_b"], 9) + self.assertEqual(record["draws"], 2) + self.assertEqual(record["win_rate_a"], 0.45) + + +if __name__ == "__main__": + unittest.main()