"""Offline regression checks for evaluate.py's paired side-swap logic.""" import unittest from unittest.mock import patch import evaluate class EvaluatePairTests(unittest.TestCase): @patch("evaluate.run_half") def test_replays_same_seed_with_models_and_masks_swapped(self, run_half) -> None: run_half.side_effect = [ {"episodes": 4, "goals_a": 3, "goals_b": 1, "draws": 0}, {"episodes": 4, "goals_a": 2, "goals_b": 1, "draws": 1}, ] record = evaluate.evaluate_pair( "godot", "candidate.json", "reference.json", 8, 16, 42, grounded_a=False, grounded_b=True, ) self.assertEqual(run_half.call_args_list[0].args, ("godot", "candidate.json", "reference.json", 4, 16, 42)) self.assertEqual(run_half.call_args_list[1].args, ("godot", "reference.json", "candidate.json", 4, 16, 42)) self.assertEqual( run_half.call_args_list[0].kwargs, {"grounded_a": False, "grounded_b": True, "team_size": 1}, ) self.assertEqual( run_half.call_args_list[1].kwargs, {"grounded_a": True, "grounded_b": False, "team_size": 1}, ) self.assertEqual(record["wins_a"], 4) self.assertEqual(record["wins_b"], 3) self.assertEqual(record["draws"], 1) self.assertEqual(record["physical_team_wins"], {"team_0": 5, "team_1": 2}) self.assertEqual(record["side_results"]["a_team_0"]["wins_a"], 3) self.assertEqual(record["side_results"]["a_team_1"]["wins_a"], 1) def test_rejects_unpaired_episode_counts(self) -> None: for episodes in (0, 1, 3, 99): with self.subTest(episodes=episodes): with self.assertRaisesRegex(ValueError, "even number"): evaluate.evaluate_pair("godot", "a", "b", episodes, 16, 1) def test_rejects_unsupported_team_size_before_launch(self) -> None: with self.assertRaisesRegex(ValueError, "team_size must be 1 or 2"): evaluate.run_half("godot", "a", "b", 2, 16, 1, team_size=3) with self.assertRaisesRegex(ValueError, "team_size must be 1 or 2"): evaluate.evaluate_pair("godot", "a", "b", 2, 16, 1, team_size=3) @patch("evaluate.subprocess.run") def test_2v2_run_passes_team_size_to_godot(self, run_process) -> None: run_process.return_value.stdout = 'EVAL_RESULT {"episodes": 2, "goals_a": 1, "goals_b": 0, "draws": 1}\n' evaluate.run_half("godot", "a", "b", 2, 16, 9, team_size=2) command = run_process.call_args.args[0] self.assertIn("--eval_team_size=2", command) @patch("evaluate.subprocess.run") def test_run_uses_portable_headless_renderer_and_writable_log(self, run_process) -> None: run_process.return_value.stdout = 'EVAL_RESULT {"episodes": 2, "goals_a": 1, "goals_b": 0, "draws": 1}\n' evaluate.run_half("godot", "a", "b", 2, 16, 9) command = run_process.call_args.args[0] for option in ("--display-driver", "headless", "--rendering-method", "gl_compatibility", "--audio-driver", "Dummy", "--log-file"): self.assertIn(option, command) @patch("evaluate.run_half") def test_2v2_evaluation_preserves_side_swap_and_team_size(self, run_half) -> None: run_half.side_effect = [ {"episodes": 2, "goals_a": 1, "goals_b": 0, "draws": 1}, {"episodes": 2, "goals_a": 0, "goals_b": 1, "draws": 1}, ] evaluate.evaluate_pair("godot", "a", "b", 4, 16, 9, team_size=2) self.assertEqual(run_half.call_args_list[0].kwargs["team_size"], 2) self.assertEqual(run_half.call_args_list[1].kwargs["team_size"], 2) @patch("evaluate.run_half") def test_identical_policy_results_cancel_physical_side_bias(self, run_half) -> None: # Replaying the same deterministic matchup must produce the same # physical-team result. Model A receives opposite sides in the two # halves, so even a large team-0 advantage cancels exactly. physical_result = {"episodes": 10, "goals_a": 8, "goals_b": 1, "draws": 1} run_half.side_effect = [physical_result, physical_result] record = evaluate.evaluate_pair("godot", "same.json", "same.json", 20, 16, 7) self.assertEqual(record["wins_a"], 9) self.assertEqual(record["wins_b"], 9) self.assertEqual(record["draws"], 2) self.assertEqual(record["win_rate_a"], 0.45) if __name__ == "__main__": unittest.main()