mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-10 16:04:04 +00:00
Josh Creek
86 lines
3.9 KiB
Python
86 lines
3.9 KiB
Python
"""Offline regression checks for evaluate.py's paired side-swap logic."""
|
|
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import evaluate
|
|
|
|
|
|
class EvaluatePairTests(unittest.TestCase):
|
|
@patch("evaluate.run_half")
|
|
def test_replays_same_seed_with_models_and_masks_swapped(self, run_half) -> None:
|
|
run_half.side_effect = [
|
|
{"episodes": 4, "goals_a": 3, "goals_b": 1, "draws": 0},
|
|
{"episodes": 4, "goals_a": 2, "goals_b": 1, "draws": 1},
|
|
]
|
|
|
|
record = evaluate.evaluate_pair(
|
|
"godot", "candidate.json", "reference.json", 8, 16, 42,
|
|
grounded_a=False, grounded_b=True,
|
|
)
|
|
|
|
self.assertEqual(run_half.call_args_list[0].args, ("godot", "candidate.json", "reference.json", 4, 16, 42))
|
|
self.assertEqual(run_half.call_args_list[1].args, ("godot", "reference.json", "candidate.json", 4, 16, 42))
|
|
self.assertEqual(
|
|
run_half.call_args_list[0].kwargs,
|
|
{"grounded_a": False, "grounded_b": True, "team_size": 1},
|
|
)
|
|
self.assertEqual(
|
|
run_half.call_args_list[1].kwargs,
|
|
{"grounded_a": True, "grounded_b": False, "team_size": 1},
|
|
)
|
|
self.assertEqual(record["wins_a"], 4)
|
|
self.assertEqual(record["wins_b"], 3)
|
|
self.assertEqual(record["draws"], 1)
|
|
self.assertEqual(record["physical_team_wins"], {"team_0": 5, "team_1": 2})
|
|
self.assertEqual(record["side_results"]["a_team_0"]["wins_a"], 3)
|
|
self.assertEqual(record["side_results"]["a_team_1"]["wins_a"], 1)
|
|
|
|
def test_rejects_unpaired_episode_counts(self) -> None:
|
|
for episodes in (0, 1, 3, 99):
|
|
with self.subTest(episodes=episodes):
|
|
with self.assertRaisesRegex(ValueError, "even number"):
|
|
evaluate.evaluate_pair("godot", "a", "b", episodes, 16, 1)
|
|
|
|
def test_rejects_unsupported_team_size_before_launch(self) -> None:
|
|
with self.assertRaisesRegex(ValueError, "team_size must be 1 or 2"):
|
|
evaluate.run_half("godot", "a", "b", 2, 16, 1, team_size=3)
|
|
with self.assertRaisesRegex(ValueError, "team_size must be 1 or 2"):
|
|
evaluate.evaluate_pair("godot", "a", "b", 2, 16, 1, team_size=3)
|
|
|
|
@patch("evaluate.subprocess.run")
|
|
def test_2v2_run_passes_team_size_to_godot(self, run_process) -> None:
|
|
run_process.return_value.stdout = 'EVAL_RESULT {"episodes": 2, "goals_a": 1, "goals_b": 0, "draws": 1}\n'
|
|
evaluate.run_half("godot", "a", "b", 2, 16, 9, team_size=2)
|
|
command = run_process.call_args.args[0]
|
|
self.assertIn("--eval_team_size=2", command)
|
|
|
|
@patch("evaluate.run_half")
|
|
def test_2v2_evaluation_preserves_side_swap_and_team_size(self, run_half) -> None:
|
|
run_half.side_effect = [
|
|
{"episodes": 2, "goals_a": 1, "goals_b": 0, "draws": 1},
|
|
{"episodes": 2, "goals_a": 0, "goals_b": 1, "draws": 1},
|
|
]
|
|
evaluate.evaluate_pair("godot", "a", "b", 4, 16, 9, team_size=2)
|
|
self.assertEqual(run_half.call_args_list[0].kwargs["team_size"], 2)
|
|
self.assertEqual(run_half.call_args_list[1].kwargs["team_size"], 2)
|
|
|
|
@patch("evaluate.run_half")
|
|
def test_identical_policy_results_cancel_physical_side_bias(self, run_half) -> None:
|
|
# Replaying the same deterministic matchup must produce the same
|
|
# physical-team result. Model A receives opposite sides in the two
|
|
# halves, so even a large team-0 advantage cancels exactly.
|
|
physical_result = {"episodes": 10, "goals_a": 8, "goals_b": 1, "draws": 1}
|
|
run_half.side_effect = [physical_result, physical_result]
|
|
|
|
record = evaluate.evaluate_pair("godot", "same.json", "same.json", 20, 16, 7)
|
|
|
|
self.assertEqual(record["wins_a"], 9)
|
|
self.assertEqual(record["wins_b"], 9)
|
|
self.assertEqual(record["draws"], 2)
|
|
self.assertEqual(record["win_rate_a"], 0.45)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|