fix(training): make policy evaluation portable

This commit is contained in:
Josh Creek
2026-09-01 18:56:31 +01:00
parent 066aee96cc
commit e56850a236
3 changed files with 25 additions and 0 deletions
+8
View File
@@ -139,6 +139,14 @@ appends to `training/eval_history.json` — the long-term progress record.
Evaluate each new candidate against the previous promoted bot and a fixed Evaluate each new candidate against the previous promoted bot and a fixed
early reference to see absolute progress over time. early reference to see absolute progress over time.
The evaluator launches Godot with the explicit headless display driver,
Compatibility renderer, dummy audio driver, and a temporary writable log path
so evaluation is reproducible on machines where the default renderer or
`user://` log location is unavailable. A two-seed smoke check on 2026-09-01
completed 10 paired episodes per seed for the current Stage 6 export versus
`hard.json` (both runs finished 35 with 2 draws); this is only a runtime
smoke and is not promotion evidence for the Stage 6 gate.
If a model was trained with the locomotion mask on (curriculum stages 1, 2, If a model was trained with the locomotion mask on (curriculum stages 1, 2,
and 5 — see below), pass `--grounded-a`/`--grounded-b` for whichever side it's on. and 5 — see below), pass `--grounded-a`/`--grounded-b` for whichever side it's on.
The eval otherwise runs `AIShipController` fully unmasked regardless of how a The eval otherwise runs `AIShipController` fully unmasked regardless of how a
+9
View File
@@ -17,6 +17,7 @@ import json
import os import os
import pathlib import pathlib
import subprocess import subprocess
import tempfile
TRAINING_DIR = pathlib.Path(__file__).resolve().parent TRAINING_DIR = pathlib.Path(__file__).resolve().parent
GAME_DIR = TRAINING_DIR.parent / "Game" GAME_DIR = TRAINING_DIR.parent / "Game"
@@ -37,6 +38,14 @@ def run_half(
) -> dict: ) -> dict:
cmd = [ cmd = [
godot_bin, godot_bin,
"--display-driver",
"headless",
"--rendering-method",
"gl_compatibility",
"--audio-driver",
"Dummy",
"--log-file",
str(pathlib.Path(tempfile.gettempdir()) / "cosmic-clash-evaluate-godot.log"),
"--path", "--path",
str(GAME_DIR), str(GAME_DIR),
TRAINING_SCENE, TRAINING_SCENE,
+8
View File
@@ -55,6 +55,14 @@ class EvaluatePairTests(unittest.TestCase):
command = run_process.call_args.args[0] command = run_process.call_args.args[0]
self.assertIn("--eval_team_size=2", command) self.assertIn("--eval_team_size=2", command)
@patch("evaluate.subprocess.run")
def test_run_uses_portable_headless_renderer_and_writable_log(self, run_process) -> None:
run_process.return_value.stdout = 'EVAL_RESULT {"episodes": 2, "goals_a": 1, "goals_b": 0, "draws": 1}\n'
evaluate.run_half("godot", "a", "b", 2, 16, 9)
command = run_process.call_args.args[0]
for option in ("--display-driver", "headless", "--rendering-method", "gl_compatibility", "--audio-driver", "Dummy", "--log-file"):
self.assertIn(option, command)
@patch("evaluate.run_half") @patch("evaluate.run_half")
def test_2v2_evaluation_preserves_side_swap_and_team_size(self, run_half) -> None: def test_2v2_evaluation_preserves_side_swap_and_team_size(self, run_half) -> None:
run_half.side_effect = [ run_half.side_effect = [