fix(*): apply the locomotion mask during in-game/eval inference, not just training

AIShipController (eval + real gameplay) ran the raw policy output unmasked
regardless of allow_vertical/allow_pitch_roll, while ShipAIController
(training) correctly discarded those axes for grounded curriculum stages.
A grounded-trained model's untrained vertical/pitch-roll output reached the
ship as noise during eval, understating it against models that were never
handicapped this way.
This commit is contained in:
Josh Creek
2026-07-21 22:23:09 +01:00
parent dd2b3c570b
commit 8c15c466ef
5 changed files with 78 additions and 8 deletions
+17 -2
View File
@@ -60,6 +60,7 @@ STAGES = [
"--attack-goal-bias", "1.0",
"--no-allow-vertical", "--no-allow-pitch-roll",
],
"grounded": True,
},
{
"name": "defend",
@@ -67,14 +68,17 @@ STAGES = [
"--opponent-mode", "self_play",
"--no-allow-vertical", "--no-allow-pitch-roll",
],
"grounded": True,
},
{
"name": "no_draws",
"flags": ["--draw-penalty", "5"],
"grounded": False,
},
{
"name": "mechanics",
"flags": [],
"grounded": False,
},
]
@@ -145,9 +149,20 @@ def run_stage_attempt(stage_index: int, attempt: int, args) -> str:
return exp
def evaluate_attempt(experiment: str, reference: str, episodes: int) -> dict:
def reference_grounded(stage_index: int) -> bool:
# rookie.json predates the locomotion mask entirely — always full 3D.
return False if stage_index == 0 else STAGES[stage_index - 1]["grounded"]
def evaluate_attempt(experiment: str, reference: str, episodes: int, stage_index: int) -> dict:
candidate = TRAINING_DIR.parent / "Game" / "bots" / f"{experiment}.json"
cmd = [".venv/bin/python", "evaluate.py", str(candidate), reference, "--episodes", str(episodes)]
# Must match how each side was actually trained — see ai_ship_controller.gd's
# allow_vertical/allow_pitch_roll and evaluate.py's --grounded-a/-b.
if STAGES[stage_index]["grounded"]:
cmd.append("--grounded-a")
if reference_grounded(stage_index):
cmd.append("--grounded-b")
print(" ".join(cmd))
subprocess.run(cmd, cwd=TRAINING_DIR, check=True)
history = json.loads(EVAL_HISTORY_PATH.read_text())
@@ -217,7 +232,7 @@ def main():
experiment = run_stage_attempt(stage_index, attempt, args)
reference = reference_bot(stage_index)
record = evaluate_attempt(experiment, reference, EVAL_EPISODES)
record = evaluate_attempt(experiment, reference, EVAL_EPISODES, stage_index)
decision = decide(record)
print(f"{experiment}: candidate {record['wins_a']}-{record['wins_b']} reference "