From 8c15c466ef485ef30a44960a1f9666784ea65501 Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:23:09 +0100 Subject: [PATCH] fix(*): apply the locomotion mask during in-game/eval inference, not just training AIShipController (eval + real gameplay) ran the raw policy output unmasked regardless of allow_vertical/allow_pitch_roll, while ShipAIController (training) correctly discarded those axes for grounded curriculum stages. A grounded-trained model's untrained vertical/pitch-roll output reached the ship as noise during eval, understating it against models that were never handicapped this way. --- Game/scripts/ai_ship_controller.gd | 16 ++++++++++++--- Game/scripts/training_mode.gd | 12 +++++++++++ TRAINING.md | 7 +++++++ training/curriculum.py | 19 ++++++++++++++++-- training/evaluate.py | 32 +++++++++++++++++++++++++++--- 5 files changed, 78 insertions(+), 8 deletions(-) diff --git a/Game/scripts/ai_ship_controller.gd b/Game/scripts/ai_ship_controller.gd index 9107a027..981fba4d 100644 --- a/Game/scripts/ai_ship_controller.gd +++ b/Game/scripts/ai_ship_controller.gd @@ -17,6 +17,16 @@ extends ShipController # Uniform noise magnitude added to each action axis (0 = play at full skill). @export_range(0.0, 1.0) var action_noise: float = 0.0 +# Must mirror whatever the model was actually trained with (see +# ShipAIController's identical exports on the training side, curriculum +# stages 1-2 in TRAINING.md). A model trained grounded (mask on) never got a +# reward gradient on these axes, so its raw output there is untrained noise — +# leaving this true for such a model doesn't make it fly well, it just lets +# that noise reach the ship instead of being discarded like it was in +# training. Set false to match a grounded-trained model's actual behaviour. +@export var allow_vertical := true +@export var allow_pitch_roll := true + var _policy: PolicyNetwork var _action := ShipAction.new() var _ticks_until_decision := 0 @@ -53,13 +63,13 @@ func _decide() -> void: # gymnasium orders by SORTED key name — rotation xyz, thrust xyz, turbo # (> 0 means on) — NOT ShipAction's thrust-first declaration order. _action.rotation = Vector3( - _axis(out[0]), + _axis(out[0]) if allow_pitch_roll else 0.0, _axis(out[1]), - _axis(out[2]) + _axis(out[2]) if allow_pitch_roll else 0.0 ) _action.thrust = Vector3( _axis(out[3]), - _axis(out[4]), + _axis(out[4]) if allow_vertical else 0.0, _axis(out[5]) ) _action.turbo = out[6] > 0.0 diff --git a/Game/scripts/training_mode.gd b/Game/scripts/training_mode.gd index c5aec249..00f5d7c1 100644 --- a/Game/scripts/training_mode.gd +++ b/Game/scripts/training_mode.gd @@ -91,6 +91,12 @@ var _agents: Array[ShipAIController] = [] # Eval mode state (see header comment) var _eval := false var _eval_models: Array[String] = ["", ""] +# Per-model locomotion mask — must match how each model was actually trained +# (see AIShipController's identical exports), so a stage 1/2 (grounded) +# candidate isn't unfairly penalized by untrained aerial noise during eval +# that its training environment never had. +var _eval_allow_vertical: Array[bool] = [true, true] +var _eval_allow_pitch_roll: Array[bool] = [true, true] var _eval_episodes := 20 var _eval_goals := {0: 0, 1: 0} var _eval_draws := 0 @@ -123,6 +129,8 @@ func _start() -> void: for team in [0, 1]: var bot := AIShipController.new() bot.model_path = _eval_models[team] + bot.allow_vertical = _eval_allow_vertical[team] + bot.allow_pitch_roll = _eval_allow_pitch_roll[team] spawn_ship(team, 0, bot) return @@ -161,6 +169,10 @@ func _parse_eval_args() -> void: _eval_models[0] = args["eval_model_a"] _eval_models[1] = args["eval_model_b"] _eval_episodes = int(args.get("eval_episodes", str(_eval_episodes))) + _eval_allow_vertical[0] = _typed_like(args.get("eval_allow_vertical_a", "true"), true) + _eval_allow_vertical[1] = _typed_like(args.get("eval_allow_vertical_b", "true"), true) + _eval_allow_pitch_roll[0] = _typed_like(args.get("eval_allow_pitch_roll_a", "true"), true) + _eval_allow_pitch_roll[1] = _typed_like(args.get("eval_allow_pitch_roll_b", "true"), true) # TrainingMode @export names a curriculum run may override from the cmdline. diff --git a/TRAINING.md b/TRAINING.md index 21fb5249..4bb55372 100644 --- a/TRAINING.md +++ b/TRAINING.md @@ -122,6 +122,13 @@ appends to `training/eval_history.json` — the long-term progress record. Evaluate each new candidate against the previous promoted bot and a fixed early reference to see absolute progress over time. +If a model was trained with the locomotion mask on (curriculum stages 1-2 — +see below), pass `--grounded-a`/`--grounded-b` for whichever side it's on. +The eval otherwise runs `AIShipController` fully unmasked regardless of how a +model was trained, so a grounded model's untrained vertical/pitch-roll output +reaches the ship as noise it never had to contend with during training — +this understates it, not a neutral comparison. + ## Difficulty tiers A bot is `(model, reaction_ticks, action_noise)` — configured on the Match diff --git a/training/curriculum.py b/training/curriculum.py index 71d10329..62df69ea 100644 --- a/training/curriculum.py +++ b/training/curriculum.py @@ -60,6 +60,7 @@ STAGES = [ "--attack-goal-bias", "1.0", "--no-allow-vertical", "--no-allow-pitch-roll", ], + "grounded": True, }, { "name": "defend", @@ -67,14 +68,17 @@ STAGES = [ "--opponent-mode", "self_play", "--no-allow-vertical", "--no-allow-pitch-roll", ], + "grounded": True, }, { "name": "no_draws", "flags": ["--draw-penalty", "5"], + "grounded": False, }, { "name": "mechanics", "flags": [], + "grounded": False, }, ] @@ -145,9 +149,20 @@ def run_stage_attempt(stage_index: int, attempt: int, args) -> str: return exp -def evaluate_attempt(experiment: str, reference: str, episodes: int) -> dict: +def reference_grounded(stage_index: int) -> bool: + # rookie.json predates the locomotion mask entirely — always full 3D. + return False if stage_index == 0 else STAGES[stage_index - 1]["grounded"] + + +def evaluate_attempt(experiment: str, reference: str, episodes: int, stage_index: int) -> dict: candidate = TRAINING_DIR.parent / "Game" / "bots" / f"{experiment}.json" cmd = [".venv/bin/python", "evaluate.py", str(candidate), reference, "--episodes", str(episodes)] + # Must match how each side was actually trained — see ai_ship_controller.gd's + # allow_vertical/allow_pitch_roll and evaluate.py's --grounded-a/-b. + if STAGES[stage_index]["grounded"]: + cmd.append("--grounded-a") + if reference_grounded(stage_index): + cmd.append("--grounded-b") print(" ".join(cmd)) subprocess.run(cmd, cwd=TRAINING_DIR, check=True) history = json.loads(EVAL_HISTORY_PATH.read_text()) @@ -217,7 +232,7 @@ def main(): experiment = run_stage_attempt(stage_index, attempt, args) reference = reference_bot(stage_index) - record = evaluate_attempt(experiment, reference, EVAL_EPISODES) + record = evaluate_attempt(experiment, reference, EVAL_EPISODES, stage_index) decision = decide(record) print(f"{experiment}: candidate {record['wins_a']}-{record['wins_b']} reference " diff --git a/training/evaluate.py b/training/evaluate.py index e7cb6ba5..af2dd6c9 100644 --- a/training/evaluate.py +++ b/training/evaluate.py @@ -23,7 +23,16 @@ TRAINING_SCENE = "res://scenes/training.tscn" DEFAULT_GODOT_MACOS = "/Applications/Godot.app/Contents/MacOS/Godot" -def run_half(godot_bin: str, model_a: str, model_b: str, episodes: int, speedup: int, seed: int) -> dict: +def run_half( + godot_bin: str, + model_a: str, + model_b: str, + episodes: int, + speedup: int, + seed: int, + grounded_a: bool = False, + grounded_b: bool = False, +) -> dict: cmd = [ godot_bin, "--path", @@ -37,6 +46,14 @@ def run_half(godot_bin: str, model_a: str, model_b: str, episodes: int, speedup: f"--speedup={speedup}", f"--env_seed={seed}", ] + # Must match how each model was actually trained (see AIShipController's + # allow_vertical/allow_pitch_roll) — a curriculum stage 1/2 model never + # got a reward gradient on these axes, so leaving them unmasked here adds + # untrained aerial noise the model's own training never had to contend with. + if grounded_a: + cmd += ["--eval_allow_vertical_a=false", "--eval_allow_pitch_roll_a=false"] + if grounded_b: + cmd += ["--eval_allow_vertical_b=false", "--eval_allow_pitch_roll_b=false"] timeout = episodes * 30 / speedup * 3 + 120 # worst case: all draws, plus margin result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) for line in result.stdout.splitlines(): @@ -57,6 +74,12 @@ def main(): ) parser.add_argument("--speedup", type=int, default=16) parser.add_argument("--history", default=str(TRAINING_DIR / "eval_history.json")) + parser.add_argument( + "--grounded-a", action="store_true", help="model_a was trained with locomotion masked (curriculum stages 1-2)" + ) + parser.add_argument( + "--grounded-b", action="store_true", help="model_b was trained with locomotion masked (curriculum stages 1-2)" + ) args = parser.parse_args() model_a = str(pathlib.Path(args.model_a).resolve()) @@ -65,8 +88,11 @@ def main(): # Half the episodes on each side to cancel any residual side asymmetry; # different seeds so the halves see different randomized episode states. - first = run_half(args.godot_bin, model_a, model_b, half, args.speedup, seed=1) - second = run_half(args.godot_bin, model_b, model_a, half, args.speedup, seed=2) + # Groundedness is per physical model, so it swaps sides along with it. + first = run_half(args.godot_bin, model_a, model_b, half, args.speedup, seed=1, + grounded_a=args.grounded_a, grounded_b=args.grounded_b) + second = run_half(args.godot_bin, model_b, model_a, half, args.speedup, seed=2, + grounded_a=args.grounded_b, grounded_b=args.grounded_a) record = { "timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"),