diff --git a/Game/scripts/training_mode.gd b/Game/scripts/training_mode.gd index e55f6da1..dd1312da 100644 --- a/Game/scripts/training_mode.gd +++ b/Game/scripts/training_mode.gd @@ -558,20 +558,45 @@ func _place_ground_start() -> void: ) +# Air-intercept drill geometry. These six ranges are not free tuning knobs — +# together they decide whether the drill is solvable at all, and the original +# values made it arithmetically impossible (see the Round 9 note in +# training/generation5.py). The constraint: the ball is only above +# ShipAIController.AIR_TOUCH_HEIGHT (5m) for a fixed window after the spawn, +# and the ship has to cross the gap within it. The ship's own numbers cap what +# it can do — vertical_thrust 120 / mass 5 = 24 m/s^2 up, less 9.8 gravity, and +# drag_coefficient 0.98/tick caps climb at roughly 12 m/s — so the window has +# to be sized against those, not chosen for how the drill looks. The values +# below were picked by simulating the spawn distribution against that flight +# envelope: an ideal interceptor now reaches the ball in ~98% of episodes and +# can do so above 5m in ~37%, versus 0% before. +const AIR_INTERCEPT_BALL_Y := Vector2(8.0, 14.0) # higher: more fall time above 5m +const AIR_INTERCEPT_BALL_SPEED := Vector2(4.0, 8.0) # slower: the ball outran the ship +const AIR_INTERCEPT_BEHIND := Vector2(4.0, 9.0) # closer: less gap to close +const AIR_INTERCEPT_LATERAL := 5.0 +const AIR_INTERCEPT_SHIP_Y := 4.0 # upper bound; FIELD_MIN_Y is the lower +# A ship in real play is already moving; spawning at a dead stop spent most of +# the drill window just building speed, which was the single largest cause of +# the old geometry being unreachable. Planar only, aimed at the ball, so the +# climb itself is still the ship's own problem to solve. +const AIR_INTERCEPT_SHIP_SPEED := Vector2(6.0, 14.0) + + # Goal-relevant aerial intercept: a high ball is already travelling toward a # randomly selected goal, while ships begin low and behind/lateral to its -# path. The generous wall clearance prevents rebound farming and an upright -# yaw-only spawn avoids wasting the short drill window on random recovery. +# path, already carrying planar speed toward it. The generous wall clearance +# prevents rebound farming and an upright yaw-only spawn avoids wasting the +# short drill window on random recovery. func _place_air_intercept() -> void: var goal := _goal_for_team(randi() % 2) var ball_position := Vector3( randf_range(-8.0, 8.0), - randf_range(6.0, minf(12.0, FIELD_MAX_Y)), + randf_range(AIR_INTERCEPT_BALL_Y.x, minf(AIR_INTERCEPT_BALL_Y.y, FIELD_MAX_Y)), randf_range(-10.0, 10.0) ) var to_goal := (goal.global_position - ball_position).normalized() var ball_velocity := (to_goal + Vector3(randf_range(-0.15, 0.15), randf_range(0.0, 0.15), 0.0)).normalized() \ - * randf_range(6.0, 11.0) + * randf_range(AIR_INTERCEPT_BALL_SPEED.x, AIR_INTERCEPT_BALL_SPEED.y) _place_body(ball, Transform3D(Basis.IDENTITY, ball_position), ball_velocity, Vector3.ZERO) var placed: Array[Vector3] = [] @@ -581,17 +606,23 @@ func _place_air_intercept() -> void: continue var ship_position := Vector3.ZERO for _attempt in 20: - var lateral := Vector3(-behind.z, 0.0, behind.x) * randf_range(-7.0, 7.0) - ship_position = ball_position + behind * randf_range(7.0, 13.0) + lateral + var lateral := Vector3(-behind.z, 0.0, behind.x) \ + * randf_range(-AIR_INTERCEPT_LATERAL, AIR_INTERCEPT_LATERAL) + ship_position = ball_position \ + + behind * randf_range(AIR_INTERCEPT_BEHIND.x, AIR_INTERCEPT_BEHIND.y) + lateral ship_position.x = clampf(ship_position.x, -FIELD_HALF_X, FIELD_HALF_X) - ship_position.y = randf_range(FIELD_MIN_Y, 3.0) + ship_position.y = randf_range(FIELD_MIN_Y, AIR_INTERCEPT_SHIP_Y) ship_position.z = clampf(ship_position.z, -FIELD_HALF_Z, FIELD_HALF_Z) if _spawn_position_clear(ship_position) and _far_enough_from(ship_position, placed): break placed.append(ship_position) var face_ball := ball_position - ship_position var yaw := atan2(-face_ball.x, -face_ball.z) - _place_body(ship, Transform3D(Basis.from_euler(Vector3(0.0, yaw, 0.0)), ship_position), Vector3.ZERO, Vector3.ZERO) + var run_up := Vector3(face_ball.x, 0.0, face_ball.z) + run_up = run_up.normalized() * randf_range( + AIR_INTERCEPT_SHIP_SPEED.x, AIR_INTERCEPT_SHIP_SPEED.y + ) if run_up.length_squared() > 0.0001 else Vector3.ZERO + _place_body(ship, Transform3D(Basis.from_euler(Vector3(0.0, yaw, 0.0)), ship_position), run_up, Vector3.ZERO) # Attacking/defending drill states: ball close to a goal, moving toward it. diff --git a/TRAINING.md b/TRAINING.md index ff4b5118..df00565e 100644 --- a/TRAINING.md +++ b/TRAINING.md @@ -690,6 +690,52 @@ own magnitude) and folded into `HANDLING_REWARD_FLAGS`. The three blocked attempts were deleted and Stage 5 restarts from Stage 4's checkpoint again with both terms active. +**Neither reward term was the problem — the drill was unsolvable.** The +third set of three attempts blocked on `productive_air_touch_fraction=0.0` +yet again, but this time the surrounding telemetry told a different story +from Rounds 7-8: the ship had measurably left the floor +(`airborne_fraction` 0.223 → 0.258, `mean_altitude` 2.59 → 3.25, +`vertical_thrust_mean` 0.004 → 0.063, `grounded_upright_fraction` 0.352 → +0.182), and watching a game confirmed it now chases and strikes the ball in +the air. The reward work had worked; the metric could not see it, because +`productive_air_touch_fraction` counts only touches with the *ball* above +`AIR_TOUCH_HEIGHT` (5 m), and `_place_air_intercept`'s spawn geometry never +produced a reachable one. + +Simulating that spawn distribution against the ship's real flight envelope +(`vertical_thrust` 120 / `mass` 5 = 24 m/s², less 9.8 gravity, with +`drag_coefficient` 0.98/tick capping climb near 12 m/s) settles it +arithmetically. A ball spawned 6-12 m up moving 6-11 m/s is above 5 m for a +median of **0.80 s**, while the ship spawned 7-13 m behind it, 3-10 m below +it, and **at a dead stop**. An *ideal* interceptor — point mass, instant +attitude, no righting torque, isotropic thrust, zero reaction delay — makes +that touch in **0.00%** of episodes and reaches the ball at all before it +lands in 0.5%. Six attempts and 360M steps were spent optimising against an +event the environment could not produce. + +The fix is in `_place_air_intercept` (see its `AIR_INTERCEPT_*` constants): +ball higher and slower, ship closer and already carrying planar speed toward +it. The dead-stop spawn was the single largest factor — a ship in real play +is already moving, and starting from rest spent most of the window just +building speed. The same simulation now puts an ideal interceptor at ~98% +reach and ~37% above 5 m, so the 0.005 floor has real headroom. +`AIR_TOUCH_HEIGHT` deliberately stays at 5.0: lowering the bar to meet a +broken drill would make the metric incomparable with earlier generations. + +Unlike every earlier round this does **not** restart from Stage 4's +checkpoint. That rule exists because a changed reward function invalidates +the learned value function; here the reward function is untouched and only +the environment's state distribution moves, so the existing policy — which +already learned to fly — is exactly what should be pointed at a now-reachable +target. `generation5_state.json` carries a one-shot `resume_override` for +this, consumed the first time `resume_checkpoint()` uses it. + +The general lesson, and the one worth carrying into later generations: when +a telemetry floor reads *exactly* zero while the behaviour it is meant to +measure is visibly happening, check that the environment can produce the +event at all before touching the reward function again. Cost of not checking +here: three rounds and 360M timesteps. + Stage 6's `league` opponent mode samples a historical exported policy at each episode reset. Each later stage preserves the preceding shaping and adds one new difficulty. diff --git a/training/generation5.py b/training/generation5.py index 8ac7db1c..a9c89816 100644 --- a/training/generation5.py +++ b/training/generation5.py @@ -210,6 +210,42 @@ STANDING_ARGS = ["--ent-coef", "0.01", "--entropy-floor"] # ~1.7x a fully-aligned ground one). Also folded into HANDLING_REWARD_FLAGS # so Stage 6 inherits it. Restarts Stage 5 from Stage 4's checkpoint again, # same reasoning as every prior mechanism change here. +# +# Round 9 (2026-08-21): the reward work in Rounds 7-8 was not the problem, and +# in fact worked. Across those three attempts the ship measurably left the +# floor -- airborne_fraction 0.223 -> 0.258, mean_altitude 2.59 -> 3.25, +# vertical_thrust_mean 0.004 -> 0.063, grounded_upright_fraction 0.352 -> +# 0.182 -- and the human watching it confirmed it now chases and strikes the +# ball in the air. productive_air_touch_fraction still read 0.0 because the +# event it counts was not reachable: it needs a touch with the *ball* above +# AIR_TOUCH_HEIGHT (5m), and _place_air_intercept's spawn geometry never +# allowed one. +# +# Simulating the spawn distribution against the ship's real flight envelope +# (vertical_thrust 120 / mass 5 = 24 m/s^2, less 9.8 gravity, with +# drag_coefficient 0.98/tick capping climb near 12 m/s) settles it +# arithmetically. The ball spawned 6-12m up and moving 6-11 m/s is above 5m +# for a median of only 0.80s, while the ship spawned 7-13m behind it, 3-10m +# below it, and at a dead stop. An *ideal* interceptor -- point mass, instant +# attitude, no righting torque, isotropic thrust, zero reaction delay -- makes +# that touch in 0.00% of episodes, and reaches the ball at all before it lands +# in 0.5%. Six attempts and 360M steps were spent optimising against an event +# the environment could not produce; the flat-at-exactly-zero metric was the +# environment's signature, not the policy's. +# +# The fix is in the drill, not the reward (see _place_air_intercept's +# constants in training_mode.gd): ball higher and slower, ship closer and +# already carrying planar speed toward it. Same simulation now puts an ideal +# interceptor at ~98% reach and ~37% above 5m, so the 0.005 floor has real +# headroom. AIR_TOUCH_HEIGHT stays 5.0 -- lowering the bar to meet a broken +# drill would make the metric incomparable with every earlier generation. +# +# Unlike Rounds 6-8 this does NOT restart from Stage 4's checkpoint. That rule +# exists because a changed reward function invalidates the learned value +# function; here the reward function is untouched and only the environment's +# state distribution moves, so retry2's policy -- which already learned to +# fly, per the telemetry above -- is exactly what should be pointed at a +# reachable target. Hence resume_override in generation5_state.json. HANDLING_REWARD_FLAGS = [ "--velocity-to-ball-weight", "0.04", "--forward-velocity-to-ball-weight", "0.15", @@ -328,7 +364,20 @@ def previous_attempt_entry(state: dict, stage_index: int, attempt: int) -> dict: raise RuntimeError(f"No previous attempt for stage index {stage_index}, attempt {attempt}") -def resume_checkpoint(state: dict, stage_index: int, attempt: int, foundation: pathlib.Path) -> pathlib.Path: +def resume_checkpoint( + state: dict, stage_index: int, attempt: int, foundation: pathlib.Path, consume: bool = True +) -> pathlib.Path: + # One-shot escape hatch for the case where a stage's attempt counter is + # reset but its accumulated policy is still worth keeping — i.e. the + # environment was fixed rather than the reward function, so the previous + # attempts' learning is still valid (see the Round 9 note above). Consumed + # on use so it can't silently pin later attempts to a stale checkpoint. + override = state.get("resume_override") + if override and override.get("stage_index") == stage_index and attempt == 0: + if consume: # --dry-run must be able to show the resume path without spending it + state.pop("resume_override") + save_state(state) + return TRAINING_DIR / "checkpoints" / override["experiment"] / "final.zip" if attempt > 0: exp = previous_attempt_entry(state, stage_index, attempt)["experiment"] return TRAINING_DIR / "checkpoints" / exp / "final.zip" @@ -389,7 +438,9 @@ def run_training(state: dict, stage_index: int, attempt: int, args) -> str: stage = STAGES[stage_index] suffix = "" if attempt == 0 else f"-retry{attempt}" experiment = f"{datetime.now().strftime('%Y%m%d-%H%M')}-gen5-s{stage['number']}-{stage['name']}{suffix}" - resume = resume_checkpoint(state, stage_index, attempt, pathlib.Path(args.foundation_checkpoint)) + resume = resume_checkpoint( + state, stage_index, attempt, pathlib.Path(args.foundation_checkpoint), consume=not args.dry_run + ) if not resume.exists(): raise FileNotFoundError(f"Resume checkpoint not found: {resume}") cmd = [ diff --git a/training/generation5_state.json b/training/generation5_state.json index 41d257dd..44e954d7 100644 --- a/training/generation5_state.json +++ b/training/generation5_state.json @@ -1,7 +1,7 @@ { "stage_index": 1, - "attempt": 2, - "status": "blocked", + "attempt": 0, + "status": "in_progress", "log": [ { "stage_index": 0, @@ -506,5 +506,10 @@ ], "decision": "fail" } - ] + ], + "resume_override": { + "stage_index": 1, + "experiment": "20260821-0056-gen5-s5-intercepts-retry2", + "reason": "Stage 5 attempts 1-3 blocked on productive_air_touch_fraction=0.0, but the cause was _place_air_intercept's unreachable spawn geometry, not the policy or the reward function (see the Round 9 note in generation5.py). The drill was fixed; the reward function is unchanged, so this lineage's learned flight behaviour is kept rather than restarted from Stage 4." + } }