mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-10 16:04:04 +00:00
fix(training): make the stage-5 air-intercept drill physically solvable
productive_air_touch_fraction sat at exactly 0.0 across nine Stage-5 attempts and 540M timesteps. Two rounds of reward shaping were aimed at it (air_approach_weight, then air_touch_bonus_weight); both worked -- airborne_fraction 0.223->0.258, mean_altitude 2.59->3.25, vertical_thrust_mean 0.004->0.063 -- and the ship now visibly plays the ball in the air. The metric could not see it because it counts only touches with the ball above AIR_TOUCH_HEIGHT (5m), and _place_air_intercept never produced a reachable one. Simulating the spawn distribution against the ship's flight envelope (vertical_thrust 120 / mass 5 = 24 m/s^2 less gravity, drag capping climb near 12 m/s): a ball spawned 6-12m up at 6-11 m/s is above 5m for a median of 0.80s, while the ship spawned 7-13m behind, 3-10m below, and at a dead stop. An ideal interceptor -- point mass, instant attitude, no righting torque, zero reaction delay -- makes that touch in 0.00% of episodes and reaches the ball at all in 0.5%. Retune the drill instead of the reward: ball higher (8-14m) and slower (4-8 m/s), ship closer (4-9m behind), narrower lateral spread, and a 6-14 m/s planar run-up rather than a standing start -- the dead stop was the largest single factor. Ideal interceptor now reaches the ball in ~98% of episodes and above 5m in ~37%, so the 0.005 floor has headroom. AIR_TOUCH_HEIGHT stays 5.0 so the metric remains comparable with earlier generations. Resume from retry2 rather than restarting from Stage 4: that rule guards against a changed reward function invalidating the value function, and the reward function is untouched here -- only the state distribution moved, so the policy that already learned to fly is what should be pointed at a reachable target. Adds a one-shot resume_override to generation5_state.json, consumed on first use.
This commit is contained in:
Josh Creek
@@ -558,20 +558,45 @@ func _place_ground_start() -> void:
|
||||
)
|
||||
|
||||
|
||||
# Air-intercept drill geometry. These six ranges are not free tuning knobs —
|
||||
# together they decide whether the drill is solvable at all, and the original
|
||||
# values made it arithmetically impossible (see the Round 9 note in
|
||||
# training/generation5.py). The constraint: the ball is only above
|
||||
# ShipAIController.AIR_TOUCH_HEIGHT (5m) for a fixed window after the spawn,
|
||||
# and the ship has to cross the gap within it. The ship's own numbers cap what
|
||||
# it can do — vertical_thrust 120 / mass 5 = 24 m/s^2 up, less 9.8 gravity, and
|
||||
# drag_coefficient 0.98/tick caps climb at roughly 12 m/s — so the window has
|
||||
# to be sized against those, not chosen for how the drill looks. The values
|
||||
# below were picked by simulating the spawn distribution against that flight
|
||||
# envelope: an ideal interceptor now reaches the ball in ~98% of episodes and
|
||||
# can do so above 5m in ~37%, versus 0% before.
|
||||
const AIR_INTERCEPT_BALL_Y := Vector2(8.0, 14.0) # higher: more fall time above 5m
|
||||
const AIR_INTERCEPT_BALL_SPEED := Vector2(4.0, 8.0) # slower: the ball outran the ship
|
||||
const AIR_INTERCEPT_BEHIND := Vector2(4.0, 9.0) # closer: less gap to close
|
||||
const AIR_INTERCEPT_LATERAL := 5.0
|
||||
const AIR_INTERCEPT_SHIP_Y := 4.0 # upper bound; FIELD_MIN_Y is the lower
|
||||
# A ship in real play is already moving; spawning at a dead stop spent most of
|
||||
# the drill window just building speed, which was the single largest cause of
|
||||
# the old geometry being unreachable. Planar only, aimed at the ball, so the
|
||||
# climb itself is still the ship's own problem to solve.
|
||||
const AIR_INTERCEPT_SHIP_SPEED := Vector2(6.0, 14.0)
|
||||
|
||||
|
||||
# Goal-relevant aerial intercept: a high ball is already travelling toward a
|
||||
# randomly selected goal, while ships begin low and behind/lateral to its
|
||||
# path. The generous wall clearance prevents rebound farming and an upright
|
||||
# yaw-only spawn avoids wasting the short drill window on random recovery.
|
||||
# path, already carrying planar speed toward it. The generous wall clearance
|
||||
# prevents rebound farming and an upright yaw-only spawn avoids wasting the
|
||||
# short drill window on random recovery.
|
||||
func _place_air_intercept() -> void:
|
||||
var goal := _goal_for_team(randi() % 2)
|
||||
var ball_position := Vector3(
|
||||
randf_range(-8.0, 8.0),
|
||||
randf_range(6.0, minf(12.0, FIELD_MAX_Y)),
|
||||
randf_range(AIR_INTERCEPT_BALL_Y.x, minf(AIR_INTERCEPT_BALL_Y.y, FIELD_MAX_Y)),
|
||||
randf_range(-10.0, 10.0)
|
||||
)
|
||||
var to_goal := (goal.global_position - ball_position).normalized()
|
||||
var ball_velocity := (to_goal + Vector3(randf_range(-0.15, 0.15), randf_range(0.0, 0.15), 0.0)).normalized() \
|
||||
* randf_range(6.0, 11.0)
|
||||
* randf_range(AIR_INTERCEPT_BALL_SPEED.x, AIR_INTERCEPT_BALL_SPEED.y)
|
||||
_place_body(ball, Transform3D(Basis.IDENTITY, ball_position), ball_velocity, Vector3.ZERO)
|
||||
|
||||
var placed: Array[Vector3] = []
|
||||
@@ -581,17 +606,23 @@ func _place_air_intercept() -> void:
|
||||
continue
|
||||
var ship_position := Vector3.ZERO
|
||||
for _attempt in 20:
|
||||
var lateral := Vector3(-behind.z, 0.0, behind.x) * randf_range(-7.0, 7.0)
|
||||
ship_position = ball_position + behind * randf_range(7.0, 13.0) + lateral
|
||||
var lateral := Vector3(-behind.z, 0.0, behind.x) \
|
||||
* randf_range(-AIR_INTERCEPT_LATERAL, AIR_INTERCEPT_LATERAL)
|
||||
ship_position = ball_position \
|
||||
+ behind * randf_range(AIR_INTERCEPT_BEHIND.x, AIR_INTERCEPT_BEHIND.y) + lateral
|
||||
ship_position.x = clampf(ship_position.x, -FIELD_HALF_X, FIELD_HALF_X)
|
||||
ship_position.y = randf_range(FIELD_MIN_Y, 3.0)
|
||||
ship_position.y = randf_range(FIELD_MIN_Y, AIR_INTERCEPT_SHIP_Y)
|
||||
ship_position.z = clampf(ship_position.z, -FIELD_HALF_Z, FIELD_HALF_Z)
|
||||
if _spawn_position_clear(ship_position) and _far_enough_from(ship_position, placed):
|
||||
break
|
||||
placed.append(ship_position)
|
||||
var face_ball := ball_position - ship_position
|
||||
var yaw := atan2(-face_ball.x, -face_ball.z)
|
||||
_place_body(ship, Transform3D(Basis.from_euler(Vector3(0.0, yaw, 0.0)), ship_position), Vector3.ZERO, Vector3.ZERO)
|
||||
var run_up := Vector3(face_ball.x, 0.0, face_ball.z)
|
||||
run_up = run_up.normalized() * randf_range(
|
||||
AIR_INTERCEPT_SHIP_SPEED.x, AIR_INTERCEPT_SHIP_SPEED.y
|
||||
) if run_up.length_squared() > 0.0001 else Vector3.ZERO
|
||||
_place_body(ship, Transform3D(Basis.from_euler(Vector3(0.0, yaw, 0.0)), ship_position), run_up, Vector3.ZERO)
|
||||
|
||||
|
||||
# Attacking/defending drill states: ball close to a goal, moving toward it.
|
||||
|
||||
+46
@@ -690,6 +690,52 @@ own magnitude) and folded into `HANDLING_REWARD_FLAGS`. The three blocked
|
||||
attempts were deleted and Stage 5 restarts from Stage 4's checkpoint again
|
||||
with both terms active.
|
||||
|
||||
**Neither reward term was the problem — the drill was unsolvable.** The
|
||||
third set of three attempts blocked on `productive_air_touch_fraction=0.0`
|
||||
yet again, but this time the surrounding telemetry told a different story
|
||||
from Rounds 7-8: the ship had measurably left the floor
|
||||
(`airborne_fraction` 0.223 → 0.258, `mean_altitude` 2.59 → 3.25,
|
||||
`vertical_thrust_mean` 0.004 → 0.063, `grounded_upright_fraction` 0.352 →
|
||||
0.182), and watching a game confirmed it now chases and strikes the ball in
|
||||
the air. The reward work had worked; the metric could not see it, because
|
||||
`productive_air_touch_fraction` counts only touches with the *ball* above
|
||||
`AIR_TOUCH_HEIGHT` (5 m), and `_place_air_intercept`'s spawn geometry never
|
||||
produced a reachable one.
|
||||
|
||||
Simulating that spawn distribution against the ship's real flight envelope
|
||||
(`vertical_thrust` 120 / `mass` 5 = 24 m/s², less 9.8 gravity, with
|
||||
`drag_coefficient` 0.98/tick capping climb near 12 m/s) settles it
|
||||
arithmetically. A ball spawned 6-12 m up moving 6-11 m/s is above 5 m for a
|
||||
median of **0.80 s**, while the ship spawned 7-13 m behind it, 3-10 m below
|
||||
it, and **at a dead stop**. An *ideal* interceptor — point mass, instant
|
||||
attitude, no righting torque, isotropic thrust, zero reaction delay — makes
|
||||
that touch in **0.00%** of episodes and reaches the ball at all before it
|
||||
lands in 0.5%. Six attempts and 360M steps were spent optimising against an
|
||||
event the environment could not produce.
|
||||
|
||||
The fix is in `_place_air_intercept` (see its `AIR_INTERCEPT_*` constants):
|
||||
ball higher and slower, ship closer and already carrying planar speed toward
|
||||
it. The dead-stop spawn was the single largest factor — a ship in real play
|
||||
is already moving, and starting from rest spent most of the window just
|
||||
building speed. The same simulation now puts an ideal interceptor at ~98%
|
||||
reach and ~37% above 5 m, so the 0.005 floor has real headroom.
|
||||
`AIR_TOUCH_HEIGHT` deliberately stays at 5.0: lowering the bar to meet a
|
||||
broken drill would make the metric incomparable with earlier generations.
|
||||
|
||||
Unlike every earlier round this does **not** restart from Stage 4's
|
||||
checkpoint. That rule exists because a changed reward function invalidates
|
||||
the learned value function; here the reward function is untouched and only
|
||||
the environment's state distribution moves, so the existing policy — which
|
||||
already learned to fly — is exactly what should be pointed at a now-reachable
|
||||
target. `generation5_state.json` carries a one-shot `resume_override` for
|
||||
this, consumed the first time `resume_checkpoint()` uses it.
|
||||
|
||||
The general lesson, and the one worth carrying into later generations: when
|
||||
a telemetry floor reads *exactly* zero while the behaviour it is meant to
|
||||
measure is visibly happening, check that the environment can produce the
|
||||
event at all before touching the reward function again. Cost of not checking
|
||||
here: three rounds and 360M timesteps.
|
||||
|
||||
Stage 6's `league` opponent mode samples a historical exported policy at each
|
||||
episode reset. Each later stage preserves the preceding shaping and adds one
|
||||
new difficulty.
|
||||
|
||||
+53
-2
@@ -210,6 +210,42 @@ STANDING_ARGS = ["--ent-coef", "0.01", "--entropy-floor"]
|
||||
# ~1.7x a fully-aligned ground one). Also folded into HANDLING_REWARD_FLAGS
|
||||
# so Stage 6 inherits it. Restarts Stage 5 from Stage 4's checkpoint again,
|
||||
# same reasoning as every prior mechanism change here.
|
||||
#
|
||||
# Round 9 (2026-08-21): the reward work in Rounds 7-8 was not the problem, and
|
||||
# in fact worked. Across those three attempts the ship measurably left the
|
||||
# floor -- airborne_fraction 0.223 -> 0.258, mean_altitude 2.59 -> 3.25,
|
||||
# vertical_thrust_mean 0.004 -> 0.063, grounded_upright_fraction 0.352 ->
|
||||
# 0.182 -- and the human watching it confirmed it now chases and strikes the
|
||||
# ball in the air. productive_air_touch_fraction still read 0.0 because the
|
||||
# event it counts was not reachable: it needs a touch with the *ball* above
|
||||
# AIR_TOUCH_HEIGHT (5m), and _place_air_intercept's spawn geometry never
|
||||
# allowed one.
|
||||
#
|
||||
# Simulating the spawn distribution against the ship's real flight envelope
|
||||
# (vertical_thrust 120 / mass 5 = 24 m/s^2, less 9.8 gravity, with
|
||||
# drag_coefficient 0.98/tick capping climb near 12 m/s) settles it
|
||||
# arithmetically. The ball spawned 6-12m up and moving 6-11 m/s is above 5m
|
||||
# for a median of only 0.80s, while the ship spawned 7-13m behind it, 3-10m
|
||||
# below it, and at a dead stop. An *ideal* interceptor -- point mass, instant
|
||||
# attitude, no righting torque, isotropic thrust, zero reaction delay -- makes
|
||||
# that touch in 0.00% of episodes, and reaches the ball at all before it lands
|
||||
# in 0.5%. Six attempts and 360M steps were spent optimising against an event
|
||||
# the environment could not produce; the flat-at-exactly-zero metric was the
|
||||
# environment's signature, not the policy's.
|
||||
#
|
||||
# The fix is in the drill, not the reward (see _place_air_intercept's
|
||||
# constants in training_mode.gd): ball higher and slower, ship closer and
|
||||
# already carrying planar speed toward it. Same simulation now puts an ideal
|
||||
# interceptor at ~98% reach and ~37% above 5m, so the 0.005 floor has real
|
||||
# headroom. AIR_TOUCH_HEIGHT stays 5.0 -- lowering the bar to meet a broken
|
||||
# drill would make the metric incomparable with every earlier generation.
|
||||
#
|
||||
# Unlike Rounds 6-8 this does NOT restart from Stage 4's checkpoint. That rule
|
||||
# exists because a changed reward function invalidates the learned value
|
||||
# function; here the reward function is untouched and only the environment's
|
||||
# state distribution moves, so retry2's policy -- which already learned to
|
||||
# fly, per the telemetry above -- is exactly what should be pointed at a
|
||||
# reachable target. Hence resume_override in generation5_state.json.
|
||||
HANDLING_REWARD_FLAGS = [
|
||||
"--velocity-to-ball-weight", "0.04",
|
||||
"--forward-velocity-to-ball-weight", "0.15",
|
||||
@@ -328,7 +364,20 @@ def previous_attempt_entry(state: dict, stage_index: int, attempt: int) -> dict:
|
||||
raise RuntimeError(f"No previous attempt for stage index {stage_index}, attempt {attempt}")
|
||||
|
||||
|
||||
def resume_checkpoint(state: dict, stage_index: int, attempt: int, foundation: pathlib.Path) -> pathlib.Path:
|
||||
def resume_checkpoint(
|
||||
state: dict, stage_index: int, attempt: int, foundation: pathlib.Path, consume: bool = True
|
||||
) -> pathlib.Path:
|
||||
# One-shot escape hatch for the case where a stage's attempt counter is
|
||||
# reset but its accumulated policy is still worth keeping — i.e. the
|
||||
# environment was fixed rather than the reward function, so the previous
|
||||
# attempts' learning is still valid (see the Round 9 note above). Consumed
|
||||
# on use so it can't silently pin later attempts to a stale checkpoint.
|
||||
override = state.get("resume_override")
|
||||
if override and override.get("stage_index") == stage_index and attempt == 0:
|
||||
if consume: # --dry-run must be able to show the resume path without spending it
|
||||
state.pop("resume_override")
|
||||
save_state(state)
|
||||
return TRAINING_DIR / "checkpoints" / override["experiment"] / "final.zip"
|
||||
if attempt > 0:
|
||||
exp = previous_attempt_entry(state, stage_index, attempt)["experiment"]
|
||||
return TRAINING_DIR / "checkpoints" / exp / "final.zip"
|
||||
@@ -389,7 +438,9 @@ def run_training(state: dict, stage_index: int, attempt: int, args) -> str:
|
||||
stage = STAGES[stage_index]
|
||||
suffix = "" if attempt == 0 else f"-retry{attempt}"
|
||||
experiment = f"{datetime.now().strftime('%Y%m%d-%H%M')}-gen5-s{stage['number']}-{stage['name']}{suffix}"
|
||||
resume = resume_checkpoint(state, stage_index, attempt, pathlib.Path(args.foundation_checkpoint))
|
||||
resume = resume_checkpoint(
|
||||
state, stage_index, attempt, pathlib.Path(args.foundation_checkpoint), consume=not args.dry_run
|
||||
)
|
||||
if not resume.exists():
|
||||
raise FileNotFoundError(f"Resume checkpoint not found: {resume}")
|
||||
cmd = [
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"stage_index": 1,
|
||||
"attempt": 2,
|
||||
"status": "blocked",
|
||||
"attempt": 0,
|
||||
"status": "in_progress",
|
||||
"log": [
|
||||
{
|
||||
"stage_index": 0,
|
||||
@@ -506,5 +506,10 @@
|
||||
],
|
||||
"decision": "fail"
|
||||
}
|
||||
]
|
||||
],
|
||||
"resume_override": {
|
||||
"stage_index": 1,
|
||||
"experiment": "20260821-0056-gen5-s5-intercepts-retry2",
|
||||
"reason": "Stage 5 attempts 1-3 blocked on productive_air_touch_fraction=0.0, but the cause was _place_air_intercept's unreachable spawn geometry, not the policy or the reward function (see the Round 9 note in generation5.py). The drill was fixed; the reward function is unchanged, so this lineage's learned flight behaviour is kept rather than restarted from Stage 4."
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user