mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-13 04:32:06 +00:00
fix(training): make the stage-5 air-intercept drill physically solvable
productive_air_touch_fraction sat at exactly 0.0 across nine Stage-5 attempts and 540M timesteps. Two rounds of reward shaping were aimed at it (air_approach_weight, then air_touch_bonus_weight); both worked -- airborne_fraction 0.223->0.258, mean_altitude 2.59->3.25, vertical_thrust_mean 0.004->0.063 -- and the ship now visibly plays the ball in the air. The metric could not see it because it counts only touches with the ball above AIR_TOUCH_HEIGHT (5m), and _place_air_intercept never produced a reachable one. Simulating the spawn distribution against the ship's flight envelope (vertical_thrust 120 / mass 5 = 24 m/s^2 less gravity, drag capping climb near 12 m/s): a ball spawned 6-12m up at 6-11 m/s is above 5m for a median of 0.80s, while the ship spawned 7-13m behind, 3-10m below, and at a dead stop. An ideal interceptor -- point mass, instant attitude, no righting torque, zero reaction delay -- makes that touch in 0.00% of episodes and reaches the ball at all in 0.5%. Retune the drill instead of the reward: ball higher (8-14m) and slower (4-8 m/s), ship closer (4-9m behind), narrower lateral spread, and a 6-14 m/s planar run-up rather than a standing start -- the dead stop was the largest single factor. Ideal interceptor now reaches the ball in ~98% of episodes and above 5m in ~37%, so the 0.005 floor has headroom. AIR_TOUCH_HEIGHT stays 5.0 so the metric remains comparable with earlier generations. Resume from retry2 rather than restarting from Stage 4: that rule guards against a changed reward function invalidating the value function, and the reward function is untouched here -- only the state distribution moved, so the policy that already learned to fly is what should be pointed at a reachable target. Adds a one-shot resume_override to generation5_state.json, consumed on first use.
This commit is contained in:
+53
-2
@@ -210,6 +210,42 @@ STANDING_ARGS = ["--ent-coef", "0.01", "--entropy-floor"]
|
||||
# ~1.7x a fully-aligned ground one). Also folded into HANDLING_REWARD_FLAGS
|
||||
# so Stage 6 inherits it. Restarts Stage 5 from Stage 4's checkpoint again,
|
||||
# same reasoning as every prior mechanism change here.
|
||||
#
|
||||
# Round 9 (2026-08-21): the reward work in Rounds 7-8 was not the problem, and
|
||||
# in fact worked. Across those three attempts the ship measurably left the
|
||||
# floor -- airborne_fraction 0.223 -> 0.258, mean_altitude 2.59 -> 3.25,
|
||||
# vertical_thrust_mean 0.004 -> 0.063, grounded_upright_fraction 0.352 ->
|
||||
# 0.182 -- and the human watching it confirmed it now chases and strikes the
|
||||
# ball in the air. productive_air_touch_fraction still read 0.0 because the
|
||||
# event it counts was not reachable: it needs a touch with the *ball* above
|
||||
# AIR_TOUCH_HEIGHT (5m), and _place_air_intercept's spawn geometry never
|
||||
# allowed one.
|
||||
#
|
||||
# Simulating the spawn distribution against the ship's real flight envelope
|
||||
# (vertical_thrust 120 / mass 5 = 24 m/s^2, less 9.8 gravity, with
|
||||
# drag_coefficient 0.98/tick capping climb near 12 m/s) settles it
|
||||
# arithmetically. The ball spawned 6-12m up and moving 6-11 m/s is above 5m
|
||||
# for a median of only 0.80s, while the ship spawned 7-13m behind it, 3-10m
|
||||
# below it, and at a dead stop. An *ideal* interceptor -- point mass, instant
|
||||
# attitude, no righting torque, isotropic thrust, zero reaction delay -- makes
|
||||
# that touch in 0.00% of episodes, and reaches the ball at all before it lands
|
||||
# in 0.5%. Six attempts and 360M steps were spent optimising against an event
|
||||
# the environment could not produce; the flat-at-exactly-zero metric was the
|
||||
# environment's signature, not the policy's.
|
||||
#
|
||||
# The fix is in the drill, not the reward (see _place_air_intercept's
|
||||
# constants in training_mode.gd): ball higher and slower, ship closer and
|
||||
# already carrying planar speed toward it. Same simulation now puts an ideal
|
||||
# interceptor at ~98% reach and ~37% above 5m, so the 0.005 floor has real
|
||||
# headroom. AIR_TOUCH_HEIGHT stays 5.0 -- lowering the bar to meet a broken
|
||||
# drill would make the metric incomparable with every earlier generation.
|
||||
#
|
||||
# Unlike Rounds 6-8 this does NOT restart from Stage 4's checkpoint. That rule
|
||||
# exists because a changed reward function invalidates the learned value
|
||||
# function; here the reward function is untouched and only the environment's
|
||||
# state distribution moves, so retry2's policy -- which already learned to
|
||||
# fly, per the telemetry above -- is exactly what should be pointed at a
|
||||
# reachable target. Hence resume_override in generation5_state.json.
|
||||
HANDLING_REWARD_FLAGS = [
|
||||
"--velocity-to-ball-weight", "0.04",
|
||||
"--forward-velocity-to-ball-weight", "0.15",
|
||||
@@ -328,7 +364,20 @@ def previous_attempt_entry(state: dict, stage_index: int, attempt: int) -> dict:
|
||||
raise RuntimeError(f"No previous attempt for stage index {stage_index}, attempt {attempt}")
|
||||
|
||||
|
||||
def resume_checkpoint(state: dict, stage_index: int, attempt: int, foundation: pathlib.Path) -> pathlib.Path:
|
||||
def resume_checkpoint(
|
||||
state: dict, stage_index: int, attempt: int, foundation: pathlib.Path, consume: bool = True
|
||||
) -> pathlib.Path:
|
||||
# One-shot escape hatch for the case where a stage's attempt counter is
|
||||
# reset but its accumulated policy is still worth keeping — i.e. the
|
||||
# environment was fixed rather than the reward function, so the previous
|
||||
# attempts' learning is still valid (see the Round 9 note above). Consumed
|
||||
# on use so it can't silently pin later attempts to a stale checkpoint.
|
||||
override = state.get("resume_override")
|
||||
if override and override.get("stage_index") == stage_index and attempt == 0:
|
||||
if consume: # --dry-run must be able to show the resume path without spending it
|
||||
state.pop("resume_override")
|
||||
save_state(state)
|
||||
return TRAINING_DIR / "checkpoints" / override["experiment"] / "final.zip"
|
||||
if attempt > 0:
|
||||
exp = previous_attempt_entry(state, stage_index, attempt)["experiment"]
|
||||
return TRAINING_DIR / "checkpoints" / exp / "final.zip"
|
||||
@@ -389,7 +438,9 @@ def run_training(state: dict, stage_index: int, attempt: int, args) -> str:
|
||||
stage = STAGES[stage_index]
|
||||
suffix = "" if attempt == 0 else f"-retry{attempt}"
|
||||
experiment = f"{datetime.now().strftime('%Y%m%d-%H%M')}-gen5-s{stage['number']}-{stage['name']}{suffix}"
|
||||
resume = resume_checkpoint(state, stage_index, attempt, pathlib.Path(args.foundation_checkpoint))
|
||||
resume = resume_checkpoint(
|
||||
state, stage_index, attempt, pathlib.Path(args.foundation_checkpoint), consume=not args.dry_run
|
||||
)
|
||||
if not resume.exists():
|
||||
raise FileNotFoundError(f"Resume checkpoint not found: {resume}")
|
||||
cmd = [
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"stage_index": 1,
|
||||
"attempt": 2,
|
||||
"status": "blocked",
|
||||
"attempt": 0,
|
||||
"status": "in_progress",
|
||||
"log": [
|
||||
{
|
||||
"stage_index": 0,
|
||||
@@ -506,5 +506,10 @@
|
||||
],
|
||||
"decision": "fail"
|
||||
}
|
||||
]
|
||||
],
|
||||
"resume_override": {
|
||||
"stage_index": 1,
|
||||
"experiment": "20260821-0056-gen5-s5-intercepts-retry2",
|
||||
"reason": "Stage 5 attempts 1-3 blocked on productive_air_touch_fraction=0.0, but the cause was _place_air_intercept's unreachable spawn geometry, not the policy or the reward function (see the Round 9 note in generation5.py). The drill was fixed; the reward function is unchanged, so this lineage's learned flight behaviour is kept rather than restarted from Stage 4."
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user