chore(training): add air_approach_weight and restart stage-5 intercepts

Stage 5 blocked all three attempts on productive_air_touch_fraction
stuck exactly at 0.0 across a continuous 180M-step lineage, while
goal_rate/upright_fraction/forward_motion_fraction kept improving on
the same budget. forward_velocity_to_ball_weight (the term that solved
Stage 4's ground pursuit) is hard-gated below GROUND_HANDLING_HEIGHT
and does nothing in the air, so Stage 5's air_intercept_chance had no
matching aerial incentive to learn from. air_approach_weight adds the
airborne mirror (nose-first 3D closing speed, no uprightness
multiplier) and folds into HANDLING_REWARD_FLAGS so Stage 6 inherits
it too. Deleted the three blocked attempts and reset state to resume
Stage 5 from the Stage-4 checkpoint with the new term.
This commit is contained in:
Josh Creek
2026-08-18 16:03:56 +01:00
parent 0e685edcf8
commit 88591e031f
15 changed files with 93 additions and 254 deletions
+31
View File
@@ -151,9 +151,40 @@ STANDING_ARGS = ["--ent-coef", "0.01", "--entropy-floor"]
# floor, and air-drill-chance goes to 0. The reward terms already built
# are left exactly as they were — they should finally pull in a direction
# the ship can go.
#
# Round 7 (2026-08-18): Stage 4 closed by human override (see TRAINING.md).
# Stage 5 (intercepts) then blocked all three attempts on the same single
# floor every time — rollout/productive_air_touch_fraction stayed exactly
# 0.0 across a continuous 180M-step lineage (each retry resumes the
# previous attempt's checkpoint, not a fresh run), while air_touch_fraction
# sat at noise level (0.00008 -> 0.00006 -> 0.00006) and goal_rate/
# upright_fraction/forward_motion_fraction all kept improving on the same
# budget. A dead-flat metric across that much continued training, next to
# metrics that keep moving, is the missing-mechanism signature from Round 6
# again, not a slow-learning one: forward_velocity_to_ball_weight -- the
# term that actually solved ground handling -- is hard-gated to
# ship.global_position.y < GROUND_HANDLING_HEIGHT and does nothing in the
# air, so air_intercept_chance (added for Stage 5) was asking for aerial
# pursuit with only the generic, orientation-agnostic velocity_to_ball_
# weight (0.04) to learn it from -- the same class of gap as Stage 4's
# missing ground-tilt/non-forward pressure before those were added.
#
# air_approach_weight (ship_ai_controller.gd) is the airborne mirror:
# nose-first 3D closing speed on the ball, active above
# GROUND_HANDLING_HEIGHT instead of below it (mutually exclusive with
# forward_velocity_to_ball_weight by altitude), with no uprightness
# multiplier since a real aerial requires pitching away from level. Set to
# 0.15 to match forward_velocity_to_ball_weight's proven-effective
# magnitude; added to HANDLING_REWARD_FLAGS (not just Stage 5's flags) so
# it also carries into Stage 6, which reuses these flags and its own
# air_intercept_chance. Stage 5 restarts from Stage 4's checkpoint rather
# than continuing retry2's, same reasoning as every previous mechanism
# change in this file: don't resume a policy shaped by an absent term into
# one where it now exists.
HANDLING_REWARD_FLAGS = [
"--velocity-to-ball-weight", "0.04",
"--forward-velocity-to-ball-weight", "0.15",
"--air-approach-weight", "0.15",
"--ball-distance-penalty", "0.01",
"--ball-touch-reward", "0.7",
"--ball-velocity-to-goal-weight", "0.06",
+2 -250
View File
@@ -1,7 +1,7 @@
{
"stage_index": 1,
"attempt": 2,
"status": "blocked",
"attempt": 0,
"status": "in_progress",
"log": [
{
"stage_index": 0,
@@ -258,254 +258,6 @@
],
"reason": "Human override. Only miss was the training goal-rate floor, at 0.7731 vs 0.80; every evaluation gate passed (65-22-13 vs promoted/easy.json, 87% non-draw, 12.6% physical-side imbalance) and the goal rate improved monotonically across all three attempts (0.537 -> 0.683 -> 0.773). Also promoted to Game/bots/promoted/medium.json on the same evidence. Stage 5 resumes from this checkpoint."
}
},
{
"stage_index": 1,
"stage_number": 5,
"stage_name": "intercepts",
"experiment": "20260817-0750-gen5-s5-intercepts",
"attempt": 0,
"telemetry_tail": {
"rollout/air_touch_fraction": 7.999999821186065e-05,
"rollout/airborne_fraction": 0.24998261865973473,
"rollout/ep_len_mean": 106.38228009033203,
"rollout/ep_rew_mean": 10.283133600234985,
"rollout/forward_motion_fraction": 0.3337678092420101,
"rollout/goal_rate": 0.7703999981880189,
"rollout/grounded_upright_fraction": 0.24178000067174435,
"rollout/mean_altitude": 3.19321390914917,
"rollout/productive_air_touch_fraction": 0.0,
"rollout/upright_fraction": 0.7911632860898972,
"rollout/vertical_thrust_mean": 0.03387699818262831
},
"telemetry_failures": [
"rollout/productive_air_touch_fraction=0.0000 < 0.0050"
],
"evaluation_goal_failures": [],
"side_balance_failures": [
"20260816-2126-gen5-s4-handling-retry2.json: physical_side_imbalance=0.290 > 0.200"
],
"eval": {
"timestamp": "2026-08-17T16:01:22+00:00",
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260817-0750-gen5-s5-intercepts.json",
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260816-2126-gen5-s4-handling-retry2.json",
"seed": 1,
"episodes": 100,
"wins_a": 51,
"wins_b": 32,
"draws": 17,
"side_results": {
"a_team_0": {
"wins_a": 18,
"wins_b": 23,
"draws": 9
},
"a_team_1": {
"wins_a": 33,
"wins_b": 9,
"draws": 8
}
},
"physical_team_wins": {
"team_0": 27,
"team_1": 56
},
"win_rate_a": 0.51
},
"evals": [
{
"timestamp": "2026-08-17T16:01:22+00:00",
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260817-0750-gen5-s5-intercepts.json",
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260816-2126-gen5-s4-handling-retry2.json",
"seed": 1,
"episodes": 100,
"wins_a": 51,
"wins_b": 32,
"draws": 17,
"side_results": {
"a_team_0": {
"wins_a": 18,
"wins_b": 23,
"draws": 9
},
"a_team_1": {
"wins_a": 33,
"wins_b": 9,
"draws": 8
}
},
"physical_team_wins": {
"team_0": 27,
"team_1": 56
},
"win_rate_a": 0.51
}
],
"decision": "fail"
},
{
"stage_index": 1,
"stage_number": 5,
"stage_name": "intercepts",
"experiment": "20260817-1701-gen5-s5-intercepts-retry1",
"attempt": 1,
"telemetry_tail": {
"rollout/air_touch_fraction": 5.9999998658895494e-05,
"rollout/airborne_fraction": 0.25157142809033395,
"rollout/ep_len_mean": 95.91548010253906,
"rollout/ep_rew_mean": 10.333840370178223,
"rollout/forward_motion_fraction": 0.3461968092918396,
"rollout/goal_rate": 0.8227999938726425,
"rollout/grounded_upright_fraction": 0.20529999974370003,
"rollout/mean_altitude": 3.198590657234192,
"rollout/productive_air_touch_fraction": 0.0,
"rollout/upright_fraction": 0.7962958091497421,
"rollout/vertical_thrust_mean": 0.053312998113571666
},
"telemetry_failures": [
"rollout/productive_air_touch_fraction=0.0000 < 0.0050"
],
"evaluation_goal_failures": [],
"side_balance_failures": [],
"eval": {
"timestamp": "2026-08-18T01:10:15+00:00",
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260817-1701-gen5-s5-intercepts-retry1.json",
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260816-2126-gen5-s4-handling-retry2.json",
"seed": 1,
"episodes": 100,
"wins_a": 58,
"wins_b": 30,
"draws": 12,
"side_results": {
"a_team_0": {
"wins_a": 26,
"wins_b": 20,
"draws": 4
},
"a_team_1": {
"wins_a": 32,
"wins_b": 10,
"draws": 8
}
},
"physical_team_wins": {
"team_0": 36,
"team_1": 52
},
"win_rate_a": 0.58
},
"evals": [
{
"timestamp": "2026-08-18T01:10:15+00:00",
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260817-1701-gen5-s5-intercepts-retry1.json",
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260816-2126-gen5-s4-handling-retry2.json",
"seed": 1,
"episodes": 100,
"wins_a": 58,
"wins_b": 30,
"draws": 12,
"side_results": {
"a_team_0": {
"wins_a": 26,
"wins_b": 20,
"draws": 4
},
"a_team_1": {
"wins_a": 32,
"wins_b": 10,
"draws": 8
}
},
"physical_team_wins": {
"team_0": 36,
"team_1": 52
},
"win_rate_a": 0.58
}
],
"decision": "fail"
},
{
"stage_index": 1,
"stage_number": 5,
"stage_name": "intercepts",
"experiment": "20260818-0210-gen5-s5-intercepts-retry2",
"attempt": 2,
"telemetry_tail": {
"rollout/air_touch_fraction": 5.9999998658895494e-05,
"rollout/airborne_fraction": 0.24753280937671662,
"rollout/ep_len_mean": 103.67796003723144,
"rollout/ep_rew_mean": 11.149126368522644,
"rollout/forward_motion_fraction": 0.363580904841423,
"rollout/goal_rate": 0.7755999964475632,
"rollout/grounded_upright_fraction": 0.23967999944090843,
"rollout/mean_altitude": 3.170829694747925,
"rollout/productive_air_touch_fraction": 0.0,
"rollout/upright_fraction": 0.800830381155014,
"rollout/vertical_thrust_mean": 0.024330998776718044
},
"telemetry_failures": [
"rollout/productive_air_touch_fraction=0.0000 < 0.0050"
],
"evaluation_goal_failures": [],
"side_balance_failures": [],
"eval": {
"timestamp": "2026-08-18T10:17:39+00:00",
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260818-0210-gen5-s5-intercepts-retry2.json",
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260816-2126-gen5-s4-handling-retry2.json",
"seed": 1,
"episodes": 100,
"wins_a": 53,
"wins_b": 33,
"draws": 14,
"side_results": {
"a_team_0": {
"wins_a": 23,
"wins_b": 20,
"draws": 7
},
"a_team_1": {
"wins_a": 30,
"wins_b": 13,
"draws": 7
}
},
"physical_team_wins": {
"team_0": 36,
"team_1": 50
},
"win_rate_a": 0.53
},
"evals": [
{
"timestamp": "2026-08-18T10:17:39+00:00",
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260818-0210-gen5-s5-intercepts-retry2.json",
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260816-2126-gen5-s4-handling-retry2.json",
"seed": 1,
"episodes": 100,
"wins_a": 53,
"wins_b": 33,
"draws": 14,
"side_results": {
"a_team_0": {
"wins_a": 23,
"wins_b": 20,
"draws": 7
},
"a_team_1": {
"wins_a": 30,
"wins_b": 13,
"draws": 7
}
},
"physical_team_wins": {
"team_0": 36,
"team_1": 50
},
"win_rate_a": 0.53
}
],
"decision": "fail"
}
]
}
+6
View File
@@ -355,6 +355,11 @@ def parse_args():
"--forward-velocity-to-ball-weight", type=float, default=None,
help="Low-altitude dense reward for nose-led planar approach toward the ball",
)
curriculum.add_argument(
"--air-approach-weight", type=float, default=None,
help="Aerial mirror of forward-velocity-to-ball-weight: high-altitude dense reward for "
"nose-led 3D closing speed toward the ball",
)
curriculum.add_argument(
"--ball-distance-penalty", type=float, default=None,
help="Overrides ShipAIController.ball_distance_penalty (dense per-tick cost scaled by distance to the ball)",
@@ -419,6 +424,7 @@ def _curriculum_kwargs(args) -> dict:
"ai_grounded_upright_reward": args.grounded_upright_reward,
"ai_velocity_to_ball_weight": args.velocity_to_ball_weight,
"ai_forward_velocity_to_ball_weight": args.forward_velocity_to_ball_weight,
"ai_air_approach_weight": args.air_approach_weight,
"ai_ball_distance_penalty": args.ball_distance_penalty,
"ai_ball_touch_reward": args.ball_touch_reward,
"ai_airborne_penalty": args.airborne_penalty,