mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-10 16:04:04 +00:00
chore(training): add air_approach_weight and restart stage-5 intercepts
Stage 5 blocked all three attempts on productive_air_touch_fraction stuck exactly at 0.0 across a continuous 180M-step lineage, while goal_rate/upright_fraction/forward_motion_fraction kept improving on the same budget. forward_velocity_to_ball_weight (the term that solved Stage 4's ground pursuit) is hard-gated below GROUND_HANDLING_HEIGHT and does nothing in the air, so Stage 5's air_intercept_chance had no matching aerial incentive to learn from. air_approach_weight adds the airborne mirror (nose-first 3D closing speed, no uprightness multiplier) and folds into HANDLING_REWARD_FLAGS so Stage 6 inherits it too. Deleted the three blocked attempts and reset state to resume Stage 5 from the Stage-4 checkpoint with the new term.
This commit is contained in:
Josh Creek
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -46,6 +46,20 @@ extends AIController3D
|
||||
# frozen checkpoints keep their original objective; generation 5 handling
|
||||
# turns it on while reducing the orientation-agnostic term.
|
||||
@export var forward_velocity_to_ball_weight := 0.0
|
||||
# Aerial mirror of forward_velocity_to_ball_weight: nose-first closing speed
|
||||
# on the ball, active above GROUND_HANDLING_HEIGHT instead of below it (the
|
||||
# two are mutually exclusive by altitude, never both active on the same
|
||||
# tick). Generation 5's intercepts stage added air_intercept_chance without
|
||||
# an airborne equivalent of the term that actually solved ground handling;
|
||||
# above 3m the only remaining approach incentive was the generic, orientation
|
||||
# -agnostic velocity_to_ball_weight (0.02-0.04), which three consecutive
|
||||
# 60M-step attempts (180M cumulative, resuming each time) showed produces
|
||||
# zero learnable gradient toward touching an aerial ball at all —
|
||||
# productive_air_touch_fraction stayed exactly 0.0 the whole time while every
|
||||
# other metric kept improving on the same budget. Uses the full 3D nose
|
||||
# vector rather than the planar-only one, since a real aerial requires
|
||||
# pitching away from level.
|
||||
@export var air_approach_weight := 0.0
|
||||
@export var ball_velocity_to_goal_weight := 0.004
|
||||
# Per-tick penalty scaled by distance to the ball (full value at the arena's
|
||||
# far diagonal, 0 on top of the ball). Run04 lesson: with idling worth a flat
|
||||
@@ -357,6 +371,21 @@ func _physics_process(delta):
|
||||
reward += forward_velocity_to_ball_weight * forward_speed * facing_ball \
|
||||
* approach_uprightness * handling_ground_factor
|
||||
|
||||
# Aerial shaping: nose-first 3D closing speed on the ball (see
|
||||
# air_approach_weight). Mirrors the ground block above but with the full
|
||||
# nose vector instead of the planar one, and no uprightness multiplier —
|
||||
# a genuine aerial approach requires pitching away from level, so paying
|
||||
# only while upright would oppose the exact behaviour this rewards.
|
||||
if air_approach_weight > 0.0 and ship.global_position.y >= GROUND_HANDLING_HEIGHT \
|
||||
and to_ball.length_squared() > 0.0001:
|
||||
var nose_forward := -ship.global_transform.basis.z
|
||||
if nose_forward.length_squared() > 0.0001:
|
||||
nose_forward = nose_forward.normalized()
|
||||
var to_ball_dir := to_ball.normalized()
|
||||
var air_facing_ball: float = maxf(nose_forward.dot(to_ball_dir), 0.0)
|
||||
var air_closing_speed: float = maxf(ship.linear_velocity.dot(to_ball_dir), 0.0) / ship.max_speed
|
||||
reward += air_approach_weight * air_closing_speed * air_facing_ball
|
||||
|
||||
# Dense penalty: distance to the ball, so idling far away bleeds reward
|
||||
# instead of scoring a safe zero (see ball_distance_penalty).
|
||||
if ball_distance_penalty > 0.0:
|
||||
|
||||
@@ -265,7 +265,7 @@ const TRAINING_MODE_OVERRIDES := [
|
||||
const SHIP_AI_OVERRIDES := [
|
||||
"ball_touch_reward", "ball_touch_cooldown_ticks", "ball_touch_direction_floor",
|
||||
"velocity_to_ball_weight", "ball_velocity_to_goal_weight", "ball_distance_penalty",
|
||||
"forward_velocity_to_ball_weight", "wall_contact_penalty", "tilt_penalty",
|
||||
"forward_velocity_to_ball_weight", "air_approach_weight", "wall_contact_penalty", "tilt_penalty",
|
||||
"ground_tilt_penalty", "non_forward_penalty", "grounded_upright_reward",
|
||||
"speed_reward_weight", "time_penalty", "airborne_penalty",
|
||||
]
|
||||
@@ -323,6 +323,7 @@ func _ai_default(name: String) -> Variant:
|
||||
"ball_touch_direction_floor": return 0.3
|
||||
"velocity_to_ball_weight": return 0.02
|
||||
"forward_velocity_to_ball_weight": return 0.0
|
||||
"air_approach_weight": return 0.0
|
||||
"ball_velocity_to_goal_weight": return 0.004
|
||||
"ball_distance_penalty": return 0.002
|
||||
"wall_contact_penalty": return 0.0025
|
||||
|
||||
+23
@@ -639,6 +639,29 @@ marking anything as passing, so the run dies immediately with `RuntimeError:
|
||||
No passing generation-5 stage index 0`. Note that retry2 already clears Stage
|
||||
5's own 0.75 goal-rate floor; the 0.80 Stage-4 figure was always the stricter
|
||||
of the two.
|
||||
|
||||
**Stage 5 (`intercepts`) blocked after its own three attempts on 2026-08-18**,
|
||||
all on the same single floor: `rollout/productive_air_touch_fraction` stayed
|
||||
exactly 0.0 across a continuous 180M-step lineage (retries resume the
|
||||
previous attempt's checkpoint, so this is one training run, not three), while
|
||||
`air_touch_fraction` sat at noise level (0.00008 → 0.00006 → 0.00006) and
|
||||
`goal_rate`/`upright_fraction`/`forward_motion_fraction` all kept improving on
|
||||
the same budget — a dead-flat metric next to ones that keep moving, the same
|
||||
missing-mechanism signature as Stage 4's original plateau, not a slow-learning
|
||||
one. The cause: `forward_velocity_to_ball_weight`, the term that actually
|
||||
taught ground pursuit, is hard-gated below `GROUND_HANDLING_HEIGHT` and does
|
||||
nothing in the air, so Stage 5's `air_intercept_chance` was asking for aerial
|
||||
pursuit with only the generic, orientation-agnostic `velocity_to_ball_weight`
|
||||
(0.04) to learn it from. `air_approach_weight` (`ship_ai_controller.gd`) adds
|
||||
the airborne mirror — nose-first 3D closing speed on the ball, active above
|
||||
`GROUND_HANDLING_HEIGHT`, no uprightness multiplier since a real aerial
|
||||
requires pitching away from level — set to 0.15 to match
|
||||
`forward_velocity_to_ball_weight`'s proven magnitude, and folded into
|
||||
`HANDLING_REWARD_FLAGS` so Stage 6 inherits it too. The three blocked attempts
|
||||
were deleted and Stage 5 restarts from Stage 4's checkpoint with the new term,
|
||||
same reasoning as every previous mechanism change: don't resume a policy
|
||||
shaped by an absent term into one where it now exists.
|
||||
|
||||
Stage 6's `league` opponent mode samples a historical exported policy at each
|
||||
episode reset. Each later stage preserves the preceding shaping and adds one
|
||||
new difficulty.
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -151,9 +151,40 @@ STANDING_ARGS = ["--ent-coef", "0.01", "--entropy-floor"]
|
||||
# floor, and air-drill-chance goes to 0. The reward terms already built
|
||||
# are left exactly as they were — they should finally pull in a direction
|
||||
# the ship can go.
|
||||
#
|
||||
# Round 7 (2026-08-18): Stage 4 closed by human override (see TRAINING.md).
|
||||
# Stage 5 (intercepts) then blocked all three attempts on the same single
|
||||
# floor every time — rollout/productive_air_touch_fraction stayed exactly
|
||||
# 0.0 across a continuous 180M-step lineage (each retry resumes the
|
||||
# previous attempt's checkpoint, not a fresh run), while air_touch_fraction
|
||||
# sat at noise level (0.00008 -> 0.00006 -> 0.00006) and goal_rate/
|
||||
# upright_fraction/forward_motion_fraction all kept improving on the same
|
||||
# budget. A dead-flat metric across that much continued training, next to
|
||||
# metrics that keep moving, is the missing-mechanism signature from Round 6
|
||||
# again, not a slow-learning one: forward_velocity_to_ball_weight -- the
|
||||
# term that actually solved ground handling -- is hard-gated to
|
||||
# ship.global_position.y < GROUND_HANDLING_HEIGHT and does nothing in the
|
||||
# air, so air_intercept_chance (added for Stage 5) was asking for aerial
|
||||
# pursuit with only the generic, orientation-agnostic velocity_to_ball_
|
||||
# weight (0.04) to learn it from -- the same class of gap as Stage 4's
|
||||
# missing ground-tilt/non-forward pressure before those were added.
|
||||
#
|
||||
# air_approach_weight (ship_ai_controller.gd) is the airborne mirror:
|
||||
# nose-first 3D closing speed on the ball, active above
|
||||
# GROUND_HANDLING_HEIGHT instead of below it (mutually exclusive with
|
||||
# forward_velocity_to_ball_weight by altitude), with no uprightness
|
||||
# multiplier since a real aerial requires pitching away from level. Set to
|
||||
# 0.15 to match forward_velocity_to_ball_weight's proven-effective
|
||||
# magnitude; added to HANDLING_REWARD_FLAGS (not just Stage 5's flags) so
|
||||
# it also carries into Stage 6, which reuses these flags and its own
|
||||
# air_intercept_chance. Stage 5 restarts from Stage 4's checkpoint rather
|
||||
# than continuing retry2's, same reasoning as every previous mechanism
|
||||
# change in this file: don't resume a policy shaped by an absent term into
|
||||
# one where it now exists.
|
||||
HANDLING_REWARD_FLAGS = [
|
||||
"--velocity-to-ball-weight", "0.04",
|
||||
"--forward-velocity-to-ball-weight", "0.15",
|
||||
"--air-approach-weight", "0.15",
|
||||
"--ball-distance-penalty", "0.01",
|
||||
"--ball-touch-reward", "0.7",
|
||||
"--ball-velocity-to-goal-weight", "0.06",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"stage_index": 1,
|
||||
"attempt": 2,
|
||||
"status": "blocked",
|
||||
"attempt": 0,
|
||||
"status": "in_progress",
|
||||
"log": [
|
||||
{
|
||||
"stage_index": 0,
|
||||
@@ -258,254 +258,6 @@
|
||||
],
|
||||
"reason": "Human override. Only miss was the training goal-rate floor, at 0.7731 vs 0.80; every evaluation gate passed (65-22-13 vs promoted/easy.json, 87% non-draw, 12.6% physical-side imbalance) and the goal rate improved monotonically across all three attempts (0.537 -> 0.683 -> 0.773). Also promoted to Game/bots/promoted/medium.json on the same evidence. Stage 5 resumes from this checkpoint."
|
||||
}
|
||||
},
|
||||
{
|
||||
"stage_index": 1,
|
||||
"stage_number": 5,
|
||||
"stage_name": "intercepts",
|
||||
"experiment": "20260817-0750-gen5-s5-intercepts",
|
||||
"attempt": 0,
|
||||
"telemetry_tail": {
|
||||
"rollout/air_touch_fraction": 7.999999821186065e-05,
|
||||
"rollout/airborne_fraction": 0.24998261865973473,
|
||||
"rollout/ep_len_mean": 106.38228009033203,
|
||||
"rollout/ep_rew_mean": 10.283133600234985,
|
||||
"rollout/forward_motion_fraction": 0.3337678092420101,
|
||||
"rollout/goal_rate": 0.7703999981880189,
|
||||
"rollout/grounded_upright_fraction": 0.24178000067174435,
|
||||
"rollout/mean_altitude": 3.19321390914917,
|
||||
"rollout/productive_air_touch_fraction": 0.0,
|
||||
"rollout/upright_fraction": 0.7911632860898972,
|
||||
"rollout/vertical_thrust_mean": 0.03387699818262831
|
||||
},
|
||||
"telemetry_failures": [
|
||||
"rollout/productive_air_touch_fraction=0.0000 < 0.0050"
|
||||
],
|
||||
"evaluation_goal_failures": [],
|
||||
"side_balance_failures": [
|
||||
"20260816-2126-gen5-s4-handling-retry2.json: physical_side_imbalance=0.290 > 0.200"
|
||||
],
|
||||
"eval": {
|
||||
"timestamp": "2026-08-17T16:01:22+00:00",
|
||||
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260817-0750-gen5-s5-intercepts.json",
|
||||
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260816-2126-gen5-s4-handling-retry2.json",
|
||||
"seed": 1,
|
||||
"episodes": 100,
|
||||
"wins_a": 51,
|
||||
"wins_b": 32,
|
||||
"draws": 17,
|
||||
"side_results": {
|
||||
"a_team_0": {
|
||||
"wins_a": 18,
|
||||
"wins_b": 23,
|
||||
"draws": 9
|
||||
},
|
||||
"a_team_1": {
|
||||
"wins_a": 33,
|
||||
"wins_b": 9,
|
||||
"draws": 8
|
||||
}
|
||||
},
|
||||
"physical_team_wins": {
|
||||
"team_0": 27,
|
||||
"team_1": 56
|
||||
},
|
||||
"win_rate_a": 0.51
|
||||
},
|
||||
"evals": [
|
||||
{
|
||||
"timestamp": "2026-08-17T16:01:22+00:00",
|
||||
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260817-0750-gen5-s5-intercepts.json",
|
||||
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260816-2126-gen5-s4-handling-retry2.json",
|
||||
"seed": 1,
|
||||
"episodes": 100,
|
||||
"wins_a": 51,
|
||||
"wins_b": 32,
|
||||
"draws": 17,
|
||||
"side_results": {
|
||||
"a_team_0": {
|
||||
"wins_a": 18,
|
||||
"wins_b": 23,
|
||||
"draws": 9
|
||||
},
|
||||
"a_team_1": {
|
||||
"wins_a": 33,
|
||||
"wins_b": 9,
|
||||
"draws": 8
|
||||
}
|
||||
},
|
||||
"physical_team_wins": {
|
||||
"team_0": 27,
|
||||
"team_1": 56
|
||||
},
|
||||
"win_rate_a": 0.51
|
||||
}
|
||||
],
|
||||
"decision": "fail"
|
||||
},
|
||||
{
|
||||
"stage_index": 1,
|
||||
"stage_number": 5,
|
||||
"stage_name": "intercepts",
|
||||
"experiment": "20260817-1701-gen5-s5-intercepts-retry1",
|
||||
"attempt": 1,
|
||||
"telemetry_tail": {
|
||||
"rollout/air_touch_fraction": 5.9999998658895494e-05,
|
||||
"rollout/airborne_fraction": 0.25157142809033395,
|
||||
"rollout/ep_len_mean": 95.91548010253906,
|
||||
"rollout/ep_rew_mean": 10.333840370178223,
|
||||
"rollout/forward_motion_fraction": 0.3461968092918396,
|
||||
"rollout/goal_rate": 0.8227999938726425,
|
||||
"rollout/grounded_upright_fraction": 0.20529999974370003,
|
||||
"rollout/mean_altitude": 3.198590657234192,
|
||||
"rollout/productive_air_touch_fraction": 0.0,
|
||||
"rollout/upright_fraction": 0.7962958091497421,
|
||||
"rollout/vertical_thrust_mean": 0.053312998113571666
|
||||
},
|
||||
"telemetry_failures": [
|
||||
"rollout/productive_air_touch_fraction=0.0000 < 0.0050"
|
||||
],
|
||||
"evaluation_goal_failures": [],
|
||||
"side_balance_failures": [],
|
||||
"eval": {
|
||||
"timestamp": "2026-08-18T01:10:15+00:00",
|
||||
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260817-1701-gen5-s5-intercepts-retry1.json",
|
||||
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260816-2126-gen5-s4-handling-retry2.json",
|
||||
"seed": 1,
|
||||
"episodes": 100,
|
||||
"wins_a": 58,
|
||||
"wins_b": 30,
|
||||
"draws": 12,
|
||||
"side_results": {
|
||||
"a_team_0": {
|
||||
"wins_a": 26,
|
||||
"wins_b": 20,
|
||||
"draws": 4
|
||||
},
|
||||
"a_team_1": {
|
||||
"wins_a": 32,
|
||||
"wins_b": 10,
|
||||
"draws": 8
|
||||
}
|
||||
},
|
||||
"physical_team_wins": {
|
||||
"team_0": 36,
|
||||
"team_1": 52
|
||||
},
|
||||
"win_rate_a": 0.58
|
||||
},
|
||||
"evals": [
|
||||
{
|
||||
"timestamp": "2026-08-18T01:10:15+00:00",
|
||||
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260817-1701-gen5-s5-intercepts-retry1.json",
|
||||
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260816-2126-gen5-s4-handling-retry2.json",
|
||||
"seed": 1,
|
||||
"episodes": 100,
|
||||
"wins_a": 58,
|
||||
"wins_b": 30,
|
||||
"draws": 12,
|
||||
"side_results": {
|
||||
"a_team_0": {
|
||||
"wins_a": 26,
|
||||
"wins_b": 20,
|
||||
"draws": 4
|
||||
},
|
||||
"a_team_1": {
|
||||
"wins_a": 32,
|
||||
"wins_b": 10,
|
||||
"draws": 8
|
||||
}
|
||||
},
|
||||
"physical_team_wins": {
|
||||
"team_0": 36,
|
||||
"team_1": 52
|
||||
},
|
||||
"win_rate_a": 0.58
|
||||
}
|
||||
],
|
||||
"decision": "fail"
|
||||
},
|
||||
{
|
||||
"stage_index": 1,
|
||||
"stage_number": 5,
|
||||
"stage_name": "intercepts",
|
||||
"experiment": "20260818-0210-gen5-s5-intercepts-retry2",
|
||||
"attempt": 2,
|
||||
"telemetry_tail": {
|
||||
"rollout/air_touch_fraction": 5.9999998658895494e-05,
|
||||
"rollout/airborne_fraction": 0.24753280937671662,
|
||||
"rollout/ep_len_mean": 103.67796003723144,
|
||||
"rollout/ep_rew_mean": 11.149126368522644,
|
||||
"rollout/forward_motion_fraction": 0.363580904841423,
|
||||
"rollout/goal_rate": 0.7755999964475632,
|
||||
"rollout/grounded_upright_fraction": 0.23967999944090843,
|
||||
"rollout/mean_altitude": 3.170829694747925,
|
||||
"rollout/productive_air_touch_fraction": 0.0,
|
||||
"rollout/upright_fraction": 0.800830381155014,
|
||||
"rollout/vertical_thrust_mean": 0.024330998776718044
|
||||
},
|
||||
"telemetry_failures": [
|
||||
"rollout/productive_air_touch_fraction=0.0000 < 0.0050"
|
||||
],
|
||||
"evaluation_goal_failures": [],
|
||||
"side_balance_failures": [],
|
||||
"eval": {
|
||||
"timestamp": "2026-08-18T10:17:39+00:00",
|
||||
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260818-0210-gen5-s5-intercepts-retry2.json",
|
||||
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260816-2126-gen5-s4-handling-retry2.json",
|
||||
"seed": 1,
|
||||
"episodes": 100,
|
||||
"wins_a": 53,
|
||||
"wins_b": 33,
|
||||
"draws": 14,
|
||||
"side_results": {
|
||||
"a_team_0": {
|
||||
"wins_a": 23,
|
||||
"wins_b": 20,
|
||||
"draws": 7
|
||||
},
|
||||
"a_team_1": {
|
||||
"wins_a": 30,
|
||||
"wins_b": 13,
|
||||
"draws": 7
|
||||
}
|
||||
},
|
||||
"physical_team_wins": {
|
||||
"team_0": 36,
|
||||
"team_1": 50
|
||||
},
|
||||
"win_rate_a": 0.53
|
||||
},
|
||||
"evals": [
|
||||
{
|
||||
"timestamp": "2026-08-18T10:17:39+00:00",
|
||||
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260818-0210-gen5-s5-intercepts-retry2.json",
|
||||
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260816-2126-gen5-s4-handling-retry2.json",
|
||||
"seed": 1,
|
||||
"episodes": 100,
|
||||
"wins_a": 53,
|
||||
"wins_b": 33,
|
||||
"draws": 14,
|
||||
"side_results": {
|
||||
"a_team_0": {
|
||||
"wins_a": 23,
|
||||
"wins_b": 20,
|
||||
"draws": 7
|
||||
},
|
||||
"a_team_1": {
|
||||
"wins_a": 30,
|
||||
"wins_b": 13,
|
||||
"draws": 7
|
||||
}
|
||||
},
|
||||
"physical_team_wins": {
|
||||
"team_0": 36,
|
||||
"team_1": 50
|
||||
},
|
||||
"win_rate_a": 0.53
|
||||
}
|
||||
],
|
||||
"decision": "fail"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -355,6 +355,11 @@ def parse_args():
|
||||
"--forward-velocity-to-ball-weight", type=float, default=None,
|
||||
help="Low-altitude dense reward for nose-led planar approach toward the ball",
|
||||
)
|
||||
curriculum.add_argument(
|
||||
"--air-approach-weight", type=float, default=None,
|
||||
help="Aerial mirror of forward-velocity-to-ball-weight: high-altitude dense reward for "
|
||||
"nose-led 3D closing speed toward the ball",
|
||||
)
|
||||
curriculum.add_argument(
|
||||
"--ball-distance-penalty", type=float, default=None,
|
||||
help="Overrides ShipAIController.ball_distance_penalty (dense per-tick cost scaled by distance to the ball)",
|
||||
@@ -419,6 +424,7 @@ def _curriculum_kwargs(args) -> dict:
|
||||
"ai_grounded_upright_reward": args.grounded_upright_reward,
|
||||
"ai_velocity_to_ball_weight": args.velocity_to_ball_weight,
|
||||
"ai_forward_velocity_to_ball_weight": args.forward_velocity_to_ball_weight,
|
||||
"ai_air_approach_weight": args.air_approach_weight,
|
||||
"ai_ball_distance_penalty": args.ball_distance_penalty,
|
||||
"ai_ball_touch_reward": args.ball_touch_reward,
|
||||
"ai_airborne_penalty": args.airborne_penalty,
|
||||
|
||||
Reference in New Issue
Block a user