chore(training): add air_touch_bonus_weight and restart stage-5 intercepts

air_approach_weight alone didn't move productive_air_touch_fraction after a
further 180M steps (360M cumulative across all six Stage-5 attempts): an
unredirected air-intercept ball falls short of the goal from gravity and
just lands on the floor, so the already-solved ground game collects the
same episode reward whether or not anything touched the ball in the air.
air_touch_bonus_weight adds a conjunctive event bonus on top of
ball_touch_reward for a touch that's both genuinely aerial and
goal-directed, targeting the actual measured behaviour instead of only the
approach to it.
This commit is contained in:
Josh Creek
2026-08-19 22:46:04 +01:00
parent 03f49e59c8
commit 602fa297d0
15 changed files with 100 additions and 260 deletions
+31 -6
View File
@@ -60,6 +60,29 @@ extends AIController3D
# vector rather than the planar-only one, since a real aerial requires
# pitching away from level.
@export var air_approach_weight := 0.0
# Event bonus, conjunctive with the same goal-direction alignment already
# gating ball_touch_reward: extra payout for a touch that is BOTH genuinely
# aerial (ball.y > AIR_TOUCH_HEIGHT, matching the productive_air_touch_
# fraction telemetry definition exactly) AND goal-directed. air_approach_
# weight alone did not move productive_air_touch_fraction (still 0.0 after
# a further 180M steps, 360M cumulative) because nothing in the reward ever
# made touching the ball while still airborne worth more than the
# alternative every policy already had available for free: let gravity pull
# an unredirected air-intercept ball back down (it falls well short of the
# goal's ~0-1.5m height band over the required flight distance, so it does
# not auto-score) and then collect the same goal_reward/ball_touch_reward
# via the already-dominant, already-solved ground game once it lands. This
# is deliberately NOT the standalone height-only bonus generation 4 ruled
# out (see TRAINING.md's "why no air-touch reward" note, added to close the
# RLGym wall-bounce exploit): it only pays scaled by the same alignment
# dot-product as the base term, so batting the ball in a non-productive
# direction earns nothing extra, same anti-farming shape as ball_touch_
# reward itself. Also safe from that specific exploit on distributional
# grounds: air-intercept spawns are central (x in [-8,8], z in [-10,10],
# arena half-extents 18/27) and air-drill spawns keep AIR_DRILL_BALL_WALL_
# CLEARANCE from every wall, so neither state can be solved by bouncing off
# one.
@export var air_touch_bonus_weight := 0.0
@export var ball_velocity_to_goal_weight := 0.004
# Per-tick penalty scaled by distance to the ball (full value at the arena's
# far diagonal, 0 on top of the ball). Run04 lesson: with idling worth a flat
@@ -504,14 +527,16 @@ func _on_ship_body_entered(body: Node) -> void:
var to_goal := attack_goal_position - ball.global_position
if to_goal.length_squared() > 0.0001 and ball.linear_velocity.length_squared() > 0.0001:
alignment = clampf(ball.linear_velocity.normalized().dot(to_goal.normalized()), 0.0, 1.0)
reward += ball_touch_reward * lerpf(ball_touch_direction_floor, 1.0, alignment)
var touch_payout := ball_touch_reward * lerpf(ball_touch_direction_floor, 1.0, alignment)
if air_touch_bonus_weight > 0.0 and ball.global_position.y > AIR_TOUCH_HEIGHT:
touch_payout += air_touch_bonus_weight * alignment
reward += touch_payout
_ticks_since_ball_touch = 0
# Telemetry only (see get_info's air_touch_fraction) — not a reward term.
# Generation 4 deliberately doesn't reward high touches directly (see
# TRAINING.md's "why no air-touch reward" note); this just measures
# whether the air-drill state setter is producing genuine aerial
# contests, so a future decision to add one is data-driven.
# air_touch_fraction/productive_air_touch_fraction (see get_info) share
# their AIR_TOUCH_HEIGHT/alignment definitions 1:1 with air_touch_bonus_
# weight above by design — the reward now targets exactly the behaviour
# the telemetry measures.
_touches += 1
if ball.global_position.y > AIR_TOUCH_HEIGHT:
_air_touches += 1
+2 -1
View File
@@ -265,7 +265,7 @@ const TRAINING_MODE_OVERRIDES := [
const SHIP_AI_OVERRIDES := [
"ball_touch_reward", "ball_touch_cooldown_ticks", "ball_touch_direction_floor",
"velocity_to_ball_weight", "ball_velocity_to_goal_weight", "ball_distance_penalty",
"forward_velocity_to_ball_weight", "air_approach_weight", "wall_contact_penalty", "tilt_penalty",
"forward_velocity_to_ball_weight", "air_approach_weight", "air_touch_bonus_weight", "wall_contact_penalty", "tilt_penalty",
"ground_tilt_penalty", "non_forward_penalty", "grounded_upright_reward",
"speed_reward_weight", "time_penalty", "airborne_penalty",
]
@@ -324,6 +324,7 @@ func _ai_default(name: String) -> Variant:
"velocity_to_ball_weight": return 0.02
"forward_velocity_to_ball_weight": return 0.0
"air_approach_weight": return 0.0
"air_touch_bonus_weight": return 0.0
"ball_velocity_to_goal_weight": return 0.004
"ball_distance_penalty": return 0.002
"wall_contact_penalty": return 0.0025