feat(training): add generation 5 curriculum

This commit is contained in:
Josh Creek
2026-08-08 14:56:17 +01:00
parent 33952b3cd0
commit 341a67f6da
10 changed files with 832 additions and 19 deletions
+67
View File
@@ -39,6 +39,13 @@ extends AIController3D
# the objective" framing.
@export_range(0.0, 1.0) var ball_touch_direction_floor := 0.3
@export var velocity_to_ball_weight := 0.02
# Dense reward for approaching the ball *nose first* near the floor. Unlike
# velocity_to_ball_weight, sideways/reverse closing velocity earns nothing:
# the planar ship-forward vector must face the ball and planar velocity must
# have a positive component along it. Default off so existing curricula and
# frozen checkpoints keep their original objective; generation 5 handling
# turns it on while reducing the orientation-agnostic term.
@export var forward_velocity_to_ball_weight := 0.0
@export var ball_velocity_to_goal_weight := 0.004
# Per-tick penalty scaled by distance to the ball (full value at the arena's
# far diagonal, 0 on top of the ball). Run04 lesson: with idling worth a flat
@@ -62,6 +69,11 @@ extends AIController3D
# to teach. Not removed outright — an always-inverted bot still looks bad in
# a shipped game.
@export var tilt_penalty := 0.0005
# Additional tilt cost that fades to zero over the first few metres above the
# floor. This can teach readable, upright ground handling without opposing
# pitch/roll during a real aerial. Generation 5 uses this instead of raising
# the global tilt_penalty back to its pre-flight value.
@export var ground_tilt_penalty := 0.0
# Per-tick bonus for own speed: 0 stationary, full value (+0.24/s) at
# max_speed. Run07 lesson: after the kickoff flurry both ships parked next to
# a cornered ball — with every other dense term near zero there, standing
@@ -90,6 +102,15 @@ extends AIController3D
# air-touch reward.
const AIR_TOUCH_HEIGHT := 5.0
# Generation-5 ground-handling telemetry/reward thresholds. Fixed constants
# keep the logged metrics comparable across stages; changing one starts a new
# metric definition and therefore requires a fresh baseline.
const GROUND_HANDLING_HEIGHT := 3.0
const UPRIGHT_DOT_THRESHOLD := 0.7
const FORWARD_MOTION_DOT_THRESHOLD := 0.7
const MIN_HANDLING_SPEED := 1.0
const PRODUCTIVE_AIR_TOUCH_ALIGNMENT := 0.5
# Contact normals with y above this are floor contact (exempt from the wall
# penalty); below it they read as wall (sideways) or ceiling (downward).
# Mirrors ShipObservations.FLOOR_NORMAL_MIN_Y (see that file's comment).
@@ -144,6 +165,11 @@ var _altitude_sum := 0.0
var _thrust_y_sum := 0.0
var _touches := 0
var _air_touches := 0
var _productive_air_touches := 0
var _ground_ticks := 0
var _upright_ground_ticks := 0
var _moving_ground_ticks := 0
var _forward_moving_ground_ticks := 0
# Wire up references after the ship is spawned. `attack_goal` is the goal
@@ -201,6 +227,9 @@ func get_info() -> Dictionary:
info["mean_altitude"] = _altitude_sum / _telemetry_ticks if _telemetry_ticks > 0 else 0.0
info["vertical_thrust_mean"] = _thrust_y_sum / _telemetry_ticks if _telemetry_ticks > 0 else 0.0
info["air_touch_fraction"] = float(_air_touches) / _touches if _touches > 0 else 0.0
info["productive_air_touch_fraction"] = float(_productive_air_touches) / _touches if _touches > 0 else 0.0
info["upright_fraction"] = float(_upright_ground_ticks) / _ground_ticks if _ground_ticks > 0 else 0.0
info["forward_motion_fraction"] = float(_forward_moving_ground_ticks) / _moving_ground_ticks if _moving_ground_ticks > 0 else 0.0
return info
@@ -223,6 +252,11 @@ func reset():
_thrust_y_sum = 0.0
_touches = 0
_air_touches = 0
_productive_air_touches = 0
_ground_ticks = 0
_upright_ground_ticks = 0
_moving_ground_ticks = 0
_forward_moving_ground_ticks = 0
func _physics_process(delta):
@@ -240,6 +274,20 @@ func _physics_process(delta):
var closing_speed := ship.linear_velocity.dot(to_ball.normalized())
reward += velocity_to_ball_weight * closing_speed / ship.max_speed
# Ground-handling shaping: forward planar motion while the nose faces the
# ball. It fades out with altitude so an aerial remains free to approach a
# ball using whatever body attitude is effective.
if forward_velocity_to_ball_weight > 0.0 and ship.global_position.y < GROUND_HANDLING_HEIGHT:
var planar_forward := Vector3(-ship.global_transform.basis.z.x, 0.0, -ship.global_transform.basis.z.z)
var planar_velocity := Vector3(ship.linear_velocity.x, 0.0, ship.linear_velocity.z)
var planar_to_ball := Vector3(to_ball.x, 0.0, to_ball.z)
if planar_forward.length_squared() > 0.0001 and planar_to_ball.length_squared() > 0.0001:
planar_forward = planar_forward.normalized()
var facing_ball: float = maxf(planar_forward.dot(planar_to_ball.normalized()), 0.0)
var forward_speed: float = maxf(planar_velocity.dot(planar_forward), 0.0) / ship.max_speed
var handling_ground_factor: float = 1.0 - clampf(ship.global_position.y / GROUND_HANDLING_HEIGHT, 0.0, 1.0)
reward += forward_velocity_to_ball_weight * forward_speed * facing_ball * handling_ground_factor
# Dense penalty: distance to the ball, so idling far away bleeds reward
# instead of scoring a safe zero (see ball_distance_penalty).
if ball_distance_penalty > 0.0:
@@ -270,6 +318,12 @@ func _physics_process(delta):
var uprightness: float = ship.global_transform.basis.y.dot(Vector3.UP)
reward -= tilt_penalty * (1.0 - uprightness) * 0.5
# Low-altitude-only posture pressure (see ground_tilt_penalty).
if ground_tilt_penalty > 0.0 and ship.global_position.y < GROUND_HANDLING_HEIGHT:
var ground_uprightness: float = ship.global_transform.basis.y.dot(Vector3.UP)
var tilt_ground_factor: float = 1.0 - clampf(ship.global_position.y / GROUND_HANDLING_HEIGHT, 0.0, 1.0)
reward -= ground_tilt_penalty * (1.0 - ground_uprightness) * 0.5 * tilt_ground_factor
# Dense penalty: height above the floor (see airborne_penalty). The
# floor sits at world y = 0 (see training_mode.gd's FIELD_MIN_Y/
# _escaped bounds); normalized so the worst case is pinned at the
@@ -285,6 +339,17 @@ func _physics_process(delta):
if ship.global_position.y > AIRBORNE_ALTITUDE_THRESHOLD:
_airborne_ticks += 1
_thrust_y_sum += rl_controller.action.thrust.y
if ship.global_position.y < GROUND_HANDLING_HEIGHT:
_ground_ticks += 1
if ship.global_transform.basis.y.dot(Vector3.UP) >= UPRIGHT_DOT_THRESHOLD:
_upright_ground_ticks += 1
var planar_velocity := Vector3(ship.linear_velocity.x, 0.0, ship.linear_velocity.z)
if planar_velocity.length() >= MIN_HANDLING_SPEED:
_moving_ground_ticks += 1
var planar_forward := Vector3(-ship.global_transform.basis.z.x, 0.0, -ship.global_transform.basis.z.z)
if planar_forward.length_squared() > 0.0001 \
and planar_velocity.normalized().dot(planar_forward.normalized()) >= FORWARD_MOTION_DOT_THRESHOLD:
_forward_moving_ground_ticks += 1
func _wall_or_ceiling_contact() -> bool:
@@ -311,3 +376,5 @@ func _on_ship_body_entered(body: Node) -> void:
_touches += 1
if ball.global_position.y > AIR_TOUCH_HEIGHT:
_air_touches += 1
if alignment >= PRODUCTIVE_AIR_TOUCH_ALIGNMENT:
_productive_air_touches += 1