From 7d69ac4a01ea19b1681a0ed825eff89ebf155e23 Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Mon, 20 Jul 2026 20:06:11 +0100 Subject: [PATCH] feat(*): retune scoring incentives and add finishing reps --- Game/scripts/ship_ai_controller.gd | 42 ++++++++++++++++++++++++++---- Game/scripts/training_mode.gd | 15 +++++++++-- training/eval_history.json | 10 +++++++ 3 files changed, 60 insertions(+), 7 deletions(-) diff --git a/Game/scripts/ship_ai_controller.gd b/Game/scripts/ship_ai_controller.gd index ed3b40fe..a4e35817 100644 --- a/Game/scripts/ship_ai_controller.gd +++ b/Game/scripts/ship_ai_controller.gd @@ -15,13 +15,28 @@ extends AIController3D # Reward shaping weights. Dense terms accrue per physics tick (60 sim-ticks # per sim-second); event terms fire once. Exported so tuning needs no code # edits. Goal rewards are added by TrainingMode, which owns goal events. -@export var ball_touch_reward := 1.0 +@export var ball_touch_reward := 0.4 # Ball touches pay out at most once per this many physics ticks (1 sim- # second at 60). Run07 lesson: body_entered re-fires on every micro- # separation, so pinning the ball against a surface farmed ~2 touches/s — # outearning every other term while the goal rate fell. The cooldown keeps -# touches a stepping-stone signal instead of the objective. +# touches a stepping-stone signal instead of the objective. Halved again +# after run01-vs-run02 eval (training/eval_history.json) came back 87.5% +# draws: even at 1 touch/s, a full episode's worth of touches could still +# outweigh TrainingMode's goal_reward, so scoring and ending the episode +# early was never worth it. See goal_reward's comment for the other half of +# this fix. @export var ball_touch_cooldown_ticks := 60 +# A touch pays out scaled by how goal-directed it was — full ball_touch_reward +# when the post-touch ball velocity points straight at the attack goal, down +# to this floor when it doesn't (0 = only goal-directed touches pay at all). +# Without this, any contact paid the same regardless of direction, so batting +# the ball anywhere counted the same as an actual shot on goal — reinforcing +# possession, not scoring. The floor keeps a purely defensive touch (e.g. +# clearing a shot away from your own goal) worth something as a stepping +# stone, matching ball_touch_cooldown_ticks's existing "stepping-stone, not +# the objective" framing. +@export_range(0.0, 1.0) var ball_touch_direction_floor := 0.3 @export var velocity_to_ball_weight := 0.02 @export var ball_velocity_to_goal_weight := 0.004 # Per-tick penalty scaled by distance to the ball (full value at the arena's @@ -48,6 +63,13 @@ extends AIController3D # still was a rest state. Sized well below velocity_to_ball_weight so flying # fast toward the ball still beats flying fast anywhere else. @export var speed_reward_weight := 0.004 +# Flat per-tick cost (-0.06/s, -1.8 over a full 30s episode) applied +# regardless of position or behaviour. Every other dense term can be farmed +# indefinitely by an episode that never ends in a goal; this one can't — it +# only stops accruing once the episode does, via a goal or the timeout. That +# makes running the clock out strictly worse than scoring as soon as a +# chance appears, instead of a free way to keep collecting dense reward. +@export var time_penalty := 0.001 # Contact normals with y above this are floor contact (exempt from the wall # penalty); below it they read as wall (sideways) or ceiling (downward). @@ -122,6 +144,9 @@ func _physics_process(delta): return _ticks_since_ball_touch += 1 + # Flat time cost — see time_penalty. + reward -= time_penalty + # Dense shaping: own velocity toward the ball var to_ball := ball.global_position - ship.global_position if to_ball.length_squared() > 0.0001: @@ -174,6 +199,13 @@ func _wall_or_ceiling_contact() -> bool: func _on_ship_body_entered(body: Node) -> void: - if body.is_in_group("ball") and _ticks_since_ball_touch >= ball_touch_cooldown_ticks: - reward += ball_touch_reward - _ticks_since_ball_touch = 0 + if not body.is_in_group("ball") or _ticks_since_ball_touch < ball_touch_cooldown_ticks: + return + # Contact-signal ordering means ball.linear_velocity here already reflects + # the collision impulse from this touch, not the pre-touch velocity. + var alignment := 0.0 + var to_goal := attack_goal_position - ball.global_position + if to_goal.length_squared() > 0.0001 and ball.linear_velocity.length_squared() > 0.0001: + alignment = clampf(ball.linear_velocity.normalized().dot(to_goal.normalized()), 0.0, 1.0) + reward += ball_touch_reward * lerpf(ball_touch_direction_floor, 1.0, alignment) + _ticks_since_ball_touch = 0 diff --git a/Game/scripts/training_mode.gd b/Game/scripts/training_mode.gd index 445de9d9..477bb07f 100644 --- a/Game/scripts/training_mode.gd +++ b/Game/scripts/training_mode.gd @@ -18,11 +18,22 @@ extends GameMode # "EVAL_RESULT {...}" line is printed before quitting. @export var episode_length_seconds := 30.0 -@export var goal_reward := 10.0 +# Run01-vs-run02 eval (see training/eval_history.json) came back 87.5% draws: +# with a 30s episode, the dense per-tick terms on ShipAIController can sum to +# several times this value before it was raised, so scoring and forfeiting +# the rest of the episode's farmable reward was worse than never finishing. +# Raised well above that ceiling so a real scoring chance always beats +# continuing to farm dense reward for however long is left in the episode. +@export var goal_reward := 40.0 # Episode-start state mix; remaining probability = fully random state. @export_range(0.0, 1.0) var kickoff_state_chance := 0.2 -@export_range(0.0, 1.0) var ball_near_goal_chance := 0.2 +# Raised from 0.2: fixing the reward incentive to score (see goal_reward, +# ball_touch_reward, time_penalty on ShipAIController) only helps if the +# policy also gets enough reps at actually finishing. At 0.2 that scenario +# was 1 in 5 episode starts; most training time was spent in generic +# midfield play where a finish never comes up. +@export_range(0.0, 1.0) var ball_near_goal_chance := 0.35 # Placement bounds for randomized episode starts, derived from the standard # enclosure (ArenaBoundary). The inset keeps a randomly oriented ship (1x1x4 diff --git a/training/eval_history.json b/training/eval_history.json index 08b15bf5..1ed15acb 100644 --- a/training/eval_history.json +++ b/training/eval_history.json @@ -18,5 +18,15 @@ "wins_b": 1, "draws": 35, "win_rate_a": 0.1 + }, + { + "timestamp": "2026-07-20T18:47:54+00:00", + "model_a": "/Users/jcreek/Documents/repos/GitHub/CosmicClash/Game/bots/run09.json", + "model_b": "/Users/jcreek/Documents/repos/GitHub/CosmicClash/Game/bots/run07.json", + "episodes": 40, + "wins_a": 6, + "wins_b": 9, + "draws": 25, + "win_rate_a": 0.15 } ]