mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-11 00:14:00 +00:00
fix(training): make uprightness a multiplier on the forward-approach reward
Rounds 2 and 3 showed that tuning grounded_upright_reward's magnitude only slides along a tradeoff instead of resolving it: at 0.015 upright_fraction climbed to 0.331 while goal_rate sagged to 0.542 (then farmed outright at 0.696/0.366), and at 0.004 goal_rate climbed 0.569->0.604 while upright_fraction went flat at ~0.26. An additive uprightness bonus is an alternative to playing well, so the policy just picks whichever is cheaper and no magnitude buys both behaviours. Change the mechanism rather than the number: grounded_upright_reward drops to 0, and uprightness becomes a multiplier inside the nose-led approach term, which already requires moving forward at the ball. Parked-and-upright and fast-but-sideways now both pay zero; only upright, forward, nose-on to the ball pays full. forward-velocity-to-ball rises 0.06 -> 0.15 to offset the ~2-3x expected-value cut from the new factor, and ground-tilt-penalty drops 0.05 -> 0.02 now that uprightness is paid positively during play. Delete the three blocked attempts and reset state to restart from the Stage-3 foundation.
This commit is contained in:
Josh Creek
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -95,6 +95,18 @@ extends AIController3D
|
||||
# the ball, cratering goal_rate. Keep this term's episode-long ceiling
|
||||
# (value * ~1800 ticks) below ball_distance_penalty's worst-case episode
|
||||
# cost, not just below ball_touch_reward/goal_reward.
|
||||
#
|
||||
# SUPERSEDED (2026-08-12), kept at 0 for older curricula that set it: the
|
||||
# magnitude was never the real problem. Retuning it 0.015 -> 0.004 only
|
||||
# moved along a tradeoff — at 0.015 upright_fraction climbed while
|
||||
# goal_rate sagged, at 0.004 goal_rate climbed while upright_fraction went
|
||||
# flat — because an *additive* uprightness reward is an alternative to
|
||||
# playing well, so the policy just picks whichever is cheaper. Uprightness
|
||||
# is now a multiplier inside the forward-approach term below instead, which
|
||||
# makes it conjunctive with (not competing against) moving forward at the
|
||||
# ball. Prefer that pattern for any future posture shaping; only reach for
|
||||
# a standalone additive posture bonus if there is genuinely nothing to
|
||||
# condition it on.
|
||||
@export var grounded_upright_reward := 0.0
|
||||
# Per-tick bonus for own speed: 0 stationary, full value (+0.24/s) at
|
||||
# max_speed. Run07 lesson: after the kickoff flurry both ships parked next to
|
||||
@@ -296,9 +308,22 @@ func _physics_process(delta):
|
||||
var closing_speed := ship.linear_velocity.dot(to_ball.normalized())
|
||||
reward += velocity_to_ball_weight * closing_speed / ship.max_speed
|
||||
|
||||
# Ground-handling shaping: forward planar motion while the nose faces the
|
||||
# ball. It fades out with altitude so an aerial remains free to approach a
|
||||
# ball using whatever body attitude is effective.
|
||||
# Ground-handling shaping: upright, forward planar motion while the nose
|
||||
# faces the ball. It fades out with altitude so an aerial remains free to
|
||||
# approach a ball using whatever body attitude is effective.
|
||||
#
|
||||
# Uprightness is a *multiplier* here rather than a separate additive term,
|
||||
# and that is the whole point. Stage 4's earlier rounds paid uprightness
|
||||
# additively (grounded_upright_reward): because additive terms let a
|
||||
# policy collect whichever one is cheapest, it could either play well
|
||||
# (tilted, scoring) or sit parked upright (still, not scoring) — and it
|
||||
# picked one or the other depending purely on that term's magnitude, so
|
||||
# upright_fraction and goal_rate moved in opposite directions at every
|
||||
# value tried. As a multiplier, uprightness pays only while the ship is
|
||||
# also moving forward and nose-on to the ball, so no subset of the three
|
||||
# behaviours can be farmed in isolation: parked pays zero (forward_speed
|
||||
# is zero), on-its-side pays zero (uprightness is zero), and only doing
|
||||
# all three at once pays full.
|
||||
if forward_velocity_to_ball_weight > 0.0 and ship.global_position.y < GROUND_HANDLING_HEIGHT:
|
||||
var planar_forward := Vector3(-ship.global_transform.basis.z.x, 0.0, -ship.global_transform.basis.z.z)
|
||||
var planar_velocity := Vector3(ship.linear_velocity.x, 0.0, ship.linear_velocity.z)
|
||||
@@ -307,8 +332,10 @@ func _physics_process(delta):
|
||||
planar_forward = planar_forward.normalized()
|
||||
var facing_ball: float = maxf(planar_forward.dot(planar_to_ball.normalized()), 0.0)
|
||||
var forward_speed: float = maxf(planar_velocity.dot(planar_forward), 0.0) / ship.max_speed
|
||||
var approach_uprightness: float = maxf(ship.global_transform.basis.y.dot(Vector3.UP), 0.0)
|
||||
var handling_ground_factor: float = 1.0 - clampf(ship.global_position.y / GROUND_HANDLING_HEIGHT, 0.0, 1.0)
|
||||
reward += forward_velocity_to_ball_weight * forward_speed * facing_ball * handling_ground_factor
|
||||
reward += forward_velocity_to_ball_weight * forward_speed * facing_ball \
|
||||
* approach_uprightness * handling_ground_factor
|
||||
|
||||
# Dense penalty: distance to the ball, so idling far away bleeds reward
|
||||
# instead of scoring a safe zero (see ball_distance_penalty).
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
+28
-3
@@ -75,18 +75,43 @@ STANDING_ARGS = ["--ent-coef", "0.01", "--entropy-floor"]
|
||||
# continuing from the farming checkpoint, same reasoning as the ground_tilt/
|
||||
# non_forward_penalty retune: don't resume a policy shaped by one reward
|
||||
# balance into a meaningfully different one.
|
||||
#
|
||||
# Round 3 (2026-08-12): 0.004 stopped the farming (vertical_thrust_mean
|
||||
# stayed positive, airborne_fraction flat) and goal_rate rose across the
|
||||
# chain 0.569->0.598->0.604 — but upright_fraction went flat at ~0.26, and
|
||||
# retry1 posted the best head-to-head in Stage-4 history (eval goal_rate
|
||||
# 0.820, 53-29-18). Lining rounds 2 and 3 up by attempt shows the actual
|
||||
# problem: where upright climbed goal_rate sagged, and where goal_rate
|
||||
# climbed upright went flat. An *additive* uprightness bonus is an
|
||||
# alternative to playing well, so the policy just picks whichever is
|
||||
# cheaper and the magnitude only slides along that tradeoff — no value can
|
||||
# buy both. Round 4 therefore changes the mechanism instead of the number:
|
||||
# grounded_upright_reward drops to 0, and uprightness becomes a multiplier
|
||||
# inside the nose-led approach term (ship_ai_controller.gd), which already
|
||||
# requires moving forward at the ball. Upright now pays only *while*
|
||||
# playing, so parked-and-upright and fast-but-sideways both pay zero and
|
||||
# only all three behaviours together pay full.
|
||||
#
|
||||
# forward-velocity-to-ball rises 0.06 -> 0.15 because multiplying by
|
||||
# uprightness cuts that term's expected per-tick value roughly 2-3x at
|
||||
# current behaviour; without the raise the approach incentive would quietly
|
||||
# weaken. ground-tilt-penalty drops 0.05 -> 0.02: it was the term most
|
||||
# likely to teach floor-avoidance, and with uprightness now positively paid
|
||||
# during play it only needs to remain a guardrail against travelling on the
|
||||
# roof rather than carry the whole posture signal. non-forward-penalty is
|
||||
# unchanged at 0.04 — it targets a specific behaviour and has not misfired.
|
||||
HANDLING_REWARD_FLAGS = [
|
||||
"--velocity-to-ball-weight", "0.04",
|
||||
"--forward-velocity-to-ball-weight", "0.06",
|
||||
"--forward-velocity-to-ball-weight", "0.15",
|
||||
"--ball-distance-penalty", "0.01",
|
||||
"--ball-touch-reward", "0.7",
|
||||
"--ball-velocity-to-goal-weight", "0.06",
|
||||
"--goal-reward", "80",
|
||||
"--speed-reward-weight", "0.0",
|
||||
"--tilt-penalty", "0.0002",
|
||||
"--ground-tilt-penalty", "0.05",
|
||||
"--ground-tilt-penalty", "0.02",
|
||||
"--non-forward-penalty", "0.04",
|
||||
"--grounded-upright-reward", "0.004",
|
||||
"--grounded-upright-reward", "0.0",
|
||||
]
|
||||
|
||||
STAGES = [
|
||||
|
||||
@@ -1,260 +1,6 @@
|
||||
{
|
||||
"stage_index": 0,
|
||||
"attempt": 2,
|
||||
"status": "blocked",
|
||||
"log": [
|
||||
{
|
||||
"stage_index": 0,
|
||||
"stage_number": 4,
|
||||
"stage_name": "handling",
|
||||
"experiment": "20260811-1606-gen5-s4-handling",
|
||||
"attempt": 0,
|
||||
"telemetry_tail": {
|
||||
"rollout/air_touch_fraction": 0.0004399999901652336,
|
||||
"rollout/airborne_fraction": 0.44587357181310655,
|
||||
"rollout/ep_len_mean": 139.7144799041748,
|
||||
"rollout/ep_rew_mean": -5.688299763679504,
|
||||
"rollout/forward_motion_fraction": 0.21974961887300015,
|
||||
"rollout/goal_rate": 0.5693600015640259,
|
||||
"rollout/mean_altitude": 4.424891119480133,
|
||||
"rollout/productive_air_touch_fraction": 7.999999821186065e-05,
|
||||
"rollout/upright_fraction": 0.2615443330407143,
|
||||
"rollout/vertical_thrust_mean": 0.08650099822867197
|
||||
},
|
||||
"telemetry_failures": [
|
||||
"rollout/goal_rate=0.5694 < 0.8000",
|
||||
"rollout/upright_fraction=0.2615 < 0.4500",
|
||||
"rollout/forward_motion_fraction=0.2197 < 0.2500"
|
||||
],
|
||||
"evaluation_goal_failures": [
|
||||
"easy.json: goal_rate=0.770 < 0.800"
|
||||
],
|
||||
"side_balance_failures": [],
|
||||
"eval": {
|
||||
"timestamp": "2026-08-11T21:16:10+00:00",
|
||||
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260811-1606-gen5-s4-handling.json",
|
||||
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
|
||||
"seed": 1,
|
||||
"episodes": 100,
|
||||
"wins_a": 47,
|
||||
"wins_b": 30,
|
||||
"draws": 23,
|
||||
"side_results": {
|
||||
"a_team_0": {
|
||||
"wins_a": 22,
|
||||
"wins_b": 16,
|
||||
"draws": 12
|
||||
},
|
||||
"a_team_1": {
|
||||
"wins_a": 25,
|
||||
"wins_b": 14,
|
||||
"draws": 11
|
||||
}
|
||||
},
|
||||
"physical_team_wins": {
|
||||
"team_0": 36,
|
||||
"team_1": 41
|
||||
},
|
||||
"win_rate_a": 0.47
|
||||
},
|
||||
"evals": [
|
||||
{
|
||||
"timestamp": "2026-08-11T21:16:10+00:00",
|
||||
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260811-1606-gen5-s4-handling.json",
|
||||
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
|
||||
"seed": 1,
|
||||
"episodes": 100,
|
||||
"wins_a": 47,
|
||||
"wins_b": 30,
|
||||
"draws": 23,
|
||||
"side_results": {
|
||||
"a_team_0": {
|
||||
"wins_a": 22,
|
||||
"wins_b": 16,
|
||||
"draws": 12
|
||||
},
|
||||
"a_team_1": {
|
||||
"wins_a": 25,
|
||||
"wins_b": 14,
|
||||
"draws": 11
|
||||
}
|
||||
},
|
||||
"physical_team_wins": {
|
||||
"team_0": 36,
|
||||
"team_1": 41
|
||||
},
|
||||
"win_rate_a": 0.47
|
||||
}
|
||||
],
|
||||
"decision": "fail"
|
||||
},
|
||||
{
|
||||
"stage_index": 0,
|
||||
"stage_number": 4,
|
||||
"stage_name": "handling",
|
||||
"experiment": "20260811-2216-gen5-s4-handling-retry1",
|
||||
"attempt": 1,
|
||||
"telemetry_tail": {
|
||||
"rollout/air_touch_fraction": 0.00019999999552965163,
|
||||
"rollout/airborne_fraction": 0.446684571146965,
|
||||
"rollout/ep_len_mean": 135.95963993835448,
|
||||
"rollout/ep_rew_mean": -4.255322490215302,
|
||||
"rollout/forward_motion_fraction": 0.21915800005197525,
|
||||
"rollout/goal_rate": 0.5982800037264824,
|
||||
"rollout/mean_altitude": 4.420792480945587,
|
||||
"rollout/productive_air_touch_fraction": 3.999999910593033e-05,
|
||||
"rollout/upright_fraction": 0.2567908099591732,
|
||||
"rollout/vertical_thrust_mean": 0.1063969987387536
|
||||
},
|
||||
"telemetry_failures": [
|
||||
"rollout/goal_rate=0.5983 < 0.8000",
|
||||
"rollout/upright_fraction=0.2568 < 0.4500",
|
||||
"rollout/forward_motion_fraction=0.2192 < 0.2500"
|
||||
],
|
||||
"evaluation_goal_failures": [],
|
||||
"side_balance_failures": [],
|
||||
"eval": {
|
||||
"timestamp": "2026-08-12T03:24:21+00:00",
|
||||
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260811-2216-gen5-s4-handling-retry1.json",
|
||||
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
|
||||
"seed": 1,
|
||||
"episodes": 100,
|
||||
"wins_a": 53,
|
||||
"wins_b": 29,
|
||||
"draws": 18,
|
||||
"side_results": {
|
||||
"a_team_0": {
|
||||
"wins_a": 22,
|
||||
"wins_b": 17,
|
||||
"draws": 11
|
||||
},
|
||||
"a_team_1": {
|
||||
"wins_a": 31,
|
||||
"wins_b": 12,
|
||||
"draws": 7
|
||||
}
|
||||
},
|
||||
"physical_team_wins": {
|
||||
"team_0": 34,
|
||||
"team_1": 48
|
||||
},
|
||||
"win_rate_a": 0.53
|
||||
},
|
||||
"evals": [
|
||||
{
|
||||
"timestamp": "2026-08-12T03:24:21+00:00",
|
||||
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260811-2216-gen5-s4-handling-retry1.json",
|
||||
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
|
||||
"seed": 1,
|
||||
"episodes": 100,
|
||||
"wins_a": 53,
|
||||
"wins_b": 29,
|
||||
"draws": 18,
|
||||
"side_results": {
|
||||
"a_team_0": {
|
||||
"wins_a": 22,
|
||||
"wins_b": 17,
|
||||
"draws": 11
|
||||
},
|
||||
"a_team_1": {
|
||||
"wins_a": 31,
|
||||
"wins_b": 12,
|
||||
"draws": 7
|
||||
}
|
||||
},
|
||||
"physical_team_wins": {
|
||||
"team_0": 34,
|
||||
"team_1": 48
|
||||
},
|
||||
"win_rate_a": 0.53
|
||||
}
|
||||
],
|
||||
"decision": "fail"
|
||||
},
|
||||
{
|
||||
"stage_index": 0,
|
||||
"stage_number": 4,
|
||||
"stage_name": "handling",
|
||||
"experiment": "20260812-0424-gen5-s4-handling-retry2",
|
||||
"attempt": 2,
|
||||
"telemetry_tail": {
|
||||
"rollout/air_touch_fraction": 0.00025999999418854715,
|
||||
"rollout/airborne_fraction": 0.44747342938184737,
|
||||
"rollout/ep_len_mean": 133.66904002380372,
|
||||
"rollout/ep_rew_mean": -2.8346366541385652,
|
||||
"rollout/forward_motion_fraction": 0.22339595240354537,
|
||||
"rollout/goal_rate": 0.6036400033831596,
|
||||
"rollout/mean_altitude": 4.436961054801941,
|
||||
"rollout/productive_air_touch_fraction": 7.999999821186065e-05,
|
||||
"rollout/upright_fraction": 0.2713429524004459,
|
||||
"rollout/vertical_thrust_mean": 0.05840199833672028
|
||||
},
|
||||
"telemetry_failures": [
|
||||
"rollout/goal_rate=0.6036 < 0.8000",
|
||||
"rollout/upright_fraction=0.2713 < 0.4500",
|
||||
"rollout/forward_motion_fraction=0.2234 < 0.2500"
|
||||
],
|
||||
"evaluation_goal_failures": [
|
||||
"easy.json: goal_rate=0.660 < 0.800"
|
||||
],
|
||||
"side_balance_failures": [],
|
||||
"eval": {
|
||||
"timestamp": "2026-08-12T09:31:39+00:00",
|
||||
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260812-0424-gen5-s4-handling-retry2.json",
|
||||
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
|
||||
"seed": 1,
|
||||
"episodes": 100,
|
||||
"wins_a": 37,
|
||||
"wins_b": 29,
|
||||
"draws": 34,
|
||||
"side_results": {
|
||||
"a_team_0": {
|
||||
"wins_a": 18,
|
||||
"wins_b": 20,
|
||||
"draws": 12
|
||||
},
|
||||
"a_team_1": {
|
||||
"wins_a": 19,
|
||||
"wins_b": 9,
|
||||
"draws": 22
|
||||
}
|
||||
},
|
||||
"physical_team_wins": {
|
||||
"team_0": 27,
|
||||
"team_1": 39
|
||||
},
|
||||
"win_rate_a": 0.37
|
||||
},
|
||||
"evals": [
|
||||
{
|
||||
"timestamp": "2026-08-12T09:31:39+00:00",
|
||||
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260812-0424-gen5-s4-handling-retry2.json",
|
||||
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
|
||||
"seed": 1,
|
||||
"episodes": 100,
|
||||
"wins_a": 37,
|
||||
"wins_b": 29,
|
||||
"draws": 34,
|
||||
"side_results": {
|
||||
"a_team_0": {
|
||||
"wins_a": 18,
|
||||
"wins_b": 20,
|
||||
"draws": 12
|
||||
},
|
||||
"a_team_1": {
|
||||
"wins_a": 19,
|
||||
"wins_b": 9,
|
||||
"draws": 22
|
||||
}
|
||||
},
|
||||
"physical_team_wins": {
|
||||
"team_0": 27,
|
||||
"team_1": 39
|
||||
},
|
||||
"win_rate_a": 0.37
|
||||
}
|
||||
],
|
||||
"decision": "fail"
|
||||
}
|
||||
]
|
||||
"attempt": 0,
|
||||
"status": "in_progress",
|
||||
"log": []
|
||||
}
|
||||
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
Reference in New Issue
Block a user