jcreek/CosmicClash
1
0
Fork 0
mirror of https://github.com/jcreek/CosmicClash.git synced 2026-09-10 16:04:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(training): cut grounded_upright_reward, restart stage-4 from Stage-3 foundation

Browse Source 
grounded_upright_reward at 0.015 overshot: four force-retries pushed
upright_fraction from 0.265 to a plateauing 0.331, then the fifth jumped it
to 0.696 (55% over the 0.45 floor) while goal_rate collapsed 0.542->0.366
and forward_motion_fraction fell 0.244->0.184 (vertical_thrust_mean went
negative) - the policy learned to sit pinned upright and farm the bonus
instead of chasing the ball. It was sized "comparable to
time_penalty/ball_distance_penalty" but at 0.015/tick it was actually above
ball_distance_penalty's 0.01/tick worst case, so idling near the ball beat
playing. Cut to 0.004/tick (episode ceiling ~7.2, below
ball_distance_penalty's ~18 worst case). Delete the five blocked attempts
and reset generation5_state.json so the next run starts fresh from the
Stage-3 foundation rather than continuing from the farming checkpoint.
This commit is contained in:
Josh Creek
2026-08-11 15:57:27 +01:00
parent f5b0a79cea
commit b8e2a7b57a
18 changed files with 29 additions and 439 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Game/bots/20260809-1340-gen5-s4-handling.json
-1
View File
File diff suppressed because one or more lines are too long
Game/bots/20260809-1955-gen5-s4-handling-retry1.json
-1
View File
File diff suppressed because one or more lines are too long
Game/bots/20260810-0211-gen5-s4-handling-retry2.json
-1
View File
File diff suppressed because one or more lines are too long
Game/bots/20260810-1338-gen5-s4-handling-retry3.json
-1
View File
File diff suppressed because one or more lines are too long
Game/bots/20260811-0858-gen5-s4-handling-retry4.json
-1
View File
File diff suppressed because one or more lines are too long
Game/scripts/ship_ai_controller.gd
+8 -5
View File
@@ -87,11 +87,14 @@ extends AIController3D
# policy that's still bad at ground handling could "solve" those penalties
# by just avoiding the floor rather than by getting better at handling on
# it — worsening Stage 3's already-airborne-heavy baseline instead of
# fixing it. Kept an order of magnitude below ball_touch_reward/goal_reward
# and comparable to time_penalty/ball_distance_penalty so grounding well is
# attractive without making idling upright on the spot, away from the ball,
# competitive with actually playing (see ball_distance_penalty's run04
# lesson on why a flat positional bonus needs a countervailing cost).
# fixing it. An initial 0.015 overshot this: it's a *guaranteed* per-tick
# reward, so it needs to stay below ball_distance_penalty's worst case
# (idling at the arena's far corner), not just "comparable" to it — at
# 0.015 (above ball_distance_penalty's 0.01 ceiling) a Stage-4 run
# converged on sitting pinned upright and farming this instead of chasing
# the ball, cratering goal_rate. Keep this term's episode-long ceiling
# (value * ~1800 ticks) below ball_distance_penalty's worst-case episode
# cost, not just below ball_touch_reward/goal_reward.
@export var grounded_upright_reward := 0.0
# Per-tick bonus for own speed: 0 stationary, full value (+0.24/s) at
# max_speed. Run07 lesson: after the kickoff flurry both ships parked next to
training/checkpoints/20260809-1340-gen5-s4-handling/final.zip
BIN
View File
Binary file not shown.
training/checkpoints/20260809-1955-gen5-s4-handling-retry1/final.zip
BIN
View File
Binary file not shown.
training/checkpoints/20260810-0211-gen5-s4-handling-retry2/final.zip
BIN
View File
Binary file not shown.
training/checkpoints/20260810-1338-gen5-s4-handling-retry3/final.zip
BIN
View File
Binary file not shown.
training/checkpoints/20260811-0858-gen5-s4-handling-retry4/final.zip
BIN
View File
Binary file not shown.
training/generation5.py
+18 -1
View File
@@ -58,6 +58,23 @@ STANDING_ARGS = ["--ent-coef", "0.01", "--entropy-floor"]
# reward is the positive counterpart: a bonus for genuine floor contact
# (not just low altitude) while upright, so grounding well is the locally
# profitable choice rather than merely the least-punished one.
#
# Round 2 (2026-08-11): grounded_upright_reward at 0.015 overshot. Four
# force-retries pushed upright_fraction from 0.265 to a plateauing 0.331,
# then the fifth jumped it to 0.696 (55% over the 0.45 floor) while
# goal_rate collapsed 0.542->0.366 and forward_motion_fraction fell
# 0.244->0.184 — the ship learned to sit pinned upright on the floor
# (vertical_thrust_mean went negative) and farm the bonus instead of
# playing. Root cause: 0.015/tick was actually *larger* than
# ball_distance_penalty's worst case (0.01/tick), so idling near the ball
# beat chasing it — not "comparable to time_penalty/ball_distance_penalty"
# as originally sized. Cut to 0.004/tick (a full grounded episode now caps
# at ~7.2, versus ball_distance_penalty's worst-case ~18 and a single goal's
# 80) — enough to stop "avoid the floor" without being worth farming over
# actually playing. Resets from the Stage-3 foundation again rather than
# continuing from the farming checkpoint, same reasoning as the ground_tilt/
# non_forward_penalty retune: don't resume a policy shaped by one reward
# balance into a meaningfully different one.
HANDLING_REWARD_FLAGS = [
"--velocity-to-ball-weight", "0.04",
"--forward-velocity-to-ball-weight", "0.06",
@@ -69,7 +86,7 @@ HANDLING_REWARD_FLAGS = [
"--tilt-penalty", "0.0002",
"--ground-tilt-penalty", "0.05",
"--non-forward-penalty", "0.04",
"--grounded-upright-reward", "0.015",
"--grounded-upright-reward", "0.004",
]
STAGES = [
training/generation5_state.json
+3 -428
View File
@@ -1,431 +1,6 @@
{
"stage_index": 0,
"attempt": 4,
"status": "blocked",
"log": [
{
"stage_index": 0,
"stage_number": 4,
"stage_name": "handling",
"experiment": "20260809-1340-gen5-s4-handling",
"attempt": 0,
"telemetry_tail": {
"rollout/air_touch_fraction": 0.00035999999195337296,
"rollout/airborne_fraction": 0.44166228532791135,
"rollout/ep_len_mean": 138.77144021606446,
"rollout/ep_rew_mean": -5.834176032543183,
"rollout/forward_motion_fraction": 0.21531347577273846,
"rollout/goal_rate": 0.5669200012683868,
"rollout/mean_altitude": 4.419930323600769,
"rollout/productive_air_touch_fraction": 3.999999910593033e-05,
"rollout/upright_fraction": 0.2645233328640461,
"rollout/vertical_thrust_mean": 0.09886699930951
},
"telemetry_failures": [
"rollout/goal_rate=0.5669 < 0.8000",
"rollout/upright_fraction=0.2645 < 0.4500",
"rollout/forward_motion_fraction=0.2153 < 0.2500"
],
"evaluation_goal_failures": [
"easy.json: goal_rate=0.770 < 0.800"
],
"side_balance_failures": [],
"eval": {
"timestamp": "2026-08-09T18:55:13+00:00",
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260809-1340-gen5-s4-handling.json",
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
"seed": 1,
"episodes": 100,
"wins_a": 46,
"wins_b": 31,
"draws": 23,
"side_results": {
"a_team_0": {
"wins_a": 21,
"wins_b": 17,
"draws": 12
},
"a_team_1": {
"wins_a": 25,
"wins_b": 14,
"draws": 11
}
},
"physical_team_wins": {
"team_0": 35,
"team_1": 42
},
"win_rate_a": 0.46
},
"evals": [
{
"timestamp": "2026-08-09T18:55:13+00:00",
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260809-1340-gen5-s4-handling.json",
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
"seed": 1,
"episodes": 100,
"wins_a": 46,
"wins_b": 31,
"draws": 23,
"side_results": {
"a_team_0": {
"wins_a": 21,
"wins_b": 17,
"draws": 12
},
"a_team_1": {
"wins_a": 25,
"wins_b": 14,
"draws": 11
}
},
"physical_team_wins": {
"team_0": 35,
"team_1": 42
},
"win_rate_a": 0.46
}
],
"decision": "fail"
},
{
"stage_index": 0,
"stage_number": 4,
"stage_name": "handling",
"experiment": "20260809-1955-gen5-s4-handling-retry1",
"attempt": 1,
"telemetry_tail": {
"rollout/air_touch_fraction": 0.00019999999552965163,
"rollout/airborne_fraction": 0.45090347522497176,
"rollout/ep_len_mean": 145.11324035644532,
"rollout/ep_rew_mean": -1.5461844632700086,
"rollout/forward_motion_fraction": 0.21509128576517106,
"rollout/goal_rate": 0.5097999980449677,
"rollout/mean_altitude": 4.43802117395401,
"rollout/productive_air_touch_fraction": 9.999999776482581e-05,
"rollout/upright_fraction": 0.3002223813533783,
"rollout/vertical_thrust_mean": 0.047706998658077794
},
"telemetry_failures": [
"rollout/goal_rate=0.5098 < 0.8000",
"rollout/upright_fraction=0.3002 < 0.4500",
"rollout/forward_motion_fraction=0.2151 < 0.2500"
],
"evaluation_goal_failures": [
"easy.json: goal_rate=0.690 < 0.800"
],
"side_balance_failures": [],
"eval": {
"timestamp": "2026-08-10T01:11:41+00:00",
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260809-1955-gen5-s4-handling-retry1.json",
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
"seed": 1,
"episodes": 100,
"wins_a": 42,
"wins_b": 27,
"draws": 31,
"side_results": {
"a_team_0": {
"wins_a": 17,
"wins_b": 17,
"draws": 16
},
"a_team_1": {
"wins_a": 25,
"wins_b": 10,
"draws": 15
}
},
"physical_team_wins": {
"team_0": 27,
"team_1": 42
},
"win_rate_a": 0.42
},
"evals": [
{
"timestamp": "2026-08-10T01:11:41+00:00",
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260809-1955-gen5-s4-handling-retry1.json",
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
"seed": 1,
"episodes": 100,
"wins_a": 42,
"wins_b": 27,
"draws": 31,
"side_results": {
"a_team_0": {
"wins_a": 17,
"wins_b": 17,
"draws": 16
},
"a_team_1": {
"wins_a": 25,
"wins_b": 10,
"draws": 15
}
},
"physical_team_wins": {
"team_0": 27,
"team_1": 42
},
"win_rate_a": 0.42
}
],
"decision": "fail"
},
{
"stage_index": 0,
"stage_number": 4,
"stage_name": "handling",
"experiment": "20260810-0211-gen5-s4-handling-retry2",
"attempt": 2,
"telemetry_tail": {
"rollout/air_touch_fraction": 0.00013999999687075616,
"rollout/airborne_fraction": 0.4427933331727982,
"rollout/ep_len_mean": 141.45631986999513,
"rollout/ep_rew_mean": -0.7604860214497894,
"rollout/forward_motion_fraction": 0.22091919091343878,
"rollout/goal_rate": 0.542920001089573,
"rollout/mean_altitude": 4.399289563179016,
"rollout/productive_air_touch_fraction": 3.999999910593033e-05,
"rollout/upright_fraction": 0.31741619047522546,
"rollout/vertical_thrust_mean": 0.07430099904080387
},
"telemetry_failures": [
"rollout/goal_rate=0.5429 < 0.8000",
"rollout/upright_fraction=0.3174 < 0.4500",
"rollout/forward_motion_fraction=0.2209 < 0.2500"
],
"evaluation_goal_failures": [
"easy.json: goal_rate=0.730 < 0.800"
],
"side_balance_failures": [],
"eval": {
"timestamp": "2026-08-10T07:22:52+00:00",
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260810-0211-gen5-s4-handling-retry2.json",
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
"seed": 1,
"episodes": 100,
"wins_a": 39,
"wins_b": 34,
"draws": 27,
"side_results": {
"a_team_0": {
"wins_a": 17,
"wins_b": 17,
"draws": 16
},
"a_team_1": {
"wins_a": 22,
"wins_b": 17,
"draws": 11
}
},
"physical_team_wins": {
"team_0": 34,
"team_1": 39
},
"win_rate_a": 0.39
},
"evals": [
{
"timestamp": "2026-08-10T07:22:52+00:00",
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260810-0211-gen5-s4-handling-retry2.json",
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
"seed": 1,
"episodes": 100,
"wins_a": 39,
"wins_b": 34,
"draws": 27,
"side_results": {
"a_team_0": {
"wins_a": 17,
"wins_b": 17,
"draws": 16
},
"a_team_1": {
"wins_a": 22,
"wins_b": 17,
"draws": 11
}
},
"physical_team_wins": {
"team_0": 34,
"team_1": 39
},
"win_rate_a": 0.39
}
],
"decision": "fail"
},
{
"stage_index": 0,
"stage_number": 4,
"stage_name": "handling",
"experiment": "20260810-1338-gen5-s4-handling-retry3",
"attempt": 3,
"telemetry_tail": {
"rollout/air_touch_fraction": 0.00011999999731779099,
"rollout/airborne_fraction": 0.4388641426563263,
"rollout/ep_len_mean": 141.27784002685547,
"rollout/ep_rew_mean": -0.6330312772742472,
"rollout/forward_motion_fraction": 0.24376914313435555,
"rollout/goal_rate": 0.5422000017166138,
"rollout/mean_altitude": 4.358656258583069,
"rollout/productive_air_touch_fraction": 0.0,
"rollout/upright_fraction": 0.3307955712378025,
"rollout/vertical_thrust_mean": 0.017282998460088857
},
"telemetry_failures": [
"rollout/goal_rate=0.5422 < 0.8000",
"rollout/upright_fraction=0.3308 < 0.4500",
"rollout/forward_motion_fraction=0.2438 < 0.2500"
],
"evaluation_goal_failures": [
"easy.json: goal_rate=0.760 < 0.800"
],
"side_balance_failures": [],
"eval": {
"timestamp": "2026-08-10T18:52:02+00:00",
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260810-1338-gen5-s4-handling-retry3.json",
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
"seed": 1,
"episodes": 100,
"wins_a": 41,
"wins_b": 35,
"draws": 24,
"side_results": {
"a_team_0": {
"wins_a": 20,
"wins_b": 21,
"draws": 9
},
"a_team_1": {
"wins_a": 21,
"wins_b": 14,
"draws": 15
}
},
"physical_team_wins": {
"team_0": 34,
"team_1": 42
},
"win_rate_a": 0.41
},
"evals": [
{
"timestamp": "2026-08-10T18:52:02+00:00",
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260810-1338-gen5-s4-handling-retry3.json",
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
"seed": 1,
"episodes": 100,
"wins_a": 41,
"wins_b": 35,
"draws": 24,
"side_results": {
"a_team_0": {
"wins_a": 20,
"wins_b": 21,
"draws": 9
},
"a_team_1": {
"wins_a": 21,
"wins_b": 14,
"draws": 15
}
},
"physical_team_wins": {
"team_0": 34,
"team_1": 42
},
"win_rate_a": 0.41
}
],
"decision": "fail"
},
{
"stage_index": 0,
"stage_number": 4,
"stage_name": "handling",
"experiment": "20260811-0858-gen5-s4-handling-retry4",
"attempt": 4,
"telemetry_tail": {
"rollout/air_touch_fraction": 0.0002199999950826168,
"rollout/airborne_fraction": 0.39690066695213316,
"rollout/ep_len_mean": 157.15703988647462,
"rollout/ep_rew_mean": 12.389891688346863,
"rollout/forward_motion_fraction": 0.18413566660881042,
"rollout/goal_rate": 0.36632000091671946,
"rollout/mean_altitude": 4.058596196651458,
"rollout/productive_air_touch_fraction": 9.999999776482581e-05,
"rollout/upright_fraction": 0.6957313330769539,
"rollout/vertical_thrust_mean": -0.25598900008201597
},
"telemetry_failures": [
"rollout/goal_rate=0.3663 < 0.8000",
"rollout/forward_motion_fraction=0.1841 < 0.2500"
],
"evaluation_goal_failures": [
"easy.json: goal_rate=0.650 < 0.800"
],
"side_balance_failures": [],
"eval": {
"timestamp": "2026-08-11T14:12:24+00:00",
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260811-0858-gen5-s4-handling-retry4.json",
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
"seed": 1,
"episodes": 100,
"wins_a": 32,
"wins_b": 33,
"draws": 35,
"side_results": {
"a_team_0": {
"wins_a": 15,
"wins_b": 18,
"draws": 17
},
"a_team_1": {
"wins_a": 17,
"wins_b": 15,
"draws": 18
}
},
"physical_team_wins": {
"team_0": 30,
"team_1": 35
},
"win_rate_a": 0.32
},
"evals": [
{
"timestamp": "2026-08-11T14:12:24+00:00",
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260811-0858-gen5-s4-handling-retry4.json",
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
"seed": 1,
"episodes": 100,
"wins_a": 32,
"wins_b": 33,
"draws": 35,
"side_results": {
"a_team_0": {
"wins_a": 15,
"wins_b": 18,
"draws": 17
},
"a_team_1": {
"wins_a": 17,
"wins_b": 15,
"draws": 18
}
},
"physical_team_wins": {
"team_0": 30,
"team_1": 35
},
"win_rate_a": 0.32
}
],
"decision": "fail"
}
]
"attempt": 0,
"status": "in_progress",
"log": []
}
training/logs/20260809-1340-gen5-s4-handling_0/events.out.tfevents.1786279241.ai.958630.0
BIN
View File
Binary file not shown.
training/logs/20260809-1955-gen5-s4-handling-retry1_0/events.out.tfevents.1786301738.ai.1124192.0
BIN
View File
Binary file not shown.
training/logs/20260810-0211-gen5-s4-handling-retry2_0/events.out.tfevents.1786324326.ai.1289673.0
BIN
View File
Binary file not shown.
training/logs/20260810-1338-gen5-s4-handling-retry3_0/events.out.tfevents.1786365512.ai.1593324.0
BIN
View File
Binary file not shown.
training/logs/20260811-0858-gen5-s4-handling-retry4_0/events.out.tfevents.1786435152.ai.2105380.0
BIN
View File
Binary file not shown.