jcreek/CosmicClash
1
0
Fork 0
mirror of https://github.com/jcreek/CosmicClash.git synced 2026-09-11 08:23:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(training): restore ground_tilt_penalty, add grounded_upright_fraction diagnostic

Browse Source 
Round 4 changed three things at once and two of them cut upright pressure:
grounded_upright_reward went to 0 and ground_tilt_penalty was cut 2.5x,
while the new uprightness multiplier only pays below GROUND_HANDLING_HEIGHT
*and* while moving forward *and* facing the ball - a far narrower slice of
ticks than the penalty it was meant to replace. Net pressure fell and
upright_fraction fell with it (0.268 -> 0.239 -> 0.238, the lowest of any
round). Restore ground_tilt_penalty to 0.05 and change nothing else, so
this is a genuine single-variable test of multiplier plus full tilt
pressure.

The conjunctive mechanism itself held up: forward_motion_fraction reached
its best sustained value (0.242) without goal_rate sagging, ep_rew_mean
turned positive for the first time (+0.28), and eval win rate hit 49% with
no reward hacking.

Also adds grounded_upright_fraction: a diagnostic, deliberately ungated
metric measuring uprightness over real floor-contact ticks instead of
sub-3m ticks. upright_fraction has never exceeded 0.331 across four rounds
and ~560M steps without cheating, and its denominator is dominated by
ballistic transit (airborne_fraction ~0.45, mean_altitude ~4.4m) where
attitude is not meaningfully controllable - so it likely cannot measure
what the 0.45 floor was meant to capture. Re-baseline that floor from what
this reports rather than from another round of reshaping.
This commit is contained in:
Josh Creek
2026-08-14 19:36:45 +01:00
parent 7da5b8316f
commit 8f7f672a15
13 changed files with 68 additions and 269 deletions
Download Patch File Download Diff File Expand all files Collapse all files
training/checkpoints/20260812-1307-gen5-s4-handling/final.zip
BIN
View File
Binary file not shown.
training/checkpoints/20260812-1922-gen5-s4-handling-retry1/final.zip
BIN
View File
Binary file not shown.
training/checkpoints/20260813-0134-gen5-s4-handling-retry2/final.zip
BIN
View File
Binary file not shown.
training/generation5.py
+28 -6
View File
@@ -95,11 +95,33 @@ STANDING_ARGS = ["--ent-coef", "0.01", "--entropy-floor"]
# forward-velocity-to-ball rises 0.06 -> 0.15 because multiplying by
# uprightness cuts that term's expected per-tick value roughly 2-3x at
# current behaviour; without the raise the approach incentive would quietly
# weaken. ground-tilt-penalty drops 0.05 -> 0.02: it was the term most
# likely to teach floor-avoidance, and with uprightness now positively paid
# during play it only needs to remain a guardrail against travelling on the
# roof rather than carry the whole posture signal. non-forward-penalty is
# unchanged at 0.04 — it targets a specific behaviour and has not misfired.
# weaken. non-forward-penalty is unchanged at 0.04 — it targets a specific
# behaviour and has not misfired.
#
# Round 5 (2026-08-14): round 4 also dropped ground-tilt-penalty 0.05 ->
# 0.02 on the theory that the multiplier could carry posture on its own.
# That confounded the experiment — two of the three changes *reduced*
# upright pressure at once (grounded_upright_reward to 0, tilt penalty cut
# 2.5x) while the multiplier only pays below GROUND_HANDLING_HEIGHT *and*
# while moving forward *and* facing the ball, i.e. a far narrower slice of
# ticks than the penalty it replaced. Net pressure fell and so did
# upright_fraction (0.268 -> 0.239 -> 0.238, the lowest of any round). The
# conjunctive part worked though: forward_motion_fraction reached its best
# sustained value (0.242) *without* goal_rate sagging, ep_rew_mean turned
# positive for the first time (+0.28), and the eval win rate hit 49% with
# no reward hacking. So round 5 restores ground-tilt-penalty to 0.05 and
# changes nothing else — a genuine single-variable test of multiplier plus
# full tilt pressure.
#
# Also added this round: grounded_upright_fraction, a *diagnostic, ungated*
# telemetry signal measuring uprightness over real floor-contact ticks
# rather than sub-3m ticks. upright_fraction has never exceeded 0.331
# across four rounds and ~560M steps without the policy cheating, and its
# denominator is dominated by ballistic transit (airborne_fraction ~0.45,
# mean_altitude ~4.4m) where attitude is not meaningfully controllable —
# so it likely cannot measure the behaviour the 0.45 floor was meant to
# capture. Re-baseline that floor from what the new signal reports rather
# than from another round of reshaping.
HANDLING_REWARD_FLAGS = [
"--velocity-to-ball-weight", "0.04",
"--forward-velocity-to-ball-weight", "0.15",
@@ -109,7 +131,7 @@ HANDLING_REWARD_FLAGS = [
"--goal-reward", "80",
"--speed-reward-weight", "0.0",
"--tilt-penalty", "0.0002",
"--ground-tilt-penalty", "0.02",
"--ground-tilt-penalty", "0.05",
"--non-forward-penalty", "0.04",
"--grounded-upright-reward", "0.0",
]
training/generation5_state.json
+3 -259
View File
@@ -1,262 +1,6 @@
{
"stage_index": 0,
"attempt": 2,
"status": "blocked",
"log": [
{
"stage_index": 0,
"stage_number": 4,
"stage_name": "handling",
"experiment": "20260812-1307-gen5-s4-handling",
"attempt": 0,
"telemetry_tail": {
"rollout/air_touch_fraction": 0.0004399999901652336,
"rollout/airborne_fraction": 0.4274553818106651,
"rollout/ep_len_mean": 138.04987992858887,
"rollout/ep_rew_mean": -3.075924689769745,
"rollout/forward_motion_fraction": 0.22564161923527717,
"rollout/goal_rate": 0.5803200021386147,
"rollout/mean_altitude": 4.297756896972656,
"rollout/productive_air_touch_fraction": 7.999999821186065e-05,
"rollout/upright_fraction": 0.2680214757621288,
"rollout/vertical_thrust_mean": 0.08520299872543663
},
"telemetry_failures": [
"rollout/goal_rate=0.5803 < 0.8000",
"rollout/upright_fraction=0.2680 < 0.4500",
"rollout/forward_motion_fraction=0.2256 < 0.2500"
],
"evaluation_goal_failures": [
"easy.json: goal_rate=0.780 < 0.800"
],
"side_balance_failures": [],
"eval": {
"timestamp": "2026-08-12T18:21:58+00:00",
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260812-1307-gen5-s4-handling.json",
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
"seed": 1,
"episodes": 100,
"wins_a": 47,
"wins_b": 31,
"draws": 22,
"side_results": {
"a_team_0": {
"wins_a": 21,
"wins_b": 18,
"draws": 11
},
"a_team_1": {
"wins_a": 26,
"wins_b": 13,
"draws": 11
}
},
"physical_team_wins": {
"team_0": 34,
"team_1": 44
},
"win_rate_a": 0.47
},
"evals": [
{
"timestamp": "2026-08-12T18:21:58+00:00",
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260812-1307-gen5-s4-handling.json",
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
"seed": 1,
"episodes": 100,
"wins_a": 47,
"wins_b": 31,
"draws": 22,
"side_results": {
"a_team_0": {
"wins_a": 21,
"wins_b": 18,
"draws": 11
},
"a_team_1": {
"wins_a": 26,
"wins_b": 13,
"draws": 11
}
},
"physical_team_wins": {
"team_0": 34,
"team_1": 44
},
"win_rate_a": 0.47
}
],
"decision": "fail"
},
{
"stage_index": 0,
"stage_number": 4,
"stage_name": "handling",
"experiment": "20260812-1922-gen5-s4-handling-retry1",
"attempt": 1,
"telemetry_tail": {
"rollout/air_touch_fraction": 0.00025999999418854715,
"rollout/airborne_fraction": 0.43636737924814223,
"rollout/ep_len_mean": 141.96904039001464,
"rollout/ep_rew_mean": -2.6134134426116944,
"rollout/forward_motion_fraction": 0.22919738078117372,
"rollout/goal_rate": 0.5506400021910668,
"rollout/mean_altitude": 4.357596744537354,
"rollout/productive_air_touch_fraction": 7.999999821186065e-05,
"rollout/upright_fraction": 0.2389067138582468,
"rollout/vertical_thrust_mean": 0.05124999895330984
},
"telemetry_failures": [
"rollout/goal_rate=0.5506 < 0.8000",
"rollout/upright_fraction=0.2389 < 0.4500",
"rollout/forward_motion_fraction=0.2292 < 0.2500"
],
"evaluation_goal_failures": [
"easy.json: goal_rate=0.750 < 0.800"
],
"side_balance_failures": [],
"eval": {
"timestamp": "2026-08-13T00:34:15+00:00",
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260812-1922-gen5-s4-handling-retry1.json",
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
"seed": 1,
"episodes": 100,
"wins_a": 45,
"wins_b": 30,
"draws": 25,
"side_results": {
"a_team_0": {
"wins_a": 22,
"wins_b": 16,
"draws": 12
},
"a_team_1": {
"wins_a": 23,
"wins_b": 14,
"draws": 13
}
},
"physical_team_wins": {
"team_0": 36,
"team_1": 39
},
"win_rate_a": 0.45
},
"evals": [
{
"timestamp": "2026-08-13T00:34:15+00:00",
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260812-1922-gen5-s4-handling-retry1.json",
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
"seed": 1,
"episodes": 100,
"wins_a": 45,
"wins_b": 30,
"draws": 25,
"side_results": {
"a_team_0": {
"wins_a": 22,
"wins_b": 16,
"draws": 12
},
"a_team_1": {
"wins_a": 23,
"wins_b": 14,
"draws": 13
}
},
"physical_team_wins": {
"team_0": 36,
"team_1": 39
},
"win_rate_a": 0.45
}
],
"decision": "fail"
},
{
"stage_index": 0,
"stage_number": 4,
"stage_name": "handling",
"experiment": "20260813-0134-gen5-s4-handling-retry2",
"attempt": 2,
"telemetry_tail": {
"rollout/air_touch_fraction": 0.00011999999731779099,
"rollout/airborne_fraction": 0.45285780996084213,
"rollout/ep_len_mean": 138.3078397369385,
"rollout/ep_rew_mean": 0.2780366077046783,
"rollout/forward_motion_fraction": 0.2423903810083866,
"rollout/goal_rate": 0.5787200031876564,
"rollout/mean_altitude": 4.478011214733124,
"rollout/productive_air_touch_fraction": 3.999999910593033e-05,
"rollout/upright_fraction": 0.2376179048269987,
"rollout/vertical_thrust_mean": 0.03810099822451826
},
"telemetry_failures": [
"rollout/goal_rate=0.5787 < 0.8000",
"rollout/upright_fraction=0.2376 < 0.4500",
"rollout/forward_motion_fraction=0.2424 < 0.2500"
],
"evaluation_goal_failures": [
"easy.json: goal_rate=0.770 < 0.800"
],
"side_balance_failures": [],
"eval": {
"timestamp": "2026-08-13T06:45:59+00:00",
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260813-0134-gen5-s4-handling-retry2.json",
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
"seed": 1,
"episodes": 100,
"wins_a": 49,
"wins_b": 28,
"draws": 23,
"side_results": {
"a_team_0": {
"wins_a": 24,
"wins_b": 17,
"draws": 9
},
"a_team_1": {
"wins_a": 25,
"wins_b": 11,
"draws": 14
}
},
"physical_team_wins": {
"team_0": 35,
"team_1": 42
},
"win_rate_a": 0.49
},
"evals": [
{
"timestamp": "2026-08-13T06:45:59+00:00",
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260813-0134-gen5-s4-handling-retry2.json",
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
"seed": 1,
"episodes": 100,
"wins_a": 49,
"wins_b": 28,
"draws": 23,
"side_results": {
"a_team_0": {
"wins_a": 24,
"wins_b": 17,
"draws": 9
},
"a_team_1": {
"wins_a": 25,
"wins_b": 11,
"draws": 14
}
},
"physical_team_wins": {
"team_0": 35,
"team_1": 42
},
"win_rate_a": 0.49
}
],
"decision": "fail"
}
]
"attempt": 0,
"status": "in_progress",
"log": []
}
training/logs/20260812-1307-gen5-s4-handling_0/events.out.tfevents.1786536482.ai.2856782.0
BIN
View File
Binary file not shown.
training/logs/20260812-1922-gen5-s4-handling-retry1_0/events.out.tfevents.1786558943.ai.3021139.0
BIN
View File
Binary file not shown.
training/logs/20260813-0134-gen5-s4-handling-retry2_0/events.out.tfevents.1786581281.ai.3184839.0
BIN
View File
Binary file not shown.
training/train.py
+10 -1
View File
@@ -69,7 +69,14 @@ class FlightTelemetryCallback(BaseCallback):
24h+ run later, which is what made every past generation's failure mode
expensive to diagnose. Requires VecMonitor(..., info_keywords=(...,
"airborne_fraction", "mean_altitude", "air_touch_fraction",
"vertical_thrust_mean")) — see ShipAIController.get_info."""
"vertical_thrust_mean")) — see ShipAIController.get_info.
grounded_upright_fraction is diagnostic only and deliberately ungated:
upright_fraction's denominator is sub-3m ticks, most of which are
ballistic transit rather than driving, so it understates uprightness
while the ship is actually on the floor. Adding a key here requires
adding it to VecMonitor's info_keywords below too, or the bare
info[key] lookup raises KeyError mid-run."""
_KEYS = (
"airborne_fraction",
@@ -79,6 +86,7 @@ class FlightTelemetryCallback(BaseCallback):
"productive_air_touch_fraction",
"upright_fraction",
"forward_motion_fraction",
"grounded_upright_fraction",
)
def _on_step(self) -> bool:
@@ -448,6 +456,7 @@ def main():
"productive_air_touch_fraction",
"upright_fraction",
"forward_motion_fraction",
"grounded_upright_fraction",
),
)