mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-10 16:04:04 +00:00
fix(training): restore ground_tilt_penalty, add grounded_upright_fraction diagnostic
Round 4 changed three things at once and two of them cut upright pressure: grounded_upright_reward went to 0 and ground_tilt_penalty was cut 2.5x, while the new uprightness multiplier only pays below GROUND_HANDLING_HEIGHT *and* while moving forward *and* facing the ball - a far narrower slice of ticks than the penalty it was meant to replace. Net pressure fell and upright_fraction fell with it (0.268 -> 0.239 -> 0.238, the lowest of any round). Restore ground_tilt_penalty to 0.05 and change nothing else, so this is a genuine single-variable test of multiplier plus full tilt pressure. The conjunctive mechanism itself held up: forward_motion_fraction reached its best sustained value (0.242) without goal_rate sagging, ep_rew_mean turned positive for the first time (+0.28), and eval win rate hit 49% with no reward hacking. Also adds grounded_upright_fraction: a diagnostic, deliberately ungated metric measuring uprightness over real floor-contact ticks instead of sub-3m ticks. upright_fraction has never exceeded 0.331 across four rounds and ~560M steps without cheating, and its denominator is dominated by ballistic transit (airborne_fraction ~0.45, mean_altitude ~4.4m) where attitude is not meaningfully controllable - so it likely cannot measure what the 0.45 floor was meant to capture. Re-baseline that floor from what this reports rather than from another round of reshaping.
This commit is contained in:
Josh Creek
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -204,6 +204,18 @@ var _ground_ticks := 0
|
||||
var _upright_ground_ticks := 0
|
||||
var _moving_ground_ticks := 0
|
||||
var _forward_moving_ground_ticks := 0
|
||||
# Diagnostic (non-gating) counterpart to _ground_ticks/_upright_ground_ticks.
|
||||
# Those use altitude (< GROUND_HANDLING_HEIGHT) as a proxy for "on the
|
||||
# ground", but with airborne_fraction ~0.45 and mean_altitude ~4.4m a large
|
||||
# share of sub-3m ticks are really ballistic transit — climbing, descending,
|
||||
# or tumbling after contact — where attitude is neither controllable nor
|
||||
# meaningful, so upright_fraction systematically understates how upright the
|
||||
# ship is when it is actually driving. These count only ticks with genuine
|
||||
# floor contact, which is the thing "keep the belly on the floor" actually
|
||||
# means. Kept separate from (not a replacement for) upright_fraction so the
|
||||
# gated metric's definition stays comparable across every past stage.
|
||||
var _floor_contact_ticks := 0
|
||||
var _upright_floor_contact_ticks := 0
|
||||
|
||||
|
||||
# Wire up references after the ship is spawned. `attack_goal` is the goal
|
||||
@@ -264,6 +276,12 @@ func get_info() -> Dictionary:
|
||||
info["productive_air_touch_fraction"] = float(_productive_air_touches) / _touches if _touches > 0 else 0.0
|
||||
info["upright_fraction"] = float(_upright_ground_ticks) / _ground_ticks if _ground_ticks > 0 else 0.0
|
||||
info["forward_motion_fraction"] = float(_forward_moving_ground_ticks) / _moving_ground_ticks if _moving_ground_ticks > 0 else 0.0
|
||||
# Diagnostic only — deliberately NOT in any stage's telemetry_floors (see
|
||||
# generation5.py). Unlike the counters above, _floor_contact_ticks can
|
||||
# legitimately be 0 for a whole episode (a policy that never touches down),
|
||||
# so the 0.0 default here is load-bearing, not just defensive.
|
||||
info["grounded_upright_fraction"] = \
|
||||
float(_upright_floor_contact_ticks) / _floor_contact_ticks if _floor_contact_ticks > 0 else 0.0
|
||||
return info
|
||||
|
||||
|
||||
@@ -291,6 +309,8 @@ func reset():
|
||||
_upright_ground_ticks = 0
|
||||
_moving_ground_ticks = 0
|
||||
_forward_moving_ground_ticks = 0
|
||||
_floor_contact_ticks = 0
|
||||
_upright_floor_contact_ticks = 0
|
||||
|
||||
|
||||
func _physics_process(delta):
|
||||
@@ -421,6 +441,13 @@ func _physics_process(delta):
|
||||
_altitude_sum += ship.global_position.y
|
||||
if ship.global_position.y > AIRBORNE_ALTITUDE_THRESHOLD:
|
||||
_airborne_ticks += 1
|
||||
# Diagnostic: uprightness measured only while genuinely touching the floor
|
||||
# (see _floor_contact_ticks). Same UPRIGHT_DOT_THRESHOLD as the altitude-
|
||||
# based metric so the two are directly comparable.
|
||||
if ShipObservations.is_floor_contact(ship):
|
||||
_floor_contact_ticks += 1
|
||||
if ship.global_transform.basis.y.dot(Vector3.UP) >= UPRIGHT_DOT_THRESHOLD:
|
||||
_upright_floor_contact_ticks += 1
|
||||
_thrust_y_sum += rl_controller.action.thrust.y
|
||||
if ship.global_position.y < GROUND_HANDLING_HEIGHT:
|
||||
_ground_ticks += 1
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
+28
-6
@@ -95,11 +95,33 @@ STANDING_ARGS = ["--ent-coef", "0.01", "--entropy-floor"]
|
||||
# forward-velocity-to-ball rises 0.06 -> 0.15 because multiplying by
|
||||
# uprightness cuts that term's expected per-tick value roughly 2-3x at
|
||||
# current behaviour; without the raise the approach incentive would quietly
|
||||
# weaken. ground-tilt-penalty drops 0.05 -> 0.02: it was the term most
|
||||
# likely to teach floor-avoidance, and with uprightness now positively paid
|
||||
# during play it only needs to remain a guardrail against travelling on the
|
||||
# roof rather than carry the whole posture signal. non-forward-penalty is
|
||||
# unchanged at 0.04 — it targets a specific behaviour and has not misfired.
|
||||
# weaken. non-forward-penalty is unchanged at 0.04 — it targets a specific
|
||||
# behaviour and has not misfired.
|
||||
#
|
||||
# Round 5 (2026-08-14): round 4 also dropped ground-tilt-penalty 0.05 ->
|
||||
# 0.02 on the theory that the multiplier could carry posture on its own.
|
||||
# That confounded the experiment — two of the three changes *reduced*
|
||||
# upright pressure at once (grounded_upright_reward to 0, tilt penalty cut
|
||||
# 2.5x) while the multiplier only pays below GROUND_HANDLING_HEIGHT *and*
|
||||
# while moving forward *and* facing the ball, i.e. a far narrower slice of
|
||||
# ticks than the penalty it replaced. Net pressure fell and so did
|
||||
# upright_fraction (0.268 -> 0.239 -> 0.238, the lowest of any round). The
|
||||
# conjunctive part worked though: forward_motion_fraction reached its best
|
||||
# sustained value (0.242) *without* goal_rate sagging, ep_rew_mean turned
|
||||
# positive for the first time (+0.28), and the eval win rate hit 49% with
|
||||
# no reward hacking. So round 5 restores ground-tilt-penalty to 0.05 and
|
||||
# changes nothing else — a genuine single-variable test of multiplier plus
|
||||
# full tilt pressure.
|
||||
#
|
||||
# Also added this round: grounded_upright_fraction, a *diagnostic, ungated*
|
||||
# telemetry signal measuring uprightness over real floor-contact ticks
|
||||
# rather than sub-3m ticks. upright_fraction has never exceeded 0.331
|
||||
# across four rounds and ~560M steps without the policy cheating, and its
|
||||
# denominator is dominated by ballistic transit (airborne_fraction ~0.45,
|
||||
# mean_altitude ~4.4m) where attitude is not meaningfully controllable —
|
||||
# so it likely cannot measure the behaviour the 0.45 floor was meant to
|
||||
# capture. Re-baseline that floor from what the new signal reports rather
|
||||
# than from another round of reshaping.
|
||||
HANDLING_REWARD_FLAGS = [
|
||||
"--velocity-to-ball-weight", "0.04",
|
||||
"--forward-velocity-to-ball-weight", "0.15",
|
||||
@@ -109,7 +131,7 @@ HANDLING_REWARD_FLAGS = [
|
||||
"--goal-reward", "80",
|
||||
"--speed-reward-weight", "0.0",
|
||||
"--tilt-penalty", "0.0002",
|
||||
"--ground-tilt-penalty", "0.02",
|
||||
"--ground-tilt-penalty", "0.05",
|
||||
"--non-forward-penalty", "0.04",
|
||||
"--grounded-upright-reward", "0.0",
|
||||
]
|
||||
|
||||
@@ -1,262 +1,6 @@
|
||||
{
|
||||
"stage_index": 0,
|
||||
"attempt": 2,
|
||||
"status": "blocked",
|
||||
"log": [
|
||||
{
|
||||
"stage_index": 0,
|
||||
"stage_number": 4,
|
||||
"stage_name": "handling",
|
||||
"experiment": "20260812-1307-gen5-s4-handling",
|
||||
"attempt": 0,
|
||||
"telemetry_tail": {
|
||||
"rollout/air_touch_fraction": 0.0004399999901652336,
|
||||
"rollout/airborne_fraction": 0.4274553818106651,
|
||||
"rollout/ep_len_mean": 138.04987992858887,
|
||||
"rollout/ep_rew_mean": -3.075924689769745,
|
||||
"rollout/forward_motion_fraction": 0.22564161923527717,
|
||||
"rollout/goal_rate": 0.5803200021386147,
|
||||
"rollout/mean_altitude": 4.297756896972656,
|
||||
"rollout/productive_air_touch_fraction": 7.999999821186065e-05,
|
||||
"rollout/upright_fraction": 0.2680214757621288,
|
||||
"rollout/vertical_thrust_mean": 0.08520299872543663
|
||||
},
|
||||
"telemetry_failures": [
|
||||
"rollout/goal_rate=0.5803 < 0.8000",
|
||||
"rollout/upright_fraction=0.2680 < 0.4500",
|
||||
"rollout/forward_motion_fraction=0.2256 < 0.2500"
|
||||
],
|
||||
"evaluation_goal_failures": [
|
||||
"easy.json: goal_rate=0.780 < 0.800"
|
||||
],
|
||||
"side_balance_failures": [],
|
||||
"eval": {
|
||||
"timestamp": "2026-08-12T18:21:58+00:00",
|
||||
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260812-1307-gen5-s4-handling.json",
|
||||
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
|
||||
"seed": 1,
|
||||
"episodes": 100,
|
||||
"wins_a": 47,
|
||||
"wins_b": 31,
|
||||
"draws": 22,
|
||||
"side_results": {
|
||||
"a_team_0": {
|
||||
"wins_a": 21,
|
||||
"wins_b": 18,
|
||||
"draws": 11
|
||||
},
|
||||
"a_team_1": {
|
||||
"wins_a": 26,
|
||||
"wins_b": 13,
|
||||
"draws": 11
|
||||
}
|
||||
},
|
||||
"physical_team_wins": {
|
||||
"team_0": 34,
|
||||
"team_1": 44
|
||||
},
|
||||
"win_rate_a": 0.47
|
||||
},
|
||||
"evals": [
|
||||
{
|
||||
"timestamp": "2026-08-12T18:21:58+00:00",
|
||||
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260812-1307-gen5-s4-handling.json",
|
||||
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
|
||||
"seed": 1,
|
||||
"episodes": 100,
|
||||
"wins_a": 47,
|
||||
"wins_b": 31,
|
||||
"draws": 22,
|
||||
"side_results": {
|
||||
"a_team_0": {
|
||||
"wins_a": 21,
|
||||
"wins_b": 18,
|
||||
"draws": 11
|
||||
},
|
||||
"a_team_1": {
|
||||
"wins_a": 26,
|
||||
"wins_b": 13,
|
||||
"draws": 11
|
||||
}
|
||||
},
|
||||
"physical_team_wins": {
|
||||
"team_0": 34,
|
||||
"team_1": 44
|
||||
},
|
||||
"win_rate_a": 0.47
|
||||
}
|
||||
],
|
||||
"decision": "fail"
|
||||
},
|
||||
{
|
||||
"stage_index": 0,
|
||||
"stage_number": 4,
|
||||
"stage_name": "handling",
|
||||
"experiment": "20260812-1922-gen5-s4-handling-retry1",
|
||||
"attempt": 1,
|
||||
"telemetry_tail": {
|
||||
"rollout/air_touch_fraction": 0.00025999999418854715,
|
||||
"rollout/airborne_fraction": 0.43636737924814223,
|
||||
"rollout/ep_len_mean": 141.96904039001464,
|
||||
"rollout/ep_rew_mean": -2.6134134426116944,
|
||||
"rollout/forward_motion_fraction": 0.22919738078117372,
|
||||
"rollout/goal_rate": 0.5506400021910668,
|
||||
"rollout/mean_altitude": 4.357596744537354,
|
||||
"rollout/productive_air_touch_fraction": 7.999999821186065e-05,
|
||||
"rollout/upright_fraction": 0.2389067138582468,
|
||||
"rollout/vertical_thrust_mean": 0.05124999895330984
|
||||
},
|
||||
"telemetry_failures": [
|
||||
"rollout/goal_rate=0.5506 < 0.8000",
|
||||
"rollout/upright_fraction=0.2389 < 0.4500",
|
||||
"rollout/forward_motion_fraction=0.2292 < 0.2500"
|
||||
],
|
||||
"evaluation_goal_failures": [
|
||||
"easy.json: goal_rate=0.750 < 0.800"
|
||||
],
|
||||
"side_balance_failures": [],
|
||||
"eval": {
|
||||
"timestamp": "2026-08-13T00:34:15+00:00",
|
||||
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260812-1922-gen5-s4-handling-retry1.json",
|
||||
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
|
||||
"seed": 1,
|
||||
"episodes": 100,
|
||||
"wins_a": 45,
|
||||
"wins_b": 30,
|
||||
"draws": 25,
|
||||
"side_results": {
|
||||
"a_team_0": {
|
||||
"wins_a": 22,
|
||||
"wins_b": 16,
|
||||
"draws": 12
|
||||
},
|
||||
"a_team_1": {
|
||||
"wins_a": 23,
|
||||
"wins_b": 14,
|
||||
"draws": 13
|
||||
}
|
||||
},
|
||||
"physical_team_wins": {
|
||||
"team_0": 36,
|
||||
"team_1": 39
|
||||
},
|
||||
"win_rate_a": 0.45
|
||||
},
|
||||
"evals": [
|
||||
{
|
||||
"timestamp": "2026-08-13T00:34:15+00:00",
|
||||
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260812-1922-gen5-s4-handling-retry1.json",
|
||||
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
|
||||
"seed": 1,
|
||||
"episodes": 100,
|
||||
"wins_a": 45,
|
||||
"wins_b": 30,
|
||||
"draws": 25,
|
||||
"side_results": {
|
||||
"a_team_0": {
|
||||
"wins_a": 22,
|
||||
"wins_b": 16,
|
||||
"draws": 12
|
||||
},
|
||||
"a_team_1": {
|
||||
"wins_a": 23,
|
||||
"wins_b": 14,
|
||||
"draws": 13
|
||||
}
|
||||
},
|
||||
"physical_team_wins": {
|
||||
"team_0": 36,
|
||||
"team_1": 39
|
||||
},
|
||||
"win_rate_a": 0.45
|
||||
}
|
||||
],
|
||||
"decision": "fail"
|
||||
},
|
||||
{
|
||||
"stage_index": 0,
|
||||
"stage_number": 4,
|
||||
"stage_name": "handling",
|
||||
"experiment": "20260813-0134-gen5-s4-handling-retry2",
|
||||
"attempt": 2,
|
||||
"telemetry_tail": {
|
||||
"rollout/air_touch_fraction": 0.00011999999731779099,
|
||||
"rollout/airborne_fraction": 0.45285780996084213,
|
||||
"rollout/ep_len_mean": 138.3078397369385,
|
||||
"rollout/ep_rew_mean": 0.2780366077046783,
|
||||
"rollout/forward_motion_fraction": 0.2423903810083866,
|
||||
"rollout/goal_rate": 0.5787200031876564,
|
||||
"rollout/mean_altitude": 4.478011214733124,
|
||||
"rollout/productive_air_touch_fraction": 3.999999910593033e-05,
|
||||
"rollout/upright_fraction": 0.2376179048269987,
|
||||
"rollout/vertical_thrust_mean": 0.03810099822451826
|
||||
},
|
||||
"telemetry_failures": [
|
||||
"rollout/goal_rate=0.5787 < 0.8000",
|
||||
"rollout/upright_fraction=0.2376 < 0.4500",
|
||||
"rollout/forward_motion_fraction=0.2424 < 0.2500"
|
||||
],
|
||||
"evaluation_goal_failures": [
|
||||
"easy.json: goal_rate=0.770 < 0.800"
|
||||
],
|
||||
"side_balance_failures": [],
|
||||
"eval": {
|
||||
"timestamp": "2026-08-13T06:45:59+00:00",
|
||||
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260813-0134-gen5-s4-handling-retry2.json",
|
||||
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
|
||||
"seed": 1,
|
||||
"episodes": 100,
|
||||
"wins_a": 49,
|
||||
"wins_b": 28,
|
||||
"draws": 23,
|
||||
"side_results": {
|
||||
"a_team_0": {
|
||||
"wins_a": 24,
|
||||
"wins_b": 17,
|
||||
"draws": 9
|
||||
},
|
||||
"a_team_1": {
|
||||
"wins_a": 25,
|
||||
"wins_b": 11,
|
||||
"draws": 14
|
||||
}
|
||||
},
|
||||
"physical_team_wins": {
|
||||
"team_0": 35,
|
||||
"team_1": 42
|
||||
},
|
||||
"win_rate_a": 0.49
|
||||
},
|
||||
"evals": [
|
||||
{
|
||||
"timestamp": "2026-08-13T06:45:59+00:00",
|
||||
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260813-0134-gen5-s4-handling-retry2.json",
|
||||
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/promoted/easy.json",
|
||||
"seed": 1,
|
||||
"episodes": 100,
|
||||
"wins_a": 49,
|
||||
"wins_b": 28,
|
||||
"draws": 23,
|
||||
"side_results": {
|
||||
"a_team_0": {
|
||||
"wins_a": 24,
|
||||
"wins_b": 17,
|
||||
"draws": 9
|
||||
},
|
||||
"a_team_1": {
|
||||
"wins_a": 25,
|
||||
"wins_b": 11,
|
||||
"draws": 14
|
||||
}
|
||||
},
|
||||
"physical_team_wins": {
|
||||
"team_0": 35,
|
||||
"team_1": 42
|
||||
},
|
||||
"win_rate_a": 0.49
|
||||
}
|
||||
],
|
||||
"decision": "fail"
|
||||
}
|
||||
]
|
||||
"attempt": 0,
|
||||
"status": "in_progress",
|
||||
"log": []
|
||||
}
|
||||
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
+10
-1
@@ -69,7 +69,14 @@ class FlightTelemetryCallback(BaseCallback):
|
||||
24h+ run later, which is what made every past generation's failure mode
|
||||
expensive to diagnose. Requires VecMonitor(..., info_keywords=(...,
|
||||
"airborne_fraction", "mean_altitude", "air_touch_fraction",
|
||||
"vertical_thrust_mean")) — see ShipAIController.get_info."""
|
||||
"vertical_thrust_mean")) — see ShipAIController.get_info.
|
||||
|
||||
grounded_upright_fraction is diagnostic only and deliberately ungated:
|
||||
upright_fraction's denominator is sub-3m ticks, most of which are
|
||||
ballistic transit rather than driving, so it understates uprightness
|
||||
while the ship is actually on the floor. Adding a key here requires
|
||||
adding it to VecMonitor's info_keywords below too, or the bare
|
||||
info[key] lookup raises KeyError mid-run."""
|
||||
|
||||
_KEYS = (
|
||||
"airborne_fraction",
|
||||
@@ -79,6 +86,7 @@ class FlightTelemetryCallback(BaseCallback):
|
||||
"productive_air_touch_fraction",
|
||||
"upright_fraction",
|
||||
"forward_motion_fraction",
|
||||
"grounded_upright_fraction",
|
||||
)
|
||||
|
||||
def _on_step(self) -> bool:
|
||||
@@ -448,6 +456,7 @@ def main():
|
||||
"productive_air_touch_fraction",
|
||||
"upright_fraction",
|
||||
"forward_motion_fraction",
|
||||
"grounded_upright_fraction",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user