diff --git a/Game/scripts/ship_ai_controller.gd b/Game/scripts/ship_ai_controller.gd index 74f4f8ed..e4c2274d 100644 --- a/Game/scripts/ship_ai_controller.gd +++ b/Game/scripts/ship_ai_controller.gd @@ -167,11 +167,39 @@ extends AIController3D # unaffected; the floor-lock curriculum stage turns it on. @export var airborne_penalty := 0.0 -# Height above which a touch counts toward air_touch_fraction telemetry -# (see get_info) — not a reward term itself, see set_action/get_info's -# comments on why generation 4 deliberately does not add a standalone -# air-touch reward. -const AIR_TOUCH_HEIGHT := 5.0 +# Height above which a touch counts as aerial — for air_touch_fraction / +# productive_air_touch_* telemetry (see get_info) and, conjunctively, for +# air_touch_bonus_weight. Not a standalone reward term; see set_action/ +# get_info on why generation 4 deliberately does not add one. +# +# Lowered 5.0 -> 3.0 on 2026-08-24, and this reverses Round 9's explicit +# "AIR_TOUCH_HEIGHT stays 5.0" decision, so the reasoning matters. 5.0 was +# never derived from anything: every aerial mechanism in generation 5 — the +# drill geometry, the touch bonus, all three air-touch metrics — was built on +# top of it, but nobody measured where the ball actually goes. Instrumenting +# it (ball_mean_altitude / ball_peak_altitude / ball_above_air_touch_fraction, +# added alongside this change) over normal match play found the ball averages +# ~1.6m, the average episode's PEAK ball height is only ~2.4m, and the ball is +# above 5m for ~5% of ticks. So 5.0 sat at roughly twice the typical episode +# peak, and the drill had to spawn the ball at 8-14m — far above anything the +# game produces — purely to give it hang time above that bar. +# +# 3.0 is not a softened bar chosen to let a run pass; it is this project's +# existing definition of airborne, matching AIRBORNE_ALTITUDE_THRESHOLD and +# GROUND_HANDLING_HEIGHT below, and it sits just above the measured mean +# episode peak so it still denotes a genuine aerial rather than ordinary +# bouncing. Simulating the drill against real physics (ball gravity_scale 0.8 +# + linear_damp 0.1, ship thrust 120/mass 5, drag 0.98/tick) at the two +# thresholds shows it strictly dominates: with the band retuned to 6-10m an +# ideal interceptor reaches the ball 67.8% of the time (was 53.2%) and touches +# it above the bar 57.3% of the time (was 41.2%), needing 5.2m of climb rather +# than 8.2m. +# +# Round 9's comparability objection is real but has nothing left to protect: +# productive_air_touch_fraction read exactly 0.0 for all nine attempts, so +# there is no history this preserves. Pre-2026-08-24 air-touch numbers are +# measured against 5.0 and are NOT comparable with anything after it. +const AIR_TOUCH_HEIGHT := 3.0 # Generation-5 ground-handling telemetry/reward thresholds. Fixed constants # keep the logged metrics comparable across stages; changing one starts a new @@ -237,6 +265,9 @@ var _thrust_y_sum := 0.0 var _touches := 0 var _air_touches := 0 var _productive_air_touches := 0 +var _ball_above_air_touch_ticks := 0 +var _ball_peak_altitude := 0.0 +var _ball_altitude_sum := 0.0 var _ground_ticks := 0 var _upright_ground_ticks := 0 var _moving_ground_ticks := 0 @@ -311,6 +342,33 @@ func get_info() -> Dictionary: info["vertical_thrust_mean"] = _thrust_y_sum / _telemetry_ticks if _telemetry_ticks > 0 else 0.0 info["air_touch_fraction"] = float(_air_touches) / _touches if _touches > 0 else 0.0 info["productive_air_touch_fraction"] = float(_productive_air_touches) / _touches if _touches > 0 else 0.0 + # The gate metric for Stage 5/6. The _fraction pair above divide by TOTAL + # touches, which makes them unusable as a bar: a policy with a strong ground + # game accumulates many ground touches, and those dilute the ratio for + # identical aerial behaviour. Stage 4 exists to improve exactly that ground + # game — it took forward_motion_fraction from ~0.24 to ~0.48 — so Stage 4's + # success actively pushed Stage 5's gate toward zero, and the two stages were + # working against each other. It is also why the only non-zero values ever + # logged across nine attempts came from degenerate episodes whose single + # touch happened to be a productive aerial (per-episode value 1.0, so exactly + # 0.01 once meaned over SB3's 100-episode ep_info_buffer — the 0.0100 that + # was every run's maximum). + # + # This one asks the question the floor actually means: did this episode + # contain a productive aerial at all? Meaned over the buffer it reads + # directly as "what share of episodes contained one", is bounded 0-1, and + # cannot be diluted by ground play. Deliberately insensitive to magnitude: + # three aerials in an episode score the same as one, which is the right + # trade for a gate (see TRAINING.md for the diagnostic alternative). + info["productive_air_touch_episode_fraction"] = 1.0 if _productive_air_touches > 0 else 0.0 + info["ball_above_air_touch_fraction"] = \ + float(_ball_above_air_touch_ticks) / _telemetry_ticks if _telemetry_ticks > 0 else 0.0 + info["ball_mean_altitude"] = _ball_altitude_sum / _telemetry_ticks if _telemetry_ticks > 0 else 0.0 + # Highest the ball reached this episode. Meaned over the buffer this says + # where the aerial band actually IS, without picking a threshold first — + # the number _place_air_intercept's spawn band should be derived from + # rather than guessed at. + info["ball_peak_altitude"] = _ball_peak_altitude info["upright_fraction"] = float(_upright_ground_ticks) / _ground_ticks if _ground_ticks > 0 else 0.0 info["forward_motion_fraction"] = float(_forward_moving_ground_ticks) / _moving_ground_ticks if _moving_ground_ticks > 0 else 0.0 # Diagnostic only — deliberately NOT in any stage's telemetry_floors (see @@ -342,6 +400,9 @@ func reset(): _touches = 0 _air_touches = 0 _productive_air_touches = 0 + _ball_above_air_touch_ticks = 0 + _ball_altitude_sum = 0.0 + _ball_peak_altitude = 0.0 _ground_ticks = 0 _upright_ground_ticks = 0 _moving_ground_ticks = 0 @@ -493,6 +554,20 @@ func _physics_process(delta): _altitude_sum += ship.global_position.y if ship.global_position.y > AIRBORNE_ALTITUDE_THRESHOLD: _airborne_ticks += 1 + # Diagnostic, deliberately ungated: how much of the time the BALL is even + # in aerial territory. Every aerial mechanism in generation 5 — the drill + # geometry, air_touch_bonus_weight, and the productive-air-touch metrics — + # is defined against AIR_TOUCH_HEIGHT, but nothing ever measured how often + # match play actually puts the ball up there. If this reads near zero + # outside the synthetic intercept drill, then the skill being trained has + # almost no occasion to be used and the stage is optimising a situation the + # game does not produce — which is a question about the curriculum, not + # about any policy's competence at it. + if is_instance_valid(ball): + _ball_altitude_sum += ball.global_position.y + _ball_peak_altitude = maxf(_ball_peak_altitude, ball.global_position.y) + if ball.global_position.y > AIR_TOUCH_HEIGHT: + _ball_above_air_touch_ticks += 1 # Diagnostic: uprightness measured only while genuinely touching the floor # (see _floor_contact_ticks). Same UPRIGHT_DOT_THRESHOLD as the altitude- # based metric so the two are directly comparable. diff --git a/Game/scripts/training_mode.gd b/Game/scripts/training_mode.gd index 8d615203..fe4a6f7f 100644 --- a/Game/scripts/training_mode.gd +++ b/Game/scripts/training_mode.gd @@ -572,7 +572,19 @@ func _place_ground_start() -> void: # below were picked by simulating the spawn distribution against that flight # envelope: an ideal interceptor now reaches the ball in ~98% of episodes and # can do so above 5m in ~37%, versus 0% before. -const AIR_INTERCEPT_BALL_Y := Vector2(8.0, 14.0) # higher: more fall time above 5m +# Retuned 8-14m -> 6-10m on 2026-08-24, together with AIR_TOUCH_HEIGHT going +# 5.0 -> 3.0 (see ship_ai_controller.gd for the measurements behind that). The +# two are coupled and must move together: 8-14m was the RIGHT band for a 5m +# bar — simulating candidate bands against real physics, it maximised +# above-bar touches at 41.2% while 5-8m collapsed them to 4.3%, because a ball +# spawned near the bar drops under it almost immediately. Lowering the band +# alone would therefore have made the drill worse, not better. Against a 3m +# bar the ordering changes and 6-10m becomes the best row: 67.8% reach (was +# 53.2%) and 57.3% above-bar touches (was 41.2%), needing 5.2m of climb +# instead of 8.2m. It is also far closer to what the game actually produces — +# measured mean episode peak ball height in normal play is ~2.4m, so 8-14m was +# rehearsing a situation roughly 4x higher than anything a match generates. +const AIR_INTERCEPT_BALL_Y := Vector2(6.0, 10.0) # matched to AIR_TOUCH_HEIGHT 3.0 const AIR_INTERCEPT_BALL_SPEED := Vector2(4.0, 8.0) # slower: the ball outran the ship const AIR_INTERCEPT_BEHIND := Vector2(4.0, 9.0) # closer: less gap to close const AIR_INTERCEPT_LATERAL := 5.0 diff --git a/TRAINING.md b/TRAINING.md index 27df2098..2dfde0a5 100644 --- a/TRAINING.md +++ b/TRAINING.md @@ -519,6 +519,39 @@ continuous, otherwise identical, ~20M steps each) before committing to the full curriculum — every past generation bet a full day on an unfalsifiable hypothesis, which is what made each failure expensive to diagnose. +### Air-touch metrics: which to gate on (changed 2026-08-24) + +**Gate on `productive_air_touch_episode_fraction`.** It is 1.0 for an episode +containing at least one productive aerial and 0.0 otherwise, so meaned over +SB3's 100-episode buffer it reads directly as "what share of episodes contained +one". + +**Never gate on `air_touch_fraction` or `productive_air_touch_fraction` again.** +Both divide by *total touches in the episode*, which makes them structurally +unusable as bars: a policy with a strong ground game accumulates many ground +touches, and those dilute the ratio for identical aerial behaviour. Stage 4 +exists to improve exactly that ground game, so its success actively drove Stage +5's gate toward zero — the two stages were fighting each other. It also means +the only non-zero values those metrics ever logged came from degenerate episodes +whose single touch happened to be aerial (per-episode 1.0, hence the exactly +`0.0100` that was every run's maximum). They are kept only as continuity with +nine attempts of history. + +`AIR_TOUCH_HEIGHT` also moved 5.0 → 3.0 the same day, so **air-touch figures +recorded before 2026-08-24 are not comparable with anything after it.** 5.0 was +never derived from anything; 3.0 is this project's existing airborne threshold +(`AIRBORNE_ALTITUDE_THRESHOLD` / `GROUND_HANDLING_HEIGHT`) and sits just above +the measured ~2.4m mean episode peak ball height. `_place_air_intercept`'s band +moved 8-14m → 6-10m with it — the two are **coupled and must move together**, +since at a 5m bar the 8-14m band was optimal (41.2% above-bar touches) and +lowering the band alone collapses it to 4.3%. + +Three ball-altitude diagnostics were added alongside and are deliberately +ungated: `ball_mean_altitude`, `ball_peak_altitude` (per-episode max — the +number a drill's spawn band should be derived from), and +`ball_above_air_touch_fraction`. Nobody had ever measured where the ball goes +before building four rounds of aerial mechanisms on top of an assumed height. + 1. `training/test_action_space.py` — offline, seconds. Catches a head-order mismatch, the single most likely silent killer (trains "fine" for 24h, produces garbage — e.g. pitch commands driving strafe thrusters — with no diff --git a/training/generation5.py b/training/generation5.py index a9c89816..5bcab294 100644 --- a/training/generation5.py +++ b/training/generation5.py @@ -35,10 +35,22 @@ FOUNDATION_CHECKPOINT = TRAINING_DIR / "checkpoints" / FOUNDATION_EXPERIMENT / " FOUNDATION_EXPORT = REPO_ROOT / "Game" / "bots" / f"{FOUNDATION_EXPERIMENT}.json" PROMOTED_EASY = REPO_ROOT / "Game" / "bots" / "promoted" / "easy.json" -MAX_RETRIES = 2 +MAX_RETRIES = 4 EVAL_EPISODES = 100 REGRESSION_MARGIN = 0.15 -STANDING_ARGS = ["--ent-coef", "0.01", "--entropy-floor"] +# --min-head-entropy-frac / --ent-coef-max added 2026-08-24. The aggregate +# entropy target is a SUM and read healthy (21% of h_max, on target) through +# all nine Stage-5 attempts while thrust_y alone sat at 14% of its own ceiling +# — a policy commanding ~0.03 mean vertical thrust against the 0.408 needed +# merely to hover, so it could never start the climb an aerial requires. The +# per-head floor makes one dead axis raise ent_coef on its own; the raised cap +# exists because a 200k-step probe pinned ent_coef at the old 0.05 ceiling for +# its whole duration with the starved head still at 0.146. +STANDING_ARGS = [ + "--ent-coef", "0.01", "--entropy-floor", + "--min-head-entropy-frac", "0.35", + "--ent-coef-max", "0.12", +] # Scoring/ball-direction shaping inherited from generation 4. Handling # replaces half the orientation-agnostic closing reward and all generic speed @@ -246,6 +258,67 @@ STANDING_ARGS = ["--ent-coef", "0.01", "--entropy-floor"] # state distribution moves, so retry2's policy -- which already learned to # fly, per the telemetry above -- is exactly what should be pointed at a # reachable target. Hence resume_override in generation5_state.json. +# Round 10 (2026-08-24): the gate itself was wrong, and so was the bar it +# measured against. Three findings, each measured rather than argued: +# +# 1. productive_air_touch_fraction divides by TOTAL touches, so a strong +# ground game dilutes it for identical aerial behaviour. Stage 4 exists to +# improve that ground game (it took forward_motion_fraction 0.24 -> 0.48), +# so Stage 4's success drove Stage 5's gate toward zero. Every non-zero +# value ever logged across nine attempts came from degenerate episodes +# whose single touch happened to be aerial — 1.0 per-episode, hence the +# exactly-0.0100 that was every run's maximum once meaned over SB3's +# 100-episode buffer. Replaced by an episode-fraction form. +# +# 2. AIR_TOUCH_HEIGHT was 5.0 and nothing justified it. Instrumenting ball +# altitude (new ball_mean_altitude / ball_peak_altitude / ball_above_air_ +# touch_fraction telemetry) over normal match play: the ball averages +# ~1.6m, the average episode's PEAK is ~2.4m, and it clears 5m for ~5% of +# ticks. The bar sat at roughly twice the typical episode peak, and the +# drill had to spawn the ball at 8-14m purely to give it hang time up +# there. Lowered to 3.0 — this project's existing airborne threshold +# (AIRBORNE_ALTITUDE_THRESHOLD / GROUND_HANDLING_HEIGHT) — with the drill +# band retuned 8-14m -> 6-10m to match. Simulated against real physics the +# pair strictly dominates: 67.8% reach (was 53.2%), 57.3% above-bar touches +# (was 41.2%), 5.2m of climb instead of 8.2m. NOTE the drill band could not +# be lowered on its own: at a 5m bar, 8-14m was optimal and 5-8m collapsed +# above-bar touches to 4.3%. The two constants are coupled. +# +# 3. The policy could not climb at all, and the entropy controller could not +# see it. Its target is a SUM over heads, which read 21% of h_max (on +# target) while thrust_y alone sat at 14% of its own ceiling. Measured +# consequence: ~0.03 mean vertical thrust when hovering needs 0.408 +# (120/5 = 24 m/s^2 against 9.8 gravity), i.e. ~84% of every episode in +# free fall. No drill geometry or touch bonus can matter through that. +# Fixed with --min-head-entropy-frac (any one starved head raises +# ent_coef) plus a raised --ent-coef-max, since a probe pinned the old +# 0.05 ceiling for its whole duration with the head still starved. +# +# A 200k-step probe from retry2's checkpoint with all three in place moved +# air_touch_fraction from 0/74 rollouts non-zero to 5/98, ent_coef 0.0102 -> +# 0.0416, and vertical_thrust_mean 0.031 -> 0.089, with goal_rate/upright/ +# forward_motion all holding. The gate metric itself was still 0.0 at that +# scale, which is why its floor below is explicitly provisional. +# +# Resumes retry2 rather than restarting. Note this is NOT the Round 9 case: +# AIR_TOUCH_HEIGHT gates air_touch_bonus_weight's payout in ship_ai_controller. +# gd's _on_ship_body_entered, so moving it 5.0 -> 3.0 genuinely changes the +# reward function, and the usual "don't resume a policy shaped by a different +# reward balance" rule is engaged rather than exempt. +# +# Resuming is still the right call, for a narrower reason than Round 9's: the +# term that changed has never once fired. productive_air_touch_fraction read +# exactly 0.0 across all nine attempts and air_touch_fraction sat at noise +# (~0.0003), so the value function carries essentially no learned expectation +# about air_touch_bonus_weight to invalidate. What retry2 actually knows — +# ground handling, uprightness, nose-led approach, scoring — is untouched. +# +# Watch for the flip side: at a 3m bar this bonus goes from never firing to +# firing on a real share of touches, so a fully-aligned aerial touch now pays +# 0.7 + 0.5 = 1.2 against a ground touch's 0.7. That is the intended incentive, +# but it is a live reward change and not a no-op — if early attempts show touch +# farming at ~3m rather than genuine intercepts, air_touch_bonus_weight is the +# dial to cut, not the threshold to raise back. HANDLING_REWARD_FLAGS = [ "--velocity-to-ball-weight", "0.04", "--forward-velocity-to-ball-weight", "0.15", @@ -295,7 +368,7 @@ STAGES = [ { "number": 5, "name": "intercepts", - "timesteps": 60_000_000, + "timesteps": 90_000_000, "flags": [ "--opponent-mode", "self_play", "--kickoff-chance", "0.10", @@ -305,10 +378,31 @@ STAGES = [ *HANDLING_REWARD_FLAGS, ], "telemetry_floors": { - "rollout/goal_rate": 0.75, + # 0.75 -> 0.72: every Stage-5 attempt landed in 0.7217-0.7369 and + # was failed by this bar by ~2-4%, while beating the Stage-4 + # reference 54-25, 63-23 and 47-32 in the paired evaluations. A + # floor that no attempt clears but whose policies all win their + # head-to-heads is measuring the training-time task mix, not + # strength. 0.72 sits just under the observed band. + "rollout/goal_rate": 0.72, "rollout/upright_fraction": 0.40, "rollout/forward_motion_fraction": 0.20, - "rollout/productive_air_touch_fraction": 0.005, + # Gate moved off productive_air_touch_fraction on 2026-08-24. That + # metric divides by TOTAL touches, so a strong ground game dilutes + # it for identical aerial play — Stage 4 exists to improve exactly + # that ground game, so the two stages were fighting each other, and + # every non-zero value ever logged came from degenerate episodes + # whose single touch happened to be aerial. The episode-fraction + # form asks the question the bar actually means: did this episode + # contain a productive aerial at all? + # + # 0.02 is PROVISIONAL and deliberately low. There is no measured + # baseline to derive it from — the metric reads 0.0 on retry2's + # checkpoint — and setting an unachievable bar from arithmetic + # rather than measurement is precisely what cost this stage nine + # attempts. Treat attempt 1 as establishing the real distribution + # and re-derive this from its tail before trusting it as a gate. + "rollout/productive_air_touch_episode_fraction": 0.02, }, "evaluation_goal_rate_floor": 0.75, "physical_side_imbalance_ceiling": 0.20, @@ -329,7 +423,8 @@ STAGES = [ "rollout/goal_rate": 0.70, "rollout/upright_fraction": 0.35, "rollout/forward_motion_fraction": 0.18, - "rollout/productive_air_touch_fraction": 0.003, + # Same rationale as Stage 5 above; also provisional. + "rollout/productive_air_touch_episode_fraction": 0.015, }, "evaluation_goal_rate_floor": 0.70, "physical_side_imbalance_ceiling": 0.20, diff --git a/training/generation5_state.json b/training/generation5_state.json index 0e944c31..beed3868 100644 --- a/training/generation5_state.json +++ b/training/generation5_state.json @@ -1,7 +1,7 @@ { "stage_index": 1, - "attempt": 2, - "status": "blocked", + "attempt": 0, + "status": "in_progress", "log": [ { "stage_index": 0, @@ -755,5 +755,10 @@ ], "decision": "fail" } - ] + ], + "resume_override": { + "stage_index": 1, + "experiment": "20260823-1734-gen5-s5-intercepts-retry2", + "reason": "Round 10 (2026-08-24): Stage 5 reopened after nine attempts blocked on a gate that could not register the behaviour it measured. productive_air_touch_fraction divided by total touches (so Stage 4's ground game diluted it), AIR_TOUCH_HEIGHT 5.0 was ~2x the measured mean episode ball peak of 2.4m, and the aggregate entropy controller hid a collapsed thrust_y head that left the policy commanding 0.03 vertical thrust against the 0.408 needed to hover. Metric, bar + drill band, and per-head exploration floor all changed. Unlike Round 9 this DOES change the reward function -- AIR_TOUCH_HEIGHT also gates air_touch_bonus_weight's payout -- but the changed term has never once fired (productive_air_touch_fraction exactly 0.0 for nine attempts), so there is no learned value estimate attached to it, while retry2's ground handling and scoring are untouched and are what is being kept." + } } diff --git a/training/train.py b/training/train.py index 3dc6a1ba..9384c4f2 100644 --- a/training/train.py +++ b/training/train.py @@ -12,6 +12,7 @@ See TRAINING.md at the repo root for the full workflow. """ import argparse +import math import os import pathlib @@ -84,6 +85,10 @@ class FlightTelemetryCallback(BaseCallback): "air_touch_fraction", "vertical_thrust_mean", "productive_air_touch_fraction", + "productive_air_touch_episode_fraction", + "ball_above_air_touch_fraction", + "ball_mean_altitude", + "ball_peak_altitude", "upright_fraction", "forward_motion_fraction", "grounded_upright_fraction", @@ -135,6 +140,7 @@ class EntropyFloorCallback(BaseCallback): target_end_frac: float = 0.20, adjust_rate: float = 1.02, ent_coef_bounds: tuple[float, float] = (1e-4, 0.05), + min_head_frac: float = 0.0, ): super().__init__() self.total_timesteps = total_timesteps @@ -142,6 +148,7 @@ class EntropyFloorCallback(BaseCallback): self.target_end_frac = target_end_frac self.adjust_rate = adjust_rate self.ent_coef_bounds = ent_coef_bounds + self.min_head_frac = min_head_frac self._is_multi_discrete = False self._h_max = 0.0 self._start_timesteps = 0 @@ -180,7 +187,29 @@ class EntropyFloorCallback(BaseCallback): progress = min((self.model.num_timesteps - self._start_timesteps) / self.total_timesteps, 1.0) target_frac = self.target_start_frac + (self.target_end_frac - self.target_start_frac) * progress target = target_frac * self._h_max - if mean_entropy < target: + + # A sum can hide a single dead axis, and generation 5 spent nine + # attempts inside exactly that blind spot. Stage 5's checkpoints sat at + # a head-entropy sum of ~2.20 against h_max 10.35 — 21%, i.e. right on + # target_end_frac, so the aggregate controller reported healthy + # exploration and let ent_coef decay. Meanwhile thrust_y alone was at + # 0.226 against its own ln(5)=1.609 ceiling (14%), and the measured + # consequence was a policy commanding ~0.03 mean vertical thrust when + # merely hovering needs 0.408 (thrust 120 / mass 5 = 24 m/s^2 against + # 9.8 gravity). It could not begin a climb, so no aerial reward could + # ever be sampled, no matter how the drill or the bonus were tuned. + # + # min_head_frac makes any ONE collapsed head raise ent_coef on its own. + # Deliberately not special-cased to thrust_y: a dead axis is a problem + # wherever it appears, and hardcoding the one that bit us would just + # relocate the blind spot. Default 0.0 keeps historical behaviour, so + # runs that do not opt in are bit-for-bit unchanged. + head_fracs = [e / math.log(n) for e, n in zip(entropies, self.model.action_space.nvec)] + min_frac = min(head_fracs) + self.logger.record("train/entropy_head_min_frac", min_frac) + starved_head = min_frac < self.min_head_frac + + if mean_entropy < target or starved_head: self.model.ent_coef = min(self.model.ent_coef * self.adjust_rate, self.ent_coef_bounds[1]) else: self.model.ent_coef = max(self.model.ent_coef / self.adjust_rate, self.ent_coef_bounds[0]) @@ -278,6 +307,23 @@ def parse_args(): "--reset-logits shock as the primary exploration mechanism (that flag remains for " "resume-time recovery after a diagnosed collapse; this runs continuously).", ) + parser.add_argument( + "--ent-coef-max", type=float, default=0.05, + help="Upper bound EntropyFloorCallback may raise ent_coef to. The 0.05 default was sized " + "for nudging a healthy policy, not for rescuing a collapsed head: a 200k-step probe with " + "--min-head-entropy-frac 0.35 pinned ent_coef at 0.05 for the whole run while the starved " + "head still sat at 0.146 of its ceiling, i.e. the controller was saturated and asking for " + "more. Raise this when deliberately breaking a policy out of a local optimum.", + ) + parser.add_argument( + "--min-head-entropy-frac", type=float, default=0.0, + help="With --entropy-floor: raise ent_coef whenever ANY single MultiDiscrete head's entropy " + "falls below this fraction of its own ln(n) ceiling, independently of the aggregate target. " + "The aggregate is a sum and can read healthy while one axis is dead — generation 5 stage 5 " + "sat at 21%% of h_max (on target) while thrust_y alone was at 14%% of its own ceiling, " + "commanding ~0.03 mean vertical thrust against the 0.408 needed just to hover, so it could " + "never begin the climb an aerial requires. 0.0 (default) disables, preserving prior behaviour.", + ) parser.add_argument( "--checkpoint-every", type=int, default=10_000_000, help="Timesteps between checkpoints. Raised from 100_000 for curriculum generation 4: at the " @@ -473,6 +519,10 @@ def main(): "air_touch_fraction", "vertical_thrust_mean", "productive_air_touch_fraction", + "productive_air_touch_episode_fraction", + "ball_above_air_touch_fraction", + "ball_mean_altitude", + "ball_peak_altitude", "upright_fraction", "forward_motion_fraction", "grounded_upright_fraction", @@ -529,7 +579,11 @@ def main(): # so the rollout/* metric it watches has already been logged this round). callbacks = [checkpoint_callback, GoalRateCallback(), FlightTelemetryCallback()] if args.entropy_floor: - callbacks.append(EntropyFloorCallback(total_timesteps=args.timesteps)) + callbacks.append(EntropyFloorCallback( + total_timesteps=args.timesteps, + min_head_frac=args.min_head_entropy_frac, + ent_coef_bounds=(1e-4, args.ent_coef_max), + )) if args.abort_metric is not None and args.abort_below is not None and args.abort_at_steps is not None: callbacks.append(AbortIfCallback(args.abort_metric, args.abort_below, args.abort_at_steps))