mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-10 16:04:04 +00:00
feat(training): Replace all-or-nothing unmask with a gradual ramp
Generation 2's single "unmask" stage (flip vertical/pitch-roll locomotion from grounded-only to full 3D in one step) failed 3 independent 240M-step attempts, landing at a stable 32% / 28% / 31% win rate vs curric-s5-aggression each time -- not noise, and not fixable by more training time (attempts 2-3 each continued the same checkpoint lineage for another full 240M steps with zero improvement). Every attempt shows train/std collapsing from ~0.30 to ~0.13-0.15 within the first ~10% of steps and never recovering: the policy locks the newly-opened axes back down before ever meaningfully exploring them. Replaces the boolean allow_vertical/allow_pitch_roll mask on ShipAIController with float vertical_ramp/pitch_roll_ramp multipliers (0.0-1.0), scaling axis effect in set_action() instead of gating it outright -- the action space never changes shape, so checkpoints stay resumable across ramp values. The single unmask stage in curriculum.py becomes 4: three ungated warmup stages (25%/50%/75% authority, airborne_penalty ramping in step) that train, checkpoint, and always advance with no eval gate, then the measured stage at full authority -- same reference, opponent mode, and 240M budget as the 3 failed attempts, for a direct comparison. Adds a "gated" flag/branch to main()'s loop for the ungated stages. This is generation 3 of the curriculum; generation 2's state is archived to curriculum_state_gen2.json (mirroring the earlier gen1 -> gen2 archival) and curriculum_state.json resets fresh, since its stage 0 no longer means what it used to. See TRAINING.md's "Generation 3" section for the full postmortem, stage table, and the open question about whether scaling action effect in Godot (which PPO's own entropy/exploration math never sees) actually addresses the collapse.
This commit is contained in:
Josh Creek
@@ -79,15 +79,24 @@ extends AIController3D
|
||||
# unaffected; the floor-lock curriculum stage turns it on.
|
||||
@export var airborne_penalty := 0.0
|
||||
|
||||
# Locomotion curriculum: when false, the corresponding action axes are
|
||||
# discarded in set_action before reaching the ship, so the ship stays
|
||||
# grounded and only yaws — basic scoring/defending doesn't need 3D flight.
|
||||
# This masks the *effect* of thrust.y/rotation.x/rotation.z, not the action
|
||||
# space's shape: the policy still outputs values for these axes (still
|
||||
# contributing to PPO's entropy/log-prob), they're just discarded here, so
|
||||
# checkpoints stay resumable once a later curriculum stage re-enables them.
|
||||
@export var allow_vertical := true
|
||||
@export var allow_pitch_roll := true
|
||||
# Locomotion curriculum: scales how much of the corresponding action axes
|
||||
# actually reaches the ship, from 0.0 (fully discarded, grounded-only) to
|
||||
# 1.0 (full effect) — this scales the *effect* of thrust.y/rotation.x/
|
||||
# rotation.z in set_action, not the action space's shape: the policy always
|
||||
# outputs values for these axes (always contributing to PPO's entropy/log-
|
||||
# prob), they're just attenuated here, so checkpoints stay resumable across
|
||||
# ramp values.
|
||||
#
|
||||
# A hard 0/1 flip (the original bool mask) let PPO's action-distribution
|
||||
# std collapse to ~0.13-0.15 within the first ~10% of steps, before the
|
||||
# policy ever meaningfully explored the newly-unmasked axes — 3 independent
|
||||
# 240M-step attempts at the all-or-nothing flip all landed at a stable
|
||||
# ~28-32% win rate vs curric-s5-aggression (see TRAINING.md's generation 3
|
||||
# section). A gradual ramp across several short curriculum stages, each
|
||||
# resuming from the previous ramp value's checkpoint, lets the policy adopt
|
||||
# each axis incrementally instead of all at once.
|
||||
@export_range(0.0, 1.0) var vertical_ramp := 1.0
|
||||
@export_range(0.0, 1.0) var pitch_roll_ramp := 1.0
|
||||
|
||||
# Contact normals with y above this are floor contact (exempt from the wall
|
||||
# penalty); below it they read as wall (sideways) or ceiling (downward).
|
||||
@@ -163,9 +172,9 @@ func get_action_space() -> Dictionary:
|
||||
func set_action(action) -> void:
|
||||
var thrust: Array = action["thrust"]
|
||||
var rot: Array = action["rotation"]
|
||||
var thrust_y: float = thrust[1] if allow_vertical else 0.0
|
||||
var pitch: float = rot[0] if allow_pitch_roll else 0.0
|
||||
var roll: float = rot[2] if allow_pitch_roll else 0.0
|
||||
var thrust_y: float = thrust[1] * vertical_ramp
|
||||
var pitch: float = rot[0] * pitch_roll_ramp
|
||||
var roll: float = rot[2] * pitch_roll_ramp
|
||||
rl_controller.action.thrust = Vector3(thrust[0], thrust_y, thrust[2])
|
||||
rl_controller.action.rotation = Vector3(pitch, rot[1], roll)
|
||||
rl_controller.action.turbo = int(action["turbo"]) == 1
|
||||
|
||||
@@ -188,7 +188,7 @@ const SHIP_AI_OVERRIDES := [
|
||||
"ball_touch_reward", "ball_touch_cooldown_ticks", "ball_touch_direction_floor",
|
||||
"velocity_to_ball_weight", "ball_velocity_to_goal_weight", "ball_distance_penalty",
|
||||
"wall_contact_penalty", "tilt_penalty", "speed_reward_weight", "time_penalty",
|
||||
"airborne_penalty", "allow_vertical", "allow_pitch_roll",
|
||||
"airborne_penalty", "vertical_ramp", "pitch_roll_ramp",
|
||||
]
|
||||
|
||||
|
||||
@@ -239,7 +239,7 @@ func _ai_default(name: String) -> Variant:
|
||||
"speed_reward_weight": return 0.004
|
||||
"time_penalty": return 0.001
|
||||
"airborne_penalty": return 0.0
|
||||
"allow_vertical", "allow_pitch_roll": return true
|
||||
"vertical_ramp", "pitch_roll_ramp": return 1.0
|
||||
_: return null
|
||||
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ The training pipeline is built — see `TRAINING.md` (self-play PPO via the vend
|
||||
|
||||
- [x] Promote a first tier: `curric-s6-unmask` copied into `Game/bots/promoted/easy.json` as the shipped "easy" bot (see TRAINING.md's "Promoted bots" section) — `match.tscn`/`spectate.tscn` now default there instead of `run05.json`.
|
||||
- [ ] Long training runs on the Linux/3090 box to produce actually-good bots; promote further checkpoints into `Game/bots/promoted/` as `medium`/`hard` tiers once they clear `easy.json` in `evaluate.py`.
|
||||
- [x] Staged curriculum (score → defend → avoid draws → full mechanics) via `train.py`'s `--opponent-mode`/`--draw-penalty`/`--attack-goal-bias`/`--allow-vertical`/`--allow-pitch-roll` flags — see TRAINING.md's "Curriculum training" section. `--opponent-mode=frozen` is a single-fixed-model slice of the league idea below, not the full sampled pool.
|
||||
- [x] Staged curriculum (score → defend → avoid draws → full mechanics) via `train.py`'s `--opponent-mode`/`--draw-penalty`/`--attack-goal-bias`/`--vertical-ramp`/`--pitch-roll-ramp` flags — see TRAINING.md's "Curriculum training" section. `--opponent-mode=frozen` is a single-fixed-model slice of the league idea below, not the full sampled pool.
|
||||
- [ ] Frozen-opponent league: train the live policy against a *pool* of past exported checkpoints, sampled per-episode (today's `--opponent-mode=frozen` only supports one fixed model per run) to prevent self-play strategy collapse on long runs.
|
||||
- [ ] Richer state setter / curriculum: aerial states, wall plays, rebound scenarios as skill grows (beyond the score/defend/draw staging already in place).
|
||||
- [ ] Main-menu difficulty picker (Match already takes `bot_model_path`/`bot_reaction_ticks`/`bot_action_noise` exports).
|
||||
|
||||
+70
-8
@@ -185,11 +185,13 @@ movement. Each stage is a normal chained run — a new `--experiment` resumed
|
||||
via `--resume checkpoints/<previous>/final.zip`, same as any other run —
|
||||
just with different curriculum flags.
|
||||
|
||||
`curriculum.py` has run through two generations so far. Generation 1 (below)
|
||||
ran stages 1-6 to completion/block and is archived; generation 2 (the one
|
||||
`curriculum.py` actually runs today) starts a fresh stage 1 seeded from
|
||||
generation 1's last clean pass instead of continuing to retry a stage that
|
||||
kept getting worse — see "Generation 2" below.
|
||||
`curriculum.py` has run through three generations so far. Generation 1
|
||||
(below) ran stages 1-6 to completion/block and is archived; generation 2
|
||||
started a fresh stage 1 seeded from generation 1's last clean pass instead
|
||||
of continuing to retry a stage that kept getting worse, but also failed 3
|
||||
attempts; generation 3 (the one `curriculum.py` actually runs today)
|
||||
replaces generation 2's single all-or-nothing unmask stage with a gradual
|
||||
ramp — see "Generation 3" below.
|
||||
|
||||
### Generation 1 (archived — see `curriculum_state_gen1.json`)
|
||||
|
||||
@@ -224,9 +226,9 @@ vs `curric-s5-aggression`). After 3 failed attempts the script blocked for
|
||||
human review; rather than pile up `retry4`, `retry5`, ... on a lineage that
|
||||
kept getting worse, generation 2 (below) replaces it with a fresh stage 1.
|
||||
|
||||
### Generation 2 (current)
|
||||
### Generation 2 (archived — see `curriculum_state_gen2.json`)
|
||||
|
||||
`curriculum.py`'s live `STAGES` list now contains a single stage, `unmask`
|
||||
`curriculum.py`'s `STAGES` list contained a single stage, `unmask`
|
||||
(displays as stage 1 — `curric-s1-unmask`), which picks up exactly where
|
||||
generation 1's regression analysis left off. It resumes directly from
|
||||
`FOUNDATION_EXPERIMENT` (`curric-s5-aggression`'s own checkpoint — the last
|
||||
@@ -271,12 +273,72 @@ generation 1's plain ones (both checkpoint directories and TensorBoard run
|
||||
names come straight from `--experiment`) and makes run order obvious in
|
||||
TensorBoard without cross-referencing `curriculum_state.json`.
|
||||
|
||||
**Generation 2 also failed 3 attempts in a row**, landing at a stable
|
||||
32% / 28% / 31% win rate vs `curric-s5-aggression` each time — the second
|
||||
and third attempts each continued the *same* checkpoint lineage for another
|
||||
full 240M steps with zero improvement, ruling out both the reward retune
|
||||
above and "just needs more time" as fixes. Every attempt showed `train/std`
|
||||
collapsing from ~0.30 to ~0.13-0.15 within the first ~10% of steps and never
|
||||
recovering. See "Generation 3" below for the redesign this prompted.
|
||||
|
||||
### Generation 3 (current)
|
||||
|
||||
Generation 2's failures point at the *mechanism* of the transition, not the
|
||||
reward weights: flipping `allow_vertical`/`allow_pitch_roll` from false to
|
||||
true in one step let PPO's action-distribution std collapse on those axes
|
||||
before the policy ever meaningfully explored them. Generation 3 replaces
|
||||
that boolean mask with a float ramp (`vertical_ramp`/`pitch_roll_ramp` on
|
||||
`ShipAIController`, 0.0-1.0, multiplying the axis's effect in `set_action`
|
||||
instead of gating it) and spreads the transition across 4 stages instead of
|
||||
1:
|
||||
|
||||
| Stage | `vertical-ramp`/`pitch-roll-ramp` | `airborne-penalty` | timesteps | gated |
|
||||
|---|---|---|---|---|
|
||||
| 1 — `unmask-ramp25` | 0.25 | 0.0 | 40M (~4h) | No — trains, checkpoints, always advances |
|
||||
| 2 — `unmask-ramp50` | 0.5 | 0.001 | 40M (~4h) | No |
|
||||
| 3 — `unmask-ramp75` | 0.75 | 0.002 | 40M (~4h) | No |
|
||||
| 4 — `unmask` | 1.0 | 0.003 | 240M (~24h) | **Yes** — evaluated against `curric-s5-aggression`, same 15-point regression gate as every prior attempt |
|
||||
|
||||
The 3 warmup stages are deliberately ungated: they're waypoints en route to
|
||||
the real, measured transition, not decisions in their own right, so
|
||||
`curriculum.py`'s `main()` loop trains and checkpoints them and always
|
||||
advances (no eval call, no retry logic — there's nothing to fail against).
|
||||
Only the final `unmask` stage is evaluated, with the same reference bot,
|
||||
opponent mode (`self_play`, not `frozen` — kept identical to every prior
|
||||
attempt so a pass or fail cleanly isolates the ramp as the only variable),
|
||||
and 240M-step budget as all 3 failed all-or-nothing attempts, for a direct
|
||||
comparison. `airborne_penalty` ramps in step with the axes so it doesn't
|
||||
fight a still-mostly-inert axis early on.
|
||||
|
||||
All the reward-shaping flags from generation 2's stage (`velocity-to-ball-weight`,
|
||||
`ball-distance-penalty`, `ball-touch-reward`, `ball-velocity-to-goal-weight`,
|
||||
`goal-reward`, `draw-penalty`) are unchanged and identical across all 4
|
||||
stages, so the ramp is the sole studied variable.
|
||||
|
||||
`curriculum_state.json` was reset (generation 2's log archived to
|
||||
`curriculum_state_gen2.json`) rather than continuing to log against a stage
|
||||
list whose stage 0 no longer means what it used to.
|
||||
|
||||
**Open question, not yet resolved by data:** the ramp scales the action's
|
||||
effect in Godot, which runs *after* PPO samples the action — PPO's own
|
||||
std-collapse dynamics don't directly see the ramp, only the reward it
|
||||
produces. It's possible this doesn't prevent the collapse, or even makes it
|
||||
happen faster at low ramp values (weaker reward signal on those axes gives
|
||||
less incentive to keep exploring them). Watch `train/std` per stage in
|
||||
TensorBoard rather than assuming the ramp is working. If the final gated
|
||||
stage still lands ~28-32%, that's evidence the plateau isn't an
|
||||
exploration/collapse problem at all — worth revisiting reward shaping, or
|
||||
trying `--opponent-mode frozen --opponent-model <path>` during the warmup
|
||||
stages (implemented, never yet exercised in this project) to remove
|
||||
self-play's moving-target instability while the policy first learns to use
|
||||
the new axes.
|
||||
|
||||
All curriculum flags default to leaving Godot's own `@export` defaults
|
||||
alone (`train.py` only forwards a flag when you pass it), so ordinary runs
|
||||
are unaffected. Full flag list: `--opponent-mode {self_play,inert,frozen}`,
|
||||
`--opponent-model <path>` (for `frozen`), `--draw-penalty`,
|
||||
`--attack-goal-bias`, `--kickoff-chance`, `--near-goal-chance`,
|
||||
`--allow-vertical`/`--no-allow-vertical`, `--allow-pitch-roll`/`--no-allow-pitch-roll`,
|
||||
`--vertical-ramp`, `--pitch-roll-ramp` (0.0-1.0 locomotion-unmask ramp),
|
||||
`--velocity-to-ball-weight`, `--ball-distance-penalty`, `--ball-touch-reward`,
|
||||
`--airborne-penalty`, `--ball-velocity-to-goal-weight`, `--goal-reward`.
|
||||
|
||||
|
||||
+129
-65
@@ -18,19 +18,22 @@ forever for the wrong reason. When a stage does fail MAX_RETRIES times in a
|
||||
row, the script stops and asks for a human look rather than retrying
|
||||
indefinitely or silently advancing past a bad stage.
|
||||
|
||||
This is generation 2 of the curriculum. Generation 1 (6 stages: score,
|
||||
This is generation 3 of the curriculum. Generation 1 (6 stages: score,
|
||||
defend, no_draws, mechanics, aggression, unmask) ran 2026-07-21 through
|
||||
2026-07-26 and is archived in curriculum_state_gen1.json — its final stage
|
||||
("unmask", full 3D flight on top of the aggression retune) failed 3 straight
|
||||
attempts, monotonically worsening (25% -> 20% -> 15% win rate vs
|
||||
curric-s5-aggression) because every retry resumed the same drifting
|
||||
checkpoint under identical flags instead of actually changing anything.
|
||||
Rather than let generation 1's stage numbering grow indefinitely
|
||||
(unmask-retry4, retry5, ...), generation 2 starts a fresh stage 1 seeded
|
||||
directly from curric-s5-aggression's own checkpoint (FOUNDATION_EXPERIMENT
|
||||
below) — the last stage that actually passed cleanly — carrying over its
|
||||
trained progress without re-running stages 1-5. See TRAINING.md for the
|
||||
full generation 1 history and generation 2's design.
|
||||
Generation 2 (archived in curriculum_state_gen2.json) started a fresh
|
||||
single "unmask" stage seeded directly from curric-s5-aggression's own
|
||||
checkpoint (FOUNDATION_EXPERIMENT below) with retuned reward weights — it
|
||||
also failed 3 attempts, landing at a stable 32% / 28% / 31% win rate each
|
||||
time, ruling out both "retune the reward weights" and "just give it more
|
||||
time" as fixes. Generation 3 replaces the single all-or-nothing unmask
|
||||
flip with a gradual ramp (4 stages: unmask-ramp25/50/75, then unmask at
|
||||
full authority) — see TRAINING.md's "Generation 3" section for the full
|
||||
postmortem and design.
|
||||
|
||||
Every experiment name this script generates is timestamped
|
||||
(YYYYMMDD-HHMM-<name>, applied once in run_stage_attempt) so runs stay
|
||||
@@ -61,7 +64,7 @@ EVAL_HISTORY_PATH = TRAINING_DIR / "eval_history.json"
|
||||
ROOKIE_REFERENCE = TRAINING_DIR.parent / "Game" / "bots" / "rookie.json"
|
||||
|
||||
# Generation 1's last cleanly-passing checkpoint (see curriculum_state_gen1.json)
|
||||
# — generation 2's stage 1 builds on this directly instead of re-running
|
||||
# — generations 2 and 3 both build on this directly instead of re-running
|
||||
# stages 1-5.
|
||||
FOUNDATION_EXPERIMENT = "curric-s5-aggression"
|
||||
|
||||
@@ -86,70 +89,113 @@ REGRESSION_MARGIN = 0.15
|
||||
# only applies it on --resume), ent-coef keeps it from re-collapsing.
|
||||
STANDING_ARGS = ["--reset-std", "0.3", "--ent-coef", "0.001"]
|
||||
|
||||
# Ball-chasing/scoring reward flags shared by every unmask-ramp stage
|
||||
# (generation 3 — see below): identical across all 4 stages so the ramp
|
||||
# itself is the only studied variable. Lifted from generation 2's single
|
||||
# "unmask" attempt (raised from stage 5's 0.05/0.006/0.5/0.02/60 defaults).
|
||||
_UNMASK_RAMP_SHARED_FLAGS = [
|
||||
"--opponent-mode", "self_play",
|
||||
"--velocity-to-ball-weight", "0.08",
|
||||
"--ball-distance-penalty", "0.01",
|
||||
"--ball-touch-reward", "0.7",
|
||||
"--ball-velocity-to-goal-weight", "0.06",
|
||||
"--goal-reward", "80",
|
||||
"--draw-penalty", "5",
|
||||
]
|
||||
|
||||
STAGES = [
|
||||
{
|
||||
"name": "unmask",
|
||||
# Re-opens full 3D controls (no more --no-allow-vertical/
|
||||
# --no-allow-pitch-roll) on top of the aggression retune, instead of
|
||||
# keeping locomotion masked indefinitely. The mask blocked *thrust*-
|
||||
# driven flight outright; airborne_penalty (dense, scaled by height
|
||||
# above the floor — see ship_ai_controller.gd) is meant to teach the
|
||||
# policy to prefer staying grounded through incentives rather than a
|
||||
# hard constraint, so it can start learning when the other axes are
|
||||
# actually useful (aerial saves, wall recoveries) instead of never
|
||||
# touching them.
|
||||
"name": "unmask-ramp25",
|
||||
# Generation 3, step 1/4 of a gradual locomotion-unmask ramp — see
|
||||
# TRAINING.md's "Generation 3" section for the full postmortem.
|
||||
# Generation 2's single all-or-nothing "unmask" stage (flip
|
||||
# vertical_ramp/pitch_roll_ramp 0 -> 1 in one step) failed 3
|
||||
# independent 240M-step attempts in a row, landing at a stable
|
||||
# 32% / 28% / 31% win rate vs curric-s5-aggression each time — not
|
||||
# noise (attempts 2-3 each gave the *same* checkpoint lineage
|
||||
# another full 240M steps with zero improvement) and not fixable by
|
||||
# more time. Every attempt shows train/std collapsing from ~0.30 to
|
||||
# ~0.13-0.15 within the first ~10% of steps and never recovering —
|
||||
# the policy locks the newly-opened axes back down before ever
|
||||
# meaningfully exploring them.
|
||||
#
|
||||
# Generation 1 ran this exact transition 3 times (unmask, retry1,
|
||||
# retry2) with identical flags and got monotonically worse each time
|
||||
# (25% -> 20% -> 15% win rate vs curric-s5-aggression) — a blind
|
||||
# retry just continues training the same drifting policy for
|
||||
# longer, it was never going to converge differently. An adversarial
|
||||
# review of a first patch (two modest new flags, still resuming the
|
||||
# drifted retry2 checkpoint) found that insufficient too: the resume
|
||||
# target was the worst of the three already-degraded checkpoints,
|
||||
# and the new weights were too small to compete with the unchanged
|
||||
# ball-pursuit terms. Generation 2's stage 1 instead:
|
||||
# - resumes from FOUNDATION_EXPERIMENT (curric-s5-aggression)
|
||||
# directly (resume_from_experiment below) for the first attempt.
|
||||
# - raises velocity_to_ball_weight and ball_distance_penalty
|
||||
# further (the actual ball-chasing terms, unchanged since stage
|
||||
# 5 despite three failed attempts) and ball_touch_reward
|
||||
# alongside them.
|
||||
# - raises ball_velocity_to_goal_weight (reward for moving the
|
||||
# ball toward the goal, not just touching it) and goal_reward
|
||||
# (the terminal reward for scoring) — both newly exposed via
|
||||
# train.py, previously only reachable as raw Godot cmdline args.
|
||||
# - adds draw_penalty (proven effective in generation 1's stage 3
|
||||
# against passivity), which this transition had never set:
|
||||
# previously all carrot for scoring, no stick for never scoring.
|
||||
# This stage instead scales vertical_ramp/pitch_roll_ramp to 25%
|
||||
# authority. Ungated (see "gated" below and main()'s loop): this is
|
||||
# a waypoint, not a measured transition — no eval runs, no
|
||||
# regression gate applies, it always advances after training.
|
||||
# airborne_penalty is off (0.0) here: at 25% authority the axis
|
||||
# barely does anything yet, so there's nothing to discourage.
|
||||
"flags": [
|
||||
"--opponent-mode", "self_play",
|
||||
"--velocity-to-ball-weight", "0.08", # up from 0.05
|
||||
"--ball-distance-penalty", "0.01", # up from 0.006
|
||||
"--ball-touch-reward", "0.7", # up from 0.5
|
||||
"--airborne-penalty", "0.003",
|
||||
"--ball-velocity-to-goal-weight", "0.06", # up from 0.02 (0.004 default)
|
||||
"--goal-reward", "80", # up from 60 (40 default)
|
||||
"--draw-penalty", "5",
|
||||
*_UNMASK_RAMP_SHARED_FLAGS,
|
||||
"--vertical-ramp", "0.25",
|
||||
"--pitch-roll-ramp", "0.25",
|
||||
"--airborne-penalty", "0.0",
|
||||
],
|
||||
# 2026-07-29: generation 2's own first two attempts (both independently
|
||||
# resumed from FOUNDATION_EXPERIMENT under reset_retry_checkpoint,
|
||||
# identical flags/budget) landed at 32% and 27% win rate — a real
|
||||
# regression either way, but with enough run-to-run spread that
|
||||
# "identical fresh restart" isn't a controlled test of anything. The
|
||||
# first attempt's own trajectory (ep_rew_mean climbing from -10.86
|
||||
# toward ~0 by the 240M-step cutoff, briefly touching positive) looked
|
||||
# closer to convergence than the second's, so rather than another
|
||||
# independent restart from foundation, retries now continue *that*
|
||||
# attempt's own checkpoint for another full timesteps budget — an
|
||||
# actual test of "did it just need more time," not another coin flip.
|
||||
# A third, unrelated attempt crashed immediately (see train.py's
|
||||
# GoalRateCallback KeyError fix) before contributing any real signal
|
||||
# and was discarded rather than counted.
|
||||
"grounded": False,
|
||||
"timesteps": 240_000_000, # ~24h at the standing n-parallel/speedup (20M took ~2h)
|
||||
"gated": False,
|
||||
"timesteps": 40_000_000, # ~4h at the standing n-parallel/speedup (20M took ~2h)
|
||||
# Stage 0 MUST set this explicitly — resume_checkpoint()'s stage-0
|
||||
# branch returns None (train from scratch) without it.
|
||||
"resume_from_experiment": FOUNDATION_EXPERIMENT,
|
||||
},
|
||||
{
|
||||
"name": "unmask-ramp50",
|
||||
# Step 2/4: 50% authority. airborne_penalty at 1/3 of its final
|
||||
# value — enough to start discouraging unproductive altitude, not
|
||||
# enough to fight the still-partial vertical axis outright.
|
||||
# resume_from_experiment deliberately omitted: chains from
|
||||
# ramp25's pass via _resume_source_experiment's default
|
||||
# (_passing_experiment_for_stage) — do not add an override here.
|
||||
"flags": [
|
||||
*_UNMASK_RAMP_SHARED_FLAGS,
|
||||
"--vertical-ramp", "0.5",
|
||||
"--pitch-roll-ramp", "0.5",
|
||||
"--airborne-penalty", "0.001",
|
||||
],
|
||||
"gated": False,
|
||||
"timesteps": 40_000_000,
|
||||
},
|
||||
{
|
||||
"name": "unmask-ramp75",
|
||||
# Step 3/4: 75% authority, airborne_penalty at 2/3 of its final
|
||||
# value. Also chains automatically — no resume_from_experiment.
|
||||
"flags": [
|
||||
*_UNMASK_RAMP_SHARED_FLAGS,
|
||||
"--vertical-ramp", "0.75",
|
||||
"--pitch-roll-ramp", "0.75",
|
||||
"--airborne-penalty", "0.002",
|
||||
],
|
||||
"gated": False,
|
||||
"timesteps": 40_000_000,
|
||||
},
|
||||
{
|
||||
"name": "unmask",
|
||||
# Step 4/4, the measured transition: full ramp (100% authority),
|
||||
# airborne_penalty at its full value — behaviourally and eval-wise
|
||||
# identical to generation 2's "unmask" stage's flags/config, so
|
||||
# this stage's result is a direct, apples-to-apples comparison
|
||||
# against the 3 failed all-or-nothing attempts (same reference,
|
||||
# same opponent mode, same budget). Gated (default True): evaluated
|
||||
# against FOUNDATION_EXPERIMENT exactly like every prior attempt.
|
||||
#
|
||||
# No resume_from_experiment here (deliberately, unlike generation
|
||||
# 2's single-stage version) — this stage chains from ramp75's own
|
||||
# checkpoint via the default resume path, not back to
|
||||
# FOUNDATION_EXPERIMENT; only reference_experiment (the *eval*
|
||||
# opponent) stays FOUNDATION_EXPERIMENT.
|
||||
"flags": [
|
||||
*_UNMASK_RAMP_SHARED_FLAGS,
|
||||
"--vertical-ramp", "1.0",
|
||||
"--pitch-roll-ramp", "1.0",
|
||||
"--airborne-penalty", "0.003",
|
||||
],
|
||||
"grounded": False,
|
||||
"timesteps": 240_000_000, # unchanged from the 3 failed attempts — same budget for a controlled comparison
|
||||
"reference_experiment": FOUNDATION_EXPERIMENT,
|
||||
# On retry, reset to the clean ramp75 checkpoint rather than
|
||||
# compounding a failed full-ramp attempt's own drift — mirrors the
|
||||
# generation 1 -> 2 postmortem (blind same-checkpoint retries only
|
||||
# made things worse).
|
||||
"reset_retry_checkpoint": True,
|
||||
},
|
||||
]
|
||||
|
||||
@@ -365,6 +411,24 @@ def main():
|
||||
|
||||
experiment = run_stage_attempt(stage_index, attempt, args)
|
||||
last_experiment = experiment
|
||||
|
||||
if not STAGES[stage_index].get("gated", True):
|
||||
# Ungated ramp waypoint (see the unmask-ramp2X stages): trains,
|
||||
# checkpoints, and always advances — no eval, no regression
|
||||
# gate, nothing to retry against. See TRAINING.md's
|
||||
# "Generation 3" section.
|
||||
print(f"{experiment}: ungated ramp waypoint — skipping eval, advancing unconditionally")
|
||||
state["log"].append({
|
||||
"stage_index": stage_index, "experiment": experiment, "attempt": attempt,
|
||||
"decision": "pass", "note": "ungated ramp waypoint (no eval)",
|
||||
})
|
||||
state["stage_index"] += 1
|
||||
state["attempt"] = 0
|
||||
state["status"] = "in_progress"
|
||||
save_state(state)
|
||||
commit_progress(experiment)
|
||||
continue
|
||||
|
||||
reference = reference_bot(stage_index)
|
||||
record = evaluate_attempt(experiment, reference, EVAL_EPISODES, stage_index)
|
||||
decision = decide(record)
|
||||
|
||||
@@ -1,55 +1,6 @@
|
||||
{
|
||||
"stage_index": 0,
|
||||
"attempt": 2,
|
||||
"status": "blocked",
|
||||
"log": [
|
||||
{
|
||||
"stage_index": 0,
|
||||
"experiment": "20260726-1904-curric-s1-unmask",
|
||||
"attempt": 0,
|
||||
"eval": {
|
||||
"timestamp": "2026-07-27T23:30:59+00:00",
|
||||
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260726-1904-curric-s1-unmask.json",
|
||||
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s5-aggression.json",
|
||||
"episodes": 100,
|
||||
"wins_a": 32,
|
||||
"wins_b": 50,
|
||||
"draws": 18,
|
||||
"win_rate_a": 0.32
|
||||
},
|
||||
"decision": "fail"
|
||||
},
|
||||
{
|
||||
"stage_index": 0,
|
||||
"experiment": "20260729-0837-curric-s1-unmask-retry1",
|
||||
"attempt": 1,
|
||||
"eval": {
|
||||
"timestamp": "2026-07-30T11:24:45+00:00",
|
||||
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260729-0837-curric-s1-unmask-retry1.json",
|
||||
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s5-aggression.json",
|
||||
"episodes": 100,
|
||||
"wins_a": 28,
|
||||
"wins_b": 57,
|
||||
"draws": 15,
|
||||
"win_rate_a": 0.28
|
||||
},
|
||||
"decision": "fail"
|
||||
},
|
||||
{
|
||||
"stage_index": 0,
|
||||
"experiment": "20260730-1224-curric-s1-unmask-retry2",
|
||||
"attempt": 2,
|
||||
"eval": {
|
||||
"timestamp": "2026-07-31T15:07:16+00:00",
|
||||
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260730-1224-curric-s1-unmask-retry2.json",
|
||||
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s5-aggression.json",
|
||||
"episodes": 100,
|
||||
"wins_a": 31,
|
||||
"wins_b": 56,
|
||||
"draws": 13,
|
||||
"win_rate_a": 0.31
|
||||
},
|
||||
"decision": "fail"
|
||||
}
|
||||
]
|
||||
"attempt": 0,
|
||||
"status": "in_progress",
|
||||
"log": []
|
||||
}
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
{
|
||||
"stage_index": 0,
|
||||
"attempt": 2,
|
||||
"status": "blocked",
|
||||
"log": [
|
||||
{
|
||||
"stage_index": 0,
|
||||
"experiment": "20260726-1904-curric-s1-unmask",
|
||||
"attempt": 0,
|
||||
"eval": {
|
||||
"timestamp": "2026-07-27T23:30:59+00:00",
|
||||
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260726-1904-curric-s1-unmask.json",
|
||||
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s5-aggression.json",
|
||||
"episodes": 100,
|
||||
"wins_a": 32,
|
||||
"wins_b": 50,
|
||||
"draws": 18,
|
||||
"win_rate_a": 0.32
|
||||
},
|
||||
"decision": "fail"
|
||||
},
|
||||
{
|
||||
"stage_index": 0,
|
||||
"experiment": "20260729-0837-curric-s1-unmask-retry1",
|
||||
"attempt": 1,
|
||||
"eval": {
|
||||
"timestamp": "2026-07-30T11:24:45+00:00",
|
||||
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260729-0837-curric-s1-unmask-retry1.json",
|
||||
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s5-aggression.json",
|
||||
"episodes": 100,
|
||||
"wins_a": 28,
|
||||
"wins_b": 57,
|
||||
"draws": 15,
|
||||
"win_rate_a": 0.28
|
||||
},
|
||||
"decision": "fail"
|
||||
},
|
||||
{
|
||||
"stage_index": 0,
|
||||
"experiment": "20260730-1224-curric-s1-unmask-retry2",
|
||||
"attempt": 2,
|
||||
"eval": {
|
||||
"timestamp": "2026-07-31T15:07:16+00:00",
|
||||
"model_a": "/home/jcreek/ai-training/CosmicClash/Game/bots/20260730-1224-curric-s1-unmask-retry2.json",
|
||||
"model_b": "/home/jcreek/ai-training/CosmicClash/Game/bots/curric-s5-aggression.json",
|
||||
"episodes": 100,
|
||||
"wins_a": 31,
|
||||
"wins_b": 56,
|
||||
"draws": 13,
|
||||
"win_rate_a": 0.31
|
||||
},
|
||||
"decision": "fail"
|
||||
}
|
||||
]
|
||||
}
|
||||
+6
-7
@@ -112,13 +112,12 @@ def parse_args():
|
||||
curriculum.add_argument("--kickoff-chance", type=float, default=None, help="Overrides kickoff_state_chance")
|
||||
curriculum.add_argument("--near-goal-chance", type=float, default=None, help="Overrides ball_near_goal_chance")
|
||||
curriculum.add_argument(
|
||||
"--allow-vertical", action=argparse.BooleanOptionalAction, default=None, help="Allow vertical thrust (default true)"
|
||||
"--vertical-ramp", type=float, default=None,
|
||||
help="0.0-1.0: fraction of vertical thrust that reaches the ship (locomotion-unmask ramp; default 1.0)",
|
||||
)
|
||||
curriculum.add_argument(
|
||||
"--allow-pitch-roll",
|
||||
action=argparse.BooleanOptionalAction,
|
||||
default=None,
|
||||
help="Allow pitch/roll rotation (default true)",
|
||||
"--pitch-roll-ramp", type=float, default=None,
|
||||
help="0.0-1.0: fraction of pitch/roll rotation that reaches the ship (locomotion-unmask ramp; default 1.0)",
|
||||
)
|
||||
curriculum.add_argument(
|
||||
"--velocity-to-ball-weight", type=float, default=None,
|
||||
@@ -159,8 +158,8 @@ def _curriculum_kwargs(args) -> dict:
|
||||
"attack_goal_bias": args.attack_goal_bias,
|
||||
"kickoff_state_chance": args.kickoff_chance,
|
||||
"ball_near_goal_chance": args.near_goal_chance,
|
||||
"ai_allow_vertical": args.allow_vertical,
|
||||
"ai_allow_pitch_roll": args.allow_pitch_roll,
|
||||
"ai_vertical_ramp": args.vertical_ramp,
|
||||
"ai_pitch_roll_ramp": args.pitch_roll_ramp,
|
||||
"ai_velocity_to_ball_weight": args.velocity_to_ball_weight,
|
||||
"ai_ball_distance_penalty": args.ball_distance_penalty,
|
||||
"ai_ball_touch_reward": args.ball_touch_reward,
|
||||
|
||||
Reference in New Issue
Block a user