diff --git a/Game/scripts/ship_ai_controller.gd b/Game/scripts/ship_ai_controller.gd index a4e35817..260ec4a5 100644 --- a/Game/scripts/ship_ai_controller.gd +++ b/Game/scripts/ship_ai_controller.gd @@ -71,6 +71,16 @@ extends AIController3D # chance appears, instead of a free way to keep collecting dense reward. @export var time_penalty := 0.001 +# Locomotion curriculum: when false, the corresponding action axes are +# discarded in set_action before reaching the ship, so the ship stays +# grounded and only yaws — basic scoring/defending doesn't need 3D flight. +# This masks the *effect* of thrust.y/rotation.x/rotation.z, not the action +# space's shape: the policy still outputs values for these axes (still +# contributing to PPO's entropy/log-prob), they're just discarded here, so +# checkpoints stay resumable once a later curriculum stage re-enables them. +@export var allow_vertical := true +@export var allow_pitch_roll := true + # Contact normals with y above this are floor contact (exempt from the wall # penalty); below it they read as wall (sideways) or ceiling (downward). const FLOOR_NORMAL_MIN_Y := 0.7 @@ -128,8 +138,11 @@ func get_action_space() -> Dictionary: func set_action(action) -> void: var thrust: Array = action["thrust"] var rot: Array = action["rotation"] - rl_controller.action.thrust = Vector3(thrust[0], thrust[1], thrust[2]) - rl_controller.action.rotation = Vector3(rot[0], rot[1], rot[2]) + var thrust_y: float = thrust[1] if allow_vertical else 0.0 + var pitch: float = rot[0] if allow_pitch_roll else 0.0 + var roll: float = rot[2] if allow_pitch_roll else 0.0 + rl_controller.action.thrust = Vector3(thrust[0], thrust_y, thrust[2]) + rl_controller.action.rotation = Vector3(pitch, rot[1], roll) rl_controller.action.turbo = int(action["turbo"]) == 1 diff --git a/Game/scripts/training_mode.gd b/Game/scripts/training_mode.gd index 477bb07f..c5aec249 100644 --- a/Game/scripts/training_mode.gd +++ b/Game/scripts/training_mode.gd @@ -16,6 +16,13 @@ extends GameMode # instead driven by those exported policies via AIShipController; each episode # ends at the first goal (or a draw on timeout), and a final machine-readable # "EVAL_RESULT {...}" line is printed before quitting. +# +# Curriculum mode (used by training/train.py's --opponent-mode/--draw-penalty/ +# etc. flags, see TRAINING.md): --opponent_mode=inert|frozen swaps team 1's +# live self-play agent for a do-nothing placeholder or a fixed exported +# policy; --ai_= and the TrainingMode-level overrides below let +# a run retune reward shaping / episode-start mix without touching script +# defaults. See _parse_curriculum_args. @export var episode_length_seconds := 30.0 # Run01-vs-run02 eval (see training/eval_history.json) came back 87.5% draws: @@ -25,6 +32,13 @@ extends GameMode # Raised well above that ceiling so a real scoring chance always beats # continuing to farm dense reward for however long is left in the episode. @export var goal_reward := 40.0 +# One-time penalty applied to every agent when an episode times out with no +# goal scored (see _physics_process's truncation branch) — distinct from +# ShipAIController's per-tick time_penalty, which accrues regardless of +# outcome and doesn't specifically mark "this episode ended undecided." +# Default 0 (off) so ordinary runs are unaffected; curriculum stage 3 turns +# this on via --draw_penalty to teach that a draw is still a failure. +@export var draw_penalty := 0.0 # Episode-start state mix; remaining probability = fully random state. @export_range(0.0, 1.0) var kickoff_state_chance := 0.2 @@ -34,6 +48,12 @@ extends GameMode # was 1 in 5 episode starts; most training time was spent in generic # midfield play where a finish never comes up. @export_range(0.0, 1.0) var ball_near_goal_chance := 0.35 +# Which goal _place_ball_near_goal() favors: 0.5 = uniform between both goals +# (default, matches historical behaviour). 1.0 = always the goal team 0 +# attacks — used by curriculum stage 1 (--attack_goal_bias=1.0) so a lone +# trainee's near-goal resets are always finishing chances, not a coin flip +# between attacking and defending an empty net. +@export_range(0.0, 1.0) var attack_goal_bias := 0.5 # Placement bounds for randomized episode starts, derived from the standard # enclosure (ArenaBoundary). The inset keeps a randomly oriented ship (1x1x4 @@ -77,9 +97,27 @@ var _eval_draws := 0 var _eval_episodes_done := 0 var _episode_ticks := 0 +# Curriculum mode state (see _parse_curriculum_args). "self_play" (default) +# is today's only historical behaviour: both ships are live trainees sharing +# the policy. "inert" gives team 1 a do-nothing placeholder ship (no bot +# configured, same as MatchMode's fallback) so a lone trainee can drill +# scoring against an empty net. "frozen" gives team 1 a fixed exported +# policy via AIShipController — the same wiring the eval branch above +# already uses, just for one side of a live training episode. +var _opponent_mode := "self_play" +var _opponent_model_path := "" +# ShipAIController @export overrides collected from --ai_= args, +# applied to every ShipAIController this run creates (see _attach_agent). +var _ai_overrides := {} +# Ships excluded from _attach_agent (the "inert" opponent) skip randomized +# per-episode placement in _place_ships_random so they stay parked at their +# arena spawn instead of drifting into the play area as a stray obstacle. +var _inert_ships: Array[Ship] = [] + func _start() -> void: _parse_eval_args() + _parse_curriculum_args() spawn_ball() if _eval: for team in [0, 1]: @@ -87,18 +125,37 @@ func _start() -> void: bot.model_path = _eval_models[team] spawn_ship(team, 0, bot) return + var ship_team0 := spawn_ship(0, 0, RLShipController.new()) - var ship_team1 := spawn_ship(1, 0, RLShipController.new()) + var ship_team1: Ship + match _opponent_mode: + "inert": + ship_team1 = spawn_ship(1, 0, ShipController.new()) + _inert_ships.append(ship_team1) + "frozen": + var bot := AIShipController.new() + bot.model_path = _opponent_model_path + ship_team1 = spawn_ship(1, 0, bot) + _: + ship_team1 = spawn_ship(1, 0, RLShipController.new()) + _attach_agent(ship_team0, ship_team1) - _attach_agent(ship_team1, ship_team0) + if _opponent_mode == "self_play": + _attach_agent(ship_team1, ship_team0) -func _parse_eval_args() -> void: +# Shared "--key=value" cmdline scan used by both eval and curriculum parsing. +func _cmdline_kv_args() -> Dictionary: var args := {} for argument in OS.get_cmdline_args(): if argument.begins_with("--") and argument.find("=") > -1: var key_value := argument.lstrip("--").split("=", true, 1) args[key_value[0]] = key_value[1] + return args + + +func _parse_eval_args() -> void: + var args := _cmdline_kv_args() if args.has("eval_model_a") and args.has("eval_model_b"): _eval = true _eval_models[0] = args["eval_model_a"] @@ -106,22 +163,96 @@ func _parse_eval_args() -> void: _eval_episodes = int(args.get("eval_episodes", str(_eval_episodes))) +# TrainingMode @export names a curriculum run may override from the cmdline. +# Explicit allow-list (not reflection) so a typo'd flag fails loudly instead +# of silently matching an unrelated inherited export. +const TRAINING_MODE_OVERRIDES := [ + "goal_reward", "draw_penalty", "kickoff_state_chance", + "ball_near_goal_chance", "attack_goal_bias", +] +# ShipAIController @export names a curriculum run may override, read as +# --ai_= to avoid colliding with the names above. +const SHIP_AI_OVERRIDES := [ + "ball_touch_reward", "ball_touch_cooldown_ticks", "ball_touch_direction_floor", + "velocity_to_ball_weight", "ball_velocity_to_goal_weight", "ball_distance_penalty", + "wall_contact_penalty", "tilt_penalty", "speed_reward_weight", "time_penalty", + "allow_vertical", "allow_pitch_roll", +] + + +func _parse_curriculum_args() -> void: + var args := _cmdline_kv_args() + if args.has("opponent_mode"): + _opponent_mode = args["opponent_mode"] + _opponent_model_path = args.get("opponent_model", _opponent_model_path) + + for name in TRAINING_MODE_OVERRIDES: + if args.has(name): + set(name, _typed_like(args[name], get(name))) + + for name in SHIP_AI_OVERRIDES: + var key := "ai_%s" % name + if args.has(key): + _ai_overrides[name] = _typed_like(args[key], _ai_default(name)) + + +# Parses a cmdline string into the same Variant type as `sample` (bool/int/ +# float pass through Godot's str()-based conversions; anything else stays a +# String), so callers can `set()` it straight onto a typed @export var. +func _typed_like(value: String, sample) -> Variant: + match typeof(sample): + TYPE_BOOL: + return value.to_lower() in ["1", "true", "yes"] + TYPE_INT: + return value.to_int() + TYPE_FLOAT: + return value.to_float() + _: + return value + + +# ShipAIController isn't in the scene tree until _attach_agent instantiates +# one, so overrides need a default to type-match against up front; this +# mirrors ship_ai_controller.gd's own @export defaults. +func _ai_default(name: String) -> Variant: + match name: + "ball_touch_reward": return 0.4 + "ball_touch_cooldown_ticks": return 60 + "ball_touch_direction_floor": return 0.3 + "velocity_to_ball_weight": return 0.02 + "ball_velocity_to_goal_weight": return 0.004 + "ball_distance_penalty": return 0.002 + "wall_contact_penalty": return 0.0025 + "tilt_penalty": return 0.002 + "speed_reward_weight": return 0.004 + "time_penalty": return 0.001 + "allow_vertical", "allow_pitch_roll": return true + _: return null + + func _attach_agent(ship: Ship, opponent: Ship) -> void: var agent := ShipAIController.new() agent.name = "ShipAIController" agent.reset_after = int(episode_length_seconds * TICKS_PER_SIM_SECOND) + for key in _ai_overrides: + agent.set(key, _ai_overrides[key]) ship.add_child(agent) agent.setup(ship, ship.controller as RLShipController, ball, opponent, _attack_goal_position(ship.team)) _agents.append(agent) -# The goal this team scores into: the one the opponent defends/concedes. -func _attack_goal_position(team: int) -> Vector3: +# The goal a team scores into: the one the opponent defends/concedes. +func _goal_for_team(team: int) -> Goal: for goal in arena.get_goals(): if goal.team == 1 - team: - return goal.global_position + return goal push_error("TrainingMode: no goal found for team %d to attack" % team) - return Vector3.ZERO + return null + + +func _attack_goal_position(team: int) -> Vector3: + var goal := _goal_for_team(team) + return goal.global_position if goal else Vector3.ZERO func _physics_process(_delta): @@ -144,6 +275,7 @@ func _physics_process(_delta): if needs_reset: if truncated: for agent in _agents: + agent.reward -= draw_penalty agent.done = true _reset_episode() return @@ -219,9 +351,10 @@ func _place_ball_random() -> void: # Attacking/defending drill states: ball close to a goal, moving toward it. +# Which goal is picked is biased by attack_goal_bias (0.5 = uniform between +# both, matching historical behaviour; 1.0 = always the goal team 0 attacks). func _place_ball_near_goal() -> void: - var goals := arena.get_goals() - var goal: Goal = goals[randi() % goals.size()] + var goal := _goal_for_team(0) if randf() < attack_goal_bias else _goal_for_team(1) var toward_centre := -signf(goal.global_position.z) var position := Vector3( randf_range(-4.0, 4.0), @@ -235,6 +368,11 @@ func _place_ball_near_goal() -> void: func _place_ships_random() -> void: for ship in ships: + # Inert opponents (opponent_mode=inert) stay parked at their arena + # spawn instead of drifting into the play area as a stray obstacle — + # see _inert_ships. + if ship in _inert_ships: + continue var orientation := Basis.from_euler(Vector3( randf_range(-0.4, 0.4), randf_range(-PI, PI), diff --git a/TODO.md b/TODO.md index 53a83281..79c601ba 100644 --- a/TODO.md +++ b/TODO.md @@ -7,8 +7,9 @@ Deferred work, in rough priority order. The current architecture (ShipAction/Shi The training pipeline is built — see `TRAINING.md` (self-play PPO via the vendored godot_rl_agents bridge, JSON policy export, in-game GDScript inference, eval ladder). Remaining: - [ ] Long training runs on the Linux/3090 box to produce actually-good bots; promote checkpoints into `Game/bots/` as `easy`/`medium`/`hard` tiers. -- [ ] Frozen-opponent league: train the live policy against a pool of past exported checkpoints (via `AIShipController` on the opponent ship in TrainingMode) to prevent self-play strategy collapse on long runs. -- [ ] Richer state setter / curriculum: aerial states, wall plays, rebound scenarios as skill grows. +- [x] Staged curriculum (score → defend → avoid draws → full mechanics) via `train.py`'s `--opponent-mode`/`--draw-penalty`/`--attack-goal-bias`/`--allow-vertical`/`--allow-pitch-roll` flags — see TRAINING.md's "Curriculum training" section. `--opponent-mode=frozen` is a single-fixed-model slice of the league idea below, not the full sampled pool. +- [ ] Frozen-opponent league: train the live policy against a *pool* of past exported checkpoints, sampled per-episode (today's `--opponent-mode=frozen` only supports one fixed model per run) to prevent self-play strategy collapse on long runs. +- [ ] Richer state setter / curriculum: aerial states, wall plays, rebound scenarios as skill grows (beyond the score/defend/draw staging already in place). - [ ] Main-menu difficulty picker (Match already takes `bot_model_path`/`bot_reaction_ticks`/`bot_action_noise` exports). - [ ] Optional: exported headless Linux build for faster parallel training instances (train.py currently runs the project from source, which is fine but re-parses scripts per instance). diff --git a/TRAINING.md b/TRAINING.md index 0ff10148..21fb5249 100644 --- a/TRAINING.md +++ b/TRAINING.md @@ -135,10 +135,75 @@ mode (`bot_model_path`, `bot_reaction_ticks`, `bot_action_noise` in reactions, easier. - **action_noise**: adds execution error, easier. +## Curriculum training + +Training from scratch with self-play alone hands the network every skill +at once — finishing, defending, positioning, not stalling to a draw — off a +sparse ±40 goal reward. `train.py` has a `curriculum` flag group that stages +this the way you'd coach a human: score first, then also defend, then learn +that a draw is still a failure, and only then spend compute polishing general +movement. Each stage is a normal chained run — a new `--experiment` resumed +via `--resume checkpoints//final.zip`, same as any other run — +just with different curriculum flags. + +| Stage | Flags | What it teaches | +|---|---|---| +| 1 — score | `--opponent-mode inert --attack-goal-bias 1.0 --no-allow-vertical --no-allow-pitch-roll` | Team 1 is a do-nothing placeholder ship parked at its spawn (an effectively empty net); near-goal resets always target the goal the trainee attacks; the ship can't fly or pitch/roll, only drive and yaw. Isolated finishing practice. | +| 2 — defend too | `--opponent-mode self_play --no-allow-vertical --no-allow-pitch-roll` | Reintroduces a live opponent (self-play) and the default episode-start mix — the same near-goal state is now simultaneously a finishing chance for one side and a defensive save for the other. Locomotion stays grounded. | +| 3 — no draws | `--draw-penalty 5 --reset-std 0.3` | Training episodes are golden-goal (end at the *first* goal), so there's no in-episode goal-margin to penalize — `draw_penalty` is the closest available signal: a one-time penalty when an episode times out with no goal at all, on top of the existing per-tick `time_penalty`. Also lifts the locomotion mask (full 3D controls) by omitting `--allow-vertical`/`--allow-pitch-roll`; pair that with `--reset-std` since the policy never got a reward gradient on those axes before now, so expect a brief re-exploration wobble. | +| 4 — mechanics/refinement | *(no curriculum flags — plain `next_run.sh`)* | Stock self-play, full controls, default reward/start-state mix. This is what all runs before this feature already did. | + +All curriculum flags default to leaving Godot's own `@export` defaults +alone (`train.py` only forwards a flag when you pass it), so ordinary runs +are unaffected. Full flag list: `--opponent-mode {self_play,inert,frozen}`, +`--opponent-model ` (for `frozen`), `--draw-penalty`, +`--attack-goal-bias`, `--kickoff-chance`, `--near-goal-chance`, +`--allow-vertical`/`--no-allow-vertical`, `--allow-pitch-roll`/`--no-allow-pitch-roll`. + +### Running it automatically + +`training/curriculum.py` (started via `curriculum.sh`, same detached-tmux +pattern as `start_training.sh`) drives all four stages end to end: for each +stage it runs `run_training.sh` (pull, train, export, commit+push) with that +stage's flags, then evaluates the resulting checkpoint against a reference +bot — the fixed `rookie.json` baseline for stage 1, or the previous stage's +promoted checkpoint for stages 2-4 — over 100 episodes. + +```bash +cd training +./curriculum.sh # start/resume the curriculum +./curriculum.sh --seed-checkpoint checkpoints/run11/final.zip # seed stage 1 instead of a fresh policy +``` + +The gate is deliberately lenient: it blocks a stage only on a **clear +regression** (the reference beating the candidate by 15+ points of win +rate), not "must show improvement." A 40-episode eval already misled us once +in this project — run11 was the first model to deliberately score a goal, +but its head-to-head eval read as a loss on sample noise alone. A strict +gate would have retried that stage forever for the wrong reason; a loose one +still catches a genuinely broken stage. Progress and every attempt's eval +result are logged to `curriculum_state.json` (committed alongside +`eval_history.json` after each attempt). + +A stage gets up to 2 retries (3 attempts total, each resuming from that +stage's own previous attempt with a fresh `--reset-std`) before the script +stops and asks for a human look — it will not retry indefinitely or advance +past a stage that keeps failing on its own. Once you've looked at why (more +timesteps? a flag needs adjusting? the eval itself was misleading?), re-run +with `--force-retry` to try again or `--skip-to-next-stage` if you judge the +result good enough despite the gate. + +Running a stage by hand (e.g. to experiment with flags before trusting the +orchestrator) still works exactly as the table above describes — just call +`next_run.sh`/`run_training.sh` directly with that stage's flags. + ## Self-play notes -Both ships share the live policy (mirrored, team-relative observations — see -`ship_observations.gd`), so training is always against the current self. -Fixed-opponent training against frozen checkpoints (league play, to avoid -strategy collapse on long runs) is deferred — see TODO.md; the pieces -(exported JSON bots + `AIShipController`) already exist. +By default both ships share the live policy (mirrored, team-relative +observations — see `ship_observations.gd`), so training is against the +current self. `--opponent-mode inert`/`frozen` (see Curriculum training +above) replace that with a placeholder or a fixed exported policy for one +side of a run; `frozen` is a single-fixed-model slice of full league play. +Fixed-opponent training against a *pool* of past checkpoints sampled per +episode (to avoid strategy collapse on long self-play runs) is still +deferred — see TODO.md. diff --git a/training/curriculum.py b/training/curriculum.py new file mode 100644 index 00000000..71d10329 --- /dev/null +++ b/training/curriculum.py @@ -0,0 +1,258 @@ +"""Orchestrate the staged curriculum (see TRAINING.md's "Curriculum training" +section): run each stage, evaluate the result against a reference bot, and +either advance to the next stage or retry the same one. + +State is persisted to curriculum_state.json (committed to git) so the script +is safe to Ctrl-C and re-run — it picks up exactly where it left off. Each +attempt reuses run_training.sh (pull, train, export, commit+push) so every +attempt's checkpoint, log, and exported policy is versioned like any other +run; this script additionally evaluates the result and commits the updated +eval_history.json + curriculum_state.json. + +The gate is deliberately lenient ("block only on a clear regression," not +"require improvement") — see TRAINING.md. A 40-episode eval can call a real +improvement a regression on sample noise alone (this happened with run11: +it was the first model to deliberately score, but lost its head-to-head +evals). A strict improvement-required gate would have retried that stage +forever for the wrong reason. When a stage does fail MAX_RETRIES times in a +row, the script stops and asks for a human look rather than retrying +indefinitely or silently advancing past a bad stage. + +Usage: + .venv/bin/python curriculum.py # run/resume the curriculum + .venv/bin/python curriculum.py --seed-checkpoint checkpoints/run11/final.zip + .venv/bin/python curriculum.py --force-retry # after fixing something, retry the blocked stage + .venv/bin/python curriculum.py --skip-to-next-stage # human judgment call: good enough, move on anyway + +Typically started via curriculum.sh, which runs this in a detached tmux +session the way start_training.sh does for a single run. +""" + +import argparse +import json +import pathlib +import subprocess +import sys + +TRAINING_DIR = pathlib.Path(__file__).resolve().parent +STATE_PATH = TRAINING_DIR / "curriculum_state.json" +EVAL_HISTORY_PATH = TRAINING_DIR / "eval_history.json" +ROOKIE_REFERENCE = TRAINING_DIR.parent / "Game" / "bots" / "rookie.json" + +MAX_RETRIES = 2 +EVAL_EPISODES = 100 +# "Clear regression" = the reference beats the candidate by at least this +# many percentage points of win rate. Below this, noise in a 100-episode +# sample is a more likely explanation than the stage actually failing (see +# module docstring) — advance rather than retry. +REGRESSION_MARGIN = 0.15 + +# Standing flags applied to every attempt, mirroring next_run.sh: reset-std +# reopens exploration every attempt (harmless on fresh starts — train.py +# only applies it on --resume), ent-coef keeps it from re-collapsing. +STANDING_ARGS = ["--reset-std", "0.3", "--ent-coef", "0.001"] + +STAGES = [ + { + "name": "score", + "flags": [ + "--opponent-mode", "inert", + "--attack-goal-bias", "1.0", + "--no-allow-vertical", "--no-allow-pitch-roll", + ], + }, + { + "name": "defend", + "flags": [ + "--opponent-mode", "self_play", + "--no-allow-vertical", "--no-allow-pitch-roll", + ], + }, + { + "name": "no_draws", + "flags": ["--draw-penalty", "5"], + }, + { + "name": "mechanics", + "flags": [], + }, +] + + +def load_state() -> dict: + if STATE_PATH.exists(): + return json.loads(STATE_PATH.read_text()) + return {"stage_index": 0, "attempt": 0, "status": "in_progress", "log": []} + + +def save_state(state: dict) -> None: + STATE_PATH.write_text(json.dumps(state, indent=2) + "\n") + + +def experiment_name(stage_index: int, attempt: int) -> str: + name = f"curric-s{stage_index + 1}-{STAGES[stage_index]['name']}" + return name if attempt == 0 else f"{name}-retry{attempt}" + + +def resume_checkpoint(stage_index: int, attempt: int, seed_checkpoint: str | None) -> str | None: + if attempt > 0: + # Retry: keep training the same stage's own last attempt. + prev = experiment_name(stage_index, attempt - 1) + return str(TRAINING_DIR / "checkpoints" / prev / "final.zip") + if stage_index == 0: + # Deliberately fresh by default: the curriculum exists because + # resuming self-play across a regime change (run10, run11) didn't + # work, so stage 1 starts from a random policy under its own + # regime unless --seed-checkpoint says otherwise. + return seed_checkpoint + prev_stage = STAGES[stage_index - 1]["name"] + prev_experiment = _passing_experiment_for_stage(stage_index - 1) + return str(TRAINING_DIR / "checkpoints" / prev_experiment / "final.zip") + + +def reference_bot(stage_index: int) -> str: + if stage_index == 0: + return str(ROOKIE_REFERENCE) + prev_experiment = _passing_experiment_for_stage(stage_index - 1) + return str(TRAINING_DIR.parent / "Game" / "bots" / f"{prev_experiment}.json") + + +def _passing_experiment_for_stage(stage_index: int) -> str: + state = load_state() + for entry in state["log"]: + if entry["stage_index"] == stage_index and entry["decision"] == "pass": + return entry["experiment"] + raise RuntimeError(f"No passing attempt recorded for stage {stage_index} ({STAGES[stage_index]['name']})") + + +def run_stage_attempt(stage_index: int, attempt: int, args) -> str: + exp = experiment_name(stage_index, attempt) + resume = resume_checkpoint(stage_index, attempt, args.seed_checkpoint) + cmd = [ + "./run_training.sh", exp, + "--timesteps", str(args.timesteps), + "--n-parallel", str(args.n_parallel), + "--speedup", str(args.speedup), + *STANDING_ARGS, + *STAGES[stage_index]["flags"], + ] + if resume: + cmd += ["--resume", resume] + print(f"\n=== Stage {stage_index + 1}/{len(STAGES)} ({STAGES[stage_index]['name']}), " + f"attempt {attempt + 1}/{MAX_RETRIES + 1}: {exp} ===") + print(" ".join(cmd)) + subprocess.run(cmd, cwd=TRAINING_DIR, check=True) + return exp + + +def evaluate_attempt(experiment: str, reference: str, episodes: int) -> dict: + candidate = TRAINING_DIR.parent / "Game" / "bots" / f"{experiment}.json" + cmd = [".venv/bin/python", "evaluate.py", str(candidate), reference, "--episodes", str(episodes)] + print(" ".join(cmd)) + subprocess.run(cmd, cwd=TRAINING_DIR, check=True) + history = json.loads(EVAL_HISTORY_PATH.read_text()) + return history[-1] + + +def decide(record: dict) -> str: + win_rate_candidate = record["wins_a"] / record["episodes"] + win_rate_reference = record["wins_b"] / record["episodes"] + if win_rate_reference - win_rate_candidate >= REGRESSION_MARGIN: + return "fail" + return "pass" + + +def commit_progress(experiment: str) -> None: + subprocess.run(["git", "add", "curriculum_state.json", "eval_history.json"], cwd=TRAINING_DIR, check=True) + result = subprocess.run(["git", "diff", "--cached", "--quiet"], cwd=TRAINING_DIR) + if result.returncode == 0: + return + subprocess.run( + ["git", "commit", "-m", f"chore(training): curriculum progress after {experiment}"], + cwd=TRAINING_DIR, check=True, + ) + subprocess.run(["git", "push"], cwd=TRAINING_DIR, check=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--timesteps", type=int, default=20_000_000) + parser.add_argument("--n-parallel", type=int, default=14) + parser.add_argument("--speedup", type=int, default=16) + parser.add_argument("--seed-checkpoint", default=None, help="Resume stage 1 from this checkpoint instead of a fresh policy") + parser.add_argument("--force-retry", action="store_true", help="Retry a blocked stage after human review") + parser.add_argument("--skip-to-next-stage", action="store_true", help="Human judgment call: treat the blocked stage as good enough, advance anyway") + args = parser.parse_args() + + state = load_state() + + if state["status"] == "blocked": + if args.skip_to_next_stage: + print(f"Human override: advancing past stage {state['stage_index'] + 1} " + f"({STAGES[state['stage_index']]['name']}) despite exhausted retries.") + state["log"].append({ + "stage_index": state["stage_index"], "experiment": experiment_name(state["stage_index"], state["attempt"]), + "attempt": state["attempt"], "decision": "pass", "override": "skip_to_next_stage", + }) + state["stage_index"] += 1 + state["attempt"] = 0 + state["status"] = "in_progress" + save_state(state) + elif args.force_retry: + print(f"Human override: retrying stage {state['stage_index'] + 1} " + f"({STAGES[state['stage_index']]['name']}) after review.") + state["attempt"] += 1 + state["status"] = "in_progress" + save_state(state) + else: + print(f"BLOCKED at stage {state['stage_index'] + 1} ({STAGES[state['stage_index']]['name']}) " + f"after {MAX_RETRIES + 1} attempts — see curriculum_state.json's log for eval results.") + print("Re-run with --force-retry (after adjusting flags/timesteps) or " + "--skip-to-next-stage (advance anyway) once you've looked at why.") + sys.exit(1) + + while state["stage_index"] < len(STAGES): + stage_index = state["stage_index"] + attempt = state["attempt"] + + experiment = run_stage_attempt(stage_index, attempt, args) + reference = reference_bot(stage_index) + record = evaluate_attempt(experiment, reference, EVAL_EPISODES) + decision = decide(record) + + print(f"{experiment}: candidate {record['wins_a']}-{record['wins_b']} reference " + f"({record['draws']} draws) over {record['episodes']} episodes -> {decision}") + + state["log"].append({ + "stage_index": stage_index, "experiment": experiment, "attempt": attempt, + "eval": record, "decision": decision, + }) + + if decision == "pass": + state["stage_index"] += 1 + state["attempt"] = 0 + state["status"] = "in_progress" + save_state(state) + commit_progress(experiment) + continue + + if attempt >= MAX_RETRIES: + state["status"] = "blocked" + save_state(state) + commit_progress(experiment) + print(f"\nBLOCKED: stage {stage_index + 1} ({STAGES[stage_index]['name']}) failed " + f"{MAX_RETRIES + 1} attempts in a row. Stopping for human review — see " + f"curriculum_state.json. Re-run with --force-retry or --skip-to-next-stage.") + sys.exit(1) + + state["attempt"] += 1 + save_state(state) + commit_progress(experiment) + + print("\nCurriculum complete — all stages passed.") + state["status"] = "done" + save_state(state) + + +if __name__ == "__main__": + main() diff --git a/training/curriculum.sh b/training/curriculum.sh new file mode 100755 index 00000000..5eea2879 --- /dev/null +++ b/training/curriculum.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Runs the staged curriculum (see TRAINING.md, curriculum.py) detached from +# your SSH session, the same way start_training.sh does for a single run — +# safe to disconnect any time. Uses its own tmux session ("cosmic-curriculum") +# so it doesn't collide with a manual next_run.sh/start_training.sh run. +# +# Usage: ./curriculum.sh [curriculum.py args...] +# e.g.: ./curriculum.sh +# ./curriculum.sh --seed-checkpoint checkpoints/run11/final.zip +# ./curriculum.sh --force-retry +set -euo pipefail +cd "$(dirname "$0")" + +SESSION="cosmic-curriculum" +TB_PORT=6006 + +command -v tmux >/dev/null 2>&1 || { echo "tmux is required: sudo apt install tmux" >&2; exit 1; } + +if tmux has-session -t "$SESSION" 2>/dev/null; then + echo "Session '$SESSION' already running — attaching (detach with Ctrl-B then D)." + exec tmux attach -t "$SESSION" +fi + +tmux new-session -d -s "$SESSION" -n curriculum \ + ".venv/bin/python curriculum.py $*; echo; echo '=== curriculum.py exited — press Enter to close ==='; read" + +if ! (exec 3<>"/dev/tcp/127.0.0.1/$TB_PORT") 2>/dev/null; then + tmux new-window -d -t "$SESSION" -n dashboard \ + ".venv/bin/tensorboard --logdir logs --host 0.0.0.0 --port $TB_PORT" +fi + +IP=$(hostname -I 2>/dev/null | awk '{print $1}') +echo "Curriculum started in tmux session '$SESSION' — SSH disconnects won't touch it." +echo " watch it: tmux attach -t $SESSION (detach again with Ctrl-B then D)" +echo " dashboard: http://${IP:-}:$TB_PORT" +echo " progress: cat curriculum_state.json" diff --git a/training/train.py b/training/train.py index 995a30b5..c84e7208 100644 --- a/training/train.py +++ b/training/train.py @@ -51,9 +51,60 @@ def parse_args(): parser.add_argument("--checkpoint-every", type=int, default=100_000, help="Timesteps between checkpoints") parser.add_argument("--viz", action="store_true", help="Show game windows (debugging; slow)") parser.add_argument("--wandb", action="store_true", help="Also log to Weights & Biases") + + curriculum = parser.add_argument_group( + "curriculum", "Stage the training run — see TRAINING.md's Curriculum training section" + ) + curriculum.add_argument( + "--opponent-mode", + choices=["self_play", "inert", "frozen"], + default=None, + help="self_play (default): both ships are live trainees. inert: team 1 is a " + "do-nothing placeholder (isolated scoring practice). frozen: team 1 runs a " + "fixed exported policy (--opponent-model)", + ) + curriculum.add_argument("--opponent-model", default=None, help="Exported policy .json for --opponent-mode=frozen") + curriculum.add_argument( + "--draw-penalty", type=float, default=None, help="One-time penalty when an episode times out with no goal" + ) + curriculum.add_argument( + "--attack-goal-bias", + type=float, + default=None, + help="0.5 = uniform between both goals (default); 1.0 = near-goal resets always target the goal team 0 attacks", + ) + curriculum.add_argument("--kickoff-chance", type=float, default=None, help="Overrides kickoff_state_chance") + curriculum.add_argument("--near-goal-chance", type=float, default=None, help="Overrides ball_near_goal_chance") + curriculum.add_argument( + "--allow-vertical", action=argparse.BooleanOptionalAction, default=None, help="Allow vertical thrust (default true)" + ) + curriculum.add_argument( + "--allow-pitch-roll", + action=argparse.BooleanOptionalAction, + default=None, + help="Allow pitch/roll rotation (default true)", + ) + return parser.parse_args() +def _curriculum_kwargs(args) -> dict: + """Maps train.py's curriculum flags to the --key=value args training_mode.gd's + _parse_curriculum_args() reads, omitting anything not explicitly passed so + unset flags leave Godot's own @export defaults in place.""" + mapping = { + "opponent_mode": args.opponent_mode, + "opponent_model": args.opponent_model, + "draw_penalty": args.draw_penalty, + "attack_goal_bias": args.attack_goal_bias, + "kickoff_state_chance": args.kickoff_chance, + "ball_near_goal_chance": args.near_goal_chance, + "ai_allow_vertical": args.allow_vertical, + "ai_allow_pitch_roll": args.allow_pitch_roll, + } + return {key: value for key, value in mapping.items() if value is not None} + + def main(): args = parse_args() log_dir = TRAINING_DIR / "logs" @@ -72,6 +123,7 @@ def main(): port=args.port, show_window=args.viz, speedup=args.speedup, + **_curriculum_kwargs(args), ) env = VecMonitor(env)