diff --git a/Game/scripts/ai_ship_controller.gd b/Game/scripts/ai_ship_controller.gd index 886ece7e..3dd5df27 100644 --- a/Game/scripts/ai_ship_controller.gd +++ b/Game/scripts/ai_ship_controller.gd @@ -34,6 +34,11 @@ var _policy: PolicyNetwork var _action := ShipAction.new() var _ticks_until_decision := 0 +# League mode revisits a small model pool every episode. Policies are +# immutable after load, so share parsed networks by path instead of reading +# and flattening the same 100KB+ JSON on every reset for every frozen ship. +static var _policy_cache: Dictionary = {} + var _ship: Ship var _teammates: Array[Ship] = [] var _opponents: Array[Ship] = [] @@ -44,7 +49,24 @@ var _scene_refs_ready := false func _ready(): if not model_path.is_empty(): + load_policy(model_path) + + +# League training swaps a frozen opponent's policy between episodes without +# despawning the ship. Reset the held action/decision cadence with the model +# so no command from the previous opponent leaks into the next episode. +func load_policy(path: String) -> void: + model_path = path + if model_path.is_empty(): + _policy = null + elif _policy_cache.has(model_path): + _policy = _policy_cache[model_path] + else: _policy = PolicyNetwork.load_from_file(model_path) + if _policy != null: + _policy_cache[model_path] = _policy + _action = ShipAction.new() + _ticks_until_decision = 0 func get_action() -> ShipAction: diff --git a/Game/scripts/ship_ai_controller.gd b/Game/scripts/ship_ai_controller.gd index 12b04c37..ab26eedb 100644 --- a/Game/scripts/ship_ai_controller.gd +++ b/Game/scripts/ship_ai_controller.gd @@ -39,6 +39,13 @@ extends AIController3D # the objective" framing. @export_range(0.0, 1.0) var ball_touch_direction_floor := 0.3 @export var velocity_to_ball_weight := 0.02 +# Dense reward for approaching the ball *nose first* near the floor. Unlike +# velocity_to_ball_weight, sideways/reverse closing velocity earns nothing: +# the planar ship-forward vector must face the ball and planar velocity must +# have a positive component along it. Default off so existing curricula and +# frozen checkpoints keep their original objective; generation 5 handling +# turns it on while reducing the orientation-agnostic term. +@export var forward_velocity_to_ball_weight := 0.0 @export var ball_velocity_to_goal_weight := 0.004 # Per-tick penalty scaled by distance to the ball (full value at the arena's # far diagonal, 0 on top of the ball). Run04 lesson: with idling worth a flat @@ -62,6 +69,11 @@ extends AIController3D # to teach. Not removed outright — an always-inverted bot still looks bad in # a shipped game. @export var tilt_penalty := 0.0005 +# Additional tilt cost that fades to zero over the first few metres above the +# floor. This can teach readable, upright ground handling without opposing +# pitch/roll during a real aerial. Generation 5 uses this instead of raising +# the global tilt_penalty back to its pre-flight value. +@export var ground_tilt_penalty := 0.0 # Per-tick bonus for own speed: 0 stationary, full value (+0.24/s) at # max_speed. Run07 lesson: after the kickoff flurry both ships parked next to # a cornered ball — with every other dense term near zero there, standing @@ -90,6 +102,15 @@ extends AIController3D # air-touch reward. const AIR_TOUCH_HEIGHT := 5.0 +# Generation-5 ground-handling telemetry/reward thresholds. Fixed constants +# keep the logged metrics comparable across stages; changing one starts a new +# metric definition and therefore requires a fresh baseline. +const GROUND_HANDLING_HEIGHT := 3.0 +const UPRIGHT_DOT_THRESHOLD := 0.7 +const FORWARD_MOTION_DOT_THRESHOLD := 0.7 +const MIN_HANDLING_SPEED := 1.0 +const PRODUCTIVE_AIR_TOUCH_ALIGNMENT := 0.5 + # Contact normals with y above this are floor contact (exempt from the wall # penalty); below it they read as wall (sideways) or ceiling (downward). # Mirrors ShipObservations.FLOOR_NORMAL_MIN_Y (see that file's comment). @@ -144,6 +165,11 @@ var _altitude_sum := 0.0 var _thrust_y_sum := 0.0 var _touches := 0 var _air_touches := 0 +var _productive_air_touches := 0 +var _ground_ticks := 0 +var _upright_ground_ticks := 0 +var _moving_ground_ticks := 0 +var _forward_moving_ground_ticks := 0 # Wire up references after the ship is spawned. `attack_goal` is the goal @@ -201,6 +227,9 @@ func get_info() -> Dictionary: info["mean_altitude"] = _altitude_sum / _telemetry_ticks if _telemetry_ticks > 0 else 0.0 info["vertical_thrust_mean"] = _thrust_y_sum / _telemetry_ticks if _telemetry_ticks > 0 else 0.0 info["air_touch_fraction"] = float(_air_touches) / _touches if _touches > 0 else 0.0 + info["productive_air_touch_fraction"] = float(_productive_air_touches) / _touches if _touches > 0 else 0.0 + info["upright_fraction"] = float(_upright_ground_ticks) / _ground_ticks if _ground_ticks > 0 else 0.0 + info["forward_motion_fraction"] = float(_forward_moving_ground_ticks) / _moving_ground_ticks if _moving_ground_ticks > 0 else 0.0 return info @@ -223,6 +252,11 @@ func reset(): _thrust_y_sum = 0.0 _touches = 0 _air_touches = 0 + _productive_air_touches = 0 + _ground_ticks = 0 + _upright_ground_ticks = 0 + _moving_ground_ticks = 0 + _forward_moving_ground_ticks = 0 func _physics_process(delta): @@ -240,6 +274,20 @@ func _physics_process(delta): var closing_speed := ship.linear_velocity.dot(to_ball.normalized()) reward += velocity_to_ball_weight * closing_speed / ship.max_speed + # Ground-handling shaping: forward planar motion while the nose faces the + # ball. It fades out with altitude so an aerial remains free to approach a + # ball using whatever body attitude is effective. + if forward_velocity_to_ball_weight > 0.0 and ship.global_position.y < GROUND_HANDLING_HEIGHT: + var planar_forward := Vector3(-ship.global_transform.basis.z.x, 0.0, -ship.global_transform.basis.z.z) + var planar_velocity := Vector3(ship.linear_velocity.x, 0.0, ship.linear_velocity.z) + var planar_to_ball := Vector3(to_ball.x, 0.0, to_ball.z) + if planar_forward.length_squared() > 0.0001 and planar_to_ball.length_squared() > 0.0001: + planar_forward = planar_forward.normalized() + var facing_ball: float = maxf(planar_forward.dot(planar_to_ball.normalized()), 0.0) + var forward_speed: float = maxf(planar_velocity.dot(planar_forward), 0.0) / ship.max_speed + var handling_ground_factor: float = 1.0 - clampf(ship.global_position.y / GROUND_HANDLING_HEIGHT, 0.0, 1.0) + reward += forward_velocity_to_ball_weight * forward_speed * facing_ball * handling_ground_factor + # Dense penalty: distance to the ball, so idling far away bleeds reward # instead of scoring a safe zero (see ball_distance_penalty). if ball_distance_penalty > 0.0: @@ -270,6 +318,12 @@ func _physics_process(delta): var uprightness: float = ship.global_transform.basis.y.dot(Vector3.UP) reward -= tilt_penalty * (1.0 - uprightness) * 0.5 + # Low-altitude-only posture pressure (see ground_tilt_penalty). + if ground_tilt_penalty > 0.0 and ship.global_position.y < GROUND_HANDLING_HEIGHT: + var ground_uprightness: float = ship.global_transform.basis.y.dot(Vector3.UP) + var tilt_ground_factor: float = 1.0 - clampf(ship.global_position.y / GROUND_HANDLING_HEIGHT, 0.0, 1.0) + reward -= ground_tilt_penalty * (1.0 - ground_uprightness) * 0.5 * tilt_ground_factor + # Dense penalty: height above the floor (see airborne_penalty). The # floor sits at world y = 0 (see training_mode.gd's FIELD_MIN_Y/ # _escaped bounds); normalized so the worst case is pinned at the @@ -285,6 +339,17 @@ func _physics_process(delta): if ship.global_position.y > AIRBORNE_ALTITUDE_THRESHOLD: _airborne_ticks += 1 _thrust_y_sum += rl_controller.action.thrust.y + if ship.global_position.y < GROUND_HANDLING_HEIGHT: + _ground_ticks += 1 + if ship.global_transform.basis.y.dot(Vector3.UP) >= UPRIGHT_DOT_THRESHOLD: + _upright_ground_ticks += 1 + var planar_velocity := Vector3(ship.linear_velocity.x, 0.0, ship.linear_velocity.z) + if planar_velocity.length() >= MIN_HANDLING_SPEED: + _moving_ground_ticks += 1 + var planar_forward := Vector3(-ship.global_transform.basis.z.x, 0.0, -ship.global_transform.basis.z.z) + if planar_forward.length_squared() > 0.0001 \ + and planar_velocity.normalized().dot(planar_forward.normalized()) >= FORWARD_MOTION_DOT_THRESHOLD: + _forward_moving_ground_ticks += 1 func _wall_or_ceiling_contact() -> bool: @@ -311,3 +376,5 @@ func _on_ship_body_entered(body: Node) -> void: _touches += 1 if ball.global_position.y > AIR_TOUCH_HEIGHT: _air_touches += 1 + if alignment >= PRODUCTIVE_AIR_TOUCH_ALIGNMENT: + _productive_air_touches += 1 diff --git a/Game/scripts/training_mode.gd b/Game/scripts/training_mode.gd index 0ee76d9e..67e7c4f9 100644 --- a/Game/scripts/training_mode.gd +++ b/Game/scripts/training_mode.gd @@ -64,6 +64,11 @@ extends GameMode # stayed floor-pinned even though the initial one wasn't. See # _place_air_drill. @export_range(0.0, 1.0) var air_drill_chance := 0.0 +# Moving-ball aerial interception branch used by generation 5. Unlike the +# stationary/random air drill, the ball follows a reachable trajectory toward +# a real goal and ships start low behind/lateral to it, so a useful touch is +# naturally reinforced by the existing goal-directed ball rewards. +@export_range(0.0, 1.0) var air_intercept_chance := 0.0 # Ships per team. Default 1 preserves every existing curriculum script's 1v1 # behaviour unchanged; up to 5 matches ShipObservations.MAX_TEAMMATES/ @@ -129,6 +134,8 @@ var _episode_ticks := 0 # already uses, just for one side of a live training episode. var _opponent_mode := "self_play" var _opponent_model_path := "" +var _opponent_model_pool: Array[String] = [] +var _frozen_opponent_bots: Array[AIShipController] = [] # ShipAIController @export overrides collected from --ai_= args, # applied to every ShipAIController this run creates (see _attach_agent). var _ai_overrides := {} @@ -164,7 +171,7 @@ func _start() -> void: team0_ships.append(spawn_ship(0, i, RLShipController.new())) # The opponent_mode branch applies uniformly to every ship on team 1: an - # "inert"/"frozen" run means the whole opposing team gets that treatment, + # "inert"/"frozen"/"league" run means the whole opposing team gets that treatment, # not just one ship. var team1_ships: Array[Ship] = [] for i in team_size: @@ -177,6 +184,12 @@ func _start() -> void: var bot := AIShipController.new() bot.model_path = _opponent_model_path ship1 = spawn_ship(1, i, bot) + _frozen_opponent_bots.append(bot) + "league": + var bot := AIShipController.new() + bot.model_path = _opponent_model_pool[0] if not _opponent_model_pool.is_empty() else "" + ship1 = spawn_ship(1, i, bot) + _frozen_opponent_bots.append(bot) _: ship1 = spawn_ship(1, i, RLShipController.new()) team1_ships.append(ship1) @@ -230,13 +243,15 @@ func _parse_eval_args() -> void: const TRAINING_MODE_OVERRIDES := [ "goal_reward", "draw_penalty", "kickoff_state_chance", "ball_near_goal_chance", "attack_goal_bias", "air_drill_chance", + "air_intercept_chance", "team_size", ] # ShipAIController @export names a curriculum run may override, read as # --ai_= to avoid colliding with the names above. const SHIP_AI_OVERRIDES := [ "ball_touch_reward", "ball_touch_cooldown_ticks", "ball_touch_direction_floor", "velocity_to_ball_weight", "ball_velocity_to_goal_weight", "ball_distance_penalty", - "wall_contact_penalty", "tilt_penalty", "speed_reward_weight", "time_penalty", + "forward_velocity_to_ball_weight", "wall_contact_penalty", "tilt_penalty", + "ground_tilt_penalty", "speed_reward_weight", "time_penalty", "airborne_penalty", ] @@ -246,10 +261,20 @@ func _parse_curriculum_args() -> void: if args.has("opponent_mode"): _opponent_mode = args["opponent_mode"] _opponent_model_path = args.get("opponent_model", _opponent_model_path) + if args.has("opponent_model_pool"): + for path in String(args["opponent_model_pool"]).split(",", false): + if not path.is_empty(): + _opponent_model_pool.append(path) + if _opponent_mode == "league" and _opponent_model_pool.is_empty(): + push_error("TrainingMode: opponent_mode=league requires --opponent_model_pool=path,path") for name in TRAINING_MODE_OVERRIDES: if args.has(name): set(name, _typed_like(args[name], get(name))) + var start_probability := kickoff_state_chance + ball_near_goal_chance \ + + air_drill_chance + air_intercept_chance + if start_probability > 1.0: + push_error("TrainingMode: episode-start probabilities sum to %.3f (> 1.0)" % start_probability) for name in SHIP_AI_OVERRIDES: var key := "ai_%s" % name @@ -281,10 +306,12 @@ func _ai_default(name: String) -> Variant: "ball_touch_cooldown_ticks": return 60 "ball_touch_direction_floor": return 0.3 "velocity_to_ball_weight": return 0.02 + "forward_velocity_to_ball_weight": return 0.0 "ball_velocity_to_goal_weight": return 0.004 "ball_distance_penalty": return 0.002 "wall_contact_penalty": return 0.0025 "tilt_penalty": return 0.0005 + "ground_tilt_penalty": return 0.0 "speed_reward_weight": return 0.004 "time_penalty": return 0.001 "airborne_penalty": return 0.0 @@ -386,6 +413,7 @@ func _end_eval_episode() -> void: func _reset_episode() -> void: for agent in _agents: agent.reset() + _select_league_opponent() var roll := randf() if roll < kickoff_state_chance: @@ -396,6 +424,8 @@ func _reset_episode() -> void: _place_ball_near_goal() elif roll < kickoff_state_chance + ball_near_goal_chance + air_drill_chance: _place_air_drill() + elif roll < kickoff_state_chance + ball_near_goal_chance + air_drill_chance + air_intercept_chance: + _place_air_intercept() else: _place_ships_random() _place_ball_random() @@ -416,6 +446,15 @@ func _place_ball_random() -> void: # out via reward shaping. const AIR_DRILL_BALL_WALL_CLEARANCE := 5.0 + +func _select_league_opponent() -> void: + if _opponent_mode != "league" or _opponent_model_pool.is_empty(): + return + var path := _opponent_model_pool[randi() % _opponent_model_pool.size()] + for bot in _frozen_opponent_bots: + bot.load_policy(path) + + # Air drill state (see air_drill_chance): ball spawned high, both ships # spawned low and lateral, so the state is unsolvable without climbing. func _place_air_drill() -> void: @@ -456,6 +495,42 @@ func _place_air_drill() -> void: _place_body(ship, Transform3D(orientation, ship_position), Vector3.ZERO, Vector3.ZERO) +# Goal-relevant aerial intercept: a high ball is already travelling toward a +# randomly selected goal, while ships begin low and behind/lateral to its +# path. The generous wall clearance prevents rebound farming and an upright +# yaw-only spawn avoids wasting the short drill window on random recovery. +func _place_air_intercept() -> void: + var goal := _goal_for_team(randi() % 2) + var ball_position := Vector3( + randf_range(-8.0, 8.0), + randf_range(6.0, minf(12.0, FIELD_MAX_Y)), + randf_range(-10.0, 10.0) + ) + var to_goal := (goal.global_position - ball_position).normalized() + var ball_velocity := (to_goal + Vector3(randf_range(-0.15, 0.15), randf_range(0.0, 0.15), 0.0)).normalized() \ + * randf_range(6.0, 11.0) + _place_body(ball, Transform3D(Basis.IDENTITY, ball_position), ball_velocity, Vector3.ZERO) + + var placed: Array[Vector3] = [] + var behind := -Vector3(ball_velocity.x, 0.0, ball_velocity.z).normalized() + for ship in ships: + if ship in _inert_ships: + continue + var ship_position := Vector3.ZERO + for _attempt in 20: + var lateral := Vector3(-behind.z, 0.0, behind.x) * randf_range(-7.0, 7.0) + ship_position = ball_position + behind * randf_range(7.0, 13.0) + lateral + ship_position.x = clampf(ship_position.x, -FIELD_HALF_X, FIELD_HALF_X) + ship_position.y = randf_range(FIELD_MIN_Y, 3.0) + ship_position.z = clampf(ship_position.z, -FIELD_HALF_Z, FIELD_HALF_Z) + if _spawn_position_clear(ship_position) and _far_enough_from(ship_position, placed): + break + placed.append(ship_position) + var face_ball := ball_position - ship_position + var yaw := atan2(-face_ball.x, -face_ball.z) + _place_body(ship, Transform3D(Basis.from_euler(Vector3(0.0, yaw, 0.0)), ship_position), Vector3.ZERO, Vector3.ZERO) + + # Attacking/defending drill states: ball close to a goal, moving toward it. # Which goal is picked is biased by attack_goal_bias (0.5 = uniform between # both, matching historical behaviour; 1.0 = always the goal team 0 attacks). diff --git a/TODO.md b/TODO.md index 80fd9af2..b3d78ad1 100644 --- a/TODO.md +++ b/TODO.md @@ -6,9 +6,11 @@ Deferred work, in rough priority order. The current architecture (ShipAction/Shi The training pipeline is built — see `TRAINING.md` (self-play PPO via the vendored godot_rl_agents bridge, JSON policy export, in-game GDScript inference, eval ladder). Remaining: -- [ ] Long training runs on the Linux/3090 box to produce actually-good bots; promote further checkpoints into `Game/bots/promoted/` as `medium`/`hard` tiers once they clear `easy.json` in `evaluate.py`. -- [ ] Frozen-opponent league: train the live policy against a *pool* of past exported checkpoints, sampled per-episode (today's `--opponent-mode=frozen` only supports one fixed model per run) to prevent self-play strategy collapse on long runs. -- [ ] Richer state setter / curriculum: aerial states, wall plays, rebound scenarios as skill grows (beyond the score/defend/draw staging already in place). +- [x] Promote generation 4's stage-3 gauntlet policy as the new `Game/bots/promoted/easy.json` baseline. +- [ ] Run the generation-5 handling/intercepts/league/teamplay curriculum described in `TRAINING.md`; promote later checkpoints as `medium`/`hard` only after they clear the match and behaviour gates. +- [x] Frozen-opponent league plumbing: `--opponent-mode=league` samples a past exported checkpoint per episode; generation 5 Stage 6 supplies the curated pool. +- [ ] Extend generation 5's moving aerial-intercept states with wall plays and rebound scenarios after Stage 5 establishes a productive-air-touch baseline. +- [ ] Design team-credit rewards and paired 2v2 evaluation before enabling the deferred teamplay stage. ## Presentation / AAA polish diff --git a/TRAINING.md b/TRAINING.md index c6782c47..1da151ed 100644 --- a/TRAINING.md +++ b/TRAINING.md @@ -162,11 +162,9 @@ flat files there, never touching subdirectories. `Game/bots/promoted/.json` is the small, curated, hand-maintained set actually referenced by the shipped game — currently `easy.json` (promoted -2026-07-24 from `curric-s6-unmask`, the strongest checkpoint at the -time — note `curric-s6-unmask` was itself generation 1's *failed* unmask -stage, so `easy.json` is weaker than `reference-grounded.json` below; a -strong generation 4 result should promote a real replacement, plus -`medium.json`/`hard.json`) and `reference-grounded.json` (added for +2026-08-08 from generation 4's `20260806-1939-curric-s3-gauntlet`; this is +the 320M-step MultiDiscrete policy and the foundation for the planned +generation-5 curriculum below) and `reference-grounded.json` (added for generation 4 — a copy of generation 3's `curric-s5-aggression`, made before the flat `Game/bots/` dump was scrapped for the redesign, kept as the strongest grounded-era artifact and the fixed yardstick generations 1-3 were @@ -176,6 +174,11 @@ promoted file is never touched by training scripts, never overwritten by a same-named future export, and never disturbed by pruning old experiment files from the flat dump. +Until distinct `medium.json` and `hard.json` policies earn promotion, the +three menu tiers all run this same `easy.json` policy at its full trained +cadence (`reaction_ticks=8`, `action_noise=0`). The tiers are labels only; +the game does not manufacture difficulty gaps by handicapping this model. + To promote a new bot into a tier: copy the chosen `Game/bots/.json` to `Game/bots/promoted/.json` (overwriting the old one), and note the source experiment + date in this section. Do this for `medium.json`/ @@ -493,10 +496,13 @@ alone (`train.py` only forwards a flag when you pass it), so ordinary runs are unaffected. Full flag list: `--opponent-mode {self_play,inert,frozen}`, `--opponent-model ` (for `frozen`), `--draw-penalty`, `--attack-goal-bias`, `--kickoff-chance`, `--near-goal-chance`, -`--air-drill-chance` (generation 4's state-setter aerial curriculum), -`--velocity-to-ball-weight`, `--ball-distance-penalty`, `--ball-touch-reward`, -`--airborne-penalty`, `--tilt-penalty`, `--ball-velocity-to-goal-weight`, -`--goal-reward`. (`--vertical-ramp`/`--pitch-roll-ramp` are gone — generation +`--air-drill-chance`, `--air-intercept-chance`, `--team-size`, +`--velocity-to-ball-weight`, `--forward-velocity-to-ball-weight`, +`--ball-distance-penalty`, `--ball-touch-reward`, `--airborne-penalty`, +`--tilt-penalty`, `--ground-tilt-penalty`, `--speed-reward-weight`, +`--ball-velocity-to-goal-weight`, `--goal-reward`, and +`--opponent-pool` with `--opponent-mode=league`. +(`--vertical-ramp`/`--pitch-roll-ramp` are gone — generation 4 has no locomotion mask/ramp to control.) ### Running it automatically @@ -546,6 +552,75 @@ Running a stage by hand (e.g. to experiment with flags before trusting the orchestrator) still works exactly as the table above describes — just call `next_run.sh`/`run_training.sh` directly with that stage's flags. +### Generation 5 follow-on + +Generation 4's stage-3 export is the foundation rather than a throwaway +baseline: all generation-5 stages resume from +`checkpoints/20260806-1939-curric-s3-gauntlet/final.zip`. Its match results +are strong, but playtesting and its final telemetry expose the next learning +targets: it spends about 39% of play above the airborne threshold while only +about 0.04% of episode-level touches are aerial, and it often travels on its +side and strikes the ball with its roof. This is a successful scoring policy +that now needs control quality and a more productive use of flight. + +Turbo remains forward-only for players and policies: it activates only with +positive forward thrust and multiplies the resulting combined thrust vector. +Generation 5 preserves the same control contract Stage 3 was trained under. + +Generation 5 adds three episode telemetry signals to TensorBoard: +`upright_fraction` (low-altitude ticks with the +ship's up vector substantially upright), `forward_motion_fraction` +(low-altitude moving ticks whose planar velocity points broadly along the +nose), and `productive_air_touch_fraction` (touches above the aerial height +that send the ball toward the attack goal). The automatic gates are +deliberately conservative catastrophe floors; every stage records its final +500-rollout tail means in `generation5_state.json` so later threshold changes +can be based on evidence instead of a single watched match. + +| Stage | Regime | Budget | Learning target | Advancement gate | +|---|---|---:|---|---| +| 4 — `handling` | Self-play, current balanced start mix | 40M (~4h) | Prefer upright, nose-led travel near the floor. Replace the orientation-agnostic speed bonus with low-altitude forward-motion shaping, and apply the stronger tilt cost only near the floor so pitch/roll remain free in genuine aerial play. | Before Stage 5: at least 80% training goal rate, at least 80% non-draw rate in the paired evaluation versus promoted Stage 3, no clear head-to-head regression, no more than 20% physical-side win imbalance, and the upright/forward-motion telemetry floors. | +| 5 — `intercepts` | Self-play with 40–50% improved air-intercept starts | 60M (~6h) | Convert existing vertical movement into useful aerial touches. Spawn a moving high ball on reachable attacking and defensive trajectories, away from walls, so contact is instrumental to scoring or saving rather than independently rewarded. | No clear regression versus Stage 4; productive aerial-touch telemetry must improve materially without reducing upright/forward-motion telemetry back to the Stage-3 baseline. | +| 6 — `league` | Live policy against a frozen opponent sampled per episode from Stage 3, Stage 4, and Stage 5 | 100M (~10h) | Prevent a narrow self-play equilibrium and consolidate ground handling, aerial interception, attack, and defence against distinct styles. | No clear head-to-head regression against any pool member plus conservative handling/aerial telemetry floors. Promote the passing result to `medium.json` after these recorded evaluations support it. | + +Stage 7 teamplay remains deliberately unconfigured. The fixed roster +observation and `team_size` plumbing can run 2v2, but there is no paired 2v2 +evaluation or team-credit reward yet; spending 120M steps without those gates +would make a pass meaningless. + +`training/generation5.py` implements Stages 4–6 separately from the completed +generation-4 orchestrator and state. It always begins Stage 4 from +`checkpoints/20260806-1939-curric-s3-gauntlet/final.zip`, then resumes each +later stage from its passing predecessor. `generation5.sh` runs it detached, +and retries/blocks use the same restart-safe pattern as the earlier +curriculum: + +```bash +cd training +.venv/bin/python generation5.py --dry-run # print and validate the next command only +./generation5.sh # run/resume in tmux +tmux attach -t cosmic-generation5 +cat generation5_state.json +``` + +Stage 4 removes the generic speed bonus, halves the old orientation-agnostic +closing reward, and adds a nose-led planar approach reward plus a tilt cost +that fades to zero by 3m altitude. Its scoring gates deliberately run before +Stage 5: becoming upright is not progress if the resulting policy stops +finishing goals. Stage 5 adds moving high-ball intercept +starts aimed toward real goals rather than a standalone air-touch reward. +Stage 6's `league` opponent mode samples a historical exported policy at each +episode reset. Each later stage preserves the preceding shaping and adds one +new difficulty. + +The physical-side gate is separate from the model-vs-model score. A paired +side swap can make an identical policy appear perfectly balanced overall even +when the Player 2 ship never functions. This caught the canonical action-frame +bug exposed by Stage 3's pitch/roll use: team 1 observations are rotated 180° +about Y, but rotation commands feed world-space torque, so team 1 pitch and +roll must be rotated back (X/Z signs inverted). Thrust remains unchanged +because it is applied through the ship's local basis. + ## Self-play notes By default both ships share the live policy (mirrored, team-relative diff --git a/training/generation5.py b/training/generation5.py new file mode 100644 index 00000000..1fe785df --- /dev/null +++ b/training/generation5.py @@ -0,0 +1,407 @@ +"""Run the post-generation-4 curriculum from the promoted Stage-3 policy. + +This is intentionally separate from curriculum.py/curriculum_state.json: +generation 4 is a completed lineage and its final checkpoint is generation +5's fixed foundation. Stages 4-6 add one difficulty at a time: + + 4 handling -- upright, nose-led low-altitude movement + 5 intercepts -- useful moving-ball aerial interceptions + 6 league -- robustness against a pool of frozen historical styles + +Each stage resumes from its passing predecessor, exports through the normal +run_training.sh parity check, records tail telemetry, and runs a paired +100-episode regression evaluation. State is restart-safe in +generation5_state.json. +""" + +from __future__ import annotations + +import argparse +import json +import pathlib +import subprocess +import sys +from datetime import datetime + +from tensorboard.backend.event_processing.event_accumulator import EventAccumulator + +TRAINING_DIR = pathlib.Path(__file__).resolve().parent +REPO_ROOT = TRAINING_DIR.parent +STATE_PATH = TRAINING_DIR / "generation5_state.json" +EVAL_HISTORY_PATH = TRAINING_DIR / "eval_history.json" + +FOUNDATION_EXPERIMENT = "20260806-1939-curric-s3-gauntlet" +FOUNDATION_CHECKPOINT = TRAINING_DIR / "checkpoints" / FOUNDATION_EXPERIMENT / "final.zip" +FOUNDATION_EXPORT = REPO_ROOT / "Game" / "bots" / f"{FOUNDATION_EXPERIMENT}.json" +PROMOTED_EASY = REPO_ROOT / "Game" / "bots" / "promoted" / "easy.json" + +MAX_RETRIES = 2 +EVAL_EPISODES = 100 +REGRESSION_MARGIN = 0.15 +STANDING_ARGS = ["--ent-coef", "0.01", "--entropy-floor"] + +# Scoring/ball-direction shaping inherited from generation 4. Handling +# replaces half the orientation-agnostic closing reward and all generic speed +# reward with nose-led ground approach, while keeping global tilt pressure +# small enough for flight and adding a stronger floor-local term. +HANDLING_REWARD_FLAGS = [ + "--velocity-to-ball-weight", "0.04", + "--forward-velocity-to-ball-weight", "0.06", + "--ball-distance-penalty", "0.01", + "--ball-touch-reward", "0.7", + "--ball-velocity-to-goal-weight", "0.06", + "--goal-reward", "80", + "--speed-reward-weight", "0.0", + "--tilt-penalty", "0.0002", + "--ground-tilt-penalty", "0.003", +] + +STAGES = [ + { + "number": 4, + "name": "handling", + "timesteps": 40_000_000, + "flags": [ + "--opponent-mode", "self_play", + "--kickoff-chance", "0.15", + "--near-goal-chance", "0.25", + "--air-drill-chance", "0.20", + "--air-intercept-chance", "0.0", + *HANDLING_REWARD_FLAGS, + ], + # Conservative catastrophe floors, not claims of mastery. Tail values + # are recorded in state so later thresholds can be based on evidence. + "telemetry_floors": { + "rollout/goal_rate": 0.80, + "rollout/upright_fraction": 0.45, + "rollout/forward_motion_fraction": 0.25, + }, + # At least 80% of the paired candidate-vs-Stage-3 episodes must end + # in a goal. This is separate from win-rate regression: a draw-heavy + # handling policy must not advance merely because neither bot won. + "evaluation_goal_rate_floor": 0.80, + # The paired side swap also measures physical spawn/team bias. This + # catches a broken team-frame action mapping even when model A's + # aggregate result looks balanced because it plays both sides. + "physical_side_imbalance_ceiling": 0.20, + }, + { + "number": 5, + "name": "intercepts", + "timesteps": 60_000_000, + "flags": [ + "--opponent-mode", "self_play", + "--kickoff-chance", "0.10", + "--near-goal-chance", "0.20", + "--air-drill-chance", "0.10", + "--air-intercept-chance", "0.45", + *HANDLING_REWARD_FLAGS, + ], + "telemetry_floors": { + "rollout/goal_rate": 0.75, + "rollout/upright_fraction": 0.40, + "rollout/forward_motion_fraction": 0.20, + "rollout/productive_air_touch_fraction": 0.005, + }, + "evaluation_goal_rate_floor": 0.75, + "physical_side_imbalance_ceiling": 0.20, + }, + { + "number": 6, + "name": "league", + "timesteps": 100_000_000, + "flags": [ + "--opponent-mode", "league", + "--kickoff-chance", "0.15", + "--near-goal-chance", "0.25", + "--air-drill-chance", "0.15", + "--air-intercept-chance", "0.25", + *HANDLING_REWARD_FLAGS, + ], + "telemetry_floors": { + "rollout/goal_rate": 0.70, + "rollout/upright_fraction": 0.35, + "rollout/forward_motion_fraction": 0.18, + "rollout/productive_air_touch_fraction": 0.003, + }, + "evaluation_goal_rate_floor": 0.70, + "physical_side_imbalance_ceiling": 0.20, + "league_pool": True, + }, +] + + +def fresh_state() -> dict: + return {"stage_index": 0, "attempt": 0, "status": "in_progress", "log": []} + + +def load_state() -> dict: + return json.loads(STATE_PATH.read_text()) if STATE_PATH.exists() else fresh_state() + + +def save_state(state: dict) -> None: + STATE_PATH.write_text(json.dumps(state, indent=2) + "\n") + + +def passing_entry(state: dict, stage_index: int) -> dict: + for entry in state["log"]: + if entry["stage_index"] == stage_index and entry["decision"] == "pass": + return entry + raise RuntimeError(f"No passing generation-5 stage index {stage_index}") + + +def previous_attempt_entry(state: dict, stage_index: int, attempt: int) -> dict: + for entry in reversed(state["log"]): + if entry["stage_index"] == stage_index and entry["attempt"] == attempt - 1: + return entry + raise RuntimeError(f"No previous attempt for stage index {stage_index}, attempt {attempt}") + + +def resume_checkpoint(state: dict, stage_index: int, attempt: int, foundation: pathlib.Path) -> pathlib.Path: + if attempt > 0: + exp = previous_attempt_entry(state, stage_index, attempt)["experiment"] + return TRAINING_DIR / "checkpoints" / exp / "final.zip" + if stage_index == 0: + return foundation + exp = passing_entry(state, stage_index - 1)["experiment"] + return TRAINING_DIR / "checkpoints" / exp / "final.zip" + + +def reference_export(state: dict, stage_index: int) -> pathlib.Path: + if stage_index == 0: + return PROMOTED_EASY + exp = passing_entry(state, stage_index - 1)["experiment"] + return REPO_ROOT / "Game" / "bots" / f"{exp}.json" + + +def league_pool(state: dict) -> list[pathlib.Path]: + stage4 = passing_entry(state, 0)["experiment"] + stage5 = passing_entry(state, 1)["experiment"] + return [ + FOUNDATION_EXPORT, + REPO_ROOT / "Game" / "bots" / f"{stage4}.json", + REPO_ROOT / "Game" / "bots" / f"{stage5}.json", + ] + + +def telemetry_tail(experiment: str, count: int = 500) -> dict[str, float]: + log_dirs = sorted((TRAINING_DIR / "logs").glob(f"{experiment}_*")) + if not log_dirs: + return {} + event_files = sorted(log_dirs[-1].glob("events.out.tfevents.*")) + if not event_files: + return {} + accumulator = EventAccumulator(str(event_files[-1]), size_guidance={"scalars": 0}) + accumulator.Reload() + result = {} + for tag in accumulator.Tags().get("scalars", []): + if not tag.startswith("rollout/"): + continue + values = [point.value for point in accumulator.Scalars(tag)[-count:]] + if values: + result[tag] = sum(values) / len(values) + return result + + +def telemetry_passes(stage: dict, telemetry: dict[str, float]) -> tuple[bool, list[str]]: + failures = [] + for metric, floor in stage.get("telemetry_floors", {}).items(): + value = telemetry.get(metric) + if value is None: + failures.append(f"{metric} missing") + elif value < floor: + failures.append(f"{metric}={value:.4f} < {floor:.4f}") + return not failures, failures + + +def run_training(state: dict, stage_index: int, attempt: int, args) -> str: + stage = STAGES[stage_index] + suffix = "" if attempt == 0 else f"-retry{attempt}" + experiment = f"{datetime.now().strftime('%Y%m%d-%H%M')}-gen5-s{stage['number']}-{stage['name']}{suffix}" + resume = resume_checkpoint(state, stage_index, attempt, pathlib.Path(args.foundation_checkpoint)) + if not resume.exists(): + raise FileNotFoundError(f"Resume checkpoint not found: {resume}") + cmd = [ + "./run_training.sh", experiment, + "--timesteps", str(stage["timesteps"]), + "--n-parallel", str(args.n_parallel), + "--speedup", str(args.speedup), + "--resume", str(resume), + *STANDING_ARGS, + *stage["flags"], + ] + if stage.get("league_pool"): + pool = league_pool(state) + missing = [str(path) for path in pool if not path.exists()] + if missing: + raise FileNotFoundError(f"League pool models missing: {missing}") + cmd += ["--opponent-pool", ",".join(str(path) for path in pool)] + print(f"\n=== Generation 5 Stage {stage['number']} {stage['name']} attempt {attempt + 1} ===") + print(" ".join(cmd)) + if args.dry_run: + return experiment + subprocess.run(cmd, cwd=TRAINING_DIR, check=True) + return experiment + + +def evaluate(experiment: str, reference: pathlib.Path, args) -> dict: + candidate = REPO_ROOT / "Game" / "bots" / f"{experiment}.json" + cmd = [ + ".venv/bin/python", "evaluate.py", str(candidate), str(reference), + "--episodes", str(EVAL_EPISODES), "--speedup", str(args.speedup), + ] + if args.godot_bin: + cmd += ["--godot_bin", args.godot_bin] + subprocess.run(cmd, cwd=TRAINING_DIR, check=True) + return json.loads(EVAL_HISTORY_PATH.read_text())[-1] + + +def match_passes(record: dict) -> bool: + candidate = record["wins_a"] / record["episodes"] + reference = record["wins_b"] / record["episodes"] + return reference - candidate < REGRESSION_MARGIN + + +def evaluation_goal_rate(record: dict) -> float: + """Fraction of paired evaluation episodes that ended in either bot scoring.""" + return (record["wins_a"] + record["wins_b"]) / record["episodes"] + + +def physical_side_imbalance(record: dict) -> float: + """Absolute physical-team win margin as a fraction of all episodes.""" + physical = record["physical_team_wins"] + return abs(physical["team_0"] - physical["team_1"]) / record["episodes"] + + +def commit_progress(experiment: str) -> None: + subprocess.run(["git", "add", STATE_PATH.name, EVAL_HISTORY_PATH.name], cwd=TRAINING_DIR, check=True) + if subprocess.run(["git", "diff", "--cached", "--quiet"], cwd=TRAINING_DIR).returncode == 0: + return + subprocess.run( + ["git", "commit", "-m", f"chore(training): generation 5 progress after {experiment}"], + cwd=TRAINING_DIR, + check=True, + ) + subprocess.run(["git", "push"], cwd=TRAINING_DIR, check=True) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--n-parallel", type=int, default=14) + parser.add_argument("--speedup", type=int, default=16) + parser.add_argument("--godot-bin", default=None, help="Godot binary for post-stage evaluation") + parser.add_argument("--foundation-checkpoint", default=str(FOUNDATION_CHECKPOINT)) + parser.add_argument("--force-retry", action="store_true") + parser.add_argument("--skip-to-next-stage", action="store_true") + parser.add_argument("--dry-run", action="store_true", help="Print the next run command without executing it") + args = parser.parse_args() + + state = load_state() + if state["status"] == "done": + print("Generation 5 is already complete.") + return + if state["status"] == "blocked": + if args.force_retry: + state["attempt"] += 1 + state["status"] = "in_progress" + save_state(state) + elif args.skip_to_next_stage: + state["stage_index"] += 1 + state["attempt"] = 0 + state["status"] = "in_progress" + save_state(state) + else: + stage = STAGES[state["stage_index"]] + print(f"BLOCKED at Stage {stage['number']} {stage['name']}; inspect {STATE_PATH.name}.") + print("Use --force-retry after adjustment or --skip-to-next-stage after human review.") + sys.exit(1) + + while state["stage_index"] < len(STAGES): + stage_index = state["stage_index"] + attempt = state["attempt"] + stage = STAGES[stage_index] + experiment = run_training(state, stage_index, attempt, args) + if args.dry_run: + return + + telemetry = telemetry_tail(experiment) + telemetry_ok, telemetry_failures = telemetry_passes(stage, telemetry) + references = [reference_export(state, stage_index)] + if stage.get("league_pool"): + references.extend(league_pool(state)) + # Preserve order while avoiding a duplicate Stage-5 evaluation in + # the league stage (its predecessor is also in the pool). + references = list(dict.fromkeys(references)) + records = [evaluate(experiment, reference, args) for reference in references] + match_ok = all(match_passes(record) for record in records) + evaluation_goal_floor = stage.get("evaluation_goal_rate_floor", 0.0) + evaluation_goal_failures = [ + f"{pathlib.Path(record['model_b']).name}: goal_rate={evaluation_goal_rate(record):.3f} " + f"< {evaluation_goal_floor:.3f}" + for record in records + if evaluation_goal_rate(record) < evaluation_goal_floor + ] + scoring_ok = not evaluation_goal_failures + side_imbalance_ceiling = stage.get("physical_side_imbalance_ceiling", 1.0) + side_balance_failures = [ + f"{pathlib.Path(record['model_b']).name}: physical_side_imbalance=" + f"{physical_side_imbalance(record):.3f} > {side_imbalance_ceiling:.3f}" + for record in records + if physical_side_imbalance(record) > side_imbalance_ceiling + ] + side_balance_ok = not side_balance_failures + decision = "pass" if match_ok and telemetry_ok else "fail" + if not scoring_ok or not side_balance_ok: + decision = "fail" + entry = { + "stage_index": stage_index, + "stage_number": stage["number"], + "stage_name": stage["name"], + "experiment": experiment, + "attempt": attempt, + "telemetry_tail": telemetry, + "telemetry_failures": telemetry_failures, + "evaluation_goal_failures": evaluation_goal_failures, + "side_balance_failures": side_balance_failures, + "eval": records[0], + "evals": records, + "decision": decision, + } + state["log"].append(entry) + print( + f"{experiment}: match={'pass' if match_ok else 'fail'}, " + f"scoring={'pass' if scoring_ok else 'fail'}, " + f"side_balance={'pass' if side_balance_ok else 'fail'}, " + f"telemetry={'pass' if telemetry_ok else 'fail'} -> {decision}" + ) + for failure in telemetry_failures: + print(f" {failure}") + for failure in evaluation_goal_failures: + print(f" {failure}") + for failure in side_balance_failures: + print(f" {failure}") + + if decision == "pass": + state["stage_index"] += 1 + state["attempt"] = 0 + save_state(state) + commit_progress(experiment) + continue + if attempt >= MAX_RETRIES: + state["status"] = "blocked" + save_state(state) + commit_progress(experiment) + print(f"BLOCKED after {MAX_RETRIES + 1} attempts at Stage {stage['number']}.") + sys.exit(1) + state["attempt"] += 1 + save_state(state) + commit_progress(experiment) + + state["status"] = "done" + save_state(state) + commit_progress(state["log"][-1]["experiment"]) + print("Generation 5 complete: handling, intercepts, and league stages passed.") + + +if __name__ == "__main__": + main() diff --git a/training/generation5.sh b/training/generation5.sh new file mode 100755 index 00000000..bea449de --- /dev/null +++ b/training/generation5.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Run/resume the post-Stage-3 generation-5 curriculum in a detached tmux +# session. Safe to disconnect; rerunning attaches to the live session. +set -euo pipefail +cd "$(dirname "$0")" + +SESSION="cosmic-generation5" +TB_PORT=6006 + +command -v tmux >/dev/null 2>&1 || { echo "tmux is required: sudo apt install tmux" >&2; exit 1; } + +if tmux has-session -t "$SESSION" 2>/dev/null; then + echo "Session '$SESSION' already running — attaching (detach with Ctrl-B then D)." + exec tmux attach -t "$SESSION" +fi + +tmux new-session -d -s "$SESSION" -n curriculum \ + ".venv/bin/python generation5.py $*; echo; echo '=== generation5.py exited — press Enter to close ==='; read" + +if ! (exec 3<>"/dev/tcp/127.0.0.1/$TB_PORT") 2>/dev/null; then + tmux new-window -d -t "$SESSION" -n dashboard \ + ".venv/bin/tensorboard --logdir logs --host 0.0.0.0 --port $TB_PORT" +fi + +IP=$(hostname -I 2>/dev/null | awk '{print $1}') +echo "Generation 5 started in tmux session '$SESSION'." +echo " watch it: tmux attach -t $SESSION" +echo " dashboard: http://${IP:-}:$TB_PORT" +echo " progress: cat generation5_state.json" diff --git a/training/generation5_state.json b/training/generation5_state.json new file mode 100644 index 00000000..30c376dc --- /dev/null +++ b/training/generation5_state.json @@ -0,0 +1,6 @@ +{ + "stage_index": 0, + "attempt": 0, + "status": "in_progress", + "log": [] +} diff --git a/training/test_generation5.py b/training/test_generation5.py new file mode 100644 index 00000000..b3304bbb --- /dev/null +++ b/training/test_generation5.py @@ -0,0 +1,89 @@ +"""Offline checks for the generation-5 stage configuration and gates.""" + +import unittest + +import generation5 + + +def flag_value(flags: list[str], name: str) -> str: + index = flags.index(name) + return flags[index + 1] + + +class Generation5ConfigTests(unittest.TestCase): + def test_stage_sequence_and_lineage(self) -> None: + self.assertEqual([stage["number"] for stage in generation5.STAGES], [4, 5, 6]) + state = generation5.fresh_state() + checkpoint = generation5.resume_checkpoint( + state, stage_index=0, attempt=0, foundation=generation5.FOUNDATION_CHECKPOINT + ) + self.assertEqual(checkpoint, generation5.FOUNDATION_CHECKPOINT) + + def test_start_state_probabilities_leave_random_remainder(self) -> None: + for stage in generation5.STAGES: + flags = stage["flags"] + total = sum( + float(flag_value(flags, name)) + for name in ( + "--kickoff-chance", + "--near-goal-chance", + "--air-drill-chance", + "--air-intercept-chance", + ) + ) + with self.subTest(stage=stage["name"]): + self.assertLessEqual(total, 1.0) + + def test_only_league_stage_requests_pool(self) -> None: + self.assertEqual( + [stage["name"] for stage in generation5.STAGES if stage.get("league_pool")], + ["league"], + ) + self.assertEqual(flag_value(generation5.STAGES[2]["flags"], "--opponent-mode"), "league") + + def test_telemetry_floors_fail_closed_on_missing_metric(self) -> None: + ok, failures = generation5.telemetry_passes( + generation5.STAGES[0], {"rollout/upright_fraction": 1.0} + ) + self.assertFalse(ok) + self.assertIn("rollout/forward_motion_fraction missing", failures) + + def test_match_gate_only_blocks_clear_regression(self) -> None: + self.assertTrue( + generation5.match_passes({"wins_a": 40, "wins_b": 54, "episodes": 100}) + ) + self.assertFalse( + generation5.match_passes({"wins_a": 40, "wins_b": 55, "episodes": 100}) + ) + + def test_evaluation_goal_rate_counts_either_scorer(self) -> None: + self.assertEqual( + generation5.evaluation_goal_rate( + {"wins_a": 45, "wins_b": 35, "draws": 20, "episodes": 100} + ), + 0.8, + ) + + def test_physical_side_imbalance_exposes_broken_player_slot(self) -> None: + self.assertEqual( + generation5.physical_side_imbalance( + { + "physical_team_wins": {"team_0": 90, "team_1": 5}, + "episodes": 100, + } + ), + 0.85, + ) + self.assertEqual( + generation5.physical_side_imbalance( + { + "physical_team_wins": {"team_0": 42, "team_1": 38}, + "episodes": 100, + } + ), + 0.04, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/training/train.py b/training/train.py index 4cb76f64..5c569a3b 100644 --- a/training/train.py +++ b/training/train.py @@ -62,8 +62,7 @@ class GoalRateCallback(BaseCallback): class FlightTelemetryCallback(BaseCallback): - """Logs rollout/{airborne_fraction,mean_altitude,air_touch_fraction, - vertical_thrust_mean} — leading indicators for curriculum generation 4's + """Logs flight and handling telemetry — leading indicators for curriculum generation 4's core hypothesis (a discrete action space lets the policy actually hold a sustained vertical set-point, e.g. hovering), visible from the very first rollout instead of only in a win-rate number measured a full @@ -72,7 +71,15 @@ class FlightTelemetryCallback(BaseCallback): "airborne_fraction", "mean_altitude", "air_touch_fraction", "vertical_thrust_mean")) — see ShipAIController.get_info.""" - _KEYS = ("airborne_fraction", "mean_altitude", "air_touch_fraction", "vertical_thrust_mean") + _KEYS = ( + "airborne_fraction", + "mean_altitude", + "air_touch_fraction", + "vertical_thrust_mean", + "productive_air_touch_fraction", + "upright_fraction", + "forward_motion_fraction", + ) def _on_step(self) -> bool: return True @@ -287,13 +294,18 @@ def parse_args(): ) curriculum.add_argument( "--opponent-mode", - choices=["self_play", "inert", "frozen"], + choices=["self_play", "inert", "frozen", "league"], default=None, help="self_play (default): both ships are live trainees. inert: team 1 is a " "do-nothing placeholder (isolated scoring practice). frozen: team 1 runs a " - "fixed exported policy (--opponent-model)", + "fixed exported policy (--opponent-model); league: sample a fixed policy per episode " + "from --opponent-pool", ) curriculum.add_argument("--opponent-model", default=None, help="Exported policy .json for --opponent-mode=frozen") + curriculum.add_argument( + "--opponent-pool", default=None, + help="Comma-separated exported policy paths for --opponent-mode=league; one is sampled per episode", + ) curriculum.add_argument( "--draw-penalty", type=float, default=None, help="One-time penalty when an episode times out with no goal" ) @@ -310,6 +322,14 @@ def parse_args(): help="Overrides air_drill_chance: ball spawned high, both ships spawned low and lateral — " "unsolvable without climbing (curriculum generation 4's state-setter aerial curriculum)", ) + curriculum.add_argument( + "--air-intercept-chance", type=float, default=None, + help="Moving high-ball interception starts aimed at a real goal (generation-5 aerial stage)", + ) + curriculum.add_argument( + "--team-size", type=int, choices=range(1, 6), default=None, + help="Ships per team (1-5); generation-5 automated stages remain 1v1 until 2v2 evaluation exists", + ) curriculum.add_argument( "--tilt-penalty", type=float, default=None, help="Overrides ShipAIController.tilt_penalty (dense per-tick cost scaled by non-upright tilt)", @@ -318,6 +338,10 @@ def parse_args(): "--velocity-to-ball-weight", type=float, default=None, help="Overrides ShipAIController.velocity_to_ball_weight (dense reward for closing speed toward the ball)", ) + curriculum.add_argument( + "--forward-velocity-to-ball-weight", type=float, default=None, + help="Low-altitude dense reward for nose-led planar approach toward the ball", + ) curriculum.add_argument( "--ball-distance-penalty", type=float, default=None, help="Overrides ShipAIController.ball_distance_penalty (dense per-tick cost scaled by distance to the ball)", @@ -330,6 +354,14 @@ def parse_args(): "--airborne-penalty", type=float, default=None, help="Overrides ShipAIController.airborne_penalty (dense per-tick cost scaled by height above the floor)", ) + curriculum.add_argument( + "--ground-tilt-penalty", type=float, default=None, + help="Low-altitude-only tilt cost that fades to zero by the handling-height threshold", + ) + curriculum.add_argument( + "--speed-reward-weight", type=float, default=None, + help="Overrides the orientation-agnostic own-speed reward (generation 5 handling sets it to zero)", + ) curriculum.add_argument( "--ball-velocity-to-goal-weight", type=float, default=None, help="Overrides ShipAIController.ball_velocity_to_goal_weight (dense reward for the ball's velocity toward the attack goal)", @@ -349,16 +381,22 @@ def _curriculum_kwargs(args) -> dict: mapping = { "opponent_mode": args.opponent_mode, "opponent_model": args.opponent_model, + "opponent_model_pool": args.opponent_pool, "draw_penalty": args.draw_penalty, "attack_goal_bias": args.attack_goal_bias, "kickoff_state_chance": args.kickoff_chance, "ball_near_goal_chance": args.near_goal_chance, "air_drill_chance": args.air_drill_chance, + "air_intercept_chance": args.air_intercept_chance, + "team_size": args.team_size, "ai_tilt_penalty": args.tilt_penalty, + "ai_ground_tilt_penalty": args.ground_tilt_penalty, "ai_velocity_to_ball_weight": args.velocity_to_ball_weight, + "ai_forward_velocity_to_ball_weight": args.forward_velocity_to_ball_weight, "ai_ball_distance_penalty": args.ball_distance_penalty, "ai_ball_touch_reward": args.ball_touch_reward, "ai_airborne_penalty": args.airborne_penalty, + "ai_speed_reward_weight": args.speed_reward_weight, "ai_ball_velocity_to_goal_weight": args.ball_velocity_to_goal_weight, "goal_reward": args.goal_reward, } @@ -395,6 +433,9 @@ def main(): "mean_altitude", "air_touch_fraction", "vertical_thrust_mean", + "productive_air_touch_fraction", + "upright_fraction", + "forward_motion_fraction", ), )