feat(*): add staged curriculum training with an automated stage-by-stage orchestrator

2026-09-11 00:14:00 +00:00 · 2026-07-21 12:38:51 +01:00
parent 1d539cc8c7
commit 8e3fafcc8b
7 changed files with 581 additions and 18 deletions
@@ -71,6 +71,16 @@ extends AIController3D
 # chance appears, instead of a free way to keep collecting dense reward.
@export var time_penalty := 0.001

+# Locomotion curriculum: when false, the corresponding action axes are
+# discarded in set_action before reaching the ship, so the ship stays
+# grounded and only yaws — basic scoring/defending doesn't need 3D flight.
+# This masks the *effect* of thrust.y/rotation.x/rotation.z, not the action
+# space's shape: the policy still outputs values for these axes (still
+# contributing to PPO's entropy/log-prob), they're just discarded here, so
+# checkpoints stay resumable once a later curriculum stage re-enables them.
+@export var allow_vertical := true
+@export var allow_pitch_roll := true
+
 # Contact normals with y above this are floor contact (exempt from the wall
 # penalty); below it they read as wall (sideways) or ceiling (downward).
 const FLOOR_NORMAL_MIN_Y := 0.7
@@ -128,8 +138,11 @@ func get_action_space() -> Dictionary:
 func set_action(action) -> void:
 	var thrust: Array = action["thrust"]
 	var rot: Array = action["rotation"]
-	rl_controller.action.thrust = Vector3(thrust[0], thrust[1], thrust[2])
-	rl_controller.action.rotation = Vector3(rot[0], rot[1], rot[2])
+	var thrust_y: float = thrust[1] if allow_vertical else 0.0
+	var pitch: float = rot[0] if allow_pitch_roll else 0.0
+	var roll: float = rot[2] if allow_pitch_roll else 0.0
+	rl_controller.action.thrust = Vector3(thrust[0], thrust_y, thrust[2])
+	rl_controller.action.rotation = Vector3(pitch, rot[1], roll)
 	rl_controller.action.turbo = int(action["turbo"]) == 1


@@ -16,6 +16,13 @@ extends GameMode
 # instead driven by those exported policies via AIShipController; each episode
 # ends at the first goal (or a draw on timeout), and a final machine-readable
 # "EVAL_RESULT {...}" line is printed before quitting.
+#
+# Curriculum mode (used by training/train.py's --opponent-mode/--draw-penalty/
+# etc. flags, see TRAINING.md): --opponent_mode=inert|frozen swaps team 1's
+# live self-play agent for a do-nothing placeholder or a fixed exported
+# policy; --ai_<name>=<value> and the TrainingMode-level overrides below let
+# a run retune reward shaping / episode-start mix without touching script
+# defaults. See _parse_curriculum_args.

@export var episode_length_seconds := 30.0
 # Run01-vs-run02 eval (see training/eval_history.json) came back 87.5% draws:
@@ -25,6 +32,13 @@ extends GameMode
 # Raised well above that ceiling so a real scoring chance always beats
 # continuing to farm dense reward for however long is left in the episode.
@export var goal_reward := 40.0
+# One-time penalty applied to every agent when an episode times out with no
+# goal scored (see _physics_process's truncation branch) — distinct from
+# ShipAIController's per-tick time_penalty, which accrues regardless of
+# outcome and doesn't specifically mark "this episode ended undecided."
+# Default 0 (off) so ordinary runs are unaffected; curriculum stage 3 turns
+# this on via --draw_penalty to teach that a draw is still a failure.
+@export var draw_penalty := 0.0

 # Episode-start state mix; remaining probability = fully random state.
@export_range(0.0, 1.0) var kickoff_state_chance := 0.2
@@ -34,6 +48,12 @@ extends GameMode
 # was 1 in 5 episode starts; most training time was spent in generic
 # midfield play where a finish never comes up.
@export_range(0.0, 1.0) var ball_near_goal_chance := 0.35
+# Which goal _place_ball_near_goal() favors: 0.5 = uniform between both goals
+# (default, matches historical behaviour). 1.0 = always the goal team 0
+# attacks — used by curriculum stage 1 (--attack_goal_bias=1.0) so a lone
+# trainee's near-goal resets are always finishing chances, not a coin flip
+# between attacking and defending an empty net.
+@export_range(0.0, 1.0) var attack_goal_bias := 0.5

 # Placement bounds for randomized episode starts, derived from the standard
 # enclosure (ArenaBoundary). The inset keeps a randomly oriented ship (1x1x4
@@ -77,9 +97,27 @@ var _eval_draws := 0
 var _eval_episodes_done := 0
 var _episode_ticks := 0

+# Curriculum mode state (see _parse_curriculum_args). "self_play" (default)
+# is today's only historical behaviour: both ships are live trainees sharing
+# the policy. "inert" gives team 1 a do-nothing placeholder ship (no bot
+# configured, same as MatchMode's fallback) so a lone trainee can drill
+# scoring against an empty net. "frozen" gives team 1 a fixed exported
+# policy via AIShipController — the same wiring the eval branch above
+# already uses, just for one side of a live training episode.
+var _opponent_mode := "self_play"
+var _opponent_model_path := ""
+# ShipAIController @export overrides collected from --ai_<name>=<value> args,
+# applied to every ShipAIController this run creates (see _attach_agent).
+var _ai_overrides := {}
+# Ships excluded from _attach_agent (the "inert" opponent) skip randomized
+# per-episode placement in _place_ships_random so they stay parked at their
+# arena spawn instead of drifting into the play area as a stray obstacle.
+var _inert_ships: Array[Ship] = []
+

 func _start() -> void:
 	_parse_eval_args()
+	_parse_curriculum_args()
 	spawn_ball()
 	if _eval:
 		for team in [0, 1]:
@@ -87,18 +125,37 @@ func _start() -> void:
 			bot.model_path = _eval_models[team]
 			spawn_ship(team, 0, bot)
 		return
+
 	var ship_team0 := spawn_ship(0, 0, RLShipController.new())
-	var ship_team1 := spawn_ship(1, 0, RLShipController.new())
+	var ship_team1: Ship
+	match _opponent_mode:
+		"inert":
+			ship_team1 = spawn_ship(1, 0, ShipController.new())
+			_inert_ships.append(ship_team1)
+		"frozen":
+			var bot := AIShipController.new()
+			bot.model_path = _opponent_model_path
+			ship_team1 = spawn_ship(1, 0, bot)
+		_:
+			ship_team1 = spawn_ship(1, 0, RLShipController.new())
+
 	_attach_agent(ship_team0, ship_team1)
-	_attach_agent(ship_team1, ship_team0)
+	if _opponent_mode == "self_play":
+		_attach_agent(ship_team1, ship_team0)


-func _parse_eval_args() -> void:
+# Shared "--key=value" cmdline scan used by both eval and curriculum parsing.
+func _cmdline_kv_args() -> Dictionary:
 	var args := {}
 	for argument in OS.get_cmdline_args():
 		if argument.begins_with("--") and argument.find("=") > -1:
 			var key_value := argument.lstrip("--").split("=", true, 1)
 			args[key_value[0]] = key_value[1]
+	return args
+
+
+func _parse_eval_args() -> void:
+	var args := _cmdline_kv_args()
 	if args.has("eval_model_a") and args.has("eval_model_b"):
 		_eval = true
 		_eval_models[0] = args["eval_model_a"]
@@ -106,22 +163,96 @@ func _parse_eval_args() -> void:
 		_eval_episodes = int(args.get("eval_episodes", str(_eval_episodes)))


+# TrainingMode @export names a curriculum run may override from the cmdline.
+# Explicit allow-list (not reflection) so a typo'd flag fails loudly instead
+# of silently matching an unrelated inherited export.
+const TRAINING_MODE_OVERRIDES := [
+	"goal_reward", "draw_penalty", "kickoff_state_chance",
+	"ball_near_goal_chance", "attack_goal_bias",
+]
+# ShipAIController @export names a curriculum run may override, read as
+# --ai_<name>=<value> to avoid colliding with the names above.
+const SHIP_AI_OVERRIDES := [
+	"ball_touch_reward", "ball_touch_cooldown_ticks", "ball_touch_direction_floor",
+	"velocity_to_ball_weight", "ball_velocity_to_goal_weight", "ball_distance_penalty",
+	"wall_contact_penalty", "tilt_penalty", "speed_reward_weight", "time_penalty",
+	"allow_vertical", "allow_pitch_roll",
+]
+
+
+func _parse_curriculum_args() -> void:
+	var args := _cmdline_kv_args()
+	if args.has("opponent_mode"):
+		_opponent_mode = args["opponent_mode"]
+	_opponent_model_path = args.get("opponent_model", _opponent_model_path)
+
+	for name in TRAINING_MODE_OVERRIDES:
+		if args.has(name):
+			set(name, _typed_like(args[name], get(name)))
+
+	for name in SHIP_AI_OVERRIDES:
+		var key := "ai_%s" % name
+		if args.has(key):
+			_ai_overrides[name] = _typed_like(args[key], _ai_default(name))
+
+
+# Parses a cmdline string into the same Variant type as `sample` (bool/int/
+# float pass through Godot's str()-based conversions; anything else stays a
+# String), so callers can `set()` it straight onto a typed @export var.
+func _typed_like(value: String, sample) -> Variant:
+	match typeof(sample):
+		TYPE_BOOL:
+			return value.to_lower() in ["1", "true", "yes"]
+		TYPE_INT:
+			return value.to_int()
+		TYPE_FLOAT:
+			return value.to_float()
+		_:
+			return value
+
+
+# ShipAIController isn't in the scene tree until _attach_agent instantiates
+# one, so overrides need a default to type-match against up front; this
+# mirrors ship_ai_controller.gd's own @export defaults.
+func _ai_default(name: String) -> Variant:
+	match name:
+		"ball_touch_reward": return 0.4
+		"ball_touch_cooldown_ticks": return 60
+		"ball_touch_direction_floor": return 0.3
+		"velocity_to_ball_weight": return 0.02
+		"ball_velocity_to_goal_weight": return 0.004
+		"ball_distance_penalty": return 0.002
+		"wall_contact_penalty": return 0.0025
+		"tilt_penalty": return 0.002
+		"speed_reward_weight": return 0.004
+		"time_penalty": return 0.001
+		"allow_vertical", "allow_pitch_roll": return true
+		_: return null
+
+
 func _attach_agent(ship: Ship, opponent: Ship) -> void:
 	var agent := ShipAIController.new()
 	agent.name = "ShipAIController"
 	agent.reset_after = int(episode_length_seconds * TICKS_PER_SIM_SECOND)
+	for key in _ai_overrides:
+		agent.set(key, _ai_overrides[key])
 	ship.add_child(agent)
 	agent.setup(ship, ship.controller as RLShipController, ball, opponent, _attack_goal_position(ship.team))
 	_agents.append(agent)


-# The goal this team scores into: the one the opponent defends/concedes.
-func _attack_goal_position(team: int) -> Vector3:
+# The goal a team scores into: the one the opponent defends/concedes.
+func _goal_for_team(team: int) -> Goal:
 	for goal in arena.get_goals():
 		if goal.team == 1 - team:
-			return goal.global_position
+			return goal
 	push_error("TrainingMode: no goal found for team %d to attack" % team)
-	return Vector3.ZERO
+	return null
+
+
+func _attack_goal_position(team: int) -> Vector3:
+	var goal := _goal_for_team(team)
+	return goal.global_position if goal else Vector3.ZERO


 func _physics_process(_delta):
@@ -144,6 +275,7 @@ func _physics_process(_delta):
 	if needs_reset:
 		if truncated:
 			for agent in _agents:
+				agent.reward -= draw_penalty
 				agent.done = true
 		_reset_episode()
 		return
@@ -219,9 +351,10 @@ func _place_ball_random() -> void:


 # Attacking/defending drill states: ball close to a goal, moving toward it.
+# Which goal is picked is biased by attack_goal_bias (0.5 = uniform between
+# both, matching historical behaviour; 1.0 = always the goal team 0 attacks).
 func _place_ball_near_goal() -> void:
-	var goals := arena.get_goals()
-	var goal: Goal = goals[randi() % goals.size()]
+	var goal := _goal_for_team(0) if randf() < attack_goal_bias else _goal_for_team(1)
 	var toward_centre := -signf(goal.global_position.z)
 	var position := Vector3(
 		randf_range(-4.0, 4.0),
@@ -235,6 +368,11 @@ func _place_ball_near_goal() -> void:

 func _place_ships_random() -> void:
 	for ship in ships:
+		# Inert opponents (opponent_mode=inert) stay parked at their arena
+		# spawn instead of drifting into the play area as a stray obstacle —
+		# see _inert_ships.
+		if ship in _inert_ships:
+			continue
 		var orientation := Basis.from_euler(Vector3(
 			randf_range(-0.4, 0.4),
 			randf_range(-PI, PI),