From 3049c42867515cf5978b167669daba5614414800 Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Wed, 5 Aug 2026 09:17:56 +0100 Subject: [PATCH] feat(training): support N-vs-M matches with persistent per-ship spawn IDs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends ShipObservations beyond the old self+1-opponent layout to padded teammate/opponent arrays (MAX_TEAMMATES=4, MAX_OPPONENTS=5, SIZE=83), zero-filling slots past the real roster size the same way the old single- opponent slot was zero-filled when absent. Slot stability across ticks requires a persistent identity: Ship gains spawn_index (set once by GameMode.spawn_ship, never reassigned — there's no despawn path anywhere in this codebase, so a roster is fixed for the whole episode/match). ai_ship_controller.gd's opponent discovery is rewritten from "first non-self ship" to classify every other ship by team and sort by spawn_index; training_mode.gd/ship_ai_controller.gd carry the equivalent sorted lists through the training path so both agree on slot assignment for the same roster. training_mode.gd and match_mode.gd both gain a team_size export (default 1, so every existing curriculum script and match keeps today's 1v1 behaviour unchanged). This is plumbing only: no 2v2+ curriculum or reward design, and no match-mode UI to pick team size, has been done yet. The two checkpoints in Game/bots/promoted/ are fitted to the old 35-float layout and are not migrated — expected to go stale until the next training run. --- Game/scripts/ai_ship_controller.gd | 38 ++++++++++--- Game/scripts/game_mode.gd | 1 + Game/scripts/match_mode.gd | 11 +++- Game/scripts/ship.gd | 7 +++ Game/scripts/ship_ai_controller.gd | 15 +++-- Game/scripts/ship_observations.gd | 88 ++++++++++++++++++++---------- Game/scripts/training_mode.gd | 64 ++++++++++++++++------ 7 files changed, 164 insertions(+), 60 deletions(-) diff --git a/Game/scripts/ai_ship_controller.gd b/Game/scripts/ai_ship_controller.gd index 26b16c87..a3156b7b 100644 --- a/Game/scripts/ai_ship_controller.gd +++ b/Game/scripts/ai_ship_controller.gd @@ -35,7 +35,8 @@ var _action := ShipAction.new() var _ticks_until_decision := 0 var _ship: Ship -var _opponent: Ship +var _teammates: Array[Ship] = [] +var _opponents: Array[Ship] = [] var _ball: RigidBody3D var _attack_goal_position: Vector3 var _scene_refs_ready := false @@ -60,7 +61,7 @@ func get_action() -> ShipAction: func _decide() -> void: - var obs := ShipObservations.build(_ship, _opponent, _ball, _attack_goal_position) + var obs := ShipObservations.build(_ship, _teammates, _opponents, _ball, _attack_goal_position) var out := _policy.forward(obs) # See ShipActionCodec for the decode — the single source of truth shared # with the training side, so this must never reimplement layout/ordering @@ -76,21 +77,40 @@ func _decide() -> void: _action = ShipActionCodec.from_logits(out, action_noise) -# Find ship/ball/opponent/goal once everything is spawned. ShipAction axes -# are body-frame so only observations need team context (ShipObservations). +# Find ship/ball/teammates/opponents/goal once everything is spawned. +# ShipAction axes are body-frame so only observations need team context +# (ShipObservations). Rosters never change mid-match (no despawn path exists +# anywhere in this codebase), so this only needs to run once — sorted by +# spawn_index so a given ship keeps the same observation slot for the whole +# match, matching TrainingMode's identically-sorted lists. func _discover_scene_refs() -> bool: _ship = get_parent() as Ship if _ship == null or not is_inside_tree(): return false _ball = get_tree().get_first_node_in_group("ball") + if _ball == null: + return false + # Cleared, not just appended to: if an earlier call reached this point but + # a later check still failed, a retry must not re-append onto whatever it + # already collected — that would duplicate every ship in the roster. + _teammates.clear() + _opponents.clear() for node in get_tree().get_nodes_in_group("ship"): - if node != _ship: - _opponent = node - break + var other := node as Ship + if other == _ship: + continue + if other.team == _ship.team: + _teammates.append(other) + else: + _opponents.append(other) + _teammates.sort_custom(_by_spawn_index) + _opponents.sort_custom(_by_spawn_index) for goal in get_tree().get_nodes_in_group("goal"): if goal.team == 1 - _ship.team: _attack_goal_position = goal.global_position - if _ball == null: - return false _scene_refs_ready = true return true + + +static func _by_spawn_index(a: Ship, b: Ship) -> bool: + return a.spawn_index < b.spawn_index diff --git a/Game/scripts/game_mode.gd b/Game/scripts/game_mode.gd index 60417adf..bf8daf11 100644 --- a/Game/scripts/game_mode.gd +++ b/Game/scripts/game_mode.gd @@ -90,6 +90,7 @@ func spawn_ship(team: int, spawn_index: int = 0, controller: ShipController = nu var spawn_transform := spawns[spawn_index] if spawn_index < spawns.size() else Transform3D.IDENTITY ship.global_transform = spawn_transform ship.team = team + ship.spawn_index = spawn_index if controller: ship.set_controller(controller) ships.append(ship) diff --git a/Game/scripts/match_mode.gd b/Game/scripts/match_mode.gd index 1bf58bee..f1d7a43b 100644 --- a/Game/scripts/match_mode.gd +++ b/Game/scripts/match_mode.gd @@ -20,6 +20,12 @@ const KICKOFF_COUNTDOWN_SECONDS := 3 @export var match_length_seconds := 150.0 +# Ships per team. The player always controls one ship on team 0; every other +# ship (teammates and the whole opposing team) is AI-controlled — no local +# multiplayer input. Plumbing only for this pass: there's no menu UI yet to +# pick a team size above 1. +@export_range(1, 5) var team_size: int = 1 + @export_group("AI opponent") # Trained policy for the opponent; empty = inert placeholder ship. @export_file("*.json") var bot_model_path: String = "" @@ -41,7 +47,10 @@ func _start() -> void: spawn_ball() var player_ship := spawn_ship(0, 0, PlayerShipController.new()) spawn_camera_rig(player_ship) - spawn_ship(1, 0, _make_opponent_controller()) + for i in range(1, team_size): + spawn_ship(0, i, _make_opponent_controller()) + for i in team_size: + spawn_ship(1, i, _make_opponent_controller()) await _run_kickoff_countdown() diff --git a/Game/scripts/ship.gd b/Game/scripts/ship.gd index 788faeee..049e314a 100644 --- a/Game/scripts/ship.gd +++ b/Game/scripts/ship.gd @@ -58,6 +58,13 @@ var team: int = 0: team = value _apply_team_color() +# This ship's index within its team's roster (0, 1, 2, ...), set once by +# GameMode.spawn_ship and never changed afterward. The stable identity +# AIShipController/ShipAIController sort teammates/opponents by, so both +# training and in-game inference assign the same ship to the same +# observation-vector slot for the whole match (see ShipObservations). +var spawn_index: int = -1 + # Shared per-team accent material, built once per team and reused by every # ship — avoids allocating a fresh StandardMaterial3D from both _ready and # the team setter (previously ran at least twice per ship). diff --git a/Game/scripts/ship_ai_controller.gd b/Game/scripts/ship_ai_controller.gd index aeea7aea..62e85d1a 100644 --- a/Game/scripts/ship_ai_controller.gd +++ b/Game/scripts/ship_ai_controller.gd @@ -106,7 +106,8 @@ const MAX_BALL_DISTANCE := sqrt( var ship: Ship var rl_controller: RLShipController var ball: RigidBody3D -var opponent: Ship +var teammates: Array[Ship] = [] +var opponents: Array[Ship] = [] var attack_goal_position: Vector3 # Set directly by TrainingMode (_on_goal_scored / the timeout branch in @@ -146,12 +147,16 @@ var _air_touches := 0 # Wire up references after the ship is spawned. `attack_goal` is the goal -# this ship scores into (goal.team == opponent's team). -func setup(p_ship: Ship, p_rl_controller: RLShipController, p_ball: RigidBody3D, p_opponent: Ship, p_attack_goal_position: Vector3) -> void: +# this ship scores into (goal.team == the opposing team's team). +func setup( + p_ship: Ship, p_rl_controller: RLShipController, p_ball: RigidBody3D, + p_teammates: Array[Ship], p_opponents: Array[Ship], p_attack_goal_position: Vector3 +) -> void: ship = p_ship rl_controller = p_rl_controller ball = p_ball - opponent = p_opponent + teammates = p_teammates + opponents = p_opponents attack_goal_position = p_attack_goal_position init(ship) @@ -161,7 +166,7 @@ func setup(p_ship: Ship, p_rl_controller: RLShipController, p_ball: RigidBody3D, func get_obs() -> Dictionary: - return {"obs": ShipObservations.build(ship, opponent, ball, attack_goal_position)} + return {"obs": ShipObservations.build(ship, teammates, opponents, ball, attack_goal_position)} func get_reward() -> float: diff --git a/Game/scripts/ship_observations.gd b/Game/scripts/ship_observations.gd index 97306352..90035e1f 100644 --- a/Game/scripts/ship_observations.gd +++ b/Game/scripts/ship_observations.gd @@ -11,13 +11,13 @@ extends RefCounted # The same rotation must be inverted when interpreting actions (see canon — # it is its own inverse). -# Normalization scales. Standard arena volume (see ArenaBoundary): x ±12, -# z ±18, height 12, goals at z ±18 (flush with the end walls); positions are +# Normalization scales. Standard arena volume (see ArenaBoundary): x ±18, +# z ±27, height 18, goals at z ±27 (flush with the end walls); positions are # soft-normalized to roughly [-1, 1]. Do not retune without retraining every # model in Game/bots/. -const POSITION_SCALE := Vector3(20.0, 10.0, 20.0) +const POSITION_SCALE := Vector3(30.0, 15.0, 30.0) const BALL_SPEED_SCALE := 30.0 -const GOAL_DISTANCE_SCALE := 40.0 +const GOAL_DISTANCE_SCALE := 60.0 # Contact normals with y above this are floor contact; below it they read as # wall (sideways) or ceiling (downward) — mirrors @@ -26,13 +26,23 @@ const GOAL_DISTANCE_SCALE := 40.0 # counts as "in contact" for the reward/observation to stay consistent). const FLOOR_NORMAL_MIN_Y := 0.7 -# Number of floats build() returns; the policy input size. APPEND-ONLY: new -# features go on the end and existing indices never move, so an old exported -# model (whose network was trained against a shorter SIZE) still decodes its -# first N inputs identically when SIZE grows — see PolicyNetwork.forward's -# input_size slice/guard. Do not retune an *existing* index without -# retraining every model in Game/bots/. -const SIZE := 35 +# Fixed roster caps for the padded teammate/opponent slots below — the +# largest supported match size is 5v5. Slots beyond the real teammate/ +# opponent count are zero-filled, mirroring the old single-opponent's +# null-zero-fill (see build()). Callers must pass teammates/opponents already +# sorted by Ship.spawn_index, so a given ship occupies the same slot in every +# tick's observation for the whole match, in both training and in-game +# inference (see AIShipController._discover_scene_refs / +# TrainingMode._start). +const MAX_TEAMMATES := 4 +const MAX_OPPONENTS := 5 + +# Number of floats build() returns; the policy input size. +# 15 (own) + 6 (ball) + 6*MAX_TEAMMATES + 6*MAX_OPPONENTS + 4 (goal) + 4 (contact) +# Game/bots/promoted/*.json were exported against the old single-opponent, +# SIZE=35 layout and are not migrated — this is a from-scratch retrain, so +# those checkpoints are expected to go stale rather than keep decoding. +const SIZE := 15 + 6 + 6 * MAX_TEAMMATES + 6 * MAX_OPPONENTS + 4 + 4 # 180° rotation about Y for team 1; identity for team 0. A proper rotation @@ -43,8 +53,14 @@ static func canon(v: Vector3, team: int) -> Vector3: # attack_goal_position: centre of the goal this ship is trying to score in -# (the goal whose `team` == the opponent's team). -static func build(ship: Ship, opponent: Ship, ball: RigidBody3D, attack_goal_position: Vector3) -> Array: +# (the goal whose `team` == the opponent's team). teammates/opponents must +# already be sorted by Ship.spawn_index (ascending) by the caller — see +# MAX_TEAMMATES/MAX_OPPONENTS's comment for why slot stability matters; this +# function only pads/truncates to the fixed cap, it doesn't sort. +static func build( + ship: Ship, teammates: Array[Ship], opponents: Array[Ship], + ball: RigidBody3D, attack_goal_position: Vector3 +) -> Array: var team := ship.team var obs := [] @@ -60,27 +76,23 @@ static func build(ship: Ship, opponent: Ship, ball: RigidBody3D, attack_goal_pos _append(obs, canon(ball_rel, team) / POSITION_SCALE) _append(obs, canon(ball.linear_velocity, team) / BALL_SPEED_SCALE) - # Opponent, relative to self (zeros if absent, e.g. a 1-ship drill) - if is_instance_valid(opponent): - var opp_rel := opponent.global_position - ship.global_position - _append(obs, canon(opp_rel, team) / POSITION_SCALE) - _append(obs, canon(opponent.linear_velocity, team) / ship.max_speed) - else: - _append(obs, Vector3.ZERO) - _append(obs, Vector3.ZERO) + # Teammates and opponents, relative to self, each padded/truncated to a + # fixed slot count (zeros past the real roster size, e.g. a 1v1 match or + # a solo drill) so the vector shape never depends on match size. + _append_ship_slots(obs, ship, team, teammates, MAX_TEAMMATES) + _append_ship_slots(obs, ship, team, opponents, MAX_OPPONENTS) # Goal we are attacking, relative to self var goal_rel := attack_goal_position - ship.global_position _append(obs, canon(goal_rel, team) / POSITION_SCALE) obs.append(goal_rel.length() / GOAL_DISTANCE_SCALE) - # Own contact state (appended — see SIZE's append-only invariant). - # Added for generation 4: ShipAIController's wall_contact_penalty used to - # fire on a condition the observation vector couldn't see coming, - # leaving the value function to predict a reward with no supporting - # signal. Also gives the policy a direct "am I resting on a surface" - # signal it can use to push off (a real aerial mechanic), distinct from - # inferring it indirectly from position/up-vector. + # Own contact state. Added for generation 4: ShipAIController's + # wall_contact_penalty used to fire on a condition the observation vector + # couldn't see coming, leaving the value function to predict a reward + # with no supporting signal. Also gives the policy a direct "am I resting + # on a surface" signal it can use to push off (a real aerial mechanic), + # distinct from inferring it indirectly from position/up-vector. var normal := contact_normal(ship) _append(obs, canon(normal, team)) obs.append(1.0 if normal != Vector3.ZERO else 0.0) @@ -88,6 +100,26 @@ static func build(ship: Ship, opponent: Ship, ball: RigidBody3D, attack_goal_pos return obs +# Appends up to `slot_count` other ships' (relative position, relative +# velocity) — 6 floats each — zero-filling any slots beyond the real roster +# size, or beyond slot_count if the roster somehow has more (sorted-by- +# spawn_index order means truncation drops the highest spawn_index ships, +# not the nearest ones — acceptable since slot_count already covers the +# largest supported match size, 5v5). +static func _append_ship_slots( + obs: Array, ship: Ship, team: int, others: Array[Ship], slot_count: int +) -> void: + for i in slot_count: + if i < others.size() and is_instance_valid(others[i]): + var other := others[i] + var rel := other.global_position - ship.global_position + _append(obs, canon(rel, team) / POSITION_SCALE) + _append(obs, canon(other.linear_velocity, team) / ship.max_speed) + else: + _append(obs, Vector3.ZERO) + _append(obs, Vector3.ZERO) + + static func _append(obs: Array, v: Vector3) -> void: obs.append(v.x) obs.append(v.y) diff --git a/Game/scripts/training_mode.gd b/Game/scripts/training_mode.gd index 582b764f..0ee76d9e 100644 --- a/Game/scripts/training_mode.gd +++ b/Game/scripts/training_mode.gd @@ -65,6 +65,12 @@ extends GameMode # _place_air_drill. @export_range(0.0, 1.0) var air_drill_chance := 0.0 +# Ships per team. Default 1 preserves every existing curriculum script's 1v1 +# behaviour unchanged; up to 5 matches ShipObservations.MAX_TEAMMATES/ +# MAX_OPPONENTS. Plumbing only for this pass — no 2v2+ curriculum/reward +# design has been done, so a run above 1 is untested territory. +@export_range(1, 5) var team_size: int = 1 + # Placement bounds for randomized episode starts, derived from the standard # enclosure (ArenaBoundary). The inset keeps a randomly oriented ship (1x1x4 # box, worst-case half-extent ~2.05) from spawning intersecting the walls, @@ -153,22 +159,46 @@ func _start() -> void: spawn_ship(team, 0, bot) return - var ship_team0 := spawn_ship(0, 0, RLShipController.new()) - var ship_team1: Ship - match _opponent_mode: - "inert": - ship_team1 = spawn_ship(1, 0, ShipController.new()) - _inert_ships.append(ship_team1) - "frozen": - var bot := AIShipController.new() - bot.model_path = _opponent_model_path - ship_team1 = spawn_ship(1, 0, bot) - _: - ship_team1 = spawn_ship(1, 0, RLShipController.new()) + var team0_ships: Array[Ship] = [] + for i in team_size: + team0_ships.append(spawn_ship(0, i, RLShipController.new())) - _attach_agent(ship_team0, ship_team1) + # The opponent_mode branch applies uniformly to every ship on team 1: an + # "inert"/"frozen" run means the whole opposing team gets that treatment, + # not just one ship. + var team1_ships: Array[Ship] = [] + for i in team_size: + var ship1: Ship + match _opponent_mode: + "inert": + ship1 = spawn_ship(1, i, ShipController.new()) + _inert_ships.append(ship1) + "frozen": + var bot := AIShipController.new() + bot.model_path = _opponent_model_path + ship1 = spawn_ship(1, i, bot) + _: + ship1 = spawn_ship(1, i, RLShipController.new()) + team1_ships.append(ship1) + + # All ships spawn before any agent attaches, so every agent's + # teammates/opponents lists see the other side's full roster. + for ship in team0_ships: + _attach_agent(ship, _other_ships(team0_ships, ship), team1_ships) if _opponent_mode == "self_play": - _attach_agent(ship_team1, ship_team0) + for ship in team1_ships: + _attach_agent(ship, _other_ships(team1_ships, ship), team0_ships) + + +# `roster` minus `ship`, preserving order — rosters are built by spawn_index +# already, so this stays spawn_index-sorted (see ShipObservations' slot- +# stability requirement). +func _other_ships(roster: Array[Ship], ship: Ship) -> Array[Ship]: + var others: Array[Ship] = [] + for s in roster: + if s != ship: + others.append(s) + return others # Shared "--key=value" cmdline scan used by both eval and curriculum parsing. @@ -261,14 +291,14 @@ func _ai_default(name: String) -> Variant: _: return null -func _attach_agent(ship: Ship, opponent: Ship) -> void: +func _attach_agent(ship: Ship, teammates: Array[Ship], opponents: Array[Ship]) -> void: var agent := ShipAIController.new() agent.name = "ShipAIController" agent.reset_after = int(episode_length_seconds * TICKS_PER_SIM_SECOND) for key in _ai_overrides: agent.set(key, _ai_overrides[key]) ship.add_child(agent) - agent.setup(ship, ship.controller as RLShipController, ball, opponent, _attack_goal_position(ship.team)) + agent.setup(ship, ship.controller as RLShipController, ball, teammates, opponents, _attack_goal_position(ship.team)) _agents.append(agent) @@ -316,7 +346,7 @@ func _physics_process(_delta): # A goal (_on_goal_scored) does NOT do this — a goal is a # genuine terminal, V(s)=0 is correct there. agent.truncated_this_episode = true - agent.terminal_obs = ShipObservations.build(agent.ship, agent.opponent, agent.ball, agent.attack_goal_position) + agent.terminal_obs = ShipObservations.build(agent.ship, agent.teammates, agent.opponents, agent.ball, agent.attack_goal_position) _reset_episode() return