feat(*): Add self-play RL training pipeline with PPO trainer, in-game GDScript policy inference, and bot opponent support in Match mode

This commit is contained in:
Josh Creek
2026-07-18 19:32:51 +01:00
parent 328831df1f
commit 85f96eb15e
81 changed files with 3934 additions and 10 deletions
+90
View File
@@ -0,0 +1,90 @@
class_name AIShipController
extends ShipController
# Drives a ship from a trained self-play policy (see TRAINING.md). Builds the
# same canonical observation as training (ShipObservations) and runs the
# policy MLP in GDScript (PolicyNetwork) — the shipped bot has no Python,
# .NET, or network dependency.
#
# Difficulty is (model, reaction_ticks, action_noise): weaker checkpoints make
# easier bots outright, and the two knobs handicap a given model further —
# slower reactions and noisier execution. Models live in res://bots/.
@export_file("*.json") var model_path: String = ""
# Decide a new action every N physics ticks, holding the last one between
# decisions. 8 matches the training action_repeat; larger = slower reactions.
@export_range(1, 60) var reaction_ticks: int = 8
# Uniform noise magnitude added to each action axis (0 = play at full skill).
@export_range(0.0, 1.0) var action_noise: float = 0.0
var _policy: PolicyNetwork
var _action := ShipAction.new()
var _ticks_until_decision := 0
var _ship: Ship
var _opponent: Ship
var _ball: RigidBody3D
var _attack_goal_position: Vector3
var _scene_refs_ready := false
func _ready():
if not model_path.is_empty():
_policy = PolicyNetwork.load_from_file(model_path)
func get_action() -> ShipAction:
if _policy == null:
return _action # unloaded model: behaves like the inert placeholder
if not _scene_refs_ready and not _discover_scene_refs():
return _action
_ticks_until_decision -= 1
if _ticks_until_decision <= 0:
_ticks_until_decision = reaction_ticks
_decide()
return _action
func _decide() -> void:
var obs := ShipObservations.build(_ship, _opponent, _ball, _attack_goal_position)
var out := _policy.forward(obs)
# Output layout matches the flattened training action space (Box(7)):
# thrust xyz, rotation xyz, turbo (> 0 means on).
_action.thrust = Vector3(
_axis(out[0]),
_axis(out[1]),
_axis(out[2])
)
_action.rotation = Vector3(
_axis(out[3]),
_axis(out[4]),
_axis(out[5])
)
_action.turbo = out[6] > 0.0
func _axis(value: float) -> float:
if action_noise > 0.0:
value += randf_range(-action_noise, action_noise)
return clampf(value, -1.0, 1.0)
# Find ship/ball/opponent/goal once everything is spawned. ShipAction axes
# are body-frame so only observations need team context (ShipObservations).
func _discover_scene_refs() -> bool:
_ship = get_parent() as Ship
if _ship == null or not is_inside_tree():
return false
_ball = get_tree().get_first_node_in_group("ball")
for node in get_tree().get_nodes_in_group("ship"):
if node != _ship:
_opponent = node
break
for goal in get_tree().get_nodes_in_group("goal"):
if goal.team == 1 - _ship.team:
_attack_goal_position = goal.global_position
if _ball == null:
return false
_scene_refs_ready = true
return true
+1
View File
@@ -0,0 +1 @@
uid://4emuhiolrkb2
+23 -4
View File
@@ -1,15 +1,22 @@
extends GameMode
# Timed match: two teams, score tracking, kickoff resets after each goal.
# The opponent ship is currently inert (base ShipController, zero action) —
# it becomes the AI opponent once an AIShipController exists (see TODO.md),
# and additional player ships once multiplayer lands.
# The opponent is a trained AI bot when a policy model is configured
# (see TRAINING.md for training and promoting models into res://bots/),
# otherwise an inert placeholder ship.
signal timer_updated(minutes: int, seconds: int)
signal score_changed(score: Dictionary)
@export var match_length_seconds := 150.0
@export_group("AI opponent")
# Trained policy for the opponent; empty = inert placeholder ship.
@export_file("*.json") var bot_model_path: String = ""
# Difficulty handicaps, applied on top of the model (see AIShipController).
@export_range(1, 60) var bot_reaction_ticks: int = 8
@export_range(0.0, 1.0) var bot_action_noise: float = 0.0
var score := {0: 0, 1: 0}
var match_timer: Timer
@@ -18,7 +25,7 @@ func _start() -> void:
spawn_ball()
var player_ship := spawn_ship(0, 0, PlayerShipController.new())
spawn_camera_rig(player_ship)
spawn_ship(1, 0, ShipController.new()) # inert placeholder opponent
spawn_ship(1, 0, _make_opponent_controller())
match_timer = Timer.new()
match_timer.one_shot = true
@@ -28,6 +35,18 @@ func _start() -> void:
match_timer.start()
func _make_opponent_controller() -> ShipController:
if not bot_model_path.is_empty() and FileAccess.file_exists(bot_model_path):
var bot := AIShipController.new()
bot.model_path = bot_model_path
bot.reaction_ticks = bot_reaction_ticks
bot.action_noise = bot_action_noise
return bot
if not bot_model_path.is_empty():
push_warning("MatchMode: bot model not found at %s, spawning inert opponent" % bot_model_path)
return ShipController.new() # inert placeholder
func _process(_delta):
if match_timer and match_timer.time_left > 0:
var remaining := ceili(match_timer.time_left)
+73
View File
@@ -0,0 +1,73 @@
class_name PolicyNetwork
extends RefCounted
# Minimal MLP forward pass for running trained policies in pure GDScript —
# no .NET build or ONNX runtime needed. Weights come from a JSON file written
# by training/export_policy.py (see TRAINING.md). The policy net is tiny
# (31 → 64 → 64 → 7 by default), and the bot only thinks every few physics
# ticks, so GDScript is plenty fast.
#
# JSON shape:
# {
# "input_size": 31,
# "layers": [
# {"weights": [[out x in floats]], "biases": [out floats], "activation": "tanh" | "linear"},
# ...
# ]
# }
var input_size: int = 0
var _layers: Array = []
static func load_from_file(path: String) -> PolicyNetwork:
if not FileAccess.file_exists(path):
push_error("PolicyNetwork: model file not found: %s" % path)
return null
var text := FileAccess.get_file_as_string(path)
var data: Variant = JSON.parse_string(text)
if data == null or not (data is Dictionary) or not data.has("layers"):
push_error("PolicyNetwork: invalid model file: %s" % path)
return null
var net := PolicyNetwork.new()
net.input_size = int(data.get("input_size", 0))
for layer in data["layers"]:
# Flatten each layer's weights into a PackedFloat64Array for speed
var out_size: int = layer["biases"].size()
var in_size: int = layer["weights"][0].size()
var flat := PackedFloat64Array()
flat.resize(out_size * in_size)
var i := 0
for row in layer["weights"]:
for value in row:
flat[i] = value
i += 1
var biases := PackedFloat64Array(layer["biases"])
net._layers.append({
"weights": flat,
"biases": biases,
"in_size": in_size,
"out_size": out_size,
"tanh": layer.get("activation", "linear") == "tanh",
})
return net
func forward(observation: Array) -> Array:
var x := PackedFloat64Array(observation)
for layer in _layers:
var in_size: int = layer["in_size"]
var out_size: int = layer["out_size"]
var weights: PackedFloat64Array = layer["weights"]
var biases: PackedFloat64Array = layer["biases"]
var y := PackedFloat64Array()
y.resize(out_size)
for row in out_size:
var sum := biases[row]
var offset := row * in_size
for col in in_size:
sum += weights[offset + col] * x[col]
y[row] = tanh(sum) if layer["tanh"] else sum
x = y
return Array(x)
+1
View File
@@ -0,0 +1 @@
uid://biixjudn05ib2
+14
View File
@@ -0,0 +1,14 @@
class_name RLShipController
extends ShipController
# Controller for externally-driven ships (RL training and trained-policy
# inference). Something else — a ShipAIController during training, an
# AIShipController at play time — writes into `action`; the ship pulls it
# each physics tick like any other controller. The ship never knows it is
# being trained.
var action: ShipAction = ShipAction.new()
func get_action() -> ShipAction:
return action
+1
View File
@@ -0,0 +1 @@
uid://bctvr5djofbrm
+90
View File
@@ -0,0 +1,90 @@
class_name ShipAIController
extends AIController3D
# Training-side bridge between godot_rl_agents and a ship. This is the only
# class that touches plugin types (AIController3D / the Sync node protocol) —
# everything else stays behind the ShipController seam: actions received from
# the trainer are written into an RLShipController, which the ship pulls like
# any other controller.
#
# Action space is ShipAction verbatim: 6 continuous axes (thrust xyz,
# rotation xyz, each -1..1) + binary turbo. ShipAction axes are ship-local
# (body frame), so they need no team mirroring — only observations do
# (see ShipObservations.canon).
# Reward shaping weights. Dense terms accrue per physics tick (60 sim-ticks
# per sim-second); event terms fire once. Exported so tuning needs no code
# edits. Goal rewards are added by TrainingMode, which owns goal events.
@export var ball_touch_reward := 0.1
@export var velocity_to_ball_weight := 0.001
@export var ball_velocity_to_goal_weight := 0.004
var ship: Ship
var rl_controller: RLShipController
var ball: RigidBody3D
var opponent: Ship
var attack_goal_position: Vector3
# Wire up references after the ship is spawned. `attack_goal` is the goal
# this ship scores into (goal.team == opponent's team).
func setup(p_ship: Ship, p_rl_controller: RLShipController, p_ball: RigidBody3D, p_opponent: Ship, p_attack_goal_position: Vector3) -> void:
ship = p_ship
rl_controller = p_rl_controller
ball = p_ball
opponent = p_opponent
attack_goal_position = p_attack_goal_position
init(ship)
# Contact monitoring for the ball-touch reward (training-only cost;
# the shipped game leaves contact_monitor off).
ship.contact_monitor = true
ship.max_contacts_reported = 8
ship.body_entered.connect(_on_ship_body_entered)
func get_obs() -> Dictionary:
return {"obs": ShipObservations.build(ship, opponent, ball, attack_goal_position)}
func get_reward() -> float:
return reward
func get_action_space() -> Dictionary:
return {
"thrust": {"size": 3, "action_type": "continuous"},
"rotation": {"size": 3, "action_type": "continuous"},
"turbo": {"size": 2, "action_type": "discrete"},
}
func set_action(action) -> void:
var thrust: Array = action["thrust"]
var rot: Array = action["rotation"]
rl_controller.action.thrust = Vector3(thrust[0], thrust[1], thrust[2])
rl_controller.action.rotation = Vector3(rot[0], rot[1], rot[2])
rl_controller.action.turbo = int(action["turbo"]) == 1
func _physics_process(delta):
super(delta)
if not is_instance_valid(ship) or not is_instance_valid(ball):
return
# Dense shaping: own velocity toward the ball
var to_ball := ball.global_position - ship.global_position
if to_ball.length_squared() > 0.0001:
var closing_speed := ship.linear_velocity.dot(to_ball.normalized())
reward += velocity_to_ball_weight * closing_speed / ship.max_speed
# Dense shaping: ball velocity toward the goal we attack
var ball_to_goal := attack_goal_position - ball.global_position
if ball_to_goal.length_squared() > 0.0001:
var ball_progress := ball.linear_velocity.dot(ball_to_goal.normalized())
reward += ball_velocity_to_goal_weight * ball_progress / ShipObservations.BALL_SPEED_SCALE
func _on_ship_body_entered(body: Node) -> void:
if body.is_in_group("ball"):
reward += ball_touch_reward
+1
View File
@@ -0,0 +1 @@
uid://bql1ixmk23u77
+69
View File
@@ -0,0 +1,69 @@
class_name ShipObservations
extends RefCounted
# Canonical, team-relative observation builder. Shared by training
# (ShipAIController) and in-game inference (AIShipController) so a trained
# policy sees byte-identical inputs in both contexts — do not fork this logic.
#
# Self-play trick: observations for team 1 are rotated 180° about Y
# (x → -x, z → -z), so every ship perceives itself attacking toward -Z
# regardless of which side it spawned on. One policy can then play both teams.
# The same rotation must be inverted when interpreting actions (see canon —
# it is its own inverse).
# Normalization scales. Arena bounds: goals at z ≈ ±15.56, ship spawns at
# z = ±12; positions are soft-normalized to roughly [-1, 1].
const POSITION_SCALE := Vector3(20.0, 10.0, 20.0)
const BALL_SPEED_SCALE := 30.0
const GOAL_DISTANCE_SCALE := 40.0
# Number of floats build() returns; the policy input size.
const SIZE := 31
# 180° rotation about Y for team 1; identity for team 0. A proper rotation
# (preserves handedness), and its own inverse — used for both observations
# and mapping canonical-frame actions back to world intent.
static func canon(v: Vector3, team: int) -> Vector3:
return v if team == 0 else Vector3(-v.x, v.y, -v.z)
# attack_goal_position: centre of the goal this ship is trying to score in
# (the goal whose `team` == the opponent's team).
static func build(ship: Ship, opponent: Ship, ball: RigidBody3D, attack_goal_position: Vector3) -> Array:
var team := ship.team
var obs := []
# Own kinematics
_append(obs, canon(ship.global_position, team) / POSITION_SCALE)
_append(obs, canon(-ship.global_transform.basis.z, team)) # forward
_append(obs, canon(ship.global_transform.basis.y, team)) # up
_append(obs, canon(ship.linear_velocity, team) / ship.max_speed)
_append(obs, canon(ship.angular_velocity, team) / ship.max_angular_speed)
# Ball, relative to self
var ball_rel := ball.global_position - ship.global_position
_append(obs, canon(ball_rel, team) / POSITION_SCALE)
_append(obs, canon(ball.linear_velocity, team) / BALL_SPEED_SCALE)
# Opponent, relative to self (zeros if absent, e.g. a 1-ship drill)
if is_instance_valid(opponent):
var opp_rel := opponent.global_position - ship.global_position
_append(obs, canon(opp_rel, team) / POSITION_SCALE)
_append(obs, canon(opponent.linear_velocity, team) / ship.max_speed)
else:
_append(obs, Vector3.ZERO)
_append(obs, Vector3.ZERO)
# Goal we are attacking, relative to self
var goal_rel := attack_goal_position - ship.global_position
_append(obs, canon(goal_rel, team) / POSITION_SCALE)
obs.append(goal_rel.length() / GOAL_DISTANCE_SCALE)
return obs
static func _append(obs: Array, v: Vector3) -> void:
obs.append(v.x)
obs.append(v.y)
obs.append(v.z)
+1
View File
@@ -0,0 +1 @@
uid://bqux8vc4eou73
+244
View File
@@ -0,0 +1,244 @@
class_name TrainingMode
extends GameMode
# Headless self-play training mode: two RL-driven ships, no HUD, no camera.
# The scene also contains the godot_rl_agents Sync node, which speaks TCP to
# the Python trainer; this mode owns the environment rules — episodes, goal
# rewards, and randomized episode-start states (the RLGym "state setter"
# lesson: varied starts massively speed up learning versus kickoff-only).
#
# Run: godot --headless --path Game res://scenes/training.tscn
# (started automatically by training/train.py; boots into idle ships with a
# warning if no trainer is listening).
#
# Eval mode (used by training/evaluate.py): pass --eval_model_a=<path> and
# --eval_model_b=<path> (+ optional --eval_episodes=N) and both ships are
# instead driven by those exported policies via AIShipController; each episode
# ends at the first goal (or a draw on timeout), and a final machine-readable
# "EVAL_RESULT {...}" line is printed before quitting.
@export var episode_length_seconds := 30.0
@export var goal_reward := 10.0
# Episode-start state mix; remaining probability = fully random state.
@export_range(0.0, 1.0) var kickoff_state_chance := 0.2
@export_range(0.0, 1.0) var ball_near_goal_chance := 0.2
# Placement bounds, inset from the arena (goals at z ≈ ±15.56).
const FIELD_HALF_X := 10.0
const FIELD_HALF_Z := 13.0
const FIELD_MIN_Y := 1.5
const FIELD_MAX_Y := 8.0
const MAX_RANDOM_BALL_SPEED := 12.0
const MAX_RANDOM_SHIP_SPEED := 8.0
# Sim runs at 60 physics ticks per sim-second regardless of speedup.
const TICKS_PER_SIM_SECOND := 60.0
# The arena has no walls/ceiling yet (see TODO.md), so an untrained policy can
# simply fly away. Leaving this volume ends the episode with a small penalty.
const BOUNDS_HALF_X := 25.0
const BOUNDS_HALF_Z := 25.0
const BOUNDS_MIN_Y := -5.0
const BOUNDS_MAX_Y := 25.0
@export var out_of_bounds_penalty := 1.0
var _agents: Array[ShipAIController] = []
# Eval mode state (see header comment)
var _eval := false
var _eval_models: Array[String] = ["", ""]
var _eval_episodes := 20
var _eval_goals := {0: 0, 1: 0}
var _eval_draws := 0
var _eval_episodes_done := 0
var _episode_ticks := 0
func _start() -> void:
_parse_eval_args()
spawn_ball()
if _eval:
for team in [0, 1]:
var bot := AIShipController.new()
bot.model_path = _eval_models[team]
spawn_ship(team, 0, bot)
return
var ship_team0 := spawn_ship(0, 0, RLShipController.new())
var ship_team1 := spawn_ship(1, 0, RLShipController.new())
_attach_agent(ship_team0, ship_team1)
_attach_agent(ship_team1, ship_team0)
func _parse_eval_args() -> void:
var args := {}
for argument in OS.get_cmdline_args():
if argument.begins_with("--") and argument.find("=") > -1:
var key_value := argument.lstrip("--").split("=", true, 1)
args[key_value[0]] = key_value[1]
if args.has("eval_model_a") and args.has("eval_model_b"):
_eval = true
_eval_models[0] = args["eval_model_a"]
_eval_models[1] = args["eval_model_b"]
_eval_episodes = int(args.get("eval_episodes", str(_eval_episodes)))
func _attach_agent(ship: Ship, opponent: Ship) -> void:
var agent := ShipAIController.new()
agent.name = "ShipAIController"
agent.reset_after = int(episode_length_seconds * TICKS_PER_SIM_SECOND)
ship.add_child(agent)
agent.setup(ship, ship.controller as RLShipController, ball, opponent, _attack_goal_position(ship.team))
_agents.append(agent)
# The goal this team scores into: the one the opponent defends/concedes.
func _attack_goal_position(team: int) -> Vector3:
for goal in arena.get_goals():
if goal.team == 1 - team:
return goal.global_position
push_error("TrainingMode: no goal found for team %d to attack" % team)
return Vector3.ZERO
func _physics_process(_delta):
if _eval:
_episode_ticks += 1
for ship in ships:
if is_instance_valid(ship) and _out_of_bounds(ship.global_position):
_place_body(ship, _ship_spawn_transforms[ship], Vector3.ZERO, Vector3.ZERO)
var ball_lost := is_instance_valid(ball) and _out_of_bounds(ball.global_position)
if _episode_ticks > int(episode_length_seconds * TICKS_PER_SIM_SECOND) or ball_lost:
_eval_draws += 1
_end_eval_episode()
return
# Both trainer-requested resets and truncation (reset_after ticks elapsed)
# surface as needs_reset. Only truncation is an episode end the trainer
# must be told about via done — a trainer-requested reset already knows.
var needs_reset := false
var truncated := false
for agent in _agents:
needs_reset = needs_reset or agent.needs_reset
truncated = truncated or agent.n_steps > agent.reset_after
if needs_reset:
if truncated:
for agent in _agents:
agent.done = true
_reset_episode()
return
# A ship leaving the play volume is penalized and respawned at its kickoff
# spawn (the episode continues); a lost ball ends the episode for both.
for agent in _agents:
if _out_of_bounds(agent.ship.global_position):
agent.reward -= out_of_bounds_penalty
_place_body(agent.ship, _ship_spawn_transforms[agent.ship], Vector3.ZERO, Vector3.ZERO)
if is_instance_valid(ball) and _out_of_bounds(ball.global_position) and not _agents.is_empty():
for agent in _agents:
agent.done = true
_reset_episode()
func _out_of_bounds(position: Vector3) -> bool:
return absf(position.x) > BOUNDS_HALF_X \
or absf(position.z) > BOUNDS_HALF_Z \
or position.y < BOUNDS_MIN_Y \
or position.y > BOUNDS_MAX_Y
func _on_goal_scored(conceding_team: int) -> void:
if _eval:
_eval_goals[1 - conceding_team] += 1
_end_eval_episode()
return
for agent in _agents:
agent.reward += goal_reward if agent.ship.team != conceding_team else -goal_reward
agent.done = true
_reset_episode()
func _end_eval_episode() -> void:
_eval_episodes_done += 1
_episode_ticks = 0
if _eval_episodes_done >= _eval_episodes:
print("EVAL_RESULT " + JSON.stringify({
"model_a": _eval_models[0],
"model_b": _eval_models[1],
"episodes": _eval_episodes_done,
"goals_a": _eval_goals[0],
"goals_b": _eval_goals[1],
"draws": _eval_draws,
}))
get_tree().quit()
return
# Randomized states (not kickoff): deterministic policies would otherwise
# replay the identical episode every time.
_reset_episode()
func _reset_episode() -> void:
for agent in _agents:
agent.reset()
var roll := randf()
if roll < kickoff_state_chance:
reset_ball()
reset_ships()
elif roll < kickoff_state_chance + ball_near_goal_chance:
_place_ships_random()
_place_ball_near_goal()
else:
_place_ships_random()
_place_ball_random()
func _place_ball_random() -> void:
var velocity := _random_direction() * randf_range(0.0, MAX_RANDOM_BALL_SPEED)
_place_body(ball, Transform3D(Basis.IDENTITY, _random_position()), velocity, Vector3.ZERO)
# Attacking/defending drill states: ball close to a goal, moving toward it.
func _place_ball_near_goal() -> void:
var goals := arena.get_goals()
var goal: Goal = goals[randi() % goals.size()]
var toward_centre := -signf(goal.global_position.z)
var position := Vector3(
randf_range(-4.0, 4.0),
randf_range(FIELD_MIN_Y, 4.0),
goal.global_position.z + toward_centre * randf_range(3.0, 6.0)
)
var to_goal := (goal.global_position - position).normalized()
var velocity := (to_goal + _random_direction() * 0.3).normalized() * randf_range(2.0, MAX_RANDOM_BALL_SPEED)
_place_body(ball, Transform3D(Basis.IDENTITY, position), velocity, Vector3.ZERO)
func _place_ships_random() -> void:
for ship in ships:
var orientation := Basis.from_euler(Vector3(
randf_range(-0.4, 0.4),
randf_range(-PI, PI),
randf_range(-0.4, 0.4)
))
var velocity := _random_direction() * randf_range(0.0, MAX_RANDOM_SHIP_SPEED)
_place_body(ship, Transform3D(orientation, _random_position()), velocity, Vector3.ZERO)
func _random_position() -> Vector3:
return Vector3(
randf_range(-FIELD_HALF_X, FIELD_HALF_X),
randf_range(FIELD_MIN_Y, FIELD_MAX_Y),
randf_range(-FIELD_HALF_Z, FIELD_HALF_Z)
)
func _random_direction() -> Vector3:
var direction := Vector3(randf_range(-1, 1), randf_range(-1, 1), randf_range(-1, 1))
return direction.normalized() if direction.length_squared() > 0.001 else Vector3.FORWARD
func _place_body(body: RigidBody3D, to: Transform3D, linear_velocity: Vector3, angular_velocity: Vector3) -> void:
# Deferred: a RigidBody3D transform can't be set mid-physics-step
body.set_deferred("global_transform", to)
body.set_deferred("linear_velocity", linear_velocity)
body.set_deferred("angular_velocity", angular_velocity)
+1
View File
@@ -0,0 +1 @@
uid://d2kfcobanp6iv