mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-10 16:04:04 +00:00
feat(*): Add self-play RL training pipeline with PPO trainer, in-game GDScript policy inference, and bot opponent support in Match mode
This commit is contained in:
@@ -0,0 +1,90 @@
|
||||
class_name AIShipController
|
||||
extends ShipController
|
||||
|
||||
# Drives a ship from a trained self-play policy (see TRAINING.md). Builds the
|
||||
# same canonical observation as training (ShipObservations) and runs the
|
||||
# policy MLP in GDScript (PolicyNetwork) — the shipped bot has no Python,
|
||||
# .NET, or network dependency.
|
||||
#
|
||||
# Difficulty is (model, reaction_ticks, action_noise): weaker checkpoints make
|
||||
# easier bots outright, and the two knobs handicap a given model further —
|
||||
# slower reactions and noisier execution. Models live in res://bots/.
|
||||
|
||||
@export_file("*.json") var model_path: String = ""
|
||||
# Decide a new action every N physics ticks, holding the last one between
|
||||
# decisions. 8 matches the training action_repeat; larger = slower reactions.
|
||||
@export_range(1, 60) var reaction_ticks: int = 8
|
||||
# Uniform noise magnitude added to each action axis (0 = play at full skill).
|
||||
@export_range(0.0, 1.0) var action_noise: float = 0.0
|
||||
|
||||
var _policy: PolicyNetwork
|
||||
var _action := ShipAction.new()
|
||||
var _ticks_until_decision := 0
|
||||
|
||||
var _ship: Ship
|
||||
var _opponent: Ship
|
||||
var _ball: RigidBody3D
|
||||
var _attack_goal_position: Vector3
|
||||
var _scene_refs_ready := false
|
||||
|
||||
|
||||
func _ready():
|
||||
if not model_path.is_empty():
|
||||
_policy = PolicyNetwork.load_from_file(model_path)
|
||||
|
||||
|
||||
func get_action() -> ShipAction:
|
||||
if _policy == null:
|
||||
return _action # unloaded model: behaves like the inert placeholder
|
||||
if not _scene_refs_ready and not _discover_scene_refs():
|
||||
return _action
|
||||
|
||||
_ticks_until_decision -= 1
|
||||
if _ticks_until_decision <= 0:
|
||||
_ticks_until_decision = reaction_ticks
|
||||
_decide()
|
||||
return _action
|
||||
|
||||
|
||||
func _decide() -> void:
|
||||
var obs := ShipObservations.build(_ship, _opponent, _ball, _attack_goal_position)
|
||||
var out := _policy.forward(obs)
|
||||
# Output layout matches the flattened training action space (Box(7)):
|
||||
# thrust xyz, rotation xyz, turbo (> 0 means on).
|
||||
_action.thrust = Vector3(
|
||||
_axis(out[0]),
|
||||
_axis(out[1]),
|
||||
_axis(out[2])
|
||||
)
|
||||
_action.rotation = Vector3(
|
||||
_axis(out[3]),
|
||||
_axis(out[4]),
|
||||
_axis(out[5])
|
||||
)
|
||||
_action.turbo = out[6] > 0.0
|
||||
|
||||
|
||||
func _axis(value: float) -> float:
|
||||
if action_noise > 0.0:
|
||||
value += randf_range(-action_noise, action_noise)
|
||||
return clampf(value, -1.0, 1.0)
|
||||
|
||||
|
||||
# Find ship/ball/opponent/goal once everything is spawned. ShipAction axes
|
||||
# are body-frame so only observations need team context (ShipObservations).
|
||||
func _discover_scene_refs() -> bool:
|
||||
_ship = get_parent() as Ship
|
||||
if _ship == null or not is_inside_tree():
|
||||
return false
|
||||
_ball = get_tree().get_first_node_in_group("ball")
|
||||
for node in get_tree().get_nodes_in_group("ship"):
|
||||
if node != _ship:
|
||||
_opponent = node
|
||||
break
|
||||
for goal in get_tree().get_nodes_in_group("goal"):
|
||||
if goal.team == 1 - _ship.team:
|
||||
_attack_goal_position = goal.global_position
|
||||
if _ball == null:
|
||||
return false
|
||||
_scene_refs_ready = true
|
||||
return true
|
||||
@@ -0,0 +1 @@
|
||||
uid://4emuhiolrkb2
|
||||
@@ -1,15 +1,22 @@
|
||||
extends GameMode
|
||||
|
||||
# Timed match: two teams, score tracking, kickoff resets after each goal.
|
||||
# The opponent ship is currently inert (base ShipController, zero action) —
|
||||
# it becomes the AI opponent once an AIShipController exists (see TODO.md),
|
||||
# and additional player ships once multiplayer lands.
|
||||
# The opponent is a trained AI bot when a policy model is configured
|
||||
# (see TRAINING.md for training and promoting models into res://bots/),
|
||||
# otherwise an inert placeholder ship.
|
||||
|
||||
signal timer_updated(minutes: int, seconds: int)
|
||||
signal score_changed(score: Dictionary)
|
||||
|
||||
@export var match_length_seconds := 150.0
|
||||
|
||||
@export_group("AI opponent")
|
||||
# Trained policy for the opponent; empty = inert placeholder ship.
|
||||
@export_file("*.json") var bot_model_path: String = ""
|
||||
# Difficulty handicaps, applied on top of the model (see AIShipController).
|
||||
@export_range(1, 60) var bot_reaction_ticks: int = 8
|
||||
@export_range(0.0, 1.0) var bot_action_noise: float = 0.0
|
||||
|
||||
var score := {0: 0, 1: 0}
|
||||
var match_timer: Timer
|
||||
|
||||
@@ -18,7 +25,7 @@ func _start() -> void:
|
||||
spawn_ball()
|
||||
var player_ship := spawn_ship(0, 0, PlayerShipController.new())
|
||||
spawn_camera_rig(player_ship)
|
||||
spawn_ship(1, 0, ShipController.new()) # inert placeholder opponent
|
||||
spawn_ship(1, 0, _make_opponent_controller())
|
||||
|
||||
match_timer = Timer.new()
|
||||
match_timer.one_shot = true
|
||||
@@ -28,6 +35,18 @@ func _start() -> void:
|
||||
match_timer.start()
|
||||
|
||||
|
||||
func _make_opponent_controller() -> ShipController:
|
||||
if not bot_model_path.is_empty() and FileAccess.file_exists(bot_model_path):
|
||||
var bot := AIShipController.new()
|
||||
bot.model_path = bot_model_path
|
||||
bot.reaction_ticks = bot_reaction_ticks
|
||||
bot.action_noise = bot_action_noise
|
||||
return bot
|
||||
if not bot_model_path.is_empty():
|
||||
push_warning("MatchMode: bot model not found at %s, spawning inert opponent" % bot_model_path)
|
||||
return ShipController.new() # inert placeholder
|
||||
|
||||
|
||||
func _process(_delta):
|
||||
if match_timer and match_timer.time_left > 0:
|
||||
var remaining := ceili(match_timer.time_left)
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
class_name PolicyNetwork
|
||||
extends RefCounted
|
||||
|
||||
# Minimal MLP forward pass for running trained policies in pure GDScript —
|
||||
# no .NET build or ONNX runtime needed. Weights come from a JSON file written
|
||||
# by training/export_policy.py (see TRAINING.md). The policy net is tiny
|
||||
# (31 → 64 → 64 → 7 by default), and the bot only thinks every few physics
|
||||
# ticks, so GDScript is plenty fast.
|
||||
#
|
||||
# JSON shape:
|
||||
# {
|
||||
# "input_size": 31,
|
||||
# "layers": [
|
||||
# {"weights": [[out x in floats]], "biases": [out floats], "activation": "tanh" | "linear"},
|
||||
# ...
|
||||
# ]
|
||||
# }
|
||||
|
||||
var input_size: int = 0
|
||||
var _layers: Array = []
|
||||
|
||||
|
||||
static func load_from_file(path: String) -> PolicyNetwork:
|
||||
if not FileAccess.file_exists(path):
|
||||
push_error("PolicyNetwork: model file not found: %s" % path)
|
||||
return null
|
||||
var text := FileAccess.get_file_as_string(path)
|
||||
var data: Variant = JSON.parse_string(text)
|
||||
if data == null or not (data is Dictionary) or not data.has("layers"):
|
||||
push_error("PolicyNetwork: invalid model file: %s" % path)
|
||||
return null
|
||||
|
||||
var net := PolicyNetwork.new()
|
||||
net.input_size = int(data.get("input_size", 0))
|
||||
for layer in data["layers"]:
|
||||
# Flatten each layer's weights into a PackedFloat64Array for speed
|
||||
var out_size: int = layer["biases"].size()
|
||||
var in_size: int = layer["weights"][0].size()
|
||||
var flat := PackedFloat64Array()
|
||||
flat.resize(out_size * in_size)
|
||||
var i := 0
|
||||
for row in layer["weights"]:
|
||||
for value in row:
|
||||
flat[i] = value
|
||||
i += 1
|
||||
var biases := PackedFloat64Array(layer["biases"])
|
||||
net._layers.append({
|
||||
"weights": flat,
|
||||
"biases": biases,
|
||||
"in_size": in_size,
|
||||
"out_size": out_size,
|
||||
"tanh": layer.get("activation", "linear") == "tanh",
|
||||
})
|
||||
return net
|
||||
|
||||
|
||||
func forward(observation: Array) -> Array:
|
||||
var x := PackedFloat64Array(observation)
|
||||
for layer in _layers:
|
||||
var in_size: int = layer["in_size"]
|
||||
var out_size: int = layer["out_size"]
|
||||
var weights: PackedFloat64Array = layer["weights"]
|
||||
var biases: PackedFloat64Array = layer["biases"]
|
||||
var y := PackedFloat64Array()
|
||||
y.resize(out_size)
|
||||
for row in out_size:
|
||||
var sum := biases[row]
|
||||
var offset := row * in_size
|
||||
for col in in_size:
|
||||
sum += weights[offset + col] * x[col]
|
||||
y[row] = tanh(sum) if layer["tanh"] else sum
|
||||
x = y
|
||||
return Array(x)
|
||||
@@ -0,0 +1 @@
|
||||
uid://biixjudn05ib2
|
||||
@@ -0,0 +1,14 @@
|
||||
class_name RLShipController
|
||||
extends ShipController
|
||||
|
||||
# Controller for externally-driven ships (RL training and trained-policy
|
||||
# inference). Something else — a ShipAIController during training, an
|
||||
# AIShipController at play time — writes into `action`; the ship pulls it
|
||||
# each physics tick like any other controller. The ship never knows it is
|
||||
# being trained.
|
||||
|
||||
var action: ShipAction = ShipAction.new()
|
||||
|
||||
|
||||
func get_action() -> ShipAction:
|
||||
return action
|
||||
@@ -0,0 +1 @@
|
||||
uid://bctvr5djofbrm
|
||||
@@ -0,0 +1,90 @@
|
||||
class_name ShipAIController
|
||||
extends AIController3D
|
||||
|
||||
# Training-side bridge between godot_rl_agents and a ship. This is the only
|
||||
# class that touches plugin types (AIController3D / the Sync node protocol) —
|
||||
# everything else stays behind the ShipController seam: actions received from
|
||||
# the trainer are written into an RLShipController, which the ship pulls like
|
||||
# any other controller.
|
||||
#
|
||||
# Action space is ShipAction verbatim: 6 continuous axes (thrust xyz,
|
||||
# rotation xyz, each -1..1) + binary turbo. ShipAction axes are ship-local
|
||||
# (body frame), so they need no team mirroring — only observations do
|
||||
# (see ShipObservations.canon).
|
||||
|
||||
# Reward shaping weights. Dense terms accrue per physics tick (60 sim-ticks
|
||||
# per sim-second); event terms fire once. Exported so tuning needs no code
|
||||
# edits. Goal rewards are added by TrainingMode, which owns goal events.
|
||||
@export var ball_touch_reward := 0.1
|
||||
@export var velocity_to_ball_weight := 0.001
|
||||
@export var ball_velocity_to_goal_weight := 0.004
|
||||
|
||||
var ship: Ship
|
||||
var rl_controller: RLShipController
|
||||
var ball: RigidBody3D
|
||||
var opponent: Ship
|
||||
var attack_goal_position: Vector3
|
||||
|
||||
|
||||
# Wire up references after the ship is spawned. `attack_goal` is the goal
|
||||
# this ship scores into (goal.team == opponent's team).
|
||||
func setup(p_ship: Ship, p_rl_controller: RLShipController, p_ball: RigidBody3D, p_opponent: Ship, p_attack_goal_position: Vector3) -> void:
|
||||
ship = p_ship
|
||||
rl_controller = p_rl_controller
|
||||
ball = p_ball
|
||||
opponent = p_opponent
|
||||
attack_goal_position = p_attack_goal_position
|
||||
init(ship)
|
||||
|
||||
# Contact monitoring for the ball-touch reward (training-only cost;
|
||||
# the shipped game leaves contact_monitor off).
|
||||
ship.contact_monitor = true
|
||||
ship.max_contacts_reported = 8
|
||||
ship.body_entered.connect(_on_ship_body_entered)
|
||||
|
||||
|
||||
func get_obs() -> Dictionary:
|
||||
return {"obs": ShipObservations.build(ship, opponent, ball, attack_goal_position)}
|
||||
|
||||
|
||||
func get_reward() -> float:
|
||||
return reward
|
||||
|
||||
|
||||
func get_action_space() -> Dictionary:
|
||||
return {
|
||||
"thrust": {"size": 3, "action_type": "continuous"},
|
||||
"rotation": {"size": 3, "action_type": "continuous"},
|
||||
"turbo": {"size": 2, "action_type": "discrete"},
|
||||
}
|
||||
|
||||
|
||||
func set_action(action) -> void:
|
||||
var thrust: Array = action["thrust"]
|
||||
var rot: Array = action["rotation"]
|
||||
rl_controller.action.thrust = Vector3(thrust[0], thrust[1], thrust[2])
|
||||
rl_controller.action.rotation = Vector3(rot[0], rot[1], rot[2])
|
||||
rl_controller.action.turbo = int(action["turbo"]) == 1
|
||||
|
||||
|
||||
func _physics_process(delta):
|
||||
super(delta)
|
||||
if not is_instance_valid(ship) or not is_instance_valid(ball):
|
||||
return
|
||||
|
||||
# Dense shaping: own velocity toward the ball
|
||||
var to_ball := ball.global_position - ship.global_position
|
||||
if to_ball.length_squared() > 0.0001:
|
||||
var closing_speed := ship.linear_velocity.dot(to_ball.normalized())
|
||||
reward += velocity_to_ball_weight * closing_speed / ship.max_speed
|
||||
|
||||
# Dense shaping: ball velocity toward the goal we attack
|
||||
var ball_to_goal := attack_goal_position - ball.global_position
|
||||
if ball_to_goal.length_squared() > 0.0001:
|
||||
var ball_progress := ball.linear_velocity.dot(ball_to_goal.normalized())
|
||||
reward += ball_velocity_to_goal_weight * ball_progress / ShipObservations.BALL_SPEED_SCALE
|
||||
|
||||
|
||||
func _on_ship_body_entered(body: Node) -> void:
|
||||
if body.is_in_group("ball"):
|
||||
reward += ball_touch_reward
|
||||
@@ -0,0 +1 @@
|
||||
uid://bql1ixmk23u77
|
||||
@@ -0,0 +1,69 @@
|
||||
class_name ShipObservations
|
||||
extends RefCounted
|
||||
|
||||
# Canonical, team-relative observation builder. Shared by training
|
||||
# (ShipAIController) and in-game inference (AIShipController) so a trained
|
||||
# policy sees byte-identical inputs in both contexts — do not fork this logic.
|
||||
#
|
||||
# Self-play trick: observations for team 1 are rotated 180° about Y
|
||||
# (x → -x, z → -z), so every ship perceives itself attacking toward -Z
|
||||
# regardless of which side it spawned on. One policy can then play both teams.
|
||||
# The same rotation must be inverted when interpreting actions (see canon —
|
||||
# it is its own inverse).
|
||||
|
||||
# Normalization scales. Arena bounds: goals at z ≈ ±15.56, ship spawns at
|
||||
# z = ±12; positions are soft-normalized to roughly [-1, 1].
|
||||
const POSITION_SCALE := Vector3(20.0, 10.0, 20.0)
|
||||
const BALL_SPEED_SCALE := 30.0
|
||||
const GOAL_DISTANCE_SCALE := 40.0
|
||||
|
||||
# Number of floats build() returns; the policy input size.
|
||||
const SIZE := 31
|
||||
|
||||
|
||||
# 180° rotation about Y for team 1; identity for team 0. A proper rotation
|
||||
# (preserves handedness), and its own inverse — used for both observations
|
||||
# and mapping canonical-frame actions back to world intent.
|
||||
static func canon(v: Vector3, team: int) -> Vector3:
|
||||
return v if team == 0 else Vector3(-v.x, v.y, -v.z)
|
||||
|
||||
|
||||
# attack_goal_position: centre of the goal this ship is trying to score in
|
||||
# (the goal whose `team` == the opponent's team).
|
||||
static func build(ship: Ship, opponent: Ship, ball: RigidBody3D, attack_goal_position: Vector3) -> Array:
|
||||
var team := ship.team
|
||||
var obs := []
|
||||
|
||||
# Own kinematics
|
||||
_append(obs, canon(ship.global_position, team) / POSITION_SCALE)
|
||||
_append(obs, canon(-ship.global_transform.basis.z, team)) # forward
|
||||
_append(obs, canon(ship.global_transform.basis.y, team)) # up
|
||||
_append(obs, canon(ship.linear_velocity, team) / ship.max_speed)
|
||||
_append(obs, canon(ship.angular_velocity, team) / ship.max_angular_speed)
|
||||
|
||||
# Ball, relative to self
|
||||
var ball_rel := ball.global_position - ship.global_position
|
||||
_append(obs, canon(ball_rel, team) / POSITION_SCALE)
|
||||
_append(obs, canon(ball.linear_velocity, team) / BALL_SPEED_SCALE)
|
||||
|
||||
# Opponent, relative to self (zeros if absent, e.g. a 1-ship drill)
|
||||
if is_instance_valid(opponent):
|
||||
var opp_rel := opponent.global_position - ship.global_position
|
||||
_append(obs, canon(opp_rel, team) / POSITION_SCALE)
|
||||
_append(obs, canon(opponent.linear_velocity, team) / ship.max_speed)
|
||||
else:
|
||||
_append(obs, Vector3.ZERO)
|
||||
_append(obs, Vector3.ZERO)
|
||||
|
||||
# Goal we are attacking, relative to self
|
||||
var goal_rel := attack_goal_position - ship.global_position
|
||||
_append(obs, canon(goal_rel, team) / POSITION_SCALE)
|
||||
obs.append(goal_rel.length() / GOAL_DISTANCE_SCALE)
|
||||
|
||||
return obs
|
||||
|
||||
|
||||
static func _append(obs: Array, v: Vector3) -> void:
|
||||
obs.append(v.x)
|
||||
obs.append(v.y)
|
||||
obs.append(v.z)
|
||||
@@ -0,0 +1 @@
|
||||
uid://bqux8vc4eou73
|
||||
@@ -0,0 +1,244 @@
|
||||
class_name TrainingMode
|
||||
extends GameMode
|
||||
|
||||
# Headless self-play training mode: two RL-driven ships, no HUD, no camera.
|
||||
# The scene also contains the godot_rl_agents Sync node, which speaks TCP to
|
||||
# the Python trainer; this mode owns the environment rules — episodes, goal
|
||||
# rewards, and randomized episode-start states (the RLGym "state setter"
|
||||
# lesson: varied starts massively speed up learning versus kickoff-only).
|
||||
#
|
||||
# Run: godot --headless --path Game res://scenes/training.tscn
|
||||
# (started automatically by training/train.py; boots into idle ships with a
|
||||
# warning if no trainer is listening).
|
||||
#
|
||||
# Eval mode (used by training/evaluate.py): pass --eval_model_a=<path> and
|
||||
# --eval_model_b=<path> (+ optional --eval_episodes=N) and both ships are
|
||||
# instead driven by those exported policies via AIShipController; each episode
|
||||
# ends at the first goal (or a draw on timeout), and a final machine-readable
|
||||
# "EVAL_RESULT {...}" line is printed before quitting.
|
||||
|
||||
@export var episode_length_seconds := 30.0
|
||||
@export var goal_reward := 10.0
|
||||
|
||||
# Episode-start state mix; remaining probability = fully random state.
|
||||
@export_range(0.0, 1.0) var kickoff_state_chance := 0.2
|
||||
@export_range(0.0, 1.0) var ball_near_goal_chance := 0.2
|
||||
|
||||
# Placement bounds, inset from the arena (goals at z ≈ ±15.56).
|
||||
const FIELD_HALF_X := 10.0
|
||||
const FIELD_HALF_Z := 13.0
|
||||
const FIELD_MIN_Y := 1.5
|
||||
const FIELD_MAX_Y := 8.0
|
||||
const MAX_RANDOM_BALL_SPEED := 12.0
|
||||
const MAX_RANDOM_SHIP_SPEED := 8.0
|
||||
|
||||
# Sim runs at 60 physics ticks per sim-second regardless of speedup.
|
||||
const TICKS_PER_SIM_SECOND := 60.0
|
||||
|
||||
# The arena has no walls/ceiling yet (see TODO.md), so an untrained policy can
|
||||
# simply fly away. Leaving this volume ends the episode with a small penalty.
|
||||
const BOUNDS_HALF_X := 25.0
|
||||
const BOUNDS_HALF_Z := 25.0
|
||||
const BOUNDS_MIN_Y := -5.0
|
||||
const BOUNDS_MAX_Y := 25.0
|
||||
@export var out_of_bounds_penalty := 1.0
|
||||
|
||||
var _agents: Array[ShipAIController] = []
|
||||
|
||||
# Eval mode state (see header comment)
|
||||
var _eval := false
|
||||
var _eval_models: Array[String] = ["", ""]
|
||||
var _eval_episodes := 20
|
||||
var _eval_goals := {0: 0, 1: 0}
|
||||
var _eval_draws := 0
|
||||
var _eval_episodes_done := 0
|
||||
var _episode_ticks := 0
|
||||
|
||||
|
||||
func _start() -> void:
|
||||
_parse_eval_args()
|
||||
spawn_ball()
|
||||
if _eval:
|
||||
for team in [0, 1]:
|
||||
var bot := AIShipController.new()
|
||||
bot.model_path = _eval_models[team]
|
||||
spawn_ship(team, 0, bot)
|
||||
return
|
||||
var ship_team0 := spawn_ship(0, 0, RLShipController.new())
|
||||
var ship_team1 := spawn_ship(1, 0, RLShipController.new())
|
||||
_attach_agent(ship_team0, ship_team1)
|
||||
_attach_agent(ship_team1, ship_team0)
|
||||
|
||||
|
||||
func _parse_eval_args() -> void:
|
||||
var args := {}
|
||||
for argument in OS.get_cmdline_args():
|
||||
if argument.begins_with("--") and argument.find("=") > -1:
|
||||
var key_value := argument.lstrip("--").split("=", true, 1)
|
||||
args[key_value[0]] = key_value[1]
|
||||
if args.has("eval_model_a") and args.has("eval_model_b"):
|
||||
_eval = true
|
||||
_eval_models[0] = args["eval_model_a"]
|
||||
_eval_models[1] = args["eval_model_b"]
|
||||
_eval_episodes = int(args.get("eval_episodes", str(_eval_episodes)))
|
||||
|
||||
|
||||
func _attach_agent(ship: Ship, opponent: Ship) -> void:
|
||||
var agent := ShipAIController.new()
|
||||
agent.name = "ShipAIController"
|
||||
agent.reset_after = int(episode_length_seconds * TICKS_PER_SIM_SECOND)
|
||||
ship.add_child(agent)
|
||||
agent.setup(ship, ship.controller as RLShipController, ball, opponent, _attack_goal_position(ship.team))
|
||||
_agents.append(agent)
|
||||
|
||||
|
||||
# The goal this team scores into: the one the opponent defends/concedes.
|
||||
func _attack_goal_position(team: int) -> Vector3:
|
||||
for goal in arena.get_goals():
|
||||
if goal.team == 1 - team:
|
||||
return goal.global_position
|
||||
push_error("TrainingMode: no goal found for team %d to attack" % team)
|
||||
return Vector3.ZERO
|
||||
|
||||
|
||||
func _physics_process(_delta):
|
||||
if _eval:
|
||||
_episode_ticks += 1
|
||||
for ship in ships:
|
||||
if is_instance_valid(ship) and _out_of_bounds(ship.global_position):
|
||||
_place_body(ship, _ship_spawn_transforms[ship], Vector3.ZERO, Vector3.ZERO)
|
||||
var ball_lost := is_instance_valid(ball) and _out_of_bounds(ball.global_position)
|
||||
if _episode_ticks > int(episode_length_seconds * TICKS_PER_SIM_SECOND) or ball_lost:
|
||||
_eval_draws += 1
|
||||
_end_eval_episode()
|
||||
return
|
||||
|
||||
# Both trainer-requested resets and truncation (reset_after ticks elapsed)
|
||||
# surface as needs_reset. Only truncation is an episode end the trainer
|
||||
# must be told about via done — a trainer-requested reset already knows.
|
||||
var needs_reset := false
|
||||
var truncated := false
|
||||
for agent in _agents:
|
||||
needs_reset = needs_reset or agent.needs_reset
|
||||
truncated = truncated or agent.n_steps > agent.reset_after
|
||||
if needs_reset:
|
||||
if truncated:
|
||||
for agent in _agents:
|
||||
agent.done = true
|
||||
_reset_episode()
|
||||
return
|
||||
|
||||
# A ship leaving the play volume is penalized and respawned at its kickoff
|
||||
# spawn (the episode continues); a lost ball ends the episode for both.
|
||||
for agent in _agents:
|
||||
if _out_of_bounds(agent.ship.global_position):
|
||||
agent.reward -= out_of_bounds_penalty
|
||||
_place_body(agent.ship, _ship_spawn_transforms[agent.ship], Vector3.ZERO, Vector3.ZERO)
|
||||
if is_instance_valid(ball) and _out_of_bounds(ball.global_position) and not _agents.is_empty():
|
||||
for agent in _agents:
|
||||
agent.done = true
|
||||
_reset_episode()
|
||||
|
||||
|
||||
func _out_of_bounds(position: Vector3) -> bool:
|
||||
return absf(position.x) > BOUNDS_HALF_X \
|
||||
or absf(position.z) > BOUNDS_HALF_Z \
|
||||
or position.y < BOUNDS_MIN_Y \
|
||||
or position.y > BOUNDS_MAX_Y
|
||||
|
||||
|
||||
func _on_goal_scored(conceding_team: int) -> void:
|
||||
if _eval:
|
||||
_eval_goals[1 - conceding_team] += 1
|
||||
_end_eval_episode()
|
||||
return
|
||||
for agent in _agents:
|
||||
agent.reward += goal_reward if agent.ship.team != conceding_team else -goal_reward
|
||||
agent.done = true
|
||||
_reset_episode()
|
||||
|
||||
|
||||
func _end_eval_episode() -> void:
|
||||
_eval_episodes_done += 1
|
||||
_episode_ticks = 0
|
||||
if _eval_episodes_done >= _eval_episodes:
|
||||
print("EVAL_RESULT " + JSON.stringify({
|
||||
"model_a": _eval_models[0],
|
||||
"model_b": _eval_models[1],
|
||||
"episodes": _eval_episodes_done,
|
||||
"goals_a": _eval_goals[0],
|
||||
"goals_b": _eval_goals[1],
|
||||
"draws": _eval_draws,
|
||||
}))
|
||||
get_tree().quit()
|
||||
return
|
||||
# Randomized states (not kickoff): deterministic policies would otherwise
|
||||
# replay the identical episode every time.
|
||||
_reset_episode()
|
||||
|
||||
|
||||
func _reset_episode() -> void:
|
||||
for agent in _agents:
|
||||
agent.reset()
|
||||
|
||||
var roll := randf()
|
||||
if roll < kickoff_state_chance:
|
||||
reset_ball()
|
||||
reset_ships()
|
||||
elif roll < kickoff_state_chance + ball_near_goal_chance:
|
||||
_place_ships_random()
|
||||
_place_ball_near_goal()
|
||||
else:
|
||||
_place_ships_random()
|
||||
_place_ball_random()
|
||||
|
||||
|
||||
func _place_ball_random() -> void:
|
||||
var velocity := _random_direction() * randf_range(0.0, MAX_RANDOM_BALL_SPEED)
|
||||
_place_body(ball, Transform3D(Basis.IDENTITY, _random_position()), velocity, Vector3.ZERO)
|
||||
|
||||
|
||||
# Attacking/defending drill states: ball close to a goal, moving toward it.
|
||||
func _place_ball_near_goal() -> void:
|
||||
var goals := arena.get_goals()
|
||||
var goal: Goal = goals[randi() % goals.size()]
|
||||
var toward_centre := -signf(goal.global_position.z)
|
||||
var position := Vector3(
|
||||
randf_range(-4.0, 4.0),
|
||||
randf_range(FIELD_MIN_Y, 4.0),
|
||||
goal.global_position.z + toward_centre * randf_range(3.0, 6.0)
|
||||
)
|
||||
var to_goal := (goal.global_position - position).normalized()
|
||||
var velocity := (to_goal + _random_direction() * 0.3).normalized() * randf_range(2.0, MAX_RANDOM_BALL_SPEED)
|
||||
_place_body(ball, Transform3D(Basis.IDENTITY, position), velocity, Vector3.ZERO)
|
||||
|
||||
|
||||
func _place_ships_random() -> void:
|
||||
for ship in ships:
|
||||
var orientation := Basis.from_euler(Vector3(
|
||||
randf_range(-0.4, 0.4),
|
||||
randf_range(-PI, PI),
|
||||
randf_range(-0.4, 0.4)
|
||||
))
|
||||
var velocity := _random_direction() * randf_range(0.0, MAX_RANDOM_SHIP_SPEED)
|
||||
_place_body(ship, Transform3D(orientation, _random_position()), velocity, Vector3.ZERO)
|
||||
|
||||
|
||||
func _random_position() -> Vector3:
|
||||
return Vector3(
|
||||
randf_range(-FIELD_HALF_X, FIELD_HALF_X),
|
||||
randf_range(FIELD_MIN_Y, FIELD_MAX_Y),
|
||||
randf_range(-FIELD_HALF_Z, FIELD_HALF_Z)
|
||||
)
|
||||
|
||||
|
||||
func _random_direction() -> Vector3:
|
||||
var direction := Vector3(randf_range(-1, 1), randf_range(-1, 1), randf_range(-1, 1))
|
||||
return direction.normalized() if direction.length_squared() > 0.001 else Vector3.FORWARD
|
||||
|
||||
|
||||
func _place_body(body: RigidBody3D, to: Transform3D, linear_velocity: Vector3, angular_velocity: Vector3) -> void:
|
||||
# Deferred: a RigidBody3D transform can't be set mid-physics-step
|
||||
body.set_deferred("global_transform", to)
|
||||
body.set_deferred("linear_velocity", linear_velocity)
|
||||
body.set_deferred("angular_velocity", angular_velocity)
|
||||
@@ -0,0 +1 @@
|
||||
uid://d2kfcobanp6iv
|
||||
Reference in New Issue
Block a user