diff --git a/Game/scripts/ship_ai_controller.gd b/Game/scripts/ship_ai_controller.gd index a91f6091..1d77805e 100644 --- a/Game/scripts/ship_ai_controller.gd +++ b/Game/scripts/ship_ai_controller.gd @@ -107,6 +107,15 @@ var ball: RigidBody3D var opponent: Ship var attack_goal_position: Vector3 +# Set directly by TrainingMode (_on_goal_scored / the timeout branch in +# _physics_process) at the same time as `done = true`. Deliberately NOT +# cleared in reset(): TrainingMode's _reset_episode() (which calls reset()) +# runs synchronously, immediately after done is set, before the Sync node +# ever reads get_info()/get_done() for that terminal tick — clearing it here +# would wipe the value that read needs. Both call sites always overwrite +# (true on goal, false on timeout) rather than toggle, so no reset is needed. +var goal_scored_this_episode := false + var _ticks_since_ball_touch := 1 << 30 # large so the first touch always pays @@ -135,6 +144,14 @@ func get_reward() -> float: return reward +# Symmetric across both self-play agents: reports whether this episode ended +# in a goal at all, not which team scored — a clean "goal rate" signal +# distinct from rollout/ep_rew_mean, which mixes this with dense shaping +# (ball chasing/touching). See train.py's GoalRateCallback. +func get_info() -> Dictionary: + return {"goal_scored": goal_scored_this_episode} + + func get_action_space() -> Dictionary: return { "thrust": {"size": 3, "action_type": "continuous"}, diff --git a/Game/scripts/training_mode.gd b/Game/scripts/training_mode.gd index 7c641c7e..6461efd9 100644 --- a/Game/scripts/training_mode.gd +++ b/Game/scripts/training_mode.gd @@ -290,6 +290,7 @@ func _physics_process(_delta): for agent in _agents: agent.reward -= draw_penalty agent.done = true + agent.goal_scored_this_episode = false _reset_episode() return @@ -320,6 +321,7 @@ func _on_goal_scored(conceding_team: int) -> void: for agent in _agents: agent.reward += goal_reward if agent.ship.team != conceding_team else -goal_reward agent.done = true + agent.goal_scored_this_episode = true _reset_episode() diff --git a/TRAINING.md b/TRAINING.md index bab5e2ed..b08573ee 100644 --- a/TRAINING.md +++ b/TRAINING.md @@ -89,7 +89,11 @@ real behaviour needs tens of millions of steps (hours on the 3090 box). Key curves: `rollout/ep_rew_mean` (should trend up), `rollout/ep_len_mean` (should trend *down* from 225 as goals end episodes early — 225 action steps -= the 30s episode timeout). += the 30s episode timeout), `rollout/goal_rate` (fraction of recent episodes +that ended in an actual goal rather than timing out as a draw — the live +signal for "is the policy actually finishing more episodes by scoring", +since `ep_rew_mean` mixes that with dense reward-shaping (ball chasing/ +touching) and doesn't isolate it). ### Reward/observation tuning diff --git a/training/train.py b/training/train.py index fce04844..c6553cec 100644 --- a/training/train.py +++ b/training/train.py @@ -16,7 +16,8 @@ import os import pathlib from stable_baselines3 import PPO -from stable_baselines3.common.callbacks import CheckpointCallback +from stable_baselines3.common.callbacks import BaseCallback, CheckpointCallback +from stable_baselines3.common.utils import safe_mean from stable_baselines3.common.vec_env.vec_monitor import VecMonitor from cosmic_env import CosmicClashVecEnv @@ -25,6 +26,27 @@ TRAINING_DIR = pathlib.Path(__file__).resolve().parent DEFAULT_GODOT_MACOS = "/Applications/Godot.app/Contents/MacOS/Godot" +class GoalRateCallback(BaseCallback): + """Logs rollout/goal_rate: the fraction of completed episodes in the + current ep_info_buffer that ended in an actual goal, vs. timing out as a + draw. rollout/ep_rew_mean mixes dense reward-shaping (ball chasing/ + touching) with the sparse terminal goal reward, so it can trend up from + better shaping alone without the policy finishing more episodes by + actually scoring — this isolates that. Requires VecMonitor(..., + info_keywords=("goal_scored",)), which copies ShipAIController.get_info() + into each completed episode's info["episode"] dict (see + training_mode.gd's _on_goal_scored / timeout branch).""" + + def _on_step(self) -> bool: + return True + + def _on_rollout_end(self) -> None: + if len(self.model.ep_info_buffer) == 0: + return + goal_rate = safe_mean([ep_info["goal_scored"] for ep_info in self.model.ep_info_buffer]) + self.logger.record("rollout/goal_rate", goal_rate) + + def parse_args(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -164,7 +186,7 @@ def main(): speedup=args.speedup, **_curriculum_kwargs(args), ) - env = VecMonitor(env) + env = VecMonitor(env, info_keywords=("goal_scored",)) if args.resume: model = PPO.load( @@ -204,11 +226,12 @@ def main(): save_path=str(checkpoint_dir), name_prefix="ppo", ) + goal_rate_callback = GoalRateCallback() try: model.learn( args.timesteps, - callback=checkpoint_callback, + callback=[checkpoint_callback, goal_rate_callback], tb_log_name=args.experiment, reset_num_timesteps=not args.resume, )