feat(*): Add self-play RL training pipeline with PPO trainer, in-game GDScript policy inference, and bot opponent support in Match mode

training/cosmic_env.py

@@ -0,0 +1,76 @@
+"""Godot RL Agents environment wrappers that run Cosmic Clash from source.
+
+Stock GodotEnv expects an *exported* game executable and rewrites its path
+per-platform. These subclasses launch the project straight from the repo with
+a Godot binary instead (no export step), pointing it at the training scene.
+Each Godot instance contributes two agents (one ship per team) that share the
+learning policy: self-play by construction.
+"""
+
+import pathlib
+import subprocess
+
+from godot_rl.core.godot_env import GodotEnv
+from godot_rl.wrappers.stable_baselines_wrapper import StableBaselinesGodotEnv
+
+REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent
+GAME_DIR = REPO_ROOT / "Game"
+TRAINING_SCENE = "res://scenes/training.tscn"
+
+
+class CosmicClashEnv(GodotEnv):
+    """GodotEnv that launches `godot --path Game res://scenes/training.tscn`."""
+
+    # env_path is a Godot binary, not an exported game: skip the suffix and
+    # platform checks stock GodotEnv applies to exported executables.
+    def _set_platform_suffix(self, env_path: str) -> str:
+        return env_path
+
+    def check_platform(self, filename: str):
+        pass
+
+    def _launch_env(self, env_path, port, show_window, framerate, seed, action_repeat, speedup, **kwargs):
+        # sync.gd reads --key=value pairs from the raw command line; they must
+        # NOT go after a `--` separator or OS.get_cmdline_args() drops them.
+        cmd = [
+            env_path,
+            "--path",
+            str(GAME_DIR),
+            TRAINING_SCENE,
+            f"--port={port}",
+            f"--env_seed={seed}",
+        ]
+        if not show_window:
+            cmd += ["--headless", "--disable-render-loop"]
+        if framerate is not None:
+            cmd += ["--fixed-fps", str(framerate)]
+        if action_repeat is not None:
+            cmd.append(f"--action_repeat={action_repeat}")
+        if speedup is not None:
+            cmd.append(f"--speedup={speedup}")
+        for key, value in kwargs.items():
+            cmd.append(f"--{key}={value}")
+        self.proc = subprocess.Popen(cmd, start_new_session=True)
+
+
+class CosmicClashVecEnv(StableBaselinesGodotEnv):
+    """SB3 VecEnv over N parallel CosmicClashEnv instances.
+
+    convert_action_space=True flattens the env's (Box(6), Discrete(2)) action
+    space into a single Box(7): thrust xyz, rotation xyz, turbo (>0 means on).
+    """
+
+    def __init__(self, godot_bin: str, n_parallel: int = 1, seed: int = 0, port: int = GodotEnv.DEFAULT_PORT, **kwargs):
+        self.envs = [
+            CosmicClashEnv(
+                env_path=godot_bin,
+                convert_action_space=True,
+                port=port + p,
+                seed=seed + p,
+                **kwargs,
+            )
+            for p in range(n_parallel)
+        ]
+        self.n_parallel = n_parallel
+        self._check_valid_action_space()
+        self.results = None

training/eval_history.json

@@ -0,0 +1,12 @@
+[
+  {
+    "timestamp": "2026-07-18T18:27:59+00:00",
+    "model_a": "/Users/jcreek/Documents/repos/GitHub/CosmicClash/Game/bots/rookie.json",
+    "model_b": "/Users/jcreek/Documents/repos/GitHub/CosmicClash/Game/bots/rookie.json",
+    "episodes": 6,
+    "wins_a": 1,
+    "wins_b": 0,
+    "draws": 5,
+    "win_rate_a": 0.167
+  }
+]

training/evaluate.py

@@ -0,0 +1,96 @@
+"""Pit two exported policies against each other and record the result.
+
+Uses the same in-Godot inference path that ships in the game
+(AIShipController + PolicyNetwork), so eval strength = in-game strength.
+Episodes are golden-goal: first goal wins, timeout is a draw. Half the
+episodes are played with sides swapped for fairness. Results are appended to
+eval_history.json — the bot-progress-over-time record.
+
+Example:
+    .venv/bin/python evaluate.py ../Game/bots/rookie.json checkpoints/run01/candidate.json --episodes 40
+"""
+
+import argparse
+import datetime
+import json
+import os
+import pathlib
+import subprocess
+
+TRAINING_DIR = pathlib.Path(__file__).resolve().parent
+GAME_DIR = TRAINING_DIR.parent / "Game"
+TRAINING_SCENE = "res://scenes/training.tscn"
+DEFAULT_GODOT_MACOS = "/Applications/Godot.app/Contents/MacOS/Godot"
+
+
+def run_half(godot_bin: str, model_a: str, model_b: str, episodes: int, speedup: int, seed: int) -> dict:
+    cmd = [
+        godot_bin,
+        "--path",
+        str(GAME_DIR),
+        TRAINING_SCENE,
+        "--headless",
+        "--disable-render-loop",
+        f"--eval_model_a={model_a}",
+        f"--eval_model_b={model_b}",
+        f"--eval_episodes={episodes}",
+        f"--speedup={speedup}",
+        f"--env_seed={seed}",
+    ]
+    timeout = episodes * 30 / speedup * 3 + 120  # worst case: all draws, plus margin
+    result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
+    for line in result.stdout.splitlines():
+        if line.startswith("EVAL_RESULT "):
+            return json.loads(line[len("EVAL_RESULT "):])
+    raise RuntimeError(f"No EVAL_RESULT in godot output:\n{result.stdout[-2000:]}\n{result.stderr[-2000:]}")
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("model_a", help="Path to first exported policy .json")
+    parser.add_argument("model_b", help="Path to second exported policy .json")
+    parser.add_argument("--episodes", type=int, default=20, help="Total episodes (split across side swap)")
+    parser.add_argument(
+        "--godot_bin",
+        default=os.environ.get("GODOT_BIN", DEFAULT_GODOT_MACOS),
+        help="Path to the Godot binary (or set GODOT_BIN)",
+    )
+    parser.add_argument("--speedup", type=int, default=16)
+    parser.add_argument("--history", default=str(TRAINING_DIR / "eval_history.json"))
+    args = parser.parse_args()
+
+    model_a = str(pathlib.Path(args.model_a).resolve())
+    model_b = str(pathlib.Path(args.model_b).resolve())
+    half = max(args.episodes // 2, 1)
+
+    # Half the episodes on each side to cancel any residual side asymmetry;
+    # different seeds so the halves see different randomized episode states.
+    first = run_half(args.godot_bin, model_a, model_b, half, args.speedup, seed=1)
+    second = run_half(args.godot_bin, model_b, model_a, half, args.speedup, seed=2)
+
+    record = {
+        "timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"),
+        "model_a": model_a,
+        "model_b": model_b,
+        "episodes": first["episodes"] + second["episodes"],
+        "wins_a": first["goals_a"] + second["goals_b"],
+        "wins_b": first["goals_b"] + second["goals_a"],
+        "draws": first["draws"] + second["draws"],
+    }
+    record["win_rate_a"] = round(record["wins_a"] / record["episodes"], 3)
+
+    history_path = pathlib.Path(args.history)
+    history = json.loads(history_path.read_text()) if history_path.exists() else []
+    history.append(record)
+    history_path.write_text(json.dumps(history, indent=2) + "\n")
+
+    print(
+        f"{pathlib.Path(model_a).name} vs {pathlib.Path(model_b).name} over {record['episodes']} episodes: "
+        f"{record['wins_a']}-{record['wins_b']} ({record['draws']} draws), "
+        f"win rate A = {record['win_rate_a']:.0%}"
+    )
+    print(f"Appended to {history_path}")
+
+
+if __name__ == "__main__":
+    main()

training/export_policy.py

@@ -0,0 +1,86 @@
+"""Export a trained SB3 checkpoint to the JSON format PolicyNetwork.gd loads.
+
+The exported file contains the deterministic policy MLP (obs -> action means);
+the game clamps outputs to [-1, 1] and treats the last value as turbo (> 0).
+A parity self-check compares the JSON forward pass against SB3's own
+deterministic prediction before writing.
+
+Example:
+    .venv/bin/python export_policy.py checkpoints/smoke/final.zip ../Game/bots/rookie.json
+"""
+
+import argparse
+import json
+import pathlib
+
+import numpy as np
+import torch
+from stable_baselines3 import PPO
+
+
+def linear_to_layer(linear: torch.nn.Linear, activation: str) -> dict:
+    return {
+        "weights": linear.weight.detach().cpu().numpy().tolist(),
+        "biases": linear.bias.detach().cpu().numpy().tolist(),
+        "activation": activation,
+    }
+
+
+def extract_layers(policy) -> list[dict]:
+    # Features extractor must be a passthrough (flatten) for this export to
+    # be faithful; it has no parameters for our flat "obs" Box space.
+    n_extractor_params = sum(p.numel() for p in policy.features_extractor.parameters())
+    assert n_extractor_params == 0, "features extractor has weights; export logic needs updating"
+
+    layers = []
+    modules = list(policy.mlp_extractor.policy_net)
+    for i, module in enumerate(modules):
+        if isinstance(module, torch.nn.Linear):
+            next_is_tanh = i + 1 < len(modules) and isinstance(modules[i + 1], torch.nn.Tanh)
+            assert next_is_tanh or i + 1 >= len(modules), (
+                f"unsupported activation after layer {i}: {modules[i + 1] if i + 1 < len(modules) else None}"
+            )
+            layers.append(linear_to_layer(module, "tanh" if next_is_tanh else "linear"))
+        elif not isinstance(module, torch.nn.Tanh):
+            raise AssertionError(f"unsupported module in policy net: {module}")
+    layers.append(linear_to_layer(policy.action_net, "linear"))
+    return layers
+
+
+def json_forward(layers: list[dict], obs: np.ndarray) -> np.ndarray:
+    x = obs
+    for layer in layers:
+        x = np.asarray(layer["weights"]) @ x + np.asarray(layer["biases"])
+        if layer["activation"] == "tanh":
+            x = np.tanh(x)
+    return x
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("checkpoint", help="SB3 checkpoint .zip (e.g. checkpoints/smoke/final.zip)")
+    parser.add_argument("output", help="Output JSON path (e.g. ../Game/bots/rookie.json)")
+    args = parser.parse_args()
+
+    model = PPO.load(args.checkpoint, device="cpu")
+    policy = model.policy
+    layers = extract_layers(policy)
+    input_size = model.observation_space["obs"].shape[0]
+
+    # Parity check: JSON forward pass must match SB3's deterministic action
+    rng = np.random.default_rng(0)
+    for _ in range(16):
+        obs = rng.uniform(-1, 1, input_size).astype(np.float32)
+        expected, _ = model.predict({"obs": obs}, deterministic=True)
+        actual = np.clip(json_forward(layers, obs), -1.0, 1.0)
+        assert np.allclose(actual, expected, atol=1e-5), f"parity check failed: {actual} vs {expected}"
+
+    output = pathlib.Path(args.output)
+    output.parent.mkdir(parents=True, exist_ok=True)
+    with open(output, "w") as f:
+        json.dump({"input_size": int(input_size), "layers": layers}, f)
+    print(f"Exported {args.checkpoint} -> {output} (input size {input_size}, {len(layers)} layers, parity OK)")
+
+
+if __name__ == "__main__":
+    main()

training/requirements.txt

@@ -0,0 +1,5 @@
+godot-rl
+stable-baselines3
+tensorboard
+# Optional, for --wandb logging:
+# wandb

training/train.py

@@ -0,0 +1,105 @@
+"""Train the Cosmic Clash self-play PPO policy.
+
+Example (smoke run):
+    .venv/bin/python train.py --experiment smoke --timesteps 100000
+
+Long run on the Linux/CUDA box:
+    GODOT_BIN=~/godot/Godot_v4.7.1-stable_linux.x86_64 \
+        .venv/bin/python train.py --experiment run01 --timesteps 20000000 \
+        --n-parallel 6 --speedup 16
+
+See TRAINING.md at the repo root for the full workflow.
+"""
+
+import argparse
+import os
+import pathlib
+
+from stable_baselines3 import PPO
+from stable_baselines3.common.callbacks import CheckpointCallback
+from stable_baselines3.common.vec_env.vec_monitor import VecMonitor
+
+from cosmic_env import CosmicClashVecEnv
+
+TRAINING_DIR = pathlib.Path(__file__).resolve().parent
+DEFAULT_GODOT_MACOS = "/Applications/Godot.app/Contents/MacOS/Godot"
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--godot_bin",
+        default=os.environ.get("GODOT_BIN", DEFAULT_GODOT_MACOS),
+        help="Path to the Godot binary (or set GODOT_BIN)",
+    )
+    parser.add_argument("--experiment", default="default", help="Run name for logs/checkpoints")
+    parser.add_argument("--timesteps", type=int, default=200_000)
+    parser.add_argument("--n-parallel", type=int, default=2, help="Parallel Godot instances (2 agents each)")
+    parser.add_argument("--speedup", type=int, default=8, help="Physics speedup factor inside Godot")
+    parser.add_argument("--port", type=int, default=11008, help="Base TCP port (one per instance)")
+    parser.add_argument("--seed", type=int, default=0)
+    parser.add_argument("--resume", default=None, help="Checkpoint .zip to resume from")
+    parser.add_argument("--checkpoint-every", type=int, default=100_000, help="Timesteps between checkpoints")
+    parser.add_argument("--viz", action="store_true", help="Show game windows (debugging; slow)")
+    parser.add_argument("--wandb", action="store_true", help="Also log to Weights & Biases")
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    log_dir = TRAINING_DIR / "logs"
+    checkpoint_dir = TRAINING_DIR / "checkpoints" / args.experiment
+    checkpoint_dir.mkdir(parents=True, exist_ok=True)
+
+    if args.wandb:
+        import wandb
+
+        wandb.init(project="cosmic-clash-rl", name=args.experiment, sync_tensorboard=True)
+
+    env = CosmicClashVecEnv(
+        godot_bin=args.godot_bin,
+        n_parallel=args.n_parallel,
+        seed=args.seed,
+        port=args.port,
+        show_window=args.viz,
+        speedup=args.speedup,
+    )
+    env = VecMonitor(env)
+
+    if args.resume:
+        model = PPO.load(args.resume, env=env, tensorboard_log=str(log_dir))
+        print(f"Resumed from {args.resume} at {model.num_timesteps} timesteps")
+    else:
+        model = PPO(
+            "MultiInputPolicy",
+            env,
+            verbose=1,
+            ent_coef=0.0001,
+            n_steps=256,
+            batch_size=256,
+            learning_rate=3e-4,
+            tensorboard_log=str(log_dir),
+        )
+
+    checkpoint_callback = CheckpointCallback(
+        save_freq=max(args.checkpoint_every // env.num_envs, 1),
+        save_path=str(checkpoint_dir),
+        name_prefix="ppo",
+    )
+
+    try:
+        model.learn(
+            args.timesteps,
+            callback=checkpoint_callback,
+            tb_log_name=args.experiment,
+            reset_num_timesteps=not args.resume,
+        )
+    finally:
+        final_path = checkpoint_dir / "final.zip"
+        model.save(str(final_path))
+        print(f"Saved {final_path}")
+        env.close()
+
+
+if __name__ == "__main__":
+    main()