feat(*): Add exported Linux binary training path for faster parallel instances

.gitignore

@@ -5,3 +5,7 @@
 training/.venv/
 training/smoke_run.log
 training/__pycache__/
+
+# Exported training binary: a regenerable build artifact (rebuilt by
+# export_linux.sh / run_training.sh), not a training result.
+training/build/

Game/export_presets.cfg

@@ -0,0 +1,28 @@
+[preset.0]
+
+name="Linux Training"
+platform="Linux"
+runnable=true
+dedicated_server=false
+custom_features="training"
+export_filter="all_resources"
+include_filter=""
+exclude_filter=""
+export_path="../training/build/CosmicClash.x86_64"
+encryption_include_filters=""
+encryption_exclude_filters=""
+encrypt_pck=false
+encrypt_directory=false
+script_encryption_key=""
+
+[preset.0.options]
+
+custom_template/debug=""
+custom_template/release=""
+debug/export_console_script=1
+binary_format/embed_pck=true
+texture_format/bptc=true
+texture_format/s3tc=true
+texture_format/etc=false
+texture_format/etc2=false
+binary_format/architecture="x86_64"

Game/project.godot

@@ -18,6 +18,7 @@ config/name="Cosmic Clash"
 config/description="A fast-paced, physics-based sports game set in space. From Raymond Studios."
 config/version="0.0.1"
 run/main_scene="uid://bcq14356s3e2i"
+run/main_scene.training="res://scenes/training.tscn"
 config/features=PackedStringArray("4.7", "Forward Plus")
 config/icon="res://icon.svg"
 

TRAINING.md

@@ -17,7 +17,10 @@ Python, no .NET, no network.
   observation builder — training and in-game inference must stay identical,
   so never fork it.
 - `training/train.py` — PPO trainer; launches N parallel headless Godot
-  instances (2 agents each).
+  instances (2 agents each) — from source by default, or from a pre-built
+  binary via `--exported-binary` (see TRAINING_LINUX.md's "Exported-binary
+  training" section; `training/export_linux.sh` builds it from
+  `Game/export_presets.cfg`'s "Linux Training" preset).
 - `training/export_policy.py` — SB3 checkpoint → JSON policy for the game.
 - `scripts/ai_ship_controller.gd` + `scripts/policy_network.gd` — in-game
   inference (GDScript MLP forward pass).

TRAINING_LINUX.md

@@ -112,6 +112,56 @@ On the Mac (or anywhere), collecting the results is just `git pull`. A
 20M-step run adds roughly 40 MB of checkpoints — acceptable growth for the
 guarantee that training is never lost with a machine.
 
+## Exported-binary training (faster parallel startup)
+
+By default (and in every example above) `train.py` runs the project from
+source via `--godot_bin` — each of the `--n-parallel` instances re-parses
+project settings and re-imports scripts/resources on launch. An **exported**
+build skips that: resources are pre-imported and packed once at export time,
+so each instance just loads a binary. Worth it once `--n-parallel` is large
+enough that per-instance startup overhead adds up (i.e. this box, not the Mac
+mini's `--n-parallel 6`).
+
+Opt in once — `run_training.sh` (and so `start_training.sh`/`next_run.sh`/
+`curriculum.sh`, which all funnel through it) takes it from there automatically:
+
+```bash
+./export_linux.sh    # one-time opt-in: builds training/build/CosmicClash.x86_64
+./next_run.sh         # from here on, every standing/curriculum run uses it
+```
+
+You don't need to (and shouldn't) pass `--exported-binary` yourself through
+those entry points — `run_training.sh` adds it whenever `training/build/`
+exists, after re-exporting against whatever `git pull` just fetched. Calling
+`train.py` directly still takes it explicitly, same as any other flag:
+
+```bash
+.venv/bin/python train.py --experiment run04 --exported-binary build/CosmicClash.x86_64 \
+    --timesteps 20000000 --n-parallel 14 --speedup 24
+```
+
+To go back to a source run permanently, delete `training/build/` — with it
+gone, `run_training.sh` stops re-exporting and stops adding the flag, so
+`next_run.sh`/`curriculum.sh` revert to plain source runs with no code changes.
+
+`export_linux.sh` builds from the "Linux Training" preset in
+`Game/export_presets.cfg`, which is training-only — its
+`custom_features="training"` activates project.godot's
+`run/main_scene.training` override, so the resulting binary boots straight
+into `training.tscn` on its own. This indirection is required, not
+incidental: official Godot export templates have path/scene overrides
+compiled out, so passing `--scene` at launch time (the way the source run
+does) hard-aborts an exported binary with "compiled without support for path
+overrides" — there's no way to redirect an exported build to a different
+scene at runtime. Because the main scene is baked in at export time, this
+preset can't later double as a normal "ship the game" Linux build (which
+would need `main_menu.tscn` and no training feature tag) — a real game export
+would need its own separate preset.
+
+`setup_linux.sh` installs the export templates this needs alongside the
+Godot binary. If you never opt in (no `training/build/` directory), this
+costs nothing — training stays a plain source run.
+
 ## Dashboard over the network
 
 `start_training.sh` already serves TensorBoard on all interfaces — browse to

training/cosmic_env.py

@@ -19,10 +19,27 @@ TRAINING_SCENE = "res://scenes/training.tscn"
 
 
 class CosmicClashEnv(GodotEnv):
-    """GodotEnv that launches `godot --path Game res://scenes/training.tscn`."""
+    """GodotEnv that launches either the project from source or an exported binary.
 
-    # env_path is a Godot binary, not an exported game: skip the suffix and
+    Source mode (default): `godot --path Game res://scenes/training.tscn` — the
-    # platform checks stock GodotEnv applies to exported executables.
+    positional scene argument overrides the project's normal main scene.
+    Exported mode (`exported=True`): `env_path` is a pre-built game executable
+    (see training/export_linux.sh, "Linux Training" preset) — no `--path` and
+    no scene override needed or possible: official Godot export templates have
+    path/scene overrides compiled out (`--scene`/a positional scene argument
+    hard-aborts with "compiled without support for path overrides"), so the
+    binary instead boots straight into training.tscn on its own via
+    project.godot's `run/main_scene.training` feature-tag override, activated
+    by that preset's `custom_features="training"`.
+    """
+
+    def __init__(self, *args, exported: bool = False, **kwargs):
+        self.exported = exported
+        super().__init__(*args, **kwargs)
+
+    # env_path is a Godot binary (or, in exported mode, a game executable we
+    # built ourselves), not a stock godot_rl exported-project path: skip the
+    # suffix and platform checks stock GodotEnv applies to those.
     def _set_platform_suffix(self, env_path: str) -> str:
         return env_path
 
@@ -32,11 +49,10 @@ class CosmicClashEnv(GodotEnv):
     def _launch_env(self, env_path, port, show_window, framerate, seed, action_repeat, speedup, **kwargs):
         # sync.gd reads --key=value pairs from the raw command line; they must
         # NOT go after a `--` separator or OS.get_cmdline_args() drops them.
-        cmd = [
+        cmd = [env_path]
-            env_path,
+        if not self.exported:
-            "--path",
+            cmd += ["--path", str(GAME_DIR), TRAINING_SCENE]
-            str(GAME_DIR),
+        cmd += [
-            TRAINING_SCENE,
             f"--port={port}",
             f"--env_seed={seed}",
         ]

training/export_linux.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Build the exported Linux training binary from the "Linux Training" preset in
+# Game/export_presets.cfg. That preset's custom_features="training" activates
+# project.godot's `run/main_scene.training` override, so the resulting binary
+# boots straight into training.tscn on its own — official Godot export
+# templates have path/scene overrides compiled out, so this can't be done at
+# launch time via --scene. Re-run any time training-relevant scripts/scenes
+# change — an exported binary is a snapshot, not a live view of the source.
+# See TRAINING_LINUX.md.
+set -euo pipefail
+
+cd "$(dirname "$0")"
+
+GODOT_VERSION="4.7.1"
+GODOT_BIN="${GODOT_BIN:-$HOME/ai-training/godot/Godot_v${GODOT_VERSION}-stable_linux.x86_64}"
+OUT="build/CosmicClash.x86_64"
+
+mkdir -p build
+"$GODOT_BIN" --headless --path ../Game --export-release "Linux Training" "$(pwd)/$OUT"
+chmod +x "$OUT"
+echo "Exported: training/$OUT"

training/run_training.sh

@@ -16,10 +16,28 @@ shift
 # Train on the latest code and checkpoints from any machine
 git pull --rebase
 
+# Rebuild the exported training binary against the code we just pulled, so a
+# --exported-binary run never trains on a stale snapshot. Only when the
+# caller has already opted into the exported-binary flow (training/build/
+# exists from a prior export) — a source-run caller pays no extra latency.
+# Fails the whole run rather than silently falling through to train.py
+# against a stale or partially-rebuilt binary (export_linux.sh sets -e, but
+# that only exits *it*, not this script).
+EXTRA_ARGS=()
+if [ -d build ]; then
+    ./export_linux.sh || { echo "export_linux.sh failed — aborting" >&2; exit 1; }
+    # Standing entry points (next_run.sh, curriculum.py) don't know about the
+    # exported binary and never pass --exported-binary themselves; default it
+    # here so opting in (by ever running export_linux.sh once) actually gets
+    # used, not just kept up to date. An explicit --exported-binary in "$@"
+    # still wins (argparse: later occurrence overrides).
+    EXTRA_ARGS=(--exported-binary build/CosmicClash.x86_64)
+fi
+
 # Let Ctrl-C stop train.py without killing this script, so the export and
 # commit below still run
 trap ':' INT
-.venv/bin/python train.py --experiment "$EXP" "$@"
+.venv/bin/python train.py --experiment "$EXP" "${EXTRA_ARGS[@]}" "$@"
 trap - INT
 
 if [ ! -f "checkpoints/$EXP/final.zip" ]; then

training/setup_linux.sh

@@ -20,6 +20,24 @@ if [ ! -x "$GODOT_BIN" ]; then
 fi
 echo "Godot: $GODOT_BIN"
 
+# Export templates — needed to build the exported training binary
+# (export_linux.sh); the editor binary alone can't produce release exports.
+TEMPLATES_DIR="$HOME/.local/share/godot/export_templates/${GODOT_VERSION}.stable"
+if [ ! -d "$TEMPLATES_DIR" ]; then
+    echo "Downloading Godot $GODOT_VERSION export templates to $TEMPLATES_DIR"
+    TMP_DL="$(mktemp -d)"
+    wget -q "https://github.com/godotengine/godot/releases/download/${GODOT_VERSION}-stable/Godot_v${GODOT_VERSION}-stable_export_templates.tpz" \
+        -O "$TMP_DL/templates.tpz"
+    unzip -o -q "$TMP_DL/templates.tpz" -d "$TMP_DL"
+    mkdir -p "$(dirname "$TEMPLATES_DIR")"
+    # Single mv creates $TEMPLATES_DIR only on full success — if this script is
+    # interrupted any earlier, the guard above correctly sees "not installed"
+    # and retries, instead of finding a half-populated dir and skipping.
+    mv "$TMP_DL/templates" "$TEMPLATES_DIR"
+    rm -rf "$TMP_DL"
+fi
+echo "Export templates: $TEMPLATES_DIR"
+
 # Python env
 [ -d .venv ] || python3 -m venv .venv
 .venv/bin/pip install -q -r requirements.txt

training/train.py

@@ -32,6 +32,13 @@ def parse_args():
         default=os.environ.get("GODOT_BIN", DEFAULT_GODOT_MACOS),
         help="Path to the Godot binary (or set GODOT_BIN)",
     )
+    parser.add_argument(
+        "--exported-binary",
+        default=None,
+        help="Path to a pre-built game executable (see export_linux.sh) instead of running the "
+        "project from source — skips per-instance script/resource import for faster parallel "
+        "startup. Overrides --godot_bin when set.",
+    )
     parser.add_argument("--experiment", default="default", help="Run name for logs/checkpoints")
     parser.add_argument("--timesteps", type=int, default=200_000)
     parser.add_argument("--n-parallel", type=int, default=2, help="Parallel Godot instances (2 agents each)")
@@ -146,8 +153,10 @@ def main():
 
         wandb.init(project="cosmic-clash-rl", name=args.experiment, sync_tensorboard=True)
 
+    exported_binary = args.exported_binary or None
     env = CosmicClashVecEnv(
-        godot_bin=args.godot_bin,
+        godot_bin=exported_binary or args.godot_bin,
+        exported=exported_binary is not None,
         n_parallel=args.n_parallel,
         seed=args.seed,
         port=args.port,