feat(*): Add bot-vs-bot Spectate mode with main-menu entry, entropy-control flags (--ent-coef, --reset-std) for resumed training runs, and a Linux/3090 remote-training guide (TRAINING_LINUX.md)

Game/scenes/main_menu.tscn

@@ -39,5 +39,11 @@ custom_minimum_size = Vector2(330, 60)
 layout_mode = 2
 text = "Match"
 
+[node name="SpectateButton" type="Button" parent="CenterContainer/VBoxContainer"]
+custom_minimum_size = Vector2(330, 60)
+layout_mode = 2
+text = "Spectate (Bot vs Bot)"
+
 [connection signal="pressed" from="CenterContainer/VBoxContainer/FreePlayButton" to="." method="_on_free_play_pressed"]
 [connection signal="pressed" from="CenterContainer/VBoxContainer/MatchButton" to="." method="_on_match_pressed"]
+[connection signal="pressed" from="CenterContainer/VBoxContainer/SpectateButton" to="." method="_on_spectate_pressed"]

Game/scenes/spectate.tscn

@@ -0,0 +1,14 @@
+[gd_scene load_steps=4 format=3]
+
+[ext_resource type="Script" path="res://scripts/spectate_mode.gd" id="1_s"]
+[ext_resource type="PackedScene" path="res://scenes/arena_01.tscn" id="2_s"]
+[ext_resource type="PackedScene" uid="uid://c8kak2l3m4n5" path="res://scenes/HUD.tscn" id="3_s"]
+
+[node name="Spectate" type="Node3D"]
+script = ExtResource("1_s")
+bot_a_model_path = "res://bots/rookie.json"
+bot_b_model_path = "res://bots/rookie.json"
+
+[node name="Arena" parent="." instance=ExtResource("2_s")]
+
+[node name="HUD" parent="." instance=ExtResource("3_s")]

Game/scripts/main_menu.gd

@@ -10,3 +10,7 @@ func _on_free_play_pressed() -> void:
 
 func _on_match_pressed() -> void:
 	get_tree().change_scene_to_file("res://scenes/match.tscn")
+
+
+func _on_spectate_pressed() -> void:
+	get_tree().change_scene_to_file("res://scenes/spectate.tscn")

Game/scripts/spectate_mode.gd

@@ -0,0 +1,54 @@
+extends GameMode
+
+# Spectate: bot vs bot exhibition — both ships are AI-driven so two trained
+# policies can be watched playing each other. Score tracking and kickoff
+# resets like Match, but no timer. R resets the ball, Esc returns to the menu.
+
+signal score_changed(score: Dictionary)
+
+@export_group("Team 0 bot")
+@export_file("*.json") var bot_a_model_path: String = ""
+@export_range(1, 60) var bot_a_reaction_ticks: int = 8
+@export_range(0.0, 1.0) var bot_a_action_noise: float = 0.0
+
+@export_group("Team 1 bot")
+@export_file("*.json") var bot_b_model_path: String = ""
+@export_range(1, 60) var bot_b_reaction_ticks: int = 8
+@export_range(0.0, 1.0) var bot_b_action_noise: float = 0.0
+
+var score := {0: 0, 1: 0}
+
+
+func _start() -> void:
+	spawn_ball()
+	var ship_a := spawn_ship(0, 0, _make_bot(bot_a_model_path, bot_a_reaction_ticks, bot_a_action_noise))
+	spawn_ship(1, 0, _make_bot(bot_b_model_path, bot_b_reaction_ticks, bot_b_action_noise))
+	spawn_camera_rig(ship_a)
+
+
+func _make_bot(model_path: String, reaction_ticks: int, action_noise: float) -> ShipController:
+	if not model_path.is_empty() and FileAccess.file_exists(model_path):
+		var bot := AIShipController.new()
+		bot.model_path = model_path
+		bot.reaction_ticks = reaction_ticks
+		bot.action_noise = action_noise
+		return bot
+	if not model_path.is_empty():
+		push_warning("SpectateMode: bot model not found at %s, spawning inert ship" % model_path)
+	return ShipController.new()  # inert placeholder
+
+
+func _on_goal_scored(conceding_team: int) -> void:
+	var scoring_team := 1 - conceding_team
+	score[scoring_team] += 1
+	score_changed.emit(score.duplicate())
+	print("Goal for team %d! Score: %d - %d" % [scoring_team, score[0], score[1]])
+	reset_ball()
+	reset_ships()
+
+
+func _unhandled_input(event):
+	if event.is_action_pressed("reset_ball"):
+		reset_ball()
+	else:
+		super(event)

Game/scripts/spectate_mode.gd.uid

@@ -0,0 +1 @@
+uid://r86jia8qr60x

TRAINING.md

@@ -38,6 +38,10 @@ The environment is our own headless Godot sim — fully cross-platform:
 There is no hard GPU requirement (unlike Rocket League tooling) — a GPU
 mainly speeds up learning updates on long runs.
 
+For the Linux/3090 remote-training workflow (setup, throughput tuning,
+auto-copying results back to the Mac, dashboard over the network), see
+[TRAINING_LINUX.md](TRAINING_LINUX.md).
+
 ## Setup
 
 Needs Python 3.10+ and a Godot 4.7 binary.

TRAINING_LINUX.md

@@ -0,0 +1,129 @@
+# Training on the Linux / RTX 3090 box
+
+Remote-training workflow: run long training sessions on the Linux machine,
+watch the dashboard from any machine on the network, and ship the trained
+model back to the Mac mini automatically when the run finishes. General
+training concepts and the export/evaluate workflow live in
+[TRAINING.md](TRAINING.md) — this doc is only what differs on the Linux box.
+
+## One-time setup
+
+```bash
+# GitHub auth (one-time): git-over-HTTPS no longer accepts account passwords,
+# so clone over SSH. Generate a key, then add the printed public key at
+# github.com/settings/keys → "New SSH key".
+ssh-keygen -t ed25519        # accept the defaults
+cat ~/.ssh/id_ed25519.pub
+
+cd ~/ai-training
+git clone git@github.com:jcreek/CosmicClash.git
+# (submodules are editor tooling only — training doesn't need them)
+
+# Godot 4.7.1 Linux binary
+mkdir -p ~/ai-training/godot && cd ~/ai-training/godot
+wget https://github.com/godotengine/godot/releases/download/4.7.1-stable/Godot_v4.7.1-stable_linux.x86_64.zip
+unzip Godot_v4.7.1-stable_linux.x86_64.zip
+echo 'export GODOT_BIN=~/ai-training/godot/Godot_v4.7.1-stable_linux.x86_64' >> ~/.bashrc && source ~/.bashrc
+
+# Python env
+cd ~/ai-training/CosmicClash/training
+python3 -m venv .venv
+.venv/bin/pip install -r requirements.txt
+
+# CUDA sanity check — should print True
+.venv/bin/python -c "import torch; print(torch.cuda.is_available())"
+
+# Headless smoke test — the game must boot without rendering
+$GODOT_BIN --headless --path ../Game res://scenes/free_play.tscn --quit-after 300
+```
+
+If the CUDA check prints `False`, reinstall torch from the CUDA index
+(`pip install torch --index-url https://download.pytorch.org/whl/cu121`).
+
+## Maximising throughput
+
+Env stepping is CPU-bound (each `--n-parallel` instance is one headless Godot
+process simulating 2 agents); the 3090 only accelerates the PPO updates. So
+the two levers are instance count and in-engine speedup:
+
+- **`--n-parallel`**: start at `nproc` minus 2 (leave headroom for the
+  trainer process itself). The Mac mini sustains 6; a many-core box should
+  take considerably more. Each instance opens its own TCP port upward from
+  `--port` (default 11008).
+- **`--speedup`**: in-engine physics time-scale. 16 is proven; try 24–32 and
+  keep raising while `time/fps` in the console/TensorBoard still scales up.
+  Back off if fps stops improving (CPU saturated) or physics glitches appear
+  (ball tunnelling, ships escaping the arena — watch for respawn warnings in
+  the Godot output).
+
+Tune by watching `time/fps`: run a 2-minute smoke run per setting and keep
+the best. Reference: 6 instances × speedup 16 ≈ 1,385 steps/s on an M4 Mac
+mini — a 20M-step run in ~4 h. Doubling fps halves that.
+
+## Start a run
+
+Fresh run:
+
+```bash
+cd ~/ai-training/CosmicClash/training
+.venv/bin/python train.py --experiment run03 --timesteps 20000000 \
+    --n-parallel 14 --speedup 24
+```
+
+Resuming a previous policy (continues its timestep counter; `--timesteps` is
+*additional* steps). Lessons from run01/run02 hard-coded into flags:
+
+```bash
+.venv/bin/python train.py --experiment run03 --timesteps 20000000 \
+    --n-parallel 14 --speedup 24 \
+    --resume checkpoints/run02/final.zip --ent-coef 0.001 --reset-std 0.3
+```
+
+- `--ent-coef` — entropy bonus. `0.0001` collapsed the policy std to 0.075 by
+  20M steps (no exploration left); `0.005` blew it up to 3.0 (random play).
+  `0.001` is the current middle. Healthy `train/std` drifts between ~0.2 and
+  ~1.0 — check it 30–45 min in before committing to a long run.
+- `--reset-std` — on resume, restores exploration a collapsed checkpoint lost.
+
+Run inside `tmux`/`screen` so an SSH disconnect doesn't kill training.
+Ctrl-C is safe: `final.zip` is written on the way out.
+
+## Auto-copy the result to the Mac mini when training finishes
+
+One-time: enable **System Settings → General → Sharing → Remote Login** on
+the Mac mini, and `ssh-copy-id jcreek@Joshs-Mac-mini.local` from the Linux
+box so rsync runs unattended.
+
+Chain export + copy onto the training command (`;` not `&&`, so the copy
+still happens after a Ctrl-C — `final.zip` exists either way):
+
+```bash
+EXP=run03
+.venv/bin/python train.py --experiment $EXP --timesteps 20000000 --n-parallel 14 --speedup 24 ; \
+.venv/bin/python export_policy.py checkpoints/$EXP/final.zip ../Game/bots/$EXP.json && \
+rsync -av checkpoints/$EXP/final.zip \
+    jcreek@Joshs-Mac-mini.local:~/Documents/repos/GitHub/CosmicClash/training/checkpoints/$EXP/ && \
+rsync -av ../Game/bots/$EXP.json \
+    jcreek@Joshs-Mac-mini.local:~/Documents/repos/GitHub/CosmicClash/Game/bots/
+```
+
+That lands both the raw checkpoint (for future `--resume` / evaluation on the
+Mac) and the exported JSON policy (immediately playable — point Match or
+Spectate mode at `res://bots/<exp>.json`). Add a third rsync of `logs/` if
+you also want the TensorBoard history archived on the Mac.
+
+## Dashboard over the network
+
+On the Linux box, bind TensorBoard to all interfaces instead of localhost:
+
+```bash
+cd ~/ai-training/CosmicClash/training
+.venv/bin/tensorboard --logdir logs --host 0.0.0.0 --port 6006
+```
+
+Then from the Mac (or anything on the LAN): `http://<linux-box-hostname>:6006`.
+
+- If `ufw` is active on the box: `sudo ufw allow 6006/tcp`.
+- If you'd rather not open a port, tunnel instead:
+  `ssh -L 6006:localhost:6006 <linux-box>` from the Mac, then browse
+  `http://localhost:6006`.

training/train.py

@@ -39,6 +39,13 @@ def parse_args():
     parser.add_argument("--port", type=int, default=11008, help="Base TCP port (one per instance)")
     parser.add_argument("--seed", type=int, default=0)
     parser.add_argument("--resume", default=None, help="Checkpoint .zip to resume from")
+    parser.add_argument("--ent-coef", type=float, default=0.0001, help="Entropy bonus coefficient (applied on resume too)")
+    parser.add_argument(
+        "--reset-std",
+        type=float,
+        default=None,
+        help="On resume, reset the policy action std to this value (recovers exploration after entropy collapse)",
+    )
     parser.add_argument("--checkpoint-every", type=int, default=100_000, help="Timesteps between checkpoints")
     parser.add_argument("--viz", action="store_true", help="Show game windows (debugging; slow)")
     parser.add_argument("--wandb", action="store_true", help="Also log to Weights & Biases")
@@ -67,14 +74,22 @@ def main():
     env = VecMonitor(env)
 
     if args.resume:
-        model = PPO.load(args.resume, env=env, tensorboard_log=str(log_dir))
+        model = PPO.load(args.resume, env=env, tensorboard_log=str(log_dir), ent_coef=args.ent_coef)
-        print(f"Resumed from {args.resume} at {model.num_timesteps} timesteps")
+        print(f"Resumed from {args.resume} at {model.num_timesteps} timesteps (ent_coef={args.ent_coef})")
+        if args.reset_std is not None:
+            import math
+
+            import torch
+
+            with torch.no_grad():
+                model.policy.log_std.fill_(math.log(args.reset_std))
+            print(f"Reset policy action std to {args.reset_std}")
     else:
         model = PPO(
             "MultiInputPolicy",
             env,
             verbose=1,
-            ent_coef=0.0001,
+            ent_coef=args.ent_coef,
             n_steps=256,
             batch_size=256,
             learning_rate=3e-4,