diff --git a/Game/scenes/main_menu.tscn b/Game/scenes/main_menu.tscn index fe13d39e..4d4f663b 100644 --- a/Game/scenes/main_menu.tscn +++ b/Game/scenes/main_menu.tscn @@ -39,5 +39,11 @@ custom_minimum_size = Vector2(330, 60) layout_mode = 2 text = "Match" +[node name="SpectateButton" type="Button" parent="CenterContainer/VBoxContainer"] +custom_minimum_size = Vector2(330, 60) +layout_mode = 2 +text = "Spectate (Bot vs Bot)" + [connection signal="pressed" from="CenterContainer/VBoxContainer/FreePlayButton" to="." method="_on_free_play_pressed"] [connection signal="pressed" from="CenterContainer/VBoxContainer/MatchButton" to="." method="_on_match_pressed"] +[connection signal="pressed" from="CenterContainer/VBoxContainer/SpectateButton" to="." method="_on_spectate_pressed"] diff --git a/Game/scenes/spectate.tscn b/Game/scenes/spectate.tscn new file mode 100644 index 00000000..0b148c1e --- /dev/null +++ b/Game/scenes/spectate.tscn @@ -0,0 +1,14 @@ +[gd_scene load_steps=4 format=3] + +[ext_resource type="Script" path="res://scripts/spectate_mode.gd" id="1_s"] +[ext_resource type="PackedScene" path="res://scenes/arena_01.tscn" id="2_s"] +[ext_resource type="PackedScene" uid="uid://c8kak2l3m4n5" path="res://scenes/HUD.tscn" id="3_s"] + +[node name="Spectate" type="Node3D"] +script = ExtResource("1_s") +bot_a_model_path = "res://bots/rookie.json" +bot_b_model_path = "res://bots/rookie.json" + +[node name="Arena" parent="." instance=ExtResource("2_s")] + +[node name="HUD" parent="." instance=ExtResource("3_s")] diff --git a/Game/scripts/main_menu.gd b/Game/scripts/main_menu.gd index 8f3cfb35..1f9b7cbf 100644 --- a/Game/scripts/main_menu.gd +++ b/Game/scripts/main_menu.gd @@ -10,3 +10,7 @@ func _on_free_play_pressed() -> void: func _on_match_pressed() -> void: get_tree().change_scene_to_file("res://scenes/match.tscn") + + +func _on_spectate_pressed() -> void: + get_tree().change_scene_to_file("res://scenes/spectate.tscn") diff --git a/Game/scripts/spectate_mode.gd b/Game/scripts/spectate_mode.gd new file mode 100644 index 00000000..53f30859 --- /dev/null +++ b/Game/scripts/spectate_mode.gd @@ -0,0 +1,54 @@ +extends GameMode + +# Spectate: bot vs bot exhibition — both ships are AI-driven so two trained +# policies can be watched playing each other. Score tracking and kickoff +# resets like Match, but no timer. R resets the ball, Esc returns to the menu. + +signal score_changed(score: Dictionary) + +@export_group("Team 0 bot") +@export_file("*.json") var bot_a_model_path: String = "" +@export_range(1, 60) var bot_a_reaction_ticks: int = 8 +@export_range(0.0, 1.0) var bot_a_action_noise: float = 0.0 + +@export_group("Team 1 bot") +@export_file("*.json") var bot_b_model_path: String = "" +@export_range(1, 60) var bot_b_reaction_ticks: int = 8 +@export_range(0.0, 1.0) var bot_b_action_noise: float = 0.0 + +var score := {0: 0, 1: 0} + + +func _start() -> void: + spawn_ball() + var ship_a := spawn_ship(0, 0, _make_bot(bot_a_model_path, bot_a_reaction_ticks, bot_a_action_noise)) + spawn_ship(1, 0, _make_bot(bot_b_model_path, bot_b_reaction_ticks, bot_b_action_noise)) + spawn_camera_rig(ship_a) + + +func _make_bot(model_path: String, reaction_ticks: int, action_noise: float) -> ShipController: + if not model_path.is_empty() and FileAccess.file_exists(model_path): + var bot := AIShipController.new() + bot.model_path = model_path + bot.reaction_ticks = reaction_ticks + bot.action_noise = action_noise + return bot + if not model_path.is_empty(): + push_warning("SpectateMode: bot model not found at %s, spawning inert ship" % model_path) + return ShipController.new() # inert placeholder + + +func _on_goal_scored(conceding_team: int) -> void: + var scoring_team := 1 - conceding_team + score[scoring_team] += 1 + score_changed.emit(score.duplicate()) + print("Goal for team %d! Score: %d - %d" % [scoring_team, score[0], score[1]]) + reset_ball() + reset_ships() + + +func _unhandled_input(event): + if event.is_action_pressed("reset_ball"): + reset_ball() + else: + super(event) diff --git a/Game/scripts/spectate_mode.gd.uid b/Game/scripts/spectate_mode.gd.uid new file mode 100644 index 00000000..6e0b87e6 --- /dev/null +++ b/Game/scripts/spectate_mode.gd.uid @@ -0,0 +1 @@ +uid://r86jia8qr60x diff --git a/TRAINING.md b/TRAINING.md index 7d661c4b..0ff10148 100644 --- a/TRAINING.md +++ b/TRAINING.md @@ -38,6 +38,10 @@ The environment is our own headless Godot sim — fully cross-platform: There is no hard GPU requirement (unlike Rocket League tooling) — a GPU mainly speeds up learning updates on long runs. +For the Linux/3090 remote-training workflow (setup, throughput tuning, +auto-copying results back to the Mac, dashboard over the network), see +[TRAINING_LINUX.md](TRAINING_LINUX.md). + ## Setup Needs Python 3.10+ and a Godot 4.7 binary. diff --git a/TRAINING_LINUX.md b/TRAINING_LINUX.md new file mode 100644 index 00000000..bec0b53a --- /dev/null +++ b/TRAINING_LINUX.md @@ -0,0 +1,129 @@ +# Training on the Linux / RTX 3090 box + +Remote-training workflow: run long training sessions on the Linux machine, +watch the dashboard from any machine on the network, and ship the trained +model back to the Mac mini automatically when the run finishes. General +training concepts and the export/evaluate workflow live in +[TRAINING.md](TRAINING.md) — this doc is only what differs on the Linux box. + +## One-time setup + +```bash +# GitHub auth (one-time): git-over-HTTPS no longer accepts account passwords, +# so clone over SSH. Generate a key, then add the printed public key at +# github.com/settings/keys → "New SSH key". +ssh-keygen -t ed25519 # accept the defaults +cat ~/.ssh/id_ed25519.pub + +cd ~/ai-training +git clone git@github.com:jcreek/CosmicClash.git +# (submodules are editor tooling only — training doesn't need them) + +# Godot 4.7.1 Linux binary +mkdir -p ~/ai-training/godot && cd ~/ai-training/godot +wget https://github.com/godotengine/godot/releases/download/4.7.1-stable/Godot_v4.7.1-stable_linux.x86_64.zip +unzip Godot_v4.7.1-stable_linux.x86_64.zip +echo 'export GODOT_BIN=~/ai-training/godot/Godot_v4.7.1-stable_linux.x86_64' >> ~/.bashrc && source ~/.bashrc + +# Python env +cd ~/ai-training/CosmicClash/training +python3 -m venv .venv +.venv/bin/pip install -r requirements.txt + +# CUDA sanity check — should print True +.venv/bin/python -c "import torch; print(torch.cuda.is_available())" + +# Headless smoke test — the game must boot without rendering +$GODOT_BIN --headless --path ../Game res://scenes/free_play.tscn --quit-after 300 +``` + +If the CUDA check prints `False`, reinstall torch from the CUDA index +(`pip install torch --index-url https://download.pytorch.org/whl/cu121`). + +## Maximising throughput + +Env stepping is CPU-bound (each `--n-parallel` instance is one headless Godot +process simulating 2 agents); the 3090 only accelerates the PPO updates. So +the two levers are instance count and in-engine speedup: + +- **`--n-parallel`**: start at `nproc` minus 2 (leave headroom for the + trainer process itself). The Mac mini sustains 6; a many-core box should + take considerably more. Each instance opens its own TCP port upward from + `--port` (default 11008). +- **`--speedup`**: in-engine physics time-scale. 16 is proven; try 24–32 and + keep raising while `time/fps` in the console/TensorBoard still scales up. + Back off if fps stops improving (CPU saturated) or physics glitches appear + (ball tunnelling, ships escaping the arena — watch for respawn warnings in + the Godot output). + +Tune by watching `time/fps`: run a 2-minute smoke run per setting and keep +the best. Reference: 6 instances × speedup 16 ≈ 1,385 steps/s on an M4 Mac +mini — a 20M-step run in ~4 h. Doubling fps halves that. + +## Start a run + +Fresh run: + +```bash +cd ~/ai-training/CosmicClash/training +.venv/bin/python train.py --experiment run03 --timesteps 20000000 \ + --n-parallel 14 --speedup 24 +``` + +Resuming a previous policy (continues its timestep counter; `--timesteps` is +*additional* steps). Lessons from run01/run02 hard-coded into flags: + +```bash +.venv/bin/python train.py --experiment run03 --timesteps 20000000 \ + --n-parallel 14 --speedup 24 \ + --resume checkpoints/run02/final.zip --ent-coef 0.001 --reset-std 0.3 +``` + +- `--ent-coef` — entropy bonus. `0.0001` collapsed the policy std to 0.075 by + 20M steps (no exploration left); `0.005` blew it up to 3.0 (random play). + `0.001` is the current middle. Healthy `train/std` drifts between ~0.2 and + ~1.0 — check it 30–45 min in before committing to a long run. +- `--reset-std` — on resume, restores exploration a collapsed checkpoint lost. + +Run inside `tmux`/`screen` so an SSH disconnect doesn't kill training. +Ctrl-C is safe: `final.zip` is written on the way out. + +## Auto-copy the result to the Mac mini when training finishes + +One-time: enable **System Settings → General → Sharing → Remote Login** on +the Mac mini, and `ssh-copy-id jcreek@Joshs-Mac-mini.local` from the Linux +box so rsync runs unattended. + +Chain export + copy onto the training command (`;` not `&&`, so the copy +still happens after a Ctrl-C — `final.zip` exists either way): + +```bash +EXP=run03 +.venv/bin/python train.py --experiment $EXP --timesteps 20000000 --n-parallel 14 --speedup 24 ; \ +.venv/bin/python export_policy.py checkpoints/$EXP/final.zip ../Game/bots/$EXP.json && \ +rsync -av checkpoints/$EXP/final.zip \ + jcreek@Joshs-Mac-mini.local:~/Documents/repos/GitHub/CosmicClash/training/checkpoints/$EXP/ && \ +rsync -av ../Game/bots/$EXP.json \ + jcreek@Joshs-Mac-mini.local:~/Documents/repos/GitHub/CosmicClash/Game/bots/ +``` + +That lands both the raw checkpoint (for future `--resume` / evaluation on the +Mac) and the exported JSON policy (immediately playable — point Match or +Spectate mode at `res://bots/.json`). Add a third rsync of `logs/` if +you also want the TensorBoard history archived on the Mac. + +## Dashboard over the network + +On the Linux box, bind TensorBoard to all interfaces instead of localhost: + +```bash +cd ~/ai-training/CosmicClash/training +.venv/bin/tensorboard --logdir logs --host 0.0.0.0 --port 6006 +``` + +Then from the Mac (or anything on the LAN): `http://:6006`. + +- If `ufw` is active on the box: `sudo ufw allow 6006/tcp`. +- If you'd rather not open a port, tunnel instead: + `ssh -L 6006:localhost:6006 ` from the Mac, then browse + `http://localhost:6006`. diff --git a/training/train.py b/training/train.py index e84db71c..04f43cbd 100644 --- a/training/train.py +++ b/training/train.py @@ -39,6 +39,13 @@ def parse_args(): parser.add_argument("--port", type=int, default=11008, help="Base TCP port (one per instance)") parser.add_argument("--seed", type=int, default=0) parser.add_argument("--resume", default=None, help="Checkpoint .zip to resume from") + parser.add_argument("--ent-coef", type=float, default=0.0001, help="Entropy bonus coefficient (applied on resume too)") + parser.add_argument( + "--reset-std", + type=float, + default=None, + help="On resume, reset the policy action std to this value (recovers exploration after entropy collapse)", + ) parser.add_argument("--checkpoint-every", type=int, default=100_000, help="Timesteps between checkpoints") parser.add_argument("--viz", action="store_true", help="Show game windows (debugging; slow)") parser.add_argument("--wandb", action="store_true", help="Also log to Weights & Biases") @@ -67,14 +74,22 @@ def main(): env = VecMonitor(env) if args.resume: - model = PPO.load(args.resume, env=env, tensorboard_log=str(log_dir)) - print(f"Resumed from {args.resume} at {model.num_timesteps} timesteps") + model = PPO.load(args.resume, env=env, tensorboard_log=str(log_dir), ent_coef=args.ent_coef) + print(f"Resumed from {args.resume} at {model.num_timesteps} timesteps (ent_coef={args.ent_coef})") + if args.reset_std is not None: + import math + + import torch + + with torch.no_grad(): + model.policy.log_std.fill_(math.log(args.reset_std)) + print(f"Reset policy action std to {args.reset_std}") else: model = PPO( "MultiInputPolicy", env, verbose=1, - ent_coef=0.0001, + ent_coef=args.ent_coef, n_steps=256, batch_size=256, learning_rate=3e-4,