From f4722dbd8d1877a00f3738e46232e0b028fe362a Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Sun, 19 Jul 2026 10:27:21 +0100 Subject: [PATCH] feat(*): Add idempotent tmux-based training scripts for the Linux box --- TRAINING_LINUX.md | 39 +++++++++++++++++----------------- training/start_training.sh | 43 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 19 deletions(-) create mode 100755 training/start_training.sh diff --git a/TRAINING_LINUX.md b/TRAINING_LINUX.md index a5cd080b..31b71900 100644 --- a/TRAINING_LINUX.md +++ b/TRAINING_LINUX.md @@ -62,25 +62,31 @@ mini — a 20M-step run in ~4 h. Doubling fps halves that. ## Run training -`run_training.sh [train.py args...]` wraps the whole cycle: -`git pull` → train → export the policy JSON → commit and push checkpoints, -logs, and the exported bot. Run it inside `tmux`/`screen` so an SSH -disconnect doesn't kill training. Ctrl-C is safe: the trainer writes -`final.zip` on the way out, and the script still exports, commits, and -pushes what it has. - -Fresh run: +One command does everything (needs `tmux`: `sudo apt install tmux`): ```bash cd ~/ai-training/CosmicClash/training -./run_training.sh run03 --timesteps 20000000 --n-parallel 14 --speedup 24 +./start_training.sh run03 --timesteps 20000000 --n-parallel 14 --speedup 24 ``` +`start_training.sh` launches a detached tmux session with two windows — +**training survives SSH disconnects** — and prints the dashboard URL: + +- `train` runs `run_training.sh`: `git pull` → train → export the policy + JSON → commit and push checkpoints, logs, and the exported bot; +- `dashboard` serves TensorBoard on `0.0.0.0:6006` for the whole network + (reused if one is already running). + +Re-running `start_training.sh` while a session exists just attaches you to +it (detach again with `Ctrl-B` then `D`) — it will never start a second +trainer. To stop training, attach and Ctrl-C: the trainer writes `final.zip` +on the way out and the script still exports, commits, and pushes what it has. + Resuming a previous policy (continues its timestep counter; `--timesteps` is *additional* steps). Lessons from run01/run02 hard-coded into flags: ```bash -./run_training.sh run03 --timesteps 20000000 --n-parallel 14 --speedup 24 \ +./start_training.sh run03 --timesteps 20000000 --n-parallel 14 --speedup 24 \ --resume checkpoints/run02/final.zip --ent-coef 0.001 --reset-std 0.3 ``` @@ -108,16 +114,11 @@ guarantee that training is never lost with a machine. ## Dashboard over the network -On the Linux box, bind TensorBoard to all interfaces instead of localhost: - -```bash -cd ~/ai-training/CosmicClash/training -.venv/bin/tensorboard --logdir logs --host 0.0.0.0 --port 6006 -``` - -Then from the Mac (or anything on the LAN): `http://:6006`. +`start_training.sh` already serves TensorBoard on all interfaces — browse to +the URL it prints (`http://10.0.0.20:6006`) from the Mac or anything on +the LAN. - If `ufw` is active on the box: `sudo ufw allow 6006/tcp`. - If you'd rather not open a port, tunnel instead: - `ssh -L 6006:localhost:6006 ` from the Mac, then browse + `ssh -L 6006:localhost:6006 10.0.0.20` from the Mac, then browse `http://localhost:6006`. diff --git a/training/start_training.sh b/training/start_training.sh new file mode 100755 index 00000000..c8d09e2f --- /dev/null +++ b/training/start_training.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# The one command to run on the training box. Starts, fully detached from +# your SSH session (safe to disconnect any time): +# +# window "train": run_training.sh — git pull, train, export, commit+push +# window "dashboard": TensorBoard on 0.0.0.0:6006 (LAN-accessible) +# +# Idempotent: if a training session already exists it attaches to it instead +# of starting a second trainer, and an already-running dashboard is reused. +# +# Usage: ./start_training.sh [train.py args...] +# e.g.: ./start_training.sh run02 --timesteps 20000000 --n-parallel 14 --speedup 16 \ +# --resume checkpoints/run01/final.zip --ent-coef 0.001 --reset-std 0.3 +set -euo pipefail +cd "$(dirname "$0")" + +SESSION="cosmic-train" +TB_PORT=6006 + +command -v tmux >/dev/null 2>&1 || { echo "tmux is required: sudo apt install tmux" >&2; exit 1; } + +if tmux has-session -t "$SESSION" 2>/dev/null; then + echo "Session '$SESSION' already running — attaching (detach with Ctrl-B then D)." + exec tmux attach -t "$SESSION" +fi + +EXP="${1:?usage: start_training.sh [train.py args...]}" + +# Keep the window open after the run so its output and exit status stay +# readable; the results themselves are already safe in git by then. +tmux new-session -d -s "$SESSION" -n train \ + "./run_training.sh $*; echo; echo '=== run_training.sh exited — results are committed; press Enter to close ==='; read" + +# Dashboard, unless something (a previous session) is already serving it +if ! (exec 3<>"/dev/tcp/127.0.0.1/$TB_PORT") 2>/dev/null; then + tmux new-window -d -t "$SESSION" -n dashboard \ + ".venv/bin/tensorboard --logdir logs --host 0.0.0.0 --port $TB_PORT" +fi + +IP=$(hostname -I 2>/dev/null | awk '{print $1}') +echo "Training '$EXP' started in tmux session '$SESSION' — SSH disconnects won't touch it." +echo " watch it: tmux attach -t $SESSION (detach again with Ctrl-B then D)" +echo " dashboard: http://${IP:-}:$TB_PORT"