feat(*): Add idempotent tmux-based training scripts for the Linux box

training/start_training.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# The one command to run on the training box. Starts, fully detached from
+# your SSH session (safe to disconnect any time):
+#
+#   window "train":     run_training.sh — git pull, train, export, commit+push
+#   window "dashboard": TensorBoard on 0.0.0.0:6006 (LAN-accessible)
+#
+# Idempotent: if a training session already exists it attaches to it instead
+# of starting a second trainer, and an already-running dashboard is reused.
+#
+# Usage: ./start_training.sh <experiment> [train.py args...]
+# e.g.:  ./start_training.sh run02 --timesteps 20000000 --n-parallel 14 --speedup 16 \
+#            --resume checkpoints/run01/final.zip --ent-coef 0.001 --reset-std 0.3
+set -euo pipefail
+cd "$(dirname "$0")"
+
+SESSION="cosmic-train"
+TB_PORT=6006
+
+command -v tmux >/dev/null 2>&1 || { echo "tmux is required: sudo apt install tmux" >&2; exit 1; }
+
+if tmux has-session -t "$SESSION" 2>/dev/null; then
+    echo "Session '$SESSION' already running — attaching (detach with Ctrl-B then D)."
+    exec tmux attach -t "$SESSION"
+fi
+
+EXP="${1:?usage: start_training.sh <experiment> [train.py args...]}"
+
+# Keep the window open after the run so its output and exit status stay
+# readable; the results themselves are already safe in git by then.
+tmux new-session -d -s "$SESSION" -n train \
+    "./run_training.sh $*; echo; echo '=== run_training.sh exited — results are committed; press Enter to close ==='; read"
+
+# Dashboard, unless something (a previous session) is already serving it
+if ! (exec 3<>"/dev/tcp/127.0.0.1/$TB_PORT") 2>/dev/null; then
+    tmux new-window -d -t "$SESSION" -n dashboard \
+        ".venv/bin/tensorboard --logdir logs --host 0.0.0.0 --port $TB_PORT"
+fi
+
+IP=$(hostname -I 2>/dev/null | awk '{print $1}')
+echo "Training '$EXP' started in tmux session '$SESSION' — SSH disconnects won't touch it."
+echo "  watch it:   tmux attach -t $SESSION   (detach again with Ctrl-B then D)"
+echo "  dashboard:  http://${IP:-<this-box>}:$TB_PORT"