mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-10 16:04:04 +00:00
Josh Creek
44 lines
1.9 KiB
Bash
Executable File
44 lines
1.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# The one command to run on the training box. Starts, fully detached from
|
|
# your SSH session (safe to disconnect any time):
|
|
#
|
|
# window "train": run_training.sh — git pull, train, export, commit+push
|
|
# window "dashboard": TensorBoard on 0.0.0.0:6006 (LAN-accessible)
|
|
#
|
|
# Idempotent: if a training session already exists it attaches to it instead
|
|
# of starting a second trainer, and an already-running dashboard is reused.
|
|
#
|
|
# Usage: ./start_training.sh <experiment> [train.py args...]
|
|
# e.g.: ./start_training.sh run02 --timesteps 20000000 --n-parallel 14 --speedup 16 \
|
|
# --resume checkpoints/run01/final.zip --ent-coef 0.001 --reset-std 0.3
|
|
set -euo pipefail
|
|
cd "$(dirname "$0")"
|
|
|
|
SESSION="cosmic-train"
|
|
TB_PORT=6006
|
|
|
|
command -v tmux >/dev/null 2>&1 || { echo "tmux is required: sudo apt install tmux" >&2; exit 1; }
|
|
|
|
if tmux has-session -t "$SESSION" 2>/dev/null; then
|
|
echo "Session '$SESSION' already running — attaching (detach with Ctrl-B then D)."
|
|
exec tmux attach -t "$SESSION"
|
|
fi
|
|
|
|
EXP="${1:?usage: start_training.sh <experiment> [train.py args...]}"
|
|
|
|
# Keep the window open after the run so its output and exit status stay
|
|
# readable; the results themselves are already safe in git by then.
|
|
tmux new-session -d -s "$SESSION" -n train \
|
|
"./run_training.sh $*; echo; echo '=== run_training.sh exited — results are committed; press Enter to close ==='; read"
|
|
|
|
# Dashboard, unless something (a previous session) is already serving it
|
|
if ! (exec 3<>"/dev/tcp/127.0.0.1/$TB_PORT") 2>/dev/null; then
|
|
tmux new-window -d -t "$SESSION" -n dashboard \
|
|
".venv/bin/tensorboard --logdir logs --host 0.0.0.0 --port $TB_PORT"
|
|
fi
|
|
|
|
IP=$(hostname -I 2>/dev/null | awk '{print $1}')
|
|
echo "Training '$EXP' started in tmux session '$SESSION' — SSH disconnects won't touch it."
|
|
echo " watch it: tmux attach -t $SESSION (detach again with Ctrl-B then D)"
|
|
echo " dashboard: http://${IP:-<this-box>}:$TB_PORT"
|