#!/usr/bin/env bash
# The one command to run on the training box. Starts, fully detached from
# your SSH session (safe to disconnect any time):
#
#   window "train":     run_training.sh — git pull, train, export, commit+push
#   window "dashboard": TensorBoard on 0.0.0.0:6006 (LAN-accessible)
#
# Idempotent: if a training session already exists it attaches to it instead
# of starting a second trainer, and an already-running dashboard is reused.
#
# Usage: ./start_training.sh <experiment> [train.py args...]
# e.g.:  ./start_training.sh run02 --timesteps 20000000 --n-parallel 14 --speedup 16 \
#            --resume checkpoints/run01/final.zip --ent-coef 0.001 --reset-std 0.3
set -euo pipefail
cd "$(dirname "$0")"

SESSION="cosmic-train"
TB_PORT=6006

command -v tmux >/dev/null 2>&1 || { echo "tmux is required: sudo apt install tmux" >&2; exit 1; }

if tmux has-session -t "$SESSION" 2>/dev/null; then
    echo "Session '$SESSION' already running — attaching (detach with Ctrl-B then D)."
    exec tmux attach -t "$SESSION"
fi

EXP="${1:?usage: start_training.sh <experiment> [train.py args...]}"

# Keep the window open after the run so its output and exit status stay
# readable; the results themselves are already safe in git by then.
tmux new-session -d -s "$SESSION" -n train \
    "./run_training.sh $*; echo; echo '=== run_training.sh exited — results are committed; press Enter to close ==='; read"

# Dashboard, unless something (a previous session) is already serving it
if ! (exec 3<>"/dev/tcp/127.0.0.1/$TB_PORT") 2>/dev/null; then
    tmux new-window -d -t "$SESSION" -n dashboard \
        ".venv/bin/tensorboard --logdir logs --host 0.0.0.0 --port $TB_PORT"
fi

IP=$(hostname -I 2>/dev/null | awk '{print $1}')
echo "Training '$EXP' started in tmux session '$SESSION' — SSH disconnects won't touch it."
echo "  watch it:   tmux attach -t $SESSION   (detach again with Ctrl-B then D)"
echo "  dashboard:  http://${IP:-<this-box>}:$TB_PORT"