mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-10 16:04:04 +00:00
Josh Creek
37 lines
1.5 KiB
Bash
Executable File
37 lines
1.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Runs the staged curriculum (see TRAINING.md, curriculum.py) detached from
|
|
# your SSH session, the same way start_training.sh does for a single run —
|
|
# safe to disconnect any time. Uses its own tmux session ("cosmic-curriculum")
|
|
# so it doesn't collide with a manual next_run.sh/start_training.sh run.
|
|
#
|
|
# Usage: ./curriculum.sh [curriculum.py args...]
|
|
# e.g.: ./curriculum.sh
|
|
# ./curriculum.sh --seed-checkpoint checkpoints/run11/final.zip
|
|
# ./curriculum.sh --force-retry
|
|
set -euo pipefail
|
|
cd "$(dirname "$0")"
|
|
|
|
SESSION="cosmic-curriculum"
|
|
TB_PORT=6006
|
|
|
|
command -v tmux >/dev/null 2>&1 || { echo "tmux is required: sudo apt install tmux" >&2; exit 1; }
|
|
|
|
if tmux has-session -t "$SESSION" 2>/dev/null; then
|
|
echo "Session '$SESSION' already running — attaching (detach with Ctrl-B then D)."
|
|
exec tmux attach -t "$SESSION"
|
|
fi
|
|
|
|
tmux new-session -d -s "$SESSION" -n curriculum \
|
|
".venv/bin/python curriculum.py $*; echo; echo '=== curriculum.py exited — press Enter to close ==='; read"
|
|
|
|
if ! (exec 3<>"/dev/tcp/127.0.0.1/$TB_PORT") 2>/dev/null; then
|
|
tmux new-window -d -t "$SESSION" -n dashboard \
|
|
".venv/bin/tensorboard --logdir logs --host 0.0.0.0 --port $TB_PORT"
|
|
fi
|
|
|
|
IP=$(hostname -I 2>/dev/null | awk '{print $1}')
|
|
echo "Curriculum started in tmux session '$SESSION' — SSH disconnects won't touch it."
|
|
echo " watch it: tmux attach -t $SESSION (detach again with Ctrl-B then D)"
|
|
echo " dashboard: http://${IP:-<this-box>}:$TB_PORT"
|
|
echo " progress: cat curriculum_state.json"
|