mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-15 18:02:04 +00:00
feat(*): add staged curriculum training with an automated stage-by-stage orchestrator
This commit is contained in:
Executable
+36
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env bash
|
||||
# Runs the staged curriculum (see TRAINING.md, curriculum.py) detached from
|
||||
# your SSH session, the same way start_training.sh does for a single run —
|
||||
# safe to disconnect any time. Uses its own tmux session ("cosmic-curriculum")
|
||||
# so it doesn't collide with a manual next_run.sh/start_training.sh run.
|
||||
#
|
||||
# Usage: ./curriculum.sh [curriculum.py args...]
|
||||
# e.g.: ./curriculum.sh
|
||||
# ./curriculum.sh --seed-checkpoint checkpoints/run11/final.zip
|
||||
# ./curriculum.sh --force-retry
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
SESSION="cosmic-curriculum"
|
||||
TB_PORT=6006
|
||||
|
||||
command -v tmux >/dev/null 2>&1 || { echo "tmux is required: sudo apt install tmux" >&2; exit 1; }
|
||||
|
||||
if tmux has-session -t "$SESSION" 2>/dev/null; then
|
||||
echo "Session '$SESSION' already running — attaching (detach with Ctrl-B then D)."
|
||||
exec tmux attach -t "$SESSION"
|
||||
fi
|
||||
|
||||
tmux new-session -d -s "$SESSION" -n curriculum \
|
||||
".venv/bin/python curriculum.py $*; echo; echo '=== curriculum.py exited — press Enter to close ==='; read"
|
||||
|
||||
if ! (exec 3<>"/dev/tcp/127.0.0.1/$TB_PORT") 2>/dev/null; then
|
||||
tmux new-window -d -t "$SESSION" -n dashboard \
|
||||
".venv/bin/tensorboard --logdir logs --host 0.0.0.0 --port $TB_PORT"
|
||||
fi
|
||||
|
||||
IP=$(hostname -I 2>/dev/null | awk '{print $1}')
|
||||
echo "Curriculum started in tmux session '$SESSION' — SSH disconnects won't touch it."
|
||||
echo " watch it: tmux attach -t $SESSION (detach again with Ctrl-B then D)"
|
||||
echo " dashboard: http://${IP:-<this-box>}:$TB_PORT"
|
||||
echo " progress: cat curriculum_state.json"
|
||||
Reference in New Issue
Block a user