#!/usr/bin/env bash # Train, export the policy, and commit every artifact to git so no training # is ever stranded on one machine. Idempotent and re-runnable: pulls latest # before training, commits only when there is something new, and a Ctrl-C'd # run still exports and commits (final.zip is written on the way out). # # Usage: ./run_training.sh [train.py args...] # e.g.: ./run_training.sh run02 --timesteps 20000000 --n-parallel 14 --speedup 16 \ # --resume checkpoints/run01/final.zip --ent-coef 0.001 --reset-std 0.3 set -uo pipefail cd "$(dirname "$0")" EXP="${1:?usage: run_training.sh [train.py args...]}" shift # Train on the latest code and checkpoints from any machine git pull --rebase # Let Ctrl-C stop train.py without killing this script, so the export and # commit below still run trap ':' INT .venv/bin/python train.py --experiment "$EXP" "$@" trap - INT if [ ! -f "checkpoints/$EXP/final.zip" ]; then echo "No checkpoints/$EXP/final.zip — nothing to export or commit" >&2 exit 1 fi # Export for in-game use (parity-checked); models live in Game/bots/ .venv/bin/python export_policy.py "checkpoints/$EXP/final.zip" "../Game/bots/$EXP.json" git add -A checkpoints logs eval_history.json "../Game/bots" if git diff --cached --quiet; then echo "Nothing new to commit" else git commit -m "chore(training): Add $EXP checkpoints, logs, and exported policy" git push fi