mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-11 08:23:45 +00:00
Josh Creek
40 lines
1.4 KiB
Bash
Executable File
40 lines
1.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Train, export the policy, and commit every artifact to git so no training
|
|
# is ever stranded on one machine. Idempotent and re-runnable: pulls latest
|
|
# before training, commits only when there is something new, and a Ctrl-C'd
|
|
# run still exports and commits (final.zip is written on the way out).
|
|
#
|
|
# Usage: ./run_training.sh <experiment> [train.py args...]
|
|
# e.g.: ./run_training.sh run02 --timesteps 20000000 --n-parallel 14 --speedup 16 \
|
|
# --resume checkpoints/run01/final.zip --ent-coef 0.001 --reset-std 0.3
|
|
set -uo pipefail
|
|
|
|
cd "$(dirname "$0")"
|
|
EXP="${1:?usage: run_training.sh <experiment> [train.py args...]}"
|
|
shift
|
|
|
|
# Train on the latest code and checkpoints from any machine
|
|
git pull --rebase
|
|
|
|
# Let Ctrl-C stop train.py without killing this script, so the export and
|
|
# commit below still run
|
|
trap ':' INT
|
|
.venv/bin/python train.py --experiment "$EXP" "$@"
|
|
trap - INT
|
|
|
|
if [ ! -f "checkpoints/$EXP/final.zip" ]; then
|
|
echo "No checkpoints/$EXP/final.zip — nothing to export or commit" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Export for in-game use (parity-checked); models live in Game/bots/
|
|
.venv/bin/python export_policy.py "checkpoints/$EXP/final.zip" "../Game/bots/$EXP.json"
|
|
|
|
git add -A checkpoints logs eval_history.json "../Game/bots"
|
|
if git diff --cached --quiet; then
|
|
echo "Nothing new to commit"
|
|
else
|
|
git commit -m "chore(training): Add $EXP checkpoints, logs, and exported policy"
|
|
git push
|
|
fi
|