mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-11 08:23:45 +00:00
chore(training): Track training artifacts in git, add idempotent Linux setup/run scripts, and commit run01 results
This commit is contained in:
Executable
+39
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
# Train, export the policy, and commit every artifact to git so no training
|
||||
# is ever stranded on one machine. Idempotent and re-runnable: pulls latest
|
||||
# before training, commits only when there is something new, and a Ctrl-C'd
|
||||
# run still exports and commits (final.zip is written on the way out).
|
||||
#
|
||||
# Usage: ./run_training.sh <experiment> [train.py args...]
|
||||
# e.g.: ./run_training.sh run02 --timesteps 20000000 --n-parallel 14 --speedup 16 \
|
||||
# --resume checkpoints/run01/final.zip --ent-coef 0.001 --reset-std 0.3
|
||||
set -uo pipefail
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
EXP="${1:?usage: run_training.sh <experiment> [train.py args...]}"
|
||||
shift
|
||||
|
||||
# Train on the latest code and checkpoints from any machine
|
||||
git pull --rebase
|
||||
|
||||
# Let Ctrl-C stop train.py without killing this script, so the export and
|
||||
# commit below still run
|
||||
trap ':' INT
|
||||
.venv/bin/python train.py --experiment "$EXP" "$@"
|
||||
trap - INT
|
||||
|
||||
if [ ! -f "checkpoints/$EXP/final.zip" ]; then
|
||||
echo "No checkpoints/$EXP/final.zip — nothing to export or commit" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Export for in-game use (parity-checked); models live in Game/bots/
|
||||
.venv/bin/python export_policy.py "checkpoints/$EXP/final.zip" "../Game/bots/$EXP.json"
|
||||
|
||||
git add -A checkpoints logs eval_history.json "../Game/bots"
|
||||
if git diff --cached --quiet; then
|
||||
echo "Nothing new to commit"
|
||||
else
|
||||
git commit -m "chore(training): Add $EXP checkpoints, logs, and exported policy"
|
||||
git push
|
||||
fi
|
||||
Reference in New Issue
Block a user