feat(*): Add exported Linux binary training path for faster parallel instances

This commit is contained in:
Josh Creek
2026-07-28 20:56:14 +01:00
parent 4408ec3ecd
commit 01dbfc7ede
10 changed files with 179 additions and 11 deletions
+19 -1
View File
@@ -16,10 +16,28 @@ shift
# Train on the latest code and checkpoints from any machine
git pull --rebase
# Rebuild the exported training binary against the code we just pulled, so a
# --exported-binary run never trains on a stale snapshot. Only when the
# caller has already opted into the exported-binary flow (training/build/
# exists from a prior export) — a source-run caller pays no extra latency.
# Fails the whole run rather than silently falling through to train.py
# against a stale or partially-rebuilt binary (export_linux.sh sets -e, but
# that only exits *it*, not this script).
EXTRA_ARGS=()
if [ -d build ]; then
./export_linux.sh || { echo "export_linux.sh failed — aborting" >&2; exit 1; }
# Standing entry points (next_run.sh, curriculum.py) don't know about the
# exported binary and never pass --exported-binary themselves; default it
# here so opting in (by ever running export_linux.sh once) actually gets
# used, not just kept up to date. An explicit --exported-binary in "$@"
# still wins (argparse: later occurrence overrides).
EXTRA_ARGS=(--exported-binary build/CosmicClash.x86_64)
fi
# Let Ctrl-C stop train.py without killing this script, so the export and
# commit below still run
trap ':' INT
.venv/bin/python train.py --experiment "$EXP" "$@"
.venv/bin/python train.py --experiment "$EXP" "${EXTRA_ARGS[@]}" "$@"
trap - INT
if [ ! -f "checkpoints/$EXP/final.zip" ]; then