feat(erp-tune): run 6 on pfi-gx10 — jenerallee78 ARA-abliterated base (index 33c59654) pulled + byte-verified, run-5 recipe byte-held, launched under operator-2026-09-08-rnd-run6
- scripts/erp-tune-gx10/pull-verify-jenerallee78.sh + base-pin-jenerallee78-shards.txt: revision-pinned root-shard pull, 32/32 sha256+size vs brokkr-smithy pins, index set-equal to stock, STOCK tokenizer set installed over the repo's (which bakes in a 256-token truncation); repo originals kept as *.repo - scripts/erp-tune-gx10/run-06-gx10.json + launch-run-06.sh: run-05 config with the base swapped, recipe-r6, survivors-r5 verbatim, stock template path - docs/runbooks/gx10-run-06.md: pull/verify record, free-check result (encode reproduces run 5 exactly), hf download --include gotcha, gate naming (erp-seat-base-ara / erp-tune-v6)
This commit is contained in:
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env bash
|
||||
# Launch ERP-seat SFT run 6 on pfi-gx10 (NVIDIA GB10, aarch64, sm_121).
|
||||
#
|
||||
# Run this ON pfi-gx10 as infra-ops. It detaches the job from the invoking
|
||||
# shell and logs to the box, so a reaped SSH session cannot take the run with
|
||||
# it -- the failure mode that lost the first probe launch on 2026-09-01.
|
||||
#
|
||||
# Run 6 = run 5 recipe UNCHANGED on the jenerallee78 ARA-abliterated base (the single variable).
|
||||
# 8,212 survivors -> 524 optimizer steps, encode must match run 5 exactly.
|
||||
# Checkpoints every 50 steps.
|
||||
set -euo pipefail
|
||||
|
||||
ROOT=/home/infra-ops/erp-tune
|
||||
HARNESS=$ROOT/eitri-smithy
|
||||
VENV=/home/infra-ops/ml/.venv/bin/python
|
||||
CONFIG=$ROOT/run-06-gx10.json
|
||||
LOG=$ROOT/run-06.log
|
||||
|
||||
# --- Preconditions, asserted rather than assumed -----------------------------
|
||||
|
||||
# A stuck orphan holding unified memory while PyTorch reports zero allocated
|
||||
# already doomed three relaunches on this box and got blamed on the new run
|
||||
# each time. Assert the GPU is clear.
|
||||
apps=$(nvidia-smi --query-compute-apps=pid --format=csv,noheader | tr -d '[:space:]')
|
||||
if [ -n "$apps" ]; then
|
||||
echo "REFUSING: GPU is not clear -- compute apps still resident:" >&2
|
||||
nvidia-smi --query-compute-apps=pid,process_name,used_memory --format=csv >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Deliberately NOT `pgrep -f erp_sft_harness`: run this over ssh and the
|
||||
# pattern appears in the invoking shell's own argv, so the guard matches
|
||||
# itself and refuses every launch. The pidfile is exact and cannot self-match;
|
||||
# the GPU assertion above catches an orphan under any name.
|
||||
if [ -f "$ROOT/run-06.pid" ] && kill -0 "$(cat "$ROOT/run-06.pid")" 2>/dev/null; then
|
||||
echo "REFUSING: run-06.pid names a live process $(cat "$ROOT/run-06.pid"):" >&2
|
||||
ps -p "$(cat "$ROOT/run-06.pid")" -o pid,etime,cmd >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -e "$LOG" ]; then
|
||||
echo "REFUSING: $LOG exists. Move it aside first so two runs cannot share a log." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for p in "$HARNESS/erp_sft_harness/__main__.py" "$VENV" "$CONFIG"; do
|
||||
[ -e "$p" ] || { echo "REFUSING: missing $p" >&2; exit 1; }
|
||||
done
|
||||
|
||||
# Free space for checkpoints, with headroom.
|
||||
avail=$(df --output=avail -BG "$ROOT" | tail -1 | tr -dc '0-9')
|
||||
if [ "$avail" -lt 40 ]; then
|
||||
echo "REFUSING: only ${avail}G free under $ROOT; want >=40G for checkpoints." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- Launch ------------------------------------------------------------------
|
||||
|
||||
cd "$HARNESS"
|
||||
{
|
||||
echo "# launched $(date -Is) on $(hostname) by ${USER}"
|
||||
echo "# harness $(git rev-parse --short HEAD) config $CONFIG"
|
||||
} > "$LOG"
|
||||
|
||||
setsid nohup "$VENV" -m erp_sft_harness --config "$CONFIG" >> "$LOG" 2>&1 < /dev/null &
|
||||
pid=$!
|
||||
echo "$pid" > "$ROOT/run-06.pid"
|
||||
|
||||
echo "launched pid $pid -> $LOG"
|
||||
echo
|
||||
echo "watch: tail -f $LOG | tr '\\r' '\\n'"
|
||||
echo "stop: kill \$(cat $ROOT/run-06.pid) # by PID -- never pkill -f over ssh"
|
||||
Reference in New Issue
Block a user