feat(erp-tune): run 7 launched on pfi-gx10 (opening-split slot + mask union) — config, launcher, builder, runbook; erp-seat: pin vLLM nightly 311b3513 (forced tool_choice 1/9 -> 6/9, round trip clean), README on the gemma4-parser design limit

2026-09-08 23:31:09 -07:00
parent 3c54f59878
commit a34a72fc09
8 changed files with 251 additions and 2 deletions
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+# Launch ERP-seat SFT run 7 on pfi-gx10 (NVIDIA GB10, aarch64, sm_121).
+#
+# Run this ON pfi-gx10 as infra-ops. It detaches the job from the invoking
+# shell and logs to the box, so a reaped SSH session cannot take the run with
+# it -- the failure mode that lost the first probe launch on 2026-09-01.
+#
+# Run 6 = run 5 recipe UNCHANGED on the jenerallee78 ARA-abliterated base (the single variable).
+# 8,505 survivors -> ~543 optimizer steps; held targets must match run 6, slot adds 293 records.
+# Checkpoints every 50 steps.
+set -euo pipefail
+
+ROOT=/home/infra-ops/erp-tune
+HARNESS=$ROOT/eitri-smithy
+VENV=/home/infra-ops/ml/.venv/bin/python
+CONFIG=$ROOT/run-07-gx10.json
+LOG=$ROOT/run-07.log
+
+# --- Preconditions, asserted rather than assumed -----------------------------
+
+# A stuck orphan holding unified memory while PyTorch reports zero allocated
+# already doomed three relaunches on this box and got blamed on the new run
+# each time. Assert the GPU is clear.
+apps=$(nvidia-smi --query-compute-apps=pid --format=csv,noheader | tr -d '[:space:]')
+if [ -n "$apps" ]; then
+  echo "REFUSING: GPU is not clear -- compute apps still resident:" >&2
+  nvidia-smi --query-compute-apps=pid,process_name,used_memory --format=csv >&2
+  exit 1
+fi
+
+# Deliberately NOT `pgrep -f erp_sft_harness`: run this over ssh and the
+# pattern appears in the invoking shell's own argv, so the guard matches
+# itself and refuses every launch. The pidfile is exact and cannot self-match;
+# the GPU assertion above catches an orphan under any name.
+if [ -f "$ROOT/run-07.pid" ] && kill -0 "$(cat "$ROOT/run-07.pid")" 2>/dev/null; then
+  echo "REFUSING: run-07.pid names a live process $(cat "$ROOT/run-07.pid"):" >&2
+  ps -p "$(cat "$ROOT/run-07.pid")" -o pid,etime,cmd >&2
+  exit 1
+fi
+
+if [ -e "$LOG" ]; then
+  echo "REFUSING: $LOG exists. Move it aside first so two runs cannot share a log." >&2
+  exit 1
+fi
+
+for p in "$HARNESS/erp_sft_harness/__main__.py" "$VENV" "$CONFIG"; do
+  [ -e "$p" ] || { echo "REFUSING: missing $p" >&2; exit 1; }
+done
+
+# Free space for checkpoints, with headroom.
+avail=$(df --output=avail -BG "$ROOT" | tail -1 | tr -dc '0-9')
+if [ "$avail" -lt 40 ]; then
+  echo "REFUSING: only ${avail}G free under $ROOT; want >=40G for checkpoints." >&2
+  exit 1
+fi
+
+# --- Launch ------------------------------------------------------------------
+
+cd "$HARNESS"
+{
+  echo "# launched $(date -Is) on $(hostname) by ${USER}"
+  echo "# harness $(git rev-parse --short HEAD) config $CONFIG"
+} > "$LOG"
+
+setsid nohup "$VENV" -m erp_sft_harness --config "$CONFIG" >> "$LOG" 2>&1 < /dev/null &
+pid=$!
+echo "$pid" > "$ROOT/run-07.pid"
+
+echo "launched pid $pid -> $LOG"
+echo
+echo "watch:   tail -f $LOG | tr '\\r' '\\n'"
+echo "stop:    kill \$(cat $ROOT/run-07.pid)      # by PID -- never pkill -f over ssh"