diff --git a/services/flash-next-mtp-bench/run-campaign.sh b/services/flash-next-mtp-bench/run-campaign.sh index 4cf3dc3..7cbdad8 100755 --- a/services/flash-next-mtp-bench/run-campaign.sh +++ b/services/flash-next-mtp-bench/run-campaign.sh @@ -46,15 +46,24 @@ # retyped, which is what stops a stray flag from becoming the real independent # variable. # -# The production container is stopped for the duration: same GPU, same port. The -# seat has no consumers yet, so this costs nothing. It is restored at the end. +# ⚠ RUNS ON GPU 3, PORT 8023 -- operator-directed 2026-09-13 so the campaign runs in +# PARALLEL with live testing on the production seat (GPU 2, :8022). This script must +# therefore NEVER touch the production container: there is no compose down, no compose +# up, and nothing addressed by the production container name. The only container it +# creates or removes is `fn-mtp-bench`. +# +# ⚠ POWER. This puts TWO of the four cards under load at once, which is the condition +# that tripped the Anaheim rack breaker on 2026-08-26 and 2026-09-11, and the Fountain +# Valley circuit was specced while every record still said the box had two GPUs rather +# than four. The operator accepted that risk explicitly. Per-GPU power draw is sampled +# to power.log throughout so there is a record either way. set -uo pipefail STACK_DIR=${STACK_DIR:-/opt/docker/compose/flash-next-seat} OUT=${OUT:-/tank/aimodels/flash-next-mtp-bench} BENCH=${BENCH:-$OUT/concbench.py} -PORT=${PORT:-8022} -GPU=${GPU:-2} +PORT=${PORT:-8023} +GPU=${GPU:-3} REPS=${REPS:-3} CONCS=${CONCS:-"1 4 8"} MAXTOK=${MAXTOK:-400} @@ -67,6 +76,13 @@ BOOT_TIMEOUT=${BOOT_TIMEOUT:-2400} mkdir -p "$OUT" log(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$OUT/campaign.log"; } +# Per-GPU power + memory every 10 s for the life of the campaign. Two cards under +# load is the risk the operator accepted; this is the record of what it actually drew. +( while :; do + echo "$(date -u +%H:%M:%S) $(nvidia-smi --query-gpu=index,power.draw,memory.used,utilization.gpu --format=csv,noheader | tr '\n' '|')" >> "$OUT/power.log" + sleep 10 + done ) & POWER_PID=$! + # --- derive the production argv + run opts straight from the compose file ----- read_compose() { sudo -n docker compose -f "$STACK_DIR/compose.yaml" --env-file "$STACK_DIR/.env" config --format json @@ -109,7 +125,8 @@ import json; print(json.load(open('$OUT/resolved-compose.json'))['services']['vl log "image=$IMAGE model=$MODEL_DIR" stop_bench(){ sudo -n docker rm -f "$NAME" >/dev/null 2>&1 || true; } -trap 'stop_bench' EXIT +cleanup(){ stop_bench; [ -n "${POWER_PID:-}" ] && kill "$POWER_PID" 2>/dev/null; } +trap 'cleanup' EXIT boot(){ # boot [spec-json] local arm="$1"; shift @@ -161,5 +178,8 @@ run_arm k3 "{\"method\":\"$METHOD\",\"num_speculative_tokens\":3}" run_arm off_B "" stop_bench log "########## CAMPAIGN DONE -- results in $OUT ##########" -log "restoring the production container" -sudo -n docker compose -f "$STACK_DIR/compose.yaml" --env-file "$STACK_DIR/.env" up -d 2>&1 | tail -3 | tee -a "$OUT/campaign.log" +# Deliberately does NOT touch the production seat. It is live on GPU 2 / :8022 and +# serving the operator's `gen-large` traffic throughout; this campaign only ever +# created and removed `fn-mtp-bench` on GPU 3. +log "peak power draw seen per card:" +awk -F'|' '{for(i=1;i<=NF;i++) if($i ~ /W/){split($i,a,", "); gsub(/^ +/,"",a[1]); gsub(/ W/,"",a[2]); if(a[2]+0>m[a[1]]) m[a[1]]=a[2]+0}} END{for(g in m) printf " GPU %s peak %.0f W\n", g, m[g]}' "$OUT/power.log" | sort | tee -a "$OUT/campaign.log"