diff --git a/services/flash-next-mtp-bench/run-campaign.sh b/services/flash-next-mtp-bench/run-campaign.sh index a64433e..4e4e340 100755 --- a/services/flash-next-mtp-bench/run-campaign.sh +++ b/services/flash-next-mtp-bench/run-campaign.sh @@ -124,7 +124,23 @@ IMAGE=$(python3 -c " import json; print(json.load(open('$OUT/resolved-compose.json'))['services']['vllm-flash-next']['image'])") log "image=$IMAGE model=$MODEL_DIR" -stop_bench(){ sudo -n docker rm -f "$NAME" >/dev/null 2>&1 || true; } +stop_bench(){ + sudo -n docker rm -f "$NAME" >/dev/null 2>&1 || true + # `docker rm -f` returns once removal is INITIATED, not once it is done. A + # container holding ~92 GB of GPU memory plus the offloaded PLE mapping keeps + # the name reserved for several seconds while the device is released, so the + # next `docker run --name` loses a race and dies with a name Conflict. That + # surfaces as `arm SKIPPED (boot failed)` -- an entire arm silently missing + # from a campaign that still reports DONE. Observed 2026-09-13 on arm k1. + local t0=$SECONDS + while sudo -n docker ps -a --format "{{.Names}}" | grep -qx "$NAME"; do + if [ $((SECONDS-t0)) -ge 120 ]; then + log " WARN: $NAME still present 120s after rm -f; continuing anyway" + break + fi + sleep 2 + done +} cleanup(){ stop_bench; [ -n "${POWER_PID:-}" ] && kill "$POWER_PID" 2>/dev/null; } trap 'cleanup' EXIT @@ -177,7 +193,17 @@ run_arm k2 "{\"method\":\"$METHOD\",\"num_speculative_tokens\":2}" run_arm k3 "{\"method\":\"$METHOD\",\"num_speculative_tokens\":3}" run_arm off_B "" stop_bench -log "########## CAMPAIGN DONE -- results in $OUT ##########" +missing=0 +for a in off_A k1 k2 k3 off_B; do + for r in $(seq 1 "$REPS"); do + [ -s "$OUT/res-$a-rep$r.json" ] || { log " MISSING RESULT: $a rep$r"; missing=$((missing+1)); } + done +done +if [ "$missing" -gt 0 ]; then + log "########## CAMPAIGN INCOMPLETE -- $missing missing result(s); DO NOT treat as a finished run ##########" +else + log "########## CAMPAIGN DONE -- results in $OUT ##########" +fi # Deliberately does NOT touch the production seat. It is live on GPU 2 / :8022 and # serving the operator's `gen-large` traffic throughout; this campaign only ever # created and removed `fn-mtp-bench` on GPU 3.