From 67a7dc539e59dee4f7c5ee7723b48220c0193bde Mon Sep 17 00:00:00 2001
From: Vuong Hoang <vh@phasefinal.com>
Date: Sun, 13 Sep 2026 13:29:17 -0700
Subject: [PATCH] fix(mtp-bench): stop_bench raced docker name release,
 silently dropping arms

`docker rm -f` returns once removal is INITIATED, not once complete. The
bench container holds ~92 GB of GPU memory plus the offloaded PLE mapping, so
the name stays reserved for several seconds while the device is released. The
next `docker run --name` then dies with a name Conflict.

That failure was near-invisible: run_arm treats a failed boot as
`arm SKIPPED`, the campaign continues, and it still prints CAMPAIGN DONE.
Observed 2026-09-13 -- arm k1 lost the race after off_A's loaded container,
while k2 won it only because k1 had never started a container to tear down.
Every arm that follows one which actually ran is exposed, so k3 and off_B were
both on track to vanish from a run that would have reported success.

stop_bench now polls until the name is released (120s ceiling, warns and
continues). Added a completeness gate: the campaign asserts every expected
res-<arm>-rep<n>.json exists and reports CAMPAIGN INCOMPLETE naming each
missing result rather than DONE, so a gutted run cannot look like a clean one.

Verified the docker ps format string empirically -- an earlier draft nested
quotes so the template rendered as '{{.Names}}' with literal quotes, which
grep -qx could never match, making the wait a no-op that passed bash -n.
---
 services/flash-next-mtp-bench/run-campaign.sh | 30 +++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/services/flash-next-mtp-bench/run-campaign.sh b/services/flash-next-mtp-bench/run-campaign.sh
index a64433e..4e4e340 100755
--- a/services/flash-next-mtp-bench/run-campaign.sh
+++ b/services/flash-next-mtp-bench/run-campaign.sh
@@ -124,7 +124,23 @@ IMAGE=$(python3 -c "
 import json; print(json.load(open('$OUT/resolved-compose.json'))['services']['vllm-flash-next']['image'])")
 log "image=$IMAGE model=$MODEL_DIR"
 
-stop_bench(){ sudo -n docker rm -f "$NAME" >/dev/null 2>&1 || true; }
+stop_bench(){
+  sudo -n docker rm -f "$NAME" >/dev/null 2>&1 || true
+  # `docker rm -f` returns once removal is INITIATED, not once it is done. A
+  # container holding ~92 GB of GPU memory plus the offloaded PLE mapping keeps
+  # the name reserved for several seconds while the device is released, so the
+  # next `docker run --name` loses a race and dies with a name Conflict. That
+  # surfaces as `arm SKIPPED (boot failed)` -- an entire arm silently missing
+  # from a campaign that still reports DONE. Observed 2026-09-13 on arm k1.
+  local t0=$SECONDS
+  while sudo -n docker ps -a --format "{{.Names}}" | grep -qx "$NAME"; do
+    if [ $((SECONDS-t0)) -ge 120 ]; then
+      log "  WARN: $NAME still present 120s after rm -f; continuing anyway"
+      break
+    fi
+    sleep 2
+  done
+}
 cleanup(){ stop_bench; [ -n "${POWER_PID:-}" ] && kill "$POWER_PID" 2>/dev/null; }
 trap 'cleanup' EXIT
 
@@ -177,7 +193,17 @@ run_arm k2 "{\"method\":\"$METHOD\",\"num_speculative_tokens\":2}"
 run_arm k3 "{\"method\":\"$METHOD\",\"num_speculative_tokens\":3}"
 run_arm off_B ""
 stop_bench
-log "########## CAMPAIGN DONE -- results in $OUT ##########"
+missing=0
+for a in off_A k1 k2 k3 off_B; do
+  for r in $(seq 1 "$REPS"); do
+    [ -s "$OUT/res-$a-rep$r.json" ] || { log "  MISSING RESULT: $a rep$r"; missing=$((missing+1)); }
+  done
+done
+if [ "$missing" -gt 0 ]; then
+  log "########## CAMPAIGN INCOMPLETE -- $missing missing result(s); DO NOT treat as a finished run ##########"
+else
+  log "########## CAMPAIGN DONE -- results in $OUT ##########"
+fi
 # Deliberately does NOT touch the production seat. It is live on GPU 2 / :8022 and
 # serving the operator's `gen-large` traffic throughout; this campaign only ever
 # created and removed `fn-mtp-bench` on GPU 3.