From aa5ee7e0a6ff875f95426f9af24cbdd329bc1fd1 Mon Sep 17 00:00:00 2001
From: Vuong Hoang <vh@phasefinal.com>
Date: Sun, 13 Sep 2026 13:40:59 -0700
Subject: [PATCH] fix(mtp-bench): docker ps -a is the wrong probe for name
 release; wait on GPU memory and retry the conflict

The previous fix polled `docker ps -a` until the bench container's name
disappeared. That probe is useless here and the fix was a no-op: measured
2026-09-13, the container stops being listed while the daemon still holds the
name reservation, so the poll returns false early and `docker run` hits a
Conflict naming a container ID that `docker inspect` already reports as
'no such object'. Arm k1 was lost twice this way.

Two changes, each aimed at something actually observed:

- stop_bench waits on GPU memory falling below 1000 MiB rather than on the
  container listing. That is the resource the next container genuinely needs,
  and nvidia-smi reports it truthfully.
- boot() retries `docker run` while stderr matches 'already in use', up to 12
  attempts at 5s. The daemon's name lag is transient, and forfeiting an arm
  over it is silent data loss -- run_arm turns a failed boot into a skipped
  arm that still lets the campaign report success.

Both probes verified against real artifacts before deploy: grep -c
'already in use' on the captured k1.runerr returns 1, and the nvidia-smi
query returns a bare integer that compares correctly under [ -gt ]. The
earlier fix passed bash -n and was still inert, so syntax is not the check
that matters here.
---
 services/flash-next-mtp-bench/run-campaign.sh | 32 +++++++++++++++----
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/services/flash-next-mtp-bench/run-campaign.sh b/services/flash-next-mtp-bench/run-campaign.sh
index 4e4e340..6209893 100755
--- a/services/flash-next-mtp-bench/run-campaign.sh
+++ b/services/flash-next-mtp-bench/run-campaign.sh
@@ -132,10 +132,15 @@ stop_bench(){
   # next `docker run --name` loses a race and dies with a name Conflict. That
   # surfaces as `arm SKIPPED (boot failed)` -- an entire arm silently missing
   # from a campaign that still reports DONE. Observed 2026-09-13 on arm k1.
+  # NOTE: polling `docker ps -a` for the name is USELESS -- verified 2026-09-13.
+  # The container stops being listed while the daemon still holds the name
+  # reservation, so the poll goes false early and the next `docker run` still
+  # hits a Conflict against an ID that no longer exists. Wait instead on the
+  # resource that actually has to be free: the GPU's memory.
   local t0=$SECONDS
-  while sudo -n docker ps -a --format "{{.Names}}" | grep -qx "$NAME"; do
+  while [ "$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i "$GPU" 2>/dev/null || echo 0)" -gt 1000 ]; do
     if [ $((SECONDS-t0)) -ge 120 ]; then
-      log "  WARN: $NAME still present 120s after rm -f; continuing anyway"
+      log "  WARN: GPU $GPU still holding memory 120s after rm -f; continuing anyway"
       break
     fi
     sleep 2
@@ -149,11 +154,24 @@ boot(){   # boot <arm-label> [spec-json]
   local extra=(); [ $# -gt 0 ] && [ -n "${1:-}" ] && extra=(--speculative-config "$1")
   stop_bench
   log "=== boot arm=$arm ${extra[*]:-(no spec-config)}"
-  sudo -n docker run -d --name "$NAME" --ipc host --ulimit memlock=-1 \
-    --gpus "\"device=$GPU\"" -p "$PORT:8000" \
-    -v "$HFCACHE:/hfcache" -v "$MODEL_DIR:/model:ro" \
-    "${ENVARGS[@]}" "$IMAGE" "${ARGV[@]}" "${extra[@]}" \
-    > "$OUT/$arm.cid" 2>"$OUT/$arm.runerr" || { log "  docker run FAILED: $(cat "$OUT/$arm.runerr")"; return 1; }
+  # The daemon can still hold the name after the container is gone from
+  # `docker ps -a`, so a Conflict here is transient -- retry rather than
+  # forfeiting the whole arm (a skipped arm is silent data loss).
+  local attempt=1
+  while :; do
+    if sudo -n docker run -d --name "$NAME" --ipc host --ulimit memlock=-1 \
+      --gpus "\"device=$GPU\"" -p "$PORT:8000" \
+      -v "$HFCACHE:/hfcache" -v "$MODEL_DIR:/model:ro" \
+      "${ENVARGS[@]}" "$IMAGE" "${ARGV[@]}" "${extra[@]}" \
+      > "$OUT/$arm.cid" 2>"$OUT/$arm.runerr"; then
+      break
+    fi
+    if grep -q "already in use" "$OUT/$arm.runerr" && [ "$attempt" -lt 12 ]; then
+      log "  name still held by daemon (attempt $attempt) -- retrying in 5s"
+      attempt=$((attempt+1)); sleep 5; continue
+    fi
+    log "  docker run FAILED: $(cat "$OUT/$arm.runerr")"; return 1
+  done
   local t0=$SECONDS
   while [ $((SECONDS-t0)) -lt "$BOOT_TIMEOUT" ]; do
     if curl -sf "http://127.0.0.1:$PORT/health" >/dev/null 2>&1; then