From f9a8f176caadb00faf6a0c141098e82ffc150b78 Mon Sep 17 00:00:00 2001
From: Vuong Hoang <vh@phasefinal.com>
Date: Sun, 13 Sep 2026 13:51:47 -0700
Subject: [PATCH] fix(mtp-bench): give each arm its own container name instead
 of tuning teardown waits

Container teardown latency here is unpredictable, not merely slow. Measured
2026-09-13 on the same container in the same session: once ~55s, once 0s.
Removing a container holding ~92 GB of GPU memory plus the offloaded PLE
mapping leaves an Exited record owning its name for that whole window, so any
fixed wait or retry budget is a guess that will eventually be wrong -- a 12x5s
retry lost arm k1 by roughly two attempts.

Arms now use $NAME_BASE-$arm, set in boot() after stop_bench has torn down the
previous arm. Names are never reused, so the collision is impossible by
construction and teardown latency stops being load-bearing. cleanup() reaps
every fn-mtp-bench-* container at exit.

stop_bench keeps the GPU-memory wait -- the next container genuinely needs the
device, and unlike the container record nvidia-smi clears promptly and reports
truthfully. It no longer waits on the container listing at all.

Three earlier hypotheses were wrong and are recorded so they are not retried:
the name is not held by a phantom (the container is present and Exited), the
removal does not fail (rm -f succeeds; it is asynchronous), and GPU memory
release does not track name release (memory frees first, by a wide margin).
---
 services/flash-next-mtp-bench/run-campaign.sh | 38 +++++++++++--------
 1 file changed, 22 insertions(+), 16 deletions(-)

diff --git a/services/flash-next-mtp-bench/run-campaign.sh b/services/flash-next-mtp-bench/run-campaign.sh
index 6209893..cbd4dc6 100755
--- a/services/flash-next-mtp-bench/run-campaign.sh
+++ b/services/flash-next-mtp-bench/run-campaign.sh
@@ -70,7 +70,8 @@ MAXTOK=${MAXTOK:-400}
 RPS=${RPS:-4}               # requests per stream, per concbench
 METHOD=${METHOD:-qwen4_exp_mtp}
 MODEL=${MODEL:-qwen3.8-flash-next-uncensored}
-NAME=fn-mtp-bench
+NAME_BASE=fn-mtp-bench
+NAME=""   # set per-arm by boot(); never reused, so the name can never collide
 BOOT_TIMEOUT=${BOOT_TIMEOUT:-2400}
 
 mkdir -p "$OUT"
@@ -125,34 +126,39 @@ import json; print(json.load(open('$OUT/resolved-compose.json'))['services']['vl
 log "image=$IMAGE model=$MODEL_DIR"
 
 stop_bench(){
-  sudo -n docker rm -f "$NAME" >/dev/null 2>&1 || true
-  # `docker rm -f` returns once removal is INITIATED, not once it is done. A
-  # container holding ~92 GB of GPU memory plus the offloaded PLE mapping keeps
-  # the name reserved for several seconds while the device is released, so the
-  # next `docker run --name` loses a race and dies with a name Conflict. That
-  # surfaces as `arm SKIPPED (boot failed)` -- an entire arm silently missing
-  # from a campaign that still reports DONE. Observed 2026-09-13 on arm k1.
-  # NOTE: polling `docker ps -a` for the name is USELESS -- verified 2026-09-13.
-  # The container stops being listed while the daemon still holds the name
-  # reservation, so the poll goes false early and the next `docker run` still
-  # hits a Conflict against an ID that no longer exists. Wait instead on the
-  # resource that actually has to be free: the GPU's memory.
+  # Remove the PREVIOUS arm's container (if any). Deliberately does NOT wait for
+  # the container record to disappear: measured 2026-09-13, tearing down a
+  # container holding ~92 GB of GPU memory plus the offloaded PLE mapping leaves
+  # an Exited record holding its name for anywhere between 0s and ~60s. That
+  # latency is unpredictable, so arms get unique names (see boot) and the record
+  # is simply left to be reaped by cleanup at the end.
+  [ -n "${NAME:-}" ] && sudo -n docker rm -f "$NAME" >/dev/null 2>&1 || true
+  # Do wait on GPU memory: that IS what the next container needs, and unlike the
+  # container record it clears promptly and reports truthfully.
   local t0=$SECONDS
   while [ "$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i "$GPU" 2>/dev/null || echo 0)" -gt 1000 ]; do
-    if [ $((SECONDS-t0)) -ge 120 ]; then
-      log "  WARN: GPU $GPU still holding memory 120s after rm -f; continuing anyway"
+    if [ $((SECONDS-t0)) -ge 180 ]; then
+      log "  WARN: GPU $GPU still holding memory after 180s; continuing anyway"
       break
     fi
     sleep 2
   done
 }
-cleanup(){ stop_bench; [ -n "${POWER_PID:-}" ] && kill "$POWER_PID" 2>/dev/null; }
+cleanup(){
+  stop_bench
+  # reap every per-arm container this campaign created
+  for n in $(sudo -n docker ps -a --format '{{.Names}}' 2>/dev/null | grep "^${NAME_BASE}-" || true); do
+    sudo -n docker rm -f "$n" >/dev/null 2>&1 || true
+  done
+  [ -n "${POWER_PID:-}" ] && kill "$POWER_PID" 2>/dev/null
+}
 trap 'cleanup' EXIT
 
 boot(){   # boot <arm-label> [spec-json]
   local arm="$1"; shift
   local extra=(); [ $# -gt 0 ] && [ -n "${1:-}" ] && extra=(--speculative-config "$1")
   stop_bench
+  NAME="${NAME_BASE}-${arm}"   # unique per arm: collision is impossible by construction
   log "=== boot arm=$arm ${extra[*]:-(no spec-config)}"
   # The daemon can still hold the name after the container is gone from
   # `docker ps -a`, so a Conflict here is transient -- retry rather than