diff --git a/services/flash-next-mtp-bench/run-campaign.sh b/services/flash-next-mtp-bench/run-campaign.sh index 4e4e340..6209893 100755 --- a/services/flash-next-mtp-bench/run-campaign.sh +++ b/services/flash-next-mtp-bench/run-campaign.sh @@ -132,10 +132,15 @@ stop_bench(){ # next `docker run --name` loses a race and dies with a name Conflict. That # surfaces as `arm SKIPPED (boot failed)` -- an entire arm silently missing # from a campaign that still reports DONE. Observed 2026-09-13 on arm k1. + # NOTE: polling `docker ps -a` for the name is USELESS -- verified 2026-09-13. + # The container stops being listed while the daemon still holds the name + # reservation, so the poll goes false early and the next `docker run` still + # hits a Conflict against an ID that no longer exists. Wait instead on the + # resource that actually has to be free: the GPU's memory. local t0=$SECONDS - while sudo -n docker ps -a --format "{{.Names}}" | grep -qx "$NAME"; do + while [ "$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i "$GPU" 2>/dev/null || echo 0)" -gt 1000 ]; do if [ $((SECONDS-t0)) -ge 120 ]; then - log " WARN: $NAME still present 120s after rm -f; continuing anyway" + log " WARN: GPU $GPU still holding memory 120s after rm -f; continuing anyway" break fi sleep 2 @@ -149,11 +154,24 @@ boot(){ # boot [spec-json] local extra=(); [ $# -gt 0 ] && [ -n "${1:-}" ] && extra=(--speculative-config "$1") stop_bench log "=== boot arm=$arm ${extra[*]:-(no spec-config)}" - sudo -n docker run -d --name "$NAME" --ipc host --ulimit memlock=-1 \ - --gpus "\"device=$GPU\"" -p "$PORT:8000" \ - -v "$HFCACHE:/hfcache" -v "$MODEL_DIR:/model:ro" \ - "${ENVARGS[@]}" "$IMAGE" "${ARGV[@]}" "${extra[@]}" \ - > "$OUT/$arm.cid" 2>"$OUT/$arm.runerr" || { log " docker run FAILED: $(cat "$OUT/$arm.runerr")"; return 1; } + # The daemon can still hold the name after the container is gone from + # `docker ps -a`, so a Conflict here is transient -- retry rather than + # forfeiting the whole arm (a skipped arm is silent data loss). + local attempt=1 + while :; do + if sudo -n docker run -d --name "$NAME" --ipc host --ulimit memlock=-1 \ + --gpus "\"device=$GPU\"" -p "$PORT:8000" \ + -v "$HFCACHE:/hfcache" -v "$MODEL_DIR:/model:ro" \ + "${ENVARGS[@]}" "$IMAGE" "${ARGV[@]}" "${extra[@]}" \ + > "$OUT/$arm.cid" 2>"$OUT/$arm.runerr"; then + break + fi + if grep -q "already in use" "$OUT/$arm.runerr" && [ "$attempt" -lt 12 ]; then + log " name still held by daemon (attempt $attempt) -- retrying in 5s" + attempt=$((attempt+1)); sleep 5; continue + fi + log " docker run FAILED: $(cat "$OUT/$arm.runerr")"; return 1 + done local t0=$SECONDS while [ $((SECONDS-t0)) -lt "$BOOT_TIMEOUT" ]; do if curl -sf "http://127.0.0.1:$PORT/health" >/dev/null 2>&1; then