#!/usr/bin/env bash
# Run the NVFP4A16 quant with `vllm-gen` temporarily stopped to free GPU1,
# and ALWAYS bring gen back - crash, OOM, kill, or success.
#
# Operator authorised downing GPU1 residents overnight (2026-08-25) on the
# condition they are restored. The restore therefore must NOT depend on the
# calling session surviving, so it lives in a trap rather than in the caller.
#
# `docker start` (not `compose up`) is deliberate: it restarts the EXISTING
# container with its exact original config, so there is no chance of compose
# recreating the seat with drifted settings or a different image tag.
#
# gen serves `qwen3.8-27b-uncensored` + `-thinking`, and is the backing seat
# for the fleet-wide `summarizer` / `classifier` aliases. Keep the window short.
set -uo pipefail

GEN=vllm-gen
LOG=/tank/erp-tune/serve/quant.log
OUT=/tank/erp-tune/serve/nvfp4a16-test

restore() {
  echo "[restore] starting $GEN ..." | tee -a "$LOG"
  docker start "$GEN" >/dev/null 2>&1
  for i in $(seq 1 60); do
    st=$(docker inspect -f '{{.State.Health.Status}}' "$GEN" 2>/dev/null || echo unknown)
    run=$(docker inspect -f '{{.State.Running}}' "$GEN" 2>/dev/null || echo false)
    if [ "$st" = "healthy" ]; then echo "[restore] $GEN healthy" | tee -a "$LOG"; return 0; fi
    if [ "$run" != "true" ] && [ "$i" -gt 3 ]; then
      echo "[restore] ⚠ $GEN NOT RUNNING - MANUAL ACTION NEEDED" | tee -a "$LOG"; return 1
    fi
    sleep 10
  done
  echo "[restore] ⚠ $GEN started but not healthy after 600s - CHECK IT" | tee -a "$LOG"
  return 1
}
trap restore EXIT INT TERM

echo "[gen] stopping $GEN to free GPU1" | tee -a "$LOG"
docker stop "$GEN" >/dev/null 2>&1
sleep 8
nvidia-smi --query-gpu=index,memory.used,memory.total --format=csv,noheader | tee -a "$LOG"

rm -rf "$OUT"
cd /tank/erp-tune/serve
CUDA_VISIBLE_DEVICES=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
  /tank/aimodels/quant-work/.venv/bin/python -u quant_nvfp4a16.py \
  --model /tank/erp-tune/serve/merged-test \
  --out "$OUT" \
  --calib-cache /tank/erp-tune/run-01/encode-cache/encoded-a4b0796de1260930.jsonl \
  --num-calib "${NUM_CALIB:-16}" --seqlen "${SEQLEN:-4096}" >> "$LOG" 2>&1
rc=$?
echo "[quant] exit rc=$rc" | tee -a "$LOG"
exit $rc