#!/bin/bash
# Switch the decision seat to h2o-lightning-4b (the staged compose.h2o.yaml), run ON fv-ml1:
#   sudo bash /opt/docker/compose/intern-decision/switch-to-h2o.sh
# Only once albok-dev has re-fitted Nemi's judge for h2o (2026-10-12 rollback; see the stack README).
# It stops INC, checks GPU 1 has room for h2o's measured peak, and either starts h2o or puts INC back.
# h2o takes ~3 min to healthy (graph capture). Undo with rollback-to-inc.sh.
set -euo pipefail
cd /opt/docker/compose/intern-decision
NEED_MIB=16900   # h2o seat budget 16,847 MiB (peak 16,164 under 4 x 16q x 32k on 2026-10-12)
docker compose down
FREE=$(nvidia-smi -i 1 --query-gpu=memory.free --format=csv,noheader,nounits | tr -d ' ')
if [ "$FREE" -lt "$NEED_MIB" ]; then
  echo "GPU 1 free ${FREE} MiB < ${NEED_MIB}: not starting h2o; restarting INC"; docker compose up -d; exit 2
fi
cp compose.h2o.yaml compose.yaml
docker compose up -d
for i in $(seq 1 100); do curl -sf http://127.0.0.1:8033/health >/dev/null && { echo "h2o healthy"; exit 0; }; sleep 3; done
echo "h2o not healthy after 300 s: run rollback-to-inc.sh"; exit 1