#!/bin/bash
# fv-ml1-gpucheck.sh — the vLLM boot-check arithmetic, run before (re)creating a seat.
#
#   fv-ml1-gpucheck.sh <gpu-index> <util> [container]
#
# vLLM refuses to start unless the card's free memory >= util x total (97,249 MiB on these
# cards). Passes when nvidia-smi Free on <gpu-index>, plus <container>'s own memory ON THAT
# CARD (it is freed when the container is recreated), clears util x 97,249 + 512 MiB.
# A container that is moving here from another card counts nothing, so leave it off.
# Uploaded and run by playbooks/fv-ml1-rebalance-20261005.yaml.
set -u
gpu=$1 util=$2 own_c=${3:-}
free=$(nvidia-smi -i "$gpu" --query-gpu=memory.free --format=csv,noheader,nounits)
own=0
if [ -n "$own_c" ]; then
  uuid=$(nvidia-smi -i "$gpu" --query-gpu=uuid --format=csv,noheader)
  apps=$(nvidia-smi --query-compute-apps=pid,gpu_uuid,used_memory --format=csv,noheader,nounits)
  for p in $(docker top "$own_c" -eo pid 2>/dev/null | tail -n +2); do
    m=$(printf '%s\n' "$apps" | awk -F', ' -v p="$p" -v u="$uuid" '$1==p && $2==u {print $3}')
    own=$((own + ${m:-0}))
  done
fi
need=$(awk -v u="$util" 'BEGIN{printf "%d", u*97249+512}')
echo "GPU$gpu free=$free own=$own need=$need"
[ $((free + own)) -ge "$need" ]