#!/usr/bin/env bash
# diariz-nightly-restart — restart Diariz's two workers about 0300 so they drop their cached models (Prime 2026-10-19:
# GPU 3 is OK for Diariz as an exception to the "no always-resident tenant" rule, with a nightly restart). The workers
# load Whisper/pyannote/ECAPA lazily on their first job and keep them for later jobs; a restart is the only unload
# (except the live worker, which unloads itself when live transcription is switched off in the app).
#
# IDLE GATE (it gates, it does not just print): GPU 3 is sampled every 2 s for 20 s; any sample above 10% means work is
# running there (a Diariz job, or a Scriberr job on the same card), so wait 15 min and try again, until 0500. Then skip
# tonight and say so in the journal.
set -uo pipefail
GPU=3; DEADLINE=$(date -d '05:00' +%s); WORKERS=(diariz-worker-1 diariz-live-worker-1)
busy() { local s; for _ in $(seq 10); do s=$(nvidia-smi -i $GPU --query-gpu=utilization.gpu --format=csv,noheader,nounits | tr -d ' '); [ "${s:-100}" -gt 10 ] && return 0; sleep 2; done; return 1; }
while busy; do
  if [ "$(date +%s)" -ge "$DEADLINE" ]; then echo "GPU $GPU still busy at $(date +%H%M); skipping tonight's restart"; exit 0; fi
  echo "GPU $GPU busy at $(date +%H%M); retrying in 15 min"; sleep 900
done
before=$(nvidia-smi -i $GPU --query-gpu=memory.used --format=csv,noheader,nounits | tr -d ' ')
present=(); for c in "${WORKERS[@]}"; do docker inspect "$c" >/dev/null 2>&1 && present+=("$c"); done
[ ${#present[@]} -gt 0 ] || { echo "no Diariz worker containers found; nothing to restart"; exit 0; }
docker restart "${present[@]}" >/dev/null
sleep 20
after=$(nvidia-smi -i $GPU --query-gpu=memory.used --format=csv,noheader,nounits | tr -d ' ')
echo "restarted ${present[*]}; GPU $GPU memory ${before} MiB -> ${after} MiB"