24 lines
1.8 KiB
Bash
24 lines
1.8 KiB
Bash
#!/usr/bin/env bash
|
|
# diariz-nightly-restart — restart Diariz's two workers about 0300 so they drop their cached models (Prime 2026-10-19:
|
|
# GPU 3 is OK for Diariz as an exception to the "no always-resident tenant" rule, with a nightly restart). The workers
|
|
# load Whisper/pyannote/ECAPA lazily on their first job and keep them for later jobs; a restart is the only unload
|
|
# (except the live worker, which unloads itself when live transcription is switched off in the app).
|
|
#
|
|
# IDLE GATE (it gates, it does not just print): GPU 3 is sampled every 2 s for 20 s; any sample above 10% means work is
|
|
# running there (a Diariz job, or a Scriberr job on the same card), so wait 15 min and try again, until 0500. Then skip
|
|
# tonight and say so in the journal.
|
|
set -uo pipefail
|
|
GPU=3; DEADLINE=$(date -d '05:00' +%s); WORKERS=(diariz-worker-1 diariz-live-worker-1)
|
|
busy() { local s; for _ in $(seq 10); do s=$(nvidia-smi -i $GPU --query-gpu=utilization.gpu --format=csv,noheader,nounits | tr -d ' '); [ "${s:-100}" -gt 10 ] && return 0; sleep 2; done; return 1; }
|
|
while busy; do
|
|
if [ "$(date +%s)" -ge "$DEADLINE" ]; then echo "GPU $GPU still busy at $(date +%H%M); skipping tonight's restart"; exit 0; fi
|
|
echo "GPU $GPU busy at $(date +%H%M); retrying in 15 min"; sleep 900
|
|
done
|
|
before=$(nvidia-smi -i $GPU --query-gpu=memory.used --format=csv,noheader,nounits | tr -d ' ')
|
|
present=(); for c in "${WORKERS[@]}"; do docker inspect "$c" >/dev/null 2>&1 && present+=("$c"); done
|
|
[ ${#present[@]} -gt 0 ] || { echo "no Diariz worker containers found; nothing to restart"; exit 0; }
|
|
docker restart "${present[@]}" >/dev/null
|
|
sleep 20
|
|
after=$(nvidia-smi -i $GPU --query-gpu=memory.used --format=csv,noheader,nounits | tr -d ' ')
|
|
echo "restarted ${present[*]}; GPU $GPU memory ${before} MiB -> ${after} MiB"
|