feat(diariz): pilot stack on fv-ml1 GPU 3 — override (GPU pin, api port), vault-rendered .env, nightly idle-gated worker restart, fleet caddy site

2026-10-19 10:50:56 -07:00
parent 2929bfc60c
commit 9aecb677a8
10 changed files with 245 additions and 0 deletions
@@ -25,6 +25,15 @@
 			flush_interval -1
 		}
 	}
+	# Diariz pilot (fv-ml1:8081, plain HTTP; 2026-10-19, Prime). Live capture needs a SecureContext (getUserMedia), and
+	# APP_PUBLIC_URL is this name (it is also Diariz's OAuth issuer). The web container's nginx serves /api and the
+	# SignalR /hubs itself; caddy proxies WebSockets as-is. No buffering: live captions stream.
+	@diariz host diariz.nh3.phasefinal.com
+	handle @diariz {
+		reverse_proxy 10.251.50.54:8081 {
+			flush_interval -1
+		}
+	}
 	# SVOS U13 (2026-10-10, Prime-approved; svoperatingsystem docs/operations/u13-svos-edit.md "Before the edit" 2,
 	# svos docs/u13-cutover-runbook.md B3/B4). Two app origins on nh3-dev's LOOPBACK, reachable only from nh3-svos
 	# (10.100.50.43), whose kernel signs every call it makes to them. Until each app runs (the U13 edit) they answer 502.
@@ -0,0 +1,13 @@
+# fv-ml1: the Diariz workers' nightly restart (~0300, idle-gated on GPU 3). Prime 2026-10-19.
+steps:
+  - {name: script, upload: {src: stacks/diariz/diariz-nightly-restart, dest: /usr/local/sbin/diariz-nightly-restart, mode: "0755"}, sudo: true}
+  - {name: service, upload: {src: stacks/diariz/diariz-nightly-restart.service, dest: /etc/systemd/system/diariz-nightly-restart.service, mode: "0644"}, sudo: true}
+  - {name: timer, upload: {src: stacks/diariz/diariz-nightly-restart.timer, dest: /etc/systemd/system/diariz-nightly-restart.timer, mode: "0644"}, sudo: true}
+  - name: Enable the timer
+    sudo: true
+    shell: systemctl daemon-reload && systemctl enable --now diariz-nightly-restart.timer && systemctl list-timers diariz-nightly-restart.timer --no-pager | head -3
+verify:
+  - name: Timer scheduled for 0300
+    sudo: true
+    shell: systemctl is-enabled diariz-nightly-restart.timer && systemctl show diariz-nightly-restart.timer -p NextElapseUSecRealtime --value
+    changed_when: "false"
@@ -0,0 +1,26 @@
+# fv-ml1: provision the Diariz pilot (kenhayward/Diariz @ 5a96cade, AGPL-3.0) beside Scriberr. Brokkr's request for
+# Prime (2026-10-19, thread 01M5AH9QCVW7ZQ6VVXGWQGFPRS); Prime 1035: GPU 3, with a nightly worker restart ~0300.
+# Source: a clean `git archive` of the pin, extracted to /opt/docker/src/diariz-5a96cad (owned by infra-ops).
+# The fleet override (stacks/diariz/docker-compose.override.yml) goes beside upstream's compose; /opt/docker/compose/diariz
+# is a symlink to that deploy dir. The .env is rendered from the vault separately (stacks/diariz/render-env.sh) and
+# piped to the host; it is never on disk on nh3-dev. This playbook does NOT build or start anything.
+vars:
+  src: /opt/docker/src/diariz-5a96cad
+steps:
+  - {name: stage source archive, upload: {src: /tmp/claude-1000/-home-lkraven-development-eshpfi-management/e0be18fd-0734-4786-b933-904f0a74d2a2/scratchpad/diariz-5a96cad.tar.gz, dest: /tmp/diariz-5a96cad.tar.gz, mode: "0644"}}
+  - name: Extract the pinned source (once)
+    sudo: true
+    shell: |
+      set -e
+      install -d -o infra-ops -g infra-ops -m 0755 {{src}}
+      runuser -u infra-ops -- tar -xzf /tmp/diariz-5a96cad.tar.gz -C {{src}}
+      test -f {{src}}/deploy/docker-compose.yml && rm -f /tmp/diariz-5a96cad.tar.gz
+    creates: /opt/docker/src/diariz-5a96cad/deploy/docker-compose.yml
+  - {name: fleet override, upload: {src: stacks/diariz/docker-compose.override.yml, dest: /opt/docker/src/diariz-5a96cad/deploy/docker-compose.override.yml, mode: "0644"}}
+  - name: /opt/docker/compose/diariz -> the deploy dir
+    sudo: true
+    shell: ln -sfn {{src}}/deploy /opt/docker/compose/diariz && ls -ld /opt/docker/compose/diariz
+verify:
+  - name: Source pinned and the override in place
+    shell: cd {{src}}/deploy && ls docker-compose.yml docker-compose.override.yml && grep -c device_ids docker-compose.override.yml
+    changed_when: "false"
@@ -0,0 +1,28 @@
+# Diariz .env for fv-ml1 (/opt/docker/src/diariz-<sha7>/deploy/.env, mode 0600). The REAL file is rendered from the
+# vault by stacks/diariz/render-env.sh and never committed. Only the fleet-specific values are listed here; every
+# other key keeps upstream's default from deploy/.env.example.
+COMPOSE_PROFILES=live-worker
+POSTGRES_PASSWORD=<vault diariz/postgres-password>
+REDIS_PASSWORD=<vault diariz/redis-password>
+JWT_KEY=<vault diariz/jwt-key>
+CALLBACK_SECRET=<vault diariz/callback-secret>
+S3_ROOT_ACCESS_KEY=<vault diariz/s3-root-access-key>
+S3_ROOT_SECRET_KEY=<vault diariz/s3-root-secret-key>
+S3_APP_ACCESS_KEY=<vault diariz/s3-app-access-key>
+S3_APP_SECRET_KEY=<vault diariz/s3-app-secret-key>
+SEED_EMAIL=lkraven@lkraven.com
+SEED_PASSWORD=<vault diariz/seed-password>
+SEED_FULL_NAME=Vuong Hoang
+HF_TOKEN=<vault fv-ml1/hf-token>            # pyannote/speaker-diarization-3.1 is gated
+APP_PUBLIC_URL=https://diariz.nh3.phasefinal.com   # fleet TLS caddy on nh3-dev -> fv-ml1:8081
+MCP_OAUTH_ENABLED=false                    # no claude.ai connector for the pilot (Prime can turn it on)
+API_BIND=127.0.0.1
+WEB_BIND=0.0.0.0
+SUMMARY_API_BASE=http://10.250.50.70:4000/v1
+SUMMARY_API_KEY=<vault diariz/litellm-key>  # LiteLLM alias "diariz": models gen + whisper-1 only
+SUMMARY_MODEL=gen                          # flash-next on fv-ml1 GPU 2, max-model-len 262144
+CHAT_CONTEXT_LENGTH=245760                 # 240k of the 262,144 window; 16k left for the reply
+DICTATION_API_BASE=http://10.250.50.70:4000/v1
+DICTATION_API_KEY=<vault diariz/litellm-key>
+DICTATION_MODEL=whisper-1                  # the gateway's Parakeet alias; chat dictation only
+# EMBED_* left unset: the vector column is 768-d and the gateway's qwen3-embedding is 1024-d; search falls back to lexical.
@@ -0,0 +1,63 @@
+# Diariz (pilot) — live diarized meeting transcription on fv-ml1 GPU 3
+
+**What:** [kenhayward/Diariz](https://github.com/kenhayward/Diariz) (AGPL-3.0). It does live speaker-labelled captions in the
+browser, audio/video uploads with a full-recording re-pass, and summaries/minutes/actions through the LiteLLM gateway.
+It is a pilot **alongside Scriberr, never instead of it**. Prime uses Scriberr live in meetings, and its idle gate stands.
+Brokkr's request for Prime (2026-10-19, thread `01M5AH9QCVW7ZQ6VVXGWQGFPRS`). Prime chose GPU 3 at 1035 2026-10-19.
+
+**URL:** https://diariz.nh3.phasefinal.com (fleet TLS caddy on nh3-dev → `fv-ml1:8081`). Login `lkraven@lkraven.com`,
+password `secret get diariz/seed-password`.
+
+## Layout on fv-ml1
+
+| | |
+|---|---|
+| source | `/opt/docker/src/diariz-5a96cad`: a clean `git archive` of main `5a96cade77ba` (2026-10-16), owned by infra-ops. Upstream ships no images, so it is built locally |
+| compose | upstream `deploy/docker-compose.yml` + our `deploy/docker-compose.override.yml` (this dir). `/opt/docker/compose/diariz` is a **symlink** to that deploy dir |
+| .env | `deploy/.env` (0600). Rendered from the vault by `render-env.sh` and piped over ssh; never on disk on nh3-dev. Key list: `.env.example` |
+| project | `diariz`: postgres (pgvector pg16), redis 8, SeaweedFS S3, api (.NET), web (nginx), worker, live-worker (profile on) |
+| GPU | worker + live-worker pinned to **GPU 3** by `device_ids` (upstream reserves `count: all`). Scriberr shares the card (about 5.5 GB per job) |
+| ports | web `0.0.0.0:8081`; api `127.0.0.1:18080` (upstream's 8080 is Scriberr's); postgres `127.0.0.1:5433`; S3 `127.0.0.1:9002` |
+| data | named volumes `diariz_pgdata`, `diariz_s3data` (recordings), `diariz_workercache` (model weights), `diariz_apikeys`, `diariz_redisdata`. ⚠ Not in the restic set yet: it's a pilot |
+
+## Fleet settings (the why)
+
+- **Summaries:** `gen` (flash-next, 262,144 window) with `CHAT_CONTEXT_LENGTH=245760`, which leaves 16k for the reply. The key is LiteLLM alias **`diariz`**
+  (models `gen` + `whisper-1` only), vaulted as `diariz/litellm-key`.
+- **Dictation** (chat voice input only): `whisper-1` = the gateway's Parakeet alias. Meeting transcription runs on the worker's own
+  WhisperX large-v3 and does **not** use the Parakeet seat.
+- **Embeddings off.** Diariz's vector column is 768-d and the gateway's `qwen3-embedding` is 1024-d, so `EMBED_*` is unset and search is lexical.
+- **MCP OAuth off** (`MCP_OAUTH_ENABLED=false`). Prime can turn it on.
+- **HF token:** `fv-ml1/hf-token`, because `pyannote/speaker-diarization-3.1` is gated.
+
+## Memory behaviour and the nightly restart
+
+Read at source (`src/Diariz.Worker/pipeline.py`, `worker.py`, `gpu_memory.py`):
+- The models load **lazily on the first job** and stay loaded. Each job's working memory is freed afterwards.
+- The live worker drops its copy only when live transcription is switched off in the app.
+
+So `diariz-nightly-restart.timer` (03:00) restarts both workers to drop the models. It is **idle-gated**: GPU 3 is sampled for
+20 s, and any sample above 10% (a Diariz or a Scriberr job) means wait 15 min, retrying until 0500, then skip.
+Playbook: `playbooks/fv-ml1-diariz-nightly-restart.yaml`. Log: `journalctl -u diariz-nightly-restart`.
+
+## Operate
+
+```bash
+ssh infra-ops@10.251.50.54
+cd /opt/docker/compose/diariz        # = /opt/docker/src/diariz-5a96cad/deploy
+docker compose ps
+docker compose logs -f worker live-worker
+docker compose up -d                 # after a .env change (re-render it from nh3-dev first)
+```
+
+**Upgrade:** a new `git archive` of a new pin into `/opt/docker/src/diariz-<sha7>`. Copy the override and re-render the `.env`,
+`docker compose build`, `down` in the old dir, `up -d` in the new one (same project name, so the volumes carry over), then repoint the symlink.
+Upstream ships very often: pin a commit and read its migrations notes first.
+
+**Rollback / remove:** `docker compose down` in the deploy dir (add `-v` only to throw the data away). Disable the timer, then remove
+the `@diariz` block from `configs/fleet-tls-caddy/Caddyfile`.
+
+## Known limits (not deploy bugs)
+
+No YouTube/URL ingest (files only). Upstream issue #736: speaker-count instability. Desktop capture apps are Windows/macOS only;
+on Linux, capture runs in the browser. Quality is unmeasured, which is the point of the pilot.
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# diariz-nightly-restart — restart Diariz's two workers about 0300 so they drop their cached models (Prime 2026-10-19:
+# GPU 3 is OK for Diariz as an exception to the "no always-resident tenant" rule, with a nightly restart). The workers
+# load Whisper/pyannote/ECAPA lazily on their first job and keep them for later jobs; a restart is the only unload
+# (except the live worker, which unloads itself when live transcription is switched off in the app).
+#
+# IDLE GATE (it gates, it does not just print): GPU 3 is sampled every 2 s for 20 s; any sample above 10% means work is
+# running there (a Diariz job, or a Scriberr job on the same card), so wait 15 min and try again, until 0500. Then skip
+# tonight and say so in the journal.
+set -uo pipefail
+GPU=3; DEADLINE=$(date -d '05:00' +%s); WORKERS=(diariz-worker-1 diariz-live-worker-1)
+busy() { local s; for _ in $(seq 10); do s=$(nvidia-smi -i $GPU --query-gpu=utilization.gpu --format=csv,noheader,nounits | tr -d ' '); [ "${s:-100}" -gt 10 ] && return 0; sleep 2; done; return 1; }
+while busy; do
+  if [ "$(date +%s)" -ge "$DEADLINE" ]; then echo "GPU $GPU still busy at $(date +%H%M); skipping tonight's restart"; exit 0; fi
+  echo "GPU $GPU busy at $(date +%H%M); retrying in 15 min"; sleep 900
+done
+before=$(nvidia-smi -i $GPU --query-gpu=memory.used --format=csv,noheader,nounits | tr -d ' ')
+present=(); for c in "${WORKERS[@]}"; do docker inspect "$c" >/dev/null 2>&1 && present+=("$c"); done
+[ ${#present[@]} -gt 0 ] || { echo "no Diariz worker containers found; nothing to restart"; exit 0; }
+docker restart "${present[@]}" >/dev/null
+sleep 20
+after=$(nvidia-smi -i $GPU --query-gpu=memory.used --format=csv,noheader,nounits | tr -d ' ')
+echo "restarted ${present[*]}; GPU $GPU memory ${before} MiB -> ${after} MiB"
@@ -0,0 +1,9 @@
+[Unit]
+Description=Restart the Diariz workers so they drop cached models (idle-gated on GPU 3)
+After=docker.service
+Requires=docker.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/diariz-nightly-restart
+TimeoutStartSec=3h
@@ -0,0 +1,9 @@
+[Unit]
+Description=Nightly Diariz worker restart, about 0300
+
+[Timer]
+OnCalendar=*-*-* 03:00:00
+Persistent=false
+
+[Install]
+WantedBy=timers.target
@@ -0,0 +1,35 @@
+# Diariz on fv-ml1 — the fleet's override of upstream deploy/docker-compose.yml (kenhayward/Diariz @ 5a96cad).
+# Lives BESIDE the upstream file in /opt/docker/src/diariz-<sha7>/deploy/ (Compose loads docker-compose.override.yml
+# automatically); /opt/docker/compose/diariz is a symlink to that deploy dir. Canonical copy: eshpfi stacks/diariz/.
+services:
+  api:
+    # Upstream pins host port 8080, which Scriberr already holds on fv-ml1 (0.0.0.0:8080). The browser never needs
+    # it: the web container's nginx proxies /api and /hubs to api:8080 on the stack network. Loopback only, for debugging.
+    ports: !override
+      - "127.0.0.1:18080:8080"
+  web:
+    labels:
+      - homepage.group=AI - Studios
+      - homepage.name=Diariz
+      - homepage.icon=mdi-account-voice
+      - homepage.description=Live diarized meeting transcription (pilot; fv-ml1, GPU3)
+      - homepage.href=https://diariz.nh3.phasefinal.com
+  # GPU 3 by explicit device id (upstream reserves `count: all`, which would expose all four cards, and torch would
+  # default to cuda:0 = the vLLM seats' GPU 0). Prime 2026-10-19: GPU 3 is OK for Diariz as an exception to the
+  # "no always-resident tenant on GPU 3" rule; the workers are restarted nightly (~0300) to drop their cached models.
+  worker:
+    deploy:
+      resources:
+        reservations:
+          devices: !override
+            - driver: nvidia
+              device_ids: ["3"]
+              capabilities: [gpu]
+  live-worker:
+    deploy:
+      resources:
+        reservations:
+          devices: !override
+            - driver: nvidia
+              device_ids: ["3"]
+              capabilities: [gpu]
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Render Diariz's .env from the vault to stdout (never echo it; pipe it straight to the host). Run on nh3-dev.
+set -euo pipefail
+v() { secret get "$1"; }
+cat <<ENV
+COMPOSE_PROFILES=live-worker
+POSTGRES_PASSWORD=$(v diariz/postgres-password)
+REDIS_PASSWORD=$(v diariz/redis-password)
+JWT_KEY=$(v diariz/jwt-key)
+CALLBACK_SECRET=$(v diariz/callback-secret)
+S3_ROOT_ACCESS_KEY=$(v diariz/s3-root-access-key)
+S3_ROOT_SECRET_KEY=$(v diariz/s3-root-secret-key)
+S3_APP_ACCESS_KEY=$(v diariz/s3-app-access-key)
+S3_APP_SECRET_KEY=$(v diariz/s3-app-secret-key)
+SEED_EMAIL=lkraven@lkraven.com
+SEED_PASSWORD=$(v diariz/seed-password)
+SEED_FULL_NAME=Vuong Hoang
+HF_TOKEN=$(v fv-ml1/hf-token)
+APP_PUBLIC_URL=https://diariz.nh3.phasefinal.com
+MCP_OAUTH_ENABLED=false
+API_BIND=127.0.0.1
+WEB_BIND=0.0.0.0
+SUMMARY_API_BASE=http://10.250.50.70:4000/v1
+SUMMARY_API_KEY=$(v diariz/litellm-key)
+SUMMARY_MODEL=gen
+CHAT_CONTEXT_LENGTH=245760
+DICTATION_API_BASE=http://10.250.50.70:4000/v1
+DICTATION_API_KEY=$(v diariz/litellm-key)
+DICTATION_MODEL=whisper-1
+ENV