From 9aecb677a89c3e4ece57f1342ab89ebea5828c1d Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Mon, 19 Oct 2026 10:50:56 -0700 Subject: [PATCH] =?UTF-8?q?feat(diariz):=20pilot=20stack=20on=20fv-ml1=20G?= =?UTF-8?q?PU=203=20=E2=80=94=20override=20(GPU=20pin,=20api=20port),=20va?= =?UTF-8?q?ult-rendered=20.env,=20nightly=20idle-gated=20worker=20restart,?= =?UTF-8?q?=20fleet=20caddy=20site?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- configs/fleet-tls-caddy/Caddyfile | 9 +++ playbooks/fv-ml1-diariz-nightly-restart.yaml | 13 ++++ playbooks/fv-ml1-diariz.yaml | 26 ++++++++ stacks/diariz/.env.example | 28 +++++++++ stacks/diariz/README.md | 63 ++++++++++++++++++++ stacks/diariz/diariz-nightly-restart | 23 +++++++ stacks/diariz/diariz-nightly-restart.service | 9 +++ stacks/diariz/diariz-nightly-restart.timer | 9 +++ stacks/diariz/docker-compose.override.yml | 35 +++++++++++ stacks/diariz/render-env.sh | 30 ++++++++++ 10 files changed, 245 insertions(+) create mode 100644 playbooks/fv-ml1-diariz-nightly-restart.yaml create mode 100644 playbooks/fv-ml1-diariz.yaml create mode 100644 stacks/diariz/.env.example create mode 100644 stacks/diariz/README.md create mode 100644 stacks/diariz/diariz-nightly-restart create mode 100644 stacks/diariz/diariz-nightly-restart.service create mode 100644 stacks/diariz/diariz-nightly-restart.timer create mode 100644 stacks/diariz/docker-compose.override.yml create mode 100755 stacks/diariz/render-env.sh diff --git a/configs/fleet-tls-caddy/Caddyfile b/configs/fleet-tls-caddy/Caddyfile index 92468e8..45689b8 100644 --- a/configs/fleet-tls-caddy/Caddyfile +++ b/configs/fleet-tls-caddy/Caddyfile @@ -25,6 +25,15 @@ flush_interval -1 } } + # Diariz pilot (fv-ml1:8081, plain HTTP; 2026-10-19, Prime). Live capture needs a SecureContext (getUserMedia), and + # APP_PUBLIC_URL is this name (it is also Diariz's OAuth issuer). The web container's nginx serves /api and the + # SignalR /hubs itself; caddy proxies WebSockets as-is. No buffering: live captions stream. + @diariz host diariz.nh3.phasefinal.com + handle @diariz { + reverse_proxy 10.251.50.54:8081 { + flush_interval -1 + } + } # SVOS U13 (2026-10-10, Prime-approved; svoperatingsystem docs/operations/u13-svos-edit.md "Before the edit" 2, # svos docs/u13-cutover-runbook.md B3/B4). Two app origins on nh3-dev's LOOPBACK, reachable only from nh3-svos # (10.100.50.43), whose kernel signs every call it makes to them. Until each app runs (the U13 edit) they answer 502. diff --git a/playbooks/fv-ml1-diariz-nightly-restart.yaml b/playbooks/fv-ml1-diariz-nightly-restart.yaml new file mode 100644 index 0000000..2b2c841 --- /dev/null +++ b/playbooks/fv-ml1-diariz-nightly-restart.yaml @@ -0,0 +1,13 @@ +# fv-ml1: the Diariz workers' nightly restart (~0300, idle-gated on GPU 3). Prime 2026-10-19. +steps: + - {name: script, upload: {src: stacks/diariz/diariz-nightly-restart, dest: /usr/local/sbin/diariz-nightly-restart, mode: "0755"}, sudo: true} + - {name: service, upload: {src: stacks/diariz/diariz-nightly-restart.service, dest: /etc/systemd/system/diariz-nightly-restart.service, mode: "0644"}, sudo: true} + - {name: timer, upload: {src: stacks/diariz/diariz-nightly-restart.timer, dest: /etc/systemd/system/diariz-nightly-restart.timer, mode: "0644"}, sudo: true} + - name: Enable the timer + sudo: true + shell: systemctl daemon-reload && systemctl enable --now diariz-nightly-restart.timer && systemctl list-timers diariz-nightly-restart.timer --no-pager | head -3 +verify: + - name: Timer scheduled for 0300 + sudo: true + shell: systemctl is-enabled diariz-nightly-restart.timer && systemctl show diariz-nightly-restart.timer -p NextElapseUSecRealtime --value + changed_when: "false" diff --git a/playbooks/fv-ml1-diariz.yaml b/playbooks/fv-ml1-diariz.yaml new file mode 100644 index 0000000..6bd0762 --- /dev/null +++ b/playbooks/fv-ml1-diariz.yaml @@ -0,0 +1,26 @@ +# fv-ml1: provision the Diariz pilot (kenhayward/Diariz @ 5a96cade, AGPL-3.0) beside Scriberr. Brokkr's request for +# Prime (2026-10-19, thread 01M5AH9QCVW7ZQ6VVXGWQGFPRS); Prime 1035: GPU 3, with a nightly worker restart ~0300. +# Source: a clean `git archive` of the pin, extracted to /opt/docker/src/diariz-5a96cad (owned by infra-ops). +# The fleet override (stacks/diariz/docker-compose.override.yml) goes beside upstream's compose; /opt/docker/compose/diariz +# is a symlink to that deploy dir. The .env is rendered from the vault separately (stacks/diariz/render-env.sh) and +# piped to the host; it is never on disk on nh3-dev. This playbook does NOT build or start anything. +vars: + src: /opt/docker/src/diariz-5a96cad +steps: + - {name: stage source archive, upload: {src: /tmp/claude-1000/-home-lkraven-development-eshpfi-management/e0be18fd-0734-4786-b933-904f0a74d2a2/scratchpad/diariz-5a96cad.tar.gz, dest: /tmp/diariz-5a96cad.tar.gz, mode: "0644"}} + - name: Extract the pinned source (once) + sudo: true + shell: | + set -e + install -d -o infra-ops -g infra-ops -m 0755 {{src}} + runuser -u infra-ops -- tar -xzf /tmp/diariz-5a96cad.tar.gz -C {{src}} + test -f {{src}}/deploy/docker-compose.yml && rm -f /tmp/diariz-5a96cad.tar.gz + creates: /opt/docker/src/diariz-5a96cad/deploy/docker-compose.yml + - {name: fleet override, upload: {src: stacks/diariz/docker-compose.override.yml, dest: /opt/docker/src/diariz-5a96cad/deploy/docker-compose.override.yml, mode: "0644"}} + - name: /opt/docker/compose/diariz -> the deploy dir + sudo: true + shell: ln -sfn {{src}}/deploy /opt/docker/compose/diariz && ls -ld /opt/docker/compose/diariz +verify: + - name: Source pinned and the override in place + shell: cd {{src}}/deploy && ls docker-compose.yml docker-compose.override.yml && grep -c device_ids docker-compose.override.yml + changed_when: "false" diff --git a/stacks/diariz/.env.example b/stacks/diariz/.env.example new file mode 100644 index 0000000..76f2f26 --- /dev/null +++ b/stacks/diariz/.env.example @@ -0,0 +1,28 @@ +# Diariz .env for fv-ml1 (/opt/docker/src/diariz-/deploy/.env, mode 0600). The REAL file is rendered from the +# vault by stacks/diariz/render-env.sh and never committed. Only the fleet-specific values are listed here; every +# other key keeps upstream's default from deploy/.env.example. +COMPOSE_PROFILES=live-worker +POSTGRES_PASSWORD= +REDIS_PASSWORD= +JWT_KEY= +CALLBACK_SECRET= +S3_ROOT_ACCESS_KEY= +S3_ROOT_SECRET_KEY= +S3_APP_ACCESS_KEY= +S3_APP_SECRET_KEY= +SEED_EMAIL=lkraven@lkraven.com +SEED_PASSWORD= +SEED_FULL_NAME=Vuong Hoang +HF_TOKEN= # pyannote/speaker-diarization-3.1 is gated +APP_PUBLIC_URL=https://diariz.nh3.phasefinal.com # fleet TLS caddy on nh3-dev -> fv-ml1:8081 +MCP_OAUTH_ENABLED=false # no claude.ai connector for the pilot (Prime can turn it on) +API_BIND=127.0.0.1 +WEB_BIND=0.0.0.0 +SUMMARY_API_BASE=http://10.250.50.70:4000/v1 +SUMMARY_API_KEY= # LiteLLM alias "diariz": models gen + whisper-1 only +SUMMARY_MODEL=gen # flash-next on fv-ml1 GPU 2, max-model-len 262144 +CHAT_CONTEXT_LENGTH=245760 # 240k of the 262,144 window; 16k left for the reply +DICTATION_API_BASE=http://10.250.50.70:4000/v1 +DICTATION_API_KEY= +DICTATION_MODEL=whisper-1 # the gateway's Parakeet alias; chat dictation only +# EMBED_* left unset: the vector column is 768-d and the gateway's qwen3-embedding is 1024-d; search falls back to lexical. diff --git a/stacks/diariz/README.md b/stacks/diariz/README.md new file mode 100644 index 0000000..a1e50c1 --- /dev/null +++ b/stacks/diariz/README.md @@ -0,0 +1,63 @@ +# Diariz (pilot) — live diarized meeting transcription on fv-ml1 GPU 3 + +**What:** [kenhayward/Diariz](https://github.com/kenhayward/Diariz) (AGPL-3.0). It does live speaker-labelled captions in the +browser, audio/video uploads with a full-recording re-pass, and summaries/minutes/actions through the LiteLLM gateway. +It is a pilot **alongside Scriberr, never instead of it**. Prime uses Scriberr live in meetings, and its idle gate stands. +Brokkr's request for Prime (2026-10-19, thread `01M5AH9QCVW7ZQ6VVXGWQGFPRS`). Prime chose GPU 3 at 1035 2026-10-19. + +**URL:** https://diariz.nh3.phasefinal.com (fleet TLS caddy on nh3-dev → `fv-ml1:8081`). Login `lkraven@lkraven.com`, +password `secret get diariz/seed-password`. + +## Layout on fv-ml1 + +| | | +|---|---| +| source | `/opt/docker/src/diariz-5a96cad`: a clean `git archive` of main `5a96cade77ba` (2026-10-16), owned by infra-ops. Upstream ships no images, so it is built locally | +| compose | upstream `deploy/docker-compose.yml` + our `deploy/docker-compose.override.yml` (this dir). `/opt/docker/compose/diariz` is a **symlink** to that deploy dir | +| .env | `deploy/.env` (0600). Rendered from the vault by `render-env.sh` and piped over ssh; never on disk on nh3-dev. Key list: `.env.example` | +| project | `diariz`: postgres (pgvector pg16), redis 8, SeaweedFS S3, api (.NET), web (nginx), worker, live-worker (profile on) | +| GPU | worker + live-worker pinned to **GPU 3** by `device_ids` (upstream reserves `count: all`). Scriberr shares the card (about 5.5 GB per job) | +| ports | web `0.0.0.0:8081`; api `127.0.0.1:18080` (upstream's 8080 is Scriberr's); postgres `127.0.0.1:5433`; S3 `127.0.0.1:9002` | +| data | named volumes `diariz_pgdata`, `diariz_s3data` (recordings), `diariz_workercache` (model weights), `diariz_apikeys`, `diariz_redisdata`. ⚠ Not in the restic set yet: it's a pilot | + +## Fleet settings (the why) + +- **Summaries:** `gen` (flash-next, 262,144 window) with `CHAT_CONTEXT_LENGTH=245760`, which leaves 16k for the reply. The key is LiteLLM alias **`diariz`** + (models `gen` + `whisper-1` only), vaulted as `diariz/litellm-key`. +- **Dictation** (chat voice input only): `whisper-1` = the gateway's Parakeet alias. Meeting transcription runs on the worker's own + WhisperX large-v3 and does **not** use the Parakeet seat. +- **Embeddings off.** Diariz's vector column is 768-d and the gateway's `qwen3-embedding` is 1024-d, so `EMBED_*` is unset and search is lexical. +- **MCP OAuth off** (`MCP_OAUTH_ENABLED=false`). Prime can turn it on. +- **HF token:** `fv-ml1/hf-token`, because `pyannote/speaker-diarization-3.1` is gated. + +## Memory behaviour and the nightly restart + +Read at source (`src/Diariz.Worker/pipeline.py`, `worker.py`, `gpu_memory.py`): +- The models load **lazily on the first job** and stay loaded. Each job's working memory is freed afterwards. +- The live worker drops its copy only when live transcription is switched off in the app. + +So `diariz-nightly-restart.timer` (03:00) restarts both workers to drop the models. It is **idle-gated**: GPU 3 is sampled for +20 s, and any sample above 10% (a Diariz or a Scriberr job) means wait 15 min, retrying until 0500, then skip. +Playbook: `playbooks/fv-ml1-diariz-nightly-restart.yaml`. Log: `journalctl -u diariz-nightly-restart`. + +## Operate + +```bash +ssh infra-ops@10.251.50.54 +cd /opt/docker/compose/diariz # = /opt/docker/src/diariz-5a96cad/deploy +docker compose ps +docker compose logs -f worker live-worker +docker compose up -d # after a .env change (re-render it from nh3-dev first) +``` + +**Upgrade:** a new `git archive` of a new pin into `/opt/docker/src/diariz-`. Copy the override and re-render the `.env`, +`docker compose build`, `down` in the old dir, `up -d` in the new one (same project name, so the volumes carry over), then repoint the symlink. +Upstream ships very often: pin a commit and read its migrations notes first. + +**Rollback / remove:** `docker compose down` in the deploy dir (add `-v` only to throw the data away). Disable the timer, then remove +the `@diariz` block from `configs/fleet-tls-caddy/Caddyfile`. + +## Known limits (not deploy bugs) + +No YouTube/URL ingest (files only). Upstream issue #736: speaker-count instability. Desktop capture apps are Windows/macOS only; +on Linux, capture runs in the browser. Quality is unmeasured, which is the point of the pilot. diff --git a/stacks/diariz/diariz-nightly-restart b/stacks/diariz/diariz-nightly-restart new file mode 100644 index 0000000..b9b5247 --- /dev/null +++ b/stacks/diariz/diariz-nightly-restart @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# diariz-nightly-restart — restart Diariz's two workers about 0300 so they drop their cached models (Prime 2026-10-19: +# GPU 3 is OK for Diariz as an exception to the "no always-resident tenant" rule, with a nightly restart). The workers +# load Whisper/pyannote/ECAPA lazily on their first job and keep them for later jobs; a restart is the only unload +# (except the live worker, which unloads itself when live transcription is switched off in the app). +# +# IDLE GATE (it gates, it does not just print): GPU 3 is sampled every 2 s for 20 s; any sample above 10% means work is +# running there (a Diariz job, or a Scriberr job on the same card), so wait 15 min and try again, until 0500. Then skip +# tonight and say so in the journal. +set -uo pipefail +GPU=3; DEADLINE=$(date -d '05:00' +%s); WORKERS=(diariz-worker-1 diariz-live-worker-1) +busy() { local s; for _ in $(seq 10); do s=$(nvidia-smi -i $GPU --query-gpu=utilization.gpu --format=csv,noheader,nounits | tr -d ' '); [ "${s:-100}" -gt 10 ] && return 0; sleep 2; done; return 1; } +while busy; do + if [ "$(date +%s)" -ge "$DEADLINE" ]; then echo "GPU $GPU still busy at $(date +%H%M); skipping tonight's restart"; exit 0; fi + echo "GPU $GPU busy at $(date +%H%M); retrying in 15 min"; sleep 900 +done +before=$(nvidia-smi -i $GPU --query-gpu=memory.used --format=csv,noheader,nounits | tr -d ' ') +present=(); for c in "${WORKERS[@]}"; do docker inspect "$c" >/dev/null 2>&1 && present+=("$c"); done +[ ${#present[@]} -gt 0 ] || { echo "no Diariz worker containers found; nothing to restart"; exit 0; } +docker restart "${present[@]}" >/dev/null +sleep 20 +after=$(nvidia-smi -i $GPU --query-gpu=memory.used --format=csv,noheader,nounits | tr -d ' ') +echo "restarted ${present[*]}; GPU $GPU memory ${before} MiB -> ${after} MiB" diff --git a/stacks/diariz/diariz-nightly-restart.service b/stacks/diariz/diariz-nightly-restart.service new file mode 100644 index 0000000..55e5ee6 --- /dev/null +++ b/stacks/diariz/diariz-nightly-restart.service @@ -0,0 +1,9 @@ +[Unit] +Description=Restart the Diariz workers so they drop cached models (idle-gated on GPU 3) +After=docker.service +Requires=docker.service + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/diariz-nightly-restart +TimeoutStartSec=3h diff --git a/stacks/diariz/diariz-nightly-restart.timer b/stacks/diariz/diariz-nightly-restart.timer new file mode 100644 index 0000000..38eff3b --- /dev/null +++ b/stacks/diariz/diariz-nightly-restart.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Nightly Diariz worker restart, about 0300 + +[Timer] +OnCalendar=*-*-* 03:00:00 +Persistent=false + +[Install] +WantedBy=timers.target diff --git a/stacks/diariz/docker-compose.override.yml b/stacks/diariz/docker-compose.override.yml new file mode 100644 index 0000000..7159f7a --- /dev/null +++ b/stacks/diariz/docker-compose.override.yml @@ -0,0 +1,35 @@ +# Diariz on fv-ml1 — the fleet's override of upstream deploy/docker-compose.yml (kenhayward/Diariz @ 5a96cad). +# Lives BESIDE the upstream file in /opt/docker/src/diariz-/deploy/ (Compose loads docker-compose.override.yml +# automatically); /opt/docker/compose/diariz is a symlink to that deploy dir. Canonical copy: eshpfi stacks/diariz/. +services: + api: + # Upstream pins host port 8080, which Scriberr already holds on fv-ml1 (0.0.0.0:8080). The browser never needs + # it: the web container's nginx proxies /api and /hubs to api:8080 on the stack network. Loopback only, for debugging. + ports: !override + - "127.0.0.1:18080:8080" + web: + labels: + - homepage.group=AI - Studios + - homepage.name=Diariz + - homepage.icon=mdi-account-voice + - homepage.description=Live diarized meeting transcription (pilot; fv-ml1, GPU3) + - homepage.href=https://diariz.nh3.phasefinal.com + # GPU 3 by explicit device id (upstream reserves `count: all`, which would expose all four cards, and torch would + # default to cuda:0 = the vLLM seats' GPU 0). Prime 2026-10-19: GPU 3 is OK for Diariz as an exception to the + # "no always-resident tenant on GPU 3" rule; the workers are restarted nightly (~0300) to drop their cached models. + worker: + deploy: + resources: + reservations: + devices: !override + - driver: nvidia + device_ids: ["3"] + capabilities: [gpu] + live-worker: + deploy: + resources: + reservations: + devices: !override + - driver: nvidia + device_ids: ["3"] + capabilities: [gpu] diff --git a/stacks/diariz/render-env.sh b/stacks/diariz/render-env.sh new file mode 100755 index 0000000..e1b6c03 --- /dev/null +++ b/stacks/diariz/render-env.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Render Diariz's .env from the vault to stdout (never echo it; pipe it straight to the host). Run on nh3-dev. +set -euo pipefail +v() { secret get "$1"; } +cat <