diff --git a/stacks/chatterbox-fast/.env.example b/stacks/chatterbox-fast/.env.example new file mode 100644 index 0000000..7b628a7 --- /dev/null +++ b/stacks/chatterbox-fast/.env.example @@ -0,0 +1,47 @@ +# chatterbox-fast stack tunables. Copy to `.env` on irv-ml1 before deploying. + +# ── image / build ──────────────────────────────────────────────────── +# Local image tag for this stack. Bump to force a fresh layer build. +CBF_TAG=v1 + +# Base image tag — the sibling `chatterbox` stack's local image, which +# carries the chatterbox lib + torch + fastapi. Must exist on irv-ml1 +# (built by the `chatterbox` stack). Bump in lockstep if that rebuilds. +CBF_BASE_TAG=v1 + +# ── network ────────────────────────────────────────────────────────── +# Host port. Container listens on 8197 internally. +# Reserved on irv-ml1: 8188 ComfyUI, 8190 CosyVoice, 8191 Qwen3-TTS, +# 8192 IndexTTS-2, 8193 Kokoro, 8194 VibeVoice, 8196 Chatterbox, +# 8765 Parakeet. 8197 picked here. +CBF_PORT=8197 + +# Bind address. 0.0.0.0 exposes on all interfaces (incl. the WG tunnel +# interface 10.100.79.3); 127.0.0.1 restricts to local-only. +CBF_BIND=0.0.0.0 + +# ── runtime / GPU ──────────────────────────────────────────────────── +# Device visible inside the container. +# 1 = RTX A6000 (~12 GB free; the safe default). +# 0 = RTX 3090 — TIGHT: turbo loads FP32 (not fp16), and the 3090 idles +# ~20.5 GB used (shared dev stack), leaving ~3.8 GB free. Measure the +# actual footprint before pinning here; it likely will NOT fit in fp32. +CBF_GPU_DEVICES=1 + +# Default reference voice (a *.wav stem in CBF_REFERENCE_DIR, or an absolute +# path). The server prepares this at startup so first request is warm. +CBF_DEFAULT_VOICE=glados_25s + +# Perf levers (Ampere-safe, free). Off with 0. Measured: they don't move TTFA +# (AR-decode-bound) but don't hurt; bf16 is deferred (fp32 model, no clean cast). +CBF_TF32=1 +CBF_SDPA_FLASH=1 + +# ── persistent storage on the host ─────────────────────────────────── +# Reference / predefined voice wavs (mounted at /refs). Shared with the +# `chatterbox` stack. `_`-prefixed files (bench/A-B scratch) are ignored. +CBF_REFERENCE_DIR=/worktank/chatterbox/reference_audio + +# HuggingFace cache — Chatterbox-Turbo weights. Reused from the `chatterbox` +# stack (already populated, ~3.8 GB); no re-download. Excluded from restic. +CBF_CACHE_DIR=/worktank/chatterbox/cache diff --git a/stacks/chatterbox-fast/Dockerfile b/stacks/chatterbox-fast/Dockerfile new file mode 100644 index 0000000..34b7804 --- /dev/null +++ b/stacks/chatterbox-fast/Dockerfile @@ -0,0 +1,19 @@ +# chatterbox-fast — streaming TTS server. +# +# Layers our streaming server (app.py + scheduler.py) on top of the proven +# chatterbox base image (devnen's, already built locally by the sibling +# `chatterbox` stack). That base carries the chatterbox lib + torch + +# fastapi/uvicorn/pydantic — verified present — so this stays a thin overlay. +# +# Build on irv-ml1, where local/chatterbox: exists. Bump the base via the +# CHATTERBOX_BASE build arg (compose passes it from .env: CBF_BASE_TAG). +ARG CHATTERBOX_BASE=local/chatterbox:v1 +FROM ${CHATTERBOX_BASE} + +# Only the two runtime modules — tests/bench/README stay out of the image. +COPY scheduler.py app.py /cbf/ +WORKDIR /cbf + +# The base image sets devnen's own entrypoint; clear it and run our server. +ENTRYPOINT [] +CMD ["python", "app.py"] diff --git a/stacks/chatterbox-fast/README.md b/stacks/chatterbox-fast/README.md index 7c13fb4..d441b41 100644 --- a/stacks/chatterbox-fast/README.md +++ b/stacks/chatterbox-fast/README.md @@ -39,8 +39,21 @@ regardless of chunking. That is why this is the chatterbox-specific answer. | `test_scheduler.py` | GPU-free simulation: asserts no-starvation + ratchet. `python test_scheduler.py` or `pytest`. | | `app.py` | FastAPI server: model holder + `POST /tts` (StreamingResponse) + `GET /health`. | | `bench.py` | Client: ground-truth TTFB + real 1×-consumer starvation check; saves `.wav` for A/B. | +| `Dockerfile` | Thin overlay: `FROM local/chatterbox:v1` + our two modules. | +| `compose.yaml` · `.env.example` | Deploy on irv-ml1 alongside the live `chatterbox`. | -Phase 3 will add `compose.yaml`, `Dockerfile`, `.env.example`. +## Deploy (Phase 3) + +```bash +scripts/deploy-stack.sh irv-ml1 chatterbox-fast # push compose+code to the host +# then on irv-ml1, in /opt/docker/compose/chatterbox-fast/ (after copying .env): +docker compose build && docker compose up -d +``` + +`Dockerfile` is `FROM local/chatterbox:v1` (the sibling stack's image — must exist +on irv-ml1) + `COPY scheduler.py app.py`. GPU pin and voices/cache paths come from +`.env` (see `.env.example`). Default GPU is **device 1 (A6000)** — turbo loads fp32, +so the 3090's tight free VRAM likely won't fit; measure before pinning device 0. ## API diff --git a/stacks/chatterbox-fast/compose.yaml b/stacks/chatterbox-fast/compose.yaml new file mode 100644 index 0000000..882d547 --- /dev/null +++ b/stacks/chatterbox-fast/compose.yaml @@ -0,0 +1,55 @@ +# chatterbox-fast — custom streaming TTS server (sub-second time-to-first-audio +# via adaptive buffer-ratchet chunking on Chatterbox-Turbo). Deployed ALONGSIDE +# the live `chatterbox` (:8196) on irv-ml1 — burn in, then flip the catalog. +# +# Convention note: mirrors the sibling `chatterbox` stack on this host — GPU via +# `runtime: nvidia` + NVIDIA_VISIBLE_DEVICES (not the repo's generic device_ids), +# and accessed by host IP:port (no traefik-net; these GPU TTS services aren't +# traefik-fronted). Kept consistent with the proven sibling over the generic rule. +# +# All tunables live in .env — edit that, not this file. + +services: + chatterbox-fast: + image: local/chatterbox-fast:${CBF_TAG:-v1} + build: + context: . + dockerfile: Dockerfile + args: + CHATTERBOX_BASE: local/chatterbox:${CBF_BASE_TAG:-v1} + container_name: chatterbox-fast + restart: unless-stopped + runtime: nvidia + ports: + - "${CBF_BIND:-0.0.0.0}:${CBF_PORT:-8197}:8197" + environment: + # GPU pin. Default device 1 (A6000) — turbo loads fp32 (~not the fp16 old + # notes assumed), so the 3090's tight free VRAM may not fit; see .env.example. + - NVIDIA_VISIBLE_DEVICES=${CBF_GPU_DEVICES:-1} + - HF_HOME=/app/hf_cache + - CBF_MODEL_DEVICE=cuda + - CBF_VOICES_DIR=/refs + - CBF_DEFAULT_VOICE=${CBF_DEFAULT_VOICE:-glados_25s} + - CBF_BIND=0.0.0.0 + - CBF_PORT=8197 + - CBF_TF32=${CBF_TF32:-1} + - CBF_SDPA_FLASH=${CBF_SDPA_FLASH:-1} + volumes: + - ${CBF_REFERENCE_DIR:-/worktank/chatterbox/reference_audio}:/refs + - ${CBF_CACHE_DIR:-/worktank/chatterbox/cache}:/app/hf_cache + healthcheck: + # Our app exposes /health → {"status":"ok",...} once the model is loaded. + # python urllib (devnen base ships no curl); strip spaces so the match is + # formatting-agnostic. 127.0.0.1 explicitly (uvicorn binds IPv4 only). + test: ["CMD-SHELL", "python3 -c \"import urllib.request,sys; b=urllib.request.urlopen('http://127.0.0.1:8197/health',timeout=5).read(); sys.exit(0 if b'\\\"status\\\":\\\"ok\\\"' in b.replace(b' ',b'') else 1)\""] + interval: 30s + timeout: 10s + retries: 3 + # Weights are HF-cached already (~16s load measured); generous anyway. + start_period: 120s + labels: + - homepage.group=AI Systems + - homepage.name=Chatterbox Fast + - homepage.icon=mdi-account-music-outline + - homepage.description=Streaming TTS — sub-second first-audio, adaptive-chunk (irv-ml1) + - homepage.href=http://10.100.79.3:${CBF_PORT:-8197}