diff --git a/playbooks/deploy-chatterbox.yaml b/playbooks/deploy-chatterbox.yaml new file mode 100644 index 0000000..f79c785 --- /dev/null +++ b/playbooks/deploy-chatterbox.yaml @@ -0,0 +1,89 @@ +# Deploy Chatterbox Turbo (Resemble AI's low-latency English TTS w/ +# voice cloning) via the devnen/Chatterbox-TTS-Server wrapper to +# irv-ml1. +# +# Builds the image locally from devnen's Dockerfile.gpu via docker +# buildx git URL context. ~8-10 min cold build (CUDA + torch + +# Chatterbox deps). First start pulls Chatterbox-Turbo weights (~6 GB) +# into the HF cache. +# +# Usage: +# scripts/elway irv-ml1 --playbook playbooks/deploy-chatterbox.yaml +# +# Idempotent — every step is creates-/when-gated; rerun is safe. + +vars: + compose_dir: /opt/docker/compose/chatterbox + reference_dir: /worktank/chatterbox/reference_audio + cache_dir: /worktank/chatterbox/cache + host_port: "8196" + +steps: + # ── host-side dirs ────────────────────────────────────────────────── + + - name: Ensure /worktank/chatterbox root exists (one-time, sudo) + shell: mkdir -p /worktank/chatterbox + sudo: true + creates: /worktank/chatterbox + + - name: Chown /worktank/chatterbox to lkraven + shell: chown lkraven:lkraven /worktank/chatterbox + sudo: true + when: '[ "$(stat -c %U /worktank/chatterbox)" != lkraven ]' + + - name: Ensure reference-audio dir exists + shell: mkdir -p {{ reference_dir }} + creates: "{{ reference_dir }}" + + - name: Ensure cache dir exists + shell: mkdir -p {{ cache_dir }} + creates: "{{ cache_dir }}" + + - name: Ensure compose dir exists + shell: mkdir -p {{ compose_dir }} + creates: "{{ compose_dir }}" + + # ── deploy compose files ──────────────────────────────────────────── + + - name: Upload compose.yaml + upload: + src: stacks/chatterbox/compose.yaml + dest: "{{ compose_dir }}/compose.yaml" + mode: "0644" + + - name: Seed .env from template (only if absent) + upload: + src: stacks/chatterbox/.env.example + dest: "{{ compose_dir }}/.env" + mode: "0644" + when: "[ ! -f {{ compose_dir }}/.env ]" + + # ── build + bring up ──────────────────────────────────────────────── + + - name: docker compose build (~8-10 min first time; cached after) + shell: cd {{ compose_dir }} && docker compose build + + - name: docker compose up -d + shell: cd {{ compose_dir }} && docker compose up -d + + - name: Wait for /health to respond (allow ~15 min for model download + warmup) + shell: | + for i in $(seq 1 180); do + curl -sf -o /dev/null --max-time 3 http://localhost:{{ host_port }}/health && exit 0 + sleep 5 + done + exit 1 + changed_when: "false" + +verify: + - name: /health returns 200 + shell: curl -sf -o /dev/null http://localhost:{{ host_port }}/health + changed_when: "false" + + - name: /v1/audio/voices returns valid JSON + shell: curl -sf http://localhost:{{ host_port }}/v1/audio/voices | grep -q 'voice\|alloy\|echo' + changed_when: "false" + + - name: Container is running + shell: docker inspect chatterbox --format '{{.State.Status}}' | grep -q running + changed_when: "false" diff --git a/playbooks/deploy-kokoro.yaml b/playbooks/deploy-kokoro.yaml new file mode 100644 index 0000000..e0c603f --- /dev/null +++ b/playbooks/deploy-kokoro.yaml @@ -0,0 +1,86 @@ +# Deploy Kokoro-FastAPI to irv-ml1. +# +# Image is published on GHCR — no Dockerfile to maintain, no first-run +# model download (Kokoro-82M weights are baked in). Stage compose + +# .env, pull, bring up. ~6.5 GB pull on cold cache, ~2-5 min. +# +# Usage: +# scripts/elway irv-ml1 --playbook playbooks/deploy-kokoro.yaml +# +# Idempotent — every step is creates-/when-gated; rerun is safe. + +vars: + compose_dir: /opt/docker/compose/kokoro + user_voices_dir: /worktank/kokoro/user_voices + voices_dir: /worktank/kokoro/voices + host_port: "8193" + +steps: + # ── host-side dirs ────────────────────────────────────────────────── + + - name: Ensure /worktank/kokoro root exists (one-time, sudo) + shell: mkdir -p /worktank/kokoro + sudo: true + creates: /worktank/kokoro + + - name: Chown /worktank/kokoro to lkraven + shell: chown lkraven:lkraven /worktank/kokoro + sudo: true + when: '[ "$(stat -c %U /worktank/kokoro)" != lkraven ]' + + - name: Ensure user-voices dir exists + shell: mkdir -p {{ user_voices_dir }} + creates: "{{ user_voices_dir }}" + + - name: Ensure voices dir exists (used only if compose mount is enabled) + shell: mkdir -p {{ voices_dir }} + creates: "{{ voices_dir }}" + + - name: Ensure compose dir exists + shell: mkdir -p {{ compose_dir }} + creates: "{{ compose_dir }}" + + # ── deploy compose files ──────────────────────────────────────────── + + - name: Upload compose.yaml + upload: + src: stacks/kokoro/compose.yaml + dest: "{{ compose_dir }}/compose.yaml" + mode: "0644" + + - name: Seed .env from template (only if absent) + upload: + src: stacks/kokoro/.env.example + dest: "{{ compose_dir }}/.env" + mode: "0644" + when: "[ ! -f {{ compose_dir }}/.env ]" + + # ── pull + bring up ───────────────────────────────────────────────── + + - name: docker compose pull (first run: ~6.5 GB from GHCR) + shell: cd {{ compose_dir }} && docker compose pull + + - name: docker compose up -d + shell: cd {{ compose_dir }} && docker compose up -d + + - name: Wait for /v1/audio/voices to respond + shell: | + for i in $(seq 1 60); do + curl -sf -o /dev/null --max-time 3 http://localhost:{{ host_port }}/v1/audio/voices && exit 0 + sleep 5 + done + exit 1 + changed_when: "false" + +verify: + - name: /v1/audio/voices returns 200 + shell: curl -sf -o /dev/null http://localhost:{{ host_port }}/v1/audio/voices + changed_when: "false" + + - name: /v1/audio/voices includes at least one built-in (af_bella) + shell: curl -sf http://localhost:{{ host_port }}/v1/audio/voices | grep -q af_bella + changed_when: "false" + + - name: Container is running + shell: docker inspect kokoro --format '{{.State.Status}}' | grep -q running + changed_when: "false" diff --git a/playbooks/deploy-vibevoice.yaml b/playbooks/deploy-vibevoice.yaml new file mode 100644 index 0000000..35c0048 --- /dev/null +++ b/playbooks/deploy-vibevoice.yaml @@ -0,0 +1,88 @@ +# Deploy VibeVoice 1.5B (long-form) to irv-ml1. +# +# Builds the image locally from groxaxo/VibeVoice-FastAPI1 via docker +# buildx git URL context. ~12 min cold build (CUDA 12.8 + torch 2.8 + +# flash-attn). First start downloads VibeVoice-1.5B (~7 GB) into the +# bind-mounted HF cache. Generous /healthz wait deadline accommodates +# both. +# +# Usage: +# scripts/elway irv-ml1 --playbook playbooks/deploy-vibevoice.yaml +# +# Idempotent — every step is creates-/when-gated; rerun is safe. + +vars: + compose_dir: /opt/docker/compose/vibevoice + voices_dir: /worktank/vibevoice/voices + cache_dir: /worktank/vibevoice/cache + host_port: "8194" + +steps: + # ── host-side dirs ────────────────────────────────────────────────── + + - name: Ensure /worktank/vibevoice root exists (one-time, sudo) + shell: mkdir -p /worktank/vibevoice + sudo: true + creates: /worktank/vibevoice + + - name: Chown /worktank/vibevoice to lkraven + shell: chown lkraven:lkraven /worktank/vibevoice + sudo: true + when: '[ "$(stat -c %U /worktank/vibevoice)" != lkraven ]' + + - name: Ensure voices dir exists + shell: mkdir -p {{ voices_dir }} + creates: "{{ voices_dir }}" + + - name: Ensure cache dir exists + shell: mkdir -p {{ cache_dir }} + creates: "{{ cache_dir }}" + + - name: Ensure compose dir exists + shell: mkdir -p {{ compose_dir }} + creates: "{{ compose_dir }}" + + # ── deploy compose files ──────────────────────────────────────────── + + - name: Upload compose.yaml + upload: + src: stacks/vibevoice/compose.yaml + dest: "{{ compose_dir }}/compose.yaml" + mode: "0644" + + - name: Seed .env from template (only if absent) + upload: + src: stacks/vibevoice/.env.example + dest: "{{ compose_dir }}/.env" + mode: "0644" + when: "[ ! -f {{ compose_dir }}/.env ]" + + # ── build + bring up ──────────────────────────────────────────────── + + - name: docker compose build (~12 min first time; cached after) + shell: cd {{ compose_dir }} && docker compose build + + - name: docker compose up -d + shell: cd {{ compose_dir }} && docker compose up -d + + - name: Wait for /health to respond (allow ~20 min for model download + warmup) + shell: | + for i in $(seq 1 240); do + curl -sf -o /dev/null --max-time 3 http://localhost:{{ host_port }}/health && exit 0 + sleep 5 + done + exit 1 + changed_when: "false" + +verify: + - name: /health returns 200 + shell: curl -sf -o /dev/null http://localhost:{{ host_port }}/health + changed_when: "false" + + - name: /v1/audio/voices returns valid JSON + shell: curl -sf http://localhost:{{ host_port }}/v1/audio/voices | grep -q '"voices"\|"voice"\|alloy\|Carter' + changed_when: "false" + + - name: Container is running + shell: docker inspect vibevoice --format '{{.State.Status}}' | grep -q running + changed_when: "false" diff --git a/stacks/chatterbox/.env.example b/stacks/chatterbox/.env.example new file mode 100644 index 0000000..93cfef9 --- /dev/null +++ b/stacks/chatterbox/.env.example @@ -0,0 +1,52 @@ +# Chatterbox Turbo stack tunables. Copy to `.env` on irv-ml1 before +# deploying. + +# ── build pin ──────────────────────────────────────────────────────── +# SHA of devnen/Chatterbox-TTS-Server. Bump + rebuild when you want +# upstream wrapper updates. Pin a SHA — the wrapper has no tagged +# releases yet. +CHATTERBOX_SHA=main + +# Local image tag — bump when you change build context to force a +# fresh layer build. +CHATTERBOX_TAG=v1 + +# ── network ────────────────────────────────────────────────────────── +# Host port. Container listens on 8004 internally. +# Reserved on irv-ml1: 8188 ComfyUI, 8190 CosyVoice, 8191 Qwen3-TTS, +# 8192 IndexTTS-2, 8193 Kokoro, 8194 VibeVoice, 8765 Parakeet. +# 8196 picked here. +CHATTERBOX_PORT=8196 + +# Bind address. 0.0.0.0 exposes on all interfaces (incl. WG tunnel +# interface 10.100.79.3); 127.0.0.1 restricts to local-only. +CHATTERBOX_BIND=0.0.0.0 + +# ── runtime / GPU ──────────────────────────────────────────────────── +# Devices visible inside the container. "0" pins to the RTX 3090. +# Chatterbox Turbo is small (~2.5 GB fp16) — the 3090 is plenty. +CHATTERBOX_GPU_DEVICES=0 + +# Model checkpoint. Options: +# ResembleAI/chatterbox-turbo — flagship Turbo, 350M, EN-only, +# ~2.5 GB fp16, ~75 ms latency +# ResembleAI/chatterbox — base Chatterbox, 500M, EN-only, +# ~3.5 GB fp16, slower but with +# exaggeration/CFG-weight knobs +# ResembleAI/chatterbox-multilingual — 23 languages, slower than Turbo +CHATTERBOX_MODEL_REPO=ResembleAI/chatterbox-turbo + +# ── persistent storage on the host ─────────────────────────────────── +# Reference audio dir for voice cloning. Drop short (~5 s) reference +# WAVs in here; the wrapper picks them up by filename. Cloned voices +# need the original reference to recreate — included in restic. +CHATTERBOX_REFERENCE_DIR=/worktank/chatterbox/reference_audio + +# HuggingFace cache — Chatterbox Turbo weights (~6 GB) land here on +# first start. Bind-mounted so they survive container recreate. +# Excluded from restic (regenerable from HF). +CHATTERBOX_CACHE_DIR=/worktank/chatterbox/cache + +# Optional: mount a host config.yaml for hot-edit. Leave commented +# out in compose.yaml unless you actively want this. +# CHATTERBOX_CONFIG=/worktank/chatterbox/config.yaml diff --git a/stacks/chatterbox/README.md b/stacks/chatterbox/README.md new file mode 100644 index 0000000..c01b606 --- /dev/null +++ b/stacks/chatterbox/README.md @@ -0,0 +1,127 @@ +# Chatterbox Turbo + +Resemble AI's 350M-param low-latency English TTS with zero-shot voice +cloning, served via [devnen/Chatterbox-TTS-Server](https://github.com/devnen/Chatterbox-TTS-Server) +— the most actively-maintained OpenAI-compat wrapper supporting Turbo. + +Model: [ResembleAI/chatterbox-turbo](https://huggingface.co/ResembleAI/chatterbox-turbo) +— released April 2026, ~6× realtime, ~75 ms latency, MIT-licensed. + +## Why this stack exists + +Fills the **low-latency English voice-cloning** slot none of the other +TTS own cleanly: Kokoro is fast but fixed-voice; IndexTTS-2 clones +beautifully but is slow; Qwen3-TTS-Base clones at a higher quality bar +but isn't tuned for sub-second latency. Chatterbox Turbo trades some +fidelity for **6× realtime + 5-second-reference cloning**, ideal for +real-time voice-agent use cases. + +| | use case | +|---|---| +| **Chatterbox Turbo** | low-latency English w/ voice cloning + paralinguistic tags | +| Kokoro | low-latency English, fixed voice library | +| IndexTTS-2 | English voice cloning + emotion vector / text control | +| Qwen3-TTS-1.7B-Base | high-quality English voice cloning | +| CosyVoice 3 | multilingual (Chinese-leaning) | +| VibeVoice 1.5B | long-form / multi-speaker dialogue | + +## Headline features + +- **Zero-shot voice cloning from ~5 s reference** (base Chatterbox + needs ~10 s; Turbo cuts that in half). +- **Native paralinguistic tags inline in text** — drop these into your + prompt and the model honors them: + ``` + [laugh] [cough] [sigh] [gasp] [whisper] [breath] + ``` + Different shape from IndexTTS-2's 8-vector emotion control: cleaner + for "say it like this" markup directly in the prompt. +- **Mandatory PerTh watermark** on outputs (Resemble policy, cannot + be disabled). Non-issue for internal use; mention it if you ever + ship Chatterbox-generated audio externally. + +## API + +OpenAI-compat at `http://10.100.79.3:8196`: + +```bash +# Built-in voices. +curl http://10.100.79.3:8196/v1/audio/voices + +# Single-shot synthesis with a built-in voice. +curl -fsS -X POST http://10.100.79.3:8196/v1/audio/speech \ + -H 'Content-Type: application/json' \ + -d '{"model":"chatterbox-turbo","input":"Hello there. [laugh] What a day.","voice":"alloy","response_format":"wav"}' \ + > out.wav + +# Voice cloning — drop a 5 s reference WAV into +# /worktank/chatterbox/reference_audio/glados.wav, then: +curl -fsS -X POST http://10.100.79.3:8196/v1/audio/speech \ + -H 'Content-Type: application/json' \ + -d '{"model":"chatterbox-turbo","input":"I have all the time in the world.","voice":"glados"}' \ + > glados.wav + +# Streaming (where supported by the wrapper). +curl -fsS -X POST http://10.100.79.3:8196/v1/audio/speech \ + -H 'Content-Type: application/json' \ + -d '{"model":"chatterbox-turbo","input":"long passage…","voice":"alloy","stream":true}' \ + | mpv --no-cache - +``` + +Native `/tts` endpoint (devnen wrapper extension) and OpenAPI at `/docs`. +Healthcheck at `/health`. + +## Voice library + +Drop reference WAV / MP3 / FLAC into +`/worktank/chatterbox/reference_audio/` on the host. The wrapper +discovers new files on next request — no restart needed. Use clean +~5 s clips, single speaker. + +Built-in OpenAI-style voice aliases (alloy, echo, fable, onyx, nova, +shimmer) map to bundled presets — useful for OpenAI SDK clients that +hardcode those names. + +## Deploy + +```bash +scripts/elway irv-ml1 --playbook playbooks/deploy-chatterbox.yaml +``` + +Cold deploy budget: +- Image build: ~5-8 GB (CUDA + torch + Chatterbox deps) +- Model download: ~6 GB (Chatterbox Turbo weights, first run) +- **Total: ~12 GB on /worktank/chatterbox/** + +First build: ~8-10 min. First synthesis: ~10-30 s warmup. + +## Switching the model + +```bash +ssh irv-ml1 ' + cd /opt/docker/compose/chatterbox + sed -i "s|^CHATTERBOX_MODEL_REPO=.*|CHATTERBOX_MODEL_REPO=ResembleAI/chatterbox|" .env + docker compose up -d +' +``` + +Options for `CHATTERBOX_MODEL_REPO`: +- `ResembleAI/chatterbox-turbo` — flagship Turbo (default, fastest) +- `ResembleAI/chatterbox` — base Chatterbox, 500M, slower but with + exaggeration / CFG-weight knobs Turbo doesn't expose +- `ResembleAI/chatterbox-multilingual` — 23 languages (slower than + Turbo, useful if you need non-English on this stack vs CosyVoice 3) + +## Gotchas + +- **Python 3.10 only** (devnen wrapper). Image bakes that in; not + something you'd hit unless you fork the Dockerfile. +- **PerTh watermark** is unconditional. Can't disable. +- **Turbo loses some knobs vs base Chatterbox** — no `exaggeration` + or CFG-weight tuning. If you need expressive amplitude control, + flip to base Chatterbox via the config swap above. +- **Repo is fresh** (~weekly commits). Pin to a SHA in `.env` + (`CHATTERBOX_SHA=...`) and rebuild monthly to ride upstream + bug-fix progress. +- **License**: wrapper MIT; weights MIT (Resemble) — including the + watermark requirement. diff --git a/stacks/chatterbox/compose.yaml b/stacks/chatterbox/compose.yaml new file mode 100644 index 0000000..ac89496 --- /dev/null +++ b/stacks/chatterbox/compose.yaml @@ -0,0 +1,60 @@ +# Chatterbox Turbo (Resemble AI's 350M-param low-latency English TTS +# with zero-shot voice cloning) served via devnen/Chatterbox-TTS-Server +# — the most actively-maintained OpenAI-compat wrapper that supports +# the Turbo checkpoint. +# +# Why this stack exists alongside the other TTS: +# * Low-latency English with VOICE CLONING (Kokoro is fast but +# fixed-voice; this fills the speed-AND-cloning slot). +# * Native paralinguistic tags inline in text: +# [laugh] [cough] [sigh] [gasp] [whisper] [breath] +# Different shape from IndexTTS-2's emotion vector — cleaner for +# "say it like this" markup directly in the prompt. +# * MIT weights + code; ~2.5 GB VRAM at fp16; ~75 ms latency. +# * Mandatory PerTh watermark on outputs (Resemble policy, can't +# be disabled). Non-issue for internal use. +# +# Image is built locally from the upstream Dockerfile via docker +# buildx git-context (no source vendored on the host). +# +# All tunables live in .env — edit that, not this file. + +services: + chatterbox: + image: local/chatterbox:${CHATTERBOX_TAG} + build: + context: https://github.com/devnen/Chatterbox-TTS-Server.git#${CHATTERBOX_SHA} + dockerfile: docker/Dockerfile.gpu + container_name: chatterbox + restart: unless-stopped + runtime: nvidia + ports: + - "${CHATTERBOX_BIND:-0.0.0.0}:${CHATTERBOX_PORT}:8004" + environment: + - NVIDIA_VISIBLE_DEVICES=${CHATTERBOX_GPU_DEVICES:-0} + # Switch to ResembleAI/chatterbox-turbo (default) or the base + # ResembleAI/chatterbox / ResembleAI/chatterbox-multilingual + # via the wrapper's config hot-swap. + - CHATTERBOX_MODEL_REPO=${CHATTERBOX_MODEL_REPO:-ResembleAI/chatterbox-turbo} + - HF_HOME=/app/hf_cache + volumes: + - ${CHATTERBOX_REFERENCE_DIR}:/app/reference_audio + - ${CHATTERBOX_CACHE_DIR}:/app/hf_cache + # Optional: mount config.yaml as a host file for hot-edit. Default + # is to use the in-image config + env var overrides. + # - ${CHATTERBOX_CONFIG}:/app/config.yaml:ro + healthcheck: + # Devnen wrapper exposes /health; the OpenAPI/docs path is /docs. + test: ["CMD-SHELL", "curl -fsS -o /dev/null http://localhost:8004/health || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + # First boot pulls Chatterbox-Turbo (~6 GB total HF assets) and + # warms torch.compile — give it a generous deadline. + start_period: 600s + labels: + - homepage.group=AI Systems + - homepage.name=Chatterbox Turbo + - homepage.icon=mdi-account-music-outline + - homepage.description=Low-latency English TTS w/ voice cloning + paralinguistics (irv-ml1) + - homepage.href=http://10.100.79.3:${CHATTERBOX_PORT} diff --git a/stacks/kokoro/.env.example b/stacks/kokoro/.env.example new file mode 100644 index 0000000..6d59ce9 --- /dev/null +++ b/stacks/kokoro/.env.example @@ -0,0 +1,37 @@ +# Kokoro-FastAPI stack tunables. Copy to `.env` on irv-ml1 before deploying. + +# ── image pin ──────────────────────────────────────────────────────── +# Tagged release on GHCR. Avoid `latest` — upstream warns it can move +# without notice. v0.2.4-master = 2025-12-13 release with Kokoro-82M v1.0 +# baked in (commit 9901c2b). +KOKORO_TAG=v0.2.4-master + +# ── network ────────────────────────────────────────────────────────── +# Host port. Container listens on 8880 internally. +# Reserved on irv-ml1: 8188 ComfyUI, 8190 CosyVoice, 8191 Qwen3-TTS, +# 8192 IndexTTS-2, 8765 Parakeet. 8193 picked here. +KOKORO_PORT=8193 + +# Bind address. 0.0.0.0 exposes on all interfaces (incl. WG tunnel +# interface 10.100.79.3); 127.0.0.1 restricts to local-only. +KOKORO_BIND=0.0.0.0 + +# ── runtime / GPU ──────────────────────────────────────────────────── +# Devices visible inside the container. "0" pins to the RTX 3090 +# (Kokoro is tiny — ~1 GB VRAM — and doesn't need the A6000). Use +# "all" if you want the model swap to either GPU. +KOKORO_GPU_DEVICES=0 + +# Logging level for the FastAPI app. INFO is the upstream default. +KOKORO_LOG_LEVEL=INFO + +# ── persistent storage on the host ─────────────────────────────────── +# Voicepacks dir — bind-mount target IF the (commented-out) override +# is enabled in compose.yaml. Default: leave empty and use the +# in-image voicepacks. +KOKORO_VOICES_DIR=/worktank/kokoro/voices + +# User-voices dir — a parallel directory the wrapper *also* scans for +# voicepacks alongside the in-image ones. Always mounted (cheap, empty +# by default). Drop your own .pt files here if you train Kokoro voices. +KOKORO_USER_VOICES_DIR=/worktank/kokoro/user_voices diff --git a/stacks/kokoro/README.md b/stacks/kokoro/README.md new file mode 100644 index 0000000..f98d372 --- /dev/null +++ b/stacks/kokoro/README.md @@ -0,0 +1,80 @@ +# Kokoro + +[hexgrad/Kokoro-82M](https://huggingface.co/hexgrad/Kokoro-82M) served +via [remsky/Kokoro-FastAPI](https://github.com/remsky/Kokoro-FastAPI). + +## Why this stack exists + +Lowest-latency English TTS in the fleet by a wide margin — ~300 ms +time-to-first-audio on GPU, 35-100x realtime, ~1 GB VRAM at fp16. +Native streaming (Kokoro's `KPipeline.__call__` is a per-phrase +generator) and the wrapper exposes it via OpenAI-compat `stream=true` +over chunked HTTP — drop-in for any OpenAI SDK client. + +Complementary to the rest of the TTS slate: + +| | use case | +|---|---| +| **Kokoro** | low-latency English, fixed voice library | +| **Chatterbox Turbo** | low-latency English w/ voice cloning + paralinguistic tags | +| **IndexTTS-2** | English voice cloning + emotion vector / text control | +| **Qwen3-TTS-1.7B-Base** | high-quality English voice cloning | +| **CosyVoice 3** | multilingual (Chinese-leaning) | +| **VibeVoice 1.5B** | long-form podcast / multi-speaker dialogue | + +## API + +OpenAI-compat at `http://10.100.79.3:8193`: + +```bash +# List built-in voices (~60 of them, named like af_bella, am_adam, jf_*, zf_*). +curl http://10.100.79.3:8193/v1/audio/voices + +# Single-shot synthesis. +curl -fsS -X POST http://10.100.79.3:8193/v1/audio/speech \ + -H 'Content-Type: application/json' \ + -d '{"model":"kokoro","input":"Hello there.","voice":"af_bella","response_format":"wav"}' \ + > out.wav + +# Streaming — pipe straight into a player. +curl -fsS -X POST http://10.100.79.3:8193/v1/audio/speech \ + -H 'Content-Type: application/json' \ + -d '{"model":"kokoro","input":"long passage here…","voice":"af_bella","stream":true}' \ + | mpv --no-cache - + +# Voice mixing — sum voicepacks with weights. +curl -fsS -X POST http://10.100.79.3:8193/v1/audio/speech \ + -H 'Content-Type: application/json' \ + -d '{"model":"kokoro","input":"hello","voice":"af_bella(2)+af_heart(1)","response_format":"wav"}' \ + > mix.wav +``` + +Web UI at `/web` (browse voices + try in-place); OpenAPI at `/docs`. + +Supported `response_format`: `mp3 | wav | opus | flac | pcm`. + +## Voices + +- **Built-in**: 60+ in 8 languages (en-US, en-GB, ja, zh, es, fr, hi, it). + Discoverable via `GET /v1/audio/voices` — naming convention is + `_` (e.g. `af_bella`, `am_adam`, + `jf_alpha`, `zf_xiaobei`). +- **Custom**: drop `.pt` voicepacks into `/worktank/kokoro/user_voices/` + on the host. Wrapper auto-discovers them on next request (no + restart). Training Kokoro voices is non-trivial — consult the + hexgrad community for how-to. + +## Deploy + +```bash +scripts/elway irv-ml1 --playbook playbooks/deploy-kokoro.yaml +``` + +First deploy: ~6.5 GB image pull from GHCR (~2-5 min on a fast link). +No model download on first run — Kokoro-82M weights are baked in. +Subsequent starts: a few seconds. + +## License + +Apache-2.0 for both the wrapper code (remsky/Kokoro-FastAPI) and the +Kokoro-82M weights (hexgrad). diff --git a/stacks/kokoro/compose.yaml b/stacks/kokoro/compose.yaml new file mode 100644 index 0000000..2fea66e --- /dev/null +++ b/stacks/kokoro/compose.yaml @@ -0,0 +1,52 @@ +# Kokoro-82M served via remsky/Kokoro-FastAPI — the de-facto OpenAI- +# compatible wrapper for hexgrad's Kokoro-82M TTS. +# +# Why this stack exists alongside the other TTS: +# * Lowest-latency English in the fleet — ~300 ms TTFA on GPU, +# RTF 35-100x on a 4060 Ti class card. +# * Native streaming via OpenAI-compat `stream=true` over HTTP +# chunked transfer (Kokoro's KPipeline is a per-phrase generator). +# * Apache-2.0 weights + code; ~1 GB VRAM at fp16. +# * 60+ built-in voices (no cloning — for that use IndexTTS-2 or +# Chatterbox Turbo). Voices combinable via "voice(weight)+..." syntax. +# +# Image is a published GHCR build; no Dockerfile to maintain. Models +# baked into the image, no first-run download. Deploy is a pull + up. +# +# All tunables live in .env — edit that, not this file. + +services: + kokoro: + image: ghcr.io/remsky/kokoro-fastapi-gpu:${KOKORO_TAG} + container_name: kokoro + restart: unless-stopped + runtime: nvidia + ports: + - "${KOKORO_BIND:-0.0.0.0}:${KOKORO_PORT}:8880" + environment: + - NVIDIA_VISIBLE_DEVICES=${KOKORO_GPU_DEVICES:-0} + - USE_GPU=true + - API_LOG_LEVEL=${KOKORO_LOG_LEVEL:-INFO} + volumes: + # Optional voice-overlay mount — drop a custom .pt into the + # host dir to make it available alongside the 60+ built-ins. The + # image already ships voicepacks at this path, so the bind mount + # SHADOWS them — only do this if you actually want to manage the + # full voice library yourself. For most deploys, leave the mount + # commented out and use the in-image voices. + # - ${KOKORO_VOICES_DIR}:/app/api/src/voices/v1_0 + - ${KOKORO_USER_VOICES_DIR}:/app/user_voices + healthcheck: + # The image is python-based with curl available. /v1/audio/voices + # is a no-arg GET that exercises the full API path. + test: ["CMD-SHELL", "curl -fsS -o /dev/null http://localhost:8880/v1/audio/voices || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 90s + labels: + - homepage.group=AI Systems + - homepage.name=Kokoro + - homepage.icon=mdi-microphone-message + - homepage.description=Low-latency English TTS w/ streaming (irv-ml1) + - homepage.href=http://10.100.79.3:${KOKORO_PORT} diff --git a/stacks/vibevoice/.env.example b/stacks/vibevoice/.env.example new file mode 100644 index 0000000..8f0d3ae --- /dev/null +++ b/stacks/vibevoice/.env.example @@ -0,0 +1,76 @@ +# VibeVoice 1.5B (long-form) stack tunables. +# Copy to `.env` on irv-ml1 before deploying. + +# ── build pin ──────────────────────────────────────────────────────── +# SHA of groxaxo/VibeVoice-FastAPI1 (a more current fork of +# ncoder-ai/VibeVoice-FastAPI). Bump + rebuild when you want upstream +# wrapper updates. +VIBEVOICE_SHA=7614c469a145 + +# Local image tag — bump when you change build context to force a +# fresh layer build. +VIBEVOICE_TAG=v1 + +# ── network ────────────────────────────────────────────────────────── +# Host port. Container listens on 8001 internally. +# Reserved on irv-ml1: 8188 ComfyUI, 8190 CosyVoice, 8191 Qwen3-TTS, +# 8192 IndexTTS-2, 8193 Kokoro, 8765 Parakeet. 8194 picked here. +VIBEVOICE_PORT=8194 + +# Bind address. 0.0.0.0 exposes on all interfaces (incl. WG tunnel +# interface 10.100.79.3); 127.0.0.1 restricts to local-only. +VIBEVOICE_BIND=0.0.0.0 + +# ── runtime / GPU ──────────────────────────────────────────────────── +# Devices visible inside the container. "1" pins to the RTX A6000 — +# 1.5B fits easily on the 3090 too, but pinning to the bigger card +# leaves headroom if you later flip VIBEVOICE_MODEL to the 7B variant. +VIBEVOICE_GPU_DEVICES=1 + +# Model. Options (per groxaxo/ncoder-ai docs): +# microsoft/VibeVoice-1.5B — flagship, ~7 GB bf16 VRAM +# rsxdalv/VibeVoice-Large — 7B variant, ~18 GB bf16 VRAM +# (need device_ids="1" / A6000) +# FabioSarracino/VibeVoice-Large-Q8 — 7B int8 quantized, ~10 GB +VIBEVOICE_MODEL=microsoft/VibeVoice-1.5B + +# Number of denoising inference steps. Default 10 is a good +# quality/speed tradeoff. Lower = faster but lower quality. +VIBEVOICE_INFERENCE_STEPS=10 + +# Compute dtype. bfloat16 default (best speed/quality on Ampere+). +# Use float16 for older GPUs without bf16 support. +VIBEVOICE_DTYPE=bfloat16 + +# Attention impl. flash_attention_2 is fastest if installed (bundled +# in the upstream image build). Fall back to "sdpa" if it errors. +VIBEVOICE_ATTN=flash_attention_2 + +# Quantization. Empty = none. "int8_torchao" saves ~40% VRAM at a +# small quality cost — useful if you want to run 7B on a smaller GPU. +VIBEVOICE_QUANT= + +# torch.compile. Bumps cold-start by ~3-5 min the first time but +# trims per-generation latency. Disable if you're iterating quickly. +VIBEVOICE_TORCH_COMPILE=false +VIBEVOICE_TORCH_COMPILE_MODE=default + +# CFG (classifier-free guidance) scale. Default 1.8 from upstream; +# higher = stronger adherence to text/voice, lower = more free. +VIBEVOICE_CFG_SCALE=1.8 + +# Max generation length in audio frames. 5400 = ~6 minutes at the +# native rate. Bump for longer podcasts (each frame takes work). +VIBEVOICE_MAX_GEN_LEN=5400 + +# ── persistent storage on the host ─────────────────────────────────── +# Voice library — flat dir of .wav/.mp3/.flac/.m4a files. Mounted +# read-only into the container. Drop a file in, restart container, +# voice is available. (Restart needed because the upstream wrapper +# scans on init, not per-request.) +VIBEVOICE_VOICES_DIR=/worktank/vibevoice/voices + +# HuggingFace cache. Holds VibeVoice weights + any aux models pulled. +# Bind-mounted so model state survives container recreate. Excluded +# from restic (regenerable from HF). +VIBEVOICE_CACHE_DIR=/worktank/vibevoice/cache diff --git a/stacks/vibevoice/README.md b/stacks/vibevoice/README.md new file mode 100644 index 0000000..6fb1b53 --- /dev/null +++ b/stacks/vibevoice/README.md @@ -0,0 +1,115 @@ +# VibeVoice + +Microsoft's diffusion-based long-form TTS, served via +[groxaxo/VibeVoice-FastAPI1](https://github.com/groxaxo/VibeVoice-FastAPI1) +(a recent fork of [ncoder-ai/VibeVoice-FastAPI](https://github.com/ncoder-ai/VibeVoice-FastAPI) +which moves faster than upstream). + +Model: [microsoft/VibeVoice-1.5B](https://huggingface.co/microsoft/VibeVoice-1.5B) +by default. Switch to the 7B variant via `.env` if you want the +bigger checkpoint. + +## Why this stack exists + +Long-form / podcast-quality TTS with native multi-speaker dialogue +support. Designed for one-shot generation of multi-minute scripts +where conversation flow matters. **Not** for low-latency single-line +synthesis — for that use Kokoro or Chatterbox Turbo. + +| | use case | +|---|---| +| **VibeVoice 1.5B** | long-form, multi-speaker dialogue (this stack) | +| Kokoro | low-latency English, fixed voice library | +| Chatterbox Turbo | low-latency English w/ voice cloning | +| IndexTTS-2 | English voice cloning + emotion control | +| Qwen3-TTS-1.7B-Base | high-quality English voice cloning | +| CosyVoice 3 | multilingual (Chinese-leaning) | + +## API + +OpenAI-compat at `http://10.100.79.3:8194`: + +```bash +# Single-speaker (OpenAI-style). +curl -fsS -X POST http://10.100.79.3:8194/v1/audio/speech \ + -H 'Content-Type: application/json' \ + -d '{"model":"vibevoice","input":"Hello there.","voice":"voice-name","response_format":"wav"}' \ + > out.wav + +# Multi-speaker dialogue — the headline feature. Format the input +# as a script with `Speaker N:` prefixes (0-indexed). The wrapper's +# extended /v1/vibevoice/generate endpoint handles voice switching. +curl -fsS -X POST http://10.100.79.3:8194/v1/vibevoice/generate \ + -H 'Content-Type: application/json' \ + -d '{ + "script":"Speaker 0: Welcome to the show.\nSpeaker 1: Glad to be here.\nSpeaker 0: Today we discuss…", + "voices":["voice-host","voice-guest"], + "stream":true + }' > podcast.wav + +# List available voices. +curl http://10.100.79.3:8194/v1/audio/voices +``` + +OpenAPI / docs at `/docs`. Healthcheck at `/health`. + +`stream=true` is honored on the multi-speaker endpoint; the +single-shot OpenAI endpoint returns the full file in one go. + +## Voices + +Drop `.wav` / `.mp3` / `.flac` / `.m4a` into +`/worktank/vibevoice/voices/` on the host (mounted read-only into +the container). Restart the container after adding; the wrapper +scans the dir at init, not per-request: + +```bash +ssh irv-ml1 'cd /opt/docker/compose/vibevoice && docker compose restart' +``` + +VibeVoice also has built-in voice presets (Carter, Davis, Emma, +Frank, Grace, Mike, Samuel) accessible by name. Microsoft has not +released the cloning tooling so you can't add new "trained" voices +— but the bundled ones already cover most podcast use cases. + +## Deploy + +```bash +scripts/elway irv-ml1 --playbook playbooks/deploy-vibevoice.yaml +``` + +Cold deploy budget: +- ~12 GB image build (CUDA 12.8 + torch 2.8 + flash-attn) +- ~7 GB model download (VibeVoice-1.5B) on first start +- **Total: ~19 GB on /worktank/vibevoice/** + +First build: ~12 min. First generation: ~30-60 s warmup. + +## Switching to the 7B variant + +```bash +ssh irv-ml1 ' + cd /opt/docker/compose/vibevoice + sed -i "s|^VIBEVOICE_MODEL=.*|VIBEVOICE_MODEL=rsxdalv/VibeVoice-Large|" .env + docker compose up -d +' +``` + +The 7B model auto-downloads on next start (~18 GB). VRAM jumps from +~7 GB to ~18 GB bf16 — keep `VIBEVOICE_GPU_DEVICES=1` (A6000) for it. +For lower VRAM at slight quality cost, set `VIBEVOICE_QUANT=int8_torchao` +which brings 7B down to ~10 GB. + +## Gotchas + +- **Not streaming-friendly for single-line use.** The diffusion head + has to denoise the whole latent before vocoding. Streaming on + `/v1/vibevoice/generate` works at script-segment granularity + (paragraph-ish), not token-by-token. +- **Voice cloning isn't published.** Microsoft released the inference + models but not the training pipeline. Use the built-in voices, or + pick another stack (IndexTTS-2 / Qwen3-TTS / Chatterbox Turbo). +- **`flash_attention_2`** is the upstream default; if your GPU/torch + combo doesn't have it built, set `VIBEVOICE_ATTN=sdpa` in `.env` + to fall back to PyTorch's scaled-dot-product attention. +- **License**: VibeVoice MIT (Microsoft); wrapper MIT. diff --git a/stacks/vibevoice/compose.yaml b/stacks/vibevoice/compose.yaml new file mode 100644 index 0000000..f488551 --- /dev/null +++ b/stacks/vibevoice/compose.yaml @@ -0,0 +1,68 @@ +# VibeVoice 1.5B long-form TTS via groxaxo/VibeVoice-FastAPI1 +# (fork of ncoder-ai/VibeVoice-FastAPI). Multi-speaker dialogue support +# via the extended /v1/vibevoice/generate endpoint with a Speaker N: +# script format. OpenAI-compat /v1/audio/speech also exposed. +# +# Why this stack exists alongside the other TTS: +# * Long-form / podcast-quality slot — VibeVoice is Microsoft's +# diffusion-based long-form TTS designed for multi-speaker output. +# * Dialogue mode: feed `Speaker 0: ... \n Speaker 1: ...` and the +# model handles voice switching natively. +# * Trade-off: NOT streaming-friendly — generation is single-shot +# latent denoising over the whole sequence, then vocode. For +# low-latency English, use Kokoro or Chatterbox Turbo instead. +# +# Image is built locally from the upstream Dockerfile via docker +# buildx git-context (no source vendored on the host). Pinned to a +# SHA in .env so rebuilds are reproducible. +# +# Default model is VibeVoice-1.5B (~7 GB bf16 VRAM). Switch to +# rsxdalv/VibeVoice-Large for the 7B variant (~18 GB) — pin to A6000 +# in that case. +# +# All tunables live in .env — edit that, not this file. + +services: + vibevoice: + image: local/vibevoice:${VIBEVOICE_TAG} + build: + context: https://github.com/groxaxo/VibeVoice-FastAPI1.git#${VIBEVOICE_SHA} + dockerfile: Dockerfile + container_name: vibevoice + restart: unless-stopped + runtime: nvidia + ports: + - "${VIBEVOICE_BIND:-0.0.0.0}:${VIBEVOICE_PORT}:8001" + environment: + - NVIDIA_VISIBLE_DEVICES=${VIBEVOICE_GPU_DEVICES:-1} + - VIBEVOICE_MODEL_PATH=${VIBEVOICE_MODEL:-microsoft/VibeVoice-1.5B} + - VIBEVOICE_DEVICE=cuda + - VIBEVOICE_INFERENCE_STEPS=${VIBEVOICE_INFERENCE_STEPS:-10} + - VIBEVOICE_DTYPE=${VIBEVOICE_DTYPE:-bfloat16} + - VIBEVOICE_ATTN_IMPLEMENTATION=${VIBEVOICE_ATTN:-flash_attention_2} + - VIBEVOICE_QUANTIZATION=${VIBEVOICE_QUANT:-} + - TORCH_COMPILE=${VIBEVOICE_TORCH_COMPILE:-false} + - TORCH_COMPILE_MODE=${VIBEVOICE_TORCH_COMPILE_MODE:-default} + - VOICES_DIR=/app/voices + - DEFAULT_CFG_SCALE=${VIBEVOICE_CFG_SCALE:-1.8} + - MAX_GENERATION_LENGTH=${VIBEVOICE_MAX_GEN_LEN:-5400} + - HF_HOME=/root/.cache/huggingface + volumes: + - ${VIBEVOICE_VOICES_DIR}:/app/voices:ro + - ${VIBEVOICE_CACHE_DIR}:/root/.cache/huggingface + healthcheck: + # Upstream Dockerfile exposes /health. + test: ["CMD-SHELL", "curl -fsS -o /dev/null http://localhost:8001/health || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + # First boot pulls VibeVoice-1.5B (~7 GB) on a cold cache and + # may also build flash-attn / torch.compile JIT cache on the + # first inference. Generous deadline. + start_period: 900s + labels: + - homepage.group=AI Systems + - homepage.name=VibeVoice + - homepage.icon=mdi-podcast + - homepage.description=Long-form / multi-speaker dialogue TTS (irv-ml1) + - homepage.href=http://10.100.79.3:${VIBEVOICE_PORT}