diff --git a/stacks/cosyvoice/.env.example b/stacks/cosyvoice/.env.example new file mode 100644 index 0000000..b2984b8 --- /dev/null +++ b/stacks/cosyvoice/.env.example @@ -0,0 +1,42 @@ +# CosyVoice stack tunables. Copy to `.env` on irv-ml1 before deploying. +# +# cp .env.example .env +# # edit as needed +# docker compose up -d + +# Image tag. `v1.3.2` (2026-01-18) is the latest wrapper and ships +# Fun-CosyVoice3-0.5B-2512 (CosyVoice 3 model). The older `v3.x` +# tag family (2025-12-18) also ships CosyVoice 3 — the wrapper +# version numbering diverged from the model version, which is +# confusing but deliberate upstream. Stay on v1.3.x. +# +# See https://hub.docker.com/r/neosun/cosyvoice/tags for updates. +COSYVOICE_VERSION=v1.3.2 + +# Host port for the web UI + REST API (container listens on 8188). +# Avoiding 8188 on the host since ComfyUI already has it. +COSYVOICE_PORT=8190 + +# Bind address. 0.0.0.0 exposes on all interfaces including the WG +# tunnel IP (10.100.79.3). Use 127.0.0.1 to restrict to local-only. +COSYVOICE_BIND=0.0.0.0 + +# Path inside the container for the active model. The image places +# Fun-CosyVoice3-0.5B at pretrained_models/Fun-CosyVoice3-0.5B on +# first run — keep this default unless upstream ships alternate +# model variants. +COSYVOICE_MODEL_DIR=pretrained_models/Fun-CosyVoice3-0.5B + +# Seconds of idleness before the server releases GPU VRAM. Reloading +# takes a few seconds; trade off fast re-use vs sharing GPUs with +# other workloads (llama-swap style). Default 600 (10 min). +COSYVOICE_GPU_IDLE_TIMEOUT=600 + +# Host paths for persistent data. Must exist and be writable by the +# container before first `up`. +# voices/ cloned speaker profiles — precious, restic-covered +# input/ uploaded source audio — scratch +# output/ synthesized clips — scratch, regenerable +COSYVOICE_VOICES_DIR=/worktank/cosyvoice/voices +COSYVOICE_INPUT_DIR=/worktank/cosyvoice/input +COSYVOICE_OUTPUT_DIR=/worktank/cosyvoice/output diff --git a/stacks/cosyvoice/README.md b/stacks/cosyvoice/README.md new file mode 100644 index 0000000..18b8426 --- /dev/null +++ b/stacks/cosyvoice/README.md @@ -0,0 +1,104 @@ +# CosyVoice + +Multilingual expressive TTS with zero-shot voice cloning, served by +the `neosun/cosyvoice` wrapper around FunAudioLLM's +Fun-CosyVoice3-0.5B-2512. + +**Server:** irv-ml1 (Irvine, WireGuard-only) +**Port:** 8190 (configurable via `.env`; container listens on 8188 +internally but host port moved to avoid ComfyUI's 8188) +**GPUs:** both exposed (`NVIDIA_VISIBLE_DEVICES=all`); image reads +`CUDA_VISIBLE_DEVICES` for pinning +**Upstream wrapper:** [neosun100/cosyvoice-docker](https://github.com/neosun100/cosyvoice-docker) +**Upstream model:** [FunAudioLLM/Fun-CosyVoice3-0.5B-2512](https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512) + +## Why CosyVoice 3 (and not Kokoro / v2) + +Emotive content was the deal-breaker for Kokoro. CosyVoice 3 extended +the v2 instruction-following dataset from 1,500 → 5,000 hours +specifically covering emotions, speed, tones, dialects, accents, and +role-playing. Streaming TTFB stays at ~150 ms. + +Two ways to request emotion / style: + +- **XML tags** — `That's my line!`, `…`, + `…`, `…`, `…`, + `…` +- **Instruction prompts** — `You are a helpful assistant. 请用尽可能快地语速说一句话。<|endofprompt|>` + gives finer control via natural-language directives in the + instruction channel + +Language center of gravity is Mandarin + Cantonese (18+ Chinese +dialects) and then 8 other languages (English, Japanese, Korean, +German, Spanish, French, Italian, Russian). English works fine but +don't expect ElevenLabs-grade English prosody polish — ear-test with +your actual content. + +## API endpoints + +| Method + path | Purpose | +|---|---| +| `POST /v1/audio/speech` | OpenAI drop-in for TTS | +| `POST /v1/voices/create` | Clone a speaker from reference audio (auto-transcription via built-in ASR) | +| `GET /v1/voices` | List cloned voices by `voice_id` | +| `GET /health` | Health probe (used by docker healthcheck) | + +## Path layout + +| Host path | Container path | Purpose | Restic? | +|---|---|---|---| +| `/worktank/cosyvoice/voices/` | `/data/voices` | Cloned speaker profiles | **included** (precious — reproducing a clone needs the original reference audio) | +| `/worktank/cosyvoice/input/` | `/data/input` | Scratch for uploaded source audio | excluded | +| `/worktank/cosyvoice/output/` | `/data/output` | Synthesized clips | excluded (regenerable) | + +**Model weights (~2–3 GB) are NOT bind-mounted.** The image places +them at `pretrained_models/Fun-CosyVoice3-0.5B/` inside the container +on first run. Docker's image-layer cache keeps them across normal +`compose up -d` recreates; a `docker image rm` or tag bump re-downloads. + +## First-time deploy on irv-ml1 + +```bash +# 1. Push compose + env template +scripts/deploy-stack.sh irv-ml1 cosyvoice + +# 2. Create the host dirs. One-time sudo — /worktank is root-owned. +ssh -t irv-ml1 'sudo mkdir -p /worktank/cosyvoice/{voices,input,output} && \ + sudo chown -R lkraven:lkraven /worktank/cosyvoice' + +# 3. Pull + up. First `up` downloads ~2–3 GB of model weights inside +# the container; allow a few minutes before the health probe passes. +ssh irv-ml1 ' + cd /opt/docker/compose/cosyvoice && \ + cp -n .env.example .env && \ + docker compose config >/dev/null && \ + docker compose pull && \ + docker compose up -d && \ + docker compose logs -f --tail=30 +' +``` + +## Smoke test + +```bash +# From the workstation over WG — OpenAI-compatible request +curl -X POST http://10.100.79.3:8190/v1/audio/speech \ + -H 'Content-Type: application/json' \ + -d '{"model":"cosyvoice","voice":"default","input":"Hello there.","response_format":"wav"}' \ + -o /tmp/out.wav +``` + +## Version bump + +```bash +# Pick a new tag from https://hub.docker.com/r/neosun/cosyvoice/tags +ssh irv-ml1 ' + cd /opt/docker/compose/cosyvoice && \ + sed -i "s/^COSYVOICE_VERSION=.*/COSYVOICE_VERSION=/" .env && \ + docker compose pull && \ + docker compose up -d +' +``` + +Voices / input / output persist across bumps. Model weights inside +the image re-download on first run of the new tag. diff --git a/stacks/cosyvoice/compose.yaml b/stacks/cosyvoice/compose.yaml new file mode 100644 index 0000000..28d89dd --- /dev/null +++ b/stacks/cosyvoice/compose.yaml @@ -0,0 +1,60 @@ +# CosyVoice — multilingual expressive TTS with voice cloning. +# +# Ships the Fun-CosyVoice3-0.5B-2512 model from FunAudioLLM (latest +# as of 2026-04). Streaming PCM chunks with ~150 ms TTFB. Emotional +# control via either XML tags (text) or instruction +# prompts (`You are a helpful assistant. <|endofprompt|>` syntax). +# +# Runs on irv-ml1 (dual GPU). Both GPUs exposed via +# NVIDIA_VISIBLE_DEVICES=all; image reads CUDA_VISIBLE_DEVICES if +# you later want to pin. +# +# Path split: +# /worktank/cosyvoice/voices → /data/voices cloned speaker profiles +# /worktank/cosyvoice/input → /data/input scratch for uploaded source audio +# /worktank/cosyvoice/output → /data/output synthesized clips +# +# Models (~2–3 GB) download on first run into the image's internal +# pretrained_models/ path. Not bind-mounted (the image expects an +# exact layout we don't have authoritative docs for); recreating the +# container without the cached image will re-download. Cached image +# layer persists through `compose up -d` recreates. +# +# API routes (OpenAI-compatible where marked): +# POST /v1/audio/speech — OpenAI drop-in for TTS +# POST /v1/voices/create — voice cloning (reference audio in) +# GET /v1/voices — list cloned voices +# GET /health — health probe +# +# All tunables live in .env — edit that, not this file. + +services: + cosyvoice: + image: neosun/cosyvoice:${COSYVOICE_VERSION} + container_name: cosyvoice + restart: unless-stopped + runtime: nvidia + ports: + - "${COSYVOICE_BIND:-0.0.0.0}:${COSYVOICE_PORT}:8188" + environment: + - NVIDIA_VISIBLE_DEVICES=all + - MODEL_DIR=${COSYVOICE_MODEL_DIR:-pretrained_models/Fun-CosyVoice3-0.5B} + - PORT=8188 + - GPU_IDLE_TIMEOUT=${COSYVOICE_GPU_IDLE_TIMEOUT:-600} + volumes: + - ${COSYVOICE_VOICES_DIR}:/data/voices + - ${COSYVOICE_INPUT_DIR}:/data/input + - ${COSYVOICE_OUTPUT_DIR}:/data/output + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8188/health >/dev/null || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + # First boot pulls ~2–3 GB of model weights. + start_period: 300s + labels: + - homepage.group=AI Systems + - homepage.name=CosyVoice + - homepage.icon=mdi-account-voice + - homepage.description=Expressive multilingual TTS + cloning (irv-ml1) + - homepage.href=http://10.100.79.3:${COSYVOICE_PORT} diff --git a/stacks/parakeet/.env.example b/stacks/parakeet/.env.example new file mode 100644 index 0000000..8319035 --- /dev/null +++ b/stacks/parakeet/.env.example @@ -0,0 +1,49 @@ +# Parakeet ASR stack tunables. Copy to `.env` on irv-ml1 before deploying. +# +# cp .env.example .env +# # edit as needed +# docker compose build +# docker compose up -d + +# Pinned git SHA to build from. Bump + rebuild when you want upstream +# fixes. `main` latest as of 2026-04: +# https://github.com/Shadowfita/parakeet-tdt-0.6b-v2-fastapi +PARAKEET_SHA=31c5652b62d09653ad5ea8190c0ad0d35394174d + +# Host port for the FastAPI server (container listens on 8000) +PARAKEET_PORT=8765 + +# Bind address. 0.0.0.0 exposes on all interfaces including the WG +# tunnel IP (10.100.79.3). Use 127.0.0.1 to restrict to local-only. +PARAKEET_BIND=0.0.0.0 + +# Host path for the HuggingFace cache (parakeet-tdt-0.6b-v2 weights +# ~2.5 GB). Persistent across container recreates. Must exist before +# first `up` with ownership matching the container user (root inside +# this image — no UID juggling needed, but the host dir needs to be +# writable by the container). +PARAKEET_MODELS_DIR=/worktank/parakeet/models + +# Inference precision. fp16 halves VRAM and is lossless for parakeet +# in practice; use fp32 only if fp16 shows degraded WER for your +# domain audio. +PARAKEET_MODEL_PRECISION=fp16 + +# Batch size for the transcribe queue. Larger = better throughput +# under load at the cost of per-request latency. +PARAKEET_BATCH_SIZE=4 + +# Max single-clip duration (seconds). Longer inputs get rejected +# by the server with 400. Upstream default. +PARAKEET_MAX_AUDIO_DURATION=30 + +# Silero VAD threshold (0–1). Higher = stricter about what counts +# as speech (fewer false wake-ups on silence, more chance of clipping +# soft speech). 0.5 is upstream default. +PARAKEET_VAD_THRESHOLD=0.5 + +# End-to-end processing timeout per request (seconds). +PARAKEET_PROCESSING_TIMEOUT=60 + +# Log level: DEBUG | INFO | WARNING | ERROR +PARAKEET_LOG_LEVEL=INFO diff --git a/stacks/parakeet/README.md b/stacks/parakeet/README.md new file mode 100644 index 0000000..0cbe4da --- /dev/null +++ b/stacks/parakeet/README.md @@ -0,0 +1,88 @@ +# Parakeet ASR + +NVIDIA Parakeet-TDT 0.6B v2 speech-to-text served via a FastAPI +wrapper with Silero VAD and WebSocket streaming. + +**Server:** irv-ml1 (Irvine, WireGuard-only) +**Port:** 8765 (configurable via `.env`) +**GPUs:** both exposed (`NVIDIA_VISIBLE_DEVICES=all`); upstream +respects `CUDA_VISIBLE_DEVICES` if later pinning is needed +**Upstream:** [Shadowfita/parakeet-tdt-0.6b-v2-fastapi](https://github.com/Shadowfita/parakeet-tdt-0.6b-v2-fastapi) +**Image:** built locally from a pinned git SHA via docker buildx's +git URL context — no source vendored into this workspace + +## API endpoints + +| Method + path | Purpose | +|---|---| +| `POST /transcribe` | Batch transcription (multipart file upload) | +| `WS /ws/transcribe` | Streaming with Silero VAD — partial + final segments | +| `GET /healthz` | Health probe (used by docker healthcheck) | + +Not the literal OpenAI `/v1/audio/transcriptions` path. If you have +a downstream client that demands that URL shape, either point its +base URL at `/transcribe`, or add a Traefik/nginx path alias in +front. + +## Path layout + +| Host path | Container path | Purpose | Restic? | +|---|---|---|---| +| `/worktank/parakeet/models/` | `/models` (`HF_HOME`) | HF cache for parakeet-tdt-0.6b-v2 weights (~2.5 GB) | excluded (regenerable from HF) | + +The container runs as root internally; host dir just needs to exist +and be writable. + +## First-time deploy on irv-ml1 + +```bash +# 1. Push compose + env template +scripts/deploy-stack.sh irv-ml1 parakeet + +# 2. Create the models dir on the host. One-time sudo — /worktank +# itself is root-owned. +ssh -t irv-ml1 'sudo mkdir -p /worktank/parakeet/models && \ + sudo chown -R lkraven:lkraven /worktank/parakeet' + +# 3. Build the image (first time only; ~5–10 min for torch + NeMo +# wheels). Then `up`. +ssh irv-ml1 ' + cd /opt/docker/compose/parakeet && \ + cp -n .env.example .env && \ + docker compose config >/dev/null && \ + docker compose build && \ + docker compose up -d && \ + docker compose logs -f --tail=30 +' +``` + +First `/transcribe` request downloads parakeet-tdt-0.6b-v2 weights +to `/worktank/parakeet/models/` (~2.5 GB). + +## Smoke test + +```bash +# From the workstation over WG +curl -F "file=@sample.wav" http://10.100.79.3:8765/transcribe +``` + +## Rebuild against a newer upstream commit + +```bash +ssh irv-ml1 ' + cd /opt/docker/compose/parakeet && \ + sed -i "s/^PARAKEET_SHA=.*/PARAKEET_SHA=/" .env && \ + docker compose build && \ + docker compose up -d +' +``` + +Models dir is unaffected. + +## Deploy updates + +```bash +# After editing compose.yaml or .env.example here +scripts/deploy-stack.sh irv-ml1 parakeet +ssh irv-ml1 'cd /opt/docker/compose/parakeet && docker compose up -d' +``` diff --git a/stacks/parakeet/compose.yaml b/stacks/parakeet/compose.yaml new file mode 100644 index 0000000..9b35fb5 --- /dev/null +++ b/stacks/parakeet/compose.yaml @@ -0,0 +1,67 @@ +# Parakeet ASR — NVIDIA Parakeet-TDT 0.6B v2 speech-to-text. +# +# Wraps Shadowfita/parakeet-tdt-0.6b-v2-fastapi (FastAPI + Silero VAD + +# WebSocket streaming). Upstream provides no prebuilt image, so we +# build from a pinned git commit via docker buildx's git URL context +# — no source files vendored into this repo. +# +# Runs on irv-ml1 (dual GPU). Both GPUs exposed; upstream respects +# CUDA_VISIBLE_DEVICES if you want to pin later. +# +# HF weights (~2.5 GB for parakeet-tdt-0.6b-v2) cache to +# ${PARAKEET_MODELS_DIR} via HF_HOME=/models, persistent across +# container recreates. +# +# API routes (per upstream README): +# POST /transcribe — batch transcription +# WS /ws/transcribe — streaming with Silero VAD +# GET /healthz +# +# Not a literal OpenAI `/v1/audio/transcriptions` path; point clients +# at /transcribe directly, or add a reverse-proxy alias if drop-in +# compat is needed later. +# +# First `up` triggers a fresh docker build (python:3.10-slim + +# torch + NeMo ≈ 5–10 min). Subsequent starts reuse the cached +# image unless PARAKEET_SHA changes. +# +# All tunables live in .env — edit that, not this file. + +services: + parakeet: + image: local/parakeet:${PARAKEET_SHA} + build: + context: https://github.com/Shadowfita/parakeet-tdt-0.6b-v2-fastapi.git#${PARAKEET_SHA} + container_name: parakeet + restart: unless-stopped + runtime: nvidia + ports: + - "${PARAKEET_BIND:-0.0.0.0}:${PARAKEET_PORT}:8000" + environment: + - NVIDIA_VISIBLE_DEVICES=all + - HF_HOME=/models + - DEVICE=cuda + - MODEL_PRECISION=${PARAKEET_MODEL_PRECISION:-fp16} + - BATCH_SIZE=${PARAKEET_BATCH_SIZE:-4} + - TARGET_SAMPLE_RATE=16000 + - MAX_AUDIO_DURATION=${PARAKEET_MAX_AUDIO_DURATION:-30} + - VAD_THRESHOLD=${PARAKEET_VAD_THRESHOLD:-0.5} + - PROCESSING_TIMEOUT=${PARAKEET_PROCESSING_TIMEOUT:-60} + - LOG_LEVEL=${PARAKEET_LOG_LEVEL:-INFO} + volumes: + - ${PARAKEET_MODELS_DIR}:/models + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8000/healthz >/dev/null || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + # First `up` may spend several minutes on torch/NeMo install + # during the image build phase; after the image exists, startup + # is ~30-60s (NeMo model load). + start_period: 180s + labels: + - homepage.group=AI Systems + - homepage.name=Parakeet ASR + - homepage.icon=mdi-microphone + - homepage.description=Parakeet-TDT speech-to-text (irv-ml1) + - homepage.href=http://10.100.79.3:${PARAKEET_PORT}