diff --git a/playbooks/deploy-fish-cpp.yaml b/playbooks/deploy-fish-cpp.yaml new file mode 100644 index 0000000..72832fb --- /dev/null +++ b/playbooks/deploy-fish-cpp.yaml @@ -0,0 +1,130 @@ +# Deploy fish-cpp (Fish s2-pro via s2.cpp + GGML CUDA inference) to irv-ml1. +# +# Builds the image locally — multi-stage CUDA devel base (CMake + s2.cpp +# compile, ~10 min cold) → CUDA runtime base + binary + python shim. +# Pre-pulls rodrigomt/s2-pro-gguf weights (q6_k default, ~5 GB) into +# the bind-mounted weights dir. +# +# Usage: +# scripts/elway irv-ml1 --playbook playbooks/deploy-fish-cpp.yaml +# +# Idempotent — every step is creates-/when-gated; rerun is safe. + +vars: + compose_dir: /opt/docker/compose/fish-cpp + references_dir: /worktank/fish-cpp/references + weights_dir: /worktank/fish-cpp/weights + host_port: "8199" + weights_repo: rodrigomt/s2-pro-gguf + default_quant: s2-pro-q6_k.gguf + +steps: + # ── host-side dirs ────────────────────────────────────────────────── + + - name: Ensure /worktank/fish-cpp root exists (one-time, sudo) + shell: mkdir -p /worktank/fish-cpp + sudo: true + creates: /worktank/fish-cpp + + - name: Chown /worktank/fish-cpp to lkraven + shell: chown -R lkraven:lkraven /worktank/fish-cpp + sudo: true + when: "[ \"$(stat -c %U /worktank/fish-cpp)\" != \"lkraven\" ]" + + - name: Ensure references dir exists + shell: mkdir -p {{ references_dir }} + creates: "{{ references_dir }}" + + - name: Ensure weights dir exists + shell: mkdir -p {{ weights_dir }} + creates: "{{ weights_dir }}" + + - name: Ensure compose dir exists + shell: mkdir -p {{ compose_dir }} + creates: "{{ compose_dir }}" + + # ── deploy build context ──────────────────────────────────────────── + # s2.cpp is built INSIDE the docker image, but the Dockerfile + shim + # need to be present in the compose dir so `docker compose build` + # can find them. + + - name: Upload compose.yaml + upload: + src: stacks/fish-cpp/compose.yaml + dest: "{{ compose_dir }}/compose.yaml" + mode: "0644" + + - name: Upload Dockerfile + upload: + src: stacks/fish-cpp/Dockerfile + dest: "{{ compose_dir }}/Dockerfile" + mode: "0644" + + - name: Upload server.py (FastAPI shim) + upload: + src: stacks/fish-cpp/server.py + dest: "{{ compose_dir }}/server.py" + mode: "0644" + + - name: Seed .env from template (only if absent) + upload: + src: stacks/fish-cpp/.env.example + dest: "{{ compose_dir }}/.env" + mode: "0644" + when: "[ ! -f {{ compose_dir }}/.env ]" + + # ── pre-pull weights ──────────────────────────────────────────────── + # q6_k + tokenizer.json (~5 GB total). Same one-shot + # python:3.12-slim + huggingface_hub.snapshot_download + hf_transfer + # pattern we've used for fish-s2, voxtral, etc. Idempotent on rerun + # via `creates:` on the model file. + + - name: Pre-pull rodrigomt/s2-pro-gguf weights (q6_k + tokenizer, ~5 GB) + shell: | + docker run --rm --user 1000:1000 \ + -e HOME=/tmp/h -e HF_HUB_ENABLE_HF_TRANSFER=1 \ + -v {{ weights_dir }}:/dest \ + python:3.12-slim sh -c 'set -e; mkdir -p /tmp/h /tmp/pip /tmp/site; PIP_CACHE_DIR=/tmp/pip pip install --quiet --target /tmp/site huggingface_hub hf_transfer; PYTHONPATH=/tmp/site python -c "from huggingface_hub import snapshot_download; snapshot_download(repo_id=\"{{ weights_repo }}\", local_dir=\"/dest\", allow_patterns=[\"{{ default_quant }}\",\"tokenizer.json\"])"' + creates: "{{ weights_dir }}/{{ default_quant }}" + + # ── build + bring up ──────────────────────────────────────────────── + + - name: docker compose build (~10 min first time; CUDA toolchain + s2.cpp compile) + shell: | + set -o pipefail + cd {{ compose_dir }} && docker compose build 2>&1 \ + | grep -vE '^#[0-9]+ |^ => |^=> |Collecting|Downloading|Requirement|Using cached|Installing collected|Successfully (installed|built)|━' + + - name: docker compose up -d + shell: cd {{ compose_dir }} && docker compose up -d + + - name: Wait for /v1/health to respond + shell: | + for i in $(seq 1 60); do + curl -sf -o /dev/null --max-time 3 http://localhost:{{ host_port }}/v1/health && exit 0 + sleep 5 + done + exit 1 + changed_when: "false" + +verify: + - name: /v1/health returns 200 + reports model loaded + shell: | + curl -sf http://localhost:{{ host_port }}/v1/health \ + | python3 -c "import json,sys; d=json.load(sys.stdin); assert d.get('status')=='ok' and d.get('model')" + changed_when: "false" + + - name: /v1/tts returns a real WAV (POST with text body) + shell: | + out=$(mktemp --suffix=.wav) + curl -sf -X POST http://localhost:{{ host_port }}/v1/tts \ + -H 'Content-Type: application/json' \ + -d '{"text":"Verify."}' \ + -o "$out" --max-time 60 + file -b "$out" | grep -q '^RIFF.*WAVE' + rm -f "$out" + changed_when: "false" + + - name: Container is running + shell: docker inspect fish-cpp --format '{{.State.Status}}' | grep -q running + changed_when: "false" diff --git a/stacks/fish-cpp/.env.example b/stacks/fish-cpp/.env.example new file mode 100644 index 0000000..ba9ba64 --- /dev/null +++ b/stacks/fish-cpp/.env.example @@ -0,0 +1,42 @@ +# fish-cpp stack tunables. Copy to `.env` on irv-ml1 before deploying. + +# ── build pin ──────────────────────────────────────────────────────── +# SHA of rodrigomatta/s2.cpp to build from. The repo is alpha — pin a +# specific SHA so future upstream churn doesn't break our build. Update +# deliberately when you want upstream improvements. +FISH_CPP_S2_SHA=e48ce8e02d8335bd9a0ba94679f605724b31d123 + +# Local image tag — bump when you change build context. +FISH_CPP_TAG=v1 + +# ── network ────────────────────────────────────────────────────────── +# Host port. Container listens on 8000 internally. +# Reservations on irv-ml1: 8188 ComfyUI, 8190 CosyVoice, 8191 Qwen3-TTS, +# 8192 IndexTTS-2, 8193 Kokoro, 8194 VibeVoice, 8195 fish-s2 (HF), +# 8196 Chatterbox, 8197 Voxtral, 8765 Parakeet ASR. 8199 picked here. +FISH_CPP_PORT=8199 +FISH_CPP_BIND=0.0.0.0 + +# ── runtime / GPU ──────────────────────────────────────────────────── +# GPU pinning. "0" = RTX 3090 (24 GB), "1" = RTX A6000 (48 GB). +# Pinned to GPU 1 to share with fish-s2 (HF) for direct A/B benching; +# q6_k weights need ~5 GB + 3 GB runtime ≈ 8 GB. A6000 has plenty. +FISH_CPP_GPU_DEVICES=1 + +# Quantization variant to load. Available files in +# rodrigomt/s2-pro-gguf: +# s2-pro-q4_k_m.gguf — smaller / faster, quality drops noticeably +# s2-pro-q5_k_m.gguf — middle ground +# s2-pro-q6_k.gguf — sweet spot (default), near-bf16 quality +# s2-pro-q8_0.gguf — closest to bf16, larger / slower +# s2-pro-f16.gguf — full precision, no quantization win +FISH_CPP_MODEL=s2-pro-q6_k.gguf + +# ── persistent storage on the host ─────────────────────────────────── +# Weights — pre-pulled by the deploy playbook (~5 GB for q6_k + +# ~12 MB tokenizer.json). +FISH_CPP_WEIGHTS_DIR=/worktank/fish-cpp/weights + +# Reference audio for voice cloning. Drop ~5-15 s WAV/MP3/FLAC clips +# here; reference them by basename in the API request body. +FISH_CPP_REFERENCE_DIR=/worktank/fish-cpp/references diff --git a/stacks/fish-cpp/Dockerfile b/stacks/fish-cpp/Dockerfile new file mode 100644 index 0000000..9bb88a1 --- /dev/null +++ b/stacks/fish-cpp/Dockerfile @@ -0,0 +1,63 @@ +# fish-cpp — s2.cpp (pure C++/GGML inference for Fish s2-pro GGUFs) + +# tiny FastAPI shim exposing Fish's /v1/tts contract. +# +# Two-stage build: +# 1. builder — compiles s2.cpp with CUDA backend +# 2. runtime — slim image with the s2 binary + Python shim +# +# The s2.cpp binary is the actual inference engine; the Python shim is +# just an HTTP-to-CLI bridge so this stack drops into the same fleet +# pattern as the other TTS (POST /v1/tts, Fish-shaped request body). + +# ── Stage 1: build s2.cpp with CUDA ──────────────────────────────────── +FROM nvidia/cuda:12.6.0-devel-ubuntu24.04 AS builder + +ARG S2_CPP_SHA=e48ce8e02d8335bd9a0ba94679f605724b31d123 +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git ca-certificates cmake ninja-build build-essential pkg-config \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /src +RUN git clone --recurse-submodules https://github.com/rodrigomatta/s2.cpp.git \ + && cd s2.cpp \ + && git checkout ${S2_CPP_SHA} \ + && git submodule update --init --recursive + +WORKDIR /src/s2.cpp +RUN cmake -G Ninja -B build -DCMAKE_BUILD_TYPE=Release -DS2_CUDA=ON \ + && cmake --build build --parallel $(nproc) --target s2 + +# ── Stage 2: runtime — slim image with the binary + python shim ──────── +FROM nvidia/cuda:12.6.0-runtime-ubuntu24.04 + +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 python3-pip python3-venv tini \ + && rm -rf /var/lib/apt/lists/* + +# Pull the shim deps into an isolated venv so we don't fight system pip. +RUN python3 -m venv /opt/venv +ENV PATH="/opt/venv/bin:${PATH}" +RUN pip install --no-cache-dir 'fastapi>=0.115' 'uvicorn[standard]>=0.30' 'pydantic>=2' + +# Copy the s2 binary + GGML runtime libs from the builder stage. +COPY --from=builder /src/s2.cpp/build/s2 /usr/local/bin/s2 +COPY --from=builder /src/s2.cpp/build/ggml/src/libggml*.so /usr/local/lib/ +COPY --from=builder /src/s2.cpp/build/ggml/src/ggml-cuda/libggml-cuda.so /usr/local/lib/ +RUN ldconfig + +WORKDIR /app +COPY server.py /app/server.py + +# Bind-mounted at runtime: weights at /weights, references at /references. +VOLUME /weights +VOLUME /references + +EXPOSE 8000 +ENTRYPOINT ["/usr/bin/tini", "--"] +CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8000", "--no-access-log"] diff --git a/stacks/fish-cpp/README.md b/stacks/fish-cpp/README.md new file mode 100644 index 0000000..69b90ad --- /dev/null +++ b/stacks/fish-cpp/README.md @@ -0,0 +1,108 @@ +# fish-cpp + +Fish s2-pro served via [s2.cpp](https://github.com/rodrigomatta/s2.cpp) +— a pure C++/GGML inference engine for Fish s2-pro, with weights from +[rodrigomt/s2-pro-gguf](https://huggingface.co/rodrigomt/s2-pro-gguf). +Wrapped by a tiny FastAPI shim exposing Fish's `/v1/tts` HTTP contract +so it slots into the same bench harness + client patterns as `fish-s2`. + +## Why this stack alongside `fish-s2` + +`fish-s2` (HF transformers wrapper) measured at **0.78× realtime** on +the A6000 — phenomenal quality but sub-realtime, meaning streaming +clients hit buffer underruns on phrases longer than ~3-4 seconds of +audio. `fish-cpp` targets the same s2-pro architecture but runs it +through s2.cpp's C++/GGML/CUDA inference path with q6_k quantization +— typically 2-5× faster than HF transformers for equivalent precision +(GGML is what makes llama.cpp fast). + +**Goal**: hit ≥ 1× realtime on the A6000 so streaming actually flows +without stutters, while keeping Fish's quality near-equivalent to BF16. + +## Status: alpha + +s2.cpp is alpha software per its upstream README. Pin the SHA in +`.env`, don't track main blindly — community alpha projects break +weekly. + +## What works (and what doesn't) vs `fish-s2` + +| feature | fish-s2 (HF) | fish-cpp (this) | +|---|---|---| +| `/v1/tts` POST endpoint | ✓ | ✓ | +| `text` body field | ✓ | ✓ | +| `references` body field (cloning) | ✓ | ✓ (single ref only) | +| `streaming: true` | ✓ (TTFB → 26 ms) | ✗ accepted but ignored | +| Paralinguistic tags | ✓ | should work (same weights) | +| Quantization | bf16 | q4_k_m / q5_k_m / **q6_k** (default) / q8_0 | +| Realtime factor | 0.78× | targeting 1-1.5× | + +The streaming gap matters: fish-s2 with `streaming: true` returns the +first audio chunk in 26 ms (perceived latency feels instant). fish-cpp +returns nothing until generation completes. So fish-cpp's appeal is +RAW THROUGHPUT, not perceived latency. Combined with realtime+ +generation, total wall time stays low enough that polling clients +don't notice. + +## API + +OpenAPI shape mirrors `fish-s2`: + +```bash +# Basic — text only, default voice +curl -fsS -X POST http://10.100.79.3:8199/v1/tts \ + -H 'Content-Type: application/json' \ + -d '{"text":"Hello there."}' \ + > out.wav + +# With voice cloning — base64-encoded reference audio inline +B64=$(base64 -w 0 /worktank/fish-cpp/references/glados.wav) # on irv-ml1 +echo "{\"text\":\"Welcome.\",\"references\":[{\"audio\":\"$B64\",\"text\":\"transcript\"}]}" \ + | curl -fsS -X POST http://10.100.79.3:8199/v1/tts \ + -H 'Content-Type: application/json' --data-binary @- \ + > out.wav +``` + +Health probe at `GET /v1/health`. + +## Deploy + +```bash +scripts/elway irv-ml1 --playbook playbooks/deploy-fish-cpp.yaml +``` + +Cold deploy ~30-45 min: ~10 min image build (CUDA dev toolchain + +CMake + s2.cpp compile), ~3 min weights pull (~5 GB for q6_k + +12 MB tokenizer), ~1 min container boot. + +## Hardware footprint + +- **VRAM**: ~8 GB practical for q6_k (5 GB weights + 3 GB runtime). + Pinned to GPU 1 (A6000) by default to share with `fish-s2` for + direct A/B comparison. Could also run on GPU 0 (3090) with room + to spare. +- **Disk**: ~5 GB for q6_k checkpoint + tokenizer. + +## Bench plan + +Same 3-phrase suite as the other TTS: + +``` +P1 = "Hello, this is a test of the voice synthesis system. The quick brown fox jumps over the lazy dog." +P2 = "Oh my god, I cannot believe what just happened. That was absolutely incredible!" +P3 = "What the hell is going on. This is some bullshit and I am not putting up with it." +``` + +Compare: +- TTFB / total wall-clock per phrase +- audio-seconds / wall-seconds (realtime factor) +- Quality (ear test) vs `fish-s2` BF16 baseline + +If fish-cpp lands ≥ 1× realtime AND the q6_k quality holds up under +ear test, this stack becomes the default Fish path. fish-s2 stays +deployed for paralinguistic tag fidelity reference + streaming +(if that turns out to matter for any specific use case). + +## Lessons learned + +(Populate after deploy iteration.) diff --git a/stacks/fish-cpp/compose.yaml b/stacks/fish-cpp/compose.yaml new file mode 100644 index 0000000..293a706 --- /dev/null +++ b/stacks/fish-cpp/compose.yaml @@ -0,0 +1,55 @@ +# fish-cpp — Fish s2-pro served via s2.cpp (pure C++/GGML inference, +# CUDA backend) with a tiny FastAPI shim exposing Fish's /v1/tts +# contract. Built locally from rodrigomatta/s2.cpp + a pinned SHA. +# +# Why this stack alongside fish-s2: +# * fish-s2 (HF transformers wrapper): ~7-8 s TTFB, 0.78× realtime. +# Phenomenal quality but sub-realtime, buffer-underruns on long +# phrases even with streaming. +# * fish-cpp (this): targets ~2-5× speedup from the GGML inference +# path + q6_k quantization. Goal: hit realtime for streaming use. +# +# CAVEATS: +# * s2.cpp is alpha software (per upstream README). Pin the SHA; +# don't track main blindly. +# * Per-request subprocess spawn — each /v1/tts call forks the s2 +# binary. Adds ~50-100 ms over a long-running daemon. Negligible +# vs the multi-second generation cost. +# * No streaming — s2.cpp writes a complete WAV before returning, +# so the shim's `streaming: true` field is accepted but ignored. +# TTFB ≈ total wall time (fast generation is the only path to +# low-latency here, not chunked output). + +services: + fish-cpp: + image: local/fish-cpp:${FISH_CPP_TAG} + build: + context: . + dockerfile: Dockerfile + args: + S2_CPP_SHA: ${FISH_CPP_S2_SHA:-e48ce8e02d8335bd9a0ba94679f605724b31d123} + container_name: fish-cpp + restart: unless-stopped + runtime: nvidia + ports: + - "${FISH_CPP_BIND:-0.0.0.0}:${FISH_CPP_PORT}:8000" + environment: + - NVIDIA_VISIBLE_DEVICES=${FISH_CPP_GPU_DEVICES:-1} + - FISH_CPP_MODEL=${FISH_CPP_MODEL:-s2-pro-q6_k.gguf} + - FISH_CPP_TOKENIZER=tokenizer.json + - FISH_CPP_DEVICE=0 + volumes: + - ${FISH_CPP_WEIGHTS_DIR}:/weights:ro + - ${FISH_CPP_REFERENCE_DIR}:/references:ro + healthcheck: + test: ["CMD-SHELL", "python3 -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8000/v1/health', timeout=5).status==200 else 1)\""] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + labels: + - homepage.group=AI Systems + - homepage.name=Fish (s2.cpp) + - homepage.icon=mdi-fish + - homepage.description=Fish s2-pro via s2.cpp/GGML — quantized for realtime (irv-ml1) + - homepage.href=http://10.100.79.3:${FISH_CPP_PORT} diff --git a/stacks/fish-cpp/server.py b/stacks/fish-cpp/server.py new file mode 100644 index 0000000..324bf88 --- /dev/null +++ b/stacks/fish-cpp/server.py @@ -0,0 +1,131 @@ +"""fish-cpp — HTTP shim wrapping the s2.cpp CLI in Fish's /v1/tts API contract. + +s2.cpp is a pure C++/GGML inference engine for Fish s2-pro GGUFs. It's +CLI-only (alpha), so this shim turns each HTTP POST into a subprocess +invocation that: + + 1. Writes the optional reference WAV to /tmp/.wav + 2. Runs: s2 -m -t + [-pa /tmp/ref.wav -pt ""] + -text "..." -o /tmp/.out.wav + 3. Streams the resulting WAV back to the client. + +The contract matches Fish's wrapper close enough that bench harnesses +written for fish-s2 work against fish-cpp without changes — same +POST /v1/tts, same {text, references[{audio,text}], format} request body. + +What's NOT supported (vs Fish HF): + * `streaming: true` — s2.cpp writes a complete WAV before returning. + The shim accepts the field but ignores it. TTFB ≈ total wall time. + If/when s2.cpp grows incremental output, wire it through here. + * Paralinguistic tags — s2.cpp uses the same model weights, so tags + SHOULD work, but quality may differ from the HF wrapper. +""" + +from __future__ import annotations + +import base64 +import os +import subprocess +import tempfile +import uuid +from pathlib import Path +from typing import Optional + +from fastapi import FastAPI, HTTPException +from fastapi.responses import FileResponse +from pydantic import BaseModel + +S2_BINARY = os.environ.get("S2_BINARY", "/usr/local/bin/s2") +WEIGHTS_DIR = Path(os.environ.get("WEIGHTS_DIR", "/weights")) +MODEL_FILE = os.environ.get("FISH_CPP_MODEL", "s2-pro-q6_k.gguf") +TOKENIZER_FILE = os.environ.get("FISH_CPP_TOKENIZER", "tokenizer.json") +DEVICE_INDEX = os.environ.get("FISH_CPP_DEVICE", "0") # CUDA device id + +MODEL_PATH = WEIGHTS_DIR / MODEL_FILE +TOKENIZER_PATH = WEIGHTS_DIR / TOKENIZER_FILE + +app = FastAPI(title="fish-cpp") + + +class ReferenceAudio(BaseModel): + audio: str # base64-encoded WAV bytes (matches Fish HF schema) + text: str # transcript of the reference clip + + +class TTSRequest(BaseModel): + text: str + references: list[ReferenceAudio] = [] + format: str = "wav" + streaming: bool = False # accepted for contract parity; ignored + + +@app.get("/v1/health") +def health() -> dict: + if not MODEL_PATH.exists(): + raise HTTPException(503, f"model missing: {MODEL_PATH}") + if not TOKENIZER_PATH.exists(): + raise HTTPException(503, f"tokenizer missing: {TOKENIZER_PATH}") + return {"status": "ok", "model": MODEL_FILE} + + +@app.post("/v1/tts") +def tts(req: TTSRequest) -> FileResponse: + if req.format not in ("wav",): + raise HTTPException(400, f"only wav format supported, got {req.format}") + + job = uuid.uuid4().hex + out_path = Path(tempfile.gettempdir()) / f"fish-cpp-{job}.wav" + ref_path: Optional[Path] = None + + cmd: list[str] = [ + S2_BINARY, + "-m", str(MODEL_PATH), + "-t", str(TOKENIZER_PATH), + "-text", req.text, + "-o", str(out_path), + # -v selects voice mode; 0 = default (no preset). Preset voices + # would need s2.cpp to ship a voice library; clone via -pa/-pt. + "-v", "0", + ] + + if req.references: + # s2.cpp accepts a single reference at a time via -pa/-pt. If the + # client sent multiple, use the first; the upstream API doesn't + # appear to support multi-reference cloning. + ref = req.references[0] + ref_path = Path(tempfile.gettempdir()) / f"fish-cpp-{job}-ref.wav" + ref_path.write_bytes(base64.b64decode(ref.audio)) + cmd.extend(["-pa", str(ref_path), "-pt", ref.text]) + + env = os.environ.copy() + env["CUDA_VISIBLE_DEVICES"] = DEVICE_INDEX + + try: + result = subprocess.run( + cmd, env=env, capture_output=True, text=True, timeout=180, + ) + except subprocess.TimeoutExpired: + raise HTTPException(504, "s2 binary timed out (180s)") + finally: + if ref_path and ref_path.exists(): + ref_path.unlink() + + if result.returncode != 0: + raise HTTPException( + 500, + f"s2 binary failed (rc={result.returncode}): {result.stderr[-500:]}", + ) + + if not out_path.exists() or out_path.stat().st_size == 0: + raise HTTPException(500, "s2 binary produced no output") + + # FileResponse streams the file + cleans up after sending. Caller + # gets WAV bytes immediately; we reap the temp file via background + # task once the response finishes. + from fastapi import BackgroundTasks + bg = BackgroundTasks() + bg.add_task(out_path.unlink, missing_ok=True) + return FileResponse( + out_path, media_type="audio/wav", filename=f"{job}.wav", background=bg, + )