diff --git a/playbooks/deploy-fish-cpp.yaml b/playbooks/deploy-fish-cpp.yaml index 8c50aaf..cf9613a 100644 --- a/playbooks/deploy-fish-cpp.yaml +++ b/playbooks/deploy-fish-cpp.yaml @@ -66,6 +66,12 @@ steps: dest: "{{ compose_dir }}/server.py" mode: "0644" + - name: Upload entrypoint.sh (starts s2 server + uvicorn shim) + upload: + src: stacks/fish-cpp/entrypoint.sh + dest: "{{ compose_dir }}/entrypoint.sh" + mode: "0755" + - name: Seed .env from template (only if absent) upload: src: stacks/fish-cpp/.env.example diff --git a/stacks/fish-cpp/Dockerfile b/stacks/fish-cpp/Dockerfile index 9ba714d..2dfd35b 100644 --- a/stacks/fish-cpp/Dockerfile +++ b/stacks/fish-cpp/Dockerfile @@ -50,18 +50,15 @@ ENV DEBIAN_FRONTEND=noninteractive \ RUN apt-get update && apt-get install -y --no-install-recommends \ python3 python3-pip python3-venv tini \ - libgomp1 \ + libgomp1 curl \ && rm -rf /var/lib/apt/lists/* -# libgomp1 = GNU OpenMP runtime — required by the s2 binary at runtime. -# CMake auto-enabled OpenMP at build time (gcc has -fopenmp), so the -# binary dynamically links libgomp.so.1; the slim cuda:runtime base -# doesn't include it by default. Without this, s2 fails on first -# invocation with "error while loading shared libraries: libgomp.so.1". +# libgomp1 = GNU OpenMP runtime — required by the s2 binary. +# curl = entrypoint uses it to wait for s2 server to bind :3030. # Pull the shim deps into an isolated venv so we don't fight system pip. RUN python3 -m venv /opt/venv ENV PATH="/opt/venv/bin:${PATH}" -RUN pip install --no-cache-dir 'fastapi>=0.115' 'uvicorn[standard]>=0.30' 'pydantic>=2' +RUN pip install --no-cache-dir 'fastapi>=0.115' 'uvicorn[standard]>=0.30' 'pydantic>=2' 'httpx>=0.27' # Copy the s2 binary + GGML runtime libs from the builder stage. COPY --from=builder /src/s2.cpp/build/s2 /usr/local/bin/s2 @@ -71,11 +68,16 @@ RUN ldconfig WORKDIR /app COPY server.py /app/server.py +COPY entrypoint.sh /app/entrypoint.sh +RUN chmod +x /app/entrypoint.sh # Bind-mounted at runtime: weights at /weights, references at /references. VOLUME /weights VOLUME /references EXPOSE 8000 +# tini supervises the entrypoint script which manages both s2 server +# (bound to localhost:3030, model resident) and the uvicorn shim +# (bound to 0.0.0.0:8000, proxies /v1/tts → s2's /generate). ENTRYPOINT ["/usr/bin/tini", "--"] -CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8000", "--no-access-log"] +CMD ["/app/entrypoint.sh"] diff --git a/stacks/fish-cpp/entrypoint.sh b/stacks/fish-cpp/entrypoint.sh new file mode 100644 index 0000000..f9060d8 --- /dev/null +++ b/stacks/fish-cpp/entrypoint.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# fish-cpp entrypoint — start s2 server + uvicorn shim. +# +# s2 server holds the model resident on the GPU; uvicorn proxies +# Fish-shaped /v1/tts JSON requests to s2's multipart /generate. +# Both processes share the container; tini supervises both via +# `wait` after backgrounding s2. + +set -euo pipefail + +MODEL_PATH="${WEIGHTS_DIR:-/weights}/${FISH_CPP_MODEL:-s2-pro-q6_k.gguf}" +TOKENIZER_PATH="${WEIGHTS_DIR:-/weights}/${FISH_CPP_TOKENIZER:-tokenizer.json}" +DEVICE="${FISH_CPP_DEVICE:-0}" + +if [ ! -f "$MODEL_PATH" ]; then echo "missing model: $MODEL_PATH" >&2; exit 1; fi +if [ ! -f "$TOKENIZER_PATH" ]; then echo "missing tokenizer: $TOKENIZER_PATH" >&2; exit 1; fi + +# Background s2 server. Loads model on GPU once, then accepts +# multipart POSTs on localhost:3030/generate. +echo "[entrypoint] starting s2 server on :3030 with CUDA device $DEVICE" +/usr/local/bin/s2 \ + --server -H 127.0.0.1 -P 3030 \ + -m "$MODEL_PATH" \ + -t "$TOKENIZER_PATH" \ + -c "$DEVICE" & +S2_PID=$! + +# Wait for s2 to bind 3030 before starting the shim. Avoids the +# obvious /v1/health 502 race on first boot. +echo "[entrypoint] waiting for s2 server to bind :3030" +for i in $(seq 1 60); do + if curl -sf -o /dev/null --max-time 1 http://127.0.0.1:3030/ 2>/dev/null; then + echo "[entrypoint] s2 server up after ${i}s" + break + fi + if ! kill -0 "$S2_PID" 2>/dev/null; then + echo "[entrypoint] s2 server died during startup" >&2 + exit 1 + fi + sleep 1 +done + +# Foreground uvicorn. tini (PID 1) gets uvicorn signals; on SIGTERM +# uvicorn exits, then s2 gets reaped via `wait` below. +echo "[entrypoint] starting uvicorn shim on :8000" +uvicorn server:app --host 0.0.0.0 --port 8000 --no-access-log & +UV_PID=$! + +# Block until either child exits; propagate exit code. +wait -n "$S2_PID" "$UV_PID" +EXIT=$? +echo "[entrypoint] one child exited (rc=$EXIT) — shutting down peer" +kill "$S2_PID" "$UV_PID" 2>/dev/null || true +wait || true +exit "$EXIT" diff --git a/stacks/fish-cpp/server.py b/stacks/fish-cpp/server.py index a55cbf4..bc2b7cd 100644 --- a/stacks/fish-cpp/server.py +++ b/stacks/fish-cpp/server.py @@ -1,46 +1,40 @@ -"""fish-cpp — HTTP shim wrapping the s2.cpp CLI in Fish's /v1/tts API contract. +"""fish-cpp — HTTP shim wrapping s2.cpp's built-in server in Fish's +/v1/tts API contract. -s2.cpp is a pure C++/GGML inference engine for Fish s2-pro GGUFs. It's -CLI-only (alpha), so this shim turns each HTTP POST into a subprocess -invocation that: +Architecture: an `s2 --server` instance runs IN THE SAME CONTAINER on +localhost:3030 with the model loaded once on the GPU; this FastAPI +shim translates each Fish-shape JSON POST into a multipart form POST +against localhost:3030/generate and streams the WAV back. - 1. Writes the optional reference WAV to /tmp/.wav - 2. Runs: s2 -m -t - [-pa /tmp/ref.wav -pt ""] - -text "..." -o /tmp/.out.wav - 3. Streams the resulting WAV back to the client. +The earlier subprocess-per-request design paid CUDA init + model load +(~10-20s) on every call, which dominated wall time. Keeping s2 server +resident moves the cost to startup (paid once) so per-request latency +matches what the model can actually do. -The contract matches Fish's wrapper close enough that bench harnesses -written for fish-s2 work against fish-cpp without changes — same -POST /v1/tts, same {text, references[{audio,text}], format} request body. - -What's NOT supported (vs Fish HF): - * `streaming: true` — s2.cpp writes a complete WAV before returning. - The shim accepts the field but ignores it. TTFB ≈ total wall time. - If/when s2.cpp grows incremental output, wire it through here. - * Paralinguistic tags — s2.cpp uses the same model weights, so tags - SHOULD work, but quality may differ from the HF wrapper. +The shim still accepts (and ignores) `streaming: true` in the request +body for contract parity with fish-s2 — s2.cpp's /generate is +synchronous, so TTFB ≈ total wall time. If/when s2.cpp grows +incremental output, wire it through here. """ from __future__ import annotations import base64 +import io import os -import subprocess import tempfile -import uuid from pathlib import Path from typing import Optional +import httpx from fastapi import FastAPI, HTTPException -from fastapi.responses import FileResponse +from fastapi.responses import Response from pydantic import BaseModel -S2_BINARY = os.environ.get("S2_BINARY", "/usr/local/bin/s2") +S2_BASE_URL = os.environ.get("S2_BASE_URL", "http://127.0.0.1:3030") WEIGHTS_DIR = Path(os.environ.get("WEIGHTS_DIR", "/weights")) MODEL_FILE = os.environ.get("FISH_CPP_MODEL", "s2-pro-q6_k.gguf") TOKENIZER_FILE = os.environ.get("FISH_CPP_TOKENIZER", "tokenizer.json") -DEVICE_INDEX = os.environ.get("FISH_CPP_DEVICE", "0") # CUDA device id MODEL_PATH = WEIGHTS_DIR / MODEL_FILE TOKENIZER_PATH = WEIGHTS_DIR / TOKENIZER_FILE @@ -49,7 +43,7 @@ app = FastAPI(title="fish-cpp") class ReferenceAudio(BaseModel): - audio: str # base64-encoded WAV bytes (matches Fish HF schema) + audio: str # base64-encoded WAV bytes text: str # transcript of the reference clip @@ -57,7 +51,7 @@ class TTSRequest(BaseModel): text: str references: list[ReferenceAudio] = [] format: str = "wav" - streaming: bool = False # accepted for contract parity; ignored + streaming: bool = False # accepted for parity; ignored @app.get("/v1/health") @@ -66,70 +60,46 @@ def health() -> dict: raise HTTPException(503, f"model missing: {MODEL_PATH}") if not TOKENIZER_PATH.exists(): raise HTTPException(503, f"tokenizer missing: {TOKENIZER_PATH}") - return {"status": "ok", "model": MODEL_FILE} + # Probe the s2 server backend — readiness, not just liveness. + try: + r = httpx.get(f"{S2_BASE_URL}/", timeout=2.0) + except httpx.HTTPError as exc: + raise HTTPException(503, f"s2 server unreachable: {exc}") + return {"status": "ok", "model": MODEL_FILE, "s2_status": r.status_code} @app.post("/v1/tts") -def tts(req: TTSRequest) -> FileResponse: +def tts(req: TTSRequest) -> Response: if req.format not in ("wav",): raise HTTPException(400, f"only wav format supported, got {req.format}") - job = uuid.uuid4().hex - out_path = Path(tempfile.gettempdir()) / f"fish-cpp-{job}.wav" - ref_path: Optional[Path] = None - - cmd: list[str] = [ - S2_BINARY, - "-m", str(MODEL_PATH), - "-t", str(TOKENIZER_PATH), - "-text", req.text, - "-o", str(out_path), - # -c selects CUDA backend on the given device id. The - # README example uses `-v 0` (which is --vulkan 0 — easy to - # misread as "voice 0"); we want CUDA so that the work hits - # the A6000 tensor cores instead of falling back to CPU. - # Without a backend flag, s2 prints "NPU not compiled, falling - # back to CPU" and runs at single-digit RTF on a 4-core CPU. - "-c", "0", - ] - + # Translate Fish-shaped JSON to s2.cpp's multipart form. + files: dict[str, tuple] = { + "text": (None, req.text), + } if req.references: - # s2.cpp accepts a single reference at a time via -pa/-pt. If the - # client sent multiple, use the first; the upstream API doesn't - # appear to support multi-reference cloning. + # s2.cpp accepts a single reference at a time. If the client + # sent multiple, use the first. ref = req.references[0] - ref_path = Path(tempfile.gettempdir()) / f"fish-cpp-{job}-ref.wav" - ref_path.write_bytes(base64.b64decode(ref.audio)) - cmd.extend(["-pa", str(ref_path), "-pt", ref.text]) - - env = os.environ.copy() - env["CUDA_VISIBLE_DEVICES"] = DEVICE_INDEX + files["prompt_text"] = (None, ref.text) + files["prompt_audio"] = ("ref.wav", base64.b64decode(ref.audio), "audio/wav") try: - result = subprocess.run( - cmd, env=env, capture_output=True, text=True, timeout=180, + # Long timeout — s2 generation is sub-realtime on q6_k. The + # /v1/health probe upstream already vetted s2 server is live. + r = httpx.post( + f"{S2_BASE_URL}/generate", files=files, timeout=180.0, ) - except subprocess.TimeoutExpired: - raise HTTPException(504, "s2 binary timed out (180s)") - finally: - if ref_path and ref_path.exists(): - ref_path.unlink() + except httpx.HTTPError as exc: + raise HTTPException(502, f"s2 server request failed: {exc}") - if result.returncode != 0: + if r.status_code != 200: + # s2 returns JSON `{"error": "..."}` on failure raise HTTPException( - 500, - f"s2 binary failed (rc={result.returncode}): {result.stderr[-500:]}", + 500, f"s2 server returned {r.status_code}: {r.text[:500]}" ) - if not out_path.exists() or out_path.stat().st_size == 0: - raise HTTPException(500, "s2 binary produced no output") - - # FileResponse streams the file + cleans up after sending. Caller - # gets WAV bytes immediately; we reap the temp file via background - # task once the response finishes. - from fastapi import BackgroundTasks - bg = BackgroundTasks() - bg.add_task(out_path.unlink, missing_ok=True) - return FileResponse( - out_path, media_type="audio/wav", filename=f"{job}.wav", background=bg, + return Response( + content=r.content, + media_type=r.headers.get("content-type", "audio/wav"), )