vh
14f052461e
New stack scaffolding for the Fish quantized-realtime experiment. Not deployed yet — this commit lands the canonical files; deploy follows. Architecture decisions made in Phase 1: * CUDA backend, NOT Vulkan. s2.cpp's CMakeLists exposes both -DS2_VULKAN and -DS2_CUDA; the most recent upstream commit (2026-04-12) was specifically about CUDA improvements, and CUDA on the A6000 will be substantially faster than Vulkan for ML matmul. -DS2_CUDA=ON in the Dockerfile build args. * Pinned to s2.cpp commit e48ce8e02d8335bd9a0ba94679f605724b31d12 (2026-04-12 HEAD of main). Repo is alpha software per README; pin tightly so future churn doesn't break our build. Bump deliberately when wanting upstream improvements. * Multi-stage Dockerfile: nvidia/cuda:12.6.0-devel for build (needs CMake + ninja + git + the CUDA toolchain) → nvidia/cuda:12.6.0-runtime for serve (slimmer; just the s2 binary + GGML libs + a small Python shim). Cuts image size by ~50% vs single-stage devel. * FastAPI shim (server.py) wraps s2.cpp CLI in Fish's `/v1/tts` contract so the same bench harness + clients work against fish-cpp with no changes. Per-request flow: decode optional reference WAV from base64 → write to temp → subprocess.run the s2 binary → stream resulting WAV back. Adds ~50-100ms per-request fork+exec overhead; negligible vs the multi-second generation cost. * `streaming: true` accepted in request body but IGNORED — s2.cpp writes a complete WAV before returning, so chunked output isn't available. Unlike fish-s2 (HF wrapper) where streaming drops TTFB to 26ms, fish-cpp's TTFB ≈ total wall time. Speed depends entirely on raw generation throughput. * q6_k as default quant — sweet spot per typical GGUF guidance: near-bf16 quality at ~5GB. Other variants (q4_k_m, q5_k_m, q8_0, f16) selectable via FISH_CPP_MODEL env. * Pinned to GPU 1 (A6000) by default to share with fish-s2 for direct A/B benching. q6_k weights ~5GB + runtime ~3GB ≈ 8GB — comfortable on either GPU. * Port 8199 (next free in the irv-ml1 TTS slate). Phase 2 (next) is the actual deploy + first build. Reserved 30-45 min for cold-cache build + weights pull.
64 lines
2.4 KiB
Docker
64 lines
2.4 KiB
Docker
# fish-cpp — s2.cpp (pure C++/GGML inference for Fish s2-pro GGUFs) +
|
|
# tiny FastAPI shim exposing Fish's /v1/tts contract.
|
|
#
|
|
# Two-stage build:
|
|
# 1. builder — compiles s2.cpp with CUDA backend
|
|
# 2. runtime — slim image with the s2 binary + Python shim
|
|
#
|
|
# The s2.cpp binary is the actual inference engine; the Python shim is
|
|
# just an HTTP-to-CLI bridge so this stack drops into the same fleet
|
|
# pattern as the other TTS (POST /v1/tts, Fish-shaped request body).
|
|
|
|
# ── Stage 1: build s2.cpp with CUDA ────────────────────────────────────
|
|
FROM nvidia/cuda:12.6.0-devel-ubuntu24.04 AS builder
|
|
|
|
ARG S2_CPP_SHA=e48ce8e02d8335bd9a0ba94679f605724b31d123
|
|
ENV DEBIAN_FRONTEND=noninteractive
|
|
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
git ca-certificates cmake ninja-build build-essential pkg-config \
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
|
|
WORKDIR /src
|
|
RUN git clone --recurse-submodules https://github.com/rodrigomatta/s2.cpp.git \
|
|
&& cd s2.cpp \
|
|
&& git checkout ${S2_CPP_SHA} \
|
|
&& git submodule update --init --recursive
|
|
|
|
WORKDIR /src/s2.cpp
|
|
RUN cmake -G Ninja -B build -DCMAKE_BUILD_TYPE=Release -DS2_CUDA=ON \
|
|
&& cmake --build build --parallel $(nproc) --target s2
|
|
|
|
# ── Stage 2: runtime — slim image with the binary + python shim ────────
|
|
FROM nvidia/cuda:12.6.0-runtime-ubuntu24.04
|
|
|
|
ENV DEBIAN_FRONTEND=noninteractive \
|
|
PYTHONUNBUFFERED=1 \
|
|
PYTHONDONTWRITEBYTECODE=1
|
|
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
python3 python3-pip python3-venv tini \
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
|
|
# Pull the shim deps into an isolated venv so we don't fight system pip.
|
|
RUN python3 -m venv /opt/venv
|
|
ENV PATH="/opt/venv/bin:${PATH}"
|
|
RUN pip install --no-cache-dir 'fastapi>=0.115' 'uvicorn[standard]>=0.30' 'pydantic>=2'
|
|
|
|
# Copy the s2 binary + GGML runtime libs from the builder stage.
|
|
COPY --from=builder /src/s2.cpp/build/s2 /usr/local/bin/s2
|
|
COPY --from=builder /src/s2.cpp/build/ggml/src/libggml*.so /usr/local/lib/
|
|
COPY --from=builder /src/s2.cpp/build/ggml/src/ggml-cuda/libggml-cuda.so /usr/local/lib/
|
|
RUN ldconfig
|
|
|
|
WORKDIR /app
|
|
COPY server.py /app/server.py
|
|
|
|
# Bind-mounted at runtime: weights at /weights, references at /references.
|
|
VOLUME /weights
|
|
VOLUME /references
|
|
|
|
EXPOSE 8000
|
|
ENTRYPOINT ["/usr/bin/tini", "--"]
|
|
CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8000", "--no-access-log"]
|