diff --git a/stacks/fish-cpp/Dockerfile b/stacks/fish-cpp/Dockerfile index 2dfd35b..b1afff3 100644 --- a/stacks/fish-cpp/Dockerfile +++ b/stacks/fish-cpp/Dockerfile @@ -17,7 +17,12 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ git ca-certificates cmake ninja-build build-essential pkg-config \ + libvulkan-dev glslc \ && rm -rf /var/lib/apt/lists/* +# libvulkan-dev = Vulkan headers + loader for build-time linking. +# glslc = GLSL→SPIR-V compiler; GGML's Vulkan backend compiles its +# shaders with this at build time. Without it, the cmake configure +# step disables Vulkan silently. WORKDIR /src RUN git clone --recurse-submodules https://github.com/rodrigomatta/s2.cpp.git \ @@ -38,7 +43,14 @@ WORKDIR /src/s2.cpp RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/lib/libcuda.so \ && ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/lib/libcuda.so.1 \ && ldconfig -RUN cmake -G Ninja -B build -DCMAKE_BUILD_TYPE=Release -DS2_CUDA=ON \ +# Build with BOTH CUDA and Vulkan backends — runtime selects via the +# entrypoint (-c for CUDA, -v for Vulkan). We've established +# that s2.cpp's CUDA path silently falls back to CPU on this model +# (alpha software, incomplete op coverage); Vulkan is the README's +# canonical example so it likely has more complete op coverage on +# fish-speech. Keep both built so we can flip without rebuilding. +RUN cmake -G Ninja -B build -DCMAKE_BUILD_TYPE=Release \ + -DS2_CUDA=ON -DS2_VULKAN=ON \ && cmake --build build --parallel $(nproc) --target s2 # ── Stage 2: runtime — slim image with the binary + python shim ──────── @@ -51,9 +63,13 @@ ENV DEBIAN_FRONTEND=noninteractive \ RUN apt-get update && apt-get install -y --no-install-recommends \ python3 python3-pip python3-venv tini \ libgomp1 curl \ + libvulkan1 \ && rm -rf /var/lib/apt/lists/* -# libgomp1 = GNU OpenMP runtime — required by the s2 binary. -# curl = entrypoint uses it to wait for s2 server to bind :3030. +# libgomp1 = GNU OpenMP runtime — required by the s2 binary. +# curl = entrypoint uses it to wait for s2 server to bind :3030. +# libvulkan1 = Vulkan loader. The NVIDIA Vulkan ICD itself comes from +# the host driver via NVIDIA container runtime when +# NVIDIA_DRIVER_CAPABILITIES includes "graphics". # Pull the shim deps into an isolated venv so we don't fight system pip. RUN python3 -m venv /opt/venv @@ -64,6 +80,7 @@ RUN pip install --no-cache-dir 'fastapi>=0.115' 'uvicorn[standard]>=0.30' 'pydan COPY --from=builder /src/s2.cpp/build/s2 /usr/local/bin/s2 COPY --from=builder /src/s2.cpp/build/ggml/src/libggml*.so /usr/local/lib/ COPY --from=builder /src/s2.cpp/build/ggml/src/ggml-cuda/libggml-cuda.so /usr/local/lib/ +COPY --from=builder /src/s2.cpp/build/ggml/src/ggml-vulkan/libggml-vulkan.so /usr/local/lib/ RUN ldconfig WORKDIR /app diff --git a/stacks/fish-cpp/compose.yaml b/stacks/fish-cpp/compose.yaml index 293a706..94ea93b 100644 --- a/stacks/fish-cpp/compose.yaml +++ b/stacks/fish-cpp/compose.yaml @@ -35,9 +35,18 @@ services: - "${FISH_CPP_BIND:-0.0.0.0}:${FISH_CPP_PORT}:8000" environment: - NVIDIA_VISIBLE_DEVICES=${FISH_CPP_GPU_DEVICES:-1} + # Default nvidia-container-toolkit only mounts compute libs. + # Vulkan needs the graphics ICD too (libGLX_nvidia, vulkan ICD + # JSON). Without this, Vulkan init in the container fails with + # "no Vulkan ICD" even though the GPU is present. + - NVIDIA_DRIVER_CAPABILITIES=compute,utility,graphics - FISH_CPP_MODEL=${FISH_CPP_MODEL:-s2-pro-q6_k.gguf} - FISH_CPP_TOKENIZER=tokenizer.json - FISH_CPP_DEVICE=0 + # Backend selection for the s2 server in entrypoint.sh: + # cuda — -c ; alpha CUDA path, GPU 0% util on fish-speech (broken) + # vulkan — -v ; README's canonical example, more battle-tested + - FISH_CPP_BACKEND=${FISH_CPP_BACKEND:-vulkan} volumes: - ${FISH_CPP_WEIGHTS_DIR}:/weights:ro - ${FISH_CPP_REFERENCE_DIR}:/references:ro diff --git a/stacks/fish-cpp/entrypoint.sh b/stacks/fish-cpp/entrypoint.sh index f9060d8..2a1cb3d 100644 --- a/stacks/fish-cpp/entrypoint.sh +++ b/stacks/fish-cpp/entrypoint.sh @@ -11,18 +11,26 @@ set -euo pipefail MODEL_PATH="${WEIGHTS_DIR:-/weights}/${FISH_CPP_MODEL:-s2-pro-q6_k.gguf}" TOKENIZER_PATH="${WEIGHTS_DIR:-/weights}/${FISH_CPP_TOKENIZER:-tokenizer.json}" DEVICE="${FISH_CPP_DEVICE:-0}" +BACKEND="${FISH_CPP_BACKEND:-vulkan}" + +case "$BACKEND" in + cuda) BACKEND_FLAG=(-c "$DEVICE") ;; + vulkan) BACKEND_FLAG=(-v "$DEVICE") ;; + cpu) BACKEND_FLAG=() ;; # no flag → s2 stays on CPU + *) echo "[entrypoint] unknown FISH_CPP_BACKEND=$BACKEND (use cuda/vulkan/cpu)" >&2; exit 2 ;; +esac if [ ! -f "$MODEL_PATH" ]; then echo "missing model: $MODEL_PATH" >&2; exit 1; fi if [ ! -f "$TOKENIZER_PATH" ]; then echo "missing tokenizer: $TOKENIZER_PATH" >&2; exit 1; fi -# Background s2 server. Loads model on GPU once, then accepts -# multipart POSTs on localhost:3030/generate. -echo "[entrypoint] starting s2 server on :3030 with CUDA device $DEVICE" +# Background s2 server. Loads model on GPU once (per backend), then +# accepts multipart POSTs on localhost:3030/generate. +echo "[entrypoint] starting s2 server on :3030 with $BACKEND backend, device $DEVICE" /usr/local/bin/s2 \ --server -H 127.0.0.1 -P 3030 \ -m "$MODEL_PATH" \ -t "$TOKENIZER_PATH" \ - -c "$DEVICE" & + "${BACKEND_FLAG[@]}" & S2_PID=$! # Wait for s2 to bind 3030 before starting the shim. Avoids the