vh
67813bbef4
CUDA backend confirmed broken for fish-speech ops on s2.cpp v0.x — alpha, incomplete op coverage, GPU stays at 0% during generation despite ggml_cuda_init succeeding. Vulkan was the original README example (`-v 0`), so likely the more battle-tested path. Build the image with BOTH backends so we can flip via env without rebuilding: * libvulkan-dev + glslc in the build stage (GGML's Vulkan backend compiles its shaders with glslc at build time; without it the cmake configure silently disables Vulkan). * libvulkan1 + the libggml-vulkan.so copy in the runtime stage. * compose env NVIDIA_DRIVER_CAPABILITIES=compute,utility,graphics — default nvidia-container-toolkit only mounts compute libs; Vulkan needs the graphics ICD (libGLX_nvidia + nvidia_icd.json) too. * entrypoint reads FISH_CPP_BACKEND (cuda/vulkan/cpu) and selects the appropriate -c/-v/no-flag invocation. * Default backend = vulkan.
64 lines
2.2 KiB
Bash
64 lines
2.2 KiB
Bash
#!/bin/bash
|
|
# fish-cpp entrypoint — start s2 server + uvicorn shim.
|
|
#
|
|
# s2 server holds the model resident on the GPU; uvicorn proxies
|
|
# Fish-shaped /v1/tts JSON requests to s2's multipart /generate.
|
|
# Both processes share the container; tini supervises both via
|
|
# `wait` after backgrounding s2.
|
|
|
|
set -euo pipefail
|
|
|
|
MODEL_PATH="${WEIGHTS_DIR:-/weights}/${FISH_CPP_MODEL:-s2-pro-q6_k.gguf}"
|
|
TOKENIZER_PATH="${WEIGHTS_DIR:-/weights}/${FISH_CPP_TOKENIZER:-tokenizer.json}"
|
|
DEVICE="${FISH_CPP_DEVICE:-0}"
|
|
BACKEND="${FISH_CPP_BACKEND:-vulkan}"
|
|
|
|
case "$BACKEND" in
|
|
cuda) BACKEND_FLAG=(-c "$DEVICE") ;;
|
|
vulkan) BACKEND_FLAG=(-v "$DEVICE") ;;
|
|
cpu) BACKEND_FLAG=() ;; # no flag → s2 stays on CPU
|
|
*) echo "[entrypoint] unknown FISH_CPP_BACKEND=$BACKEND (use cuda/vulkan/cpu)" >&2; exit 2 ;;
|
|
esac
|
|
|
|
if [ ! -f "$MODEL_PATH" ]; then echo "missing model: $MODEL_PATH" >&2; exit 1; fi
|
|
if [ ! -f "$TOKENIZER_PATH" ]; then echo "missing tokenizer: $TOKENIZER_PATH" >&2; exit 1; fi
|
|
|
|
# Background s2 server. Loads model on GPU once (per backend), then
|
|
# accepts multipart POSTs on localhost:3030/generate.
|
|
echo "[entrypoint] starting s2 server on :3030 with $BACKEND backend, device $DEVICE"
|
|
/usr/local/bin/s2 \
|
|
--server -H 127.0.0.1 -P 3030 \
|
|
-m "$MODEL_PATH" \
|
|
-t "$TOKENIZER_PATH" \
|
|
"${BACKEND_FLAG[@]}" &
|
|
S2_PID=$!
|
|
|
|
# Wait for s2 to bind 3030 before starting the shim. Avoids the
|
|
# obvious /v1/health 502 race on first boot.
|
|
echo "[entrypoint] waiting for s2 server to bind :3030"
|
|
for i in $(seq 1 60); do
|
|
if curl -sf -o /dev/null --max-time 1 http://127.0.0.1:3030/ 2>/dev/null; then
|
|
echo "[entrypoint] s2 server up after ${i}s"
|
|
break
|
|
fi
|
|
if ! kill -0 "$S2_PID" 2>/dev/null; then
|
|
echo "[entrypoint] s2 server died during startup" >&2
|
|
exit 1
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
# Foreground uvicorn. tini (PID 1) gets uvicorn signals; on SIGTERM
|
|
# uvicorn exits, then s2 gets reaped via `wait` below.
|
|
echo "[entrypoint] starting uvicorn shim on :8000"
|
|
uvicorn server:app --host 0.0.0.0 --port 8000 --no-access-log &
|
|
UV_PID=$!
|
|
|
|
# Block until either child exits; propagate exit code.
|
|
wait -n "$S2_PID" "$UV_PID"
|
|
EXIT=$?
|
|
echo "[entrypoint] one child exited (rc=$EXIT) — shutting down peer"
|
|
kill "$S2_PID" "$UV_PID" 2>/dev/null || true
|
|
wait || true
|
|
exit "$EXIT"
|