vh
8c1088af1f
Subprocess-per-request architecture forced CUDA + model load on every /v1/tts call (~10-20s init, then 5-15s generation). Even though CUDA is now actually being used (`-c 0` fix landed), 32s for "Verify." proved per-request init was the bottleneck. s2.cpp ships a built-in HTTP server (`--server -H -P`) that keeps the model resident on the GPU. Refactor: * entrypoint.sh — backgrounds `s2 --server -P 3030 -c 0 -m ... -t ...`, waits for it to bind 3030, then foregrounds uvicorn. tini supervises via `wait -n` so either child dying takes down the container. * server.py — drops subprocess.run; instead httpx-POSTs Fish-shaped /v1/tts JSON to s2's localhost:3030/generate (multipart form: text + optional prompt_text/prompt_audio for cloning). Model load + CUDA init now happen once at container start, not per-request. * Dockerfile — added httpx (shim dep), curl (entrypoint readiness probe), and the entrypoint.sh COPY+chmod. CMD now invokes entrypoint.sh instead of uvicorn directly. * deploy-fish-cpp.yaml — uploads entrypoint.sh alongside server.py.
56 lines
1.9 KiB
Bash
56 lines
1.9 KiB
Bash
#!/bin/bash
|
|
# fish-cpp entrypoint — start s2 server + uvicorn shim.
|
|
#
|
|
# s2 server holds the model resident on the GPU; uvicorn proxies
|
|
# Fish-shaped /v1/tts JSON requests to s2's multipart /generate.
|
|
# Both processes share the container; tini supervises both via
|
|
# `wait` after backgrounding s2.
|
|
|
|
set -euo pipefail
|
|
|
|
MODEL_PATH="${WEIGHTS_DIR:-/weights}/${FISH_CPP_MODEL:-s2-pro-q6_k.gguf}"
|
|
TOKENIZER_PATH="${WEIGHTS_DIR:-/weights}/${FISH_CPP_TOKENIZER:-tokenizer.json}"
|
|
DEVICE="${FISH_CPP_DEVICE:-0}"
|
|
|
|
if [ ! -f "$MODEL_PATH" ]; then echo "missing model: $MODEL_PATH" >&2; exit 1; fi
|
|
if [ ! -f "$TOKENIZER_PATH" ]; then echo "missing tokenizer: $TOKENIZER_PATH" >&2; exit 1; fi
|
|
|
|
# Background s2 server. Loads model on GPU once, then accepts
|
|
# multipart POSTs on localhost:3030/generate.
|
|
echo "[entrypoint] starting s2 server on :3030 with CUDA device $DEVICE"
|
|
/usr/local/bin/s2 \
|
|
--server -H 127.0.0.1 -P 3030 \
|
|
-m "$MODEL_PATH" \
|
|
-t "$TOKENIZER_PATH" \
|
|
-c "$DEVICE" &
|
|
S2_PID=$!
|
|
|
|
# Wait for s2 to bind 3030 before starting the shim. Avoids the
|
|
# obvious /v1/health 502 race on first boot.
|
|
echo "[entrypoint] waiting for s2 server to bind :3030"
|
|
for i in $(seq 1 60); do
|
|
if curl -sf -o /dev/null --max-time 1 http://127.0.0.1:3030/ 2>/dev/null; then
|
|
echo "[entrypoint] s2 server up after ${i}s"
|
|
break
|
|
fi
|
|
if ! kill -0 "$S2_PID" 2>/dev/null; then
|
|
echo "[entrypoint] s2 server died during startup" >&2
|
|
exit 1
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
# Foreground uvicorn. tini (PID 1) gets uvicorn signals; on SIGTERM
|
|
# uvicorn exits, then s2 gets reaped via `wait` below.
|
|
echo "[entrypoint] starting uvicorn shim on :8000"
|
|
uvicorn server:app --host 0.0.0.0 --port 8000 --no-access-log &
|
|
UV_PID=$!
|
|
|
|
# Block until either child exits; propagate exit code.
|
|
wait -n "$S2_PID" "$UV_PID"
|
|
EXIT=$?
|
|
echo "[entrypoint] one child exited (rc=$EXIT) — shutting down peer"
|
|
kill "$S2_PID" "$UV_PID" 2>/dev/null || true
|
|
wait || true
|
|
exit "$EXIT"
|