diff --git a/stacks/fish-s2/compose.yaml b/stacks/fish-s2/compose.yaml index 8543500..9282674 100644 --- a/stacks/fish-s2/compose.yaml +++ b/stacks/fish-s2/compose.yaml @@ -80,6 +80,11 @@ services: - ${FISH_S2_REFERENCE_DIR}:/app/references # Persistent HF cache so model re-pull only happens on first deploy. - ${FISH_S2_CACHE_DIR}:/app/hf_cache + # Pass --half to start_server.sh → enables fp16 inference on the + # LLM half. Speeds up the autoregressive loop (the dominant cost + # in TTFB). Also enables the streaming path's faster total time. + # build_compile_args() echoes unknown args back to the exec line. + command: ["--half"] healthcheck: # Fish ships /v1/health on the API server. python urllib instead # of curl because the upstream image is python-based and may not diff --git a/stacks/voxtral/compose.yaml b/stacks/voxtral/compose.yaml index 2c089df..6964df2 100644 --- a/stacks/voxtral/compose.yaml +++ b/stacks/voxtral/compose.yaml @@ -37,6 +37,12 @@ services: volumes: - ${VOXTRAL_CACHE_DIR}:/root/.cache/huggingface - ${VOXTRAL_VOICES_DIR}:/voices:ro + # Patched stage config — bundled YAML hardcodes + # gpu_memory_utilization: 0.8 on the language_model stage which + # OOMs anywhere we have other models resident on the same GPU. + # Mount our own copy at a custom path; the --stage-configs-path + # flag below points at it. + - /opt/docker/conf/voxtral/voxtral_tts.yaml:/etc/voxtral/voxtral_tts.yaml:ro # vllm/vllm-omni image has no default ENTRYPOINT or CMD — the # container init expected --model=... as argv[0]. Set entrypoint # to `vllm serve` (the standard CLI) and pass model as positional @@ -54,7 +60,7 @@ services: entrypoint: ["vllm-omni", "serve"] command: - "${VOXTRAL_MODEL:-mistralai/Voxtral-4B-TTS-2603}" - - "--stage-configs-path=vllm_omni/model_executor/stage_configs/voxtral_tts.yaml" + - "--stage-configs-path=/etc/voxtral/voxtral_tts.yaml" - "--host=0.0.0.0" - "--port=8000" - "--gpu-memory-utilization=${VOXTRAL_GPU_UTIL:-0.45}"