diff --git a/stacks/voxtral/.env.example b/stacks/voxtral/.env.example index 7850f9f..db1bbaf 100644 --- a/stacks/voxtral/.env.example +++ b/stacks/voxtral/.env.example @@ -20,15 +20,17 @@ VOXTRAL_BIND=0.0.0.0 # ── runtime / GPU ──────────────────────────────────────────────────── # GPU pinning. "0" = RTX 3090 (24 GB), "1" = RTX A6000 (48 GB). -# Voxtral 4B BF16 needs ~16 GB practical (model + KV + activation). -# Pinned to A6000 by default for headroom. The 3090 fits but is tight -# for long streaming sessions. -VOXTRAL_GPU_DEVICES=1 +# Pinned to GPU 0 (3090) — the A6000 is fully booked by Fish S2-Pro +# (~17 GB) + Qwen3-TTS / IndexTTS-2 / VibeVoice slots. Voxtral 4B BF16 +# needs ~10-12 GB practical (model + small KV); the 3090's 24 GB is +# enough alongside Kokoro (~1 GB) + Chatterbox (~3 GB) + Kyutai (~6 GB). +VOXTRAL_GPU_DEVICES=0 -# vLLM GPU memory utilization fraction (0.0-1.0). 0.85 = leave 15% -# headroom for other processes / KV cache spikes. Lower if running -# alongside other GPU workloads on the same device. -VOXTRAL_GPU_UTIL=0.85 +# vLLM GPU memory utilization fraction (0.0-1.0). On the 3090 alongside +# Kokoro/Chatterbox/Kyutai (~10 GB used), 0.5 = ~12 GB target gives +# Voxtral enough room for weights + KV. Bump to 0.85 if Voxtral ever +# moves to a dedicated GPU. +VOXTRAL_GPU_UTIL=0.5 # ── HuggingFace auth ───────────────────────────────────────────────── # HF_TOKEN — required to dodge 429 rate limits on Voxtral download. diff --git a/stacks/voxtral/compose.yaml b/stacks/voxtral/compose.yaml index 3d23024..2c089df 100644 --- a/stacks/voxtral/compose.yaml +++ b/stacks/voxtral/compose.yaml @@ -41,19 +41,25 @@ services: # container init expected --model=... as argv[0]. Set entrypoint # to `vllm serve` (the standard CLI) and pass model as positional # + tuning flags via command. - entrypoint: ["vllm", "serve"] + # Voxtral TTS is a STAGE-BASED pipeline (language_model → + # acoustic_transformer → audio output), not a flat + # MistralForCausalLM. vllm-omni's `--omni` mode + a stage config + # YAML drives this. The standard `vllm serve` errors with + # "no module named 'acoustic_transformer'" because it tries to + # load Voxtral as a vanilla Mistral causal LM. + # + # Pattern lifted from /workspace/vllm-omni/examples/online_serving/ + # qwen3_tts/run_server.sh (closest analog example in the image). + # Stage config path is relative to WORKDIR=/workspace/vllm-omni. + entrypoint: ["vllm-omni", "serve"] command: - "${VOXTRAL_MODEL:-mistralai/Voxtral-4B-TTS-2603}" + - "--stage-configs-path=vllm_omni/model_executor/stage_configs/voxtral_tts.yaml" + - "--host=0.0.0.0" - "--port=8000" - - "--dtype=bfloat16" - - "--gpu-memory-utilization=${VOXTRAL_GPU_UTIL:-0.85}" - # Voxtral uses the native Mistral model format (params.json, - # tekken.json tokenizer, consolidated.safetensors) — NOT HF - # transformers format. vLLM rejects it with "ensure presence - # of params.json" unless these three flags are set. - - "--load-format=mistral" - - "--tokenizer-mode=mistral" - - "--config-format=mistral" + - "--gpu-memory-utilization=${VOXTRAL_GPU_UTIL:-0.45}" + - "--trust-remote-code" + - "--omni" healthcheck: # vLLM-Omni exposes /health for liveness + /v1/models for readiness. # /health 200 means the server's listening; /v1/models 200 means