From fe01f73d8463e8ff64864e205a35741ecad8761d Mon Sep 17 00:00:00 2001
From: Vuong Hoang <vh@phasefinal.com>
Date: Tue, 28 Apr 2026 00:17:21 -0700
Subject: [PATCH] =?UTF-8?q?voxtral:=20switch=20to=20`vllm-omni=20serve=20-?=
 =?UTF-8?q?-omni`=20with=20stage=20config=20=E2=80=94=20Voxtral=20is=20a?=
 =?UTF-8?q?=20multi-stage=20pipeline?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fourth attempt finally found the right invocation. Voxtral is a
two-stage TTS pipeline (language_model → acoustic_transformer →
audio output), not a flat MistralForCausalLM. Standard `vllm serve`
errored with "no module named 'acoustic_transformer'" because it
loads the model as a vanilla Mistral causal LM.

Pattern from /workspace/vllm-omni/examples/online_serving/
qwen3_tts/run_server.sh (closest in-image analog):

  vllm-omni serve <MODEL> \
    --stage-configs-path vllm_omni/model_executor/stage_configs/voxtral_tts.yaml \
    --host 0.0.0.0 --port 8000 \
    --gpu-memory-utilization 0.45 \
    --trust-remote-code --omni

Key differences from previous attempt:
  * `vllm-omni` binary, not `vllm`
  * `--omni` flag activates multi-stage pipeline
  * `--stage-configs-path` points at the bundled YAML that maps
    stages to GPU + scheduler + worker classes
  * Dropped --load-format/--tokenizer-mode/--config-format=mistral
    flags — the stage config handles tokenizer_mode internally
  * --trust-remote-code is required for the acoustic_transformer
    custom code path

Default .env.example now: GPU 0 (3090) with util 0.45 (~10.6 GB
target on 24 GB GPU). The A6000 is fully booked by Fish s2-pro.
---
 stacks/voxtral/.env.example | 18 ++++++++++--------
 stacks/voxtral/compose.yaml | 26 ++++++++++++++++----------
 2 files changed, 26 insertions(+), 18 deletions(-)

diff --git a/stacks/voxtral/.env.example b/stacks/voxtral/.env.example
index 7850f9f..db1bbaf 100644
--- a/stacks/voxtral/.env.example
+++ b/stacks/voxtral/.env.example
@@ -20,15 +20,17 @@ VOXTRAL_BIND=0.0.0.0
 
 # ── runtime / GPU ────────────────────────────────────────────────────
 # GPU pinning. "0" = RTX 3090 (24 GB), "1" = RTX A6000 (48 GB).
-# Voxtral 4B BF16 needs ~16 GB practical (model + KV + activation).
-# Pinned to A6000 by default for headroom. The 3090 fits but is tight
-# for long streaming sessions.
-VOXTRAL_GPU_DEVICES=1
+# Pinned to GPU 0 (3090) — the A6000 is fully booked by Fish S2-Pro
+# (~17 GB) + Qwen3-TTS / IndexTTS-2 / VibeVoice slots. Voxtral 4B BF16
+# needs ~10-12 GB practical (model + small KV); the 3090's 24 GB is
+# enough alongside Kokoro (~1 GB) + Chatterbox (~3 GB) + Kyutai (~6 GB).
+VOXTRAL_GPU_DEVICES=0
 
-# vLLM GPU memory utilization fraction (0.0-1.0). 0.85 = leave 15%
-# headroom for other processes / KV cache spikes. Lower if running
-# alongside other GPU workloads on the same device.
-VOXTRAL_GPU_UTIL=0.85
+# vLLM GPU memory utilization fraction (0.0-1.0). On the 3090 alongside
+# Kokoro/Chatterbox/Kyutai (~10 GB used), 0.5 = ~12 GB target gives
+# Voxtral enough room for weights + KV. Bump to 0.85 if Voxtral ever
+# moves to a dedicated GPU.
+VOXTRAL_GPU_UTIL=0.5
 
 # ── HuggingFace auth ─────────────────────────────────────────────────
 # HF_TOKEN — required to dodge 429 rate limits on Voxtral download.
diff --git a/stacks/voxtral/compose.yaml b/stacks/voxtral/compose.yaml
index 3d23024..2c089df 100644
--- a/stacks/voxtral/compose.yaml
+++ b/stacks/voxtral/compose.yaml
@@ -41,19 +41,25 @@ services:
     # container init expected --model=... as argv[0]. Set entrypoint
     # to `vllm serve` (the standard CLI) and pass model as positional
     # + tuning flags via command.
-    entrypoint: ["vllm", "serve"]
+    # Voxtral TTS is a STAGE-BASED pipeline (language_model →
+    # acoustic_transformer → audio output), not a flat
+    # MistralForCausalLM. vllm-omni's `--omni` mode + a stage config
+    # YAML drives this. The standard `vllm serve` errors with
+    # "no module named 'acoustic_transformer'" because it tries to
+    # load Voxtral as a vanilla Mistral causal LM.
+    #
+    # Pattern lifted from /workspace/vllm-omni/examples/online_serving/
+    # qwen3_tts/run_server.sh (closest analog example in the image).
+    # Stage config path is relative to WORKDIR=/workspace/vllm-omni.
+    entrypoint: ["vllm-omni", "serve"]
     command:
       - "${VOXTRAL_MODEL:-mistralai/Voxtral-4B-TTS-2603}"
+      - "--stage-configs-path=vllm_omni/model_executor/stage_configs/voxtral_tts.yaml"
+      - "--host=0.0.0.0"
       - "--port=8000"
-      - "--dtype=bfloat16"
-      - "--gpu-memory-utilization=${VOXTRAL_GPU_UTIL:-0.85}"
-      # Voxtral uses the native Mistral model format (params.json,
-      # tekken.json tokenizer, consolidated.safetensors) — NOT HF
-      # transformers format. vLLM rejects it with "ensure presence
-      # of params.json" unless these three flags are set.
-      - "--load-format=mistral"
-      - "--tokenizer-mode=mistral"
-      - "--config-format=mistral"
+      - "--gpu-memory-utilization=${VOXTRAL_GPU_UTIL:-0.45}"
+      - "--trust-remote-code"
+      - "--omni"
     healthcheck:
       # vLLM-Omni exposes /health for liveness + /v1/models for readiness.
       # /health 200 means the server's listening; /v1/models 200 means