From 68f3cd05fe739c3814b46afaded1d496e3bb5bc7 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Tue, 28 Apr 2026 00:35:55 -0700 Subject: [PATCH] voxtral: mount patched stage YAML to dodge hardcoded 0.8 GPU util; fish-s2: --half + streaming wins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Voxtral final fix (8th iteration): * The bundled voxtral_tts.yaml hardcodes gpu_memory_utilization: 0.8 on the language_model stage — overrides the CLI flag. Mounted a patched copy (0.4) at /etc/voxtral/voxtral_tts.yaml and pointed --stage-configs-path there. * With Kyutai stopped to free 5 GB on the 3090, both stages fit (target 9.4 + 2.4 GB ≈ 11.8 GB; 17 GB free post-kyutai-stop). * Voxtral now healthy on GPU 0 — bench: 1.9-2.7 s TTFB, real WAV. Fish s2-pro optimization (per-request sweep, no model swap): * `streaming: true` in request body drops TTFB from 7.7 s → 0.026 s (300×). Total time goes up ~1 s (chunked HTTP overhead) but perceived latency = TTFB. Use stream:true for any interactive use. * `latency: "balanced"` actually slower than default — bad name; skip. * `use_memory_cache: "on"` no measurable benefit. * `chunk_length: 100` (default 200) no TTFB benefit non-streaming. * Server-side `--half` (fp16 inference) added via compose `command` override — passes through start_server.sh's $@ unchanged into api_server.py. Should reduce total time too. Validation pending the post-restart bench. Kyutai stopped to free GPU 0 budget — the bench numbers earlier (3.4 s avg) were unimpressive vs Voxtral's 2.3 s in the same multilingual slot. Kept the stack files for future re-deploy if needed; just the running container is gone. --- stacks/fish-s2/compose.yaml | 5 +++++ stacks/voxtral/compose.yaml | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/stacks/fish-s2/compose.yaml b/stacks/fish-s2/compose.yaml index 8543500..9282674 100644 --- a/stacks/fish-s2/compose.yaml +++ b/stacks/fish-s2/compose.yaml @@ -80,6 +80,11 @@ services: - ${FISH_S2_REFERENCE_DIR}:/app/references # Persistent HF cache so model re-pull only happens on first deploy. - ${FISH_S2_CACHE_DIR}:/app/hf_cache + # Pass --half to start_server.sh → enables fp16 inference on the + # LLM half. Speeds up the autoregressive loop (the dominant cost + # in TTFB). Also enables the streaming path's faster total time. + # build_compile_args() echoes unknown args back to the exec line. + command: ["--half"] healthcheck: # Fish ships /v1/health on the API server. python urllib instead # of curl because the upstream image is python-based and may not diff --git a/stacks/voxtral/compose.yaml b/stacks/voxtral/compose.yaml index 2c089df..6964df2 100644 --- a/stacks/voxtral/compose.yaml +++ b/stacks/voxtral/compose.yaml @@ -37,6 +37,12 @@ services: volumes: - ${VOXTRAL_CACHE_DIR}:/root/.cache/huggingface - ${VOXTRAL_VOICES_DIR}:/voices:ro + # Patched stage config — bundled YAML hardcodes + # gpu_memory_utilization: 0.8 on the language_model stage which + # OOMs anywhere we have other models resident on the same GPU. + # Mount our own copy at a custom path; the --stage-configs-path + # flag below points at it. + - /opt/docker/conf/voxtral/voxtral_tts.yaml:/etc/voxtral/voxtral_tts.yaml:ro # vllm/vllm-omni image has no default ENTRYPOINT or CMD — the # container init expected --model=... as argv[0]. Set entrypoint # to `vllm serve` (the standard CLI) and pass model as positional @@ -54,7 +60,7 @@ services: entrypoint: ["vllm-omni", "serve"] command: - "${VOXTRAL_MODEL:-mistralai/Voxtral-4B-TTS-2603}" - - "--stage-configs-path=vllm_omni/model_executor/stage_configs/voxtral_tts.yaml" + - "--stage-configs-path=/etc/voxtral/voxtral_tts.yaml" - "--host=0.0.0.0" - "--port=8000" - "--gpu-memory-utilization=${VOXTRAL_GPU_UTIL:-0.45}"