diff --git a/stacks/voxtral/.env.example b/stacks/voxtral/.env.example index bef2e6f..7850f9f 100644 --- a/stacks/voxtral/.env.example +++ b/stacks/voxtral/.env.example @@ -30,6 +30,14 @@ VOXTRAL_GPU_DEVICES=1 # alongside other GPU workloads on the same device. VOXTRAL_GPU_UTIL=0.85 +# ── HuggingFace auth ───────────────────────────────────────────────── +# HF_TOKEN — required to dodge 429 rate limits on Voxtral download. +# HF aggressively throttles unauthenticated IPs that pull large repos. +# Generate one at https://huggingface.co/settings/tokens (a read-only +# token is sufficient). Without this, the first model download fails +# mid-stream and vLLM aborts engine init. +VOXTRAL_HF_TOKEN= + # ── persistent storage on the host ─────────────────────────────────── # HF cache — first start pulls the Voxtral checkpoint (~8 GB) into # this dir. Persistent across container recreates. diff --git a/stacks/voxtral/compose.yaml b/stacks/voxtral/compose.yaml index 591a1c0..3d23024 100644 --- a/stacks/voxtral/compose.yaml +++ b/stacks/voxtral/compose.yaml @@ -28,6 +28,10 @@ services: environment: - NVIDIA_VISIBLE_DEVICES=${VOXTRAL_GPU_DEVICES:-1} - HF_HOME=/root/.cache/huggingface + # HF_TOKEN required to dodge 429 rate limits on the model + # download (HF aggressively throttles unauthenticated IPs that + # pull large repos repeatedly). Set in .env — see .env.example. + - HF_TOKEN=${VOXTRAL_HF_TOKEN} # vLLM serving args — see https://docs.vllm.ai for full list. # We override the default model via cmd args below. volumes: @@ -43,6 +47,13 @@ services: - "--port=8000" - "--dtype=bfloat16" - "--gpu-memory-utilization=${VOXTRAL_GPU_UTIL:-0.85}" + # Voxtral uses the native Mistral model format (params.json, + # tekken.json tokenizer, consolidated.safetensors) — NOT HF + # transformers format. vLLM rejects it with "ensure presence + # of params.json" unless these three flags are set. + - "--load-format=mistral" + - "--tokenizer-mode=mistral" + - "--config-format=mistral" healthcheck: # vLLM-Omni exposes /health for liveness + /v1/models for readiness. # /health 200 means the server's listening; /v1/models 200 means