diff --git a/stacks/litellm/conf/config.yaml b/stacks/litellm/conf/config.yaml index 77b1206..48a19f7 100644 --- a/stacks/litellm/conf/config.yaml +++ b/stacks/litellm/conf/config.yaml @@ -67,6 +67,19 @@ model_list: model_info: mode: chat + # --- Mistral Small 4 (official NVFP4) — creative-writing / general text. 119B + # MoE (6.5B active), vLLM on ana-ml2 GPU 0 (dedicated 96 GB Blackwell), :8010. + # Explicit entry shadows the "*" wildcard. TEXT-ONLY for now — vLLM 0.23.0's + # Mistral multimodal processor crashes at startup (loaded with image/video + # limit 0); vision returns when vLLM patches it. Deployed 2026-06-15. --- + - model_name: mistral-small-4 + litellm_params: + model: hosted_vllm/mistral-small-4 + api_base: http://10.250.50.54:8010/v1 + api_key: os.environ/VLLM_API_KEY + model_info: + mode: chat + # --- Qwen3 embeddings --- - model_name: qwen3-embedding litellm_params: diff --git a/stacks/mistral-small-4/.env.example b/stacks/mistral-small-4/.env.example new file mode 100644 index 0000000..5b1d72a --- /dev/null +++ b/stacks/mistral-small-4/.env.example @@ -0,0 +1,32 @@ +# Mistral Small 4 (official NVFP4) on ana-ml2 GPU 0 — copy to .env on the host. +# Real .env lives on ana-ml2 at /opt/docker/compose/mistral-small-4/.env (gitignored). +# +# See compose.yaml header for the NVFP4/TP=1/MLA rationale and the vLLM>=0.20 floor. + +# PINNED by digest, not :latest — this model is version-sensitive (needs vLLM +# >= 0.20 for day-0 support; the related nvidia-ModelOpt NVFP4 MoE path broke on +# 0.19.1/0.22.0). Pin protects against a :latest regression. This digest = vLLM +# 0.23.0, the version validated to load this checkpoint. Bump deliberately. +MISTRAL_IMAGE=vllm/vllm-openai@sha256:6d8429e38e3747723ca07ee1b17972e09bb9c51c4032b266f24fb1cc3b22ed8f + +MISTRAL_CONTAINER_NAME=vllm-mistral4 +MISTRAL_MODEL=mistralai/Mistral-Small-4-119B-2603-NVFP4 +MISTRAL_PORT=8010 + +# GPU 0 = the free 96 GB Blackwell card, dedicated single-tenant to this model +# (74.4 GB weights leave no room to co-tenant). GPU 1 holds qwen36 + granite + +# the embed/rerank/reward trio. +MISTRAL_GPU_ID=0 + +# util 0.93 (~89 GB budget) — 74.4 GB weights + ~5 GB CUDA/graph leaves ~10 GB +# KV. MLA keeps KV compressed so 131072 ctx fits; raise toward native 256K only +# after measuring real KV headroom. Dedicated card, so 0.93 is safe. +MISTRAL_GPU_MEM_UTIL=0.93 +MISTRAL_MAX_MODEL_LEN=131072 +# Single-card KV is tighter than the official TP=2 setup → cap concurrency at 64 +# (official used 128 across two cards). +MISTRAL_MAX_NUM_SEQS=64 + +# Optional — model is ungated (Apache-2.0), no token needed. +HF_TOKEN= +API_KEY= diff --git a/stacks/mistral-small-4/compose.yaml b/stacks/mistral-small-4/compose.yaml new file mode 100644 index 0000000..58c125c --- /dev/null +++ b/stacks/mistral-small-4/compose.yaml @@ -0,0 +1,116 @@ +# mistral-small-4 — Mistral-Small-4-119B-2603 (official NVFP4) on ana-ml2 GPU 0. +# +# Mistral Small 4 is a 119B-total / 6.5B-active MoE (128 experts, 4 active), +# 256K context, multimodal, Apache-2.0 (released 2026-03). This serves the +# OFFICIAL NVFP4 checkpoint (mistralai/Mistral-Small-4-119B-2603-NVFP4) — 74.4 GB +# of compressed-tensors (llm-compressor, a vLLM + Red Hat collaboration, day-0 +# vLLM support). It is the GPU-0 tenant (the slot formerly reserved for a +# creative-writing pick — operator reassigned 2026-06-15; tune-for-creative- +# writing comes after base-characteristic probing). +# +# WHY NVFP4 (not FP8/bf16): on a SINGLE 96 GB card, NVFP4 (74.4 GB weights) is +# the only variant that fits at TP=1 — FP8 (~119 GB) and bf16 (~238 GB) need both +# GPUs. The card is Blackwell (sm_120) with FP4 tensor cores, so NVFP4 gets a real +# speedup, not just a VRAM save. NOTE: this is the COMPRESSED-TENSORS NVFP4 path +# (vendor-shipped, vLLM-tested) — distinct from the nvidia-ModelOpt NVFP4 MoE +# loader that broke on Qwen3.6 (#44081); different code path, day-0 supported. +# +# WHY TP=1 here: Mistral's official card uses --tensor-parallel-size 2 (their +# reference 80 GB cards can't fit 74.4 GB + context on one). The 96 GB Blackwell +# flips that to single-card: 74.4 GB weights + ~5 GB overhead leaves ~17 GB for +# KV. Mistral Small 4 uses MLA attention (TRITON_MLA) so KV is compressed/cheap — +# big context stays affordable even on a constrained KV pool. max-model-len is +# capped to 131072 on first bring-up (raise toward the native 256K once real KV +# headroom is measured). +# +# vLLM FLOOR: needs >= 0.20 (Mistral Small 4 day-0 support); validated on 0.23.0. +# Do NOT reuse the qwen36-vl 0.19.1 image — it predates this model. +# +# Serve flags mirror Mistral's official command (cited in README), adapted for +# single-card: TP 2->1, util 0.8->0.93, max-len 262144->131072, max-num-seqs +# 128->64. All tunables live in .env — edit that, not this file. + +name: mistral-small-4 + +services: + vllm-mistral4: + image: ${MISTRAL_IMAGE} + container_name: ${MISTRAL_CONTAINER_NAME} + restart: unless-stopped + ipc: host + ports: + - "${MISTRAL_PORT}:8000" + volumes: + - /tank/aimodels/huggingface:/hfcache + environment: + - HF_HOME=/hfcache + - HF_HUB_CACHE=/hfcache/hub + - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-} + - VLLM_API_KEY=${API_KEY:-} + command: + - ${MISTRAL_MODEL} + # Pre-quantized NVFP4 (compressed-tensors) — vLLM auto-detects the quant; + # no --quantization flag. + - --served-model-name + - mistral-small-4 + - --host + - 0.0.0.0 + - --port + - "8000" + - --tensor-parallel-size + - "1" + - --gpu-memory-utilization + - ${MISTRAL_GPU_MEM_UTIL} + - --max-model-len + - ${MISTRAL_MAX_MODEL_LEN} + # MLA attention backend (DeepSeek-style latent KV → compressed, cheap KV). + - --attention-backend + - TRITON_MLA + # Mistral tool-calling + configurable reasoning (per the official card). + - --tool-call-parser + - mistral + - --enable-auto-tool-choice + - --reasoning-parser + - mistral + - --max-num-seqs + - ${MISTRAL_MAX_NUM_SEQS} + # TEXT-ONLY (2026-06-15): vLLM 0.23.0's Mistral multimodal processor crashes + # at startup dummy-image profiling — `MistralCommonImageProcessor has no + # attribute fetch_images` (vLLM↔mistral_common incompat; same class hit + # Mistral-3.1/Devstral/Magistral). Setting image/video limit to 0 skips the + # vision profiling so the model loads text-only — which is all the creative- + # writing use needs. REMOVE this flag to restore vision once vLLM patches the + # Mistral mm path (track: the model is natively multimodal). + - --limit-mm-per-prompt + - '{"image":0,"video":0}' + - --dtype + - auto + - --enable-prefix-caching + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: + - "${MISTRAL_GPU_ID}" + capabilities: + - gpu + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 600s + networks: + - tnet + labels: + - homepage.group=AI Systems + - homepage.name=Mistral Small 4 (NVFP4) + - homepage.icon=mdi-creation + - homepage.description=Mistral-Small-4-119B-2603 MoE (NVFP4) via vLLM (ana-ml2 GPU 0) + - homepage.href=http://10.250.50.54:${MISTRAL_PORT}/docs + +networks: + tnet: + name: traefik-net + external: true