diff --git a/stacks/litellm/conf/config.yaml b/stacks/litellm/conf/config.yaml index 48a19f7..547c72c 100644 --- a/stacks/litellm/conf/config.yaml +++ b/stacks/litellm/conf/config.yaml @@ -80,6 +80,21 @@ model_list: model_info: mode: chat + # mistral-small-4-reasoning: same upstream checkpoint, reasoning ON (2026-06-15, + # operator wanted "medium" — but Mistral's reasoning_effort is BINARY, only + # 'none' or 'high' (a medium/low request 400s). 'high' is the sole reasoning-ON + # level, so it carries the -reasoning intent. LiteLLM forwards extra_body to + # vLLM; the mistral reasoning-parser splits [THINK]…[/THINK] into reasoning_content. + - model_name: mistral-small-4-reasoning + litellm_params: + model: hosted_vllm/mistral-small-4 + api_base: http://10.250.50.54:8010/v1 + api_key: os.environ/VLLM_API_KEY + extra_body: + reasoning_effort: high + model_info: + mode: chat + # --- Qwen3 embeddings --- - model_name: qwen3-embedding litellm_params: diff --git a/stacks/mistral-small-4/.env.example b/stacks/mistral-small-4/.env.example index 5b1d72a..7c0599e 100644 --- a/stacks/mistral-small-4/.env.example +++ b/stacks/mistral-small-4/.env.example @@ -3,11 +3,15 @@ # # See compose.yaml header for the NVFP4/TP=1/MLA rationale and the vLLM>=0.20 floor. -# PINNED by digest, not :latest — this model is version-sensitive (needs vLLM -# >= 0.20 for day-0 support; the related nvidia-ModelOpt NVFP4 MoE path broke on -# 0.19.1/0.22.0). Pin protects against a :latest regression. This digest = vLLM -# 0.23.0, the version validated to load this checkpoint. Bump deliberately. -MISTRAL_IMAGE=vllm/vllm-openai@sha256:6d8429e38e3747723ca07ee1b17972e09bb9c51c4032b266f24fb1cc3b22ed8f +# PINNED to v0.22.0 — the last release BEFORE the Mistral multimodal regression +# (#44911, MistralCommonImageProcessor.fetch_images, landed ~0.22.1; 0.23.0 is +# affected). v0.22.0 loads the NVFP4 (compressed-tensors) AND serves VISION — +# verified: half-blue/half-red image read correctly ("left blue, right red"). +# This gives a working vision tower as the abliteration/tuning baseline. Do NOT +# bump to 0.23.0 (breaks vision). reasoning_effort works (none/high only) but +# reasoning_content-splitting is unreliable on this version — vision is the +# priority. Revisit when vLLM patches the Mistral mm path on a newer release. +MISTRAL_IMAGE=vllm/vllm-openai:v0.22.0 MISTRAL_CONTAINER_NAME=vllm-mistral4 MISTRAL_MODEL=mistralai/Mistral-Small-4-119B-2603-NVFP4 diff --git a/stacks/mistral-small-4/compose.yaml b/stacks/mistral-small-4/compose.yaml index 58c125c..7df87c0 100644 --- a/stacks/mistral-small-4/compose.yaml +++ b/stacks/mistral-small-4/compose.yaml @@ -74,15 +74,14 @@ services: - mistral - --max-num-seqs - ${MISTRAL_MAX_NUM_SEQS} - # TEXT-ONLY (2026-06-15): vLLM 0.23.0's Mistral multimodal processor crashes - # at startup dummy-image profiling — `MistralCommonImageProcessor has no - # attribute fetch_images` (vLLM↔mistral_common incompat; same class hit - # Mistral-3.1/Devstral/Magistral). Setting image/video limit to 0 skips the - # vision profiling so the model loads text-only — which is all the creative- - # writing use needs. REMOVE this flag to restore vision once vLLM patches the - # Mistral mm path (track: the model is natively multimodal). - - --limit-mm-per-prompt - - '{"image":0,"video":0}' + # VISION ENABLED. vLLM is pinned to v0.22.0 in .env — the last release BEFORE + # the Mistral multimodal regression (#44911, `MistralCommonImageProcessor has + # no attribute fetch_images`, landed ~0.22.1+; 0.23.0 is affected). v0.22.0 + # still has Mistral-Small-4 arch + compressed-tensors NVFP4 support (the + # #44081 ModelOpt-NVFP4 bug on 0.22.0 is a DIFFERENT quant path, doesn't touch + # this compressed-tensors checkpoint). Gives a verified working vision tower + # as the abliteration/tuning baseline. (qwen36 stays on 0.23.0 — separate + # container; it NEEDS 0.23.0 for its ModelOpt NVFP4.) - --dtype - auto - --enable-prefix-caching