From f566f61b244f5c8905634364cc542d33b69c8f4d Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Wed, 17 Jun 2026 22:33:29 -0700 Subject: [PATCH] feat(stacks): mistral-small-4-heretic drop-in (abliterated NVFP4 backend swap) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Serves the in-house abliterated Mistral Small 4 (heretic NVFP4, vision-intact) under --served-model-name mistral-small-4 on ana-ml2 GPU0:8010 — a true drop-in for the official mistral-small-4 backend. Both litellm entries (mistral-small-4 + mistral-small-4-reasoning) route here with no litellm change. GPU0 fits one mistral-class model, so this is a backend swap, not a co-tenant: bring up after downing the official stack; revert by downing this and up-ing the official. Verified live through the gateway: standard returns clean answers, reasoning populates reasoning_content (the [THINK] split). Checkpoint built per tools/mistral-small4-nvfp4/. --- stacks/mistral-small-4-heretic/.env.example | 24 +++++ stacks/mistral-small-4-heretic/compose.yaml | 101 ++++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 stacks/mistral-small-4-heretic/.env.example create mode 100644 stacks/mistral-small-4-heretic/compose.yaml diff --git a/stacks/mistral-small-4-heretic/.env.example b/stacks/mistral-small-4-heretic/.env.example new file mode 100644 index 0000000..9fe5a3b --- /dev/null +++ b/stacks/mistral-small-4-heretic/.env.example @@ -0,0 +1,24 @@ +# mistral-small-4-heretic .env — copy to .env on ana-ml2 (gitignored). +# Easiest: copy the official stack's live .env and change MISTRAL_CONTAINER_NAME, +# so API_KEY (== litellm's VLLM_API_KEY) and HF_TOKEN carry over unchanged — +# that key match is what lets litellm authenticate to this backend with no edit. + +# v0.22.0 = the last vLLM before the Mistral multimodal regression (#44911); +# verified working vision tower. Matches the official stack's pin. +MISTRAL_IMAGE=vllm/vllm-openai:v0.22.0 + +# Distinct from the official's vllm-mistral4 so both can exist on disk; only one +# runs at a time (shared GPU0 + port 8010). +MISTRAL_CONTAINER_NAME=vllm-mistral4-heretic + +# Same port + GPU as the official → true drop-in after the official is downed. +MISTRAL_PORT=8010 +MISTRAL_GPU_ID=0 +MISTRAL_GPU_MEM_UTIL=0.93 +MISTRAL_MAX_MODEL_LEN=262144 +MISTRAL_MAX_NUM_SEQS=32 + +# Must equal the official stack's API_KEY (== litellm VLLM_API_KEY env). Carried +# over by copying the official .env. Real value lives on the host, gitignored. +API_KEY= +HF_TOKEN= diff --git a/stacks/mistral-small-4-heretic/compose.yaml b/stacks/mistral-small-4-heretic/compose.yaml new file mode 100644 index 0000000..5ffaf6e --- /dev/null +++ b/stacks/mistral-small-4-heretic/compose.yaml @@ -0,0 +1,101 @@ +# mistral-small-4-heretic — abliterated Mistral Small 4 (heretic NVFP4) as a +# DROP-IN for the official mistral-small-4 backend. +# +# Serves darkc0de/Mistral-Small-4-119B-2603-heretic, quantized to NVFP4 in-house +# (vision tower kept bf16) and converted to Mistral native format. Built + validated +# 2026-06-17 — see tools/mistral-small4-nvfp4/ for the build pipeline. +# +# WHY a separate stack: GPU0 fits only one mistral-class model (~65-70 GB), so this +# is a backend SWAP, not a co-tenant. Bring it up only after downing the official +# mistral-small-4 stack. It serves under --served-model-name mistral-small-4 on the +# SAME port (8010), so litellm's mistral-small-4 + mistral-small-4-reasoning entries +# route here with NO litellm change. Revert = down this, `up -d` the official stack. +# +# DIFFERENCES vs the official compose (everything else mirrors it for a faithful +# drop-in — TP=1, util 0.93, MLA, reasoning + tool-call parsers, prefix caching): +# - model is a LOCAL native dir (mounted /model), not an HF id, so it needs the +# native loader flags: --config-format/--load-format/--tokenizer-mode mistral. +# - distinct container_name (vllm-mistral4-heretic) so it can be staged without +# colliding with the official container. + +name: mistral-small-4-heretic + +services: + vllm-mistral4-heretic: + image: ${MISTRAL_IMAGE} + container_name: ${MISTRAL_CONTAINER_NAME} + restart: unless-stopped + ipc: host + ports: + - "${MISTRAL_PORT}:8000" + volumes: + - /tank/aimodels/huggingface:/hfcache + # The in-house heretic native NVFP4 checkpoint (read-only). + - /tank/aimodels/quant-work/heretic-native-nvfp4:/model:ro + environment: + - HF_HOME=/hfcache + - HF_HUB_CACHE=/hfcache/hub + - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-} + - VLLM_API_KEY=${API_KEY:-} + command: + - /model + # Native Mistral format (params.json + consolidated*.safetensors + tekken). + - --config-format + - mistral + - --load-format + - mistral + - --tokenizer-mode + - mistral + # SAME served name as the official → litellm routes here unchanged. + - --served-model-name + - mistral-small-4 + - --host + - 0.0.0.0 + - --port + - "8000" + - --tensor-parallel-size + - "1" + - --gpu-memory-utilization + - ${MISTRAL_GPU_MEM_UTIL} + - --max-model-len + - ${MISTRAL_MAX_MODEL_LEN} + - --attention-backend + - TRITON_MLA + - --tool-call-parser + - mistral + - --enable-auto-tool-choice + - --reasoning-parser + - mistral + - --max-num-seqs + - ${MISTRAL_MAX_NUM_SEQS} + - --dtype + - auto + - --enable-prefix-caching + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: + - "${MISTRAL_GPU_ID}" + capabilities: + - gpu + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 600s + networks: + - tnet + labels: + - homepage.group=AI Systems + - homepage.name=Mistral Small 4 (heretic NVFP4) + - homepage.icon=mdi-creation + - homepage.description=Abliterated Mistral-Small-4 (heretic NVFP4) drop-in via vLLM (ana-ml2 GPU 0) + - homepage.href=http://10.250.50.54:${MISTRAL_PORT}/docs + +networks: + tnet: + name: traefik-net + external: true