diff --git a/stacks/heretic2-charrp-reasoning/.env.example b/stacks/heretic2-charrp-reasoning/.env.example new file mode 100644 index 0000000..b359f89 --- /dev/null +++ b/stacks/heretic2-charrp-reasoning/.env.example @@ -0,0 +1,32 @@ +# heretic2-charrp-reasoning on ana-ml2. Real .env lives on the host, never committed. +# +# Every value below is the compose default — this file exists to name the knobs, +# not to change them. The live host currently sets only the bottom three. + +# --- image / container --- +REASONING_IMAGE=vllm/vllm-openai:v0.24.0 +REASONING_CONTAINER=vllm-charrp-reasoning-nvfp4 +REASONING_PORT=8018 + +# --- model --- +REASONING_MODEL=/tank/aimodels/heretic2-nvfp4-work/heretic2-modelopt-nvfp4-mtp +# Speculative tokens for the qwen3_5_mtp draft head. This is the MTP seat's whole +# point (~77 tok/s, ~1.3x over the retired GGUF seat); dropping it to 0 disables +# spec-decode and the workaround mount stops mattering. +SPEC_TOKENS=3 + +# --- auth --- +# vLLM bearer token. Empty = unauthenticated on the LAN, which is how the other +# ana-ml2 seats run; set it if this seat is ever exposed beyond the fleet. +API_KEY= + +# --- placement / VRAM (the three the host actually overrides) --- +# GPU0 is shared with vllm-aeon-gen (gen) and llama-charrp (char-rp). NVFP4 27B +# weights are ~26 GB + KV, so util is held low to fit the ~33 GB free alongside +# them — and that is why max-model-len is 32768 rather than the 256K the lighter +# Q5 GGUF seat managed. Raising either without rebalancing GPU0 will OOM the +# neighbours, not just this container. +REASONING_GPU_ID=0 +REASONING_GPU_MEM_UTIL=0.30 +REASONING_MAX_MODEL_LEN=32768 +REASONING_MAX_NUM_SEQS=4 diff --git a/stacks/heretic2-charrp-reasoning/README.md b/stacks/heretic2-charrp-reasoning/README.md new file mode 100644 index 0000000..78fd58a --- /dev/null +++ b/stacks/heretic2-charrp-reasoning/README.md @@ -0,0 +1,48 @@ +# heretic2-charrp-reasoning — NVFP4 + native MTP reasoning seat (ana-ml2) + +The `char-rp-reasoning` seat: NEO-CODE Heretic2 27B quantized to modelopt NVFP4 +with a grafted BF16 MTP head, served by vLLM on **ana-ml2 GPU0**, port **8018**. +Replaces the retired GGUF seat (`llama-charrp-reasoning`) at roughly **77 tok/s +(~1.3x)** via `qwen3_5_mtp` speculative decode. + +- **Served model name:** `char-rp-reasoning` — the gateway alias consumers use +- **Endpoint:** `http://10.250.50.54:8018` (`/docs` for the card link) + +## ⚠️ It does not boot without the MTP workaround + +vLLM 0.24.0 does not propagate modelopt's `exclude_modules` to the **draft** +model in a spec-decode config, so the BF16 MTP head gets quantized along with +everything else and the engine dies at load on a shape mismatch. + +`conf/mtp-workaround/sitecustomize.py` is mounted at `PYTHONPATH` and patches +`is_layer_skipped` to force-skip `mtp.*`, keeping the head BF16. **The mount and +the `PYTHONPATH` env are both load-bearing** — remove either and the seat +crash-loops at startup with an error that looks like a bad quant rather than a +missing shim. + +Canonical copy of the shim and the reasoning behind it live in +`services/heretic2-nvfp4-quant/`; the full build-and-serve recipe, including the +other landmines hit on the way, is in +[`docs/runbooks/heretic2-nvfp4-mtp-seat.md`](../../docs/runbooks/heretic2-nvfp4-mtp-seat.md). +Read the runbook before changing anything here — this README is the pointer, not +the spec. + +## VRAM is the constraint, and it is shared + +GPU0 also hosts `vllm-aeon-gen` (`gen`) and `llama-charrp` (`char-rp`). NVFP4 +27B weights are ~26 GB plus KV, which is why `--gpu-memory-utilization` sits at +**0.30** and `--max-model-len` at **32768** — the old GGUF seat managed 256K on +much lighter Q5 weights. **Raising either without rebalancing GPU0 first will +OOM the neighbours, not just this container.** Tunables are in `.env`; see +`.env.example`. + +## Deploy + +```bash +scripts/deploy-stack.sh ana-ml2 heretic2-charrp-reasoning +ssh infra-ops@10.250.50.54 \ + 'cd /opt/docker/compose/heretic2-charrp-reasoning && sudo docker compose up -d' +``` + +First start is slow — the healthcheck allows a 600s `start_period` because +loading NVFP4 weights plus the draft head takes minutes. diff --git a/stacks/heretic2-charrp-reasoning/compose.yaml b/stacks/heretic2-charrp-reasoning/compose.yaml new file mode 100644 index 0000000..d4565a4 --- /dev/null +++ b/stacks/heretic2-charrp-reasoning/compose.yaml @@ -0,0 +1,85 @@ +# heretic2-charrp-reasoning — modelopt NVFP4 + native MTP fast char-rp-reasoning seat on +# ana-ml2 GPU0, replacing the GGUF NEO-CODE reasoning seat (llama-charrp-reasoning, now retired). +# Same Heretic2/NEO-CODE model; ~77 tok/s (~1.3x over GGUF) via qwen3_5_mtp spec-decode. +# +# ⚠️ REQUIRES the MTP workaround: vLLM 0.24.0 doesn't propagate modelopt exclude_modules to the +# spec-decode DRAFT model, so the BF16 mtp head gets quantized -> shape crash. The mounted +# sitecustomize.py (conf/mtp-workaround/) force-skips mtp.* in is_layer_skipped. Without it the +# engine dies at load. Full recipe: eshpfi docs/runbooks/heretic2-nvfp4-mtp-seat.md. +# +# Co-located on GPU0 with vllm-aeon-gen (gen) + llama-charrp (char-rp). VRAM: NVFP4 27B weights +# ~26GB + KV. util 0.30 fits the ~33GB free alongside gen+char-rp -> max-model-len capped at +# 32768 (the GGUF seat did 256K on lighter Q5 weights; NVFP4 is heavier, so context is reduced +# until VRAM is rebalanced). Tunables in .env. + +name: heretic2-charrp-reasoning + +services: + vllm-charrp-reasoning: + image: ${REASONING_IMAGE:-vllm/vllm-openai:v0.24.0} + container_name: ${REASONING_CONTAINER:-vllm-charrp-reasoning-nvfp4} + restart: unless-stopped + ipc: host + ports: + - "${REASONING_PORT:-8018}:8000" + volumes: + - /tank/aimodels:/tank/aimodels + # The MTP draft-model quant workaround (sitecustomize.py). PYTHONPATH loads it in the + # engine-core subprocess. See runbook landmine #4. + - ./conf/mtp-workaround:/mtp-workaround:ro + environment: + - PYTHONPATH=/mtp-workaround + - VLLM_API_KEY=${API_KEY:-} + command: + - ${REASONING_MODEL:-/tank/aimodels/heretic2-nvfp4-work/heretic2-modelopt-nvfp4-mtp} + - --quantization + - modelopt + - --speculative-config + - '{"method": "qwen3_5_mtp", "num_speculative_tokens": ${SPEC_TOKENS:-3}}' + - --language-model-only + - --mamba-cache-dtype + - float32 + - --reasoning-parser + - qwen3 + - --tool-call-parser + - qwen3_coder + - --enable-auto-tool-choice + - --served-model-name + - char-rp-reasoning + - --max-model-len + - "${REASONING_MAX_MODEL_LEN:-32768}" + - --max-num-seqs + - "${REASONING_MAX_NUM_SEQS:-4}" + - --gpu-memory-utilization + - "${REASONING_GPU_MEM_UTIL:-0.30}" + - --kv-cache-dtype + - fp8 + - --trust-remote-code + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: + - "${REASONING_GPU_ID:-0}" + capabilities: + - gpu + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 600s + networks: + - tnet + labels: + - homepage.group=AI - Inference + - homepage.name=char-rp-reasoning (Heretic2 NVFP4+MTP) + - homepage.icon=mdi-rocket-launch + - homepage.description=NEO-CODE Heretic2 NVFP4 + native MTP, ~77 tok/s (ana-ml2 GPU0) + - homepage.href=http://10.250.50.54:${REASONING_PORT:-8018}/docs + +networks: + tnet: + name: traefik-net + external: true diff --git a/stacks/heretic2-charrp-reasoning/conf/mtp-workaround/sitecustomize.py b/stacks/heretic2-charrp-reasoning/conf/mtp-workaround/sitecustomize.py new file mode 100644 index 0000000..f5835ee --- /dev/null +++ b/stacks/heretic2-charrp-reasoning/conf/mtp-workaround/sitecustomize.py @@ -0,0 +1,42 @@ +# MTP draft-model quant workaround for vLLM 0.24.0 (mounted at PYTHONPATH). Canonical copy + +# rationale: eshpfi services/heretic2-nvfp4-quant/sitecustomize-mtp-workaround.py + the runbook. +# vLLM 0.24.0 doesn't propagate modelopt exclude_modules to the spec-decode DRAFT model -> the +# BF16 mtp head gets quantized -> shape crash. Force-skip mtp.* in is_layer_skipped to keep it BF16. +import importlib.abc +import importlib.util +import sys + +TARGET = "vllm.model_executor.layers.quantization.utils.quant_utils" + + +class _Finder(importlib.abc.MetaPathFinder): + def find_spec(self, name, path, target=None): + if name != TARGET: + return None + sys.meta_path.remove(self) + try: + spec = importlib.util.find_spec(name) + finally: + sys.meta_path.insert(0, self) + if not spec or not spec.loader: + return None + _orig_exec = spec.loader.exec_module + + def exec_module(module): + _orig_exec(module) + _orig_isls = module.is_layer_skipped + + def is_layer_skipped(prefix, ignored_layers, *args, **kwargs): + pl = str(prefix) + if pl.startswith("mtp.") or ".mtp." in pl: + return True + return _orig_isls(prefix, ignored_layers, *args, **kwargs) + + module.is_layer_skipped = is_layer_skipped + print("[mtp-workaround] is_layer_skipped force-skip for mtp.* installed", flush=True) + + spec.loader.exec_module = exec_module + return spec + + +sys.meta_path.insert(0, _Finder())