feat(heretic2-nvfp4): WORKING modelopt NVFP4+MTP seat + full recipe runbook

The fast char-rp-reasoning seat works: ~77 tok/s (vs GGUF ~59.5, base NVFP4 ~53),
MTP draft-acceptance 32-40%, mean acceptance length 2.19. Same Heretic2/NEO-CODE
model, NVFP4 + native qwen3_5_mtp spec-decode.

Full end-to-end recipe + the four landmines in docs/runbooks/heretic2-nvfp4-mtp-seat.md:
(1) load as AutoModelForImageTextToText not AutoModelForCausalLM (namespace/gibberish);
(2) modelopt format not compressed-tensors (compressed-tensors MTP = 0% accept);
(3) modelopt 0.45 <-> transformers 5.12.1 FusedMoE crash (guarded in quant_modelopt.py);
(4) vLLM 0.24.0 does NOT propagate modelopt exclude_modules to the spec-decode draft
model -> BF16 mtp head gets quantized -> shape crash; no checkpoint config fixes it
(is_layer_skipped is exact-membership not glob) -> fix is a mounted sitecustomize that
force-skips mtp.* in is_layer_skipped (upstream vLLM bug to report).

Scripts: quant_modelopt.py (FusedMoE guard + single-shard export + multimodal load),
finalize_modelopt_mtp.py (splice bf16 mtp), serve_modelopt_mtp.sh, run_quant_modelopt.sh,
sitecustomize-mtp-workaround.py.

services/heretic2-nvfp4-quant/finalize_modelopt_mtp.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+"""Splice the 15 BF16 mtp.* tensors into the modelopt NVFP4 quant output → the servable seat.
+
+Runs AFTER quant_modelopt.py. Copies heretic2-modelopt-nvfp4 → heretic2-modelopt-nvfp4-mtp,
+splices the grafted BF16 mtp head into the single shard (transformers never builds an mtp module
+at load, so mtp is always post-quant-spliced — same as AEON/pantheon), and adds the mtp module
+names to config.json exclude_modules for tidiness. NOTE: the actual thing that keeps the mtp head
+BF16 at serve time is the sitecustomize MTP workaround (see runbook landmine #4); the config
+exclude here is belt-and-suspenders and does NOT by itself prevent the drafter-quant crash.
+
+Run in a vLLM container (root; /tank/aimodels files are root-owned):
+  docker run --rm -v /tank/aimodels:/tank/aimodels -v /home/lkraven:/lk \
+    --entrypoint python3 vllm/vllm-openai:v0.24.0 /lk/finalize_modelopt_mtp.py
+"""
+import json
+import os
+import shutil
+
+from safetensors import safe_open
+from safetensors.torch import save_file
+
+WORK = "/tank/aimodels/heretic2-nvfp4-work"
+SRC = f"{WORK}/heretic2-modelopt-nvfp4"
+DST = f"{WORK}/heretic2-modelopt-nvfp4-mtp"
+GRAFT = f"{WORK}/heretic2-mtp-bf16"
+
+if os.path.exists(DST):
+    shutil.rmtree(DST)
+print(f"copying {SRC} -> {DST}", flush=True)
+shutil.copytree(SRC, DST)
+
+out_st = f"{DST}/model.safetensors"   # single shard (quant_modelopt.py forces max_shard_size huge)
+mtp_st = f"{GRAFT}/model-mtp.safetensors"
+tensors = {}
+with safe_open(out_st, framework="pt") as f:
+    for k in f.keys():
+        tensors[k] = f.get_tensor(k)
+n_main = len(tensors)
+with safe_open(mtp_st, framework="pt") as f:
+    mtp_keys = list(f.keys())
+    for k in mtp_keys:
+        tensors[k] = f.get_tensor(k)
+assert not any("mtp" in k.lower() for k in list(tensors)[:n_main]), "output already had mtp?"
+save_file(tensors, out_st, metadata={"format": "pt"})
+print(f"spliced {len(mtp_keys)} bf16 mtp tensors -> {len(tensors)} total", flush=True)
+
+cfgp = f"{DST}/config.json"
+cfg = json.load(open(cfgp))
+qc = cfg.setdefault("quantization_config", {})
+exc = qc.setdefault("exclude_modules", [])
+mtp_mods = sorted({k.rsplit(".", 1)[0] for k in mtp_keys})
+exc.extend(m for m in mtp_mods if m not in exc)
+json.dump(cfg, open(cfgp, "w"), indent=2)
+print(f"config exclude_modules += {len(mtp_mods)} mtp modules; total {len(exc)}", flush=True)
+print(f"DONE: {DST}", flush=True)

services/heretic2-nvfp4-quant/quant_modelopt.py

@@ -93,6 +93,22 @@ def main() -> int:
     import modelopt.torch.quantization as mtq
     from modelopt.torch.export import export_hf_checkpoint
 
+    # modelopt 0.45 + transformers 5.12.1 compat guard. transformers 5.x exposes `FusedMoE` as a
+    # FUNCTION, but modelopt registers it in QuantModuleRegistry expecting an nn.Module class, so the
+    # registry scan (register_fused_experts_on_the_fly -> _get_registered_nn_class) does
+    # `issubclass(nn_cls, <function FusedMoE>)` and dies with "arg 2 must be a class". Our model is
+    # DENSE (no FusedMoE) so skipping non-class registry entries is safe. Guard the scan:
+    from modelopt.torch.opt import dynamic as _mo_dyn
+
+    def _grnc_safe(self, nn_cls):
+        for nn_cls_ in self._registry:
+            if (isinstance(nn_cls_, type) and issubclass(nn_cls, nn_cls_)
+                    and nn_cls.forward is nn_cls_.forward):
+                return nn_cls_
+        return None
+
+    _mo_dyn._DMRegistryCls._get_registered_nn_class = _grnc_safe
+
     print(f"loading grafted model (multimodal ConditionalGeneration): {args.model}", flush=True)
     model = AutoModelForImageTextToText.from_pretrained(
         args.model, torch_dtype="auto", device_map="auto", trust_remote_code=True,
@@ -120,7 +136,9 @@ def main() -> int:
     mtq.quantize(model, cfg, forward_loop=forward_loop)
 
     print(f"exporting modelopt HF checkpoint -> {args.out}", flush=True)
-    export_hf_checkpoint(model, export_dir=args.out)
+    # Force a SINGLE shard (default max_shard_size 10GB would split the ~14GB output into 3 shards,
+    # but splice_mtp.py expects a single <out>/model.safetensors to add the bf16 mtp.* into).
+    export_hf_checkpoint(model, export_dir=args.out, max_shard_size="1TB")
     tok.save_pretrained(args.out)
     print("DONE. Next: splice_mtp.py <out> <graft>  then serve "
           "--quantization modelopt --speculative-config "

services/heretic2-nvfp4-quant/run_quant_modelopt.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+# Launch the modelopt NVFP4 quant of the grafted Heretic2 on ana-ml2 GPU0 (detached, survives ssh
+# drop). bare nvidia-modelopt (0.45). The FusedMoE-compat guard + single-shard export + the
+# multimodal load class are all inside quant_modelopt.py. ~18 min. Output: heretic2-modelopt-nvfp4
+# (no mtp yet — run finalize_modelopt_mtp.py after). quant_modelopt.py must be at /lk/quant_modelopt.py
+# (mount /home/lkraven as /lk, or scp it there first).
+set -euo pipefail
+docker rm -f vllm-heretic2-modelopt-quant 2>/dev/null || true
+rm -rf /tank/aimodels/heretic2-nvfp4-work/heretic2-modelopt-nvfp4 2>/dev/null || true
+docker run -d --name vllm-heretic2-modelopt-quant --gpus '"device=0"' --ipc host \
+  -v /tank/aimodels:/tank/aimodels -v /home/lkraven:/lk \
+  --entrypoint bash vllm/vllm-openai:v0.24.0 -c '
+    set -e
+    pip install -q nvidia-modelopt tiktoken sentencepiece 2>&1 | tail -1
+    python3 /lk/quant_modelopt.py \
+      --model /tank/aimodels/heretic2-nvfp4-work/heretic2-mtp-bf16 \
+      --calib-mode chat --calib /tank/aimodels/heretic2-nvfp4-work/production_calib_512.jsonl \
+      --num-samples 512 --seqlen 8192 \
+      --out /tank/aimodels/heretic2-nvfp4-work/heretic2-modelopt-nvfp4'
+echo "LAUNCHED: $(docker ps --filter name=vllm-heretic2-modelopt-quant --format '{{.Status}}')"

services/heretic2-nvfp4-quant/serve_modelopt_mtp.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# Serve the modelopt NVFP4 + MTP Heretic2 seat (the WORKING fast char-rp-reasoning seat).
+# ~77 tok/s, MTP acceptance 32-40%. Requires: (1) heretic2-modelopt-nvfp4-mtp built (quant ->
+# finalize), (2) the sitecustomize MTP workaround mounted on PYTHONPATH (vLLM 0.24 draft-model
+# exclude bug — see runbook landmine #4; without it the engine crashes on a shape mismatch).
+set -euo pipefail
+MODEL="${1:-/tank/aimodels/heretic2-nvfp4-work/heretic2-modelopt-nvfp4-mtp}"
+# Dir containing sitecustomize.py (a copy of sitecustomize-mtp-workaround.py named sitecustomize.py):
+WORKAROUND_DIR="${MTP_WORKAROUND_DIR:-/home/lkraven/isls_debug}"
+docker rm -f vllm-charrp-modelopt 2>/dev/null || true
+docker run -d --name vllm-charrp-modelopt --gpus '"device=0"' --ipc host \
+  -v /tank/aimodels:/tank/aimodels \
+  -v "${WORKAROUND_DIR}":/lk_debug -e PYTHONPATH=/lk_debug \
+  -p 8018:8000 \
+  vllm/vllm-openai:v0.24.0 \
+  "$MODEL" \
+  --quantization modelopt \
+  --speculative-config '{"method":"qwen3_5_mtp","num_speculative_tokens":3}' \
+  --language-model-only \
+  --mamba-cache-dtype float32 \
+  --reasoning-parser qwen3 --tool-call-parser qwen3_coder --enable-auto-tool-choice \
+  --served-model-name char-rp-reasoning \
+  --max-model-len 40960 --max-num-seqs 32 --gpu-memory-utilization 0.5 --trust-remote-code
+echo "started: $(docker ps --filter name=vllm-charrp-modelopt --format '{{.Status}}')"
+echo "verify MTP: docker logs vllm-charrp-modelopt 2>&1 | grep -E 'mtp-workaround|SpecDecoding'"

services/heretic2-nvfp4-quant/sitecustomize-mtp-workaround.py

@@ -0,0 +1,53 @@
+# MTP draft-model quant workaround for vLLM 0.24.0 — MUST be named sitecustomize.py and be on
+# PYTHONPATH so it loads in the vLLM engine-core subprocess. Mount its directory into the serve
+# container and set -e PYTHONPATH=<mount>.
+#
+# THE BUG: vLLM 0.24.0 does not propagate the main model's modelopt `exclude_modules` to the
+# spec-decode DRAFT model (Qwen3_5MTP). So the drafter builds its own qkv_proj/gate_up_proj as
+# NVFP4-quantized while the grafted MTP head is BF16 → `AssertionError: param_data.shape ==
+# loaded_weight.shape` in qwen3_5_mtp.py:256, engine-core dies during weight load. Instrumenting
+# is_layer_skipped proved the drafter's exclude list only ever contains the *main* model's
+# entries, never the mtp ones — so no checkpoint config can fix it. (Also: is_layer_skipped does
+# exact string membership, not glob — wildcards like `mtp.layers.0.*` match nothing.)
+#
+# THE FIX: force is_layer_skipped to return True (skip = keep BF16) for any `mtp.*` layer, so the
+# draft head stays unquantized and its BF16 weights load. Report upstream: draft-model quant
+# config should inherit the target model's exclude_modules.
+import importlib.abc
+import importlib.util
+import sys
+
+TARGET = "vllm.model_executor.layers.quantization.utils.quant_utils"
+
+
+class _Finder(importlib.abc.MetaPathFinder):
+    def find_spec(self, name, path, target=None):
+        if name != TARGET:
+            return None
+        sys.meta_path.remove(self)
+        try:
+            spec = importlib.util.find_spec(name)
+        finally:
+            sys.meta_path.insert(0, self)
+        if not spec or not spec.loader:
+            return None
+        _orig_exec = spec.loader.exec_module
+
+        def exec_module(module):
+            _orig_exec(module)
+            _orig_isls = module.is_layer_skipped
+
+            def is_layer_skipped(prefix, ignored_layers, *args, **kwargs):
+                pl = str(prefix)
+                if pl.startswith("mtp.") or ".mtp." in pl:
+                    return True  # keep the mtp draft head BF16
+                return _orig_isls(prefix, ignored_layers, *args, **kwargs)
+
+            module.is_layer_skipped = is_layer_skipped
+            print("[mtp-workaround] is_layer_skipped force-skip for mtp.* installed", flush=True)
+
+        spec.loader.exec_module = exec_module
+        return spec
+
+
+sys.meta_path.insert(0, _Finder())