# MTP draft-model quant workaround for vLLM 0.24.0 — MUST be named sitecustomize.py and be on # PYTHONPATH so it loads in the vLLM engine-core subprocess. Mount its directory into the serve # container and set -e PYTHONPATH=. # # THE BUG: vLLM 0.24.0 does not propagate the main model's modelopt `exclude_modules` to the # spec-decode DRAFT model (Qwen3_5MTP). So the drafter builds its own qkv_proj/gate_up_proj as # NVFP4-quantized while the grafted MTP head is BF16 → `AssertionError: param_data.shape == # loaded_weight.shape` in qwen3_5_mtp.py:256, engine-core dies during weight load. Instrumenting # is_layer_skipped proved the drafter's exclude list only ever contains the *main* model's # entries, never the mtp ones — so no checkpoint config can fix it. (Also: is_layer_skipped does # exact string membership, not glob — wildcards like `mtp.layers.0.*` match nothing.) # # THE FIX: force is_layer_skipped to return True (skip = keep BF16) for any `mtp.*` layer, so the # draft head stays unquantized and its BF16 weights load. Report upstream: draft-model quant # config should inherit the target model's exclude_modules. import importlib.abc import importlib.util import sys TARGET = "vllm.model_executor.layers.quantization.utils.quant_utils" class _Finder(importlib.abc.MetaPathFinder): def find_spec(self, name, path, target=None): if name != TARGET: return None sys.meta_path.remove(self) try: spec = importlib.util.find_spec(name) finally: sys.meta_path.insert(0, self) if not spec or not spec.loader: return None _orig_exec = spec.loader.exec_module def exec_module(module): _orig_exec(module) _orig_isls = module.is_layer_skipped def is_layer_skipped(prefix, ignored_layers, *args, **kwargs): pl = str(prefix) if pl.startswith("mtp.") or ".mtp." in pl: return True # keep the mtp draft head BF16 return _orig_isls(prefix, ignored_layers, *args, **kwargs) module.is_layer_skipped = is_layer_skipped print("[mtp-workaround] is_layer_skipped force-skip for mtp.* installed", flush=True) spec.loader.exec_module = exec_module return spec sys.meta_path.insert(0, _Finder())