snapshot: NVFP4+MTP fast-seat quant recipe + failure state (gibberish, unisolated)
Captures the full pipeline recipe (graft->quant->splice->config->serve) with every gotcha found this session, the 3 gibberish suspects, and the diagnostic ladder (validate native-config no-MTP coherence FIRST) for a fresh session to finish the chase. Also stages the NVFP4 scripts + 512-row calib. Recent decisions: NEO-CODE seat swap (R36), webhook ALLOWED_HOST_LIST fix. Lessons: validate-tracer-bullet-first, mtp-graft-dropped-at-load, gitea-204-red-herring.
This commit is contained in:
@@ -1,39 +1,30 @@
|
||||
#!/usr/bin/env python3
|
||||
"""NVFP4 quantize the MTP-grafted Heretic2 model (llm-compressor / compressed-tensors).
|
||||
|
||||
Runs AFTER graft_mtp.py. Uses the fleet-proven compressed-tensors NVFP4 path
|
||||
(the pantheon-27b-mtp-nvfp4 serving pattern already works on our Blackwell) with
|
||||
the ignore-list from robbatt's on-fleet deckard-nvfp4 recipe PLUS the MTP head:
|
||||
keep GDN/linear-attn, vision tower, lm-head, all norms, AND mtp.* in BF16;
|
||||
NVFP4 only the dense Linear layers. (R36 fast-seat spike, brokkr calib spec.)
|
||||
Runs AFTER graft_mtp.py. Fleet-proven compressed-tensors NVFP4 path (pantheon-27b-
|
||||
mtp-nvfp4 serves this on our Blackwell) with the ignore-list from robbatt's on-fleet
|
||||
deckard-nvfp4 recipe PLUS the MTP head: keep GDN/linear-attn, vision, lm-head, all
|
||||
norms, AND mtp.* in BF16; NVFP4 only the dense Linear layers. (R36 fast-seat spike.)
|
||||
|
||||
⚠️ NEEDS: (a) an env with llmcompressor + a CUDA torch (run inside a vLLM
|
||||
container: `pip install llmcompressor` on vllm/vllm-openai:v0.24.0), (b) a freed
|
||||
Blackwell GPU (~55 GB — both ana-ml2 GPUs are normally full; needs an off-peak
|
||||
window). API validated against llm-compressor at run time — treat the exact
|
||||
symbol names as first-draft until a dry import confirms them.
|
||||
API validated against llmcompressor 0.12.0: oneshot(model, dataset, recipe,
|
||||
num_calibration_samples, max_seq_length) — dataset is a pre-tokenized datasets.Dataset.
|
||||
|
||||
Two calib modes (brokkr's two artifacts):
|
||||
--calib-mode text : AEON-baseline control (neuralmagic/calibration LLM split)
|
||||
--calib-mode chat : production 512-row mix (JSONL rows {messages, tools});
|
||||
each row rendered via apply_chat_template(enable_thinking=True)
|
||||
so the forward-pass sees the qwen3_coder tool-call XML =
|
||||
the seat's native activations (the #355-preservation point).
|
||||
Two calib modes:
|
||||
--calib-mode text : AEON-baseline control (neuralmagic/calibration LLM split)
|
||||
--calib-mode chat : production 512-row mix (JSONL {messages, tools}); each row
|
||||
rendered via apply_chat_template(enable_thinking=True) so the
|
||||
forward-pass sees the qwen3_coder tool-call XML the seat emits.
|
||||
|
||||
Usage:
|
||||
python3 quant_nvfp4.py --model /tank/aimodels/heretic2-mtp-bf16 \
|
||||
--calib-mode text --calib neuralmagic/calibration --num-samples 160 \
|
||||
--out /tank/aimodels/heretic2-mtp-nvfp4-baseline
|
||||
python3 quant_nvfp4.py --model /tank/aimodels/heretic2-mtp-bf16 \
|
||||
--calib-mode chat --calib /path/to/production_calib_512.jsonl \
|
||||
--out /tank/aimodels/heretic2-mtp-nvfp4-prod
|
||||
Run in a vLLM container on the freed GPU0:
|
||||
docker run --gpus '"device=0"' --ipc host -v /tank/aimodels:/tank/aimodels \
|
||||
--entrypoint bash vllm/vllm-openai:v0.24.0 -c \
|
||||
"pip install -q llmcompressor tiktoken sentencepiece && python3 quant_nvfp4.py ..."
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
|
||||
# Ignore list = robbatt deckard-nvfp4 recipe + MTP head (brokkr: keep mtp.* BF16).
|
||||
# Keeps in BF16: lm-head, embeddings, vision tower, GDN/linear-attn, all norms, MTP.
|
||||
IGNORE = [
|
||||
"lm_head",
|
||||
"re:.*embed_tokens$",
|
||||
@@ -43,27 +34,31 @@ IGNORE = [
|
||||
"re:.*norm.*",
|
||||
"re:.*q_norm.*",
|
||||
"re:.*k_norm.*",
|
||||
"re:mtp.*", # MTP head stays BF16 for the qwen3_5_mtp spec-decode head
|
||||
"re:.*mtp.*", # MTP head stays BF16 for qwen3_5_mtp spec-decode (match anywhere:
|
||||
# module path is model.mtp.*, so an anchored re:mtp.* misses it)
|
||||
]
|
||||
|
||||
|
||||
def _tok_rows_to_dataset(tok_rows):
|
||||
from datasets import Dataset
|
||||
return Dataset.from_list(tok_rows)
|
||||
|
||||
|
||||
def load_calib_text(name, tokenizer, n, seqlen):
|
||||
from datasets import load_dataset
|
||||
ds = load_dataset(name, split="train").shuffle(seed=42).select(range(n))
|
||||
ds = load_dataset(name, split="train").shuffle(seed=42).select(range(min(n, 100000)))
|
||||
col = "text" if "text" in ds.column_names else ds.column_names[0]
|
||||
return [tokenizer(x[col], truncation=True, max_length=seqlen) for x in ds]
|
||||
rows = [tokenizer(x[col], truncation=True, max_length=seqlen) for x in ds.select(range(n))]
|
||||
return _tok_rows_to_dataset(rows)
|
||||
|
||||
|
||||
def load_calib_chat(path, tokenizer, seqlen):
|
||||
"""Render {messages, tools} JSONL rows through the chat template with thinking on.
|
||||
The assistant tool_calls (OpenAI form) serialize to qwen3_coder XML here, so the
|
||||
calibration forward-pass sees the EXACT tool-call token distribution the seat emits."""
|
||||
rows = [json.loads(l) for l in open(path) if l.strip()]
|
||||
def load_calib_chat(path, tokenizer, seqlen, n):
|
||||
rows = [json.loads(l) for l in open(path) if l.strip()][:n]
|
||||
out = []
|
||||
for r in rows:
|
||||
# Tool_call arguments arrive as OpenAI wire-form JSON *strings*; the Qwen3.6
|
||||
# template does .items() on them → needs a dict. Parse string→dict so the
|
||||
# forward-pass sees the qwen3_coder XML the seat emits. (render_verify finding.)
|
||||
# Tool_call arguments may be OpenAI wire-form JSON strings; the Qwen3.6 template
|
||||
# does .items() on them → needs a dict. Parse string→dict (render_verify finding;
|
||||
# belt-and-suspenders even though brokkr canonicalized the rows to dicts).
|
||||
for m in r["messages"]:
|
||||
for tc in (m.get("tool_calls") or []):
|
||||
a = tc.get("function", {}).get("arguments")
|
||||
@@ -74,14 +69,14 @@ def load_calib_chat(path, tokenizer, seqlen):
|
||||
tokenize=False, add_generation_prompt=False, enable_thinking=True,
|
||||
)
|
||||
out.append(tokenizer(text, truncation=True, max_length=seqlen))
|
||||
return out
|
||||
return _tok_rows_to_dataset(out)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--model", required=True, help="grafted BF16 model dir (graft_mtp.py output)")
|
||||
ap.add_argument("--model", required=True)
|
||||
ap.add_argument("--calib-mode", choices=["text", "chat"], required=True)
|
||||
ap.add_argument("--calib", required=True, help="HF dataset name (text) or JSONL path (chat)")
|
||||
ap.add_argument("--calib", required=True)
|
||||
ap.add_argument("--out", required=True)
|
||||
ap.add_argument("--num-samples", type=int, default=512)
|
||||
ap.add_argument("--seqlen", type=int, default=8192)
|
||||
@@ -91,34 +86,33 @@ def main() -> int:
|
||||
from llmcompressor import oneshot
|
||||
from llmcompressor.modifiers.quantization import QuantizationModifier
|
||||
|
||||
print(f"loading grafted model: {args.model}")
|
||||
print(f"loading grafted model: {args.model}", flush=True)
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
args.model, torch_dtype="auto", device_map="auto", trust_remote_code=True,
|
||||
)
|
||||
tok = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
|
||||
|
||||
print(f"building calibration ({args.calib_mode}, {args.num_samples} samples, seq {args.seqlen})")
|
||||
print(f"building calibration ({args.calib_mode}, up to {args.num_samples} @ seq {args.seqlen})", flush=True)
|
||||
if args.calib_mode == "text":
|
||||
calib = load_calib_text(args.calib, tok, args.num_samples, args.seqlen)
|
||||
else:
|
||||
calib = load_calib_chat(args.calib, tok, args.seqlen)
|
||||
print(f" {len(calib)} calibration rows")
|
||||
calib = load_calib_chat(args.calib, tok, args.seqlen, args.num_samples)
|
||||
print(f" {len(calib)} calibration rows", flush=True)
|
||||
|
||||
recipe = QuantizationModifier(targets="Linear", scheme="NVFP4", ignore=IGNORE)
|
||||
|
||||
print("running NVFP4 oneshot (Linear-only; GDN/vision/lm-head/norms/MTP kept BF16)")
|
||||
print("running NVFP4 oneshot (Linear-only; GDN/vision/lm-head/norms/MTP kept BF16)", flush=True)
|
||||
oneshot(
|
||||
model=model, dataset=calib, recipe=recipe,
|
||||
max_seq_length=args.seqlen, num_calibration_samples=len(calib),
|
||||
num_calibration_samples=len(calib), max_seq_length=args.seqlen,
|
||||
)
|
||||
|
||||
print(f"saving -> {args.out}")
|
||||
print(f"saving -> {args.out}", flush=True)
|
||||
model.save_pretrained(args.out, save_compressed=True)
|
||||
tok.save_pretrained(args.out)
|
||||
print("DONE. serve: vllm --quantization compressed-tensors "
|
||||
"--speculative-config '{\"method\":\"qwen3_5_mtp\",\"num_speculative_tokens\":3}' "
|
||||
"--reasoning-parser qwen3 --tool-call-parser qwen3_coder --enable-auto-tool-choice")
|
||||
print("NEXT: ping brokkr -> P00 rig (soong 9-tool k5); acceptance = hold ~0.967/perfect attach_tool")
|
||||
"--reasoning-parser qwen3 --tool-call-parser qwen3_coder --enable-auto-tool-choice", flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user