vh
74f596b1d3
Re-quantizes the fleet `gen` seat from weight-only NVFP4A16 to a mixed-precision build: NVFP4 W4A4 for layers 0-55 MLPs, FP8 W8A8 for the attention projections / linear_attn / lm_head / layers 56-63 MLPs, FP8 KV cache. Replicates the scheme of unsloth/Qwen3.8-27B-NVFP4 on the abliterated weights. The queued task named this "W4A8" (NVFP4 weights + FP8 activations). That checkpoint cannot be served: vLLM 0.24's compressed-tensors dispatcher (compressed_tensors.py:704-713) accepts NVFP4 weights with either no input quantization (W4A16, which forces the Marlin kernel) or NVFP4 input quantization (W4A4) -- anything else, FP8 included, raises ValueError at load. CompressedTensorsW4A8Fp8 is INT4 weights gated on an exact-sm90 check, so it is closed on Blackwell twice over. The ~20% intuition was correct; the scheme name was not. Getting FP8 into the mix has to be done per-layer-group. Established the gain before spending GPU time: unsloth's build was already on-box, so serving it as a probe measured +19.1% over our seat at identical MTP acceptance -- a kernel-level result, no requant needed to learn it. Measured, cache-busted, bs=1: decode 80.12 -> 94.53 tok/s (+18.0%) MTP acceptance 47.8% -> 47.7% (unchanged) perplexity (n=6) 6.941 -> 7.059 (+1.7%) abliteration 4/4 -> 4/4 (preserved) weights on disk 27.7 -> 22.5 GB (-19%) Surface test green on the live seat: plain chat, vision, tool calling, thinking split, 36K-token needle retrieval, streaming. All 7 LiteLLM aliases verified routing. GEN_GPU_MEM_UTIL 0.45 -> 0.43: the new weights are 5.2 GB smaller, and at 0.45 the seat absorbed that slack as KV, leaving meromero-charrp 0.18 GiB short of its budget on the shared GPU0 -- it crash-looped. Handing the space back leaves gen 422K tokens of KV (1.6x its 262K context) and both seats co-resident at 89.8/97.9 GB. Also records two measured negatives so they are not re-chased: GEN_SPEC_TOKENS is already optimal at 3 (swept 2/3/4/5 -> 77.1/80.1/78.7/ 75.9 tok/s), and vLLM's prompt_logprobs are ~uniform while speculative decoding is on, so perplexity must be measured with spec off. Pipeline, acceptance harness and raw measurements land in services/gen-seat-mixed-quant/. Rollback is one .env line; the previous build is untouched at /tank/aimodels/qwen38-27b-uncensored-nvfp4.
64 lines
2.5 KiB
Python
64 lines
2.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Dry-run the quant target regexes against the real module names from the
|
|
safetensors index. Verifies: full coverage of Linear weights, zero overlap
|
|
between groups, and that everything intentionally excluded is excluded."""
|
|
import json, re, sys, collections
|
|
|
|
BASE = sys.argv[1]
|
|
G0 = [r".*self_attn\.(q|k|v|o)_proj$",
|
|
r".*linear_attn\.(in_proj_qkv|in_proj_z|out_proj)$",
|
|
r".*lm_head",
|
|
r".*layers\.(56|57|58|59|60|61|62|63)\.mlp\.(gate|up|down)_proj$"]
|
|
G1 = [r".*layers\.([0-9]|[1-4][0-9]|5[0-5])\.mlp\.(gate|up|down)_proj$"]
|
|
IG = [r".*visual.*", r".*linear_attn\.(norm|in_proj_a|in_proj_b)$", r"^mtp.*"]
|
|
|
|
idx = json.load(open(BASE + "/model.safetensors.index.json"))
|
|
mods = sorted({k.rsplit(".", 1)[0] for k in idx["weight_map"] if k.endswith(".weight")})
|
|
|
|
def hit(pats, name):
|
|
return any(re.fullmatch(p, name) for p in pats)
|
|
|
|
g0 = [m for m in mods if hit(G0, m)]
|
|
g1 = [m for m in mods if hit(G1, m)]
|
|
ig = [m for m in mods if hit(IG, m)]
|
|
overlap = sorted(set(g0) & set(g1))
|
|
covered = set(g0) | set(g1) | set(ig)
|
|
uncov = [m for m in mods if m not in covered]
|
|
|
|
print(f"total modules with .weight : {len(mods)}")
|
|
print(f"group_0 (FP8 W8A8) : {len(g0)}")
|
|
print(f"group_1 (NVFP4 W4A4) : {len(g1)}")
|
|
print(f"ignored : {len(ig)}")
|
|
print(f"OVERLAP g0&g1 : {len(overlap)} {'<-- BUG' if overlap else 'OK'}")
|
|
if overlap:
|
|
print(" ", overlap[:10])
|
|
|
|
# sanity: which layers landed in which MLP group
|
|
def layers_of(lst, kind):
|
|
out = set()
|
|
for m in lst:
|
|
mm = re.search(r"layers\.(\d+)\.mlp\.", m)
|
|
if mm:
|
|
out.add(int(mm.group(1)))
|
|
return sorted(out)
|
|
|
|
l0, l1 = layers_of(g0, "g0"), layers_of(g1, "g1")
|
|
print(f"\nMLP layers -> FP8 : {l0[:3]}..{l0[-3:] if l0 else []} (n={len(l0)})")
|
|
print(f"MLP layers -> NVFP4 : {l1[:3]}..{l1[-3:] if l1 else []} (n={len(l1)})")
|
|
print(f"MLP layer union covers 0-63: {sorted(set(l0)|set(l1)) == list(range(64))}")
|
|
|
|
print("\nuncovered modules (neither quantized nor explicitly ignored):", len(uncov))
|
|
buck = collections.Counter()
|
|
for m in uncov:
|
|
if "visual" in m: buck["visual"] += 1
|
|
elif "mtp" in m: buck["mtp"] += 1
|
|
elif "norm" in m: buck["norm"] += 1
|
|
elif "embed" in m: buck["embed"] += 1
|
|
elif "linear_attn" in m: buck["linear_attn"] += 1
|
|
else: buck["OTHER:" + m.split(".")[-1]] += 1
|
|
for k, v in buck.most_common():
|
|
print(f" {k}: {v}")
|
|
oth = [m for m in uncov if not any(s in m for s in ("visual", "mtp", "norm", "embed", "linear_attn"))]
|
|
if oth:
|
|
print(" uncovered non-norm/embed sample:", oth[:12])
|