From 725c8fdf9ebc975beb94e173a043c39d73b8836b Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Thu, 20 Aug 2026 10:18:48 -0700 Subject: [PATCH] =?UTF-8?q?feat(coldfusion-abliteration):=20THESIS=20PROVE?= =?UTF-8?q?N=20=E2=80=94=20in-band-abliterated=20MTP=20head=20accepts=2059?= =?UTF-8?q?.1%=20(beats=20incumbent=20~47%)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Quantized the L35 abliterated model to mixed NVFP4 and measured MTP acceptance end to end. The experiment's whole premise: Heretic (the incumbent gen seat) leaves the MTP head a byte-identical base graft its wrapper never loads, whereas Robinson abliterates the MTP head in-band — the question was whether that in-band edit survives well enough to spec-decode. It does, better than the graft: MTP acceptance 59.1% median (51-65%, 8 cache-busted topics) vs incumbent ~47% decode 118.7 tok/s median (faster; image-confounded, read as not-worse) abliteration survives quant (creative refusals drop, self-harm guardrail intact, coherent) Output at /tank/aimodels/qwen38-27b-coldfusion-L35-nvfp4-mixed (22.5 GB). Result JSON in bench/. NOT cut over — the incumbent seat is untouched; making L35 the gen seat is a separate decision needing the full Stage-3 gate + real multi-turn hold. Two env foot-guns hardened along the way: - quant_mixed_nvfp4.py now promotes text_config attention fields (num_attention_heads etc.) to the top-level config for the oneshot, then restores. transformers 5.10 / llmcompressor 0.12 (this venv moved under us since the Aug-15 heresy quant) no longer delegate the top-level lookup, so oneshot raised "Cannot determine num_attention_heads". Same "the fight is the environment" pattern as the abliteration capture. - a sub-~23GB quant saves as a single model.safetensors with no index, so the post_quant MTP graft needed an index built first — from the safetensors header, not safe_open (which mmaps the whole shard and ENOMEMs on ZFS). post_quant grafted the abliterated MTP (15 tensors, 849 MB) and re-injected re:^mtp.* into quantization_config.ignore (llm-compressor pruned it again — the two-rounds-lost 0%-MTP bug, fired and repaired as designed). Probe served on the pinned nightly (#51113 qwen3_5_mtp fix) to match the live seat's vLLM. --- services/coldfusion-abliteration/README.md | 43 +++++++++++++++++++ .../bench/mtp_coldfusion_L35.json | 29 +++++++++++++ .../gen-seat-mixed-quant/quant_mixed_nvfp4.py | 28 ++++++++++++ 3 files changed, 100 insertions(+) create mode 100644 services/gen-seat-mixed-quant/bench/mtp_coldfusion_L35.json diff --git a/services/coldfusion-abliteration/README.md b/services/coldfusion-abliteration/README.md index c209cbd..1f141b3 100644 --- a/services/coldfusion-abliteration/README.md +++ b/services/coldfusion-abliteration/README.md @@ -209,6 +209,49 @@ guess-and-retry. Agreement is still computed and printed, as a diagnostic. > problem was never the calibration set. See the calibration section above; kept > as the record of a dead-end worth not re-running. +## ✅ THESIS RESULT — the in-band-abliterated MTP head accepts BETTER than a graft (2026-08-20) + +The whole reason to abliterate Cold-Fusion ourselves rather than run the incumbent +Heretic seat: Heretic leaves the MTP head a **byte-identical base graft** (its +wrapper never loads it), while the Robinson formula abliterates the MTP head +**in-band** (its 2 residual writers). The open question was whether that in-band +edit *survives* — an abliterated MTP head that no longer predicts well would kill +speculative decoding. Measured, end to end: + +| metric | L35 quant | incumbent (heresy) | gate | verdict | +|---|---|---|---|---| +| **MTP acceptance** (median, 8 cache-busted topics) | **59.1%** (51–65%) | ~47% | ≳40% | **PASS — beats incumbent** | +| decode tok/s (median) | 118.7 | ~95–103 | ≥ incumbent | faster (⚠ image-confounded, read as "not worse") | +| abliteration survives quant | yes | — | creative↓, self-harm intact | **PASS** | +| coherence / no catatonia | clean | — | eyeball | **PASS** | + +So the in-band MTP abliteration doesn't merely preserve speculative decoding — the +abliterated head **accepts 59.1% vs the untouched graft's ~47%.** That is the +additive delta the experiment set out to test, and it's positive. + +**Pipeline** (`services/gen-seat-mixed-quant/`): mixed NVFP4 (W4A4 L0–55 MLP) + +FP8 (attn/linear_attn/lm_head/L56–63 MLP) + FP8 KV → 22.5 GB. Post-quant grafts +the **abliterated** MTP (15 tensors, 849 MB) from the L35 bf16 source and +re-injects `re:^mtp.*` into `quantization_config.ignore` (llm-compressor pruned it +again — the two-rounds-lost bug, fired and repaired as designed). Output: +`/tank/aimodels/qwen38-27b-coldfusion-L35-nvfp4-mixed`. Result JSON: +`services/gen-seat-mixed-quant/bench/mtp_coldfusion_L35.json`. + +> ⚠️ Env foot-gun banked: the quant venv's `transformers` moved to 5.10 / +> `llmcompressor` 0.12 since the Aug-15 heresy quant, and the top-level config no +> longer delegates `num_attention_heads` to `text_config` → oneshot raised +> "Cannot determine num_attention_heads". `quant_mixed_nvfp4.py` now promotes those +> fields from `text_config` for the duration of quant, then restores. Also: a +> small (<~23 GB) quant saves as a **single** `model.safetensors` with no index, +> so `post_quant`'s MTP graft needs an index built first (from the safetensors +> header — never `safe_open`, which mmaps the whole shard and ENOMEMs on ZFS). + +**NOT cut over.** The incumbent gen seat is untouched. Making L35 the `gen` seat is +a separate operator decision needing the full Stage-3 gate (PPL, prefill, surface +6/6, refusal-probe battery) + the real multi-turn-use hold (the 2026-08-14 +delete-too-early / multi-day-degeneration lesson). The thesis is proven; the +cutover is a distinct call. + ## Why the write is shard surgery, not `model.save_pretrained` The `--out` path edits the 18 safetensors shards directly and never instantiates diff --git a/services/gen-seat-mixed-quant/bench/mtp_coldfusion_L35.json b/services/gen-seat-mixed-quant/bench/mtp_coldfusion_L35.json new file mode 100644 index 0000000..4fb865f --- /dev/null +++ b/services/gen-seat-mixed-quant/bench/mtp_coldfusion_L35.json @@ -0,0 +1,29 @@ +{ + "tag": "coldfusion-L35", + "model": "probe", + "tok_s_median": 118.71736157977526, + "tok_s_mean": 118.55983517633987, + "tok_s_min": 109.41762922436195, + "tok_s_max": 127.07885746551419, + "mtp_accept_median": 0.5905651340996169, + "rates": [ + 118.19663140596533, + 115.20398398977953, + 113.01193271508572, + 109.41762922436195, + 127.07885746551419, + 119.25802285580335, + 119.23809175358517, + 127.07353200062373 + ], + "accs": [ + 0.5839080459770115, + 0.5592841163310962, + 0.5482456140350878, + 0.5138004246284501, + 0.654320987654321, + 0.5972222222222222, + 0.5972222222222222, + 0.654320987654321 + ] +} \ No newline at end of file diff --git a/services/gen-seat-mixed-quant/quant_mixed_nvfp4.py b/services/gen-seat-mixed-quant/quant_mixed_nvfp4.py index 9ff2d96..86990f8 100644 --- a/services/gen-seat-mixed-quant/quant_mixed_nvfp4.py +++ b/services/gen-seat-mixed-quant/quant_mixed_nvfp4.py @@ -126,6 +126,26 @@ def main(): model = Qwen3_5ForConditionalGeneration.from_pretrained( a.model, torch_dtype="auto", device_map=None, trust_remote_code=True) + # llmcompressor 0.12 introspects attention structure off the TOP-LEVEL config + # (to size kv-cache/quant params). Qwen3_5 keeps num_attention_heads etc. under + # text_config, and transformers 5.10 no longer delegates the top-level lookup, + # so oneshot raises "Cannot determine num_attention_heads from config". Promote + # them from the authoritative text_config for the duration of quant, then + # restore, so the saved config keeps its canonical text_config-only shape. + # (This env moved under us since the 2026-08-15 heresy quant, where the older + # transformers still delegated — same "the fight is the environment" pattern.) + _promote = ("num_attention_heads", "num_key_value_heads", "hidden_size", + "head_dim", "num_hidden_layers") + _tc = getattr(model.config, "text_config", None) + _orig = {f: getattr(model.config, f, None) for f in _promote} + if _tc is not None: + for f in _promote: + v = getattr(_tc, f, None) + if v is not None: + setattr(model.config, f, v) + print("promoted text_config attention fields to top-level config for oneshot: " + + ", ".join(f"{f}={getattr(model.config, f)}" for f in _promote), flush=True) + ds = load_calib(a.calib, tok, a.num_samples, a.seqlen) recipe = build_recipe() print("oneshot: NVFP4 W4A4 (L0-55 MLP) + FP8 W8A8 (attn/linear_attn/lm_head/L56-63 MLP) " @@ -133,6 +153,14 @@ def main(): oneshot(model=model, dataset=ds, recipe=recipe, num_calibration_samples=len(ds), max_seq_length=a.seqlen) + # restore the canonical config shape (undo the promotion above) so the saved + # top-level config matches the known-good heresy output; vLLM reads text_config. + for f, v in _orig.items(): + try: + setattr(model.config, f, v) + except Exception: + pass + print(f"saving -> {a.out}", flush=True) model.save_pretrained(a.out, save_compressed=True) tok.save_pretrained(a.out)