diff --git a/scripts/erp-tune-serve/merge_lora.py b/scripts/erp-tune-serve/merge_lora.py index 7c77a99..ea68c6b 100755 --- a/scripts/erp-tune-serve/merge_lora.py +++ b/scripts/erp-tune-serve/merge_lora.py @@ -97,6 +97,50 @@ def main() -> int: n_lines = len(src.read_text().splitlines()) print(f"[merge] chat_template.jinja <- {src} ({n_lines} lines)", flush=True) + # ⚠ CARRY THE PROCESSOR FILES. This is a multimodal checkpoint, so vLLM + # builds a feature extractor at startup and dies without them: + # OSError: Can't load feature extractor for '' + # `save_pretrained` on the merged model writes tokenizer files only, so + # anything else the base ships as auxiliary config must be copied across. + # Verified against the served nvfp4a16 artifact, which carries exactly this. + for aux in ("processor_config.json", "preprocessor_config.json", + "video_preprocessor_config.json", "special_tokens_map.json"): + src_aux = Path(a.base) / aux + if src_aux.exists() and not (out / aux).exists(): + shutil.copy2(src_aux, out / aux) + print(f"[merge] carried {aux}", flush=True) + + # ⚠⚠ CONFIG SCHEMA DOWNGRADE. transformers 5.15 MIGRATES Gemma-4's + # heterogeneous-attention config on save: it drops `global_head_dim` / + # `num_global_key_value_heads` and writes a `per_layer_config` dict instead. + # Older transformers (5.10, which is what the llmcompressor venv pins) does + # not understand the new key and resolves `config.num_key_value_heads` to + # None, dying with: + # TypeError: unsupported operand type(s) for //: 'int' and 'NoneType' + # Every WORKING artifact on this box - the bf16 base, the served nvfp4 prod + # seat, and the nvfp4a16 build - uses the OLD schema. Merging a LoRA changes + # weights, not architecture, so the base's expression of the architecture is + # the correct one to ship. + cfg_path = out / "config.json" + cfg = json.loads(cfg_path.read_text()) + base_cfg = json.loads((Path(a.base) / "config.json").read_text()) + ct, bt = cfg.get("text_config", cfg), base_cfg.get("text_config", base_cfg) + if "per_layer_config" in ct: + ct.pop("per_layer_config") + for k in ("global_head_dim", "num_global_key_value_heads"): + if k in bt: + ct[k] = bt[k] + cfg_path.write_text(json.dumps(cfg, indent=2) + "\n") + print("[merge] config schema downgraded to match the base " + "(per_layer_config -> global_head_dim/num_global_key_value_heads)", + flush=True) + ct2 = json.loads(cfg_path.read_text()).get("text_config", {}) + for k in ("global_head_dim", "num_global_key_value_heads"): + if bt.get(k) is not None and ct2.get(k) != bt.get(k): + print(f"REFUSING: {k} is {ct2.get(k)}, base says {bt.get(k)}", + file=sys.stderr) + return 6 + tj = out / "tokenizer.json" if tj.exists() and json.loads(tj.read_text()).get("truncation"): print("REFUSING: shipped tokenizer carries a truncation cap", file=sys.stderr) diff --git a/scripts/erp-tune-serve/quant_nvfp4a16.py b/scripts/erp-tune-serve/quant_nvfp4a16.py index ff56014..64bb361 100755 --- a/scripts/erp-tune-serve/quant_nvfp4a16.py +++ b/scripts/erp-tune-serve/quant_nvfp4a16.py @@ -146,8 +146,14 @@ def main() -> int: ) print("[oneshot] starting", flush=True) + # ⚠ `processor` must be passed EXPLICITLY. This is a multimodal + # (Gemma4ForConditionalGeneration) checkpoint, and llmcompressor's + # auto-init fails on it with "An error occurred when attempting to + # initialize model processor, which is required when a dataset is + # provided." Calibration here is text-only - the records come from the + # training encode cache - so the tokenizer is the correct processor. oneshot( - model=model, dataset=ds, recipe=recipe, + model=model, dataset=ds, recipe=recipe, processor=tok, max_seq_length=a.seqlen, num_calibration_samples=len(ds), output_dir=str(out), ) @@ -155,10 +161,14 @@ def main() -> int: # ⚠ playbook 3.14 - NEVER ship the calibration tokenizer. Re-read pristine. AutoTokenizer.from_pretrained(a.model, trust_remote_code=True).save_pretrained(out) - src_tpl = Path(a.model) / "chat_template.jinja" - if src_tpl.exists(): - (out / "chat_template.jinja").write_text(src_tpl.read_text()) - print("[post] chat_template.jinja carried over", flush=True) + import shutil as _sh + for aux in ("chat_template.jinja", "processor_config.json", + "preprocessor_config.json", "video_preprocessor_config.json", + "special_tokens_map.json", "generation_config.json"): + src_aux = Path(a.model) / aux + if src_aux.exists(): + _sh.copy2(src_aux, out / aux) + print(f"[post] carried {aux}", flush=True) tj = out / "tokenizer.json" if tj.exists() and json.loads(tj.read_text()).get("truncation"): diff --git a/scripts/erp-tune-serve/quant_with_gen_down.sh b/scripts/erp-tune-serve/quant_with_gen_down.sh new file mode 100644 index 0000000..4707cfa --- /dev/null +++ b/scripts/erp-tune-serve/quant_with_gen_down.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Run the NVFP4A16 quant with `vllm-gen` temporarily stopped to free GPU1, +# and ALWAYS bring gen back - crash, OOM, kill, or success. +# +# Operator authorised downing GPU1 residents overnight (2026-08-25) on the +# condition they are restored. The restore therefore must NOT depend on the +# calling session surviving, so it lives in a trap rather than in the caller. +# +# `docker start` (not `compose up`) is deliberate: it restarts the EXISTING +# container with its exact original config, so there is no chance of compose +# recreating the seat with drifted settings or a different image tag. +# +# gen serves `qwen3.8-27b-uncensored` + `-thinking`, and is the backing seat +# for the fleet-wide `summarizer` / `classifier` aliases. Keep the window short. +set -uo pipefail + +GEN=vllm-gen +LOG=/tank/erp-tune/serve/quant.log +OUT=/tank/erp-tune/serve/nvfp4a16-test + +restore() { + echo "[restore] starting $GEN ..." | tee -a "$LOG" + docker start "$GEN" >/dev/null 2>&1 + for i in $(seq 1 60); do + st=$(docker inspect -f '{{.State.Health.Status}}' "$GEN" 2>/dev/null || echo unknown) + run=$(docker inspect -f '{{.State.Running}}' "$GEN" 2>/dev/null || echo false) + if [ "$st" = "healthy" ]; then echo "[restore] $GEN healthy" | tee -a "$LOG"; return 0; fi + if [ "$run" != "true" ] && [ "$i" -gt 3 ]; then + echo "[restore] ⚠ $GEN NOT RUNNING - MANUAL ACTION NEEDED" | tee -a "$LOG"; return 1 + fi + sleep 10 + done + echo "[restore] ⚠ $GEN started but not healthy after 600s - CHECK IT" | tee -a "$LOG" + return 1 +} +trap restore EXIT INT TERM + +echo "[gen] stopping $GEN to free GPU1" | tee -a "$LOG" +docker stop "$GEN" >/dev/null 2>&1 +sleep 8 +nvidia-smi --query-gpu=index,memory.used,memory.total --format=csv,noheader | tee -a "$LOG" + +rm -rf "$OUT" +cd /tank/erp-tune/serve +CUDA_VISIBLE_DEVICES=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \ + /tank/aimodels/quant-work/.venv/bin/python -u quant_nvfp4a16.py \ + --model /tank/erp-tune/serve/merged-test \ + --out "$OUT" \ + --calib-cache /tank/erp-tune/run-01/encode-cache/encoded-a4b0796de1260930.jsonl \ + --num-calib "${NUM_CALIB:-16}" --seqlen "${SEQLEN:-4096}" >> "$LOG" 2>&1 +rc=$? +echo "[quant] exit rc=$rc" | tee -a "$LOG" +exit $rc