vh
b972bef10e
Captures the full pipeline recipe (graft->quant->splice->config->serve) with every gotcha found this session, the 3 gibberish suspects, and the diagnostic ladder (validate native-config no-MTP coherence FIRST) for a fresh session to finish the chase. Also stages the NVFP4 scripts + 512-row calib. Recent decisions: NEO-CODE seat swap (R36), webhook ALLOWED_HOST_LIST fix. Lessons: validate-tracer-bullet-first, mtp-graft-dropped-at-load, gitea-204-red-herring.
21 lines
821 B
Bash
21 lines
821 B
Bash
#!/bin/bash
|
|
# Run the Heretic2 NVFP4 production quant on ana-ml2 GPU0 (freed for the window).
|
|
# llmcompressor + tokenizer deps pip-installed into a vLLM container (torch/CUDA ready).
|
|
set -euo pipefail
|
|
WORK=/tank/aimodels/heretic2-nvfp4-work
|
|
MODE="${1:-chat}" # chat = production 512-row mix; text = AEON baseline
|
|
CALIB="${2:-$WORK/production_calib_512.jsonl}"
|
|
OUT="${3:-$WORK/heretic2-mtp-nvfp4-prod}"
|
|
|
|
docker run --rm --gpus '"device=0"' --ipc host \
|
|
-v /tank/aimodels:/tank/aimodels \
|
|
--entrypoint bash vllm/vllm-openai:v0.24.0 -c "
|
|
set -e
|
|
pip install -q llmcompressor tiktoken sentencepiece 2>&1 | tail -1
|
|
python3 $WORK/quant_nvfp4.py \
|
|
--model $WORK/heretic2-mtp-bf16 \
|
|
--calib-mode $MODE --calib $CALIB \
|
|
--num-samples 512 --seqlen 8192 \
|
|
--out $OUT
|
|
"
|