vh
b972bef10e
Captures the full pipeline recipe (graft->quant->splice->config->serve) with every gotcha found this session, the 3 gibberish suspects, and the diagnostic ladder (validate native-config no-MTP coherence FIRST) for a fresh session to finish the chase. Also stages the NVFP4 scripts + 512-row calib. Recent decisions: NEO-CODE seat swap (R36), webhook ALLOWED_HOST_LIST fix. Lessons: validate-tracer-bullet-first, mtp-graft-dropped-at-load, gitea-204-red-herring.
21 lines
1.0 KiB
Bash
21 lines
1.0 KiB
Bash
#!/bin/bash
|
|
# Serve the NVFP4+MTP Heretic2 seat on ana-ml2 GPU0 for P00 validation.
|
|
# vLLM compressed-tensors NVFP4 + qwen3_5_mtp spec-decode (pantheon pattern).
|
|
# Served as char-rp-reasoning on :8018 (the GGUF seat's port; that seat is stopped).
|
|
set -euo pipefail
|
|
WORK=/tank/aimodels/heretic2-nvfp4-work
|
|
docker rm -f vllm-charrp-nvfp4-test 2>/dev/null || true
|
|
|
|
docker run -d --name vllm-charrp-nvfp4-test --gpus '"device=0"' --ipc host \
|
|
-v /tank/aimodels:/tank/aimodels -p 8018:8000 \
|
|
vllm/vllm-openai:v0.24.0 \
|
|
"$WORK/heretic2-mtp-nvfp4-prod" \
|
|
--quantization compressed-tensors \
|
|
--speculative-config '{"method":"qwen3_5_mtp","num_speculative_tokens":3}' \
|
|
--reasoning-parser qwen3 --tool-call-parser qwen3_coder --enable-auto-tool-choice \
|
|
--language-model-only \
|
|
--mamba-cache-dtype float32 \
|
|
--served-model-name char-rp-reasoning \
|
|
--max-model-len 40960 --max-num-seqs 256 --gpu-memory-utilization 0.45 --trust-remote-code
|
|
echo "container started: $(docker ps --filter name=vllm-charrp-nvfp4-test --format '{{.Status}}')"
|