vh
74f596b1d3
Re-quantizes the fleet `gen` seat from weight-only NVFP4A16 to a mixed-precision build: NVFP4 W4A4 for layers 0-55 MLPs, FP8 W8A8 for the attention projections / linear_attn / lm_head / layers 56-63 MLPs, FP8 KV cache. Replicates the scheme of unsloth/Qwen3.8-27B-NVFP4 on the abliterated weights. The queued task named this "W4A8" (NVFP4 weights + FP8 activations). That checkpoint cannot be served: vLLM 0.24's compressed-tensors dispatcher (compressed_tensors.py:704-713) accepts NVFP4 weights with either no input quantization (W4A16, which forces the Marlin kernel) or NVFP4 input quantization (W4A4) -- anything else, FP8 included, raises ValueError at load. CompressedTensorsW4A8Fp8 is INT4 weights gated on an exact-sm90 check, so it is closed on Blackwell twice over. The ~20% intuition was correct; the scheme name was not. Getting FP8 into the mix has to be done per-layer-group. Established the gain before spending GPU time: unsloth's build was already on-box, so serving it as a probe measured +19.1% over our seat at identical MTP acceptance -- a kernel-level result, no requant needed to learn it. Measured, cache-busted, bs=1: decode 80.12 -> 94.53 tok/s (+18.0%) MTP acceptance 47.8% -> 47.7% (unchanged) perplexity (n=6) 6.941 -> 7.059 (+1.7%) abliteration 4/4 -> 4/4 (preserved) weights on disk 27.7 -> 22.5 GB (-19%) Surface test green on the live seat: plain chat, vision, tool calling, thinking split, 36K-token needle retrieval, streaming. All 7 LiteLLM aliases verified routing. GEN_GPU_MEM_UTIL 0.45 -> 0.43: the new weights are 5.2 GB smaller, and at 0.45 the seat absorbed that slack as KV, leaving meromero-charrp 0.18 GiB short of its budget on the shared GPU0 -- it crash-looped. Handing the space back leaves gen 422K tokens of KV (1.6x its 262K context) and both seats co-resident at 89.8/97.9 GB. Also records two measured negatives so they are not re-chased: GEN_SPEC_TOKENS is already optimal at 3 (swept 2/3/4/5 -> 77.1/80.1/78.7/ 75.9 tok/s), and vLLM's prompt_logprobs are ~uniform while speculative decoding is on, so perplexity must be measured with spec off. Pipeline, acceptance harness and raw measurements land in services/gen-seat-mixed-quant/. Rollback is one .env line; the previous build is untouched at /tank/aimodels/qwen38-27b-uncensored-nvfp4.
80 lines
3.5 KiB
Python
80 lines
3.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Cache-busted bs=1 decode bench against a vLLM OpenAI seat + MTP acceptance."""
|
|
import json, sys, time, urllib.request, statistics, random, argparse
|
|
|
|
TOPICS = ["a coral reef ecosystem", "the Roman aqueduct system", "how lithium-ion cells degrade",
|
|
"the history of the printing press", "how radar altimeters work", "glacier mass balance",
|
|
"the design of the Saturn V F-1 engine", "how sourdough fermentation works"]
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--base", default="http://10.250.50.54:8015")
|
|
ap.add_argument("--model", default="qwen3.8-27b-uncensored")
|
|
ap.add_argument("--tag", default="run")
|
|
ap.add_argument("--max-tokens", type=int, default=400)
|
|
ap.add_argument("--out", default=None)
|
|
ap.add_argument("--quiet", action="store_true")
|
|
a = ap.parse_args()
|
|
|
|
def post(p):
|
|
req = urllib.request.Request(a.base + "/v1/chat/completions",
|
|
data=json.dumps(p).encode(),
|
|
headers={"Content-Type": "application/json"})
|
|
with urllib.request.urlopen(req, timeout=900) as r:
|
|
return json.load(r)
|
|
|
|
def spec():
|
|
try:
|
|
with urllib.request.urlopen(a.base + "/metrics", timeout=30) as r:
|
|
t = r.read().decode()
|
|
except Exception:
|
|
return {}
|
|
o = {}
|
|
for k in ("vllm:spec_decode_num_draft_tokens_total",
|
|
"vllm:spec_decode_num_accepted_tokens_total"):
|
|
s = 0.0
|
|
for ln in t.splitlines():
|
|
if ln.startswith(k) and not ln.startswith("#"):
|
|
try:
|
|
s += float(ln.rsplit(" ", 1)[1])
|
|
except Exception:
|
|
pass
|
|
o[k] = s
|
|
return o
|
|
|
|
random.seed(1234)
|
|
# warmup
|
|
post({"model": a.model, "messages": [{"role": "user", "content": "Say hello."}],
|
|
"max_tokens": 16, "temperature": 0})
|
|
|
|
rates, accs = [], []
|
|
for i, t in enumerate(TOPICS):
|
|
nonce = random.randint(10**9, 10**10)
|
|
prompt = f"[session {nonce}] Write a detailed technical explanation of {t}. Be thorough and specific."
|
|
b = spec(); t0 = time.perf_counter()
|
|
r = post({"model": a.model, "messages": [{"role": "user", "content": prompt}],
|
|
"max_tokens": a.max_tokens, "temperature": 0, "stream": False})
|
|
w = time.perf_counter() - t0; af = spec()
|
|
ct = r["usage"]["completion_tokens"]
|
|
dd = af.get("vllm:spec_decode_num_draft_tokens_total", 0) - b.get("vllm:spec_decode_num_draft_tokens_total", 0)
|
|
da = af.get("vllm:spec_decode_num_accepted_tokens_total", 0) - b.get("vllm:spec_decode_num_accepted_tokens_total", 0)
|
|
ar = da / dd if dd else 0.0
|
|
rates.append(ct / w); accs.append(ar)
|
|
if not a.quiet:
|
|
print(f" {i+1}. gen={ct} {w:.2f}s -> {ct/w:6.2f} tok/s MTP={ar*100:5.1f}%", flush=True)
|
|
|
|
res = {"tag": a.tag, "model": a.model,
|
|
"tok_s_median": statistics.median(rates),
|
|
"tok_s_mean": statistics.mean(rates),
|
|
"tok_s_min": min(rates), "tok_s_max": max(rates),
|
|
"mtp_accept_median": statistics.median(accs),
|
|
"rates": rates, "accs": accs}
|
|
print(f"\n[{a.tag}] median {res['tok_s_median']:.2f} tok/s mean {res['tok_s_mean']:.2f} "
|
|
f"(min {res['tok_s_min']:.2f}/max {res['tok_s_max']:.2f}) MTP {res['mtp_accept_median']*100:.1f}%")
|
|
if a.out:
|
|
json.dump(res, open(a.out, "w"), indent=2)
|
|
return 0
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|