diff --git a/docs/pfi/fv-ml1-gpu-seat-inventory.md b/docs/pfi/fv-ml1-gpu-seat-inventory.md index f94f8fb..3c721ed 100644 --- a/docs/pfi/fv-ml1-gpu-seat-inventory.md +++ b/docs/pfi/fv-ml1-gpu-seat-inventory.md @@ -6,7 +6,7 @@ Hand-edits are overwritten and, worse, drift silently while looking authoritative — which is exactly the failure this file replaced. --> -**Generated 2026-09-14 17:24 UTC** by `scripts/seat-inventory.py`, read from the running +**Generated 2026-09-14 17:45 UTC** by `scripts/seat-inventory.py`, read from the running containers on `100.64.0.7` — `docker inspect`, `nvidia-smi`, each model's own `config.json`, and the `.PROVENANCE.txt` siblings on `/tank`. @@ -18,11 +18,11 @@ containers on `100.64.0.7` — `docker inspect`, `nvidia-smi`, each model's own | GPU | seat | VRAM | weights | KV tokens | ctx | concurrency | util | |---|---|---|---|---|---|---|---| | 0 | `vllm-cyberprev` | 46.0 GiB | 25.11 GiB | 359,372 | 262144 | **1.37×** | 0.40 | -| 0 | `vllm-gen-small` | 35.3 GiB | 23.98 GiB | 670,142 | 262144 | **2.56×** | 0.48 | -| 1 | `vllm-meromero-rp` | 41.3 GiB | 19.51 GiB | 320,774 | 262144 | **1.22×** | 0.52 | -| 1 | `vllm-erp-seat` | 26.4 GiB | 15.9 GiB | 534,649 | 262144 | **2.04×** | 0.24 | +| 0 | `vllm-gen-small` | 35.8 GiB | 23.98 GiB | 670,142 | 262144 | **2.56×** | 0.48 | +| 1 | `vllm-meromero-rp` | 43.0 GiB | 19.51 GiB | 320,774 | 262144 | **1.22×** | 0.52 | +| 1 | `vllm-erp-seat` | 27.1 GiB | 15.9 GiB | 534,649 | 262144 | **2.04×** | 0.24 | | 1 | `vllm-reward` | 9.0 GiB | 4.41 GiB | 26,224 | 16384 | **1.60×** | 0.10 | -| 1 | `vllm-coder` | 5.5 GiB | 2.98 GiB | 77,056 | 16384 | **4.70×** | 0.055 | +| 1 | `vllm-coder` | 6.1 GiB | 2.98 GiB | 77,056 | 16384 | **4.70×** | 0.055 | | 1 | `vllm-embed` | 3.4 GiB | 1.12 GiB | 10,272 | 8192 | **1.25×** | 0.03 | | 1 | `vllm-rerank-a3` | 2.1 GiB | 1.06 GiB | — | 8192 | — | 0.03 | | 2 | `vllm-flash-next` | 93.1 GiB | 76.82 GiB | 344,155 | 262144 | **1.31×** | 0.96 | @@ -133,6 +133,8 @@ context. --- +*Lineage, provenance, model cards, measured tok/s and depth results live in the hand-curated companion [`llm-seat-catalog.md`](llm-seat-catalog.md).* + *Regenerate with `scripts/seat-inventory.py` after ANY seat change — model swap, quant change, context or utilization edit, or speculative-decoding change. Run `--check` in CI to catch a stale document.* diff --git a/docs/pfi/llm-seat-catalog.md b/docs/pfi/llm-seat-catalog.md new file mode 100644 index 0000000..91adb9a --- /dev/null +++ b/docs/pfi/llm-seat-catalog.md @@ -0,0 +1,159 @@ +# LLM seat catalog — fv-ml1 + +The durable, curated record of **what each LLM seat IS**: lineage, provenance, +model-card facts, quantization, speculative decoding, and measured speed/depth. + +**This file is hand-curated and complements the generated inventory.** The two split +by volatility: + +| file | owns | updated by | +|---|---|---| +| [`fv-ml1-gpu-seat-inventory.md`](fv-ml1-gpu-seat-inventory.md) | **live** placement, VRAM, KV tokens, concurrency, quant format, gateway aliases | `scripts/seat-inventory.py` (read-only, auto) | +| **this file** | **durable** lineage, provenance, cards, licenses, measured tok/s + depth results, "why this seat" | **by hand**, on seat change or re-bench | + +Where they overlap (ctx, concurrency, VRAM), the **inventory is authoritative for the +live number**; the values here are point-in-time and dated. If they disagree, the +inventory won and this file is stale — fix it. + +## Keeping this current + +Update this file whenever a seat changes — model swap, quant change, context/KV edit, +or a new seat. Two commands regenerate the inputs: + +```bash +scripts/seat-inventory.py # live placement/KV/concurrency (docs/pfi/fv-ml1-gpu-seat-inventory.md) +scripts/seat-bench.py # warm tok/s + deep-prefill OOM check (serial; prints the numbers below) +``` + +⚠ **Speed and depth numbers are measurements, not facts about the weights** — they +carry a date and a harness. Re-run `seat-bench.py` after any context/KV/quant/spec +change and update the table with the new date. A number without its harness is not a +result (see the measurement-discipline note at the bottom). + +--- + +## Summary (measured 2026-09-14, harness below) + +| seat | GPU | model | ctx | conc. | warm tok/s | VRAM | depth verified | OOM | +|---|---|---|---|---|---|---|---|---| +| **cyberprev** (sec) | 0 | Qwen3.8-27B cyber-SFT (abliterated base) | 262,144 | 1.37× | 136.6 | 47.1 GiB | 259,722 tok | none | +| **gen-small** | 0 | Qwen3.6-35B-A3B Heretic | 262,144 | 2.56× | 254.8 | 36.1 GiB | 254,526 tok | none | +| **gen** (flash-next) | 2 | Qwen3.8-Flash-Next (orcarouter) | 262,144 | 1.31× | 170.7 | 95.3 GiB | 254,273 tok | none | +| **char-rp** | 1 | G4-MeroMero-v2-31B (Gemma4 dense) | 262,144 | 1.22× | 62.7 | 42.3 GiB | 254,858 tok | none | +| **char-rp-fast** | 1 | G4-MeroMero-26B-A4B (Gemma4 MoE) | 262,144 | 2.04× | 225.2 | 27.0 GiB | 254,823 tok | none | +| **coder** | 1 | Qwen2.5-Coder-1.5B (base) | 16,384 | 4.70× | 337.3 | 5.6 GiB | 15,905 tok | none | + +Support (non-generative): **reward** Skywork-Reward-V2-Llama-3.1-8B (9.2 GiB, GPU1), +**embed** Qwen3-Embedding-0.6B (3.4 GiB, GPU1), **rerank** bge-reranker-v2-m3 +(2.1 GiB, GPU1). **gen (flash-next) is on GPU2 and off-limits to rearrangement; +GPU3 is reserved scratch (empty).** + +--- + +## Seats + +### cyberprev — the `sec` / `sec-reasoning` seat (GPU 0, :8025) + +- **Serves gateway:** `sec`, `sec-reasoning`. Displaced mog-sec 2026-09-14. +- **Lineage:** `Qwen/Qwen3.8-27B` → `hotdogs/Qwen3.8-27B-abliterated` (abliterated base) + → **offensive-security tool-calling LoRA** trained on `hotdogs/cyber-sft-agent-qwen38` + (8,400 rows, 22 pentest tools: nmap/sqlmap/metasploit/hydra/…), merged @ scale 1.0 + = `hotdogs/Qwen3.8-27B-abliterated-cyber-preview` → **in-house name-repair** (the raw + export shipped 850/1199 body tensors with a tripled `language_model.` prefix — an + unsloth export bug, see [[reference_unsloth_tripled_prefix_export_bug]]) → **house + mixed NVFP4 quant** → `/tank/aimodels/cyberprev-nvfp4-mixed-v2`. +- **It is a cyber SFT finetune**, NOT "an abliteration" — the abliteration is inherited + from the base; the new capability is the cyber tool-calling SFT. Card: tool-call + format 0/6→6/6, correct tool 0/6→6/6, general capability held (KL 0.04 general / + 0.81 tool = surgical re-target). "Will not refuse" per its card. +- **Quant:** compressed-tensors mixed — NVFP4 W4A4 on MLP layers 0-55, FP8 W8A8 on + attn/linear_attn/lm_head/MLP 56-63, FP8 KV. 15 MTP tensors grafted (bf16), vision + tower + `re:^mtp.*` in ignore. Pipeline: `services/gen-seat-mixed-quant/`. +- **Speculative:** dflash k=7 (drafter `qwen38-27b-dflash2-drafter`), ~2.77 mean accept len. +- **Context:** native 262,144, **depth-probed clean to 259,722 tokens** (non-repeating + prompt). Notable: same base arch as the retired mog-sec, which crashed above ~163,840 + on processing depth — cyberprev does not. +- **License:** apache-2.0. + +### gen-small — the fast A3B tier (GPU 0, :8026) + +- **Serves gateway:** `gen-small`, `gen-small-reasoning`, and backs `summarizer` + + `classifier` (the bulk/triage aliases). Stood up 2026-09-14. +- **Model:** `llmfan46/Qwen3.6-35B-A3B-uncensored-heretic-Native-MTP-Preserved-NVFP4-Experts-Only` + → `/tank/aimodels/qwen36-35b-a3b-heretic-nvfp4`. +- **Lineage:** `Qwen/Qwen3.6-35B-A3B` (3B-active MoE, 256 experts, hybrid GDN+attn) → + **Heretic v1.3.0** abliteration (MPOA — Magnitude-Preserving Orthogonal Ablation, the + house-favorite method) → NVFP4 experts-only quant. **3.8 was requested but there is no + general Qwen3.8 A3B** (the 3.8 MoEs are Flash-Next and the 2.4T), so this is the 3.6 + fallback. +- **Card:** 88% fewer refusals (10/100 vs 83/100 original) at 0.0015 KL divergence + (quality preserved). 19 MTP tensors preserved (native). +- **Quant:** modelopt NVFP4, experts-only (256 experts NVFP4; attn/shared-expert/ + linear_attn bf16). Serves as-is, no re-quant. +- **Speculative:** qwen3_5_mtp k=3, **measured 69.6% acceptance / 3.09 mean length** — MTP + is a clear win here. +- **Context:** native 262,144, depth-clean to 254,526. Cheap KV (A3B + fp8) → 2.56× + concurrency on only 8 GiB KV. +- **License:** apache-2.0 (per base `Qwen/Qwen3.6-35B-A3B`). +- **Why:** high-volume, low-caliber work (summarization, classification, triage) belongs + on a fast 3B-active seat, not the premium gen seat. + +### gen — flash-next (GPU 2, :8022) — off-limits to rearrangement + +- **Serves gateway:** `gen`, `gen-large`, `gen-reasoning`, `summarizer-large`, + `classifier-large`, `image-judge`, `chat-judge`, `qwen-image-bench`. +- **Model:** `orcarouter/Qwen3.8-Flash-Next-Uncensored-NVFP4` → **in-house PLE bf16→FP8 + conversion + `layer_types` rename** → `/tank/aimodels/qwen38-flash-next-orcarouter-nvfp4-plefp8`. + See [[reference_qwen4exp_ple_loader_branch_order]]. +- **Lineage:** `Qwen/Qwen3.8-Flash-Next` (176B total: 125B main + 51B n-gram PLE table, + ~6B active) → orcarouter uncensored NVFP4 → in-house PLE→FP8 so vLLM's + `Qwen4ExpPLEEmbeddingMethod` selects the FP8 path. +- **Architecture:** the only seat whose weights don't fit its card — the 51B PLE table + lives in **pinned host RAM**, read over CUDA UVA. GDN linear-attn + QSA hybrid. +- **Quant:** compressed-tensors mixed (W8A16 attn / W4A16 experts) + FP8 PLE. +- **Speculative:** MTP k=3, ~60.4% acceptance. +- **Context:** 262,144, depth-clean to 254,273. Fastest prefill of the fleet (27.7s/254K). + +### char-rp — the char quality tier (GPU 1, :8016) + +- **Serves gateway:** `char-rp`, `char-rp-reasoning`. +- **Model:** `/tank/aimodels/meromero-v2-nvfp4-work/G4-MeroMero-v2-31B-NVFP4A16` — + in-house build, **Gemma4 dense, 60 layers**, heretic-abliterated, compressed-tensors + NVFP4 W4A16. +- **Speed:** 62.7 tok/s warm — the slowest seat, inherent to a dense 31B; this is the + **quality** tier. Deep prefill 267s/254K (slowest). Gemma4 sliding-window attn + (window 1024) keeps KV cheap at depth. +- **Context:** 262,144 @ 1.22×, depth-clean to 254,858. + +### char-rp-fast — the char speed tier (GPU 1, :8021) + +- **Serves gateway:** `char-rp-fast`. +- **Model:** `/tank/aimodels/G4-MeroMero-26B-A4B-it-uncensored-heretic-NVFP4A16` — + in-house build, **Gemma4 MoE (26B-A4B)**, heretic-abliterated, compressed-tensors + NVFP4A16. +- **Speed:** 225.2 tok/s warm — the throughput answer to char-rp's quality. 2.04× + concurrency (KV pinned ~8.5 GiB). Depth-clean to 254,823. +- **Context:** 262,144 @ 2.04×. + +### coder — FIM code-completion (GPU 1, :8020) + +- **Serves gateway:** `coder-fast`. Backs Zed edit-predictions. +- **Model:** `Qwen/Qwen2.5-Coder-1.5B` (base, unquantized), fp8 KV. +- **Speed:** 337.3 tok/s (smallest model, fastest). Context 16,384 @ 4.70×. +- **Note:** util-sized; the ~4.7× overshoots the "2-3×" intent because the 1.5B + weight+overhead floor (~4.2 GiB) sits just under the util knob's resolution. Hitting + ≤3× reliably needs a `--kv-cache-memory` byte pin (compose change), deferred. + +--- + +## Benchmark harness (state it with any number above) + +- **warm decode tok/s:** greedy (temperature 0), **conc=1** (single stream), **n=3** + reps, median, fixed ~40-word prompt → 300 output tokens. Decode throughput — + generation is never prefix-cached, so reps are valid; spread was <1% on every seat. +- **deep prefill / OOM:** one **non-repeating** random prompt at ~0.97× max-model-len, + 8 output tokens. PASS = returns AND the seat's allocator log shows **no OOM / CUBLAS / + illegal-memory** across the probe window (verified 2026-09-14, 0 hits on all seats). +- **Measured serially** (one seat at a time) — no cross-seat contention. These are clean, + uncontended, single-stream **ceilings**; real aggregate throughput under concurrency is + higher per-GPU and lower per-request. Re-derive with `scripts/seat-bench.py`. diff --git a/scripts/seat-bench.py b/scripts/seat-bench.py new file mode 100755 index 0000000..8e5b99b --- /dev/null +++ b/scripts/seat-bench.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Warm decode tok/s + deep-prefill OOM check for the fv-ml1 LLM seats. + +Complements scripts/seat-inventory.py: the inventory reads STATIC state +(placement, KV, concurrency, quant) with zero load; this applies LOAD to +measure warm decode throughput and to prove each seat survives a near-max-context +prefill without OOM. The numbers it prints are what docs/pfi/llm-seat-catalog.md +records, with the harness stated so they travel honestly. + + scripts/seat-bench.py # bench every generative seat, serially + scripts/seat-bench.py --host fv-ml1 + +⚠ SERIAL BY DESIGN. Two deep prefills at once contend for GPU memory/compute and +would both confound the OOM result and depress tok/s. One seat at a time. + +HARNESS (state it with any number this prints): + warm decode = greedy, temperature 0, conc=1 (single stream), n=3 reps, median, + a fixed ~40-word prompt generating 300 tokens (decode throughput; generation + is never prefix-cached, so the reps are valid). + deep prefill = one non-repeating random prompt at ~0.97x max-model-len, 8 output + tokens; PASS = returns with no error AND the seat's allocator log shows no + OOM / CUBLAS / illegal-memory across the probe window. + +These are clean, uncontended, single-stream figures — a ceiling, not a loaded +number. Aggregate throughput under real concurrency is higher per-GPU and lower +per-request. +""" +import argparse, json, time, random, statistics, subprocess, urllib.request, urllib.error + +# (label, served-model-name, port, max_model_len). Refresh from seat-inventory if seats change. +SEATS = [ + ("cyberprev (sec)", "cyberprev-27b", 8025, 262144), + ("gen-small", "gen-small", 8026, 262144), + ("char-rp", "char-rp", 8016, 262144), + ("char-rp-fast", "G4-MeroMero-26B-A4B-it-uncensored-heretic-NVFP4A16", 8021, 262144), + ("gen (flash-next)","qwen3.8-flash-next-uncensored", 8022, 262144), + ("coder", "qwen2.5-coder-1.5b", 8020, 16384), +] +DECODE_PROMPT = ("Write a detailed technical explanation of how a modern CPU branch predictor " + "works, covering the pattern history table, the branch target buffer, and " + "misprediction cost.") + +def run(host, cmd): + r = subprocess.run(["ssh","-o","BatchMode=yes","-o","ConnectTimeout=10", + f"infra-ops@{host}", cmd], capture_output=True, text=True, timeout=1900) + return r.stdout + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--host", default="10.251.50.54") + a = ap.parse_args() + # This driver runs the HTTP calls ON the host (loopback to each seat) via a pushed helper. + helper = "/tmp/_seat_bench_http.py" + open("/tmp/_seat_bench_http.py","w").write(_HTTP_DRIVER) + subprocess.run(["scp","-o","BatchMode=yes","/tmp/_seat_bench_http.py", + f"infra-ops@{a.host}:{helper}"], check=True) + print(run(a.host, f"python3 {helper}")) + +_HTTP_DRIVER = r''' +import json,time,random,statistics,urllib.request,urllib.error +WORDS=[f"{random.Random(i).randint(0,1<<30):x}" for i in range(320000)] +SEATS=[("cyberprev (sec)","cyberprev-27b",8025,262144), + ("gen-small","gen-small",8026,262144), + ("char-rp","char-rp",8016,262144), + ("char-rp-fast","G4-MeroMero-26B-A4B-it-uncensored-heretic-NVFP4A16",8021,262144), + ("gen (flash-next)","qwen3.8-flash-next-uncensored",8022,262144), + ("coder","qwen2.5-coder-1.5b",8020,16384)] +P=("Write a detailed technical explanation of how a modern CPU branch predictor works, " + "covering the pattern history table, the branch target buffer, and misprediction cost.") +def call(port,model,prompt,mx,to=1800): + b=json.dumps({"model":model,"prompt":prompt,"max_tokens":mx,"temperature":0}).encode() + r=urllib.request.Request(f"http://127.0.0.1:{port}/v1/completions",data=b,headers={"Content-Type":"application/json"}) + t0=time.time() + try: d=json.loads(urllib.request.urlopen(r,timeout=to).read()) + except urllib.error.HTTPError as e: return None,time.time()-t0,f"HTTP {e.code}: {e.read().decode()[:120]}" + except Exception as e: return None,time.time()-t0,f"{type(e).__name__}: {str(e)[:100]}" + return d.get("usage",{}),time.time()-t0,None +for name,model,port,mc in SEATS: + print(f"\n===== {name} (:{port}, max {mc}) =====",flush=True) + call(port,model,P,64,to=120) + rates=[] + for i in range(3): + u,dt,err=call(port,model,P,300,to=180) + if err: print(f" rep{i+1} ERR {err}",flush=True); continue + t=u.get("completion_tokens",0) + if t: rates.append(t/dt); print(f" rep{i+1}: {t}/{dt:.2f}s = {t/dt:.1f} tok/s",flush=True) + if rates: print(f" >> warm decode MEDIAN {statistics.median(rates):.1f} tok/s (n={len(rates)})",flush=True) + nw=int(mc*0.97/7.9); r=random.Random(port*13+int(time.time())%997) + dp=" ".join(r.choice(WORDS) for _ in range(nw)) + u,dt,err=call(port,model,dp,8,to=1800) + print((" >> DEEP PREFILL FAIL "+err) if err else f" >> DEEP PREFILL: {u.get('prompt_tokens'):,} tok in {dt:.1f}s OK",flush=True) +print("\nBENCH DONE",flush=True) +''' + +if __name__ == "__main__": + main() diff --git a/scripts/seat-inventory.py b/scripts/seat-inventory.py index 03cec4a..04e6652 100755 --- a/scripts/seat-inventory.py +++ b/scripts/seat-inventory.py @@ -209,7 +209,10 @@ def render(seats, als, host): L.append("|---|---|") for a, p in als: L.append(f"| `{a}` | {p} |") - L += ["", "---", "", "*Regenerate with `scripts/seat-inventory.py` after ANY seat change —", + L += ["", "---", "", + "*Lineage, provenance, model cards, measured tok/s and depth results live in the " + "hand-curated companion [`llm-seat-catalog.md`](llm-seat-catalog.md).*", "", + "*Regenerate with `scripts/seat-inventory.py` after ANY seat change —", "model swap, quant change, context or utilization edit, or speculative-decoding", "change. Run `--check` in CI to catch a stale document.*", ""] return "\n".join(L)