From 52612cbe96b403e00b635459179a4d66708b2e31 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Fri, 25 Sep 2026 09:09:48 -0700 Subject: [PATCH] feat(reward-seat): move Skywork reward seat from fv-ml1 to esh-ml1; audit finds nothing superseding it - Audit: Skywork-Reward-V2-Llama-3.1-8B is still #1 of 188 on AllenAI's RewardBench 2 per-sample results; no Skywork V3; the -40M sibling is vendor-marked experimental. Our AWQ W4A16 quant: 0.847 vs published bf16 0.860 on a 150-prompt sample (within +/-2.9 pt SE), 96.2% pairwise agreement. Double BOS from vLLM on pre-templated text costs a further ~2.7 pts; callers must send add_special_tokens=false. - No working consumer: 0 requests since 2026-09-13; Worldtree Domari points at a dead IP with a non-vLLM schema (reported to worldtree-dev). - Move: sha256-identical model copy; vLLM v0.24.0 on esh-ml1 :8003 at 0.55 util (KV 1.30x of a 16k request). Parity vs fv-ml1: 149/150 verdicts, 99.8% pairwise signs, raw |delta| median 0.049. - Gateway /scalar-judge passthrough -> 10.0.50.80:8003; fv-ml1 vllm-reward removed (~10.2 GB freed on GPU 1). stacks/vllm now holds only vllm-coder. --- docs/pfi/fv-ml1-gpu-seat-inventory.md | 4 ++ servers/esh-ml1/README.md | 11 +++- stacks/litellm/conf/config.yaml | 12 ++-- stacks/reward-seat/.env.example | 18 ++++++ stacks/reward-seat/README.md | 81 +++++++++++++++++++++++++++ stacks/reward-seat/compose.yaml | 75 +++++++++++++++++++++++++ stacks/vllm/.env.example | 6 +- stacks/vllm/compose.yaml | 72 ++++-------------------- 8 files changed, 207 insertions(+), 72 deletions(-) create mode 100644 stacks/reward-seat/.env.example create mode 100644 stacks/reward-seat/README.md create mode 100644 stacks/reward-seat/compose.yaml diff --git a/docs/pfi/fv-ml1-gpu-seat-inventory.md b/docs/pfi/fv-ml1-gpu-seat-inventory.md index fee363b..a5a6696 100644 --- a/docs/pfi/fv-ml1-gpu-seat-inventory.md +++ b/docs/pfi/fv-ml1-gpu-seat-inventory.md @@ -140,3 +140,7 @@ change. Run `--check` in CI to catch a stale document.* > **2026-09-25:** `qwen3-embedding` (:8001) and `reranker` (:8013) LEFT fv-ml1 — > their vLLM seats were retired and the names now route to TEI on esh-ml1 > (`stacks/embed-rerank`). ~6.1 GB freed on GPU 1. + +> **2026-09-25:** `vllm-reward` (Skywork, :8003, `/scalar-judge`) also LEFT fv-ml1 for +> esh-ml1 (`stacks/reward-seat`); ~10.2 GB more freed on GPU 1 (93.1 → 77.8 GB used +> across the day). The per-seat entry above describing it is historical. diff --git a/servers/esh-ml1/README.md b/servers/esh-ml1/README.md index 785ed3e..9434fad 100644 --- a/servers/esh-ml1/README.md +++ b/servers/esh-ml1/README.md @@ -32,9 +32,14 @@ vLLM here after a bake-off; see | `tei-embed` | `Qwen/Qwen3-Embedding-0.6B` | 8001 | `qwen3-embedding` (`hosted_vllm/`, `/v1`) | | `tei-rerank` | `BAAI/bge-reranker-v2-m3` | 8013 | `reranker`, `reranker-a3-bge-v2-m3` (`huggingface/`, no `/v1`) | -VRAM ~2.6 GB for both; the rest of the 16 GB is free for further utility seats. -The vLLM image (`vllm/vllm-openai:v0.24.0`, ~30 GB) is still on disk: kept in -case a vLLM-only seat (the reward classifier) moves here. +VRAM ~2.6 GB for both. + +**Also here since 2026-09-25: the reward seat** — `stacks/reward-seat`, +`vllm-reward` (Skywork-Reward-V2-Llama-3.1-8B AWQ, vLLM v0.24.0) on `:8003`, behind +the gateway passthrough `/scalar-judge`. It is the one vLLM seat on the box, +because TEI cannot serve a Llama classifier. Audit and parity: +[`stacks/reward-seat/README.md`](../../stacks/reward-seat/README.md). GPU total +with all three: ~10.4 of 16.4 GB. **Cut-over verified 2026-09-25** through the gateway against fv-ml1's vLLM seats just before they were retired: embed cosine median 0.999927 / min 0.999881 diff --git a/stacks/litellm/conf/config.yaml b/stacks/litellm/conf/config.yaml index f0a9a7a..c735a9c 100644 --- a/stacks/litellm/conf/config.yaml +++ b/stacks/litellm/conf/config.yaml @@ -1047,15 +1047,19 @@ general_settings: # unbounded. Names verified against LiteLLM docs (proxy/spend_logs_deletion). maximum_spend_logs_retention_period: "7d" maximum_spend_logs_retention_interval: "1d" - # scalar-judge → Skywork-Reward-V2 (scalar reward model; vLLM pooling on - # fv-ml1:8003). LiteLLM has no reward/pooling MODE, so this is a passthrough, + # scalar-judge → Skywork-Reward-V2-Llama-3.1-8B (AWQ; vLLM pooling on esh-ml1 + # :8003, moved from fv-ml1 2026-09-25, same files, verdicts identical 149/150). + # LiteLLM has no reward/pooling MODE, so this is a passthrough, # not a model_list alias. Gateway-key-gated. Consumers POST the reward body to - # /scalar-judge/ (e.g. /pooling or /classify), forwarded to :8003. + # /scalar-judge/ (e.g. /classify), forwarded to :8003. # SWAP-SENSITIVE: a different reward model shifts the score scale, so consumers # must recalibrate thresholds after a backing swap. + # ⚠ Send already-templated text with "add_special_tokens": false. Otherwise vLLM + # prepends a SECOND <|begin_of_text|>, which cost ~2.7 pts on RewardBench 2 + # (stacks/reward-seat/README.md). pass_through_endpoints: - path: "/scalar-judge" - target: "http://10.251.50.54:8003" + target: "http://10.0.50.80:8003" forward_headers: true include_subpath: true diff --git a/stacks/reward-seat/.env.example b/stacks/reward-seat/.env.example new file mode 100644 index 0000000..2e91ddb --- /dev/null +++ b/stacks/reward-seat/.env.example @@ -0,0 +1,18 @@ +# reward-seat tunables (esh-ml1). Copy to `.env` on the server. + +# Same vLLM as fv-ml1's seat had, so the move changed only the GPU. The image +# is already on esh-ml1 from the vLLM embed/rerank era. +VLLM_VERSION=v0.24.0 + +# Same port as on fv-ml1, so the gateway passthrough changed only its host. +REWARD_PORT=8003 + +# Fraction of the Ada's TOTAL 16,380 MiB (≈9.0 GB). vLLM checks it against +# free memory at start; TEI embed+rerank hold ~2.6 GB of the rest. The KV pool +# must hold one full 16,384-token sequence (~2.1 GB for Llama-3.1-8B in bf16). +REWARD_GPU_MEM_UTIL=0.55 +# Server-side cap; Worldtree's Domari budgets 14,000 tokens against it. +REWARD_MAX_MODEL_LEN=16384 + +# Matches fv-ml1: no key (the gateway passthrough fronts it). +API_KEY= diff --git a/stacks/reward-seat/README.md b/stacks/reward-seat/README.md new file mode 100644 index 0000000..e1fe2ff --- /dev/null +++ b/stacks/reward-seat/README.md @@ -0,0 +1,81 @@ +# reward-seat + +The fleet's scalar reward model on **esh-ml1** (CT 110 on esh-pve, RTX 2000E +Ada): **Skywork-Reward-V2-Llama-3.1-8B**, our own AWQ W4A16 quant, on vLLM +v0.24.0, `:8003`. Reached through the gateway passthrough +`http://10.250.50.70:4000/scalar-judge/` (gateway key required). +Moved from fv-ml1 on 2026-09-25 (Prime: utility seats off the Blackwells). + +## Audit, 2026-09-25 — has anything superseded it? No. + +- **Still the best model of its kind.** In AllenAI's official RewardBench 2 + per-sample results (188 models, last updated 2025-12-11), this model ranks + **#1** by mean accuracy over the five non-Ties subsets (84.7; our aggregation + approximates the official metric). The rest of Skywork's own V2 family fills + #2–#4. There is **no Skywork-Reward-V3**. The sibling + `Skywork-Reward-V2-Llama-3.1-8B-40M` scores higher on paper, but Skywork calls + it *experimental* (about a third of its training pairs were flipped) and + recommends the plain 8B for production. No credible 2026 release beat it in + the HF or leaderboard searches, but models never submitted to RewardBench 2 + would not show up there. +- **Our quant is close to the original.** RewardBench 2 sample: 150 prompts, 30 + per non-Ties subset, 600 texts. Reference = the published bf16 scores. + - Accuracy: AWQ **0.847** vs bf16 **0.860**. The standard error at n=150 is + ±2.9 pts, so the difference is not resolved. + - Pairwise preference-sign agreement 96.2%; raw-score Pearson 0.991; mean + shift −0.60. + - Largest per-subset gap: Precise IF 0.67 vs 0.77 (n=30). + - Part of the remaining difference may be harness, not quant: the published + run used HF transformers. +- ⚠ **Double BOS costs accuracy.** RewardBench texts already start with + `<|begin_of_text|>`, and vLLM's `/classify` adds another unless the request + sets `"add_special_tokens": false`. With the double BOS, accuracy fell to + **0.820** (−2.7 pts) and Pearson to 0.981. Callers sending chat-templated + text must set that flag. +- **It has no working consumer.** fv-ml1's seat served 0 requests between its + 2026-09-13 start and this move. The only code caller, Worldtree's Domari + (`tools/domari/score_pair_skywork.py`), is broken twice over: + - it hardcodes ana-ml2's dead address `10.250.50.54:8003`; + - it sends `{"prompt", "responses"}` and reads `{"rewards"}`, which is not + vLLM's `/classify` schema (`input` → `data[].probs`). It could never have + succeeded against this seat, and its docstring says it falls back to the + Selene/chat-judge path on error. + + Reported to worldtree-dev 2026-09-25. + +## Move, 2026-09-25 + +- Model files: a sha256-identical copy of fv-ml1's + `/tank/aimodels/llm/Skywork-Reward-V2-Llama-3.1-8B-AWQ` (7 files, 4.4 GB), now at + `/opt/aimodels/llm/` on esh-ml1. fv-ml1 keeps the canonical copy. +- Same vLLM image digest and flags. `--gpu-memory-utilization 0.55` (≈9.0 GB of + 16,380 MiB) → KV 21,328 tokens = 1.30× a full 16,384-token request. GPU total + with TEI: ~10.4 GB used. +- **Parity vs fv-ml1** (the same 600 texts, 2 runs per site): + + | | raw-score \|Δ\| median | p99 | max | + |---|---|---|---| + | fv-ml1 vs itself | 0.000 | 0.198 | 0.297 | + | esh-ml1 vs itself | 0.002 | 0.157 | 0.199 | + | **esh-ml1 vs fv-ml1** | **0.049** | 0.221 | 0.321 | + + 149/150 prompt verdicts are identical, and 99.8% of pairwise preference + signs. Different GPUs add a small offset (median 0.05 against a score range + of tens), but the tails match each site's own noise. +- Gateway `/scalar-judge` repointed to `10.0.50.80:8003`; verified 3/3 through + the gateway. fv-ml1's `vllm-reward` was removed, freeing ~10.2 GB on GPU 1. + +## Smoke test + +```bash +curl -s http://10.0.50.80:8003/classify -H 'content-type: application/json' \ + -d '{"model":"Skywork/Skywork-Reward-V2-Llama-3.1-8B-AWQ","input":"User: hello\nAssistant: hi there","use_activation":false}' \ + | jq '.data[0].probs[0]' # raw reward, ~4.27 +``` + +## Options not taken + +- **FP8 (W8A8) quant**, native on Ada, would likely track bf16 more closely + at ~8.5 GB of weights (it fits beside TEI, tightly). Not pursued: the quant + already sits within the measurement's resolution, nothing consumes the seat, + and any quant swap shifts the score scale for consumers. diff --git a/stacks/reward-seat/compose.yaml b/stacks/reward-seat/compose.yaml new file mode 100644 index 0000000..c561118 --- /dev/null +++ b/stacks/reward-seat/compose.yaml @@ -0,0 +1,75 @@ +# reward-seat — the fleet's scalar reward model, on esh-ml1 (RTX 2000E Ada, 16 GB). +# +# Moved here from fv-ml1's `vllm` stack on 2026-09-25 (Prime: utility seats off +# the Blackwells). Same model files (sha256-identical copy of fv-ml1's local AWQ +# output), same vLLM version and flags. Only the GPU and the host changed. +# +# vllm-reward Skywork-Reward-V2-Llama-3.1-8B, AWQ W4A16 (llm-compressor, +# score head kept full precision) → /classify :8003 +# +# vLLM, not TEI: TEI cannot serve a Llama sequence-classification head. That +# makes this the one vLLM seat on esh-ml1; embed/rerank are TEI +# (stacks/embed-rerank). +# +# Scores are SWAP-SENSITIVE: a different model or quant shifts the scale and +# every consumer threshold with it. /classify returns sigmoid probs by default; +# send "use_activation": false for the raw reward. +# +# Consumers reach it through the gateway passthrough `/scalar-judge` on +# ana-docker:4000 (stacks/litellm, pass_through_endpoints). Worldtree's Domari +# hardcodes an address instead (see README). +# +# Model files: /opt/aimodels/llm/Skywork-Reward-V2-Llama-3.1-8B-AWQ (local quant, +# not on HF Hub; the canonical copy also lives on fv-ml1 /tank/aimodels/llm). + +name: reward-seat + +services: + vllm-reward: + image: vllm/vllm-openai:${VLLM_VERSION} + container_name: vllm-reward + restart: unless-stopped + ipc: host + ports: + - "${REWARD_PORT}:8000" + volumes: + - /opt/aimodels/llm:/local-models:ro + environment: + - VLLM_API_KEY=${API_KEY:-} + command: + - /local-models/Skywork-Reward-V2-Llama-3.1-8B-AWQ + - --served-model-name + - Skywork/Skywork-Reward-V2-Llama-3.1-8B-AWQ + # config.json declares LlamaForSequenceClassification; the pooling + # runner serves it as a single-label classifier. + - --runner + - pooling + - --host + - 0.0.0.0 + - --port + - "8000" + - --gpu-memory-utilization + - ${REWARD_GPU_MEM_UTIL} + - --max-model-len + - ${REWARD_MAX_MODEL_LEN} + - --dtype + - auto + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: ["0"] + capabilities: [gpu] + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 240s + labels: + - homepage.group=AI - Eval & Retrieval + - homepage.name=Reward — Skywork V2 8B (esh-ml1) + - homepage.icon=mdi-scale-balance + - homepage.description=Scalar reward classifier (/scalar-judge) via vLLM on esh-ml1 + - homepage.href=http://10.0.50.80:${REWARD_PORT}/docs diff --git a/stacks/vllm/.env.example b/stacks/vllm/.env.example index 60177b4..7a5beb9 100644 --- a/stacks/vllm/.env.example +++ b/stacks/vllm/.env.example @@ -14,7 +14,7 @@ VLLM_VERSION=v0.24.0 # Host ports (container always listens on 8000 internally) # (EMBED_PORT 8001 / RERANK_PORT 8013 retired 2026-09-25 with their seats — # embedding + reranking moved to TEI on esh-ml1, stacks/embed-rerank.) -REWARD_PORT=8003 +# REWARD_PORT=8003 (retired here 2026-09-25 — the reward seat moved to esh-ml1, stacks/reward-seat) # GPU assignment — all services share this GPU # (ana-ml2 has 0 and 1; default 1 keeps 0 free for heavy LLM work in llama-swap) @@ -54,13 +54,13 @@ GPU_ID=1 # each at 0.05 (mostly util-reservation waste); 0.03 (~3.6 GB) fits weights + # CUDA context with room, freeing ~4 GB back to granite. Recreate them ONE AT A # TIME — concurrent recreate races the memory-profiling assertion. -REWARD_GPU_MEM_UTIL=0.10 +# REWARD_GPU_MEM_UTIL=0.10 (retired here 2026-09-25 — the reward seat moved to esh-ml1, stacks/reward-seat) # Context length caps — lower these if VRAM is tight. # Qwen3-Embedding supports up to 32k; reranker up to 32k. # Skywork capped at 16k server-side as defense-in-depth; JudgeClient also # enforces the cap at dispatch time per spec. -REWARD_MAX_MODEL_LEN=16384 +# REWARD_MAX_MODEL_LEN=16384 (retired here 2026-09-25 — the reward seat moved to esh-ml1, stacks/reward-seat) # Optional API key — leave blank for no auth (fine on the internal network). # If set, all three services require `Authorization: Bearer `. diff --git a/stacks/vllm/compose.yaml b/stacks/vllm/compose.yaml index 17ab2e1..8b5d09d 100644 --- a/stacks/vllm/compose.yaml +++ b/stacks/vllm/compose.yaml @@ -1,4 +1,4 @@ -# vLLM — utility seats on fv-ml1: Skywork Reward-V2 classifier + the coder FIM seat. +# vLLM — the coder FIM seat on fv-ml1 (the last utility seat left here). # # Originally created to replace the unmaintained Infinity stack (embed + # rerank); generalized 2026-05-13 to host any vLLM-served model on fv-ml1, @@ -7,13 +7,14 @@ # # ⚠ Embedding + reranking LEFT this stack 2026-09-25 (Prime): they now run on TEI # on esh-ml1 (stacks/embed-rerank), and TEI is the fleet's embed/rerank engine -# from now on. Do not re-add them here. +# from now on. The reward classifier moved to esh-ml1 the same day +# (stacks/reward-seat). Do not re-add them here. # # All tunables live in .env — edit that, not this file. # -# Skywork-Reward-V2-Llama-3.1-8B-AWQ is a locally-quantized model — lives at -# /tank/aimodels/llm/Skywork-Reward-V2-Llama-3.1-8B-AWQ on fv-ml1 and is -# bind-mounted into the reward service at /local-models. Not from HF Hub. +# Skywork-Reward-V2-Llama-3.1-8B-AWQ is a locally-quantized model. The canonical +# copy still lives at /tank/aimodels/llm/Skywork-Reward-V2-Llama-3.1-8B-AWQ on +# fv-ml1; the serving copy is on esh-ml1. Not from HF Hub. services: # vllm-embed (Qwen3-Embedding-0.6B, :8001) and vllm-rerank-a3 @@ -23,63 +24,10 @@ services: # and `reranker-a3-bge-v2-m3` did not change. Their definitions are in git # history before this commit if a rollback is ever needed. - vllm-reward: - image: vllm/vllm-openai:${VLLM_VERSION} - container_name: vllm-reward - restart: unless-stopped - ipc: host - ports: - - "${REWARD_PORT}:8000" - volumes: - # AWQ output lives in the legacy llama-swap models tree, not the HF cache - # — bind-mount the LLM models dir read-only so the reward service can - # load it as a local-path HF-format model. - - /tank/aimodels/llm:/local-models:ro - environment: - - VLLM_API_KEY=${API_KEY:-} - command: - - /local-models/Skywork-Reward-V2-Llama-3.1-8B-AWQ - - --served-model-name - - Skywork/Skywork-Reward-V2-Llama-3.1-8B-AWQ - # vLLM 0.19.1 deprecated --task in favor of --runner. The model's - # config.json declares `LlamaForSequenceClassification` so the - # pooling runner uses it as a classifier (single-label reward score) - # without needing an explicit task flag. - - --runner - - pooling - - --host - - 0.0.0.0 - - --port - - "8000" - - --gpu-memory-utilization - - ${REWARD_GPU_MEM_UTIL} - - --max-model-len - - ${REWARD_MAX_MODEL_LEN} - - --dtype - - auto - deploy: - resources: - reservations: - devices: - - driver: nvidia - device_ids: - - "${GPU_ID}" - capabilities: - - gpu - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8000/health"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 240s - networks: - - tnet - labels: - - homepage.group=AI - Eval & Retrieval - - homepage.name=vLLM Reward (Skywork) - - homepage.icon=mdi-scale-balance - - homepage.description=Skywork-Reward-V2 8B classifier via vLLM (fv-ml1) - - homepage.href=http://10.251.50.54:${REWARD_PORT}/docs + # vllm-reward (Skywork-Reward-V2-Llama-3.1-8B AWQ, :8003) — MOVED to esh-ml1 + # 2026-09-25 (stacks/reward-seat); the gateway `/scalar-judge` passthrough + # follows it. The model files stay at /tank/aimodels/llm/ on fv-ml1 as the + # canonical copy of the local quant. # vllm-granite (ibm-granite/granite-4.1-8b-fp8, :8004) — RETIRED 2026-08-12, # service block removed 2026-08-20. It was the fleet summarizer until the