diff --git a/stacks/litellm/conf/config.yaml b/stacks/litellm/conf/config.yaml index 37c0ee1..7e13103 100644 --- a/stacks/litellm/conf/config.yaml +++ b/stacks/litellm/conf/config.yaml @@ -30,12 +30,13 @@ model_list: model_info: mode: chat - # --- Qwen3.5-9B vision-language (FP8) — vision + chat. vLLM on ana-ml2 GPU 1, - # nightly-pinned (vision-FP8 exclusion fix), :8007. Explicit entry shadows - # the "*" wildcard llama-swap route. --- - - model_name: qwen3.5-9b-fp8 + # --- Qwen3.6-35B-A3B vision-language MoE (official FP8) — vision + chat. vLLM + # on ana-ml2 GPU 1, :8007. Explicit entry shadows the "*" wildcard llama-swap + # route. REPLACED qwen3.5-9b-fp8 2026-06-14 (the 9B is retired; this is a + # 35B-A3B MoE — served under its TRUE name, never aliased under the old one). --- + - model_name: qwen3.6-35b-a3b litellm_params: - model: hosted_vllm/qwen3.5-9b-fp8 + model: hosted_vllm/qwen3.6-35b-a3b api_base: http://10.250.50.54:8007/v1 api_key: os.environ/VLLM_API_KEY model_info: diff --git a/stacks/llama-swap/conf/config.yaml b/stacks/llama-swap/conf/config.yaml index eb3ae4b..c4d7d3c 100644 --- a/stacks/llama-swap/conf/config.yaml +++ b/stacks/llama-swap/conf/config.yaml @@ -117,24 +117,10 @@ models: --presence-penalty 1.5 --chat-template-kwargs '{"enable_thinking":true}' - "qwen3.5-9b": - name: "Qwen 3.5 9B UD-Q4_K_XL" - description: "Dense 9B model. Lightweight general-purpose chat and reasoning." - ttl: 0 # pinned — member of the `pinned` group, never unloads - cmd: | - /app/llama-server - --context-shift - --model /models/unsloth_Qwen3.5-9B-GGUF/Qwen3.5-9B-UD-Q4_K_XL.gguf - --port ${PORT} - --n-gpu-layers 999 - --ctx-size 32768 - --flash-attn on - --temp 1.0 - --top-p 0.95 - --top-k 20 - --min-p 0.00 - --presence-penalty 1.5 - --chat-template-kwargs '{"enable_thinking":true}' + # "qwen3.5-9b" REMOVED 2026-06-14 — the GPU-0 GGUF pin is dropped. General + # chat + vision now served by Qwen3.6-35B-A3B (official FP8) on GPU 1 via vLLM + # (stacks/qwen36-vl). GPU 0 is freed for the creative-writing hot-swap card. + # The GGUF stays on disk (/models/unsloth_Qwen3.5-9B-GGUF/) if ever wanted. # -------------------------------------------------------------------------- # Qwen 3.6 — uses -hf syntax, reads from HF_HOME=/hfcache (host pre-download) @@ -630,25 +616,10 @@ groups: - "qwen3-embedding-0.6B" - "qwen3-reranker-0.6B" - # Pinned general-purpose / utility models. Coexist in VRAM, never - # unload. Members also have ttl: 0 individually so idle-timeout can't - # drop them. - # - # Current pins: - # qwen3.5-9b — ~6 GB at Q4 + KV. General-purpose chat baseline. - # VRAM budget: ~6 GB persistent in the pin slot. llama-swap is pinned to - # GPU 0 (a single RTX PRO 6000 Blackwell, 96 GB), so this leaves ~90 GB - # for whichever non-pinned model the user invokes alongside. - # - # granite-4-small WAS pinned here; removed 2026-06-04 — superseded by - # phi4-mini (vLLM FP8, stacks/vllm → vllm-phi4). Freed ~24 GB (120K KV). - # - # qwen3.6-35-a3b WAS in this group; removed 2026-04-27 because its - # ~29 GB at Q6_K_XL pushed concurrent loads OOM. Now lives outside - # with ttl: 0 — never idle-unloads but evictable under memory pressure. - "pinned": - swap: false - exclusive: false - persistent: true - members: - - "qwen3.5-9b" + # Pinned group RETIRED 2026-06-14 — its only pin (qwen3.5-9b) was dropped, so + # there is no active `pinned` group. General chat/vision moved to Qwen3.6-35B- + # A3B FP8 on GPU 1 (vLLM), not a GGUF pin on GPU 0; GPU 0 is now fully free for + # the creative-writing hot-swap card. Re-add a `pinned` group here if a + # persistent GPU-0 model is ever wanted again. + # (History: granite-4-small removed 2026-06-04 → phi4-mini vLLM FP8; + # qwen3.6-35-a3b removed 2026-04-27 — ~29 GB Q6_K_XL OOM'd concurrent loads.) diff --git a/stacks/qwen35-vl/.env.example b/stacks/qwen35-vl/.env.example deleted file mode 100644 index ad3d44a..0000000 --- a/stacks/qwen35-vl/.env.example +++ /dev/null @@ -1,28 +0,0 @@ -# Qwen3.5-9B VL (FP8) on ana-ml2 — copy to .env on the host and fill. -# Real .env lives on ana-ml2 at /opt/docker/compose/qwen35-vl/.env (gitignored). - -# Pinned nightly digest — carries the Qwen3.5-VL vision-FP8 exclusion fix that -# :latest (v0.19.1) lacks. Re-pin to :latest once the fix reaches a stable -# release (see README + compose header). -QWEN_IMAGE=vllm/vllm-openai@sha256:49211ab2155b21a2dc35f3583f5b545f5e55e77daf8f86df49977c71d5f2f528 - -QWEN_CONTAINER_NAME=vllm-qwen35 -QWEN_MODEL=Qwen/Qwen3.5-9B -QWEN_SERVED_NAME=qwen3.5-9b-fp8 -QWEN_PORT=8007 - -# GPU 1 = shared with the granite summarizer + embed/rerank/reward trio. -# GPU 0 is kept free for hot-reloading large models. -QWEN_GPU_ID=1 - -# util 0.35 (~33.6 GB) / max-len 65536 — GPU-1 rebalance 2026-06-13. Qwen was -# wildly over-provisioned (20x conc @ 32k); trimmed to free room for granite while -# DOUBLING qwen's own context (32k->65k, still ~8x conc). On this shared card vLLM -# needs free >= util*total (cap ~0.51 here); floor to start at 65k is ~0.34. -# Raising max-len is free for short requests (PagedAttention = KV per actual token). -QWEN_GPU_MEM_UTIL=0.35 -QWEN_MAX_MODEL_LEN=65536 - -# Optional -HF_TOKEN= -API_KEY= diff --git a/stacks/qwen35-vl/README.md b/stacks/qwen35-vl/README.md deleted file mode 100644 index dfc9dcf..0000000 --- a/stacks/qwen35-vl/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# qwen35-vl — Qwen3.5-9B vision-language (FP8) on ana-ml2 - -Qwen3.5-9B, a hybrid GDN + vision-language model, served **FP8** on ana-ml2 -**GPU 1**, fronted by the LiteLLM gateway as `qwen3.5-9b-fp8`. Image + video -understanding and chat. The language model is FP8; the **vision tower stays -BF16** (see below). - -## Placement -- **GPU 1**, co-located with the granite summarizer + embed/rerank/reward trio. - **GPU 0 is deliberately kept free** for hot-reloading large models. -- Port **8007**. Gateway: `qwen3.5-9b-fp8` via LiteLLM (`ana-docker:4000`). -- Container `vllm-qwen35`, compose project `qwen35-vl`. - -## Why a pinned nightly digest (not :latest) -vLLM `:latest` (v0.19.1) quantizes the Qwen3.5-VL **vision tower** under -`--quantization fp8` → garbage vision (the LM answers text fine but "sees" -noise — verified: it described the two-cats COCO image as "a 6×6 grid of gray -squares"). The **nightly** correctly excludes the vision tower from FP8, so -vision works while the LM still gets the FP8 throughput/VRAM win (BF16 vision -read perfectly: "two cats on a bright pink surface… two remote controls"). - -We pin the exact nightly digest (`sha256:49211ab2…`) for reproducibility — a -moving `:nightly` tag would silently change the engine. **WATCH:** when the -vision-FP8 exclusion lands in a stable release, re-pin to `:latest` and delete -this note. - -## Why util 0.40 -The model needs ~34 GB just to **start** at 32k context (FP8 weights + BF16 -vision tower + CUDA-graph capture + 32k memory profiling). On shared GPU 1 -(prod uses ~46 GB, ~48 GB free) this vLLM build requires `free >= util*total`, -capping util at ~0.51 here; **0.40 (~38 GB)** sits above the ~34 GB floor with -~10 GB card headroom and reports ~20× max concurrency at 32k. (Empty-GPU floor -was 0.35; below ~0.33 it crashes with "no KV blocks".) To shrink the footprint, -lower `QWEN_MAX_MODEL_LEN` (vision queries rarely need 32k) rather than util. - -## Deploy -``` -scripts/deploy-stack.sh ana-ml2 qwen35-vl # or scp compose to the host -# on ana-ml2: create /opt/docker/compose/qwen35-vl/.env from .env.example, then: -cd /opt/docker/compose/qwen35-vl && docker compose up -d -``` - -## Smoke test (incl. vision) -```bash -curl -s http://10.250.50.54:8007/v1/chat/completions -H 'Content-Type: application/json' \ - -d '{"model":"qwen3.5-9b-fp8","messages":[{"role":"user","content":[ - {"type":"text","text":"How many cats and what surface are they on?"}, - {"type":"image_url","image_url":{"url":"http://images.cocodataset.org/val2017/000000039769.jpg"}}]}], - "max_tokens":120,"chat_template_kwargs":{"enable_thinking":false}}' -``` -It's a **thinking** model (emits a reasoning trace by default) — pass -`chat_template_kwargs:{"enable_thinking":false}` for terse answers. - -## Related -- The NVFP4 path for this model was abandoned — FP8 is the answer on Blackwell - (W4A4 collapses, weight-only 4-bit doesn't accelerate). The AxionML NVFP4 - community quant + joninco SGLang fork were the NVFP4 attempt; not used. diff --git a/stacks/qwen35-vl/compose.yaml b/stacks/qwen35-vl/compose.yaml deleted file mode 100644 index 3cbbb59..0000000 --- a/stacks/qwen35-vl/compose.yaml +++ /dev/null @@ -1,87 +0,0 @@ -# qwen35-vl — Qwen3.5-9B vision-language model (FP8) on ana-ml2. -# -# Co-located on GPU 1 with the granite summarizer + embed/rerank/reward trio -# (GPU 0 is deliberately kept free for hot-reloading large models). Serves on -# :8007, fronted by the LiteLLM gateway as `qwen3.5-9b-fp8`. -# -# WHY A PINNED NIGHTLY DIGEST (not :latest): vLLM :latest (v0.19.1) quantizes -# the Qwen3.5-VL *vision tower* under --quantization fp8, producing garbage -# vision output (the language model is unaffected — it answers text fine but -# "sees" noise). The nightly correctly excludes the vision tower, so vision -# works while the LM still gets the FP8 throughput/VRAM win. We pin the exact -# nightly digest for reproducibility — a moving :nightly tag would silently -# change the engine. WATCH: once the vision-FP8 exclusion lands in a stable -# release, re-pin to :latest and drop this note. -# -# WHY util 0.40 (not the trio's tiny values): the model needs ~34 GB just to -# start at 32k context (FP8 weights + BF16 vision tower + graph capture + 32k -# profiling). On shared GPU 1 (prod uses ~46 GB, ~48 GB free) this vLLM build -# requires free >= util*total, capping util at ~0.51 here; 0.40 (~38 GB) sits -# above the ~34 GB floor with ~10 GB card headroom. -# -# All tunables live in .env — edit that, not this file. - -name: qwen35-vl - -services: - vllm-qwen35: - image: ${QWEN_IMAGE} - container_name: ${QWEN_CONTAINER_NAME} - restart: unless-stopped - ipc: host - ports: - - "${QWEN_PORT}:8000" - volumes: - - /tank/aimodels/huggingface:/hfcache - environment: - - HF_HOME=/hfcache - - HF_HUB_CACHE=/hfcache/hub - - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-} - - VLLM_API_KEY=${API_KEY:-} - command: - - ${QWEN_MODEL} - - --served-model-name - - ${QWEN_SERVED_NAME} - - --quantization - - fp8 - - --host - - 0.0.0.0 - - --port - - "8000" - - --gpu-memory-utilization - - ${QWEN_GPU_MEM_UTIL} - - --max-model-len - - ${QWEN_MAX_MODEL_LEN} - - --dtype - - auto - # Prefix caching pinned ON (the nightly defaults it OFF). Free win for the - # text-chat path; marginal for vision (each image is a distinct prefix). - - --enable-prefix-caching - deploy: - resources: - reservations: - devices: - - driver: nvidia - device_ids: - - "${QWEN_GPU_ID}" - capabilities: - - gpu - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8000/health"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 300s - networks: - - tnet - labels: - - homepage.group=AI Systems - - homepage.name=Qwen3.5-9B VL (FP8) - - homepage.icon=mdi-image-search - - homepage.description=Qwen3.5-9B vision-language (FP8) via vLLM (ana-ml2) - - homepage.href=http://10.250.50.54:${QWEN_PORT}/docs - -networks: - tnet: - name: traefik-net - external: true diff --git a/stacks/qwen36-vl/.env.example b/stacks/qwen36-vl/.env.example new file mode 100644 index 0000000..0771721 --- /dev/null +++ b/stacks/qwen36-vl/.env.example @@ -0,0 +1,34 @@ +# Qwen3.6-35B-A3B VL (official FP8) on ana-ml2 — copy to .env on the host and fill. +# Real .env lives on ana-ml2 at /opt/docker/compose/qwen36-vl/.env (gitignored). +# +# Replaces qwen35-vl (Qwen3.5-9B) 2026-06-14. See compose.yaml header for the +# FP8-over-NVFP4 rationale (vLLM NVFP4 MoE loader broken, #44081) and why this +# uses plain :latest with NO --quantization (pre-quantized checkpoint; a forced +# flag would noise-quantize the vision tower like the old qwen35-vl). + +# :latest is fine — the official FP8 checkpoint loads + serves vision correctly +# on 0.19.1 (validated 2026-06-14). NO pinned nightly digest needed. +QWEN_IMAGE=vllm/vllm-openai:latest + +QWEN_CONTAINER_NAME=vllm-qwen36 +QWEN_MODEL=Qwen/Qwen3.6-35B-A3B-FP8 +QWEN_PORT=8007 + +# GPU 1 = shared with the granite summarizer + embed/rerank/reward trio. +# GPU 0 is kept free for the llama-swap creative-writing hot-swap card. +QWEN_GPU_ID=1 + +# util 0.42 (~40 GB) — official FP8 weights load in ~34.2 GiB; 0.42 covers +# weights + CUDA-graph + a generous KV pool. Hybrid attn (10 of 40 layers full- +# attn, ~10 KB/tok KV) makes long context nearly free, so max-len is generous. +# Budget (2026-06-14): qwen36 0.42 + granite 0.28 + trio 0.20 = 0.90 total, +# ~10 GB graph headroom. Bring qwen36 up LAST so capture sees the free room. +QWEN_GPU_MEM_UTIL=0.42 +QWEN_MAX_MODEL_LEN=131072 +# Sampler-warmup OOM guard on the shared GPU (248K vocab × default 1024 seqs is +# a huge transient). 32 is plenty for a vision endpoint. +QWEN_MAX_NUM_SEQS=32 + +# Optional +HF_TOKEN= +API_KEY= diff --git a/stacks/qwen36-vl/compose.yaml b/stacks/qwen36-vl/compose.yaml new file mode 100644 index 0000000..a30eaea --- /dev/null +++ b/stacks/qwen36-vl/compose.yaml @@ -0,0 +1,102 @@ +# qwen36-vl — Qwen3.6-35B-A3B vision-language MoE (official FP8) on ana-ml2. +# +# Replaces the qwen35-vl stack (Qwen3.5-9B) 2026-06-14. Co-located on GPU 1 with +# the granite summarizer + embed/rerank/reward trio (GPU 0 stays free for the +# llama-swap creative-writing hot-swap card). Serves on :8007. +# +# WHY official FP8 (not NVFP4): NVFP4 (nvidia/Qwen3.6-35B-A3B-NVFP4, ~21 GB) is the +# lighter fit but its vLLM ModelOpt-MoE loader is BROKEN as of 0.19.1/0.22.0 +# (KeyError w2_input_scale / lm_head.input_scale — vLLM #44081). The official +# Qwen pre-quantized FP8 (~34 GB weights) loads clean on :latest and — unlike +# the old qwen35-vl — needs NO pinned nightly digest: that hack existed because +# vLLM DYNAMIC `--quantization fp8` quantized the vision tower to noise. This +# checkpoint is PRE-quantized, so we OMIT --quantization (vLLM auto-detects the +# checkpoint's own fp8) and the vision tower is preserved. Validated 2026-06-14 +# on GPU 0: loads in 34.2 GiB, image test returns correct ("Blue"). Revisit +# NVFP4 (frees ~13 GB) once vLLM's loader is fixed. +# +# NAMING: served ONLY as its TRUE name `qwen3.6-35b-a3b`. A model is never aliased +# under a prior model's name — a caller asking for `qwen3.5-9b-fp8` (a 9B dense) +# must NOT be silently handed this 35B-A3B MoE; that's a downstream-confusion +# footgun. The legacy `qwen3.5-9b-fp8` name is RETIRED. Consumers (Arbo's vision +# hero-judge, stacks/arbo v0.11.3+) migrate to `qwen3.6-35b-a3b` — they 404 on the +# old name until they repoint, which is the correct loud signal (notified 2026-06-14). +# +# WHY util 0.42 / max-len 131072: hybrid attn (10 of 40 layers full-attn, ~10 KB/ +# tok KV) → KV is cheap, so big context is nearly free; the 34 GB weights are the +# cost. 0.42 (~40 GB) = weights + graph + generous KV. Granite drops to 0.25/64K +# to make room (the FP8-vs-maxed-granite tradeoff, operator-approved 2026-06-14). +# +# All tunables live in .env — edit that, not this file. + +name: qwen36-vl + +services: + vllm-qwen36: + image: ${QWEN_IMAGE} + container_name: ${QWEN_CONTAINER_NAME} + restart: unless-stopped + ipc: host + ports: + - "${QWEN_PORT}:8000" + volumes: + - /tank/aimodels/huggingface:/hfcache + environment: + - HF_HOME=/hfcache + - HF_HUB_CACHE=/hfcache/hub + - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-} + - VLLM_API_KEY=${API_KEY:-} + command: + - ${QWEN_MODEL} + # Pre-quantized FP8 checkpoint → NO --quantization (vLLM auto-detects; a + # forced flag would re-quantize the vision tower to noise, see header). + - --served-model-name + - qwen3.6-35b-a3b + - --host + - 0.0.0.0 + - --port + - "8000" + - --gpu-memory-utilization + - ${QWEN_GPU_MEM_UTIL} + - --max-model-len + - ${QWEN_MAX_MODEL_LEN} + # Cap concurrency: vLLM warms the sampler with max_num_seqs dummy requests, + # and this model's 248K vocab makes that warmup tensor huge — the default + # 1024 OOMs on a shared GPU even though weights+KV fit. 32 is ample for a + # vision endpoint (the summarizer carries the concurrency, not this). + - --max-num-seqs + - ${QWEN_MAX_NUM_SEQS} + - --kv-cache-dtype + - fp8 + - --trust-remote-code + - --dtype + - auto + - --enable-prefix-caching + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: + - "${QWEN_GPU_ID}" + capabilities: + - gpu + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 300s + networks: + - tnet + labels: + - homepage.group=AI Systems + - homepage.name=Qwen3.6-35B-A3B VL (FP8) + - homepage.icon=mdi-image-search + - homepage.description=Qwen3.6-35B-A3B vision-language MoE (FP8) via vLLM (ana-ml2) + - homepage.href=http://10.250.50.54:${QWEN_PORT}/docs + +networks: + tnet: + name: traefik-net + external: true diff --git a/stacks/vllm/.env.example b/stacks/vllm/.env.example index dbe5696..ce1d18e 100644 --- a/stacks/vllm/.env.example +++ b/stacks/vllm/.env.example @@ -48,8 +48,12 @@ RERANK_MODEL=Qwen/Qwen3-Reranker-0.6B # reward 0.10 (~9.6 GB) — the real over-provision was here (0.18 -> 0.10). # Net: ~11 GB freed on GPU 1. (Fractions are of TOTAL card VRAM — re-floor if # the cards change again.) -EMBED_GPU_MEM_UTIL=0.05 -RERANK_GPU_MEM_UTIL=0.05 +# 0.03 each — REBALANCED 2026-06-14 (was 0.05): the 0.6B models sat at ~5.5 GB +# each at 0.05 (mostly util-reservation waste); 0.03 (~3.6 GB) fits weights + +# CUDA context with room, freeing ~4 GB back to granite. Recreate them ONE AT A +# TIME — concurrent recreate races the memory-profiling assertion. +EMBED_GPU_MEM_UTIL=0.03 +RERANK_GPU_MEM_UTIL=0.03 REWARD_GPU_MEM_UTIL=0.10 # Context length caps — lower these if VRAM is tight. @@ -89,11 +93,18 @@ GRANITE_SERVED_NAME=granite-4.1-8b # PagedAttention allocates KV per ACTUAL token, so 131072 is only a CEILING — a 1k # summarize turn uses ~1k tokens, so the pool holds ~300 concurrently; the "2.33x" # headline is worst-case (every request maxing 131k). Granite 4.1 supports 131072. -GRANITE_MAX_MODEL_LEN=131072 +# 65536 — REDUCED 2026-06-14 (was 131072) to free GPU-1 room for the FP8 vision +# model (Qwen3.6-35B-A3B, stacks/qwen36-vl, ~34 GB weights). Summarizer load is +# short parallel calls, so the 64K cap is ample. +GRANITE_MAX_MODEL_LEN=65536 # FP8 KV cache (native on Blackwell cc 12.0). At 50K ≈ ~4.2 GB (vs ~8.4 GB at fp16). GRANITE_KV_CACHE_DTYPE=fp8 # util 0.35 (~33.6 GB) — tuned 2026-06-13 to leave ~3.5 GB free on GPU 1 alongside # the trio + qwen co-tenants. On this shared card vLLM needs free >= util*total at # startup, and START ORDER matters: trim qwen FIRST, then grow granite, else granite # OOMs against the full card. (0.37 overshot to 1.7 GB free; 0.35 lands ~3.7 GB.) -GRANITE_GPU_MEM_UTIL=0.35 +# 0.24 — REBALANCED 2026-06-14 (was 0.35) for the FP8 vision cutover. GPU-1 budget: +# qwen36-vl 0.46 + granite 0.24 + reward 0.10 + embed/rerank 0.03 ≈ 0.90 total, +# ~7.5 GB headroom (the OOM buffer; held under 20-concurrent load test). granite +# gets a 169K-token KV pool = 2.58x concurrency @ 64K. Bring qwen36 up LAST. +GRANITE_GPU_MEM_UTIL=0.24