diff --git a/stacks/litellm/conf/config.yaml b/stacks/litellm/conf/config.yaml index 22d6c1a..37c0ee1 100644 --- a/stacks/litellm/conf/config.yaml +++ b/stacks/litellm/conf/config.yaml @@ -30,6 +30,17 @@ model_list: model_info: mode: chat + # --- Qwen3.5-9B vision-language (FP8) — vision + chat. vLLM on ana-ml2 GPU 1, + # nightly-pinned (vision-FP8 exclusion fix), :8007. Explicit entry shadows + # the "*" wildcard llama-swap route. --- + - model_name: qwen3.5-9b-fp8 + litellm_params: + model: hosted_vllm/qwen3.5-9b-fp8 + api_base: http://10.250.50.54:8007/v1 + api_key: os.environ/VLLM_API_KEY + model_info: + mode: chat + # --- Qwen3 embeddings --- - model_name: qwen3-embedding litellm_params: diff --git a/stacks/qwen35-vl/.env.example b/stacks/qwen35-vl/.env.example new file mode 100644 index 0000000..5b145d0 --- /dev/null +++ b/stacks/qwen35-vl/.env.example @@ -0,0 +1,25 @@ +# Qwen3.5-9B VL (FP8) on ana-ml2 — copy to .env on the host and fill. +# Real .env lives on ana-ml2 at /opt/docker/compose/qwen35-vl/.env (gitignored). + +# Pinned nightly digest — carries the Qwen3.5-VL vision-FP8 exclusion fix that +# :latest (v0.19.1) lacks. Re-pin to :latest once the fix reaches a stable +# release (see README + compose header). +QWEN_IMAGE=vllm/vllm-openai@sha256:49211ab2155b21a2dc35f3583f5b545f5e55e77daf8f86df49977c71d5f2f528 + +QWEN_CONTAINER_NAME=vllm-qwen35 +QWEN_MODEL=Qwen/Qwen3.5-9B +QWEN_SERVED_NAME=qwen3.5-9b-fp8 +QWEN_PORT=8007 + +# GPU 1 = shared with the granite summarizer + embed/rerank/reward trio. +# GPU 0 is kept free for hot-reloading large models. +QWEN_GPU_ID=1 + +# 0.40 (~38 GB) — above the ~34 GB start floor, ~10 GB card headroom over prod. +# On this shared card vLLM needs free >= util*total, so util is capped ~0.51. +QWEN_GPU_MEM_UTIL=0.40 +QWEN_MAX_MODEL_LEN=32768 + +# Optional +HF_TOKEN= +API_KEY= diff --git a/stacks/qwen35-vl/README.md b/stacks/qwen35-vl/README.md new file mode 100644 index 0000000..dfc9dcf --- /dev/null +++ b/stacks/qwen35-vl/README.md @@ -0,0 +1,57 @@ +# qwen35-vl — Qwen3.5-9B vision-language (FP8) on ana-ml2 + +Qwen3.5-9B, a hybrid GDN + vision-language model, served **FP8** on ana-ml2 +**GPU 1**, fronted by the LiteLLM gateway as `qwen3.5-9b-fp8`. Image + video +understanding and chat. The language model is FP8; the **vision tower stays +BF16** (see below). + +## Placement +- **GPU 1**, co-located with the granite summarizer + embed/rerank/reward trio. + **GPU 0 is deliberately kept free** for hot-reloading large models. +- Port **8007**. Gateway: `qwen3.5-9b-fp8` via LiteLLM (`ana-docker:4000`). +- Container `vllm-qwen35`, compose project `qwen35-vl`. + +## Why a pinned nightly digest (not :latest) +vLLM `:latest` (v0.19.1) quantizes the Qwen3.5-VL **vision tower** under +`--quantization fp8` → garbage vision (the LM answers text fine but "sees" +noise — verified: it described the two-cats COCO image as "a 6×6 grid of gray +squares"). The **nightly** correctly excludes the vision tower from FP8, so +vision works while the LM still gets the FP8 throughput/VRAM win (BF16 vision +read perfectly: "two cats on a bright pink surface… two remote controls"). + +We pin the exact nightly digest (`sha256:49211ab2…`) for reproducibility — a +moving `:nightly` tag would silently change the engine. **WATCH:** when the +vision-FP8 exclusion lands in a stable release, re-pin to `:latest` and delete +this note. + +## Why util 0.40 +The model needs ~34 GB just to **start** at 32k context (FP8 weights + BF16 +vision tower + CUDA-graph capture + 32k memory profiling). On shared GPU 1 +(prod uses ~46 GB, ~48 GB free) this vLLM build requires `free >= util*total`, +capping util at ~0.51 here; **0.40 (~38 GB)** sits above the ~34 GB floor with +~10 GB card headroom and reports ~20× max concurrency at 32k. (Empty-GPU floor +was 0.35; below ~0.33 it crashes with "no KV blocks".) To shrink the footprint, +lower `QWEN_MAX_MODEL_LEN` (vision queries rarely need 32k) rather than util. + +## Deploy +``` +scripts/deploy-stack.sh ana-ml2 qwen35-vl # or scp compose to the host +# on ana-ml2: create /opt/docker/compose/qwen35-vl/.env from .env.example, then: +cd /opt/docker/compose/qwen35-vl && docker compose up -d +``` + +## Smoke test (incl. vision) +```bash +curl -s http://10.250.50.54:8007/v1/chat/completions -H 'Content-Type: application/json' \ + -d '{"model":"qwen3.5-9b-fp8","messages":[{"role":"user","content":[ + {"type":"text","text":"How many cats and what surface are they on?"}, + {"type":"image_url","image_url":{"url":"http://images.cocodataset.org/val2017/000000039769.jpg"}}]}], + "max_tokens":120,"chat_template_kwargs":{"enable_thinking":false}}' +``` +It's a **thinking** model (emits a reasoning trace by default) — pass +`chat_template_kwargs:{"enable_thinking":false}` for terse answers. + +## Related +- The NVFP4 path for this model was abandoned — FP8 is the answer on Blackwell + (W4A4 collapses, weight-only 4-bit doesn't accelerate). The AxionML NVFP4 + community quant + joninco SGLang fork were the NVFP4 attempt; not used. diff --git a/stacks/qwen35-vl/compose.yaml b/stacks/qwen35-vl/compose.yaml new file mode 100644 index 0000000..5377998 --- /dev/null +++ b/stacks/qwen35-vl/compose.yaml @@ -0,0 +1,84 @@ +# qwen35-vl — Qwen3.5-9B vision-language model (FP8) on ana-ml2. +# +# Co-located on GPU 1 with the granite summarizer + embed/rerank/reward trio +# (GPU 0 is deliberately kept free for hot-reloading large models). Serves on +# :8007, fronted by the LiteLLM gateway as `qwen3.5-9b-fp8`. +# +# WHY A PINNED NIGHTLY DIGEST (not :latest): vLLM :latest (v0.19.1) quantizes +# the Qwen3.5-VL *vision tower* under --quantization fp8, producing garbage +# vision output (the language model is unaffected — it answers text fine but +# "sees" noise). The nightly correctly excludes the vision tower, so vision +# works while the LM still gets the FP8 throughput/VRAM win. We pin the exact +# nightly digest for reproducibility — a moving :nightly tag would silently +# change the engine. WATCH: once the vision-FP8 exclusion lands in a stable +# release, re-pin to :latest and drop this note. +# +# WHY util 0.40 (not the trio's tiny values): the model needs ~34 GB just to +# start at 32k context (FP8 weights + BF16 vision tower + graph capture + 32k +# profiling). On shared GPU 1 (prod uses ~46 GB, ~48 GB free) this vLLM build +# requires free >= util*total, capping util at ~0.51 here; 0.40 (~38 GB) sits +# above the ~34 GB floor with ~10 GB card headroom. +# +# All tunables live in .env — edit that, not this file. + +name: qwen35-vl + +services: + vllm-qwen35: + image: ${QWEN_IMAGE} + container_name: ${QWEN_CONTAINER_NAME} + restart: unless-stopped + ipc: host + ports: + - "${QWEN_PORT}:8000" + volumes: + - /tank/aimodels/huggingface:/hfcache + environment: + - HF_HOME=/hfcache + - HF_HUB_CACHE=/hfcache/hub + - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-} + - VLLM_API_KEY=${API_KEY:-} + command: + - ${QWEN_MODEL} + - --served-model-name + - ${QWEN_SERVED_NAME} + - --quantization + - fp8 + - --host + - 0.0.0.0 + - --port + - "8000" + - --gpu-memory-utilization + - ${QWEN_GPU_MEM_UTIL} + - --max-model-len + - ${QWEN_MAX_MODEL_LEN} + - --dtype + - auto + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: + - "${QWEN_GPU_ID}" + capabilities: + - gpu + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 300s + networks: + - tnet + labels: + - homepage.group=AI Systems + - homepage.name=Qwen3.5-9B VL (FP8) + - homepage.icon=mdi-image-search + - homepage.description=Qwen3.5-9B vision-language (FP8) via vLLM (ana-ml2) + - homepage.href=http://10.250.50.54:${QWEN_PORT}/docs + +networks: + tnet: + name: traefik-net + external: true