vh
2e3dcc2d3d
Qwen3.5-9B vision-language served FP8 on ana-ml2 GPU1 (co-located with granite + the embed/rerank/reward trio; GPU0 kept free for hot-loading large models), :8007, fronted by LiteLLM as qwen3.5-9b-fp8. Pinned to vllm/vllm-openai nightly@sha256:49211ab2 — :latest (v0.19.1) quantizes the VL vision tower under fp8 and garbles vision; the nightly correctly excludes it (LM stays FP8, vision tower BF16). util 0.40 (~38GB) on the shared card (vLLM needs free>=util*total here). Vision verified end-to-end through the gateway.
85 lines
2.7 KiB
YAML
85 lines
2.7 KiB
YAML
# qwen35-vl — Qwen3.5-9B vision-language model (FP8) on ana-ml2.
|
|
#
|
|
# Co-located on GPU 1 with the granite summarizer + embed/rerank/reward trio
|
|
# (GPU 0 is deliberately kept free for hot-reloading large models). Serves on
|
|
# :8007, fronted by the LiteLLM gateway as `qwen3.5-9b-fp8`.
|
|
#
|
|
# WHY A PINNED NIGHTLY DIGEST (not :latest): vLLM :latest (v0.19.1) quantizes
|
|
# the Qwen3.5-VL *vision tower* under --quantization fp8, producing garbage
|
|
# vision output (the language model is unaffected — it answers text fine but
|
|
# "sees" noise). The nightly correctly excludes the vision tower, so vision
|
|
# works while the LM still gets the FP8 throughput/VRAM win. We pin the exact
|
|
# nightly digest for reproducibility — a moving :nightly tag would silently
|
|
# change the engine. WATCH: once the vision-FP8 exclusion lands in a stable
|
|
# release, re-pin to :latest and drop this note.
|
|
#
|
|
# WHY util 0.40 (not the trio's tiny values): the model needs ~34 GB just to
|
|
# start at 32k context (FP8 weights + BF16 vision tower + graph capture + 32k
|
|
# profiling). On shared GPU 1 (prod uses ~46 GB, ~48 GB free) this vLLM build
|
|
# requires free >= util*total, capping util at ~0.51 here; 0.40 (~38 GB) sits
|
|
# above the ~34 GB floor with ~10 GB card headroom.
|
|
#
|
|
# All tunables live in .env — edit that, not this file.
|
|
|
|
name: qwen35-vl
|
|
|
|
services:
|
|
vllm-qwen35:
|
|
image: ${QWEN_IMAGE}
|
|
container_name: ${QWEN_CONTAINER_NAME}
|
|
restart: unless-stopped
|
|
ipc: host
|
|
ports:
|
|
- "${QWEN_PORT}:8000"
|
|
volumes:
|
|
- /tank/aimodels/huggingface:/hfcache
|
|
environment:
|
|
- HF_HOME=/hfcache
|
|
- HF_HUB_CACHE=/hfcache/hub
|
|
- HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-}
|
|
- VLLM_API_KEY=${API_KEY:-}
|
|
command:
|
|
- ${QWEN_MODEL}
|
|
- --served-model-name
|
|
- ${QWEN_SERVED_NAME}
|
|
- --quantization
|
|
- fp8
|
|
- --host
|
|
- 0.0.0.0
|
|
- --port
|
|
- "8000"
|
|
- --gpu-memory-utilization
|
|
- ${QWEN_GPU_MEM_UTIL}
|
|
- --max-model-len
|
|
- ${QWEN_MAX_MODEL_LEN}
|
|
- --dtype
|
|
- auto
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
device_ids:
|
|
- "${QWEN_GPU_ID}"
|
|
capabilities:
|
|
- gpu
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 3
|
|
start_period: 300s
|
|
networks:
|
|
- tnet
|
|
labels:
|
|
- homepage.group=AI Systems
|
|
- homepage.name=Qwen3.5-9B VL (FP8)
|
|
- homepage.icon=mdi-image-search
|
|
- homepage.description=Qwen3.5-9B vision-language (FP8) via vLLM (ana-ml2)
|
|
- homepage.href=http://10.250.50.54:${QWEN_PORT}/docs
|
|
|
|
networks:
|
|
tnet:
|
|
name: traefik-net
|
|
external: true
|