diff --git a/stacks/litellm/conf/config.yaml b/stacks/litellm/conf/config.yaml index 768a1a3..affee6a 100644 --- a/stacks/litellm/conf/config.yaml +++ b/stacks/litellm/conf/config.yaml @@ -41,40 +41,25 @@ model_list: model_info: mode: chat - # --- Qwen3.6-35B-A3B vision-language MoE (official FP8) — vision + chat. vLLM - # on ana-ml2 GPU 1, :8007. Explicit entry shadows the "*" wildcard llama-swap - # route. REPLACED qwen3.5-9b-fp8 2026-06-14 (the 9B is retired; this is a - # 35B-A3B MoE — served under its TRUE name, never aliased under the old one). - # - # THINKING SPLIT (2026-06-15, operator call — mirrors the glm-5.1 pattern - # above). One hybrid checkpoint; the per-request `enable_thinking` switch - # picks the mode. LiteLLM forwards extra_body verbatim to vLLM, where - # chat_template_kwargs lands in the chat template. vLLM runs - # --reasoning-parser qwen3 so reasoning surfaces as reasoning_content. --- - # qwen3.6-35b-a3b: thinking DISABLED by default. The checkpoint defaults - # thinking ON; enable_thinking=false forces the empty block. - # Reasoning is opt-in via qwen3.6-35b-a3b-thinking below. - - model_name: qwen3.6-35b-a3b + # --- Qwen-Image-Bench (NVFP4) — text-to-image quality JUDGE, vision-intact. vLLM + # on ana-ml2 GPU 1, :8014. REPLACED qwen3.6-35b-a3b (operator 2026-06-19; the + # qwen36 VL judge was downed + its FP8 weights freed ~33GB). Multimodal — image + # + prompt in, a 5-dim verdict out (overall quality / prompt match / aesthetic / + # LoRA activation / confidence). qwen-image-judge is a duplicate-entry alias. + # ⚠️ arbo's hero-judge was bound to qwen3.6-35b-a3b — comfy-dev notified to + # repoint (different I/O than the old general VL judge). --- + - model_name: qwen-image-bench litellm_params: - model: hosted_vllm/qwen3.6-35b-a3b - api_base: http://10.250.50.54:8007/v1 + model: hosted_vllm/qwen-image-bench + api_base: http://10.250.50.54:8014/v1 api_key: os.environ/VLLM_API_KEY - extra_body: - chat_template_kwargs: - enable_thinking: false model_info: mode: chat - # qwen3.6-35b-a3b-thinking: identical upstream checkpoint, thinking ENABLED - # (opt-in reasoning). The qwen3 reasoning-parser splits into - # reasoning_content; content holds just the answer. - - model_name: qwen3.6-35b-a3b-thinking + - model_name: qwen-image-judge litellm_params: - model: hosted_vllm/qwen3.6-35b-a3b - api_base: http://10.250.50.54:8007/v1 + model: hosted_vllm/qwen-image-bench + api_base: http://10.250.50.54:8014/v1 api_key: os.environ/VLLM_API_KEY - extra_body: - chat_template_kwargs: - enable_thinking: true model_info: mode: chat diff --git a/stacks/qwen-image-bench/.env.example b/stacks/qwen-image-bench/.env.example new file mode 100644 index 0000000..355d4c9 --- /dev/null +++ b/stacks/qwen-image-bench/.env.example @@ -0,0 +1,20 @@ +# qwen-image-bench (flukethoughts/Qwen-Image-Bench-NVFP4) — ana-ml2 GPU 1 tunables. +# Real .env lives at /opt/docker/compose/qwen-image-bench/.env. + +QIB_IMAGE=vllm/vllm-openai:latest +QIB_CONTAINER_NAME=vllm-qwen-image-bench + +# Own port (8007=qwen36 [downed], 8010=qwen-gen, 8011=selene — 8014 free). +QIB_PORT=8014 +QIB_GPU_ID=1 + +# Shared GPU 1 (granite/selene/embed/rerank/reward + this). util is a fraction of +# TOTAL (96GB), so it must clear the ~20GB weight floor + leave KV: 0.22 (~21GB) left +# ~nothing for KV and crash-looped ("no available memory for cache blocks"). 0.32 +# (~31GB → ~9GB KV) fits the ~37GB freed by downing qwen36 (cf. qwen36's own 0.34). +QIB_GPU_MEM_UTIL=0.32 +QIB_MAX_MODEL_LEN=32768 +QIB_MAX_NUM_SEQS=8 + +# Optional upstream vLLM API key (empty = no auth; internal net only). +API_KEY= diff --git a/stacks/qwen-image-bench/compose.yaml b/stacks/qwen-image-bench/compose.yaml new file mode 100644 index 0000000..4305761 --- /dev/null +++ b/stacks/qwen-image-bench/compose.yaml @@ -0,0 +1,84 @@ +# qwen-image-bench — flukethoughts/Qwen-Image-Bench-NVFP4 on ana-ml2 GPU 1, +# REPLACING qwen3.6-35b-a3b (operator 2026-06-19). Qwen's text-to-image quality +# JUDGE model (vision-language, NVFP4 weights / vision tower bf16). NOT generative — +# it scores T2I outputs on 5 dims (overall quality, prompt match, aesthetic, LoRA +# activation, confidence). +# +# Arch: Qwen3_5ForConditionalGeneration (dense Qwen3.5 hybrid SSM+attn + vision), +# ~17B / ~20GB NVFP4. VISION-INTACT → served as multimodal; NO text-only patch +# (unlike the qwen3.5-122b gen model, which had text-only weights). vLLM +# production-validated per the model card. +# +# ⚠️ qwen3.6-35b-a3b was arbo's hero-judge (comfy-dev consumer). Downing it breaks +# arbo's judging until comfy-dev repoints to qwen-image-bench (different I/O — a +# 5-dim verdict vs a general VL judge). comfy-dev notified. +# +# DISPLACEMENT: GPU 1 is shared (granite/selene/embed/rerank/reward). qwen3.6 used +# util 0.34 (~33GB); down it first, then this fits at util ~0.22 (~21GB). REVERT = +# down this, `up -d` the qwen36 stack. +# +# Tunables in .env. + +name: qwen-image-bench + +services: + vllm-qwen-image-bench: + image: ${QIB_IMAGE:-vllm/vllm-openai:latest} + container_name: ${QIB_CONTAINER_NAME:-vllm-qwen-image-bench} + restart: unless-stopped + ipc: host + ports: + - "${QIB_PORT:-8014}:8000" + volumes: + - /tank/aimodels/huggingface:/hfcache + - /tank/aimodels/qwen-image-bench-nvfp4:/model:ro + environment: + - HF_HOME=/hfcache + - HF_HUB_CACHE=/hfcache/hub + - VLLM_API_KEY=${API_KEY:-} + command: + - /model + - --served-model-name + - qwen-image-bench + - --host + - 0.0.0.0 + - --port + - "8000" + - --gpu-memory-utilization + - ${QIB_GPU_MEM_UTIL:-0.32} + - --max-model-len + - ${QIB_MAX_MODEL_LEN:-32768} + - --max-num-seqs + - ${QIB_MAX_NUM_SEQS:-8} + - --trust-remote-code + - --dtype + - auto + - --enable-prefix-caching + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: + - "${QIB_GPU_ID:-1}" + capabilities: + - gpu + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 600s + networks: + - tnet + labels: + - homepage.group=AI Systems + - homepage.name=Qwen-Image-Bench (T2I judge, NVFP4) + - homepage.icon=mdi-image-check + - homepage.description=Qwen text-to-image quality judge (NVFP4, vision-intact) on ana-ml2 GPU 1 + - homepage.href=http://10.250.50.54:${QIB_PORT:-8014}/docs + +networks: + tnet: + name: traefik-net + external: true