esh-pfi-infrastructure/stacks/qwen36-vl/compose.yaml

# qwen36-vl — Qwen3.6-35B-A3B vision-language MoE (official NVFP4) on ana-ml2.
#
# Replaces the qwen35-vl stack (Qwen3.5-9B) 2026-06-14. Co-located on GPU 1 with
# the granite summarizer + embed/rerank/reward trio. Serves on :8007.
#
# WHY NVFP4 now (swapped FROM FP8 2026-06-15): the nvidia ModelOpt NVFP4 MoE that
# was BROKEN on vLLM 0.19.1/0.22.0 (#44081, lm_head.input_scale) loads clean on
# 0.23.0 — the ModelOpt lm_head fix landed. So we cut FP8→NVFP4: ~20.4 GiB weights
# vs FP8's ~34 GiB (~40% lighter, ~13 GB reclaimed on GPU 1), faster single-stream
# on Blackwell's FP4 tensor cores, and the freed room funds fp16 KV + a granite
# context restore (see the rebalance note below). The NVFP4 checkpoint preserves
# the vision tower (ModelOpt leaves it high-precision) — VALIDATED by comfy-dev's
# real anatomy-judge A/B on 16 prod images: PASS, holds the load-bearing
# discrimination (gross-deformity reject + clean-pass), only shuffles already-
# unreliable sub-ceiling borderline-hand calls. brokkr's text/speed arm: parity
# except a minor multi-step chained-numeric-reasoning slip (W4A4 tell) — doesn't
# bite the vision-judge role; flag for any gateway consumer doing chained math.
# Requires vLLM >= 0.23.0 (pinned by digest in .env). NO --quantization flag
# (vLLM auto-detects the checkpoint's NVFP4).
#
# NAMING: served ONLY as its TRUE name `qwen3.6-35b-a3b`. A model is never aliased
# under a prior model's name — a caller asking for `qwen3.5-9b-fp8` (a 9B dense)
# must NOT be silently handed this 35B-A3B MoE; that's a downstream-confusion
# footgun. The legacy `qwen3.5-9b-fp8` name is RETIRED. Consumers (Arbo's vision
# hero-judge, stacks/arbo v0.11.3+) migrate to `qwen3.6-35b-a3b` — they 404 on the
# old name until they repoint, which is the correct loud signal (notified 2026-06-14).
#
# GPU-1 REBALANCE (2026-06-15, pinned): the NVFP4 swap freed ~13 GB, redistributed —
#   qwen36 NVFP4: util 0.46→0.32 (~31 GB: 20.4 GB weights + fp16 KV + graph).
#     fp16 KV (we DROPPED --kv-cache-dtype fp8) — the freed room buys back full-
#     precision KV; hybrid attn (10/40 full-attn) keeps even fp16 KV affordable.
#   granite: RESTORED 0.24→0.34, max-len 65536→131072 (gives back the context
#     sacrificed for FP8 qwen — the FP8-vs-maxed-granite tradeoff is now undone).
#   trio (embed/rerank/reward) unchanged at floor.
#   Total GPU-1 util ~0.82 → ~17 GB headroom (was a tight ~5 GB).
#
# THINKING TOGGLE: this is ONE hybrid checkpoint (not separate Instruct/Thinking
# downloads) with a Qwen3-style per-request `enable_thinking` switch. The chat
# template defaults thinking ON (`<think>\n`); passing
# `chat_template_kwargs={"enable_thinking":false}` emits the empty
# `<think>\n\n</think>\n\n` block (no reasoning). We run --reasoning-parser qwen3
# (model-matched — its vLLM docstring describes THIS checkpoint) so ONE endpoint
# serves BOTH modes cleanly: thinking-ON splits <think>…</think> into
# reasoning_content; thinking-OFF routes everything to content. The gateway
# selects the mode per model_name (stacks/litellm/conf/config.yaml):
#   qwen3.6-35b-a3b          → enable_thinking:false  (non-thinking DEFAULT)
#   qwen3.6-35b-a3b-thinking → enable_thinking:true   (opt-in reasoning)
#
# All tunables live in .env — edit that, not this file.

name: qwen36-vl

services:
  vllm-qwen36:
    image: ${QWEN_IMAGE}
    container_name: ${QWEN_CONTAINER_NAME}
    restart: unless-stopped
    ipc: host
    ports:
      - "${QWEN_PORT}:8000"
    volumes:
      - /tank/aimodels/huggingface:/hfcache
    environment:
      - HF_HOME=/hfcache
      - HF_HUB_CACHE=/hfcache/hub
      - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-}
      - VLLM_API_KEY=${API_KEY:-}
    command:
      - ${QWEN_MODEL}
      # Pre-quantized NVFP4 (ModelOpt) checkpoint → NO --quantization (vLLM auto-
      # detects; the vision tower is left high-precision by the producer).
      - --served-model-name
      - qwen3.6-35b-a3b
      - --host
      - 0.0.0.0
      - --port
      - "8000"
      - --gpu-memory-utilization
      - ${QWEN_GPU_MEM_UTIL}
      - --max-model-len
      - ${QWEN_MAX_MODEL_LEN}
      # Cap concurrency: vLLM warms the sampler with max_num_seqs dummy requests,
      # and this model's 248K vocab makes that warmup tensor huge — the default
      # 1024 OOMs on a shared GPU even though weights+KV fit. 32 is ample for a
      # vision endpoint (the summarizer carries the concurrency, not this).
      - --max-num-seqs
      - ${QWEN_MAX_NUM_SEQS}
      # fp16 KV (no --kv-cache-dtype): the NVFP4 swap freed enough room to run
      # full-precision KV — better than the fp8 KV the FP8 build needed to fit.
      - --trust-remote-code
      - --dtype
      - auto
      - --enable-prefix-caching
      # Model-matched reasoning parser for the hybrid thinking toggle (see header).
      # Splits <think>…</think> into reasoning_content when thinking is ON; routes
      # all output to content when the empty think-block signals thinking OFF — so
      # this single :8007 endpoint serves both the non-thinking default and the
      # qwen3.6-35b-a3b-thinking gateway variant.
      - --reasoning-parser
      - qwen3
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids:
                - "${QWEN_GPU_ID}"
              capabilities:
                - gpu
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 300s
    networks:
      - tnet
    labels:
      - homepage.group=AI Systems
      - homepage.name=Qwen3.6-35B-A3B VL (NVFP4)
      - homepage.icon=mdi-image-search
      - homepage.description=Qwen3.6-35B-A3B vision-language MoE (NVFP4) via vLLM (ana-ml2)
      - homepage.href=http://10.250.50.54:${QWEN_PORT}/docs

networks:
  tnet:
    name: traefik-net
    external: true