# qwen35-vl — Qwen3.5-9B vision-language model (FP8) on ana-ml2.
#
# Co-located on GPU 1 with the granite summarizer + embed/rerank/reward trio
# (GPU 0 is deliberately kept free for hot-reloading large models). Serves on
# :8007, fronted by the LiteLLM gateway as `qwen3.5-9b-fp8`.
#
# WHY A PINNED NIGHTLY DIGEST (not :latest): vLLM :latest (v0.19.1) quantizes
# the Qwen3.5-VL *vision tower* under --quantization fp8, producing garbage
# vision output (the language model is unaffected — it answers text fine but
# "sees" noise). The nightly correctly excludes the vision tower, so vision
# works while the LM still gets the FP8 throughput/VRAM win. We pin the exact
# nightly digest for reproducibility — a moving :nightly tag would silently
# change the engine. WATCH: once the vision-FP8 exclusion lands in a stable
# release, re-pin to :latest and drop this note.
#
# WHY util 0.40 (not the trio's tiny values): the model needs ~34 GB just to
# start at 32k context (FP8 weights + BF16 vision tower + graph capture + 32k
# profiling). On shared GPU 1 (prod uses ~46 GB, ~48 GB free) this vLLM build
# requires free >= util*total, capping util at ~0.51 here; 0.40 (~38 GB) sits
# above the ~34 GB floor with ~10 GB card headroom.
#
# All tunables live in .env — edit that, not this file.

name: qwen35-vl

services:
  vllm-qwen35:
    image: ${QWEN_IMAGE}
    container_name: ${QWEN_CONTAINER_NAME}
    restart: unless-stopped
    ipc: host
    ports:
      - "${QWEN_PORT}:8000"
    volumes:
      - /tank/aimodels/huggingface:/hfcache
    environment:
      - HF_HOME=/hfcache
      - HF_HUB_CACHE=/hfcache/hub
      - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-}
      - VLLM_API_KEY=${API_KEY:-}
    command:
      - ${QWEN_MODEL}
      - --served-model-name
      - ${QWEN_SERVED_NAME}
      - --quantization
      - fp8
      - --host
      - 0.0.0.0
      - --port
      - "8000"
      - --gpu-memory-utilization
      - ${QWEN_GPU_MEM_UTIL}
      - --max-model-len
      - ${QWEN_MAX_MODEL_LEN}
      - --dtype
      - auto
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids:
                - "${QWEN_GPU_ID}"
              capabilities:
                - gpu
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 300s
    networks:
      - tnet
    labels:
      - homepage.group=AI Systems
      - homepage.name=Qwen3.5-9B VL (FP8)
      - homepage.icon=mdi-image-search
      - homepage.description=Qwen3.5-9B vision-language (FP8) via vLLM (ana-ml2)
      - homepage.href=http://10.250.50.54:${QWEN_PORT}/docs

networks:
  tnet:
    name: traefik-net
    external: true