# qwen3.5-122b — bjk110/Qwen3.5-122B-A10B-abliterated-NVFP4 on fv-ml1 GPU 0,
# REPLACING mistral-small-4 (operator 2026-06-19: down the heretic, serve this as
# the new general/`gen` model). Abliterated Qwen3.5 MoE (256 experts, 10B active),
# NVFP4 (compressed-tensors), HF format.
#
# SERVING (per the repo's serving/): Qwen3.5 MoE is a MULTIMODAL arch but this
# checkpoint is text-only, so vLLM needs the repo's text-only PATCH applied before
# startup. We reuse the repo's entrypoint.sh (applies the patch, then runs vLLM) and
# vllm_patches/, mounted from the downloaded model dir — keeps patch+checkpoint
# version-coupled. Thinking split via litellm extra_body chat_template_kwargs
# (enable_thinking) + --reasoning-parser qwen3 (mirrors the qwen3.6-35b-a3b pattern).
#
# DISPLACEMENT: GPU 0 fits one mistral-class model; bring this up only after downing
# mistral-small-4-heretic. REVERT = down this, `up -d` the heretic stack.
#
# NOTE: image is vllm/vllm-openai:latest per the repo (the patch targets latest) —
# MUTABLE tag; pin a digest once a known-good version is established.
#
# Tunables in .env.

name: qwen3.5-122b

services:
  vllm-qwen35-122b:
    image: ${QWEN35_IMAGE:-vllm/vllm-openai:latest}
    container_name: ${QWEN35_CONTAINER_NAME:-vllm-qwen35-122b}
    restart: unless-stopped
    ipc: host
    ports:
      - "${QWEN35_PORT:-8013}:8000"
    environment:
      - ROLE=head
      - TP_SIZE=1
      - MODEL_CONTAINER_PATH=/models/qwen
      - SERVED_MODEL_NAME=${QWEN35_SERVED_NAME:-qwen3.5-122-a10b}
      - HOST_PORT=8000
      - MAX_MODEL_LEN=${QWEN35_MAX_MODEL_LEN:-131072}
      - MAX_NUM_SEQS=${QWEN35_MAX_NUM_SEQS:-8}
      - GPU_MEMORY_UTILIZATION=${QWEN35_GPU_MEM_UTIL:-0.95}
      - MAX_NUM_BATCHED_TOKENS=${QWEN35_MAX_NUM_BATCHED_TOKENS:-32768}
      # --reasoning-parser qwen3 surfaces <think>…</think> as reasoning_content;
      # the thinking on/off itself is per-request (litellm chat_template_kwargs).
      # --enable-auto-tool-choice + --tool-call-parser: Qwen3.5 emits XML tool calls
      # <tool_call><function=NAME><parameter=K>V</parameter></function></tool_call>
      # (NOT Hermes JSON), so the parser is qwen3_xml. Without these flags vLLM never
      # parses tool calls (tool-calling is broken). The bjk110 repo command omitted them.
      - VLLM_EXTRA_ARGS=--reasoning-parser qwen3 --enable-chunked-prefill --enable-auto-tool-choice --tool-call-parser qwen3_xml
      - NVIDIA_VISIBLE_DEVICES=0
      - PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
      - VLLM_API_KEY=${API_KEY:-}
    volumes:
      # checkpoint + the repo's entrypoint/patch (downloaded with the model).
      - /tank/aimodels/qwen3.5-122b-a10b-nvfp4:/models/qwen:ro
      - /tank/aimodels/qwen3.5-122b-a10b-nvfp4/serving/entrypoint.sh:/entrypoint.sh:ro
      - /tank/aimodels/qwen3.5-122b-a10b-nvfp4/vllm_patches:/patches:ro
      - /tank/aimodels/qwen3.5-122b-a10b-nvfp4/.cache/vllm:/root/.cache/vllm
    entrypoint: ["bash", "/entrypoint.sh"]
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids:
                - "${QWEN35_GPU_ID:-0}"
              capabilities:
                - gpu
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 900s
    networks:
      - tnet
    labels:
      - homepage.group=AI Systems
      - homepage.name=Qwen3.5-122B-A10B (abliterated NVFP4)
      - homepage.icon=mdi-creation
      - homepage.description=Abliterated Qwen3.5 122B-A10B NVFP4, the new `gen` model (fv-ml1 GPU 0)
      - homepage.href=http://10.251.50.54:${QWEN35_PORT:-8013}/docs

networks:
  tnet:
    name: traefik-net
    external: true