esh-pfi-infrastructure/stacks/vllm-qwen3/compose.yaml

# vLLM — Qwen3 Embedding + Reranker (one stack, two services).
#
# Replaces the unmaintained Infinity stack. vLLM runs one model per process,
# so this stack brings up two containers sharing a single GPU:
#
#   vllm-embed   — Qwen3-Embedding served as an OpenAI /v1/embeddings server
#   vllm-rerank  — Qwen3-Reranker served as a /rerank + /score server
#
# The reranker is a causal-LM checkpoint; --hf-overrides re-maps it to
# Qwen3ForSequenceClassification so vLLM's reranking endpoints work and the
# model only emits two class logits (no/yes) instead of the full 151k vocab.
#
# All tunables live in .env — edit that, not this file.
#
# Pre-download models to avoid first-run delay:
#   HF_HOME=/tank/aimodels/huggingface hf download Qwen/Qwen3-Embedding-0.6B
#   HF_HOME=/tank/aimodels/huggingface hf download Qwen/Qwen3-Reranker-0.6B

services:
  vllm-embed:
    image: vllm/vllm-openai:${VLLM_VERSION}
    container_name: vllm-embed
    restart: unless-stopped
    ipc: host
    ports:
      - "${EMBED_PORT}:8000"
    volumes:
      - /tank/aimodels/huggingface:/hfcache
    environment:
      - HF_HOME=/hfcache
      - HF_HUB_CACHE=/hfcache/hub
      - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-}
      - VLLM_API_KEY=${API_KEY:-}
    command:
      - ${EMBED_MODEL}
      - --served-model-name
      - ${EMBED_MODEL}
      - --runner
      - pooling
      - --host
      - 0.0.0.0
      - --port
      - "8000"
      - --gpu-memory-utilization
      - ${EMBED_GPU_MEM_UTIL}
      - --max-model-len
      - ${EMBED_MAX_MODEL_LEN}
      - --dtype
      - auto
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids:
                - "${GPU_ID}"
              capabilities:
                - gpu
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 180s
    networks:
      - tnet
    labels:
      - homepage.group=AI Systems
      - homepage.name=vLLM Embed (Qwen3)
      - homepage.icon=mdi-vector-arrange-below
      - homepage.description=Qwen3 Embedding via vLLM (ana-ml2)
      - homepage.href=http://10.250.50.54:${EMBED_PORT}/docs

  vllm-rerank:
    image: vllm/vllm-openai:${VLLM_VERSION}
    container_name: vllm-rerank
    restart: unless-stopped
    ipc: host
    ports:
      - "${RERANK_PORT}:8000"
    volumes:
      - /tank/aimodels/huggingface:/hfcache
    environment:
      - HF_HOME=/hfcache
      - HF_HUB_CACHE=/hfcache/hub
      - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-}
      - VLLM_API_KEY=${API_KEY:-}
    command:
      - ${RERANK_MODEL}
      - --served-model-name
      - ${RERANK_MODEL}
      - --runner
      - pooling
      - --hf-overrides
      - '{"architectures":["Qwen3ForSequenceClassification"],"classifier_from_token":["no","yes"],"is_original_qwen3_reranker":true}'
      - --host
      - 0.0.0.0
      - --port
      - "8000"
      - --gpu-memory-utilization
      - ${RERANK_GPU_MEM_UTIL}
      - --max-model-len
      - ${RERANK_MAX_MODEL_LEN}
      - --dtype
      - auto
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids:
                - "${GPU_ID}"
              capabilities:
                - gpu
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 180s
    networks:
      - tnet
    labels:
      - homepage.group=AI Systems
      - homepage.name=vLLM Rerank (Qwen3)
      - homepage.icon=mdi-sort-variant
      - homepage.description=Qwen3 Reranker via vLLM (ana-ml2)
      - homepage.href=http://10.250.50.54:${RERANK_PORT}/docs

networks:
  tnet:
    name: traefik-net
    external: true