esh-pfi-infrastructure/stacks/embed-rerank/compose.yaml

# embed-rerank — the fleet's embedding + reranking models, served locally at ESH
# on esh-ml1 (CT 110 on esh-pve, RTX 2000E Ada, 16 GB).
#
# WHY: Prime's decision 2026-09-24 — the Ada card offloads embedding and
# reranking so ESH consumers (Open WebUI RAG, Paperless) are not tied to one
# seat on fv-ml1. It serves the SAME models as fv-ml1's `vllm` stack, with the
# SAME vLLM version and flags, because embedding vectors are model-specific:
# a different embedding model here would silently poison every index built
# against fv-ml1's. Mirror stacks/vllm/compose.yaml when that changes.
#
#   vllm-embed        Qwen/Qwen3-Embedding-0.6B  → /v1/embeddings   :8001
#   vllm-rerank-bge   BAAI/bge-reranker-v2-m3    → /rerank, /score  :8013
#
# Ports match fv-ml1 on purpose, so a LiteLLM deployment for either site
# differs only in the host part of api_base.
#
# `vllm-rerank-bge`, not fv-ml1's `vllm-rerank-a3`: that name carries R43
# bake-off provenance for THAT container; and plain `vllm-rerank` was the
# retired Qwen3-Reranker that measured harmful. Name the model instead.
#
# NO `tnet`/traefik-net: esh-ml1 runs no traefik, and an external network
# that does not exist would stop the stack from starting. Consumers reach the
# published ports directly.
#
# Host setup (driver, LXC, docker, toolkit): playbooks/esh-pve-nvidia-host.yaml
# then playbooks/esh-ml1-lxc.yaml. Tunables live in .env — edit that, not this.

name: embed-rerank

services:
  vllm-embed:
    image: vllm/vllm-openai:${VLLM_VERSION}
    container_name: vllm-embed
    restart: unless-stopped
    ipc: host
    ports:
      - "${EMBED_PORT}:8000"
    volumes:
      - /opt/aimodels/huggingface:/hfcache
    environment:
      - HF_HOME=/hfcache
      - HF_HUB_CACHE=/hfcache/hub
      - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-}
      - VLLM_API_KEY=${API_KEY:-}
    command:
      - ${EMBED_MODEL}
      - --served-model-name
      - ${EMBED_MODEL}
      - --runner
      - pooling
      - --host
      - 0.0.0.0
      - --port
      - "8000"
      - --gpu-memory-utilization
      - ${EMBED_GPU_MEM_UTIL}
      - --max-model-len
      - ${EMBED_MAX_MODEL_LEN}
      - --dtype
      - auto
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ["0"]
              capabilities: [gpu]
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 180s
    labels:
      - homepage.group=AI - Eval & Retrieval
      - homepage.name=vLLM Embed (esh-ml1)
      - homepage.icon=mdi-vector-arrange-below
      - homepage.description=Qwen3 Embedding 0.6B via vLLM (esh-ml1, RTX 2000E Ada)
      - homepage.href=http://10.0.50.80:${EMBED_PORT}/docs

  vllm-rerank-bge:
    image: vllm/vllm-openai:${VLLM_VERSION}
    container_name: vllm-rerank-bge
    restart: unless-stopped
    ipc: host
    ports:
      - "${RERANK_PORT}:8000"
    volumes:
      - /opt/aimodels/huggingface:/hfcache
    environment:
      - HF_HOME=/hfcache
      - HF_HUB_CACHE=/hfcache/hub
      - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-}
      - VLLM_API_KEY=${API_KEY:-}
    command:
      - ${RERANK_MODEL}
      - --served-model-name
      - ${RERANK_MODEL}
      - --runner
      - pooling
      # bge-reranker-v2-m3 is natively a cross-encoder; no --hf-overrides.
      - --host
      - 0.0.0.0
      - --port
      - "8000"
      - --gpu-memory-utilization
      - ${RERANK_GPU_MEM_UTIL}
      - --max-model-len
      - ${RERANK_MAX_MODEL_LEN}
      - --dtype
      - auto
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ["0"]
              capabilities: [gpu]
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 180s
    labels:
      - homepage.group=AI - Eval & Retrieval
      - homepage.name=vLLM Rerank bge-v2-m3 (esh-ml1)
      - homepage.icon=mdi-sort-variant
      - homepage.description=bge-reranker-v2-m3 via vLLM (esh-ml1, RTX 2000E Ada)
      - homepage.href=http://10.0.50.80:${RERANK_PORT}/docs