esh-pfi-infrastructure/stacks/embed-rerank/compose.yaml

# embed-rerank — THE fleet's embedding + reranking service, on esh-ml1 (CT 110 on
# esh-pve, RTX 2000E Ada, 16 GB). Served by Hugging Face Text Embeddings
# Inference (TEI). A second instance runs on nh3-ml1 (CT 109 on nh3-pve, the same
# card) since 2026-09-25; the per-host bits are HOST_NAME / HOST_IP in .env, and
# their defaults are esh-ml1's, so esh-ml1's live .env needs no change.
#
# Prime, 2026-09-25: "TEI is embed/reranker server for esh-ml1 and the FLEET in
# general, in future." It replaced vLLM here the same day, after a side-by-side
# bake-off on this card (docs/pfi/embed-rerank-tei-vs-vllm-bakeoff.md). TEI
# gives the same vectors as the old vLLM seats (no re-embedding), but it is
# ~1.3x slower on bulk work on this card. It is much lighter (2.6 GB VRAM for
# both, 8 GB image, ~4 s restart). fv-ml1's vLLM embed/rerank seats were
# retired after this went live.
#
#   tei-embed    Qwen/Qwen3-Embedding-0.6B → /v1/embeddings (OpenAI), /embed  :8001
#   tei-rerank   BAAI/bge-reranker-v2-m3   → /rerank  (body: query + texts)   :8013
#
# Ports kept from the vLLM era (and fv-ml1), so the embedding gateway entry did
# not change address. ⚠ The RERANK gateway entry must use LiteLLM's
# `huggingface/` provider: `hosted_vllm/` sends `documents` and TEI answers 422
# "missing field texts".
#
# ⚠ Embedding vectors are model-specific. Never change EMBED_MODEL without a
# re-embedding plan for every index built on it (Worldtree, nevermore, Open WebUI).
#
# FAIL-CLOSED truncation (--auto-truncate false). TEI's default silently
# embedded the first 16,384 tokens of a ~40k-token input and returned 200.
# Turning it off requires --max-batch-tokens >= the model's max input (32,768
# for Qwen3-Embedding), or TEI refuses to start.
#
# NO `tnet`/traefik-net: esh-ml1 runs no traefik; consumers reach the published
# ports, and in practice only the LiteLLM gateway does (verified from the seats'
# logs 2026-09-25: every request matched a gateway spend-log row).
#
# Host setup: playbooks/pve-nvidia-host.yaml, then playbooks/gpu-lxc.yaml.
# Tunables live in .env.

name: embed-rerank

services:
  tei-embed:
    image: ghcr.io/huggingface/text-embeddings-inference:${TEI_TAG}
    container_name: tei-embed
    restart: unless-stopped
    ports:
      - "${EMBED_PORT}:80"
    volumes:
      - /opt/aimodels/tei-cache:/data
    environment:
      - HF_TOKEN=${HF_TOKEN:-}
    command:
      - --model-id
      - ${EMBED_MODEL}
      - --served-model-name
      - ${EMBED_MODEL}
      # TEI on CUDA is float16-only; parity vs the bf16 vLLM vectors was measured.
      - --dtype
      - float16
      # Default 32; vLLM had no cap and callers batch 64.
      - --max-client-batch-size
      - "${MAX_CLIENT_BATCH_SIZE}"
      - --auto-truncate
      - "false"
      - --max-batch-tokens
      - "32768"
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ["0"]
              capabilities: [gpu]
    healthcheck:
      test: ["CMD", "curl", "-fsS", "http://localhost:80/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 60s
    labels:
      - homepage.group=AI - Eval & Retrieval
      - homepage.name=Embed — Qwen3 0.6B (TEI, ${HOST_NAME:-esh-ml1})
      - homepage.icon=mdi-vector-arrange-below
      - homepage.description=Fleet embeddings (qwen3-embedding) via TEI on ${HOST_NAME:-esh-ml1}
      - homepage.href=http://${HOST_IP:-10.0.50.80}:${EMBED_PORT}/docs

  tei-rerank:
    image: ghcr.io/huggingface/text-embeddings-inference:${TEI_TAG}
    container_name: tei-rerank
    restart: unless-stopped
    ports:
      - "${RERANK_PORT}:80"
    volumes:
      - /opt/aimodels/tei-cache:/data
    environment:
      - HF_TOKEN=${HF_TOKEN:-}
    command:
      - --model-id
      - ${RERANK_MODEL}
      - --dtype
      - float16
      - --max-client-batch-size
      - "${MAX_CLIENT_BATCH_SIZE}"
      # Fail-closed; bge-reranker-v2-m3's max input (8,192) fits the default
      # max-batch-tokens (16,384).
      - --auto-truncate
      - "false"
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ["0"]
              capabilities: [gpu]
    healthcheck:
      test: ["CMD", "curl", "-fsS", "http://localhost:80/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 60s
    labels:
      - homepage.group=AI - Eval & Retrieval
      - homepage.name=Rerank — bge-v2-m3 (TEI, ${HOST_NAME:-esh-ml1})
      - homepage.icon=mdi-sort-variant
      - homepage.description=Fleet reranker (reranker) via TEI on ${HOST_NAME:-esh-ml1}
      - homepage.href=http://${HOST_IP:-10.0.50.80}:${RERANK_PORT}/docs