70 lines
2.2 KiB
YAML
70 lines
2.2 KiB
YAML
# tei-bakeoff — Hugging Face Text Embeddings Inference (TEI) serving the SAME two
|
||
# models as stacks/embed-rerank (vLLM), side by side on esh-ml1, so the two
|
||
# engines can be compared on identical hardware. Prime, 2026-09-25: "run the
|
||
# TEI bake-off". Not wired into the gateway. Temporary: either this becomes the
|
||
# embed-rerank stack or it is torn down.
|
||
#
|
||
# tei-embed Qwen/Qwen3-Embedding-0.6B → /v1/embeddings, /embed :8081
|
||
# tei-rerank BAAI/bge-reranker-v2-m3 → /rerank :8083
|
||
#
|
||
# 89-* = the Ada Lovelace (sm_89) build. Pinned to an exact release.
|
||
# TEI on CUDA offers float16/float32 only (no bfloat16), while vLLM serves
|
||
# Qwen3-Embedding in bf16 — the parity measurement is what decides whether
|
||
# that matters.
|
||
# (--max-batch-tokens 32768 was tried on tei-embed: whole-novel time unchanged,
|
||
# 39.2–40.4 s vs 39.9–40.3 s, so it stays at the default.)
|
||
# --max-client-batch-size 128: the default is 32; vLLM has no such cap and the
|
||
# bake-off sends 64 per request.
|
||
# Separate HF cache from vLLM's, so neither engine can disturb the other's files.
|
||
|
||
name: tei-bakeoff
|
||
|
||
services:
|
||
tei-embed:
|
||
image: ghcr.io/huggingface/text-embeddings-inference:89-1.9.4
|
||
container_name: tei-embed
|
||
restart: unless-stopped
|
||
ports:
|
||
- "8081:80"
|
||
volumes:
|
||
- /opt/aimodels/tei-cache:/data
|
||
command:
|
||
- --model-id
|
||
- Qwen/Qwen3-Embedding-0.6B
|
||
- --served-model-name
|
||
- Qwen/Qwen3-Embedding-0.6B
|
||
- --dtype
|
||
- float16
|
||
- --max-client-batch-size
|
||
- "128"
|
||
deploy:
|
||
resources:
|
||
reservations:
|
||
devices:
|
||
- driver: nvidia
|
||
device_ids: ["0"]
|
||
capabilities: [gpu]
|
||
|
||
tei-rerank:
|
||
image: ghcr.io/huggingface/text-embeddings-inference:89-1.9.4
|
||
container_name: tei-rerank
|
||
restart: unless-stopped
|
||
ports:
|
||
- "8083:80"
|
||
volumes:
|
||
- /opt/aimodels/tei-cache:/data
|
||
command:
|
||
- --model-id
|
||
- BAAI/bge-reranker-v2-m3
|
||
- --dtype
|
||
- float16
|
||
- --max-client-batch-size
|
||
- "128"
|
||
deploy:
|
||
resources:
|
||
reservations:
|
||
devices:
|
||
- driver: nvidia
|
||
device_ids: ["0"]
|
||
capabilities: [gpu]
|