feat(tei-bakeoff): TEI 1.9.4 vs vLLM on esh-ml1 — parity holds, not faster, much lighter

2026-09-25 08:05:17 -07:00
parent a1885718e9
commit 582b150131
4 changed files with 178 additions and 0 deletions
@@ -0,0 +1,69 @@
+# tei-bakeoff — Hugging Face Text Embeddings Inference (TEI) serving the SAME two
+# models as stacks/embed-rerank (vLLM), side by side on esh-ml1, so the two
+# engines can be compared on identical hardware. Prime, 2026-09-25: "run the
+# TEI bake-off". Not wired into the gateway. Temporary: either this becomes the
+# embed-rerank stack or it is torn down.
+#
+#   tei-embed    Qwen/Qwen3-Embedding-0.6B  → /v1/embeddings, /embed   :8081
+#   tei-rerank   BAAI/bge-reranker-v2-m3    → /rerank                  :8083
+#
+# 89-* = the Ada Lovelace (sm_89) build. Pinned to an exact release.
+# TEI on CUDA offers float16/float32 only (no bfloat16), while vLLM serves
+# Qwen3-Embedding in bf16 — the parity measurement is what decides whether
+# that matters.
+# (--max-batch-tokens 32768 was tried on tei-embed: whole-novel time unchanged,
+# 39.2–40.4 s vs 39.9–40.3 s, so it stays at the default.)
+# --max-client-batch-size 128: the default is 32; vLLM has no such cap and the
+# bake-off sends 64 per request.
+# Separate HF cache from vLLM's, so neither engine can disturb the other's files.
+
+name: tei-bakeoff
+
+services:
+  tei-embed:
+    image: ghcr.io/huggingface/text-embeddings-inference:89-1.9.4
+    container_name: tei-embed
+    restart: unless-stopped
+    ports:
+      - "8081:80"
+    volumes:
+      - /opt/aimodels/tei-cache:/data
+    command:
+      - --model-id
+      - Qwen/Qwen3-Embedding-0.6B
+      - --served-model-name
+      - Qwen/Qwen3-Embedding-0.6B
+      - --dtype
+      - float16
+      - --max-client-batch-size
+      - "128"
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              device_ids: ["0"]
+              capabilities: [gpu]
+
+  tei-rerank:
+    image: ghcr.io/huggingface/text-embeddings-inference:89-1.9.4
+    container_name: tei-rerank
+    restart: unless-stopped
+    ports:
+      - "8083:80"
+    volumes:
+      - /opt/aimodels/tei-cache:/data
+    command:
+      - --model-id
+      - BAAI/bge-reranker-v2-m3
+      - --dtype
+      - float16
+      - --max-client-batch-size
+      - "128"
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              device_ids: ["0"]
+              capabilities: [gpu]