From 391244b18b766536f1437d22e2fa515f4046e462 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Thu, 14 May 2026 15:32:43 -0700 Subject: [PATCH] docs: delete stacks/infinity (retired, replaced by vllm) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stack was retired and replaced by the vllm stack (originally vllm-qwen3, renamed 2026-05-13). Its README still framed it as a current solution while ana-ml2's README + vllm's README both documented the retirement. stacks/vllm/README.md "Migrating off Infinity" step 3 explicitly said "Delete stacks/infinity/ from this workspace" — actioning that now. No backwards-compat shims (PRACTICES §4): contract of a deleted system has no historical value the next contributor needs; the replacement path is documented in stacks/vllm/README.md. Surfaced by /tend-docs audit 2026-05-14. --- stacks/infinity/.env.example | 30 ----------------- stacks/infinity/README.md | 65 ------------------------------------ stacks/infinity/compose.yaml | 63 ---------------------------------- 3 files changed, 158 deletions(-) delete mode 100644 stacks/infinity/.env.example delete mode 100644 stacks/infinity/README.md delete mode 100644 stacks/infinity/compose.yaml diff --git a/stacks/infinity/.env.example b/stacks/infinity/.env.example deleted file mode 100644 index 47c4157..0000000 --- a/stacks/infinity/.env.example +++ /dev/null @@ -1,30 +0,0 @@ -# Infinity stack tunables. Copy this to `.env` on the server before deploying. -# -# cp .env.example .env -# # edit .env with real values -# docker compose up -d - -# Image version — pin for reproducibility (`latest` for edge) -INFINITY_VERSION=latest - -# Port exposed on host -INFINITY_PORT=7997 - -# GPU assignment (ana-ml2 has 0 and 1; default 1 keeps 0 free for heavy LLM work) -GPU_ID=1 - -# Models — both served simultaneously; reference by the full repo name in requests -EMBED_MODEL=Qwen/Qwen3-Embedding-0.6B -RERANK_MODEL=Qwen/Qwen3-Reranker-0.6B - -# Inference engine: torch (widest support) or optimum (ONNX, sometimes faster) -ENGINE=torch - -# Batch size — 32 is a safe default; bump for throughput if VRAM allows -BATCH_SIZE=32 - -# Optional API key — leave blank for no auth (fine on the internal network) -API_KEY= - -# HuggingFace token — only needed for gated models -HF_TOKEN= diff --git a/stacks/infinity/README.md b/stacks/infinity/README.md deleted file mode 100644 index c4e3db8..0000000 --- a/stacks/infinity/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# infinity - -OpenAI-compatible embeddings + reranker server. One container serves both embedding and reranker models simultaneously. - -**Server:** ana-ml2 -**Port:** 7997 (infinity default) -**GPU:** pinned to GPU 1 by default (configurable via `.env`) - -## What it replaces / supersedes - -- `qwen3-embedding-0.6B` entry in llama-swap (llama.cpp GGUF → infinity transformer) -- `qwen3-reranker-0.6B` entry in llama-swap - -Once infinity is verified stable, retire those two entries from `stacks/llama-swap/config.yaml`. - -## Deploy - -```bash -# On ana-ml2: -sudo mkdir -p /opt/docker/compose/infinity -sudo chown $USER /opt/docker/compose/infinity -cd /opt/docker/compose/infinity - -# Copy compose.yaml + .env.example here (e.g. via scp from this workspace) -# Then: -cp .env.example .env -# edit .env — pick GPU, models, etc. - -# Pre-download models into the shared HF cache (optional, speeds first boot) -HF_HOME=/tank/aimodels/huggingface hf download "$(grep ^EMBED_MODEL .env | cut -d= -f2)" -HF_HOME=/tank/aimodels/huggingface hf download "$(grep ^RERANK_MODEL .env | cut -d= -f2)" - -# Dry-parse -docker compose config - -# Launch -docker compose up -d -docker compose logs -f -``` - -## Verify - -```bash -# Health -curl -s http://localhost:7997/health - -# Embedding -curl -s http://localhost:7997/embeddings \ - -H "Content-Type: application/json" \ - -d '{"model":"Qwen/Qwen3-Embedding-0.6B","input":["hello world"]}' | jq . - -# Reranker -curl -s http://localhost:7997/rerank \ - -H "Content-Type: application/json" \ - -d '{"model":"Qwen/Qwen3-Reranker-0.6B","query":"what is a cat","documents":["cats are mammals","dogs bark"]}' | jq . - -# Listed models -curl -s http://localhost:7997/models | jq . -``` - -## Scaling knobs - -- **`BATCH_SIZE`** in `.env` — bigger = higher throughput, more VRAM. 32 is safe; try 64 or 128 if you have headroom. -- **Model size** — Qwen3-Embedding/Reranker come in 0.6B / 4B / 8B. Pick based on quality-vs-latency tradeoff. On RTX 6000 Ada 46 GB, the 8B pair fits easily (~20 GB VRAM). -- **`ENGINE=optimum`** — uses ONNX runtime, sometimes faster. Requires the model to have ONNX weights available; fall back to `torch` if it errors on startup. diff --git a/stacks/infinity/compose.yaml b/stacks/infinity/compose.yaml deleted file mode 100644 index 16e799f..0000000 --- a/stacks/infinity/compose.yaml +++ /dev/null @@ -1,63 +0,0 @@ -# Infinity — OpenAI-compatible embeddings + reranker server. -# -# Serves embedding and reranker models simultaneously from one container -# on port 7997 (HTTP). Consumers: AIPA agents (search/retrieval), LibreChat -# RAG, anything that needs vector embeddings. -# -# All tunables live in .env — edit that, not this file. -# -# Pre-download models to avoid first-run delay: -# HF_HOME=/tank/aimodels/huggingface hf download Qwen/Qwen3-Embedding-0.6B -# HF_HOME=/tank/aimodels/huggingface hf download Qwen/Qwen3-Reranker-0.6B - -services: - infinity: - image: michaelf34/infinity:${INFINITY_VERSION} - container_name: infinity - restart: unless-stopped - ports: - - "${INFINITY_PORT}:7997" - volumes: - - /tank/aimodels/huggingface:/hfcache - environment: - - HF_HOME=/hfcache - - HF_HUB_CACHE=/hfcache/hub - - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-} - command: > - v2 - --model-id ${EMBED_MODEL} - --model-id ${RERANK_MODEL} - --engine ${ENGINE} - --device cuda - --batch-size ${BATCH_SIZE} - --host 0.0.0.0 - --port 7997 - --api-key ${API_KEY:-} - deploy: - resources: - reservations: - devices: - - driver: nvidia - device_ids: - - "${GPU_ID}" - capabilities: - - gpu - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:7997/health"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 120s - networks: - - tnet - labels: - - homepage.group=AI Systems - - homepage.name=Infinity - - homepage.icon=mdi-vector-arrange-below - - homepage.description=Embeddings + Reranker API (ana-ml2) - - homepage.href=http://10.250.50.54:7997/docs - -networks: - tnet: - name: traefik-net - external: true