vh
957c8f177d
ext-tts / tts-1 / tts-1-hd / gpt-4o-mini-tts (all openai/zonos) were pinned to irv-ml1's pre-move address 10.100.79.3:8198, dead since the 2026-09-06 headscale cutover to 10.6.110.50 — so ext-tts through the gateway hung. Reported by tts-dev/svos-dev. The 4 aliases are DB-backed (store_model_in_db), so their api_base was updated via the admin API to the DNS name http://irv-ml1.nh3.internal:8198/v1 (not a fresh IP — that just re-arms the trap on the next move). The container cannot resolve *.nh3.internal via split-horizon DNS, so this adds an extra_hosts entry mapping the name to 10.6.110.50. Verified: container resolves the name and a live ext-tts/sindra call returns 200 + valid MP3.
121 lines
5.2 KiB
YAML
121 lines
5.2 KiB
YAML
# LiteLLM — OpenAI-compatible gateway fronting the vLLM services on
|
|
# ana-ml2, standing in the request path to LOG every request + response
|
|
# so they're inspectable in a browser (the thing vLLM itself does not
|
|
# give us; Dozzle only shows connection metadata).
|
|
#
|
|
# litellm — proxy + Logs UI (http://10.250.50.70:4000/ui)
|
|
# litellm-db — Postgres: spend logs (incl. full prompts/responses),
|
|
# virtual keys, model config
|
|
#
|
|
# Why this exists: phi4-mini is a production summarizer + "dreaming"
|
|
# agent on ana-ml2; "what was it asked, what did it answer" must be
|
|
# inspectable. LiteLLM captures full req/resp per call with a Logs UI.
|
|
#
|
|
# Langfuse-ready: this is the lean first cut. To ship full traces to a
|
|
# Langfuse instance later, uncomment the success/failure_callback block
|
|
# in conf/config.yaml and fill LANGFUSE_* in .env — no re-architecture,
|
|
# the gateway and all consumers stay pointed here.
|
|
#
|
|
# Consumers (nevermore, the dreaming agent, asset-engine, …) re-point
|
|
# their OpenAI base_url at this gateway and select a model by name
|
|
# (phi4-mini / qwen3-embedding / qwen3-reranker). The gateway forwards
|
|
# to the right vLLM port on ana-ml2.
|
|
#
|
|
# All tunables live in .env — edit that, not this file.
|
|
|
|
services:
|
|
litellm:
|
|
image: ghcr.io/berriai/litellm:${LITELLM_TAG:-main-stable}
|
|
container_name: litellm
|
|
restart: unless-stopped
|
|
# irv-ml1 moved 10.100.79.3 -> 10.6.110.50 on 2026-09-06 (headscale
|
|
# cutover). The container cannot resolve *.nh3.internal via split-horizon
|
|
# DNS, so map the fleet-TTS gateway name here rather than re-hardcoding an
|
|
# IP in the DB model configs (ext-tts / tts-1 / tts-1-hd / gpt-4o-mini-tts,
|
|
# all openai/zonos). Update this IP if irv-ml1 moves again.
|
|
extra_hosts:
|
|
- "irv-ml1.nh3.internal:10.6.110.50"
|
|
ports:
|
|
- "${LITELLM_BIND:-0.0.0.0}:${LITELLM_PORT:-4000}:4000"
|
|
volumes:
|
|
- /opt/docker/conf/litellm/config.yaml:/app/config.yaml:ro
|
|
# Custom pre-call hook (strip empty `tools: []` before forwarding to vLLM).
|
|
# Must sit beside config.yaml — LiteLLM loads callbacks relative to the
|
|
# config file's directory, so this lands at /app/strip_empty_tools.py.
|
|
- /opt/docker/conf/litellm/strip_empty_tools.py:/app/strip_empty_tools.py:ro
|
|
# Second pre-call hook: per-model `reasoning_effort` translation. Same
|
|
# beside-the-config requirement as the hook above.
|
|
- /opt/docker/conf/litellm/reasoning_effort_map.py:/app/reasoning_effort_map.py:ro
|
|
environment:
|
|
# master_key gates the proxy + admin UI login. Must start with sk-.
|
|
- LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY}
|
|
# salt for encrypting any virtual/model keys stored in the DB.
|
|
- LITELLM_SALT_KEY=${LITELLM_SALT_KEY}
|
|
- DATABASE_URL=postgresql://${POSTGRES_USER:-litellm}:${POSTGRES_PASSWORD}@litellm-db:5432/${POSTGRES_DB:-litellm}
|
|
- STORE_MODEL_IN_DB=True
|
|
# Upstream vLLM API key (empty in the vllm stack's .env by default →
|
|
# leave blank; LiteLLM still needs the var to exist).
|
|
- VLLM_API_KEY=${VLLM_API_KEY:-}
|
|
# Cloud API keys fronted by the gateway for unified logging (z.ai GLM,
|
|
# Moonshot Kimi, etc.). Paid — only gateway-keyed callers reach them,
|
|
# but they spend.
|
|
- Z_AI_API_KEY=${Z_AI_API_KEY:-}
|
|
# Kimi: KIMI_CODE_API_KEY = the coding endpoint (api.kimi.com/coding, the
|
|
# primary kimi-k3 arm); MOONSHOT_API_KEY = the general api.moonshot.ai
|
|
# endpoint (the kimi-k3-gen-api variant).
|
|
- KIMI_CODE_API_KEY=${KIMI_CODE_API_KEY:-}
|
|
- MOONSHOT_API_KEY=${MOONSHOT_API_KEY:-}
|
|
# Langfuse-ready: blank until you bolt Langfuse on. Filling these +
|
|
# uncommenting the callback in config.yaml is the entire upgrade.
|
|
- LANGFUSE_PUBLIC_KEY=${LANGFUSE_PUBLIC_KEY:-}
|
|
- LANGFUSE_SECRET_KEY=${LANGFUSE_SECRET_KEY:-}
|
|
- LANGFUSE_HOST=${LANGFUSE_HOST:-}
|
|
command: ["--config", "/app/config.yaml"]
|
|
depends_on:
|
|
litellm-db:
|
|
condition: service_healthy
|
|
healthcheck:
|
|
# python is guaranteed in the image (curl is not); urlopen raises
|
|
# on non-200. 127.0.0.1 dodges the IPv6-first localhost trap.
|
|
test: ["CMD-SHELL", "python -c 'import urllib.request; urllib.request.urlopen(\"http://127.0.0.1:4000/health/liveliness\")' || exit 1"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 3
|
|
start_period: 40s
|
|
networks:
|
|
- tnet
|
|
labels:
|
|
- homepage.group=AI - Gateways & Chat
|
|
- homepage.name=LiteLLM Gateway
|
|
- homepage.icon=mdi-router-network
|
|
- homepage.description=vLLM request/response logging gateway (ana-docker)
|
|
- homepage.href=http://10.250.50.70:${LITELLM_PORT:-4000}/ui
|
|
|
|
litellm-db:
|
|
image: postgres:16-alpine
|
|
container_name: litellm-db
|
|
restart: unless-stopped
|
|
environment:
|
|
- POSTGRES_USER=${POSTGRES_USER:-litellm}
|
|
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
|
- POSTGRES_DB=${POSTGRES_DB:-litellm}
|
|
volumes:
|
|
- litellm_db:/var/lib/postgresql/data
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-litellm} -d ${POSTGRES_DB:-litellm}"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 5
|
|
start_period: 20s
|
|
networks:
|
|
- tnet
|
|
|
|
volumes:
|
|
litellm_db:
|
|
name: litellm_db
|
|
|
|
networks:
|
|
tnet:
|
|
name: traefik-net
|
|
external: true
|