vh
c7ee8884c6
Adds explicit gateway entries for the four z.ai GLM models (glm-5.1, glm-5-turbo, glm-4.7, glm-4.5-air) routed to api.z.ai with Z_AI_API_KEY, plus the compose env passthrough + .env.example doc. Explicit entries win over the llama-swap wildcard (distinct IDs, no collision). Extends the gateway's unified logging to cloud inference, not just local vLLM/llama-swap. Cost note: paid API — only gateway-keyed callers reach these, but calls spend z.ai credits (documented in config + compose comments).
101 lines
3.9 KiB
YAML
101 lines
3.9 KiB
YAML
# LiteLLM — OpenAI-compatible gateway fronting the vLLM services on
|
|
# ana-ml2, standing in the request path to LOG every request + response
|
|
# so they're inspectable in a browser (the thing vLLM itself does not
|
|
# give us; Dozzle only shows connection metadata).
|
|
#
|
|
# litellm — proxy + Logs UI (http://10.250.50.70:4000/ui)
|
|
# litellm-db — Postgres: spend logs (incl. full prompts/responses),
|
|
# virtual keys, model config
|
|
#
|
|
# Why this exists: phi4-mini is a production summarizer + "dreaming"
|
|
# agent on ana-ml2; "what was it asked, what did it answer" must be
|
|
# inspectable. LiteLLM captures full req/resp per call with a Logs UI.
|
|
#
|
|
# Langfuse-ready: this is the lean first cut. To ship full traces to a
|
|
# Langfuse instance later, uncomment the success/failure_callback block
|
|
# in conf/config.yaml and fill LANGFUSE_* in .env — no re-architecture,
|
|
# the gateway and all consumers stay pointed here.
|
|
#
|
|
# Consumers (nevermore, the dreaming agent, asset-engine, …) re-point
|
|
# their OpenAI base_url at this gateway and select a model by name
|
|
# (phi4-mini / qwen3-embedding / qwen3-reranker). The gateway forwards
|
|
# to the right vLLM port on ana-ml2.
|
|
#
|
|
# All tunables live in .env — edit that, not this file.
|
|
|
|
services:
|
|
litellm:
|
|
image: ghcr.io/berriai/litellm:${LITELLM_TAG:-main-stable}
|
|
container_name: litellm
|
|
restart: unless-stopped
|
|
ports:
|
|
- "${LITELLM_BIND:-0.0.0.0}:${LITELLM_PORT:-4000}:4000"
|
|
volumes:
|
|
- /opt/docker/conf/litellm/config.yaml:/app/config.yaml:ro
|
|
environment:
|
|
# master_key gates the proxy + admin UI login. Must start with sk-.
|
|
- LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY}
|
|
# salt for encrypting any virtual/model keys stored in the DB.
|
|
- LITELLM_SALT_KEY=${LITELLM_SALT_KEY}
|
|
- DATABASE_URL=postgresql://${POSTGRES_USER:-litellm}:${POSTGRES_PASSWORD}@litellm-db:5432/${POSTGRES_DB:-litellm}
|
|
- STORE_MODEL_IN_DB=True
|
|
# Upstream vLLM API key (empty in the vllm stack's .env by default →
|
|
# leave blank; LiteLLM still needs the var to exist).
|
|
- VLLM_API_KEY=${VLLM_API_KEY:-}
|
|
# Cloud API keys fronted by the gateway for unified logging (z.ai GLM,
|
|
# etc.). Paid — only gateway-keyed callers reach them, but they spend.
|
|
- Z_AI_API_KEY=${Z_AI_API_KEY:-}
|
|
# Langfuse-ready: blank until you bolt Langfuse on. Filling these +
|
|
# uncommenting the callback in config.yaml is the entire upgrade.
|
|
- LANGFUSE_PUBLIC_KEY=${LANGFUSE_PUBLIC_KEY:-}
|
|
- LANGFUSE_SECRET_KEY=${LANGFUSE_SECRET_KEY:-}
|
|
- LANGFUSE_HOST=${LANGFUSE_HOST:-}
|
|
command: ["--config", "/app/config.yaml"]
|
|
depends_on:
|
|
litellm-db:
|
|
condition: service_healthy
|
|
healthcheck:
|
|
# python is guaranteed in the image (curl is not); urlopen raises
|
|
# on non-200. 127.0.0.1 dodges the IPv6-first localhost trap.
|
|
test: ["CMD-SHELL", "python -c 'import urllib.request; urllib.request.urlopen(\"http://127.0.0.1:4000/health/liveliness\")' || exit 1"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 3
|
|
start_period: 40s
|
|
networks:
|
|
- tnet
|
|
labels:
|
|
- homepage.group=AI Systems
|
|
- homepage.name=LiteLLM Gateway
|
|
- homepage.icon=mdi-router-network
|
|
- homepage.description=vLLM request/response logging gateway (ana-docker)
|
|
- homepage.href=http://10.250.50.70:${LITELLM_PORT:-4000}/ui
|
|
|
|
litellm-db:
|
|
image: postgres:16-alpine
|
|
container_name: litellm-db
|
|
restart: unless-stopped
|
|
environment:
|
|
- POSTGRES_USER=${POSTGRES_USER:-litellm}
|
|
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
|
- POSTGRES_DB=${POSTGRES_DB:-litellm}
|
|
volumes:
|
|
- litellm_db:/var/lib/postgresql/data
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-litellm} -d ${POSTGRES_DB:-litellm}"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 5
|
|
start_period: 20s
|
|
networks:
|
|
- tnet
|
|
|
|
volumes:
|
|
litellm_db:
|
|
name: litellm_db
|
|
|
|
networks:
|
|
tnet:
|
|
name: traefik-net
|
|
external: true
|