vh
926fc2fb7a
The gen-reasoning seat accepts only xhigh/medium/low and 400s on anything else — including `high`, which is the default of several clients, so the seat presented as broken rather than as one enum value out of step. The DeepSeek Harness failed every request on its default setting, and the only working client value was `low`: the seat's WEAKEST reasoning tier, while its own default is xhigh. conf/reasoning_effort_map.py is a pre-call hook in the same shape as the existing strip_empty_tools hook. It is scoped to one model group, measured rather than assumed: gen-reasoning rejects `high`; gen, sec, char-rp-reasoning and summarizer all accept it and are left alone. Paid passthroughs were not probed, because probing them spends vendor credits, and are not mapped. Verified after deploy: high and max now succeed on gen-reasoning, low and xhigh still work, a request with no effort param still works, `gen` with `high` still passes through unmapped, and the harness completes a real file-edit task at full reasoning. Needed a compose change as well as a conf push — callbacks are bind-mounted per file, so the volume only attaches on container create. Recreated the litellm service by name so the DB was not bounced with it.
114 lines
4.8 KiB
YAML
114 lines
4.8 KiB
YAML
# LiteLLM — OpenAI-compatible gateway fronting the vLLM services on
|
|
# ana-ml2, standing in the request path to LOG every request + response
|
|
# so they're inspectable in a browser (the thing vLLM itself does not
|
|
# give us; Dozzle only shows connection metadata).
|
|
#
|
|
# litellm — proxy + Logs UI (http://10.250.50.70:4000/ui)
|
|
# litellm-db — Postgres: spend logs (incl. full prompts/responses),
|
|
# virtual keys, model config
|
|
#
|
|
# Why this exists: phi4-mini is a production summarizer + "dreaming"
|
|
# agent on ana-ml2; "what was it asked, what did it answer" must be
|
|
# inspectable. LiteLLM captures full req/resp per call with a Logs UI.
|
|
#
|
|
# Langfuse-ready: this is the lean first cut. To ship full traces to a
|
|
# Langfuse instance later, uncomment the success/failure_callback block
|
|
# in conf/config.yaml and fill LANGFUSE_* in .env — no re-architecture,
|
|
# the gateway and all consumers stay pointed here.
|
|
#
|
|
# Consumers (nevermore, the dreaming agent, asset-engine, …) re-point
|
|
# their OpenAI base_url at this gateway and select a model by name
|
|
# (phi4-mini / qwen3-embedding / qwen3-reranker). The gateway forwards
|
|
# to the right vLLM port on ana-ml2.
|
|
#
|
|
# All tunables live in .env — edit that, not this file.
|
|
|
|
services:
|
|
litellm:
|
|
image: ghcr.io/berriai/litellm:${LITELLM_TAG:-main-stable}
|
|
container_name: litellm
|
|
restart: unless-stopped
|
|
ports:
|
|
- "${LITELLM_BIND:-0.0.0.0}:${LITELLM_PORT:-4000}:4000"
|
|
volumes:
|
|
- /opt/docker/conf/litellm/config.yaml:/app/config.yaml:ro
|
|
# Custom pre-call hook (strip empty `tools: []` before forwarding to vLLM).
|
|
# Must sit beside config.yaml — LiteLLM loads callbacks relative to the
|
|
# config file's directory, so this lands at /app/strip_empty_tools.py.
|
|
- /opt/docker/conf/litellm/strip_empty_tools.py:/app/strip_empty_tools.py:ro
|
|
# Second pre-call hook: per-model `reasoning_effort` translation. Same
|
|
# beside-the-config requirement as the hook above.
|
|
- /opt/docker/conf/litellm/reasoning_effort_map.py:/app/reasoning_effort_map.py:ro
|
|
environment:
|
|
# master_key gates the proxy + admin UI login. Must start with sk-.
|
|
- LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY}
|
|
# salt for encrypting any virtual/model keys stored in the DB.
|
|
- LITELLM_SALT_KEY=${LITELLM_SALT_KEY}
|
|
- DATABASE_URL=postgresql://${POSTGRES_USER:-litellm}:${POSTGRES_PASSWORD}@litellm-db:5432/${POSTGRES_DB:-litellm}
|
|
- STORE_MODEL_IN_DB=True
|
|
# Upstream vLLM API key (empty in the vllm stack's .env by default →
|
|
# leave blank; LiteLLM still needs the var to exist).
|
|
- VLLM_API_KEY=${VLLM_API_KEY:-}
|
|
# Cloud API keys fronted by the gateway for unified logging (z.ai GLM,
|
|
# Moonshot Kimi, etc.). Paid — only gateway-keyed callers reach them,
|
|
# but they spend.
|
|
- Z_AI_API_KEY=${Z_AI_API_KEY:-}
|
|
# Kimi: KIMI_CODE_API_KEY = the coding endpoint (api.kimi.com/coding, the
|
|
# primary kimi-k3 arm); MOONSHOT_API_KEY = the general api.moonshot.ai
|
|
# endpoint (the kimi-k3-gen-api variant).
|
|
- KIMI_CODE_API_KEY=${KIMI_CODE_API_KEY:-}
|
|
- MOONSHOT_API_KEY=${MOONSHOT_API_KEY:-}
|
|
# Langfuse-ready: blank until you bolt Langfuse on. Filling these +
|
|
# uncommenting the callback in config.yaml is the entire upgrade.
|
|
- LANGFUSE_PUBLIC_KEY=${LANGFUSE_PUBLIC_KEY:-}
|
|
- LANGFUSE_SECRET_KEY=${LANGFUSE_SECRET_KEY:-}
|
|
- LANGFUSE_HOST=${LANGFUSE_HOST:-}
|
|
command: ["--config", "/app/config.yaml"]
|
|
depends_on:
|
|
litellm-db:
|
|
condition: service_healthy
|
|
healthcheck:
|
|
# python is guaranteed in the image (curl is not); urlopen raises
|
|
# on non-200. 127.0.0.1 dodges the IPv6-first localhost trap.
|
|
test: ["CMD-SHELL", "python -c 'import urllib.request; urllib.request.urlopen(\"http://127.0.0.1:4000/health/liveliness\")' || exit 1"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 3
|
|
start_period: 40s
|
|
networks:
|
|
- tnet
|
|
labels:
|
|
- homepage.group=AI - Gateways & Chat
|
|
- homepage.name=LiteLLM Gateway
|
|
- homepage.icon=mdi-router-network
|
|
- homepage.description=vLLM request/response logging gateway (ana-docker)
|
|
- homepage.href=http://10.250.50.70:${LITELLM_PORT:-4000}/ui
|
|
|
|
litellm-db:
|
|
image: postgres:16-alpine
|
|
container_name: litellm-db
|
|
restart: unless-stopped
|
|
environment:
|
|
- POSTGRES_USER=${POSTGRES_USER:-litellm}
|
|
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
|
|
- POSTGRES_DB=${POSTGRES_DB:-litellm}
|
|
volumes:
|
|
- litellm_db:/var/lib/postgresql/data
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-litellm} -d ${POSTGRES_DB:-litellm}"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 5
|
|
start_period: 20s
|
|
networks:
|
|
- tnet
|
|
|
|
volumes:
|
|
litellm_db:
|
|
name: litellm_db
|
|
|
|
networks:
|
|
tnet:
|
|
name: traefik-net
|
|
external: true
|