From 83b2ec1a8aa6dc64649660053413dede73d7ceee Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Thu, 4 Jun 2026 01:17:50 -0700 Subject: [PATCH] feat(litellm): add vLLM request/response logging gateway on ana-docker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LiteLLM proxy fronting the vLLM services on ana-ml2 so every request + response is captured and inspectable in a browser Logs UI — the visibility vLLM itself lacks (Dozzle shows only connection metadata). - compose: litellm (proxy + /ui Logs) + litellm-db (Postgres store) - conf/config.yaml: routes phi4-mini (chat, :8004), qwen3-embedding (:8001), qwen3-reranker (:8002); store_prompts_in_spend_logs persists full prompt/completion text. reward classifier (:8003) stays direct (no first-class LiteLLM route). - Langfuse-ready: lean first cut intentionally skips Langfuse's heavy v3 stack; graduating is one env-var + callback step, no re-architecture. - roadmap: mark the vLLM-observability item's first cut as shipped. Lean first cut of docs/roadmap.md "Observability for the vLLM stack". --- docs/roadmap.md | 11 +++- stacks/litellm/.env.example | 38 +++++++++++ stacks/litellm/README.md | 109 ++++++++++++++++++++++++++++++++ stacks/litellm/compose.yaml | 97 ++++++++++++++++++++++++++++ stacks/litellm/conf/config.yaml | 65 +++++++++++++++++++ 5 files changed, 318 insertions(+), 2 deletions(-) create mode 100644 stacks/litellm/.env.example create mode 100644 stacks/litellm/README.md create mode 100644 stacks/litellm/compose.yaml create mode 100644 stacks/litellm/conf/config.yaml diff --git a/docs/roadmap.md b/docs/roadmap.md index 4d81202..d211e66 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -41,5 +41,12 @@ coarse host/GPU stats but no app-layer inference metrics. KV-cache-bound, queueing, or has latency regressions — the operational view llama-swap's UI only hinted at. -**Status:** deferred during the phi4-mini deploy; do after phi4 is live. -Langfuse first (the operator specifically wants full req/resp visibility). +**Status:** phi4 is live. **Lean first cut shipped 2026-06-04** — the +`litellm` stack (`stacks/litellm/`, ana-docker) is the req/resp logging +gateway: full prompt/completion captured in a browser Logs UI, fronting +the vLLM services on ana-ml2. It is **Langfuse-ready** (one env-var + +callback step graduates it to full Langfuse traces, no re-architecture). +Remaining: (a) re-point consumers (nevermore, dreaming agent, +asset-engine) at the gateway; (b) stand up Langfuse + flip the callback +when the polished trace UI is wanted; (c) the Prometheus + Grafana +operational-metrics layer above. diff --git a/stacks/litellm/.env.example b/stacks/litellm/.env.example new file mode 100644 index 0000000..5d35125 --- /dev/null +++ b/stacks/litellm/.env.example @@ -0,0 +1,38 @@ +# LiteLLM gateway — copy to .env on ana-docker and fill the secrets. +# Real .env lives on the server only (gitignored); never commit it. + +# Image tag. main-stable is the rolling stable; pin to a dated/SHA tag +# (e.g. main-v1.74.0-stable) once a known-good build is confirmed. +LITELLM_TAG=main-stable + +# Publish. Bind to all interfaces on the LAN; 4000 is the LiteLLM default +# (proxy API + admin/Logs UI at /ui). +LITELLM_BIND=0.0.0.0 +LITELLM_PORT=4000 + +# Proxy + admin-UI master key. MUST start with "sk-". Generate one: +# openssl rand -hex 24 | sed 's/^/sk-/' +LITELLM_MASTER_KEY= + +# Salt for encrypting any virtual/model keys stored in Postgres. Generate: +# openssl rand -hex 32 +# Changing this after keys are stored makes them undecryptable — set once. +LITELLM_SALT_KEY= + +# Postgres (backing store for spend logs + keys). Password is required; +# user/db default to "litellm". +POSTGRES_USER=litellm +POSTGRES_DB=litellm +POSTGRES_PASSWORD= + +# Upstream vLLM API key. The vllm stack on ana-ml2 ships API_KEY= (empty) +# by default → leave this blank. Set it ONLY if you set API_KEY in the +# vllm stack's .env. +VLLM_API_KEY= + +# --- Langfuse-ready (leave blank for the lean first cut) --- +# Fill these AND uncomment success/failure_callback in conf/config.yaml +# to ship full traces to a Langfuse instance. That is the whole upgrade. +LANGFUSE_PUBLIC_KEY= +LANGFUSE_SECRET_KEY= +LANGFUSE_HOST= diff --git a/stacks/litellm/README.md b/stacks/litellm/README.md new file mode 100644 index 0000000..72f9a90 --- /dev/null +++ b/stacks/litellm/README.md @@ -0,0 +1,109 @@ +# litellm + +OpenAI-compatible **gateway** in front of the vLLM services on ana-ml2, +standing in the request path so every request + response is **logged and +inspectable in a browser**. This is the thing vLLM does not give us: +Dozzle shows vLLM's stdout (connection/request metadata) but not the full +prompt/completion bodies. LiteLLM captures both, per call, with a Logs UI. + +**Server:** ana-docker (`10.250.50.70`) +**Port:** `4000` (proxy API + admin/Logs UI at `/ui`) — configurable in `.env` +**Backs:** the `vllm` stack on ana-ml2 (`10.250.50.54`) + +## Why it exists + +phi4-mini is becoming a production summarizer + "dreaming" agent. Being +able to read exactly what it was asked and what it answered is the +difference between debuggable and opaque. See `docs/roadmap.md` → +"Observability for the vLLM stack". This is the **lean first cut** of that +roadmap item — see *Langfuse-ready* below for the upgrade path. + +## What routes through it + +Consumers point their OpenAI `base_url` at `http://10.250.50.70:4000` and +pick a model **by name**; the gateway forwards to the right vLLM port and +logs the round-trip. + +| model name (here) | upstream | vLLM port | logged | +|---|---|---|---| +| `phi4-mini` | generative chat | `:8004` | **full prompt + completion** | +| `qwen3-embedding` | `/v1/embeddings` | `:8001` | input + vector metadata | +| `qwen3-reranker` | `/rerank` | `:8002` | query + docs + scores | + +**Not routed:** the `vllm-reward` Skywork classifier (`:8003`) is a pooling +`/classify` endpoint with no first-class LiteLLM route — callers hit it +directly for now. The generative model is the high-value target for +req/resp visibility and it routes cleanly here. (If reward logging is +wanted later, LiteLLM `pass_through_endpoints` can cover it.) + +## The log switch + +Full prompt/response text shows in the Logs UI because of +`store_prompts_in_spend_logs: true` in `conf/config.yaml`. Without it you'd +get metadata only (tokens, latency, model name) — not the text. The +Postgres sidecar (`litellm-db`) is the store. + +## Langfuse-ready + +This deliberately does **not** stand up Langfuse's heavy v3 stack +(ClickHouse + Redis + MinIO + Postgres + app containers). To graduate to +full Langfuse traces later: + +1. Stand up (or point at) a Langfuse instance. +2. Set `LANGFUSE_PUBLIC_KEY` / `LANGFUSE_SECRET_KEY` / `LANGFUSE_HOST` in `.env`. +3. Uncomment `success_callback` / `failure_callback` in `conf/config.yaml`. +4. `docker compose up -d` to restart. + +No re-architecture: the gateway and every consumer stay pointed here. + +## Deploy + +```bash +# 1. Sync canonical → ana-docker (compose + conf/config.yaml) +scripts/deploy-stack.sh ana-docker litellm + +# 2. On the server: create .env from the template and fill secrets +ssh ana-docker 'cd /opt/docker/compose/litellm && cp -n .env.example .env' +# generate the keys: +# openssl rand -hex 24 | sed 's/^/sk-/' # LITELLM_MASTER_KEY +# openssl rand -hex 32 # LITELLM_SALT_KEY +# openssl rand -hex 24 # POSTGRES_PASSWORD +$EDITOR # fill .env on the server + +# 3. Sanity-parse then launch +ssh ana-docker 'cd /opt/docker/compose/litellm && docker compose config >/dev/null && docker compose up -d && docker compose ps' +``` + +> `.env.example` is the only env file in git. The real `.env` (master key, +> salt, Postgres password) lives on the server and is gitignored. + +## Smoke test + +```bash +# liveness (no auth) +curl -fsS http://10.250.50.70:4000/health/liveliness # -> "I'm alive!" + +# a chat round-trip (uses the master key), then look for it in the Logs UI +curl -s http://10.250.50.70:4000/v1/chat/completions \ + -H "Authorization: Bearer $LITELLM_MASTER_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model":"phi4-mini","messages":[{"role":"user","content":"say hi"}]}' + +# embeddings +curl -s http://10.250.50.70:4000/v1/embeddings \ + -H "Authorization: Bearer $LITELLM_MASTER_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model":"qwen3-embedding","input":"hello"}' +``` + +Then open `http://10.250.50.70:4000/ui` (log in with the master key) → +**Logs** tab → the calls appear with full request + response. + +## Notes + +- Both boxes are Anaheim (`10.250.0.0/16`) so the ana-docker → ana-ml2 hop + is LAN-local; negligible added latency. +- `VLLM_API_KEY` is blank by default because the `vllm` stack ships + `API_KEY=` empty. Set it here only if you set it there. +- `LITELLM_SALT_KEY` must be set **once** and never changed — rotating it + makes any keys stored in Postgres undecryptable. diff --git a/stacks/litellm/compose.yaml b/stacks/litellm/compose.yaml new file mode 100644 index 0000000..2690501 --- /dev/null +++ b/stacks/litellm/compose.yaml @@ -0,0 +1,97 @@ +# LiteLLM — OpenAI-compatible gateway fronting the vLLM services on +# ana-ml2, standing in the request path to LOG every request + response +# so they're inspectable in a browser (the thing vLLM itself does not +# give us; Dozzle only shows connection metadata). +# +# litellm — proxy + Logs UI (http://10.250.50.70:4000/ui) +# litellm-db — Postgres: spend logs (incl. full prompts/responses), +# virtual keys, model config +# +# Why this exists: phi4-mini is a production summarizer + "dreaming" +# agent on ana-ml2; "what was it asked, what did it answer" must be +# inspectable. LiteLLM captures full req/resp per call with a Logs UI. +# +# Langfuse-ready: this is the lean first cut. To ship full traces to a +# Langfuse instance later, uncomment the success/failure_callback block +# in conf/config.yaml and fill LANGFUSE_* in .env — no re-architecture, +# the gateway and all consumers stay pointed here. +# +# Consumers (nevermore, the dreaming agent, asset-engine, …) re-point +# their OpenAI base_url at this gateway and select a model by name +# (phi4-mini / qwen3-embedding / qwen3-reranker). The gateway forwards +# to the right vLLM port on ana-ml2. +# +# All tunables live in .env — edit that, not this file. + +services: + litellm: + image: ghcr.io/berriai/litellm:${LITELLM_TAG:-main-stable} + container_name: litellm + restart: unless-stopped + ports: + - "${LITELLM_BIND:-0.0.0.0}:${LITELLM_PORT:-4000}:4000" + volumes: + - /opt/docker/conf/litellm/config.yaml:/app/config.yaml:ro + environment: + # master_key gates the proxy + admin UI login. Must start with sk-. + - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY} + # salt for encrypting any virtual/model keys stored in the DB. + - LITELLM_SALT_KEY=${LITELLM_SALT_KEY} + - DATABASE_URL=postgresql://${POSTGRES_USER:-litellm}:${POSTGRES_PASSWORD}@litellm-db:5432/${POSTGRES_DB:-litellm} + - STORE_MODEL_IN_DB=True + # Upstream vLLM API key (empty in the vllm stack's .env by default → + # leave blank; LiteLLM still needs the var to exist). + - VLLM_API_KEY=${VLLM_API_KEY:-} + # Langfuse-ready: blank until you bolt Langfuse on. Filling these + + # uncommenting the callback in config.yaml is the entire upgrade. + - LANGFUSE_PUBLIC_KEY=${LANGFUSE_PUBLIC_KEY:-} + - LANGFUSE_SECRET_KEY=${LANGFUSE_SECRET_KEY:-} + - LANGFUSE_HOST=${LANGFUSE_HOST:-} + command: ["--config", "/app/config.yaml"] + depends_on: + litellm-db: + condition: service_healthy + healthcheck: + # python is guaranteed in the image (curl is not); urlopen raises + # on non-200. 127.0.0.1 dodges the IPv6-first localhost trap. + test: ["CMD-SHELL", "python -c 'import urllib.request; urllib.request.urlopen(\"http://127.0.0.1:4000/health/liveliness\")' || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + networks: + - tnet + labels: + - homepage.group=AI Systems + - homepage.name=LiteLLM Gateway + - homepage.icon=mdi-router-network + - homepage.description=vLLM request/response logging gateway (ana-docker) + - homepage.href=http://10.250.50.70:${LITELLM_PORT:-4000}/ui + + litellm-db: + image: postgres:16-alpine + container_name: litellm-db + restart: unless-stopped + environment: + - POSTGRES_USER=${POSTGRES_USER:-litellm} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - POSTGRES_DB=${POSTGRES_DB:-litellm} + volumes: + - litellm_db:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-litellm} -d ${POSTGRES_DB:-litellm}"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 20s + networks: + - tnet + +volumes: + litellm_db: + name: litellm_db + +networks: + tnet: + name: traefik-net + external: true diff --git a/stacks/litellm/conf/config.yaml b/stacks/litellm/conf/config.yaml new file mode 100644 index 0000000..415109f --- /dev/null +++ b/stacks/litellm/conf/config.yaml @@ -0,0 +1,65 @@ +# LiteLLM gateway config — fronts the vLLM services on ana-ml2 +# (10.250.50.54) and logs every request + response so they're +# inspectable in the Logs UI at http://10.250.50.70:4000/ui. +# +# Deploys to /opt/docker/conf/litellm/config.yaml (mounted read-only +# into the container at /app/config.yaml). +# +# Model-name → upstream vLLM port mapping: +# phi4-mini → :8004 (generative chat — the priority target) +# qwen3-embedding → :8001 (/v1/embeddings) +# qwen3-reranker → :8002 (/rerank) +# +# The vllm-reward classifier (:8003) is a pooling /classify endpoint +# with no first-class LiteLLM route — left direct; see README. The +# generative model is the one whose prompts/completions we most need +# to read, and it routes cleanly here. + +model_list: + # --- Phi-4-mini (generative chat) — production summarizer + dreaming + # agent. Full prompt + completion captured per call. --- + - model_name: phi4-mini + litellm_params: + model: hosted_vllm/phi4-mini + api_base: http://10.250.50.54:8004/v1 + api_key: os.environ/VLLM_API_KEY + model_info: + mode: chat + + # --- Qwen3 embeddings --- + - model_name: qwen3-embedding + litellm_params: + model: hosted_vllm/Qwen/Qwen3-Embedding-0.6B + api_base: http://10.250.50.54:8001/v1 + api_key: os.environ/VLLM_API_KEY + model_info: + mode: embedding + + # --- Qwen3 reranker (proxy /rerank route) --- + - model_name: qwen3-reranker + litellm_params: + model: hosted_vllm/Qwen/Qwen3-Reranker-0.6B + api_base: http://10.250.50.54:8002/v1 + api_key: os.environ/VLLM_API_KEY + model_info: + mode: rerank + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + database_url: os.environ/DATABASE_URL + store_model_in_db: true + # THE log switch: persists full request messages + response bodies into + # SpendLogs so they render in the Logs UI. Without this you get metadata + # (tokens, latency, model) but not the prompt/completion text. + store_prompts_in_spend_logs: true + +litellm_settings: + # vLLM rejects some OpenAI params other backends accept; drop silently + # rather than 400 the caller. + drop_params: true + # --- Langfuse-ready: uncomment to ship full traces to a Langfuse + # instance (set LANGFUSE_PUBLIC_KEY / LANGFUSE_SECRET_KEY / + # LANGFUSE_HOST in .env). This is the entire upgrade — the gateway + # and every consumer stay pointed here. --- + # success_callback: ["langfuse"] + # failure_callback: ["langfuse"]