vh
9171e6a20f
LLM observability for the fleet — pretty trace UI over the gateway: prompts, completions, reasoning, latency, token counts. The pretty layer LiteLLM's spend_logs lacked. - stacks/langfuse: v3 self-host stack (web/worker/postgres/clickhouse/redis/ minio) on ana-docker, adapted from upstream. UI on :3001 (gitea owns :3000). Project + API keys auto-provisioned via LANGFUSE_INIT_*. HOSTNAME=0.0.0.0 on langfuse-web so it's reachable via the published port while also on tnet. - litellm: enabled success_callback/failure_callback: ["langfuse"] (the passthrough env was already wired); keys + host go in the litellm .env. Verified: stack healthy, project keys authenticate, and a real gateway call landed a litellm-acompletion trace in Langfuse within ~6s. Secrets live only in the server .env (never committed).
113 lines
4.7 KiB
YAML
113 lines
4.7 KiB
YAML
# LiteLLM gateway config — fronts the vLLM services on ana-ml2
|
|
# (10.250.50.54) and logs every request + response so they're
|
|
# inspectable in the Logs UI at http://10.250.50.70:4000/ui.
|
|
#
|
|
# Deploys to /opt/docker/conf/litellm/config.yaml (mounted read-only
|
|
# into the container at /app/config.yaml).
|
|
#
|
|
# Model-name → upstream mapping:
|
|
# phi4-mini → vLLM :8004 (generative chat)
|
|
# qwen3-embedding → vLLM :8001 (/v1/embeddings)
|
|
# qwen3-reranker → vLLM :8002 (/rerank)
|
|
# * (wildcard) → llama-swap :9292 (the swappable generative zoo)
|
|
#
|
|
# The wildcard fronts llama-swap so its whole model zoo logs through the
|
|
# gateway without per-model registration. The vllm-reward classifier
|
|
# (:8003) is a pooling /classify endpoint with no first-class LiteLLM
|
|
# route — left direct; see README.
|
|
|
|
model_list:
|
|
# --- Granite 4.1 8B (generative chat) — production summarizer + dreaming
|
|
# agent. Replaced phi4-mini 2026-06-05 (beat it on precision in brokkr's
|
|
# R15 P03 eval). vLLM on ana-ml2 GPU 1, official FP8, 50K ctx. Explicit
|
|
# entry shadows the "*" wildcard's llama-swap route for this name. Full
|
|
# prompt + completion captured per call. ---
|
|
- model_name: granite-4.1-8b
|
|
litellm_params:
|
|
model: hosted_vllm/granite-4.1-8b
|
|
api_base: http://10.250.50.54:8004/v1
|
|
api_key: os.environ/VLLM_API_KEY
|
|
model_info:
|
|
mode: chat
|
|
|
|
# --- Qwen3 embeddings ---
|
|
- model_name: qwen3-embedding
|
|
litellm_params:
|
|
model: hosted_vllm/Qwen/Qwen3-Embedding-0.6B
|
|
api_base: http://10.250.50.54:8001/v1
|
|
api_key: os.environ/VLLM_API_KEY
|
|
model_info:
|
|
mode: embedding
|
|
|
|
# --- Qwen3 reranker (proxy /rerank route) ---
|
|
- model_name: qwen3-reranker
|
|
litellm_params:
|
|
model: hosted_vllm/Qwen/Qwen3-Reranker-0.6B
|
|
api_base: http://10.250.50.54:8002/v1
|
|
api_key: os.environ/VLLM_API_KEY
|
|
model_info:
|
|
mode: rerank
|
|
|
|
# --- z.ai GLM (cloud API) — fronted for unified logging across local
|
|
# + cloud inference. Explicit entries, so they win over the "*"
|
|
# wildcard below (no collision with llama-swap's glm4.7-flash etc.
|
|
# — different model IDs). NOTE: paid API; only gateway-keyed callers
|
|
# can reach these, but they DO spend z.ai credits. Key in .env. ---
|
|
- model_name: glm-5.1
|
|
litellm_params:
|
|
model: openai/glm-5.1
|
|
api_base: https://api.z.ai/api/coding/paas/v4
|
|
api_key: os.environ/Z_AI_API_KEY
|
|
- model_name: glm-5-turbo
|
|
litellm_params:
|
|
model: openai/glm-5-turbo
|
|
api_base: https://api.z.ai/api/coding/paas/v4
|
|
api_key: os.environ/Z_AI_API_KEY
|
|
- model_name: glm-4.7
|
|
litellm_params:
|
|
model: openai/glm-4.7
|
|
api_base: https://api.z.ai/api/coding/paas/v4
|
|
api_key: os.environ/Z_AI_API_KEY
|
|
- model_name: glm-4.5-air
|
|
litellm_params:
|
|
model: openai/glm-4.5-air
|
|
api_base: https://api.z.ai/api/coding/paas/v4
|
|
api_key: os.environ/Z_AI_API_KEY
|
|
|
|
# --- llama-swap passthrough (the swappable generative LLM zoo on
|
|
# ana-ml2:9292) ---
|
|
# Wildcard: any model name NOT matched by an exact entry above routes to
|
|
# llama-swap, which swaps the requested model into GPU on demand. This lets
|
|
# the gateway front the WHOLE swappable zoo (artemis / selene / qwen3.x / …)
|
|
# for logging + auth WITHOUT registering each model here — keep adding and
|
|
# swapping models in llama-swap freely; litellm logs them all. litellm does
|
|
# no inference; llama-swap still does all the model loading + serving.
|
|
# Exact matches above (phi4-mini / qwen3-embedding / qwen3-reranker) win;
|
|
# this only catches everything else. `openai/*` forwards the requested model
|
|
# name verbatim to llama-swap's OpenAI-compatible endpoint.
|
|
- model_name: "*"
|
|
litellm_params:
|
|
model: openai/*
|
|
api_base: http://10.250.50.54:9292/v1
|
|
api_key: "noauth" # llama-swap takes no auth; placeholder bearer
|
|
|
|
general_settings:
|
|
master_key: os.environ/LITELLM_MASTER_KEY
|
|
database_url: os.environ/DATABASE_URL
|
|
store_model_in_db: true
|
|
# THE log switch: persists full request messages + response bodies into
|
|
# SpendLogs so they render in the Logs UI. Without this you get metadata
|
|
# (tokens, latency, model) but not the prompt/completion text.
|
|
store_prompts_in_spend_logs: true
|
|
|
|
litellm_settings:
|
|
# vLLM rejects some OpenAI params other backends accept; drop silently
|
|
# rather than 400 the caller.
|
|
drop_params: true
|
|
# --- Langfuse trace export (live 2026-06-05). Full prompt/completion +
|
|
# reasoning + tok-derivable latency traces ship to the Langfuse stack on
|
|
# ana-docker (project "gateway"). Keys + host in .env. The gateway and
|
|
# every consumer stay pointed here — this callback is the whole upgrade. ---
|
|
success_callback: ["langfuse"]
|
|
failure_callback: ["langfuse"]
|