410 lines
16 KiB
YAML
410 lines
16 KiB
YAML
# LiteLLM gateway config — fronts the vLLM services on ana-ml2
|
|
# (10.250.50.54) and logs every request + response so they're
|
|
# inspectable in the Logs UI at http://10.250.50.70:4000/ui.
|
|
#
|
|
# Deploys to /opt/docker/conf/litellm/config.yaml (mounted read-only
|
|
# into the container at /app/config.yaml).
|
|
#
|
|
# Model-name → upstream mapping:
|
|
# phi4-mini → vLLM :8004 (generative chat)
|
|
# qwen3-embedding → vLLM :8001 (/v1/embeddings)
|
|
# qwen3-reranker → vLLM :8002 (/rerank)
|
|
# * (wildcard) → llama-swap :9292 (the swappable generative zoo)
|
|
#
|
|
# The wildcard fronts llama-swap so its whole model zoo logs through the
|
|
# gateway without per-model registration. The vllm-reward classifier
|
|
# (:8003) is a pooling /classify endpoint with no first-class LiteLLM
|
|
# route — left direct; see README.
|
|
|
|
model_list:
|
|
# --- Granite 4.1 8B (generative chat) — production summarizer + dreaming
|
|
# agent. Replaced phi4-mini 2026-06-05 (beat it on precision in brokkr's
|
|
# R15 P03 eval). vLLM on ana-ml2 GPU 1, official FP8, 50K ctx. Explicit
|
|
# entry shadows the "*" wildcard's llama-swap route for this name. Full
|
|
# prompt + completion captured per call. ---
|
|
- model_name: granite-4.1-8b
|
|
litellm_params:
|
|
model: hosted_vllm/granite-4.1-8b
|
|
api_base: http://10.250.50.54:8004/v1
|
|
api_key: os.environ/VLLM_API_KEY
|
|
temperature: 0
|
|
model_info:
|
|
mode: chat
|
|
|
|
# alias: summarizer -> granite-4.1-8b (operator 2026-06-19). Duplicate-entry alias
|
|
# (not router_settings.model_group_alias — that's hidden from /v1/models and can be
|
|
# silently ignored in config per litellm #15020/#5524). Keep api_base in sync above.
|
|
- model_name: summarizer
|
|
litellm_params:
|
|
model: hosted_vllm/granite-4.1-8b
|
|
api_base: http://10.250.50.54:8004/v1
|
|
api_key: os.environ/VLLM_API_KEY
|
|
temperature: 0
|
|
model_info:
|
|
mode: chat
|
|
|
|
# alias: classifier -> granite-4.1-8b (operator 2026-06-19). Light/fast classification
|
|
# + triage endpoint; same backend as summarizer. Keep api_base in sync above.
|
|
- model_name: classifier
|
|
litellm_params:
|
|
model: hosted_vllm/granite-4.1-8b
|
|
api_base: http://10.250.50.54:8004/v1
|
|
api_key: os.environ/VLLM_API_KEY
|
|
temperature: 0
|
|
model_info:
|
|
mode: chat
|
|
|
|
# alias: summarizer-large -> gen / qwen3.5-122-a10b (operator 2026-06-19). For heavier
|
|
# summarization that wants the 122B Qwopus instead of granite-8b. Thinking OFF (matches
|
|
# gen). Keep api_base (:8013) + enable_thinking in sync with the gen record below.
|
|
- model_name: summarizer-large
|
|
litellm_params:
|
|
model: hosted_vllm/qwen3.5-122-a10b
|
|
api_base: http://10.250.50.54:8013/v1
|
|
api_key: os.environ/VLLM_API_KEY
|
|
presence_penalty: 1.0
|
|
temperature: 0.7
|
|
top_p: 0.8
|
|
extra_body:
|
|
top_k: 20
|
|
chat_template_kwargs:
|
|
enable_thinking: false
|
|
model_info:
|
|
mode: chat
|
|
|
|
# --- Qwen-Image-Bench (NVFP4) — text-to-image quality JUDGE, vision-intact. vLLM
|
|
# on ana-ml2 GPU 1, :8014. REPLACED qwen3.6-35b-a3b (operator 2026-06-19; the
|
|
# qwen36 VL judge was downed + its FP8 weights freed ~33GB). Multimodal — image
|
|
# + prompt in, a 5-dim verdict out (overall quality / prompt match / aesthetic /
|
|
# LoRA activation / confidence). image-judge is a duplicate-entry alias.
|
|
# ⚠️ arbo's hero-judge was bound to qwen3.6-35b-a3b — comfy-dev notified to
|
|
# repoint (different I/O than the old general VL judge). ---
|
|
- model_name: qwen-image-bench
|
|
litellm_params:
|
|
model: hosted_vllm/qwen-image-bench
|
|
api_base: http://10.250.50.54:8014/v1
|
|
api_key: os.environ/VLLM_API_KEY
|
|
temperature: 0
|
|
top_p: 1.0
|
|
extra_body:
|
|
top_k: 1
|
|
repetition_penalty: 1.05
|
|
model_info:
|
|
mode: chat
|
|
- model_name: image-judge
|
|
litellm_params:
|
|
model: hosted_vllm/qwen-image-bench
|
|
api_base: http://10.250.50.54:8014/v1
|
|
api_key: os.environ/VLLM_API_KEY
|
|
temperature: 0
|
|
top_p: 1.0
|
|
extra_body:
|
|
top_k: 1
|
|
repetition_penalty: 1.05
|
|
model_info:
|
|
mode: chat
|
|
|
|
# --- Qwopus3.5-122B-A10B (Kimi-distilled, abliterated, NVFP4, VISION-INTACT) — the
|
|
# general / `gen` model on ana-ml2 GPU 0. Replaced the bjk110 text-only qwen3.5-122b
|
|
# 2026-06-19 (which had replaced mistral-small-4). Served on :8013 via vLLM as plain
|
|
# multimodal (no text-only patch), served-name qwen3.5-122-a10b — so these records
|
|
# route UNCHANGED. Full 256K (262144) @ fp8 KV + CUDA graphs (92.7 tok/s warm);
|
|
# tool-calling via qwen3_coder. Thinking split = chat_template_kwargs.enable_thinking
|
|
# + --reasoning-parser qwen3. One upstream fanned out under qwen3.5-122-a10b[-reasoning]
|
|
# + aliases qwen-large[-reasoning] + gen[-reasoning]; -reasoning variants enable
|
|
# thinking. Keep api_base in sync.
|
|
# presence_penalty: 1.0 on ALL these qwen3.5-122-a10b records (+ summarizer-large
|
|
# above) — anti-repetition-loop damper for the abliterated/NVFP4 tendency (operator
|
|
# 2026-06-27). Gateway-tunable default (callers can override); bake the validated
|
|
# value into the vLLM serving def (stacks/qwen3.5-122b, --override-generation-config)
|
|
# once confirmed, to also cover direct (non-gateway) callers.
|
|
# ⚠️ Worldtree CHARACTER backend (was bound to mistral-small-4) is dark until
|
|
# repointed — operator-acknowledged. ---
|
|
- model_name: qwen3.5-122-a10b
|
|
litellm_params:
|
|
model: hosted_vllm/qwen3.5-122-a10b
|
|
api_base: http://10.250.50.54:8013/v1
|
|
api_key: os.environ/VLLM_API_KEY
|
|
presence_penalty: 1.0
|
|
temperature: 0.7
|
|
top_p: 0.8
|
|
extra_body:
|
|
top_k: 20
|
|
chat_template_kwargs:
|
|
enable_thinking: false
|
|
model_info:
|
|
mode: chat
|
|
- model_name: qwen3.5-122-a10b-reasoning
|
|
litellm_params:
|
|
model: hosted_vllm/qwen3.5-122-a10b
|
|
api_base: http://10.250.50.54:8013/v1
|
|
api_key: os.environ/VLLM_API_KEY
|
|
presence_penalty: 1.0
|
|
temperature: 0.6
|
|
top_p: 0.95
|
|
extra_body:
|
|
top_k: 20
|
|
chat_template_kwargs:
|
|
enable_thinking: true
|
|
model_info:
|
|
mode: chat
|
|
- model_name: qwen-large
|
|
litellm_params:
|
|
model: hosted_vllm/qwen3.5-122-a10b
|
|
api_base: http://10.250.50.54:8013/v1
|
|
api_key: os.environ/VLLM_API_KEY
|
|
presence_penalty: 1.0
|
|
temperature: 0.7
|
|
top_p: 0.8
|
|
extra_body:
|
|
top_k: 20
|
|
chat_template_kwargs:
|
|
enable_thinking: false
|
|
model_info:
|
|
mode: chat
|
|
- model_name: qwen-large-reasoning
|
|
litellm_params:
|
|
model: hosted_vllm/qwen3.5-122-a10b
|
|
api_base: http://10.250.50.54:8013/v1
|
|
api_key: os.environ/VLLM_API_KEY
|
|
presence_penalty: 1.0
|
|
temperature: 0.6
|
|
top_p: 0.95
|
|
extra_body:
|
|
top_k: 20
|
|
chat_template_kwargs:
|
|
enable_thinking: true
|
|
model_info:
|
|
mode: chat
|
|
- model_name: gen
|
|
litellm_params:
|
|
model: hosted_vllm/qwen3.5-122-a10b
|
|
api_base: http://10.250.50.54:8013/v1
|
|
api_key: os.environ/VLLM_API_KEY
|
|
presence_penalty: 1.0
|
|
temperature: 0.7
|
|
top_p: 0.8
|
|
extra_body:
|
|
top_k: 20
|
|
chat_template_kwargs:
|
|
enable_thinking: false
|
|
model_info:
|
|
mode: chat
|
|
- model_name: gen-reasoning
|
|
litellm_params:
|
|
model: hosted_vllm/qwen3.5-122-a10b
|
|
api_base: http://10.250.50.54:8013/v1
|
|
api_key: os.environ/VLLM_API_KEY
|
|
presence_penalty: 1.0
|
|
temperature: 0.6
|
|
top_p: 0.95
|
|
extra_body:
|
|
top_k: 20
|
|
chat_template_kwargs:
|
|
enable_thinking: true
|
|
model_info:
|
|
mode: chat
|
|
|
|
# --- Selene 1 Mini 8B (AtlaAI judge, FP8) — restored on GPU1 after the
|
|
# llama-swap teardown (was the Q6_K GGUF in the swap zoo). vLLM dynamic fp8,
|
|
# :8011. Explicit entry shadows the "*" wildcard (which used to reach it via
|
|
# llama-swap). Hallucination/RAG-faithfulness judge; callers set temp ~0.01. ---
|
|
- model_name: selene-1-mini-8b
|
|
litellm_params:
|
|
model: hosted_vllm/selene-1-mini-8b
|
|
api_base: http://10.250.50.54:8011/v1
|
|
api_key: os.environ/VLLM_API_KEY
|
|
temperature: 0.6
|
|
top_p: 0.9
|
|
model_info:
|
|
mode: chat
|
|
|
|
# --- Qwen3 embeddings ---
|
|
- model_name: qwen3-embedding
|
|
litellm_params:
|
|
model: hosted_vllm/Qwen/Qwen3-Embedding-0.6B
|
|
api_base: http://10.250.50.54:8001/v1
|
|
api_key: os.environ/VLLM_API_KEY
|
|
model_info:
|
|
mode: embedding
|
|
|
|
# --- Qwen3 reranker (proxy /rerank route) ---
|
|
- model_name: qwen3-reranker
|
|
litellm_params:
|
|
model: hosted_vllm/Qwen/Qwen3-Reranker-0.6B
|
|
api_base: http://10.250.50.54:8002/v1
|
|
api_key: os.environ/VLLM_API_KEY
|
|
model_info:
|
|
mode: rerank
|
|
|
|
# --- Worldtree capability aliases (role→capability gateway swaps, ADR-0012).
|
|
# Stable role-named aliases so consumers bind the CAPABILITY, not a concrete
|
|
# model; swap the backing model here and callers are unaffected. NO generic
|
|
# `embedding` alias ON PURPOSE — embedding vectors are model-specific (not
|
|
# swap-transparent), so that capability stays `qwen3-embedding` above. ---
|
|
# chat-judge → generative LLM-as-judge (Selene); WT selene-judgment role.
|
|
- model_name: chat-judge
|
|
litellm_params:
|
|
model: hosted_vllm/selene-1-mini-8b
|
|
api_base: http://10.250.50.54:8011/v1
|
|
api_key: os.environ/VLLM_API_KEY
|
|
temperature: 0.6
|
|
top_p: 0.9
|
|
model_info:
|
|
mode: chat
|
|
# reranker → generic capability name for rerank (currently qwen3-reranker).
|
|
- model_name: reranker
|
|
litellm_params:
|
|
model: hosted_vllm/Qwen/Qwen3-Reranker-0.6B
|
|
api_base: http://10.250.50.54:8002/v1
|
|
api_key: os.environ/VLLM_API_KEY
|
|
model_info:
|
|
mode: rerank
|
|
|
|
# --- z.ai GLM (cloud API) — fronted for unified logging across local
|
|
# + cloud inference. Explicit entries, so they win over the "*"
|
|
# wildcard below (no collision with llama-swap's glm4.7-flash etc.
|
|
# — different model IDs). NOTE: paid API; only gateway-keyed callers
|
|
# can reach these, but they DO spend z.ai credits. Key in .env. ---
|
|
# glm-5.1: thinking DISABLED by default (2026-06-11, operator call). LiteLLM
|
|
# strips a top-level `thinking` param (drop_params), but forwards `extra_body`
|
|
# verbatim to z.ai, where the native thinking:{type:disabled} control lands —
|
|
# verified reasoning_tokens→0. Reasoning is opt-in via glm-5.1-reasoning below.
|
|
- model_name: glm-5.1
|
|
litellm_params:
|
|
model: openai/glm-5.1
|
|
api_base: https://api.z.ai/api/coding/paas/v4
|
|
api_key: os.environ/Z_AI_API_KEY
|
|
temperature: 1.0
|
|
top_p: 0.95
|
|
extra_body:
|
|
thinking:
|
|
type: disabled
|
|
# glm-5.1-reasoning: identical upstream, thinking ENABLED (opt-in reasoning).
|
|
- model_name: glm-5.1-reasoning
|
|
litellm_params:
|
|
model: openai/glm-5.1
|
|
api_base: https://api.z.ai/api/coding/paas/v4
|
|
api_key: os.environ/Z_AI_API_KEY
|
|
temperature: 1.0
|
|
top_p: 0.95
|
|
extra_body:
|
|
thinking:
|
|
type: enabled
|
|
# glm-5.2 (released ~2026-06; canonical z.ai id `glm-5.2`, confirmed via /models +
|
|
# a live completion with our key). Mirrors the glm-5.1 pattern: thinking DISABLED
|
|
# by default (consistency with the 2026-06-11 operator call), opt-in reasoning via
|
|
# glm-5.2-reasoning. extra_body.thinking is forwarded verbatim to z.ai.
|
|
- model_name: glm-5.2
|
|
litellm_params:
|
|
model: openai/glm-5.2
|
|
api_base: https://api.z.ai/api/coding/paas/v4
|
|
api_key: os.environ/Z_AI_API_KEY
|
|
temperature: 1.0
|
|
top_p: 0.95
|
|
extra_body:
|
|
thinking:
|
|
type: disabled
|
|
- model_name: glm-5.2-reasoning
|
|
litellm_params:
|
|
model: openai/glm-5.2
|
|
api_base: https://api.z.ai/api/coding/paas/v4
|
|
api_key: os.environ/Z_AI_API_KEY
|
|
temperature: 1.0
|
|
top_p: 0.95
|
|
extra_body:
|
|
thinking:
|
|
type: enabled
|
|
# gen-frontier / gen-frontier-reasoning: capability aliases for the PAID
|
|
# frontier tier (GLM 5.2 @ z.ai), mirroring glm-5.2 / glm-5.2-reasoning
|
|
# (thinking off / on). Worldtree binds these for frontier-grade generation
|
|
# / reasoning; swap the backing frontier model here, callers unaffected.
|
|
# PAID — only all-proxy-models / explicitly-scoped keys reach them; the free
|
|
# all-agents-local key is fenced off z.ai spend and cannot.
|
|
- model_name: gen-frontier
|
|
litellm_params:
|
|
model: openai/glm-5.2
|
|
api_base: https://api.z.ai/api/coding/paas/v4
|
|
api_key: os.environ/Z_AI_API_KEY
|
|
temperature: 1.0
|
|
top_p: 0.95
|
|
extra_body:
|
|
thinking:
|
|
type: disabled
|
|
- model_name: gen-frontier-reasoning
|
|
litellm_params:
|
|
model: openai/glm-5.2
|
|
api_base: https://api.z.ai/api/coding/paas/v4
|
|
api_key: os.environ/Z_AI_API_KEY
|
|
temperature: 1.0
|
|
top_p: 0.95
|
|
extra_body:
|
|
thinking:
|
|
type: enabled
|
|
- model_name: glm-5-turbo
|
|
litellm_params:
|
|
model: openai/glm-5-turbo
|
|
api_base: https://api.z.ai/api/coding/paas/v4
|
|
api_key: os.environ/Z_AI_API_KEY
|
|
temperature: 1.0
|
|
top_p: 0.95
|
|
- model_name: glm-4.7
|
|
litellm_params:
|
|
model: openai/glm-4.7
|
|
api_base: https://api.z.ai/api/coding/paas/v4
|
|
api_key: os.environ/Z_AI_API_KEY
|
|
temperature: 1.0
|
|
top_p: 0.95
|
|
- model_name: glm-4.5-air
|
|
litellm_params:
|
|
model: openai/glm-4.5-air
|
|
api_base: https://api.z.ai/api/coding/paas/v4
|
|
api_key: os.environ/Z_AI_API_KEY
|
|
temperature: 0.6
|
|
top_p: 0.95
|
|
|
|
# --- (removed 2026-06-20, operator call) the `*` wildcard → llama-swap
|
|
# (ana-ml2:9292). llama-swap is decommissioned (:9292 confirmed down), so
|
|
# the wildcard routed every unmatched / typo'd / stale model name to a DEAD
|
|
# backend → a misleading "Connection error" instead of a clean "model not
|
|
# found". This is the footgun that silently swallowed Worldtree's defunct
|
|
# model names. Removed so unknown models now fail loudly (404). Re-add an
|
|
# explicit per-model entry if a swappable zoo ever returns. ---
|
|
|
|
general_settings:
|
|
master_key: os.environ/LITELLM_MASTER_KEY
|
|
database_url: os.environ/DATABASE_URL
|
|
store_model_in_db: true
|
|
# THE log switch: persists full request messages + response bodies into
|
|
# SpendLogs so they render in the Logs UI. Without this you get metadata
|
|
# (tokens, latency, model) but not the prompt/completion text.
|
|
store_prompts_in_spend_logs: true
|
|
# scalar-judge → Skywork-Reward-V2 (scalar reward model; vLLM pooling on
|
|
# ana-ml2:8003). LiteLLM has no reward/pooling MODE, so this is a passthrough,
|
|
# not a model_list alias. Gateway-key-gated. Consumers POST the reward body to
|
|
# /scalar-judge/<route> (e.g. /pooling or /classify), forwarded to :8003.
|
|
# SWAP-SENSITIVE: a different reward model shifts the score scale, so consumers
|
|
# must recalibrate thresholds after a backing swap.
|
|
pass_through_endpoints:
|
|
- path: "/scalar-judge"
|
|
target: "http://10.250.50.54:8003"
|
|
forward_headers: true
|
|
include_subpath: true
|
|
|
|
litellm_settings:
|
|
# vLLM rejects some OpenAI params other backends accept; drop silently
|
|
# rather than 400 the caller.
|
|
drop_params: true
|
|
# Custom pre-call hook: strip an empty `tools: []` (+ orphaned tool_choice)
|
|
# before forwarding upstream. vLLM 400s on empty tools arrays ("tools must
|
|
# not be an empty array"); drop_params doesn't catch empty VALUES, only
|
|
# unsupported params. Runs on every request → fixes it for all vLLM models.
|
|
# File mounted at /app/strip_empty_tools.py; reference is module.instance,
|
|
# resolved relative to this config's directory.
|
|
callbacks: ["strip_empty_tools.strip_empty_tools_instance"]
|
|
# Langfuse trace export RETIRED 2026-06-20 (operator). Its ClickHouse member spewed
|
|
# ~94 GB of unrotated logs and filled ana-docker's disk; the trace UI was redundant
|
|
# with LiteLLM's native spend_logs (store_prompts_in_spend_logs: true → full
|
|
# prompts/responses/tokens/cost at :4000/ui). Gateway observability stays fully
|
|
# native. Re-add success_callback/failure_callback here if Langfuse ever returns.
|