# LiteLLM gateway config — fronts the vLLM services on ana-ml2 # (10.250.50.54) and logs every request + response so they're # inspectable in the Logs UI at http://10.250.50.70:4000/ui. # # Deploys to /opt/docker/conf/litellm/config.yaml (mounted read-only # into the container at /app/config.yaml). # # Model-name → upstream mapping: # phi4-mini → vLLM :8004 (generative chat) # qwen3-embedding → vLLM :8001 (/v1/embeddings) # reranker → vLLM :8013 (/rerank; bge-v2-m3. The old qwen3-reranker # alias on :8002 was retired 2026-08-20.) # * (wildcard) → llama-swap :9292 (the swappable generative zoo) # # The wildcard fronts llama-swap so its whole model zoo logs through the # gateway without per-model registration. The vllm-reward classifier # (:8003) is a pooling /classify endpoint with no first-class LiteLLM # route — left direct; see README. model_list: # --- Granite 4.1 8B (generative chat) — production summarizer + dreaming # agent. Replaced phi4-mini 2026-06-05 (beat it on precision in brokkr's # R15 P03 eval). vLLM on ana-ml2 GPU 1, official FP8, 50K ctx. Explicit # entry shadows the "*" wildcard's llama-swap route for this name. Full # prompt + completion captured per call. --- # --- granite-4.1-8b RETIRED 2026-08-12 (seat downed, GPU1 reclaimed for RP context) --- # - model_name: granite-4.1-8b # litellm_params: # model: hosted_vllm/granite-4.1-8b # api_base: http://10.250.50.54:8004/v1 # api_key: os.environ/VLLM_API_KEY # temperature: 0 # model_info: # mode: chat # alias: summarizer -> gen (repointed 2026-08-12, granite retired) (operator 2026-06-19). Duplicate-entry alias # (not router_settings.model_group_alias — that's hidden from /v1/models and can be # silently ignored in config per litellm #15020/#5524). Keep api_base in sync above. - model_name: summarizer litellm_params: model: hosted_vllm/qwen3.8-27b-uncensored api_base: http://10.250.50.54:8015/v1 api_key: os.environ/VLLM_API_KEY temperature: 0 extra_body: chat_template_kwargs: enable_thinking: false model_info: mode: chat # alias: classifier -> gen (repointed 2026-08-12, granite retired) (operator 2026-06-19). Light/fast classification # + triage endpoint; same backend as summarizer. Keep api_base in sync above. - model_name: classifier litellm_params: model: hosted_vllm/qwen3.8-27b-uncensored api_base: http://10.250.50.54:8015/v1 api_key: os.environ/VLLM_API_KEY temperature: 0 extra_body: chat_template_kwargs: enable_thinking: false model_info: mode: chat # alias: summarizer-large -> gen / qwen3.8-27b-uncensored (operator 2026-07-05). For heavier # summarization that wants the 35B-A3B heretic `gen` model instead of granite-8b. Thinking OFF # (matches gen). Keep api_base (:8015) + enable_thinking in sync with the gen record below. - model_name: summarizer-large litellm_params: model: hosted_vllm/qwen3.8-27b-uncensored api_base: http://10.250.50.54:8015/v1 api_key: os.environ/VLLM_API_KEY presence_penalty: 1.5 temperature: 0.7 top_p: 0.8 extra_body: top_k: 20 chat_template_kwargs: enable_thinking: false model_info: mode: chat # --- image-judge / qwen-image-bench — T2I quality JUDGE aliases. The dedicated # Qwen-Image-Bench NVFP4 backend (ana-ml2 GPU 1, :8014) was RETIRED 2026-07-15 # (operator: reclaim ~30GB GPU1) after the arbo→gen hero-judge switch. Both # aliases now REPOINT to the gen backend (:8015, qwen3.8-27b-uncensored, # vision-intact), held at deterministic judge sampling (temp 0 / top_k 1) with # enable_thinking:false (a reasoning preamble breaks json_object). Revert = # `docker compose start` stacks/qwen-image-bench on ana-ml2 + repoint api_base # back to :8014 + model hosted_vllm/qwen-image-bench. --- - model_name: qwen-image-bench litellm_params: model: hosted_vllm/qwen3.8-27b-uncensored api_base: http://10.250.50.54:8015/v1 api_key: os.environ/VLLM_API_KEY temperature: 0 top_p: 1.0 extra_body: top_k: 1 repetition_penalty: 1.05 chat_template_kwargs: enable_thinking: false model_info: mode: chat - model_name: image-judge litellm_params: model: hosted_vllm/qwen3.8-27b-uncensored api_base: http://10.250.50.54:8015/v1 api_key: os.environ/VLLM_API_KEY temperature: 0 top_p: 1.0 extra_body: top_k: 1 repetition_penalty: 1.05 chat_template_kwargs: enable_thinking: false model_info: mode: chat # --- Qwen3.6-35B-A3B heretic (llmfan46, uncensored, NVFP4 ModelOpt Experts-Only, VISION-INTACT) # — the general / `gen` model on ana-ml2 GPU 0, at /tank/aimodels/qwen36-35b-a3b-heretic-nvfp4. # Displaced AEON-27B 2026-07-08 (which had displaced qwopus3.5-122b 2026-07-05). MoE 35B-A3B # (256 experts / 8 active), qwen3_5_moe GDN-hybrid, native MTP preserved but served MTP-OFF # (spec-decode hurts concurrent aggregate). Served on :8015 via vLLM, # served-name qwen3.8-27b-uncensored. Thinking split = chat_template_kwargs.enable_thinking + # --reasoning-parser qwen3; tool-calling qwen3_coder. gen / gen-reasoning + summarizer- # large route here; -reasoning enables thinking. Keep api_base (:8015) in sync. # RETIRED with the displacement (→ 404, callers migrate to gen): qwen3.5-122-a10b # [-reasoning] + qwen-large[-reasoning] — they named a 122B that no longer exists; # aliasing a 27B under those is the naming footgun the qwen36-vl stack warns against. # presence_penalty: 1.5 — Qwen3.6 README anti-repetition rec for BOTH non-thinking and # thinking (dvalin-smithy canonical 2026-07-08, validated vs Qwen guidance). gen # non-thinking temp 0.7/top_p 0.8; gen-reasoning thinking temp 1.0/top_p 0.95 (the # GENERAL thinking profile, not the 0.6 coding sub-profile). docs/pfi/model-sampler-defaults.md. --- # CANONICAL Qwen3.8 INSTRUCT (non-thinking) sampling set, verified 2026-08-16 # against BOTH upstreams, which are byte-identical on this: # Qwen/Qwen3.8-27B card "Best Practices" §1 and unsloth/Qwen3.8-27B §1 # temperature=0.7, top_p=0.80, top_k=20, min_p=0.0, # presence_penalty=1.5, repetition_penalty=1.0 # # ⚠️ presence_penalty=1.5 is canonical BUT is the one value upstream itself # hedges on, verbatim: "you can adjust the presence_penalty parameter between # 0 and 2 to reduce endless repetition. However, using a higher value may # occasionally result in LANGUAGE MIXING and a slight decrease in model # performance." 1.5 sits high in that 0-2 band. If short/degraded replies # reappear on long multi-turn conversations, THIS is the first dial to move # (try 0.0-0.5) — operator's own hypothesis 2026-08-16, and upstream's caveat # supports it. Left at canonical for now so the baseline is defensible rather # than hand-tuned. - model_name: gen litellm_params: model: hosted_vllm/qwen3.8-27b-uncensored api_base: http://10.250.50.54:8015/v1 api_key: os.environ/VLLM_API_KEY presence_penalty: 1.5 temperature: 0.7 top_p: 0.8 extra_body: top_k: 20 min_p: 0.0 repetition_penalty: 1.0 chat_template_kwargs: enable_thinking: false model_info: mode: chat - model_name: gen-reasoning litellm_params: # Distinct served-name so a thinking-off `gen` request can't mutate this deployment's # enable_thinking (shared-config-mutation footgun). Same backend :8015, different model id. model: hosted_vllm/qwen3.8-27b-uncensored-thinking api_base: http://10.250.50.54:8015/v1 api_key: os.environ/VLLM_API_KEY # CANONICAL Qwen3.8 THINKING sampling set (Qwen + unsloth "Best Practices" # §1, identical in both): temperature=1.0, top_p=0.95, top_k=20, # min_p=0.0, presence_penalty=0.0, repetition_penalty=1.0. # # ⚠️ presence_penalty was 1.5 here until 2026-08-16 — the INSTRUCT-mode # value applied to a THINKING deployment. Canonical for thinking mode is # 0.0, and upstream warns a high presence_penalty can cause language # mixing and degrade performance. Corrected to 0.0. presence_penalty: 0.0 temperature: 1.0 top_p: 0.95 extra_body: top_k: 20 min_p: 0.0 repetition_penalty: 1.0 chat_template_kwargs: enable_thinking: true model_info: mode: chat # char-rp -> MeroMero-v2 NON-THINKING prose seat (:8016, vLLM, meromero-charrp stack on # ana-ml2 GPU 0). G4-MeroMero-v2-31B NVFP4A16, **Gemma-4 base** (google/gemma-4-31B-it), # 256K ctx, in-house quant. Replaced the GGUF/llama.cpp Magidonia-24B seat 2026-08-12. # # THIS SEAT EXISTS BECAUSE THE QWEN BASE THINKS INCESSANTLY. char-rp-reasoning is a # Qwen3.x derivative and emits ~5-6k chars of CoT per turn no matter which Qwen RP tune # is loaded — that is the base family, not the finetune, and no swap within it fixes it # (measured 2026-08-16: Dark-Scarlett 6036 ch vs Fable-Fusion 5323 ch on the same # prompts). Gemma-4 gives a genuinely non-thinking prose seat. Reach for THIS one when # you want prose without a reasoning trace; reach for char-rp-reasoning when you want # the deliberation. Best-of-breed per seat — deliberately NOT the same model. # # Serving flags are load-bearing (commit b8f0f4c): `--tool-call-parser gemma4 # --enable-auto-tool-choice --reasoning-parser gemma4` AND # `--default-chat-template-kwargs '{"enable_thinking": false}'`. That last flag is # MANDATORY, not decorative — the gemma4 parser defaults enable_thinking to True, which # pre-initialises the engine to REASONING and returns null `content` for all plain RP # prose. Before it was set, every tools-bearing request also 400'd (no parser at all). # Verified 0 chars reasoning / clean prose end-to-end 2026-08-16. # # Sampler note: the temp 1.1 / min_p 0.10 / top_k 0 values below were A/B-tuned # 2026-07-08 against the retired Mistral-family Magidonia seat, NOT against MeroMero. # They have not been re-tuned for Gemma-4 — treat as inherited, not canonical. # Callers may override. docs/pfi/model-sampler-defaults.md; stacks/meromero-charrp/. - model_name: char-rp litellm_params: model: hosted_vllm/char-rp api_base: http://10.250.50.54:8016/v1 api_key: os.environ/VLLM_API_KEY temperature: 1.1 top_p: 0.95 extra_body: min_p: 0.10 top_k: 0 model_info: mode: chat # char-rp-reasoning -> GGUF managed-REASONING seat (:8018, llama.cpp, char-rp-gguf stack). # Qwen3.5-27B-Deckard-PKD-Heretic-Uncensored-Thinking i1-Q5_K_M — DavidAU creative tune. # Reasoning ON server-side (--reasoning on): CoT surfaces in reasoning_content, content stays # clean prose, budget-capped. DRY server-side (sampler order = dry after temperature) tames looping. # A/B WINNER 2026-07-08: 0/30 loops + 0/30 refusals; beat RpR-v4 (1/30 loop, forbids DRY), # Pantheon-27B (7/30 explicit refusals), Snowdrop + Gembrain (llama.cpp template-incompat). # Deckard decode: temp 1.0, top_p 0.95, top_k 40, min_p 0.05 (dvalin-CONFIRMED canonical 2026-07-08; # NO presence/rep penalty; DRY 0.8 server-side). Tuning ladder: flat prose→min_p 0.08, loops→DRY 0.9, # over-damped→DRY 0.6/off. Do NOT import RpR/QwQ sampler rules (different family). NOT the same model as # char-rp (best-of-breed per seat) — see stacks/char-rp-gguf/README.md. # ⚠️ TEMPORARY REPOINT 2026-08-16 (operator-directed evaluation window). # char-rp-reasoning currently resolves to FABLE-FUSION 711 on :8019, NOT to # Dark-Scarlett. DS v1.0 is DOWN — GPU1 is zero-sum and Fable-Fusion occupies # her slot. This is a deliberate, explicit substitution for hands-on testing; # it is NOT a silent alias swap, and it is not the permanent seat decision. # Address the seat unambiguously as `char-rp-fable` below; `char-rp-reasoning` # is kept live only so existing consumers keep working during the window. # # WHY: DS v1.0 is a plain finetune of stock Qwen3.6-27B with NO abliteration, # so cold prompts revert to safety-tuned base behaviour. Measured three-arm # A/B (services/refusal-probe/): under a bare instruction with no character # card, DS refuses 92.5% (37/40) and Fable-Fusion 15.8% (6/38); with a # character card both sit at ~0%. Fable-Fusion is Heretic-abliterated. # # ROLLBACK (restores Dark-Scarlett): # ssh infra-ops@10.250.50.54 'cd /opt/docker/compose/fablefusion-charrp-probe && sudo docker compose down' # ssh infra-ops@10.250.50.54 'cd /opt/docker/compose/darkscarlett-charrp-reasoning && sudo docker compose up -d' # then revert this block to api_base :8018 / model hosted_vllm/char-rp-reasoning # and restart litellm (~52s). # # Samplers below are the model card's thinking-mode recommendation (temp 1.0 / # top_p 0.95 / top_k 20) and are unchanged from the DS entry. Verified the FF # chat template honours `enable_thinking` (chat_template.jinja:44) rather than # ignoring it — the mismatch that returned null content on the MeroMero seat. - model_name: char-rp-reasoning litellm_params: model: hosted_vllm/char-rp-probe api_base: http://10.250.50.54:8019/v1 api_key: os.environ/VLLM_API_KEY temperature: 1.0 top_p: 0.95 extra_body: top_k: 20 chat_template_kwargs: enable_thinking: true model_info: mode: chat # char-rp-fable -> the SAME Fable-Fusion seat under its own honest name, so the # evaluation can address it without relying on the temporary repoint above. # Distinct model_name = distinct litellm_params object, which avoids the # shared-deployment param mutation that bleeds sampler overrides between # variants. - model_name: char-rp-fable litellm_params: model: hosted_vllm/char-rp-probe api_base: http://10.250.50.54:8019/v1 api_key: os.environ/VLLM_API_KEY temperature: 1.0 top_p: 0.95 extra_body: top_k: 20 chat_template_kwargs: enable_thinking: true model_info: mode: chat # --- Selene 1 Mini 8B (AtlaAI judge, FP8) — restored on GPU1 after the # llama-swap teardown (was the Q6_K GGUF in the swap zoo). vLLM dynamic fp8, # :8011. Explicit entry shadows the "*" wildcard (which used to reach it via # llama-swap). Hallucination/RAG-faithfulness judge; callers set temp ~0.01. --- - model_name: selene-1-mini-8b litellm_params: model: hosted_vllm/selene-1-mini-8b api_base: http://10.250.50.54:8011/v1 api_key: os.environ/VLLM_API_KEY temperature: 0.6 top_p: 0.9 model_info: mode: chat # --- Qwen3 embeddings --- - model_name: qwen3-embedding litellm_params: model: hosted_vllm/Qwen/Qwen3-Embedding-0.6B api_base: http://10.250.50.54:8001/v1 api_key: os.environ/VLLM_API_KEY model_info: mode: embedding # --- qwen3-reranker RETIRED 2026-08-20. It named Qwen3-Reranker-0.6B on :8002, # the incumbent the R43 bake-off replaced on 2026-08-06 after measuring it # HARMING 80/90 fleet queries (no-reranker beat it 89/90 vs 56/90). The alias # was kept as the rollback path and, for 13 days, was the ONLY reranker # actually receiving traffic: nevermore was pinned to it by name, so the # cutover moved `reranker` but never moved nevermore. Fixed at the consumer # (nevermore now pins `reranker`), then the seat and this alias were retired. # Use `reranker` -> bge-reranker-v2-m3 :8013. --- # --- Worldtree capability aliases (role→capability gateway swaps, ADR-0012). # Stable role-named aliases so consumers bind the CAPABILITY, not a concrete # model; swap the backing model here and callers are unaffected. NO generic # `embedding` alias ON PURPOSE — embedding vectors are model-specific (not # swap-transparent), so that capability stays `qwen3-embedding` above. --- # chat-judge → generative LLM-as-judge (Selene); WT selene-judgment role. - model_name: chat-judge litellm_params: model: hosted_vllm/selene-1-mini-8b api_base: http://10.250.50.54:8011/v1 api_key: os.environ/VLLM_API_KEY temperature: 0.6 top_p: 0.9 model_info: mode: chat # reranker → generic capability name for rerank. THE reranker alias — the only # one left as of 2026-08-20. Every consumer pins this name, never a model name. - model_name: reranker litellm_params: model: hosted_vllm/BAAI/bge-reranker-v2-m3 api_base: http://10.250.50.54:8013/v1 api_key: os.environ/VLLM_API_KEY model_info: mode: rerank # --- coder-fast → Qwen2.5-Coder-1.5B (BASE), FIM code-completion seat (ana-ml2 # GPU1 :8020, vLLM; deep-research pick 2026-07-27). For Zed editor inline # edit-predictions via the LEGACY /v1/completions endpoint with Qwen FIM # markers (<|fim_prefix|>/<|fim_suffix|>/<|fim_middle|>). BASE not -Instruct # (FIM is a pretraining objective; base completions are cleaner). Apache-2.0. # mode: completion — this is text-completion, not chat. Reached KEYLESS from # Vuong's Mac via the zed-fim-proxy (separate port on ana-docker) which injects # a coder-fast-scoped virtual key; the proxy's model-allowlist + the scoped key # bound the blast radius. Runner-up was Qwen2.5-Coder-3B (higher HumanEval-FIM, # non-commercial Qwen-Research license). --- - model_name: coder-fast litellm_params: model: hosted_vllm/qwen2.5-coder-1.5b api_base: http://10.250.50.54:8020/v1 api_key: os.environ/VLLM_API_KEY model_info: mode: completion # --- z.ai GLM (cloud API) — fronted for unified logging across local # + cloud inference. Explicit entries, so they win over the "*" # wildcard below (no collision with llama-swap's glm4.7-flash etc. # — different model IDs). NOTE: paid API; only gateway-keyed callers # can reach these, but they DO spend z.ai credits. Key in .env. --- # glm-5.1: thinking DISABLED by default (2026-06-11, operator call). LiteLLM # strips a top-level `thinking` param (drop_params), but forwards `extra_body` # verbatim to z.ai, where the native thinking:{type:disabled} control lands — # verified reasoning_tokens→0. Reasoning is opt-in via glm-5.1-reasoning below. - model_name: glm-5.1 litellm_params: model: openai/glm-5.1 api_base: https://api.z.ai/api/coding/paas/v4 api_key: os.environ/Z_AI_API_KEY temperature: 1.0 top_p: 0.95 extra_body: thinking: type: disabled # glm-5.1-reasoning: identical upstream, thinking ENABLED (opt-in reasoning). - model_name: glm-5.1-reasoning litellm_params: model: openai/glm-5.1 api_base: https://api.z.ai/api/coding/paas/v4 api_key: os.environ/Z_AI_API_KEY temperature: 1.0 top_p: 0.95 extra_body: thinking: type: enabled # glm-5.2 (released ~2026-06; canonical z.ai id `glm-5.2`, confirmed via /models + # a live completion with our key). Mirrors the glm-5.1 pattern: thinking DISABLED # by default (consistency with the 2026-06-11 operator call), opt-in reasoning via # glm-5.2-reasoning. extra_body.thinking is forwarded verbatim to z.ai. # CANONICAL LIMITS (probed live vs z.ai 2026-07-05): 1,048,576-token (1M, 2^20) # INPUT context; 131,072 (128K) MAX OUTPUT (z.ai max_tokens range [1,131072]). # NO gateway-side cap — pure z.ai passthrough, so these are the effective limits. - model_name: glm-5.2 litellm_params: model: openai/glm-5.2 api_base: https://api.z.ai/api/coding/paas/v4 api_key: os.environ/Z_AI_API_KEY temperature: 1.0 top_p: 0.95 extra_body: thinking: type: disabled - model_name: glm-5.2-reasoning litellm_params: model: openai/glm-5.2 api_base: https://api.z.ai/api/coding/paas/v4 api_key: os.environ/Z_AI_API_KEY temperature: 1.0 top_p: 0.95 extra_body: thinking: type: enabled # gen-frontier / gen-frontier-reasoning: capability aliases for the PAID # frontier tier (GLM 5.2 @ z.ai), mirroring glm-5.2 / glm-5.2-reasoning # (thinking off / on). Worldtree binds these for frontier-grade generation # / reasoning; swap the backing frontier model here, callers unaffected. # PAID — only all-proxy-models / explicitly-scoped keys reach them; the free # all-agents-local key is fenced off z.ai spend and cannot. - model_name: gen-frontier litellm_params: model: openai/glm-5.2 api_base: https://api.z.ai/api/coding/paas/v4 api_key: os.environ/Z_AI_API_KEY temperature: 1.0 top_p: 0.95 extra_body: thinking: type: disabled - model_name: gen-frontier-reasoning litellm_params: model: openai/glm-5.2 api_base: https://api.z.ai/api/coding/paas/v4 api_key: os.environ/Z_AI_API_KEY temperature: 1.0 top_p: 0.95 extra_body: thinking: type: enabled - model_name: glm-5-turbo litellm_params: model: openai/glm-5-turbo api_base: https://api.z.ai/api/coding/paas/v4 api_key: os.environ/Z_AI_API_KEY temperature: 1.0 top_p: 0.95 - model_name: glm-4.7 litellm_params: model: openai/glm-4.7 api_base: https://api.z.ai/api/coding/paas/v4 api_key: os.environ/Z_AI_API_KEY temperature: 1.0 top_p: 0.95 - model_name: glm-4.5-air litellm_params: model: openai/glm-4.5-air api_base: https://api.z.ai/api/coding/paas/v4 api_key: os.environ/Z_AI_API_KEY temperature: 0.6 top_p: 0.95 # --- Kimi K3 — CODING endpoint (Kimi Code / Vivace membership). THE PRIMARY # Kimi arm the Heid cross-frontier panel plan uses. OpenAI-compatible base # https://api.kimi.com/coding/v1 → openai/ provider, upstream model id `k3` # (1M-context; the coding lineup also carries k3-256k, kimi-for-coding, # kimi-for-coding-highspeed — ids confirmed live via /models 2026-07-25). # PAID (Vivace subscription); key KIMI_CODE_API_KEY in .env. CONSTRAINT # (verified live 2026-07-25): k3 accepts ONLY temperature=1 — any other value # 400s ("only 1 is allowed for this model") — so it is pinned here; callers # must NOT override it. k3 is also a REASONING model (thinking-effort tiers # low/high/max per Kimi Code docs): CoT returns in `reasoning_content`, the # answer in `content` — give it adequate max_tokens or content returns EMPTY # (reasoning eats a tiny budget). --- - model_name: kimi-k3 litellm_params: model: openai/k3 api_base: https://api.kimi.com/coding/v1 api_key: os.environ/KIMI_CODE_API_KEY temperature: 1 model_info: mode: chat # --- Kimi K3 — GENERAL Moonshot API endpoint (https://api.moonshot.ai/v1), # kept as the `-gen-api` variant. The plan uses the CODING endpoint above; # this is the general-platform route (originally wired then demoted when the # coding endpoint became canonical). OpenAI-compatible, upstream `kimi-k3`, # key MOONSHOT_API_KEY. Same temperature=1 + reasoning-model constraints as # the coding k3 (verified live through the gateway 2026-07-25, 17+25→"42"). --- - model_name: kimi-k3-gen-api litellm_params: model: openai/kimi-k3 api_base: https://api.moonshot.ai/v1 api_key: os.environ/MOONSHOT_API_KEY temperature: 1 model_info: mode: chat # --- (removed 2026-06-20, operator call) the `*` wildcard → llama-swap # (ana-ml2:9292). llama-swap is decommissioned (:9292 confirmed down), so # the wildcard routed every unmatched / typo'd / stale model name to a DEAD # backend → a misleading "Connection error" instead of a clean "model not # found". This is the footgun that silently swallowed Worldtree's defunct # model names. Removed so unknown models now fail loudly (404). Re-add an # explicit per-model entry if a swappable zoo ever returns. --- # --- lfm2.5-2.6b -> RETIRED PERMANENTLY 2026-08-20 (operator directive). The # LiquidAI LFM2.5-2.6B seat (ana-ml2 GPU1 :8021) was an EVAL-ONLY bake-off # against granite-4.1-8b that never got its operator ruling; its comparator # was retired 2026-08-15 and spend logs showed 0 calls in the 4 days to # 2026-08-21. Container removed, service deleted from stacks/vllm. The alias # is deleted rather than repointed so the name 404s cleanly. --- general_settings: master_key: os.environ/LITELLM_MASTER_KEY database_url: os.environ/DATABASE_URL store_model_in_db: true # THE log switch: persists full request messages + response bodies into # SpendLogs so they render in the Logs UI. Without this you get metadata # (tokens, latency, model) but not the prompt/completion text. # # ⚠️ TURNED OFF 2026-08-16 (operator: "I don't need any of that information"). # With this TRUE the SpendLogs table stored every prompt+completion body and # grew to 6.0 GB (of a 6.08 GB DB). Off = lightweight cost/usage rows only # (tokens, latency, model, cost) — the cross-project spend tracking survives, # the bulky bodies do not. Re-enable ONLY for a bounded debugging window, not # standing. store_prompts_in_spend_logs: false # HARD CAP on SpendLogs growth (operator: "if there's a way to cap it, CAP # it"). The retention job deletes rows older than the period on the interval # cadence, so the table is bounded by ~7 days of lightweight rows rather than # unbounded. Names verified against LiteLLM docs (proxy/spend_logs_deletion). maximum_spend_logs_retention_period: "7d" maximum_spend_logs_retention_interval: "1d" # scalar-judge → Skywork-Reward-V2 (scalar reward model; vLLM pooling on # ana-ml2:8003). LiteLLM has no reward/pooling MODE, so this is a passthrough, # not a model_list alias. Gateway-key-gated. Consumers POST the reward body to # /scalar-judge/ (e.g. /pooling or /classify), forwarded to :8003. # SWAP-SENSITIVE: a different reward model shifts the score scale, so consumers # must recalibrate thresholds after a backing swap. pass_through_endpoints: - path: "/scalar-judge" target: "http://10.250.50.54:8003" forward_headers: true include_subpath: true litellm_settings: # vLLM rejects some OpenAI params other backends accept; drop silently # rather than 400 the caller. drop_params: true # Custom pre-call hook: strip an empty `tools: []` (+ orphaned tool_choice) # before forwarding upstream. vLLM 400s on empty tools arrays ("tools must # not be an empty array"); drop_params doesn't catch empty VALUES, only # unsupported params. Runs on every request → fixes it for all vLLM models. # File mounted at /app/strip_empty_tools.py; reference is module.instance, # resolved relative to this config's directory. callbacks: ["strip_empty_tools.strip_empty_tools_instance"] # Langfuse trace export RETIRED 2026-06-20 (operator). Its ClickHouse member spewed # ~94 GB of unrotated logs and filled ana-docker's disk; the trace UI was redundant # with LiteLLM's native spend_logs (store_prompts_in_spend_logs: true → full # prompts/responses/tokens/cost at :4000/ui). Gateway observability stays fully # native. Re-add success_callback/failure_callback here if Langfuse ever returns.