esh-pfi-infrastructure/stacks/litellm/conf/config.yaml

# LiteLLM gateway config — fronts the vLLM services on ana-ml2
# (10.250.50.54) and logs every request + response so they're
# inspectable in the Logs UI at http://10.250.50.70:4000/ui.
#
# Deploys to /opt/docker/conf/litellm/config.yaml (mounted read-only
# into the container at /app/config.yaml).
#
# Model-name → upstream mapping:
#   phi4-mini        → vLLM :8004  (generative chat)
#   qwen3-embedding  → vLLM :8001  (/v1/embeddings)
#   qwen3-reranker   → vLLM :8002  (/rerank)
#   *  (wildcard)    → llama-swap :9292  (the swappable generative zoo)
#
# The wildcard fronts llama-swap so its whole model zoo logs through the
# gateway without per-model registration. The vllm-reward classifier
# (:8003) is a pooling /classify endpoint with no first-class LiteLLM
# route — left direct; see README.

model_list:
  # --- Granite 4.1 8B (generative chat) — production summarizer + dreaming
  #     agent. Replaced phi4-mini 2026-06-05 (beat it on precision in brokkr's
  #     R15 P03 eval). vLLM on ana-ml2 GPU 1, official FP8, 50K ctx. Explicit
  #     entry shadows the "*" wildcard's llama-swap route for this name. Full
  #     prompt + completion captured per call. ---
  - model_name: granite-4.1-8b
    litellm_params:
      model: hosted_vllm/granite-4.1-8b
      api_base: http://10.250.50.54:8004/v1
      api_key: os.environ/VLLM_API_KEY
      temperature: 0
    model_info:
      mode: chat

  # alias: summarizer -> granite-4.1-8b (operator 2026-06-19). Duplicate-entry alias
  #   (not router_settings.model_group_alias — that's hidden from /v1/models and can be
  #   silently ignored in config per litellm #15020/#5524). Keep api_base in sync above.
  - model_name: summarizer
    litellm_params:
      model: hosted_vllm/granite-4.1-8b
      api_base: http://10.250.50.54:8004/v1
      api_key: os.environ/VLLM_API_KEY
      temperature: 0
    model_info:
      mode: chat

  # alias: classifier -> granite-4.1-8b (operator 2026-06-19). Light/fast classification
  #   + triage endpoint; same backend as summarizer. Keep api_base in sync above.
  - model_name: classifier
    litellm_params:
      model: hosted_vllm/granite-4.1-8b
      api_base: http://10.250.50.54:8004/v1
      api_key: os.environ/VLLM_API_KEY
      temperature: 0
    model_info:
      mode: chat

  # alias: summarizer-large -> gen / qwen3.6-27b-aeon (operator 2026-07-05). For heavier
  #   summarization that wants the AEON 27B `gen` model instead of granite-8b. Thinking OFF
  #   (matches gen). Keep api_base (:8015) + enable_thinking in sync with the gen record below.
  - model_name: summarizer-large
    litellm_params:
      model: hosted_vllm/qwen3.6-27b-aeon
      api_base: http://10.250.50.54:8015/v1
      api_key: os.environ/VLLM_API_KEY
      presence_penalty: 1.5
      temperature: 0.7
      top_p: 0.8
      extra_body:
        top_k: 20
        chat_template_kwargs:
          enable_thinking: false
    model_info:
      mode: chat

  # --- Qwen-Image-Bench (NVFP4) — text-to-image quality JUDGE, vision-intact. vLLM
  #     on ana-ml2 GPU 1, :8014. REPLACED qwen3.6-35b-a3b (operator 2026-06-19; the
  #     qwen36 VL judge was downed + its FP8 weights freed ~33GB). Multimodal — image
  #     + prompt in, a 5-dim verdict out (overall quality / prompt match / aesthetic /
  #     LoRA activation / confidence). image-judge is a duplicate-entry alias.
  #     ⚠️ arbo's hero-judge was bound to qwen3.6-35b-a3b — comfy-dev notified to
  #     repoint (different I/O than the old general VL judge). ---
  - model_name: qwen-image-bench
    litellm_params:
      model: hosted_vllm/qwen-image-bench
      api_base: http://10.250.50.54:8014/v1
      api_key: os.environ/VLLM_API_KEY
      temperature: 0
      top_p: 1.0
      extra_body:
        top_k: 1
        repetition_penalty: 1.05
    model_info:
      mode: chat
  - model_name: image-judge
    litellm_params:
      model: hosted_vllm/qwen-image-bench
      api_base: http://10.250.50.54:8014/v1
      api_key: os.environ/VLLM_API_KEY
      temperature: 0
      top_p: 1.0
      extra_body:
        top_k: 1
        repetition_penalty: 1.05
    model_info:
      mode: chat

  # --- Qwen3.6-27B AEON (uncensored/abliterated, NVFP4 ModelOpt, VISION-INTACT image+video)
  #     — the general / `gen` model on ana-ml2 GPU 0. REPLACED qwopus3.5-122b 2026-07-05
  #     (operator: displace qwopus, this assumes gen/gen-reasoning). Dense 27B, qwen3_5
  #     GDN-hybrid (qwopus's little sibling), native MTP head. Served on :8015 via vLLM,
  #     served-name qwen3.6-27b-aeon, MTP OFF (spec-decode hurts concurrent aggregate — the
  #     MTP twin is char-rp below). Thinking split = chat_template_kwargs.enable_thinking +
  #     --reasoning-parser qwen3; tool-calling qwen3_coder. gen / gen-reasoning + summarizer-
  #     large route here; -reasoning enables thinking. Keep api_base (:8015) in sync.
  #     RETIRED with the displacement (→ 404, callers migrate to gen): qwen3.5-122-a10b
  #     [-reasoning] + qwen-large[-reasoning] — they named a 122B that no longer exists;
  #     aliasing a 27B under those is the naming footgun the qwen36-vl stack warns against.
  #     presence_penalty: 1.5 — Qwen3.6 README anti-repetition rec for BOTH non-thinking and
  #     thinking (dvalin-smithy canonical 2026-07-08, validated vs Qwen guidance). gen
  #     non-thinking temp 0.7/top_p 0.8; gen-reasoning thinking temp 1.0/top_p 0.95 (the
  #     GENERAL thinking profile, not the 0.6 coding sub-profile). docs/pfi/model-sampler-defaults.md. ---
  - model_name: gen
    litellm_params:
      model: hosted_vllm/qwen3.6-27b-aeon
      api_base: http://10.250.50.54:8015/v1
      api_key: os.environ/VLLM_API_KEY
      presence_penalty: 1.5
      temperature: 0.7
      top_p: 0.8
      extra_body:
        top_k: 20
        chat_template_kwargs:
          enable_thinking: false
    model_info:
      mode: chat
  - model_name: gen-reasoning
    litellm_params:
      # Distinct served-name so a thinking-off `gen` request can't mutate this deployment's
      # enable_thinking (shared-config-mutation footgun). Same backend :8015, different model id.
      model: hosted_vllm/qwen3.6-27b-aeon-thinking
      api_base: http://10.250.50.54:8015/v1
      api_key: os.environ/VLLM_API_KEY
      presence_penalty: 1.5
      temperature: 1.0
      top_p: 0.95
      extra_body:
        top_k: 20
        chat_template_kwargs:
          enable_thinking: true
    model_info:
      mode: chat
  # char-rp -> GGUF dark-romantasy PROSE seat (:8016, llama.cpp, char-rp-gguf stack on
  #   ana-ml2 GPU 0). TheDrummer Magidonia-24B-v4.3 Q6_K — Magistral (Mistral) RP tune.
  #   NON-thinking: elite literary prose, zero refusal on dark/explicit scenes, ~65 tok/s,
  #   tight POV/instruction adherence (live-tested 2026-07-08). Replaced the broken Angel
  #   NVFP4 serve AND the earlier AEON-rp MTP twin. Sampler A/B-tuned 2026-07-08 vs the
  #   dvalin-smithy canonical: temp 1.1 / top_p 0.95 / min_p 0.10 / top_k 0, NO repetition
  #   penalty. min_p 0.10 richened imagery vs 0.03; rep 1.05 REJECTED (injected a stray
  #   markdown title in a grief scene — rep-style penalties hurt Drummer RP, matching the
  #   card). No enable_thinking kwarg — meaningless to the Mistral template. Alt prose model
  #   (swap via the stack .env): MS3.2-PaintedFantasy-v4.1-24B. Callers may override.
  #   docs/pfi/model-sampler-defaults.md.
  - model_name: char-rp
    litellm_params:
      model: hosted_vllm/magidonia-24b-v4.3
      api_base: http://10.250.50.54:8016/v1
      api_key: os.environ/VLLM_API_KEY
      temperature: 1.1
      top_p: 0.95
      extra_body:
        min_p: 0.10
        top_k: 0
    model_info:
      mode: chat
  # char-rp-reasoning -> GGUF managed-REASONING seat (:8018, llama.cpp, char-rp-gguf stack).
  #   Qwen3.5-27B-Deckard-PKD-Heretic-Uncensored-Thinking i1-Q5_K_M — DavidAU creative tune.
  #   Reasoning ON server-side (--reasoning on): CoT surfaces in reasoning_content, content stays
  #   clean prose, budget-capped. DRY server-side (sampler order = dry after temperature) tames looping.
  #   A/B WINNER 2026-07-08: 0/30 loops + 0/30 refusals; beat RpR-v4 (1/30 loop, forbids DRY),
  #   Pantheon-27B (7/30 explicit refusals), Snowdrop + Gembrain (llama.cpp template-incompat).
  #   Deckard decode: temp 1.0, top_p 0.95, top_k 40, min_p 0.05 (dvalin-CONFIRMED canonical 2026-07-08;
  #   NO presence/rep penalty; DRY 0.8 server-side). Tuning ladder: flat prose→min_p 0.08, loops→DRY 0.9,
  #   over-damped→DRY 0.6/off. Do NOT import RpR/QwQ sampler rules (different family). NOT the same model as
  #   char-rp (best-of-breed per seat) — see stacks/char-rp-gguf/README.md.
  - model_name: char-rp-reasoning
    litellm_params:
      model: hosted_vllm/deckard-pkd-27b
      api_base: http://10.250.50.54:8018/v1
      api_key: os.environ/VLLM_API_KEY
      temperature: 1.0
      top_p: 0.95
      extra_body:
        top_k: 40
        min_p: 0.05
    model_info:
      mode: chat

  # --- Selene 1 Mini 8B (AtlaAI judge, FP8) — restored on GPU1 after the
  #     llama-swap teardown (was the Q6_K GGUF in the swap zoo). vLLM dynamic fp8,
  #     :8011. Explicit entry shadows the "*" wildcard (which used to reach it via
  #     llama-swap). Hallucination/RAG-faithfulness judge; callers set temp ~0.01. ---
  - model_name: selene-1-mini-8b
    litellm_params:
      model: hosted_vllm/selene-1-mini-8b
      api_base: http://10.250.50.54:8011/v1
      api_key: os.environ/VLLM_API_KEY
      temperature: 0.6
      top_p: 0.9
    model_info:
      mode: chat

  # --- Qwen3 embeddings ---
  - model_name: qwen3-embedding
    litellm_params:
      model: hosted_vllm/Qwen/Qwen3-Embedding-0.6B
      api_base: http://10.250.50.54:8001/v1
      api_key: os.environ/VLLM_API_KEY
    model_info:
      mode: embedding

  # --- Qwen3 reranker (proxy /rerank route) ---
  - model_name: qwen3-reranker
    litellm_params:
      model: hosted_vllm/Qwen/Qwen3-Reranker-0.6B
      api_base: http://10.250.50.54:8002/v1
      api_key: os.environ/VLLM_API_KEY
    model_info:
      mode: rerank

  # --- Worldtree capability aliases (role→capability gateway swaps, ADR-0012).
  #     Stable role-named aliases so consumers bind the CAPABILITY, not a concrete
  #     model; swap the backing model here and callers are unaffected. NO generic
  #     `embedding` alias ON PURPOSE — embedding vectors are model-specific (not
  #     swap-transparent), so that capability stays `qwen3-embedding` above. ---
  # chat-judge → generative LLM-as-judge (Selene); WT selene-judgment role.
  - model_name: chat-judge
    litellm_params:
      model: hosted_vllm/selene-1-mini-8b
      api_base: http://10.250.50.54:8011/v1
      api_key: os.environ/VLLM_API_KEY
      temperature: 0.6
      top_p: 0.9
    model_info:
      mode: chat
  # reranker → generic capability name for rerank (currently qwen3-reranker).
  - model_name: reranker
    litellm_params:
      model: hosted_vllm/Qwen/Qwen3-Reranker-0.6B
      api_base: http://10.250.50.54:8002/v1
      api_key: os.environ/VLLM_API_KEY
    model_info:
      mode: rerank

  # --- z.ai GLM (cloud API) — fronted for unified logging across local
  #     + cloud inference. Explicit entries, so they win over the "*"
  #     wildcard below (no collision with llama-swap's glm4.7-flash etc.
  #     — different model IDs). NOTE: paid API; only gateway-keyed callers
  #     can reach these, but they DO spend z.ai credits. Key in .env. ---
  # glm-5.1: thinking DISABLED by default (2026-06-11, operator call). LiteLLM
  #   strips a top-level `thinking` param (drop_params), but forwards `extra_body`
  #   verbatim to z.ai, where the native thinking:{type:disabled} control lands —
  #   verified reasoning_tokens→0. Reasoning is opt-in via glm-5.1-reasoning below.
  - model_name: glm-5.1
    litellm_params:
      model: openai/glm-5.1
      api_base: https://api.z.ai/api/coding/paas/v4
      api_key: os.environ/Z_AI_API_KEY
      temperature: 1.0
      top_p: 0.95
      extra_body:
        thinking:
          type: disabled
  # glm-5.1-reasoning: identical upstream, thinking ENABLED (opt-in reasoning).
  - model_name: glm-5.1-reasoning
    litellm_params:
      model: openai/glm-5.1
      api_base: https://api.z.ai/api/coding/paas/v4
      api_key: os.environ/Z_AI_API_KEY
      temperature: 1.0
      top_p: 0.95
      extra_body:
        thinking:
          type: enabled
  # glm-5.2 (released ~2026-06; canonical z.ai id `glm-5.2`, confirmed via /models +
  #   a live completion with our key). Mirrors the glm-5.1 pattern: thinking DISABLED
  #   by default (consistency with the 2026-06-11 operator call), opt-in reasoning via
  #   glm-5.2-reasoning. extra_body.thinking is forwarded verbatim to z.ai.
  #   CANONICAL LIMITS (probed live vs z.ai 2026-07-05): 1,048,576-token (1M, 2^20)
  #   INPUT context; 131,072 (128K) MAX OUTPUT (z.ai max_tokens range [1,131072]).
  #   NO gateway-side cap — pure z.ai passthrough, so these are the effective limits.
  - model_name: glm-5.2
    litellm_params:
      model: openai/glm-5.2
      api_base: https://api.z.ai/api/coding/paas/v4
      api_key: os.environ/Z_AI_API_KEY
      temperature: 1.0
      top_p: 0.95
      extra_body:
        thinking:
          type: disabled
  - model_name: glm-5.2-reasoning
    litellm_params:
      model: openai/glm-5.2
      api_base: https://api.z.ai/api/coding/paas/v4
      api_key: os.environ/Z_AI_API_KEY
      temperature: 1.0
      top_p: 0.95
      extra_body:
        thinking:
          type: enabled
  # gen-frontier / gen-frontier-reasoning: capability aliases for the PAID
  #   frontier tier (GLM 5.2 @ z.ai), mirroring glm-5.2 / glm-5.2-reasoning
  #   (thinking off / on). Worldtree binds these for frontier-grade generation
  #   / reasoning; swap the backing frontier model here, callers unaffected.
  #   PAID — only all-proxy-models / explicitly-scoped keys reach them; the free
  #   all-agents-local key is fenced off z.ai spend and cannot.
  - model_name: gen-frontier
    litellm_params:
      model: openai/glm-5.2
      api_base: https://api.z.ai/api/coding/paas/v4
      api_key: os.environ/Z_AI_API_KEY
      temperature: 1.0
      top_p: 0.95
      extra_body:
        thinking:
          type: disabled
  - model_name: gen-frontier-reasoning
    litellm_params:
      model: openai/glm-5.2
      api_base: https://api.z.ai/api/coding/paas/v4
      api_key: os.environ/Z_AI_API_KEY
      temperature: 1.0
      top_p: 0.95
      extra_body:
        thinking:
          type: enabled
  - model_name: glm-5-turbo
    litellm_params:
      model: openai/glm-5-turbo
      api_base: https://api.z.ai/api/coding/paas/v4
      api_key: os.environ/Z_AI_API_KEY
      temperature: 1.0
      top_p: 0.95
  - model_name: glm-4.7
    litellm_params:
      model: openai/glm-4.7
      api_base: https://api.z.ai/api/coding/paas/v4
      api_key: os.environ/Z_AI_API_KEY
      temperature: 1.0
      top_p: 0.95
  - model_name: glm-4.5-air
    litellm_params:
      model: openai/glm-4.5-air
      api_base: https://api.z.ai/api/coding/paas/v4
      api_key: os.environ/Z_AI_API_KEY
      temperature: 0.6
      top_p: 0.95

  # --- (removed 2026-06-20, operator call) the `*` wildcard → llama-swap
  #     (ana-ml2:9292). llama-swap is decommissioned (:9292 confirmed down), so
  #     the wildcard routed every unmatched / typo'd / stale model name to a DEAD
  #     backend → a misleading "Connection error" instead of a clean "model not
  #     found". This is the footgun that silently swallowed Worldtree's defunct
  #     model names. Removed so unknown models now fail loudly (404). Re-add an
  #     explicit per-model entry if a swappable zoo ever returns. ---

general_settings:
  master_key: os.environ/LITELLM_MASTER_KEY
  database_url: os.environ/DATABASE_URL
  store_model_in_db: true
  # THE log switch: persists full request messages + response bodies into
  # SpendLogs so they render in the Logs UI. Without this you get metadata
  # (tokens, latency, model) but not the prompt/completion text.
  store_prompts_in_spend_logs: true
  # scalar-judge → Skywork-Reward-V2 (scalar reward model; vLLM pooling on
  #   ana-ml2:8003). LiteLLM has no reward/pooling MODE, so this is a passthrough,
  #   not a model_list alias. Gateway-key-gated. Consumers POST the reward body to
  #   /scalar-judge/<route> (e.g. /pooling or /classify), forwarded to :8003.
  #   SWAP-SENSITIVE: a different reward model shifts the score scale, so consumers
  #   must recalibrate thresholds after a backing swap.
  pass_through_endpoints:
    - path: "/scalar-judge"
      target: "http://10.250.50.54:8003"
      forward_headers: true
      include_subpath: true

litellm_settings:
  # vLLM rejects some OpenAI params other backends accept; drop silently
  # rather than 400 the caller.
  drop_params: true
  # Custom pre-call hook: strip an empty `tools: []` (+ orphaned tool_choice)
  # before forwarding upstream. vLLM 400s on empty tools arrays ("tools must
  # not be an empty array"); drop_params doesn't catch empty VALUES, only
  # unsupported params. Runs on every request → fixes it for all vLLM models.
  # File mounted at /app/strip_empty_tools.py; reference is module.instance,
  # resolved relative to this config's directory.
  callbacks: ["strip_empty_tools.strip_empty_tools_instance"]
  # Langfuse trace export RETIRED 2026-06-20 (operator). Its ClickHouse member spewed
  # ~94 GB of unrotated logs and filled ana-docker's disk; the trace UI was redundant
  # with LiteLLM's native spend_logs (store_prompts_in_spend_logs: true → full
  # prompts/responses/tokens/cost at :4000/ui). Gateway observability stays fully
  # native. Re-add success_callback/failure_callback here if Langfuse ever returns.