esh-pfi-infrastructure/stacks/litellm/conf/config.yaml

# LiteLLM gateway config — fronts the vLLM services on ana-ml2
# (10.250.50.54) and logs every request + response so they're
# inspectable in the Logs UI at http://10.250.50.70:4000/ui.
#
# Deploys to /opt/docker/conf/litellm/config.yaml (mounted read-only
# into the container at /app/config.yaml).
#
# Model-name → upstream mapping:
#   phi4-mini        → vLLM :8004  (generative chat)
#   qwen3-embedding  → vLLM :8001  (/v1/embeddings)
#   qwen3-reranker   → vLLM :8002  (/rerank)
#   *  (wildcard)    → llama-swap :9292  (the swappable generative zoo)
#
# The wildcard fronts llama-swap so its whole model zoo logs through the
# gateway without per-model registration. The vllm-reward classifier
# (:8003) is a pooling /classify endpoint with no first-class LiteLLM
# route — left direct; see README.

model_list:
  # --- Granite 4.1 8B (generative chat) — production summarizer + dreaming
  #     agent. Replaced phi4-mini 2026-06-05 (beat it on precision in brokkr's
  #     R15 P03 eval). vLLM on ana-ml2 GPU 1, official FP8, 50K ctx. Explicit
  #     entry shadows the "*" wildcard's llama-swap route for this name. Full
  #     prompt + completion captured per call. ---
  - model_name: granite-4.1-8b
    litellm_params:
      model: hosted_vllm/granite-4.1-8b
      api_base: http://10.250.50.54:8004/v1
      api_key: os.environ/VLLM_API_KEY
    model_info:
      mode: chat

  # --- Qwen3 embeddings ---
  - model_name: qwen3-embedding
    litellm_params:
      model: hosted_vllm/Qwen/Qwen3-Embedding-0.6B
      api_base: http://10.250.50.54:8001/v1
      api_key: os.environ/VLLM_API_KEY
    model_info:
      mode: embedding

  # --- Qwen3 reranker (proxy /rerank route) ---
  - model_name: qwen3-reranker
    litellm_params:
      model: hosted_vllm/Qwen/Qwen3-Reranker-0.6B
      api_base: http://10.250.50.54:8002/v1
      api_key: os.environ/VLLM_API_KEY
    model_info:
      mode: rerank

  # --- z.ai GLM (cloud API) — fronted for unified logging across local
  #     + cloud inference. Explicit entries, so they win over the "*"
  #     wildcard below (no collision with llama-swap's glm4.7-flash etc.
  #     — different model IDs). NOTE: paid API; only gateway-keyed callers
  #     can reach these, but they DO spend z.ai credits. Key in .env. ---
  # glm-5.1: thinking DISABLED by default (2026-06-11, operator call). LiteLLM
  #   strips a top-level `thinking` param (drop_params), but forwards `extra_body`
  #   verbatim to z.ai, where the native thinking:{type:disabled} control lands —
  #   verified reasoning_tokens→0. Reasoning is opt-in via glm-5.1-reasoning below.
  - model_name: glm-5.1
    litellm_params:
      model: openai/glm-5.1
      api_base: https://api.z.ai/api/coding/paas/v4
      api_key: os.environ/Z_AI_API_KEY
      extra_body:
        thinking:
          type: disabled
  # glm-5.1-reasoning: identical upstream, thinking ENABLED (opt-in reasoning).
  - model_name: glm-5.1-reasoning
    litellm_params:
      model: openai/glm-5.1
      api_base: https://api.z.ai/api/coding/paas/v4
      api_key: os.environ/Z_AI_API_KEY
      extra_body:
        thinking:
          type: enabled
  - model_name: glm-5-turbo
    litellm_params:
      model: openai/glm-5-turbo
      api_base: https://api.z.ai/api/coding/paas/v4
      api_key: os.environ/Z_AI_API_KEY
  - model_name: glm-4.7
    litellm_params:
      model: openai/glm-4.7
      api_base: https://api.z.ai/api/coding/paas/v4
      api_key: os.environ/Z_AI_API_KEY
  - model_name: glm-4.5-air
    litellm_params:
      model: openai/glm-4.5-air
      api_base: https://api.z.ai/api/coding/paas/v4
      api_key: os.environ/Z_AI_API_KEY

  # --- llama-swap passthrough (the swappable generative LLM zoo on
  #     ana-ml2:9292) ---
  # Wildcard: any model name NOT matched by an exact entry above routes to
  # llama-swap, which swaps the requested model into GPU on demand. This lets
  # the gateway front the WHOLE swappable zoo (artemis / selene / qwen3.x / …)
  # for logging + auth WITHOUT registering each model here — keep adding and
  # swapping models in llama-swap freely; litellm logs them all. litellm does
  # no inference; llama-swap still does all the model loading + serving.
  # Exact matches above (phi4-mini / qwen3-embedding / qwen3-reranker) win;
  # this only catches everything else. `openai/*` forwards the requested model
  # name verbatim to llama-swap's OpenAI-compatible endpoint.
  - model_name: "*"
    litellm_params:
      model: openai/*
      api_base: http://10.250.50.54:9292/v1
      api_key: "noauth"   # llama-swap takes no auth; placeholder bearer

general_settings:
  master_key: os.environ/LITELLM_MASTER_KEY
  database_url: os.environ/DATABASE_URL
  store_model_in_db: true
  # THE log switch: persists full request messages + response bodies into
  # SpendLogs so they render in the Logs UI. Without this you get metadata
  # (tokens, latency, model) but not the prompt/completion text.
  store_prompts_in_spend_logs: true

litellm_settings:
  # vLLM rejects some OpenAI params other backends accept; drop silently
  # rather than 400 the caller.
  drop_params: true
  # --- Langfuse trace export (live 2026-06-05). Full prompt/completion +
  #     reasoning + tok-derivable latency traces ship to the Langfuse stack on
  #     ana-docker (project "gateway"). Keys + host in .env. The gateway and
  #     every consumer stay pointed here — this callback is the whole upgrade. ---
  success_callback: ["langfuse"]
  failure_callback: ["langfuse"]