diff --git a/stacks/litellm/.env.example b/stacks/litellm/.env.example index 91355a7..5579e55 100644 --- a/stacks/litellm/.env.example +++ b/stacks/litellm/.env.example @@ -35,6 +35,9 @@ VLLM_API_KEY= # (glm-5.1 / glm-5-turbo / glm-4.7 / glm-4.5-air). Z_AI_API_KEY= +# Moonshot AI (Kimi) key — fronts kimi-k3 (https://api.moonshot.ai/v1). PAID. +MOONSHOT_API_KEY= + # --- Langfuse-ready (leave blank for the lean first cut) --- # Fill these AND uncomment success/failure_callback in conf/config.yaml # to ship full traces to a Langfuse instance. That is the whole upgrade. diff --git a/stacks/litellm/compose.yaml b/stacks/litellm/compose.yaml index a9b3374..de1c3aa 100644 --- a/stacks/litellm/compose.yaml +++ b/stacks/litellm/compose.yaml @@ -47,8 +47,10 @@ services: # leave blank; LiteLLM still needs the var to exist). - VLLM_API_KEY=${VLLM_API_KEY:-} # Cloud API keys fronted by the gateway for unified logging (z.ai GLM, - # etc.). Paid — only gateway-keyed callers reach them, but they spend. + # Moonshot Kimi, etc.). Paid — only gateway-keyed callers reach them, + # but they spend. - Z_AI_API_KEY=${Z_AI_API_KEY:-} + - MOONSHOT_API_KEY=${MOONSHOT_API_KEY:-} # Langfuse-ready: blank until you bolt Langfuse on. Filling these + # uncommenting the callback in config.yaml is the entire upgrade. - LANGFUSE_PUBLIC_KEY=${LANGFUSE_PUBLIC_KEY:-} diff --git a/stacks/litellm/conf/config.yaml b/stacks/litellm/conf/config.yaml index 9ed4915..0835bfe 100644 --- a/stacks/litellm/conf/config.yaml +++ b/stacks/litellm/conf/config.yaml @@ -362,6 +362,30 @@ model_list: temperature: 0.6 top_p: 0.95 + # --- Kimi K3 (Moonshot AI, cloud API) — paid passthrough fronted for unified + # logging alongside the local + z.ai inference. OpenAI-compatible endpoint + # (https://api.moonshot.ai/v1) → openai/ provider. Flagship long-horizon + # coding + knowledge model, 1M-token context (platform.kimi.ai docs). Model + # id `kimi-k3` confirmed live via /v1/models 2026-07-25 (siblings kimi-k2.6, + # kimi-k2.7-code, kimi-k2.7-code-highspeed — add explicit entries if wanted). + # PAID — spends Moonshot credits; reachable by any gateway key scoped to it + # (the shared all-agents key spans all proxy models). Key in .env + # (MOONSHOT_API_KEY). CONSTRAINT (Moonshot, verified live 2026-07-25): K3 + # ONLY accepts temperature=1 — any other value 400s ("only 1 is allowed for + # this model"). So temperature is pinned to 1 here as the default; callers + # must NOT override it with another value. K3 is also a REASONING model: + # the CoT comes back in `reasoning_content`, the answer in `content` — give + # it adequate max_tokens or content returns EMPTY (reasoning eats a tiny + # budget). Verified live through the gateway 2026-07-25 (17+25→"42"). --- + - model_name: kimi-k3 + litellm_params: + model: openai/kimi-k3 + api_base: https://api.moonshot.ai/v1 + api_key: os.environ/MOONSHOT_API_KEY + temperature: 1 + model_info: + mode: chat + # --- (removed 2026-06-20, operator call) the `*` wildcard → llama-swap # (ana-ml2:9292). llama-swap is decommissioned (:9292 confirmed down), so # the wildcard routed every unmatched / typo'd / stale model name to a DEAD