From edaa9a9c50874455ecf7c1a96575504b601bbcf1 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Sat, 25 Jul 2026 10:53:43 -0700 Subject: [PATCH] feat(litellm): front Kimi K3 (Moonshot) as a paid gateway passthrough MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds model_name kimi-k3 → openai/kimi-k3 @ https://api.moonshot.ai/v1 (OpenAI-compatible), keyed by MOONSHOT_API_KEY (compose env + .env.example placeholder; real key on server only). Verified live through the gateway. Two Moonshot constraints captured in the config comment + pinned: K3 accepts ONLY temperature=1 (else 400), and it is a reasoning model (CoT in reasoning_content, answer in content — needs adequate max_tokens or content returns empty). Model id confirmed via /v1/models. --- stacks/litellm/.env.example | 3 +++ stacks/litellm/compose.yaml | 4 +++- stacks/litellm/conf/config.yaml | 24 ++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/stacks/litellm/.env.example b/stacks/litellm/.env.example index 91355a7..5579e55 100644 --- a/stacks/litellm/.env.example +++ b/stacks/litellm/.env.example @@ -35,6 +35,9 @@ VLLM_API_KEY= # (glm-5.1 / glm-5-turbo / glm-4.7 / glm-4.5-air). Z_AI_API_KEY= +# Moonshot AI (Kimi) key — fronts kimi-k3 (https://api.moonshot.ai/v1). PAID. +MOONSHOT_API_KEY= + # --- Langfuse-ready (leave blank for the lean first cut) --- # Fill these AND uncomment success/failure_callback in conf/config.yaml # to ship full traces to a Langfuse instance. That is the whole upgrade. diff --git a/stacks/litellm/compose.yaml b/stacks/litellm/compose.yaml index a9b3374..de1c3aa 100644 --- a/stacks/litellm/compose.yaml +++ b/stacks/litellm/compose.yaml @@ -47,8 +47,10 @@ services: # leave blank; LiteLLM still needs the var to exist). - VLLM_API_KEY=${VLLM_API_KEY:-} # Cloud API keys fronted by the gateway for unified logging (z.ai GLM, - # etc.). Paid — only gateway-keyed callers reach them, but they spend. + # Moonshot Kimi, etc.). Paid — only gateway-keyed callers reach them, + # but they spend. - Z_AI_API_KEY=${Z_AI_API_KEY:-} + - MOONSHOT_API_KEY=${MOONSHOT_API_KEY:-} # Langfuse-ready: blank until you bolt Langfuse on. Filling these + # uncommenting the callback in config.yaml is the entire upgrade. - LANGFUSE_PUBLIC_KEY=${LANGFUSE_PUBLIC_KEY:-} diff --git a/stacks/litellm/conf/config.yaml b/stacks/litellm/conf/config.yaml index 9ed4915..0835bfe 100644 --- a/stacks/litellm/conf/config.yaml +++ b/stacks/litellm/conf/config.yaml @@ -362,6 +362,30 @@ model_list: temperature: 0.6 top_p: 0.95 + # --- Kimi K3 (Moonshot AI, cloud API) — paid passthrough fronted for unified + # logging alongside the local + z.ai inference. OpenAI-compatible endpoint + # (https://api.moonshot.ai/v1) → openai/ provider. Flagship long-horizon + # coding + knowledge model, 1M-token context (platform.kimi.ai docs). Model + # id `kimi-k3` confirmed live via /v1/models 2026-07-25 (siblings kimi-k2.6, + # kimi-k2.7-code, kimi-k2.7-code-highspeed — add explicit entries if wanted). + # PAID — spends Moonshot credits; reachable by any gateway key scoped to it + # (the shared all-agents key spans all proxy models). Key in .env + # (MOONSHOT_API_KEY). CONSTRAINT (Moonshot, verified live 2026-07-25): K3 + # ONLY accepts temperature=1 — any other value 400s ("only 1 is allowed for + # this model"). So temperature is pinned to 1 here as the default; callers + # must NOT override it with another value. K3 is also a REASONING model: + # the CoT comes back in `reasoning_content`, the answer in `content` — give + # it adequate max_tokens or content returns EMPTY (reasoning eats a tiny + # budget). Verified live through the gateway 2026-07-25 (17+25→"42"). --- + - model_name: kimi-k3 + litellm_params: + model: openai/kimi-k3 + api_base: https://api.moonshot.ai/v1 + api_key: os.environ/MOONSHOT_API_KEY + temperature: 1 + model_info: + mode: chat + # --- (removed 2026-06-20, operator call) the `*` wildcard → llama-swap # (ana-ml2:9292). llama-swap is decommissioned (:9292 confirmed down), so # the wildcard routed every unmatched / typo'd / stale model name to a DEAD