From c7ee8884c64904f0b0bbcba1f1d9ebd5ec6ab527 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Thu, 4 Jun 2026 11:13:32 -0700 Subject: [PATCH] feat(litellm): front z.ai GLM cloud models through the gateway MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds explicit gateway entries for the four z.ai GLM models (glm-5.1, glm-5-turbo, glm-4.7, glm-4.5-air) routed to api.z.ai with Z_AI_API_KEY, plus the compose env passthrough + .env.example doc. Explicit entries win over the llama-swap wildcard (distinct IDs, no collision). Extends the gateway's unified logging to cloud inference, not just local vLLM/llama-swap. Cost note: paid API — only gateway-keyed callers reach these, but calls spend z.ai credits (documented in config + compose comments). --- stacks/litellm/.env.example | 5 +++++ stacks/litellm/compose.yaml | 3 +++ stacks/litellm/conf/config.yaml | 26 ++++++++++++++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/stacks/litellm/.env.example b/stacks/litellm/.env.example index 5d35125..91355a7 100644 --- a/stacks/litellm/.env.example +++ b/stacks/litellm/.env.example @@ -30,6 +30,11 @@ POSTGRES_PASSWORD= # vllm stack's .env. VLLM_API_KEY= +# Cloud-provider API keys fronted by the gateway (PAID — spend on use; only +# gateway-keyed callers can reach them). z.ai GLM models +# (glm-5.1 / glm-5-turbo / glm-4.7 / glm-4.5-air). +Z_AI_API_KEY= + # --- Langfuse-ready (leave blank for the lean first cut) --- # Fill these AND uncomment success/failure_callback in conf/config.yaml # to ship full traces to a Langfuse instance. That is the whole upgrade. diff --git a/stacks/litellm/compose.yaml b/stacks/litellm/compose.yaml index 2690501..507225c 100644 --- a/stacks/litellm/compose.yaml +++ b/stacks/litellm/compose.yaml @@ -42,6 +42,9 @@ services: # Upstream vLLM API key (empty in the vllm stack's .env by default → # leave blank; LiteLLM still needs the var to exist). - VLLM_API_KEY=${VLLM_API_KEY:-} + # Cloud API keys fronted by the gateway for unified logging (z.ai GLM, + # etc.). Paid — only gateway-keyed callers reach them, but they spend. + - Z_AI_API_KEY=${Z_AI_API_KEY:-} # Langfuse-ready: blank until you bolt Langfuse on. Filling these + # uncommenting the callback in config.yaml is the entire upgrade. - LANGFUSE_PUBLIC_KEY=${LANGFUSE_PUBLIC_KEY:-} diff --git a/stacks/litellm/conf/config.yaml b/stacks/litellm/conf/config.yaml index 1f73c3c..0f433de 100644 --- a/stacks/litellm/conf/config.yaml +++ b/stacks/litellm/conf/config.yaml @@ -45,6 +45,32 @@ model_list: model_info: mode: rerank + # --- z.ai GLM (cloud API) — fronted for unified logging across local + # + cloud inference. Explicit entries, so they win over the "*" + # wildcard below (no collision with llama-swap's glm4.7-flash etc. + # — different model IDs). NOTE: paid API; only gateway-keyed callers + # can reach these, but they DO spend z.ai credits. Key in .env. --- + - model_name: glm-5.1 + litellm_params: + model: openai/glm-5.1 + api_base: https://api.z.ai/api/coding/paas/v4 + api_key: os.environ/Z_AI_API_KEY + - model_name: glm-5-turbo + litellm_params: + model: openai/glm-5-turbo + api_base: https://api.z.ai/api/coding/paas/v4 + api_key: os.environ/Z_AI_API_KEY + - model_name: glm-4.7 + litellm_params: + model: openai/glm-4.7 + api_base: https://api.z.ai/api/coding/paas/v4 + api_key: os.environ/Z_AI_API_KEY + - model_name: glm-4.5-air + litellm_params: + model: openai/glm-4.5-air + api_base: https://api.z.ai/api/coding/paas/v4 + api_key: os.environ/Z_AI_API_KEY + # --- llama-swap passthrough (the swappable generative LLM zoo on # ana-ml2:9292) --- # Wildcard: any model name NOT matched by an exact entry above routes to