From d0eb09cac1399b85c1fed5b4aa92069f0f42669a Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Sat, 20 Jun 2026 10:08:20 -0700 Subject: [PATCH] =?UTF-8?q?fix(litellm):=20remove=20the=20`*`=20=E2=86=92?= =?UTF-8?q?=20llama-swap=20wildcard=20(decommissioned=20backend)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit llama-swap (ana-ml2:9292) is decommissioned (:9292 confirmed down), so the catch-all wildcard routed every unmatched / typo'd / stale model name to a DEAD backend, surfacing a misleading "Connection error" instead of a clean "model not found". This is the exact footgun that silently swallowed Worldtree's defunct model names (mistral-small-4 etc.) instead of erroring. Removed (operator call) so unknown models now 404 loudly. Verified: gateway healthy post-restart, a bogus model name now returns a clean not-found error, real aliases (gen) still serve. Re-add explicit per-model entries if a swappable zoo ever returns. --- stacks/litellm/conf/config.yaml | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/stacks/litellm/conf/config.yaml b/stacks/litellm/conf/config.yaml index d06ac1e..0bc5ad5 100644 --- a/stacks/litellm/conf/config.yaml +++ b/stacks/litellm/conf/config.yaml @@ -273,22 +273,13 @@ model_list: api_base: https://api.z.ai/api/coding/paas/v4 api_key: os.environ/Z_AI_API_KEY - # --- llama-swap passthrough (the swappable generative LLM zoo on - # ana-ml2:9292) --- - # Wildcard: any model name NOT matched by an exact entry above routes to - # llama-swap, which swaps the requested model into GPU on demand. This lets - # the gateway front the WHOLE swappable zoo (artemis / selene / qwen3.x / …) - # for logging + auth WITHOUT registering each model here — keep adding and - # swapping models in llama-swap freely; litellm logs them all. litellm does - # no inference; llama-swap still does all the model loading + serving. - # Exact matches above (phi4-mini / qwen3-embedding / qwen3-reranker) win; - # this only catches everything else. `openai/*` forwards the requested model - # name verbatim to llama-swap's OpenAI-compatible endpoint. - - model_name: "*" - litellm_params: - model: openai/* - api_base: http://10.250.50.54:9292/v1 - api_key: "noauth" # llama-swap takes no auth; placeholder bearer + # --- (removed 2026-06-20, operator call) the `*` wildcard → llama-swap + # (ana-ml2:9292). llama-swap is decommissioned (:9292 confirmed down), so + # the wildcard routed every unmatched / typo'd / stale model name to a DEAD + # backend → a misleading "Connection error" instead of a clean "model not + # found". This is the footgun that silently swallowed Worldtree's defunct + # model names. Removed so unknown models now fail loudly (404). Re-add an + # explicit per-model entry if a swappable zoo ever returns. --- general_settings: master_key: os.environ/LITELLM_MASTER_KEY