From 349b3614db3f8a19f78112b2d87b5431ee3f6cb0 Mon Sep 17 00:00:00 2001
From: Vuong Hoang <vh@phasefinal.com>
Date: Thu, 4 Jun 2026 11:02:35 -0700
Subject: [PATCH] feat(litellm): front llama-swap through the gateway via
 wildcard passthrough
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Adds a `model_name: "*"` entry routing any unmatched model to llama-swap
(ana-ml2:9292) so its whole swappable LLM zoo logs through the gateway
without per-model registration — add/swap models in llama-swap freely,
litellm logs them all. Exact entries (phi4-mini/qwen3-embedding/
qwen3-reranker → vLLM) still win; the wildcard only catches the rest.

litellm does no inference; llama-swap keeps loading + serving. Enables
routing worldtree-personal's generative chat through the gateway for
full req/resp logging while preserving llama-swap's on-demand swapping.
---
 stacks/litellm/conf/config.yaml | 34 +++++++++++++++++++++++++--------
 1 file changed, 26 insertions(+), 8 deletions(-)

diff --git a/stacks/litellm/conf/config.yaml b/stacks/litellm/conf/config.yaml
index 415109f..1f73c3c 100644
--- a/stacks/litellm/conf/config.yaml
+++ b/stacks/litellm/conf/config.yaml
@@ -5,15 +5,16 @@
 # Deploys to /opt/docker/conf/litellm/config.yaml (mounted read-only
 # into the container at /app/config.yaml).
 #
-# Model-name → upstream vLLM port mapping:
-#   phi4-mini        → :8004  (generative chat — the priority target)
-#   qwen3-embedding  → :8001  (/v1/embeddings)
-#   qwen3-reranker   → :8002  (/rerank)
+# Model-name → upstream mapping:
+#   phi4-mini        → vLLM :8004  (generative chat)
+#   qwen3-embedding  → vLLM :8001  (/v1/embeddings)
+#   qwen3-reranker   → vLLM :8002  (/rerank)
+#   *  (wildcard)    → llama-swap :9292  (the swappable generative zoo)
 #
-# The vllm-reward classifier (:8003) is a pooling /classify endpoint
-# with no first-class LiteLLM route — left direct; see README. The
-# generative model is the one whose prompts/completions we most need
-# to read, and it routes cleanly here.
+# The wildcard fronts llama-swap so its whole model zoo logs through the
+# gateway without per-model registration. The vllm-reward classifier
+# (:8003) is a pooling /classify endpoint with no first-class LiteLLM
+# route — left direct; see README.
 
 model_list:
   # --- Phi-4-mini (generative chat) — production summarizer + dreaming
@@ -44,6 +45,23 @@ model_list:
     model_info:
       mode: rerank
 
+  # --- llama-swap passthrough (the swappable generative LLM zoo on
+  #     ana-ml2:9292) ---
+  # Wildcard: any model name NOT matched by an exact entry above routes to
+  # llama-swap, which swaps the requested model into GPU on demand. This lets
+  # the gateway front the WHOLE swappable zoo (artemis / selene / qwen3.x / …)
+  # for logging + auth WITHOUT registering each model here — keep adding and
+  # swapping models in llama-swap freely; litellm logs them all. litellm does
+  # no inference; llama-swap still does all the model loading + serving.
+  # Exact matches above (phi4-mini / qwen3-embedding / qwen3-reranker) win;
+  # this only catches everything else. `openai/*` forwards the requested model
+  # name verbatim to llama-swap's OpenAI-compatible endpoint.
+  - model_name: "*"
+    litellm_params:
+      model: openai/*
+      api_base: http://10.250.50.54:9292/v1
+      api_key: "noauth"   # llama-swap takes no auth; placeholder bearer
+
 general_settings:
   master_key: os.environ/LITELLM_MASTER_KEY
   database_url: os.environ/DATABASE_URL