From d3721034c1b9816bd42c05824c604f234b055a15 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Sat, 20 Jun 2026 09:53:09 -0700 Subject: [PATCH] feat(litellm): Worldtree capability aliases (chat-judge, reranker, scalar-judge) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stand up the gateway-side capability aliases for the role→capability model indirection (worldtree-dev's transparent-swap direction; operator: no wt- prefix, reuse the existing summarizer/classifier/gen alias convention). - chat-judge -> selene-1-mini-8b (mode chat) — WT selene-judgment role. - reranker -> qwen3-reranker (mode rerank) — generic name for the cap. - scalar-judge -> Skywork-Reward-V2 via a pass_through_endpoint to ana-ml2:8003 (LiteLLM has no reward/pooling MODE, so it's a passthrough, gateway-key-gated; consumers hit /scalar-judge/ e.g. /score|/pooling|/classify). Deliberately NO generic `embedding` alias: embedding vectors are model-specific (not swap-transparent), so that capability stays `qwen3-embedding` — the model- specific name is the guardrail against treating it as freely swappable. Verified all three live (chat-judge 200, reranker present, scalar-judge passthrough 200 returning a Skywork reward). Deployed + gateway health-gated. --- stacks/litellm/conf/config.yaml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/stacks/litellm/conf/config.yaml b/stacks/litellm/conf/config.yaml index 7a70218..d06ac1e 100644 --- a/stacks/litellm/conf/config.yaml +++ b/stacks/litellm/conf/config.yaml @@ -189,6 +189,28 @@ model_list: model_info: mode: rerank + # --- Worldtree capability aliases (role→capability gateway swaps, ADR-0012). + # Stable role-named aliases so consumers bind the CAPABILITY, not a concrete + # model; swap the backing model here and callers are unaffected. NO generic + # `embedding` alias ON PURPOSE — embedding vectors are model-specific (not + # swap-transparent), so that capability stays `qwen3-embedding` above. --- + # chat-judge → generative LLM-as-judge (Selene); WT selene-judgment role. + - model_name: chat-judge + litellm_params: + model: hosted_vllm/selene-1-mini-8b + api_base: http://10.250.50.54:8011/v1 + api_key: os.environ/VLLM_API_KEY + model_info: + mode: chat + # reranker → generic capability name for rerank (currently qwen3-reranker). + - model_name: reranker + litellm_params: + model: hosted_vllm/Qwen/Qwen3-Reranker-0.6B + api_base: http://10.250.50.54:8002/v1 + api_key: os.environ/VLLM_API_KEY + model_info: + mode: rerank + # --- z.ai GLM (cloud API) — fronted for unified logging across local # + cloud inference. Explicit entries, so they win over the "*" # wildcard below (no collision with llama-swap's glm4.7-flash etc. @@ -276,6 +298,17 @@ general_settings: # SpendLogs so they render in the Logs UI. Without this you get metadata # (tokens, latency, model) but not the prompt/completion text. store_prompts_in_spend_logs: true + # scalar-judge → Skywork-Reward-V2 (scalar reward model; vLLM pooling on + # ana-ml2:8003). LiteLLM has no reward/pooling MODE, so this is a passthrough, + # not a model_list alias. Gateway-key-gated. Consumers POST the reward body to + # /scalar-judge/ (e.g. /pooling or /classify), forwarded to :8003. + # SWAP-SENSITIVE: a different reward model shifts the score scale, so consumers + # must recalibrate thresholds after a backing swap. + pass_through_endpoints: + - path: "/scalar-judge" + target: "http://10.250.50.54:8003" + forward_headers: true + include_subpath: true litellm_settings: # vLLM rejects some OpenAI params other backends accept; drop silently