From 2e5ab72e2cf7e618b0b1d983d31a0e4b5032afaf Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Fri, 19 Jun 2026 11:35:48 -0700 Subject: [PATCH] feat(litellm): add gen-nt / gen-reasoning-nt (noop-tool + tool_choice:none compat variants) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same Qwopus gen model as gen / gen-reasoning (served-name qwen3.5-122-a10b @ :8013, thinking off/on respectively), but each bakes a dummy 'noop' function tool + tool_choice:none into litellm_params so a NON-EMPTY tools array always reaches vLLM — for consumers where the global strip_empty_tools hook isn't the right fix (they need a valid tools structure present, not stripped). tool_choice:none means the noop is never called. api_base = the real LAN endpoint http://10.250.50.54:8013 (the requested http://vllm:8000 template wouldn't resolve from the ana-docker litellm container). Verified: gen-nt + gen-reasoning-nt both survive a client tools:[] send; noop never invoked; reasoning split intact. --- stacks/litellm/conf/config.yaml | 49 +++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/stacks/litellm/conf/config.yaml b/stacks/litellm/conf/config.yaml index 6ab7296..d5c3f35 100644 --- a/stacks/litellm/conf/config.yaml +++ b/stacks/litellm/conf/config.yaml @@ -134,6 +134,55 @@ model_list: enable_thinking: true model_info: mode: chat + # --- gen-nt / gen-reasoning-nt: same Qwopus `gen` model, but with a dummy `noop` + # tool + tool_choice:"none" baked into the record so a NON-EMPTY tools array + # always reaches vLLM (which 400s on an empty `tools:[]`). For consumers where + # the global strip_empty_tools hook isn't the right fix — i.e. they need a valid + # tools structure PRESENT rather than stripped. tool_choice:none disables actual + # calling, so the noop is never invoked. api_base = the real LAN endpoint (the + # operator's `http://vllm:8000` template wouldn't resolve from ana-docker). --- + - model_name: gen-nt + litellm_params: + model: hosted_vllm/qwen3.5-122-a10b + api_base: http://10.250.50.54:8013/v1 + api_key: os.environ/VLLM_API_KEY + extra_body: + chat_template_kwargs: + enable_thinking: false + tools: + - type: function + function: + name: noop + description: "No-op tool. Do not call this tool; it exists only for compatibility." + parameters: + type: object + properties: {} + required: [] + additionalProperties: false + tool_choice: "none" + model_info: + mode: chat + - model_name: gen-reasoning-nt + litellm_params: + model: hosted_vllm/qwen3.5-122-a10b + api_base: http://10.250.50.54:8013/v1 + api_key: os.environ/VLLM_API_KEY + extra_body: + chat_template_kwargs: + enable_thinking: true + tools: + - type: function + function: + name: noop + description: "No-op tool. Do not call this tool; it exists only for compatibility." + parameters: + type: object + properties: {} + required: [] + additionalProperties: false + tool_choice: "none" + model_info: + mode: chat # --- Selene 1 Mini 8B (AtlaAI judge, FP8) — restored on GPU1 after the # llama-swap teardown (was the Q6_K GGUF in the swap zoo). vLLM dynamic fp8,