From 6de08443239af80aeefeb6490b9bd1425b678310 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Mon, 15 Jun 2026 13:55:11 -0700 Subject: [PATCH] =?UTF-8?q?feat(qwen36-vl):=20split=20thinking=20=E2=80=94?= =?UTF-8?q?=20non-thinking=20default=20+=20qwen3.6-35b-a3b-thinking=20vari?= =?UTF-8?q?ant?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The qwen3.6-35b-a3b VL checkpoint is a single hybrid model with a per- request enable_thinking switch (Qwen3-style), defaulting thinking ON. Make the default non-thinking and add an opt-in reasoning variant, mirroring the existing glm-5.1 / glm-5.1-reasoning gateway split. - qwen36-vl compose: add --reasoning-parser qwen3 (model-matched) so the single :8007 endpoint splits into reasoning_content when on and routes all output to content when off — serving both modes cleanly. - litellm gateway: base qwen3.6-35b-a3b pins chat_template_kwargs enable_thinking=false (non-thinking default); new qwen3.6-35b-a3b-thinking pins enable_thinking=true (opt-in reasoning). Same upstream checkpoint, no extra VRAM/container. Deployed + verified on ana-ml2 (vLLM recreated, healthy) and ana-docker (litellm reloaded): default returns a direct answer with no reasoning_content; -thinking returns cleanly-separated reasoning_content, no raw tag leak. --- stacks/litellm/conf/config.yaml | 27 ++++++++++++++++++++++++++- stacks/qwen36-vl/compose.yaml | 19 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/stacks/litellm/conf/config.yaml b/stacks/litellm/conf/config.yaml index 7e13103..77b1206 100644 --- a/stacks/litellm/conf/config.yaml +++ b/stacks/litellm/conf/config.yaml @@ -33,12 +33,37 @@ model_list: # --- Qwen3.6-35B-A3B vision-language MoE (official FP8) — vision + chat. vLLM # on ana-ml2 GPU 1, :8007. Explicit entry shadows the "*" wildcard llama-swap # route. REPLACED qwen3.5-9b-fp8 2026-06-14 (the 9B is retired; this is a - # 35B-A3B MoE — served under its TRUE name, never aliased under the old one). --- + # 35B-A3B MoE — served under its TRUE name, never aliased under the old one). + # + # THINKING SPLIT (2026-06-15, operator call — mirrors the glm-5.1 pattern + # above). One hybrid checkpoint; the per-request `enable_thinking` switch + # picks the mode. LiteLLM forwards extra_body verbatim to vLLM, where + # chat_template_kwargs lands in the chat template. vLLM runs + # --reasoning-parser qwen3 so reasoning surfaces as reasoning_content. --- + # qwen3.6-35b-a3b: thinking DISABLED by default. The checkpoint defaults + # thinking ON; enable_thinking=false forces the empty block. + # Reasoning is opt-in via qwen3.6-35b-a3b-thinking below. - model_name: qwen3.6-35b-a3b litellm_params: model: hosted_vllm/qwen3.6-35b-a3b api_base: http://10.250.50.54:8007/v1 api_key: os.environ/VLLM_API_KEY + extra_body: + chat_template_kwargs: + enable_thinking: false + model_info: + mode: chat + # qwen3.6-35b-a3b-thinking: identical upstream checkpoint, thinking ENABLED + # (opt-in reasoning). The qwen3 reasoning-parser splits … into + # reasoning_content; content holds just the answer. + - model_name: qwen3.6-35b-a3b-thinking + litellm_params: + model: hosted_vllm/qwen3.6-35b-a3b + api_base: http://10.250.50.54:8007/v1 + api_key: os.environ/VLLM_API_KEY + extra_body: + chat_template_kwargs: + enable_thinking: true model_info: mode: chat diff --git a/stacks/qwen36-vl/compose.yaml b/stacks/qwen36-vl/compose.yaml index a30eaea..38bcf5f 100644 --- a/stacks/qwen36-vl/compose.yaml +++ b/stacks/qwen36-vl/compose.yaml @@ -27,6 +27,18 @@ # cost. 0.42 (~40 GB) = weights + graph + generous KV. Granite drops to 0.25/64K # to make room (the FP8-vs-maxed-granite tradeoff, operator-approved 2026-06-14). # +# THINKING TOGGLE: this is ONE hybrid checkpoint (not separate Instruct/Thinking +# downloads) with a Qwen3-style per-request `enable_thinking` switch. The chat +# template defaults thinking ON (`\n`); passing +# `chat_template_kwargs={"enable_thinking":false}` emits the empty +# `\n\n\n\n` block (no reasoning). We run --reasoning-parser qwen3 +# (model-matched — its vLLM docstring describes THIS checkpoint) so ONE endpoint +# serves BOTH modes cleanly: thinking-ON splits … into +# reasoning_content; thinking-OFF routes everything to content. The gateway +# selects the mode per model_name (stacks/litellm/conf/config.yaml): +# qwen3.6-35b-a3b → enable_thinking:false (non-thinking DEFAULT) +# qwen3.6-35b-a3b-thinking → enable_thinking:true (opt-in reasoning) +# # All tunables live in .env — edit that, not this file. name: qwen36-vl @@ -72,6 +84,13 @@ services: - --dtype - auto - --enable-prefix-caching + # Model-matched reasoning parser for the hybrid thinking toggle (see header). + # Splits … into reasoning_content when thinking is ON; routes + # all output to content when the empty think-block signals thinking OFF — so + # this single :8007 endpoint serves both the non-thinking default and the + # qwen3.6-35b-a3b-thinking gateway variant. + - --reasoning-parser + - qwen3 deploy: resources: reservations: