From 3462b5336c88d24418755ed33470bd23bcdb4363 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Sun, 16 Aug 2026 16:01:22 -0700 Subject: [PATCH] config(litellm): apply canonical Qwen3.8 sampling; fix presence_penalty on the thinking alias Sourced from upstream rather than tuned by hand. Qwen/Qwen3.8-27B card 'Best Practices' 1 and unsloth/Qwen3.8-27B 1 are byte-identical: Thinking: temperature=1.0 top_p=0.95 top_k=20 min_p=0.0 presence_penalty=0.0 repetition_penalty=1.0 Instruct: temperature=0.7 top_p=0.80 top_k=20 min_p=0.0 presence_penalty=1.5 repetition_penalty=1.0 REAL BUG FIXED: gen-reasoning carried presence_penalty=1.5 -- the INSTRUCT-mode value applied to a THINKING deployment, where canonical is 0.0. Corrected. gen was already canonical; added the missing explicit min_p and repetition_penalty so the full set is visible at the call site rather than relying on backend defaults that happen to agree. DELIBERATELY NOT canonicalised: summarizer, classifier, image-judge and qwen-image-bench run temperature=0 (and the judges top_k=1, repetition_penalty=1.05) because determinism is the point of those seats. Forcing temperature=0.7 on a classifier to match a chat preset would break their contract, so canonical is applied only where the alias is actually doing open-ended generation. Recorded against presence_penalty=1.5, which upstream itself hedges on verbatim: 'you can adjust the presence_penalty parameter between 0 and 2 to reduce endless repetition. However, using a higher value may occasionally result in language mixing and a slight decrease in model performance.' 1.5 is high in that band and is the operator's suspected trigger for the multi-turn degradation. Left at canonical so the baseline is defensible, with the caveat and the 0.0-0.5 fallback documented inline as the first dial to move if it recurs. --- stacks/litellm/conf/config.yaml | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/stacks/litellm/conf/config.yaml b/stacks/litellm/conf/config.yaml index 95f6ad0..3ef7fe5 100644 --- a/stacks/litellm/conf/config.yaml +++ b/stacks/litellm/conf/config.yaml @@ -131,6 +131,21 @@ model_list: # thinking (dvalin-smithy canonical 2026-07-08, validated vs Qwen guidance). gen # non-thinking temp 0.7/top_p 0.8; gen-reasoning thinking temp 1.0/top_p 0.95 (the # GENERAL thinking profile, not the 0.6 coding sub-profile). docs/pfi/model-sampler-defaults.md. --- + # CANONICAL Qwen3.8 INSTRUCT (non-thinking) sampling set, verified 2026-08-16 + # against BOTH upstreams, which are byte-identical on this: + # Qwen/Qwen3.8-27B card "Best Practices" §1 and unsloth/Qwen3.8-27B §1 + # temperature=0.7, top_p=0.80, top_k=20, min_p=0.0, + # presence_penalty=1.5, repetition_penalty=1.0 + # + # ⚠️ presence_penalty=1.5 is canonical BUT is the one value upstream itself + # hedges on, verbatim: "you can adjust the presence_penalty parameter between + # 0 and 2 to reduce endless repetition. However, using a higher value may + # occasionally result in LANGUAGE MIXING and a slight decrease in model + # performance." 1.5 sits high in that 0-2 band. If short/degraded replies + # reappear on long multi-turn conversations, THIS is the first dial to move + # (try 0.0-0.5) — operator's own hypothesis 2026-08-16, and upstream's caveat + # supports it. Left at canonical for now so the baseline is defensible rather + # than hand-tuned. - model_name: gen litellm_params: model: hosted_vllm/qwen3.8-27b-uncensored @@ -141,6 +156,8 @@ model_list: top_p: 0.8 extra_body: top_k: 20 + min_p: 0.0 + repetition_penalty: 1.0 chat_template_kwargs: enable_thinking: false model_info: @@ -152,11 +169,21 @@ model_list: model: hosted_vllm/qwen3.8-27b-uncensored-thinking api_base: http://10.250.50.54:8015/v1 api_key: os.environ/VLLM_API_KEY - presence_penalty: 1.5 + # CANONICAL Qwen3.8 THINKING sampling set (Qwen + unsloth "Best Practices" + # §1, identical in both): temperature=1.0, top_p=0.95, top_k=20, + # min_p=0.0, presence_penalty=0.0, repetition_penalty=1.0. + # + # ⚠️ presence_penalty was 1.5 here until 2026-08-16 — the INSTRUCT-mode + # value applied to a THINKING deployment. Canonical for thinking mode is + # 0.0, and upstream warns a high presence_penalty can cause language + # mixing and degrade performance. Corrected to 0.0. + presence_penalty: 0.0 temperature: 1.0 top_p: 0.95 extra_body: top_k: 20 + min_p: 0.0 + repetition_penalty: 1.0 chat_template_kwargs: enable_thinking: true model_info: