From 4f094fa6539e48713918e4bb3341136150570d2e Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Sat, 27 Jun 2026 08:26:46 -0700 Subject: [PATCH] litellm: canonical general-use sampling defaults across gateway models Per operator directive (overrideable defaults => sane optimal general-use sampling on every served model, for quality not just repetition): - qwen3.5-122-a10b: thinking-split canonicals (non-thinking temp 0.7/top_p 0.8; thinking temp 0.6/top_p 0.95; top_k 20 both), keep presence_penalty 1.0. Fixes the non-thinking routes previously running thinking-mode sampling. - qwen-image-bench / image-judge: deterministic judge profile temp 0 / top_k 1 / top_p 1.0 / repetition_penalty 1.05. - selene-1-mini-8b / chat-judge: temp 0.6 / top_p 0.9 (gen_config). vLLM-only params (top_k, repetition_penalty) in extra_body to survive drop_params. Values dvalin-researched + KB-corroborated (docs/pfi/ recommended-model-settings.md). granite + GLM pending dvalin deeper search. --- stacks/litellm/conf/config.yaml | 35 +++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/stacks/litellm/conf/config.yaml b/stacks/litellm/conf/config.yaml index e602c8d..d1d85c8 100644 --- a/stacks/litellm/conf/config.yaml +++ b/stacks/litellm/conf/config.yaml @@ -60,7 +60,10 @@ model_list: api_base: http://10.250.50.54:8013/v1 api_key: os.environ/VLLM_API_KEY presence_penalty: 1.0 + temperature: 0.7 + top_p: 0.8 extra_body: + top_k: 20 chat_template_kwargs: enable_thinking: false model_info: @@ -78,6 +81,11 @@ model_list: model: hosted_vllm/qwen-image-bench api_base: http://10.250.50.54:8014/v1 api_key: os.environ/VLLM_API_KEY + temperature: 0 + top_p: 1.0 + extra_body: + top_k: 1 + repetition_penalty: 1.05 model_info: mode: chat - model_name: image-judge @@ -85,6 +93,11 @@ model_list: model: hosted_vllm/qwen-image-bench api_base: http://10.250.50.54:8014/v1 api_key: os.environ/VLLM_API_KEY + temperature: 0 + top_p: 1.0 + extra_body: + top_k: 1 + repetition_penalty: 1.05 model_info: mode: chat @@ -110,7 +123,10 @@ model_list: api_base: http://10.250.50.54:8013/v1 api_key: os.environ/VLLM_API_KEY presence_penalty: 1.0 + temperature: 0.7 + top_p: 0.8 extra_body: + top_k: 20 chat_template_kwargs: enable_thinking: false model_info: @@ -121,7 +137,10 @@ model_list: api_base: http://10.250.50.54:8013/v1 api_key: os.environ/VLLM_API_KEY presence_penalty: 1.0 + temperature: 0.6 + top_p: 0.95 extra_body: + top_k: 20 chat_template_kwargs: enable_thinking: true model_info: @@ -132,7 +151,10 @@ model_list: api_base: http://10.250.50.54:8013/v1 api_key: os.environ/VLLM_API_KEY presence_penalty: 1.0 + temperature: 0.7 + top_p: 0.8 extra_body: + top_k: 20 chat_template_kwargs: enable_thinking: false model_info: @@ -143,7 +165,10 @@ model_list: api_base: http://10.250.50.54:8013/v1 api_key: os.environ/VLLM_API_KEY presence_penalty: 1.0 + temperature: 0.6 + top_p: 0.95 extra_body: + top_k: 20 chat_template_kwargs: enable_thinking: true model_info: @@ -154,7 +179,10 @@ model_list: api_base: http://10.250.50.54:8013/v1 api_key: os.environ/VLLM_API_KEY presence_penalty: 1.0 + temperature: 0.7 + top_p: 0.8 extra_body: + top_k: 20 chat_template_kwargs: enable_thinking: false model_info: @@ -165,7 +193,10 @@ model_list: api_base: http://10.250.50.54:8013/v1 api_key: os.environ/VLLM_API_KEY presence_penalty: 1.0 + temperature: 0.6 + top_p: 0.95 extra_body: + top_k: 20 chat_template_kwargs: enable_thinking: true model_info: @@ -180,6 +211,8 @@ model_list: model: hosted_vllm/selene-1-mini-8b api_base: http://10.250.50.54:8011/v1 api_key: os.environ/VLLM_API_KEY + temperature: 0.6 + top_p: 0.9 model_info: mode: chat @@ -212,6 +245,8 @@ model_list: model: hosted_vllm/selene-1-mini-8b api_base: http://10.250.50.54:8011/v1 api_key: os.environ/VLLM_API_KEY + temperature: 0.6 + top_p: 0.9 model_info: mode: chat # reranker → generic capability name for rerank (currently qwen3-reranker).