diff --git a/stacks/litellm/README.md b/stacks/litellm/README.md index a5ab983..3379bdd 100644 --- a/stacks/litellm/README.md +++ b/stacks/litellm/README.md @@ -56,6 +56,35 @@ full Langfuse traces later: No re-architecture: the gateway and every consumer stay pointed here. +## ⚠ `reasoning_effort` is not a universal vocabulary + +`gen-reasoning` accepts **only** `xhigh` (its default), `medium` and `low`, and +returns HTTP 400 on anything else: + + Unexpected reasoning effort high. Supported types are xhigh (default), + medium, and low. + +That is the *default* value of several clients, so the seat presents as broken +rather than as one enum value out of step. `conf/reasoning_effort_map.py` is a +pre-call hook that maps `high` and `max` onto `xhigh` for that model group only. + +Measured 2026-09-02 across every local seat before scoping it: + +| model | `reasoning_effort: high` | +|---|---| +| `gen-reasoning` | **rejected** → mapped | +| `gen`, `sec`, `char-rp-reasoning`, `summarizer` | accepted → untouched | + +Paid passthroughs (`gen-frontier*`, `glm*`, `kimi*`) were deliberately **not** +probed — they spend vendor credits — and are not mapped. **Add a model to +`EFFORT_MAP` only after measuring that it actually rejects the value.** + +⚠ **A hook file needs a compose change, not just a conf push.** Callbacks are +bind-mounted per-file beside `config.yaml`, so a new hook requires a new volume +line and `docker compose up -d litellm` (a `restart` will not pick it up — the +volume only attaches at container creation). Target the service by name; a bare +`up -d` bounces the DB too. + ## Deploy ```bash diff --git a/stacks/litellm/compose.yaml b/stacks/litellm/compose.yaml index 7923bc8..9b874e6 100644 --- a/stacks/litellm/compose.yaml +++ b/stacks/litellm/compose.yaml @@ -36,6 +36,9 @@ services: # Must sit beside config.yaml — LiteLLM loads callbacks relative to the # config file's directory, so this lands at /app/strip_empty_tools.py. - /opt/docker/conf/litellm/strip_empty_tools.py:/app/strip_empty_tools.py:ro + # Second pre-call hook: per-model `reasoning_effort` translation. Same + # beside-the-config requirement as the hook above. + - /opt/docker/conf/litellm/reasoning_effort_map.py:/app/reasoning_effort_map.py:ro environment: # master_key gates the proxy + admin UI login. Must start with sk-. - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY} diff --git a/stacks/litellm/conf/config.yaml b/stacks/litellm/conf/config.yaml index bbec88c..eb7ecea 100644 --- a/stacks/litellm/conf/config.yaml +++ b/stacks/litellm/conf/config.yaml @@ -894,7 +894,13 @@ litellm_settings: # unsupported params. Runs on every request → fixes it for all vLLM models. # File mounted at /app/strip_empty_tools.py; reference is module.instance, # resolved relative to this config's directory. - callbacks: ["strip_empty_tools.strip_empty_tools_instance"] + # Second pre-call hook: translate `reasoning_effort` values a backend does not + # accept (gen-reasoning takes only xhigh/medium/low and 400s on `high`, which + # is the DEFAULT of several clients). Scoped per model group inside the file; + # measured, not assumed. Added 2026-09-02. + callbacks: + - "strip_empty_tools.strip_empty_tools_instance" + - "reasoning_effort_map.reasoning_effort_map_instance" # Langfuse trace export RETIRED 2026-06-20 (operator). Its ClickHouse member spewed # ~94 GB of unrotated logs and filled ana-docker's disk; the trace UI was redundant # with LiteLLM's native spend_logs (store_prompts_in_spend_logs: true → full diff --git a/stacks/litellm/conf/reasoning_effort_map.py b/stacks/litellm/conf/reasoning_effort_map.py new file mode 100644 index 0000000..c043e45 --- /dev/null +++ b/stacks/litellm/conf/reasoning_effort_map.py @@ -0,0 +1,65 @@ +# reasoning_effort_map.py +# +# LiteLLM proxy pre-call hook: translate `reasoning_effort` values that a +# backend does not accept into the ones it does, per model group. +# +# Why: the OpenAI-shaped `reasoning_effort` vocabulary is not universal. Our +# `gen-reasoning` seat (qwen3.8-27b-uncensored-thinking on ana-ml2 :8015) +# accepts ONLY `xhigh` (its default), `medium`, and `low`, and 400s on anything +# else: +# "Unexpected reasoning effort high. Supported types are xhigh (default), +# medium, and low." +# Clients that emit the common OpenAI trio (low/medium/high) or their own +# vocabulary therefore fail on their DEFAULT setting, which reads as "the seat +# is broken" rather than "one enum value differs". +# +# Concrete case that motivated this (2026-09-02): the DeepSeek Harness +# (@deepseek-ai/dsh) emits off/low/high/max and defaults to `high`. Every +# request failed. Without this map the only working client setting was `low` — +# the seat's WEAKEST reasoning tier — while its own default is `xhigh`. +# +# ⚠ SCOPED DELIBERATELY. Measured 2026-09-02 against every local seat: +# gen-reasoning high -> REJECTED +# gen, sec, char-rp-reasoning, summarizer high -> accepted +# so only `gen-reasoning` is remapped. Paid passthroughs (gen-frontier*, glm*, +# kimi*) were NOT probed — they spend vendor credits — and are not mapped. +# Add a model here only after measuring that it actually rejects the value. +# +# Registered via litellm_settings.callbacks. LiteLLM loads callbacks RELATIVE +# TO THE CONFIG FILE'S DIRECTORY, so this file must sit beside config.yaml — +# mounted into the container at /app/reasoning_effort_map.py. + +from litellm.integrations.custom_logger import CustomLogger +from litellm._logging import verbose_proxy_logger + +# model group -> {value the client sent: value the backend accepts} +# Values already accepted by the backend are simply absent and pass through. +EFFORT_MAP = { + "gen-reasoning": { + # Map the two "as hard as you can think" spellings onto the seat's own + # maximum, which is also its default. `medium` and `low` are native. + "high": "xhigh", + "max": "xhigh", + }, +} + + +class ReasoningEffortMap(CustomLogger): + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + if not isinstance(data, dict): + return data + mapping = EFFORT_MAP.get(data.get("model")) + if not mapping: + return data + sent = data.get("reasoning_effort") + mapped = mapping.get(sent) + if mapped is not None: + data["reasoning_effort"] = mapped + verbose_proxy_logger.debug( + "reasoning_effort_map: %s %r -> %r (call_type=%s)", + data.get("model"), sent, mapped, call_type, + ) + return data + + +reasoning_effort_map_instance = ReasoningEffortMap()