From d1bea1399406b58b9efcb3bbf571c0c9e92a1ede Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Tue, 16 Jun 2026 00:43:57 -0700 Subject: [PATCH] fix(litellm): strip empty tools:[] before forwarding to vLLM vLLM's OpenAI server 400s on an empty tools array ("tools must not be an empty array"), which broke every gateway call carrying tools:[] (clients that send it to mean "no tools" -- OpenAI tolerates it, vLLM does not). drop_params doesn't help: it drops unsupported PARAMS, not empty VALUES. Add a CustomLogger async_pre_call_hook (conf/strip_empty_tools.py) that pops an empty/None tools field (+ orphaned tool_choice) before forwarding, registered globally via litellm_settings.callbacks so it covers every vLLM-backed model, not just mistral-small-4. Mounted at /app/strip_empty_tools.py beside config.yaml (LiteLLM resolves callbacks relative to the config dir). Surgical: only fires when tools is present and empty; real tools pass through untouched. Verified on live gateway (1.87.0): mistral-small-4 and granite-4.1-8b with tools:[] now 200 (were 400); no-tools baseline unchanged; a real tool still passes through. --- stacks/litellm/README.md | 10 ++++++ stacks/litellm/compose.yaml | 4 +++ stacks/litellm/conf/config.yaml | 7 ++++ stacks/litellm/conf/strip_empty_tools.py | 44 ++++++++++++++++++++++++ 4 files changed, 65 insertions(+) create mode 100644 stacks/litellm/conf/strip_empty_tools.py diff --git a/stacks/litellm/README.md b/stacks/litellm/README.md index 72f9a90..a5ab983 100644 --- a/stacks/litellm/README.md +++ b/stacks/litellm/README.md @@ -107,3 +107,13 @@ Then open `http://10.250.50.70:4000/ui` (log in with the master key) → `API_KEY=` empty. Set it here only if you set it there. - `LITELLM_SALT_KEY` must be set **once** and never changed — rotating it makes any keys stored in Postgres undecryptable. +- **Empty `tools: []` stripping** — `conf/strip_empty_tools.py` is a pre-call + hook (registered via `litellm_settings.callbacks`) that drops an empty/None + `tools` field (and any orphaned `tool_choice`) before forwarding. vLLM 400s on + `tools: []` ("tools must not be an empty array"); `drop_params` doesn't catch + empty *values*, only unsupported params. It runs on **every** request, so all + vLLM-backed models are covered, and only fires when `tools` is present-and-empty + (real tools pass through untouched). The file mounts at `/app/strip_empty_tools.py` + beside `config.yaml` because LiteLLM resolves callbacks relative to the config + dir. Note: real tool-calls additionally need the upstream vLLM server launched + with `--enable-auto-tool-choice` — a vLLM-side flag, separate from this gateway. diff --git a/stacks/litellm/compose.yaml b/stacks/litellm/compose.yaml index 507225c..00442b0 100644 --- a/stacks/litellm/compose.yaml +++ b/stacks/litellm/compose.yaml @@ -32,6 +32,10 @@ services: - "${LITELLM_BIND:-0.0.0.0}:${LITELLM_PORT:-4000}:4000" volumes: - /opt/docker/conf/litellm/config.yaml:/app/config.yaml:ro + # Custom pre-call hook (strip empty `tools: []` before forwarding to vLLM). + # Must sit beside config.yaml — LiteLLM loads callbacks relative to the + # config file's directory, so this lands at /app/strip_empty_tools.py. + - /opt/docker/conf/litellm/strip_empty_tools.py:/app/strip_empty_tools.py:ro environment: # master_key gates the proxy + admin UI login. Must start with sk-. - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY} diff --git a/stacks/litellm/conf/config.yaml b/stacks/litellm/conf/config.yaml index 90dccf9..bbd79c7 100644 --- a/stacks/litellm/conf/config.yaml +++ b/stacks/litellm/conf/config.yaml @@ -197,6 +197,13 @@ litellm_settings: # vLLM rejects some OpenAI params other backends accept; drop silently # rather than 400 the caller. drop_params: true + # Custom pre-call hook: strip an empty `tools: []` (+ orphaned tool_choice) + # before forwarding upstream. vLLM 400s on empty tools arrays ("tools must + # not be an empty array"); drop_params doesn't catch empty VALUES, only + # unsupported params. Runs on every request → fixes it for all vLLM models. + # File mounted at /app/strip_empty_tools.py; reference is module.instance, + # resolved relative to this config's directory. + callbacks: ["strip_empty_tools.strip_empty_tools_instance"] # --- Langfuse trace export (live 2026-06-05). Full prompt/completion + # reasoning + tok-derivable latency traces ship to the Langfuse stack on # ana-docker (project "gateway"). Keys + host in .env. The gateway and diff --git a/stacks/litellm/conf/strip_empty_tools.py b/stacks/litellm/conf/strip_empty_tools.py new file mode 100644 index 0000000..26c8521 --- /dev/null +++ b/stacks/litellm/conf/strip_empty_tools.py @@ -0,0 +1,44 @@ +# strip_empty_tools.py +# +# LiteLLM proxy pre-call hook: drop an empty / falsy `tools` field (and any +# now-orphaned `tool_choice`) before the request is forwarded upstream. +# +# Why: vLLM's OpenAI server validates that `tools`, when present, is non-empty. +# It 400s on `tools: []`: +# "Value error, tools must not be an empty array. Either provide at least one +# tool or omit the field entirely." +# Several clients send `tools: []` to mean "no tools" — OpenAI and most backends +# tolerate it; vLLM does not. `drop_params: true` does NOT help (it drops +# provider-UNSUPPORTED params, not empty VALUES), so we strip it at the gateway. +# +# Registered globally via litellm_settings.callbacks, so it runs on EVERY request +# regardless of model — covers all vLLM-backed routes (granite / qwen3.6 / mistral +# / selene / the llama-swap wildcard), not just one model. Harmless on backends +# that accept empty tools (we only remove a no-op field), and it only fires when +# the `tools` key is actually present and empty, so normal tool-less calls are +# untouched. +# +# LiteLLM loads callbacks RELATIVE TO THE CONFIG FILE'S DIRECTORY, so this file +# must sit beside config.yaml — mounted into the container at +# /app/strip_empty_tools.py (config.yaml is at /app/config.yaml). + +from litellm.integrations.custom_logger import CustomLogger +from litellm._logging import verbose_proxy_logger + + +class StripEmptyTools(CustomLogger): + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + # Only act when `tools` is present AND falsy ([] or None). Leaving the + # key absent for normal requests means this is a no-op on that path. + if isinstance(data, dict) and "tools" in data and not data["tools"]: + data.pop("tools", None) + data.pop("tool_choice", None) + verbose_proxy_logger.debug( + "strip_empty_tools: dropped empty `tools` (model=%s, call_type=%s)", + data.get("model"), + call_type, + ) + return data + + +strip_empty_tools_instance = StripEmptyTools()