From 9e696394826d894fe04932446acf64d64a289dbe Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Thu, 16 Jul 2026 10:47:15 -0700 Subject: [PATCH] fix(vllm): pin granite --max-num-seqs=1024 (was implicit default 128) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit granite (fleet fan-out summarizer/classifier) had no explicit --max-num-seqs, so vLLM V1 resolved it to 128 — which caps concurrency BELOW granite's own KV bound (~192 concurrent @ 1K-token calls, more for shorter classify calls). Pinned it very high (1024) so the KV pool is the only bound; VRAM-neutral (the KV pool is util-bound, unchanged). Added the flag to the granite command + GRANITE_MAX_NUM_SEQS to the env template. Live applied + verified (resolved max_num_seqs=1024, seat healthy). --- stacks/vllm/.env.example | 4 ++++ stacks/vllm/compose.yaml | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/stacks/vllm/.env.example b/stacks/vllm/.env.example index 7340fd3..f72a5ef 100644 --- a/stacks/vllm/.env.example +++ b/stacks/vllm/.env.example @@ -119,3 +119,7 @@ GRANITE_KV_CACHE_DTYPE=fp8 # leftover room on full-chapter context (max-len 131072). KV 15.0 GiB = 196,560 tokens # = 1.50x @ 131072; GPU-1 lands ~6.7 GB headroom (char-rp 30 + granite 27 + selene 17 + trio). GRANITE_GPU_MEM_UTIL=0.27 +# Concurrency cap — set VERY HIGH 2026-07-16 (was vLLM default 128) so the KV pool is +# the only bound. granite is the fleet fan-out summarizer/classifier (many concurrent +# SHORT calls); default 128 capped below the KV bound (~192 @ 1K-tok). VRAM-neutral. +GRANITE_MAX_NUM_SEQS=1024 diff --git a/stacks/vllm/compose.yaml b/stacks/vllm/compose.yaml index 9db106c..527aa2f 100644 --- a/stacks/vllm/compose.yaml +++ b/stacks/vllm/compose.yaml @@ -233,6 +233,12 @@ services: - ${GRANITE_GPU_MEM_UTIL} - --max-model-len - ${GRANITE_MAX_MODEL_LEN} + # Very high so the KV pool (not the seq cap) is the only concurrency bound — + # granite is the fleet fan-out summarizer/classifier (many concurrent SHORT + # calls). vLLM's default resolves to 128, capping below the KV bound + # (~192 @ 1K-tok); 1024 unblocks it (VRAM-neutral — KV pool is util-bound). + - --max-num-seqs + - ${GRANITE_MAX_NUM_SEQS} - --dtype - auto # CUDA graphs ENABLED (no --enforce-eager) for decode throughput. Made