diff --git a/stacks/vllm/.env.example b/stacks/vllm/.env.example index 7340fd3..f72a5ef 100644 --- a/stacks/vllm/.env.example +++ b/stacks/vllm/.env.example @@ -119,3 +119,7 @@ GRANITE_KV_CACHE_DTYPE=fp8 # leftover room on full-chapter context (max-len 131072). KV 15.0 GiB = 196,560 tokens # = 1.50x @ 131072; GPU-1 lands ~6.7 GB headroom (char-rp 30 + granite 27 + selene 17 + trio). GRANITE_GPU_MEM_UTIL=0.27 +# Concurrency cap — set VERY HIGH 2026-07-16 (was vLLM default 128) so the KV pool is +# the only bound. granite is the fleet fan-out summarizer/classifier (many concurrent +# SHORT calls); default 128 capped below the KV bound (~192 @ 1K-tok). VRAM-neutral. +GRANITE_MAX_NUM_SEQS=1024 diff --git a/stacks/vllm/compose.yaml b/stacks/vllm/compose.yaml index 9db106c..527aa2f 100644 --- a/stacks/vllm/compose.yaml +++ b/stacks/vllm/compose.yaml @@ -233,6 +233,12 @@ services: - ${GRANITE_GPU_MEM_UTIL} - --max-model-len - ${GRANITE_MAX_MODEL_LEN} + # Very high so the KV pool (not the seq cap) is the only concurrency bound — + # granite is the fleet fan-out summarizer/classifier (many concurrent SHORT + # calls). vLLM's default resolves to 128, capping below the KV bound + # (~192 @ 1K-tok); 1024 unblocks it (VRAM-neutral — KV pool is util-bound). + - --max-num-seqs + - ${GRANITE_MAX_NUM_SEQS} - --dtype - auto # CUDA graphs ENABLED (no --enforce-eager) for decode throughput. Made