From a9a2be706008fbf8b22c08a5f5291065ef5acc7b Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Sat, 13 Jun 2026 12:30:20 -0700 Subject: [PATCH] tune(vllm): pin --enable-prefix-caching on granite + qwen Benched granite prefix caching at ~6.5x faster TTFT (45ms cached vs 292ms uncached) on a shared ~4.5k-token summarizer template. granite already had it on by vLLM-v1 default; pinned explicit so a version flip can't silently disable it. qwen (nightly) defaulted it OFF -> flipped on (free for the text-chat path, marginal for vision where each image is a distinct prefix). Soft/evictable KV, zero memory change (GPU1 still ~3.7GB free), all 5 services healthy. --- stacks/qwen35-vl/compose.yaml | 3 +++ stacks/vllm/compose.yaml | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/stacks/qwen35-vl/compose.yaml b/stacks/qwen35-vl/compose.yaml index 5377998..3cbbb59 100644 --- a/stacks/qwen35-vl/compose.yaml +++ b/stacks/qwen35-vl/compose.yaml @@ -54,6 +54,9 @@ services: - ${QWEN_MAX_MODEL_LEN} - --dtype - auto + # Prefix caching pinned ON (the nightly defaults it OFF). Free win for the + # text-chat path; marginal for vision (each image is a distinct prefix). + - --enable-prefix-caching deploy: resources: reservations: diff --git a/stacks/vllm/compose.yaml b/stacks/vllm/compose.yaml index afe3d7e..186ee4b 100644 --- a/stacks/vllm/compose.yaml +++ b/stacks/vllm/compose.yaml @@ -243,6 +243,11 @@ services: # FP8 KV cache — halves KV memory; near-lossless on Ada (cc 8.9). - --kv-cache-dtype - ${GRANITE_KV_CACHE_DTYPE} + # Prefix caching pinned EXPLICIT (vLLM v1 defaults it on, but pin so a + # version flip can't silently disable it). Benched 2026-06-13: ~6.5x faster + # TTFT (45ms vs 292ms) on a shared ~4.5k-token summarizer template; soft/ + # evictable KV, neutral when prefixes don't repeat — pure win for granite. + - --enable-prefix-caching deploy: resources: reservations: