tune(qwen3.5-122b): gpu-mem-util 0.90->0.95, max-num-seqs 4->8 (KV 260K->446K tokens, 3.4x concurrency @131K, no OOM)

stacks/qwen3.5-122b/.env.example

@@ -15,8 +15,10 @@ QWEN35_GPU_ID=0
 QWEN35_SERVED_NAME=qwen3.5-122-a10b
 
 QWEN35_MAX_MODEL_LEN=131072
-QWEN35_MAX_NUM_SEQS=4
-QWEN35_GPU_MEM_UTIL=0.90
+QWEN35_MAX_NUM_SEQS=8
+# Single-tenant 96GB card; 0.95 is safe-aggressive (leaves ~5GB for activation
+# spikes + fragmentation, esp. with 8 concurrent seqs). 0.97+ risks OOM under load.
+QWEN35_GPU_MEM_UTIL=0.95
 QWEN35_MAX_NUM_BATCHED_TOKENS=32768
 
 # Optional upstream vLLM API key (empty = no auth; internal net only).

stacks/qwen3.5-122b/compose.yaml

@@ -35,8 +35,8 @@ services:
       - SERVED_MODEL_NAME=${QWEN35_SERVED_NAME:-qwen3.5-122-a10b}
       - HOST_PORT=8000
       - MAX_MODEL_LEN=${QWEN35_MAX_MODEL_LEN:-131072}
-      - MAX_NUM_SEQS=${QWEN35_MAX_NUM_SEQS:-4}
-      - GPU_MEMORY_UTILIZATION=${QWEN35_GPU_MEM_UTIL:-0.90}
+      - MAX_NUM_SEQS=${QWEN35_MAX_NUM_SEQS:-8}
+      - GPU_MEMORY_UTILIZATION=${QWEN35_GPU_MEM_UTIL:-0.95}
       - MAX_NUM_BATCHED_TOKENS=${QWEN35_MAX_NUM_BATCHED_TOKENS:-32768}
       # --reasoning-parser qwen3 surfaces <think>…</think> as reasoning_content;
       # the thinking on/off itself is per-request (litellm chat_template_kwargs).