diff --git a/stacks/qwen3.5-122b/.env.example b/stacks/qwen3.5-122b/.env.example index 6ed557d..bcf21d6 100644 --- a/stacks/qwen3.5-122b/.env.example +++ b/stacks/qwen3.5-122b/.env.example @@ -15,8 +15,10 @@ QWEN35_GPU_ID=0 QWEN35_SERVED_NAME=qwen3.5-122-a10b QWEN35_MAX_MODEL_LEN=131072 -QWEN35_MAX_NUM_SEQS=4 -QWEN35_GPU_MEM_UTIL=0.90 +QWEN35_MAX_NUM_SEQS=8 +# Single-tenant 96GB card; 0.95 is safe-aggressive (leaves ~5GB for activation +# spikes + fragmentation, esp. with 8 concurrent seqs). 0.97+ risks OOM under load. +QWEN35_GPU_MEM_UTIL=0.95 QWEN35_MAX_NUM_BATCHED_TOKENS=32768 # Optional upstream vLLM API key (empty = no auth; internal net only). diff --git a/stacks/qwen3.5-122b/compose.yaml b/stacks/qwen3.5-122b/compose.yaml index 4838721..a414c6b 100644 --- a/stacks/qwen3.5-122b/compose.yaml +++ b/stacks/qwen3.5-122b/compose.yaml @@ -35,8 +35,8 @@ services: - SERVED_MODEL_NAME=${QWEN35_SERVED_NAME:-qwen3.5-122-a10b} - HOST_PORT=8000 - MAX_MODEL_LEN=${QWEN35_MAX_MODEL_LEN:-131072} - - MAX_NUM_SEQS=${QWEN35_MAX_NUM_SEQS:-4} - - GPU_MEMORY_UTILIZATION=${QWEN35_GPU_MEM_UTIL:-0.90} + - MAX_NUM_SEQS=${QWEN35_MAX_NUM_SEQS:-8} + - GPU_MEMORY_UTILIZATION=${QWEN35_GPU_MEM_UTIL:-0.95} - MAX_NUM_BATCHED_TOKENS=${QWEN35_MAX_NUM_BATCHED_TOKENS:-32768} # --reasoning-parser qwen3 surfaces … as reasoning_content; # the thinking on/off itself is per-request (litellm chat_template_kwargs).