diff --git a/stacks/litellm/conf/config.yaml b/stacks/litellm/conf/config.yaml index 0f433de..dff73b9 100644 --- a/stacks/litellm/conf/config.yaml +++ b/stacks/litellm/conf/config.yaml @@ -17,11 +17,14 @@ # route — left direct; see README. model_list: - # --- Phi-4-mini (generative chat) — production summarizer + dreaming - # agent. Full prompt + completion captured per call. --- - - model_name: phi4-mini + # --- Granite 4.1 8B (generative chat) — production summarizer + dreaming + # agent. Replaced phi4-mini 2026-06-05 (beat it on precision in brokkr's + # R15 P03 eval). vLLM on ana-ml2 GPU 1, official FP8, 50K ctx. Explicit + # entry shadows the "*" wildcard's llama-swap route for this name. Full + # prompt + completion captured per call. --- + - model_name: granite-4.1-8b litellm_params: - model: hosted_vllm/phi4-mini + model: hosted_vllm/granite-4.1-8b api_base: http://10.250.50.54:8004/v1 api_key: os.environ/VLLM_API_KEY model_info: diff --git a/stacks/vllm/.env.example b/stacks/vllm/.env.example index 58808b4..ffdc53d 100644 --- a/stacks/vllm/.env.example +++ b/stacks/vllm/.env.example @@ -29,13 +29,18 @@ RERANK_MODEL=Qwen/Qwen3-Reranker-0.6B # Setting any one too low causes that container to OOM on KV cache allocation # with `Available KV cache memory: -X.XX GiB`. # -# Layout on a 48 GB Ada (~30% headroom kept free; matches current production): -# EMBED 0.20 (~9.6 GB) — 0.6B Qwen3 embed at 8k ctx; comfortable -# RERANK 0.20 (~9.6 GB) — 0.6B Qwen3 rerank at 8k ctx; comfortable -# REWARD 0.30 (~14 GB) — 8B Skywork AWQ at 16k ctx classify; tune up if OOM -EMBED_GPU_MEM_UTIL=0.20 -RERANK_GPU_MEM_UTIL=0.20 -REWARD_GPU_MEM_UTIL=0.30 +# Layout on a 48 GB Ada, RIGHT-SIZED 2026-06-05 to free room for the granite +# summarizer's CUDA graphs + a LoRA hedge. The trio was wildly over-provisioned: +# embed ran a 5.89x KV pool, reward 3.90x — pure waste for utility models that +# need ~1.5x. Trimmed to free ~10 GB. (Utilization = fraction of TOTAL GPU mem; +# each service profiles independently. Too low → that container OOMs on KV with +# `Available KV cache memory: -X.XX GiB`.) +# EMBED 0.07 (~3.4 GB) — 0.6B Qwen3 embed @8k; 1.1GB weights + ~2GB KV (~3x) +# RERANK 0.07 (~3.4 GB) — 0.6B Qwen3 rerank @8k; comfortable +# REWARD 0.18 (~8.6 GB) — 8B Skywork AWQ @16k; 4.4GB weights + ~3.5GB KV (~1.5x) +EMBED_GPU_MEM_UTIL=0.07 +RERANK_GPU_MEM_UTIL=0.07 +REWARD_GPU_MEM_UTIL=0.18 # Context length caps — lower these if VRAM is tight. # Qwen3-Embedding supports up to 32k; reranker up to 32k. @@ -53,27 +58,28 @@ API_KEY= # services (embed/rerank). Reward is local-path, ignores this. HF_TOKEN= -# === phi4-mini (summarizer / dreaming agent; supersedes llama-swap granite-4-small) === -PHI4_PORT=8004 -# GPU 1 — co-locate with the embed/rerank/reward trio so GPU 0 stays clear for -# llama-swap's heavy swap-ins. Post-granite GPU 1 has ~10.4 GB free; phi4 at 50K -# + FP8 KV needs ~9 GB → ~1.2 GB margin (tight). NOTE: llama-swap also uses GPU 1 -# dynamically (a model's ~10 GB share lives here), so a large swap-in could -# contend — for clean separation, pin llama-swap to GPU 0 (follow-up). -PHI4_GPU_ID=1 -# FP8 path. Default = vLLM-native dynamic FP8 from the official BF16 weights -# (most robust on Ada/Hopper, near-lossless, ~1.2x). Alternative = the -# pre-quantized checkpoint: PHI4_MODEL=pytorch/Phi-4-mini-instruct-FP8 + PHI4_QUANT=torchao. -PHI4_MODEL=microsoft/Phi-4-mini-instruct -PHI4_QUANT=fp8 -PHI4_SERVED_NAME=phi4-mini -# 50K ctx (51200) — dropped from the 128K canonical max to fit GPU 1's ~10.4 GB -# free post-granite (64K was ~10 GB = no margin). Ample for feed summarization + -# the dreaming agent; Phi-4-mini supports up to 131072 if it ever moves to GPU 0. -PHI4_MAX_MODEL_LEN=51200 -# FP8 KV cache (native on Ada cc 8.9). At 50K ≈ ~3.1 GB KV (vs ~6.3 GB at fp16). -PHI4_KV_CACHE_DTYPE=fp8 -# ~9 GB total: ~4 GB FP8 weights + ~3.1 GB FP8 KV @50K + overhead. 0.20×46≈9.2 GB. -# vLLM prints the real KV size on first boot — tune after; do NOT raise without -# first confirming GPU 1 freed more (it's only ~1.2 GB over budget here). -PHI4_GPU_MEM_UTIL=0.20 +# === granite-4.1-8b (production summarizer / dreaming agent) === +# Replaced phi4-mini 2026-06-05 (Granite 4.1 8B beat phi4 on precision in +# brokkr's R15 P03 model-fitness eval). Same GPU-1 slot, reusing phi4's port. +GRANITE_PORT=8004 +# GPU 1 — co-located with the embed/rerank/reward trio. With phi4 retired, GPU 1 +# has ~19 GB free; granite at 56K + FP8 KV needs ~15-16 GB → ~2-3 GB margin. +# NOTE: llama-swap also uses GPU 1 dynamically — a large swap-in could contend; +# pin llama-swap to GPU 0 for clean separation (follow-up). +GRANITE_GPU_ID=1 +# Official IBM pre-quantized FP8 (compressed-tensors) — calibrated, ~9.6 GB, +# loaded directly (FP8 native on Ada cc 8.9). Fallback to vLLM-native dynamic FP8 +# from BF16: GRANITE_MODEL=ibm-granite/granite-4.1-8b + GRANITE_QUANT=fp8. +GRANITE_MODEL=ibm-granite/granite-4.1-8b-fp8 +GRANITE_QUANT=compressed-tensors +GRANITE_SERVED_NAME=granite-4.1-8b +# 50K ctx (51200) — matches phi4's window. Deliberately NOT maxed: leaves ~3 GB +# on GPU 1 as a hedge for future Granite-LoRA serving (--enable-lora needs ~2-3 GB) +# / a holistic GPU-1 retune of the embed/rerank/reward trio. KV @50K fp8 ≈ 4.2 GB; +# +9.6 weights +~2.4 overhead ≈ 16.2 GB. Granite 4.1 supports up to 131072. +GRANITE_MAX_MODEL_LEN=51200 +# FP8 KV cache (native on Ada cc 8.9). At 50K ≈ ~4.2 GB (vs ~8.4 GB at fp16). +GRANITE_KV_CACHE_DTYPE=fp8 +# Target ~16.2 GB of GPU 1's ~19 GB free: weights 9.6 + KV 4.2 + overhead ~2.4. +# 0.36×46≈16.6 GB budget (covers it with slack). Confirm vs vLLM's first-boot KV log. +GRANITE_GPU_MEM_UTIL=0.36 diff --git a/stacks/vllm/compose.yaml b/stacks/vllm/compose.yaml index 1a0334d..afe3d7e 100644 --- a/stacks/vllm/compose.yaml +++ b/stacks/vllm/compose.yaml @@ -201,13 +201,13 @@ services: # llama-swap granite-4-small pin. Generative chat model (OpenAI # /v1/chat/completions), so NO --runner pooling. FP8 on RTX 6000 Ada # (cc 8.9): near-lossless, ~1.2x, ~6 GB. - vllm-phi4: + vllm-granite: image: vllm/vllm-openai:${VLLM_VERSION} - container_name: vllm-phi4 + container_name: vllm-granite restart: unless-stopped ipc: host ports: - - "${PHI4_PORT}:8000" + - "${GRANITE_PORT}:8000" volumes: - /tank/aimodels/huggingface:/hfcache environment: @@ -216,34 +216,40 @@ services: - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-} - VLLM_API_KEY=${API_KEY:-} command: - # Default = vLLM-native dynamic FP8 from the official BF16 weights (most - # robust on Ada). Alt = pre-quantized checkpoint: PHI4_MODEL= - # pytorch/Phi-4-mini-instruct-FP8 + PHI4_QUANT=torchao (see .env/README). - - ${PHI4_MODEL} + # Production summarizer (replaced phi4-mini 2026-06-05). Default = official + # IBM pre-quantized FP8 (compressed-tensors), loaded directly; FP8 is native + # on the RTX 6000 Ada (cc 8.9). Fallback to vLLM-native dynamic FP8 from + # BF16: GRANITE_MODEL=ibm-granite/granite-4.1-8b + GRANITE_QUANT=fp8. + - ${GRANITE_MODEL} - --served-model-name - - ${PHI4_SERVED_NAME} + - ${GRANITE_SERVED_NAME} - --quantization - - ${PHI4_QUANT} + - ${GRANITE_QUANT} - --host - 0.0.0.0 - --port - "8000" - --gpu-memory-utilization - - ${PHI4_GPU_MEM_UTIL} + - ${GRANITE_GPU_MEM_UTIL} - --max-model-len - - ${PHI4_MAX_MODEL_LEN} + - ${GRANITE_MAX_MODEL_LEN} - --dtype - auto - # FP8 KV cache — halves KV memory at 128K ctx on Ada (cc 8.9); near-lossless. + # CUDA graphs ENABLED (no --enforce-eager) for decode throughput. Made + # room 2026-06-05 by right-sizing the embed/rerank/reward trio's KV pools + # (they were over-provisioned at 5.9x/2.0x/3.9x concurrency); GPU 1 now has + # ~17 GB free after granite, so graph-capture buffers fit. If the trio + # ever grows back, granite may need --enforce-eager again on this card. + # FP8 KV cache — halves KV memory; near-lossless on Ada (cc 8.9). - --kv-cache-dtype - - ${PHI4_KV_CACHE_DTYPE} + - ${GRANITE_KV_CACHE_DTYPE} deploy: resources: reservations: devices: - driver: nvidia device_ids: - - "${PHI4_GPU_ID}" + - "${GRANITE_GPU_ID}" capabilities: - gpu healthcheck: @@ -256,10 +262,10 @@ services: - tnet labels: - homepage.group=AI Systems - - homepage.name=vLLM Phi-4-mini (summarizer/dreamer) + - homepage.name=vLLM Granite 4.1 8B (summarizer) - homepage.icon=mdi-text-box-outline - - homepage.description=Phi-4-mini FP8 via vLLM (ana-ml2) - - homepage.href=http://10.250.50.54:${PHI4_PORT}/docs + - homepage.description=Granite 4.1 8B FP8 via vLLM (ana-ml2) + - homepage.href=http://10.250.50.54:${GRANITE_PORT}/docs networks: tnet: