diff --git a/stacks/llama-swap/conf/config.yaml b/stacks/llama-swap/conf/config.yaml index 8bc680c..226a73f 100644 --- a/stacks/llama-swap/conf/config.yaml +++ b/stacks/llama-swap/conf/config.yaml @@ -419,7 +419,7 @@ models: "granite-4-small": name: "Granite 4.0 Small Q4_K_M" description: "IBM Granite 4.0 Small. Deterministic utility model for structured tasks." - ttl: 0 + ttl: 0 # pinned — member of the `pinned` group, never unloads cmd: | /app/llama-server --context-shift @@ -545,17 +545,27 @@ groups: - "qwen3-embedding-0.6B" - "qwen3-reranker-0.6B" - # Pinned general-purpose chat models. Coexist in VRAM, never unload. - # Members also have ttl: 0 individually so idle-timeout can't drop them. - # Currently just qwen3.5-9b — small enough (~6 GB at Q4 + KV) to be - # cheap to keep resident. qwen3.6-35-a3b was previously here but its - # ~29 GB at Q6_K_XL made other model loads OOM under memory pressure - # (persistent: true exempts members from eviction). It now sits with - # ttl: 0 outside any group — never idle-unloads, but the scheduler - # CAN evict it when another non-coexistent model is requested. + # Pinned general-purpose / utility models. Coexist in VRAM, never + # unload. Members also have ttl: 0 individually so idle-timeout can't + # drop them. + # + # Current pins: + # qwen3.5-9b — ~6 GB at Q4 + KV. General-purpose chat baseline. + # granite-4-small — ~5-6 GB at Q4_K_M + 120K KV. Used by news-digest + # curator twice daily; pinning avoids the cold-load + # latency and prevents qwen3.6-27b (and similar) + # from evicting it when both are needed concurrently. + # VRAM budget: ~12 GB persistent in the pin slot. Single RTX 6000 Ada + # is 48 GB, so this leaves ~36 GB for whichever non-pinned model the + # user invokes alongside (qwen3.6-27b at ~30 GB fits cleanly). + # + # qwen3.6-35-a3b WAS in this group; removed 2026-04-27 because its + # ~29 GB at Q6_K_XL pushed concurrent loads OOM. Now lives outside + # with ttl: 0 — never idle-unloads but evictable under memory pressure. "pinned": swap: false exclusive: false persistent: true members: - "qwen3.5-9b" + - "granite-4-small"