From 7bc648672fbc709b3a4818400a777fcc86de59f8 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Mon, 27 Apr 2026 18:14:59 -0700 Subject: [PATCH] =?UTF-8?q?llama-swap:=20pin=20granite-4-small=20alongside?= =?UTF-8?q?=20qwen3.5-9b=20=E2=80=94=20stop=20swapping=20with=20qwen3.6-27?= =?UTF-8?q?b?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Symptom: granite-4-small and qwen3.6-27b were evicting each other when called in alternation. granite is the news-digest curator (fires twice daily on cron) — being evicted means a cold reload (~5s) on every digest tick, plus visible churn whenever the user uses 27b concurrently. Added granite-4-small to the `pinned` group as a persistent member. ~5-6 GB at Q4_K_M + 120K KV ≈ comfortable inside the existing pin budget (qwen3.5-9b ~6 GB → ~12 GB total persistent). Single RTX 6000 Ada is 48 GB, leaves ~36 GB headroom for whichever non-pinned model the user invokes (qwen3.6-27b at ~30 GB fits cleanly). Updated the pinned group's docstring to capture the current member set + VRAM math + the historical context (qwen3.6-35-a3b was here, was too heavy, got removed yesterday). Marked the granite ttl: 0 with the matching "pinned — never unloads" comment as the other group members. --- stacks/llama-swap/conf/config.yaml | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/stacks/llama-swap/conf/config.yaml b/stacks/llama-swap/conf/config.yaml index 8bc680c..226a73f 100644 --- a/stacks/llama-swap/conf/config.yaml +++ b/stacks/llama-swap/conf/config.yaml @@ -419,7 +419,7 @@ models: "granite-4-small": name: "Granite 4.0 Small Q4_K_M" description: "IBM Granite 4.0 Small. Deterministic utility model for structured tasks." - ttl: 0 + ttl: 0 # pinned — member of the `pinned` group, never unloads cmd: | /app/llama-server --context-shift @@ -545,17 +545,27 @@ groups: - "qwen3-embedding-0.6B" - "qwen3-reranker-0.6B" - # Pinned general-purpose chat models. Coexist in VRAM, never unload. - # Members also have ttl: 0 individually so idle-timeout can't drop them. - # Currently just qwen3.5-9b — small enough (~6 GB at Q4 + KV) to be - # cheap to keep resident. qwen3.6-35-a3b was previously here but its - # ~29 GB at Q6_K_XL made other model loads OOM under memory pressure - # (persistent: true exempts members from eviction). It now sits with - # ttl: 0 outside any group — never idle-unloads, but the scheduler - # CAN evict it when another non-coexistent model is requested. + # Pinned general-purpose / utility models. Coexist in VRAM, never + # unload. Members also have ttl: 0 individually so idle-timeout can't + # drop them. + # + # Current pins: + # qwen3.5-9b — ~6 GB at Q4 + KV. General-purpose chat baseline. + # granite-4-small — ~5-6 GB at Q4_K_M + 120K KV. Used by news-digest + # curator twice daily; pinning avoids the cold-load + # latency and prevents qwen3.6-27b (and similar) + # from evicting it when both are needed concurrently. + # VRAM budget: ~12 GB persistent in the pin slot. Single RTX 6000 Ada + # is 48 GB, so this leaves ~36 GB for whichever non-pinned model the + # user invokes alongside (qwen3.6-27b at ~30 GB fits cleanly). + # + # qwen3.6-35-a3b WAS in this group; removed 2026-04-27 because its + # ~29 GB at Q6_K_XL pushed concurrent loads OOM. Now lives outside + # with ttl: 0 — never idle-unloads but evictable under memory pressure. "pinned": swap: false exclusive: false persistent: true members: - "qwen3.5-9b" + - "granite-4-small"