diff --git a/stacks/llama-swap/conf/config.yaml b/stacks/llama-swap/conf/config.yaml index 891b3df..8bc680c 100644 --- a/stacks/llama-swap/conf/config.yaml +++ b/stacks/llama-swap/conf/config.yaml @@ -129,7 +129,8 @@ models: "qwen3.6-35-a3b": name: "Qwen 3.6 35B-A3B UD-Q6_K_XL" description: "Qwen3.6 MoE, 3B active of 35B. Stock unsloth Q6_K_XL (~29GB). Thinking on by default." - ttl: 0 # pinned — member of the `pinned` group, never unloads + ttl: 0 # never idle-unload, but evictable under memory pressure + # (out of `pinned` group as of 2026-04-27 — was OOM'ing other loads). cmd: | /app/llama-server --context-shift @@ -546,12 +547,15 @@ groups: # Pinned general-purpose chat models. Coexist in VRAM, never unload. # Members also have ttl: 0 individually so idle-timeout can't drop them. - # VRAM budget check: qwen3.5-9b (~6 GB at Q4) + qwen3.6-35-a3b (~29 GB - # at Q6_K_XL) + 128K KV cache for each ≈ 50-60 GB. Size for your GPU. + # Currently just qwen3.5-9b — small enough (~6 GB at Q4 + KV) to be + # cheap to keep resident. qwen3.6-35-a3b was previously here but its + # ~29 GB at Q6_K_XL made other model loads OOM under memory pressure + # (persistent: true exempts members from eviction). It now sits with + # ttl: 0 outside any group — never idle-unloads, but the scheduler + # CAN evict it when another non-coexistent model is requested. "pinned": swap: false exclusive: false persistent: true members: - "qwen3.5-9b" - - "qwen3.6-35-a3b"