diff --git a/stacks/llama-swap/config.yaml b/stacks/llama-swap/config.yaml index 8b4c30b..c3888bd 100644 --- a/stacks/llama-swap/config.yaml +++ b/stacks/llama-swap/config.yaml @@ -24,6 +24,9 @@ # Swapped abliterated entry from IIEleven11 Heretic Q8_0 to # mradermacher abliterated i1-Q6_K (already cached). # Bumped both Qwen 3.6 entries ctx-size 32768 → 131072. +# 2026-04-22: New `pinned` group — qwen3.5-9b + qwen3.6-35-a3b coexist +# in VRAM with persistent=true and ttl=0. Means the two +# can be called concurrently and never idle-unload. # ============================================================================ # Default 1200 seconds (20 min) to wait for model to be available to load. @@ -97,7 +100,7 @@ models: "qwen3.5-9b": name: "Qwen 3.5 9B UD-Q4_K_XL" description: "Dense 9B model. Lightweight general-purpose chat and reasoning." - ttl: 600 + ttl: 0 # pinned — member of the `pinned` group, never unloads cmd: | /app/llama-server --context-shift @@ -120,7 +123,7 @@ models: "qwen3.6-35-a3b": name: "Qwen 3.6 35B-A3B UD-Q6_K_XL" description: "Qwen3.6 MoE, 3B active of 35B. Stock unsloth Q6_K_XL (~29GB). Thinking on by default." - ttl: 600 + ttl: 0 # pinned — member of the `pinned` group, never unloads cmd: | /app/llama-server --context-shift @@ -487,3 +490,15 @@ groups: - "embeddinggemma-300M" - "qwen3-embedding-0.6B" - "qwen3-reranker-0.6B" + + # Pinned general-purpose chat models. Coexist in VRAM, never unload. + # Members also have ttl: 0 individually so idle-timeout can't drop them. + # VRAM budget check: qwen3.5-9b (~6 GB at Q4) + qwen3.6-35-a3b (~29 GB + # at Q6_K_XL) + 128K KV cache for each ≈ 50-60 GB. Size for your GPU. + "pinned": + swap: false + exclusive: false + persistent: true + members: + - "qwen3.5-9b" + - "qwen3.6-35-a3b"