llama-swap: pin qwen3.5-9b + qwen3.6-35-a3b as a coexistence group
Adds a new `pinned` group with swap: false (models coexist in VRAM), exclusive: false (group shares with other groups), persistent: true (never unload). Each member also gets ttl: 0 so the per-model idle-timeout can't drop them either — belt + suspenders. Pair is currently qwen3.5-9b (~6 GB Q4) + qwen3.6-35-a3b (~29 GB Q6). Plus the 128K KV caches, roughly 50-60 GB VRAM resident. Appropriate for an A6000/H100-class card; verify fit after deploy. Committed as a canonical change; push + restart still needed on ana-ml2.
This commit is contained in:
@@ -24,6 +24,9 @@
|
|||||||
# Swapped abliterated entry from IIEleven11 Heretic Q8_0 to
|
# Swapped abliterated entry from IIEleven11 Heretic Q8_0 to
|
||||||
# mradermacher abliterated i1-Q6_K (already cached).
|
# mradermacher abliterated i1-Q6_K (already cached).
|
||||||
# Bumped both Qwen 3.6 entries ctx-size 32768 → 131072.
|
# Bumped both Qwen 3.6 entries ctx-size 32768 → 131072.
|
||||||
|
# 2026-04-22: New `pinned` group — qwen3.5-9b + qwen3.6-35-a3b coexist
|
||||||
|
# in VRAM with persistent=true and ttl=0. Means the two
|
||||||
|
# can be called concurrently and never idle-unload.
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
|
|
||||||
# Default 1200 seconds (20 min) to wait for model to be available to load.
|
# Default 1200 seconds (20 min) to wait for model to be available to load.
|
||||||
@@ -97,7 +100,7 @@ models:
|
|||||||
"qwen3.5-9b":
|
"qwen3.5-9b":
|
||||||
name: "Qwen 3.5 9B UD-Q4_K_XL"
|
name: "Qwen 3.5 9B UD-Q4_K_XL"
|
||||||
description: "Dense 9B model. Lightweight general-purpose chat and reasoning."
|
description: "Dense 9B model. Lightweight general-purpose chat and reasoning."
|
||||||
ttl: 600
|
ttl: 0 # pinned — member of the `pinned` group, never unloads
|
||||||
cmd: |
|
cmd: |
|
||||||
/app/llama-server
|
/app/llama-server
|
||||||
--context-shift
|
--context-shift
|
||||||
@@ -120,7 +123,7 @@ models:
|
|||||||
"qwen3.6-35-a3b":
|
"qwen3.6-35-a3b":
|
||||||
name: "Qwen 3.6 35B-A3B UD-Q6_K_XL"
|
name: "Qwen 3.6 35B-A3B UD-Q6_K_XL"
|
||||||
description: "Qwen3.6 MoE, 3B active of 35B. Stock unsloth Q6_K_XL (~29GB). Thinking on by default."
|
description: "Qwen3.6 MoE, 3B active of 35B. Stock unsloth Q6_K_XL (~29GB). Thinking on by default."
|
||||||
ttl: 600
|
ttl: 0 # pinned — member of the `pinned` group, never unloads
|
||||||
cmd: |
|
cmd: |
|
||||||
/app/llama-server
|
/app/llama-server
|
||||||
--context-shift
|
--context-shift
|
||||||
@@ -487,3 +490,15 @@ groups:
|
|||||||
- "embeddinggemma-300M"
|
- "embeddinggemma-300M"
|
||||||
- "qwen3-embedding-0.6B"
|
- "qwen3-embedding-0.6B"
|
||||||
- "qwen3-reranker-0.6B"
|
- "qwen3-reranker-0.6B"
|
||||||
|
|
||||||
|
# Pinned general-purpose chat models. Coexist in VRAM, never unload.
|
||||||
|
# Members also have ttl: 0 individually so idle-timeout can't drop them.
|
||||||
|
# VRAM budget check: qwen3.5-9b (~6 GB at Q4) + qwen3.6-35-a3b (~29 GB
|
||||||
|
# at Q6_K_XL) + 128K KV cache for each ≈ 50-60 GB. Size for your GPU.
|
||||||
|
"pinned":
|
||||||
|
swap: false
|
||||||
|
exclusive: false
|
||||||
|
persistent: true
|
||||||
|
members:
|
||||||
|
- "qwen3.5-9b"
|
||||||
|
- "qwen3.6-35-a3b"
|
||||||
|
|||||||
Reference in New Issue
Block a user