vh/esh-pfi-infrastructure
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects Releases Wiki Activity
Files
graphify-tooling
esh-pfi-infrastructure/stacks/llama-swap/conf/config.yaml
T
vh 03d73a212e feat(llama-swap): serve Granite 4.1 3B + 8B instruct for brokkr R15 P03 
IBM Granite 4.1 dense instruct (3B + 8B), unsloth Q4_K_M GGUF via -hf
syntax, 64K ctx with q8 KV. Auto-exposed through the LiteLLM gateway
wildcard (ana-docker:4000) and direct on llama-swap (:9292).

Replaces the retired irv-ml1 Ollama granite4.1 — Ollama is now banned
fleet-wide; serving consolidates onto the sanctioned llama-swap/vLLM
substrate.
2026-06-05 00:31:39 -07:00

655 lines
24 KiB
YAML
Raw Permalink Blame History

# ============================================================================
# llama-swap configuration for PFI-ANA
# Optimized and synchronized with /models disk inventory
# Last updated: 2026-04-10
#
# KB Sources:
# - reference/nemotron-3-super-running-parameters.md
# - reference/nemotron-3-nano-running-parameters.md
# - reference/qwen3.5-running-parameters.md
# - reference/qwen3-coder-next-running-parameters.md
# - reference/gemma-4-running-parameters.md
# - reference/qwen3-embedding-running-parameters.md
# - reference/qwen3-reranker-running-parameters.md
#
# Changelog:
# 2025-07-22: Removed jina-reranker-v3 (unused, out of rotation).
# 2026-04-10: Fixed Qwen3-Embedding pooling (mean→last; causal LM uses last-token
# pooling). Fixed ctx-size 4096→8192 for embedding+reranker. Fixed
# reranker: removed --embeddings flag (not an embedding model).
# 2026-04-17: Added Qwen3.6-35B-A3B Abliterated Heretic Q8_0 via -hf syntax.
# Requires HF_HOME=/hfcache in compose (see docker-compose.yml).
# New convention: use -hf repo[:quant] instead of --model /path.
# 2026-04-20: Added stock unsloth Qwen3.6-35B-A3B at Q6_K_XL via -hf syntax.
# Swapped abliterated entry from IIEleven11 Heretic Q8_0 to
# mradermacher abliterated i1-Q6_K (already cached).
# Bumped both Qwen 3.6 entries ctx-size 32768 → 131072.
# 2026-04-22: New `pinned` group — qwen3.5-9b + qwen3.6-35-a3b coexist
# in VRAM with persistent=true and ttl=0. Means the two
# can be called concurrently and never idle-unload.
# 2026-04-26: Added qwen3.6-35-a3b-heretic (llmfan46 uncensored heretic
# Q6_K, ~28GB) and qwen3.6-27b (unsloth dynamic UD-Q6_K_XL,
# ~26GB). Both pre-pulled into /tank/aimodels/huggingface
# via huggingface_hub.snapshot_download with hf_transfer.
# Both repos include mmproj-BF16.gguf — llama-server's -hf
# flag auto-loads it, so vision works without --mmproj.
# 2026-05-13: Added selene-1-mini-8b (AtlaAI judge model, mradermacher
# imatrix Q6_K, ~6.5GB). New JUDGE / EVAL section. Pre-pulled
# via playbooks/pull-hf-repo.yaml (canonical replacement
# for ad-hoc huggingface_hub.snapshot_download calls).
# 2026-05-25: Added artemis-31b-v1i (BeaverAI Artemis-31B-v1i Gemma 4
# dense fine-tune, Q6_K ~28.6GB) via -hf syntax. Pre-pulled
# via playbooks/pull-hf-repo.yaml. Sampler defaults mirrored
# from gemma4-31b-dense (temp 1.0, top-p 0.95, top-k 64,
# repeat_penalty 1.0); ctx 32K starting per Gemma 4 KB.
# 2026-06-05: Added granite-4.1-3b + granite-4.1-8b (IBM Granite 4.1 dense
# INSTRUCT, unsloth Q4_K_M, -hf syntax, ctx 64K, q8 KV). Served
# for brokkr R15 model-fitness. Replaces the retired irv-ml1
# Ollama granite4.1 — Ollama is now banned fleet-wide. Auto-
# exposed via the LiteLLM gateway wildcard (ana-docker:4000).
# ============================================================================
# Default 1200 seconds (20 min) to wait for model to be available to load.
healthCheckTimeout: 1200
# logLevel: sets the logging value
# - optional, default: info
# - Valid log levels: debug, info, warn, error
logLevel: info
# metricsMaxInMemory: maximum number of metrics to keep in memory
# - optional, default: 1000
metricsMaxInMemory: 1000
# startPort: sets the starting port number for the automatic ${PORT} macro.
# - optional, default: 5800
# - the ${PORT} macro can be used in model.cmd and model.proxy settings
# - it is automatically incremented for every model that uses it
# startPort: 10001
models:
# ==========================================================================
# QWEN 3.5 MODELS (KB-recommended settings)
# - Thinking mode: temp 1.0, top-p 0.95, top-k 20, min-p 0.0, presence_penalty 1.5
# - Coding (precise): temp 0.6, top-p 0.95, top-k 20, min-p 0.0, presence_penalty 0.0
# - Non-thinking general: temp 0.7, top-p 0.8, top-k 20, min-p 0.0, presence_penalty 1.5
# - Context: 256K native (start 16K-32K for responsiveness)
# - Gibberish fix: add --cache-type-k bf16 --cache-type-v bf16
# - No Ollama support for Qwen3.5 GGUFs — use llama.cpp only
# ==========================================================================
"qwen3.5-35-a3b":
name: "Qwen 3.5 35B-A3B Thinking"
description: "MoE reasoning model. 3B active params, general-purpose thinking/chat."
ttl: 600
cmd: |
/app/llama-server
--context-shift
--model /models/unsloth_Qwen3.5-35B-A3B-GGUF/Qwen3.5-35B-A3B-UD-Q4_K_XL.gguf
--port ${PORT}
--n-gpu-layers 999
--ctx-size 32768
--flash-attn on
--temp 1.0
--top-p 0.95
--top-k 20
--min-p 0.00
--presence-penalty 1.5
--chat-template-kwargs '{"enable_thinking":true}'
"qwen3.5-122b-a10b":
name: "Qwen 3.5 122B-A10B UD-Q4_K_XL"
description: "Large MoE reasoning model. 10B active params, heavy reasoning tasks."
ttl: 600
cmd: |
/app/llama-server
--context-shift
--model /models/unsloth_Qwen3.5-122B-A10B-GGUF/UD-Q4_K_XL/Qwen3.5-122B-A10B-UD-Q4_K_XL-00001-of-00003.gguf
--port ${PORT}
--n-gpu-layers 999
--ctx-size 32768
--flash-attn on
--temp 1.0
--top-p 0.95
--top-k 20
--min-p 0.00
--presence-penalty 1.5
--chat-template-kwargs '{"enable_thinking":true}'
"qwen3.5-9b":
name: "Qwen 3.5 9B UD-Q4_K_XL"
description: "Dense 9B model. Lightweight general-purpose chat and reasoning."
ttl: 0 # pinned — member of the `pinned` group, never unloads
cmd: |
/app/llama-server
--context-shift
--model /models/unsloth_Qwen3.5-9B-GGUF/Qwen3.5-9B-UD-Q4_K_XL.gguf
--port ${PORT}
--n-gpu-layers 999
--ctx-size 32768
--flash-attn on
--temp 1.0
--top-p 0.95
--top-k 20
--min-p 0.00
--presence-penalty 1.5
--chat-template-kwargs '{"enable_thinking":true}'
# --------------------------------------------------------------------------
# Qwen 3.6 — uses -hf syntax, reads from HF_HOME=/hfcache (host pre-download)
# --------------------------------------------------------------------------
"qwen3.6-35-a3b":
name: "Qwen 3.6 35B-A3B UD-Q6_K_XL"
description: "Qwen3.6 MoE, 3B active of 35B. Stock unsloth Q6_K_XL (~29GB). Thinking on by default."
ttl: 0 # never idle-unload, but evictable under memory pressure
# (out of `pinned` group as of 2026-04-27 — was OOM'ing other loads).
cmd: |
/app/llama-server
--context-shift
--jinja
-hf unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q6_K_XL
--port ${PORT}
--n-gpu-layers 999
--ctx-size 131072
--flash-attn on
--temp 1.0
--top-p 0.95
--top-k 20
--min-p 0.00
--presence-penalty 1.5
--repeat-penalty 1.0
--reasoning on
--reasoning-format deepseek
"qwen3.6-35-a3b-abliterated":
name: "Qwen 3.6 35B-A3B Abliterated i1-Q6_K"
description: "Qwen3.6 MoE, 3B active of 35B. mradermacher abliterated imatrix Q6_K (~27GB)."
# No ttl — stays loaded across idle. Still gets evicted by the
# normal swap when another non-pinned model is requested.
cmd: |
/app/llama-server
--context-shift
--jinja
-hf mradermacher/Qwen3.6-35B-A3B-abliterated-i1-GGUF:i1-Q6_K
--port ${PORT}
--n-gpu-layers 999
--ctx-size 131072
--flash-attn on
--temp 1.0
--top-p 0.95
--top-k 20
--min-p 0.00
--presence-penalty 1.5
--repeat-penalty 1.0
--reasoning on
--reasoning-format deepseek
"qwen3.6-35-a3b-heretic":
name: "Qwen 3.6 35B-A3B Uncensored Heretic Q6_K (vision)"
description: "Qwen3.6 MoE, 3B active of 35B. llmfan46 uncensored 'heretic' Q6_K (~28GB) + auto-loaded BF16 mmproj for vision."
# No ttl — stays loaded across idle. Still gets evicted by the
# normal swap when another non-pinned model is requested.
cmd: |
/app/llama-server
--context-shift
--jinja
-hf llmfan46/Qwen3.6-35B-A3B-uncensored-heretic-GGUF:Q6_K
--port ${PORT}
--n-gpu-layers 999
--ctx-size 131072
--flash-attn on
--temp 1.0
--top-p 0.95
--top-k 20
--min-p 0.00
--presence-penalty 1.5
--repeat-penalty 1.0
--reasoning on
--reasoning-format deepseek
"qwen3.6-27b":
name: "Qwen 3.6 27B UD-Q6_K_XL (vision)"
description: "Qwen3.6 dense 27B. Unsloth dynamic Q6_K_XL (~26GB) + auto-loaded BF16 mmproj for vision."
# No ttl — stays loaded across idle. Still gets evicted by the
# normal swap when another non-pinned model is requested.
cmd: |
/app/llama-server
--context-shift
--jinja
-hf unsloth/Qwen3.6-27B-GGUF:UD-Q6_K_XL
--port ${PORT}
--n-gpu-layers 999
--ctx-size 131072
--flash-attn on
--temp 1.0
--top-p 0.95
--top-k 20
--min-p 0.00
--presence-penalty 1.5
--repeat-penalty 1.0
--reasoning on
--reasoning-format deepseek
# ==========================================================================
# NEMOTRON MODELS (KB-recommended settings)
# - General Chat: temp 1.0, top-p 1.0, min_p 0.01
# - Tool Calling: temp 0.6, top-p 0.95, min_p 0.01
# - NoPE architecture: no YaRN needed
# - DEPRECATED --special flag for reasoning tokens
# - --special flag causes issues.
# - Start ctx 16K-32K, increase cautiously
# ==========================================================================
"nemotron-3-super-120b":
name: "NVIDIA Nemotron 3 Super 120B-A12B UD-Q4_K_XL"
description: "Flagship NVIDIA reasoning model. 12B active of 120B, MoE. 64-72GB VRAM at Q4."
ttl: 600
cmd: |
/app/llama-server
--context-shift
--model /models/unsloth_NVIDIA-Nemotron-3-Super-120B-A12B-GGUF/UD-Q4_K_XL/NVIDIA-Nemotron-3-Super-120B-A12B-UD-Q4_K_XL-00001-of-00003.gguf
--port ${PORT}
--n-gpu-layers 999
--ctx-size 16384
--flash-attn on
--temp 1.0
--top-p 1.0
--min-p 0.01
--seed 3407
"nemotron-3-nano-30b":
name: "NVIDIA Nemotron 3 Nano 30B-A3B UD-Q4_K_XL"
description: "Compact Nemotron. 3B active of 30B, MoE. ~24GB at Q4. Best performance/size on 24GB GPUs."
ttl: 600
cmd: |
/app/llama-server
--context-shift
--special
--model /models/unsloth_Nemotron-3-Nano-30B-A3B-GGUF/Nemotron-3-Nano-30B-A3B-UD-Q4_K_XL.gguf
--port ${PORT}
--n-gpu-layers 999
--ctx-size 32768
--flash-attn on
--temp 1.0
--top-p 1.0
--min-p 0.01
--seed 3407
# ==========================================================================
# GEMMA 4 MODELS (KB-recommended settings)
# - All variants: temp 1.0, top-p 0.95, top-k 64, repeat_penalty 1.0
# - Thinking: enable via --chat-template-kwargs '{"enable_thinking":true}'
# - Multi-turn: only keep final visible answer in history (not thought blocks)
# - Context: E2B/E4B=128K, 26B-A4B/31B=256K. Start at 32K.
# - ⚠️ Do NOT use CUDA 13.2 runtime — causes poor outputs
# - Use llama-server (not llama-cli) for thinking control
# ==========================================================================
"gemma4-26b-a4b":
name: "Gemma 4 26B-A4B"
description: "Google DeepMind Gemma 4 MoE, 4B active params, 256K context. Best speed/quality tradeoff."
ttl: 600
cmd: |
/app/llama-server
--context-shift
--model /models/unsloth_gemma-4-26B-A4B-it-GGUF/gemma-4-26B-A4B-it-UD-Q4_K_XL.gguf
--port ${PORT}
--n-gpu-layers 999
--ctx-size 32768
--flash-attn on
--temp 1.0
--top-p 0.95
--top-k 64
--repeat-penalty 1.0
--chat-template-kwargs '{"enable_thinking":true}'
"gemma4-31b-dense":
name: "Gemma 4 31B Dense"
description: "Google DeepMind Gemma 4 dense 31B. Maximum quality for complex reasoning, 256K context."
ttl: 600
cmd: |
/app/llama-server
--context-shift
--model /models/unsloth_gemma-4-31B-it-GGUF/gemma-4-31B-it-UD-Q4_K_XL.gguf
--port ${PORT}
--n-gpu-layers 999
--ctx-size 32768
--flash-attn on
--temp 1.0
--top-p 0.95
--top-k 64
--repeat-penalty 1.0
--chat-template-kwargs '{"enable_thinking":true}'
"artemis-31b-v1i":
name: "Artemis 31B v1i Q6_K"
description: "BeaverAI Artemis-31B-v1i — Gemma 4 31B dense fine-tune, Q6_K (~28.6GB). 256K context native; served at 128K."
ttl: 600
cmd: |
/app/llama-server
--context-shift
-hf BeaverAI/Artemis-31B-v1i-GGUF:Q6_K
--port ${PORT}
--n-gpu-layers 999
--ctx-size 131072
--flash-attn on
--temp 1.0
--top-p 0.95
--top-k 64
--repeat-penalty 1.0
--chat-template-kwargs '{"enable_thinking":true}'
--reasoning on
--reasoning-format deepseek
# ==========================================================================
# GLM MODELS
# ==========================================================================
"glm4.7-flash":
name: "GLM 4.7 Flash UD-Q4_K_XL"
description: "THUDM GLM 4.7 Flash. Fast inference, general-purpose chat."
ttl: 600
cmd: |
/app/llama-server
--context-shift
--model /models/unsloth_GLM-4.7-Flash-GGUF/GLM-4.7-Flash-UD-Q4_K_XL.gguf
--port ${PORT}
--n-gpu-layers 999
--ctx-size 40000
--flash-attn on
--temp 0.6
--top-p 0.95
"glm-steam-106b":
name: "GLM Steam 106B-A12B Q4_K_M"
description: "TheDrummer GLM Steam MoE. 12B active of 106B. Creative and RP-focused."
ttl: 600
cmd: |
/app/llama-server
--context-shift
--model /models/RP/bartowski_TheDrummer_GLM-Steam-106B-A12B-v1-GGUF/TheDrummer_GLM-Steam-106B-A12B-v1-Q4_K_M-00001-of-00002.gguf
--port ${PORT}
--n-gpu-layers 999
--ctx-size 40000
--flash-attn on
--temp 0.6
--top-p 0.95
# ==========================================================================
# SKYFALL MODELS
# ==========================================================================
"skyfall-r1-31b-q6k":
name: "Skyfall R1 31B v4 Q6_K_L"
description: "TheDrummer Skyfall R1 31B v4. General-purpose reasoning."
ttl: 600
cmd: |
/app/llama-server
--context-shift
--model /models/bartowski_TheDrummer_Skyfall-31B-v4-GGUF/TheDrummer_Skyfall-31B-v4-Q6_K_L.gguf
--port ${PORT}
--n-gpu-layers 999
--ctx-size 40000
--flash-attn on
"skyfall-r1-31b-v4a":
name: "Skyfall R1 31B v4a Q6_K (RP)"
description: "BeaverAI Skyfall R1 v4a variant. RP/creative-focused."
ttl: 600
cmd: |
/app/llama-server
--context-shift
--model /models/RP/BeaverAI_Skyfall-R1-31B-v4a-GGUF/Skyfall-R1-31B-v4a-Q6_K.gguf
--port ${PORT}
--n-gpu-layers 999
--ctx-size 40000
--flash-attn on
# ==========================================================================
# CODER MODELS
# ==========================================================================
"qwen3-coder-next":
name: "Qwen3 Coder Next UD-Q4_K_XL"
description: "Latest Qwen3 Coder. Non-reasoning model, optimized for code gen. KB: temp 1.0, top-k 40, min-p 0.01."
ttl: 600
cmd: |
/app/llama-server
--context-shift
--model /models/unsloth_Qwen3-Coder-Next-GGUF/Qwen3-Coder-Next-UD-Q4_K_XL.gguf
--port ${PORT}
--n-gpu-layers 999
--ctx-size 32768
--flash-attn on
--temp 1.0
--top-p 0.95
--top-k 40
--min-p 0.01
--repeat-penalty 1.0
# ==========================================================================
# LARGE / SPECIAL-PURPOSE MODELS
# ==========================================================================
"kimik2-q2kxl":
name: "Kimi K2 Instruct UD-Q2_K_XL"
description: "Moonshot Kimi K2. Huge MoE model (8-shard Q2). Limited GPU layers due to size."
ttl: 600
cmd: |
/app/llama-server
--context-shift
--model /models/unsloth_Kimi-K2-Instruct-0905-GGUF/UD-Q2_K_XL/Kimi-K2-Instruct-0905-UD-Q2_K_XL-00001-of-00008.gguf
--port ${PORT}
--n-gpu-layers 2
--temp 0.6
--top-p 0.95
# ==========================================================================
# GRANITE MODELS (IBM)
# ==========================================================================
# ── "granite-4-small" REMOVED 2026-06-04 ──────────────────────────────────
# Superseded by phi4-mini (summarizer + dreaming agent), now served via vLLM
# FP8 on ana-ml2 (stacks/vllm → vllm-phi4, :8004). It was pinned at 120K ctx
# (~24 GB resident: ~6 GB weights + ~18 GB KV); removing it reclaims that VRAM.
# DOWNSTREAM: repoint the news-digest curator from this llama-swap endpoint to
# the phi4-mini vLLM endpoint at/before deploy.
# ── "granite-4-micro" config REMOVED 2026-06-04 ───────────────────────────
# Retired from llama-swap alongside granite-4-small (both superseded by
# phi4-mini). Per operator: CONFIG ONLY — the GGUF stays on disk at
# /models/ibm-granite_granite-4.0-micro-GGUF/granite-4.0-micro-Q4_K_M.gguf
# (NOT deleted), so this entry can be restored later if needed.
# ==========================================================================
# GRANITE 4.1 — dense INSTRUCT (general). Served for brokkr R15 model-fitness.
# - IBM Granite 4.1 instruct (the plain `granite-4.1-Nb` name IS the instruct
# variant, finetuned from `-Base`). unsloth GGUF w/ chat-template fixes.
# Apache-2.0. Tool-use + instruction-following + long-context.
# - Replaces the retired irv-ml1 Ollama granite4.1:3b/8b (Ollama banned
# fleet-wide 2026-06-05). Auto-exposed via the LiteLLM gateway wildcard.
# - ctx 64K is well above brokkr's ~30K "full" cell; q8 KV keeps VRAM modest
# so granite coexists with the swap pool. No sampling pinned — clients
# drive temp/top-p per request (eval reproducibility is the caller's call).
# ==========================================================================
"granite-4.1-3b":
name: "Granite 4.1 3B Instruct (Q4_K_M)"
description: "IBM Granite 4.1 3B dense instruct. unsloth Q4_K_M (~2.1GB) with chat-template fixes. Apache-2.0. 64K ctx."
ttl: 600
cmd: |
/app/llama-server
--context-shift
--jinja
-hf unsloth/granite-4.1-3b-GGUF:Q4_K_M
--port ${PORT}
--n-gpu-layers 999
--ctx-size 65536
--cache-type-k q8_0
--cache-type-v q8_0
--flash-attn on
"granite-4.1-8b":
name: "Granite 4.1 8B Instruct (Q4_K_M)"
description: "IBM Granite 4.1 8B dense instruct. unsloth Q4_K_M (~5.35GB) with chat-template fixes. Apache-2.0. 64K ctx."
ttl: 600
cmd: |
/app/llama-server
--context-shift
--jinja
-hf unsloth/granite-4.1-8b-GGUF:Q4_K_M
--port ${PORT}
--n-gpu-layers 999
--ctx-size 65536
--cache-type-k q8_0
--cache-type-v q8_0
--flash-attn on
# ==========================================================================
# JUDGE / EVAL MODELS
# - Designed for evaluation/scoring rather than chat/reasoning.
# - Deterministic-leaning defaults (low temp, no repeat penalty) so the
# same prompt scores consistently across re-runs.
# ==========================================================================
"selene-1-mini-8b":
name: "Selene 1 Mini Llama 3.1 8B iMat-Q6_K"
description: "AtlaAI's small judge model. Llama 3.1 8B base, iMatrix-weighted Q6_K (~6.5GB; quality-leaning quant). Apache-2.0. 84% on RAGTruth hallucination eval per Atla cookbook with these defaults."
ttl: 600
cmd: |
/app/llama-server
--jinja
-hf mradermacher/Selene-1-Mini-Llama-3.1-8B-i1-GGUF:Q6_K
--port ${PORT}
--n-gpu-layers 999
--ctx-size 32768
--cache-type-k q8_0
--cache-type-v q8_0
--flash-attn on
--temp 0.01
--top-p 1.0
--repeat-penalty 1.0
# ==========================================================================
# EMBEDDING MODELS (persistent, always loaded)
# ==========================================================================
"embeddinggemma-300M":
name: "Embedding Gemma 300M"
description: "Google Embedding Gemma for vectorization."
ttl: 0
cmd: |
/app/llama-server
--embedding
--pooling cls
--model /models/ggml-org_embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf
--port ${PORT}
--n-gpu-layers 0
--ctx-size 2048
--batch-size 1024
--no-mmap
--ubatch-size 1024
--cont-batching
--threads 24
"qwen3-embedding-0.6B":
name: "Qwen3 Embedding 0.6B"
description: "Qwen3 Embedding model for vectorization. 32K context, last-token pooling (decoder/causal LM)."
ttl: 0
cmd: |
/app/llama-server
--embeddings
--pooling last
--model /models/Qwen_Qwen3-Embedding-0.6B-GGUF/Qwen3-Embedding-0.6B-Q8_0.gguf
--port ${PORT}
--n-gpu-layers 0
--ctx-size 8192
--batch-size 8192
--ubatch-size 2048
--no-mmap
--cont-batching
--threads 24
# ==========================================================================
# RERANKING MODELS (persistent, always loaded)
# ==========================================================================
"qwen3-reranker-0.6B":
name: "Qwen3 Reranker 0.6B"
description: "Qwen3 Reranker for retrieval reranking. Causal LM scoring yes/no logits at last token. Replaces BGE v2."
ttl: 0
cmd: |
/app/llama-server
--reranking
--pooling rank
--model /models/ggml-org_Qwen3-Reranker-0.6B-Q8_0-GGUF/qwen3-reranker-0.6b-q8_0.gguf
--port ${PORT}
--n-gpu-layers 0
--ctx-size 8192
--batch-size 8192
--ubatch-size 2048
--cont-batching
--threads 24
# NOTE: jina-reranker-v3 removed 2025-07-22 — unused, out of rotation.
# Qwen3 Reranker handles all reranking duties.
# ============================================================================
# GROUPS
# - swap: false = models in group can coexist in memory
# - exclusive: false = group can share memory with other groups
# - persistent: true = models never unload (for utility/embedding)
# ============================================================================
groups:
"high-reasoning":
swap: false
exclusive: false
members:
- "qwen3.5-35-a3b"
- "gemma4-31b-dense"
- "nemotron-3-nano-30b"
"heavy-moe":
swap: false
exclusive: false
members:
- "qwen3.5-122b-a10b"
- "nemotron-3-super-120b"
- "kimik2-q2kxl"
- "glm-steam-106b"
"utility":
swap: false
exclusive: false
persistent: true
members:
- "embeddinggemma-300M"
- "qwen3-embedding-0.6B"
- "qwen3-reranker-0.6B"
# Pinned general-purpose / utility models. Coexist in VRAM, never
# unload. Members also have ttl: 0 individually so idle-timeout can't
# drop them.
#
# Current pins:
# qwen3.5-9b — ~6 GB at Q4 + KV. General-purpose chat baseline.
# VRAM budget: ~6 GB persistent in the pin slot. Single RTX 6000 Ada
# is 48 GB, so this leaves ~40 GB for whichever non-pinned model the
# user invokes alongside.
#
# granite-4-small WAS pinned here; removed 2026-06-04 — superseded by
# phi4-mini (vLLM FP8, stacks/vllm → vllm-phi4). Freed ~24 GB (120K KV).
#
# qwen3.6-35-a3b WAS in this group; removed 2026-04-27 because its
# ~29 GB at Q6_K_XL pushed concurrent loads OOM. Now lives outside
# with ttl: 0 — never idle-unloads but evictable under memory pressure.
"pinned":
swap: false
exclusive: false
persistent: true
members:
- "qwen3.5-9b"
Reference in New Issue View Git Blame Copy Permalink