# ============================================================================
# llama-swap configuration for PFI-ANA
# Optimized and synchronized with /models disk inventory
# Last updated: 2026-04-10
#
# KB Sources:
#   - reference/nemotron-3-super-running-parameters.md
#   - reference/nemotron-3-nano-running-parameters.md
#   - reference/qwen3.5-running-parameters.md
#   - reference/qwen3-coder-next-running-parameters.md
#   - reference/gemma-4-running-parameters.md
#   - reference/qwen3-embedding-running-parameters.md
#   - reference/qwen3-reranker-running-parameters.md
#
# Changelog:
#   2025-07-22: Removed jina-reranker-v3 (unused, out of rotation).
#   2026-04-10: Fixed Qwen3-Embedding pooling (mean→last; causal LM uses last-token
#               pooling). Fixed ctx-size 4096→8192 for embedding+reranker. Fixed
#               reranker: removed --embeddings flag (not an embedding model).
#   2026-04-17: Added Qwen3.6-35B-A3B Abliterated Heretic Q8_0 via -hf syntax.
#               Requires HF_HOME=/hfcache in compose (see docker-compose.yml).
#               New convention: use -hf repo[:quant] instead of --model /path.
#   2026-04-20: Added stock unsloth Qwen3.6-35B-A3B at Q6_K_XL via -hf syntax.
#               Swapped abliterated entry from IIEleven11 Heretic Q8_0 to
#               mradermacher abliterated i1-Q6_K (already cached).
#               Bumped both Qwen 3.6 entries ctx-size 32768 → 131072.
#   2026-04-22: New `pinned` group — qwen3.5-9b + qwen3.6-35-a3b coexist
#               in VRAM with persistent=true and ttl=0. Means the two
#               can be called concurrently and never idle-unload.
#   2026-04-26: Added qwen3.6-35-a3b-heretic (llmfan46 uncensored heretic
#               Q6_K, ~28GB) and qwen3.6-27b (unsloth dynamic UD-Q6_K_XL,
#               ~26GB). Both pre-pulled into /tank/aimodels/huggingface
#               via huggingface_hub.snapshot_download with hf_transfer.
#               Both repos include mmproj-BF16.gguf — llama-server's -hf
#               flag auto-loads it, so vision works without --mmproj.
#   2026-05-13: Added selene-1-mini-8b (AtlaAI judge model, mradermacher
#               imatrix Q6_K, ~6.5GB). New JUDGE / EVAL section. Pre-pulled
#               via playbooks/pull-hf-repo.yaml (canonical replacement
#               for ad-hoc huggingface_hub.snapshot_download calls).
#   2026-05-25: Added artemis-31b-v1i (BeaverAI Artemis-31B-v1i Gemma 4
#               dense fine-tune, Q6_K ~28.6GB) via -hf syntax. Pre-pulled
#               via playbooks/pull-hf-repo.yaml. Sampler defaults mirrored
#               from gemma4-31b-dense (temp 1.0, top-p 0.95, top-k 64,
#               repeat_penalty 1.0); ctx 32K starting per Gemma 4 KB.
#   2026-06-05: Added granite-4.1-3b + granite-4.1-8b (IBM Granite 4.1 dense
#               INSTRUCT, unsloth Q4_K_M, -hf syntax, ctx 64K, q8 KV). Served
#               for brokkr R15 model-fitness. Replaces the retired irv-ml1
#               Ollama granite4.1 — Ollama is now banned fleet-wide. Auto-
#               exposed via the LiteLLM gateway wildcard (ana-docker:4000).
# ============================================================================

# Default 1200 seconds (20 min) to wait for model to be available to load.
healthCheckTimeout: 1200

# logLevel: sets the logging value
# - optional, default: info
# - Valid log levels: debug, info, warn, error
logLevel: info

# metricsMaxInMemory: maximum number of metrics to keep in memory
# - optional, default: 1000
metricsMaxInMemory: 1000

# startPort: sets the starting port number for the automatic ${PORT} macro.
# - optional, default: 5800
# - the ${PORT} macro can be used in model.cmd and model.proxy settings
# - it is automatically incremented for every model that uses it
# startPort: 10001

models:

  # ==========================================================================
  # QWEN 3.5 MODELS (KB-recommended settings)
  # - Thinking mode: temp 1.0, top-p 0.95, top-k 20, min-p 0.0, presence_penalty 1.5
  # - Coding (precise): temp 0.6, top-p 0.95, top-k 20, min-p 0.0, presence_penalty 0.0
  # - Non-thinking general: temp 0.7, top-p 0.8, top-k 20, min-p 0.0, presence_penalty 1.5
  # - Context: 256K native (start 16K-32K for responsiveness)
  # - Gibberish fix: add --cache-type-k bf16 --cache-type-v bf16
  # - No Ollama support for Qwen3.5 GGUFs — use llama.cpp only
  # ==========================================================================

  "qwen3.5-35-a3b":
    name: "Qwen 3.5 35B-A3B Thinking"
    description: "MoE reasoning model. 3B active params, general-purpose thinking/chat."
    ttl: 600
    cmd: |
      /app/llama-server
      --context-shift
      --model /models/unsloth_Qwen3.5-35B-A3B-GGUF/Qwen3.5-35B-A3B-UD-Q4_K_XL.gguf
      --port ${PORT}
      --n-gpu-layers 999
      --ctx-size 32768
      --flash-attn on
      --temp 1.0
      --top-p 0.95
      --top-k 20
      --min-p 0.00
      --presence-penalty 1.5
      --chat-template-kwargs '{"enable_thinking":true}'

  "qwen3.5-122b-a10b":
    name: "Qwen 3.5 122B-A10B UD-Q4_K_XL"
    description: "Large MoE reasoning model. 10B active params, heavy reasoning tasks."
    ttl: 600
    cmd: |
      /app/llama-server
      --context-shift
      --model /models/unsloth_Qwen3.5-122B-A10B-GGUF/UD-Q4_K_XL/Qwen3.5-122B-A10B-UD-Q4_K_XL-00001-of-00003.gguf
      --port ${PORT}
      --n-gpu-layers 999
      --ctx-size 32768
      --flash-attn on
      --temp 1.0
      --top-p 0.95
      --top-k 20
      --min-p 0.00
      --presence-penalty 1.5
      --chat-template-kwargs '{"enable_thinking":true}'

  "qwen3.5-9b":
    name: "Qwen 3.5 9B UD-Q4_K_XL"
    description: "Dense 9B model. Lightweight general-purpose chat and reasoning."
    ttl: 0   # pinned — member of the `pinned` group, never unloads
    cmd: |
      /app/llama-server
      --context-shift
      --model /models/unsloth_Qwen3.5-9B-GGUF/Qwen3.5-9B-UD-Q4_K_XL.gguf
      --port ${PORT}
      --n-gpu-layers 999
      --ctx-size 32768
      --flash-attn on
      --temp 1.0
      --top-p 0.95
      --top-k 20
      --min-p 0.00
      --presence-penalty 1.5
      --chat-template-kwargs '{"enable_thinking":true}'

  # --------------------------------------------------------------------------
  # Qwen 3.6 — uses -hf syntax, reads from HF_HOME=/hfcache (host pre-download)
  # --------------------------------------------------------------------------

  "qwen3.6-35-a3b":
    name: "Qwen 3.6 35B-A3B UD-Q6_K_XL"
    description: "Qwen3.6 MoE, 3B active of 35B. Stock unsloth Q6_K_XL (~29GB). Thinking on by default."
    ttl: 0   # never idle-unload, but evictable under memory pressure
             # (out of `pinned` group as of 2026-04-27 — was OOM'ing other loads).
    cmd: |
      /app/llama-server
      --context-shift
      --jinja
      -hf unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q6_K_XL
      --port ${PORT}
      --n-gpu-layers 999
      --ctx-size 131072
      --flash-attn on
      --temp 1.0
      --top-p 0.95
      --top-k 20
      --min-p 0.00
      --presence-penalty 1.5
      --repeat-penalty 1.0
      --reasoning on
      --reasoning-format deepseek

  "qwen3.6-35-a3b-abliterated":
    name: "Qwen 3.6 35B-A3B Abliterated i1-Q6_K"
    description: "Qwen3.6 MoE, 3B active of 35B. mradermacher abliterated imatrix Q6_K (~27GB)."
    # No ttl — stays loaded across idle. Still gets evicted by the
    # normal swap when another non-pinned model is requested.
    cmd: |
      /app/llama-server
      --context-shift
      --jinja
      -hf mradermacher/Qwen3.6-35B-A3B-abliterated-i1-GGUF:i1-Q6_K
      --port ${PORT}
      --n-gpu-layers 999
      --ctx-size 131072
      --flash-attn on
      --temp 1.0
      --top-p 0.95
      --top-k 20
      --min-p 0.00
      --presence-penalty 1.5
      --repeat-penalty 1.0
      --reasoning on
      --reasoning-format deepseek

  "qwen3.6-35-a3b-heretic":
    name: "Qwen 3.6 35B-A3B Uncensored Heretic Q6_K (vision)"
    description: "Qwen3.6 MoE, 3B active of 35B. llmfan46 uncensored 'heretic' Q6_K (~28GB) + auto-loaded BF16 mmproj for vision."
    # No ttl — stays loaded across idle. Still gets evicted by the
    # normal swap when another non-pinned model is requested.
    cmd: |
      /app/llama-server
      --context-shift
      --jinja
      -hf llmfan46/Qwen3.6-35B-A3B-uncensored-heretic-GGUF:Q6_K
      --port ${PORT}
      --n-gpu-layers 999
      --ctx-size 131072
      --flash-attn on
      --temp 1.0
      --top-p 0.95
      --top-k 20
      --min-p 0.00
      --presence-penalty 1.5
      --repeat-penalty 1.0
      --reasoning on
      --reasoning-format deepseek

  "qwen3.6-27b":
    name: "Qwen 3.6 27B UD-Q6_K_XL (vision)"
    description: "Qwen3.6 dense 27B. Unsloth dynamic Q6_K_XL (~26GB) + auto-loaded BF16 mmproj for vision."
    # No ttl — stays loaded across idle. Still gets evicted by the
    # normal swap when another non-pinned model is requested.
    cmd: |
      /app/llama-server
      --context-shift
      --jinja
      -hf unsloth/Qwen3.6-27B-GGUF:UD-Q6_K_XL
      --port ${PORT}
      --n-gpu-layers 999
      --ctx-size 131072
      --flash-attn on
      --temp 1.0
      --top-p 0.95
      --top-k 20
      --min-p 0.00
      --presence-penalty 1.5
      --repeat-penalty 1.0
      --reasoning on
      --reasoning-format deepseek

  # ==========================================================================
  # NEMOTRON MODELS (KB-recommended settings)
  # - General Chat: temp 1.0, top-p 1.0, min_p 0.01
  # - Tool Calling: temp 0.6, top-p 0.95, min_p 0.01
  # - NoPE architecture: no YaRN needed
  # - DEPRECATED --special flag for reasoning tokens
  # - --special flag causes issues.
  # - Start ctx 16K-32K, increase cautiously
  # ==========================================================================

  "nemotron-3-super-120b":
    name: "NVIDIA Nemotron 3 Super 120B-A12B UD-Q4_K_XL"
    description: "Flagship NVIDIA reasoning model. 12B active of 120B, MoE. 64-72GB VRAM at Q4."
    ttl: 600
    cmd: |
      /app/llama-server
      --context-shift
      --model /models/unsloth_NVIDIA-Nemotron-3-Super-120B-A12B-GGUF/UD-Q4_K_XL/NVIDIA-Nemotron-3-Super-120B-A12B-UD-Q4_K_XL-00001-of-00003.gguf
      --port ${PORT}
      --n-gpu-layers 999
      --ctx-size 16384
      --flash-attn on
      --temp 1.0
      --top-p 1.0
      --min-p 0.01
      --seed 3407

  "nemotron-3-nano-30b":
    name: "NVIDIA Nemotron 3 Nano 30B-A3B UD-Q4_K_XL"
    description: "Compact Nemotron. 3B active of 30B, MoE. ~24GB at Q4. Best performance/size on 24GB GPUs."
    ttl: 600
    cmd: |
      /app/llama-server
      --context-shift
      --special
      --model /models/unsloth_Nemotron-3-Nano-30B-A3B-GGUF/Nemotron-3-Nano-30B-A3B-UD-Q4_K_XL.gguf
      --port ${PORT}
      --n-gpu-layers 999
      --ctx-size 32768
      --flash-attn on
      --temp 1.0
      --top-p 1.0
      --min-p 0.01
      --seed 3407

  # ==========================================================================
  # GEMMA 4 MODELS (KB-recommended settings)
  # - All variants: temp 1.0, top-p 0.95, top-k 64, repeat_penalty 1.0
  # - Thinking: enable via --chat-template-kwargs '{"enable_thinking":true}'
  # - Multi-turn: only keep final visible answer in history (not thought blocks)
  # - Context: E2B/E4B=128K, 26B-A4B/31B=256K. Start at 32K.
  # - ⚠️ Do NOT use CUDA 13.2 runtime — causes poor outputs
  # - Use llama-server (not llama-cli) for thinking control
  # ==========================================================================

  "gemma4-26b-a4b":
    name: "Gemma 4 26B-A4B"
    description: "Google DeepMind Gemma 4 MoE, 4B active params, 256K context. Best speed/quality tradeoff."
    ttl: 600
    cmd: |
      /app/llama-server
      --context-shift
      --model /models/unsloth_gemma-4-26B-A4B-it-GGUF/gemma-4-26B-A4B-it-UD-Q4_K_XL.gguf
      --port ${PORT}
      --n-gpu-layers 999
      --ctx-size 32768
      --flash-attn on
      --temp 1.0
      --top-p 0.95
      --top-k 64
      --repeat-penalty 1.0
      --chat-template-kwargs '{"enable_thinking":true}'

  "gemma4-31b-dense":
    name: "Gemma 4 31B Dense"
    description: "Google DeepMind Gemma 4 dense 31B. Maximum quality for complex reasoning, 256K context."
    ttl: 600
    cmd: |
      /app/llama-server
      --context-shift
      --model /models/unsloth_gemma-4-31B-it-GGUF/gemma-4-31B-it-UD-Q4_K_XL.gguf
      --port ${PORT}
      --n-gpu-layers 999
      --ctx-size 32768
      --flash-attn on
      --temp 1.0
      --top-p 0.95
      --top-k 64
      --repeat-penalty 1.0
      --chat-template-kwargs '{"enable_thinking":true}'

  "artemis-31b-v1i":
    name: "Artemis 31B v1i Q6_K"
    description: "BeaverAI Artemis-31B-v1i — Gemma 4 31B dense fine-tune, Q6_K (~28.6GB). 256K context native; served at 128K."
    ttl: 600
    cmd: |
      /app/llama-server
      --context-shift
      -hf BeaverAI/Artemis-31B-v1i-GGUF:Q6_K
      --port ${PORT}
      --n-gpu-layers 999
      --ctx-size 131072
      --flash-attn on
      --temp 1.0
      --top-p 0.95
      --top-k 64
      --repeat-penalty 1.0
      --chat-template-kwargs '{"enable_thinking":true}'
      --reasoning on
      --reasoning-format deepseek

  # ==========================================================================
  # GLM MODELS
  # ==========================================================================

  "glm4.7-flash":
    name: "GLM 4.7 Flash UD-Q4_K_XL"
    description: "THUDM GLM 4.7 Flash. Fast inference, general-purpose chat."
    ttl: 600
    cmd: |
      /app/llama-server
      --context-shift
      --model /models/unsloth_GLM-4.7-Flash-GGUF/GLM-4.7-Flash-UD-Q4_K_XL.gguf
      --port ${PORT}
      --n-gpu-layers 999
      --ctx-size 40000
      --flash-attn on
      --temp 0.6
      --top-p 0.95

  "glm-steam-106b":
    name: "GLM Steam 106B-A12B Q4_K_M"
    description: "TheDrummer GLM Steam MoE. 12B active of 106B. Creative and RP-focused."
    ttl: 600
    cmd: |
      /app/llama-server
      --context-shift
      --model /models/RP/bartowski_TheDrummer_GLM-Steam-106B-A12B-v1-GGUF/TheDrummer_GLM-Steam-106B-A12B-v1-Q4_K_M-00001-of-00002.gguf
      --port ${PORT}
      --n-gpu-layers 999
      --ctx-size 40000
      --flash-attn on
      --temp 0.6
      --top-p 0.95

  # ==========================================================================
  # SKYFALL MODELS
  # ==========================================================================

  "skyfall-r1-31b-q6k":
    name: "Skyfall R1 31B v4 Q6_K_L"
    description: "TheDrummer Skyfall R1 31B v4. General-purpose reasoning."
    ttl: 600
    cmd: |
      /app/llama-server
      --context-shift
      --model /models/bartowski_TheDrummer_Skyfall-31B-v4-GGUF/TheDrummer_Skyfall-31B-v4-Q6_K_L.gguf
      --port ${PORT}
      --n-gpu-layers 999
      --ctx-size 40000
      --flash-attn on

  "skyfall-r1-31b-v4a":
    name: "Skyfall R1 31B v4a Q6_K (RP)"
    description: "BeaverAI Skyfall R1 v4a variant. RP/creative-focused."
    ttl: 600
    cmd: |
      /app/llama-server
      --context-shift
      --model /models/RP/BeaverAI_Skyfall-R1-31B-v4a-GGUF/Skyfall-R1-31B-v4a-Q6_K.gguf
      --port ${PORT}
      --n-gpu-layers 999
      --ctx-size 40000
      --flash-attn on

  # ==========================================================================
  # CODER MODELS
  # ==========================================================================

  "qwen3-coder-next":
    name: "Qwen3 Coder Next UD-Q4_K_XL"
    description: "Latest Qwen3 Coder. Non-reasoning model, optimized for code gen. KB: temp 1.0, top-k 40, min-p 0.01."
    ttl: 600
    cmd: |
      /app/llama-server
      --context-shift
      --model /models/unsloth_Qwen3-Coder-Next-GGUF/Qwen3-Coder-Next-UD-Q4_K_XL.gguf
      --port ${PORT}
      --n-gpu-layers 999
      --ctx-size 32768
      --flash-attn on
      --temp 1.0
      --top-p 0.95
      --top-k 40
      --min-p 0.01
      --repeat-penalty 1.0

  # ==========================================================================
  # LARGE / SPECIAL-PURPOSE MODELS
  # ==========================================================================

  "kimik2-q2kxl":
    name: "Kimi K2 Instruct UD-Q2_K_XL"
    description: "Moonshot Kimi K2. Huge MoE model (8-shard Q2). Limited GPU layers due to size."
    ttl: 600
    cmd: |
      /app/llama-server
      --context-shift
      --model /models/unsloth_Kimi-K2-Instruct-0905-GGUF/UD-Q2_K_XL/Kimi-K2-Instruct-0905-UD-Q2_K_XL-00001-of-00008.gguf
      --port ${PORT}
      --n-gpu-layers 2
      --temp 0.6
      --top-p 0.95

  # ==========================================================================
  # GRANITE MODELS (IBM)
  # ==========================================================================

  # ── "granite-4-small" REMOVED 2026-06-04 ──────────────────────────────────
  # Superseded by phi4-mini (summarizer + dreaming agent), now served via vLLM
  # FP8 on ana-ml2 (stacks/vllm → vllm-phi4, :8004). It was pinned at 120K ctx
  # (~24 GB resident: ~6 GB weights + ~18 GB KV); removing it reclaims that VRAM.
  # DOWNSTREAM: repoint the news-digest curator from this llama-swap endpoint to
  # the phi4-mini vLLM endpoint at/before deploy.

  # ── "granite-4-micro" config REMOVED 2026-06-04 ───────────────────────────
  # Retired from llama-swap alongside granite-4-small (both superseded by
  # phi4-mini). Per operator: CONFIG ONLY — the GGUF stays on disk at
  # /models/ibm-granite_granite-4.0-micro-GGUF/granite-4.0-micro-Q4_K_M.gguf
  # (NOT deleted), so this entry can be restored later if needed.

  # ==========================================================================
  # GRANITE 4.1 — dense INSTRUCT (general). Served for brokkr R15 model-fitness.
  # - IBM Granite 4.1 instruct (the plain `granite-4.1-Nb` name IS the instruct
  #   variant, finetuned from `-Base`). unsloth GGUF w/ chat-template fixes.
  #   Apache-2.0. Tool-use + instruction-following + long-context.
  # - Replaces the retired irv-ml1 Ollama granite4.1:3b/8b (Ollama banned
  #   fleet-wide 2026-06-05). Auto-exposed via the LiteLLM gateway wildcard.
  # - ctx 64K is well above brokkr's ~30K "full" cell; q8 KV keeps VRAM modest
  #   so granite coexists with the swap pool. No sampling pinned — clients
  #   drive temp/top-p per request (eval reproducibility is the caller's call).
  # ==========================================================================

  "granite-4.1-3b":
    name: "Granite 4.1 3B Instruct (Q4_K_M)"
    description: "IBM Granite 4.1 3B dense instruct. unsloth Q4_K_M (~2.1GB) with chat-template fixes. Apache-2.0. 64K ctx."
    ttl: 600
    cmd: |
      /app/llama-server
      --context-shift
      --jinja
      -hf unsloth/granite-4.1-3b-GGUF:Q4_K_M
      --port ${PORT}
      --n-gpu-layers 999
      --ctx-size 65536
      --cache-type-k q8_0
      --cache-type-v q8_0
      --flash-attn on

  "granite-4.1-8b":
    name: "Granite 4.1 8B Instruct (Q4_K_M)"
    description: "IBM Granite 4.1 8B dense instruct. unsloth Q4_K_M (~5.35GB) with chat-template fixes. Apache-2.0. 64K ctx."
    ttl: 600
    cmd: |
      /app/llama-server
      --context-shift
      --jinja
      -hf unsloth/granite-4.1-8b-GGUF:Q4_K_M
      --port ${PORT}
      --n-gpu-layers 999
      --ctx-size 65536
      --cache-type-k q8_0
      --cache-type-v q8_0
      --flash-attn on

  # ==========================================================================
  # JUDGE / EVAL MODELS
  # - Designed for evaluation/scoring rather than chat/reasoning.
  # - Deterministic-leaning defaults (low temp, no repeat penalty) so the
  #   same prompt scores consistently across re-runs.
  # ==========================================================================

  "selene-1-mini-8b":
    name: "Selene 1 Mini Llama 3.1 8B iMat-Q6_K"
    description: "AtlaAI's small judge model. Llama 3.1 8B base, iMatrix-weighted Q6_K (~6.5GB; quality-leaning quant). Apache-2.0. 84% on RAGTruth hallucination eval per Atla cookbook with these defaults."
    ttl: 600
    cmd: |
      /app/llama-server
      --jinja
      -hf mradermacher/Selene-1-Mini-Llama-3.1-8B-i1-GGUF:Q6_K
      --port ${PORT}
      --n-gpu-layers 999
      --ctx-size 32768
      --cache-type-k q8_0
      --cache-type-v q8_0
      --flash-attn on
      --temp 0.01
      --top-p 1.0
      --repeat-penalty 1.0

  # ==========================================================================
  # EMBEDDING MODELS (persistent, always loaded)
  # ==========================================================================

  "embeddinggemma-300M":
    name: "Embedding Gemma 300M"
    description: "Google Embedding Gemma for vectorization."
    ttl: 0
    cmd: |
      /app/llama-server
      --embedding
      --pooling cls
      --model /models/ggml-org_embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf
      --port ${PORT}
      --n-gpu-layers 0
      --ctx-size 2048
      --batch-size 1024
      --no-mmap
      --ubatch-size 1024
      --cont-batching
      --threads 24

  "qwen3-embedding-0.6B":
    name: "Qwen3 Embedding 0.6B"
    description: "Qwen3 Embedding model for vectorization. 32K context, last-token pooling (decoder/causal LM)."
    ttl: 0
    cmd: |
      /app/llama-server
      --embeddings
      --pooling last
      --model /models/Qwen_Qwen3-Embedding-0.6B-GGUF/Qwen3-Embedding-0.6B-Q8_0.gguf
      --port ${PORT}
      --n-gpu-layers 0
      --ctx-size 8192
      --batch-size 8192
      --ubatch-size 2048
      --no-mmap
      --cont-batching
      --threads 24

  # ==========================================================================
  # RERANKING MODELS (persistent, always loaded)
  # ==========================================================================

  "qwen3-reranker-0.6B":
    name: "Qwen3 Reranker 0.6B"
    description: "Qwen3 Reranker for retrieval reranking. Causal LM scoring yes/no logits at last token. Replaces BGE v2."
    ttl: 0
    cmd: |
      /app/llama-server
      --reranking
      --pooling rank
      --model /models/ggml-org_Qwen3-Reranker-0.6B-Q8_0-GGUF/qwen3-reranker-0.6b-q8_0.gguf
      --port ${PORT}
      --n-gpu-layers 0
      --ctx-size 8192
      --batch-size 8192
      --ubatch-size 2048
      --cont-batching
      --threads 24

  # NOTE: jina-reranker-v3 removed 2025-07-22 — unused, out of rotation.
  # Qwen3 Reranker handles all reranking duties.

# ============================================================================
# GROUPS
# - swap: false = models in group can coexist in memory
# - exclusive: false = group can share memory with other groups
# - persistent: true = models never unload (for utility/embedding)
# ============================================================================

groups:
  "high-reasoning":
    swap: false
    exclusive: false
    members:
      - "qwen3.5-35-a3b"
      - "gemma4-31b-dense"
      - "nemotron-3-nano-30b"

  "heavy-moe":
    swap: false
    exclusive: false
    members:
      - "qwen3.5-122b-a10b"
      - "nemotron-3-super-120b"
      - "kimik2-q2kxl"
      - "glm-steam-106b"

  "utility":
    swap: false
    exclusive: false
    persistent: true
    members:
      - "embeddinggemma-300M"
      - "qwen3-embedding-0.6B"
      - "qwen3-reranker-0.6B"

  # Pinned general-purpose / utility models. Coexist in VRAM, never
  # unload. Members also have ttl: 0 individually so idle-timeout can't
  # drop them.
  #
  # Current pins:
  #   qwen3.5-9b      — ~6 GB at Q4 + KV. General-purpose chat baseline.
  # VRAM budget: ~6 GB persistent in the pin slot. Single RTX 6000 Ada
  # is 48 GB, so this leaves ~40 GB for whichever non-pinned model the
  # user invokes alongside.
  #
  # granite-4-small WAS pinned here; removed 2026-06-04 — superseded by
  # phi4-mini (vLLM FP8, stacks/vllm → vllm-phi4). Freed ~24 GB (120K KV).
  #
  # qwen3.6-35-a3b WAS in this group; removed 2026-04-27 because its
  # ~29 GB at Q6_K_XL pushed concurrent loads OOM. Now lives outside
  # with ttl: 0 — never idle-unloads but evictable under memory pressure.
  "pinned":
    swap: false
    exclusive: false
    persistent: true
    members:
      - "qwen3.5-9b"