From 03d73a212e2615338424e51fb77e0f2f1bdd79b8 Mon Sep 17 00:00:00 2001
From: Vuong Hoang <vh@phasefinal.com>
Date: Fri, 5 Jun 2026 00:31:39 -0700
Subject: [PATCH] feat(llama-swap): serve Granite 4.1 3B + 8B instruct for
 brokkr R15 P03
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

IBM Granite 4.1 dense instruct (3B + 8B), unsloth Q4_K_M GGUF via -hf
syntax, 64K ctx with q8 KV. Auto-exposed through the LiteLLM gateway
wildcard (ana-docker:4000) and direct on llama-swap (:9292).

Replaces the retired irv-ml1 Ollama granite4.1 — Ollama is now banned
fleet-wide; serving consolidates onto the sanctioned llama-swap/vLLM
substrate.
---
 stacks/llama-swap/conf/config.yaml | 49 ++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/stacks/llama-swap/conf/config.yaml b/stacks/llama-swap/conf/config.yaml
index 247d4c9..a6d3c96 100644
--- a/stacks/llama-swap/conf/config.yaml
+++ b/stacks/llama-swap/conf/config.yaml
@@ -42,6 +42,11 @@
 #               via playbooks/pull-hf-repo.yaml. Sampler defaults mirrored
 #               from gemma4-31b-dense (temp 1.0, top-p 0.95, top-k 64,
 #               repeat_penalty 1.0); ctx 32K starting per Gemma 4 KB.
+#   2026-06-05: Added granite-4.1-3b + granite-4.1-8b (IBM Granite 4.1 dense
+#               INSTRUCT, unsloth Q4_K_M, -hf syntax, ctx 64K, q8 KV). Served
+#               for brokkr R15 model-fitness. Replaces the retired irv-ml1
+#               Ollama granite4.1 — Ollama is now banned fleet-wide. Auto-
+#               exposed via the LiteLLM gateway wildcard (ana-docker:4000).
 # ============================================================================
 
 # Default 1200 seconds (20 min) to wait for model to be available to load.
@@ -458,6 +463,50 @@ models:
   # /models/ibm-granite_granite-4.0-micro-GGUF/granite-4.0-micro-Q4_K_M.gguf
   # (NOT deleted), so this entry can be restored later if needed.
 
+  # ==========================================================================
+  # GRANITE 4.1 — dense INSTRUCT (general). Served for brokkr R15 model-fitness.
+  # - IBM Granite 4.1 instruct (the plain `granite-4.1-Nb` name IS the instruct
+  #   variant, finetuned from `-Base`). unsloth GGUF w/ chat-template fixes.
+  #   Apache-2.0. Tool-use + instruction-following + long-context.
+  # - Replaces the retired irv-ml1 Ollama granite4.1:3b/8b (Ollama banned
+  #   fleet-wide 2026-06-05). Auto-exposed via the LiteLLM gateway wildcard.
+  # - ctx 64K is well above brokkr's ~30K "full" cell; q8 KV keeps VRAM modest
+  #   so granite coexists with the swap pool. No sampling pinned — clients
+  #   drive temp/top-p per request (eval reproducibility is the caller's call).
+  # ==========================================================================
+
+  "granite-4.1-3b":
+    name: "Granite 4.1 3B Instruct (Q4_K_M)"
+    description: "IBM Granite 4.1 3B dense instruct. unsloth Q4_K_M (~2.1GB) with chat-template fixes. Apache-2.0. 64K ctx."
+    ttl: 600
+    cmd: |
+      /app/llama-server
+      --context-shift
+      --jinja
+      -hf unsloth/granite-4.1-3b-GGUF:Q4_K_M
+      --port ${PORT}
+      --n-gpu-layers 999
+      --ctx-size 65536
+      --cache-type-k q8_0
+      --cache-type-v q8_0
+      --flash-attn on
+
+  "granite-4.1-8b":
+    name: "Granite 4.1 8B Instruct (Q4_K_M)"
+    description: "IBM Granite 4.1 8B dense instruct. unsloth Q4_K_M (~5.35GB) with chat-template fixes. Apache-2.0. 64K ctx."
+    ttl: 600
+    cmd: |
+      /app/llama-server
+      --context-shift
+      --jinja
+      -hf unsloth/granite-4.1-8b-GGUF:Q4_K_M
+      --port ${PORT}
+      --n-gpu-layers 999
+      --ctx-size 65536
+      --cache-type-k q8_0
+      --cache-type-v q8_0
+      --flash-attn on
+
   # ==========================================================================
   # JUDGE / EVAL MODELS
   # - Designed for evaluation/scoring rather than chat/reasoning.