feat(vllm,litellm): lfm2.5-2.6b non-prod bake-off alias for brokkr

vllm-lfm25 on ana-ml2 GPU1 :8021 (LiquidAI/LFM2.5-2.6B, util 0.09 into unreserved slack, max-len 16384, no reasoning-parser so content is non-empty). LiteLLM alias lfm2.5-2.6b with vendor sampling baked as default (temp 0.1; top_k 50 + repetition_penalty 1.1 via extra_body). Eval-only, not in any routing chain, pending operator ruling on LFM Open License production use.
2026-08-10 07:13:37 -07:00
parent c8acf60449
commit edc9f42da1
4 changed files with 100 additions and 1 deletions
@@ -426,6 +426,23 @@ model_list:
  #     model names. Removed so unknown models now fail loudly (404). Re-add an
  #     explicit per-model entry if a swappable zoo ever returns. ---

+  # --- lfm2.5-2.6b -> LiquidAI LFM2.5-2.6B (ana-ml2 GPU1 :8021, vLLM). NON-PROD bake-off
+  #     vs granite-4.1-8b (brokkr R-target 2026-08-10). LFM Open License v1.0 (<USD 10M-rev
+  #     commercial) - EVAL-ONLY pending operator ruling; NOT in any default/fallback chain.
+  #     Reasoning model served raw (no vLLM reasoning-parser) so content is non-empty.
+  #     Vendor sampling (temp 0.1 / top_k 50 / rep_pen 1.1) baked as the alias default. ---
+  - model_name: lfm2.5-2.6b
+    litellm_params:
+      model: hosted_vllm/lfm2.5-2.6b
+      api_base: http://10.250.50.54:8021/v1
+      api_key: os.environ/VLLM_API_KEY
+      temperature: 0.1
+      extra_body:
+        top_k: 50
+        repetition_penalty: 1.1
+    model_info:
+      mode: chat
+
 general_settings:
  master_key: os.environ/LITELLM_MASTER_KEY
  database_url: os.environ/DATABASE_URL
@@ -142,3 +142,18 @@ CODER_MAX_MODEL_LEN=8192
 CODER_KV_CACHE_DTYPE=fp8
 CODER_GPU_MEM_UTIL=0.06
 CODER_MAX_NUM_SEQS=32
+
+# LFM2.5-2.6B — NON-PRODUCTION bake-off alias vs granite (brokkr R-target 2026-08-10).
+# LiquidAI LFM Open License v1.0 (<USD 10M-rev commercial, not OSI) — eval-only pending
+# an operator production ruling. Reasoning model (emits <think>); served raw (no vLLM
+# reasoning-parser) so content is non-empty. util 0.09 (~8.6GB) fits GPU1's free slack
+# without touching production reservations; max-len 16384 keeps KV small (bake-off
+# doesn't need the 131k ceiling). Vendor sampling lives in the LiteLLM alias.
+LFM25_PORT=8021
+LFM25_GPU_ID=1
+LFM25_MODEL=LiquidAI/LFM2.5-2.6B
+LFM25_SERVED_NAME=lfm2.5-2.6b
+LFM25_MAX_MODEL_LEN=16384
+LFM25_KV_CACHE_DTYPE=auto
+LFM25_GPU_MEM_UTIL=0.09
+LFM25_MAX_NUM_SEQS=8
@@ -342,6 +342,73 @@ services:
      - homepage.description=Qwen2.5-Coder-1.5B FIM code-completion (ana-ml2, Zed edit-predictions)
      - homepage.href=http://10.250.50.54:${CODER_PORT}/docs

+  vllm-lfm25:
+    image: vllm/vllm-openai:${VLLM_VERSION}
+    container_name: vllm-lfm25
+    restart: unless-stopped
+    ipc: host
+    ports:
+      - "${LFM25_PORT}:8000"
+    volumes:
+      - /tank/aimodels/huggingface:/hfcache
+    environment:
+      - HF_HOME=/hfcache
+      - HF_HUB_CACHE=/hfcache/hub
+      - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-}
+      - VLLM_API_KEY=${API_KEY:-}
+    command:
+      # LiquidAI/LFM2.5-2.6B (BF16, Lfm2ForCausalLM). NON-PRODUCTION bake-off alias
+      # vs granite-4.1-8b on structured extraction / classification / tool-call
+      # formatting (brokkr R-target, 2026-08-10). LFM Open License v1.0 (<USD 10M-rev
+      # commercial, not OSI) — EVAL-ONLY pending an operator ruling on production; NOT
+      # wired into any default/fallback routing chain. Reasoning model: the chat
+      # template opens <think>; deliberately NO --reasoning-parser, so the full
+      # generation (thinking + answer) lands in `content` non-empty — brokkr's explicit
+      # requirement (an empty content with the answer stranded in reasoning_content
+      # reads as a degenerate model). Vendor sampling (temp 0.1 / top_k 50 / rep_pen 1.1)
+      # is the LiteLLM alias default, not a launch arg.
+      - ${LFM25_MODEL}
+      - --served-model-name
+      - ${LFM25_SERVED_NAME}
+      - --host
+      - 0.0.0.0
+      - --port
+      - "8000"
+      - --gpu-memory-utilization
+      - ${LFM25_GPU_MEM_UTIL}
+      - --max-model-len
+      - ${LFM25_MAX_MODEL_LEN}
+      - --max-num-seqs
+      - ${LFM25_MAX_NUM_SEQS}
+      - --dtype
+      - auto
+      - --kv-cache-dtype
+      - ${LFM25_KV_CACHE_DTYPE}
+      - --enable-prefix-caching
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              device_ids:
+                - "${LFM25_GPU_ID}"
+              capabilities:
+                - gpu
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 300s
+    networks:
+      - tnet
+    labels:
+      - homepage.group=AI - Inference
+      - homepage.name=vLLM LFM2.5-2.6B (bake-off)
+      - homepage.icon=mdi-flask-outline
+      - homepage.description=LiquidAI LFM2.5-2.6B non-prod bake-off vs granite (ana-ml2)
+      - homepage.href=http://10.250.50.54:${LFM25_PORT}/docs
+
 networks:
  tnet:
    name: traefik-net