diff --git a/docs/roadmap.md b/docs/roadmap.md new file mode 100644 index 0000000..4d81202 --- /dev/null +++ b/docs/roadmap.md @@ -0,0 +1,45 @@ +# PFI infra roadmap + +Deferred / planned infrastructure work. Not a ticket tracker — a durable +list of "we decided to do this, later" items so they don't get lost. + +## Observability for the vLLM stack (ana-ml2) + +Context: surfaced 2026-06-04 during the phi4-mini (summarizer/dreaming +agent) deploy. vLLM has **no built-in web UI** for logs or model state the +way llama-swap does. Dozzle (already running) shows vLLM's stdout = +connection/request-metadata only — **not** full request/response bodies. +Two complementary layers fill the gap: + +### 1. Langfuse — request/response tracing (PRIORITY) + +The thing llama-swap's UI gave us and vLLM doesn't: **see the entire +request and response** per call, in a browser. Langfuse (open-source, +self-hostable) captures every call's full prompt + full completion + +tokens + latency + cost, with a polished trace UI — a strict upgrade over +llama-swap's raw log dump. + +- **Where:** a stack on ana-docker (alongside the other hubs — Dozzle, + Beszel, gitea, etc.). +- **How:** vLLM services instrumented or fronted by it. Either point the + consuming agents (nevermore, the dreaming agent, etc.) at a **LiteLLM + proxy** that logs to Langfuse, or instrument the clients directly. +- **Why it matters:** phi4-mini is becoming a production summarizer + + dreaming agent; being able to inspect exactly what it was asked and what + it answered is the difference between debuggable and opaque. + +### 2. Prometheus + Grafana — operational metrics + +vLLM natively exposes a Prometheus `/metrics` endpoint (throughput, +time-to-first-token, KV-cache utilization, queue depth, running/waiting +requests) and ships **official Grafana dashboards**. We have Beszel for +coarse host/GPU stats but no app-layer inference metrics. + +- **Where:** Prometheus + Grafana stack on ana-docker, scraping ana-ml2's + vLLM `:metrics` ports (and reusable for any future vLLM service). +- **Why:** tells us if phi4 (or the embed/rerank/reward trio) is + KV-cache-bound, queueing, or has latency regressions — the operational + view llama-swap's UI only hinted at. + +**Status:** deferred during the phi4-mini deploy; do after phi4 is live. +Langfuse first (the operator specifically wants full req/resp visibility). diff --git a/stacks/llama-swap/conf/config.yaml b/stacks/llama-swap/conf/config.yaml index 1ac775a..247d4c9 100644 --- a/stacks/llama-swap/conf/config.yaml +++ b/stacks/llama-swap/conf/config.yaml @@ -445,36 +445,18 @@ models: # GRANITE MODELS (IBM) # ========================================================================== - "granite-4-small": - name: "Granite 4.0 Small Q4_K_M" - description: "IBM Granite 4.0 Small. Deterministic utility model for structured tasks." - ttl: 0 # pinned — member of the `pinned` group, never unloads - cmd: | - /app/llama-server - --context-shift - --model /models/unsloth_granite-4.0-h-small-GGUF/granite-4.0-h-small-Q4_K_M.gguf - --port ${PORT} - --n-gpu-layers 999 - --ctx-size 120000 - --flash-attn on - --top-p 1.0 - --temp 0.0 - --top-k 0 + # ── "granite-4-small" REMOVED 2026-06-04 ────────────────────────────────── + # Superseded by phi4-mini (summarizer + dreaming agent), now served via vLLM + # FP8 on ana-ml2 (stacks/vllm → vllm-phi4, :8004). It was pinned at 120K ctx + # (~24 GB resident: ~6 GB weights + ~18 GB KV); removing it reclaims that VRAM. + # DOWNSTREAM: repoint the news-digest curator from this llama-swap endpoint to + # the phi4-mini vLLM endpoint at/before deploy. - "granite-4-micro": - name: "Granite 4.0 Micro Q4_K_M" - description: "IBM Granite 4.0 Micro. Ultra-lightweight for fast structured responses." - ttl: 600 - cmd: | - /app/llama-server - --context-shift - --model /models/ibm-granite_granite-4.0-micro-GGUF/granite-4.0-micro-Q4_K_M.gguf - --port ${PORT} - --n-gpu-layers 999 - --ctx-size 32768 - --flash-attn on - --temp 0.0 - --top-p 1.0 + # ── "granite-4-micro" config REMOVED 2026-06-04 ─────────────────────────── + # Retired from llama-swap alongside granite-4-small (both superseded by + # phi4-mini). Per operator: CONFIG ONLY — the GGUF stays on disk at + # /models/ibm-granite_granite-4.0-micro-GGUF/granite-4.0-micro-Q4_K_M.gguf + # (NOT deleted), so this entry can be restored later if needed. # ========================================================================== # JUDGE / EVAL MODELS @@ -605,13 +587,12 @@ groups: # # Current pins: # qwen3.5-9b — ~6 GB at Q4 + KV. General-purpose chat baseline. - # granite-4-small — ~5-6 GB at Q4_K_M + 120K KV. Used by news-digest - # curator twice daily; pinning avoids the cold-load - # latency and prevents qwen3.6-27b (and similar) - # from evicting it when both are needed concurrently. - # VRAM budget: ~12 GB persistent in the pin slot. Single RTX 6000 Ada - # is 48 GB, so this leaves ~36 GB for whichever non-pinned model the - # user invokes alongside (qwen3.6-27b at ~30 GB fits cleanly). + # VRAM budget: ~6 GB persistent in the pin slot. Single RTX 6000 Ada + # is 48 GB, so this leaves ~40 GB for whichever non-pinned model the + # user invokes alongside. + # + # granite-4-small WAS pinned here; removed 2026-06-04 — superseded by + # phi4-mini (vLLM FP8, stacks/vllm → vllm-phi4). Freed ~24 GB (120K KV). # # qwen3.6-35-a3b WAS in this group; removed 2026-04-27 because its # ~29 GB at Q6_K_XL pushed concurrent loads OOM. Now lives outside @@ -622,4 +603,3 @@ groups: persistent: true members: - "qwen3.5-9b" - - "granite-4-small" diff --git a/stacks/vllm/.env.example b/stacks/vllm/.env.example index 016d7dc..58808b4 100644 --- a/stacks/vllm/.env.example +++ b/stacks/vllm/.env.example @@ -52,3 +52,28 @@ API_KEY= # HuggingFace token — only needed for gated models in the HF-Hub-loaded # services (embed/rerank). Reward is local-path, ignores this. HF_TOKEN= + +# === phi4-mini (summarizer / dreaming agent; supersedes llama-swap granite-4-small) === +PHI4_PORT=8004 +# GPU 1 — co-locate with the embed/rerank/reward trio so GPU 0 stays clear for +# llama-swap's heavy swap-ins. Post-granite GPU 1 has ~10.4 GB free; phi4 at 50K +# + FP8 KV needs ~9 GB → ~1.2 GB margin (tight). NOTE: llama-swap also uses GPU 1 +# dynamically (a model's ~10 GB share lives here), so a large swap-in could +# contend — for clean separation, pin llama-swap to GPU 0 (follow-up). +PHI4_GPU_ID=1 +# FP8 path. Default = vLLM-native dynamic FP8 from the official BF16 weights +# (most robust on Ada/Hopper, near-lossless, ~1.2x). Alternative = the +# pre-quantized checkpoint: PHI4_MODEL=pytorch/Phi-4-mini-instruct-FP8 + PHI4_QUANT=torchao. +PHI4_MODEL=microsoft/Phi-4-mini-instruct +PHI4_QUANT=fp8 +PHI4_SERVED_NAME=phi4-mini +# 50K ctx (51200) — dropped from the 128K canonical max to fit GPU 1's ~10.4 GB +# free post-granite (64K was ~10 GB = no margin). Ample for feed summarization + +# the dreaming agent; Phi-4-mini supports up to 131072 if it ever moves to GPU 0. +PHI4_MAX_MODEL_LEN=51200 +# FP8 KV cache (native on Ada cc 8.9). At 50K ≈ ~3.1 GB KV (vs ~6.3 GB at fp16). +PHI4_KV_CACHE_DTYPE=fp8 +# ~9 GB total: ~4 GB FP8 weights + ~3.1 GB FP8 KV @50K + overhead. 0.20×46≈9.2 GB. +# vLLM prints the real KV size on first boot — tune after; do NOT raise without +# first confirming GPU 1 freed more (it's only ~1.2 GB over budget here). +PHI4_GPU_MEM_UTIL=0.20 diff --git a/stacks/vllm/compose.yaml b/stacks/vllm/compose.yaml index 5508679..1a0334d 100644 --- a/stacks/vllm/compose.yaml +++ b/stacks/vllm/compose.yaml @@ -197,6 +197,70 @@ services: - homepage.description=Skywork-Reward-V2 8B classifier via vLLM (ana-ml2) - homepage.href=http://10.250.50.54:${REWARD_PORT}/docs + # Phi-4-mini (FP8) — summarizer + "dreaming" agent. Supersedes the + # llama-swap granite-4-small pin. Generative chat model (OpenAI + # /v1/chat/completions), so NO --runner pooling. FP8 on RTX 6000 Ada + # (cc 8.9): near-lossless, ~1.2x, ~6 GB. + vllm-phi4: + image: vllm/vllm-openai:${VLLM_VERSION} + container_name: vllm-phi4 + restart: unless-stopped + ipc: host + ports: + - "${PHI4_PORT}:8000" + volumes: + - /tank/aimodels/huggingface:/hfcache + environment: + - HF_HOME=/hfcache + - HF_HUB_CACHE=/hfcache/hub + - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-} + - VLLM_API_KEY=${API_KEY:-} + command: + # Default = vLLM-native dynamic FP8 from the official BF16 weights (most + # robust on Ada). Alt = pre-quantized checkpoint: PHI4_MODEL= + # pytorch/Phi-4-mini-instruct-FP8 + PHI4_QUANT=torchao (see .env/README). + - ${PHI4_MODEL} + - --served-model-name + - ${PHI4_SERVED_NAME} + - --quantization + - ${PHI4_QUANT} + - --host + - 0.0.0.0 + - --port + - "8000" + - --gpu-memory-utilization + - ${PHI4_GPU_MEM_UTIL} + - --max-model-len + - ${PHI4_MAX_MODEL_LEN} + - --dtype + - auto + # FP8 KV cache — halves KV memory at 128K ctx on Ada (cc 8.9); near-lossless. + - --kv-cache-dtype + - ${PHI4_KV_CACHE_DTYPE} + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: + - "${PHI4_GPU_ID}" + capabilities: + - gpu + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 180s + networks: + - tnet + labels: + - homepage.group=AI Systems + - homepage.name=vLLM Phi-4-mini (summarizer/dreamer) + - homepage.icon=mdi-text-box-outline + - homepage.description=Phi-4-mini FP8 via vLLM (ana-ml2) + - homepage.href=http://10.250.50.54:${PHI4_PORT}/docs + networks: tnet: name: traefik-net