diff --git a/stacks/litellm/conf/config.yaml b/stacks/litellm/conf/config.yaml index 547c72c..90dccf9 100644 --- a/stacks/litellm/conf/config.yaml +++ b/stacks/litellm/conf/config.yaml @@ -95,6 +95,18 @@ model_list: model_info: mode: chat + # --- Selene 1 Mini 8B (AtlaAI judge, FP8) — restored on GPU1 after the + # llama-swap teardown (was the Q6_K GGUF in the swap zoo). vLLM dynamic fp8, + # :8011. Explicit entry shadows the "*" wildcard (which used to reach it via + # llama-swap). Hallucination/RAG-faithfulness judge; callers set temp ~0.01. --- + - model_name: selene-1-mini-8b + litellm_params: + model: hosted_vllm/selene-1-mini-8b + api_base: http://10.250.50.54:8011/v1 + api_key: os.environ/VLLM_API_KEY + model_info: + mode: chat + # --- Qwen3 embeddings --- - model_name: qwen3-embedding litellm_params: diff --git a/stacks/mistral-small-4/.env.example b/stacks/mistral-small-4/.env.example index 7c0599e..5bf4a3d 100644 --- a/stacks/mistral-small-4/.env.example +++ b/stacks/mistral-small-4/.env.example @@ -22,14 +22,17 @@ MISTRAL_PORT=8010 # the embed/rerank/reward trio. MISTRAL_GPU_ID=0 -# util 0.93 (~89 GB budget) — 74.4 GB weights + ~5 GB CUDA/graph leaves ~10 GB -# KV. MLA keeps KV compressed so 131072 ctx fits; raise toward native 256K only -# after measuring real KV headroom. Dedicated card, so 0.93 is safe. +# util 0.93 (~89 GB budget) — measured 66.1 GiB weights + 0.8 GiB graph + ~18.5 GiB +# KV. MLA keeps KV compressed, so the FULL native 256K context fits (verified). MISTRAL_GPU_MEM_UTIL=0.93 -MISTRAL_MAX_MODEL_LEN=131072 -# Single-card KV is tighter than the official TP=2 setup → cap concurrency at 64 -# (official used 128 across two cards). -MISTRAL_MAX_NUM_SEQS=64 +# 262144 = native 256K (for novel-length consistency-checking). The KV pool is +# util-bound (~862K tokens) regardless of max-len, so 256K costs no extra VRAM — +# it just lets one request use up to 256K (→ max concurrency 3.29x at full length). +MISTRAL_MAX_MODEL_LEN=262144 +# 32 (halved from 64 when going to 256K): the warmup transient scales with +# max-num-seqs × max-model-len, so halving seqs while doubling len keeps it flat +# and fits the tight card. 32 is ample — this is a low-concurrency creative model. +MISTRAL_MAX_NUM_SEQS=32 # Optional — model is ungated (Apache-2.0), no token needed. HF_TOKEN= diff --git a/stacks/mistral-small-4/compose.yaml b/stacks/mistral-small-4/compose.yaml index 7df87c0..0c2d067 100644 --- a/stacks/mistral-small-4/compose.yaml +++ b/stacks/mistral-small-4/compose.yaml @@ -19,16 +19,17 @@ # reference 80 GB cards can't fit 74.4 GB + context on one). The 96 GB Blackwell # flips that to single-card: 74.4 GB weights + ~5 GB overhead leaves ~17 GB for # KV. Mistral Small 4 uses MLA attention (TRITON_MLA) so KV is compressed/cheap — -# big context stays affordable even on a constrained KV pool. max-model-len is -# capped to 131072 on first bring-up (raise toward the native 256K once real KV -# headroom is measured). +# big context stays affordable even on a constrained KV pool. We serve the FULL +# native 256K (max-model-len 262144) — the KV pool is util-bound (~862K tokens) +# so 256K costs no extra VRAM, it just lets one request use up to 256K (max +# concurrency 3.29x at full length). For novel-length consistency-checking. # -# vLLM FLOOR: needs >= 0.20 (Mistral Small 4 day-0 support); validated on 0.23.0. -# Do NOT reuse the qwen36-vl 0.19.1 image — it predates this model. +# vLLM PIN: v0.22.0 (in .env) — the last release with WORKING Mistral vision +# (#44911 fetch_images regression hit 0.22.1+/0.23.0). See the .env header. # # Serve flags mirror Mistral's official command (cited in README), adapted for -# single-card: TP 2->1, util 0.8->0.93, max-len 262144->131072, max-num-seqs -# 128->64. All tunables live in .env — edit that, not this file. +# single-card: TP 2->1, util 0.8->0.93, max-num-seqs 128->32 (32 keeps the +# 256K warmup transient flat). All tunables live in .env — edit that, not this file. name: mistral-small-4 diff --git a/stacks/selene/.env.example b/stacks/selene/.env.example new file mode 100644 index 0000000..d702a95 --- /dev/null +++ b/stacks/selene/.env.example @@ -0,0 +1,26 @@ +# Selene 1 Mini 8B judge (FP8) on ana-ml2 GPU 1 — copy to .env on the host. +# Real .env lives on ana-ml2 at /opt/docker/compose/selene/.env (gitignored). +# +# See compose.yaml header for the FP8-over-NVFP4 (judge-fidelity) rationale. + +# 0.23.0 — Llama 3.1 + dynamic fp8 is rock-solid here (same digest as qwen36). +SELENE_IMAGE=vllm/vllm-openai@sha256:6d8429e38e3747723ca07ee1b17972e09bb9c51c4032b266f24fb1cc3b22ed8f + +SELENE_CONTAINER_NAME=vllm-selene +SELENE_MODEL=AtlaAI/Selene-1-Mini-Llama-3.1-8B +SELENE_PORT=8011 + +# GPU 1 = shared with qwen36 (NVFP4) + granite + embed/rerank/reward. +SELENE_GPU_ID=1 + +# util 0.13 (~12.5 GB) — measured: 8.51 GiB FP8 weights + ~1.5 GiB graph + +# 2.53 GiB fp8 KV (41,456-token pool, 1.27x concurrency at full 32K). util 0.12 +# was too thin (1.86 GiB KV < the 2.0 GiB a single 32K request needs → crash). +# Keeps GPU 1 total ~0.95 → ~5 GB buffer; ctx 32768 mirrors the old judge config. +SELENE_GPU_MEM_UTIL=0.13 +SELENE_MAX_MODEL_LEN=32768 +SELENE_MAX_NUM_SEQS=16 + +# Optional — checkpoint is ungated (Apache-2.0). +HF_TOKEN= +API_KEY= diff --git a/stacks/selene/compose.yaml b/stacks/selene/compose.yaml new file mode 100644 index 0000000..bd60765 --- /dev/null +++ b/stacks/selene/compose.yaml @@ -0,0 +1,86 @@ +# selene — AtlaAI Selene 1 Mini (Llama 3.1 8B) judge/eval model on ana-ml2 GPU 1. +# +# Restores the judge that went offline when llama-swap was downed (it was the +# Q6_K GGUF `selene-1-mini-8b` in the llama-swap zoo). Re-served on vLLM at the +# operator's request, FP8 (NVFP4 had no pre-made checkpoint and W4A4 is too +# aggressive for a precision judge validated at Q6_K — FP8 ≥ Q6_K fidelity). +# +# FP8 = vLLM DYNAMIC --quantization fp8 (W8A8) of the bf16 AtlaAI checkpoint — +# no offline quant needed, near-lossless, and Selene is text-only Llama 3.1 so +# there's NO vision tower for dynamic fp8 to noise-quantize (the qwen35-VL +# footgun doesn't apply here). ~8 GiB weights on GPU 1's headroom. +# +# Co-tenant on GPU 1 with qwen36 (NVFP4) + granite + embed/rerank/reward. Sized +# to fit the ~24 GB headroom while leaving GPU 1 a safe buffer (see .env). +# Served ONLY as `selene-1-mini-8b` (the name its consumers know). All tunables +# live in .env. + +name: selene + +services: + vllm-selene: + image: ${SELENE_IMAGE} + container_name: ${SELENE_CONTAINER_NAME} + restart: unless-stopped + ipc: host + ports: + - "${SELENE_PORT}:8000" + volumes: + - /tank/aimodels/huggingface:/hfcache + environment: + - HF_HOME=/hfcache + - HF_HUB_CACHE=/hfcache/hub + - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-} + - VLLM_API_KEY=${API_KEY:-} + command: + - ${SELENE_MODEL} + # Dynamic FP8 (W8A8) from the bf16 checkpoint — no pre-quant needed. + - --quantization + - fp8 + - --served-model-name + - selene-1-mini-8b + - --host + - 0.0.0.0 + - --port + - "8000" + - --gpu-memory-utilization + - ${SELENE_GPU_MEM_UTIL} + - --max-model-len + - ${SELENE_MAX_MODEL_LEN} + - --max-num-seqs + - ${SELENE_MAX_NUM_SEQS} + # fp8 KV — matches the judge's old q8 KV posture + keeps the pool compact + # on the shared card. + - --kv-cache-dtype + - fp8 + - --dtype + - auto + - --enable-prefix-caching + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: + - "${SELENE_GPU_ID}" + capabilities: + - gpu + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 180s + networks: + - tnet + labels: + - homepage.group=AI Systems + - homepage.name=Selene 1 Mini 8B (judge, FP8) + - homepage.icon=mdi-gavel + - homepage.description=AtlaAI Selene 1 Mini Llama-3.1-8B judge (FP8) via vLLM (ana-ml2 GPU1) + - homepage.href=http://10.250.50.54:${SELENE_PORT}/docs + +networks: + tnet: + name: traefik-net + external: true