diff --git a/stacks/gemma4-charrp/.env.example b/stacks/gemma4-charrp/.env.example new file mode 100644 index 0000000..03e8638 --- /dev/null +++ b/stacks/gemma4-charrp/.env.example @@ -0,0 +1,31 @@ +# gemma4-charrp — char-rp seat on ana-ml2 GPU0. Real .env lives on the host. + +# ⚠ GPU0 IS SHARED WITH `vllm-gen`. gen runs at --gpu-memory-utilization 0.43 +# but actually holds ~45.6 GiB of the 94.97 GiB card — that flag sizes the KV +# cache and does NOT cover CUDA context, graphs and non-torch overhead. The +# predecessor seat sat at 0.51, the pair summed to 0.94, and on 2026-08-24 it +# stopped fitting and crash-looped 13 times with +# `torch.OutOfMemoryError: ... 195.19 MiB is free`. +# +# 0.47 keeps ~4.8 GiB of real margin. This model's weights are only ~15.3 GiB +# (NVFP4) against the predecessor's ~19.5 GiB, so the same budget buys MORE KV +# cache than before, not less. Raising this means lowering gen's in the same +# change — and check the real numbers, not the flags: +# nvidia-smi --query-compute-apps=pid,used_memory --format=csv +GEMMA4_GPU_MEM_UTIL=0.47 + +# Native context. config.json declares max_position_embeddings 262144, same as +# the outgoing seat, so this is a straight-across swap on context too. +GEMMA4_MAX_MODEL_LEN=262144 +GEMMA4_MAX_NUM_SEQS=32 + +# ⚠ THE NVFP4 QUANT, NOT THE BF16. /tank/aimodels/gemma4-26b-a4b-it-bf16 is the +# QLoRA tuning base and is 48.10 GiB of weights — it does not fit beside gen. +GEMMA4_MODEL=/tank/aimodels/gemma4-26b-a4b-it-nvfp4 + +GEMMA4_PORT=8016 +GEMMA4_GPU_ID=0 +GEMMA4_CONTAINER=vllm-gemma4-charrp + +# Same value as every other vLLM seat on this host — the gateway presents it. +API_KEY= diff --git a/stacks/gemma4-charrp/compose.yaml b/stacks/gemma4-charrp/compose.yaml new file mode 100644 index 0000000..10f7071 --- /dev/null +++ b/stacks/gemma4-charrp/compose.yaml @@ -0,0 +1,120 @@ +# gemma4-charrp — google/gemma-4-26B-A4B-it NVFP4 char-rp seat on ana-ml2 GPU0. +# +# Straight-across replacement for `meromero-charrp` (operator, 2026-08-24): same +# port, same served-model-names, so every gateway route and consumer is +# unchanged. The outgoing G4-MeroMero-v2-31B-NVFP4A16 stack is retained stopped +# for rollback — see stacks/meromero-charrp/. +# +# WHY THE SWAP: the seat's requirements now include chain-of-thought, which makes +# throughput MORE critical rather than less — the user waits through the whole +# reasoning block before the first visible token. The incumbent was the DENSE +# 31B at ~40.7 tok/s @32K; this is the 26B-A4B MoE at ~114 tok/s @32K +# (brokkr-smithy-dev's measurements). Same Gemma-4 family, so the parser and +# chat-template machinery below carries over unchanged. +# +# ⚠ THIS IS THE QUANT, NOT THE TUNING BASE. The BF16 weights live at +# /tank/aimodels/gemma4-26b-a4b-it-bf16 and are for QLoRA — 48.10 GiB of BF16 +# weights CANNOT be served here: GPU0 is shared with vllm-gen, which holds +# ~45.6 GiB of the 94.97 GiB card, leaving ~49 GiB. Weights alone would eat all +# of it and the engine would die at KV-cache allocation. Do not "simplify" this +# to the BF16 path. +# +# ⚠ SCHEME IS W4A4, NOT W4A16. The RedHatAI quant declares 4-bit +# input_activations (compressed-tensors, format nvfp4-pack-quantized). The +# incumbent was NVFP4**A16**. Faster, and NOT a like-for-like quality +# comparison on the activation axis — say so when benching the two. +# +# Tunables in .env. + +name: gemma4-charrp + +services: + vllm-gemma4-charrp: + image: ${GEMMA4_IMAGE:-vllm/vllm-openai:latest} + container_name: ${GEMMA4_CONTAINER:-vllm-gemma4-charrp} + restart: unless-stopped + ipc: host + ports: + - "${GEMMA4_PORT:-8016}:8000" + volumes: + - /tank/aimodels:/tank/aimodels + environment: + - PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + - VLLM_API_KEY=${API_KEY:-} + command: + - ${GEMMA4_MODEL:-/tank/aimodels/gemma4-26b-a4b-it-nvfp4} + - --quantization + - compressed-tensors + # UNCHANGED FROM THE OUTGOING SEAT, ON PURPOSE. Both names are live + # LiteLLM routes at http://10.250.50.54:8016/v1 — `char-rp` -> + # hosted_vllm/char-rp and `char-rp-reasoning` -> hosted_vllm/char-rp-thinking. + # They are two aliases onto ONE seat, not two seats. Renaming either + # breaks the gateway silently. + - --served-model-name + - char-rp + - char-rp-thinking + # Tool-calling: Gemma-4 emits its OWN native syntax + # (<|tool_call>call:name{...}), NOT the qwen3_coder XML the + # other seats use. Without these two flags any tools-bearing request 400s + # outright. Architecture-level, so it carries over from MeroMero unchanged. + - --tool-call-parser + - gemma4 + - --enable-auto-tool-choice + # The gemma4 REASONING parser absorbs the <|channel>... thought + # markers; without it they leak into `content` verbatim on the post-tool + # turn (upstream vllm #45834). + - --reasoning-parser + - gemma4 + # MANDATORY COMPANION TO THE REASONING PARSER. The parser reads + # enable_thinking from chat_template_kwargs and DEFAULTS IT TO TRUE. True + # makes is_reasoning_end() return False at a new turn, which pre-initialises + # the engine to REASONING — so ALL plain RP prose lands in + # reasoning_content with a NULL content, breaking every char-rp consumer. + # Do not remove. Thinking is still reachable per-request via + # chat_template_kwargs {"enable_thinking": true}, which is what the + # `char-rp-reasoning` alias exercises. + - --default-chat-template-kwargs + - '{"enable_thinking": false}' + # ⚠ NO --chat-template. MeroMero pointed at a hand-patched jinja under + # meromero-v2-nvfp4-work/; that patch was cut against THAT checkpoint and + # does not describe this one. This model ships its own chat_template.jinja + # and vLLM picks it up. If thinking mode ever misbehaves, diff this + # model's template against the MeroMero patch before reaching for a + # copy — do not assume the patch transfers. + - --max-model-len + - "${GEMMA4_MAX_MODEL_LEN:-262144}" + - --max-num-seqs + - "${GEMMA4_MAX_NUM_SEQS:-32}" + - --gpu-memory-utilization + - "${GEMMA4_GPU_MEM_UTIL:-0.47}" + - --kv-cache-dtype + - fp8 + - --trust-remote-code + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: + - "${GEMMA4_GPU_ID:-0}" + capabilities: + - gpu + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 600s + networks: + - tnet + labels: + - homepage.group=AI - Inference + - homepage.name=char-rp (Gemma-4 26B-A4B NVFP4, MoE) + - homepage.icon=mdi-drama-masks + - homepage.description=gemma-4-26B-A4B-it NVFP4 MoE prose+CoT seat, 256K (ana-ml2 GPU0) + - homepage.href=http://10.250.50.54:${GEMMA4_PORT:-8016}/docs + +networks: + tnet: + name: traefik-net + external: true diff --git a/stacks/meromero-charrp/compose.yaml b/stacks/meromero-charrp/compose.yaml index 063c4c7..04308ac 100644 --- a/stacks/meromero-charrp/compose.yaml +++ b/stacks/meromero-charrp/compose.yaml @@ -86,10 +86,14 @@ services: networks: - tnet labels: - - homepage.group=AI - Inference - - homepage.name=char-rp (MeroMero-v2 NVFP4, multimodal) + # STOPPED 2026-08-24 — superseded by stacks/gemma4-charrp (gemma-4-26B-A4B + # -it NVFP4 MoE) on the same port and the same served-model-names. Kept as + # the rollback seat: `docker compose up -d vllm-meromero-rp` here after + # stopping the gemma4 one, since both want :8016. + - homepage.group=AI - Dormant + - homepage.name=char-rp (MeroMero-v2 NVFP4) — rollback - homepage.icon=mdi-drama-masks - - homepage.description=G4-MeroMero-v2-31B NVFP4A16 non-thinking prose seat, vision-enabled, 256K (ana-ml2 GPU0) + - homepage.description=Superseded 2026-08-24 by the Gemma-4 26B-A4B MoE seat; dense 31B NVFP4A16, kept for rollback (ana-ml2 GPU0) - homepage.href=http://10.250.50.54:${MEROMERO_PORT:-8016}/docs networks: