esh-pfi-infrastructure/stacks/gemma4-charrp/compose.yaml

# gemma4-charrp — google/gemma-4-26B-A4B-it NVFP4 char-rp seat on ana-ml2 GPU0.
#
# Straight-across replacement for `meromero-charrp` (operator, 2026-08-24): same
# port, same served-model-names, so every gateway route and consumer is
# unchanged. The outgoing G4-MeroMero-v2-31B-NVFP4A16 stack is retained stopped
# for rollback — see stacks/meromero-charrp/.
#
# WHY THE SWAP: the seat's requirements now include chain-of-thought, which makes
# throughput MORE critical rather than less — the user waits through the whole
# reasoning block before the first visible token. The incumbent was the DENSE
# 31B at ~40.7 tok/s @32K; this is the 26B-A4B MoE at ~114 tok/s @32K
# (brokkr-smithy-dev's measurements). Same Gemma-4 family, so the parser and
# chat-template machinery below carries over unchanged.
#
# ⚠ THIS IS THE QUANT, NOT THE TUNING BASE. The BF16 weights live at
# /tank/aimodels/gemma4-26b-a4b-it-bf16 and are for QLoRA — 48.10 GiB of BF16
# weights CANNOT be served here: GPU0 is shared with vllm-gen, which holds
# ~45.6 GiB of the 94.97 GiB card, leaving ~49 GiB. Weights alone would eat all
# of it and the engine would die at KV-cache allocation. Do not "simplify" this
# to the BF16 path.
#
# ⚠ SCHEME IS W4A4, NOT W4A16. The RedHatAI quant declares 4-bit
# input_activations (compressed-tensors, format nvfp4-pack-quantized). The
# incumbent was NVFP4**A16**. Faster, and NOT a like-for-like quality
# comparison on the activation axis — say so when benching the two.
#
# Tunables in .env.

name: gemma4-charrp

services:
  vllm-gemma4-charrp:
    image: ${GEMMA4_IMAGE:-vllm/vllm-openai:latest}
    container_name: ${GEMMA4_CONTAINER:-vllm-gemma4-charrp}
    restart: unless-stopped
    ipc: host
    ports:
      - "${GEMMA4_PORT:-8016}:8000"
    volumes:
      - /tank/aimodels:/tank/aimodels
    environment:
      - PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
      - VLLM_API_KEY=${API_KEY:-}
    command:
      - ${GEMMA4_MODEL:-/tank/aimodels/gemma4-26b-a4b-it-nvfp4}
      - --quantization
      - compressed-tensors
      # UNCHANGED FROM THE OUTGOING SEAT, ON PURPOSE. Both names are live
      # LiteLLM routes at http://10.250.50.54:8016/v1 — `char-rp` ->
      # hosted_vllm/char-rp and `char-rp-reasoning` -> hosted_vllm/char-rp-thinking.
      # They are two aliases onto ONE seat, not two seats. Renaming either
      # breaks the gateway silently.
      - --served-model-name
      - char-rp
      - char-rp-thinking
      # Tool-calling: Gemma-4 emits its OWN native syntax
      # (<|tool_call>call:name{...}<tool_call|>), NOT the qwen3_coder XML the
      # other seats use. Without these two flags any tools-bearing request 400s
      # outright. Architecture-level, so it carries over from MeroMero unchanged.
      - --tool-call-parser
      - gemma4
      - --enable-auto-tool-choice
      # The gemma4 REASONING parser absorbs the <|channel>...<channel|> thought
      # markers; without it they leak into `content` verbatim on the post-tool
      # turn (upstream vllm #45834).
      - --reasoning-parser
      - gemma4
      # MANDATORY COMPANION TO THE REASONING PARSER. The parser reads
      # enable_thinking from chat_template_kwargs and DEFAULTS IT TO TRUE. True
      # makes is_reasoning_end() return False at a new turn, which pre-initialises
      # the engine to REASONING — so ALL plain RP prose lands in
      # reasoning_content with a NULL content, breaking every char-rp consumer.
      # Do not remove. Thinking is still reachable per-request via
      # chat_template_kwargs {"enable_thinking": true}, which is what the
      # `char-rp-reasoning` alias exercises.
      - --default-chat-template-kwargs
      - '{"enable_thinking": false}'
      # ⚠ NO --chat-template. MeroMero pointed at a hand-patched jinja under
      # meromero-v2-nvfp4-work/; that patch was cut against THAT checkpoint and
      # does not describe this one. This model ships its own chat_template.jinja
      # and vLLM picks it up. If thinking mode ever misbehaves, diff this
      # model's template against the MeroMero patch before reaching for a
      # copy — do not assume the patch transfers.
      - --max-model-len
      - "${GEMMA4_MAX_MODEL_LEN:-262144}"
      - --max-num-seqs
      - "${GEMMA4_MAX_NUM_SEQS:-32}"
      - --gpu-memory-utilization
      - "${GEMMA4_GPU_MEM_UTIL:-0.47}"
      - --kv-cache-dtype
      - fp8
      - --trust-remote-code
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids:
                - "${GEMMA4_GPU_ID:-0}"
              capabilities:
                - gpu
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 600s
    networks:
      - tnet
    labels:
      - homepage.group=AI - Inference
      - homepage.name=char-rp (Gemma-4 26B-A4B NVFP4, MoE)
      - homepage.icon=mdi-drama-masks
      - homepage.description=gemma-4-26B-A4B-it NVFP4 MoE prose+CoT seat, 256K (ana-ml2 GPU0)
      - homepage.href=http://10.250.50.54:${GEMMA4_PORT:-8016}/docs

networks:
  tnet:
    name: traefik-net
    external: true