From 019ccff7e88bea971c388f2c3e7dd0ee92cbe9f9 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Mon, 24 Aug 2026 14:19:49 -0700 Subject: [PATCH] feat(gemma4-trainee-bench): BF16 bench stack; record that gen's footprint grows with uptime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an ephemeral stack for serving the BF16 trainee base on :8016 under the char-rp aliases, so the abliterated base can be measured on the same battery and the same gateway routes as the served seat with no harness edit. It is a separate stack rather than another variable on gemma4-charrp because that compose hardcodes `--quantization compressed-tensors` for the NVFP4 build. Pointing it at unquantized BF16 weights crash-loops immediately — `TypeError: CompressedTensorsConfig.__init__() missing 3 required positional arguments: 'target_scheme_map', 'ignore', 'quant_format'` — vLLM trying to read a quantization config out of a checkpoint that has none. 35 restarts before it was caught. `restart: "no"` here so a bench seat cannot resurrect itself and block gen's restore, and no homepage labels so it leaves no permanently-offline dashboard card. It cannot coexist with gen and says so: 48.07 GiB of BF16 weights plus gen's footprint exceeds the 94.97 GiB card before any KV cache. Running it means gen is stopped. THE MORE USEFUL FINDING is in the meromero env note: gen's memory footprint GROWS WITH UPTIME. Measured today at 46,726 MiB (45.6 GiB) after ~3 days up, and 39,424 MiB (38.5 GiB) immediately after a restart — same container, same --gpu-memory-utilization 0.43, ~7 GiB apart. That is the missing half of this afternoon's crash-loop: the char-rp seat "fit on the 21st and stopped fitting on the 24th" because nothing about char-rp changed and gen crept up underneath it. Headroom arithmetic done against a long-running gen is measuring a moving number, so the note now says to measure against a freshly-restarted one. Operator's requested end state reached and verified through the gateway: gen and summarizer both 200, char-rp down deliberately to hold GPU0 headroom for the upcoming trainee run, bench seat stopped. --- stacks/gemma4-trainee-bench/compose.yaml | 105 +++++++++++++++++++++++ stacks/meromero-charrp/.env.example | 9 ++ 2 files changed, 114 insertions(+) create mode 100644 stacks/gemma4-trainee-bench/compose.yaml diff --git a/stacks/gemma4-trainee-bench/compose.yaml b/stacks/gemma4-trainee-bench/compose.yaml new file mode 100644 index 0000000..af71d11 --- /dev/null +++ b/stacks/gemma4-trainee-bench/compose.yaml @@ -0,0 +1,105 @@ +# gemma4-trainee-bench — BF16 abliterated trainee base, served for benchmarking. +# +# EPHEMERAL BY DESIGN. This is not a production seat. It exists so the trainee +# base can be measured on the same battery as the served char-rp seat, and it +# takes GPU0 to itself while it runs. +# +# ⚠ IT CANNOT COEXIST WITH `vllm-gen`. The weights are BF16 — 48.07 GiB of a +# 94.97 GiB card — and gen actually holds ~45.6 GiB. 48.07 + 45.6 = 93.7 GiB +# before a single byte of KV cache, activations, CUDA graphs or non-torch +# overhead. Running this means gen is stopped, and gen coming back means this +# is stopped. There is no arrangement where both are up. +# +# ⚠ NO `--quantization` FLAG, AND THAT IS THE POINT. +# The production gemma4-charrp compose hardcodes `--quantization +# compressed-tensors` because it serves an NVFP4 build. Pointing that stack at +# these BF16 weights crash-loops immediately: +# TypeError: CompressedTensorsConfig.__init__() missing 3 required +# positional arguments: 'target_scheme_map', 'ignore', 'quant_format' +# vLLM tries to read a quantization config out of a checkpoint that has none. +# 35 restarts before it was caught. Hence a separate stack rather than another +# variable on the production one. +# +# SAME PORT AND SAME SERVED-MODEL-NAMES AS char-rp, deliberately: brokkr's +# battery targets the `char-rp` / `char-rp-reasoning` gateway aliases, so +# serving under those names means their harness needs no edit and the numbers +# are directly comparable to the runs already recorded. Only one of this stack +# and gemma4-charrp can be up at a time — both bind :8016. +# +# Tunables in .env. + +name: gemma4-trainee-bench + +services: + vllm-gemma4-trainee-bench: + image: ${BENCH_IMAGE:-vllm/vllm-openai:v0.26.0} + container_name: ${BENCH_CONTAINER:-vllm-gemma4-trainee-bench} + # `no`, not unless-stopped. A bench seat that resurrects itself after the + # window closes would silently hold 48 GiB and block gen's restore. + restart: "no" + ipc: host + ports: + - "${BENCH_PORT:-8016}:8000" + volumes: + - /tank/aimodels:/tank/aimodels + environment: + - PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + - VLLM_API_KEY=${API_KEY:-} + command: + - ${BENCH_MODEL:-/tank/aimodels/gemma4-26b-a4b-it-heretic-bf16} + - --served-model-name + - char-rp + - char-rp-thinking + # Architecture-level Gemma-4 flags, identical to the production seat — + # see stacks/gemma4-charrp/README.md for why each is load-bearing. + - --tool-call-parser + - gemma4 + - --enable-auto-tool-choice + - --reasoning-parser + - gemma4 + - --default-chat-template-kwargs + - '{"enable_thinking": false}' + # ⚠ UPSTREAM TEMPLATE, NOT THE ONE THIS CHECKPOINT SHIPS. + # Every third-party Gemma-4 derivative carries a stale template: this + # abliterated build ships a 365-line file, the stock upstream is 390 + # lines. Benching through a different template than production serves + # would make the comparison meaningless — the same reason the production + # seat pins it. + - --chat-template + - ${BENCH_CHAT_TEMPLATE:-/tank/aimodels/gemma4-26b-a4b-it-nvfp4/chat_template.jinja} + - --max-model-len + - "${BENCH_MAX_MODEL_LEN:-262144}" + - --max-num-seqs + - "${BENCH_MAX_NUM_SEQS:-32}" + # Card to itself. 0.92 leaves ~7 GiB of real headroom on top of 48 GiB of + # weights — deliberately not 0.95, after a seat crash-looped this + # afternoon on 0.6 GiB of margin. + - --gpu-memory-utilization + - "${BENCH_GPU_MEM_UTIL:-0.92}" + - --kv-cache-dtype + - fp8 + - --trust-remote-code + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: + - "${BENCH_GPU_ID:-0}" + capabilities: + - gpu + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 900s + networks: + - tnet + # No homepage labels. This is a transient bench seat; giving it a dashboard + # card would leave a permanently-offline entry once the window closes. + +networks: + tnet: + name: traefik-net + external: true diff --git a/stacks/meromero-charrp/.env.example b/stacks/meromero-charrp/.env.example index f6844ad..a3bde83 100644 --- a/stacks/meromero-charrp/.env.example +++ b/stacks/meromero-charrp/.env.example @@ -17,6 +17,15 @@ # # Raising this again means lowering gen's in the same change. Check the real # numbers, not the flags: `nvidia-smi --query-compute-apps=pid,used_memory`. +# +# ⚠ AND CHECK THEM ON A FRESHLY-RESTARTED gen. Measured 2026-08-24: gen held +# 46,726 MiB (45.6 GiB) after ~3 days of uptime, and 39,424 MiB (38.5 GiB) +# immediately after a restart — the same container, the same +# `--gpu-memory-utilization 0.43`, ~7 GiB apart. Its footprint GROWS WITH +# UPTIME, which is the missing half of why this seat "fit on the 21st and +# stopped fitting on the 24th": nothing about this seat changed, gen crept up +# underneath it. Headroom arithmetic done against a long-running gen is +# measuring a moving number. MEROMERO_GPU_MEM_UTIL=0.47 MEROMERO_MAX_MODEL_LEN=262144 MEROMERO_GPU_ID=0