diff --git a/stacks/gen-small-seat/.env.example b/stacks/gen-small-seat/.env.example index 2776090..8660230 100644 --- a/stacks/gen-small-seat/.env.example +++ b/stacks/gen-small-seat/.env.example @@ -20,7 +20,11 @@ GEN_SMALL_SERVED_NAME_THINK=gen-small-thinking # ── Memory ── FIRST-BOOT values, expected to be tuned to a sane concurrency band # (1.2-2.5x) by reading the boot log. A3B attention + fp8 KV is cheap, so 8 GiB likely # over-provisions; pin down after the first boot reports its token count. -GEN_SMALL_GPU_MEM_UTIL=0.55 +# 2026-10-01: 0.33 live (.env on fv-ml1). The KV is byte-pinned by GEN_SMALL_KV_CACHE_MEMORY, so util only sets +# the boot check (Free at start must be >= util x total). Lowering it freed GPU 0 slack for parakeet-nemo +# at zero KV cost (still 670,142 tokens / 2.56x). Before ANY restart, check the arithmetic: +# util x 95.6 GiB <= measured Free + this seat's own resident memory. Do not trial-boot. +GEN_SMALL_GPU_MEM_UTIL=0.33 GEN_SMALL_KV_CACHE_MEMORY=8589934592 GEN_SMALL_MAX_MODEL_LEN=262144 GEN_SMALL_MAX_NUM_SEQS=16 diff --git a/stacks/parakeet-nemo/compose.yaml b/stacks/parakeet-nemo/compose.yaml index 40a660a..ec96ad2 100644 --- a/stacks/parakeet-nemo/compose.yaml +++ b/stacks/parakeet-nemo/compose.yaml @@ -6,9 +6,10 @@ # # HOST: fv-ml1, GPU 0. # -# ⚠ GPU 0 room came from gen-small's KV: its --gpu-memory-utilization dropped 0.48 -> 0.46 -# (measured boot, 2026-09-30: the seat rests ~2.5 GB, served peak ~2.8 GB, load peak ~3.0 GB; -# the old seat held 1,690 MiB). Do not raise that util back without re-measuring nvidia-smi Free +# ⚠ GPU 0 room came from gen-small's util SLACK, not its KV: util went 0.48 -> 0.36 (now 0.33 in its .env), +# and the KV is byte-pinned (unchanged at 670,142 tokens). Plan GPU 0 against this seat's STEADY STATE of +# 3,582 MiB (the torch cache after a long, windowed request), not its at-rest figure; that leaves Free ~385 MiB +# (the old seat held 1,690 MiB). Do not raise that util back without re-measuring nvidia-smi Free # on GPU 0 — util does not predict resident VRAM (see the 09-15 note in gen-small-seat/.env). # # ⚠ Weights: /tank/aimodels/huggingface mounted READ-ONLY. nvidia/parakeet-unified-en-0.6b @