diff --git a/stacks/qwen36-vl/.env.example b/stacks/qwen36-vl/.env.example index 020fb85..6a47c1a 100644 --- a/stacks/qwen36-vl/.env.example +++ b/stacks/qwen36-vl/.env.example @@ -22,9 +22,11 @@ QWEN_GPU_ID=1 # fp16 KV + CUDA-graph. fp16 KV (compose drops --kv-cache-dtype fp8): the NVFP4 # swap freed enough room to run full-precision KV. Hybrid attn (10/40 full-attn) # keeps even fp16 KV cheap. GPU-1 budget (2026-06-15 rebalance, pinned): qwen36 -# 0.32 + granite 0.34/131072 (RESTORED from the FP8-era 0.24/64K) + trio 0.16 = -# ~0.82, ~24 GB free headroom. Recreate ONE service at a time (profiling race). -QWEN_GPU_MEM_UTIL=0.32 +# GPU-1 budget (2026-06-16): qwen36 0.34 (grown from 0.32 for the arbo-judge + +# worldtree actor/echo + gateway load) + granite 0.34/131072 + selene 0.17 + trio +# 0.16. Nominal sum >1.0 but the pooling models + granite under-use their util, so +# it fits with ~5-6 GB physical free. Recreate ONE service at a time (profiling race). +QWEN_GPU_MEM_UTIL=0.34 QWEN_MAX_MODEL_LEN=131072 # Sampler-warmup OOM guard on the shared GPU (248K vocab × default 1024 seqs is # a huge transient). 32 is plenty for a vision endpoint. diff --git a/stacks/selene/.env.example b/stacks/selene/.env.example index d702a95..71d11cc 100644 --- a/stacks/selene/.env.example +++ b/stacks/selene/.env.example @@ -13,11 +13,11 @@ SELENE_PORT=8011 # GPU 1 = shared with qwen36 (NVFP4) + granite + embed/rerank/reward. SELENE_GPU_ID=1 -# util 0.13 (~12.5 GB) — measured: 8.51 GiB FP8 weights + ~1.5 GiB graph + -# 2.53 GiB fp8 KV (41,456-token pool, 1.27x concurrency at full 32K). util 0.12 -# was too thin (1.86 GiB KV < the 2.0 GiB a single 32K request needs → crash). -# Keeps GPU 1 total ~0.95 → ~5 GB buffer; ctx 32768 mirrors the old judge config. -SELENE_GPU_MEM_UTIL=0.13 +# util 0.17 (~16.3 GB) — grown from 0.13 to give Domari's now-live judge role real +# concurrency headroom (KV ~6.3 GiB → ~3.2x at 32K, far more at short eval ctx). +# 8.51 GiB FP8 weights + graph + the rest is KV. (0.12 crashed — KV < one 32K req; +# 0.13 was the minimum-fit; 0.17 is the comfortable judge size.) ctx 32768. +SELENE_GPU_MEM_UTIL=0.17 SELENE_MAX_MODEL_LEN=32768 SELENE_MAX_NUM_SEQS=16