# Rebalance fv-ml1 GPUs 0 and 1 (2026-10-05, Prime: "accept all that", plus "all models need # context lengths up to their published maximums if at all possible"). GPU 2 (flash-next) and # GPU 3 (clear for ad-hoc work) are not touched; meromero-rp and intern-decision are not touched. # # seat before after # gen-small GPU 0, KV 5.5 GiB, ctx 262,144 GPU 1, same KV/ctx, default output cap 32,768 # erp-seat GPU 1, KV 8.49 GiB, util 0.24 GPU 0, KV 5.5 GiB, util 0.25 # coder GPU 1, ctx 16,384, util 0.055 GPU 0, ctx 32,768 (card maximum), util 0.06 # voices GPU 0, ctx 8,192, KV 1.5 GiB GPU 0, ctx 40,960 (config/card native), KV 6 GiB, util 0.16 # cyberprev GPU 0, KV 16 GiB GPU 0, KV 15 GiB # # Order keeps every boot check satisfied and never stacks a seat onto a card without room: # stop erp + coder (frees GPU 1) -> gen-small to GPU 1 -> coder to GPU 0 -> voices in place -> # erp to GPU 0 -> cyberprev in place. Each seat must come back healthy with the expected KV/ctx # line before the next is touched; elway halts on the first failure. # scripts/elway infra-ops@10.251.50.54 --playbook playbooks/fv-ml1-rebalance-20261005.yaml # Rollback per seat: restore its .env.bak-20261005-rebal (+ compose.yaml from git), then # `docker compose up -d --no-deps ` in the stack dir, mind the order above. vars: base: /opt/docker/compose gens_dir: /opt/docker/compose/gen-small-seat erp_dir: /opt/docker/compose/erp-seat vllm_dir: /opt/docker/compose/vllm voices_dir: /opt/docker/compose/voices-seat cyber_dir: /opt/docker/compose/cyberprev-seat check: /tmp/rebal-gpucheck.sh wait: /tmp/rebal-wait-healthy.sh steps: - name: upload helpers upload: src: scripts/oneoff/fv-ml1-gpucheck.sh dest: "{{ check }}" mode: "0755" - name: upload wait helper upload: src: scripts/oneoff/wait-healthy.sh dest: "{{ wait }}" mode: "0755" - name: back up the five .env files once sudo: true shell: for d in {{ gens_dir }} {{ erp_dir }} {{ vllm_dir }} {{ voices_dir }} {{ cyber_dir }}; do cp -p $d/.env $d/.env.bak-20261005-rebal; done creates: "{{ cyber_dir }}/.env.bak-20261005-rebal" # ---------- 1. free GPU 1: stop erp + coder (both near idle: 55 and 4 gateway calls/7d) ---------- - name: stop erp-seat (until it is recreated on GPU 0) shell: docker stop vllm-erp-seat when: "! docker inspect vllm-erp-seat | grep -q 5905580032" - name: stop coder (until it is recreated on GPU 0) shell: docker stop vllm-coder when: "! docker inspect vllm-coder | grep -q '\"32768\"'" # ---------- 2. gen-small -> GPU 1, default output cap ---------- - name: gen-small .env GPU 0 -> 1 sudo: true shell: sed -i 's/^GEN_SMALL_GPU_ID=.*/GEN_SMALL_GPU_ID=1/' {{ gens_dir }}/.env when: "! sudo -n grep -qx 'GEN_SMALL_GPU_ID=1' {{ gens_dir }}/.env" - name: gen-small upload compose (adds --override-generation-config) sudo: true upload: src: stacks/gen-small-seat/compose.yaml dest: "{{ gens_dir }}/compose.yaml" mode: "0644" owner: "root:root" - name: gen-small dry-parse; rendered config carries the cap and GPU 1 sudo: true shell: cd {{ gens_dir }} && docker compose config -q && docker compose config | grep -q max_new_tokens && docker compose config | grep -A3 device_ids | grep -q '"1"' changed_when: "false" - name: gen-small boot check on GPU 1 (util 0.33) shell: "{{ check }} 1 0.33" when: "! docker inspect vllm-gen-small | grep -q max_new_tokens" changed_when: "false" - name: gen-small recreate on GPU 1 sudo: true shell: cd {{ gens_dir }} && docker compose up -d --no-deps vllm-gen-small when: "! docker inspect vllm-gen-small | grep -q max_new_tokens" - name: gen-small healthy shell: "{{ wait }} vllm-gen-small 1500" changed_when: "false" - name: gen-small KV line and output cap in the log shell: | docker logs vllm-gen-small 2>&1 | grep -E 'GPU KV cache size|max_tokens' | tail -3 docker logs vllm-gen-small 2>&1 | grep -q 'reserved 5.5 GiB memory for KV Cache' changed_when: "false" # ---------- 3. coder -> GPU 0, context 32,768 ---------- - name: coder .env GPU 0, ctx 32768, util 0.06 sudo: true shell: sed -i -e 's/^CODER_GPU_ID=.*/CODER_GPU_ID=0/' -e 's/^CODER_MAX_MODEL_LEN=.*/CODER_MAX_MODEL_LEN=32768/' -e 's/^CODER_GPU_MEM_UTIL=.*/CODER_GPU_MEM_UTIL=0.06/' {{ vllm_dir }}/.env when: "! grep -qx 'CODER_MAX_MODEL_LEN=32768' {{ vllm_dir }}/.env" - name: coder dry-parse sudo: true shell: cd {{ vllm_dir }} && docker compose config -q && docker compose config | grep -q '"32768"' changed_when: "false" - name: coder boot check on GPU 0 (util 0.06) shell: "{{ check }} 0 0.06" when: "! docker inspect vllm-coder | grep -q '\"32768\"'" changed_when: "false" - name: coder recreate on GPU 0 sudo: true shell: cd {{ vllm_dir }} && docker compose up -d --no-deps vllm-coder when: "! docker inspect vllm-coder | grep -q '\"32768\"'" - name: coder healthy shell: "{{ wait }} vllm-coder 600" changed_when: "false" - name: coder KV line at 32,768 shell: | docker logs vllm-coder 2>&1 | grep -E 'GPU KV cache size|Maximum concurrency' | tail -2 docker logs vllm-coder 2>&1 | grep -q 'GPU KV cache size: 77,056 tokens' changed_when: "false" # ---------- 4. voices in place: context 40,960, KV 6 GiB ---------- - name: voices .env ctx 40960, KV 6 GiB, util 0.16 sudo: true shell: | f={{ voices_dir }}/.env sed -i -e 's/^VOICES_KV_CACHE_MEMORY=.*/VOICES_KV_CACHE_MEMORY=6442450944/' -e 's/^VOICES_GPU_MEM_UTIL=.*/VOICES_GPU_MEM_UTIL=0.16/' $f grep -q '^VOICES_MAX_MODEL_LEN=' $f || sed -i '$a VOICES_MAX_MODEL_LEN=40960' $f when: "! grep -qx 'VOICES_MAX_MODEL_LEN=40960' {{ voices_dir }}/.env" - name: voices upload compose (defaults follow the .env) sudo: true upload: src: stacks/voices-seat/compose.yaml dest: "{{ voices_dir }}/compose.yaml" mode: "0644" owner: "infra-ops:infra-ops" - name: voices dry-parse sudo: true shell: cd {{ voices_dir }} && docker compose config -q && docker compose config | grep -q '"40960"' && docker compose config | grep -q 6442450944 changed_when: "false" - name: voices boot check on GPU 0 (util 0.16, own memory counts) shell: "{{ check }} 0 0.16 vllm-voices" when: "! docker inspect vllm-voices | grep -q 6442450944" changed_when: "false" - name: voices recreate sudo: true shell: cd {{ voices_dir }} && docker compose up -d --no-deps vllm-voices when: "! docker inspect vllm-voices | grep -q 6442450944" - name: voices healthy shell: "{{ wait }} vllm-voices 1200" changed_when: "false" - name: voices KV line at 40,960 shell: | docker logs vllm-voices 2>&1 | grep -E 'GPU KV cache size' | tail -1 docker logs vllm-voices 2>&1 | grep -q 'reserved 6.0 GiB memory for KV Cache' changed_when: "false" # ---------- 5. erp-seat -> GPU 0, KV 5.5 GiB ---------- - name: erp .env GPU 0, KV 5.5 GiB, util 0.25 sudo: true shell: sed -i -e 's/^ERP_GPU_ID=.*/ERP_GPU_ID=0/' -e 's/^ERP_KV_CACHE_MEMORY=.*/ERP_KV_CACHE_MEMORY=5905580032/' -e 's/^ERP_GPU_MEM_UTIL=.*/ERP_GPU_MEM_UTIL=0.25/' {{ erp_dir }}/.env when: "! grep -qx 'ERP_KV_CACHE_MEMORY=5905580032' {{ erp_dir }}/.env" - name: erp upload compose (label says GPU0) sudo: true upload: src: stacks/erp-seat/compose.yaml dest: "{{ erp_dir }}/compose.yaml" mode: "0644" owner: "lkraven:lkraven" - name: erp dry-parse sudo: true shell: cd {{ erp_dir }} && docker compose config -q && docker compose config | grep -q 5905580032 changed_when: "false" - name: erp boot check on GPU 0 (util 0.25) shell: "{{ check }} 0 0.25" when: "! docker inspect vllm-erp-seat | grep -q 5905580032" changed_when: "false" - name: erp recreate on GPU 0 sudo: true shell: cd {{ erp_dir }} && docker compose up -d --no-deps vllm-erp-seat when: "! docker inspect vllm-erp-seat | grep -q 5905580032" - name: erp healthy shell: "{{ wait }} vllm-erp-seat 1200" changed_when: "false" - name: erp KV line shell: | docker logs vllm-erp-seat 2>&1 | grep -E 'GPU KV cache size' | tail -1 docker logs vllm-erp-seat 2>&1 | grep -q 'reserved 5.5 GiB memory for KV Cache' changed_when: "false" # ---------- 6. cyberprev in place: KV 16 -> 15 GiB ---------- - name: cyberprev .env KV 15 GiB sudo: true shell: sed -i 's/^CYBER_KV_CACHE_MEMORY=.*/CYBER_KV_CACHE_MEMORY=16106127360/' {{ cyber_dir }}/.env when: "! sudo -n grep -qx 'CYBER_KV_CACHE_MEMORY=16106127360' {{ cyber_dir }}/.env" - name: cyberprev dry-parse sudo: true shell: cd {{ cyber_dir }} && docker compose config -q && docker compose config | grep -q 16106127360 changed_when: "false" - name: cyberprev boot check on GPU 0 (util 0.40, own memory counts) shell: "{{ check }} 0 0.40 vllm-cyberprev" when: "! docker inspect vllm-cyberprev | grep -q 16106127360" changed_when: "false" - name: cyberprev recreate sudo: true shell: cd {{ cyber_dir }} && docker compose up -d --no-deps vllm-cyberprev when: "! docker inspect vllm-cyberprev | grep -q 16106127360" - name: cyberprev healthy shell: "{{ wait }} vllm-cyberprev 1800" changed_when: "false" - name: cyberprev KV line shell: | docker logs vllm-cyberprev 2>&1 | grep -E 'GPU KV cache size' | tail -1 docker logs vllm-cyberprev 2>&1 | grep -q 'reserved 15.0 GiB memory for KV Cache' changed_when: "false" verify: - name: per-GPU free memory shell: nvidia-smi --query-gpu=index,memory.used,memory.free --format=csv,noheader changed_when: "false" - name: every fv-ml1 GPU seat healthy shell: for c in vllm-cyberprev vllm-gen-small vllm-voices parakeet-nemo intern-decision vllm-meromero-rp vllm-erp-seat vllm-coder vllm-flash-next; do docker ps -q --filter name=^$c$ --filter health=healthy | grep -q . || { echo "NOT healthy $c"; exit 1; }; done changed_when: "false" - name: .env owners and modes unchanged sudo: true shell: | test "$(stat -c '%U %a' {{ gens_dir }}/.env)" = "root 600" && test "$(stat -c '%U %a' {{ cyber_dir }}/.env)" = "root 600" && test "$(stat -c '%U %a' {{ erp_dir }}/.env)" = "lkraven 644" && test "$(stat -c '%U %a' {{ vllm_dir }}/.env)" = "lkraven 644" && test "$(stat -c '%U %a' {{ voices_dir }}/.env)" = "infra-ops 664" changed_when: "false"