esh-pfi-infrastructure/playbooks/fv-ml1-rebalance-20261005.yaml

# Rebalance fv-ml1 GPUs 0 and 1 (2026-10-05, Prime: "accept all that", plus "all models need
# context lengths up to their published maximums if at all possible"). GPU 2 (flash-next) and
# GPU 3 (clear for ad-hoc work) are not touched; meromero-rp and intern-decision are not touched.
#
#   seat        before                          after
#   gen-small   GPU 0, KV 5.5 GiB, ctx 262,144  GPU 1, same KV/ctx, default output cap 32,768
#   erp-seat    GPU 1, KV 8.49 GiB, util 0.24   GPU 0, KV 5.5 GiB, util 0.25
#   coder       GPU 1, ctx 16,384, util 0.055   GPU 0, ctx 32,768 (card maximum), util 0.06
#   voices      GPU 0, ctx 8,192, KV 1.5 GiB    GPU 0, ctx 40,960 (config/card native), KV 6 GiB, util 0.16
#   cyberprev   GPU 0, KV 16 GiB                GPU 0, KV 15 GiB
#
# Order keeps every boot check satisfied and never stacks a seat onto a card without room:
# stop erp + coder (frees GPU 1) -> gen-small to GPU 1 -> coder to GPU 0 -> voices in place ->
# erp to GPU 0 -> cyberprev in place. Each seat must come back healthy with the expected KV/ctx
# line before the next is touched; elway halts on the first failure.
#   scripts/elway infra-ops@10.251.50.54 --playbook playbooks/fv-ml1-rebalance-20261005.yaml
# Rollback per seat: restore its .env.bak-20261005-rebal (+ compose.yaml from git), then
# `docker compose up -d --no-deps <service>` in the stack dir, mind the order above.
vars:
  base: /opt/docker/compose
  gens_dir: /opt/docker/compose/gen-small-seat
  erp_dir: /opt/docker/compose/erp-seat
  vllm_dir: /opt/docker/compose/vllm
  voices_dir: /opt/docker/compose/voices-seat
  cyber_dir: /opt/docker/compose/cyberprev-seat
  check: /tmp/rebal-gpucheck.sh
  wait: /tmp/rebal-wait-healthy.sh

steps:
  - name: upload helpers
    upload:
      src: scripts/oneoff/fv-ml1-gpucheck.sh
      dest: "{{ check }}"
      mode: "0755"
  - name: upload wait helper
    upload:
      src: scripts/oneoff/wait-healthy.sh
      dest: "{{ wait }}"
      mode: "0755"

  - name: back up the five .env files once
    sudo: true
    shell: for d in {{ gens_dir }} {{ erp_dir }} {{ vllm_dir }} {{ voices_dir }} {{ cyber_dir }}; do cp -p $d/.env $d/.env.bak-20261005-rebal; done
    creates: "{{ cyber_dir }}/.env.bak-20261005-rebal"

  # ---------- 1. free GPU 1: stop erp + coder (both near idle: 55 and 4 gateway calls/7d) ----------
  - name: stop erp-seat (until it is recreated on GPU 0)
    shell: docker stop vllm-erp-seat
    when: "! docker inspect vllm-erp-seat | grep -q 5905580032"
  - name: stop coder (until it is recreated on GPU 0)
    shell: docker stop vllm-coder
    when: "! docker inspect vllm-coder | grep -q '\"32768\"'"

  # ---------- 2. gen-small -> GPU 1, default output cap ----------
  - name: gen-small .env GPU 0 -> 1
    sudo: true
    shell: sed -i 's/^GEN_SMALL_GPU_ID=.*/GEN_SMALL_GPU_ID=1/' {{ gens_dir }}/.env
    when: "! sudo -n grep -qx 'GEN_SMALL_GPU_ID=1' {{ gens_dir }}/.env"
  - name: gen-small upload compose (adds --override-generation-config)
    sudo: true
    upload:
      src: stacks/gen-small-seat/compose.yaml
      dest: "{{ gens_dir }}/compose.yaml"
      mode: "0644"
      owner: "root:root"
  - name: gen-small dry-parse; rendered config carries the cap and GPU 1
    sudo: true
    shell: cd {{ gens_dir }} && docker compose config -q && docker compose config | grep -q max_new_tokens && docker compose config | grep -A3 device_ids | grep -q '"1"'
    changed_when: "false"
  - name: gen-small boot check on GPU 1 (util 0.33)
    shell: "{{ check }} 1 0.33"
    when: "! docker inspect vllm-gen-small | grep -q max_new_tokens"
    changed_when: "false"
  - name: gen-small recreate on GPU 1
    sudo: true
    shell: cd {{ gens_dir }} && docker compose up -d --no-deps vllm-gen-small
    when: "! docker inspect vllm-gen-small | grep -q max_new_tokens"
  - name: gen-small healthy
    shell: "{{ wait }} vllm-gen-small 1500"
    changed_when: "false"
  - name: gen-small KV line and output cap in the log
    shell: |
      docker logs vllm-gen-small 2>&1 | grep -E 'GPU KV cache size|max_tokens' | tail -3
      docker logs vllm-gen-small 2>&1 | grep -q 'reserved 5.5 GiB memory for KV Cache'
    changed_when: "false"

  # ---------- 3. coder -> GPU 0, context 32,768 ----------
  - name: coder .env GPU 0, ctx 32768, util 0.06
    sudo: true
    shell: sed -i -e 's/^CODER_GPU_ID=.*/CODER_GPU_ID=0/' -e 's/^CODER_MAX_MODEL_LEN=.*/CODER_MAX_MODEL_LEN=32768/' -e 's/^CODER_GPU_MEM_UTIL=.*/CODER_GPU_MEM_UTIL=0.06/' {{ vllm_dir }}/.env
    when: "! grep -qx 'CODER_MAX_MODEL_LEN=32768' {{ vllm_dir }}/.env"
  - name: coder dry-parse
    sudo: true
    shell: cd {{ vllm_dir }} && docker compose config -q && docker compose config | grep -q '"32768"'
    changed_when: "false"
  - name: coder boot check on GPU 0 (util 0.06)
    shell: "{{ check }} 0 0.06"
    when: "! docker inspect vllm-coder | grep -q '\"32768\"'"
    changed_when: "false"
  - name: coder recreate on GPU 0
    sudo: true
    shell: cd {{ vllm_dir }} && docker compose up -d --no-deps vllm-coder
    when: "! docker inspect vllm-coder | grep -q '\"32768\"'"
  - name: coder healthy
    shell: "{{ wait }} vllm-coder 600"
    changed_when: "false"
  - name: coder KV line at 32,768
    shell: |
      docker logs vllm-coder 2>&1 | grep -E 'GPU KV cache size|Maximum concurrency' | tail -2
      docker logs vllm-coder 2>&1 | grep -q 'GPU KV cache size: 77,056 tokens'
    changed_when: "false"

  # ---------- 4. voices in place: context 40,960, KV 6 GiB ----------
  - name: voices .env ctx 40960, KV 6 GiB, util 0.16
    sudo: true
    shell: |
      f={{ voices_dir }}/.env
      sed -i -e 's/^VOICES_KV_CACHE_MEMORY=.*/VOICES_KV_CACHE_MEMORY=6442450944/' -e 's/^VOICES_GPU_MEM_UTIL=.*/VOICES_GPU_MEM_UTIL=0.16/' $f
      grep -q '^VOICES_MAX_MODEL_LEN=' $f || sed -i '$a VOICES_MAX_MODEL_LEN=40960' $f
    when: "! grep -qx 'VOICES_MAX_MODEL_LEN=40960' {{ voices_dir }}/.env"
  - name: voices upload compose (defaults follow the .env)
    sudo: true
    upload:
      src: stacks/voices-seat/compose.yaml
      dest: "{{ voices_dir }}/compose.yaml"
      mode: "0644"
      owner: "infra-ops:infra-ops"
  - name: voices dry-parse
    sudo: true
    shell: cd {{ voices_dir }} && docker compose config -q && docker compose config | grep -q '"40960"' && docker compose config | grep -q 6442450944
    changed_when: "false"
  - name: voices boot check on GPU 0 (util 0.16, own memory counts)
    shell: "{{ check }} 0 0.16 vllm-voices"
    when: "! docker inspect vllm-voices | grep -q 6442450944"
    changed_when: "false"
  - name: voices recreate
    sudo: true
    shell: cd {{ voices_dir }} && docker compose up -d --no-deps vllm-voices
    when: "! docker inspect vllm-voices | grep -q 6442450944"
  - name: voices healthy
    shell: "{{ wait }} vllm-voices 1200"
    changed_when: "false"
  - name: voices KV line at 40,960
    shell: |
      docker logs vllm-voices 2>&1 | grep -E 'GPU KV cache size' | tail -1
      docker logs vllm-voices 2>&1 | grep -q 'reserved 6.0 GiB memory for KV Cache'
    changed_when: "false"

  # ---------- 5. erp-seat -> GPU 0, KV 5.5 GiB ----------
  - name: erp .env GPU 0, KV 5.5 GiB, util 0.25
    sudo: true
    shell: sed -i -e 's/^ERP_GPU_ID=.*/ERP_GPU_ID=0/' -e 's/^ERP_KV_CACHE_MEMORY=.*/ERP_KV_CACHE_MEMORY=5905580032/' -e 's/^ERP_GPU_MEM_UTIL=.*/ERP_GPU_MEM_UTIL=0.25/' {{ erp_dir }}/.env
    when: "! grep -qx 'ERP_KV_CACHE_MEMORY=5905580032' {{ erp_dir }}/.env"
  - name: erp upload compose (label says GPU0)
    sudo: true
    upload:
      src: stacks/erp-seat/compose.yaml
      dest: "{{ erp_dir }}/compose.yaml"
      mode: "0644"
      owner: "lkraven:lkraven"
  - name: erp dry-parse
    sudo: true
    shell: cd {{ erp_dir }} && docker compose config -q && docker compose config | grep -q 5905580032
    changed_when: "false"
  - name: erp boot check on GPU 0 (util 0.25)
    shell: "{{ check }} 0 0.25"
    when: "! docker inspect vllm-erp-seat | grep -q 5905580032"
    changed_when: "false"
  - name: erp recreate on GPU 0
    sudo: true
    shell: cd {{ erp_dir }} && docker compose up -d --no-deps vllm-erp-seat
    when: "! docker inspect vllm-erp-seat | grep -q 5905580032"
  - name: erp healthy
    shell: "{{ wait }} vllm-erp-seat 1200"
    changed_when: "false"
  - name: erp KV line
    shell: |
      docker logs vllm-erp-seat 2>&1 | grep -E 'GPU KV cache size' | tail -1
      docker logs vllm-erp-seat 2>&1 | grep -q 'reserved 5.5 GiB memory for KV Cache'
    changed_when: "false"

  # ---------- 6. cyberprev in place: KV 16 -> 15 GiB ----------
  - name: cyberprev .env KV 15 GiB
    sudo: true
    shell: sed -i 's/^CYBER_KV_CACHE_MEMORY=.*/CYBER_KV_CACHE_MEMORY=16106127360/' {{ cyber_dir }}/.env
    when: "! sudo -n grep -qx 'CYBER_KV_CACHE_MEMORY=16106127360' {{ cyber_dir }}/.env"
  - name: cyberprev dry-parse
    sudo: true
    shell: cd {{ cyber_dir }} && docker compose config -q && docker compose config | grep -q 16106127360
    changed_when: "false"
  - name: cyberprev boot check on GPU 0 (util 0.40, own memory counts)
    shell: "{{ check }} 0 0.40 vllm-cyberprev"
    when: "! docker inspect vllm-cyberprev | grep -q 16106127360"
    changed_when: "false"
  - name: cyberprev recreate
    sudo: true
    shell: cd {{ cyber_dir }} && docker compose up -d --no-deps vllm-cyberprev
    when: "! docker inspect vllm-cyberprev | grep -q 16106127360"
  - name: cyberprev healthy
    shell: "{{ wait }} vllm-cyberprev 1800"
    changed_when: "false"
  - name: cyberprev KV line
    shell: |
      docker logs vllm-cyberprev 2>&1 | grep -E 'GPU KV cache size' | tail -1
      docker logs vllm-cyberprev 2>&1 | grep -q 'reserved 15.0 GiB memory for KV Cache'
    changed_when: "false"

verify:
  - name: per-GPU free memory
    shell: nvidia-smi --query-gpu=index,memory.used,memory.free --format=csv,noheader
    changed_when: "false"
  - name: every fv-ml1 GPU seat healthy
    shell: for c in vllm-cyberprev vllm-gen-small vllm-voices parakeet-nemo intern-decision vllm-meromero-rp vllm-erp-seat vllm-coder vllm-flash-next; do docker ps -q --filter name=^$c$ --filter health=healthy | grep -q . || { echo "NOT healthy $c"; exit 1; }; done
    changed_when: "false"
  - name: .env owners and modes unchanged
    sudo: true
    shell: |
      test "$(stat -c '%U %a' {{ gens_dir }}/.env)" = "root 600" &&
      test "$(stat -c '%U %a' {{ cyber_dir }}/.env)" = "root 600" &&
      test "$(stat -c '%U %a' {{ erp_dir }}/.env)" = "lkraven 644" &&
      test "$(stat -c '%U %a' {{ vllm_dir }}/.env)" = "lkraven 644" &&
      test "$(stat -c '%U %a' {{ voices_dir }}/.env)" = "infra-ops 664"
    changed_when: "false"