# Pin the KV cache in bytes on three fv-ml1 vLLM seats (2026-10-05, Prime's call). # # GPU 1 vllm-meromero-rp util-sized (0.52) -> pinned 22 GiB, util 0.47 (boot check only) # GPU 1 vllm-coder util-sized -> pinned 1,104,674,816 B (77,056 tokens, unchanged) # GPU 0 vllm-gen-small pinned 8 GiB -> pinned 5.5 GiB, so parakeet-nemo stays on GPU 0 # # Why: on a shared card vLLM's memory profile counts co-booting neighbours as its own, so # util-sized KV is a lottery per boot (meromero 20.37 / 26.74 / 29.04 GiB), and the 0234 # restart left intern-decision's 32k calls ~2.4 GB short. GPU 0 creeps to ~60 MiB free. # Detail: persistent-memory.md, in-flight 2026-10-05. # # One seat at a time; each is guarded by the boot-check arithmetic (Free + own resident # >= util x 97,249 MiB + 512) and must come back healthy with the expected KV line before # the next one is touched. elway halts on the first failure. # scripts/elway infra-ops@10.251.50.54 --playbook playbooks/fv-ml1-kv-pin-20261005.yaml # Rollback per seat: restore its .env.bak-20261005-kvpin and the previous compose.yaml from # git, then `docker compose up -d --no-deps ` in the stack dir. vars: mero: /opt/docker/compose/meromero-charrp vllm: /opt/docker/compose/vllm gens: /opt/docker/compose/gen-small-seat mero_kv: "23622320128" coder_kv: "1104674816" gens_kv: "5905580032" steps: # ---------------- GPU 1: meromero-rp ---------------- - name: meromero — back up .env once sudo: true shell: cp -p {{ mero }}/.env {{ mero }}/.env.bak-20261005-kvpin creates: "{{ mero }}/.env.bak-20261005-kvpin" - name: meromero — util 0.52 -> 0.47 sudo: true shell: sed -i 's/^MEROMERO_GPU_MEM_UTIL=.*/MEROMERO_GPU_MEM_UTIL=0.47/' {{ mero }}/.env when: "! grep -qx 'MEROMERO_GPU_MEM_UTIL=0.47' {{ mero }}/.env" - name: meromero — add MEROMERO_KV_CACHE_MEMORY sudo: true shell: sed -i '$a MEROMERO_KV_CACHE_MEMORY={{ mero_kv }}' {{ mero }}/.env when: "! grep -q '^MEROMERO_KV_CACHE_MEMORY=' {{ mero }}/.env" - name: meromero — upload compose.yaml sudo: true upload: src: stacks/meromero-charrp/compose.yaml dest: "{{ mero }}/compose.yaml" mode: "0644" owner: "lkraven:lkraven" - name: meromero — dry-parse, and the rendered args carry the pin sudo: true shell: cd {{ mero }} && docker compose config -q && docker compose config | grep -q '{{ mero_kv }}' changed_when: "false" - name: meromero — boot-check arithmetic (Free + own >= 0.47 x 97249 + 512) shell: | own=0; for p in $(docker top vllm-meromero-rp -eo pid | tail -n +2); do m=$(nvidia-smi --query-compute-apps=pid,used_memory --format=csv,noheader,nounits | awk -F', ' -v p="$p" '$1==p{print $2}') own=$((own + ${m:-0})); done free=$(nvidia-smi -i 1 --query-gpu=memory.free --format=csv,noheader,nounits) need=$(awk 'BEGIN{printf "%d", 0.47*97249+512}') echo "GPU1 free=$free own=$own need=$need"; [ $((free + own)) -ge "$need" ] when: "! docker inspect vllm-meromero-rp | grep -q '{{ mero_kv }}'" changed_when: "false" - name: meromero — recreate the seat sudo: true shell: cd {{ mero }} && docker compose up -d --no-deps vllm-meromero-rp when: "! docker inspect vllm-meromero-rp | grep -q '{{ mero_kv }}'" - name: meromero — wait for healthy (up to 20 min) shell: | i=0; while [ $i -lt 120 ]; do [ -n "$(docker ps -q --filter name=^vllm-meromero-rp$ --filter health=healthy)" ] && exit 0 i=$((i + 1)); sleep 10; done; exit 1 changed_when: "false" - name: meromero — KV is the pinned 22 GiB shell: docker logs vllm-meromero-rp 2>&1 | grep -E 'reserved 22.0 GiB memory for KV Cache|GPU KV cache size' | tail -2 && docker logs vllm-meromero-rp 2>&1 | grep -q 'reserved 22.0 GiB memory for KV Cache' changed_when: "false" # ---------------- GPU 1: coder ---------------- - name: coder — back up .env once sudo: true shell: cp -p {{ vllm }}/.env {{ vllm }}/.env.bak-20261005-kvpin creates: "{{ vllm }}/.env.bak-20261005-kvpin" - name: coder — add CODER_KV_CACHE_MEMORY sudo: true shell: sed -i '$a CODER_KV_CACHE_MEMORY={{ coder_kv }}' {{ vllm }}/.env when: "! grep -q '^CODER_KV_CACHE_MEMORY=' {{ vllm }}/.env" - name: coder — upload compose.yaml sudo: true upload: src: stacks/vllm/compose.yaml dest: "{{ vllm }}/compose.yaml" mode: "0644" owner: "root:docker" - name: coder — dry-parse, and the rendered args carry the pin sudo: true shell: cd {{ vllm }} && docker compose config -q && docker compose config | grep -q '{{ coder_kv }}' changed_when: "false" - name: coder — recreate the seat sudo: true shell: cd {{ vllm }} && docker compose up -d --no-deps vllm-coder when: "! docker inspect vllm-coder | grep -q '{{ coder_kv }}'" - name: coder — wait for healthy (up to 10 min) shell: | i=0; while [ $i -lt 60 ]; do [ -n "$(docker ps -q --filter name=^vllm-coder$ --filter health=healthy)" ] && exit 0 i=$((i + 1)); sleep 10; done; exit 1 changed_when: "false" - name: coder — KV is the pinned 77,056 tokens shell: | docker logs vllm-coder 2>&1 | grep -E 'GPU KV cache size' | tail -1 docker logs vllm-coder 2>&1 | grep -q 'GPU KV cache size: 77,056 tokens' changed_when: "false" # ---------------- GPU 0: gen-small ---------------- - name: gen-small — back up .env once sudo: true shell: cp -p {{ gens }}/.env {{ gens }}/.env.bak-20261005-kvpin creates: "{{ gens }}/.env.bak-20261005-kvpin" - name: gen-small — KV 8 GiB -> 5.5 GiB sudo: true shell: sed -i 's/^GEN_SMALL_KV_CACHE_MEMORY=.*/GEN_SMALL_KV_CACHE_MEMORY={{ gens_kv }}/' {{ gens }}/.env when: "! sudo -n grep -qx 'GEN_SMALL_KV_CACHE_MEMORY={{ gens_kv }}' {{ gens }}/.env" - name: gen-small — upload compose.yaml (compose default follows the .env) sudo: true upload: src: stacks/gen-small-seat/compose.yaml dest: "{{ gens }}/compose.yaml" mode: "0644" owner: "root:root" - name: gen-small — dry-parse, the rendered args carry the pin, util is 0.33 sudo: true shell: cd {{ gens }} && docker compose config -q && docker compose config | grep -q '{{ gens_kv }}' && grep -qx 'GEN_SMALL_GPU_MEM_UTIL=0.33' .env changed_when: "false" - name: gen-small — boot-check arithmetic (Free + own >= 0.33 x 97249 + 512) shell: | own=0; for p in $(docker top vllm-gen-small -eo pid | tail -n +2); do m=$(nvidia-smi --query-compute-apps=pid,used_memory --format=csv,noheader,nounits | awk -F', ' -v p="$p" '$1==p{print $2}') own=$((own + ${m:-0})); done free=$(nvidia-smi -i 0 --query-gpu=memory.free --format=csv,noheader,nounits) need=$(awk 'BEGIN{printf "%d", 0.33*97249+512}') echo "GPU0 free=$free own=$own need=$need"; [ $((free + own)) -ge "$need" ] when: "! docker inspect vllm-gen-small | grep -q '{{ gens_kv }}'" changed_when: "false" - name: gen-small — recreate the seat sudo: true shell: cd {{ gens }} && docker compose up -d --no-deps vllm-gen-small when: "! docker inspect vllm-gen-small | grep -q '{{ gens_kv }}'" - name: gen-small — wait for healthy (up to 20 min) shell: | i=0; while [ $i -lt 120 ]; do [ -n "$(docker ps -q --filter name=^vllm-gen-small$ --filter health=healthy)" ] && exit 0 i=$((i + 1)); sleep 10; done; exit 1 changed_when: "false" - name: gen-small — KV is the pinned 5.5 GiB shell: docker logs vllm-gen-small 2>&1 | grep -E 'reserved 5.5 GiB memory for KV Cache|GPU KV cache size' | tail -2 && docker logs vllm-gen-small 2>&1 | grep -q 'reserved 5.5 GiB memory for KV Cache' changed_when: "false" verify: - name: per-GPU free memory (GPU 1 must clear intern-decision's 32k need of ~6,400 MiB + margin) shell: nvidia-smi --query-gpu=index,memory.used,memory.free --format=csv,noheader && [ "$(nvidia-smi -i 1 --query-gpu=memory.free --format=csv,noheader,nounits)" -ge 8000 ] changed_when: "false" - name: .env owners and modes unchanged sudo: true shell: test "$(stat -c '%U %a' {{ mero }}/.env)" = "lkraven 644" && test "$(stat -c '%U %a' {{ vllm }}/.env)" = "lkraven 644" && test "$(stat -c '%U %a' {{ gens }}/.env)" = "root 600" changed_when: "false" - name: all nine fv-ml1 GPU seats healthy shell: for c in vllm-cyberprev vllm-gen-small vllm-voices parakeet-nemo intern-decision vllm-meromero-rp vllm-erp-seat vllm-coder vllm-flash-next; do docker ps -q --filter name=^$c$ --filter health=healthy | grep -q . || { echo "NOT healthy $c"; exit 1; }; done changed_when: "false"