vh
6851ecbd0b
meromero-rp sized its KV from --gpu-memory-utilization, and vLLM's profile counts co-booting neighbours as its own non-torch memory. Three boots gave 20.37 / 26.74 / 29.04 GiB of KV, and the 2026-10-05 Docker-move restart cut GPU 1 free memory to 3,981 MiB, below intern-decision's 32k-token budget. Coder read negative KV on every co-boot. Both are now pinned, so util only gates the boot check (meromero util 0.52 -> 0.47, just above its footprint). GPU 0 plateaus at ~60 MiB free 1-2 days after a boot (Beszel), so gen-small's KV drops to 5.5 GiB (461,215 tokens, 1.76x) and parakeet-nemo stays on GPU 0 with room for its 3,840 MiB cap. GPU 3 stays clear. Applied with playbooks/fv-ml1-kv-pin-20261005.yaml: GPU 1 free 3,981 -> 11,801 MiB, GPU 0 495 -> 5,039 MiB right after boot; all nine seats healthy.
187 lines
8.6 KiB
YAML
187 lines
8.6 KiB
YAML
# Pin the KV cache in bytes on three fv-ml1 vLLM seats (2026-10-05, Prime's call).
|
|
#
|
|
# GPU 1 vllm-meromero-rp util-sized (0.52) -> pinned 22 GiB, util 0.47 (boot check only)
|
|
# GPU 1 vllm-coder util-sized -> pinned 1,104,674,816 B (77,056 tokens, unchanged)
|
|
# GPU 0 vllm-gen-small pinned 8 GiB -> pinned 5.5 GiB, so parakeet-nemo stays on GPU 0
|
|
#
|
|
# Why: on a shared card vLLM's memory profile counts co-booting neighbours as its own, so
|
|
# util-sized KV is a lottery per boot (meromero 20.37 / 26.74 / 29.04 GiB), and the 0234
|
|
# restart left intern-decision's 32k calls ~2.4 GB short. GPU 0 creeps to ~60 MiB free.
|
|
# Detail: persistent-memory.md, in-flight 2026-10-05.
|
|
#
|
|
# One seat at a time; each is guarded by the boot-check arithmetic (Free + own resident
|
|
# >= util x 97,249 MiB + 512) and must come back healthy with the expected KV line before
|
|
# the next one is touched. elway halts on the first failure.
|
|
# scripts/elway infra-ops@10.251.50.54 --playbook playbooks/fv-ml1-kv-pin-20261005.yaml
|
|
# Rollback per seat: restore its .env.bak-20261005-kvpin and the previous compose.yaml from
|
|
# git, then `docker compose up -d --no-deps <service>` in the stack dir.
|
|
vars:
|
|
mero: /opt/docker/compose/meromero-charrp
|
|
vllm: /opt/docker/compose/vllm
|
|
gens: /opt/docker/compose/gen-small-seat
|
|
mero_kv: "23622320128"
|
|
coder_kv: "1104674816"
|
|
gens_kv: "5905580032"
|
|
|
|
steps:
|
|
# ---------------- GPU 1: meromero-rp ----------------
|
|
- name: meromero — back up .env once
|
|
sudo: true
|
|
shell: cp -p {{ mero }}/.env {{ mero }}/.env.bak-20261005-kvpin
|
|
creates: "{{ mero }}/.env.bak-20261005-kvpin"
|
|
|
|
- name: meromero — util 0.52 -> 0.47
|
|
sudo: true
|
|
shell: sed -i 's/^MEROMERO_GPU_MEM_UTIL=.*/MEROMERO_GPU_MEM_UTIL=0.47/' {{ mero }}/.env
|
|
when: "! grep -qx 'MEROMERO_GPU_MEM_UTIL=0.47' {{ mero }}/.env"
|
|
|
|
- name: meromero — add MEROMERO_KV_CACHE_MEMORY
|
|
sudo: true
|
|
shell: sed -i '$a MEROMERO_KV_CACHE_MEMORY={{ mero_kv }}' {{ mero }}/.env
|
|
when: "! grep -q '^MEROMERO_KV_CACHE_MEMORY=' {{ mero }}/.env"
|
|
|
|
- name: meromero — upload compose.yaml
|
|
sudo: true
|
|
upload:
|
|
src: stacks/meromero-charrp/compose.yaml
|
|
dest: "{{ mero }}/compose.yaml"
|
|
mode: "0644"
|
|
owner: "lkraven:lkraven"
|
|
|
|
- name: meromero — dry-parse, and the rendered args carry the pin
|
|
sudo: true
|
|
shell: cd {{ mero }} && docker compose config -q && docker compose config | grep -q '{{ mero_kv }}'
|
|
changed_when: "false"
|
|
|
|
- name: meromero — boot-check arithmetic (Free + own >= 0.47 x 97249 + 512)
|
|
shell: |
|
|
own=0; for p in $(docker top vllm-meromero-rp -eo pid | tail -n +2); do
|
|
m=$(nvidia-smi --query-compute-apps=pid,used_memory --format=csv,noheader,nounits | awk -F', ' -v p="$p" '$1==p{print $2}')
|
|
own=$((own + ${m:-0})); done
|
|
free=$(nvidia-smi -i 1 --query-gpu=memory.free --format=csv,noheader,nounits)
|
|
need=$(awk 'BEGIN{printf "%d", 0.47*97249+512}')
|
|
echo "GPU1 free=$free own=$own need=$need"; [ $((free + own)) -ge "$need" ]
|
|
when: "! docker inspect vllm-meromero-rp | grep -q '{{ mero_kv }}'"
|
|
changed_when: "false"
|
|
|
|
- name: meromero — recreate the seat
|
|
sudo: true
|
|
shell: cd {{ mero }} && docker compose up -d --no-deps vllm-meromero-rp
|
|
when: "! docker inspect vllm-meromero-rp | grep -q '{{ mero_kv }}'"
|
|
|
|
- name: meromero — wait for healthy (up to 20 min)
|
|
shell: |
|
|
i=0; while [ $i -lt 120 ]; do
|
|
[ -n "$(docker ps -q --filter name=^vllm-meromero-rp$ --filter health=healthy)" ] && exit 0
|
|
i=$((i + 1)); sleep 10; done; exit 1
|
|
changed_when: "false"
|
|
|
|
- name: meromero — KV is the pinned 22 GiB
|
|
shell: docker logs vllm-meromero-rp 2>&1 | grep -E 'reserved 22.0 GiB memory for KV Cache|GPU KV cache size' | tail -2 && docker logs vllm-meromero-rp 2>&1 | grep -q 'reserved 22.0 GiB memory for KV Cache'
|
|
changed_when: "false"
|
|
|
|
# ---------------- GPU 1: coder ----------------
|
|
- name: coder — back up .env once
|
|
sudo: true
|
|
shell: cp -p {{ vllm }}/.env {{ vllm }}/.env.bak-20261005-kvpin
|
|
creates: "{{ vllm }}/.env.bak-20261005-kvpin"
|
|
|
|
- name: coder — add CODER_KV_CACHE_MEMORY
|
|
sudo: true
|
|
shell: sed -i '$a CODER_KV_CACHE_MEMORY={{ coder_kv }}' {{ vllm }}/.env
|
|
when: "! grep -q '^CODER_KV_CACHE_MEMORY=' {{ vllm }}/.env"
|
|
|
|
- name: coder — upload compose.yaml
|
|
sudo: true
|
|
upload:
|
|
src: stacks/vllm/compose.yaml
|
|
dest: "{{ vllm }}/compose.yaml"
|
|
mode: "0644"
|
|
owner: "root:docker"
|
|
|
|
- name: coder — dry-parse, and the rendered args carry the pin
|
|
sudo: true
|
|
shell: cd {{ vllm }} && docker compose config -q && docker compose config | grep -q '{{ coder_kv }}'
|
|
changed_when: "false"
|
|
|
|
- name: coder — recreate the seat
|
|
sudo: true
|
|
shell: cd {{ vllm }} && docker compose up -d --no-deps vllm-coder
|
|
when: "! docker inspect vllm-coder | grep -q '{{ coder_kv }}'"
|
|
|
|
- name: coder — wait for healthy (up to 10 min)
|
|
shell: |
|
|
i=0; while [ $i -lt 60 ]; do
|
|
[ -n "$(docker ps -q --filter name=^vllm-coder$ --filter health=healthy)" ] && exit 0
|
|
i=$((i + 1)); sleep 10; done; exit 1
|
|
changed_when: "false"
|
|
|
|
- name: coder — KV is the pinned 77,056 tokens
|
|
shell: |
|
|
docker logs vllm-coder 2>&1 | grep -E 'GPU KV cache size' | tail -1
|
|
docker logs vllm-coder 2>&1 | grep -q 'GPU KV cache size: 77,056 tokens'
|
|
changed_when: "false"
|
|
|
|
# ---------------- GPU 0: gen-small ----------------
|
|
- name: gen-small — back up .env once
|
|
sudo: true
|
|
shell: cp -p {{ gens }}/.env {{ gens }}/.env.bak-20261005-kvpin
|
|
creates: "{{ gens }}/.env.bak-20261005-kvpin"
|
|
|
|
- name: gen-small — KV 8 GiB -> 5.5 GiB
|
|
sudo: true
|
|
shell: sed -i 's/^GEN_SMALL_KV_CACHE_MEMORY=.*/GEN_SMALL_KV_CACHE_MEMORY={{ gens_kv }}/' {{ gens }}/.env
|
|
when: "! sudo -n grep -qx 'GEN_SMALL_KV_CACHE_MEMORY={{ gens_kv }}' {{ gens }}/.env"
|
|
|
|
- name: gen-small — upload compose.yaml (compose default follows the .env)
|
|
sudo: true
|
|
upload:
|
|
src: stacks/gen-small-seat/compose.yaml
|
|
dest: "{{ gens }}/compose.yaml"
|
|
mode: "0644"
|
|
owner: "root:root"
|
|
|
|
- name: gen-small — dry-parse, the rendered args carry the pin, util is 0.33
|
|
sudo: true
|
|
shell: cd {{ gens }} && docker compose config -q && docker compose config | grep -q '{{ gens_kv }}' && grep -qx 'GEN_SMALL_GPU_MEM_UTIL=0.33' .env
|
|
changed_when: "false"
|
|
|
|
- name: gen-small — boot-check arithmetic (Free + own >= 0.33 x 97249 + 512)
|
|
shell: |
|
|
own=0; for p in $(docker top vllm-gen-small -eo pid | tail -n +2); do
|
|
m=$(nvidia-smi --query-compute-apps=pid,used_memory --format=csv,noheader,nounits | awk -F', ' -v p="$p" '$1==p{print $2}')
|
|
own=$((own + ${m:-0})); done
|
|
free=$(nvidia-smi -i 0 --query-gpu=memory.free --format=csv,noheader,nounits)
|
|
need=$(awk 'BEGIN{printf "%d", 0.33*97249+512}')
|
|
echo "GPU0 free=$free own=$own need=$need"; [ $((free + own)) -ge "$need" ]
|
|
when: "! docker inspect vllm-gen-small | grep -q '{{ gens_kv }}'"
|
|
changed_when: "false"
|
|
|
|
- name: gen-small — recreate the seat
|
|
sudo: true
|
|
shell: cd {{ gens }} && docker compose up -d --no-deps vllm-gen-small
|
|
when: "! docker inspect vllm-gen-small | grep -q '{{ gens_kv }}'"
|
|
|
|
- name: gen-small — wait for healthy (up to 20 min)
|
|
shell: |
|
|
i=0; while [ $i -lt 120 ]; do
|
|
[ -n "$(docker ps -q --filter name=^vllm-gen-small$ --filter health=healthy)" ] && exit 0
|
|
i=$((i + 1)); sleep 10; done; exit 1
|
|
changed_when: "false"
|
|
|
|
- name: gen-small — KV is the pinned 5.5 GiB
|
|
shell: docker logs vllm-gen-small 2>&1 | grep -E 'reserved 5.5 GiB memory for KV Cache|GPU KV cache size' | tail -2 && docker logs vllm-gen-small 2>&1 | grep -q 'reserved 5.5 GiB memory for KV Cache'
|
|
changed_when: "false"
|
|
|
|
verify:
|
|
- name: per-GPU free memory (GPU 1 must clear intern-decision's 32k need of ~6,400 MiB + margin)
|
|
shell: nvidia-smi --query-gpu=index,memory.used,memory.free --format=csv,noheader && [ "$(nvidia-smi -i 1 --query-gpu=memory.free --format=csv,noheader,nounits)" -ge 8000 ]
|
|
changed_when: "false"
|
|
- name: .env owners and modes unchanged
|
|
sudo: true
|
|
shell: test "$(stat -c '%U %a' {{ mero }}/.env)" = "lkraven 644" && test "$(stat -c '%U %a' {{ vllm }}/.env)" = "lkraven 644" && test "$(stat -c '%U %a' {{ gens }}/.env)" = "root 600"
|
|
changed_when: "false"
|
|
- name: all nine fv-ml1 GPU seats healthy
|
|
shell: for c in vllm-cyberprev vllm-gen-small vllm-voices parakeet-nemo intern-decision vllm-meromero-rp vllm-erp-seat vllm-coder vllm-flash-next; do docker ps -q --filter name=^$c$ --filter health=healthy | grep -q . || { echo "NOT healthy $c"; exit 1; }; done
|
|
changed_when: "false"
|