vh
48e34e1849
homepage-regroup, mog-sec-move-to-gpu0, pull-hf-repo and serve-qwen3.5-122b all carried runnable 'scripts/elway ana-ml2 --playbook ...' instructions or the old 10.250.50.54 address. Each would fail today against a dead name and a dead IP, so these are corrections rather than cosmetics. homepage-regroup is renamed to match; the other three keep their names, which never carried the host.
81 lines
4.0 KiB
YAML
81 lines
4.0 KiB
YAML
# Move the `sec` pen-test seat (M.O.G.-SEC-27B) from fv-ml1 GPU 1 to GPU 0 and bring it up.
|
|
#
|
|
# Why: GPU 1 carries the five resident fleet seats (gen 46 GB + embed 9.8 + coder 8.4 +
|
|
# rerank 3.5 + reward 2.1 = ~69.9 GB of 97.9), leaving ~28 GB. This seat reserves
|
|
# MOG_GPU_MEM_UTIL=0.52 -> ~51 GB, so it could not start on GPU 1 at all. GPU 0 has been
|
|
# idle since run 3c was stopped on 2026-08-26. Operator-directed 2026-08-28.
|
|
#
|
|
# ⚠ POWER. This re-arms the two-GPU load condition that tripped the Anaheim rack breaker
|
|
# on 2026-08-26. One circuit feeds the whole rack including ana-gw and ana-wg, so a trip
|
|
# costs the site AND the way back in. Idle draw is negligible; the risk materialises when
|
|
# sec and gen are under concurrent load. Operator accepted this with the constraint stated.
|
|
#
|
|
# Labels only apply at container CREATION, so this uses `up -d`, never `restart` --
|
|
# the homepage description carries the GPU number and would otherwise stay stale.
|
|
#
|
|
# Run: scripts/elway infra-ops@fv-ml1 --playbook playbooks/mog-sec-move-to-gpu0.yaml
|
|
# Model load is slow (22 GB + 262K ctx + MTP graft); the verify phase polls rather than
|
|
# assuming readiness, and the compose healthcheck allows a 900s start_period.
|
|
|
|
vars:
|
|
stack_dir: /opt/docker/compose/mog-sec
|
|
container: vllm-mog-sec
|
|
service: vllm-mog-sec
|
|
gpu_id: "0"
|
|
port: "8019"
|
|
staging: /tmp/mog-sec-compose.yaml
|
|
|
|
steps:
|
|
- name: Stage the updated compose (GPU pin default + label now say GPU 0)
|
|
upload:
|
|
src: stacks/mog-sec/compose.yaml
|
|
dest: "{{ staging }}"
|
|
mode: "0644"
|
|
|
|
- name: Install it over the deployed copy
|
|
# /opt/docker/compose is root-owned, so the scp above lands in /tmp and this
|
|
# promotes it. Verified byte-identical against the deployed file beforehand:
|
|
# the only diff was these edits, so nothing on the host is being clobbered.
|
|
shell: sudo install -o root -g root -m 0644 {{ staging }} {{ stack_dir }}/compose.yaml
|
|
changed_when: "! sudo cmp -s {{ staging }} {{ stack_dir }}/compose.yaml"
|
|
|
|
- name: Pin the seat to GPU {{ gpu_id }} in the host .env
|
|
# The .env is the tunable surface and is NOT in git (secrets//tunables are
|
|
# excluded both directions). The compose default now matches, but the .env
|
|
# is what actually decides, so set it explicitly rather than relying on the
|
|
# default resolving.
|
|
shell: sudo sed -i 's/^MOG_GPU_ID=.*/MOG_GPU_ID={{ gpu_id }}/' {{ stack_dir }}/.env
|
|
when: "! sudo grep -qxF 'MOG_GPU_ID={{ gpu_id }}' {{ stack_dir }}/.env"
|
|
|
|
- name: Bring the seat up (up -d, not restart — labels apply at creation)
|
|
shell: cd {{ stack_dir }} && sudo docker compose up -d {{ service }}
|
|
|
|
verify:
|
|
- name: Container exists and is running
|
|
# ⚠ No `docker inspect -f` here. Go templates use {{ }} and so does elway's own
|
|
# variable substitution, so an inspect format string gets eaten before it reaches
|
|
# the host -- these two checks reported FAILED on a deploy that had in fact
|
|
# succeeded. Filter-and-grep has no such collision.
|
|
shell: sudo docker ps --filter name={{ container }} --filter status=running --quiet | grep -q .
|
|
changed_when: "false"
|
|
|
|
- name: The container is actually pinned to GPU {{ gpu_id }}
|
|
# Assert the EFFECTIVE device reservation on the running container, not the
|
|
# .env string we wrote -- the .env is an input, this is the outcome.
|
|
shell: sudo docker inspect {{ container }} | tr -d ' \n' | grep -q '"DeviceIDs":\["{{ gpu_id }}"\]'
|
|
changed_when: "false"
|
|
|
|
- name: GPU 0 now holds a vLLM process (the seat really loaded onto that card)
|
|
shell: nvidia-smi --id={{ gpu_id }} --query-compute-apps=pid,used_memory --format=csv,noheader | grep -qE '[0-9]'
|
|
changed_when: "false"
|
|
|
|
- name: Health endpoint answers
|
|
shell: curl -fsS --max-time 10 http://127.0.0.1:{{ port }}/health >/dev/null
|
|
changed_when: "false"
|
|
|
|
- name: Both served names are advertised (base + thinking)
|
|
shell: |
|
|
MODELS=$(curl -fsS --max-time 10 http://127.0.0.1:{{ port }}/v1/models)
|
|
echo "$MODELS" | grep -q 'mog-sec-27b' && echo "$MODELS" | grep -q 'mog-sec-27b-thinking'
|
|
changed_when: "false"
|