# Move the `sec` pen-test seat (M.O.G.-SEC-27B) from ana-ml2 GPU 1 to GPU 0 and bring it up.
#
# Why: GPU 1 carries the five resident fleet seats (gen 46 GB + embed 9.8 + coder 8.4 +
# rerank 3.5 + reward 2.1 = ~69.9 GB of 97.9), leaving ~28 GB. This seat reserves
# MOG_GPU_MEM_UTIL=0.52 -> ~51 GB, so it could not start on GPU 1 at all. GPU 0 has been
# idle since run 3c was stopped on 2026-08-26. Operator-directed 2026-08-28.
#
# ⚠ POWER. This re-arms the two-GPU load condition that tripped the Anaheim rack breaker
# on 2026-08-26. One circuit feeds the whole rack including ana-gw and ana-wg, so a trip
# costs the site AND the way back in. Idle draw is negligible; the risk materialises when
# sec and gen are under concurrent load. Operator accepted this with the constraint stated.
#
# Labels only apply at container CREATION, so this uses `up -d`, never `restart` --
# the homepage description carries the GPU number and would otherwise stay stale.
#
# Run:  scripts/elway infra-ops@ana-ml2 --playbook playbooks/mog-sec-move-to-gpu0.yaml
# Model load is slow (22 GB + 262K ctx + MTP graft); the verify phase polls rather than
# assuming readiness, and the compose healthcheck allows a 900s start_period.

vars:
  stack_dir: /opt/docker/compose/mog-sec
  container: vllm-mog-sec
  service: vllm-mog-sec
  gpu_id: "0"
  port: "8019"
  staging: /tmp/mog-sec-compose.yaml

steps:
  - name: Stage the updated compose (GPU pin default + label now say GPU 0)
    upload:
      src: stacks/mog-sec/compose.yaml
      dest: "{{ staging }}"
      mode: "0644"

  - name: Install it over the deployed copy
    # /opt/docker/compose is root-owned, so the scp above lands in /tmp and this
    # promotes it. Verified byte-identical against the deployed file beforehand:
    # the only diff was these edits, so nothing on the host is being clobbered.
    shell: sudo install -o root -g root -m 0644 {{ staging }} {{ stack_dir }}/compose.yaml
    changed_when: "! sudo cmp -s {{ staging }} {{ stack_dir }}/compose.yaml"

  - name: Pin the seat to GPU {{ gpu_id }} in the host .env
    # The .env is the tunable surface and is NOT in git (secrets//tunables are
    # excluded both directions). The compose default now matches, but the .env
    # is what actually decides, so set it explicitly rather than relying on the
    # default resolving.
    shell: sudo sed -i 's/^MOG_GPU_ID=.*/MOG_GPU_ID={{ gpu_id }}/' {{ stack_dir }}/.env
    when: "! sudo grep -qxF 'MOG_GPU_ID={{ gpu_id }}' {{ stack_dir }}/.env"

  - name: Bring the seat up (up -d, not restart — labels apply at creation)
    shell: cd {{ stack_dir }} && sudo docker compose up -d {{ service }}

verify:
  - name: Container exists and is running
    # ⚠ No `docker inspect -f` here. Go templates use {{ }} and so does elway's own
    # variable substitution, so an inspect format string gets eaten before it reaches
    # the host -- these two checks reported FAILED on a deploy that had in fact
    # succeeded. Filter-and-grep has no such collision.
    shell: sudo docker ps --filter name={{ container }} --filter status=running --quiet | grep -q .
    changed_when: "false"

  - name: The container is actually pinned to GPU {{ gpu_id }}
    # Assert the EFFECTIVE device reservation on the running container, not the
    # .env string we wrote -- the .env is an input, this is the outcome.
    shell: sudo docker inspect {{ container }} | tr -d ' \n' | grep -q '"DeviceIDs":\["{{ gpu_id }}"\]'
    changed_when: "false"

  - name: GPU 0 now holds a vLLM process (the seat really loaded onto that card)
    shell: nvidia-smi --id={{ gpu_id }} --query-compute-apps=pid,used_memory --format=csv,noheader | grep -qE '[0-9]'
    changed_when: "false"

  - name: Health endpoint answers
    shell: curl -fsS --max-time 10 http://127.0.0.1:{{ port }}/health >/dev/null
    changed_when: "false"

  - name: Both served names are advertised (base + thinking)
    shell: |
      MODELS=$(curl -fsS --max-time 10 http://127.0.0.1:{{ port }}/v1/models)
      echo "$MODELS" | grep -q 'mog-sec-27b' && echo "$MODELS" | grep -q 'mog-sec-27b-thinking'
    changed_when: "false"