esh-pfi-infrastructure/playbooks/ana-ml2-training-window-close.yaml

# ana-ml2 — CLOSE the ERP/RP tune window: put the fleet back the way it was.
#
#   gen GPU1 -> GPU0  ->  start mog-sec back onto GPU1
#
# The exact inverse of playbooks/ana-ml2-training-window-open.yaml.
#
# ⚠⚠ ORDER IS LOAD-BEARING, AND IT IS THE MIRROR OF THE OPEN ORDER.
# `gen` must vacate GPU1 BEFORE mog-sec is started. mog-sec runs at
# --gpu-memory-utilization 0.52 = 50,901 MiB that must be free at startup. With
# gen still resident on GPU1 only ~30,000 MiB is free, so mog-sec would fail to
# boot. gen moves back to the (empty) GPU0 first; step 4 waits for GPU1 to
# actually release before mog-sec is started at all.
#
# ⚠ FIRST STEP IS A GATE, NOT A COURTESY. If a training process is still
# resident on GPU0 this playbook REFUSES to run — moving gen back would either
# OOM the run or OOM gen. Override only when you have confirmed the run is
# finished or deliberately abandoned:
#
#   scripts/elway ana-ml2 --playbook playbooks/ana-ml2-training-window-close.yaml \
#     --var allow_busy_gpu0=true
#
#   scripts/elway ana-ml2 --playbook playbooks/ana-ml2-training-window-close.yaml

vars:
  gen_dir: /opt/docker/compose/gen-seat
  mog_dir: /opt/docker/compose/mog-sec
  gen_port: "8015"
  mog_port: "8019"
  # mog-sec's startup requirement: 0.52 x 97,887 MiB, rounded up.
  mog_required_free_mib: "50950"
  # Set to "true" to close the window even with a process still on GPU0.
  allow_busy_gpu0: "false"

steps:
  - name: "GATE — GPU0 is idle (refuses to evict a training run mid-flight)"
    sudo: true
    shell: |
      set -e
      gpu0_uuid=$(nvidia-smi --query-gpu=uuid --format=csv,noheader -i 0)
      n=$(nvidia-smi --query-compute-apps=gpu_uuid,pid --format=csv,noheader | grep -c "$gpu0_uuid" || true)
      echo "GPU0 compute procs: $n"
      if [ "$n" -eq 0 ]; then exit 0; fi
      if [ "{{ allow_busy_gpu0 }}" = "true" ]; then
        echo "GPU0 still busy but allow_busy_gpu0=true — proceeding under override"
        nvidia-smi --query-compute-apps=gpu_uuid,pid,used_memory,process_name --format=csv | grep "$gpu0_uuid" || true
        exit 0
      fi
      echo "REFUSING: a process is still resident on GPU0. Confirm the tune has"
      echo "finished, then rerun with --var allow_busy_gpu0=true"
      nvidia-smi --query-compute-apps=gpu_uuid,pid,used_memory,process_name --format=csv | grep "$gpu0_uuid" || true
      exit 1
    changed_when: "false"

  - name: "PREFLIGHT — record gen's current container id (proves the recreate later)"
    sudo: true
    shell: docker inspect vllm-gen --format '{{.Id}}' | tee /tmp/gen-container-id-before.txt
    changed_when: "false"

  - name: "Point gen back at GPU0 in its .env (GEN_GPU_ID 1 -> 0)"
    sudo: true
    # ⚠ `sudo` INSIDE the when: expression — a step's `sudo: true` does NOT
    # cover its guards, and the root-only .env makes an unsudo'd grep exit 2,
    # which silently skips the step. See the open playbook for the incident.
    shell: sed -i 's/^GEN_GPU_ID=1$/GEN_GPU_ID=0/' {{ gen_dir }}/.env
    when: "sudo grep -qx 'GEN_GPU_ID=1' {{ gen_dir }}/.env"

  - name: "Assert the EFFECTIVE device id, not the .env line"
    sudo: true
    # ⚠ Parse the JSON, do not regex the YAML — compose emits `- "0"` with
    # DOUBLE quotes. See the open playbook for the incident.
    shell: |
      docker compose --project-directory {{ gen_dir }} config --format json \
        | jq -e '.services["vllm-gen"].deploy.resources.reservations.devices[0].device_ids == ["0"]'
    changed_when: "false"

  - name: "Recreate gen onto GPU0"
    sudo: true
    shell: docker compose --project-directory {{ gen_dir }} up -d vllm-gen

  - name: "Wait for gen to serve /health on GPU0"
    sudo: true
    shell: |
      for i in $(seq 1 180); do
        if curl -sf -o /dev/null http://127.0.0.1:{{ gen_port }}/health; then
          echo "gen healthy after $((i*5))s"; exit 0
        fi
        sleep 5
      done
      echo "TIMEOUT: gen did not become healthy in 900s"; exit 1
    changed_when: "false"

  - name: "Wait for GPU1 to release gen's memory before mog-sec is started"
    sudo: true
    shell: |
      for i in $(seq 1 60); do
        free=$(nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits -i 1)
        if [ "$free" -ge {{ mog_required_free_mib }} ]; then
          echo "GPU1 free: ${free} MiB"; exit 0
        fi
        sleep 2
      done
      echo "TIMEOUT: GPU1 free is ${free} MiB, need >= {{ mog_required_free_mib }}"; exit 1
    changed_when: "false"

  - name: "Start mog-sec back up on GPU1 (`sec` / `sec-reasoning`)"
    sudo: true
    shell: docker compose --project-directory {{ mog_dir }} start vllm-mog-sec
    when: "! docker inspect -f '{{.State.Running}}' vllm-mog-sec 2>/dev/null | grep -q true"

  - name: "Wait for mog-sec to serve /health"
    sudo: true
    shell: |
      for i in $(seq 1 180); do
        if curl -sf -o /dev/null http://127.0.0.1:{{ mog_port }}/health; then
          echo "mog-sec healthy after $((i*5))s"; exit 0
        fi
        sleep 5
      done
      echo "TIMEOUT: mog-sec did not become healthy in 900s"; exit 1
    changed_when: "false"

verify:
  - name: "gen was genuinely RECREATED (container id changed)"
    sudo: true
    shell: |
      before=$(cat /tmp/gen-container-id-before.txt)
      after=$(docker inspect vllm-gen --format '{{.Id}}')
      echo "before=${before:0:12} after=${after:0:12}"
      test "$before" != "$after"
    changed_when: "false"

  - name: "gen's process is resident on GPU0 again"
    sudo: true
    # ⚠ Match by CGROUP, not by `.State.Pid` — vLLM V1's EngineCore is a CHILD
    # of the container's pid 1, and it is the child nvidia-smi reports.
    shell: |
      cid=$(docker inspect vllm-gen --format '{{.Id}}')
      gpu0_uuid=$(nvidia-smi --query-gpu=uuid --format=csv,noheader -i 0)
      found=0
      for p in $(nvidia-smi --query-compute-apps=gpu_uuid,pid,used_memory --format=csv,noheader \
                 | grep "$gpu0_uuid" | cut -d, -f2 | tr -d ' '); do
        if grep -q "$cid" /proc/$p/cgroup 2>/dev/null; then
          echo "gen pid $p resident on GPU0: $(nvidia-smi --query-compute-apps=pid,used_memory --format=csv,noheader | grep "^$p,")"
          found=1
        fi
      done
      test "$found" -eq 1
    changed_when: "false"

  - name: "gen answers a real completion"
    sudo: true
    shell: |
      . {{ gen_dir }}/.env
      curl -sf -m 120 http://127.0.0.1:{{ gen_port }}/v1/chat/completions \
        -H "Authorization: Bearer ${API_KEY}" -H 'Content-Type: application/json' \
        -d '{"model":"'"${GEN_SERVED_NAME}"'","messages":[{"role":"user","content":"reply with the single word: ok"}],"max_tokens":16}' \
        | grep -q '"content"'
    changed_when: "false"

  - name: "sec answers a real completion"
    sudo: true
    shell: |
      . {{ mog_dir }}/.env
      curl -sf -m 120 http://127.0.0.1:{{ mog_port }}/v1/chat/completions \
        -H "Authorization: Bearer ${API_KEY}" -H 'Content-Type: application/json' \
        -d '{"model":"'"${MOG_SERVED_NAME}"'","messages":[{"role":"user","content":"reply with the single word: ok"}],"max_tokens":16}' \
        | grep -q '"content"'
    changed_when: "false"

  - name: "Both cards are back to their normal tenancy"
    sudo: true
    shell: |
      nvidia-smi --query-gpu=index,memory.used,memory.free --format=csv
      nvidia-smi --query-compute-apps=gpu_uuid,pid,used_memory,process_name --format=csv
    changed_when: "false"