# esh-pve-nas — reboot the NAS hypervisor without wedging its NFS clients.
#
# Run:  scripts/elway root@esh-pve-nas --playbook playbooks/esh-pve-nas-safe-reboot.yaml --var quiesced=yes
#
# PRECONDITION: run the quiesce playbooks first and confirm the client mount
# tables are clear — this host has no ssh keys to them, so the caller attests:
#     scripts/elway infra-ops@10.0.50.45 -p playbooks/esh-cutover-1-quiesce-docker-vm.yaml
#     scripts/elway root@10.0.250.35    -p playbooks/esh-cutover-2-quiesce-esh-pve.yaml
# Restore after with esh-cutover-5-restore-docker-vm.yaml + re-enable the esh-pve
# storages.
#
# ⚠ This host is half of the 2-node `esh-pve-cluster` (quorum 2, no qdevice), so
# while it is down the OTHER node's /etc/pve is READ-ONLY. Guests there keep
# running; config changes, VM start/stop and storage edits do not work until this
# host returns. HA manages no resources, so there is no watchdog fencing risk.
#
# ⚠ There is NO auto-fallback if the boot fails, and NO IPMI/BMC/serial console on
# this box. grubenv lives on an LVM LV that GRUB can read but not write, so
# one-shot boot selection does not survive. Recovery from a failed boot means
# physically selecting the ROLLBACK entry at the GRUB menu.

vars:
  quiesced: "no"

steps:
  - name: GUARD — caller has confirmed both hard-NFS clients are unmounted
    shell: |
      test "{{ quiesced }}" = "yes" || {
        echo "quiesce the NFS clients first, then pass --var quiesced=yes"; exit 1; }
      echo "--- NFS sessions still seen by CT 103 (informational) ---"
      pct exec 103 -- ss -tnH state established '( sport = :2049 )' 2>/dev/null \
        | awk '{print $4}' | sed 's/:[0-9]*$//' | sort | uniq -c || true
    changed_when: "false"

  - name: GUARD — the boot chain is sane before we rely on it
    shell: |
      grep -q '^GRUB_DEFAULT=0' /etc/default/grub || { echo "GRUB_DEFAULT is not 0"; exit 1; }
      grep -q 'pve-ext4-rollback' /boot/grub/grub.cfg || { echo "no rollback entry"; exit 1; }
      awk '/^[[:space:]]*linux[[:space:]]/ {
             r=""; for (i=1;i<=NF;i++) if ($i ~ /^root=/) r=$i;
             if (r != "root=ZFS=nvme/ROOT/pve-1" && r != "root=/dev/mapper/pve-root") {
               print "BAD EFFECTIVE ROOT: " r; bad=1 }
           } END { exit bad?1:0 }' /boot/grub/grub.cfg
      first=$(awk '/^menuentry /{print NR; exit}' /boot/grub/grub.cfg)
      awk -v s="$first" 'NR>s && /^[[:space:]]*linux[[:space:]]/ {print "  entry 0 -> " $0; exit}' /boot/grub/grub.cfg
      echo "boot chain OK"
    changed_when: "false"

  - name: Stop the guests, CT 103 (the NAS) last
    shell: |
      for v in 105 106 107; do
        pct status $v 2>/dev/null | grep -q running && pct shutdown $v --timeout 90 || true
      done
      qm status 104 2>/dev/null | grep -q running && qm shutdown 104 --timeout 90 || true
      for i in $(seq 1 30); do
        running=$( (pct list | awk 'NR>1 && $2=="running"'; qm list | awk 'NR>1 && $3=="running"') | wc -l )
        [ "$running" -le 1 ] && break
        sleep 3
      done
      pct status 103 2>/dev/null | grep -q running && pct shutdown 103 --timeout 90 || true
      sleep 3
      echo "--- remaining ---"; pct list; qm list | tail -3
    changed_when: "true"

  - name: REBOOT — connection loss here is expected
    shell: |
      sync
      systemd-run --on-active=3 --timer-property=AccuracySec=1s systemctl reboot >/dev/null 2>&1
      echo "reboot armed (+3s)"
    changed_when: "true"