# esh-pve: after the PVE 9 dist-upgrade, BEFORE the reboot. One fix-up step, the rest is read-only.
#   scripts/elway infra-ops@10.0.250.35 --playbook playbooks/esh-pve-pve9-prereboot.yaml
# Every verify here is something that, if wrong, makes the reboot an ESH outage only someone at the console can end
# (esh-pve carries esh-scale, ESH's only mesh path, and has no AMT). Template: playbooks/pfi-pve-pve9-prereboot.yaml.
steps:
  - name: the upgrade ships pve-enterprise.sources enabled; turn it off (no subscription)
    sudo: true
    shell: |
      f=/etc/apt/sources.list.d/pve-enterprise.sources
      if grep -q '^Enabled:' $f; then sed -i 's/^Enabled:.*/Enabled: false/' $f; else echo 'Enabled: false' >> $f; fi
      cat $f
    when: "test -f /etc/apt/sources.list.d/pve-enterprise.sources && ! grep -q '^Enabled: *false' /etc/apt/sources.list.d/pve-enterprise.sources"

verify:
  - name: the detached dist-upgrade finished with EXIT 0
    sudo: true
    shell: tail -1 /root/pve9-upgrade.log | grep -qx 'EXIT 0'
    changed_when: "false"
  - name: dpkg is clean and nothing is left for apt -f
    sudo: true
    shell: |
      test -z "$(dpkg --audit 2>&1)"
      test "$(apt-get -s -f install 2>/dev/null | grep -cE '^(Inst|Conf) ')" -eq 0
    changed_when: "false"
  - name: PVE 9 userland
    sudo: true
    shell: pveversion | grep -E '^pve-manager/9\.'
    changed_when: "false"
  - name: nvidia DKMS module is built for the new kernel (esh-ml1's GPU; the nh3-pve trap)
    sudo: true
    shell: |
      new=$(ls /boot/vmlinuz-* | sed 's#/boot/vmlinuz-##' | sort -V | tail -1)
      /usr/sbin/dkms status | cat
      /usr/sbin/dkms status | grep -q "nvidia/.*, $new, x86_64: installed"
    changed_when: "false"
  - name: newest kernel has an initramfs carrying the NIC pins and dm-snapshot (root has an LVM snapshot)
    sudo: true
    shell: |
      set -e
      new=$(ls /boot/vmlinuz-* | sed 's#/boot/vmlinuz-##' | sort -V | tail -1)
      echo "newest kernel: $new"
      lsinitramfs /boot/initrd.img-$new > /tmp/pve9-initrd.lst
      grep -q 10-pin-enp3s0f0np0.link /tmp/pve9-initrd.lst
      grep -q 10-pin-enp88s0.link /tmp/pve9-initrd.lst
      grep -q 'dm-snapshot.ko' /tmp/pve9-initrd.lst
      grep -q "vmlinuz-$new" /boot/grub/grub.cfg
    changed_when: "false"
  - name: ESP rewritten by the trixie GRUB (shim path); removable copy matches; firmware boots proxmox shim first
    sudo: true
    shell: |
      set -e
      ls -la --time-style=long-iso /boot/efi/EFI/proxmox/ /boot/efi/EFI/BOOT/
      test "$(md5sum < /boot/efi/EFI/proxmox/shimx64.efi)" = "$(md5sum < /boot/efi/EFI/BOOT/BOOTx64.efi)"
      test "$(md5sum < /boot/efi/EFI/proxmox/grubx64.efi)" = "$(md5sum < /boot/efi/EFI/BOOT/grubx64.efi)"
      # The binary carries no greppable version string; the trixie grub-efi-amd64 postinst's own report is the proof.
      grep -q 'Installation finished. No error reported.' /root/pve9-upgrade.log
      first=$(efibootmgr | sed -n 's/^BootOrder: \([0-9A-F]*\).*/\1/p')
      efibootmgr -v | grep -E "^Boot${first}\*? " | grep -qi 'EFI\\proxmox\\shimx64.efi'
    changed_when: "false"
  - name: networking will come back (static vmbr0, interfaces parse, every bridge port exists, sysctl.conf has nothing to lose)
    sudo: true
    shell: |
      set -e
      dpkg-query -W -f='${Status}' ifupdown2 | grep -q 'install ok installed'
      for p in $(sed -n -E 's/^\s*bridge-ports\s+//p' /etc/network/interfaces); do test -e /sys/class/net/$p || { echo "missing $p"; exit 1; }; done
      # ifupdown2 3.3 (PVE 9) flags PVE's own `bridge-fd 0` as out of range and `-s` exits 1 on that warning alone.
      # nh3-pve and esh-nas-pve show the same warning and booted with vmbr0 fine (STP off, forward_delay 0), so
      # only a warning OTHER than that one fails this check.
      out=$(ifreload -a -s 2>&1) || true
      echo "$out"
      # (an `if`, not a bare `! pipeline`: set -e ignores a negated pipeline, so that form could never fail)
      if echo "$out" | grep -v 'bridge-fd: value of out range "0"' | grep -q .; then echo "unexpected ifupdown2 output"; exit 1; fi
      test -z "$(grep -vE '^\s*(#|$)' /etc/sysctl.conf 2>/dev/null)"
    changed_when: "false"
  - name: pve8to9 --full (post-upgrade checks)
    sudo: true
    shell: pve8to9 --full 2>&1 | grep -E '^(FAIL|WARN):|TOTAL|PASSED|SKIPPED|WARNINGS|FAILURES' | cat
    changed_when: "false"