esh-pfi-infrastructure/playbooks/esh-pve-pve9-upgrade.yaml

# esh-pve (`pve`): PVE 8.4 → 9 upgrade, the package half (Prime go 2026-10-04, on site). Does NOT reboot.
#   scripts/elway infra-ops@10.0.250.35 --playbook playbooks/esh-pve-pve9-upgrade.yaml
# Prereq: playbooks/esh-pve-pve9-prep.yaml. NO PVE 8 reboot test here: Prime chose one reboot (2026-10-04, on site),
# so the prep's next-boot changes (bridge-ports fix, NIC pins, kernel, microcode) land together with PVE 9.
# Template: playbooks/pfi-pve-pve9-upgrade.yaml. Plan: docs/runbooks/esh-pve-cluster-pve9-upgrade-plan.md.
# The dist-upgrade runs DETACHED (systemd-run → /root/pve9-upgrade.log), so a dropped ssh cannot kill it mid-dpkg.
# Poll: sudo tail /root/pve9-upgrade.log ; systemctl status pve9-dist-upgrade
# Then: playbooks/esh-pve-pve9-prereboot.yaml, then the reboot.
# Rollback: `lvconvert --merge pve/root-pre-pve9` and reboot (the merge runs at the next activation of pve/root);
# from a rescue boot if 9 does not boot. The snapshot covers / and /boot (one LV), not the ESP or guest storage.
vars:
  stage_dir: /root/pve9-prep-20261004

steps:
  - name: GUARD — pve8to9 --full reports no FAIL
    sudo: true
    shell: |
      out=$(pve8to9 --full 2>&1)
      echo "$out" | grep -E '^(FAIL|WARN):' | cat
      ! echo "$out" | grep -q '^FAIL:'
    changed_when: "false"

  - name: GUARD — prep is in (systemd-boot out, NIC pins, nvidia built for the newest 6.8)
    sudo: true
    shell: |
      set -e
      # an `if`, not `! pipeline`: set -e ignores a negated pipeline, so that line could never fail
      if dpkg-query -W -f='${Status}' systemd-boot 2>/dev/null | grep -q 'install ok installed'; then echo "systemd-boot still installed"; exit 1; fi
      test -f /etc/systemd/network/10-pin-enp3s0f0np0.link
      dpkg-query -W -f='${Status}' proxmox-default-headers | grep -q 'install ok installed'
      /usr/sbin/dkms status | grep -q "nvidia/.*, $(ls /boot/vmlinuz-* | sed 's#/boot/vmlinuz-##' | sort -V | tail -1), x86_64: installed"
    changed_when: "false"

  - name: GUARD — the VG has room for the root snapshot and dm-snapshot is in the running kernel's initramfs
    sudo: true
    shell: |
      set -e
      free=$(vgs --noheadings --units g --nosuffix -o vg_free pve | tr -d ' ' | cut -d. -f1)
      echo "pve VG free: ${free}G"
      test "$free" -ge 15
      lsinitramfs /boot/initrd.img-$(uname -r) | grep -q 'dm-snapshot.ko'
    changed_when: "false"

  - name: root snapshot for rollback (pve/root-pre-pve9, 15G COW)
    sudo: true
    shell: lvcreate -s -L 15G -n root-pre-pve9 pve/root
    when: "! lvs pve/root-pre-pve9 >/dev/null 2>&1"

  - name: switch apt sources bookworm → trixie (Debian lines rewritten; PVE repo moved to deb822 proxmox.sources)
    sudo: true
    shell: |
      set -e
      cp -p /etc/apt/sources.list {{ stage_dir }}/sources.list.bookworm
      cat > /etc/apt/sources.list <<'EOF'
      deb http://ftp.us.debian.org/debian trixie main contrib non-free-firmware
      deb http://ftp.us.debian.org/debian trixie-updates main contrib non-free-firmware
      deb http://security.debian.org/debian-security trixie-security main contrib non-free-firmware
      EOF
      cat > /etc/apt/sources.list.d/proxmox.sources <<'EOF'
      Types: deb
      URIs: http://download.proxmox.com/debian/pve
      Suites: trixie
      Components: pve-no-subscription
      Signed-By: /usr/share/keyrings/proxmox-archive-keyring.gpg
      EOF
      for f in /etc/apt/sources.list.d/pve-enterprise.list /etc/apt/sources.list.d/ceph.list; do
        if [ -e "$f" ]; then mv "$f" {{ stage_dir }}/; fi
      done
      cat /etc/apt/sources.list; ls /etc/apt/sources.list.d/
    when: "grep -q bookworm /etc/apt/sources.list"

  - name: apt update on trixie (must be clean)
    sudo: true
    shell: |
      ! apt-get update -q 2>&1 | grep -E '^(E|Err):'
    changed_when: "false"

  - name: GUARD — read the plan; refuse if it would remove anything this host cannot boot or network without
    sudo: true
    shell: |
      apt-get -s dist-upgrade > {{ stage_dir }}/dist-upgrade-plan.txt 2>&1
      grep -cE '^Inst ' {{ stage_dir }}/dist-upgrade-plan.txt | sed 's/^/  installs\/upgrades: /'
      grep -E '^Remv ' {{ stage_dir }}/dist-upgrade-plan.txt | sed 's/^/  /' || echo "  no removals"
      ! grep -E '^Remv (proxmox-ve|pve-manager|grub-efi-amd64|grub-efi-amd64-bin|grub-efi-amd64-signed|shim-signed|ifupdown2|lvm2|openssh-server|qemu-server|pve-container|dkms|proxmox-default-headers) ' {{ stage_dir }}/dist-upgrade-plan.txt
    changed_when: "false"

  - name: start the dist-upgrade DETACHED (log /root/pve9-upgrade.log)
    sudo: true
    shell: |
      systemd-run --unit=pve9-dist-upgrade --setenv=DEBIAN_FRONTEND=noninteractive \
        /bin/sh -c 'apt-get dist-upgrade -y -q -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold > /root/pve9-upgrade.log 2>&1; echo "EXIT $?" >> /root/pve9-upgrade.log'
    when: "! test -f /root/pve9-upgrade.log"

verify:
  - name: root snapshot exists
    sudo: true
    shell: lvs pve/root-pre-pve9
    changed_when: "false"
  - name: sources are trixie
    sudo: true
    shell: "! grep -rq bookworm /etc/apt/sources.list /etc/apt/sources.list.d/"
    changed_when: "false"