ops(pfi-pve): PVE 9 prep done (8.4.21, grub-efi-amd64 on the UEFI/LVM box, microcode, NIC pins); upgrade plan + playbooks

playbooks/pfi-pve-pve9-prep.yaml

@@ -0,0 +1,150 @@
+# pfi-pve: no-downtime prep for the PVE 8 → 9 upgrade (Prime 2026-10-04: "drive the ana pve upgrade to proxmox 9").
+#   scripts/elway infra-ops@10.250.250.31 --playbook playbooks/pfi-pve-pve9-prep.yaml
+# Plan + rationale: docs/runbooks/pfi-pve-pve9-upgrade-plan.md. Template: playbooks/esh-nas-pve-pve9-prep.yaml.
+# Nothing here reboots or touches a guest. The new kernel, microcode, bootloader and NIC pins all take effect at the
+# NEXT boot, which is the planned reboot test on PVE 8, BEFORE any repo switch to trixie.
+#   - latest 8.4 packages (it is on 8.3.5 / kernel 6.8.12-8; pve8to9 only exists from 8.4);
+#   - non-free-firmware + intel-microcode;
+#   - grub-pc → grub-efi-amd64. The box boots UEFI from \EFI\proxmox\grubx64.efi, but the BIOS meta package is the
+#     one installed, so that EFI binary has not been touched since the 2023-03-30 (PVE 7.4 / bullseye) install. The
+#     PVE 8→9 wiki's "UEFI systems with root on LVM may fail to boot" fix is exactly this package. The 2023 binary is
+#     copied to EFI/proxmox-2023/ as a last resort only: once grub-install replaces /boot/grub/x86_64-efi, an old core
+#     loading new modules can fail (the grub_is_lockdown class), so the real fallback is the tar of /boot/grub;
+#   - pin every NIC name by MAC. vmbr0 has four ports and only eno8303 has carrier; the two bnxt ports carry an
+#     `np0/np1` phys_port_name suffix, the class of name the wiki says a new kernel can change.
+vars:
+  stage_dir: /root/pve9-prep-20261004
+
+steps:
+  - name: back up /etc, the package list and the 2023 EFI binaries
+    sudo: true
+    shell: |
+      set -e
+      mkdir -p {{ stage_dir }}
+      tar czf {{ stage_dir }}/etc-before.tar.gz -C / etc 2>/dev/null
+      dpkg -l > {{ stage_dir }}/dpkg-before.txt
+      pveversion -v > {{ stage_dir }}/pveversion-before.txt 2>&1
+      cp -p /etc/network/interfaces {{ stage_dir }}/interfaces.before
+      cp -p /etc/apt/sources.list {{ stage_dir }}/sources.list.before
+      cp -a /boot/efi/EFI {{ stage_dir }}/EFI-2023
+      cp -a /boot/grub {{ stage_dir }}/boot-grub-before
+      ls -la {{ stage_dir }}
+    creates: "{{ stage_dir }}/etc-before.tar.gz"
+
+  - name: keep the 2023 GRUB bootable from the ESP (EFI/proxmox-2023/)
+    sudo: true
+    shell: |
+      set -e
+      mkdir -p /boot/efi/EFI/proxmox-2023
+      cp -p /boot/efi/EFI/proxmox/grubx64.efi /boot/efi/EFI/proxmox-2023/grubx64.efi
+      md5sum /boot/efi/EFI/proxmox-2023/grubx64.efi
+    creates: /boot/efi/EFI/proxmox-2023/grubx64.efi
+
+  - name: add non-free-firmware to the Debian bookworm lines (for intel-microcode)
+    sudo: true
+    shell: sed -i -E '/^deb .*debian(\.org)?(\/debian)? bookworm/{/non-free-firmware/!s/$/ non-free-firmware/}' /etc/apt/sources.list
+    when: "grep -E '^deb .*debian(\\.org)?(/debian)? bookworm' /etc/apt/sources.list | grep -qv non-free-firmware"
+
+  - name: apt update (must be clean)
+    sudo: true
+    shell: |
+      ! apt-get update -q 2>&1 | grep -E '^(E|Err):'
+    changed_when: "false"
+
+  - name: latest 8.4 packages (keep local configs)
+    sudo: true
+    shell: |
+      set -o pipefail
+      DEBIAN_FRONTEND=noninteractive apt-get full-upgrade -y -q \
+        -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold 2>&1 | tail -8
+      pveversion | cat
+
+  - name: intel-microcode
+    sudo: true
+    shell: DEBIAN_FRONTEND=noninteractive apt-get install -y -q intel-microcode 2>&1 | tail -3
+    when: "! dpkg-query -W -f='${Status}' intel-microcode 2>/dev/null | grep -q 'install ok installed'"
+
+  - name: grub-pc → grub-efi-amd64, with the removable-path copy (EFI/BOOT/BOOTX64.EFI) kept in step
+    sudo: true
+    shell: |
+      set -o pipefail
+      echo 'grub-efi-amd64 grub2/force_efi_extra_removable boolean true' | debconf-set-selections -v -u
+      DEBIAN_FRONTEND=noninteractive apt-get install -y -q --no-install-recommends grub-efi-amd64 2>&1 | tail -8
+    when: "! dpkg-query -W -f='${Status}' grub-efi-amd64 2>/dev/null | grep -q 'install ok installed'"
+
+  - name: make sure the ESP binary was actually rewritten (grub-install explicitly if the postinst skipped it)
+    sudo: true
+    shell: |
+      set -e
+      grub-install --target=x86_64-efi --efi-directory=/boot/efi --bootloader-id=proxmox --force-extra-removable 2>&1 | tail -3
+      update-grub 2>&1 | tail -3
+    when: "test \"$(md5sum < /boot/efi/EFI/proxmox/grubx64.efi | cut -d' ' -f1)\" = a4a8d8d1e75eb2dad19c22a8c1065c4c"
+
+  - name: pin every NIC name to its MAC (systemd .link; takes effect next boot)
+    sudo: true
+    shell: |
+      set -e
+      pin() {  # name mac
+        cat > /etc/systemd/network/10-pin-$1.link <<EOF
+      # Pin $1 by MAC so the PVE 9 / Debian 13 udev cannot rename it.
+      # See eshpfi playbooks/pfi-pve-pve9-prep.yaml
+      [Match]
+      MACAddress=$2
+      Type=ether
+      [Link]
+      Name=$1
+      EOF
+      }
+      pin eno8303     cc:48:3a:1b:c3:d0
+      pin eno8403     cc:48:3a:1b:c3:d1
+      pin eno12399np0 00:62:0b:c6:ca:f0
+      pin eno12409np1 00:62:0b:c6:ca:f1
+      ls -la /etc/systemd/network/
+    when: "! test -f /etc/systemd/network/10-pin-eno8303.link"
+
+  - name: rebuild the initramfs of the running and the newest kernel so the .link files apply in early boot
+    sudo: true
+    shell: |
+      set -e
+      new=$(ls /boot/vmlinuz-* | sed 's#/boot/vmlinuz-##' | sort -V | tail -1)
+      for k in $(uname -r) $new; do update-initramfs -u -k $k 2>&1 | tail -1; done
+    when: "! lsinitramfs /boot/initrd.img-$(ls /boot/vmlinuz-* | sed 's#/boot/vmlinuz-##' | sort -V | tail -1) | grep -q 10-pin-eno8303.link"
+
+verify:
+  - name: interfaces file still parses (no live reload)
+    sudo: true
+    shell: ifreload -a -s
+    changed_when: "false"
+  - name: newest kernel's initramfs carries the uplink pin
+    sudo: true
+    shell: lsinitramfs /boot/initrd.img-$(ls /boot/vmlinuz-* | sed 's#/boot/vmlinuz-##' | sort -V | tail -1) | grep -q 10-pin-eno8303.link
+    changed_when: "false"
+  - name: grub-efi-amd64 in, grub-pc out, intel-microcode in
+    sudo: true
+    shell: |
+      dpkg-query -W -f='${Status}' grub-efi-amd64 | grep -q 'install ok installed' &&
+      ! dpkg-query -W -f='${Status}' grub-pc 2>/dev/null | grep -q 'install ok installed' &&
+      dpkg-query -W -f='${Status}' intel-microcode | grep -q 'install ok installed'
+    changed_when: "false"
+  - name: ESP carries a NEW grubx64.efi, the removable copy matches it, the 2023 copy is kept
+    sudo: true
+    shell: |
+      set -e
+      a=$(md5sum < /boot/efi/EFI/proxmox/grubx64.efi | cut -d' ' -f1)
+      b=$(md5sum < /boot/efi/EFI/BOOT/BOOTX64.EFI | cut -d' ' -f1)
+      test "$a" != a4a8d8d1e75eb2dad19c22a8c1065c4c
+      test "$a" = "$b"
+      test "$(md5sum < /boot/efi/EFI/proxmox-2023/grubx64.efi | cut -d' ' -f1)" = a4a8d8d1e75eb2dad19c22a8c1065c4c
+      ls -la --time-style=long-iso /boot/efi/EFI/proxmox/ /boot/efi/EFI/BOOT/
+    changed_when: "false"
+  - name: firmware boots the proxmox entry first
+    sudo: true
+    shell: |
+      efibootmgr -v | grep -E '^BootOrder|proxmox'
+      first=$(efibootmgr | sed -n 's/^BootOrder: \([0-9A-F]*\).*/\1/p')
+      efibootmgr -v | grep -E "^Boot${first}\*? " | grep -qi 'EFI\\proxmox\\grubx64.efi'
+    changed_when: "false"
+  - name: pve8to9 --full summary
+    sudo: true
+    shell: pve8to9 --full 2>&1 | grep -E '^(FAIL|WARN):|^= SUMMARY|TOTAL|PASSED|SKIPPED|WARNINGS|FAILURES' | cat
+    changed_when: "false"

playbooks/pfi-pve-pve9-prereboot.yaml

@@ -0,0 +1,64 @@
+# pfi-pve: after the PVE 9 dist-upgrade, BEFORE the reboot. One fix-up step, the rest is read-only.
+#   scripts/elway infra-ops@10.250.250.31 --playbook playbooks/pfi-pve-pve9-prereboot.yaml
+# Every verify here is something that, if wrong, makes the reboot a full Anaheim outage with no path back in except
+# the FortiGate WAN-SSH break-glass (docs/runbooks/pfi-pve-pve9-upgrade-plan.md § Break-glass).
+steps:
+  - name: the upgrade ships pve-enterprise.sources enabled; turn it off (no subscription)
+    sudo: true
+    shell: |
+      f=/etc/apt/sources.list.d/pve-enterprise.sources
+      if grep -q '^Enabled:' $f; then sed -i 's/^Enabled:.*/Enabled: false/' $f; else echo 'Enabled: false' >> $f; fi
+      cat $f
+    when: "test -f /etc/apt/sources.list.d/pve-enterprise.sources && ! grep -q '^Enabled: *false' /etc/apt/sources.list.d/pve-enterprise.sources"
+
+verify:
+  - name: the detached dist-upgrade finished with EXIT 0
+    sudo: true
+    shell: tail -1 /root/pve9-upgrade.log | grep -qx 'EXIT 0'
+    changed_when: "false"
+  - name: dpkg is clean and nothing is left for apt -f
+    sudo: true
+    shell: |
+      test -z "$(dpkg --audit 2>&1)"
+      test "$(apt-get -s -f install 2>/dev/null | grep -cE '^(Inst|Conf) ')" -eq 0
+    changed_when: "false"
+  - name: PVE 9 userland
+    sudo: true
+    shell: pveversion | grep -E '^pve-manager/9\.'
+    changed_when: "false"
+  - name: newest kernel has an initramfs carrying the NIC pins and dm-snapshot (root has an LVM snapshot)
+    sudo: true
+    shell: |
+      set -e
+      new=$(ls /boot/vmlinuz-* | sed 's#/boot/vmlinuz-##' | sort -V | tail -1)
+      echo "newest kernel: $new"
+      lsinitramfs /boot/initrd.img-$new > /tmp/pve9-initrd.lst
+      grep -q 10-pin-eno8303.link /tmp/pve9-initrd.lst
+      grep -q 10-pin-eno12399np0.link /tmp/pve9-initrd.lst
+      grep -q 'dm-snapshot.ko' /tmp/pve9-initrd.lst
+      grep -q '/bnxt_en.ko' /tmp/pve9-initrd.lst || echo "  (bnxt_en not in initramfs; loads from root, fine)"
+      grep -q "vmlinuz-$new" /boot/grub/grub.cfg
+    changed_when: "false"
+  - name: ESP rewritten by the trixie GRUB; removable copy matches; firmware boots proxmox first
+    sudo: true
+    shell: |
+      set -e
+      ls -la --time-style=long-iso /boot/efi/EFI/proxmox/ /boot/efi/EFI/BOOT/
+      test "$(md5sum < /boot/efi/EFI/proxmox/grubx64.efi)" = "$(md5sum < /boot/efi/EFI/BOOT/BOOTX64.EFI)"
+      strings /boot/efi/EFI/proxmox/grubx64.efi | grep -m1 -E '^2\.1[2-9]' || strings /boot/efi/EFI/proxmox/grubx64.efi | grep -m1 -i 'grub 2\.1'
+      first=$(efibootmgr | sed -n 's/^BootOrder: \([0-9A-F]*\).*/\1/p')
+      efibootmgr -v | grep -E "^Boot${first}\*? " | grep -qi 'EFI\\proxmox\\grubx64.efi'
+    changed_when: "false"
+  - name: networking will come back (dhclient for vmbr0.250, interfaces parse, sysctl.conf has nothing to lose)
+    sudo: true
+    shell: |
+      set -e
+      dpkg-query -W -f='${Status}' isc-dhcp-client | grep -q 'install ok installed' || dpkg-query -W -f='${Status}' dhcpcd-base | grep -q 'install ok installed'
+      dpkg-query -W -f='${Status}' ifupdown2 | grep -q 'install ok installed'
+      ifreload -a -s
+      test -z "$(grep -vE '^\s*(#|$)' /etc/sysctl.conf 2>/dev/null)"
+    changed_when: "false"
+  - name: pve8to9 --full (post-upgrade checks)
+    sudo: true
+    shell: pve8to9 --full 2>&1 | grep -E '^(FAIL|WARN):|TOTAL|PASSED|SKIPPED|WARNINGS|FAILURES' | cat
+    changed_when: "false"

playbooks/pfi-pve-pve9-upgrade.yaml

@@ -0,0 +1,101 @@
+# pfi-pve: PVE 8.4 → 9 upgrade, the package half (Prime go 2026-10-04). Does NOT reboot.
+#   scripts/elway infra-ops@10.250.250.31 --playbook playbooks/pfi-pve-pve9-upgrade.yaml
+# Prereq: playbooks/pfi-pve-pve9-prep.yaml, then the PVE 8 reboot test (it validates the new GRUB, NIC pins and kernel
+# before trixie lands; see docs/runbooks/pfi-pve-pve9-upgrade-plan.md). Template: playbooks/esh-nas-pve-pve9-upgrade.yaml.
+# The dist-upgrade runs DETACHED (systemd-run → /root/pve9-upgrade.log), so a dropped ssh cannot kill it mid-dpkg.
+# Poll: sudo tail /root/pve9-upgrade.log ; systemctl status pve9-dist-upgrade
+# Then: playbooks/pfi-pve-pve9-prereboot.yaml, then the reboot.
+# Rollback: `lvconvert --merge pve/root-pre-pve9` and reboot (the merge runs at the next activation of pve/root);
+# from a rescue boot if 9 does not boot. The snapshot covers / and /boot (one LV), not the ESP or guest storage.
+vars:
+  stage_dir: /root/pve9-prep-20261004
+
+steps:
+  - name: GUARD — pve8to9 --full reports no FAIL
+    sudo: true
+    shell: |
+      out=$(pve8to9 --full 2>&1)
+      echo "$out" | grep -E '^(FAIL|WARN):' | cat
+      ! echo "$out" | grep -q '^FAIL:'
+    changed_when: "false"
+
+  - name: GUARD — prep is in, and the box has booted the newest 6.8 kernel since (the PVE 8 reboot test)
+    sudo: true
+    shell: |
+      set -e
+      dpkg-query -W -f='${Status}' grub-efi-amd64 | grep -q 'install ok installed'
+      test -f /etc/systemd/network/10-pin-eno8303.link
+      new=$(ls /boot/vmlinuz-* | sed 's#/boot/vmlinuz-##' | sort -V | tail -1)
+      echo "running $(uname -r), newest $new"
+      test "$(uname -r)" = "$new"
+    changed_when: "false"
+
+  - name: GUARD — the VG has room for the root snapshot and dm-snapshot is in the running kernel's initramfs
+    sudo: true
+    shell: |
+      set -e
+      free=$(vgs --noheadings --units g --nosuffix -o vg_free pve | tr -d ' ' | cut -d. -f1)
+      echo "pve VG free: ${free}G"
+      test "$free" -ge 15
+      lsinitramfs /boot/initrd.img-$(uname -r) | grep -q 'dm-snapshot.ko'
+    changed_when: "false"
+
+  - name: root snapshot for rollback (pve/root-pre-pve9, 15G COW)
+    sudo: true
+    shell: lvcreate -s -L 15G -n root-pre-pve9 pve/root
+    when: "! lvs pve/root-pre-pve9 >/dev/null 2>&1"
+
+  - name: switch apt sources bookworm → trixie (Debian lines rewritten; PVE repo moved to deb822 proxmox.sources)
+    sudo: true
+    shell: |
+      set -e
+      cp -p /etc/apt/sources.list {{ stage_dir }}/sources.list.bookworm
+      cat > /etc/apt/sources.list <<'EOF'
+      deb http://ftp.us.debian.org/debian trixie main contrib non-free-firmware
+      deb http://ftp.us.debian.org/debian trixie-updates main contrib non-free-firmware
+      deb http://security.debian.org/debian-security trixie-security main contrib non-free-firmware
+      EOF
+      cat > /etc/apt/sources.list.d/proxmox.sources <<'EOF'
+      Types: deb
+      URIs: http://download.proxmox.com/debian/pve
+      Suites: trixie
+      Components: pve-no-subscription
+      Signed-By: /usr/share/keyrings/proxmox-archive-keyring.gpg
+      EOF
+      for f in /etc/apt/sources.list.d/pve-enterprise.list /etc/apt/sources.list.d/pve-enterprise.list.dpkg-old; do
+        if [ -e "$f" ]; then mv "$f" {{ stage_dir }}/; fi
+      done
+      cat /etc/apt/sources.list; ls /etc/apt/sources.list.d/
+    when: "grep -q bookworm /etc/apt/sources.list"
+
+  - name: apt update on trixie (must be clean)
+    sudo: true
+    shell: |
+      ! apt-get update -q 2>&1 | grep -E '^(E|Err):'
+    changed_when: "false"
+
+  - name: GUARD — read the plan; refuse if it would remove anything this host cannot boot or network without
+    sudo: true
+    shell: |
+      apt-get -s dist-upgrade > {{ stage_dir }}/dist-upgrade-plan.txt 2>&1
+      grep -cE '^Inst ' {{ stage_dir }}/dist-upgrade-plan.txt | sed 's/^/  installs\/upgrades: /'
+      grep -E '^Remv ' {{ stage_dir }}/dist-upgrade-plan.txt | sed 's/^/  /' || echo "  no removals"
+      ! grep -E '^Remv (proxmox-ve|pve-manager|grub-efi-amd64|grub-efi-amd64-bin|ifupdown2|isc-dhcp-client|zfsutils-linux|lvm2|openssh-server|qemu-server|pve-container) ' {{ stage_dir }}/dist-upgrade-plan.txt
+    changed_when: "false"
+
+  - name: start the dist-upgrade DETACHED (log /root/pve9-upgrade.log)
+    sudo: true
+    shell: |
+      systemd-run --unit=pve9-dist-upgrade --setenv=DEBIAN_FRONTEND=noninteractive \
+        /bin/sh -c 'apt-get dist-upgrade -y -q -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold > /root/pve9-upgrade.log 2>&1; echo "EXIT $?" >> /root/pve9-upgrade.log'
+    when: "! test -f /root/pve9-upgrade.log"
+
+verify:
+  - name: root snapshot exists
+    sudo: true
+    shell: lvs pve/root-pre-pve9
+    changed_when: "false"
+  - name: sources are trixie
+    sudo: true
+    shell: "! grep -rq bookworm /etc/apt/sources.list /etc/apt/sources.list.d/"
+    changed_when: "false"