esh-pfi-infrastructure/playbooks/esh-pve-pve9-prep.yaml

# esh-pve (`pve`, 10.0.250.35): no-downtime prep for the PVE 8 → 9 upgrade (Prime 2026-10-04: "okay, do esh-pve").
#   scripts/elway infra-ops@10.0.250.35 --playbook playbooks/esh-pve-pve9-prep.yaml
# Plan: docs/runbooks/esh-pve-cluster-pve9-upgrade-plan.md. Template: playbooks/pfi-pve-pve9-prep.yaml.
# Nothing here reboots, reloads networking or touches a guest. Kernel, microcode, NIC pins, the bridge-port fix and
# the removable-path bootloader all take effect at the NEXT boot.
#   - latest 8.4 (8.4.20 → 8.4.21, kernel 6.8.12-43; nvidia DKMS rebuilds for it via proxmox-default-headers);
#   - non-free-firmware + intel-microcode (pve8to9 WARN);
#   - remove the systemd-boot meta package (pve8to9 FAIL; the box boots shim → GRUB);
#   - force_efi_extra_removable, so EFI/BOOT/BOOTX64.EFI (firmware entry "UEFI OS", BootOrder #2) stops being the
#     2024-05-18 copy (pve8to9 WARN);
#   - pin every wired NIC by MAC. The GPU added 2026-09-24 already renumbered the two igc ports once
#     (enp87s0/enp88s0 → enp88s0/enp89s0), and the X710 ports carry the `np0/np1` suffix the wiki warns about;
#   - fix vmbr0's bridge-ports: it still names enp87s0, which has not existed since that GPU went in, and every boot
#     since logs "bridge port enp87s0 does not exist … bridge configuration failed (missing ports)". The new list is
#     what is actually forwarding today (enp3s0f0np0 + enp3s0f1np1). enp88s0 is in the bridge but admin-DOWN now and
#     stays out: bringing a 2.5G port up in an STP-off bridge is how you build an L2 loop if a second cable is in.
vars:
  stage_dir: /root/pve9-prep-20261004

steps:
  - name: back up /etc, the package list, /boot/grub and the ESP
    sudo: true
    shell: |
      set -e
      mkdir -p {{ stage_dir }}
      tar czf {{ stage_dir }}/etc-before.tar.gz -C / etc 2>/dev/null
      dpkg -l > {{ stage_dir }}/dpkg-before.txt
      pveversion -v > {{ stage_dir }}/pveversion-before.txt 2>&1
      cp -p /etc/network/interfaces {{ stage_dir }}/interfaces.before
      cp -p /etc/apt/sources.list {{ stage_dir }}/sources.list.before
      cp -a /boot/efi/EFI {{ stage_dir }}/EFI-before
      cp -a /boot/grub {{ stage_dir }}/boot-grub-before
      ls -la {{ stage_dir }}
    creates: "{{ stage_dir }}/etc-before.tar.gz"

  - name: add non-free-firmware to the Debian bookworm lines (for intel-microcode)
    sudo: true
    shell: sed -i -E '/^deb .*debian(\.org)?(\/debian)? bookworm/{/non-free-firmware/!s/$/ non-free-firmware/}' /etc/apt/sources.list
    when: "grep -E '^deb .*debian(\\.org)?(/debian)? bookworm' /etc/apt/sources.list | grep -qv non-free-firmware"

  - name: apt update (must be clean)
    sudo: true
    shell: |
      ! apt-get update -q 2>&1 | grep -E '^(E|Err):'
    changed_when: "false"

  - name: latest 8.4 packages (keep local configs)
    sudo: true
    shell: |
      set -o pipefail
      DEBIAN_FRONTEND=noninteractive apt-get full-upgrade -y -q \
        -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold 2>&1 | tail -8
      pveversion | cat

  - name: intel-microcode
    sudo: true
    shell: DEBIAN_FRONTEND=noninteractive apt-get install -y -q intel-microcode 2>&1 | tail -3
    when: "! dpkg-query -W -f='${Status}' intel-microcode 2>/dev/null | grep -q 'install ok installed'"

  - name: remove the systemd-boot meta package (shim → GRUB is the bootloader here)
    sudo: true
    shell: DEBIAN_FRONTEND=noninteractive apt-get remove -y -q systemd-boot 2>&1 | tail -3
    when: "dpkg-query -W -f='${Status}' systemd-boot 2>/dev/null | grep -q 'install ok installed'"

  - name: keep EFI/BOOT/BOOTX64.EFI in step with GRUB (force_efi_extra_removable + reinstall)
    sudo: true
    shell: |
      set -o pipefail
      echo 'grub-efi-amd64 grub2/force_efi_extra_removable boolean true' | debconf-set-selections -v -u
      DEBIAN_FRONTEND=noninteractive apt-get install -y -q --reinstall grub-efi-amd64 2>&1 | tail -5
    when: "debconf-show grub-efi-amd64 2>/dev/null | grep -q 'force_efi_extra_removable: false'"

  - name: pin every wired NIC name to its MAC (systemd .link; takes effect next boot)
    sudo: true
    shell: |
      set -e
      mkdir -p /etc/systemd/network
      pin() {  # name mac
        cat > /etc/systemd/network/10-pin-$1.link <<EOF
      # Pin $1 by MAC so the PVE 9 / Debian 13 udev cannot rename it.
      # See eshpfi playbooks/esh-pve-pve9-prep.yaml
      [Match]
      MACAddress=$2
      Type=ether
      [Link]
      Name=$1
      EOF
      }
      pin enp3s0f0np0 58:47:ca:76:99:2f
      pin enp3s0f1np1 58:47:ca:76:99:30
      pin enp88s0     58:47:ca:76:99:31
      pin enp89s0     58:47:ca:76:99:32
      ls -la /etc/systemd/network/
    when: "! test -f /etc/systemd/network/10-pin-enp3s0f0np0.link"

  - name: vmbr0 bridge-ports = what is forwarding today; iface stanzas renamed to the post-GPU names (file only, NO reload)
    sudo: true
    shell: |
      set -e
      f=/etc/network/interfaces
      sed -i -E 's/^(\s*bridge-ports) enp3s0f0np0 enp3s0f1np1 enp87s0 enp88s0\s*$/\1 enp3s0f0np0 enp3s0f1np1/' $f
      sed -i -E 's/^iface enp88s0 inet manual$/iface enp89s0 inet manual/; s/^iface enp87s0 inet manual$/iface enp88s0 inet manual/; s/^iface wlp89s0 inet manual$/iface wlp90s0 inet manual/' $f
      diff {{ stage_dir }}/interfaces.before $f || true
    when: "grep -qE '^\\s*bridge-ports .*enp87s0' /etc/network/interfaces"

  - name: rebuild the initramfs of the running and the newest kernel so the .link files apply in early boot
    sudo: true
    shell: |
      set -e
      new=$(ls /boot/vmlinuz-* | sed 's#/boot/vmlinuz-##' | sort -V | tail -1)
      for k in $(uname -r) $new; do update-initramfs -u -k $k 2>&1 | tail -1; done
    when: "! lsinitramfs /boot/initrd.img-$(ls /boot/vmlinuz-* | sed 's#/boot/vmlinuz-##' | sort -V | tail -1) | grep -q 10-pin-enp3s0f0np0.link"

verify:
  - name: interfaces parse, and the only bridge port named is one that exists (no live reload)
    sudo: true
    shell: |
      set -e
      out=$(ifreload -a -s 2>&1) || { echo "$out"; exit 1; }
      echo "$out"
      grep -E '^\s*bridge-ports' /etc/network/interfaces
      for p in $(sed -n -E 's/^\s*bridge-ports\s+//p' /etc/network/interfaces); do test -e /sys/class/net/$p || { echo "missing $p"; exit 1; }; done
    changed_when: "false"
  - name: newest kernel's initramfs carries the uplink pin
    sudo: true
    shell: lsinitramfs /boot/initrd.img-$(ls /boot/vmlinuz-* | sed 's#/boot/vmlinuz-##' | sort -V | tail -1) | grep -q 10-pin-enp3s0f0np0.link
    changed_when: "false"
  - name: nvidia DKMS module is built for the newest installed kernel
    sudo: true
    shell: |
      new=$(ls /boot/vmlinuz-* | sed 's#/boot/vmlinuz-##' | sort -V | tail -1)
      /usr/sbin/dkms status | cat
      /usr/sbin/dkms status | grep -q "nvidia/.*, $new, x86_64: installed"
    changed_when: "false"
  - name: systemd-boot out, intel-microcode in, GRUB still installed
    sudo: true
    shell: |
      ! dpkg-query -W -f='${Status}' systemd-boot 2>/dev/null | grep -q 'install ok installed' &&
      dpkg-query -W -f='${Status}' intel-microcode | grep -q 'install ok installed' &&
      dpkg-query -W -f='${Status}' grub-efi-amd64 | grep -q 'install ok installed'
    changed_when: "false"
  - name: removable path refreshed today; firmware boots proxmox (shim) first
    sudo: true
    shell: |
      set -e
      ls -la --time-style=long-iso /boot/efi/EFI/BOOT/ /boot/efi/EFI/proxmox/
      test -n "$(find /boot/efi/EFI/BOOT -iname 'BOOTX64.EFI' -newer {{ stage_dir }}/etc-before.tar.gz)"
      first=$(efibootmgr | sed -n 's/^BootOrder: \([0-9A-F]*\).*/\1/p')
      efibootmgr -v | grep -E "^Boot${first}\*? " | grep -qi 'EFI\\proxmox\\shimx64.efi'
    changed_when: "false"
  - name: pve8to9 --full summary
    sudo: true
    shell: pve8to9 --full 2>&1 | sed 's/\x1b\[[0-9;]*m//g' | grep -E '^(FAIL|WARN):|TOTAL|PASSED|SKIPPED|WARNINGS|FAILURES' | cat
    changed_when: "false"