ops(esh-pve): upgraded to PVE 9.2.21 / kernel 7.0.14-20 (one reboot, Prime on site); ESH cluster uniform on 9

2026-10-04 07:45:05 -07:00
parent 9a6b23581e
commit 73129a0564
5 changed files with 206 additions and 5 deletions
@@ -0,0 +1,78 @@
+# esh-pve: after the PVE 9 dist-upgrade, BEFORE the reboot. One fix-up step, the rest is read-only.
+#   scripts/elway infra-ops@10.0.250.35 --playbook playbooks/esh-pve-pve9-prereboot.yaml
+# Every verify here is something that, if wrong, makes the reboot an ESH outage only someone at the console can end
+# (esh-pve carries esh-scale, ESH's only mesh path, and has no AMT). Template: playbooks/pfi-pve-pve9-prereboot.yaml.
+steps:
+  - name: the upgrade ships pve-enterprise.sources enabled; turn it off (no subscription)
+    sudo: true
+    shell: |
+      f=/etc/apt/sources.list.d/pve-enterprise.sources
+      if grep -q '^Enabled:' $f; then sed -i 's/^Enabled:.*/Enabled: false/' $f; else echo 'Enabled: false' >> $f; fi
+      cat $f
+    when: "test -f /etc/apt/sources.list.d/pve-enterprise.sources && ! grep -q '^Enabled: *false' /etc/apt/sources.list.d/pve-enterprise.sources"
+
+verify:
+  - name: the detached dist-upgrade finished with EXIT 0
+    sudo: true
+    shell: tail -1 /root/pve9-upgrade.log | grep -qx 'EXIT 0'
+    changed_when: "false"
+  - name: dpkg is clean and nothing is left for apt -f
+    sudo: true
+    shell: |
+      test -z "$(dpkg --audit 2>&1)"
+      test "$(apt-get -s -f install 2>/dev/null | grep -cE '^(Inst|Conf) ')" -eq 0
+    changed_when: "false"
+  - name: PVE 9 userland
+    sudo: true
+    shell: pveversion | grep -E '^pve-manager/9\.'
+    changed_when: "false"
+  - name: nvidia DKMS module is built for the new kernel (esh-ml1's GPU; the nh3-pve trap)
+    sudo: true
+    shell: |
+      new=$(ls /boot/vmlinuz-* | sed 's#/boot/vmlinuz-##' | sort -V | tail -1)
+      /usr/sbin/dkms status | cat
+      /usr/sbin/dkms status | grep -q "nvidia/.*, $new, x86_64: installed"
+    changed_when: "false"
+  - name: newest kernel has an initramfs carrying the NIC pins and dm-snapshot (root has an LVM snapshot)
+    sudo: true
+    shell: |
+      set -e
+      new=$(ls /boot/vmlinuz-* | sed 's#/boot/vmlinuz-##' | sort -V | tail -1)
+      echo "newest kernel: $new"
+      lsinitramfs /boot/initrd.img-$new > /tmp/pve9-initrd.lst
+      grep -q 10-pin-enp3s0f0np0.link /tmp/pve9-initrd.lst
+      grep -q 10-pin-enp88s0.link /tmp/pve9-initrd.lst
+      grep -q 'dm-snapshot.ko' /tmp/pve9-initrd.lst
+      grep -q "vmlinuz-$new" /boot/grub/grub.cfg
+    changed_when: "false"
+  - name: ESP rewritten by the trixie GRUB (shim path); removable copy matches; firmware boots proxmox shim first
+    sudo: true
+    shell: |
+      set -e
+      ls -la --time-style=long-iso /boot/efi/EFI/proxmox/ /boot/efi/EFI/BOOT/
+      test "$(md5sum < /boot/efi/EFI/proxmox/shimx64.efi)" = "$(md5sum < /boot/efi/EFI/BOOT/BOOTx64.efi)"
+      test "$(md5sum < /boot/efi/EFI/proxmox/grubx64.efi)" = "$(md5sum < /boot/efi/EFI/BOOT/grubx64.efi)"
+      # The binary carries no greppable version string; the trixie grub-efi-amd64 postinst's own report is the proof.
+      grep -q 'Installation finished. No error reported.' /root/pve9-upgrade.log
+      first=$(efibootmgr | sed -n 's/^BootOrder: \([0-9A-F]*\).*/\1/p')
+      efibootmgr -v | grep -E "^Boot${first}\*? " | grep -qi 'EFI\\proxmox\\shimx64.efi'
+    changed_when: "false"
+  - name: networking will come back (static vmbr0, interfaces parse, every bridge port exists, sysctl.conf has nothing to lose)
+    sudo: true
+    shell: |
+      set -e
+      dpkg-query -W -f='${Status}' ifupdown2 | grep -q 'install ok installed'
+      for p in $(sed -n -E 's/^\s*bridge-ports\s+//p' /etc/network/interfaces); do test -e /sys/class/net/$p || { echo "missing $p"; exit 1; }; done
+      # ifupdown2 3.3 (PVE 9) flags PVE's own `bridge-fd 0` as out of range and `-s` exits 1 on that warning alone.
+      # nh3-pve and esh-nas-pve show the same warning and booted with vmbr0 fine (STP off, forward_delay 0), so
+      # only a warning OTHER than that one fails this check.
+      out=$(ifreload -a -s 2>&1) || true
+      echo "$out"
+      # (an `if`, not a bare `! pipeline`: set -e ignores a negated pipeline, so that form could never fail)
+      if echo "$out" | grep -v 'bridge-fd: value of out range "0"' | grep -q .; then echo "unexpected ifupdown2 output"; exit 1; fi
+      test -z "$(grep -vE '^\s*(#|$)' /etc/sysctl.conf 2>/dev/null)"
+    changed_when: "false"
+  - name: pve8to9 --full (post-upgrade checks)
+    sudo: true
+    shell: pve8to9 --full 2>&1 | grep -E '^(FAIL|WARN):|TOTAL|PASSED|SKIPPED|WARNINGS|FAILURES' | cat
+    changed_when: "false"
@@ -0,0 +1,102 @@
+# esh-pve (`pve`): PVE 8.4 → 9 upgrade, the package half (Prime go 2026-10-04, on site). Does NOT reboot.
+#   scripts/elway infra-ops@10.0.250.35 --playbook playbooks/esh-pve-pve9-upgrade.yaml
+# Prereq: playbooks/esh-pve-pve9-prep.yaml. NO PVE 8 reboot test here: Prime chose one reboot (2026-10-04, on site),
+# so the prep's next-boot changes (bridge-ports fix, NIC pins, kernel, microcode) land together with PVE 9.
+# Template: playbooks/pfi-pve-pve9-upgrade.yaml. Plan: docs/runbooks/esh-pve-cluster-pve9-upgrade-plan.md.
+# The dist-upgrade runs DETACHED (systemd-run → /root/pve9-upgrade.log), so a dropped ssh cannot kill it mid-dpkg.
+# Poll: sudo tail /root/pve9-upgrade.log ; systemctl status pve9-dist-upgrade
+# Then: playbooks/esh-pve-pve9-prereboot.yaml, then the reboot.
+# Rollback: `lvconvert --merge pve/root-pre-pve9` and reboot (the merge runs at the next activation of pve/root);
+# from a rescue boot if 9 does not boot. The snapshot covers / and /boot (one LV), not the ESP or guest storage.
+vars:
+  stage_dir: /root/pve9-prep-20261004
+
+steps:
+  - name: GUARD — pve8to9 --full reports no FAIL
+    sudo: true
+    shell: |
+      out=$(pve8to9 --full 2>&1)
+      echo "$out" | grep -E '^(FAIL|WARN):' | cat
+      ! echo "$out" | grep -q '^FAIL:'
+    changed_when: "false"
+
+  - name: GUARD — prep is in (systemd-boot out, NIC pins, nvidia built for the newest 6.8)
+    sudo: true
+    shell: |
+      set -e
+      # an `if`, not `! pipeline`: set -e ignores a negated pipeline, so that line could never fail
+      if dpkg-query -W -f='${Status}' systemd-boot 2>/dev/null | grep -q 'install ok installed'; then echo "systemd-boot still installed"; exit 1; fi
+      test -f /etc/systemd/network/10-pin-enp3s0f0np0.link
+      dpkg-query -W -f='${Status}' proxmox-default-headers | grep -q 'install ok installed'
+      /usr/sbin/dkms status | grep -q "nvidia/.*, $(ls /boot/vmlinuz-* | sed 's#/boot/vmlinuz-##' | sort -V | tail -1), x86_64: installed"
+    changed_when: "false"
+
+  - name: GUARD — the VG has room for the root snapshot and dm-snapshot is in the running kernel's initramfs
+    sudo: true
+    shell: |
+      set -e
+      free=$(vgs --noheadings --units g --nosuffix -o vg_free pve | tr -d ' ' | cut -d. -f1)
+      echo "pve VG free: ${free}G"
+      test "$free" -ge 15
+      lsinitramfs /boot/initrd.img-$(uname -r) | grep -q 'dm-snapshot.ko'
+    changed_when: "false"
+
+  - name: root snapshot for rollback (pve/root-pre-pve9, 15G COW)
+    sudo: true
+    shell: lvcreate -s -L 15G -n root-pre-pve9 pve/root
+    when: "! lvs pve/root-pre-pve9 >/dev/null 2>&1"
+
+  - name: switch apt sources bookworm → trixie (Debian lines rewritten; PVE repo moved to deb822 proxmox.sources)
+    sudo: true
+    shell: |
+      set -e
+      cp -p /etc/apt/sources.list {{ stage_dir }}/sources.list.bookworm
+      cat > /etc/apt/sources.list <<'EOF'
+      deb http://ftp.us.debian.org/debian trixie main contrib non-free-firmware
+      deb http://ftp.us.debian.org/debian trixie-updates main contrib non-free-firmware
+      deb http://security.debian.org/debian-security trixie-security main contrib non-free-firmware
+      EOF
+      cat > /etc/apt/sources.list.d/proxmox.sources <<'EOF'
+      Types: deb
+      URIs: http://download.proxmox.com/debian/pve
+      Suites: trixie
+      Components: pve-no-subscription
+      Signed-By: /usr/share/keyrings/proxmox-archive-keyring.gpg
+      EOF
+      for f in /etc/apt/sources.list.d/pve-enterprise.list /etc/apt/sources.list.d/ceph.list; do
+        if [ -e "$f" ]; then mv "$f" {{ stage_dir }}/; fi
+      done
+      cat /etc/apt/sources.list; ls /etc/apt/sources.list.d/
+    when: "grep -q bookworm /etc/apt/sources.list"
+
+  - name: apt update on trixie (must be clean)
+    sudo: true
+    shell: |
+      ! apt-get update -q 2>&1 | grep -E '^(E|Err):'
+    changed_when: "false"
+
+  - name: GUARD — read the plan; refuse if it would remove anything this host cannot boot or network without
+    sudo: true
+    shell: |
+      apt-get -s dist-upgrade > {{ stage_dir }}/dist-upgrade-plan.txt 2>&1
+      grep -cE '^Inst ' {{ stage_dir }}/dist-upgrade-plan.txt | sed 's/^/  installs\/upgrades: /'
+      grep -E '^Remv ' {{ stage_dir }}/dist-upgrade-plan.txt | sed 's/^/  /' || echo "  no removals"
+      ! grep -E '^Remv (proxmox-ve|pve-manager|grub-efi-amd64|grub-efi-amd64-bin|grub-efi-amd64-signed|shim-signed|ifupdown2|lvm2|openssh-server|qemu-server|pve-container|dkms|proxmox-default-headers) ' {{ stage_dir }}/dist-upgrade-plan.txt
+    changed_when: "false"
+
+  - name: start the dist-upgrade DETACHED (log /root/pve9-upgrade.log)
+    sudo: true
+    shell: |
+      systemd-run --unit=pve9-dist-upgrade --setenv=DEBIAN_FRONTEND=noninteractive \
+        /bin/sh -c 'apt-get dist-upgrade -y -q -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold > /root/pve9-upgrade.log 2>&1; echo "EXIT $?" >> /root/pve9-upgrade.log'
+    when: "! test -f /root/pve9-upgrade.log"
+
+verify:
+  - name: root snapshot exists
+    sudo: true
+    shell: lvs pve/root-pre-pve9
+    changed_when: "false"
+  - name: sources are trixie
+    sudo: true
+    shell: "! grep -rq bookworm /etc/apt/sources.list /etc/apt/sources.list.d/"
+    changed_when: "false"