diff --git a/docs/runbooks/esh-pve-cluster-pve9-upgrade-plan.md b/docs/runbooks/esh-pve-cluster-pve9-upgrade-plan.md index 5560561..79770ce 100644 --- a/docs/runbooks/esh-pve-cluster-pve9-upgrade-plan.md +++ b/docs/runbooks/esh-pve-cluster-pve9-upgrade-plan.md @@ -168,3 +168,19 @@ There is **no supported in-place downgrade** from PVE 9 / Debian 13 back to 8 / - **Last resort:** reinstall 8.4 and restore `/etc` and guests from the pre-flight PBS backups (hours). - **Guest data is not touched by the host upgrade**, so the risk is host availability, not data loss. Both rollbacks need console access, which is why finding 2 matters. + +## esh-pve: prep DONE 2026-10-04 0729-0733 (`playbooks/esh-pve-pve9-prep.yaml`, no downtime; Prime: "okay, do esh-pve") + +- 8.4.21, kernel 6.8.12-43 with nvidia 580.178.04 DKMS built for it (`proxmox-default-headers` was already in, so the + nh3-pve headers trap does not apply); intel-microcode + non-free-firmware; `systemd-boot` removed (the pve8to9 FAIL); + `force_efi_extra_removable` true, so EFI/BOOT now carries the same shim + GRUB as EFI/proxmox (was 2024-05-18). +- NIC pins by MAC for enp3s0f0np0/enp3s0f1np1 (X710) and enp88s0/enp89s0 (igc). +- ⚠ **Found and fixed: vmbr0's `bridge-ports` still named `enp87s0`**, which has not existed since the RTX 2000E went in on + 2026-09-24 and renumbered the igc ports (enp87s0/enp88s0 → enp88s0/enp89s0). That boot logged "bridge port enp87s0 + does not exist … bridge configuration failed (missing ports)"; vmbr0 came up on the X710 anyway. bridge-ports is now + `enp3s0f0np0 enp3s0f1np1`, which is what was actually forwarding (enp88s0 was enslaved but admin-down and stays out: + a 2.5G port brought up in an STP-off bridge is an L2 loop if a second cable is ever in). The iface stanzas were renamed + to match. File only, no live reload; takes effect at the next boot. Backup `/root/pve9-prep-20261004/interfaces.before`. +- pve8to9 --full: 0 FAIL, 3 WARN (2-node cluster, running guests, dkms). +- Nightly vzdump 03:30 OK (100, 101, 111); 108 (subnet router) excluded by design; 110 esh-ml1 not backed up and not + one-off'd: rebuildable from `stacks/embed-rerank`, and embed/rerank are load-shared with nh3-ml1 since 2026-09-26. diff --git a/playbooks/esh-pve-pve9-prep.yaml b/playbooks/esh-pve-pve9-prep.yaml new file mode 100644 index 0000000..874822e --- /dev/null +++ b/playbooks/esh-pve-pve9-prep.yaml @@ -0,0 +1,154 @@ +# esh-pve (`pve`, 10.0.250.35): no-downtime prep for the PVE 8 → 9 upgrade (Prime 2026-10-04: "okay, do esh-pve"). +# scripts/elway infra-ops@10.0.250.35 --playbook playbooks/esh-pve-pve9-prep.yaml +# Plan: docs/runbooks/esh-pve-cluster-pve9-upgrade-plan.md. Template: playbooks/pfi-pve-pve9-prep.yaml. +# Nothing here reboots, reloads networking or touches a guest. Kernel, microcode, NIC pins, the bridge-port fix and +# the removable-path bootloader all take effect at the NEXT boot. +# - latest 8.4 (8.4.20 → 8.4.21, kernel 6.8.12-43; nvidia DKMS rebuilds for it via proxmox-default-headers); +# - non-free-firmware + intel-microcode (pve8to9 WARN); +# - remove the systemd-boot meta package (pve8to9 FAIL; the box boots shim → GRUB); +# - force_efi_extra_removable, so EFI/BOOT/BOOTX64.EFI (firmware entry "UEFI OS", BootOrder #2) stops being the +# 2024-05-18 copy (pve8to9 WARN); +# - pin every wired NIC by MAC. The GPU added 2026-09-24 already renumbered the two igc ports once +# (enp87s0/enp88s0 → enp88s0/enp89s0), and the X710 ports carry the `np0/np1` suffix the wiki warns about; +# - fix vmbr0's bridge-ports: it still names enp87s0, which has not existed since that GPU went in, and every boot +# since logs "bridge port enp87s0 does not exist … bridge configuration failed (missing ports)". The new list is +# what is actually forwarding today (enp3s0f0np0 + enp3s0f1np1). enp88s0 is in the bridge but admin-DOWN now and +# stays out: bringing a 2.5G port up in an STP-off bridge is how you build an L2 loop if a second cable is in. +vars: + stage_dir: /root/pve9-prep-20261004 + +steps: + - name: back up /etc, the package list, /boot/grub and the ESP + sudo: true + shell: | + set -e + mkdir -p {{ stage_dir }} + tar czf {{ stage_dir }}/etc-before.tar.gz -C / etc 2>/dev/null + dpkg -l > {{ stage_dir }}/dpkg-before.txt + pveversion -v > {{ stage_dir }}/pveversion-before.txt 2>&1 + cp -p /etc/network/interfaces {{ stage_dir }}/interfaces.before + cp -p /etc/apt/sources.list {{ stage_dir }}/sources.list.before + cp -a /boot/efi/EFI {{ stage_dir }}/EFI-before + cp -a /boot/grub {{ stage_dir }}/boot-grub-before + ls -la {{ stage_dir }} + creates: "{{ stage_dir }}/etc-before.tar.gz" + + - name: add non-free-firmware to the Debian bookworm lines (for intel-microcode) + sudo: true + shell: sed -i -E '/^deb .*debian(\.org)?(\/debian)? bookworm/{/non-free-firmware/!s/$/ non-free-firmware/}' /etc/apt/sources.list + when: "grep -E '^deb .*debian(\\.org)?(/debian)? bookworm' /etc/apt/sources.list | grep -qv non-free-firmware" + + - name: apt update (must be clean) + sudo: true + shell: | + ! apt-get update -q 2>&1 | grep -E '^(E|Err):' + changed_when: "false" + + - name: latest 8.4 packages (keep local configs) + sudo: true + shell: | + set -o pipefail + DEBIAN_FRONTEND=noninteractive apt-get full-upgrade -y -q \ + -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold 2>&1 | tail -8 + pveversion | cat + + - name: intel-microcode + sudo: true + shell: DEBIAN_FRONTEND=noninteractive apt-get install -y -q intel-microcode 2>&1 | tail -3 + when: "! dpkg-query -W -f='${Status}' intel-microcode 2>/dev/null | grep -q 'install ok installed'" + + - name: remove the systemd-boot meta package (shim → GRUB is the bootloader here) + sudo: true + shell: DEBIAN_FRONTEND=noninteractive apt-get remove -y -q systemd-boot 2>&1 | tail -3 + when: "dpkg-query -W -f='${Status}' systemd-boot 2>/dev/null | grep -q 'install ok installed'" + + - name: keep EFI/BOOT/BOOTX64.EFI in step with GRUB (force_efi_extra_removable + reinstall) + sudo: true + shell: | + set -o pipefail + echo 'grub-efi-amd64 grub2/force_efi_extra_removable boolean true' | debconf-set-selections -v -u + DEBIAN_FRONTEND=noninteractive apt-get install -y -q --reinstall grub-efi-amd64 2>&1 | tail -5 + when: "debconf-show grub-efi-amd64 2>/dev/null | grep -q 'force_efi_extra_removable: false'" + + - name: pin every wired NIC name to its MAC (systemd .link; takes effect next boot) + sudo: true + shell: | + set -e + mkdir -p /etc/systemd/network + pin() { # name mac + cat > /etc/systemd/network/10-pin-$1.link <&1 | tail -1; done + when: "! lsinitramfs /boot/initrd.img-$(ls /boot/vmlinuz-* | sed 's#/boot/vmlinuz-##' | sort -V | tail -1) | grep -q 10-pin-enp3s0f0np0.link" + +verify: + - name: interfaces parse, and the only bridge port named is one that exists (no live reload) + sudo: true + shell: | + set -e + out=$(ifreload -a -s 2>&1) || { echo "$out"; exit 1; } + echo "$out" + grep -E '^\s*bridge-ports' /etc/network/interfaces + for p in $(sed -n -E 's/^\s*bridge-ports\s+//p' /etc/network/interfaces); do test -e /sys/class/net/$p || { echo "missing $p"; exit 1; }; done + changed_when: "false" + - name: newest kernel's initramfs carries the uplink pin + sudo: true + shell: lsinitramfs /boot/initrd.img-$(ls /boot/vmlinuz-* | sed 's#/boot/vmlinuz-##' | sort -V | tail -1) | grep -q 10-pin-enp3s0f0np0.link + changed_when: "false" + - name: nvidia DKMS module is built for the newest installed kernel + sudo: true + shell: | + new=$(ls /boot/vmlinuz-* | sed 's#/boot/vmlinuz-##' | sort -V | tail -1) + /usr/sbin/dkms status | cat + /usr/sbin/dkms status | grep -q "nvidia/.*, $new, x86_64: installed" + changed_when: "false" + - name: systemd-boot out, intel-microcode in, GRUB still installed + sudo: true + shell: | + ! dpkg-query -W -f='${Status}' systemd-boot 2>/dev/null | grep -q 'install ok installed' && + dpkg-query -W -f='${Status}' intel-microcode | grep -q 'install ok installed' && + dpkg-query -W -f='${Status}' grub-efi-amd64 | grep -q 'install ok installed' + changed_when: "false" + - name: removable path refreshed today; firmware boots proxmox (shim) first + sudo: true + shell: | + set -e + ls -la --time-style=long-iso /boot/efi/EFI/BOOT/ /boot/efi/EFI/proxmox/ + test -n "$(find /boot/efi/EFI/BOOT -iname 'BOOTX64.EFI' -newer {{ stage_dir }}/etc-before.tar.gz)" + first=$(efibootmgr | sed -n 's/^BootOrder: \([0-9A-F]*\).*/\1/p') + efibootmgr -v | grep -E "^Boot${first}\*? " | grep -qi 'EFI\\proxmox\\shimx64.efi' + changed_when: "false" + - name: pve8to9 --full summary + sudo: true + shell: pve8to9 --full 2>&1 | sed 's/\x1b\[[0-9;]*m//g' | grep -E '^(FAIL|WARN):|TOTAL|PASSED|SKIPPED|WARNINGS|FAILURES' | cat + changed_when: "false"