72 lines
3.9 KiB
YAML
72 lines
3.9 KiB
YAML
# pfi-pve: after the PVE 9 dist-upgrade, BEFORE the reboot. One fix-up step, the rest is read-only.
|
|
# scripts/elway infra-ops@10.250.250.31 --playbook playbooks/pfi-pve-pve9-prereboot.yaml
|
|
# Every verify here is something that, if wrong, makes the reboot a full Anaheim outage with no path back in except
|
|
# the FortiGate WAN-SSH break-glass (docs/runbooks/pfi-pve-pve9-upgrade-plan.md § Break-glass).
|
|
steps:
|
|
- name: the upgrade ships pve-enterprise.sources enabled; turn it off (no subscription)
|
|
sudo: true
|
|
shell: |
|
|
f=/etc/apt/sources.list.d/pve-enterprise.sources
|
|
if grep -q '^Enabled:' $f; then sed -i 's/^Enabled:.*/Enabled: false/' $f; else echo 'Enabled: false' >> $f; fi
|
|
cat $f
|
|
when: "test -f /etc/apt/sources.list.d/pve-enterprise.sources && ! grep -q '^Enabled: *false' /etc/apt/sources.list.d/pve-enterprise.sources"
|
|
|
|
verify:
|
|
- name: the detached dist-upgrade finished with EXIT 0
|
|
sudo: true
|
|
shell: tail -1 /root/pve9-upgrade.log | grep -qx 'EXIT 0'
|
|
changed_when: "false"
|
|
- name: dpkg is clean and nothing is left for apt -f
|
|
sudo: true
|
|
shell: |
|
|
test -z "$(dpkg --audit 2>&1)"
|
|
test "$(apt-get -s -f install 2>/dev/null | grep -cE '^(Inst|Conf) ')" -eq 0
|
|
changed_when: "false"
|
|
- name: PVE 9 userland
|
|
sudo: true
|
|
shell: pveversion | grep -E '^pve-manager/9\.'
|
|
changed_when: "false"
|
|
- name: newest kernel has an initramfs carrying the NIC pins and dm-snapshot (root has an LVM snapshot)
|
|
sudo: true
|
|
shell: |
|
|
set -e
|
|
new=$(ls /boot/vmlinuz-* | sed 's#/boot/vmlinuz-##' | sort -V | tail -1)
|
|
echo "newest kernel: $new"
|
|
lsinitramfs /boot/initrd.img-$new > /tmp/pve9-initrd.lst
|
|
grep -q 10-pin-eno8303.link /tmp/pve9-initrd.lst
|
|
grep -q 10-pin-eno12399np0.link /tmp/pve9-initrd.lst
|
|
grep -q 'dm-snapshot.ko' /tmp/pve9-initrd.lst
|
|
grep -q '/bnxt_en.ko' /tmp/pve9-initrd.lst || echo " (bnxt_en not in initramfs; loads from root, fine)"
|
|
grep -q "vmlinuz-$new" /boot/grub/grub.cfg
|
|
changed_when: "false"
|
|
- name: ESP rewritten by the trixie GRUB; removable copy matches; firmware boots proxmox first
|
|
sudo: true
|
|
shell: |
|
|
set -e
|
|
ls -la --time-style=long-iso /boot/efi/EFI/proxmox/ /boot/efi/EFI/BOOT/
|
|
test "$(md5sum < /boot/efi/EFI/proxmox/grubx64.efi)" = "$(md5sum < /boot/efi/EFI/BOOT/BOOTX64.EFI)"
|
|
# The binary carries no greppable version string; the trixie grub-efi-amd64 postinst's own report is the proof.
|
|
grep -q 'Installation finished. No error reported.' /root/pve9-upgrade.log
|
|
first=$(efibootmgr | sed -n 's/^BootOrder: \([0-9A-F]*\).*/\1/p')
|
|
efibootmgr -v | grep -E "^Boot${first}\*? " | grep -qi 'EFI\\proxmox\\grubx64.efi'
|
|
changed_when: "false"
|
|
- name: networking will come back (dhclient for vmbr0.250, interfaces parse, sysctl.conf has nothing to lose)
|
|
sudo: true
|
|
shell: |
|
|
set -e
|
|
dpkg-query -W -f='${Status}' isc-dhcp-client | grep -q 'install ok installed' || dpkg-query -W -f='${Status}' dhcpcd-base | grep -q 'install ok installed'
|
|
dpkg-query -W -f='${Status}' ifupdown2 | grep -q 'install ok installed'
|
|
# ifupdown2 3.3 (PVE 9) flags PVE's own `bridge-fd 0` as out of range and `-s` exits 1 on that warning alone.
|
|
# nh3-pve and esh-nas-pve show the same warning and booted with vmbr0 fine (STP off, forward_delay 0), so
|
|
# only a warning OTHER than that one fails this check.
|
|
out=$(ifreload -a -s 2>&1) || true
|
|
echo "$out"
|
|
# (an `if`, not a bare `! pipeline`: set -e ignores a negated pipeline, so that form could never fail)
|
|
if echo "$out" | grep -v 'bridge-fd: value of out range "0"' | grep -q .; then echo "unexpected ifupdown2 output"; exit 1; fi
|
|
test -z "$(grep -vE '^\s*(#|$)' /etc/sysctl.conf 2>/dev/null)"
|
|
changed_when: "false"
|
|
- name: pve8to9 --full (post-upgrade checks)
|
|
sudo: true
|
|
shell: pve8to9 --full 2>&1 | grep -E '^(FAIL|WARN):|TOTAL|PASSED|SKIPPED|WARNINGS|FAILURES' | cat
|
|
changed_when: "false"
|