# Upgrade a Proxmox node's packages. DOES NOT REBOOT — reboot is a separate, # deliberate step because it has cluster and NFS consequences this playbook # cannot see. # # Run: scripts/elway root@ --playbook playbooks/pve-node-upgrade.yaml # # ⚠ Both ESH nodes are members of the 2-node `esh-pve-cluster` (quorum 2, no # qdevice). Upgrading is safe while both are up; REBOOTING makes the survivor's # /etc/pve read-only until the node returns. Do one node at a time and let the # cluster go quorate again before touching the second. corosync 3.1.9 -> 3.1.10 # is a minor bump and rolling-safe, but do not leave the pair skewed longer than # the window needs. # # ⚠ For esh-pve-nas specifically, run playbooks/esh-pve-nas-fix-grub-default.yaml # FIRST. Its boot default used to pin a single kernel, so installing a new one # would either break the default entry or silently keep booting the old kernel. # # ⚠ The corosync bump RESTARTS corosync mid-upgrade, which on a 2-node cluster is # a brief quorum event — both nodes' /etc/pve go read-only for a few seconds and # then recover. Guests are unaffected and PVE does this routinely, but do not run # it concurrently with anything that writes cluster config, and check # `pvecm status` afterwards rather than assuming. # # Conffile policy: --force-confdef + --force-confold, i.e. keep the on-disk # version wherever a package ships a changed conffile. That is the right default # for these hosts (hand-tuned /etc/default/grub, grub.d drop-ins, storage.cfg), # and it means a genuinely important upstream conffile change will be left as a # .dpkg-dist file rather than applied — the verify phase lists any that appear so # they are not silently ignored. vars: backup_dir: /root/pre-upgrade-backup steps: - name: GUARD — cluster is quorate before we start shell: | pvecm status 2>/dev/null | grep -q "Quorate:.*Yes" || { echo "cluster is NOT quorate — resolve that before upgrading"; exit 1; } echo " quorate; nodes: $(pvecm nodes 2>/dev/null | awk 'NR>2 && $3 {print $3}' | tr '\n' ' ')" changed_when: "false" - name: GUARD — enough free space on / for the unpack shell: | avail=$(df -Pk / | awk 'NR==2{print $4}') test "$avail" -gt 2097152 || { echo "less than 2G free on / — refusing"; exit 1; } echo " / free: $(df -h / | awk 'NR==2{print $4}')" if findmnt -no TARGET /boot >/dev/null 2>&1; then bavail=$(df -Pk /boot | awk 'NR==2{print $4}') test "$bavail" -gt 204800 || { echo "less than 200M free on /boot — refusing"; exit 1; } echo " /boot free: $(df -h /boot | awk 'NR==2{print $4}')" else echo " /boot is part of / on this node" fi changed_when: "false" - name: Snapshot the config that matters before touching packages shell: | mkdir -p {{ backup_dir }} tar czf {{ backup_dir }}/pre-upgrade-$(hostname)-config.tar.gz \ -C / etc/pve etc/network/interfaces etc/fstab etc/default/grub \ etc/apt etc/corosync 2>/dev/null || true dpkg -l > {{ backup_dir }}/dpkg-before.txt pveversion -v > {{ backup_dir }}/pveversion-before.txt 2>&1 ls -la {{ backup_dir }}/ creates: "{{ backup_dir }}/pre-upgrade-backup.done" # On a ZFS-root node this is the cheapest insurance available: an instant, # space-free snapshot of the entire userspace before 200+ packages land. If the # upgrade goes wrong, the recovery is a rollback and a reboot rather than an # archaeology session in dpkg. Skipped automatically on non-ZFS roots. - name: Snapshot the root dataset (ZFS-root nodes only) shell: | ds=$(findmnt -no SOURCE /) snap="${ds}@pre-upgrade-$(date -u +%Y%m%dT%H%M%SZ)" zfs snapshot "$snap" echo " created $snap" echo " rollback if needed: zfs rollback -r $snap && reboot" zfs list -t snapshot -o name,used,creation -s creation "$ds" 2>/dev/null | tail -4 when: "test \"$(findmnt -no FSTYPE /)\" = zfs" - name: Refresh package lists shell: apt-get update -qq changed_when: "true" - name: Record what is about to change shell: | apt-get -s dist-upgrade 2>/dev/null | grep -E "^Inst " > {{ backup_dir }}/planned-upgrade.txt echo " $(wc -l < {{ backup_dir }}/planned-upgrade.txt) packages planned" grep -E "kernel|corosync|pve-manager|zfs" {{ backup_dir }}/planned-upgrade.txt | sed 's/^/ /' changed_when: "false" - name: dist-upgrade shell: | DEBIAN_FRONTEND=noninteractive apt-get -y \ -o Dpkg::Options::=--force-confdef \ -o Dpkg::Options::=--force-confold \ dist-upgrade 2>&1 | tail -30 changed_when: "true" - name: Record the result shell: | pveversion -v > {{ backup_dir }}/pveversion-after.txt 2>&1 head -3 {{ backup_dir }}/pveversion-after.txt touch {{ backup_dir }}/pre-upgrade-backup.done changed_when: "true" verify: - name: dpkg is in a clean state shell: | broken=$(dpkg -l | grep -cE "^i[^i]| ^r" || true) dpkg --audit 2>&1 | head -5 test -z "$(dpkg --audit 2>/dev/null)" || { echo "dpkg --audit is not clean"; exit 1; } echo "dpkg clean" changed_when: "false" - name: No packages left half-configured shell: | n=$(apt-get -s -f install 2>/dev/null | grep -cE "^Inst |^Conf " || true) test "$n" -eq 0 || { echo "apt -f install wants to do $n things"; exit 1; } echo "nothing outstanding for apt -f install" changed_when: "false" - name: Core PVE services still active shell: | for s in pve-cluster corosync pvedaemon pveproxy pvestatd; do a=$(systemctl is-active $s 2>&1); printf " %-14s %s\n" "$s" "$a" test "$a" = "active" || bad=1 done test -z "$bad" changed_when: "false" - name: Cluster still quorate after the upgrade shell: pvecm status 2>/dev/null | grep -E "Quorate|Total votes" changed_when: "false" - name: Surface any conffiles the confold policy left unapplied shell: | found=$(find /etc -name "*.dpkg-dist" -o -name "*.dpkg-new" 2>/dev/null | head -20) if [ -n "$found" ]; then echo "REVIEW THESE — upstream shipped changes that were NOT applied:"; echo "$found" else echo "no unapplied conffiles" fi changed_when: "false" - name: Report whether a reboot is required shell: | run=$(uname -r) new=$(ls -1 /boot/vmlinuz-* 2>/dev/null | sed 's|.*/vmlinuz-||' | sort -V | tail -1) echo " running kernel: $run" echo " newest on disk: $new" [ "$run" != "$new" ] && echo " -> REBOOT REQUIRED to run $new" || echo " -> no kernel change" changed_when: "false"