diff --git a/playbooks/pve-node-upgrade.yaml b/playbooks/pve-node-upgrade.yaml new file mode 100644 index 0000000..5f6aa7e --- /dev/null +++ b/playbooks/pve-node-upgrade.yaml @@ -0,0 +1,133 @@ +# Upgrade a Proxmox node's packages. DOES NOT REBOOT — reboot is a separate, +# deliberate step because it has cluster and NFS consequences this playbook +# cannot see. +# +# Run: scripts/elway root@ --playbook playbooks/pve-node-upgrade.yaml +# +# ⚠ Both ESH nodes are members of the 2-node `esh-pve-cluster` (quorum 2, no +# qdevice). Upgrading is safe while both are up; REBOOTING makes the survivor's +# /etc/pve read-only until the node returns. Do one node at a time and let the +# cluster go quorate again before touching the second. corosync 3.1.9 -> 3.1.10 +# is a minor bump and rolling-safe, but do not leave the pair skewed longer than +# the window needs. +# +# ⚠ For esh-pve-nas specifically, run playbooks/esh-pve-nas-fix-grub-default.yaml +# FIRST. Its boot default used to pin a single kernel, so installing a new one +# would either break the default entry or silently keep booting the old kernel. +# +# Conffile policy: --force-confdef + --force-confold, i.e. keep the on-disk +# version wherever a package ships a changed conffile. That is the right default +# for these hosts (hand-tuned /etc/default/grub, grub.d drop-ins, storage.cfg), +# and it means a genuinely important upstream conffile change will be left as a +# .dpkg-dist file rather than applied — the verify phase lists any that appear so +# they are not silently ignored. + +vars: + backup_dir: /root/pre-upgrade-backup + +steps: + - name: GUARD — cluster is quorate before we start + shell: | + pvecm status 2>/dev/null | grep -q "Quorate:.*Yes" || { + echo "cluster is NOT quorate — resolve that before upgrading"; exit 1; } + echo " quorate; nodes: $(pvecm nodes 2>/dev/null | awk 'NR>2 && $3 {print $3}' | tr '\n' ' ')" + changed_when: "false" + + - name: GUARD — enough free space on / for the unpack + shell: | + avail=$(df -Pk / | awk 'NR==2{print $4}') + test "$avail" -gt 2097152 || { echo "less than 2G free on / — refusing"; exit 1; } + echo " / free: $(df -h / | awk 'NR==2{print $4}')" + if findmnt -no TARGET /boot >/dev/null 2>&1; then + bavail=$(df -Pk /boot | awk 'NR==2{print $4}') + test "$bavail" -gt 204800 || { echo "less than 200M free on /boot — refusing"; exit 1; } + echo " /boot free: $(df -h /boot | awk 'NR==2{print $4}')" + else + echo " /boot is part of / on this node" + fi + changed_when: "false" + + - name: Snapshot the config that matters before touching packages + shell: | + mkdir -p {{ backup_dir }} + tar czf {{ backup_dir }}/pre-upgrade-$(hostname)-config.tar.gz \ + -C / etc/pve etc/network/interfaces etc/fstab etc/default/grub \ + etc/apt etc/corosync 2>/dev/null || true + dpkg -l > {{ backup_dir }}/dpkg-before.txt + pveversion -v > {{ backup_dir }}/pveversion-before.txt 2>&1 + ls -la {{ backup_dir }}/ + creates: "{{ backup_dir }}/pre-upgrade-backup.done" + + - name: Refresh package lists + shell: apt-get update -qq + changed_when: "true" + + - name: Record what is about to change + shell: | + apt-get -s dist-upgrade 2>/dev/null | grep -E "^Inst " > {{ backup_dir }}/planned-upgrade.txt + echo " $(wc -l < {{ backup_dir }}/planned-upgrade.txt) packages planned" + grep -E "kernel|corosync|pve-manager|zfs" {{ backup_dir }}/planned-upgrade.txt | sed 's/^/ /' + changed_when: "false" + + - name: dist-upgrade + shell: | + DEBIAN_FRONTEND=noninteractive apt-get -y \ + -o Dpkg::Options::=--force-confdef \ + -o Dpkg::Options::=--force-confold \ + dist-upgrade 2>&1 | tail -30 + changed_when: "true" + + - name: Record the result + shell: | + pveversion -v > {{ backup_dir }}/pveversion-after.txt 2>&1 + head -3 {{ backup_dir }}/pveversion-after.txt + touch {{ backup_dir }}/pre-upgrade-backup.done + changed_when: "true" + +verify: + - name: dpkg is in a clean state + shell: | + broken=$(dpkg -l | grep -cE "^i[^i]| ^r" || true) + dpkg --audit 2>&1 | head -5 + test -z "$(dpkg --audit 2>/dev/null)" || { echo "dpkg --audit is not clean"; exit 1; } + echo "dpkg clean" + changed_when: "false" + + - name: No packages left half-configured + shell: | + n=$(apt-get -s -f install 2>/dev/null | grep -cE "^Inst |^Conf " || true) + test "$n" -eq 0 || { echo "apt -f install wants to do $n things"; exit 1; } + echo "nothing outstanding for apt -f install" + changed_when: "false" + + - name: Core PVE services still active + shell: | + for s in pve-cluster corosync pvedaemon pveproxy pvestatd; do + a=$(systemctl is-active $s 2>&1); printf " %-14s %s\n" "$s" "$a" + test "$a" = "active" || bad=1 + done + test -z "$bad" + changed_when: "false" + + - name: Cluster still quorate after the upgrade + shell: pvecm status 2>/dev/null | grep -E "Quorate|Total votes" + changed_when: "false" + + - name: Surface any conffiles the confold policy left unapplied + shell: | + found=$(find /etc -name "*.dpkg-dist" -o -name "*.dpkg-new" 2>/dev/null | head -20) + if [ -n "$found" ]; then + echo "REVIEW THESE — upstream shipped changes that were NOT applied:"; echo "$found" + else + echo "no unapplied conffiles" + fi + changed_when: "false" + + - name: Report whether a reboot is required + shell: | + run=$(uname -r) + new=$(ls -1 /boot/vmlinuz-* 2>/dev/null | sed 's|.*/vmlinuz-||' | sort -V | tail -1) + echo " running kernel: $run" + echo " newest on disk: $new" + [ "$run" != "$new" ] && echo " -> REBOOT REQUIRED to run $new" || echo " -> no kernel change" + changed_when: "false"