esh-pfi-infrastructure/playbooks/pve-node-upgrade.yaml

# Upgrade a Proxmox node's packages. DOES NOT REBOOT — reboot is a separate,
# deliberate step because it has cluster and NFS consequences this playbook
# cannot see.
#
# Run:  scripts/elway root@<node> --playbook playbooks/pve-node-upgrade.yaml
#
# ⚠ Both ESH nodes are members of the 2-node `esh-pve-cluster` (quorum 2, no
# qdevice). Upgrading is safe while both are up; REBOOTING makes the survivor's
# /etc/pve read-only until the node returns. Do one node at a time and let the
# cluster go quorate again before touching the second. corosync 3.1.9 -> 3.1.10
# is a minor bump and rolling-safe, but do not leave the pair skewed longer than
# the window needs.
#
# ⚠ For esh-pve-nas specifically, run playbooks/esh-pve-nas-fix-grub-default.yaml
# FIRST. Its boot default used to pin a single kernel, so installing a new one
# would either break the default entry or silently keep booting the old kernel.
#
# ⚠ The corosync bump RESTARTS corosync mid-upgrade, which on a 2-node cluster is
# a brief quorum event — both nodes' /etc/pve go read-only for a few seconds and
# then recover. Guests are unaffected and PVE does this routinely, but do not run
# it concurrently with anything that writes cluster config, and check
# `pvecm status` afterwards rather than assuming.
#
# Conffile policy: --force-confdef + --force-confold, i.e. keep the on-disk
# version wherever a package ships a changed conffile. That is the right default
# for these hosts (hand-tuned /etc/default/grub, grub.d drop-ins, storage.cfg),
# and it means a genuinely important upstream conffile change will be left as a
# .dpkg-dist file rather than applied — the verify phase lists any that appear so
# they are not silently ignored.

vars:
  backup_dir: /root/pre-upgrade-backup

steps:
  - name: GUARD — cluster is quorate before we start
    shell: |
      pvecm status 2>/dev/null | grep -q "Quorate:.*Yes" || {
        echo "cluster is NOT quorate — resolve that before upgrading"; exit 1; }
      echo "  quorate; nodes: $(pvecm nodes 2>/dev/null | awk 'NR>2 && $3 {print $3}' | tr '\n' ' ')"
    changed_when: "false"

  - name: GUARD — enough free space on / for the unpack
    shell: |
      avail=$(df -Pk / | awk 'NR==2{print $4}')
      test "$avail" -gt 2097152 || { echo "less than 2G free on / — refusing"; exit 1; }
      echo "  / free: $(df -h / | awk 'NR==2{print $4}')"
      if findmnt -no TARGET /boot >/dev/null 2>&1; then
        bavail=$(df -Pk /boot | awk 'NR==2{print $4}')
        test "$bavail" -gt 204800 || { echo "less than 200M free on /boot — refusing"; exit 1; }
        echo "  /boot free: $(df -h /boot | awk 'NR==2{print $4}')"
      else
        echo "  /boot is part of / on this node"
      fi
    changed_when: "false"

  - name: Snapshot the config that matters before touching packages
    shell: |
      mkdir -p {{ backup_dir }}
      tar czf {{ backup_dir }}/pre-upgrade-$(hostname)-config.tar.gz \
        -C / etc/pve etc/network/interfaces etc/fstab etc/default/grub \
        etc/apt etc/corosync 2>/dev/null || true
      dpkg -l > {{ backup_dir }}/dpkg-before.txt
      pveversion -v > {{ backup_dir }}/pveversion-before.txt 2>&1
      ls -la {{ backup_dir }}/
    creates: "{{ backup_dir }}/pre-upgrade-backup.done"

  # On a ZFS-root node this is the cheapest insurance available: an instant,
  # space-free snapshot of the entire userspace before 200+ packages land. If the
  # upgrade goes wrong, the recovery is a rollback and a reboot rather than an
  # archaeology session in dpkg. Skipped automatically on non-ZFS roots.
  - name: Snapshot the root dataset (ZFS-root nodes only)
    shell: |
      ds=$(findmnt -no SOURCE /)
      snap="${ds}@pre-upgrade-$(date -u +%Y%m%dT%H%M%SZ)"
      zfs snapshot "$snap"
      echo "  created $snap"
      echo "  rollback if needed:  zfs rollback -r $snap  &&  reboot"
      zfs list -t snapshot -o name,used,creation -s creation "$ds" 2>/dev/null | tail -4
    when: "test \"$(findmnt -no FSTYPE /)\" = zfs"

  - name: Refresh package lists
    shell: apt-get update -qq
    changed_when: "true"

  - name: Record what is about to change
    shell: |
      apt-get -s dist-upgrade 2>/dev/null | grep -E "^Inst " > {{ backup_dir }}/planned-upgrade.txt
      echo "  $(wc -l < {{ backup_dir }}/planned-upgrade.txt) packages planned"
      grep -E "kernel|corosync|pve-manager|zfs" {{ backup_dir }}/planned-upgrade.txt | sed 's/^/    /'
    changed_when: "false"

  - name: dist-upgrade
    shell: |
      DEBIAN_FRONTEND=noninteractive apt-get -y \
        -o Dpkg::Options::=--force-confdef \
        -o Dpkg::Options::=--force-confold \
        dist-upgrade 2>&1 | tail -30
    changed_when: "true"

  - name: Record the result
    shell: |
      pveversion -v > {{ backup_dir }}/pveversion-after.txt 2>&1
      head -3 {{ backup_dir }}/pveversion-after.txt
      touch {{ backup_dir }}/pre-upgrade-backup.done
    changed_when: "true"

verify:
  - name: dpkg is in a clean state
    shell: |
      broken=$(dpkg -l | grep -cE "^i[^i]| ^r" || true)
      dpkg --audit 2>&1 | head -5
      test -z "$(dpkg --audit 2>/dev/null)" || { echo "dpkg --audit is not clean"; exit 1; }
      echo "dpkg clean"
    changed_when: "false"

  - name: No packages left half-configured
    shell: |
      n=$(apt-get -s -f install 2>/dev/null | grep -cE "^Inst |^Conf " || true)
      test "$n" -eq 0 || { echo "apt -f install wants to do $n things"; exit 1; }
      echo "nothing outstanding for apt -f install"
    changed_when: "false"

  - name: Core PVE services still active
    shell: |
      for s in pve-cluster corosync pvedaemon pveproxy pvestatd; do
        a=$(systemctl is-active $s 2>&1); printf "  %-14s %s\n" "$s" "$a"
        test "$a" = "active" || bad=1
      done
      test -z "$bad"
    changed_when: "false"

  - name: Cluster still quorate after the upgrade
    shell: pvecm status 2>/dev/null | grep -E "Quorate|Total votes"
    changed_when: "false"

  - name: Surface any conffiles the confold policy left unapplied
    shell: |
      found=$(find /etc -name "*.dpkg-dist" -o -name "*.dpkg-new" 2>/dev/null | head -20)
      if [ -n "$found" ]; then
        echo "REVIEW THESE — upstream shipped changes that were NOT applied:"; echo "$found"
      else
        echo "no unapplied conffiles"
      fi
    changed_when: "false"

  - name: Report whether a reboot is required
    shell: |
      run=$(uname -r)
      new=$(ls -1 /boot/vmlinuz-* 2>/dev/null | sed 's|.*/vmlinuz-||' | sort -V | tail -1)
      echo "  running kernel: $run"
      echo "  newest on disk: $new"
      [ "$run" != "$new" ] && echo "  -> REBOOT REQUIRED to run $new" || echo "  -> no kernel change"
    changed_when: "false"