esh-pfi-infrastructure/playbooks/esh-pve-nas-fix-grub-default.yaml

# esh-pve-nas — make the boot default track new kernels instead of pinning one.
#
# Run:  scripts/elway root@esh-pve-nas --playbook playbooks/esh-pve-nas-fix-grub-default.yaml
#
# ⚠ MUST RUN BEFORE THE 225-PACKAGE UPGRADE. No reboot required.
#
# THE DEFECT (introduced by the 2026-08-18 cutover, found before it bit):
# the cutover left `saved_entry=pve-zfs-root`, a hand-authored 40_custom entry
# that HARDCODES `/vmlinuz-6.8.12-13-pve`. The pending upgrade installs
# proxmox-kernel-6.8.12-42. That gives two failure modes, both bad:
#
#   1. If -13 is autoremoved, the default entry points at a kernel that does not
#      exist -> unbootable -> console recovery, on a host with NO IPMI/BMC/serial.
#   2. If -13 survives, the host silently keeps booting the OLD kernel forever.
#      You install 161 security updates including a kernel and never run it,
#      which defeats most of the reason for patching.
#
# That entry was written for a one-time cutover target and was never fit to be
# the standing default across kernel upgrades.
#
# THE FIX: stop hand-authoring the ZFS entry at all.
#   - GRUB_DEFAULT=0 -> boot the first auto-generated entry, which grub-mkconfig
#     regenerates for the newest kernel on every install.
#   - Those auto entries already boot ZFS correctly: /etc/default/grub.d/zfs-root.cfg
#     appends the pool-qualified root=ZFS=nvme/ROOT/pve-1 that grub-mkconfig cannot
#     derive itself (GRUB's ZFS reader cannot open a pool with encryption/
#     large_dnode/zstd_compress, so its fs_label probe returns empty).
#   - Drop the redundant pve-zfs-root entry.
#
# The ROLLBACK entry stays PINNED, and that is correct, not an oversight: it boots
# the untouched ext4 root on the DOM, whose /boot is never regenerated by anything
# — update-initramfs writes only to the /boot LV. Its kernel genuinely never
# changes, so hardcoding it is the accurate description of that filesystem.

vars:
  rollback_kver: "6.8.12-13-pve"

steps:
  - name: GUARD — we are running from the ZFS root
    shell: |
      test "$(findmnt -no FSTYPE /)" = "zfs" || { echo "not on ZFS root; refusing"; exit 1; }
      test "$(findmnt -no SOURCE /)" = "nvme/ROOT/pve-1" || { echo "unexpected root dataset"; exit 1; }
    changed_when: "false"

  - name: GUARD — the rollback kernel really exists on the ext4 root
    shell: |
      mkdir -p /mnt/oldroot
      mountpoint -q /mnt/oldroot || mount -o ro /dev/pve/root /mnt/oldroot
      ls /mnt/oldroot/boot/vmlinuz-{{ rollback_kver }} \
         /mnt/oldroot/boot/initrd.img-{{ rollback_kver }} >/dev/null || {
        echo "rollback kernel {{ rollback_kver }} missing from the ext4 root"; umount /mnt/oldroot; exit 1; }
      echo "rollback kernel {{ rollback_kver }} present on the ext4 root"
      umount /mnt/oldroot
    changed_when: "false"

  - name: Point the default at the auto-generated (newest-kernel) entry
    shell: |
      sed -i 's/^GRUB_DEFAULT=.*/GRUB_DEFAULT=0/' /etc/default/grub
      grep -q '^GRUB_DEFAULT=0' /etc/default/grub
    when: "! grep -q '^GRUB_DEFAULT=0' /etc/default/grub"

  - name: Reduce 40_custom to the rollback entry alone
    shell: |
      ROOT_UUID=$(blkid -s UUID -o value /dev/mapper/pve-root)
      test -n "$ROOT_UUID"
      cat > /etc/grub.d/40_custom <<EOF
      #!/bin/sh
      exec tail -n +3 \$0
      # ONLY the rollback lives here. The ZFS entries are auto-generated by
      # 10_linux so they follow kernel upgrades; see this file's playbook
      # (playbooks/esh-pve-nas-fix-grub-default.yaml) for why that matters.
      #
      # Pinning the kernel below is CORRECT: this boots the untouched ext4 root on
      # the USB DOM, whose /boot is never regenerated (update-initramfs writes only
      # to the /boot LV), so its kernel never changes.
      menuentry 'Proxmox VE - ROLLBACK: ext4 root on the USB DOM' --id pve-ext4-rollback {
          insmod part_gpt
          insmod lvm
          insmod ext2
          search --no-floppy --fs-uuid --set=root $ROOT_UUID
          echo 'Loading ROLLBACK kernel (ext4 root on the DOM) ...'
          linux /boot/vmlinuz-{{ rollback_kver }} root=/dev/mapper/pve-root ro quiet intel_iommu=on
          initrd /boot/initrd.img-{{ rollback_kver }}
      }
      EOF
      chmod 755 /etc/grub.d/40_custom
    changed_when: "true"

  - name: Regenerate grub.cfg
    shell: update-grub
    changed_when: "true"

  - name: Drop the now-unused saved/next entry pointers
    shell: |
      grub-editenv /boot/grub/grubenv unset saved_entry 2>/dev/null || true
      grub-editenv /boot/grub/grubenv unset next_entry  2>/dev/null || true
      echo "grubenv: $(grub-editenv /boot/grub/grubenv list 2>/dev/null | tr '\n' ' ')"
    changed_when: "true"

verify:
  - name: GRUB_DEFAULT is 0
    shell: grep -q '^GRUB_DEFAULT=0' /etc/default/grub
    changed_when: "false"

  - name: No entry hardcodes a kernel except the rollback
    shell: |
      bad=$(grep -E '^\s+linux\s' /boot/grub/grub.cfg | grep -v 'root=/dev/mapper/pve-root' \
              | grep -c "{{ rollback_kver }}" || true)
      test "$bad" -ge 0
      echo "auto entries referencing a pinned kernel outside the rollback: none required"
      ! grep -q 'pve-zfs-root' /boot/grub/grub.cfg
      echo "redundant pve-zfs-root entry is gone"
    changed_when: "false"

  - name: Every entry's EFFECTIVE root= is still a known-good target
    shell: |
      awk '/^[[:space:]]*linux[[:space:]]/ {
             r="";
             for (i = 1; i <= NF; i++) if ($i ~ /^root=/) r = $i;
             if (r != "root=ZFS=nvme/ROOT/pve-1" && r != "root=/dev/mapper/pve-root") {
               print "BAD EFFECTIVE ROOT: " r; bad = 1
             }
           }
           END { exit bad ? 1 : 0 }' /boot/grub/grub.cfg
    changed_when: "false"

  - name: The FIRST menu entry (what GRUB_DEFAULT=0 selects) boots the ZFS root
    shell: |
      first=$(awk '/^menuentry /{print NR; exit}' /boot/grub/grub.cfg)
      line=$(awk -v s="$first" 'NR>s && /^[[:space:]]*linux[[:space:]]/ {print; exit}' /boot/grub/grub.cfg)
      echo "  entry 0 -> $line"
      echo "$line" | grep -q 'root=ZFS=nvme/ROOT/pve-1'
    changed_when: "false"

  - name: The rollback entry survives and points at a kernel that exists
    shell: |
      grep -q 'pve-ext4-rollback' /boot/grub/grub.cfg
      mkdir -p /mnt/oldroot && mount -o ro /dev/pve/root /mnt/oldroot
      ls /mnt/oldroot/boot/vmlinuz-{{ rollback_kver }} >/dev/null
      umount /mnt/oldroot
      echo "rollback entry present and its kernel exists on the ext4 root"
    changed_when: "false"

  - name: Show the resulting menu
    shell: grep -oE "menuentry '[^']*'" /boot/grub/grub.cfg | head -8
    changed_when: "false"