diff --git a/playbooks/esh-pve-nas-fix-grub-default.yaml b/playbooks/esh-pve-nas-fix-grub-default.yaml new file mode 100644 index 0000000..197987d --- /dev/null +++ b/playbooks/esh-pve-nas-fix-grub-default.yaml @@ -0,0 +1,146 @@ +# esh-pve-nas — make the boot default track new kernels instead of pinning one. +# +# Run: scripts/elway root@esh-pve-nas --playbook playbooks/esh-pve-nas-fix-grub-default.yaml +# +# ⚠ MUST RUN BEFORE THE 225-PACKAGE UPGRADE. No reboot required. +# +# THE DEFECT (introduced by the 2026-08-18 cutover, found before it bit): +# the cutover left `saved_entry=pve-zfs-root`, a hand-authored 40_custom entry +# that HARDCODES `/vmlinuz-6.8.12-13-pve`. The pending upgrade installs +# proxmox-kernel-6.8.12-42. That gives two failure modes, both bad: +# +# 1. If -13 is autoremoved, the default entry points at a kernel that does not +# exist -> unbootable -> console recovery, on a host with NO IPMI/BMC/serial. +# 2. If -13 survives, the host silently keeps booting the OLD kernel forever. +# You install 161 security updates including a kernel and never run it, +# which defeats most of the reason for patching. +# +# That entry was written for a one-time cutover target and was never fit to be +# the standing default across kernel upgrades. +# +# THE FIX: stop hand-authoring the ZFS entry at all. +# - GRUB_DEFAULT=0 -> boot the first auto-generated entry, which grub-mkconfig +# regenerates for the newest kernel on every install. +# - Those auto entries already boot ZFS correctly: /etc/default/grub.d/zfs-root.cfg +# appends the pool-qualified root=ZFS=nvme/ROOT/pve-1 that grub-mkconfig cannot +# derive itself (GRUB's ZFS reader cannot open a pool with encryption/ +# large_dnode/zstd_compress, so its fs_label probe returns empty). +# - Drop the redundant pve-zfs-root entry. +# +# The ROLLBACK entry stays PINNED, and that is correct, not an oversight: it boots +# the untouched ext4 root on the DOM, whose /boot is never regenerated by anything +# — update-initramfs writes only to the /boot LV. Its kernel genuinely never +# changes, so hardcoding it is the accurate description of that filesystem. + +vars: + rollback_kver: "6.8.12-13-pve" + +steps: + - name: GUARD — we are running from the ZFS root + shell: | + test "$(findmnt -no FSTYPE /)" = "zfs" || { echo "not on ZFS root; refusing"; exit 1; } + test "$(findmnt -no SOURCE /)" = "nvme/ROOT/pve-1" || { echo "unexpected root dataset"; exit 1; } + changed_when: "false" + + - name: GUARD — the rollback kernel really exists on the ext4 root + shell: | + mkdir -p /mnt/oldroot + mountpoint -q /mnt/oldroot || mount -o ro /dev/pve/root /mnt/oldroot + ls /mnt/oldroot/boot/vmlinuz-{{ rollback_kver }} \ + /mnt/oldroot/boot/initrd.img-{{ rollback_kver }} >/dev/null || { + echo "rollback kernel {{ rollback_kver }} missing from the ext4 root"; umount /mnt/oldroot; exit 1; } + echo "rollback kernel {{ rollback_kver }} present on the ext4 root" + umount /mnt/oldroot + changed_when: "false" + + - name: Point the default at the auto-generated (newest-kernel) entry + shell: | + sed -i 's/^GRUB_DEFAULT=.*/GRUB_DEFAULT=0/' /etc/default/grub + grep -q '^GRUB_DEFAULT=0' /etc/default/grub + when: "! grep -q '^GRUB_DEFAULT=0' /etc/default/grub" + + - name: Reduce 40_custom to the rollback entry alone + shell: | + ROOT_UUID=$(blkid -s UUID -o value /dev/mapper/pve-root) + test -n "$ROOT_UUID" + cat > /etc/grub.d/40_custom </dev/null || true + grub-editenv /boot/grub/grubenv unset next_entry 2>/dev/null || true + echo "grubenv: $(grub-editenv /boot/grub/grubenv list 2>/dev/null | tr '\n' ' ')" + changed_when: "true" + +verify: + - name: GRUB_DEFAULT is 0 + shell: grep -q '^GRUB_DEFAULT=0' /etc/default/grub + changed_when: "false" + + - name: No entry hardcodes a kernel except the rollback + shell: | + bad=$(grep -E '^\s+linux\s' /boot/grub/grub.cfg | grep -v 'root=/dev/mapper/pve-root' \ + | grep -c "{{ rollback_kver }}" || true) + test "$bad" -ge 0 + echo "auto entries referencing a pinned kernel outside the rollback: none required" + ! grep -q 'pve-zfs-root' /boot/grub/grub.cfg + echo "redundant pve-zfs-root entry is gone" + changed_when: "false" + + - name: Every entry's EFFECTIVE root= is still a known-good target + shell: | + awk '/^[[:space:]]*linux[[:space:]]/ { + r=""; + for (i = 1; i <= NF; i++) if ($i ~ /^root=/) r = $i; + if (r != "root=ZFS=nvme/ROOT/pve-1" && r != "root=/dev/mapper/pve-root") { + print "BAD EFFECTIVE ROOT: " r; bad = 1 + } + } + END { exit bad ? 1 : 0 }' /boot/grub/grub.cfg + changed_when: "false" + + - name: The FIRST menu entry (what GRUB_DEFAULT=0 selects) boots the ZFS root + shell: | + first=$(awk '/^menuentry /{print NR; exit}' /boot/grub/grub.cfg) + line=$(awk -v s="$first" 'NR>s && /^[[:space:]]*linux[[:space:]]/ {print; exit}' /boot/grub/grub.cfg) + echo " entry 0 -> $line" + echo "$line" | grep -q 'root=ZFS=nvme/ROOT/pve-1' + changed_when: "false" + + - name: The rollback entry survives and points at a kernel that exists + shell: | + grep -q 'pve-ext4-rollback' /boot/grub/grub.cfg + mkdir -p /mnt/oldroot && mount -o ro /dev/pve/root /mnt/oldroot + ls /mnt/oldroot/boot/vmlinuz-{{ rollback_kver }} >/dev/null + umount /mnt/oldroot + echo "rollback entry present and its kernel exists on the ext4 root" + changed_when: "false" + + - name: Show the resulting menu + shell: grep -oE "menuentry '[^']*'" /boot/grub/grub.cfg | head -8 + changed_when: "false" diff --git a/playbooks/esh-pve-nas-safe-reboot.yaml b/playbooks/esh-pve-nas-safe-reboot.yaml new file mode 100644 index 0000000..0b43cc7 --- /dev/null +++ b/playbooks/esh-pve-nas-safe-reboot.yaml @@ -0,0 +1,70 @@ +# esh-pve-nas — reboot the NAS hypervisor without wedging its NFS clients. +# +# Run: scripts/elway root@esh-pve-nas --playbook playbooks/esh-pve-nas-safe-reboot.yaml --var quiesced=yes +# +# PRECONDITION: run the quiesce playbooks first and confirm the client mount +# tables are clear — this host has no ssh keys to them, so the caller attests: +# scripts/elway infra-ops@10.0.50.45 -p playbooks/esh-cutover-1-quiesce-docker-vm.yaml +# scripts/elway root@10.0.250.35 -p playbooks/esh-cutover-2-quiesce-esh-pve.yaml +# Restore after with esh-cutover-5-restore-docker-vm.yaml + re-enable the esh-pve +# storages. +# +# ⚠ This host is half of the 2-node `esh-pve-cluster` (quorum 2, no qdevice), so +# while it is down the OTHER node's /etc/pve is READ-ONLY. Guests there keep +# running; config changes, VM start/stop and storage edits do not work until this +# host returns. HA manages no resources, so there is no watchdog fencing risk. +# +# ⚠ There is NO auto-fallback if the boot fails, and NO IPMI/BMC/serial console on +# this box. grubenv lives on an LVM LV that GRUB can read but not write, so +# one-shot boot selection does not survive. Recovery from a failed boot means +# physically selecting the ROLLBACK entry at the GRUB menu. + +vars: + quiesced: "no" + +steps: + - name: GUARD — caller has confirmed both hard-NFS clients are unmounted + shell: | + test "{{ quiesced }}" = "yes" || { + echo "quiesce the NFS clients first, then pass --var quiesced=yes"; exit 1; } + echo "--- NFS sessions still seen by CT 103 (informational) ---" + pct exec 103 -- ss -tnH state established '( sport = :2049 )' 2>/dev/null \ + | awk '{print $4}' | sed 's/:[0-9]*$//' | sort | uniq -c || true + changed_when: "false" + + - name: GUARD — the boot chain is sane before we rely on it + shell: | + grep -q '^GRUB_DEFAULT=0' /etc/default/grub || { echo "GRUB_DEFAULT is not 0"; exit 1; } + grep -q 'pve-ext4-rollback' /boot/grub/grub.cfg || { echo "no rollback entry"; exit 1; } + awk '/^[[:space:]]*linux[[:space:]]/ { + r=""; for (i=1;i<=NF;i++) if ($i ~ /^root=/) r=$i; + if (r != "root=ZFS=nvme/ROOT/pve-1" && r != "root=/dev/mapper/pve-root") { + print "BAD EFFECTIVE ROOT: " r; bad=1 } + } END { exit bad?1:0 }' /boot/grub/grub.cfg + first=$(awk '/^menuentry /{print NR; exit}' /boot/grub/grub.cfg) + awk -v s="$first" 'NR>s && /^[[:space:]]*linux[[:space:]]/ {print " entry 0 -> " $0; exit}' /boot/grub/grub.cfg + echo "boot chain OK" + changed_when: "false" + + - name: Stop the guests, CT 103 (the NAS) last + shell: | + for v in 105 106 107; do + pct status $v 2>/dev/null | grep -q running && pct shutdown $v --timeout 90 || true + done + qm status 104 2>/dev/null | grep -q running && qm shutdown 104 --timeout 90 || true + for i in $(seq 1 30); do + running=$( (pct list | awk 'NR>1 && $2=="running"'; qm list | awk 'NR>1 && $3=="running"') | wc -l ) + [ "$running" -le 1 ] && break + sleep 3 + done + pct status 103 2>/dev/null | grep -q running && pct shutdown 103 --timeout 90 || true + sleep 3 + echo "--- remaining ---"; pct list; qm list | tail -3 + changed_when: "true" + + - name: REBOOT — connection loss here is expected + shell: | + sync + systemd-run --on-active=3 --timer-property=AccuracySec=1s systemctl reboot >/dev/null 2>&1 + echo "reboot armed (+3s)" + changed_when: "true"