# esh-pve-nas — PHASE 2 of the ZFS-root migration: build the boot artifacts.
#
# Runbook: docs/runbooks/esh-pve-nas-boot-migration.md
# Run AFTER playbooks/esh-pve-nas-stage-zfs-root.yaml.
#
# ⚠ THIS PLAYBOOK DELIBERATELY DOES NOT RUN `grub-install`.
#
# That is the whole safety design. Everything expensive and error-prone — the
# ZFS-capable initramfs, the generated grub.cfg, the rollback menu entry, the
# grubenv default — is built and verified here, onto the NEW /boot LV, while the
# ESP stub on the DOM still points at the OLD /boot inside the ext4 root LV.
#
# So until cutover the host's boot path is byte-for-byte what it has been for
# 140 days. An unplanned reboot mid-staging lands exactly where it always did.
# The cutover reduces to one idempotent two-second command plus the reboot:
#
#     chroot /mnt/newroot grub-install --target=x86_64-efi \
#         --efi-directory=/boot/efi --bootloader-id=proxmox
#     chroot /mnt/newroot grub-reboot '<zfs entry id printed by verify below>'
#     reboot
#
# Why the rollback entry matters here: the ext4 root LV keeps its own /boot
# contents (the new LV is a copy, not a move), and its initrd is never
# regenerated — update-initramfs inside the chroot writes only to the new LV.
# So the rollback path is genuinely independent of anything we build.
#
# Why GRUB_DEFAULT=saved: the default stays pinned to the ext4 rollback entry.
# At cutover `grub-reboot` marks the ZFS entry to be tried EXACTLY ONCE. If the
# ZFS root fails to come up, the next reboot returns to ext4 with nobody at the
# console — which matters because a failed boot here takes CT 103 `esh-nas`
# down and wedges esh-docker-vm into unkillable D-state on hard NFS.

vars:
  newroot: /mnt/newroot
  root_dataset: nvme/ROOT/pve-1

steps:
  # ---------- guards ----------

  - name: GUARD — host must still be running from the ext4 root on the DOM
    shell: |
      test "$(findmnt -no FSTYPE /)" = "ext4" || {
        echo "root is not ext4 — already cut over; refusing"; exit 1; }
    changed_when: "false"

  - name: GUARD — phase 1 must have completed (ZFS copy populated)
    shell: |
      mountpoint -q {{ newroot }} || { echo "{{ newroot }} not mounted"; exit 1; }
      test -x {{ newroot }}/usr/bin/pveversion || { echo "ZFS copy incomplete"; exit 1; }
      test -f {{ newroot }}/etc/fstab || { echo "ZFS copy has no fstab"; exit 1; }
    changed_when: "false"

  # Tolerates either staging location: /mnt/boot-new before this playbook has
  # moved the LV, {{ newroot }}/boot after — so a rerun still passes.
  - name: GUARD — the new /boot LV must exist and carry a kernel
    shell: |
      lvs pve/boot >/dev/null 2>&1 || { echo "pve/boot missing"; exit 1; }
      ls /mnt/boot-new/vmlinuz-* >/dev/null 2>&1 || \
      ls {{ newroot }}/boot/vmlinuz-* >/dev/null 2>&1 || {
        echo "no kernel on the boot LV at either staging path"; exit 1; }
    changed_when: "false"

  # ---------- back up what we are about to regenerate ----------

  - name: Snapshot the ESP and grub defaults before touching anything
    shell: |
      mkdir -p /root/pre-zfs-boot-backup
      tar czf /root/pre-zfs-boot-backup/esp-and-grub.tar.gz \
        -C / boot/efi etc/default/grub 2>/dev/null
      ls -la /root/pre-zfs-boot-backup/
    creates: /root/pre-zfs-boot-backup/esp-and-grub.tar.gz

  # ---------- assemble the chroot ----------

  - name: Release the staging mount of the boot LV so it can move under the chroot
    shell: umount /mnt/boot-new
    when: "mountpoint -q /mnt/boot-new"

  # ESP goes in as a BIND of the live /boot/efi rather than a second mount of
  # /dev/sdq2 — same filesystem either way, but the bind leaves no ambiguity
  # about which superblock grub-install writes through at cutover.
  - name: Mount the boot LV and ESP inside the ZFS copy
    shell: |
      mount /dev/pve/boot {{ newroot }}/boot
      mkdir -p {{ newroot }}/boot/efi
      mount --bind /boot/efi {{ newroot }}/boot/efi
    when: "! mountpoint -q {{ newroot }}/boot"

  # ⚠⚠ --make-rslave IS LOAD-BEARING. Without it this cost a production outage on
  # 2026-08-18.
  #
  # On a systemd host `/` has SHARED mount propagation, so `mount --rbind /dev`
  # creates a bind that shares propagation with the original. Every later
  # `umount -R` of the chroot copy then propagates BACK to the live system and
  # unmounts the REAL /sys/fs/cgroup, /dev/pts and /dev/shm. With cgroup2 gone,
  # systemd-logind cannot create a session: sshd still completes authentication
  # and already-resident daemons keep serving from memory, but every new exec
  # hangs forever. The host looks alive and is unusable, and — this is the part
  # that wasted the most time — it looks exactly like failing root-disk I/O.
  #
  # --make-rslave makes propagation one-way: host -> chroot only. Teardown then
  # cannot reach back.
  - name: Bind the kernel filesystems into the chroot (SLAVE propagation)
    shell: |
      for d in dev proc sys; do
        mountpoint -q {{ newroot }}/$d || mount --rbind /$d {{ newroot }}/$d
        mount --make-rslave {{ newroot }}/$d
      done
      echo "--- propagation (must NOT say shared) ---"
      findmnt -o TARGET,PROPAGATION {{ newroot }}/dev {{ newroot }}/sys {{ newroot }}/proc
    changed_when: "true"

  - name: GUARD — refuse to continue if any chroot bind is still shared
    shell: |
      if findmnt -no PROPAGATION -R {{ newroot }}/dev {{ newroot }}/sys {{ newroot }}/proc \
           2>/dev/null | grep -q shared; then
        echo "chroot binds are SHARED — teardown would unmount the live host's /sys and /dev"
        exit 1
      fi
      echo "all chroot binds are private/slave — teardown cannot propagate back"
    changed_when: "false"

  # ---------- build the boot artifacts inside the chroot ----------

  - name: Pin the default boot entry to the rollback, not to ZFS
    shell: |
      sed -i -e 's/^GRUB_DEFAULT=.*/GRUB_DEFAULT=saved/' \
             -e 's/^#\?GRUB_SAVEDEFAULT=.*/GRUB_SAVEDEFAULT=false/' \
             {{ newroot }}/etc/default/grub
      grep -q '^GRUB_DEFAULT=saved' {{ newroot }}/etc/default/grub
      grep -q '^GRUB_TIMEOUT=' {{ newroot }}/etc/default/grub || \
        echo 'GRUB_TIMEOUT=5' >> {{ newroot }}/etc/default/grub
    changed_when: "true"

  # ⚠ THE POOL-NAME BUG. Left to itself, grub-mkconfig emits
  #       root=ZFS=/ROOT/pve-1
  # with the pool name MISSING, which drops the boot at an initramfs prompt.
  #
  # Cause, and it is worth understanding because it is not a typo: Debian's
  # /etc/grub.d/10_linux builds the ZFS root as ${rpool}${bootfs}, where
  #   rpool  = grub-probe --device <dev> --target=fs_label
  #   bootfs = make_system_path_relative_to_its_root /   -> /ROOT/pve-1
  # and `grub-probe --target=fs /` on this pool fails outright with "unknown
  # filesystem" — GRUB's own ZFS reader cannot open a pool with `encryption`,
  # `large_dnode` and `zstd_compress` enabled. So rpool comes back EMPTY and
  # concatenates to nothing. It is the very same feature set that forced /boot
  # to stay ext4; here it silently corrupts the kernel command line instead of
  # erroring, which is why this is caught by a verify step and not by trust.
  #
  # A drop-in is used rather than editing /etc/default/grub so a future grub
  # package upgrade cannot revert it in a conffile merge.
  - name: Override the ZFS root on the kernel command line (grub cannot derive it)
    shell: |
      mkdir -p {{ newroot }}/etc/default/grub.d
      cat > {{ newroot }}/etc/default/grub.d/zfs-root.cfg <<'EOF'
      # grub-mkconfig cannot resolve this pool's name (GRUB's ZFS reader does not
      # support encryption/large_dnode/zstd_compress) and emits a pool-less
      # root=ZFS=/ROOT/pve-1. This appends the correct value AFTER it; the kernel
      # and the zfs initramfs script both take the LAST root= on the line.
      # The explicit `pve-zfs-root` menu entry in 40_custom carries a single
      # clean root= and is what cutover targets — this drop-in exists so the
      # auto-generated entries are correct too.
      GRUB_CMDLINE_LINUX="root=ZFS=nvme/ROOT/pve-1 boot=zfs"
      EOF
    changed_when: "true"

  # Both entries are hand-authored with STABLE ids. The auto-generated ones get
  # ids derived from device paths (`gnulinux-simple-/dev/nvme0n1p1_/dev/nvme1n1p1`)
  # which change if the pool's members ever change — not something to aim
  # `grub-reboot` at during a downtime window.
  - name: Author the explicit ZFS-root and ext4-rollback menu entries
    shell: |
      ROOT_UUID=$(blkid -s UUID -o value /dev/mapper/pve-root)
      BOOT_UUID=$(blkid -s UUID -o value /dev/mapper/pve-boot)
      KVER=$(basename $(ls -1 {{ newroot }}/boot/vmlinuz-* | sort -V | tail -1) | sed 's/^vmlinuz-//')
      test -n "$ROOT_UUID" && test -n "$BOOT_UUID" && test -n "$KVER"
      cat > {{ newroot }}/etc/grub.d/40_custom <<EOF
      #!/bin/sh
      exec tail -n +3 \$0
      # Target of the cutover grub-reboot. Kernel and initrd paths are relative
      # to the /boot LV (pve-boot), which is a filesystem in its own right now —
      # hence /vmlinuz-*, not /boot/vmlinuz-*. One clean root=, no duplicate.
      menuentry 'Proxmox VE - ZFS root (nvme/ROOT/pve-1)' --id pve-zfs-root {
          insmod part_gpt
          insmod lvm
          insmod ext2
          search --no-floppy --fs-uuid --set=root $BOOT_UUID
          echo 'Loading ZFS root (nvme/ROOT/pve-1) ...'
          linux /vmlinuz-$KVER root=ZFS=nvme/ROOT/pve-1 boot=zfs ro quiet intel_iommu=on
          initrd /initrd.img-$KVER
      }
      # Rollback path: boot the original ext4 root still present on the USB DOM.
      # Its /boot contents and initrd are never regenerated by this migration
      # (update-initramfs writes only to the new LV), so this entry is genuinely
      # independent of every ZFS artifact above it. Paths are /boot/* because on
      # that filesystem /boot is still an ordinary directory.
      menuentry 'Proxmox VE - ROLLBACK: ext4 root on the USB DOM' --id pve-ext4-rollback {
          insmod part_gpt
          insmod lvm
          insmod ext2
          search --no-floppy --fs-uuid --set=root $ROOT_UUID
          echo 'Loading ROLLBACK kernel (ext4 root on the DOM) ...'
          linux /boot/vmlinuz-$KVER root=/dev/mapper/pve-root ro quiet intel_iommu=on
          initrd /boot/initrd.img-$KVER
      }
      EOF
      chmod 755 {{ newroot }}/etc/grub.d/40_custom
    changed_when: "true"

  - name: Rebuild the initramfs with ZFS root support (writes to the new /boot LV only)
    shell: chroot {{ newroot }} update-initramfs -u -k all
    changed_when: "true"

  - name: Generate grub.cfg on the new /boot LV
    shell: chroot {{ newroot }} update-grub
    changed_when: "true"

  - name: Pin grubenv's saved default to the rollback entry
    shell: chroot {{ newroot }} grub-set-default pve-ext4-rollback
    changed_when: "true"

verify:
  # The load-bearing check. Not "does the right string appear somewhere" — that
  # passed happily while every entry was still pool-less. This walks EVERY
  # `linux` line, takes the LAST root= on it (what the kernel and the zfs
  # initramfs script actually honour), and demands it be one of the two known
  # good values. A pool-less root=ZFS=/ROOT/pve-1 surviving as the effective
  # root on any entry fails the run.
  - name: Every menu entry's EFFECTIVE root= is a known-good target
    shell: |
      awk '/^[[:space:]]*linux[[:space:]]/ {
             r="";
             for (i = 1; i <= NF; i++) if ($i ~ /^root=/) r = $i;
             if (r != "root=ZFS={{ root_dataset }}" && r != "root=/dev/mapper/pve-root") {
               print "BAD EFFECTIVE ROOT: " r " on: " $0; bad = 1
             }
           }
           END { exit bad ? 1 : 0 }' {{ newroot }}/boot/grub/grub.cfg
    changed_when: "false"

  - name: The explicit ZFS entry exists and carries exactly one clean root=
    shell: |
      grep -q "pve-zfs-root" {{ newroot }}/boot/grub/grub.cfg
      n=$(grep -A6 "pve-zfs-root" {{ newroot }}/boot/grub/grub.cfg \
            | grep -cE '^[[:space:]]*linux[[:space:]].*root=ZFS={{ root_dataset }}[[:space:]]')
      test "$n" -eq 1
      grep -A6 "pve-zfs-root" {{ newroot }}/boot/grub/grub.cfg \
        | grep -E '^[[:space:]]*linux[[:space:]]' | grep -vq 'ZFS=/ROOT'
    changed_when: "false"

  - name: The rollback entry is present and points at the ext4 root
    shell: |
      grep -q "id 'pve-ext4-rollback'" {{ newroot }}/boot/grub/grub.cfg || \
      grep -q "pve-ext4-rollback" {{ newroot }}/boot/grub/grub.cfg
      grep -q 'root=/dev/mapper/pve-root' {{ newroot }}/boot/grub/grub.cfg
    changed_when: "false"

  - name: grub.cfg honours the one-shot next_entry mechanism
    shell: grep -q 'next_entry' {{ newroot }}/boot/grub/grub.cfg
    changed_when: "false"

  - name: grubenv default is the rollback entry
    shell: grep -q 'saved_entry=pve-ext4-rollback' {{ newroot }}/boot/grub/grubenv
    changed_when: "false"

  - name: The new initramfs actually contains the ZFS modules
    shell: |
      KVER=$(basename $(ls -1 {{ newroot }}/boot/vmlinuz-* | sort -V | tail -1) | sed 's/^vmlinuz-//')
      lsinitramfs {{ newroot }}/boot/initrd.img-$KVER | grep -qE 'zfs|zpool.cache'
    changed_when: "false"

  - name: The new initramfs carries the three-pool zpool.cache
    shell: |
      KVER=$(basename $(ls -1 {{ newroot }}/boot/vmlinuz-* | sort -V | tail -1) | sed 's/^vmlinuz-//')
      lsinitramfs {{ newroot }}/boot/initrd.img-$KVER | grep -q 'zpool.cache'
    changed_when: "false"

  - name: The ext4 rollback root still has its own untouched kernel and initrd
    shell: ls /boot/vmlinuz-* /boot/initrd.img-* >/dev/null
    changed_when: "false"

  - name: ESP is still the ORIGINAL stub pointing at the ext4 root (no grub-install yet)
    shell: |
      grep -q "$(blkid -s UUID -o value /dev/mapper/pve-root)" \
        {{ newroot }}/boot/efi/EFI/proxmox/grub.cfg
    changed_when: "false"

  - name: Show the cutover command and every entry's effective root
    shell: |
      echo "--- cutover one-shot:  chroot {{ newroot }} grub-reboot pve-zfs-root ---"
      echo "--- effective root= per menu entry ---"
      awk '/^[[:space:]]*menuentry/ { t = $0; sub(/^[[:space:]]*menuentry[[:space:]]*/, "", t) }
           /^[[:space:]]*linux[[:space:]]/ {
             r = "";
             for (i = 1; i <= NF; i++) if ($i ~ /^root=/) r = $i;
             printf "  %-46.46s -> %s\n", substr(t, 1, 46), r
           }' {{ newroot }}/boot/grub/grub.cfg
    changed_when: "false"