esh-pfi-infrastructure/playbooks/esh-pve-nas-stage-zfs-root.yaml

# esh-pve-nas — STAGE the PVE root migration off the USB DOM onto ZFS.
#
# Runbook: docs/runbooks/esh-pve-nas-boot-migration.md
# Design:  boot chain stays ext4 on the DOM; root moves to nvme/ROOT/pve-1.
#
# THIS PLAYBOOK DOES NOT CUT OVER. It leaves the host still running from the
# ext4 root on the DOM. Nothing here changes what the next reboot does — the
# bootloader phase is deliberately a separate playbook.
#
# What it does, all live, no downtime:
#   1. Reclaims 512 MB from the 768 MB swap LV for a dedicated /boot LV
#      (operator's call 2026-08-17: shrink swap to 256 MB rather than drop it).
#   2. Populates that LV from the current /boot.
#   3. rsyncs the live ext4 root into the ZFS dataset nvme/ROOT/pve-1.
#   4. Writes the ZFS copy's /etc/fstab for the post-cutover layout.
#
# The ext4 root LV is never modified — it stays byte-intact as the rollback,
# including its own /boot contents, which the new mount only shadows.
#
# Preconditions (verified 2026-08-17, re-asserted as guard steps below):
#   - nvme/ROOT/pve-1 exists, canmount=noauto, encryption off
#   - /etc/zfs/zpool.cache populated with ALL THREE pools (nvme, ssd, tank).
#     ⚠ A cache holding only `nvme` flips the host from import-by-scan to
#     import-by-cache and leaves ssd+tank unimported at boot — which breaks
#     CT 103 `esh-nas`, whose 12 bind mounts span all three pools.
#
# Rerunnable: every step is guarded, so a second run reports ok/skipped.

vars:
  newroot: /mnt/newroot
  bootstage: /mnt/boot-new
  boot_lv_size: 512M
  swap_lv_size: 256M
  root_dataset: nvme/ROOT/pve-1

steps:
  # ---------- guards: refuse to run against an already-migrated or unprepared host ----------

  - name: GUARD — host must still be running from the ext4 root on the DOM
    shell: |
      test "$(findmnt -no FSTYPE /)" = "ext4" || {
        echo "root is not ext4 — host already cut over; refusing"; exit 1; }
    changed_when: "false"

  - name: GUARD — ZFS root dataset must exist with canmount=noauto
    shell: |
      test "$(zfs get -H -o value canmount {{ root_dataset }})" = "noauto" || {
        echo "{{ root_dataset }} missing or canmount!=noauto; refusing"; exit 1; }
    changed_when: "false"

  - name: GUARD — zpool.cache must list all three pools
    shell: |
      for p in nvme ssd tank; do
        zdb -C -U /etc/zfs/zpool.cache 2>/dev/null | grep -q "name: '$p'" || {
          echo "pool $p missing from zpool.cache — would not import at boot"; exit 1; }
      done
    changed_when: "false"

  # ---------- phase 1: carve a /boot LV out of swap ----------

  # Gated on the ORIGINAL 768M size, not on "is swap on" — otherwise a rerun
  # swaps off the new 256M device and never turns it back on.
  - name: Disable swap so its LV can be resized
    shell: swapoff /dev/pve/swap
    when: "lvs --noheadings -o lv_size --units m pve/swap 2>/dev/null | grep -q '768'"

  - name: Remove the oversized swap LV
    shell: lvremove -y pve/swap
    when: "lvs --noheadings -o lv_size --units m pve/swap 2>/dev/null | grep -q '768'"

  - name: Create the dedicated /boot LV
    shell: lvcreate -y -L {{ boot_lv_size }} -n boot pve
    when: "! lvs pve/boot >/dev/null 2>&1"

  - name: Recreate swap at the reduced size
    shell: lvcreate -y -L {{ swap_lv_size }} -n swap pve
    when: "! lvs pve/swap >/dev/null 2>&1"

  - name: Make the /boot filesystem
    shell: mkfs.ext4 -q -L pveboot /dev/pve/boot
    when: "! blkid -s TYPE -o value /dev/pve/boot 2>/dev/null | grep -q ext4"

  - name: Make and enable the new swap
    shell: |
      blkid -s TYPE -o value /dev/pve/swap 2>/dev/null | grep -q swap || mkswap -L pveswap /dev/pve/swap
      swapon /dev/pve/swap
    when: "! swapon --show=NAME --noheadings | grep -q dm-"

  # ---------- phase 2: populate the /boot LV ----------

  - name: Stage-mount the new /boot LV
    shell: mkdir -p {{ bootstage }} && mount /dev/pve/boot {{ bootstage }}
    when: "! mountpoint -q {{ bootstage }}"

  - name: Copy the current /boot into it (ESP contents excluded — separate vfat mount)
    shell: |
      rsync -aHAX --numeric-ids --one-file-system --delete \
        --exclude='/lost+found' \
        /boot/ {{ bootstage }}/
      mkdir -p {{ bootstage }}/efi
    changed_when: "true"

  - name: Verify the kernel and initrd landed
    shell: |
      ls {{ bootstage }}/vmlinuz-* {{ bootstage }}/initrd.img-* >/dev/null
      test -f {{ bootstage }}/grub/grub.cfg
    changed_when: "false"

  # ---------- phase 3: rsync the live root into the ZFS dataset ----------

  - name: Point the ZFS root dataset at a staging mountpoint
    shell: zfs set mountpoint={{ newroot }} {{ root_dataset }}
    when: "test \"$(zfs get -H -o value mountpoint {{ root_dataset }})\" != '{{ newroot }}'"

  - name: Mount the ZFS root dataset for staging
    shell: zfs mount {{ root_dataset }}
    when: "! mountpoint -q {{ newroot }}"

  - name: rsync the ext4 root into ZFS (one-file-system — every other mount is excluded)
    shell: |
      rsync -aHAX --numeric-ids --one-file-system --delete \
        --exclude='/proc/*' --exclude='/sys/*' --exclude='/dev/*' \
        --exclude='/run/*'  --exclude='/tmp/*'  --exclude='/mnt/*' \
        --exclude='/media/*' \
        / {{ newroot }}/
      # mountpoints that --one-file-system skipped still need to exist
      mkdir -p {{ newroot }}/proc {{ newroot }}/sys {{ newroot }}/dev \
               {{ newroot }}/run  {{ newroot }}/tmp {{ newroot }}/mnt \
               {{ newroot }}/boot {{ newroot }}/boot/efi \
               {{ newroot }}/nvme {{ newroot }}/ssd {{ newroot }}/tank \
               {{ newroot }}/var/log/journal
      chmod 1777 {{ newroot }}/tmp
    changed_when: "true"

  # ---------- phase 4: fstab for the post-cutover layout ----------

  - name: Write the ZFS copy's /etc/fstab
    shell: |
      cat > {{ newroot }}/etc/fstab <<'FSTAB'
      # <file system> <mount point> <type> <options> <dump> <pass>
      # root is {{ root_dataset }} (ZFS) — mounted by the initramfs, no entry here.
      /dev/pve/boot /boot ext4 defaults 0 2
      UUID=1D32-43A5 /boot/efi vfat defaults 0 2
      /dev/pve/swap none swap sw 0 0
      proc /proc proc defaults 0 0
      FSTAB
    changed_when: "true"

verify:
  - name: LVM layout is root + boot + swap
    shell: lvs --noheadings -o lv_name pve | tr -d ' ' | sort | tr '\n' ',' | grep -qx 'boot,root,swap,'
    changed_when: "false"

  - name: Live root is still the untouched ext4 LV
    shell: test "$(findmnt -no SOURCE /)" = "/dev/mapper/pve-root"
    changed_when: "false"

  - name: Swap is active at the reduced size
    shell: swapon --show=NAME --noheadings | grep -q dm-
    changed_when: "false"

  - name: New /boot LV carries a bootable kernel set
    shell: ls {{ bootstage }}/vmlinuz-* {{ bootstage }}/initrd.img-* >/dev/null
    changed_when: "false"

  - name: ZFS root copy has a populated /usr and /etc
    shell: test -x {{ newroot }}/usr/bin/pveversion && test -f {{ newroot }}/etc/fstab
    changed_when: "false"

  - name: ZFS root copy's fstab has no root line and does have the boot line
    shell: |
      ! grep -qE '^\S+\s+/\s+' {{ newroot }}/etc/fstab
      grep -q '/dev/pve/boot /boot ext4' {{ newroot }}/etc/fstab
    changed_when: "false"

  - name: PVE cluster config copied (guest configs present)
    shell: test -d {{ newroot }}/var/lib/pve-cluster
    changed_when: "false"