esh-pfi-infrastructure/playbooks/esh-cutover-3-esh-pve-nas.yaml

# esh-pve-nas cutover, step 3 of 5 — point the ESP at the new /boot and reboot.
#
# Run:  scripts/elway root@esh-pve-nas --playbook playbooks/esh-cutover-3-esh-pve-nas.yaml
#
# PRECONDITION: steps 1 and 2 must have run. Both NFS clients hold `hard` mounts
# from CT 103 which lives on this host; taking it down with them mounted wedges
# esh-docker-vm in unkillable D-state. A guard below refuses to proceed if either
# client is still mounted.
#
# ⚠ ORDERING TRAP, and it is the reason this is a playbook and not four commands:
# `zfs set mountpoint=/` on a dataset that is CURRENTLY MOUNTED makes ZFS unmount
# and REMOUNT it at the new location — i.e. it would try to mount the ZFS root
# over the live ext4 root of a running hypervisor. canmount=noauto does not save
# you; that governs automatic mounting at import, not an explicit property change
# on a mounted dataset. The dataset must be UNMOUNTED first, which means the
# chroot binds have to come down first, which means grub-install and grub-reboot
# have to happen BEFORE any of that. Hence the sequence below is not negotiable.
#
# This playbook ENDS BY REBOOTING THE HOST. elway will lose the connection; that
# is expected, not a failure.

vars:
  newroot: /mnt/newroot
  root_dataset: nvme/ROOT/pve-1
  quiesced: "no"          # caller MUST pass --var quiesced=yes after verifying both clients

steps:
  # ---------- guards ----------

  - name: GUARD — still on the ext4 root (not already cut over)
    shell: |
      test "$(findmnt -no FSTYPE /)" = "ext4" || { echo "already on ZFS; refusing"; exit 1; }
    changed_when: "false"

  # This host has no ssh keys to the NFS clients, so the caller verifies their
  # mount tables and attests via --var quiesced=yes.
  #
  # ⚠ THE RUNBOOK'S BLAST RADIUS WAS WRONG. It named two dependents. `ss` on CT 103
  # showed FIVE distinct clients on 2026-08-18:
  #   10.0.50.45   esh-docker-vm   hard  -> quiesced by step 1
  #   10.0.250.35  esh-pve         hard  -> quiesced by step 2
  #   10.0.50.60   esh-vm-db       hard  -> DELIBERATELY LEFT MOUNTED (see below)
  #   10.0.50.154  vm-esh-nas      n/a   -> is VM 104 on THIS host; dies with it
  #   10.100.10.50 nh3-dev         soft,ro -> errors instead of blocking; safe
  #
  # esh-vm-db is left mounted on purpose. It is a backup TARGET with no live user:
  # resticprofile-backup and postgresql-dump next fire ~19h out, and a hard mount
  # with nothing actively using it blocks and then resumes when the server returns
  # — that is what `hard` is for. Unmounting it would mean an unmount/remount cycle
  # over the qemu guest agent on a host with no ssh access, where a failed remount
  # breaks backups silently. Leaving it is the lower-risk branch, not the lazy one.
  # The gate is `quiesced`, which the CALLER sets only after checking each client's
  # mount table directly (this host has no ssh to them; see step 1/2 playbooks).
  #
  # ⚠ It deliberately does NOT gate on server-side NFS session count. Measured
  # 2026-08-18: esh-docker-vm's sessions drained within ~90s, but esh-pve held 11
  # established connections to :2049 indefinitely with NO mounts in either
  # `findmnt` or `/proc/mounts` and nothing holding a cwd there. That is the Linux
  # NFSv4 client keeping its transport alive past the last unmount, and it is the
  # wrong thing to gate on: the failure this whole runbook exists to prevent is a
  # process blocking on a MOUNTED hard filesystem when the server vanishes. With no
  # mount there is nothing to block on — an idle socket to a departing server just
  # resets. Gating on sessions would have stalled the window forever on a condition
  # that never clears and never mattered.
  - name: GUARD — caller has confirmed both hard-NFS clients are unmounted
    shell: |
      test "{{ quiesced }}" = "yes" || {
        echo "run playbooks 1 and 2 and confirm client mount tables first"; exit 1; }
      echo "caller attests: esh-docker-vm and esh-pve carry no esh-nas mounts"
      echo "--- server-side sessions, informational only ---"
      pct exec 103 -- ss -tnH state established '( sport = :2049 )' 2>/dev/null \
        | awk '{print $4}' | sed 's/:[0-9]*$//' | sort | uniq -c || true
    changed_when: "false"

  - name: GUARD — staging artifacts are all present
    shell: |
      mountpoint -q {{ newroot }}      || { echo "{{ newroot }} not mounted";     exit 1; }
      mountpoint -q {{ newroot }}/boot || { echo "boot LV not in the chroot";     exit 1; }
      grep -q pve-zfs-root {{ newroot }}/boot/grub/grub.cfg || { echo "no ZFS entry"; exit 1; }
      grep -q 'saved_entry=pve-ext4-rollback' {{ newroot }}/boot/grub/grubenv     || { echo "grubenv not pinned to rollback"; exit 1; }
    changed_when: "false"

  # ---------- stop the guests, NAS last ----------

  - name: Stop the guests (reverse of startup order — CT 103, the NAS, goes last)
    shell: |
      for v in 105 106 107; do pct status $v 2>/dev/null | grep -q running && pct shutdown $v --timeout 90 || true; done
      qm status 104 2>/dev/null | grep -q running && qm shutdown 104 --timeout 90 || true
      for i in $(seq 1 30); do
        running=$( (pct list | awk 'NR>1 && $2=="running"'; qm list | awk 'NR>1 && $3=="running"') | wc -l )
        [ "$running" -le 1 ] && break
        sleep 3
      done
      pct status 103 2>/dev/null | grep -q running && pct shutdown 103 --timeout 90 || true
      sleep 3
      echo "--- remaining ---"; pct list; qm list
    changed_when: "true"

  # ---------- the actual cutover ----------

  - name: Point the ESP at the new /boot LV
    shell: |
      chroot {{ newroot }} grub-install --target=x86_64-efi \
             --efi-directory=/boot/efi --bootloader-id=proxmox
    changed_when: "true"

  - name: Verify the ESP stub now points at the /boot LV, not the ext4 root
    shell: |
      BOOT_UUID=$(blkid -s UUID -o value /dev/mapper/pve-boot)
      grep -q "$BOOT_UUID" {{ newroot }}/boot/efi/EFI/proxmox/grub.cfg || {
        echo "ESP stub does NOT reference the boot LV — aborting before reboot"; exit 1; }
      echo "ESP stub -> boot LV $BOOT_UUID"
    changed_when: "false"

  - name: Arm the ONE-SHOT ZFS boot (default stays pinned to the ext4 rollback)
    shell: |
      chroot {{ newroot }} grub-reboot pve-zfs-root
      grep -o 'next_entry=.*'  {{ newroot }}/boot/grub/grubenv
      grep -o 'saved_entry=.*' {{ newroot }}/boot/grub/grubenv
    changed_when: "true"

  # ---------- tear the chroot down so the dataset can be unmounted ----------

  - name: Unmount the chroot, innermost first
    shell: |
      for m in proc/sys/fs/binfmt_misc proc sys dev/pts dev/shm dev/mqueue dev/hugepages dev boot/efi boot; do
        mountpoint -q {{ newroot }}/$m && umount -R {{ newroot }}/$m 2>/dev/null || true
      done
      findmnt -R {{ newroot }} -o TARGET | tail -n +2 || echo "  (nothing left under {{ newroot }})"
    changed_when: "true"

  - name: Unmount the ZFS root dataset BEFORE changing its mountpoint
    shell: zfs unmount {{ root_dataset }}
    when: "mountpoint -q {{ newroot }}"

  - name: Set the dataset's final mountpoint (safe only now that it is unmounted)
    shell: |
      zfs set mountpoint=/ {{ root_dataset }}
      zfs get -H -o value mountpoint,canmount {{ root_dataset }} | tr '\n' ' '; echo
      # paranoia: the live root must STILL be the ext4 LV at this instant
      test "$(findmnt -no SOURCE /)" = "/dev/mapper/pve-root" || {
        echo "ZFS MOUNTED OVER THE LIVE ROOT — do not reboot, investigate"; exit 1; }
    changed_when: "true"

  - name: Final pre-reboot assertion
    shell: |
      echo "root now:      $(findmnt -no SOURCE,FSTYPE /)"
      echo "dataset:       $(zfs get -H -o value mounted {{ root_dataset }}) mounted, canmount=$(zfs get -H -o value canmount {{ root_dataset }})"
      echo "next_entry:    $(grep -o 'next_entry=.*' {{ newroot }}/boot/grub/grubenv 2>/dev/null || echo '(grubenv not readable — boot LV is unmounted, expected)')"
    changed_when: "false"

  - name: REBOOT — connection loss here is expected
    shell: systemd-run --on-active=3 --timer-property=AccuracySec=1s /sbin/reboot
    changed_when: "true"