From 1cbde502f494a37d68c8a8dffc2b41ae3b3b07f4 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Thu, 24 Sep 2026 13:16:18 -0700 Subject: [PATCH] fix(nh3): NFS clients mount nh3-nas shares on first access; record guest power-loss expectations After the 2026-09-24 NH3 power outage every NFS line from nh3-nas failed at boot: nh3-docker's /mnt/compose and /mnt/backup, and nh3-dev's /mnt/backup. The NAS is the slowest box to serve NFS, and the plain fstab mounts stayed down until remounted by hand. nh3-dev's /mnt/smithy already used x-systemd.automount, and the same boot shows it self-healing on its next access. playbooks/nh3-nfs-automount.yaml brings the other lines to that shape (_netdev,nofail,x-systemd.automount,x-systemd.mount-timeout=30). Hard-mount semantics are unchanged; only WHEN the mount happens moves. Applied and read back on nh3-docker and nh3-dev: every nh3-nas line carries automount, each automount unit is active, and each share is NFS-mounted after access. A second run on nh3-docker is a no-op. The fstab check judges only errors the rewrite ADDS, because nh3-dev's installer cdrom line has a pre-existing verify error. The first run caught that and aborted without writing. servers/nh3-pve/README.md now lists each guest's expected state after a power loss: nh3-laser is on-demand and stays off (Prime), pbs-nh3 now has onboot=1 (it had none and stayed down), and pfi-gx10 is bare metal that does not auto-power-on. --- playbooks/nh3-nfs-automount.yaml | 73 ++++++++++++++++++++++++++++++++ servers/nh3-pve/README.md | 25 ++++++++++- 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 playbooks/nh3-nfs-automount.yaml diff --git a/playbooks/nh3-nfs-automount.yaml b/playbooks/nh3-nfs-automount.yaml new file mode 100644 index 0000000..e0564bd --- /dev/null +++ b/playbooks/nh3-nfs-automount.yaml @@ -0,0 +1,73 @@ +# NH3 NFS clients: mount nh3-nas shares on first access, not at boot. +# +# Why (2026-09-24 NH3 power outage): after a site-wide power loss every box +# boots at once and nh3-nas is the slowest to serve NFS. Plain fstab NFS lines +# failed at boot on nh3-docker (/mnt/compose, /mnt/backup) and nh3-dev +# (/mnt/backup) and stayed down until someone remounted them by hand. +# nh3-dev's /mnt/smithy already had x-systemd.automount, and the same boot +# shows it self-healing: its first access at 1234 failed, and the next access +# at 1309 mounted it with no intervention. This brings the other lines to that +# shape. Hard-mount semantics are unchanged; only WHEN the mount happens moves. +# +# scripts/elway infra-ops@10.100.50.40 --playbook playbooks/nh3-nfs-automount.yaml +# scripts/elway infra-ops@10.100.10.50 --playbook playbooks/nh3-nfs-automount.yaml +# +# Rerunnable: lines that already carry x-systemd.automount are left alone, and +# the activate steps skip a path whose automount is already active. + +steps: + - name: Back up fstab + sudo: true + shell: cp -p /etc/fstab /etc/fstab.bak-20260924-automount + creates: /etc/fstab.bak-20260924-automount + + - name: Add automount options to nh3-nas NFS lines that lack them + sudo: true + shell: | + set -eu + awk -v OFS='\t' '$1 ~ /^10\.100\.50\.50:/ && $3 ~ /^nfs/ && $4 !~ /x-systemd\.automount/ { + $4 = $4 ",_netdev,nofail,x-systemd.automount,x-systemd.mount-timeout=30" } { print }' \ + /etc/fstab > /etc/fstab.automount-new + # Judge the rewrite by what it ADDS: nh3-dev's installer cdrom line + # (udf,iso9660 vs an attached iso9660) is a pre-existing verify error + # that has nothing to do with NFS, so "zero errors" would refuse forever. + errs() { findmnt --verify "$@" 2>&1 | sed -nE 's/.*[^0-9]([0-9]+) errors?,.*/\1/p' | tail -1; } + before=$(errs); after=$(errs --tab-file /etc/fstab.automount-new) + if [ "${after:-99}" -gt "${before:-0}" ]; then + rm /etc/fstab.automount-new; echo "rewrite adds fstab errors ($before -> $after); fstab unchanged" >&2; exit 1 + fi + cat /etc/fstab.automount-new > /etc/fstab + rm /etc/fstab.automount-new + systemctl daemon-reload + when: "awk '$1 ~ /^10\\.100\\.50\\.50:/ && $3 ~ /^nfs/ && $4 !~ /x-systemd\\.automount/' /etc/fstab | grep -q ." + + # Activate now rather than at next boot: a path that is already a plain + # mount cannot take an automount on top, so unmount (fails safely if busy), + # start the automount, then touch the path so it mounts through autofs. + - name: Switch /mnt/compose to automount + sudo: true + shell: umount /mnt/compose && systemctl start mnt-compose.automount && ls /mnt/compose >/dev/null + when: "grep -qE '\\s/mnt/compose\\s.*x-systemd.automount' /etc/fstab && ! systemctl is-active -q mnt-compose.automount" + + - name: Switch /mnt/backup to automount + sudo: true + shell: umount /mnt/backup && systemctl start mnt-backup.automount && ls /mnt/backup >/dev/null + when: "grep -qE '\\s/mnt/backup\\s.*x-systemd.automount' /etc/fstab && ! systemctl is-active -q mnt-backup.automount" + +verify: + - name: Every nh3-nas NFS line carries x-systemd.automount + shell: "! awk '$1 ~ /^10\\.100\\.50\\.50:/ && $3 ~ /^nfs/ && $4 !~ /x-systemd\\.automount/' /etc/fstab | grep -q ." + + - name: fstab has no parse errors + sudo: true + shell: findmnt --verify 2>&1 | grep -qE '(^|[^0-9])0 parse errors' + + - name: Each automount is active and its share is mounted after access + shell: | + set -e + for m in $(awk '$1 ~ /^10\.100\.50\.50:/ && $3 ~ /^nfs/ {print $2}' /etc/fstab); do + u=$(systemd-escape -p --suffix=automount "$m") + systemctl is-active -q "$u" + ls "$m" >/dev/null + findmnt -n -t nfs,nfs4 "$m" >/dev/null + done diff --git a/servers/nh3-pve/README.md b/servers/nh3-pve/README.md index 355b2f3..f99894e 100644 --- a/servers/nh3-pve/README.md +++ b/servers/nh3-pve/README.md @@ -18,7 +18,30 @@ Proxmox VE hypervisor for the NH3 site (`nh3-vmhost.phasefinal.com`). ## What it runs -`nh3-docker` (`10.100.50.40`) is a VM on this hypervisor. Other VMs (if any) aren't catalogued yet — run `qm list` on the host when you need the current inventory. +Guests as of 2026-09-24 (`qm list` / `pct list` is the live truth). **Expected state +after a power loss** is the column that matters in a recovery: a guest listed as +`stays off` is not a casualty. + +| ID | guest | kind | onboot | expected after power loss | +|---|---|---|---|---| +| 100 | nh3-docker (`10.100.50.40`) | VM | 1 | up | +| 101 | nh3-extdev (`10.100.50.42`) | VM | 1 | up | +| 102 | nh3-dev (`10.100.10.50`) | VM | 1 | up | +| 104 | nh3-laser | VM | — | **stays off — on-demand** (Prime, 2026-09-24) | +| 105 | pbs-nh3 (`10.100.50.90`) | VM | 1 | up (onboot added 2026-09-24; it had none and stayed down after the outage) | +| 108 | opnsense-lab | VM | 0 | stays off (lab) | +| 103 | nh3-wg | CT | 1 | up | +| 106 | nh3-headscale | CT | 1 | up | +| 107 | nh3-scale | CT | 1 | up (mesh subnet router + fleet egress proxy) | + +**Power-loss recovery (2026-09-24 outage).** Every guest boots at once, and +nh3-nas is the slowest to serve NFS. NFS clients now mount nh3-nas shares on +first access (`playbooks/nh3-nfs-automount.yaml`, applied to nh3-docker and +nh3-dev), so a late NAS no longer leaves shares down. pbs-nh3's datastore line +uses `bg` (background retry) instead. `qm guest exec 105` works (the guest agent is +on), which is the root path into pbs-nh3; infra-ops is not provisioned there. +**pfi-gx10 (`10.100.50.60`) is bare metal on this site, not a guest here**, and did +not power back on by itself. ## Refresh state