From 21d9a07bc335b2ba423377f8b4cec82df84763f5 Mon Sep 17 00:00:00 2001
From: Vuong Hoang <vh@phasefinal.com>
Date: Tue, 14 Jul 2026 21:53:33 -0700
Subject: [PATCH] fix(esh-nfs): order docker after the NFS mount units directly
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The prior fix (_netdev,nofail + docker.service After=remote-fs.target) looked
correct but silently failed — paperless still Exited(255) on the 2026-07-14
reboot. Root cause: `nofail` drops a mount out of remote-fs.target's blocking
set, so ordering docker After=remote-fs.target does NOT wait for the nofail
NFS mounts. Fix: add x-systemd.before=docker.service,x-systemd.mount-timeout=30
to the 4 NFS fstab lines (direct mount->docker ordering, nofail-safe). Applied
+ verified live (systemctl show docker -p After now lists all 4 mnt-*.mount).
Playbook + verify updated to canonicalize.
---
 playbooks/fix-esh-nfs-boot-ordering.yaml | 40 +++++++++++++++++++++---
 1 file changed, 35 insertions(+), 5 deletions(-)

diff --git a/playbooks/fix-esh-nfs-boot-ordering.yaml b/playbooks/fix-esh-nfs-boot-ordering.yaml
index 077bc53..6d4384b 100644
--- a/playbooks/fix-esh-nfs-boot-ordering.yaml
+++ b/playbooks/fix-esh-nfs-boot-ordering.yaml
@@ -12,8 +12,20 @@
 #   - fstab: defaults -> defaults,_netdev,nofail  (keeps `hard`)
 #       _netdev : order mount after network-online.target
 #       nofail  : NAS-down at boot doesn't wedge boot / kill DNS
-#   - docker.service drop-in: After=remote-fs.target so Docker starts
-#     after the NFS mounts have completed.
+#   - fstab: + x-systemd.before=docker.service,x-systemd.mount-timeout=30
+#       Puts Before=docker.service directly on each generated .mount unit
+#       so Docker waits for the ACTUAL mounts; mount-timeout bounds the
+#       wait if the NAS is down at boot.
+#   - docker.service drop-in: After=remote-fs.target (kept as a weaker
+#     belt-and-suspenders layer).
+#
+# WHY the drop-in alone was NOT enough (2026-07-14 reboot): `nofail`
+# removes a mount from remote-fs.target's blocking set, so ordering
+# Docker `After=remote-fs.target` does not actually wait for the nofail
+# NFS mounts -> paperless still lost the race and Exited(255) on reboot.
+# The load-bearing fix is the DIRECT mount->docker ordering from the
+# fstab `x-systemd.before` option. Verify with:
+#   systemctl show docker -p After | tr ' ' '\n' | grep mnt-   # lists all 4
 #
 # Idempotent: re-runs show ok/skipped. Does NOT reboot — the real test
 # is the next reboot, run that separately.
@@ -37,6 +49,16 @@ steps:
     # Run only if at least one unfixed NFS line remains.
     when: "grep -qE '^10\\.0\\.50\\.50:.* nfs defaults ' /etc/fstab"
 
+  - name: Order each NFS mount before docker.service (direct dep; nofail-safe)
+    # THE load-bearing fix. remote-fs.target ordering (below) is defeated
+    # by `nofail` (the mount drops out of that target's blocking set).
+    # x-systemd.before=docker.service injects Before=docker.service onto
+    # each generated .mount unit, so Docker genuinely waits for the mounts.
+    shell: sed -i -E '/^10\.0\.50\.50:/{/x-systemd.before/!s/(_netdev,nofail)/\1,x-systemd.before=docker.service,x-systemd.mount-timeout=30/}' /etc/fstab
+    sudo: true
+    # Run only if an NFS line with _netdev,nofail still lacks the ordering.
+    when: "grep -E '^10\\.0\\.50\\.50:.*_netdev,nofail' /etc/fstab | grep -qv x-systemd.before"
+
   - name: Install docker.service drop-in to order after remote-fs.target
     # Use a DISTINCT filename — esh-docker-vm already ships an
     # override.conf (dockerd ExecStart/containerd socket); systemd merges
@@ -57,14 +79,22 @@ steps:
     changed_when: "false"
 
 verify:
-  - name: All 4 NFS lines now carry _netdev,nofail
-    shell: test "$(grep -cE '^10\.0\.50\.50:.* nfs defaults,_netdev,nofail ' /etc/fstab)" -eq 4
+  - name: All 4 NFS lines carry _netdev,nofail
+    shell: test "$(grep -cE '^10\.0\.50\.50:.*nfs defaults,_netdev,nofail' /etc/fstab)" -eq 4
+    changed_when: "false"
+
+  - name: All 4 NFS lines carry x-systemd.before=docker.service
+    shell: test "$(grep -cE '^10\.0\.50\.50:.*x-systemd.before=docker.service' /etc/fstab)" -eq 4
     changed_when: "false"
 
   - name: fstab parses cleanly (findmnt --verify, no fatal errors)
     shell: findmnt --verify >/dev/null
     changed_when: "false"
 
-  - name: Docker is ordered after remote-fs.target
+  - name: Docker is ordered after the actual NFS mount units (the real fix)
+    shell: systemctl show docker -p After | tr ' ' '\n' | grep -q '^mnt-documents.mount$'
+    changed_when: "false"
+
+  - name: Docker is also ordered after remote-fs.target (belt-and-suspenders)
     shell: systemctl show docker -p After | grep -q remote-fs.target
     changed_when: "false"