diff --git a/playbooks/fix-esh-nfs-boot-ordering.yaml b/playbooks/fix-esh-nfs-boot-ordering.yaml new file mode 100644 index 0000000..e5b8530 --- /dev/null +++ b/playbooks/fix-esh-nfs-boot-ordering.yaml @@ -0,0 +1,65 @@ +# Harden NFS boot ordering on esh-docker-vm so Docker waits for the +# 10.0.50.50 NFS mounts before starting containers. +# +# Root cause (2026-05-30 incident): /etc/fstab NFS lines used `defaults` +# (no `_netdev`), so the system attempted them too early and Docker +# started NFS-bind-mount containers (paperless) before /mnt/documents +# was ready -> paperless Exited(255) on every reboot, needing a manual +# `docker start`. The same `hard` mounts also froze the celery worker in +# unkillable D-state when the NAS stalled at runtime. +# +# This playbook fixes the BOOT race only: +# - fstab: defaults -> defaults,_netdev,nofail (keeps `hard`) +# _netdev : order mount after network-online.target +# nofail : NAS-down at boot doesn't wedge boot / kill DNS +# - docker.service drop-in: After=remote-fs.target so Docker starts +# after the NFS mounts have completed. +# +# Idempotent: re-runs show ok/skipped. Does NOT reboot — the real test +# is the next reboot, run that separately. +# +# scripts/elway esh-docker-vm --playbook playbooks/fix-esh-nfs-boot-ordering.yaml + +vars: + fstab_backup: /etc/fstab.bak-20260530-nfs-boot + +steps: + - name: Back up /etc/fstab (once) + shell: cp -n /etc/fstab {{ fstab_backup }} + sudo: true + creates: "{{ fstab_backup }}" + + - name: Add _netdev,nofail to the 10.0.50.50 NFS mounts + # Match active (non-#) lines with ` nfs defaults ` and not already + # carrying _netdev; rewrite the options field in place. + shell: sed -i -E '/^10\.0\.50\.50:.* nfs defaults /{/_netdev/!s/ nfs defaults / nfs defaults,_netdev,nofail /}' /etc/fstab + sudo: true + # Run only if at least one unfixed NFS line remains. + when: "grep -qE '^10\\.0\\.50\\.50:.* nfs defaults ' /etc/fstab" + + - name: Install docker.service drop-in to order after remote-fs.target + shell: | + install -d -m 0755 /etc/systemd/system/docker.service.d + printf '[Unit]\nAfter=remote-fs.target\nWants=remote-fs.target\n' \ + > /etc/systemd/system/docker.service.d/override.conf + sudo: true + creates: /etc/systemd/system/docker.service.d/override.conf + + - name: Reload systemd so the drop-in takes effect next boot + shell: systemctl daemon-reload + sudo: true + # A reload mutates nothing observable on its own; report as ok. + changed_when: "false" + +verify: + - name: All 4 NFS lines now carry _netdev,nofail + shell: test "$(grep -cE '^10\.0\.50\.50:.* nfs defaults,_netdev,nofail ' /etc/fstab)" -eq 4 + changed_when: "false" + + - name: fstab parses cleanly (findmnt --verify, no fatal errors) + shell: findmnt --verify >/dev/null + changed_when: "false" + + - name: Docker is ordered after remote-fs.target + shell: systemctl show docker -p After | grep -q remote-fs.target + changed_when: "false"