esh-pfi-infrastructure/playbooks/fix-esh-nfs-boot-ordering.yaml

# Harden NFS boot ordering on esh-docker-vm so Docker waits for the
# 10.0.50.50 NFS mounts before starting containers.
#
# Root cause (2026-05-30 incident): /etc/fstab NFS lines used `defaults`
# (no `_netdev`), so the system attempted them too early and Docker
# started NFS-bind-mount containers (paperless) before /mnt/documents
# was ready -> paperless Exited(255) on every reboot, needing a manual
# `docker start`. The same `hard` mounts also froze the celery worker in
# unkillable D-state when the NAS stalled at runtime.
#
# This playbook fixes the BOOT race only:
#   - fstab: defaults -> defaults,_netdev,nofail  (keeps `hard`)
#       _netdev : order mount after network-online.target
#       nofail  : NAS-down at boot doesn't wedge boot / kill DNS
#   - fstab: + x-systemd.before=docker.service,x-systemd.mount-timeout=30
#       Puts Before=docker.service directly on each generated .mount unit
#       so Docker waits for the ACTUAL mounts; mount-timeout bounds the
#       wait if the NAS is down at boot.
#   - docker.service drop-in: After=remote-fs.target (kept as a weaker
#     belt-and-suspenders layer).
#
# WHY the drop-in alone was NOT enough (2026-07-14 reboot): `nofail`
# removes a mount from remote-fs.target's blocking set, so ordering
# Docker `After=remote-fs.target` does not actually wait for the nofail
# NFS mounts -> paperless still lost the race and Exited(255) on reboot.
# The load-bearing fix is the DIRECT mount->docker ordering from the
# fstab `x-systemd.before` option. Verify with:
#   systemctl show docker -p After | tr ' ' '\n' | grep mnt-   # lists all 4
#
# Idempotent: re-runs show ok/skipped. Does NOT reboot — the real test
# is the next reboot, run that separately.
#
#   scripts/elway esh-docker-vm --playbook playbooks/fix-esh-nfs-boot-ordering.yaml

vars:
  fstab_backup: /etc/fstab.bak-20260530-nfs-boot

steps:
  - name: Back up /etc/fstab (once)
    shell: cp -n /etc/fstab {{ fstab_backup }}
    sudo: true
    creates: "{{ fstab_backup }}"

  - name: Add _netdev,nofail to the 10.0.50.50 NFS mounts
    # Match active (non-#) lines with ` nfs defaults ` and not already
    # carrying _netdev; rewrite the options field in place.
    shell: sed -i -E '/^10\.0\.50\.50:.* nfs defaults /{/_netdev/!s/ nfs defaults / nfs defaults,_netdev,nofail /}' /etc/fstab
    sudo: true
    # Run only if at least one unfixed NFS line remains.
    when: "grep -qE '^10\\.0\\.50\\.50:.* nfs defaults ' /etc/fstab"

  - name: Order each NFS mount before docker.service (direct dep; nofail-safe)
    # THE load-bearing fix. remote-fs.target ordering (below) is defeated
    # by `nofail` (the mount drops out of that target's blocking set).
    # x-systemd.before=docker.service injects Before=docker.service onto
    # each generated .mount unit, so Docker genuinely waits for the mounts.
    shell: sed -i -E '/^10\.0\.50\.50:/{/x-systemd.before/!s/(_netdev,nofail)/\1,x-systemd.before=docker.service,x-systemd.mount-timeout=30/}' /etc/fstab
    sudo: true
    # Run only if an NFS line with _netdev,nofail still lacks the ordering.
    when: "grep -E '^10\\.0\\.50\\.50:.*_netdev,nofail' /etc/fstab | grep -qv x-systemd.before"

  - name: Install docker.service drop-in to order after remote-fs.target
    # Use a DISTINCT filename — esh-docker-vm already ships an
    # override.conf (dockerd ExecStart/containerd socket); systemd merges
    # all *.conf drop-ins, so a separate file augments rather than
    # clobbers it. Guard on CONTENT, not file existence, so a stale/empty
    # file can't make this silently skip.
    shell: |
      install -d -m 0755 /etc/systemd/system/docker.service.d
      printf '[Unit]\nAfter=remote-fs.target\nWants=remote-fs.target\n' \
        > /etc/systemd/system/docker.service.d/10-after-remote-fs.conf
    sudo: true
    when: "! grep -qs remote-fs.target /etc/systemd/system/docker.service.d/10-after-remote-fs.conf"

  - name: Reload systemd so the drop-in takes effect next boot
    shell: systemctl daemon-reload
    sudo: true
    # A reload mutates nothing observable on its own; report as ok.
    changed_when: "false"

verify:
  - name: All 4 NFS lines carry _netdev,nofail
    shell: test "$(grep -cE '^10\.0\.50\.50:.*nfs defaults,_netdev,nofail' /etc/fstab)" -eq 4
    changed_when: "false"

  - name: All 4 NFS lines carry x-systemd.before=docker.service
    shell: test "$(grep -cE '^10\.0\.50\.50:.*x-systemd.before=docker.service' /etc/fstab)" -eq 4
    changed_when: "false"

  - name: fstab parses cleanly (findmnt --verify, no fatal errors)
    shell: findmnt --verify >/dev/null
    changed_when: "false"

  - name: Docker is ordered after the actual NFS mount units (the real fix)
    shell: systemctl show docker -p After | tr ' ' '\n' | grep -q '^mnt-documents.mount$'
    changed_when: "false"

  - name: Docker is also ordered after remote-fs.target (belt-and-suspenders)
    shell: systemctl show docker -p After | grep -q remote-fs.target
    changed_when: "false"