# Harden NFS boot ordering on esh-docker-vm so Docker waits for the
# 10.0.50.50 NFS mounts before starting containers.
#
# Root cause (2026-05-30 incident): /etc/fstab NFS lines used `defaults`
# (no `_netdev`), so the system attempted them too early and Docker
# started NFS-bind-mount containers (paperless) before /mnt/documents
# was ready -> paperless Exited(255) on every reboot, needing a manual
# `docker start`. The same `hard` mounts also froze the celery worker in
# unkillable D-state when the NAS stalled at runtime.
#
# This playbook fixes the BOOT race only:
#   - fstab: defaults -> defaults,_netdev,nofail  (keeps `hard`)
#       _netdev : order mount after network-online.target
#       nofail  : NAS-down at boot doesn't wedge boot / kill DNS
#   - docker.service drop-in: After=remote-fs.target so Docker starts
#     after the NFS mounts have completed.
#
# Idempotent: re-runs show ok/skipped. Does NOT reboot — the real test
# is the next reboot, run that separately.
#
#   scripts/elway esh-docker-vm --playbook playbooks/fix-esh-nfs-boot-ordering.yaml

vars:
  fstab_backup: /etc/fstab.bak-20260530-nfs-boot

steps:
  - name: Back up /etc/fstab (once)
    shell: cp -n /etc/fstab {{ fstab_backup }}
    sudo: true
    creates: "{{ fstab_backup }}"

  - name: Add _netdev,nofail to the 10.0.50.50 NFS mounts
    # Match active (non-#) lines with ` nfs defaults ` and not already
    # carrying _netdev; rewrite the options field in place.
    shell: sed -i -E '/^10\.0\.50\.50:.* nfs defaults /{/_netdev/!s/ nfs defaults / nfs defaults,_netdev,nofail /}' /etc/fstab
    sudo: true
    # Run only if at least one unfixed NFS line remains.
    when: "grep -qE '^10\\.0\\.50\\.50:.* nfs defaults ' /etc/fstab"

  - name: Install docker.service drop-in to order after remote-fs.target
    # Use a DISTINCT filename — esh-docker-vm already ships an
    # override.conf (dockerd ExecStart/containerd socket); systemd merges
    # all *.conf drop-ins, so a separate file augments rather than
    # clobbers it. Guard on CONTENT, not file existence, so a stale/empty
    # file can't make this silently skip.
    shell: |
      install -d -m 0755 /etc/systemd/system/docker.service.d
      printf '[Unit]\nAfter=remote-fs.target\nWants=remote-fs.target\n' \
        > /etc/systemd/system/docker.service.d/10-after-remote-fs.conf
    sudo: true
    when: "! grep -qs remote-fs.target /etc/systemd/system/docker.service.d/10-after-remote-fs.conf"

  - name: Reload systemd so the drop-in takes effect next boot
    shell: systemctl daemon-reload
    sudo: true
    # A reload mutates nothing observable on its own; report as ok.
    changed_when: "false"

verify:
  - name: All 4 NFS lines now carry _netdev,nofail
    shell: test "$(grep -cE '^10\.0\.50\.50:.* nfs defaults,_netdev,nofail ' /etc/fstab)" -eq 4
    changed_when: "false"

  - name: fstab parses cleanly (findmnt --verify, no fatal errors)
    shell: findmnt --verify >/dev/null
    changed_when: "false"

  - name: Docker is ordered after remote-fs.target
    shell: systemctl show docker -p After | grep -q remote-fs.target
    changed_when: "false"