vh
53157b193d
override.conf already existed on esh-docker-vm (dockerd ExecStart), so the creates: guard skipped the step and the After=remote-fs.target directive was never written. Use a distinct filename (10-after-remote-fs.conf) that systemd merges alongside override.conf, and guard on content rather than file existence so a stale file can't silently skip.
71 lines
3.1 KiB
YAML
71 lines
3.1 KiB
YAML
# Harden NFS boot ordering on esh-docker-vm so Docker waits for the
|
|
# 10.0.50.50 NFS mounts before starting containers.
|
|
#
|
|
# Root cause (2026-05-30 incident): /etc/fstab NFS lines used `defaults`
|
|
# (no `_netdev`), so the system attempted them too early and Docker
|
|
# started NFS-bind-mount containers (paperless) before /mnt/documents
|
|
# was ready -> paperless Exited(255) on every reboot, needing a manual
|
|
# `docker start`. The same `hard` mounts also froze the celery worker in
|
|
# unkillable D-state when the NAS stalled at runtime.
|
|
#
|
|
# This playbook fixes the BOOT race only:
|
|
# - fstab: defaults -> defaults,_netdev,nofail (keeps `hard`)
|
|
# _netdev : order mount after network-online.target
|
|
# nofail : NAS-down at boot doesn't wedge boot / kill DNS
|
|
# - docker.service drop-in: After=remote-fs.target so Docker starts
|
|
# after the NFS mounts have completed.
|
|
#
|
|
# Idempotent: re-runs show ok/skipped. Does NOT reboot — the real test
|
|
# is the next reboot, run that separately.
|
|
#
|
|
# scripts/elway esh-docker-vm --playbook playbooks/fix-esh-nfs-boot-ordering.yaml
|
|
|
|
vars:
|
|
fstab_backup: /etc/fstab.bak-20260530-nfs-boot
|
|
|
|
steps:
|
|
- name: Back up /etc/fstab (once)
|
|
shell: cp -n /etc/fstab {{ fstab_backup }}
|
|
sudo: true
|
|
creates: "{{ fstab_backup }}"
|
|
|
|
- name: Add _netdev,nofail to the 10.0.50.50 NFS mounts
|
|
# Match active (non-#) lines with ` nfs defaults ` and not already
|
|
# carrying _netdev; rewrite the options field in place.
|
|
shell: sed -i -E '/^10\.0\.50\.50:.* nfs defaults /{/_netdev/!s/ nfs defaults / nfs defaults,_netdev,nofail /}' /etc/fstab
|
|
sudo: true
|
|
# Run only if at least one unfixed NFS line remains.
|
|
when: "grep -qE '^10\\.0\\.50\\.50:.* nfs defaults ' /etc/fstab"
|
|
|
|
- name: Install docker.service drop-in to order after remote-fs.target
|
|
# Use a DISTINCT filename — esh-docker-vm already ships an
|
|
# override.conf (dockerd ExecStart/containerd socket); systemd merges
|
|
# all *.conf drop-ins, so a separate file augments rather than
|
|
# clobbers it. Guard on CONTENT, not file existence, so a stale/empty
|
|
# file can't make this silently skip.
|
|
shell: |
|
|
install -d -m 0755 /etc/systemd/system/docker.service.d
|
|
printf '[Unit]\nAfter=remote-fs.target\nWants=remote-fs.target\n' \
|
|
> /etc/systemd/system/docker.service.d/10-after-remote-fs.conf
|
|
sudo: true
|
|
when: "! grep -qs remote-fs.target /etc/systemd/system/docker.service.d/10-after-remote-fs.conf"
|
|
|
|
- name: Reload systemd so the drop-in takes effect next boot
|
|
shell: systemctl daemon-reload
|
|
sudo: true
|
|
# A reload mutates nothing observable on its own; report as ok.
|
|
changed_when: "false"
|
|
|
|
verify:
|
|
- name: All 4 NFS lines now carry _netdev,nofail
|
|
shell: test "$(grep -cE '^10\.0\.50\.50:.* nfs defaults,_netdev,nofail ' /etc/fstab)" -eq 4
|
|
changed_when: "false"
|
|
|
|
- name: fstab parses cleanly (findmnt --verify, no fatal errors)
|
|
shell: findmnt --verify >/dev/null
|
|
changed_when: "false"
|
|
|
|
- name: Docker is ordered after remote-fs.target
|
|
shell: systemctl show docker -p After | grep -q remote-fs.target
|
|
changed_when: "false"
|