diff --git a/playbooks/install-traefik-postboot.yaml b/playbooks/install-traefik-postboot.yaml new file mode 100644 index 0000000..9e106ed --- /dev/null +++ b/playbooks/install-traefik-postboot.yaml @@ -0,0 +1,98 @@ +# install-traefik-postboot — install a Type=oneshot systemd unit that +# restarts traefik N seconds after every boot. +# +# Why this exists: +# When dockerd starts (boot, daemon swap, daemon crash-recover), 30+ +# containers come up in a burst. Two races bite traefik: +# +# 1. Events-stream debounce. Traefik's docker provider does an initial +# scan + watches `/events` for live changes. When dozens of "container +# start" events arrive in a 2-3s burst on daemon startup, the provider +# can drop / coalesce some of them. Backends end up missing from the +# routing table. +# +# 2. Network-attach race. A backend's container can be in `Created` +# state on the docker socket (so traefik scans it) but not yet +# attached to traefik-net. Traefik builds a router pointing at a +# gateway IP it can't reach. +# +# Either way, the workaround is the same: `docker restart traefik` once +# the topology has settled. This unit automates that — sleep N seconds +# after docker.service is up, then restart traefik. Runs once per boot. +# +# Usage: +# scripts/elway esh-docker-vm --playbook playbooks/install-traefik-postboot.yaml +# scripts/elway ana-docker --playbook playbooks/install-traefik-postboot.yaml +# +# Tunables (override via `--var`): +# --var delay_seconds=90 (default 60 — empirically enough for ~30 stacks) +# --var container=traefik (default 'traefik') +# +# Hosts in scope (have a traefik container, verified 2026-04-24): +# esh-docker-vm, ana-docker +# Skip on hosts without traefik (the verify phase will catch a typo). +# +# Removal: `sudo systemctl disable --now traefik-postboot && sudo rm +# /etc/systemd/system/traefik-postboot.service && sudo systemctl +# daemon-reload` + +vars: + delay_seconds: "60" + container: traefik + unit_path: /etc/systemd/system/traefik-postboot.service + +steps: + - name: Install /etc/systemd/system/traefik-postboot.service + # Unconditional rewrite — the unit is 12 lines and the cost of a + # rewrite + daemon-reload is sub-second. cmp-then-skip would be + # over-engineering. The systemctl enable below is gated separately. + shell: | + sudo tee {{ unit_path }} >/dev/null <<'UNIT' + [Unit] + Description=Restart {{ container }} {{ delay_seconds }}s after boot for clean topology scan + After=docker.service + Wants=docker.service + + [Service] + Type=oneshot + ExecStartPre=/bin/sleep {{ delay_seconds }} + ExecStart=/usr/bin/docker restart {{ container }} + + [Install] + WantedBy=multi-user.target + UNIT + sudo chmod 0644 {{ unit_path }} + sudo: true + + - name: systemctl daemon-reload + shell: systemctl daemon-reload + sudo: true + + - name: Enable traefik-postboot.service (idempotent — skip if already enabled) + shell: systemctl enable traefik-postboot.service + sudo: true + when: "! systemctl is-enabled --quiet traefik-postboot.service" + +verify: + - name: Unit file exists with mode 0644 + shell: test -f {{ unit_path }} && [ "$(stat -c %a {{ unit_path }})" = "644" ] + changed_when: "false" + + - name: Unit is enabled + shell: systemctl is-enabled --quiet traefik-postboot.service + changed_when: "false" + + - name: Unit references the right container in ExecStart + shell: grep -q 'docker restart {{ container }}' {{ unit_path }} + changed_when: "false" + + - name: Container '{{ container }}' actually exists on this host (catches typos) + shell: docker inspect {{ container }} >/dev/null 2>&1 + changed_when: "false" + + - name: systemd-analyze parses the unit cleanly + # Lints the unit for syntax / dependency / Type errors without + # actually executing it (executing would needlessly bounce traefik + # right after install on already-healthy hosts). + shell: systemd-analyze verify {{ unit_path }} 2>&1 + changed_when: "false"