# install-traefik-postboot — install a Type=oneshot systemd unit that
# restarts traefik N seconds after every boot.
#
# Why this exists:
# When dockerd starts (boot, daemon swap, daemon crash-recover), 30+
# containers come up in a burst. Two races bite traefik:
#
#   1. Events-stream debounce. Traefik's docker provider does an initial
#      scan + watches `/events` for live changes. When dozens of "container
#      start" events arrive in a 2-3s burst on daemon startup, the provider
#      can drop / coalesce some of them. Backends end up missing from the
#      routing table.
#
#   2. Network-attach race. A backend's container can be in `Created`
#      state on the docker socket (so traefik scans it) but not yet
#      attached to traefik-net. Traefik builds a router pointing at a
#      gateway IP it can't reach.
#
# Either way, the workaround is the same: `docker restart traefik` once
# the topology has settled. This unit automates that — sleep N seconds
# after docker.service is up, then restart traefik. Runs once per boot.
#
# Usage:
#   scripts/elway esh-docker-vm --playbook playbooks/install-traefik-postboot.yaml
#   scripts/elway ana-docker --playbook playbooks/install-traefik-postboot.yaml
#
# Tunables (override via `--var`):
#   --var delay_seconds=90   (default 60 — empirically enough for ~30 stacks)
#   --var container=traefik  (default 'traefik')
#
# Hosts in scope (have a traefik container, verified 2026-04-24):
#   esh-docker-vm, ana-docker
# Skip on hosts without traefik (the verify phase will catch a typo).
#
# Removal: `sudo systemctl disable --now traefik-postboot && sudo rm
# /etc/systemd/system/traefik-postboot.service && sudo systemctl
# daemon-reload`

vars:
  delay_seconds: "60"
  container: traefik
  unit_path: /etc/systemd/system/traefik-postboot.service

steps:
  - name: Install /etc/systemd/system/traefik-postboot.service
    # Unconditional rewrite — the unit is 12 lines and the cost of a
    # rewrite + daemon-reload is sub-second. cmp-then-skip would be
    # over-engineering. The systemctl enable below is gated separately.
    shell: |
      sudo tee {{ unit_path }} >/dev/null <<'UNIT'
      [Unit]
      Description=Restart {{ container }} {{ delay_seconds }}s after boot for clean topology scan
      After=docker.service
      Wants=docker.service

      [Service]
      Type=oneshot
      ExecStartPre=/bin/sleep {{ delay_seconds }}
      ExecStart=/usr/bin/docker restart {{ container }}

      [Install]
      WantedBy=multi-user.target
      UNIT
      sudo chmod 0644 {{ unit_path }}
    sudo: true

  - name: systemctl daemon-reload
    shell: systemctl daemon-reload
    sudo: true

  - name: Enable traefik-postboot.service (idempotent — skip if already enabled)
    shell: systemctl enable traefik-postboot.service
    sudo: true
    when: "! systemctl is-enabled --quiet traefik-postboot.service"

verify:
  - name: Unit file exists with mode 0644
    shell: test -f {{ unit_path }} && [ "$(stat -c %a {{ unit_path }})" = "644" ]
    changed_when: "false"

  - name: Unit is enabled
    shell: systemctl is-enabled --quiet traefik-postboot.service
    changed_when: "false"

  - name: Unit references the right container in ExecStart
    shell: grep -q 'docker restart {{ container }}' {{ unit_path }}
    changed_when: "false"

  - name: Container '{{ container }}' actually exists on this host (catches typos)
    shell: docker inspect {{ container }} >/dev/null 2>&1
    changed_when: "false"

  - name: systemd-analyze parses the unit cleanly
    # Lints the unit for syntax / dependency / Type errors without
    # actually executing it (executing would needlessly bounce traefik
    # right after install on already-healthy hosts).
    shell: systemd-analyze verify {{ unit_path }} 2>&1
    changed_when: "false"