# upgrade-docker-ce — migrate a host from Debian's docker.io (20.10.x,
# bookworm-packaged) to Docker's official docker-ce repo (28+).
#
# Bookworm's docker.io stays pinned at 20.10.24, which:
#   * uses an old client/daemon API (1.41) that newer compose clients
#     (1.52+) refuse to talk to → "client version 1.52 is too new"
#     during builds
#   * is EOL upstream — docker.io upstream doesn't ship to it anymore
#   * is missing modern buildx driver versions our newer client expects
#
# This playbook handles the migration on one host at a time. Stops
# every running stack via `docker compose stop`, removes the old
# packages (preserving /var/lib/docker/), adds Docker's signed APT
# repo, installs docker-ce + docker-compose-plugin + containerd.io,
# starts the new daemon, and brings each stack back up via
# `docker compose up -d` (compose's restart=unless-stopped also
# auto-restarts containers when the daemon comes back, but doing
# them explicitly per-stack lets us see failures cleanly).
#
# Usage:
#   scripts/elway <host> --playbook playbooks/upgrade-docker-ce.yaml
#
# Recommended host order (least → most blast radius):
#   1. nh3-docker
#   2. esh-docker-vm
#   3. ana-docker
#
# Verify after each before moving to the next:
#   ssh <host> 'docker version --format "{{.Server.Version}}"'
#   ssh <host> 'docker ps --format "{{.Names}}\t{{.Status}}" | head'
#
# Rollback (if a daemon won't start, or a container errors out):
#   ssh <host> 'sudo apt install --allow-downgrades docker.io'
#   then re-add the docker.io packages from /tmp/docker-pre-upgrade.txt
#
# /var/lib/docker/ is preserved throughout (apt remove, not purge),
# so volumes / images / containers survive the package swap. The
# overlay2 storage driver is the default on both packages, so no
# data migration needed.

vars:
  pkgs_to_remove: docker.io docker-compose docker-compose-plugin docker-buildx-plugin docker-doc docker
  pkgs_to_install: docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin

steps:
  # ── snapshot + stop ─────────────────────────────────────────────────

  - name: Snapshot current docker package versions (rollback reference)
    shell: |
      dpkg -l | awk '/^ii\s+(docker|containerd)/ {print $2 "=" $3}' \
        | sudo tee /tmp/docker-pre-upgrade.txt >/dev/null
      cat /tmp/docker-pre-upgrade.txt
    sudo: true
    changed_when: "false"

  - name: List currently-running compose stacks (snapshot for restart)
    shell: |
      docker ps --format '{{.Label "com.docker.compose.project"}}' \
        | sort -u | grep -v '^$' \
        | tee /tmp/docker-pre-upgrade-stacks.txt
    changed_when: "false"

  - name: Stop every running stack via docker compose
    # Walk each /opt/docker/compose/<stack>/ dir that has a running
    # container and `compose stop` it. Skips dirs without a running
    # stack so reruns don't error.
    shell: |
      set +e
      for dir in /opt/docker/compose/*/; do
        stack=$(basename "$dir")
        if docker ps --format '{{.Label "com.docker.compose.project"}}' | grep -q "^${stack}$"; then
          echo "  stopping $stack"
          (cd "$dir" && docker compose stop) || echo "    (no-op or failed)"
        fi
      done
      true

  # ── add Docker's official APT repo ──────────────────────────────────

  - name: Install prereqs for the new APT repo
    shell: apt-get install -y ca-certificates curl gnupg
    sudo: true

  - name: Ensure /etc/apt/keyrings exists (mode 0755)
    shell: install -d -m 0755 /etc/apt/keyrings
    sudo: true
    creates: /etc/apt/keyrings

  - name: Fetch + install Docker's signing key (only if absent)
    shell: |
      curl -fsSL https://download.docker.com/linux/debian/gpg \
        | sudo gpg --dearmor --yes -o /etc/apt/keyrings/docker.gpg
      sudo chmod a+r /etc/apt/keyrings/docker.gpg
    sudo: true
    when: '[ ! -f /etc/apt/keyrings/docker.gpg ]'

  - name: Add Docker's APT source
    # codename is whatever lsb_release says — bookworm on this fleet.
    shell: |
      codename=$(lsb_release -cs)
      arch=$(dpkg --print-architecture)
      url="https://download.docker.com/linux/debian"
      line="deb [arch=${arch} signed-by=/etc/apt/keyrings/docker.gpg] ${url} ${codename} stable"
      echo "${line}" | sudo tee /etc/apt/sources.list.d/docker.list >/dev/null
    sudo: true
    when: '[ ! -f /etc/apt/sources.list.d/docker.list ]'

  - name: apt-get update (now sees Docker's repo)
    shell: apt-get update -qq
    sudo: true

  - name: Show candidate version of docker-ce
    shell: apt-cache policy docker-ce | head -5
    sudo: true
    changed_when: "false"

  # ── swap packages ───────────────────────────────────────────────────

  - name: Remove Debian's docker.io packages (PRESERVE /var/lib/docker)
    # apt remove (not purge) keeps /var/lib/docker/* in place — images,
    # volumes, container metadata all survive the swap.
    shell: DEBIAN_FRONTEND=noninteractive apt-get remove -y {{ pkgs_to_remove }} || true
    sudo: true

  - name: Install docker-ce + plugins from Docker's repo
    shell: DEBIAN_FRONTEND=noninteractive apt-get install -y {{ pkgs_to_install }}
    sudo: true

  - name: Rewrite docker.service drop-in to match the new package layout
    # Two related fixes for the systemd unit, learned the hard way on
    # nh3-docker. The docker.io era left an
    # /etc/systemd/system/docker.service.d/override.conf hardcoding
    # ExecStart=/usr/sbin/dockerd (Debian package path) and usually
    # adding `-H tcp://...` for remote discovery.
    #
    # After the swap to docker-ce two things are wrong:
    #   1. /usr/sbin/dockerd doesn't exist — docker-ce installs at
    #      /usr/bin/dockerd. Daemon fails status=203/EXEC.
    #   2. If daemon.json defines `hosts:` (typical here for the
    #      0.0.0.0:2375 homepage-discovery binding), then ANY `-H`
    #      flag in ExecStart conflicts: dockerd refuses to start with
    #      "conflicting host options".
    #
    # The shipped docker-ce unit's ExecStart is `dockerd -H fd://
    # --containerd=/run/containerd/containerd.sock` — that `-H fd://`
    # ALSO conflicts with daemon.json hosts:. So we always need an
    # override that strips `-H` entirely when daemon.json defines
    # hosts.
    #
    # Procedure:
    #   * Back up any existing override (preserves it as .pre-upgrade)
    #   * Probe /etc/docker/daemon.json for a `hosts:` setting
    #   * If hosts: is defined → install an override that clears the
    #     base unit's ExecStart and replaces it WITHOUT -H
    #   * Otherwise no override is needed (base unit's `-H fd://` is fine)
    shell: |
      d=/etc/systemd/system/docker.service.d
      f="$d/override.conf"
      sudo install -d -m 0755 "$d"
      if [ -f "$f" ]; then
        echo "  backing up existing $f → ${f}.pre-upgrade"
        sudo mv "$f" "${f}.pre-upgrade"
      fi
      if [ -f /etc/docker/daemon.json ] && grep -q '"hosts"' /etc/docker/daemon.json; then
        echo "  daemon.json has hosts: — installing override that strips -H from ExecStart"
        sudo tee "$f" >/dev/null <<'OVR'
      [Service]
      ExecStart=
      ExecStart=/usr/bin/dockerd --containerd=/run/containerd/containerd.sock
      OVR
      else
        echo "  daemon.json has no hosts: — base unit's ExecStart is fine, no override needed"
      fi
    sudo: true

  - name: systemctl daemon-reload (so the new unit + override are loaded)
    shell: systemctl daemon-reload
    sudo: true

  - name: Reset the docker.service failed state (3 failed starts during
      the install often leaves it in start-rate-limited state — reset
      so the next start isn't immediately denied)
    shell: systemctl reset-failed docker.service
    sudo: true

  - name: Enable + start the new daemon (idempotent)
    shell: systemctl enable --now docker
    sudo: true

  - name: Confirm new daemon is responsive + report its version
    shell: docker version --format 'client={{.Client.Version}} server={{.Server.Version}} api={{.Server.APIVersion}}'
    changed_when: "false"

  # ── bring stacks back up ────────────────────────────────────────────

  - name: Start every previously-running compose stack
    # Use the snapshot we took before the swap. Stacks whose containers
    # have restart=unless-stopped will already be coming back via the
    # daemon — `compose up -d` is idempotent and surfaces failures we
    # might otherwise miss.
    shell: |
      set +e
      while read stack; do
        dir="/opt/docker/compose/$stack"
        if [ -d "$dir" ]; then
          echo "  starting $stack"
          (cd "$dir" && docker compose up -d) || echo "    FAILED — investigate $stack"
        fi
      done < /tmp/docker-pre-upgrade-stacks.txt
      true

  - name: Brief settle pause then list container states
    shell: sleep 8 && docker ps --format 'table {{.Names}}\t{{.Status}}'
    changed_when: "false"

verify:
  - name: docker daemon responding
    shell: docker info >/dev/null
    changed_when: "false"

  - name: docker server version is 24+ (was 20.10)
    shell: |
      v=$(docker version --format '{{.Server.Version}}' | cut -d. -f1)
      [ "$v" -ge 24 ]
    changed_when: "false"

  - name: docker compose plugin v2+ installed
    # docker-ce 29 ships docker-compose-plugin renumbered to v5.x (was
    # v2.x with docker-ce 26-28). Both are the same Compose v2 codebase
    # under the hood — Docker just realigned the plugin's major number.
    # Accept anything ≥ 2 so this verify keeps working across future
    # plugin renumbers.
    shell: |
      major=$(docker compose version --short | sed 's/^v//' | cut -d. -f1)
      [ "$major" -ge 2 ]
    changed_when: "false"

  - name: at least one previously-running stack is back up
    # If the host had no running stacks pre-upgrade (e.g. nh3-docker may
    # be empty), this verify passes trivially.
    shell: |
      [ ! -s /tmp/docker-pre-upgrade-stacks.txt ] && exit 0
      head -1 /tmp/docker-pre-upgrade-stacks.txt | xargs -I {} docker ps \
        --filter 'label=com.docker.compose.project={}' --format '{{.Names}}' \
        | grep -q .
    changed_when: "false"