esh-pfi-infrastructure/playbooks/upgrade-docker-ce.yaml

# upgrade-docker-ce — migrate a host from Debian's docker.io (20.10.x,
# bookworm-packaged) to Docker's official docker-ce repo (28+).
#
# Bookworm's docker.io stays pinned at 20.10.24, which:
#   * uses an old client/daemon API (1.41) that newer compose clients
#     (1.52+) refuse to talk to → "client version 1.52 is too new"
#     during builds
#   * is EOL upstream — docker.io upstream doesn't ship to it anymore
#   * is missing modern buildx driver versions our newer client expects
#
# This playbook handles the migration on one host at a time. Stops
# every running stack via `docker compose stop`, removes the old
# packages (preserving /var/lib/docker/), adds Docker's signed APT
# repo, installs docker-ce + docker-compose-plugin + containerd.io,
# starts the new daemon, and brings each stack back up via
# `docker compose up -d` (compose's restart=unless-stopped also
# auto-restarts containers when the daemon comes back, but doing
# them explicitly per-stack lets us see failures cleanly).
#
# Usage:
#   scripts/elway <host> --playbook playbooks/upgrade-docker-ce.yaml
#
# Recommended host order (least → most blast radius):
#   1. nh3-docker
#   2. esh-docker-vm
#   3. ana-docker
#
# Verify after each before moving to the next:
#   ssh <host> 'docker version --format "{{.Server.Version}}"'
#   ssh <host> 'docker ps --format "{{.Names}}\t{{.Status}}" | head'
#
# Rollback (if a daemon won't start, or a container errors out):
#   ssh <host> 'sudo apt install --allow-downgrades docker.io'
#   then re-add the docker.io packages from /tmp/docker-pre-upgrade.txt
#
# /var/lib/docker/ is preserved throughout (apt remove, not purge),
# so volumes / images / containers survive the package swap. The
# overlay2 storage driver is the default on both packages, so no
# data migration needed.

vars:
  pkgs_to_remove: docker.io docker-compose docker-compose-plugin docker-buildx-plugin docker-doc docker
  pkgs_to_install: docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin

steps:
  # ── snapshot + stop ─────────────────────────────────────────────────

  - name: Snapshot current docker package versions (rollback reference)
    shell: |
      dpkg -l | awk '/^ii\s+(docker|containerd)/ {print $2 "=" $3}' \
        | sudo tee /tmp/docker-pre-upgrade.txt >/dev/null
      cat /tmp/docker-pre-upgrade.txt
    sudo: true
    changed_when: "false"

  - name: List currently-running compose stacks (snapshot for restart)
    shell: |
      docker ps --format '{{.Label "com.docker.compose.project"}}' \
        | sort -u | grep -v '^$' \
        | tee /tmp/docker-pre-upgrade-stacks.txt
    changed_when: "false"

  - name: Stop every running stack via docker compose
    # Walk each /opt/docker/compose/<stack>/ dir that has a running
    # container and `compose stop` it. Skips dirs without a running
    # stack so reruns don't error.
    shell: |
      set +e
      for dir in /opt/docker/compose/*/; do
        stack=$(basename "$dir")
        if docker ps --format '{{.Label "com.docker.compose.project"}}' | grep -q "^${stack}$"; then
          echo "  stopping $stack"
          (cd "$dir" && docker compose stop) || echo "    (no-op or failed)"
        fi
      done
      true

  # ── add Docker's official APT repo ──────────────────────────────────

  - name: Install prereqs for the new APT repo
    shell: apt-get install -y ca-certificates curl gnupg
    sudo: true

  - name: Ensure /etc/apt/keyrings exists (mode 0755)
    shell: install -d -m 0755 /etc/apt/keyrings
    sudo: true
    creates: /etc/apt/keyrings

  - name: Fetch + install Docker's signing key (only if absent)
    shell: |
      curl -fsSL https://download.docker.com/linux/debian/gpg \
        | sudo gpg --dearmor --yes -o /etc/apt/keyrings/docker.gpg
      sudo chmod a+r /etc/apt/keyrings/docker.gpg
    sudo: true
    when: '[ ! -f /etc/apt/keyrings/docker.gpg ]'

  - name: Add Docker's APT source
    # codename is whatever lsb_release says — bookworm on this fleet.
    shell: |
      codename=$(lsb_release -cs)
      arch=$(dpkg --print-architecture)
      echo "deb [arch=${arch} signed-by=/etc/apt/keyrings/docker.gpg] \
https://download.docker.com/linux/debian ${codename} stable" \
        | sudo tee /etc/apt/sources.list.d/docker.list >/dev/null
    sudo: true
    when: '[ ! -f /etc/apt/sources.list.d/docker.list ]'

  - name: apt-get update (now sees Docker's repo)
    shell: apt-get update -qq
    sudo: true

  - name: Show candidate version of docker-ce
    shell: apt-cache policy docker-ce | head -5
    sudo: true
    changed_when: "false"

  # ── swap packages ───────────────────────────────────────────────────

  - name: Remove Debian's docker.io packages (PRESERVE /var/lib/docker)
    # apt remove (not purge) keeps /var/lib/docker/* in place — images,
    # volumes, container metadata all survive the swap.
    shell: DEBIAN_FRONTEND=noninteractive apt-get remove -y {{ pkgs_to_remove }} || true
    sudo: true

  - name: Install docker-ce + plugins from Docker's repo
    shell: DEBIAN_FRONTEND=noninteractive apt-get install -y {{ pkgs_to_install }}
    sudo: true

  - name: Enable + start the new daemon (idempotent)
    shell: systemctl enable --now docker
    sudo: true

  - name: Confirm new daemon is responsive + report its version
    shell: docker version --format 'client={{.Client.Version}} server={{.Server.Version}} api={{.Server.APIVersion}}'
    changed_when: "false"

  # ── bring stacks back up ────────────────────────────────────────────

  - name: Start every previously-running compose stack
    # Use the snapshot we took before the swap. Stacks whose containers
    # have restart=unless-stopped will already be coming back via the
    # daemon — `compose up -d` is idempotent and surfaces failures we
    # might otherwise miss.
    shell: |
      set +e
      while read stack; do
        dir="/opt/docker/compose/$stack"
        if [ -d "$dir" ]; then
          echo "  starting $stack"
          (cd "$dir" && docker compose up -d) || echo "    FAILED — investigate $stack"
        fi
      done < /tmp/docker-pre-upgrade-stacks.txt
      true

  - name: Brief settle pause then list container states
    shell: sleep 8 && docker ps --format 'table {{.Names}}\t{{.Status}}'
    changed_when: "false"

verify:
  - name: docker daemon responding
    shell: docker info >/dev/null
    changed_when: "false"

  - name: docker server version is 24+ (was 20.10)
    shell: |
      v=$(docker version --format '{{.Server.Version}}' | cut -d. -f1)
      [ "$v" -ge 24 ]
    changed_when: "false"

  - name: docker compose plugin v2 installed
    shell: docker compose version | grep -qE 'v2\.[0-9]+\.[0-9]+'
    changed_when: "false"

  - name: at least one previously-running stack is back up
    # If the host had no running stacks pre-upgrade (e.g. nh3-docker may
    # be empty), this verify passes trivially.
    shell: |
      [ ! -s /tmp/docker-pre-upgrade-stacks.txt ] && exit 0
      head -1 /tmp/docker-pre-upgrade-stacks.txt | xargs -I {} docker ps \
        --filter 'label=com.docker.compose.project={}' --format '{{.Names}}' \
        | grep -q .
    changed_when: "false"