diff --git a/playbooks/upgrade-docker-ce.yaml b/playbooks/upgrade-docker-ce.yaml new file mode 100644 index 0000000..6a59f2f --- /dev/null +++ b/playbooks/upgrade-docker-ce.yaml @@ -0,0 +1,182 @@ +# upgrade-docker-ce — migrate a host from Debian's docker.io (20.10.x, +# bookworm-packaged) to Docker's official docker-ce repo (28+). +# +# Bookworm's docker.io stays pinned at 20.10.24, which: +# * uses an old client/daemon API (1.41) that newer compose clients +# (1.52+) refuse to talk to → "client version 1.52 is too new" +# during builds +# * is EOL upstream — docker.io upstream doesn't ship to it anymore +# * is missing modern buildx driver versions our newer client expects +# +# This playbook handles the migration on one host at a time. Stops +# every running stack via `docker compose stop`, removes the old +# packages (preserving /var/lib/docker/), adds Docker's signed APT +# repo, installs docker-ce + docker-compose-plugin + containerd.io, +# starts the new daemon, and brings each stack back up via +# `docker compose up -d` (compose's restart=unless-stopped also +# auto-restarts containers when the daemon comes back, but doing +# them explicitly per-stack lets us see failures cleanly). +# +# Usage: +# scripts/elway --playbook playbooks/upgrade-docker-ce.yaml +# +# Recommended host order (least → most blast radius): +# 1. nh3-docker +# 2. esh-docker-vm +# 3. ana-docker +# +# Verify after each before moving to the next: +# ssh 'docker version --format "{{.Server.Version}}"' +# ssh 'docker ps --format "{{.Names}}\t{{.Status}}" | head' +# +# Rollback (if a daemon won't start, or a container errors out): +# ssh 'sudo apt install --allow-downgrades docker.io' +# then re-add the docker.io packages from /tmp/docker-pre-upgrade.txt +# +# /var/lib/docker/ is preserved throughout (apt remove, not purge), +# so volumes / images / containers survive the package swap. The +# overlay2 storage driver is the default on both packages, so no +# data migration needed. + +vars: + pkgs_to_remove: docker.io docker-compose docker-compose-plugin docker-buildx-plugin docker-doc docker + pkgs_to_install: docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + +steps: + # ── snapshot + stop ───────────────────────────────────────────────── + + - name: Snapshot current docker package versions (rollback reference) + shell: | + dpkg -l | awk '/^ii\s+(docker|containerd)/ {print $2 "=" $3}' \ + | sudo tee /tmp/docker-pre-upgrade.txt >/dev/null + cat /tmp/docker-pre-upgrade.txt + sudo: true + changed_when: "false" + + - name: List currently-running compose stacks (snapshot for restart) + shell: | + docker ps --format '{{.Label "com.docker.compose.project"}}' \ + | sort -u | grep -v '^$' \ + | tee /tmp/docker-pre-upgrade-stacks.txt + changed_when: "false" + + - name: Stop every running stack via docker compose + # Walk each /opt/docker/compose// dir that has a running + # container and `compose stop` it. Skips dirs without a running + # stack so reruns don't error. + shell: | + set +e + for dir in /opt/docker/compose/*/; do + stack=$(basename "$dir") + if docker ps --format '{{.Label "com.docker.compose.project"}}' | grep -q "^${stack}$"; then + echo " stopping $stack" + (cd "$dir" && docker compose stop) || echo " (no-op or failed)" + fi + done + true + + # ── add Docker's official APT repo ────────────────────────────────── + + - name: Install prereqs for the new APT repo + shell: apt-get install -y ca-certificates curl gnupg + sudo: true + + - name: Ensure /etc/apt/keyrings exists (mode 0755) + shell: install -d -m 0755 /etc/apt/keyrings + sudo: true + creates: /etc/apt/keyrings + + - name: Fetch + install Docker's signing key (only if absent) + shell: | + curl -fsSL https://download.docker.com/linux/debian/gpg \ + | sudo gpg --dearmor --yes -o /etc/apt/keyrings/docker.gpg + sudo chmod a+r /etc/apt/keyrings/docker.gpg + sudo: true + when: '[ ! -f /etc/apt/keyrings/docker.gpg ]' + + - name: Add Docker's APT source + # codename is whatever lsb_release says — bookworm on this fleet. + shell: | + codename=$(lsb_release -cs) + arch=$(dpkg --print-architecture) + echo "deb [arch=${arch} signed-by=/etc/apt/keyrings/docker.gpg] \ +https://download.docker.com/linux/debian ${codename} stable" \ + | sudo tee /etc/apt/sources.list.d/docker.list >/dev/null + sudo: true + when: '[ ! -f /etc/apt/sources.list.d/docker.list ]' + + - name: apt-get update (now sees Docker's repo) + shell: apt-get update -qq + sudo: true + + - name: Show candidate version of docker-ce + shell: apt-cache policy docker-ce | head -5 + sudo: true + changed_when: "false" + + # ── swap packages ─────────────────────────────────────────────────── + + - name: Remove Debian's docker.io packages (PRESERVE /var/lib/docker) + # apt remove (not purge) keeps /var/lib/docker/* in place — images, + # volumes, container metadata all survive the swap. + shell: DEBIAN_FRONTEND=noninteractive apt-get remove -y {{ pkgs_to_remove }} || true + sudo: true + + - name: Install docker-ce + plugins from Docker's repo + shell: DEBIAN_FRONTEND=noninteractive apt-get install -y {{ pkgs_to_install }} + sudo: true + + - name: Enable + start the new daemon (idempotent) + shell: systemctl enable --now docker + sudo: true + + - name: Confirm new daemon is responsive + report its version + shell: docker version --format 'client={{.Client.Version}} server={{.Server.Version}} api={{.Server.APIVersion}}' + changed_when: "false" + + # ── bring stacks back up ──────────────────────────────────────────── + + - name: Start every previously-running compose stack + # Use the snapshot we took before the swap. Stacks whose containers + # have restart=unless-stopped will already be coming back via the + # daemon — `compose up -d` is idempotent and surfaces failures we + # might otherwise miss. + shell: | + set +e + while read stack; do + dir="/opt/docker/compose/$stack" + if [ -d "$dir" ]; then + echo " starting $stack" + (cd "$dir" && docker compose up -d) || echo " FAILED — investigate $stack" + fi + done < /tmp/docker-pre-upgrade-stacks.txt + true + + - name: Brief settle pause then list container states + shell: sleep 8 && docker ps --format 'table {{.Names}}\t{{.Status}}' + changed_when: "false" + +verify: + - name: docker daemon responding + shell: docker info >/dev/null + changed_when: "false" + + - name: docker server version is 24+ (was 20.10) + shell: | + v=$(docker version --format '{{.Server.Version}}' | cut -d. -f1) + [ "$v" -ge 24 ] + changed_when: "false" + + - name: docker compose plugin v2 installed + shell: docker compose version | grep -qE 'v2\.[0-9]+\.[0-9]+' + changed_when: "false" + + - name: at least one previously-running stack is back up + # If the host had no running stacks pre-upgrade (e.g. nh3-docker may + # be empty), this verify passes trivially. + shell: | + [ ! -s /tmp/docker-pre-upgrade-stacks.txt ] && exit 0 + head -1 /tmp/docker-pre-upgrade-stacks.txt | xargs -I {} docker ps \ + --filter 'label=com.docker.compose.project={}' --format '{{.Names}}' \ + | grep -q . + changed_when: "false"