vh
a3ab1a7b6c
nh3-docker's daemon kept failing post-package-swap with status=203
even after daemon-reload. Root cause: a stale
/etc/systemd/system/docker.service.d/override.conf from the docker.io
era hardcoding ExecStart=/usr/sbin/dockerd. The override (a) points
at the no-longer-existing path, AND (b) typically also adds
-H tcp://... which now duplicates the hosts: setting in
/etc/docker/daemon.json — dockerd refuses to start when both define
hosts ('conflicting host options').
Daemon.json is the modern way to expose the TCP socket. The
override is redundant and wrong. Move it aside (preserve a
.pre-upgrade copy for forensics), then daemon-reload, then start.
Should let esh-docker-vm and ana-docker upgrades go through cleanly
without the manual debug loop nh3-docker required.
206 lines
8.4 KiB
YAML
206 lines
8.4 KiB
YAML
# upgrade-docker-ce — migrate a host from Debian's docker.io (20.10.x,
|
|
# bookworm-packaged) to Docker's official docker-ce repo (28+).
|
|
#
|
|
# Bookworm's docker.io stays pinned at 20.10.24, which:
|
|
# * uses an old client/daemon API (1.41) that newer compose clients
|
|
# (1.52+) refuse to talk to → "client version 1.52 is too new"
|
|
# during builds
|
|
# * is EOL upstream — docker.io upstream doesn't ship to it anymore
|
|
# * is missing modern buildx driver versions our newer client expects
|
|
#
|
|
# This playbook handles the migration on one host at a time. Stops
|
|
# every running stack via `docker compose stop`, removes the old
|
|
# packages (preserving /var/lib/docker/), adds Docker's signed APT
|
|
# repo, installs docker-ce + docker-compose-plugin + containerd.io,
|
|
# starts the new daemon, and brings each stack back up via
|
|
# `docker compose up -d` (compose's restart=unless-stopped also
|
|
# auto-restarts containers when the daemon comes back, but doing
|
|
# them explicitly per-stack lets us see failures cleanly).
|
|
#
|
|
# Usage:
|
|
# scripts/elway <host> --playbook playbooks/upgrade-docker-ce.yaml
|
|
#
|
|
# Recommended host order (least → most blast radius):
|
|
# 1. nh3-docker
|
|
# 2. esh-docker-vm
|
|
# 3. ana-docker
|
|
#
|
|
# Verify after each before moving to the next:
|
|
# ssh <host> 'docker version --format "{{.Server.Version}}"'
|
|
# ssh <host> 'docker ps --format "{{.Names}}\t{{.Status}}" | head'
|
|
#
|
|
# Rollback (if a daemon won't start, or a container errors out):
|
|
# ssh <host> 'sudo apt install --allow-downgrades docker.io'
|
|
# then re-add the docker.io packages from /tmp/docker-pre-upgrade.txt
|
|
#
|
|
# /var/lib/docker/ is preserved throughout (apt remove, not purge),
|
|
# so volumes / images / containers survive the package swap. The
|
|
# overlay2 storage driver is the default on both packages, so no
|
|
# data migration needed.
|
|
|
|
vars:
|
|
pkgs_to_remove: docker.io docker-compose docker-compose-plugin docker-buildx-plugin docker-doc docker
|
|
pkgs_to_install: docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
|
|
|
|
steps:
|
|
# ── snapshot + stop ─────────────────────────────────────────────────
|
|
|
|
- name: Snapshot current docker package versions (rollback reference)
|
|
shell: |
|
|
dpkg -l | awk '/^ii\s+(docker|containerd)/ {print $2 "=" $3}' \
|
|
| sudo tee /tmp/docker-pre-upgrade.txt >/dev/null
|
|
cat /tmp/docker-pre-upgrade.txt
|
|
sudo: true
|
|
changed_when: "false"
|
|
|
|
- name: List currently-running compose stacks (snapshot for restart)
|
|
shell: |
|
|
docker ps --format '{{.Label "com.docker.compose.project"}}' \
|
|
| sort -u | grep -v '^$' \
|
|
| tee /tmp/docker-pre-upgrade-stacks.txt
|
|
changed_when: "false"
|
|
|
|
- name: Stop every running stack via docker compose
|
|
# Walk each /opt/docker/compose/<stack>/ dir that has a running
|
|
# container and `compose stop` it. Skips dirs without a running
|
|
# stack so reruns don't error.
|
|
shell: |
|
|
set +e
|
|
for dir in /opt/docker/compose/*/; do
|
|
stack=$(basename "$dir")
|
|
if docker ps --format '{{.Label "com.docker.compose.project"}}' | grep -q "^${stack}$"; then
|
|
echo " stopping $stack"
|
|
(cd "$dir" && docker compose stop) || echo " (no-op or failed)"
|
|
fi
|
|
done
|
|
true
|
|
|
|
# ── add Docker's official APT repo ──────────────────────────────────
|
|
|
|
- name: Install prereqs for the new APT repo
|
|
shell: apt-get install -y ca-certificates curl gnupg
|
|
sudo: true
|
|
|
|
- name: Ensure /etc/apt/keyrings exists (mode 0755)
|
|
shell: install -d -m 0755 /etc/apt/keyrings
|
|
sudo: true
|
|
creates: /etc/apt/keyrings
|
|
|
|
- name: Fetch + install Docker's signing key (only if absent)
|
|
shell: |
|
|
curl -fsSL https://download.docker.com/linux/debian/gpg \
|
|
| sudo gpg --dearmor --yes -o /etc/apt/keyrings/docker.gpg
|
|
sudo chmod a+r /etc/apt/keyrings/docker.gpg
|
|
sudo: true
|
|
when: '[ ! -f /etc/apt/keyrings/docker.gpg ]'
|
|
|
|
- name: Add Docker's APT source
|
|
# codename is whatever lsb_release says — bookworm on this fleet.
|
|
shell: |
|
|
codename=$(lsb_release -cs)
|
|
arch=$(dpkg --print-architecture)
|
|
url="https://download.docker.com/linux/debian"
|
|
line="deb [arch=${arch} signed-by=/etc/apt/keyrings/docker.gpg] ${url} ${codename} stable"
|
|
echo "${line}" | sudo tee /etc/apt/sources.list.d/docker.list >/dev/null
|
|
sudo: true
|
|
when: '[ ! -f /etc/apt/sources.list.d/docker.list ]'
|
|
|
|
- name: apt-get update (now sees Docker's repo)
|
|
shell: apt-get update -qq
|
|
sudo: true
|
|
|
|
- name: Show candidate version of docker-ce
|
|
shell: apt-cache policy docker-ce | head -5
|
|
sudo: true
|
|
changed_when: "false"
|
|
|
|
# ── swap packages ───────────────────────────────────────────────────
|
|
|
|
- name: Remove Debian's docker.io packages (PRESERVE /var/lib/docker)
|
|
# apt remove (not purge) keeps /var/lib/docker/* in place — images,
|
|
# volumes, container metadata all survive the swap.
|
|
shell: DEBIAN_FRONTEND=noninteractive apt-get remove -y {{ pkgs_to_remove }} || true
|
|
sudo: true
|
|
|
|
- name: Install docker-ce + plugins from Docker's repo
|
|
shell: DEBIAN_FRONTEND=noninteractive apt-get install -y {{ pkgs_to_install }}
|
|
sudo: true
|
|
|
|
- name: Move aside any stale docker.service drop-in override
|
|
# The docker.io era often left an /etc/systemd/system/docker.service.d/
|
|
# override.conf adding `-H tcp://...` to ExecStart for remote docker
|
|
# discovery. After the swap, the override (a) hard-codes
|
|
# /usr/sbin/dockerd which no longer exists, and (b) duplicates the
|
|
# `hosts:` setting in /etc/docker/daemon.json — dockerd refuses to
|
|
# start when both are set ("conflicting host options"). daemon.json
|
|
# is the modern way; the override is redundant.
|
|
shell: |
|
|
f=/etc/systemd/system/docker.service.d/override.conf
|
|
if [ -f "$f" ]; then
|
|
echo " moving aside $f"
|
|
sudo mv "$f" "${f}.pre-upgrade"
|
|
fi
|
|
sudo: true
|
|
|
|
- name: systemctl daemon-reload (CRITICAL — package swap put dockerd at
|
|
/usr/bin/dockerd; without reload, systemd keeps the old unit's
|
|
ExecStart pointing at /usr/sbin/dockerd from docker.io and fails
|
|
with status=203/EXEC "No such file or directory")
|
|
shell: systemctl daemon-reload
|
|
sudo: true
|
|
|
|
- name: Enable + start the new daemon (idempotent)
|
|
shell: systemctl enable --now docker
|
|
sudo: true
|
|
|
|
- name: Confirm new daemon is responsive + report its version
|
|
shell: docker version --format 'client={{.Client.Version}} server={{.Server.Version}} api={{.Server.APIVersion}}'
|
|
changed_when: "false"
|
|
|
|
# ── bring stacks back up ────────────────────────────────────────────
|
|
|
|
- name: Start every previously-running compose stack
|
|
# Use the snapshot we took before the swap. Stacks whose containers
|
|
# have restart=unless-stopped will already be coming back via the
|
|
# daemon — `compose up -d` is idempotent and surfaces failures we
|
|
# might otherwise miss.
|
|
shell: |
|
|
set +e
|
|
while read stack; do
|
|
dir="/opt/docker/compose/$stack"
|
|
if [ -d "$dir" ]; then
|
|
echo " starting $stack"
|
|
(cd "$dir" && docker compose up -d) || echo " FAILED — investigate $stack"
|
|
fi
|
|
done < /tmp/docker-pre-upgrade-stacks.txt
|
|
true
|
|
|
|
- name: Brief settle pause then list container states
|
|
shell: sleep 8 && docker ps --format 'table {{.Names}}\t{{.Status}}'
|
|
changed_when: "false"
|
|
|
|
verify:
|
|
- name: docker daemon responding
|
|
shell: docker info >/dev/null
|
|
changed_when: "false"
|
|
|
|
- name: docker server version is 24+ (was 20.10)
|
|
shell: |
|
|
v=$(docker version --format '{{.Server.Version}}' | cut -d. -f1)
|
|
[ "$v" -ge 24 ]
|
|
changed_when: "false"
|
|
|
|
- name: docker compose plugin v2 installed
|
|
shell: docker compose version | grep -qE 'v2\.[0-9]+\.[0-9]+'
|
|
changed_when: "false"
|
|
|
|
- name: at least one previously-running stack is back up
|
|
# If the host had no running stacks pre-upgrade (e.g. nh3-docker may
|
|
# be empty), this verify passes trivially.
|
|
shell: |
|
|
[ ! -s /tmp/docker-pre-upgrade-stacks.txt ] && exit 0
|
|
head -1 /tmp/docker-pre-upgrade-stacks.txt | xargs -I {} docker ps \
|
|
--filter 'label=com.docker.compose.project={}' --format '{{.Names}}' \
|
|
| grep -q .
|
|
changed_when: "false"
|