vh
ccc0df6870
The restart loop runs as the deploy identity, not root, and a stack .env is allowed to be root-owned 0600. compose bails on the unreadable file before doing anything, so the stack was reported FAILED while restart=unless-stopped had already brought it back healthy — a false failure, which is worse than a quiet one because it trains readers to skim the failure lines. Retry under sudo -n before calling it a failure, and print compose's own output either way. Verified on nh3-dev against beszel: plain attempt rc=1 'open /opt/docker/compose/beszel/.env: permission denied', sudo retry rc=0 'Container beszel-agent Started', container back to healthy. The happy path is unchanged — the sudo attempt only fires after a failure. Also record that tts-dev migrated talk from ~/talk into /opt/docker/compose/talk, which removes the one stack on this host that was invisible to anything walking that path.
260 lines
11 KiB
YAML
260 lines
11 KiB
YAML
# upgrade-docker-ce — migrate a host from Debian's docker.io (20.10.x,
|
|
# bookworm-packaged) to Docker's official docker-ce repo (28+).
|
|
#
|
|
# Bookworm's docker.io stays pinned at 20.10.24, which:
|
|
# * uses an old client/daemon API (1.41) that newer compose clients
|
|
# (1.52+) refuse to talk to → "client version 1.52 is too new"
|
|
# during builds
|
|
# * is EOL upstream — docker.io upstream doesn't ship to it anymore
|
|
# * is missing modern buildx driver versions our newer client expects
|
|
#
|
|
# This playbook handles the migration on one host at a time. Stops
|
|
# every running stack via `docker compose stop`, removes the old
|
|
# packages (preserving /var/lib/docker/), adds Docker's signed APT
|
|
# repo, installs docker-ce + docker-compose-plugin + containerd.io,
|
|
# starts the new daemon, and brings each stack back up via
|
|
# `docker compose up -d` (compose's restart=unless-stopped also
|
|
# auto-restarts containers when the daemon comes back, but doing
|
|
# them explicitly per-stack lets us see failures cleanly).
|
|
#
|
|
# Usage:
|
|
# scripts/elway <host> --playbook playbooks/upgrade-docker-ce.yaml
|
|
#
|
|
# Recommended host order (least → most blast radius):
|
|
# 1. nh3-docker
|
|
# 2. esh-docker-vm
|
|
# 3. ana-docker
|
|
#
|
|
# Verify after each before moving to the next:
|
|
# ssh <host> 'docker version --format "{{.Server.Version}}"'
|
|
# ssh <host> 'docker ps --format "{{.Names}}\t{{.Status}}" | head'
|
|
#
|
|
# Rollback (if a daemon won't start, or a container errors out):
|
|
# ssh <host> 'sudo apt install --allow-downgrades docker.io'
|
|
# then re-add the docker.io packages from /tmp/docker-pre-upgrade.txt
|
|
#
|
|
# /var/lib/docker/ is preserved throughout (apt remove, not purge),
|
|
# so volumes / images / containers survive the package swap. The
|
|
# overlay2 storage driver is the default on both packages, so no
|
|
# data migration needed.
|
|
|
|
vars:
|
|
pkgs_to_remove: docker.io docker-compose docker-compose-plugin docker-buildx-plugin docker-doc docker
|
|
pkgs_to_install: docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
|
|
|
|
steps:
|
|
# ── snapshot + stop ─────────────────────────────────────────────────
|
|
|
|
- name: Snapshot current docker package versions (rollback reference)
|
|
shell: |
|
|
dpkg -l | awk '/^ii\s+(docker|containerd)/ {print $2 "=" $3}' \
|
|
| sudo tee /tmp/docker-pre-upgrade.txt >/dev/null
|
|
cat /tmp/docker-pre-upgrade.txt
|
|
sudo: true
|
|
changed_when: "false"
|
|
|
|
- name: List currently-running compose stacks (snapshot for restart)
|
|
shell: |
|
|
docker ps --format '{{.Label "com.docker.compose.project"}}' \
|
|
| sort -u | grep -v '^$' \
|
|
| tee /tmp/docker-pre-upgrade-stacks.txt
|
|
changed_when: "false"
|
|
|
|
- name: Stop every running stack via docker compose
|
|
# Walk each /opt/docker/compose/<stack>/ dir that has a running
|
|
# container and `compose stop` it. Skips dirs without a running
|
|
# stack so reruns don't error.
|
|
shell: |
|
|
set +e
|
|
for dir in /opt/docker/compose/*/; do
|
|
stack=$(basename "$dir")
|
|
if docker ps --format '{{.Label "com.docker.compose.project"}}' | grep -q "^${stack}$"; then
|
|
echo " stopping $stack"
|
|
(cd "$dir" && docker compose stop) || echo " (no-op or failed)"
|
|
fi
|
|
done
|
|
true
|
|
|
|
# ── add Docker's official APT repo ──────────────────────────────────
|
|
|
|
- name: Install prereqs for the new APT repo
|
|
shell: apt-get install -y ca-certificates curl gnupg
|
|
sudo: true
|
|
|
|
- name: Ensure /etc/apt/keyrings exists (mode 0755)
|
|
shell: install -d -m 0755 /etc/apt/keyrings
|
|
sudo: true
|
|
creates: /etc/apt/keyrings
|
|
|
|
- name: Fetch + install Docker's signing key (only if absent)
|
|
shell: |
|
|
curl -fsSL https://download.docker.com/linux/debian/gpg \
|
|
| sudo gpg --dearmor --yes -o /etc/apt/keyrings/docker.gpg
|
|
sudo chmod a+r /etc/apt/keyrings/docker.gpg
|
|
sudo: true
|
|
when: '[ ! -f /etc/apt/keyrings/docker.gpg ]'
|
|
|
|
- name: Add Docker's APT source
|
|
# codename is whatever lsb_release says — bookworm on this fleet.
|
|
shell: |
|
|
codename=$(lsb_release -cs)
|
|
arch=$(dpkg --print-architecture)
|
|
url="https://download.docker.com/linux/debian"
|
|
line="deb [arch=${arch} signed-by=/etc/apt/keyrings/docker.gpg] ${url} ${codename} stable"
|
|
echo "${line}" | sudo tee /etc/apt/sources.list.d/docker.list >/dev/null
|
|
sudo: true
|
|
when: '[ ! -f /etc/apt/sources.list.d/docker.list ]'
|
|
|
|
- name: apt-get update (now sees Docker's repo)
|
|
shell: apt-get update -qq
|
|
sudo: true
|
|
|
|
- name: Show candidate version of docker-ce
|
|
shell: apt-cache policy docker-ce | head -5
|
|
sudo: true
|
|
changed_when: "false"
|
|
|
|
# ── swap packages ───────────────────────────────────────────────────
|
|
|
|
- name: Remove Debian's docker.io packages (PRESERVE /var/lib/docker)
|
|
# apt remove (not purge) keeps /var/lib/docker/* in place — images,
|
|
# volumes, container metadata all survive the swap.
|
|
shell: DEBIAN_FRONTEND=noninteractive apt-get remove -y {{ pkgs_to_remove }} || true
|
|
sudo: true
|
|
|
|
- name: Install docker-ce + plugins from Docker's repo
|
|
shell: DEBIAN_FRONTEND=noninteractive apt-get install -y {{ pkgs_to_install }}
|
|
sudo: true
|
|
|
|
- name: Rewrite docker.service drop-in to match the new package layout
|
|
# Two related fixes for the systemd unit, learned the hard way on
|
|
# nh3-docker. The docker.io era left an
|
|
# /etc/systemd/system/docker.service.d/override.conf hardcoding
|
|
# ExecStart=/usr/sbin/dockerd (Debian package path) and usually
|
|
# adding `-H tcp://...` for remote discovery.
|
|
#
|
|
# After the swap to docker-ce two things are wrong:
|
|
# 1. /usr/sbin/dockerd doesn't exist — docker-ce installs at
|
|
# /usr/bin/dockerd. Daemon fails status=203/EXEC.
|
|
# 2. If daemon.json defines `hosts:` (typical here for the
|
|
# 0.0.0.0:2375 homepage-discovery binding), then ANY `-H`
|
|
# flag in ExecStart conflicts: dockerd refuses to start with
|
|
# "conflicting host options".
|
|
#
|
|
# The shipped docker-ce unit's ExecStart is `dockerd -H fd://
|
|
# --containerd=/run/containerd/containerd.sock` — that `-H fd://`
|
|
# ALSO conflicts with daemon.json hosts:. So we always need an
|
|
# override that strips `-H` entirely when daemon.json defines
|
|
# hosts.
|
|
#
|
|
# Procedure:
|
|
# * Back up any existing override (preserves it as .pre-upgrade)
|
|
# * Probe /etc/docker/daemon.json for a `hosts:` setting
|
|
# * If hosts: is defined → install an override that clears the
|
|
# base unit's ExecStart and replaces it WITHOUT -H
|
|
# * Otherwise no override is needed (base unit's `-H fd://` is fine)
|
|
shell: |
|
|
d=/etc/systemd/system/docker.service.d
|
|
f="$d/override.conf"
|
|
sudo install -d -m 0755 "$d"
|
|
if [ -f "$f" ]; then
|
|
echo " backing up existing $f → ${f}.pre-upgrade"
|
|
sudo mv "$f" "${f}.pre-upgrade"
|
|
fi
|
|
if [ -f /etc/docker/daemon.json ] && grep -q '"hosts"' /etc/docker/daemon.json; then
|
|
echo " daemon.json has hosts: — installing override that strips -H from ExecStart"
|
|
sudo tee "$f" >/dev/null <<'OVR'
|
|
[Service]
|
|
ExecStart=
|
|
ExecStart=/usr/bin/dockerd --containerd=/run/containerd/containerd.sock
|
|
OVR
|
|
else
|
|
echo " daemon.json has no hosts: — base unit's ExecStart is fine, no override needed"
|
|
fi
|
|
sudo: true
|
|
|
|
- name: systemctl daemon-reload (so the new unit + override are loaded)
|
|
shell: systemctl daemon-reload
|
|
sudo: true
|
|
|
|
- name: Reset the docker.service failed state (3 failed starts during
|
|
the install often leaves it in start-rate-limited state — reset
|
|
so the next start isn't immediately denied)
|
|
shell: systemctl reset-failed docker.service
|
|
sudo: true
|
|
|
|
- name: Enable + start the new daemon (idempotent)
|
|
shell: systemctl enable --now docker
|
|
sudo: true
|
|
|
|
- name: Confirm new daemon is responsive + report its version
|
|
shell: docker version --format 'client={{.Client.Version}} server={{.Server.Version}} api={{.Server.APIVersion}}'
|
|
changed_when: "false"
|
|
|
|
# ── bring stacks back up ────────────────────────────────────────────
|
|
|
|
- name: Start every previously-running compose stack
|
|
# Use the snapshot we took before the swap. Stacks whose containers
|
|
# have restart=unless-stopped will already be coming back via the
|
|
# daemon — `compose up -d` is idempotent and surfaces failures we
|
|
# might otherwise miss.
|
|
shell: |
|
|
set +e
|
|
while read stack; do
|
|
dir="/opt/docker/compose/$stack"
|
|
if [ -d "$dir" ]; then
|
|
echo " starting $stack"
|
|
out=$(cd "$dir" && docker compose up -d 2>&1); rc=$?
|
|
if [ $rc -ne 0 ]; then
|
|
# This loop runs as the deploy identity, not root, and a stack
|
|
# .env is allowed to be root-owned 0600 — unreadable to us, so
|
|
# compose bails before it does anything. Retry under sudo before
|
|
# calling it a failure. (nh3-dev 2026-09-14: beszel reported
|
|
# "FAILED — investigate beszel" on exactly this while
|
|
# restart=unless-stopped had already brought it back healthy. A
|
|
# false FAILED in automation output is worse than a quiet one —
|
|
# it trains readers to skim the failure lines.)
|
|
out=$(cd "$dir" && sudo -n docker compose up -d 2>&1); rc=$?
|
|
fi
|
|
printf '%s\n' "$out"
|
|
[ $rc -eq 0 ] || echo " FAILED — investigate $stack"
|
|
fi
|
|
done < /tmp/docker-pre-upgrade-stacks.txt
|
|
true
|
|
|
|
- name: Brief settle pause then list container states
|
|
shell: sleep 8 && docker ps --format 'table {{.Names}}\t{{.Status}}'
|
|
changed_when: "false"
|
|
|
|
verify:
|
|
- name: docker daemon responding
|
|
shell: docker info >/dev/null
|
|
changed_when: "false"
|
|
|
|
- name: docker server version is 24+ (was 20.10)
|
|
shell: |
|
|
v=$(docker version --format '{{.Server.Version}}' | cut -d. -f1)
|
|
[ "$v" -ge 24 ]
|
|
changed_when: "false"
|
|
|
|
- name: docker compose plugin v2+ installed
|
|
# docker-ce 29 ships docker-compose-plugin renumbered to v5.x (was
|
|
# v2.x with docker-ce 26-28). Both are the same Compose v2 codebase
|
|
# under the hood — Docker just realigned the plugin's major number.
|
|
# Accept anything ≥ 2 so this verify keeps working across future
|
|
# plugin renumbers.
|
|
shell: |
|
|
major=$(docker compose version --short | sed 's/^v//' | cut -d. -f1)
|
|
[ "$major" -ge 2 ]
|
|
changed_when: "false"
|
|
|
|
- name: at least one previously-running stack is back up
|
|
# If the host had no running stacks pre-upgrade (e.g. nh3-docker may
|
|
# be empty), this verify passes trivially.
|
|
shell: |
|
|
[ ! -s /tmp/docker-pre-upgrade-stacks.txt ] && exit 0
|
|
head -1 /tmp/docker-pre-upgrade-stacks.txt | xargs -I {} docker ps \
|
|
--filter 'label=com.docker.compose.project={}' --format '{{.Names}}' \
|
|
| grep -q .
|
|
changed_when: "false"
|