104 lines
6.4 KiB
Bash
Executable File
104 lines
6.4 KiB
Bash
Executable File
#!/bin/bash
|
|
# One-off, 2026-10-05 (Prime, park 99): move fv-ml1's Docker data off the root SSD mirror (zroot) onto tank,
|
|
# and patch the host in the same window so the model seats bounce once, not twice.
|
|
#
|
|
# scp this to fv-ml1, then: sudo systemd-run --unit=docker-move --collect /root/fv-ml1-docker-to-tank.sh
|
|
# progress: /var/log/docker-move-20261005.log
|
|
#
|
|
# Shape: a sparse 1 TiB zvol tank/docker, ext4, mounted AT /var/lib/docker (fstab, nofail, after
|
|
# zfs-volume-wait). The path does not change, so nothing that names /var/lib/docker (restic, volumes, compose)
|
|
# needs editing. Why a zvol and not a dataset: Docker's overlay2 driver refuses a ZFS backing filesystem, and an
|
|
# unpinned daemon would silently pick the zfs graph driver and lose every image. ext4 under overlay2 is the
|
|
# canonical setup. docker.service gets RequiresMountsFor=/var/lib/docker, so if the zvol is ever missing at boot
|
|
# docker stays down instead of starting fresh on an empty root directory.
|
|
#
|
|
# Patching: everything except the packages that could take the box or its GPUs down at the next boot, which are
|
|
# held and parked for Prime: the ZFS stack (2.3.2 -> 2.3.9 rebuilds the module root is mounted with) and
|
|
# nvidia-container-toolkit (the GPU hook every seat starts through). No reboot.
|
|
#
|
|
# The old tree stays at /var/lib/docker.old-zroot until the seats are verified; deleting it is a separate,
|
|
# literal-path step.
|
|
#
|
|
# RUN 1 (01:48) ABORTED ON ITS OWN GUARD, and docker was restarted on the untouched tree at 02:22. Cause: pass 1 ran
|
|
# while containers were up, and rsync (without -x) walked INTO a mounted overlay2/<id>/merged view and copied a
|
|
# whole container rootfs (195,206 entries, 21 G) that --delete never removed. RUN 2 below is one-filesystem (-x),
|
|
# excludes overlay2/*/merged/ content (always an empty mountpoint on disk), deletes anything excluded already copied,
|
|
# and verifies with sorted path lists instead of a bare count.
|
|
set -euo pipefail
|
|
exec > >(tee -a /var/log/docker-move-20261005.log) 2>&1
|
|
step() { echo; echo "=== $(date +%H:%M:%S) $*"; }
|
|
|
|
step "pre-state"
|
|
docker ps --format '{{.Names}}' | sort > /root/docker-move-running-before.txt
|
|
cat /root/docker-move-running-before.txt | tr '\n' ' '; echo
|
|
df -h / /tank | cat
|
|
|
|
step "zvol tank/docker (sparse 1T) + ext4"
|
|
zfs list tank/docker >/dev/null 2>&1 || zfs create -s -V 1T tank/docker
|
|
udevadm settle
|
|
for i in $(seq 1 30); do [ -e /dev/zvol/tank/docker ] && break; sleep 1; done
|
|
test -e /dev/zvol/tank/docker
|
|
blkid /dev/zvol/tank/docker >/dev/null 2>&1 || mkfs.ext4 -q -L docker -E lazy_itable_init=1,lazy_journal_init=1 /dev/zvol/tank/docker
|
|
mkdir -p /mnt/docker-new
|
|
mountpoint -q /mnt/docker-new || mount /dev/zvol/tank/docker /mnt/docker-new
|
|
|
|
step "pass 1 (live copy)"
|
|
RS=(rsync -aHAXSx --numeric-ids --exclude='/overlay2/*/merged/*')
|
|
"${RS[@]}" /var/lib/docker/ /mnt/docker-new/ || [ $? -eq 24 ] # 24 = files vanished under a live copy; pass 2 fixes it
|
|
|
|
step "stop docker"
|
|
# From here until the swap, ANY failure puts docker back on the untouched old tree by itself
|
|
# (run 1 left the seats down for 4 min waiting for a human).
|
|
SWAPPED=0
|
|
restore() { rc=$?; if [ "$SWAPPED" = 0 ] && [ "$rc" != 0 ]; then echo "!!! failed before swap (rc=$rc): restoring docker on the old tree"; umount /mnt/docker-new 2>/dev/null || true; systemctl start docker.service && echo "docker restarted on the old tree"; fi; }
|
|
trap restore EXIT
|
|
systemctl stop docker.socket docker.service
|
|
sleep 2
|
|
if pgrep -x dockerd >/dev/null; then echo 'dockerd still running, abort before the final pass'; exit 1; fi
|
|
|
|
step "pass 2 (final, --delete)"
|
|
"${RS[@]}" --delete --delete-excluded /var/lib/docker/ /mnt/docker-new/
|
|
(cd /var/lib/docker && find . -xdev -not -path './overlay2/*/merged/*' | LC_ALL=C sort) > /root/docker-move-old.list
|
|
(cd /mnt/docker-new && find . -xdev -not -path './overlay2/*/merged/*' -not -path './lost+found' | LC_ALL=C sort) > /root/docker-move-new.list
|
|
echo "entries: old=$(wc -l < /root/docker-move-old.list) new=$(wc -l < /root/docker-move-new.list)"
|
|
cmp -s /root/docker-move-old.list /root/docker-move-new.list || { echo 'path lists differ, abort (docker is stopped, old tree untouched):'; diff /root/docker-move-old.list /root/docker-move-new.list | head -20; exit 1; }
|
|
|
|
step "swap: old tree aside, zvol mounted at /var/lib/docker"
|
|
umount /mnt/docker-new
|
|
mv /var/lib/docker /var/lib/docker.old-zroot
|
|
SWAPPED=1
|
|
install -d -m 710 /var/lib/docker
|
|
grep -q '^/dev/zvol/tank/docker ' /etc/fstab || \
|
|
echo '/dev/zvol/tank/docker /var/lib/docker ext4 defaults,nofail,x-systemd.requires=zfs-volume-wait.service,x-systemd.after=zfs-volume-wait.service 0 2' >> /etc/fstab
|
|
install -d /etc/systemd/system/docker.service.d
|
|
printf '[Unit]\n# /var/lib/docker is the tank/docker zvol (2026-10-05). Never start on an empty root dir.\nRequiresMountsFor=/var/lib/docker\n' \
|
|
> /etc/systemd/system/docker.service.d/10-requires-docker-mount.conf
|
|
systemctl daemon-reload
|
|
mount /var/lib/docker
|
|
findmnt -no SOURCE,FSTYPE /var/lib/docker
|
|
|
|
step "patch (holds: ZFS stack, nvidia-container-toolkit)"
|
|
apt-mark hold zfs-dkms zfsutils-linux zfs-zed zfs-initramfs libzfs6linux libzpool6linux libnvpair3linux libuutil3linux \
|
|
nvidia-container-toolkit nvidia-container-toolkit-base libnvidia-container-tools libnvidia-container1 2>&1 | tail -3 || true
|
|
DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l apt-get -q -y -o Dpkg::Options::=--force-confold upgrade --with-new-pkgs
|
|
echo "still upgradable (held/kept back):"; apt list --upgradable 2>/dev/null | tail -n +2 | cut -d/ -f1 | tr '\n' ' '; echo
|
|
test -f /var/run/reboot-required && echo "REBOOT REQUIRED (deferred): $(cat /var/run/reboot-required.pkgs 2>/dev/null | tr '\n' ' ')"
|
|
|
|
step "start docker"
|
|
systemctl start docker.service
|
|
docker info --format 'root={{.DockerRootDir}} driver={{.Driver}} ver={{.ServerVersion}} images={{.Images}}'
|
|
|
|
step "wait for the same containers to be running + healthy (up to 25 min)"
|
|
for i in $(seq 1 150); do
|
|
docker ps --format '{{.Names}}' | sort > /root/docker-move-running-after.txt
|
|
missing=$(comm -23 /root/docker-move-running-before.txt /root/docker-move-running-after.txt | tr '\n' ' ')
|
|
# `|| true`: grep finding nothing is the SUCCESS case here, and set -e would kill the script on it
|
|
starting=$(docker ps --format '{{.Names}} {{.Status}}' | grep -E 'health: starting|unhealthy' | cut -d' ' -f1 | tr '\n' ' ' || true)
|
|
[ -z "$missing" ] && [ -z "$starting" ] && break
|
|
sleep 10
|
|
done
|
|
echo "missing: ${missing:-none}"; echo "starting/unhealthy: ${starting:-none}"
|
|
docker ps --format '{{.Names}}|{{.Status}}' | cat
|
|
df -h / /var/lib/docker | cat
|
|
step "done"
|