diff --git a/scripts/oneoff/fv-ml1-docker-to-tank-20261005.sh b/scripts/oneoff/fv-ml1-docker-to-tank-20261005.sh index a8e8b7a..9a613ce 100755 --- a/scripts/oneoff/fv-ml1-docker-to-tank-20261005.sh +++ b/scripts/oneoff/fv-ml1-docker-to-tank-20261005.sh @@ -18,6 +18,12 @@ # # The old tree stays at /var/lib/docker.old-zroot until the seats are verified; deleting it is a separate, # literal-path step. +# +# RUN 1 (01:48) ABORTED ON ITS OWN GUARD, and docker was restarted on the untouched tree at 02:22. Cause: pass 1 ran +# while containers were up, and rsync (without -x) walked INTO a mounted overlay2//merged view and copied a +# whole container rootfs (195,206 entries, 21 G) that --delete never removed. RUN 2 below is one-filesystem (-x), +# excludes overlay2/*/merged/ content (always an empty mountpoint on disk), deletes anything excluded already copied, +# and verifies with sorted path lists instead of a bare count. set -euo pipefail exec > >(tee -a /var/log/docker-move-20261005.log) 2>&1 step() { echo; echo "=== $(date +%H:%M:%S) $*"; } @@ -37,23 +43,30 @@ mkdir -p /mnt/docker-new mountpoint -q /mnt/docker-new || mount /dev/zvol/tank/docker /mnt/docker-new step "pass 1 (live copy)" -rsync -aHAXS --numeric-ids /var/lib/docker/ /mnt/docker-new/ +RS=(rsync -aHAXSx --numeric-ids --exclude='/overlay2/*/merged/*') +"${RS[@]}" /var/lib/docker/ /mnt/docker-new/ || [ $? -eq 24 ] # 24 = files vanished under a live copy; pass 2 fixes it step "stop docker" +# From here until the swap, ANY failure puts docker back on the untouched old tree by itself +# (run 1 left the seats down for 4 min waiting for a human). +SWAPPED=0 +restore() { rc=$?; if [ "$SWAPPED" = 0 ] && [ "$rc" != 0 ]; then echo "!!! failed before swap (rc=$rc): restoring docker on the old tree"; umount /mnt/docker-new 2>/dev/null || true; systemctl start docker.service && echo "docker restarted on the old tree"; fi; } +trap restore EXIT systemctl stop docker.socket docker.service sleep 2 if pgrep -x dockerd >/dev/null; then echo 'dockerd still running, abort before the final pass'; exit 1; fi step "pass 2 (final, --delete)" -rsync -aHAXS --numeric-ids --delete /var/lib/docker/ /mnt/docker-new/ -a=$(find /var/lib/docker -xdev | wc -l); b=$(find /mnt/docker-new -xdev | wc -l) -echo "entries: old=$a new=$b" -# lost+found exists only on the new ext4, so new = old + 1 -test "$((b - a))" -eq 1 || { echo 'entry count mismatch, abort (docker is stopped, old tree untouched)'; exit 1; } +"${RS[@]}" --delete --delete-excluded /var/lib/docker/ /mnt/docker-new/ +(cd /var/lib/docker && find . -xdev -not -path './overlay2/*/merged/*' | LC_ALL=C sort) > /root/docker-move-old.list +(cd /mnt/docker-new && find . -xdev -not -path './overlay2/*/merged/*' -not -path './lost+found' | LC_ALL=C sort) > /root/docker-move-new.list +echo "entries: old=$(wc -l < /root/docker-move-old.list) new=$(wc -l < /root/docker-move-new.list)" +cmp -s /root/docker-move-old.list /root/docker-move-new.list || { echo 'path lists differ, abort (docker is stopped, old tree untouched):'; diff /root/docker-move-old.list /root/docker-move-new.list | head -20; exit 1; } step "swap: old tree aside, zvol mounted at /var/lib/docker" umount /mnt/docker-new mv /var/lib/docker /var/lib/docker.old-zroot +SWAPPED=1 install -d -m 710 /var/lib/docker grep -q '^/dev/zvol/tank/docker ' /etc/fstab || \ echo '/dev/zvol/tank/docker /var/lib/docker ext4 defaults,nofail,x-systemd.requires=zfs-volume-wait.service,x-systemd.after=zfs-volume-wait.service 0 2' >> /etc/fstab