fix(oneoff): fv-ml1 docker move run 2 — one-filesystem rsync, skip overlay merged views, path-list verify, auto-restore before swap

scripts/oneoff/fv-ml1-docker-to-tank-20261005.sh

@@ -18,6 +18,12 @@
 #
 # The old tree stays at /var/lib/docker.old-zroot until the seats are verified; deleting it is a separate,
 # literal-path step.
+#
+# RUN 1 (01:48) ABORTED ON ITS OWN GUARD, and docker was restarted on the untouched tree at 02:22. Cause: pass 1 ran
+# while containers were up, and rsync (without -x) walked INTO a mounted overlay2/<id>/merged view and copied a
+# whole container rootfs (195,206 entries, 21 G) that --delete never removed. RUN 2 below is one-filesystem (-x),
+# excludes overlay2/*/merged/ content (always an empty mountpoint on disk), deletes anything excluded already copied,
+# and verifies with sorted path lists instead of a bare count.
 set -euo pipefail
 exec > >(tee -a /var/log/docker-move-20261005.log) 2>&1
 step() { echo; echo "=== $(date +%H:%M:%S) $*"; }
@@ -37,23 +43,30 @@ mkdir -p /mnt/docker-new
 mountpoint -q /mnt/docker-new || mount /dev/zvol/tank/docker /mnt/docker-new
 
 step "pass 1 (live copy)"
-rsync -aHAXS --numeric-ids /var/lib/docker/ /mnt/docker-new/
+RS=(rsync -aHAXSx --numeric-ids --exclude='/overlay2/*/merged/*')
+"${RS[@]}" /var/lib/docker/ /mnt/docker-new/ || [ $? -eq 24 ]   # 24 = files vanished under a live copy; pass 2 fixes it
 
 step "stop docker"
+# From here until the swap, ANY failure puts docker back on the untouched old tree by itself
+# (run 1 left the seats down for 4 min waiting for a human).
+SWAPPED=0
+restore() { rc=$?; if [ "$SWAPPED" = 0 ] && [ "$rc" != 0 ]; then echo "!!! failed before swap (rc=$rc): restoring docker on the old tree"; umount /mnt/docker-new 2>/dev/null || true; systemctl start docker.service && echo "docker restarted on the old tree"; fi; }
+trap restore EXIT
 systemctl stop docker.socket docker.service
 sleep 2
 if pgrep -x dockerd >/dev/null; then echo 'dockerd still running, abort before the final pass'; exit 1; fi
 
 step "pass 2 (final, --delete)"
-rsync -aHAXS --numeric-ids --delete /var/lib/docker/ /mnt/docker-new/
+"${RS[@]}" --delete --delete-excluded /var/lib/docker/ /mnt/docker-new/
-a=$(find /var/lib/docker -xdev | wc -l); b=$(find /mnt/docker-new -xdev | wc -l)
+(cd /var/lib/docker && find . -xdev -not -path './overlay2/*/merged/*' | LC_ALL=C sort) > /root/docker-move-old.list
-echo "entries: old=$a new=$b"
+(cd /mnt/docker-new && find . -xdev -not -path './overlay2/*/merged/*' -not -path './lost+found' | LC_ALL=C sort) > /root/docker-move-new.list
-# lost+found exists only on the new ext4, so new = old + 1
+echo "entries: old=$(wc -l < /root/docker-move-old.list) new=$(wc -l < /root/docker-move-new.list)"
-test "$((b - a))" -eq 1 || { echo 'entry count mismatch, abort (docker is stopped, old tree untouched)'; exit 1; }
+cmp -s /root/docker-move-old.list /root/docker-move-new.list || { echo 'path lists differ, abort (docker is stopped, old tree untouched):'; diff /root/docker-move-old.list /root/docker-move-new.list | head -20; exit 1; }
 
 step "swap: old tree aside, zvol mounted at /var/lib/docker"
 umount /mnt/docker-new
 mv /var/lib/docker /var/lib/docker.old-zroot
+SWAPPED=1
 install -d -m 710 /var/lib/docker
 grep -q '^/dev/zvol/tank/docker ' /etc/fstab || \
   echo '/dev/zvol/tank/docker /var/lib/docker ext4 defaults,nofail,x-systemd.requires=zfs-volume-wait.service,x-systemd.after=zfs-volume-wait.service 0 2' >> /etc/fstab