vh/esh-pfi-infrastructure
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(dev-backup): chmod before prune so retention actually deletes; fail the unit on a bad prune

Browse Source 
rsync -a copies a read-only source dir (0555) as read-only, so the hourly
prune's rm -rf could not unlink inside it. From 2026-07-18 every pruned
snapshot was left as a 22-entry husk while the run still logged OK. The
1,740 husks on nh3-nas were removed (0 errors; 48 full snapshots kept).

The prune now runs chmod -R u+w before rm -rf, logs its error count and
the number of snapshots kept, and exits 3 on a failed prune (exit 1 on a
failed rsync) so the systemd unit shows failed instead of passing.
This commit is contained in:
vh
2026-10-01 05:59:07 -07:00
parent 9af5a16d07
commit de11e00dfb
3 changed files with 26 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/nh3-dev-development-backup.sh
+16 -3
View File
@@ -38,9 +38,22 @@ echo "rsync rc=$RC"
# rc 0 = ok; rc 24 = some files vanished mid-transfer (benign for a live tree)
if [ "$RC" -eq 0 ] || [ "$RC" -eq 24 ]; then
ssh -o BatchMode=yes "$DEST_HOST" "ln -sfn '$DEST_BASE/$STAMP' '$DEST_BASE/latest'"
# retention: keep newest 48 hourly snapshots
ssh -o BatchMode=yes "$DEST_HOST" "ls -1d $DEST_BASE/20* 2>/dev/null | sort | head -n -48 | xargs -r rm -rf"
echo "=== $(date -Is) snapshot $STAMP OK (rc=$RC) ==="
# retention: keep newest 48 hourly snapshots.
# chmod BEFORE rm: rsync -a copies a read-only source dir (mode 0555) as read-only, and rm cannot
# unlink inside it. Without the chmod this prune failed every run from 2026-07-18 to 2026-10-01 and
# left 1,740 husk dirs behind (removed 2026-10-01), while the run still logged OK. Errors are COUNTED,
# not dumped (one failing run logged ~400 MB), and a failed prune now fails the unit.
PRUNE_LIST="ls -1d $DEST_BASE/20* 2>/dev/null | sort | head -n -48"
PRUNE_ERR="$(ssh -o BatchMode=yes "$DEST_HOST" "$PRUNE_LIST | xargs -r chmod -R u+w 2>&1; $PRUNE_LIST | xargs -r rm -rf 2>&1" | wc -l)"
KEPT="$(ssh -o BatchMode=yes "$DEST_HOST" "ls -1d $DEST_BASE/20* 2>/dev/null | wc -l")"
echo "retention prune: ${PRUNE_ERR:-?} error lines, ${KEPT:-?} snapshots on the NAS (expected 0 and <=48)"
if [ "${PRUNE_ERR:-x}" = 0 ] && [ "${KEPT:-999}" -le 48 ] 2>/dev/null; then
echo "=== $(date -Is) snapshot $STAMP OK (rc=$RC) ==="
else
echo "=== $(date -Is) snapshot $STAMP taken (rc=$RC) but RETENTION FAILED ==="
exit 3
fi
else
echo "=== $(date -Is) snapshot $STAMP FAILED rc=$RC — keeping partial for inspection ==="
exit 1
fi