diff --git a/scripts/check-backup-freshness.sh b/scripts/check-backup-freshness.sh index 99c2977..820ff8f 100755 --- a/scripts/check-backup-freshness.sh +++ b/scripts/check-backup-freshness.sh @@ -63,6 +63,23 @@ declare -A NS_NODE=( ) declare -A NS_JOBS=() # ns → one "enabled|all|vmids|excludes" line per job declare -A NS_KNOWN=() # ns → 1 once coverage was successfully read +declare -A NS_INV=() # ns → guest id set from pvesh, ONLY when the read succeeded + +# Inventory of the owning cluster ("qemu/110 lxc/109 ..."). An empty/unset set +# means UNKNOWN, never "empty cluster": an unreachable node must not make a +# live guest look destroyed (fail loud, same principle as jobs.cfg). +load_inventory() { # $1 = namespace + local ns="$1" node="${NS_NODE[$1]:-}" raw + [ -n "$node" ] || return 1 + [ -n "${NS_INV[$ns]:-}" ] && return 0 + # -n is load-bearing: load_inventory runs INSIDE the while-read loop over the + # PBS tree, and an ssh without -n swallows the loop's remaining stdin (the + # whole PBS layer silently disappeared on 2026-10-06 until that was pinned). + raw=$($SSH -n "$node" "sudo -n pvesh get /cluster/resources --type vm --output-format text" 2>/dev/null) || return 1 + raw=$(echo "$raw" | grep -oE "(qemu|lxc)/[0-9]+" | sort -u | tr "\n" " ") + [ -n "$raw" ] || return 1 + NS_INV[$ns]="$raw" +} # WHY each policy-excluded guest is out of the backup jobs (2026-10-05). The # exclusion is derived from jobs.cfg coverage; the REASON is standing policy @@ -225,6 +242,22 @@ if [ -z "$pbs" ]; then errors+=("PBS-ANA: unreachable or no snapshots"); else while read -r guest ep; do [ -n "$guest" ] || continue ns=${guest%%/*}; vmid=${guest##*/} + # Destroyed-guest check (2026-10-06): a guest deleted from the cluster stops + # writing snapshots, and its last snapshot ages into a 🔴 forever. pvesh says + # qemu/NN + lxc/NN; the PBS tree says vm/NN + ct/NN. If the owning cluster is + # readable and no longer lists this guest, its snapshots are residue, not a + # stale backup. Inventory unreadable -> fall through and alarm (fail loud). + if load_inventory "$ns"; then + # 'guest' is namespaced ("ana-pve/vm/110") — match the segment, and + # pre-set pveid so an unexpected path shape fails the MATCH, not the + # script, under set -u (infra-ops, 2026-10-06). + pveid="" + case "$guest" in */vm/*) pveid="qemu/$vmid";; */ct/*) pveid="lxc/$vmid";; esac + case " ${NS_INV[$ns]} " in + *" $pveid "*) : ;; + *) excluded+=("pbs/$guest: GUEST DESTROYED — no longer in $ns inventory; last snapshot $([ -n "$ep" ] && date -d "@$ep" '+%Y-%m-%d' || echo none) is residue, prune at leisure") ; continue ;; + esac + fi is_covered "$ns" "$vmid" case $? in 1) excluded+=("pbs/$guest: no enabled vzdump job covers it$(