From 0a348e548c94473385d3f6d053c1d6ff8e38cabe Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Tue, 6 Oct 2026 08:04:17 -0700 Subject: [PATCH] fix(check-backup-freshness): report destroyed guests as policy residue, not STALE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A guest destroyed with 'qm destroy --purge' also drops out of the vzdump job's exclude list, so its last snapshot silently ages into a false 🔴 once it passes the threshold (ana-pve/vm/110, destroyed 10-04, fired this morning). Before reporting coverage, consult the owning cluster's pvesh inventory: a guest the cluster no longer lists is reported in the policy section as residue, never as stale. Inventory unreadable fails loud (falls through to normal alarm). Two traps pinned the hard way during verification: - 'guest' is namespaced (ana-pve/vm/110); the case must match the segment (*/vm/*) and pre-set pveid so a foreign path shape fails the match, not the script, under set -u (infra-ops caught the first cut). - ssh inside the while-read loop MUST take -n; without it ssh swallows the loop's remaining stdin and the entire PBS layer silently vanishes from the report while exit code and verdict stay green. --- scripts/check-backup-freshness.sh | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/scripts/check-backup-freshness.sh b/scripts/check-backup-freshness.sh index 99c2977..820ff8f 100755 --- a/scripts/check-backup-freshness.sh +++ b/scripts/check-backup-freshness.sh @@ -63,6 +63,23 @@ declare -A NS_NODE=( ) declare -A NS_JOBS=() # ns → one "enabled|all|vmids|excludes" line per job declare -A NS_KNOWN=() # ns → 1 once coverage was successfully read +declare -A NS_INV=() # ns → guest id set from pvesh, ONLY when the read succeeded + +# Inventory of the owning cluster ("qemu/110 lxc/109 ..."). An empty/unset set +# means UNKNOWN, never "empty cluster": an unreachable node must not make a +# live guest look destroyed (fail loud, same principle as jobs.cfg). +load_inventory() { # $1 = namespace + local ns="$1" node="${NS_NODE[$1]:-}" raw + [ -n "$node" ] || return 1 + [ -n "${NS_INV[$ns]:-}" ] && return 0 + # -n is load-bearing: load_inventory runs INSIDE the while-read loop over the + # PBS tree, and an ssh without -n swallows the loop's remaining stdin (the + # whole PBS layer silently disappeared on 2026-10-06 until that was pinned). + raw=$($SSH -n "$node" "sudo -n pvesh get /cluster/resources --type vm --output-format text" 2>/dev/null) || return 1 + raw=$(echo "$raw" | grep -oE "(qemu|lxc)/[0-9]+" | sort -u | tr "\n" " ") + [ -n "$raw" ] || return 1 + NS_INV[$ns]="$raw" +} # WHY each policy-excluded guest is out of the backup jobs (2026-10-05). The # exclusion is derived from jobs.cfg coverage; the REASON is standing policy @@ -225,6 +242,22 @@ if [ -z "$pbs" ]; then errors+=("PBS-ANA: unreachable or no snapshots"); else while read -r guest ep; do [ -n "$guest" ] || continue ns=${guest%%/*}; vmid=${guest##*/} + # Destroyed-guest check (2026-10-06): a guest deleted from the cluster stops + # writing snapshots, and its last snapshot ages into a 🔴 forever. pvesh says + # qemu/NN + lxc/NN; the PBS tree says vm/NN + ct/NN. If the owning cluster is + # readable and no longer lists this guest, its snapshots are residue, not a + # stale backup. Inventory unreadable -> fall through and alarm (fail loud). + if load_inventory "$ns"; then + # 'guest' is namespaced ("ana-pve/vm/110") — match the segment, and + # pre-set pveid so an unexpected path shape fails the MATCH, not the + # script, under set -u (infra-ops, 2026-10-06). + pveid="" + case "$guest" in */vm/*) pveid="qemu/$vmid";; */ct/*) pveid="lxc/$vmid";; esac + case " ${NS_INV[$ns]} " in + *" $pveid "*) : ;; + *) excluded+=("pbs/$guest: GUEST DESTROYED — no longer in $ns inventory; last snapshot $([ -n "$ep" ] && date -d "@$ep" '+%Y-%m-%d' || echo none) is residue, prune at leisure") ; continue ;; + esac + fi is_covered "$ns" "$vmid" case $? in 1) excluded+=("pbs/$guest: no enabled vzdump job covers it$(