fix(check-backup-freshness): report destroyed guests as policy residue, not STALE

A guest destroyed with 'qm destroy --purge' also drops out of the vzdump
job's exclude list, so its last snapshot silently ages into a false
🔴 once it passes the threshold (ana-pve/vm/110, destroyed 10-04, fired
this morning). Before reporting coverage, consult the owning cluster's
pvesh inventory: a guest the cluster no longer lists is reported in the
policy section as residue, never as stale. Inventory unreadable fails
loud (falls through to normal alarm).

Two traps pinned the hard way during verification:
- 'guest' is namespaced (ana-pve/vm/110); the case must match the
  segment (*/vm/*) and pre-set pveid so a foreign path shape fails the
  match, not the script, under set -u (infra-ops caught the first cut).
- ssh inside the while-read loop MUST take -n; without it ssh swallows
  the loop's remaining stdin and the entire PBS layer silently vanishes
  from the report while exit code and verdict stay green.
This commit is contained in:
vh
2026-10-06 08:04:17 -07:00
parent 14bf8681f0
commit 0a348e548c
+33
View File
@@ -63,6 +63,23 @@ declare -A NS_NODE=(
) )
declare -A NS_JOBS=() # ns → one "enabled|all|vmids|excludes" line per job declare -A NS_JOBS=() # ns → one "enabled|all|vmids|excludes" line per job
declare -A NS_KNOWN=() # ns → 1 once coverage was successfully read declare -A NS_KNOWN=() # ns → 1 once coverage was successfully read
declare -A NS_INV=() # ns → guest id set from pvesh, ONLY when the read succeeded
# Inventory of the owning cluster ("qemu/110 lxc/109 ..."). An empty/unset set
# means UNKNOWN, never "empty cluster": an unreachable node must not make a
# live guest look destroyed (fail loud, same principle as jobs.cfg).
load_inventory() { # $1 = namespace
local ns="$1" node="${NS_NODE[$1]:-}" raw
[ -n "$node" ] || return 1
[ -n "${NS_INV[$ns]:-}" ] && return 0
# -n is load-bearing: load_inventory runs INSIDE the while-read loop over the
# PBS tree, and an ssh without -n swallows the loop's remaining stdin (the
# whole PBS layer silently disappeared on 2026-10-06 until that was pinned).
raw=$($SSH -n "$node" "sudo -n pvesh get /cluster/resources --type vm --output-format text" 2>/dev/null) || return 1
raw=$(echo "$raw" | grep -oE "(qemu|lxc)/[0-9]+" | sort -u | tr "\n" " ")
[ -n "$raw" ] || return 1
NS_INV[$ns]="$raw"
}
# WHY each policy-excluded guest is out of the backup jobs (2026-10-05). The # WHY each policy-excluded guest is out of the backup jobs (2026-10-05). The
# exclusion is derived from jobs.cfg coverage; the REASON is standing policy # exclusion is derived from jobs.cfg coverage; the REASON is standing policy
@@ -225,6 +242,22 @@ if [ -z "$pbs" ]; then errors+=("PBS-ANA: unreachable or no snapshots"); else
while read -r guest ep; do while read -r guest ep; do
[ -n "$guest" ] || continue [ -n "$guest" ] || continue
ns=${guest%%/*}; vmid=${guest##*/} ns=${guest%%/*}; vmid=${guest##*/}
# Destroyed-guest check (2026-10-06): a guest deleted from the cluster stops
# writing snapshots, and its last snapshot ages into a 🔴 forever. pvesh says
# qemu/NN + lxc/NN; the PBS tree says vm/NN + ct/NN. If the owning cluster is
# readable and no longer lists this guest, its snapshots are residue, not a
# stale backup. Inventory unreadable -> fall through and alarm (fail loud).
if load_inventory "$ns"; then
# 'guest' is namespaced ("ana-pve/vm/110") — match the segment, and
# pre-set pveid so an unexpected path shape fails the MATCH, not the
# script, under set -u (infra-ops, 2026-10-06).
pveid=""
case "$guest" in */vm/*) pveid="qemu/$vmid";; */ct/*) pveid="lxc/$vmid";; esac
case " ${NS_INV[$ns]} " in
*" $pveid "*) : ;;
*) excluded+=("pbs/$guest: GUEST DESTROYED — no longer in $ns inventory; last snapshot $([ -n "$ep" ] && date -d "@$ep" '+%Y-%m-%d' || echo none) is residue, prune at leisure") ; continue ;;
esac
fi
is_covered "$ns" "$vmid" is_covered "$ns" "$vmid"
case $? in case $? in
1) excluded+=("pbs/$guest: no enabled vzdump job covers it$( 1) excluded+=("pbs/$guest: no enabled vzdump job covers it$(