fix(check-backup-freshness): report destroyed guests as policy residue, not STALE
A guest destroyed with 'qm destroy --purge' also drops out of the vzdump
job's exclude list, so its last snapshot silently ages into a false
🔴 once it passes the threshold (ana-pve/vm/110, destroyed 10-04, fired
this morning). Before reporting coverage, consult the owning cluster's
pvesh inventory: a guest the cluster no longer lists is reported in the
policy section as residue, never as stale. Inventory unreadable fails
loud (falls through to normal alarm).
Two traps pinned the hard way during verification:
- 'guest' is namespaced (ana-pve/vm/110); the case must match the
segment (*/vm/*) and pre-set pveid so a foreign path shape fails the
match, not the script, under set -u (infra-ops caught the first cut).
- ssh inside the while-read loop MUST take -n; without it ssh swallows
the loop's remaining stdin and the entire PBS layer silently vanishes
from the report while exit code and verdict stay green.
This commit is contained in:
@@ -63,6 +63,23 @@ declare -A NS_NODE=(
|
|||||||
)
|
)
|
||||||
declare -A NS_JOBS=() # ns → one "enabled|all|vmids|excludes" line per job
|
declare -A NS_JOBS=() # ns → one "enabled|all|vmids|excludes" line per job
|
||||||
declare -A NS_KNOWN=() # ns → 1 once coverage was successfully read
|
declare -A NS_KNOWN=() # ns → 1 once coverage was successfully read
|
||||||
|
declare -A NS_INV=() # ns → guest id set from pvesh, ONLY when the read succeeded
|
||||||
|
|
||||||
|
# Inventory of the owning cluster ("qemu/110 lxc/109 ..."). An empty/unset set
|
||||||
|
# means UNKNOWN, never "empty cluster": an unreachable node must not make a
|
||||||
|
# live guest look destroyed (fail loud, same principle as jobs.cfg).
|
||||||
|
load_inventory() { # $1 = namespace
|
||||||
|
local ns="$1" node="${NS_NODE[$1]:-}" raw
|
||||||
|
[ -n "$node" ] || return 1
|
||||||
|
[ -n "${NS_INV[$ns]:-}" ] && return 0
|
||||||
|
# -n is load-bearing: load_inventory runs INSIDE the while-read loop over the
|
||||||
|
# PBS tree, and an ssh without -n swallows the loop's remaining stdin (the
|
||||||
|
# whole PBS layer silently disappeared on 2026-10-06 until that was pinned).
|
||||||
|
raw=$($SSH -n "$node" "sudo -n pvesh get /cluster/resources --type vm --output-format text" 2>/dev/null) || return 1
|
||||||
|
raw=$(echo "$raw" | grep -oE "(qemu|lxc)/[0-9]+" | sort -u | tr "\n" " ")
|
||||||
|
[ -n "$raw" ] || return 1
|
||||||
|
NS_INV[$ns]="$raw"
|
||||||
|
}
|
||||||
|
|
||||||
# WHY each policy-excluded guest is out of the backup jobs (2026-10-05). The
|
# WHY each policy-excluded guest is out of the backup jobs (2026-10-05). The
|
||||||
# exclusion is derived from jobs.cfg coverage; the REASON is standing policy
|
# exclusion is derived from jobs.cfg coverage; the REASON is standing policy
|
||||||
@@ -225,6 +242,22 @@ if [ -z "$pbs" ]; then errors+=("PBS-ANA: unreachable or no snapshots"); else
|
|||||||
while read -r guest ep; do
|
while read -r guest ep; do
|
||||||
[ -n "$guest" ] || continue
|
[ -n "$guest" ] || continue
|
||||||
ns=${guest%%/*}; vmid=${guest##*/}
|
ns=${guest%%/*}; vmid=${guest##*/}
|
||||||
|
# Destroyed-guest check (2026-10-06): a guest deleted from the cluster stops
|
||||||
|
# writing snapshots, and its last snapshot ages into a 🔴 forever. pvesh says
|
||||||
|
# qemu/NN + lxc/NN; the PBS tree says vm/NN + ct/NN. If the owning cluster is
|
||||||
|
# readable and no longer lists this guest, its snapshots are residue, not a
|
||||||
|
# stale backup. Inventory unreadable -> fall through and alarm (fail loud).
|
||||||
|
if load_inventory "$ns"; then
|
||||||
|
# 'guest' is namespaced ("ana-pve/vm/110") — match the segment, and
|
||||||
|
# pre-set pveid so an unexpected path shape fails the MATCH, not the
|
||||||
|
# script, under set -u (infra-ops, 2026-10-06).
|
||||||
|
pveid=""
|
||||||
|
case "$guest" in */vm/*) pveid="qemu/$vmid";; */ct/*) pveid="lxc/$vmid";; esac
|
||||||
|
case " ${NS_INV[$ns]} " in
|
||||||
|
*" $pveid "*) : ;;
|
||||||
|
*) excluded+=("pbs/$guest: GUEST DESTROYED — no longer in $ns inventory; last snapshot $([ -n "$ep" ] && date -d "@$ep" '+%Y-%m-%d' || echo none) is residue, prune at leisure") ; continue ;;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
is_covered "$ns" "$vmid"
|
is_covered "$ns" "$vmid"
|
||||||
case $? in
|
case $? in
|
||||||
1) excluded+=("pbs/$guest: no enabled vzdump job covers it$(
|
1) excluded+=("pbs/$guest: no enabled vzdump job covers it$(
|
||||||
|
|||||||
Reference in New Issue
Block a user