From 5be25be0812b18693eb9f10b369da5c917059694 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Sat, 19 Sep 2026 05:20:14 -0700 Subject: [PATCH] fix(backups): stop paging on guests that are deliberately not backed up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ana-scale (CT 114) is a subnet-router LXC, excluded from vzdump on 2026-09-07 after a backup lock on its ESH counterpart blackholed that entire site. The freshness check knew nothing about that and reported it 🔴 STALE every single morning, which is how an alarm teaches you to ignore it. Such guests now get their own section: printed every run, never hidden, and not counted as a fault. The subtlety is in how coverage is computed, and the obvious implementation is wrong twice over: * Reading one job's `exclude` list gets ana CT 109 (ana-nas) exactly backwards. It IS excluded from the 03:00 all-guests job AND it has its own dedicated 22:00 job. Suppressing on the exclude list would have stopped alarming on a guest that is genuinely backed up -- trading a noisy alarm for a blind one. * ESH's job uses an explicit `vmid 100..107` INCLUDE list, so esh-scale 108 is excluded by OMISSION and appears in no exclude list at all. So coverage is a union across every enabled job on the cluster, and a guest is "intentionally not backed up" only when none of them covers it. If coverage cannot be read, nothing is suppressed and the gap is reported: an unreachable PVE node means we do not know, and a backup alarm must fail loud. The SureFire namespace is never consulted (tenant property), so its guests can never be suppressed either. Verified against the live fleet on all four paths: CT 114 suppressed; CT 109 NOT suppressed despite being in an exclude list; esh-vm-workstation 102, which a job really does cover and which really is failing, still reports STALE; and with a PVE node made unreachable, 114 returns to STALE with the gap named. --- scripts/check-backup-freshness.sh | 98 ++++++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 2 deletions(-) diff --git a/scripts/check-backup-freshness.sh b/scripts/check-backup-freshness.sh index 3097679..7e87896 100755 --- a/scripts/check-backup-freshness.sh +++ b/scripts/check-backup-freshness.sh @@ -10,12 +10,89 @@ # Companion to docs/runbooks/backups.md. Read-only — only SSH stat/curl. # # BACKUP_MAX_AGE_HOURS=48 scripts/check-backup-freshness.sh +# +# POLICY AWARENESS (2026-09-19). Some guests are deliberately not backed up -- +# ana-scale (CT 114) and esh-scale (CT 108) are subnet-router LXCs excluded +# after a vzdump lock on esh-scale blackholed that entire site. Before this, +# the check reported those as 🔴 STALE every single morning, which is how an +# alarm teaches you to ignore it. Such guests now get their own section: +# printed, never hidden, but not counted as a fault. +# +# ⚠ COVERAGE IS A UNION ACROSS JOBS, NOT ONE JOB'S EXCLUDE LIST. Reading +# `exclude` alone gets ana CT 109 (ana-nas) exactly wrong: it IS excluded from +# the 03:00 all-guests job AND it has its own dedicated 22:00 job. Suppressing +# on the exclude list would have silently stopped alarming on a guest that is +# genuinely backed up -- turning a fix for a noisy alarm into a blind spot. +# The same applies in reverse for ESH, whose job uses an explicit +# `vmid 100..107` INCLUDE list, so esh-scale 108 is excluded by OMISSION and +# appears in no exclude list at all. +# A guest is "intentionally not backed up" only when NO enabled vzdump job +# covers it. +# +# ⚠ IF COVERAGE CANNOT BE DETERMINED, NOTHING IS SUPPRESSED. An unreachable +# PVE node means we do not know, and a backup alarm must fail loud. +# The SureFire namespace (sfsrv-pve) is deliberately never consulted -- those +# are tenant hosts -- so its guests can never be suppressed either. set -uo pipefail MAX_AGE_H="${BACKUP_MAX_AGE_HOURS:-48}" SSH="ssh -o ConnectTimeout=8 -o BatchMode=yes" now=$(date +%s) -stale=() ; fresh=() ; errors=() +stale=() ; fresh=() ; errors=() ; excluded=() + +# PBS namespace → a node of the PVE cluster that owns it. jobs.cfg lives in +# pmxcfs and is cluster-wide, so ESH's two nodes share one job set and either +# answers for both. sfsrv-pve is absent ON PURPOSE (tenant property). +declare -A NS_NODE=( + [ana-pve]=infra-ops@10.250.250.31 + [esh-pve]=infra-ops@10.0.250.35 + [nh3-pve]=infra-ops@10.100.250.60 +) +declare -A NS_JOBS=() # ns → one "enabled|all|vmids|excludes" line per job +declare -A NS_KNOWN=() # ns → 1 once coverage was successfully read + +load_jobs() { # $1 = namespace + local ns="$1" node="${NS_NODE[$1]:-}" raw + [ -n "$node" ] || return 1 + # Fetch the raw file and parse it HERE. The parser used to be an awk script + # embedded in the ssh command string; the quoting mangled it silently and + # every namespace came back unreadable — which at least failed loud, because + # that is how this is built. Keep the remote side a plain `cat`. + raw=$($SSH "$node" "sudo -n cat /etc/pve/jobs.cfg" 2>/dev/null) || return 1 + [ -n "$raw" ] || return 1 + local parsed + parsed=$(awk ' + function flush() { if (seen) { print e "|" a "|" v "|" x; seen=0 } } + /^vzdump:/ { flush(); seen=1; e=1; a=0; v=""; x=""; next } + /^[^ \t]/ { flush(); next } + seen && $1=="enabled" { e=$2 } + seen && $1=="all" { a=$2 } + seen && $1=="vmid" { v=$2 } + seen && $1=="exclude" { x=$2 } + END { flush() } + ' <<<"$raw") + [ -n "$parsed" ] || return 1 + NS_JOBS[$ns]="$parsed" + NS_KNOWN[$ns]=1 +} + +in_csv() { # $1=needle $2=comma list + case ",$2," in *",$1,"*) return 0 ;; *) return 1 ;; esac +} + +is_covered() { # $1=namespace $2=vmid — 0 covered, 1 not covered, 2 unknown + local ns="$1" id="$2" line enabled all vmids excludes + [ -n "${NS_KNOWN[$ns]:-}" ] || return 2 + while IFS='|' read -r enabled all vmids excludes; do + [ "$enabled" = "1" ] || continue + if [ -n "$vmids" ]; then + in_csv "$id" "$vmids" && return 0 # explicit include list + elif [ "$all" = "1" ]; then + in_csv "$id" "$excludes" || return 0 # all-guests minus its excludes + fi + done <<<"${NS_JOBS[$ns]}" + return 1 +} # newest snapshot epoch under a remote glob (echoes epoch or empty) newest_epoch() { # $1=host $2=glob @@ -47,7 +124,17 @@ pbs=$($SSH pbs-ana 'for ns in /mnt/pbs-datastore/ns/*/; do n=$(basename "$ns") [ -n "$nb" ] && echo "$n/$d/$(basename "$g") $(stat -c %Y "$nb")" done; done; done' 2>/dev/null) if [ -z "$pbs" ]; then errors+=("PBS-ANA: unreachable or no snapshots"); else - while read -r guest ep; do [ -n "$guest" ] && report "pbs/$guest" "$ep"; done <<<"$pbs" + for ns in "${!NS_NODE[@]}"; do load_jobs "$ns" || errors+=("coverage for namespace '$ns' UNREADABLE — nothing in it will be suppressed"); done + while read -r guest ep; do + [ -n "$guest" ] || continue + ns=${guest%%/*}; vmid=${guest##*/} + is_covered "$ns" "$vmid" + case $? in + 1) excluded+=("pbs/$guest: no enabled vzdump job covers it$( + [ -n "$ep" ] && printf ' (last snapshot %s)' "$(date -d "@$ep" '+%Y-%m-%d' 2>/dev/null)")") ;; + *) report "pbs/$guest" "$ep" ;; + esac + done <<<"$pbs" fi # --- rest-server endpoint health (401 = up & serving) --- @@ -59,6 +146,13 @@ done echo echo "FRESH (${#fresh[@]}):"; printf ' ✅ %s\n' "${fresh[@]}" +if [ "${#excluded[@]}" -gt 0 ]; then + # Printed every run on purpose: an exclusion that was a mistake is only + # catchable if you can see it. Silence here would trade a noisy alarm for a + # blind one. + echo; echo "NOT BACKED UP BY POLICY (${#excluded[@]}) — not a fault, but check the list is still right:" + printf ' ⏸ %s\n' "${excluded[@]}" +fi if [ "${#stale[@]}" -gt 0 ] || [ "${#errors[@]}" -gt 0 ]; then echo; echo "STALE / PROBLEMS (${#stale[@]}+${#errors[@]}):" printf ' 🔴 %s\n' "${stale[@]}" "${errors[@]}"