fix(backups): stop paging on guests that are deliberately not backed up

ana-scale (CT 114) is a subnet-router LXC, excluded from vzdump on 2026-09-07
after a backup lock on its ESH counterpart blackholed that entire site. The
freshness check knew nothing about that and reported it 🔴 STALE every single
morning, which is how an alarm teaches you to ignore it.

Such guests now get their own section: printed every run, never hidden, and
not counted as a fault.

The subtlety is in how coverage is computed, and the obvious implementation is
wrong twice over:

  * Reading one job's `exclude` list gets ana CT 109 (ana-nas) exactly
    backwards. It IS excluded from the 03:00 all-guests job AND it has its own
    dedicated 22:00 job. Suppressing on the exclude list would have stopped
    alarming on a guest that is genuinely backed up -- trading a noisy alarm
    for a blind one.
  * ESH's job uses an explicit `vmid 100..107` INCLUDE list, so esh-scale 108
    is excluded by OMISSION and appears in no exclude list at all.

So coverage is a union across every enabled job on the cluster, and a guest is
"intentionally not backed up" only when none of them covers it.

If coverage cannot be read, nothing is suppressed and the gap is reported: an
unreachable PVE node means we do not know, and a backup alarm must fail loud.
The SureFire namespace is never consulted (tenant property), so its guests can
never be suppressed either.

Verified against the live fleet on all four paths: CT 114 suppressed; CT 109
NOT suppressed despite being in an exclude list; esh-vm-workstation 102, which
a job really does cover and which really is failing, still reports STALE; and
with a PVE node made unreachable, 114 returns to STALE with the gap named.

scripts/check-backup-freshness.sh

@@ -10,12 +10,89 @@
 # Companion to docs/runbooks/backups.md. Read-only — only SSH stat/curl.
 #
 #   BACKUP_MAX_AGE_HOURS=48 scripts/check-backup-freshness.sh
+#
+# POLICY AWARENESS (2026-09-19). Some guests are deliberately not backed up --
+# ana-scale (CT 114) and esh-scale (CT 108) are subnet-router LXCs excluded
+# after a vzdump lock on esh-scale blackholed that entire site. Before this,
+# the check reported those as 🔴 STALE every single morning, which is how an
+# alarm teaches you to ignore it. Such guests now get their own section:
+# printed, never hidden, but not counted as a fault.
+#
+# ⚠ COVERAGE IS A UNION ACROSS JOBS, NOT ONE JOB'S EXCLUDE LIST. Reading
+# `exclude` alone gets ana CT 109 (ana-nas) exactly wrong: it IS excluded from
+# the 03:00 all-guests job AND it has its own dedicated 22:00 job. Suppressing
+# on the exclude list would have silently stopped alarming on a guest that is
+# genuinely backed up -- turning a fix for a noisy alarm into a blind spot.
+# The same applies in reverse for ESH, whose job uses an explicit
+# `vmid 100..107` INCLUDE list, so esh-scale 108 is excluded by OMISSION and
+# appears in no exclude list at all.
+# A guest is "intentionally not backed up" only when NO enabled vzdump job
+# covers it.
+#
+# ⚠ IF COVERAGE CANNOT BE DETERMINED, NOTHING IS SUPPRESSED. An unreachable
+# PVE node means we do not know, and a backup alarm must fail loud.
+# The SureFire namespace (sfsrv-pve) is deliberately never consulted -- those
+# are tenant hosts -- so its guests can never be suppressed either.
 set -uo pipefail
 
 MAX_AGE_H="${BACKUP_MAX_AGE_HOURS:-48}"
 SSH="ssh -o ConnectTimeout=8 -o BatchMode=yes"
 now=$(date +%s)
-stale=() ; fresh=() ; errors=()
+stale=() ; fresh=() ; errors=() ; excluded=()
+
+# PBS namespace → a node of the PVE cluster that owns it. jobs.cfg lives in
+# pmxcfs and is cluster-wide, so ESH's two nodes share one job set and either
+# answers for both. sfsrv-pve is absent ON PURPOSE (tenant property).
+declare -A NS_NODE=(
+  [ana-pve]=infra-ops@10.250.250.31
+  [esh-pve]=infra-ops@10.0.250.35
+  [nh3-pve]=infra-ops@10.100.250.60
+)
+declare -A NS_JOBS=()      # ns → one "enabled|all|vmids|excludes" line per job
+declare -A NS_KNOWN=()     # ns → 1 once coverage was successfully read
+
+load_jobs() { # $1 = namespace
+  local ns="$1" node="${NS_NODE[$1]:-}" raw
+  [ -n "$node" ] || return 1
+  # Fetch the raw file and parse it HERE. The parser used to be an awk script
+  # embedded in the ssh command string; the quoting mangled it silently and
+  # every namespace came back unreadable — which at least failed loud, because
+  # that is how this is built. Keep the remote side a plain `cat`.
+  raw=$($SSH "$node" "sudo -n cat /etc/pve/jobs.cfg" 2>/dev/null) || return 1
+  [ -n "$raw" ] || return 1
+  local parsed
+  parsed=$(awk '
+    function flush() { if (seen) { print e "|" a "|" v "|" x; seen=0 } }
+    /^vzdump:/       { flush(); seen=1; e=1; a=0; v=""; x=""; next }
+    /^[^ \t]/       { flush(); next }
+    seen && $1=="enabled" { e=$2 }
+    seen && $1=="all"     { a=$2 }
+    seen && $1=="vmid"    { v=$2 }
+    seen && $1=="exclude" { x=$2 }
+    END { flush() }
+  ' <<<"$raw")
+  [ -n "$parsed" ] || return 1
+  NS_JOBS[$ns]="$parsed"
+  NS_KNOWN[$ns]=1
+}
+
+in_csv() { # $1=needle  $2=comma list
+  case ",$2," in *",$1,"*) return 0 ;; *) return 1 ;; esac
+}
+
+is_covered() { # $1=namespace  $2=vmid — 0 covered, 1 not covered, 2 unknown
+  local ns="$1" id="$2" line enabled all vmids excludes
+  [ -n "${NS_KNOWN[$ns]:-}" ] || return 2
+  while IFS='|' read -r enabled all vmids excludes; do
+    [ "$enabled" = "1" ] || continue
+    if [ -n "$vmids" ]; then
+      in_csv "$id" "$vmids" && return 0      # explicit include list
+    elif [ "$all" = "1" ]; then
+      in_csv "$id" "$excludes" || return 0   # all-guests minus its excludes
+    fi
+  done <<<"${NS_JOBS[$ns]}"
+  return 1
+}
 
 # newest snapshot epoch under a remote glob (echoes epoch or empty)
 newest_epoch() { # $1=host  $2=glob
@@ -47,7 +124,17 @@ pbs=$($SSH pbs-ana 'for ns in /mnt/pbs-datastore/ns/*/; do n=$(basename "$ns")
     [ -n "$nb" ] && echo "$n/$d/$(basename "$g") $(stat -c %Y "$nb")"
   done; done; done' 2>/dev/null)
 if [ -z "$pbs" ]; then errors+=("PBS-ANA: unreachable or no snapshots"); else
-  while read -r guest ep; do [ -n "$guest" ] && report "pbs/$guest" "$ep"; done <<<"$pbs"
+  for ns in "${!NS_NODE[@]}"; do load_jobs "$ns" || errors+=("coverage for namespace '$ns' UNREADABLE — nothing in it will be suppressed"); done
+  while read -r guest ep; do
+    [ -n "$guest" ] || continue
+    ns=${guest%%/*}; vmid=${guest##*/}
+    is_covered "$ns" "$vmid"
+    case $? in
+      1) excluded+=("pbs/$guest: no enabled vzdump job covers it$(
+           [ -n "$ep" ] && printf ' (last snapshot %s)' "$(date -d "@$ep" '+%Y-%m-%d' 2>/dev/null)")") ;;
+      *) report "pbs/$guest" "$ep" ;;
+    esac
+  done <<<"$pbs"
 fi
 
 # --- rest-server endpoint health (401 = up & serving) ---
@@ -59,6 +146,13 @@ done
 
 echo
 echo "FRESH (${#fresh[@]}):"; printf '  ✅ %s\n' "${fresh[@]}"
+if [ "${#excluded[@]}" -gt 0 ]; then
+  # Printed every run on purpose: an exclusion that was a mistake is only
+  # catchable if you can see it. Silence here would trade a noisy alarm for a
+  # blind one.
+  echo; echo "NOT BACKED UP BY POLICY (${#excluded[@]}) — not a fault, but check the list is still right:"
+  printf '  ⏸  %s\n' "${excluded[@]}"
+fi
 if [ "${#stale[@]}" -gt 0 ] || [ "${#errors[@]}" -gt 0 ]; then
   echo; echo "STALE / PROBLEMS (${#stale[@]}+${#errors[@]}):"
   printf '  🔴 %s\n' "${stale[@]}" "${errors[@]}"