feat(backups): catch a job that runs and errors, not just one that goes stale

Snapshot age is structurally blind to a backup job that executes every night
and fails every night. Nothing new is written, so the group simply ages, and
the fault only surfaces once it crosses the 48h threshold -- days after the
first failure, with the evidence sitting in a task log nobody reads.

Two live cases, both found today and both invisible for a week by this exact
mechanism:

  * esh-nas-pve CT 107 (vm-jellyfin): a backup run died around 09-06 and left
    a stale `backup` lock, so every nightly since failed instantly with "CT is
    locked (backup)". Age named it on ~09-12. Task status would have named it
    on 09-07.
  * esh-pve VM 102 (esh-vm-workstation): failing nightly since ~09-06 with
    "timeout waiting on systemd". Same six-day gap.

PVE already records every task result in /var/log/pve/tasks/index. This reads
it on all four non-tenant PVE nodes and reports any vzdump in the last 36h
whose status is not OK, as its own section that sets the exit code.

It found a third case on its first run: esh-nas-pve's job had been reporting
`job errors` nightly while every guest on that node read 0-1h fresh, so no
age-based check could ever have flagged it.

Window is BACKUP_JOB_WINDOW_HOURS (default 36 -- longer than a daily cycle so
one missed run does not hide a failure). A node whose task log cannot be read
is reported, never assumed healthy.

scripts/check-backup-freshness.sh

@@ -51,6 +51,17 @@ declare -A NS_NODE=(
 declare -A NS_JOBS=()      # ns → one "enabled|all|vmids|excludes" line per job
 declare -A NS_KNOWN=()     # ns → 1 once coverage was successfully read
 
+# Every PVE node whose vzdump TASK RESULTS we read. Unlike jobs.cfg this is
+# per-NODE, not cluster-wide, so both ESH nodes appear. sfsrv is absent: tenant.
+PVE_NODES=(
+  "ana-pve       infra-ops@10.250.250.31"
+  "esh-pve       infra-ops@10.0.250.35"
+  "esh-nas-pve   infra-ops@10.0.50.55"
+  "nh3-pve       infra-ops@10.100.250.60"
+)
+JOB_WINDOW_H="${BACKUP_JOB_WINDOW_HOURS:-36}"
+jobfail=()
+
 load_jobs() { # $1 = namespace
   local ns="$1" node="${NS_NODE[$1]:-}" raw
   [ -n "$node" ] || return 1
@@ -137,6 +148,42 @@ if [ -z "$pbs" ]; then errors+=("PBS-ANA: unreachable or no snapshots"); else
   done <<<"$pbs"
 fi
 
+# --- Layer: did the vzdump jobs actually SUCCEED? -----------------------
+#
+# Snapshot age alone is structurally blind to a job that runs and ERRORS every
+# night: nothing new is written, so the group simply ages, and the fault only
+# surfaces once it crosses the 48h threshold — days late. esh-vm-workstation
+# (VM 102) failed nightly from ~2026-09-06 with "timeout waiting on systemd"
+# and this check would not have named it until 09-12.
+#
+# PVE records every task's result in /var/log/pve/tasks/index as
+#   <UPID> <endtime-hex> <status>
+# where the UPID's 6th colon-field is the task type and the 5th is its hex
+# start time. Anything vzdump in the window whose status is not OK is a fault
+# TODAY, not in two days.
+for entry in "${PVE_NODES[@]}"; do
+  set -- $entry; node_label="$1"; node_addr="$2"
+  idx=$($SSH "$node_addr" "sudo -n cat /var/log/pve/tasks/index" 2>/dev/null)
+  if [ -z "$idx" ]; then
+    errors+=("vzdump task log on $node_label UNREADABLE — job failures there are invisible")
+    continue
+  fi
+  bad=$(awk -v now="$now" -v win="$((JOB_WINDOW_H * 3600))" '
+    {
+      upid = $1; endh = $2; status = $0
+      sub(/^[^ ]+ [^ ]+ /, "", status)
+      n = split(upid, f, ":")
+      if (n < 7 || f[6] != "vzdump") next
+      start = strtonum("0x" f[5])
+      if (now - start > win) next
+      if (status == "OK") next
+      printf "%s%s: %s\n", (f[7] == "" ? "job" : "guest " f[7]), "", status
+    }' <<<"$idx" | sort -u)
+  [ -n "$bad" ] && while IFS= read -r b; do
+    [ -n "$b" ] && jobfail+=("$node_label $b")
+  done <<<"$bad"
+done
+
 # --- rest-server endpoint health (401 = up & serving) ---
 for ep in "rest-server-ana http://10.250.50.70:8000/" "rest-server-nh3 http://10.100.50.50:8000/"; do
   set -- $ep
@@ -153,9 +200,16 @@ if [ "${#excluded[@]}" -gt 0 ]; then
   echo; echo "NOT BACKED UP BY POLICY (${#excluded[@]}) — not a fault, but check the list is still right:"
   printf '  ⏸  %s\n' "${excluded[@]}"
 fi
+if [ "${#jobfail[@]}" -gt 0 ]; then
+  echo; echo "BACKUP JOBS THAT ERRORED in the last ${JOB_WINDOW_H}h (${#jobfail[@]}):"
+  printf '  ❌ %s\n' "${jobfail[@]}"
+fi
 if [ "${#stale[@]}" -gt 0 ] || [ "${#errors[@]}" -gt 0 ]; then
   echo; echo "STALE / PROBLEMS (${#stale[@]}+${#errors[@]}):"
-  printf '  🔴 %s\n' "${stale[@]}" "${errors[@]}"
+  [ "${#stale[@]}"  -gt 0 ] && printf '  🔴 %s\n' "${stale[@]}"
+  [ "${#errors[@]}" -gt 0 ] && printf '  🔴 %s\n' "${errors[@]}"
+fi
+if [ "${#stale[@]}" -gt 0 ] || [ "${#errors[@]}" -gt 0 ] || [ "${#jobfail[@]}" -gt 0 ]; then
   echo; echo "RESULT: STALE — see docs/runbooks/backups.md"
   exit 1
 fi