diff --git a/scripts/check-backup-freshness.sh b/scripts/check-backup-freshness.sh index 7e87896..cd7e96a 100755 --- a/scripts/check-backup-freshness.sh +++ b/scripts/check-backup-freshness.sh @@ -51,6 +51,17 @@ declare -A NS_NODE=( declare -A NS_JOBS=() # ns → one "enabled|all|vmids|excludes" line per job declare -A NS_KNOWN=() # ns → 1 once coverage was successfully read +# Every PVE node whose vzdump TASK RESULTS we read. Unlike jobs.cfg this is +# per-NODE, not cluster-wide, so both ESH nodes appear. sfsrv is absent: tenant. +PVE_NODES=( + "ana-pve infra-ops@10.250.250.31" + "esh-pve infra-ops@10.0.250.35" + "esh-nas-pve infra-ops@10.0.50.55" + "nh3-pve infra-ops@10.100.250.60" +) +JOB_WINDOW_H="${BACKUP_JOB_WINDOW_HOURS:-36}" +jobfail=() + load_jobs() { # $1 = namespace local ns="$1" node="${NS_NODE[$1]:-}" raw [ -n "$node" ] || return 1 @@ -137,6 +148,42 @@ if [ -z "$pbs" ]; then errors+=("PBS-ANA: unreachable or no snapshots"); else done <<<"$pbs" fi +# --- Layer: did the vzdump jobs actually SUCCEED? ----------------------- +# +# Snapshot age alone is structurally blind to a job that runs and ERRORS every +# night: nothing new is written, so the group simply ages, and the fault only +# surfaces once it crosses the 48h threshold — days late. esh-vm-workstation +# (VM 102) failed nightly from ~2026-09-06 with "timeout waiting on systemd" +# and this check would not have named it until 09-12. +# +# PVE records every task's result in /var/log/pve/tasks/index as +# +# where the UPID's 6th colon-field is the task type and the 5th is its hex +# start time. Anything vzdump in the window whose status is not OK is a fault +# TODAY, not in two days. +for entry in "${PVE_NODES[@]}"; do + set -- $entry; node_label="$1"; node_addr="$2" + idx=$($SSH "$node_addr" "sudo -n cat /var/log/pve/tasks/index" 2>/dev/null) + if [ -z "$idx" ]; then + errors+=("vzdump task log on $node_label UNREADABLE — job failures there are invisible") + continue + fi + bad=$(awk -v now="$now" -v win="$((JOB_WINDOW_H * 3600))" ' + { + upid = $1; endh = $2; status = $0 + sub(/^[^ ]+ [^ ]+ /, "", status) + n = split(upid, f, ":") + if (n < 7 || f[6] != "vzdump") next + start = strtonum("0x" f[5]) + if (now - start > win) next + if (status == "OK") next + printf "%s%s: %s\n", (f[7] == "" ? "job" : "guest " f[7]), "", status + }' <<<"$idx" | sort -u) + [ -n "$bad" ] && while IFS= read -r b; do + [ -n "$b" ] && jobfail+=("$node_label $b") + done <<<"$bad" +done + # --- rest-server endpoint health (401 = up & serving) --- for ep in "rest-server-ana http://10.250.50.70:8000/" "rest-server-nh3 http://10.100.50.50:8000/"; do set -- $ep @@ -153,9 +200,16 @@ if [ "${#excluded[@]}" -gt 0 ]; then echo; echo "NOT BACKED UP BY POLICY (${#excluded[@]}) — not a fault, but check the list is still right:" printf ' ⏸ %s\n' "${excluded[@]}" fi +if [ "${#jobfail[@]}" -gt 0 ]; then + echo; echo "BACKUP JOBS THAT ERRORED in the last ${JOB_WINDOW_H}h (${#jobfail[@]}):" + printf ' ❌ %s\n' "${jobfail[@]}" +fi if [ "${#stale[@]}" -gt 0 ] || [ "${#errors[@]}" -gt 0 ]; then echo; echo "STALE / PROBLEMS (${#stale[@]}+${#errors[@]}):" - printf ' 🔴 %s\n' "${stale[@]}" "${errors[@]}" + [ "${#stale[@]}" -gt 0 ] && printf ' 🔴 %s\n' "${stale[@]}" + [ "${#errors[@]}" -gt 0 ] && printf ' 🔴 %s\n' "${errors[@]}" +fi +if [ "${#stale[@]}" -gt 0 ] || [ "${#errors[@]}" -gt 0 ] || [ "${#jobfail[@]}" -gt 0 ]; then echo; echo "RESULT: STALE — see docs/runbooks/backups.md" exit 1 fi