diff --git a/scripts/backup-freshness-alert.sh b/scripts/backup-freshness-alert.sh index 9136664..c394517 100755 --- a/scripts/backup-freshness-alert.sh +++ b/scripts/backup-freshness-alert.sh @@ -26,10 +26,16 @@ # infra-ops. ⚠ It must NOT be infra-ops — this runs AS infra-ops, so that # would mail the alarm to itself, which is the mirror trap named in CLAUDE.md. # -# Exit codes: +# Exit codes — mirrored from the check, so `systemctl status` distinguishes +# the three states that matter: # 0 backups fresh # 1 backups STALE/DOWN, and the alert was delivered # 2 the ALERT PATH ITSELF FAILED (investigate this before the backups) +# 3 ERRORED-JOBS — every body fresh, a job errored recently; alert delivered +# +# 3 is deliberately not 1. A nightly job that errored is worth a look; a stale +# backup body is worth a page. Reporting both as "STALE" over a fleet whose +# every backup is current is how an alarm teaches you to ignore it. set -uo pipefail REPO=/home/lkraven/development/eshpfi-management @@ -72,14 +78,24 @@ out=$("$REPO/scripts/check-backup-freshness.sh" 2>&1); rc=$? printf '%s\n' "$out" [ "$rc" -eq 0 ] && exit 0 +# Match the words and the emoji to the finding. A job-errors-only state is not +# an outage and must not dress like one. +if [ "$rc" -eq 3 ]; then + subject="🟡 Backup jobs errored ($(date '+%Y-%m-%d')) — all bodies fresh" + lead="A vzdump job errored recently. Every backup body on the fleet is FRESH; nothing is stale." +else + subject="🔴 Backup freshness ALERT ($(date '+%Y-%m-%d'))" + lead="Automated daily backup-freshness check found STALE or DOWN backup layer(s) on the PFI fleet." +fi + printf '%s\n' \ - "Automated daily backup-freshness check found STALE or DOWN backup layer(s) on the PFI fleet." \ + "$lead" \ "Runbook: docs/runbooks/backups.md (topology, 2-min check, rest-server-ana recovery)." \ "" \ "$out" \ - | send_alert "🔴 Backup freshness ALERT ($(date '+%Y-%m-%d'))" || { - echo "FATAL: backups are STALE *and* the alert post FAILED — nobody was told." >&2 + | send_alert "$subject" || { + echo "FATAL: the check found problems *and* the alert post FAILED — nobody was told." >&2 echo " Fix the alert path first; that is the fault that hides the others." >&2 exit 2; } -exit 1 +exit "$rc" diff --git a/scripts/check-backup-freshness.sh b/scripts/check-backup-freshness.sh index cd7e96a..66df688 100755 --- a/scripts/check-backup-freshness.sh +++ b/scripts/check-backup-freshness.sh @@ -2,8 +2,16 @@ # check-backup-freshness.sh — the "are we actually backed up?" check. # # Walks every backup layer and flags anything whose newest snapshot is older -# than the threshold (default 48h) or any down endpoint. Prints a report; -# exits 0 if everything is fresh, 1 if anything is stale/down. Designed to be +# than the threshold (default 48h) or any down endpoint. Prints a report. +# +# EXIT CODES — the verdict, not just a pass/fail: +# 0 all backups fresh +# 1 STALE — a backup body is past the threshold, or an endpoint is down +# 3 ERRORED-JOBS — every body is fresh, but a vzdump job errored recently +# +# 1 and 3 are deliberately different. Collapsing them prints "STALE" over a +# fleet whose every backup is current, which is how an alarm earns being +# ignored. Designed to be # run by a daily timer that alerts on non-zero exit (see # scripts/install-backup-freshness-timer.sh), or by hand anytime. # @@ -209,9 +217,28 @@ if [ "${#stale[@]}" -gt 0 ] || [ "${#errors[@]}" -gt 0 ]; then [ "${#stale[@]}" -gt 0 ] && printf ' 🔴 %s\n' "${stale[@]}" [ "${#errors[@]}" -gt 0 ] && printf ' 🔴 %s\n' "${errors[@]}" fi -if [ "${#stale[@]}" -gt 0 ] || [ "${#errors[@]}" -gt 0 ] || [ "${#jobfail[@]}" -gt 0 ]; then +# --------- Verdict. Two different findings, two different words. ------- +# +# ⚠ These used to collapse into one "RESULT: STALE / exit 1". That is a false +# statement of fact whenever the only finding is a recent job error: on +# 2026-09-20 this printed STALE while reporting 37 FRESH layers and zero stale +# ones — every backup body provably current, the ❌ rows all yesterday's +# pre-fix runs aging out of the window. infra-hermes flagged it: a reader or a +# forwarder could page someone over a state where nothing is actually stale. +# +# STALE is a claim about backup AGE. A job that ran and errored is a different +# claim with different urgency, so it gets its own verdict and its own code. +if [ "${#stale[@]}" -gt 0 ] || [ "${#errors[@]}" -gt 0 ]; then + # Bodies stale or an endpoint down. The serious one. Job errors may ride + # along and are already printed above. echo; echo "RESULT: STALE — see docs/runbooks/backups.md" exit 1 fi +if [ "${#jobfail[@]}" -gt 0 ]; then + echo; echo "RESULT: ERRORED-JOBS — every backup body is FRESH; ${#jobfail[@]} job(s)" + echo " errored in the last ${JOB_WINDOW_H}h. Worth a look, not a page." + echo " Errors older than the window age out on their own." + exit 3 +fi echo; echo "RESULT: all backups fresh" exit 0