diff --git a/scripts/backup-freshness-alert.sh b/scripts/backup-freshness-alert.sh index 0552ceb..9136664 100755 --- a/scripts/backup-freshness-alert.sh +++ b/scripts/backup-freshness-alert.sh @@ -1,19 +1,85 @@ #!/usr/bin/env bash # backup-freshness-alert.sh — daily wrapper around check-backup-freshness.sh. -# Runs the check; on any stale/down layer (exit!=0) posts an althing alert to -# infra-ops so the silent-failure class (the 2026-05-06→06-20 ana outage that -# went unnoticed ~6.5 weeks) can't recur. Installed as a systemd user timer on -# nh3-dev via scripts/install-backup-freshness-timer.sh. +# +# Runs the check; on any stale/down layer (exit != 0) posts an althing alert so +# the silent-failure class (the 2026-05-06→06-20 ana outage that went unnoticed +# ~6.5 weeks) can't recur. Installed as a systemd user timer on nh3-dev via +# scripts/install-backup-freshness-timer.sh. +# +# ⚠ 2026-09-19 — THIS SCRIPT'S OWN ALARM WIRE WAS CUT FOR THREE WEEKS. +# It called `althing-cli`, which the v3 post-office cutover (2026-08-28) +# DELETED rather than deprecated. The check kept working and kept detecting +# stale backups; it simply could not tell anyone, and the only trace was one +# WARN line inside a unit that was already showing `failed` for the stale +# backups themselves. Two lessons are wired into the code below: +# +# 1. A BROKEN ALARM IS A WORSE FAULT THAN THE THING IT WATCHES, so it gets +# its own exit code (2) instead of being folded into the check's (1). +# `systemctl status` can now tell "backups are stale" from "nobody was +# told". +# 2. THE HEALTHY PATH MUST BE EXERCISABLE. `--test-alert` sends a real +# message through the real path on demand — a positive control, so the +# wire is provably intact without waiting for a genuine stale backup. +# +# Alert recipient is infra-hermes: day-to-day checks and routine triage are +# his half of the split (operator ruling 2026-09-19). He escalates to +# infra-ops. ⚠ It must NOT be infra-ops — this runs AS infra-ops, so that +# would mail the alarm to itself, which is the mirror trap named in CLAUDE.md. +# +# Exit codes: +# 0 backups fresh +# 1 backups STALE/DOWN, and the alert was delivered +# 2 the ALERT PATH ITSELF FAILED (investigate this before the backups) set -uo pipefail + REPO=/home/lkraven/development/eshpfi-management -ALTHING=/home/lkraven/.local/bin/althing-cli +POSTBOX=/home/lkraven/.local/bin/postbox +ALERT_TO=${BACKUP_ALERT_TO:-infra-hermes} + +# Both vars are required and postbox has no default address by design — a +# systemd user unit inherits neither, so the unit sets them. Fail loudly here +# rather than discovering it at the moment an alert is needed. +if [ -z "${ALTHING_POST_OFFICE:-}" ] || [ -z "${ALTHING_HANDLE:-}" ]; then + echo "FATAL: ALTHING_POST_OFFICE and ALTHING_HANDLE must both be set —" >&2 + echo " postbox has no default address. The alert path is DOWN." >&2 + exit 2 +fi +if [ ! -x "$POSTBOX" ]; then + echo "FATAL: $POSTBOX is missing or not executable. The alert path is DOWN." >&2 + exit 2 +fi + +send_alert() { # $1 = subject, body on stdin + "$POSTBOX" send --to "$ALERT_TO" --subject "$1" +} + +# --- positive control: prove the wire without waiting for a real fault --- +if [ "${1:-}" = "--test-alert" ]; then + printf '%s\n' \ + "Positive control for the fleet backup-freshness alarm — no action needed." \ + "" \ + "This message exists to prove the alert path is intact. If you are reading" \ + "it, backup-freshness-alert.sh can reach you. The path was silently dead" \ + "from the 2026-08-28 althing v3 cutover until 2026-09-19 because this" \ + "script still called the deleted althing-cli." \ + | send_alert "✅ Backup-freshness alarm test ($(date '+%Y-%m-%d %H:%M'))" || { + echo "FAIL: the alert path is DOWN — that is the finding." >&2; exit 2; } + echo "OK: alert path verified to $ALERT_TO." + exit 0 +fi out=$("$REPO/scripts/check-backup-freshness.sh" 2>&1); rc=$? printf '%s\n' "$out" +[ "$rc" -eq 0 ] && exit 0 -if [ "$rc" -ne 0 ]; then - printf 'Automated daily backup-freshness check found STALE or DOWN backup layer(s) on the PFI fleet.\nRunbook: docs/runbooks/backups.md (topology, 2-min check, rest-server-ana recovery).\n\n%s\n' "$out" \ - | "$ALTHING" post --to infra-ops --subject "🔴 Backup freshness ALERT ($(date '+%Y-%m-%d'))" 2>&1 \ - || echo "WARN: althing alert post failed — the check still ran (exit $rc); investigate manually." -fi -exit "$rc" +printf '%s\n' \ + "Automated daily backup-freshness check found STALE or DOWN backup layer(s) on the PFI fleet." \ + "Runbook: docs/runbooks/backups.md (topology, 2-min check, rest-server-ana recovery)." \ + "" \ + "$out" \ + | send_alert "🔴 Backup freshness ALERT ($(date '+%Y-%m-%d'))" || { + echo "FATAL: backups are STALE *and* the alert post FAILED — nobody was told." >&2 + echo " Fix the alert path first; that is the fault that hides the others." >&2 + exit 2; } + +exit 1 diff --git a/scripts/install-backup-freshness-timer.sh b/scripts/install-backup-freshness-timer.sh index 129d27b..39a30eb 100755 --- a/scripts/install-backup-freshness-timer.sh +++ b/scripts/install-backup-freshness-timer.sh @@ -15,7 +15,12 @@ After=network-online.target [Service] Type=oneshot +# Both are required: postbox has no default post-office address by design, and +# a systemd user unit inherits neither from the interactive shell. Omitting +# ALTHING_POST_OFFICE is why swapping althing-cli->postbox alone would have +# left this unit just as dead, with a different error message. Environment=ALTHING_HANDLE=infra-ops +Environment=ALTHING_POST_OFFICE=http://10.100.50.40:8390 ExecStart=$REPO/scripts/backup-freshness-alert.sh EOF