#!/usr/bin/env bash # backup-freshness-alert.sh — daily wrapper around check-backup-freshness.sh. # Runs the check; on any stale/down layer (exit!=0) posts an althing alert to # infra-ops so the silent-failure class (the 2026-05-06→06-20 ana outage that # went unnoticed ~6.5 weeks) can't recur. Installed as a systemd user timer on # nh3-dev via scripts/install-backup-freshness-timer.sh. set -uo pipefail REPO=/home/lkraven/development/eshpfi-management ALTHING=/home/lkraven/.local/bin/althing-cli out=$("$REPO/scripts/check-backup-freshness.sh" 2>&1); rc=$? printf '%s\n' "$out" if [ "$rc" -ne 0 ]; then printf 'Automated daily backup-freshness check found STALE or DOWN backup layer(s) on the PFI fleet.\nRunbook: docs/runbooks/backups.md (topology, 2-min check, rest-server-ana recovery).\n\n%s\n' "$out" \ | "$ALTHING" post --to infra-ops --subject "🔴 Backup freshness ALERT ($(date '+%Y-%m-%d'))" 2>&1 \ || echo "WARN: althing alert post failed — the check still ran (exit $rc); investigate manually." fi exit "$rc"