20 lines
1.0 KiB
Bash
Executable File
20 lines
1.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# backup-freshness-alert.sh — daily wrapper around check-backup-freshness.sh.
|
|
# Runs the check; on any stale/down layer (exit!=0) posts an althing alert to
|
|
# infra-ops so the silent-failure class (the 2026-05-06→06-20 ana outage that
|
|
# went unnoticed ~6.5 weeks) can't recur. Installed as a systemd user timer on
|
|
# nh3-dev via scripts/install-backup-freshness-timer.sh.
|
|
set -uo pipefail
|
|
REPO=/home/lkraven/development/eshpfi-management
|
|
ALTHING=/home/lkraven/.local/bin/althing-cli
|
|
|
|
out=$("$REPO/scripts/check-backup-freshness.sh" 2>&1); rc=$?
|
|
printf '%s\n' "$out"
|
|
|
|
if [ "$rc" -ne 0 ]; then
|
|
printf 'Automated daily backup-freshness check found STALE or DOWN backup layer(s) on the PFI fleet.\nRunbook: docs/runbooks/backups.md (topology, 2-min check, rest-server-ana recovery).\n\n%s\n' "$out" \
|
|
| "$ALTHING" post --to infra-ops --subject "🔴 Backup freshness ALERT ($(date '+%Y-%m-%d'))" 2>&1 \
|
|
|| echo "WARN: althing alert post failed — the check still ran (exit $rc); investigate manually."
|
|
fi
|
|
exit "$rc"
|