feat(backups): freshness check + daily alert timer; record rest-server-ana recovery, fstab hardening, esh-pve-nas gap, worldtree admin-key provisioning
This commit is contained in:
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env bash
|
||||
# backup-freshness-alert.sh — daily wrapper around check-backup-freshness.sh.
|
||||
# Runs the check; on any stale/down layer (exit!=0) posts an althing alert to
|
||||
# infra-ops so the silent-failure class (the 2026-05-06→06-20 ana outage that
|
||||
# went unnoticed ~6.5 weeks) can't recur. Installed as a systemd user timer on
|
||||
# nh3-dev via scripts/install-backup-freshness-timer.sh.
|
||||
set -uo pipefail
|
||||
REPO=/home/lkraven/development/eshpfi-management
|
||||
ALTHING=/home/lkraven/.local/bin/althing-cli
|
||||
|
||||
out=$("$REPO/scripts/check-backup-freshness.sh" 2>&1); rc=$?
|
||||
printf '%s\n' "$out"
|
||||
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
printf 'Automated daily backup-freshness check found STALE or DOWN backup layer(s) on the PFI fleet.\nRunbook: docs/runbooks/backups.md (topology, 2-min check, rest-server-ana recovery).\n\n%s\n' "$out" \
|
||||
| "$ALTHING" post --to infra-ops --subject "🔴 Backup freshness ALERT ($(date '+%Y-%m-%d'))" 2>&1 \
|
||||
|| echo "WARN: althing alert post failed — the check still ran (exit $rc); investigate manually."
|
||||
fi
|
||||
exit "$rc"
|
||||
Reference in New Issue
Block a user