feat(backups): freshness check + daily alert timer; record rest-server-ana recovery, fstab hardening, esh-pve-nas gap, worldtree admin-key provisioning

This commit is contained in:
vh
2026-06-20 16:13:13 -07:00
parent a7b4a82dec
commit 76b317ce3e
5 changed files with 170 additions and 20 deletions
+19
View File
@@ -0,0 +1,19 @@
#!/usr/bin/env bash
# backup-freshness-alert.sh — daily wrapper around check-backup-freshness.sh.
# Runs the check; on any stale/down layer (exit!=0) posts an althing alert to
# infra-ops so the silent-failure class (the 2026-05-06→06-20 ana outage that
# went unnoticed ~6.5 weeks) can't recur. Installed as a systemd user timer on
# nh3-dev via scripts/install-backup-freshness-timer.sh.
set -uo pipefail
REPO=/home/lkraven/development/eshpfi-management
ALTHING=/home/lkraven/.local/bin/althing-cli
out=$("$REPO/scripts/check-backup-freshness.sh" 2>&1); rc=$?
printf '%s\n' "$out"
if [ "$rc" -ne 0 ]; then
printf 'Automated daily backup-freshness check found STALE or DOWN backup layer(s) on the PFI fleet.\nRunbook: docs/runbooks/backups.md (topology, 2-min check, rest-server-ana recovery).\n\n%s\n' "$out" \
| "$ALTHING" post --to infra-ops --subject "🔴 Backup freshness ALERT ($(date '+%Y-%m-%d'))" 2>&1 \
|| echo "WARN: althing alert post failed — the check still ran (exit $rc); investigate manually."
fi
exit "$rc"