feat(backups): freshness check + daily alert timer; record rest-server-ana recovery, fstab hardening, esh-pve-nas gap, worldtree admin-key provisioning

2026-06-20 16:13:13 -07:00
parent a7b4a82dec
commit 76b317ce3e
5 changed files with 170 additions and 20 deletions
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# check-backup-freshness.sh — the "are we actually backed up?" check.
+#
+# Walks every backup layer and flags anything whose newest snapshot is older
+# than the threshold (default 48h) or any down endpoint. Prints a report;
+# exits 0 if everything is fresh, 1 if anything is stale/down. Designed to be
+# run by a daily timer that alerts on non-zero exit (see
+# scripts/install-backup-freshness-timer.sh), or by hand anytime.
+#
+# Companion to docs/runbooks/backups.md. Read-only — only SSH stat/curl.
+#
+#   BACKUP_MAX_AGE_HOURS=48 scripts/check-backup-freshness.sh
+set -uo pipefail
+
+MAX_AGE_H="${BACKUP_MAX_AGE_HOURS:-48}"
+SSH="ssh -o ConnectTimeout=8 -o BatchMode=yes"
+now=$(date +%s)
+stale=() ; fresh=() ; errors=()
+
+# newest snapshot epoch under a remote glob (echoes epoch or empty)
+newest_epoch() { # $1=host  $2=glob
+  $SSH "$1" "stat -c %Y $2 2>/dev/null | sort -n | tail -1" 2>/dev/null
+}
+report() { # $1=label  $2=epoch("" = none)
+  local label="$1" ep="$2"
+  if [ -z "$ep" ]; then stale+=("$label: NO SNAPSHOTS / unreachable"); return; fi
+  local age=$(( (now - ep) / 3600 ))
+  local when; when=$(date -d "@$ep" '+%Y-%m-%d %H:%M' 2>/dev/null)
+  if [ "$age" -gt "$MAX_AGE_H" ]; then stale+=("$label: ${age}h old (newest $when)")
+  else fresh+=("$label: ${age}h old (newest $when)"); fi
+}
+
+echo "=== Backup freshness (threshold ${MAX_AGE_H}h) — $(date '+%Y-%m-%d %H:%M %Z') ==="
+
+# --- Layer: restic file+DB, ANA side (rest-server-ana) ---
+for c in ana-docker ana-ml2 esh-docker-vm esh-vm-db vm-esh-nas; do
+  report "restic/ana/$c" "$(newest_epoch ana-nas "/mnt/backup/restic/repo/ana/$c/snapshots/*")"
+done
+# --- Layer: restic file+DB, NH3 side (rest-server-nh3) ---
+for c in irv-ml1 nh3-docker; do
+  report "restic/nh3/$c" "$(newest_epoch nh3-nas "/volume1/Backup/restic/$c/snapshots/*")"
+done
+# --- Layer: PBS VM images (newest per guest, all namespaces) ---
+pbs=$($SSH pbs-ana 'for ns in /mnt/pbs-datastore/ns/*/; do n=$(basename "$ns")
+  for d in vm ct; do for g in "$ns$d"/*/; do [ -d "$g" ] || continue
+    nb=$(ls -d "$g"20*T* 2>/dev/null | sort | tail -1)
+    [ -n "$nb" ] && echo "$n/$d/$(basename "$g") $(stat -c %Y "$nb")"
+  done; done; done' 2>/dev/null)
+if [ -z "$pbs" ]; then errors+=("PBS-ANA: unreachable or no snapshots"); else
+  while read -r guest ep; do [ -n "$guest" ] && report "pbs/$guest" "$ep"; done <<<"$pbs"
+fi
+
+# --- rest-server endpoint health (401 = up & serving) ---
+for ep in "rest-server-ana http://10.250.50.70:8000/" "rest-server-nh3 http://10.100.50.50:8000/"; do
+  set -- $ep
+  code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 6 "$2" 2>/dev/null)
+  [ "$code" = "401" ] && fresh+=("$1: up (401)") || stale+=("$1: endpoint code=$code (expected 401)")
+done
+
+echo
+echo "FRESH (${#fresh[@]}):"; printf '  ✅ %s\n' "${fresh[@]}"
+if [ "${#stale[@]}" -gt 0 ] || [ "${#errors[@]}" -gt 0 ]; then
+  echo; echo "STALE / PROBLEMS (${#stale[@]}+${#errors[@]}):"
+  printf '  🔴 %s\n' "${stale[@]}" "${errors[@]}"
+  echo; echo "RESULT: STALE — see docs/runbooks/backups.md"
+  exit 1
+fi
+echo; echo "RESULT: all backups fresh"
+exit 0