feat(backups): freshness check + daily alert timer; record rest-server-ana recovery, fstab hardening, esh-pve-nas gap, worldtree admin-key provisioning

scripts/backup-freshness-alert.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# backup-freshness-alert.sh — daily wrapper around check-backup-freshness.sh.
+# Runs the check; on any stale/down layer (exit!=0) posts an althing alert to
+# infra-ops so the silent-failure class (the 2026-05-06→06-20 ana outage that
+# went unnoticed ~6.5 weeks) can't recur. Installed as a systemd user timer on
+# nh3-dev via scripts/install-backup-freshness-timer.sh.
+set -uo pipefail
+REPO=/home/lkraven/development/eshpfi-management
+ALTHING=/home/lkraven/.local/bin/althing-cli
+
+out=$("$REPO/scripts/check-backup-freshness.sh" 2>&1); rc=$?
+printf '%s\n' "$out"
+
+if [ "$rc" -ne 0 ]; then
+  printf 'Automated daily backup-freshness check found STALE or DOWN backup layer(s) on the PFI fleet.\nRunbook: docs/runbooks/backups.md (topology, 2-min check, rest-server-ana recovery).\n\n%s\n' "$out" \
+    | "$ALTHING" post --to infra-ops --subject "🔴 Backup freshness ALERT ($(date '+%Y-%m-%d'))" 2>&1 \
+    || echo "WARN: althing alert post failed — the check still ran (exit $rc); investigate manually."
+fi
+exit "$rc"

scripts/check-backup-freshness.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# check-backup-freshness.sh — the "are we actually backed up?" check.
+#
+# Walks every backup layer and flags anything whose newest snapshot is older
+# than the threshold (default 48h) or any down endpoint. Prints a report;
+# exits 0 if everything is fresh, 1 if anything is stale/down. Designed to be
+# run by a daily timer that alerts on non-zero exit (see
+# scripts/install-backup-freshness-timer.sh), or by hand anytime.
+#
+# Companion to docs/runbooks/backups.md. Read-only — only SSH stat/curl.
+#
+#   BACKUP_MAX_AGE_HOURS=48 scripts/check-backup-freshness.sh
+set -uo pipefail
+
+MAX_AGE_H="${BACKUP_MAX_AGE_HOURS:-48}"
+SSH="ssh -o ConnectTimeout=8 -o BatchMode=yes"
+now=$(date +%s)
+stale=() ; fresh=() ; errors=()
+
+# newest snapshot epoch under a remote glob (echoes epoch or empty)
+newest_epoch() { # $1=host  $2=glob
+  $SSH "$1" "stat -c %Y $2 2>/dev/null | sort -n | tail -1" 2>/dev/null
+}
+report() { # $1=label  $2=epoch("" = none)
+  local label="$1" ep="$2"
+  if [ -z "$ep" ]; then stale+=("$label: NO SNAPSHOTS / unreachable"); return; fi
+  local age=$(( (now - ep) / 3600 ))
+  local when; when=$(date -d "@$ep" '+%Y-%m-%d %H:%M' 2>/dev/null)
+  if [ "$age" -gt "$MAX_AGE_H" ]; then stale+=("$label: ${age}h old (newest $when)")
+  else fresh+=("$label: ${age}h old (newest $when)"); fi
+}
+
+echo "=== Backup freshness (threshold ${MAX_AGE_H}h) — $(date '+%Y-%m-%d %H:%M %Z') ==="
+
+# --- Layer: restic file+DB, ANA side (rest-server-ana) ---
+for c in ana-docker ana-ml2 esh-docker-vm esh-vm-db vm-esh-nas; do
+  report "restic/ana/$c" "$(newest_epoch ana-nas "/mnt/backup/restic/repo/ana/$c/snapshots/*")"
+done
+# --- Layer: restic file+DB, NH3 side (rest-server-nh3) ---
+for c in irv-ml1 nh3-docker; do
+  report "restic/nh3/$c" "$(newest_epoch nh3-nas "/volume1/Backup/restic/$c/snapshots/*")"
+done
+# --- Layer: PBS VM images (newest per guest, all namespaces) ---
+pbs=$($SSH pbs-ana 'for ns in /mnt/pbs-datastore/ns/*/; do n=$(basename "$ns")
+  for d in vm ct; do for g in "$ns$d"/*/; do [ -d "$g" ] || continue
+    nb=$(ls -d "$g"20*T* 2>/dev/null | sort | tail -1)
+    [ -n "$nb" ] && echo "$n/$d/$(basename "$g") $(stat -c %Y "$nb")"
+  done; done; done' 2>/dev/null)
+if [ -z "$pbs" ]; then errors+=("PBS-ANA: unreachable or no snapshots"); else
+  while read -r guest ep; do [ -n "$guest" ] && report "pbs/$guest" "$ep"; done <<<"$pbs"
+fi
+
+# --- rest-server endpoint health (401 = up & serving) ---
+for ep in "rest-server-ana http://10.250.50.70:8000/" "rest-server-nh3 http://10.100.50.50:8000/"; do
+  set -- $ep
+  code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 6 "$2" 2>/dev/null)
+  [ "$code" = "401" ] && fresh+=("$1: up (401)") || stale+=("$1: endpoint code=$code (expected 401)")
+done
+
+echo
+echo "FRESH (${#fresh[@]}):"; printf '  ✅ %s\n' "${fresh[@]}"
+if [ "${#stale[@]}" -gt 0 ] || [ "${#errors[@]}" -gt 0 ]; then
+  echo; echo "STALE / PROBLEMS (${#stale[@]}+${#errors[@]}):"
+  printf '  🔴 %s\n' "${stale[@]}" "${errors[@]}"
+  echo; echo "RESULT: STALE — see docs/runbooks/backups.md"
+  exit 1
+fi
+echo; echo "RESULT: all backups fresh"
+exit 0

scripts/install-backup-freshness-timer.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# install-backup-freshness-timer.sh — install/refresh the daily backup-freshness
+# alert as a systemd USER timer on nh3-dev (the only host with SSH to all backup
+# stores + althing-cli). Idempotent; re-run after editing the wrapper/check.
+# Requires linger (loginctl enable-linger lkraven) so it fires without a login.
+set -euo pipefail
+UNIT_DIR="$HOME/.config/systemd/user"
+REPO=/home/lkraven/development/eshpfi-management
+mkdir -p "$UNIT_DIR"
+
+cat > "$UNIT_DIR/backup-freshness.service" <<EOF
+[Unit]
+Description=Fleet backup freshness check + althing alert
+After=network-online.target
+
+[Service]
+Type=oneshot
+Environment=ALTHING_HANDLE=infra-ops
+ExecStart=$REPO/scripts/backup-freshness-alert.sh
+EOF
+
+cat > "$UNIT_DIR/backup-freshness.timer" <<EOF
+[Unit]
+Description=Daily fleet backup freshness check (08:00)
+
+[Timer]
+OnCalendar=*-*-* 08:00:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target
+EOF
+
+systemctl --user daemon-reload
+systemctl --user enable --now backup-freshness.timer
+echo "installed. next run:"
+systemctl --user list-timers backup-freshness.timer --all --no-pager