70 lines
3.0 KiB
Bash
Executable File
70 lines
3.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# check-backup-freshness.sh — the "are we actually backed up?" check.
|
|
#
|
|
# Walks every backup layer and flags anything whose newest snapshot is older
|
|
# than the threshold (default 48h) or any down endpoint. Prints a report;
|
|
# exits 0 if everything is fresh, 1 if anything is stale/down. Designed to be
|
|
# run by a daily timer that alerts on non-zero exit (see
|
|
# scripts/install-backup-freshness-timer.sh), or by hand anytime.
|
|
#
|
|
# Companion to docs/runbooks/backups.md. Read-only — only SSH stat/curl.
|
|
#
|
|
# BACKUP_MAX_AGE_HOURS=48 scripts/check-backup-freshness.sh
|
|
set -uo pipefail
|
|
|
|
MAX_AGE_H="${BACKUP_MAX_AGE_HOURS:-48}"
|
|
SSH="ssh -o ConnectTimeout=8 -o BatchMode=yes"
|
|
now=$(date +%s)
|
|
stale=() ; fresh=() ; errors=()
|
|
|
|
# newest snapshot epoch under a remote glob (echoes epoch or empty)
|
|
newest_epoch() { # $1=host $2=glob
|
|
$SSH "$1" "stat -c %Y $2 2>/dev/null | sort -n | tail -1" 2>/dev/null
|
|
}
|
|
report() { # $1=label $2=epoch("" = none)
|
|
local label="$1" ep="$2"
|
|
if [ -z "$ep" ]; then stale+=("$label: NO SNAPSHOTS / unreachable"); return; fi
|
|
local age=$(( (now - ep) / 3600 ))
|
|
local when; when=$(date -d "@$ep" '+%Y-%m-%d %H:%M' 2>/dev/null)
|
|
if [ "$age" -gt "$MAX_AGE_H" ]; then stale+=("$label: ${age}h old (newest $when)")
|
|
else fresh+=("$label: ${age}h old (newest $when)"); fi
|
|
}
|
|
|
|
echo "=== Backup freshness (threshold ${MAX_AGE_H}h) — $(date '+%Y-%m-%d %H:%M %Z') ==="
|
|
|
|
# --- Layer: restic file+DB, ANA side (rest-server-ana) ---
|
|
for c in ana-docker ana-ml2 esh-docker-vm esh-vm-db vm-esh-nas; do
|
|
report "restic/ana/$c" "$(newest_epoch ana-nas "/mnt/backup/restic/repo/ana/$c/snapshots/*")"
|
|
done
|
|
# --- Layer: restic file+DB, NH3 side (rest-server-nh3) ---
|
|
for c in irv-ml1 nh3-docker; do
|
|
report "restic/nh3/$c" "$(newest_epoch nh3-nas "/volume1/Backup/restic/$c/snapshots/*")"
|
|
done
|
|
# --- Layer: PBS VM images (newest per guest, all namespaces) ---
|
|
pbs=$($SSH pbs-ana 'for ns in /mnt/pbs-datastore/ns/*/; do n=$(basename "$ns")
|
|
for d in vm ct; do for g in "$ns$d"/*/; do [ -d "$g" ] || continue
|
|
nb=$(ls -d "$g"20*T* 2>/dev/null | sort | tail -1)
|
|
[ -n "$nb" ] && echo "$n/$d/$(basename "$g") $(stat -c %Y "$nb")"
|
|
done; done; done' 2>/dev/null)
|
|
if [ -z "$pbs" ]; then errors+=("PBS-ANA: unreachable or no snapshots"); else
|
|
while read -r guest ep; do [ -n "$guest" ] && report "pbs/$guest" "$ep"; done <<<"$pbs"
|
|
fi
|
|
|
|
# --- rest-server endpoint health (401 = up & serving) ---
|
|
for ep in "rest-server-ana http://10.250.50.70:8000/" "rest-server-nh3 http://10.100.50.50:8000/"; do
|
|
set -- $ep
|
|
code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 6 "$2" 2>/dev/null)
|
|
[ "$code" = "401" ] && fresh+=("$1: up (401)") || stale+=("$1: endpoint code=$code (expected 401)")
|
|
done
|
|
|
|
echo
|
|
echo "FRESH (${#fresh[@]}):"; printf ' ✅ %s\n' "${fresh[@]}"
|
|
if [ "${#stale[@]}" -gt 0 ] || [ "${#errors[@]}" -gt 0 ]; then
|
|
echo; echo "STALE / PROBLEMS (${#stale[@]}+${#errors[@]}):"
|
|
printf ' 🔴 %s\n' "${stale[@]}" "${errors[@]}"
|
|
echo; echo "RESULT: STALE — see docs/runbooks/backups.md"
|
|
exit 1
|
|
fi
|
|
echo; echo "RESULT: all backups fresh"
|
|
exit 0
|