#!/usr/bin/env bash
# check-backup-freshness.sh — the "are we actually backed up?" check.
#
# Walks every backup layer and flags anything whose newest snapshot is older
# than the threshold (default 48h) or any down endpoint. Prints a report;
# exits 0 if everything is fresh, 1 if anything is stale/down. Designed to be
# run by a daily timer that alerts on non-zero exit (see
# scripts/install-backup-freshness-timer.sh), or by hand anytime.
#
# Companion to docs/runbooks/backups.md. Read-only — only SSH stat/curl.
#
#   BACKUP_MAX_AGE_HOURS=48 scripts/check-backup-freshness.sh
set -uo pipefail

MAX_AGE_H="${BACKUP_MAX_AGE_HOURS:-48}"
SSH="ssh -o ConnectTimeout=8 -o BatchMode=yes"
now=$(date +%s)
stale=() ; fresh=() ; errors=()

# newest snapshot epoch under a remote glob (echoes epoch or empty)
newest_epoch() { # $1=host  $2=glob
  $SSH "$1" "stat -c %Y $2 2>/dev/null | sort -n | tail -1" 2>/dev/null
}
report() { # $1=label  $2=epoch("" = none)
  local label="$1" ep="$2"
  if [ -z "$ep" ]; then stale+=("$label: NO SNAPSHOTS / unreachable"); return; fi
  local age=$(( (now - ep) / 3600 ))
  local when; when=$(date -d "@$ep" '+%Y-%m-%d %H:%M' 2>/dev/null)
  if [ "$age" -gt "$MAX_AGE_H" ]; then stale+=("$label: ${age}h old (newest $when)")
  else fresh+=("$label: ${age}h old (newest $when)"); fi
}

echo "=== Backup freshness (threshold ${MAX_AGE_H}h) — $(date '+%Y-%m-%d %H:%M %Z') ==="

# --- Layer: restic file+DB, ANA side (rest-server-ana) ---
for c in ana-docker ana-ml2 esh-docker-vm esh-vm-db vm-esh-nas; do
  report "restic/ana/$c" "$(newest_epoch ana-nas "/mnt/backup/restic/repo/ana/$c/snapshots/*")"
done
# --- Layer: restic file+DB, NH3 side (rest-server-nh3) ---
for c in irv-ml1 nh3-docker; do
  report "restic/nh3/$c" "$(newest_epoch nh3-nas "/volume1/Backup/restic/$c/snapshots/*")"
done
# --- Layer: PBS VM images (newest per guest, all namespaces) ---
pbs=$($SSH pbs-ana 'for ns in /mnt/pbs-datastore/ns/*/; do n=$(basename "$ns")
  for d in vm ct; do for g in "$ns$d"/*/; do [ -d "$g" ] || continue
    nb=$(ls -d "$g"20*T* 2>/dev/null | sort | tail -1)
    [ -n "$nb" ] && echo "$n/$d/$(basename "$g") $(stat -c %Y "$nb")"
  done; done; done' 2>/dev/null)
if [ -z "$pbs" ]; then errors+=("PBS-ANA: unreachable or no snapshots"); else
  while read -r guest ep; do [ -n "$guest" ] && report "pbs/$guest" "$ep"; done <<<"$pbs"
fi

# --- rest-server endpoint health (401 = up & serving) ---
for ep in "rest-server-ana http://10.250.50.70:8000/" "rest-server-nh3 http://10.100.50.50:8000/"; do
  set -- $ep
  code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 6 "$2" 2>/dev/null)
  [ "$code" = "401" ] && fresh+=("$1: up (401)") || stale+=("$1: endpoint code=$code (expected 401)")
done

echo
echo "FRESH (${#fresh[@]}):"; printf '  ✅ %s\n' "${fresh[@]}"
if [ "${#stale[@]}" -gt 0 ] || [ "${#errors[@]}" -gt 0 ]; then
  echo; echo "STALE / PROBLEMS (${#stale[@]}+${#errors[@]}):"
  printf '  🔴 %s\n' "${stale[@]}" "${errors[@]}"
  echo; echo "RESULT: STALE — see docs/runbooks/backups.md"
  exit 1
fi
echo; echo "RESULT: all backups fresh"
exit 0