esh-pfi-infrastructure/scripts/check-backup-freshness.sh

#!/usr/bin/env bash
# check-backup-freshness.sh — the "are we actually backed up?" check.
#
# Walks every backup layer and flags anything whose newest snapshot is older
# than the threshold (default 48h) or any down endpoint. Prints a report.
#
# EXIT CODES — the verdict, not just a pass/fail:
#   0  all backups fresh
#   1  STALE — a backup body is past the threshold, or an endpoint is down
#   3  ERRORED-JOBS — every body is fresh, but a vzdump job errored recently
#
# 1 and 3 are deliberately different. Collapsing them prints "STALE" over a
# fleet whose every backup is current, which is how an alarm earns being
# ignored. Designed to be
# run by a daily timer that alerts on non-zero exit (see
# scripts/install-backup-freshness-timer.sh), or by hand anytime.
#
# Companion to docs/runbooks/backups.md. Read-only — only SSH stat/curl.
#
#   BACKUP_MAX_AGE_HOURS=48 scripts/check-backup-freshness.sh
#
# POLICY AWARENESS (2026-09-19). Some guests are deliberately not backed up --
# ana-scale (CT 114) and esh-scale (CT 108) are subnet-router LXCs excluded
# after a vzdump lock on esh-scale blackholed that entire site. Before this,
# the check reported those as 🔴 STALE every single morning, which is how an
# alarm teaches you to ignore it. Such guests now get their own section:
# printed, never hidden, but not counted as a fault.
#
# ⚠ COVERAGE IS A UNION ACROSS JOBS, NOT ONE JOB'S EXCLUDE LIST. Reading
# `exclude` alone gets ana CT 109 (ana-nas) exactly wrong: it IS excluded from
# the 03:00 all-guests job AND it has its own dedicated 22:00 job. Suppressing
# on the exclude list would have silently stopped alarming on a guest that is
# genuinely backed up -- turning a fix for a noisy alarm into a blind spot.
# The same applies in reverse for ESH, whose job uses an explicit
# `vmid 100..107` INCLUDE list, so esh-scale 108 is excluded by OMISSION and
# appears in no exclude list at all.
# A guest is "intentionally not backed up" only when NO enabled vzdump job
# covers it.
#
# ⚠ IF COVERAGE CANNOT BE DETERMINED, NOTHING IS SUPPRESSED. An unreachable
# PVE node means we do not know, and a backup alarm must fail loud.
# The SureFire namespace (sfsrv-pve) is deliberately never consulted -- those
# are tenant hosts -- so its guests can never be suppressed either.
set -uo pipefail

MAX_AGE_H="${BACKUP_MAX_AGE_HOURS:-48}"
SSH="ssh -o ConnectTimeout=8 -o BatchMode=yes"
now=$(date +%s)
stale=() ; fresh=() ; errors=() ; excluded=()

# PBS namespace → a node of the PVE cluster that owns it. jobs.cfg lives in
# pmxcfs and is cluster-wide, so ESH's two nodes share one job set and either
# answers for both. sfsrv-pve is absent ON PURPOSE (tenant property).
declare -A NS_NODE=(
  [ana-pve]=infra-ops@10.250.250.31
  [esh-pve]=infra-ops@10.0.250.35
  [nh3-pve]=infra-ops@10.100.250.60
)
declare -A NS_JOBS=()      # ns → one "enabled|all|vmids|excludes" line per job
declare -A NS_KNOWN=()     # ns → 1 once coverage was successfully read

# Every PVE node whose vzdump TASK RESULTS we read. Unlike jobs.cfg this is
# per-NODE, not cluster-wide, so both ESH nodes appear. sfsrv is absent: tenant.
PVE_NODES=(
  "ana-pve       infra-ops@10.250.250.31"
  "esh-pve       infra-ops@10.0.250.35"
  "esh-nas-pve   infra-ops@10.0.50.55"
  "nh3-pve       infra-ops@10.100.250.60"
)
JOB_WINDOW_H="${BACKUP_JOB_WINDOW_HOURS:-36}"
jobfail=()

load_jobs() { # $1 = namespace
  local ns="$1" node="${NS_NODE[$1]:-}" raw
  [ -n "$node" ] || return 1
  # Fetch the raw file and parse it HERE. The parser used to be an awk script
  # embedded in the ssh command string; the quoting mangled it silently and
  # every namespace came back unreadable — which at least failed loud, because
  # that is how this is built. Keep the remote side a plain `cat`.
  raw=$($SSH "$node" "sudo -n cat /etc/pve/jobs.cfg" 2>/dev/null) || return 1
  [ -n "$raw" ] || return 1
  local parsed
  parsed=$(awk '
    function flush() { if (seen) { print e "|" a "|" v "|" x; seen=0 } }
    /^vzdump:/       { flush(); seen=1; e=1; a=0; v=""; x=""; next }
    /^[^ \t]/       { flush(); next }
    seen && $1=="enabled" { e=$2 }
    seen && $1=="all"     { a=$2 }
    seen && $1=="vmid"    { v=$2 }
    seen && $1=="exclude" { x=$2 }
    END { flush() }
  ' <<<"$raw")
  [ -n "$parsed" ] || return 1
  NS_JOBS[$ns]="$parsed"
  NS_KNOWN[$ns]=1
}

in_csv() { # $1=needle  $2=comma list
  case ",$2," in *",$1,"*) return 0 ;; *) return 1 ;; esac
}

is_covered() { # $1=namespace  $2=vmid — 0 covered, 1 not covered, 2 unknown
  local ns="$1" id="$2" line enabled all vmids excludes
  [ -n "${NS_KNOWN[$ns]:-}" ] || return 2
  while IFS='|' read -r enabled all vmids excludes; do
    [ "$enabled" = "1" ] || continue
    if [ -n "$vmids" ]; then
      in_csv "$id" "$vmids" && return 0      # explicit include list
    elif [ "$all" = "1" ]; then
      in_csv "$id" "$excludes" || return 0   # all-guests minus its excludes
    fi
  done <<<"${NS_JOBS[$ns]}"
  return 1
}

# newest snapshot epoch under a remote glob (echoes epoch or empty)
newest_epoch() { # $1=host  $2=glob
  $SSH "$1" "stat -c %Y $2 2>/dev/null | sort -n | tail -1" 2>/dev/null
}
report() { # $1=label  $2=epoch("" = none)
  local label="$1" ep="$2"
  if [ -z "$ep" ]; then stale+=("$label: NO SNAPSHOTS / unreachable"); return; fi
  local age=$(( (now - ep) / 3600 ))
  local when; when=$(date -d "@$ep" '+%Y-%m-%d %H:%M' 2>/dev/null)
  if [ "$age" -gt "$MAX_AGE_H" ]; then stale+=("$label: ${age}h old (newest $when)")
  else fresh+=("$label: ${age}h old (newest $when)"); fi
}

echo "=== Backup freshness (threshold ${MAX_AGE_H}h) — $(date '+%Y-%m-%d %H:%M %Z') ==="

# --- Layer: restic file+DB, ANA side (rest-server-ana) ---
for c in ana-docker ana-ml2 esh-docker-vm esh-vm-db vm-esh-nas; do
  report "restic/ana/$c" "$(newest_epoch ana-nas "/mnt/backup/restic/repo/ana/$c/snapshots/*")"
done
# --- Layer: restic file+DB, NH3 side (rest-server-nh3) ---
for c in irv-ml1 nh3-docker; do
  report "restic/nh3/$c" "$(newest_epoch nh3-nas "/volume1/Backup/restic/$c/snapshots/*")"
done
# --- Layer: PBS VM images (newest per guest, all namespaces) ---
pbs=$($SSH pbs-ana 'for ns in /mnt/pbs-datastore/ns/*/; do n=$(basename "$ns")
  for d in vm ct; do for g in "$ns$d"/*/; do [ -d "$g" ] || continue
    nb=$(ls -d "$g"20*T* 2>/dev/null | sort | tail -1)
    [ -n "$nb" ] && echo "$n/$d/$(basename "$g") $(stat -c %Y "$nb")"
  done; done; done' 2>/dev/null)
if [ -z "$pbs" ]; then errors+=("PBS-ANA: unreachable or no snapshots"); else
  for ns in "${!NS_NODE[@]}"; do load_jobs "$ns" || errors+=("coverage for namespace '$ns' UNREADABLE — nothing in it will be suppressed"); done
  while read -r guest ep; do
    [ -n "$guest" ] || continue
    ns=${guest%%/*}; vmid=${guest##*/}
    is_covered "$ns" "$vmid"
    case $? in
      1) excluded+=("pbs/$guest: no enabled vzdump job covers it$(
           [ -n "$ep" ] && printf ' (last snapshot %s)' "$(date -d "@$ep" '+%Y-%m-%d' 2>/dev/null)")") ;;
      *) report "pbs/$guest" "$ep" ;;
    esac
  done <<<"$pbs"
fi

# --- Layer: did the vzdump jobs actually SUCCEED? -----------------------
#
# Snapshot age alone is structurally blind to a job that runs and ERRORS every
# night: nothing new is written, so the group simply ages, and the fault only
# surfaces once it crosses the 48h threshold — days late. esh-vm-workstation
# (VM 102) failed nightly from ~2026-09-06 with "timeout waiting on systemd"
# and this check would not have named it until 09-12.
#
# PVE records every task's result in /var/log/pve/tasks/index as
#   <UPID> <endtime-hex> <status>
# where the UPID's 6th colon-field is the task type and the 5th is its hex
# start time. Anything vzdump in the window whose status is not OK is a fault
# TODAY, not in two days.
for entry in "${PVE_NODES[@]}"; do
  set -- $entry; node_label="$1"; node_addr="$2"
  idx=$($SSH "$node_addr" "sudo -n cat /var/log/pve/tasks/index" 2>/dev/null)
  if [ -z "$idx" ]; then
    errors+=("vzdump task log on $node_label UNREADABLE — job failures there are invisible")
    continue
  fi
  bad=$(awk -v now="$now" -v win="$((JOB_WINDOW_H * 3600))" '
    {
      upid = $1; endh = $2; status = $0
      sub(/^[^ ]+ [^ ]+ /, "", status)
      n = split(upid, f, ":")
      if (n < 7 || f[6] != "vzdump") next
      start = strtonum("0x" f[5])
      if (now - start > win) next
      if (status == "OK") next
      printf "%s%s: %s\n", (f[7] == "" ? "job" : "guest " f[7]), "", status
    }' <<<"$idx" | sort -u)
  [ -n "$bad" ] && while IFS= read -r b; do
    [ -n "$b" ] && jobfail+=("$node_label $b")
  done <<<"$bad"
done

# --- rest-server endpoint health (401 = up & serving) ---
for ep in "rest-server-ana http://10.250.50.70:8000/" "rest-server-nh3 http://10.100.50.50:8000/"; do
  set -- $ep
  code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 6 "$2" 2>/dev/null)
  [ "$code" = "401" ] && fresh+=("$1: up (401)") || stale+=("$1: endpoint code=$code (expected 401)")
done

echo
echo "FRESH (${#fresh[@]}):"; printf '  ✅ %s\n' "${fresh[@]}"
if [ "${#excluded[@]}" -gt 0 ]; then
  # Printed every run on purpose: an exclusion that was a mistake is only
  # catchable if you can see it. Silence here would trade a noisy alarm for a
  # blind one.
  echo; echo "NOT BACKED UP BY POLICY (${#excluded[@]}) — not a fault, but check the list is still right:"
  printf '  ⏸  %s\n' "${excluded[@]}"
fi
if [ "${#jobfail[@]}" -gt 0 ]; then
  echo; echo "BACKUP JOBS THAT ERRORED in the last ${JOB_WINDOW_H}h (${#jobfail[@]}):"
  printf '  ❌ %s\n' "${jobfail[@]}"
fi
if [ "${#stale[@]}" -gt 0 ] || [ "${#errors[@]}" -gt 0 ]; then
  echo; echo "STALE / PROBLEMS (${#stale[@]}+${#errors[@]}):"
  [ "${#stale[@]}"  -gt 0 ] && printf '  🔴 %s\n' "${stale[@]}"
  [ "${#errors[@]}" -gt 0 ] && printf '  🔴 %s\n' "${errors[@]}"
fi
# --------- Verdict. Two different findings, two different words. -------
#
# ⚠ These used to collapse into one "RESULT: STALE / exit 1". That is a false
# statement of fact whenever the only finding is a recent job error: on
# 2026-09-20 this printed STALE while reporting 37 FRESH layers and zero stale
# ones — every backup body provably current, the ❌ rows all yesterday's
# pre-fix runs aging out of the window. infra-hermes flagged it: a reader or a
# forwarder could page someone over a state where nothing is actually stale.
#
# STALE is a claim about backup AGE. A job that ran and errored is a different
# claim with different urgency, so it gets its own verdict and its own code.
if [ "${#stale[@]}" -gt 0 ] || [ "${#errors[@]}" -gt 0 ]; then
  # Bodies stale or an endpoint down. The serious one. Job errors may ride
  # along and are already printed above.
  echo; echo "RESULT: STALE — see docs/runbooks/backups.md"
  exit 1
fi
if [ "${#jobfail[@]}" -gt 0 ]; then
  echo; echo "RESULT: ERRORED-JOBS — every backup body is FRESH; ${#jobfail[@]} job(s)"
  echo "        errored in the last ${JOB_WINDOW_H}h. Worth a look, not a page."
  echo "        Errors older than the window age out on their own."
  exit 3
fi
echo; echo "RESULT: all backups fresh"
exit 0