esh-pfi-infrastructure/scripts/check-backup-freshness.sh

#!/usr/bin/env bash
# check-backup-freshness.sh — the "are we actually backed up?" check.
#
# Walks every backup layer and flags anything whose newest snapshot is older
# than the threshold (default 48h) or any down endpoint. Prints a report.
#
# EXIT CODES — the verdict, not just a pass/fail:
#   0  all backups fresh
#   1  STALE — a backup body is past the threshold, or an endpoint is down
#   3  ERRORED-JOBS — every body is fresh, but a vzdump job errored recently
#
# 1 and 3 are deliberately different. Collapsing them prints "STALE" over a
# fleet whose every backup is current, which is how an alarm earns being
# ignored. Designed to be
# run by a daily timer that alerts on non-zero exit (see
# scripts/install-backup-freshness-timer.sh), or by hand anytime.
#
# Companion to docs/runbooks/backups.md. Read-only — only SSH stat/curl.
#
#   BACKUP_MAX_AGE_HOURS=48 scripts/check-backup-freshness.sh
#
# POLICY AWARENESS (2026-09-19). Some guests are deliberately not backed up --
# ana-scale (CT 114) and esh-scale (CT 108) are subnet-router LXCs excluded
# after a vzdump lock on esh-scale blackholed that entire site. Before this,
# the check reported those as 🔴 STALE every single morning, which is how an
# alarm teaches you to ignore it. Such guests now get their own section:
# printed, never hidden, but not counted as a fault.
#
# ⚠ COVERAGE IS A UNION ACROSS JOBS, NOT ONE JOB'S EXCLUDE LIST. Reading
# `exclude` alone gets ana CT 109 (ana-nas) exactly wrong: it IS excluded from
# the 03:00 all-guests job AND it has its own dedicated 22:00 job. Suppressing
# on the exclude list would have silently stopped alarming on a guest that is
# genuinely backed up -- turning a fix for a noisy alarm into a blind spot.
# The same applies in reverse for ESH, whose job uses an explicit
# `vmid 100..107` INCLUDE list, so esh-scale 108 is excluded by OMISSION and
# appears in no exclude list at all.
# A guest is "intentionally not backed up" only when NO enabled vzdump job
# covers it.
#
# ⚠ IF COVERAGE CANNOT BE DETERMINED, NOTHING IS SUPPRESSED. An unreachable
# PVE node means we do not know, and a backup alarm must fail loud.
# The SureFire namespace (sfsrv-pve) is deliberately never consulted -- those
# are tenant hosts -- so its guests can never be suppressed either.
set -uo pipefail

MAX_AGE_H="${BACKUP_MAX_AGE_HOURS:-48}"
SSH="ssh -o ConnectTimeout=8 -o BatchMode=yes"
now=$(date +%s)
stale=() ; fresh=() ; errors=() ; excluded=()

# PBS namespace → a node of the PVE cluster that owns it. jobs.cfg lives in
# pmxcfs and is cluster-wide, so ESH's two nodes share one job set and either
# answers for both. sfsrv-pve is absent ON PURPOSE (tenant property).
declare -A NS_NODE=(
  [ana-pve]=infra-ops@10.250.250.31
  [esh-pve]=infra-ops@10.0.250.35
  [nh3-pve]=infra-ops@10.100.250.60
)
declare -A NS_JOBS=()      # ns → one "enabled|all|vmids|excludes" line per job
declare -A NS_KNOWN=()     # ns → 1 once coverage was successfully read

# Every PVE node whose vzdump TASK RESULTS we read. Unlike jobs.cfg this is
# per-NODE, not cluster-wide, so both ESH nodes appear. sfsrv is absent: tenant.
PVE_NODES=(
  "ana-pve       infra-ops@10.250.250.31"
  "esh-pve       infra-ops@10.0.250.35"
  "esh-nas-pve   infra-ops@10.0.50.55"
  "nh3-pve       infra-ops@10.100.250.60"
)
JOB_WINDOW_H="${BACKUP_JOB_WINDOW_HOURS:-36}"
jobfail=()

load_jobs() { # $1 = namespace
  local ns="$1" node="${NS_NODE[$1]:-}" raw
  [ -n "$node" ] || return 1
  # Fetch the raw file and parse it HERE. The parser used to be an awk script
  # embedded in the ssh command string; the quoting mangled it silently and
  # every namespace came back unreadable — which at least failed loud, because
  # that is how this is built. Keep the remote side a plain `cat`.
  raw=$($SSH "$node" "sudo -n cat /etc/pve/jobs.cfg" 2>/dev/null) || return 1
  [ -n "$raw" ] || return 1
  local parsed
  parsed=$(awk '
    function flush() { if (seen) { print e "|" a "|" v "|" x; seen=0 } }
    /^vzdump:/       { flush(); seen=1; e=1; a=0; v=""; x=""; next }
    /^[^ \t]/       { flush(); next }
    seen && $1=="enabled" { e=$2 }
    seen && $1=="all"     { a=$2 }
    seen && $1=="vmid"    { v=$2 }
    seen && $1=="exclude" { x=$2 }
    END { flush() }
  ' <<<"$raw")
  [ -n "$parsed" ] || return 1
  NS_JOBS[$ns]="$parsed"
  NS_KNOWN[$ns]=1
}

in_csv() { # $1=needle  $2=comma list
  case ",$2," in *",$1,"*) return 0 ;; *) return 1 ;; esac
}

is_covered() { # $1=namespace  $2=vmid — 0 covered, 1 not covered, 2 unknown
  local ns="$1" id="$2" line enabled all vmids excludes
  [ -n "${NS_KNOWN[$ns]:-}" ] || return 2
  while IFS='|' read -r enabled all vmids excludes; do
    [ "$enabled" = "1" ] || continue
    if [ -n "$vmids" ]; then
      in_csv "$id" "$vmids" && return 0      # explicit include list
    elif [ "$all" = "1" ]; then
      in_csv "$id" "$excludes" || return 0   # all-guests minus its excludes
    fi
  done <<<"${NS_JOBS[$ns]}"
  return 1
}

# newest snapshot epoch under a remote glob (echoes epoch or empty)
newest_epoch() { # $1=host  $2=glob
  $SSH "$1" "stat -c %Y $2 2>/dev/null | sort -n | tail -1" 2>/dev/null
}
report() { # $1=label  $2=epoch("" = none)
  local label="$1" ep="$2"
  if [ -z "$ep" ]; then stale+=("$label: NO SNAPSHOTS / unreachable"); return; fi
  local age=$(( (now - ep) / 3600 ))
  local when; when=$(date -d "@$ep" '+%Y-%m-%d %H:%M' 2>/dev/null)
  if [ "$age" -gt "$MAX_AGE_H" ]; then stale+=("$label: ${age}h old (newest $when)")
  else fresh+=("$label: ${age}h old (newest $when)"); fi
}

echo "=== Backup freshness (threshold ${MAX_AGE_H}h) — $(date '+%Y-%m-%d %H:%M %Z') ==="

# --- Layer: restic file+DB — DISCOVERED, not enumerated ---------------------
#
# ⚠ THIS LIST USED TO BE HAND-WRITTEN, AND ON 2026-09-22 IT WAS WRONG. It read
# `irv-ml1 nh3-docker` and omitted nh3-dev -- the repo holding every Claude Code
# transcript on that box, althing routes, hermes history and Miranda's
# conversation. The repo had always existed and always been written; it was
# simply never enumerated, so the one repository nobody could reconstruct was
# the one nothing watched.
#
# Adding nh3-dev fixed the instance. Deriving fixes the CLASS (svos-dev's
# framing): a hand-maintained list of things to watch, sitting beside a NAS that
# already knows which repos exist, means the next repo added is unwatched BY
# DEFAULT and nothing says so. The list of what to check can silently disagree
# with the set of what exists -- the same shape as every other instrument fault
# found that day, only slower-acting.
#
# So: ask the NAS what is there. A directory counts as a repository when it has
# a `snapshots/` child, which distinguishes a real repo from a container dir
# (/volume1/Backup/restic/repo/ holds ana|esh|nh3 namespaces and no snapshots
# of its own).
#
# The hand-written list survives DEMOTED, as an EXPECTED set -- used only to
# report a repo that has VANISHED. That keeps two different facts distinct that
# would otherwise both read as silence:
#     "a repo exists that nobody watches"   -> impossible now, it is discovered
#     "a repo we expected is gone"          -> reported below
discover_repos() { # $1=host  $2=parent dir — prints "name<TAB>newest_epoch"
  $SSH "$1" "for d in $2/*/; do [ -d \"\$d/snapshots\" ] || continue
    printf '%s\t%s\n' \"\$(basename \"\$d\")\" \"\$(stat -c %Y \"\$d\"snapshots/* 2>/dev/null | sort -n | tail -1)\"
  done" 2>/dev/null
}

EXPECTED_ANA="${EXPECTED_RESTIC_ANA:-ana-docker ana-ml2 esh-docker-vm esh-ml1 esh-vm-db vm-esh-nas}"
EXPECTED_NH3="${EXPECTED_RESTIC_NH3:-irv-ml1 nh3-docker nh3-dev}"

found_ana=""; found_nh3=""
while IFS=$'\t' read -r name ep; do
  [ -n "$name" ] || continue
  found_ana="$found_ana $name"
  report "restic/ana/$name" "$ep"
done < <(discover_repos ana-nas /mnt/backup/restic/repo/ana)

while IFS=$'\t' read -r name ep; do
  [ -n "$name" ] || continue
  found_nh3="$found_nh3 $name"
  report "restic/nh3/$name" "$ep"
done < <(discover_repos nh3-nas /volume1/Backup/restic)

# A repo we expected and did NOT discover is a different fault from a stale one:
# the repository is gone, not behind. Say so in those words.
for want in $EXPECTED_ANA; do
  in_csv "$want" "$(echo $found_ana | tr ' ' ',')" || \
    stale+=("restic/ana/$want: EXPECTED REPO NOT FOUND on ana-nas (deleted, renamed, or never created)")
done
for want in $EXPECTED_NH3; do
  in_csv "$want" "$(echo $found_nh3 | tr ' ' ',')" || \
    stale+=("restic/nh3/$want: EXPECTED REPO NOT FOUND on nh3-nas (deleted, renamed, or never created)")
done

# --- Layer: PBS VM images (newest per guest, all namespaces) ---
pbs=$($SSH pbs-ana 'for ns in /mnt/pbs-datastore/ns/*/; do n=$(basename "$ns")
  for d in vm ct; do for g in "$ns$d"/*/; do [ -d "$g" ] || continue
    nb=$(ls -d "$g"20*T* 2>/dev/null | sort | tail -1)
    [ -n "$nb" ] && echo "$n/$d/$(basename "$g") $(stat -c %Y "$nb")"
  done; done; done' 2>/dev/null)
if [ -z "$pbs" ]; then errors+=("PBS-ANA: unreachable or no snapshots"); else
  for ns in "${!NS_NODE[@]}"; do load_jobs "$ns" || errors+=("coverage for namespace '$ns' UNREADABLE — nothing in it will be suppressed"); done
  while read -r guest ep; do
    [ -n "$guest" ] || continue
    ns=${guest%%/*}; vmid=${guest##*/}
    is_covered "$ns" "$vmid"
    case $? in
      1) excluded+=("pbs/$guest: no enabled vzdump job covers it$(
           [ -n "$ep" ] && printf ' (last snapshot %s)' "$(date -d "@$ep" '+%Y-%m-%d' 2>/dev/null)")") ;;
      *) report "pbs/$guest" "$ep" ;;
    esac
  done <<<"$pbs"
fi

# --- Layer: did the vzdump jobs actually SUCCEED? -----------------------
#
# Snapshot age alone is structurally blind to a job that runs and ERRORS every
# night: nothing new is written, so the group simply ages, and the fault only
# surfaces once it crosses the 48h threshold — days late. esh-vm-workstation
# (VM 102) failed nightly from ~2026-09-06 with "timeout waiting on systemd"
# and this check would not have named it until 09-12.
#
# PVE records every task's result in /var/log/pve/tasks/index as
#   <UPID> <endtime-hex> <status>
# where the UPID's 6th colon-field is the task type and the 5th is its hex
# start time. Anything vzdump in the window whose status is not OK is a fault
# TODAY, not in two days.
for entry in "${PVE_NODES[@]}"; do
  set -- $entry; node_label="$1"; node_addr="$2"
  idx=$($SSH "$node_addr" "sudo -n cat /var/log/pve/tasks/index" 2>/dev/null)
  if [ -z "$idx" ]; then
    errors+=("vzdump task log on $node_label UNREADABLE — job failures there are invisible")
    continue
  fi
  bad=$(awk -v now="$now" -v win="$((JOB_WINDOW_H * 3600))" '
    {
      upid = $1; endh = $2; status = $0
      sub(/^[^ ]+ [^ ]+ /, "", status)
      n = split(upid, f, ":")
      if (n < 7 || f[6] != "vzdump") next
      start = strtonum("0x" f[5])
      if (now - start > win) next
      if (status == "OK") next
      printf "%s%s: %s\n", (f[7] == "" ? "job" : "guest " f[7]), "", status
    }' <<<"$idx" | sort -u)
  [ -n "$bad" ] && while IFS= read -r b; do
    [ -n "$b" ] && jobfail+=("$node_label $b")
  done <<<"$bad"
done

# --- restic CONTENT assertion (operator ruling 2026-09-22) -------------------
#
# The checks above read a snapshot directory's MTIME on the NAS. That proves a
# file was written recently. It does not prove the repository holds the data,
# and metadata is exactly what survives the failures worth fearing: a pruned or
# partially-written repo can still list a path whose blobs are gone.
#
# ⚠ THE ASSERTION IS CONJUNCTIVE ON PURPOSE. "a snapshot exists containing X"
# is satisfied by a three-month-old snapshot; "the latest snapshot is recent" is
# satisfied by an empty one. Age alone was the old check's problem; content
# alone is the same problem rotated. Both, or neither is worth printing.
#
# The rungs, and what each actually proves (svos-dev, 2026-09-22):
#     snapshots list    the repo answers
#     ls a path in it   the path is in the INDEX  <- metadata only
#     restic check      the repo's structure is intact
#     RESTORE a file    the bytes come back       <- the only proof of recovery
# We take the top rung for the small irreplaceable set, because it is the only
# one that answers the question a green light is taken to mean. Restoring from
# the multi-gigabyte paths is not worth it: a repo that returns one file is
# overwhelmingly likely to return others, and one that cannot is broken for
# everything.
#
# ⚠ IDENTITY: this runs as a --user timer (lkraven), which has no NOPASSWD sudo
# on nh3-dev, and the repo credentials live under /etc/restic. So the probe hops
# through infra-ops@localhost, which does. Not a workaround -- infra-ops is the
# documented ops identity and the credentials are deliberately root-only.
RESTIC_PROBE_PATHS="${RESTIC_PROBE_PATHS:-/home/lkraven/.local/state/svos}"
probe=$($SSH infra-ops@10.100.10.50 "sudo -n bash -s" <<PROBE 2>/dev/null
set -a; . /etc/restic/restic.env 2>/dev/null; set +a
export RESTIC_PASSWORD_FILE=/etc/restic/password
id=\$(restic snapshots --latest 1 --json 2>/dev/null | python3 -c 'import json,sys
d=json.load(sys.stdin)
print(d[0]["short_id"], int(__import__("datetime").datetime.fromisoformat(d[0]["time"][:19]).timestamp())) if d else print("NONE 0")' 2>/dev/null)
sid=\$(echo "\$id" | cut -d" " -f1); ep=\$(echo "\$id" | cut -d" " -f2)
[ "\$sid" = "NONE" ] && { echo "NOSNAP 0 0 0"; exit 0; }
# ⚠ grep -c '^/' NOT grep -c '.' — \`restic ls\` always prints a HEADER line
# ("snapshot <id> of [...] filtered by [...]") whether or not anything matched,
# so a path absent from the repo returns 1 line and a naive count reads it as
# found. Measured 2026-09-22: real path 6 lines, bogus path 1. That made the
# ABSENT case fall through and report "blobs gone" — telling an operator the
# repository was corrupt when the truth was a mistyped path. Only lines that
# begin with / are entries.
n=\$(restic ls "\$sid" $RESTIC_PROBE_PATHS 2>/dev/null | grep -c '^/')
tmp=\$(mktemp -d); restic restore "\$sid" --target "\$tmp" --include $RESTIC_PROBE_PATHS >/dev/null 2>&1
bytes=\$(find "\$tmp" -type f -printf '%s\n' 2>/dev/null | awk '{s+=\$1} END {print s+0}')
rm -rf -- "\$tmp"
echo "\$sid \$ep \$n \$bytes"
PROBE
)
set -- ${probe:-ERR 0 0 0}
r_sid="$1"; r_ep="$2"; r_entries="$3"; r_bytes="$4"
if [ "$r_sid" = "ERR" ] || [ -z "$probe" ]; then
  stale+=("restic/nh3-dev CONTENT: probe could not run (repo unreachable or creds unreadable)")
elif [ "$r_sid" = "NOSNAP" ]; then
  stale+=("restic/nh3-dev CONTENT: repository holds NO SNAPSHOTS")
else
  r_age=$(( (now - r_ep) / 3600 ))
  # conjunctive: recent AND indexed AND restorable
  if [ "$r_age" -gt "$MAX_AGE_H" ]; then
    stale+=("restic/nh3-dev CONTENT: latest snapshot $r_sid is ${r_age}h old (>${MAX_AGE_H}h)")
  elif [ "${r_entries:-0}" -lt 1 ]; then
    stale+=("restic/nh3-dev CONTENT: $RESTIC_PROBE_PATHS absent from snapshot $r_sid")
  elif [ "${r_bytes:-0}" -lt 1 ]; then
    stale+=("restic/nh3-dev CONTENT: $RESTIC_PROBE_PATHS listed in $r_sid but RESTORED ZERO BYTES — index intact, blobs gone")
  else
    fresh+=("restic/nh3-dev CONTENT: $r_sid ${r_age}h old, $r_entries entries, restored ${r_bytes}B")
  fi
fi

# --- rest-server endpoint health (401 = up & serving) ---
for ep in "rest-server-ana http://10.250.50.70:8000/" "rest-server-nh3 http://10.100.50.50:8000/"; do
  set -- $ep
  code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 6 "$2" 2>/dev/null)
  [ "$code" = "401" ] && fresh+=("$1: up (401)") || stale+=("$1: endpoint code=$code (expected 401)")
done

echo
echo "FRESH (${#fresh[@]}):"; printf '  ✅ %s\n' "${fresh[@]}"
if [ "${#excluded[@]}" -gt 0 ]; then
  # Printed every run on purpose: an exclusion that was a mistake is only
  # catchable if you can see it. Silence here would trade a noisy alarm for a
  # blind one.
  echo; echo "NOT BACKED UP BY POLICY (${#excluded[@]}) — not a fault, but check the list is still right:"
  printf '  ⏸  %s\n' "${excluded[@]}"
fi
if [ "${#jobfail[@]}" -gt 0 ]; then
  echo; echo "BACKUP JOBS THAT ERRORED in the last ${JOB_WINDOW_H}h (${#jobfail[@]}):"
  printf '  ❌ %s\n' "${jobfail[@]}"
fi
if [ "${#stale[@]}" -gt 0 ] || [ "${#errors[@]}" -gt 0 ]; then
  echo; echo "STALE / PROBLEMS (${#stale[@]}+${#errors[@]}):"
  [ "${#stale[@]}"  -gt 0 ] && printf '  🔴 %s\n' "${stale[@]}"
  [ "${#errors[@]}" -gt 0 ] && printf '  🔴 %s\n' "${errors[@]}"
fi
# --------- Verdict. Two different findings, two different words. -------
#
# ⚠ These used to collapse into one "RESULT: STALE / exit 1". That is a false
# statement of fact whenever the only finding is a recent job error: on
# 2026-09-20 this printed STALE while reporting 37 FRESH layers and zero stale
# ones — every backup body provably current, the ❌ rows all yesterday's
# pre-fix runs aging out of the window. infra-hermes flagged it: a reader or a
# forwarder could page someone over a state where nothing is actually stale.
#
# STALE is a claim about backup AGE. A job that ran and errored is a different
# claim with different urgency, so it gets its own verdict and its own code.
if [ "${#stale[@]}" -gt 0 ] || [ "${#errors[@]}" -gt 0 ]; then
  # Bodies stale or an endpoint down. The serious one. Job errors may ride
  # along and are already printed above.
  echo; echo "RESULT: STALE — see docs/runbooks/backups.md"
  exit 1
fi
if [ "${#jobfail[@]}" -gt 0 ]; then
  echo; echo "RESULT: ERRORED-JOBS — every backup body is FRESH; ${#jobfail[@]} job(s)"
  echo "        errored in the last ${JOB_WINDOW_H}h. Worth a look, not a page."
  echo "        Errors older than the window age out on their own."
  exit 3
fi
echo; echo "RESULT: all backups fresh"
exit 0