vh
5be25be081
ana-scale (CT 114) is a subnet-router LXC, excluded from vzdump on 2026-09-07
after a backup lock on its ESH counterpart blackholed that entire site. The
freshness check knew nothing about that and reported it 🔴 STALE every single
morning, which is how an alarm teaches you to ignore it.
Such guests now get their own section: printed every run, never hidden, and
not counted as a fault.
The subtlety is in how coverage is computed, and the obvious implementation is
wrong twice over:
* Reading one job's `exclude` list gets ana CT 109 (ana-nas) exactly
backwards. It IS excluded from the 03:00 all-guests job AND it has its own
dedicated 22:00 job. Suppressing on the exclude list would have stopped
alarming on a guest that is genuinely backed up -- trading a noisy alarm
for a blind one.
* ESH's job uses an explicit `vmid 100..107` INCLUDE list, so esh-scale 108
is excluded by OMISSION and appears in no exclude list at all.
So coverage is a union across every enabled job on the cluster, and a guest is
"intentionally not backed up" only when none of them covers it.
If coverage cannot be read, nothing is suppressed and the gap is reported: an
unreachable PVE node means we do not know, and a backup alarm must fail loud.
The SureFire namespace is never consulted (tenant property), so its guests can
never be suppressed either.
Verified against the live fleet on all four paths: CT 114 suppressed; CT 109
NOT suppressed despite being in an exclude list; esh-vm-workstation 102, which
a job really does cover and which really is failing, still reports STALE; and
with a PVE node made unreachable, 114 returns to STALE with the gap named.
164 lines
7.2 KiB
Bash
Executable File
164 lines
7.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# check-backup-freshness.sh — the "are we actually backed up?" check.
|
|
#
|
|
# Walks every backup layer and flags anything whose newest snapshot is older
|
|
# than the threshold (default 48h) or any down endpoint. Prints a report;
|
|
# exits 0 if everything is fresh, 1 if anything is stale/down. Designed to be
|
|
# run by a daily timer that alerts on non-zero exit (see
|
|
# scripts/install-backup-freshness-timer.sh), or by hand anytime.
|
|
#
|
|
# Companion to docs/runbooks/backups.md. Read-only — only SSH stat/curl.
|
|
#
|
|
# BACKUP_MAX_AGE_HOURS=48 scripts/check-backup-freshness.sh
|
|
#
|
|
# POLICY AWARENESS (2026-09-19). Some guests are deliberately not backed up --
|
|
# ana-scale (CT 114) and esh-scale (CT 108) are subnet-router LXCs excluded
|
|
# after a vzdump lock on esh-scale blackholed that entire site. Before this,
|
|
# the check reported those as 🔴 STALE every single morning, which is how an
|
|
# alarm teaches you to ignore it. Such guests now get their own section:
|
|
# printed, never hidden, but not counted as a fault.
|
|
#
|
|
# ⚠ COVERAGE IS A UNION ACROSS JOBS, NOT ONE JOB'S EXCLUDE LIST. Reading
|
|
# `exclude` alone gets ana CT 109 (ana-nas) exactly wrong: it IS excluded from
|
|
# the 03:00 all-guests job AND it has its own dedicated 22:00 job. Suppressing
|
|
# on the exclude list would have silently stopped alarming on a guest that is
|
|
# genuinely backed up -- turning a fix for a noisy alarm into a blind spot.
|
|
# The same applies in reverse for ESH, whose job uses an explicit
|
|
# `vmid 100..107` INCLUDE list, so esh-scale 108 is excluded by OMISSION and
|
|
# appears in no exclude list at all.
|
|
# A guest is "intentionally not backed up" only when NO enabled vzdump job
|
|
# covers it.
|
|
#
|
|
# ⚠ IF COVERAGE CANNOT BE DETERMINED, NOTHING IS SUPPRESSED. An unreachable
|
|
# PVE node means we do not know, and a backup alarm must fail loud.
|
|
# The SureFire namespace (sfsrv-pve) is deliberately never consulted -- those
|
|
# are tenant hosts -- so its guests can never be suppressed either.
|
|
set -uo pipefail
|
|
|
|
MAX_AGE_H="${BACKUP_MAX_AGE_HOURS:-48}"
|
|
SSH="ssh -o ConnectTimeout=8 -o BatchMode=yes"
|
|
now=$(date +%s)
|
|
stale=() ; fresh=() ; errors=() ; excluded=()
|
|
|
|
# PBS namespace → a node of the PVE cluster that owns it. jobs.cfg lives in
|
|
# pmxcfs and is cluster-wide, so ESH's two nodes share one job set and either
|
|
# answers for both. sfsrv-pve is absent ON PURPOSE (tenant property).
|
|
declare -A NS_NODE=(
|
|
[ana-pve]=infra-ops@10.250.250.31
|
|
[esh-pve]=infra-ops@10.0.250.35
|
|
[nh3-pve]=infra-ops@10.100.250.60
|
|
)
|
|
declare -A NS_JOBS=() # ns → one "enabled|all|vmids|excludes" line per job
|
|
declare -A NS_KNOWN=() # ns → 1 once coverage was successfully read
|
|
|
|
load_jobs() { # $1 = namespace
|
|
local ns="$1" node="${NS_NODE[$1]:-}" raw
|
|
[ -n "$node" ] || return 1
|
|
# Fetch the raw file and parse it HERE. The parser used to be an awk script
|
|
# embedded in the ssh command string; the quoting mangled it silently and
|
|
# every namespace came back unreadable — which at least failed loud, because
|
|
# that is how this is built. Keep the remote side a plain `cat`.
|
|
raw=$($SSH "$node" "sudo -n cat /etc/pve/jobs.cfg" 2>/dev/null) || return 1
|
|
[ -n "$raw" ] || return 1
|
|
local parsed
|
|
parsed=$(awk '
|
|
function flush() { if (seen) { print e "|" a "|" v "|" x; seen=0 } }
|
|
/^vzdump:/ { flush(); seen=1; e=1; a=0; v=""; x=""; next }
|
|
/^[^ \t]/ { flush(); next }
|
|
seen && $1=="enabled" { e=$2 }
|
|
seen && $1=="all" { a=$2 }
|
|
seen && $1=="vmid" { v=$2 }
|
|
seen && $1=="exclude" { x=$2 }
|
|
END { flush() }
|
|
' <<<"$raw")
|
|
[ -n "$parsed" ] || return 1
|
|
NS_JOBS[$ns]="$parsed"
|
|
NS_KNOWN[$ns]=1
|
|
}
|
|
|
|
in_csv() { # $1=needle $2=comma list
|
|
case ",$2," in *",$1,"*) return 0 ;; *) return 1 ;; esac
|
|
}
|
|
|
|
is_covered() { # $1=namespace $2=vmid — 0 covered, 1 not covered, 2 unknown
|
|
local ns="$1" id="$2" line enabled all vmids excludes
|
|
[ -n "${NS_KNOWN[$ns]:-}" ] || return 2
|
|
while IFS='|' read -r enabled all vmids excludes; do
|
|
[ "$enabled" = "1" ] || continue
|
|
if [ -n "$vmids" ]; then
|
|
in_csv "$id" "$vmids" && return 0 # explicit include list
|
|
elif [ "$all" = "1" ]; then
|
|
in_csv "$id" "$excludes" || return 0 # all-guests minus its excludes
|
|
fi
|
|
done <<<"${NS_JOBS[$ns]}"
|
|
return 1
|
|
}
|
|
|
|
# newest snapshot epoch under a remote glob (echoes epoch or empty)
|
|
newest_epoch() { # $1=host $2=glob
|
|
$SSH "$1" "stat -c %Y $2 2>/dev/null | sort -n | tail -1" 2>/dev/null
|
|
}
|
|
report() { # $1=label $2=epoch("" = none)
|
|
local label="$1" ep="$2"
|
|
if [ -z "$ep" ]; then stale+=("$label: NO SNAPSHOTS / unreachable"); return; fi
|
|
local age=$(( (now - ep) / 3600 ))
|
|
local when; when=$(date -d "@$ep" '+%Y-%m-%d %H:%M' 2>/dev/null)
|
|
if [ "$age" -gt "$MAX_AGE_H" ]; then stale+=("$label: ${age}h old (newest $when)")
|
|
else fresh+=("$label: ${age}h old (newest $when)"); fi
|
|
}
|
|
|
|
echo "=== Backup freshness (threshold ${MAX_AGE_H}h) — $(date '+%Y-%m-%d %H:%M %Z') ==="
|
|
|
|
# --- Layer: restic file+DB, ANA side (rest-server-ana) ---
|
|
for c in ana-docker ana-ml2 esh-docker-vm esh-vm-db vm-esh-nas; do
|
|
report "restic/ana/$c" "$(newest_epoch ana-nas "/mnt/backup/restic/repo/ana/$c/snapshots/*")"
|
|
done
|
|
# --- Layer: restic file+DB, NH3 side (rest-server-nh3) ---
|
|
for c in irv-ml1 nh3-docker; do
|
|
report "restic/nh3/$c" "$(newest_epoch nh3-nas "/volume1/Backup/restic/$c/snapshots/*")"
|
|
done
|
|
# --- Layer: PBS VM images (newest per guest, all namespaces) ---
|
|
pbs=$($SSH pbs-ana 'for ns in /mnt/pbs-datastore/ns/*/; do n=$(basename "$ns")
|
|
for d in vm ct; do for g in "$ns$d"/*/; do [ -d "$g" ] || continue
|
|
nb=$(ls -d "$g"20*T* 2>/dev/null | sort | tail -1)
|
|
[ -n "$nb" ] && echo "$n/$d/$(basename "$g") $(stat -c %Y "$nb")"
|
|
done; done; done' 2>/dev/null)
|
|
if [ -z "$pbs" ]; then errors+=("PBS-ANA: unreachable or no snapshots"); else
|
|
for ns in "${!NS_NODE[@]}"; do load_jobs "$ns" || errors+=("coverage for namespace '$ns' UNREADABLE — nothing in it will be suppressed"); done
|
|
while read -r guest ep; do
|
|
[ -n "$guest" ] || continue
|
|
ns=${guest%%/*}; vmid=${guest##*/}
|
|
is_covered "$ns" "$vmid"
|
|
case $? in
|
|
1) excluded+=("pbs/$guest: no enabled vzdump job covers it$(
|
|
[ -n "$ep" ] && printf ' (last snapshot %s)' "$(date -d "@$ep" '+%Y-%m-%d' 2>/dev/null)")") ;;
|
|
*) report "pbs/$guest" "$ep" ;;
|
|
esac
|
|
done <<<"$pbs"
|
|
fi
|
|
|
|
# --- rest-server endpoint health (401 = up & serving) ---
|
|
for ep in "rest-server-ana http://10.250.50.70:8000/" "rest-server-nh3 http://10.100.50.50:8000/"; do
|
|
set -- $ep
|
|
code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 6 "$2" 2>/dev/null)
|
|
[ "$code" = "401" ] && fresh+=("$1: up (401)") || stale+=("$1: endpoint code=$code (expected 401)")
|
|
done
|
|
|
|
echo
|
|
echo "FRESH (${#fresh[@]}):"; printf ' ✅ %s\n' "${fresh[@]}"
|
|
if [ "${#excluded[@]}" -gt 0 ]; then
|
|
# Printed every run on purpose: an exclusion that was a mistake is only
|
|
# catchable if you can see it. Silence here would trade a noisy alarm for a
|
|
# blind one.
|
|
echo; echo "NOT BACKED UP BY POLICY (${#excluded[@]}) — not a fault, but check the list is still right:"
|
|
printf ' ⏸ %s\n' "${excluded[@]}"
|
|
fi
|
|
if [ "${#stale[@]}" -gt 0 ] || [ "${#errors[@]}" -gt 0 ]; then
|
|
echo; echo "STALE / PROBLEMS (${#stale[@]}+${#errors[@]}):"
|
|
printf ' 🔴 %s\n' "${stale[@]}" "${errors[@]}"
|
|
echo; echo "RESULT: STALE — see docs/runbooks/backups.md"
|
|
exit 1
|
|
fi
|
|
echo; echo "RESULT: all backups fresh"
|
|
exit 0
|