vh
d920cf3816
PBS rewrites index.json.blob inside a snapshot directory at every verify (the next 02:30, then every 30 days per verification.cfg outdated-after 30) and on notes/protect edits, which moves the directory mtime forward. Measured: ana-pve/vm/101's 2026-10-04T10:00:04Z snapshot read as 2026-10-06. A covered guest whose backups stopped would have alarmed up to a day late, then gone quiet for ~2 days after each monthly re-verify. The age now comes from the snapshot name (its backup start time). Verified: the default run exits 0 with 28 fresh PBS groups and vm/101 dated 2026-10-04. Positive control BACKUP_MAX_AGE_HOURS=5 flags 10 groups by name age, exit 1.
498 lines
26 KiB
Bash
Executable File
498 lines
26 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# check-backup-freshness.sh — the "are we actually backed up?" check.
|
|
#
|
|
# Walks every backup layer and flags anything whose newest snapshot is older
|
|
# than the threshold (default 48h) or any down endpoint. Prints a report.
|
|
#
|
|
# EXIT CODES — the verdict, not just a pass/fail:
|
|
# 0 all backups fresh
|
|
# 1 STALE — a backup body is past the threshold, or an endpoint is down
|
|
# 3 ERRORED-JOBS — every body is fresh, but a vzdump job errored recently
|
|
# 4 DEGRADED — no job errored, but a backup in the window logged an ERROR inside a
|
|
# task PVE still calls OK; above all an fs-freeze refusal, which means the image
|
|
# is crash-consistent, NOT application-consistent (see the layer below)
|
|
#
|
|
# 1 and 3 are deliberately different. Collapsing them prints "STALE" over a
|
|
# fleet whose every backup is current, which is how an alarm earns being
|
|
# ignored. Designed to be
|
|
# run by a daily timer that alerts on non-zero exit (see
|
|
# scripts/install-backup-freshness-timer.sh), or by hand anytime.
|
|
#
|
|
# Companion to docs/runbooks/backups.md. Read-only — only SSH stat/curl.
|
|
#
|
|
# BACKUP_MAX_AGE_HOURS=48 scripts/check-backup-freshness.sh
|
|
#
|
|
# POLICY AWARENESS (2026-09-19). Some guests are deliberately not backed up --
|
|
# ana-scale (CT 114) and esh-scale (CT 108) are subnet-router LXCs excluded
|
|
# after a vzdump lock on esh-scale blackholed that entire site. Before this,
|
|
# the check reported those as 🔴 STALE every single morning, which is how an
|
|
# alarm teaches you to ignore it. Such guests now get their own section:
|
|
# printed, never hidden, but not counted as a fault.
|
|
#
|
|
# ⚠ COVERAGE IS A UNION ACROSS JOBS, NOT ONE JOB'S EXCLUDE LIST. Reading
|
|
# `exclude` alone gets ana CT 109 (ana-nas) exactly wrong: it IS excluded from
|
|
# the 03:00 all-guests job AND it has its own dedicated 22:00 job. Suppressing
|
|
# on the exclude list would have silently stopped alarming on a guest that is
|
|
# genuinely backed up -- turning a fix for a noisy alarm into a blind spot.
|
|
# The same applies in reverse for ESH, whose job uses an explicit
|
|
# `vmid 100..107` INCLUDE list, so esh-scale 108 is excluded by OMISSION and
|
|
# appears in no exclude list at all.
|
|
# A guest is "intentionally not backed up" only when NO enabled vzdump job
|
|
# covers it.
|
|
#
|
|
# ⚠ IF COVERAGE CANNOT BE DETERMINED, NOTHING IS SUPPRESSED. An unreachable
|
|
# PVE node means we do not know, and a backup alarm must fail loud.
|
|
# The SureFire namespace (sfsrv-pve) is deliberately never consulted -- those
|
|
# are tenant hosts -- so its guests can never be suppressed either.
|
|
set -uo pipefail
|
|
|
|
MAX_AGE_H="${BACKUP_MAX_AGE_HOURS:-48}"
|
|
SSH="ssh -o ConnectTimeout=8 -o BatchMode=yes"
|
|
now=$(date +%s)
|
|
stale=() ; fresh=() ; errors=() ; excluded=()
|
|
|
|
# PBS namespace → a node of the PVE cluster that owns it. jobs.cfg lives in
|
|
# pmxcfs and is cluster-wide, so ESH's two nodes share one job set and either
|
|
# answers for both. sfsrv-pve is absent ON PURPOSE (tenant property).
|
|
declare -A NS_NODE=(
|
|
[ana-pve]=infra-ops@10.250.250.31
|
|
[esh-pve]=infra-ops@10.0.250.35
|
|
[nh3-pve]=infra-ops@10.100.250.60
|
|
[nh3-pve-2]=infra-ops@10.100.250.62 # standalone (not clustered): own jobs.cfg, own namespace. Added 2026-10-05;
|
|
# until then its job results were never read at all
|
|
)
|
|
declare -A NS_JOBS=() # ns → one "enabled|all|vmids|excludes" line per job
|
|
declare -A NS_KNOWN=() # ns → 1 once coverage was successfully read
|
|
declare -A NS_INV=() # ns → guest id set from pvesh, ONLY when the read succeeded
|
|
|
|
# Inventory of the owning cluster ("qemu/110 lxc/109 ..."). An empty/unset set
|
|
# means UNKNOWN, never "empty cluster": an unreachable node must not make a
|
|
# live guest look destroyed (fail loud, same principle as jobs.cfg).
|
|
load_inventory() { # $1 = namespace
|
|
local ns="$1" node="${NS_NODE[$1]:-}" raw
|
|
[ -n "$node" ] || return 1
|
|
[ -n "${NS_INV[$ns]:-}" ] && return 0
|
|
# -n is load-bearing: load_inventory runs INSIDE the while-read loop over the
|
|
# PBS tree, and an ssh without -n swallows the loop's remaining stdin (the
|
|
# whole PBS layer silently disappeared on 2026-10-06 until that was pinned).
|
|
raw=$($SSH -n "$node" "sudo -n pvesh get /cluster/resources --type vm --output-format text" 2>/dev/null) || return 1
|
|
raw=$(echo "$raw" | grep -oE "(qemu|lxc)/[0-9]+" | sort -u | tr "\n" " ")
|
|
[ -n "$raw" ] || return 1
|
|
NS_INV[$ns]="$raw"
|
|
}
|
|
|
|
# WHY each policy-excluded guest is out of the backup jobs (2026-10-05). The
|
|
# exclusion is derived from jobs.cfg coverage; the REASON is standing policy
|
|
# and lives here so the report can say why instead of asking the reader to
|
|
# remember. Keep one line per guest with its paper trail; if an entry shows up
|
|
# in the policy section with no reason, that is the flag: undocumented hole.
|
|
declare -A EXCLUDE_WHY=(
|
|
[ana-pve/vm/101]="retired/shut down — Prime ruling 2026-10-04 (bus 01M44MEXG60); last snapshots kept"
|
|
[nh3-pve/vm/104]="retired/shut down (nh3-laser) — Prime ruling 2026-10-04 (bus 01M44MEXG60); last snapshots kept"
|
|
[ana-pve/ct/114]="subnet-router (ana-scale) — vzdump lock risk; ESH-scale lock once blackholed the site; rebuildable from config (docs/pfi/proxmox-inventory-2026-10-02.md)"
|
|
[nh3-pve/ct/107]="subnet-router (nh3-scale) — same scale-router class as ana/esh-scale; rebuildable from config"
|
|
[esh-pve/ct/108]="subnet-router (esh-scale) — its vzdump lock blackholed the ESH site; rebuildable from config"
|
|
[esh-pve/vm/102]="Windows Parsec/RDP sandbox, no needed state — Prime ruling 2026-09-19 (playbooks/esh-exclude-vm102-from-backup.yaml)"
|
|
)
|
|
|
|
# Every PVE node whose vzdump TASK RESULTS we read. Unlike jobs.cfg this is
|
|
# per-NODE, not cluster-wide, so both ESH nodes appear. sfsrv is absent: tenant.
|
|
PVE_NODES=(
|
|
"ana-pve infra-ops@10.250.250.31"
|
|
"esh-pve infra-ops@10.0.250.35"
|
|
"esh-nas-pve infra-ops@10.0.50.55"
|
|
"nh3-pve infra-ops@10.100.250.60"
|
|
"nh3-pve-2 infra-ops@10.100.250.62"
|
|
)
|
|
JOB_WINDOW_H="${BACKUP_JOB_WINDOW_HOURS:-36}"
|
|
jobfail=()
|
|
degraded=()
|
|
acked=()
|
|
# Acknowledged OK-but-ERROR tasks: a known, deliberate cause (a drill), printed every run in their own section and
|
|
# never counted. Key = the full UPID, value = why. Keep the paper trail; the entries age out with the window.
|
|
declare -A ACKED_TASKS=(
|
|
[UPID:nh3-pve-2:0004FFBA:00719F66:6AC4628F:vzdump:111:root@pam:]="2026-10-05 svos backup drill B: freeze refused ON PURPOSE by a temporary hook (servers/nh3-svos/README.md)"
|
|
)
|
|
|
|
load_jobs() { # $1 = namespace
|
|
local ns="$1" node="${NS_NODE[$1]:-}" raw
|
|
[ -n "$node" ] || return 1
|
|
# Fetch the raw file and parse it HERE. The parser used to be an awk script
|
|
# embedded in the ssh command string; the quoting mangled it silently and
|
|
# every namespace came back unreadable — which at least failed loud, because
|
|
# that is how this is built. Keep the remote side a plain `cat`.
|
|
raw=$($SSH "$node" "sudo -n cat /etc/pve/jobs.cfg" 2>/dev/null) || return 1
|
|
[ -n "$raw" ] || return 1
|
|
local parsed
|
|
parsed=$(awk '
|
|
function flush() { if (seen) { print e "|" a "|" v "|" x; seen=0 } }
|
|
/^vzdump:/ { flush(); seen=1; e=1; a=0; v=""; x=""; next }
|
|
/^[^ \t]/ { flush(); next }
|
|
seen && $1=="enabled" { e=$2 }
|
|
seen && $1=="all" { a=$2 }
|
|
seen && $1=="vmid" { v=$2 }
|
|
seen && $1=="exclude" { x=$2 }
|
|
END { flush() }
|
|
' <<<"$raw")
|
|
[ -n "$parsed" ] || return 1
|
|
NS_JOBS[$ns]="$parsed"
|
|
NS_KNOWN[$ns]=1
|
|
}
|
|
|
|
in_csv() { # $1=needle $2=comma list
|
|
case ",$2," in *",$1,"*) return 0 ;; *) return 1 ;; esac
|
|
}
|
|
|
|
is_covered() { # $1=namespace $2=vmid — 0 covered, 1 not covered, 2 unknown
|
|
local ns="$1" id="$2" line enabled all vmids excludes
|
|
[ -n "${NS_KNOWN[$ns]:-}" ] || return 2
|
|
while IFS='|' read -r enabled all vmids excludes; do
|
|
[ "$enabled" = "1" ] || continue
|
|
if [ -n "$vmids" ]; then
|
|
in_csv "$id" "$vmids" && return 0 # explicit include list
|
|
elif [ "$all" = "1" ]; then
|
|
in_csv "$id" "$excludes" || return 0 # all-guests minus its excludes
|
|
fi
|
|
done <<<"${NS_JOBS[$ns]}"
|
|
return 1
|
|
}
|
|
|
|
# newest snapshot epoch under a remote glob (echoes epoch or empty)
|
|
newest_epoch() { # $1=host $2=glob
|
|
$SSH "$1" "stat -c %Y $2 2>/dev/null | sort -n | tail -1" 2>/dev/null
|
|
}
|
|
report() { # $1=label $2=epoch("" = none)
|
|
local label="$1" ep="$2"
|
|
if [ -z "$ep" ]; then stale+=("$label: NO SNAPSHOTS / unreachable"); return; fi
|
|
local age=$(( (now - ep) / 3600 ))
|
|
local when; when=$(date -d "@$ep" '+%Y-%m-%d %H:%M' 2>/dev/null)
|
|
if [ "$age" -gt "$MAX_AGE_H" ]; then stale+=("$label: ${age}h old (newest $when)")
|
|
else fresh+=("$label: ${age}h old (newest $when)"); fi
|
|
}
|
|
|
|
echo "=== Backup freshness (threshold ${MAX_AGE_H}h) — $(date '+%Y-%m-%d %H:%M %Z') ==="
|
|
|
|
# --- Layer: restic file+DB — DISCOVERED, not enumerated ---------------------
|
|
#
|
|
# ⚠ THIS LIST USED TO BE HAND-WRITTEN, AND ON 2026-09-22 IT WAS WRONG. It read
|
|
# `irv-ml1 nh3-docker` and omitted nh3-dev -- the repo holding every Claude Code
|
|
# transcript on that box, althing routes, hermes history and Miranda's
|
|
# conversation. The repo had always existed and always been written; it was
|
|
# simply never enumerated, so the one repository nobody could reconstruct was
|
|
# the one nothing watched.
|
|
#
|
|
# Adding nh3-dev fixed the instance. Deriving fixes the CLASS (svos-dev's
|
|
# framing): a hand-maintained list of things to watch, sitting beside a NAS that
|
|
# already knows which repos exist, means the next repo added is unwatched BY
|
|
# DEFAULT and nothing says so. The list of what to check can silently disagree
|
|
# with the set of what exists -- the same shape as every other instrument fault
|
|
# found that day, only slower-acting.
|
|
#
|
|
# So: ask the NAS what is there. A directory counts as a repository when it has
|
|
# a `snapshots/` child, which distinguishes a real repo from a container dir
|
|
# (/volume1/Backup/restic/repo/ holds ana|esh|nh3 namespaces and no snapshots
|
|
# of its own).
|
|
#
|
|
# The hand-written list survives DEMOTED, as an EXPECTED set -- used only to
|
|
# report a repo that has VANISHED. That keeps two different facts distinct that
|
|
# would otherwise both read as silence:
|
|
# "a repo exists that nobody watches" -> impossible now, it is discovered
|
|
# "a repo we expected is gone" -> reported below
|
|
discover_repos() { # $1=host $2=parent dir — prints "name<TAB>newest_epoch"
|
|
$SSH "$1" "for d in $2/*/; do [ -d \"\$d/snapshots\" ] || continue
|
|
printf '%s\t%s\n' \"\$(basename \"\$d\")\" \"\$(stat -c %Y \"\$d\"snapshots/* 2>/dev/null | sort -n | tail -1)\"
|
|
done" 2>/dev/null
|
|
}
|
|
|
|
EXPECTED_ANA="${EXPECTED_RESTIC_ANA:-ana-docker ana-ml2 esh-docker-vm esh-ml1 esh-vm-db vm-esh-nas}"
|
|
EXPECTED_NH3="${EXPECTED_RESTIC_NH3:-irv-ml1 nh3-docker nh3-dev}"
|
|
|
|
found_ana=""; found_nh3=""
|
|
while IFS=$'\t' read -r name ep; do
|
|
[ -n "$name" ] || continue
|
|
found_ana="$found_ana $name"
|
|
report "restic/ana/$name" "$ep"
|
|
done < <(discover_repos ana-nas /mnt/backup/restic/repo/ana)
|
|
|
|
while IFS=$'\t' read -r name ep; do
|
|
[ -n "$name" ] || continue
|
|
found_nh3="$found_nh3 $name"
|
|
report "restic/nh3/$name" "$ep"
|
|
done < <(discover_repos nh3-nas /volume1/Backup/restic)
|
|
|
|
# A repo we expected and did NOT discover is a different fault from a stale one:
|
|
# the repository is gone, not behind. Say so in those words.
|
|
for want in $EXPECTED_ANA; do
|
|
in_csv "$want" "$(echo $found_ana | tr ' ' ',')" || \
|
|
stale+=("restic/ana/$want: EXPECTED REPO NOT FOUND on ana-nas (deleted, renamed, or never created)")
|
|
done
|
|
for want in $EXPECTED_NH3; do
|
|
in_csv "$want" "$(echo $found_nh3 | tr ' ' ',')" || \
|
|
stale+=("restic/nh3/$want: EXPECTED REPO NOT FOUND on nh3-nas (deleted, renamed, or never created)")
|
|
done
|
|
|
|
# --- Layer: PBS VM images (newest per guest, all namespaces) ---
|
|
# ⚠ The age comes from the snapshot's NAME (its backup time, e.g. 2026-10-04T10:00:04Z),
|
|
# never from the directory's mtime (2026-10-06). PBS rewrites index.json.blob inside the
|
|
# snapshot dir at every verify (first one at the next 02:30, then every 30 days, from
|
|
# verification.cfg outdated-after 30) and on notes or protect edits, which moves the
|
|
# dir mtime forward. Measured: ana-pve/vm/101's 10-04 snapshot read "2026-10-06" here.
|
|
# A guest whose backups had stopped would alarm up to a day late, then go quiet again
|
|
# for ~2 days after each monthly re-verify.
|
|
pbs=$($SSH pbs-ana 'for ns in /mnt/pbs-datastore/ns/*/; do n=$(basename "$ns")
|
|
for d in vm ct; do for g in "$ns$d"/*/; do [ -d "$g" ] || continue
|
|
nb=$(ls -d "$g"20*T* 2>/dev/null | sort | tail -1)
|
|
[ -n "$nb" ] && echo "$n/$d/$(basename "$g") $(date -d "$(basename "$nb")" +%s)"
|
|
done; done; done' 2>/dev/null)
|
|
if [ -z "$pbs" ]; then errors+=("PBS-ANA: unreachable or no snapshots"); else
|
|
for ns in "${!NS_NODE[@]}"; do load_jobs "$ns" || errors+=("coverage for namespace '$ns' UNREADABLE — nothing in it will be suppressed"); done
|
|
while read -r guest ep; do
|
|
[ -n "$guest" ] || continue
|
|
ns=${guest%%/*}; vmid=${guest##*/}
|
|
# Destroyed-guest check (2026-10-06): a guest deleted from the cluster stops
|
|
# writing snapshots, and its last snapshot ages into a 🔴 forever. pvesh says
|
|
# qemu/NN + lxc/NN; the PBS tree says vm/NN + ct/NN. If the owning cluster is
|
|
# readable and no longer lists this guest, its snapshots are residue, not a
|
|
# stale backup. Inventory unreadable -> fall through and alarm (fail loud).
|
|
if load_inventory "$ns"; then
|
|
# 'guest' is namespaced ("ana-pve/vm/110") — match the segment, and
|
|
# pre-set pveid so an unexpected path shape fails the MATCH, not the
|
|
# script, under set -u (infra-ops, 2026-10-06).
|
|
pveid=""
|
|
case "$guest" in */vm/*) pveid="qemu/$vmid";; */ct/*) pveid="lxc/$vmid";; esac
|
|
case " ${NS_INV[$ns]} " in
|
|
*" $pveid "*) : ;;
|
|
*) excluded+=("pbs/$guest: GUEST DESTROYED — no longer in $ns inventory; last snapshot $([ -n "$ep" ] && date -d "@$ep" '+%Y-%m-%d' || echo none) is residue, prune at leisure") ; continue ;;
|
|
esac
|
|
fi
|
|
is_covered "$ns" "$vmid"
|
|
case $? in
|
|
1) excluded+=("pbs/$guest: no enabled vzdump job covers it$(
|
|
[ -n "$ep" ] && printf ' (last snapshot %s)' "$(date -d "@$ep" '+%Y-%m-%d' 2>/dev/null)") — ${EXCLUDE_WHY[$guest]:-⚠ NO DOCUMENTED REASON — check this exclusion is intentional}") ;;
|
|
*) report "pbs/$guest" "$ep" ;;
|
|
esac
|
|
done <<<"$pbs"
|
|
fi
|
|
|
|
# --- Layer: did the vzdump jobs actually SUCCEED? -----------------------
|
|
#
|
|
# Snapshot age alone is structurally blind to a job that runs and ERRORS every
|
|
# night: nothing new is written, so the group simply ages, and the fault only
|
|
# surfaces once it crosses the 48h threshold — days late. esh-vm-workstation
|
|
# (VM 102) failed nightly from ~2026-09-06 with "timeout waiting on systemd"
|
|
# and this check would not have named it until 09-12.
|
|
#
|
|
# PVE records every task's result in /var/log/pve/tasks/index as
|
|
# <UPID> <endtime-hex> <status>
|
|
# where the UPID's 6th colon-field is the task type and the 5th is its hex
|
|
# start time. Anything vzdump in the window whose status is not OK is a fault
|
|
# TODAY, not in two days.
|
|
for entry in "${PVE_NODES[@]}"; do
|
|
set -- $entry; node_label="$1"; node_addr="$2"
|
|
idx=$($SSH "$node_addr" "sudo -n cat /var/log/pve/tasks/index" 2>/dev/null)
|
|
if [ -z "$idx" ]; then
|
|
errors+=("vzdump task log on $node_label UNREADABLE — job failures there are invisible")
|
|
continue
|
|
fi
|
|
bad=$(awk -v now="$now" -v win="$((JOB_WINDOW_H * 3600))" '
|
|
{
|
|
upid = $1; endh = $2; status = $0
|
|
sub(/^[^ ]+ [^ ]+ /, "", status)
|
|
n = split(upid, f, ":")
|
|
if (n < 7 || f[6] != "vzdump") next
|
|
start = strtonum("0x" f[5])
|
|
if (now - start > win) next
|
|
if (status == "OK") next
|
|
printf "%s%s: %s\n", (f[7] == "" ? "job" : "guest " f[7]), "", status
|
|
}' <<<"$idx" | sort -u)
|
|
[ -n "$bad" ] && while IFS= read -r b; do
|
|
[ -n "$b" ] && jobfail+=("$node_label $b")
|
|
done <<<"$bad"
|
|
done
|
|
|
|
# --- Layer: did the jobs that SUCCEEDED actually quiesce? --------------------
|
|
#
|
|
# ⚠ A task PVE records as OK can still have failed the thing that matters. Measured 2026-10-05 (nh3-pve-2, VM 111,
|
|
# svos backup drill): when the guest REFUSES fs-freeze, vzdump logs
|
|
# ERROR: unable to freeze guest fs - child process has failed to execute fsfreeze hook: exit status 1
|
|
# then backs up UNFROZEN and ends "Backup job finished successfully", task status OK, exit 0. The image is
|
|
# crash-consistent, not application-consistent, and the job-success layer above cannot see it. So: read the task
|
|
# LOG of every OK vzdump task in the window, and report any ERROR line. Freeze/thaw errors are named as
|
|
# application-consistency failures; anything else is reported as an ERROR inside an OK task.
|
|
# The remote side stays a plain grep per file (see load_jobs on why embedded awk is avoided). A task log that
|
|
# cannot be read is an error, never a silent pass.
|
|
for entry in "${PVE_NODES[@]}"; do
|
|
set -- $entry; node_label="$1"; node_addr="$2"
|
|
idx=$($SSH "$node_addr" "sudo -n cat /var/log/pve/tasks/index" 2>/dev/null)
|
|
[ -n "$idx" ] || continue # already reported UNREADABLE by the layer above
|
|
okups=$(awk -v now="$now" -v win="$((JOB_WINDOW_H * 3600))" '
|
|
{ n = split($1, f, ":"); if (n < 7 || f[6] != "vzdump") next
|
|
if (now - strtonum("0x" f[5]) > win) next
|
|
st = $0; sub(/^[^ ]+ [^ ]+ /, "", st); if (st == "OK") print $1 }' <<<"$idx")
|
|
[ -n "$okups" ] || continue
|
|
script=""
|
|
while IFS= read -r u; do
|
|
[ -n "$u" ] || continue
|
|
sh=$(cut -d: -f5 <<<"$u"); d=${sh: -1}
|
|
script+="f='/var/log/pve/tasks/$d/$u'; if [ -r \"\$f\" ]; then grep -E '^(INFO: Starting Backup of VM|ERROR:)' \"\$f\" | sed 's|^|$u\t|'; else printf 'MISSING\t%s\n' '$u'; fi
|
|
"
|
|
done <<<"$okups"
|
|
hits=$($SSH "$node_addr" "sudo -n bash -s" <<<"$script" 2>/dev/null) || { errors+=("vzdump task LOGS on $node_label unreadable — freeze failures there are invisible"); continue; }
|
|
# attribute each ERROR to the guest whose "Starting Backup of VM" line came last in that task
|
|
while IFS=$'\t' read -r u line; do
|
|
[ -n "$u" ] || continue
|
|
if [ "$u" = "MISSING" ]; then errors+=("vzdump task log missing on $node_label: $line"); continue; fi
|
|
case "$line" in
|
|
"INFO: Starting Backup of VM "*) cur=${line#INFO: Starting Backup of VM }; cur=${cur%% *}; continue ;;
|
|
esac
|
|
gid=${cur:-$(cut -d: -f7 <<<"$u")}
|
|
ts=$(date -d "@$((16#$(cut -d: -f5 <<<"$u")))" '+%m-%d %H:%M' 2>/dev/null)
|
|
if grep -qiE 'freeze|thaw' <<<"$line"; then
|
|
msg="$node_label guest $gid ($ts): APPLICATION-CONSISTENCY FAILURE, image is crash-consistent only :: ${line#ERROR: }"
|
|
else
|
|
msg="$node_label guest $gid ($ts): ERROR inside a task PVE calls OK :: ${line#ERROR: }"
|
|
fi
|
|
if [ -n "${ACKED_TASKS[$u]:-}" ] && [ -z "${BACKUP_IGNORE_ACKS:-}" ]; then
|
|
acked+=("$msg [ack: ${ACKED_TASKS[$u]}]")
|
|
else
|
|
degraded+=("$msg")
|
|
fi
|
|
done <<<"$hits"
|
|
cur=""
|
|
done
|
|
|
|
# --- restic CONTENT assertion (operator ruling 2026-09-22) -------------------
|
|
#
|
|
# The checks above read a snapshot directory's MTIME on the NAS. That proves a
|
|
# file was written recently. It does not prove the repository holds the data,
|
|
# and metadata is exactly what survives the failures worth fearing: a pruned or
|
|
# partially-written repo can still list a path whose blobs are gone.
|
|
#
|
|
# ⚠ THE ASSERTION IS CONJUNCTIVE ON PURPOSE. "a snapshot exists containing X"
|
|
# is satisfied by a three-month-old snapshot; "the latest snapshot is recent" is
|
|
# satisfied by an empty one. Age alone was the old check's problem; content
|
|
# alone is the same problem rotated. Both, or neither is worth printing.
|
|
#
|
|
# The rungs, and what each actually proves (svos-dev, 2026-09-22):
|
|
# snapshots list the repo answers
|
|
# ls a path in it the path is in the INDEX <- metadata only
|
|
# restic check the repo's structure is intact
|
|
# RESTORE a file the bytes come back <- the only proof of recovery
|
|
# We take the top rung for the small irreplaceable set, because it is the only
|
|
# one that answers the question a green light is taken to mean. Restoring from
|
|
# the multi-gigabyte paths is not worth it: a repo that returns one file is
|
|
# overwhelmingly likely to return others, and one that cannot is broken for
|
|
# everything.
|
|
#
|
|
# ⚠ IDENTITY: this runs as a --user timer (lkraven), which has no NOPASSWD sudo
|
|
# on nh3-dev, and the repo credentials live under /etc/restic. So the probe hops
|
|
# through infra-ops@localhost, which does. Not a workaround -- infra-ops is the
|
|
# documented ops identity and the credentials are deliberately root-only.
|
|
RESTIC_PROBE_PATHS="${RESTIC_PROBE_PATHS:-/home/lkraven/.local/state/svos}"
|
|
probe=$($SSH infra-ops@10.100.10.50 "sudo -n bash -s" <<PROBE 2>/dev/null
|
|
set -a; . /etc/restic/restic.env 2>/dev/null; set +a
|
|
export RESTIC_PASSWORD_FILE=/etc/restic/password
|
|
id=\$(restic snapshots --latest 1 --json 2>/dev/null | python3 -c 'import json,sys
|
|
d=json.load(sys.stdin)
|
|
print(d[0]["short_id"], int(__import__("datetime").datetime.fromisoformat(d[0]["time"][:19]).timestamp())) if d else print("NONE 0")' 2>/dev/null)
|
|
sid=\$(echo "\$id" | cut -d" " -f1); ep=\$(echo "\$id" | cut -d" " -f2)
|
|
[ "\$sid" = "NONE" ] && { echo "NOSNAP 0 0 0"; exit 0; }
|
|
# ⚠ grep -c '^/' NOT grep -c '.' — \`restic ls\` always prints a HEADER line
|
|
# ("snapshot <id> of [...] filtered by [...]") whether or not anything matched,
|
|
# so a path absent from the repo returns 1 line and a naive count reads it as
|
|
# found. Measured 2026-09-22: real path 6 lines, bogus path 1. That made the
|
|
# ABSENT case fall through and report "blobs gone" — telling an operator the
|
|
# repository was corrupt when the truth was a mistyped path. Only lines that
|
|
# begin with / are entries.
|
|
n=\$(restic ls "\$sid" $RESTIC_PROBE_PATHS 2>/dev/null | grep -c '^/')
|
|
tmp=\$(mktemp -d); restic restore "\$sid" --target "\$tmp" --include $RESTIC_PROBE_PATHS >/dev/null 2>&1
|
|
bytes=\$(find "\$tmp" -type f -printf '%s\n' 2>/dev/null | awk '{s+=\$1} END {print s+0}')
|
|
rm -rf -- "\$tmp"
|
|
echo "\$sid \$ep \$n \$bytes"
|
|
PROBE
|
|
)
|
|
set -- ${probe:-ERR 0 0 0}
|
|
r_sid="$1"; r_ep="$2"; r_entries="$3"; r_bytes="$4"
|
|
if [ "$r_sid" = "ERR" ] || [ -z "$probe" ]; then
|
|
stale+=("restic/nh3-dev CONTENT: probe could not run (repo unreachable or creds unreadable)")
|
|
elif [ "$r_sid" = "NOSNAP" ]; then
|
|
stale+=("restic/nh3-dev CONTENT: repository holds NO SNAPSHOTS")
|
|
else
|
|
r_age=$(( (now - r_ep) / 3600 ))
|
|
# conjunctive: recent AND indexed AND restorable
|
|
if [ "$r_age" -gt "$MAX_AGE_H" ]; then
|
|
stale+=("restic/nh3-dev CONTENT: latest snapshot $r_sid is ${r_age}h old (>${MAX_AGE_H}h)")
|
|
elif [ "${r_entries:-0}" -lt 1 ]; then
|
|
stale+=("restic/nh3-dev CONTENT: $RESTIC_PROBE_PATHS absent from snapshot $r_sid")
|
|
elif [ "${r_bytes:-0}" -lt 1 ]; then
|
|
stale+=("restic/nh3-dev CONTENT: $RESTIC_PROBE_PATHS listed in $r_sid but RESTORED ZERO BYTES — index intact, blobs gone")
|
|
else
|
|
fresh+=("restic/nh3-dev CONTENT: $r_sid ${r_age}h old, $r_entries entries, restored ${r_bytes}B")
|
|
fi
|
|
fi
|
|
|
|
# --- rest-server endpoint health (401 = up & serving) ---
|
|
for ep in "rest-server-ana http://10.250.50.70:8000/" "rest-server-nh3 http://10.100.50.50:8000/"; do
|
|
set -- $ep
|
|
code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 6 "$2" 2>/dev/null)
|
|
[ "$code" = "401" ] && fresh+=("$1: up (401)") || stale+=("$1: endpoint code=$code (expected 401)")
|
|
done
|
|
|
|
echo
|
|
echo "FRESH (${#fresh[@]}):"; printf ' ✅ %s\n' "${fresh[@]}"
|
|
if [ "${#excluded[@]}" -gt 0 ]; then
|
|
# Printed every run on purpose: an exclusion that was a mistake is only
|
|
# catchable if you can see it. Silence here would trade a noisy alarm for a
|
|
# blind one.
|
|
echo; echo "NOT BACKED UP BY POLICY (${#excluded[@]}) — not a fault, but check the list is still right:"
|
|
printf ' ⏸ %s\n' "${excluded[@]}"
|
|
fi
|
|
if [ "${#jobfail[@]}" -gt 0 ]; then
|
|
echo; echo "BACKUP JOBS THAT ERRORED in the last ${JOB_WINDOW_H}h (${#jobfail[@]}):"
|
|
printf ' ❌ %s\n' "${jobfail[@]}"
|
|
fi
|
|
if [ "${#degraded[@]}" -gt 0 ]; then
|
|
echo; echo "DEGRADED — task OK, but the log has an ERROR (last ${JOB_WINDOW_H}h, ${#degraded[@]}):"
|
|
printf ' 🟠 %s\n' "${degraded[@]}"
|
|
fi
|
|
if [ "${#acked[@]}" -gt 0 ]; then
|
|
echo; echo "ACKNOWLEDGED (known cause, not counted; BACKUP_IGNORE_ACKS=1 counts them):"
|
|
printf ' ☑ %s\n' "${acked[@]}"
|
|
fi
|
|
if [ "${#stale[@]}" -gt 0 ] || [ "${#errors[@]}" -gt 0 ]; then
|
|
echo; echo "STALE / PROBLEMS (${#stale[@]}+${#errors[@]}):"
|
|
[ "${#stale[@]}" -gt 0 ] && printf ' 🔴 %s\n' "${stale[@]}"
|
|
[ "${#errors[@]}" -gt 0 ] && printf ' 🔴 %s\n' "${errors[@]}"
|
|
fi
|
|
# --------- Verdict. Two different findings, two different words. -------
|
|
#
|
|
# ⚠ These used to collapse into one "RESULT: STALE / exit 1". That is a false
|
|
# statement of fact whenever the only finding is a recent job error: on
|
|
# 2026-09-20 this printed STALE while reporting 37 FRESH layers and zero stale
|
|
# ones — every backup body provably current, the ❌ rows all yesterday's
|
|
# pre-fix runs aging out of the window. infra-hermes flagged it: a reader or a
|
|
# forwarder could page someone over a state where nothing is actually stale.
|
|
#
|
|
# STALE is a claim about backup AGE. A job that ran and errored is a different
|
|
# claim with different urgency, so it gets its own verdict and its own code.
|
|
if [ "${#stale[@]}" -gt 0 ] || [ "${#errors[@]}" -gt 0 ]; then
|
|
# Bodies stale or an endpoint down. The serious one. Job errors may ride
|
|
# along and are already printed above.
|
|
echo; echo "RESULT: STALE — see docs/runbooks/backups.md"
|
|
exit 1
|
|
fi
|
|
if [ "${#jobfail[@]}" -gt 0 ]; then
|
|
echo; echo "RESULT: ERRORED-JOBS — every backup body is FRESH; ${#jobfail[@]} job(s)"
|
|
echo " errored in the last ${JOB_WINDOW_H}h. Worth a look, not a page."
|
|
echo " Errors older than the window age out on their own."
|
|
exit 3
|
|
fi
|
|
if [ "${#degraded[@]}" -gt 0 ]; then
|
|
echo; echo "RESULT: DEGRADED — every body is FRESH and no job errored, but ${#degraded[@]} backup(s) logged an"
|
|
echo " ERROR inside an OK task. An fs-freeze failure means that image is crash-consistent ONLY:"
|
|
echo " do not count it as application-consistent. Worth a look, not a page."
|
|
exit 4
|
|
fi
|
|
echo; echo "RESULT: all backups fresh"
|
|
exit 0
|