vh
0a348e548c
A guest destroyed with 'qm destroy --purge' also drops out of the vzdump
job's exclude list, so its last snapshot silently ages into a false
🔴 once it passes the threshold (ana-pve/vm/110, destroyed 10-04, fired
this morning). Before reporting coverage, consult the owning cluster's
pvesh inventory: a guest the cluster no longer lists is reported in the
policy section as residue, never as stale. Inventory unreadable fails
loud (falls through to normal alarm).
Two traps pinned the hard way during verification:
- 'guest' is namespaced (ana-pve/vm/110); the case must match the
segment (*/vm/*) and pre-set pveid so a foreign path shape fails the
match, not the script, under set -u (infra-ops caught the first cut).
- ssh inside the while-read loop MUST take -n; without it ssh swallows
the loop's remaining stdin and the entire PBS layer silently vanishes
from the report while exit code and verdict stay green.
491 lines
25 KiB
Bash
Executable File
491 lines
25 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# check-backup-freshness.sh — the "are we actually backed up?" check.
|
|
#
|
|
# Walks every backup layer and flags anything whose newest snapshot is older
|
|
# than the threshold (default 48h) or any down endpoint. Prints a report.
|
|
#
|
|
# EXIT CODES — the verdict, not just a pass/fail:
|
|
# 0 all backups fresh
|
|
# 1 STALE — a backup body is past the threshold, or an endpoint is down
|
|
# 3 ERRORED-JOBS — every body is fresh, but a vzdump job errored recently
|
|
# 4 DEGRADED — no job errored, but a backup in the window logged an ERROR inside a
|
|
# task PVE still calls OK; above all an fs-freeze refusal, which means the image
|
|
# is crash-consistent, NOT application-consistent (see the layer below)
|
|
#
|
|
# 1 and 3 are deliberately different. Collapsing them prints "STALE" over a
|
|
# fleet whose every backup is current, which is how an alarm earns being
|
|
# ignored. Designed to be
|
|
# run by a daily timer that alerts on non-zero exit (see
|
|
# scripts/install-backup-freshness-timer.sh), or by hand anytime.
|
|
#
|
|
# Companion to docs/runbooks/backups.md. Read-only — only SSH stat/curl.
|
|
#
|
|
# BACKUP_MAX_AGE_HOURS=48 scripts/check-backup-freshness.sh
|
|
#
|
|
# POLICY AWARENESS (2026-09-19). Some guests are deliberately not backed up --
|
|
# ana-scale (CT 114) and esh-scale (CT 108) are subnet-router LXCs excluded
|
|
# after a vzdump lock on esh-scale blackholed that entire site. Before this,
|
|
# the check reported those as 🔴 STALE every single morning, which is how an
|
|
# alarm teaches you to ignore it. Such guests now get their own section:
|
|
# printed, never hidden, but not counted as a fault.
|
|
#
|
|
# ⚠ COVERAGE IS A UNION ACROSS JOBS, NOT ONE JOB'S EXCLUDE LIST. Reading
|
|
# `exclude` alone gets ana CT 109 (ana-nas) exactly wrong: it IS excluded from
|
|
# the 03:00 all-guests job AND it has its own dedicated 22:00 job. Suppressing
|
|
# on the exclude list would have silently stopped alarming on a guest that is
|
|
# genuinely backed up -- turning a fix for a noisy alarm into a blind spot.
|
|
# The same applies in reverse for ESH, whose job uses an explicit
|
|
# `vmid 100..107` INCLUDE list, so esh-scale 108 is excluded by OMISSION and
|
|
# appears in no exclude list at all.
|
|
# A guest is "intentionally not backed up" only when NO enabled vzdump job
|
|
# covers it.
|
|
#
|
|
# ⚠ IF COVERAGE CANNOT BE DETERMINED, NOTHING IS SUPPRESSED. An unreachable
|
|
# PVE node means we do not know, and a backup alarm must fail loud.
|
|
# The SureFire namespace (sfsrv-pve) is deliberately never consulted -- those
|
|
# are tenant hosts -- so its guests can never be suppressed either.
|
|
set -uo pipefail
|
|
|
|
MAX_AGE_H="${BACKUP_MAX_AGE_HOURS:-48}"
|
|
SSH="ssh -o ConnectTimeout=8 -o BatchMode=yes"
|
|
now=$(date +%s)
|
|
stale=() ; fresh=() ; errors=() ; excluded=()
|
|
|
|
# PBS namespace → a node of the PVE cluster that owns it. jobs.cfg lives in
|
|
# pmxcfs and is cluster-wide, so ESH's two nodes share one job set and either
|
|
# answers for both. sfsrv-pve is absent ON PURPOSE (tenant property).
|
|
declare -A NS_NODE=(
|
|
[ana-pve]=infra-ops@10.250.250.31
|
|
[esh-pve]=infra-ops@10.0.250.35
|
|
[nh3-pve]=infra-ops@10.100.250.60
|
|
[nh3-pve-2]=infra-ops@10.100.250.62 # standalone (not clustered): own jobs.cfg, own namespace. Added 2026-10-05;
|
|
# until then its job results were never read at all
|
|
)
|
|
declare -A NS_JOBS=() # ns → one "enabled|all|vmids|excludes" line per job
|
|
declare -A NS_KNOWN=() # ns → 1 once coverage was successfully read
|
|
declare -A NS_INV=() # ns → guest id set from pvesh, ONLY when the read succeeded
|
|
|
|
# Inventory of the owning cluster ("qemu/110 lxc/109 ..."). An empty/unset set
|
|
# means UNKNOWN, never "empty cluster": an unreachable node must not make a
|
|
# live guest look destroyed (fail loud, same principle as jobs.cfg).
|
|
load_inventory() { # $1 = namespace
|
|
local ns="$1" node="${NS_NODE[$1]:-}" raw
|
|
[ -n "$node" ] || return 1
|
|
[ -n "${NS_INV[$ns]:-}" ] && return 0
|
|
# -n is load-bearing: load_inventory runs INSIDE the while-read loop over the
|
|
# PBS tree, and an ssh without -n swallows the loop's remaining stdin (the
|
|
# whole PBS layer silently disappeared on 2026-10-06 until that was pinned).
|
|
raw=$($SSH -n "$node" "sudo -n pvesh get /cluster/resources --type vm --output-format text" 2>/dev/null) || return 1
|
|
raw=$(echo "$raw" | grep -oE "(qemu|lxc)/[0-9]+" | sort -u | tr "\n" " ")
|
|
[ -n "$raw" ] || return 1
|
|
NS_INV[$ns]="$raw"
|
|
}
|
|
|
|
# WHY each policy-excluded guest is out of the backup jobs (2026-10-05). The
|
|
# exclusion is derived from jobs.cfg coverage; the REASON is standing policy
|
|
# and lives here so the report can say why instead of asking the reader to
|
|
# remember. Keep one line per guest with its paper trail; if an entry shows up
|
|
# in the policy section with no reason, that is the flag: undocumented hole.
|
|
declare -A EXCLUDE_WHY=(
|
|
[ana-pve/vm/101]="retired/shut down — Prime ruling 2026-10-04 (bus 01M44MEXG60); last snapshots kept"
|
|
[nh3-pve/vm/104]="retired/shut down (nh3-laser) — Prime ruling 2026-10-04 (bus 01M44MEXG60); last snapshots kept"
|
|
[ana-pve/ct/114]="subnet-router (ana-scale) — vzdump lock risk; ESH-scale lock once blackholed the site; rebuildable from config (docs/pfi/proxmox-inventory-2026-10-02.md)"
|
|
[nh3-pve/ct/107]="subnet-router (nh3-scale) — same scale-router class as ana/esh-scale; rebuildable from config"
|
|
[esh-pve/ct/108]="subnet-router (esh-scale) — its vzdump lock blackholed the ESH site; rebuildable from config"
|
|
[esh-pve/vm/102]="Windows Parsec/RDP sandbox, no needed state — Prime ruling 2026-09-19 (playbooks/esh-exclude-vm102-from-backup.yaml)"
|
|
)
|
|
|
|
# Every PVE node whose vzdump TASK RESULTS we read. Unlike jobs.cfg this is
|
|
# per-NODE, not cluster-wide, so both ESH nodes appear. sfsrv is absent: tenant.
|
|
PVE_NODES=(
|
|
"ana-pve infra-ops@10.250.250.31"
|
|
"esh-pve infra-ops@10.0.250.35"
|
|
"esh-nas-pve infra-ops@10.0.50.55"
|
|
"nh3-pve infra-ops@10.100.250.60"
|
|
"nh3-pve-2 infra-ops@10.100.250.62"
|
|
)
|
|
JOB_WINDOW_H="${BACKUP_JOB_WINDOW_HOURS:-36}"
|
|
jobfail=()
|
|
degraded=()
|
|
acked=()
|
|
# Acknowledged OK-but-ERROR tasks: a known, deliberate cause (a drill), printed every run in their own section and
|
|
# never counted. Key = the full UPID, value = why. Keep the paper trail; the entries age out with the window.
|
|
declare -A ACKED_TASKS=(
|
|
[UPID:nh3-pve-2:0004FFBA:00719F66:6AC4628F:vzdump:111:root@pam:]="2026-10-05 svos backup drill B: freeze refused ON PURPOSE by a temporary hook (servers/nh3-svos/README.md)"
|
|
)
|
|
|
|
load_jobs() { # $1 = namespace
|
|
local ns="$1" node="${NS_NODE[$1]:-}" raw
|
|
[ -n "$node" ] || return 1
|
|
# Fetch the raw file and parse it HERE. The parser used to be an awk script
|
|
# embedded in the ssh command string; the quoting mangled it silently and
|
|
# every namespace came back unreadable — which at least failed loud, because
|
|
# that is how this is built. Keep the remote side a plain `cat`.
|
|
raw=$($SSH "$node" "sudo -n cat /etc/pve/jobs.cfg" 2>/dev/null) || return 1
|
|
[ -n "$raw" ] || return 1
|
|
local parsed
|
|
parsed=$(awk '
|
|
function flush() { if (seen) { print e "|" a "|" v "|" x; seen=0 } }
|
|
/^vzdump:/ { flush(); seen=1; e=1; a=0; v=""; x=""; next }
|
|
/^[^ \t]/ { flush(); next }
|
|
seen && $1=="enabled" { e=$2 }
|
|
seen && $1=="all" { a=$2 }
|
|
seen && $1=="vmid" { v=$2 }
|
|
seen && $1=="exclude" { x=$2 }
|
|
END { flush() }
|
|
' <<<"$raw")
|
|
[ -n "$parsed" ] || return 1
|
|
NS_JOBS[$ns]="$parsed"
|
|
NS_KNOWN[$ns]=1
|
|
}
|
|
|
|
in_csv() { # $1=needle $2=comma list
|
|
case ",$2," in *",$1,"*) return 0 ;; *) return 1 ;; esac
|
|
}
|
|
|
|
is_covered() { # $1=namespace $2=vmid — 0 covered, 1 not covered, 2 unknown
|
|
local ns="$1" id="$2" line enabled all vmids excludes
|
|
[ -n "${NS_KNOWN[$ns]:-}" ] || return 2
|
|
while IFS='|' read -r enabled all vmids excludes; do
|
|
[ "$enabled" = "1" ] || continue
|
|
if [ -n "$vmids" ]; then
|
|
in_csv "$id" "$vmids" && return 0 # explicit include list
|
|
elif [ "$all" = "1" ]; then
|
|
in_csv "$id" "$excludes" || return 0 # all-guests minus its excludes
|
|
fi
|
|
done <<<"${NS_JOBS[$ns]}"
|
|
return 1
|
|
}
|
|
|
|
# newest snapshot epoch under a remote glob (echoes epoch or empty)
|
|
newest_epoch() { # $1=host $2=glob
|
|
$SSH "$1" "stat -c %Y $2 2>/dev/null | sort -n | tail -1" 2>/dev/null
|
|
}
|
|
report() { # $1=label $2=epoch("" = none)
|
|
local label="$1" ep="$2"
|
|
if [ -z "$ep" ]; then stale+=("$label: NO SNAPSHOTS / unreachable"); return; fi
|
|
local age=$(( (now - ep) / 3600 ))
|
|
local when; when=$(date -d "@$ep" '+%Y-%m-%d %H:%M' 2>/dev/null)
|
|
if [ "$age" -gt "$MAX_AGE_H" ]; then stale+=("$label: ${age}h old (newest $when)")
|
|
else fresh+=("$label: ${age}h old (newest $when)"); fi
|
|
}
|
|
|
|
echo "=== Backup freshness (threshold ${MAX_AGE_H}h) — $(date '+%Y-%m-%d %H:%M %Z') ==="
|
|
|
|
# --- Layer: restic file+DB — DISCOVERED, not enumerated ---------------------
|
|
#
|
|
# ⚠ THIS LIST USED TO BE HAND-WRITTEN, AND ON 2026-09-22 IT WAS WRONG. It read
|
|
# `irv-ml1 nh3-docker` and omitted nh3-dev -- the repo holding every Claude Code
|
|
# transcript on that box, althing routes, hermes history and Miranda's
|
|
# conversation. The repo had always existed and always been written; it was
|
|
# simply never enumerated, so the one repository nobody could reconstruct was
|
|
# the one nothing watched.
|
|
#
|
|
# Adding nh3-dev fixed the instance. Deriving fixes the CLASS (svos-dev's
|
|
# framing): a hand-maintained list of things to watch, sitting beside a NAS that
|
|
# already knows which repos exist, means the next repo added is unwatched BY
|
|
# DEFAULT and nothing says so. The list of what to check can silently disagree
|
|
# with the set of what exists -- the same shape as every other instrument fault
|
|
# found that day, only slower-acting.
|
|
#
|
|
# So: ask the NAS what is there. A directory counts as a repository when it has
|
|
# a `snapshots/` child, which distinguishes a real repo from a container dir
|
|
# (/volume1/Backup/restic/repo/ holds ana|esh|nh3 namespaces and no snapshots
|
|
# of its own).
|
|
#
|
|
# The hand-written list survives DEMOTED, as an EXPECTED set -- used only to
|
|
# report a repo that has VANISHED. That keeps two different facts distinct that
|
|
# would otherwise both read as silence:
|
|
# "a repo exists that nobody watches" -> impossible now, it is discovered
|
|
# "a repo we expected is gone" -> reported below
|
|
discover_repos() { # $1=host $2=parent dir — prints "name<TAB>newest_epoch"
|
|
$SSH "$1" "for d in $2/*/; do [ -d \"\$d/snapshots\" ] || continue
|
|
printf '%s\t%s\n' \"\$(basename \"\$d\")\" \"\$(stat -c %Y \"\$d\"snapshots/* 2>/dev/null | sort -n | tail -1)\"
|
|
done" 2>/dev/null
|
|
}
|
|
|
|
EXPECTED_ANA="${EXPECTED_RESTIC_ANA:-ana-docker ana-ml2 esh-docker-vm esh-ml1 esh-vm-db vm-esh-nas}"
|
|
EXPECTED_NH3="${EXPECTED_RESTIC_NH3:-irv-ml1 nh3-docker nh3-dev}"
|
|
|
|
found_ana=""; found_nh3=""
|
|
while IFS=$'\t' read -r name ep; do
|
|
[ -n "$name" ] || continue
|
|
found_ana="$found_ana $name"
|
|
report "restic/ana/$name" "$ep"
|
|
done < <(discover_repos ana-nas /mnt/backup/restic/repo/ana)
|
|
|
|
while IFS=$'\t' read -r name ep; do
|
|
[ -n "$name" ] || continue
|
|
found_nh3="$found_nh3 $name"
|
|
report "restic/nh3/$name" "$ep"
|
|
done < <(discover_repos nh3-nas /volume1/Backup/restic)
|
|
|
|
# A repo we expected and did NOT discover is a different fault from a stale one:
|
|
# the repository is gone, not behind. Say so in those words.
|
|
for want in $EXPECTED_ANA; do
|
|
in_csv "$want" "$(echo $found_ana | tr ' ' ',')" || \
|
|
stale+=("restic/ana/$want: EXPECTED REPO NOT FOUND on ana-nas (deleted, renamed, or never created)")
|
|
done
|
|
for want in $EXPECTED_NH3; do
|
|
in_csv "$want" "$(echo $found_nh3 | tr ' ' ',')" || \
|
|
stale+=("restic/nh3/$want: EXPECTED REPO NOT FOUND on nh3-nas (deleted, renamed, or never created)")
|
|
done
|
|
|
|
# --- Layer: PBS VM images (newest per guest, all namespaces) ---
|
|
pbs=$($SSH pbs-ana 'for ns in /mnt/pbs-datastore/ns/*/; do n=$(basename "$ns")
|
|
for d in vm ct; do for g in "$ns$d"/*/; do [ -d "$g" ] || continue
|
|
nb=$(ls -d "$g"20*T* 2>/dev/null | sort | tail -1)
|
|
[ -n "$nb" ] && echo "$n/$d/$(basename "$g") $(stat -c %Y "$nb")"
|
|
done; done; done' 2>/dev/null)
|
|
if [ -z "$pbs" ]; then errors+=("PBS-ANA: unreachable or no snapshots"); else
|
|
for ns in "${!NS_NODE[@]}"; do load_jobs "$ns" || errors+=("coverage for namespace '$ns' UNREADABLE — nothing in it will be suppressed"); done
|
|
while read -r guest ep; do
|
|
[ -n "$guest" ] || continue
|
|
ns=${guest%%/*}; vmid=${guest##*/}
|
|
# Destroyed-guest check (2026-10-06): a guest deleted from the cluster stops
|
|
# writing snapshots, and its last snapshot ages into a 🔴 forever. pvesh says
|
|
# qemu/NN + lxc/NN; the PBS tree says vm/NN + ct/NN. If the owning cluster is
|
|
# readable and no longer lists this guest, its snapshots are residue, not a
|
|
# stale backup. Inventory unreadable -> fall through and alarm (fail loud).
|
|
if load_inventory "$ns"; then
|
|
# 'guest' is namespaced ("ana-pve/vm/110") — match the segment, and
|
|
# pre-set pveid so an unexpected path shape fails the MATCH, not the
|
|
# script, under set -u (infra-ops, 2026-10-06).
|
|
pveid=""
|
|
case "$guest" in */vm/*) pveid="qemu/$vmid";; */ct/*) pveid="lxc/$vmid";; esac
|
|
case " ${NS_INV[$ns]} " in
|
|
*" $pveid "*) : ;;
|
|
*) excluded+=("pbs/$guest: GUEST DESTROYED — no longer in $ns inventory; last snapshot $([ -n "$ep" ] && date -d "@$ep" '+%Y-%m-%d' || echo none) is residue, prune at leisure") ; continue ;;
|
|
esac
|
|
fi
|
|
is_covered "$ns" "$vmid"
|
|
case $? in
|
|
1) excluded+=("pbs/$guest: no enabled vzdump job covers it$(
|
|
[ -n "$ep" ] && printf ' (last snapshot %s)' "$(date -d "@$ep" '+%Y-%m-%d' 2>/dev/null)") — ${EXCLUDE_WHY[$guest]:-⚠ NO DOCUMENTED REASON — check this exclusion is intentional}") ;;
|
|
*) report "pbs/$guest" "$ep" ;;
|
|
esac
|
|
done <<<"$pbs"
|
|
fi
|
|
|
|
# --- Layer: did the vzdump jobs actually SUCCEED? -----------------------
|
|
#
|
|
# Snapshot age alone is structurally blind to a job that runs and ERRORS every
|
|
# night: nothing new is written, so the group simply ages, and the fault only
|
|
# surfaces once it crosses the 48h threshold — days late. esh-vm-workstation
|
|
# (VM 102) failed nightly from ~2026-09-06 with "timeout waiting on systemd"
|
|
# and this check would not have named it until 09-12.
|
|
#
|
|
# PVE records every task's result in /var/log/pve/tasks/index as
|
|
# <UPID> <endtime-hex> <status>
|
|
# where the UPID's 6th colon-field is the task type and the 5th is its hex
|
|
# start time. Anything vzdump in the window whose status is not OK is a fault
|
|
# TODAY, not in two days.
|
|
for entry in "${PVE_NODES[@]}"; do
|
|
set -- $entry; node_label="$1"; node_addr="$2"
|
|
idx=$($SSH "$node_addr" "sudo -n cat /var/log/pve/tasks/index" 2>/dev/null)
|
|
if [ -z "$idx" ]; then
|
|
errors+=("vzdump task log on $node_label UNREADABLE — job failures there are invisible")
|
|
continue
|
|
fi
|
|
bad=$(awk -v now="$now" -v win="$((JOB_WINDOW_H * 3600))" '
|
|
{
|
|
upid = $1; endh = $2; status = $0
|
|
sub(/^[^ ]+ [^ ]+ /, "", status)
|
|
n = split(upid, f, ":")
|
|
if (n < 7 || f[6] != "vzdump") next
|
|
start = strtonum("0x" f[5])
|
|
if (now - start > win) next
|
|
if (status == "OK") next
|
|
printf "%s%s: %s\n", (f[7] == "" ? "job" : "guest " f[7]), "", status
|
|
}' <<<"$idx" | sort -u)
|
|
[ -n "$bad" ] && while IFS= read -r b; do
|
|
[ -n "$b" ] && jobfail+=("$node_label $b")
|
|
done <<<"$bad"
|
|
done
|
|
|
|
# --- Layer: did the jobs that SUCCEEDED actually quiesce? --------------------
|
|
#
|
|
# ⚠ A task PVE records as OK can still have failed the thing that matters. Measured 2026-10-05 (nh3-pve-2, VM 111,
|
|
# svos backup drill): when the guest REFUSES fs-freeze, vzdump logs
|
|
# ERROR: unable to freeze guest fs - child process has failed to execute fsfreeze hook: exit status 1
|
|
# then backs up UNFROZEN and ends "Backup job finished successfully", task status OK, exit 0. The image is
|
|
# crash-consistent, not application-consistent, and the job-success layer above cannot see it. So: read the task
|
|
# LOG of every OK vzdump task in the window, and report any ERROR line. Freeze/thaw errors are named as
|
|
# application-consistency failures; anything else is reported as an ERROR inside an OK task.
|
|
# The remote side stays a plain grep per file (see load_jobs on why embedded awk is avoided). A task log that
|
|
# cannot be read is an error, never a silent pass.
|
|
for entry in "${PVE_NODES[@]}"; do
|
|
set -- $entry; node_label="$1"; node_addr="$2"
|
|
idx=$($SSH "$node_addr" "sudo -n cat /var/log/pve/tasks/index" 2>/dev/null)
|
|
[ -n "$idx" ] || continue # already reported UNREADABLE by the layer above
|
|
okups=$(awk -v now="$now" -v win="$((JOB_WINDOW_H * 3600))" '
|
|
{ n = split($1, f, ":"); if (n < 7 || f[6] != "vzdump") next
|
|
if (now - strtonum("0x" f[5]) > win) next
|
|
st = $0; sub(/^[^ ]+ [^ ]+ /, "", st); if (st == "OK") print $1 }' <<<"$idx")
|
|
[ -n "$okups" ] || continue
|
|
script=""
|
|
while IFS= read -r u; do
|
|
[ -n "$u" ] || continue
|
|
sh=$(cut -d: -f5 <<<"$u"); d=${sh: -1}
|
|
script+="f='/var/log/pve/tasks/$d/$u'; if [ -r \"\$f\" ]; then grep -E '^(INFO: Starting Backup of VM|ERROR:)' \"\$f\" | sed 's|^|$u\t|'; else printf 'MISSING\t%s\n' '$u'; fi
|
|
"
|
|
done <<<"$okups"
|
|
hits=$($SSH "$node_addr" "sudo -n bash -s" <<<"$script" 2>/dev/null) || { errors+=("vzdump task LOGS on $node_label unreadable — freeze failures there are invisible"); continue; }
|
|
# attribute each ERROR to the guest whose "Starting Backup of VM" line came last in that task
|
|
while IFS=$'\t' read -r u line; do
|
|
[ -n "$u" ] || continue
|
|
if [ "$u" = "MISSING" ]; then errors+=("vzdump task log missing on $node_label: $line"); continue; fi
|
|
case "$line" in
|
|
"INFO: Starting Backup of VM "*) cur=${line#INFO: Starting Backup of VM }; cur=${cur%% *}; continue ;;
|
|
esac
|
|
gid=${cur:-$(cut -d: -f7 <<<"$u")}
|
|
ts=$(date -d "@$((16#$(cut -d: -f5 <<<"$u")))" '+%m-%d %H:%M' 2>/dev/null)
|
|
if grep -qiE 'freeze|thaw' <<<"$line"; then
|
|
msg="$node_label guest $gid ($ts): APPLICATION-CONSISTENCY FAILURE, image is crash-consistent only :: ${line#ERROR: }"
|
|
else
|
|
msg="$node_label guest $gid ($ts): ERROR inside a task PVE calls OK :: ${line#ERROR: }"
|
|
fi
|
|
if [ -n "${ACKED_TASKS[$u]:-}" ] && [ -z "${BACKUP_IGNORE_ACKS:-}" ]; then
|
|
acked+=("$msg [ack: ${ACKED_TASKS[$u]}]")
|
|
else
|
|
degraded+=("$msg")
|
|
fi
|
|
done <<<"$hits"
|
|
cur=""
|
|
done
|
|
|
|
# --- restic CONTENT assertion (operator ruling 2026-09-22) -------------------
|
|
#
|
|
# The checks above read a snapshot directory's MTIME on the NAS. That proves a
|
|
# file was written recently. It does not prove the repository holds the data,
|
|
# and metadata is exactly what survives the failures worth fearing: a pruned or
|
|
# partially-written repo can still list a path whose blobs are gone.
|
|
#
|
|
# ⚠ THE ASSERTION IS CONJUNCTIVE ON PURPOSE. "a snapshot exists containing X"
|
|
# is satisfied by a three-month-old snapshot; "the latest snapshot is recent" is
|
|
# satisfied by an empty one. Age alone was the old check's problem; content
|
|
# alone is the same problem rotated. Both, or neither is worth printing.
|
|
#
|
|
# The rungs, and what each actually proves (svos-dev, 2026-09-22):
|
|
# snapshots list the repo answers
|
|
# ls a path in it the path is in the INDEX <- metadata only
|
|
# restic check the repo's structure is intact
|
|
# RESTORE a file the bytes come back <- the only proof of recovery
|
|
# We take the top rung for the small irreplaceable set, because it is the only
|
|
# one that answers the question a green light is taken to mean. Restoring from
|
|
# the multi-gigabyte paths is not worth it: a repo that returns one file is
|
|
# overwhelmingly likely to return others, and one that cannot is broken for
|
|
# everything.
|
|
#
|
|
# ⚠ IDENTITY: this runs as a --user timer (lkraven), which has no NOPASSWD sudo
|
|
# on nh3-dev, and the repo credentials live under /etc/restic. So the probe hops
|
|
# through infra-ops@localhost, which does. Not a workaround -- infra-ops is the
|
|
# documented ops identity and the credentials are deliberately root-only.
|
|
RESTIC_PROBE_PATHS="${RESTIC_PROBE_PATHS:-/home/lkraven/.local/state/svos}"
|
|
probe=$($SSH infra-ops@10.100.10.50 "sudo -n bash -s" <<PROBE 2>/dev/null
|
|
set -a; . /etc/restic/restic.env 2>/dev/null; set +a
|
|
export RESTIC_PASSWORD_FILE=/etc/restic/password
|
|
id=\$(restic snapshots --latest 1 --json 2>/dev/null | python3 -c 'import json,sys
|
|
d=json.load(sys.stdin)
|
|
print(d[0]["short_id"], int(__import__("datetime").datetime.fromisoformat(d[0]["time"][:19]).timestamp())) if d else print("NONE 0")' 2>/dev/null)
|
|
sid=\$(echo "\$id" | cut -d" " -f1); ep=\$(echo "\$id" | cut -d" " -f2)
|
|
[ "\$sid" = "NONE" ] && { echo "NOSNAP 0 0 0"; exit 0; }
|
|
# ⚠ grep -c '^/' NOT grep -c '.' — \`restic ls\` always prints a HEADER line
|
|
# ("snapshot <id> of [...] filtered by [...]") whether or not anything matched,
|
|
# so a path absent from the repo returns 1 line and a naive count reads it as
|
|
# found. Measured 2026-09-22: real path 6 lines, bogus path 1. That made the
|
|
# ABSENT case fall through and report "blobs gone" — telling an operator the
|
|
# repository was corrupt when the truth was a mistyped path. Only lines that
|
|
# begin with / are entries.
|
|
n=\$(restic ls "\$sid" $RESTIC_PROBE_PATHS 2>/dev/null | grep -c '^/')
|
|
tmp=\$(mktemp -d); restic restore "\$sid" --target "\$tmp" --include $RESTIC_PROBE_PATHS >/dev/null 2>&1
|
|
bytes=\$(find "\$tmp" -type f -printf '%s\n' 2>/dev/null | awk '{s+=\$1} END {print s+0}')
|
|
rm -rf -- "\$tmp"
|
|
echo "\$sid \$ep \$n \$bytes"
|
|
PROBE
|
|
)
|
|
set -- ${probe:-ERR 0 0 0}
|
|
r_sid="$1"; r_ep="$2"; r_entries="$3"; r_bytes="$4"
|
|
if [ "$r_sid" = "ERR" ] || [ -z "$probe" ]; then
|
|
stale+=("restic/nh3-dev CONTENT: probe could not run (repo unreachable or creds unreadable)")
|
|
elif [ "$r_sid" = "NOSNAP" ]; then
|
|
stale+=("restic/nh3-dev CONTENT: repository holds NO SNAPSHOTS")
|
|
else
|
|
r_age=$(( (now - r_ep) / 3600 ))
|
|
# conjunctive: recent AND indexed AND restorable
|
|
if [ "$r_age" -gt "$MAX_AGE_H" ]; then
|
|
stale+=("restic/nh3-dev CONTENT: latest snapshot $r_sid is ${r_age}h old (>${MAX_AGE_H}h)")
|
|
elif [ "${r_entries:-0}" -lt 1 ]; then
|
|
stale+=("restic/nh3-dev CONTENT: $RESTIC_PROBE_PATHS absent from snapshot $r_sid")
|
|
elif [ "${r_bytes:-0}" -lt 1 ]; then
|
|
stale+=("restic/nh3-dev CONTENT: $RESTIC_PROBE_PATHS listed in $r_sid but RESTORED ZERO BYTES — index intact, blobs gone")
|
|
else
|
|
fresh+=("restic/nh3-dev CONTENT: $r_sid ${r_age}h old, $r_entries entries, restored ${r_bytes}B")
|
|
fi
|
|
fi
|
|
|
|
# --- rest-server endpoint health (401 = up & serving) ---
|
|
for ep in "rest-server-ana http://10.250.50.70:8000/" "rest-server-nh3 http://10.100.50.50:8000/"; do
|
|
set -- $ep
|
|
code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 6 "$2" 2>/dev/null)
|
|
[ "$code" = "401" ] && fresh+=("$1: up (401)") || stale+=("$1: endpoint code=$code (expected 401)")
|
|
done
|
|
|
|
echo
|
|
echo "FRESH (${#fresh[@]}):"; printf ' ✅ %s\n' "${fresh[@]}"
|
|
if [ "${#excluded[@]}" -gt 0 ]; then
|
|
# Printed every run on purpose: an exclusion that was a mistake is only
|
|
# catchable if you can see it. Silence here would trade a noisy alarm for a
|
|
# blind one.
|
|
echo; echo "NOT BACKED UP BY POLICY (${#excluded[@]}) — not a fault, but check the list is still right:"
|
|
printf ' ⏸ %s\n' "${excluded[@]}"
|
|
fi
|
|
if [ "${#jobfail[@]}" -gt 0 ]; then
|
|
echo; echo "BACKUP JOBS THAT ERRORED in the last ${JOB_WINDOW_H}h (${#jobfail[@]}):"
|
|
printf ' ❌ %s\n' "${jobfail[@]}"
|
|
fi
|
|
if [ "${#degraded[@]}" -gt 0 ]; then
|
|
echo; echo "DEGRADED — task OK, but the log has an ERROR (last ${JOB_WINDOW_H}h, ${#degraded[@]}):"
|
|
printf ' 🟠 %s\n' "${degraded[@]}"
|
|
fi
|
|
if [ "${#acked[@]}" -gt 0 ]; then
|
|
echo; echo "ACKNOWLEDGED (known cause, not counted; BACKUP_IGNORE_ACKS=1 counts them):"
|
|
printf ' ☑ %s\n' "${acked[@]}"
|
|
fi
|
|
if [ "${#stale[@]}" -gt 0 ] || [ "${#errors[@]}" -gt 0 ]; then
|
|
echo; echo "STALE / PROBLEMS (${#stale[@]}+${#errors[@]}):"
|
|
[ "${#stale[@]}" -gt 0 ] && printf ' 🔴 %s\n' "${stale[@]}"
|
|
[ "${#errors[@]}" -gt 0 ] && printf ' 🔴 %s\n' "${errors[@]}"
|
|
fi
|
|
# --------- Verdict. Two different findings, two different words. -------
|
|
#
|
|
# ⚠ These used to collapse into one "RESULT: STALE / exit 1". That is a false
|
|
# statement of fact whenever the only finding is a recent job error: on
|
|
# 2026-09-20 this printed STALE while reporting 37 FRESH layers and zero stale
|
|
# ones — every backup body provably current, the ❌ rows all yesterday's
|
|
# pre-fix runs aging out of the window. infra-hermes flagged it: a reader or a
|
|
# forwarder could page someone over a state where nothing is actually stale.
|
|
#
|
|
# STALE is a claim about backup AGE. A job that ran and errored is a different
|
|
# claim with different urgency, so it gets its own verdict and its own code.
|
|
if [ "${#stale[@]}" -gt 0 ] || [ "${#errors[@]}" -gt 0 ]; then
|
|
# Bodies stale or an endpoint down. The serious one. Job errors may ride
|
|
# along and are already printed above.
|
|
echo; echo "RESULT: STALE — see docs/runbooks/backups.md"
|
|
exit 1
|
|
fi
|
|
if [ "${#jobfail[@]}" -gt 0 ]; then
|
|
echo; echo "RESULT: ERRORED-JOBS — every backup body is FRESH; ${#jobfail[@]} job(s)"
|
|
echo " errored in the last ${JOB_WINDOW_H}h. Worth a look, not a page."
|
|
echo " Errors older than the window age out on their own."
|
|
exit 3
|
|
fi
|
|
if [ "${#degraded[@]}" -gt 0 ]; then
|
|
echo; echo "RESULT: DEGRADED — every body is FRESH and no job errored, but ${#degraded[@]} backup(s) logged an"
|
|
echo " ERROR inside an OK task. An fs-freeze failure means that image is crash-consistent ONLY:"
|
|
echo " do not count it as application-consistent. Worth a look, not a page."
|
|
exit 4
|
|
fi
|
|
echo; echo "RESULT: all backups fresh"
|
|
exit 0
|