feat(svos-front,backups): SVOS hosting bounds and previous-release assets; backup check flags fs-freeze failures and covers nh3-pve-2
Front door: request bodies capped at 1 MiB, headers within 5 s, body within 15 s, no response deadlines (SSE stays open). Validated live: 413 over the cap, stalled body cut at 15 s, signed ping and an SSE stream unaffected. svos-front-inventory also serves the previous release's hashed assets credential-free from that release's static dir, hash-checked. Backup check: reads the logs of vzdump tasks PVE records as OK and reports ERROR lines; an fs-freeze/thaw error is an application-consistency failure, new verdict DEGRADED (exit 4), wired into the alert wrapper. nh3-pve-2 added to the node and namespace lists (its job results were never read). The svos drill task is acknowledged by UPID; BACKUP_IGNORE_ACKS=1 counts it (positive control: exit 4).
This commit is contained in:
@@ -8,6 +8,9 @@
|
||||
# 0 all backups fresh
|
||||
# 1 STALE — a backup body is past the threshold, or an endpoint is down
|
||||
# 3 ERRORED-JOBS — every body is fresh, but a vzdump job errored recently
|
||||
# 4 DEGRADED — no job errored, but a backup in the window logged an ERROR inside a
|
||||
# task PVE still calls OK; above all an fs-freeze refusal, which means the image
|
||||
# is crash-consistent, NOT application-consistent (see the layer below)
|
||||
#
|
||||
# 1 and 3 are deliberately different. Collapsing them prints "STALE" over a
|
||||
# fleet whose every backup is current, which is how an alarm earns being
|
||||
@@ -55,6 +58,8 @@ declare -A NS_NODE=(
|
||||
[ana-pve]=infra-ops@10.250.250.31
|
||||
[esh-pve]=infra-ops@10.0.250.35
|
||||
[nh3-pve]=infra-ops@10.100.250.60
|
||||
[nh3-pve-2]=infra-ops@10.100.250.62 # standalone (not clustered): own jobs.cfg, own namespace. Added 2026-10-05;
|
||||
# until then its job results were never read at all
|
||||
)
|
||||
declare -A NS_JOBS=() # ns → one "enabled|all|vmids|excludes" line per job
|
||||
declare -A NS_KNOWN=() # ns → 1 once coverage was successfully read
|
||||
@@ -80,9 +85,17 @@ PVE_NODES=(
|
||||
"esh-pve infra-ops@10.0.250.35"
|
||||
"esh-nas-pve infra-ops@10.0.50.55"
|
||||
"nh3-pve infra-ops@10.100.250.60"
|
||||
"nh3-pve-2 infra-ops@10.100.250.62"
|
||||
)
|
||||
JOB_WINDOW_H="${BACKUP_JOB_WINDOW_HOURS:-36}"
|
||||
jobfail=()
|
||||
degraded=()
|
||||
acked=()
|
||||
# Acknowledged OK-but-ERROR tasks: a known, deliberate cause (a drill), printed every run in their own section and
|
||||
# never counted. Key = the full UPID, value = why. Keep the paper trail; the entries age out with the window.
|
||||
declare -A ACKED_TASKS=(
|
||||
[UPID:nh3-pve-2:0004FFBA:00719F66:6AC4628F:vzdump:111:root@pam:]="2026-10-05 svos backup drill B: freeze refused ON PURPOSE by a temporary hook (servers/nh3-svos/README.md)"
|
||||
)
|
||||
|
||||
load_jobs() { # $1 = namespace
|
||||
local ns="$1" node="${NS_NODE[$1]:-}" raw
|
||||
@@ -257,6 +270,57 @@ for entry in "${PVE_NODES[@]}"; do
|
||||
done <<<"$bad"
|
||||
done
|
||||
|
||||
# --- Layer: did the jobs that SUCCEEDED actually quiesce? --------------------
|
||||
#
|
||||
# ⚠ A task PVE records as OK can still have failed the thing that matters. Measured 2026-10-05 (nh3-pve-2, VM 111,
|
||||
# svos backup drill): when the guest REFUSES fs-freeze, vzdump logs
|
||||
# ERROR: unable to freeze guest fs - child process has failed to execute fsfreeze hook: exit status 1
|
||||
# then backs up UNFROZEN and ends "Backup job finished successfully", task status OK, exit 0. The image is
|
||||
# crash-consistent, not application-consistent, and the job-success layer above cannot see it. So: read the task
|
||||
# LOG of every OK vzdump task in the window, and report any ERROR line. Freeze/thaw errors are named as
|
||||
# application-consistency failures; anything else is reported as an ERROR inside an OK task.
|
||||
# The remote side stays a plain grep per file (see load_jobs on why embedded awk is avoided). A task log that
|
||||
# cannot be read is an error, never a silent pass.
|
||||
for entry in "${PVE_NODES[@]}"; do
|
||||
set -- $entry; node_label="$1"; node_addr="$2"
|
||||
idx=$($SSH "$node_addr" "sudo -n cat /var/log/pve/tasks/index" 2>/dev/null)
|
||||
[ -n "$idx" ] || continue # already reported UNREADABLE by the layer above
|
||||
okups=$(awk -v now="$now" -v win="$((JOB_WINDOW_H * 3600))" '
|
||||
{ n = split($1, f, ":"); if (n < 7 || f[6] != "vzdump") next
|
||||
if (now - strtonum("0x" f[5]) > win) next
|
||||
st = $0; sub(/^[^ ]+ [^ ]+ /, "", st); if (st == "OK") print $1 }' <<<"$idx")
|
||||
[ -n "$okups" ] || continue
|
||||
script=""
|
||||
while IFS= read -r u; do
|
||||
[ -n "$u" ] || continue
|
||||
sh=$(cut -d: -f5 <<<"$u"); d=${sh: -1}
|
||||
script+="f='/var/log/pve/tasks/$d/$u'; if [ -r \"\$f\" ]; then grep -E '^(INFO: Starting Backup of VM|ERROR:)' \"\$f\" | sed 's|^|$u\t|'; else printf 'MISSING\t%s\n' '$u'; fi
|
||||
"
|
||||
done <<<"$okups"
|
||||
hits=$($SSH "$node_addr" "sudo -n bash -s" <<<"$script" 2>/dev/null) || { errors+=("vzdump task LOGS on $node_label unreadable — freeze failures there are invisible"); continue; }
|
||||
# attribute each ERROR to the guest whose "Starting Backup of VM" line came last in that task
|
||||
while IFS=$'\t' read -r u line; do
|
||||
[ -n "$u" ] || continue
|
||||
if [ "$u" = "MISSING" ]; then errors+=("vzdump task log missing on $node_label: $line"); continue; fi
|
||||
case "$line" in
|
||||
"INFO: Starting Backup of VM "*) cur=${line#INFO: Starting Backup of VM }; cur=${cur%% *}; continue ;;
|
||||
esac
|
||||
gid=${cur:-$(cut -d: -f7 <<<"$u")}
|
||||
ts=$(date -d "@$((16#$(cut -d: -f5 <<<"$u")))" '+%m-%d %H:%M' 2>/dev/null)
|
||||
if grep -qiE 'freeze|thaw' <<<"$line"; then
|
||||
msg="$node_label guest $gid ($ts): APPLICATION-CONSISTENCY FAILURE, image is crash-consistent only :: ${line#ERROR: }"
|
||||
else
|
||||
msg="$node_label guest $gid ($ts): ERROR inside a task PVE calls OK :: ${line#ERROR: }"
|
||||
fi
|
||||
if [ -n "${ACKED_TASKS[$u]:-}" ] && [ -z "${BACKUP_IGNORE_ACKS:-}" ]; then
|
||||
acked+=("$msg [ack: ${ACKED_TASKS[$u]}]")
|
||||
else
|
||||
degraded+=("$msg")
|
||||
fi
|
||||
done <<<"$hits"
|
||||
cur=""
|
||||
done
|
||||
|
||||
# --- restic CONTENT assertion (operator ruling 2026-09-22) -------------------
|
||||
#
|
||||
# The checks above read a snapshot directory's MTIME on the NAS. That proves a
|
||||
@@ -347,6 +411,14 @@ if [ "${#jobfail[@]}" -gt 0 ]; then
|
||||
echo; echo "BACKUP JOBS THAT ERRORED in the last ${JOB_WINDOW_H}h (${#jobfail[@]}):"
|
||||
printf ' ❌ %s\n' "${jobfail[@]}"
|
||||
fi
|
||||
if [ "${#degraded[@]}" -gt 0 ]; then
|
||||
echo; echo "DEGRADED — task OK, but the log has an ERROR (last ${JOB_WINDOW_H}h, ${#degraded[@]}):"
|
||||
printf ' 🟠 %s\n' "${degraded[@]}"
|
||||
fi
|
||||
if [ "${#acked[@]}" -gt 0 ]; then
|
||||
echo; echo "ACKNOWLEDGED (known cause, not counted; BACKUP_IGNORE_ACKS=1 counts them):"
|
||||
printf ' ☑ %s\n' "${acked[@]}"
|
||||
fi
|
||||
if [ "${#stale[@]}" -gt 0 ] || [ "${#errors[@]}" -gt 0 ]; then
|
||||
echo; echo "STALE / PROBLEMS (${#stale[@]}+${#errors[@]}):"
|
||||
[ "${#stale[@]}" -gt 0 ] && printf ' 🔴 %s\n' "${stale[@]}"
|
||||
@@ -375,5 +447,11 @@ if [ "${#jobfail[@]}" -gt 0 ]; then
|
||||
echo " Errors older than the window age out on their own."
|
||||
exit 3
|
||||
fi
|
||||
if [ "${#degraded[@]}" -gt 0 ]; then
|
||||
echo; echo "RESULT: DEGRADED — every body is FRESH and no job errored, but ${#degraded[@]} backup(s) logged an"
|
||||
echo " ERROR inside an OK task. An fs-freeze failure means that image is crash-consistent ONLY:"
|
||||
echo " do not count it as application-consistent. Worth a look, not a page."
|
||||
exit 4
|
||||
fi
|
||||
echo; echo "RESULT: all backups fresh"
|
||||
exit 0
|
||||
|
||||
Reference in New Issue
Block a user