vh
1210b4a0d5
Front door: request bodies capped at 1 MiB, headers within 5 s, body within 15 s, no response deadlines (SSE stays open). Validated live: 413 over the cap, stalled body cut at 15 s, signed ping and an SSE stream unaffected. svos-front-inventory also serves the previous release's hashed assets credential-free from that release's static dir, hash-checked. Backup check: reads the logs of vzdump tasks PVE records as OK and reports ERROR lines; an fs-freeze/thaw error is an application-consistency failure, new verdict DEGRADED (exit 4), wired into the alert wrapper. nh3-pve-2 added to the node and namespace lists (its job results were never read). The svos drill task is acknowledged by UPID; BACKUP_IGNORE_ACKS=1 counts it (positive control: exit 4).
107 lines
5.1 KiB
Bash
Executable File
107 lines
5.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# backup-freshness-alert.sh — daily wrapper around check-backup-freshness.sh.
|
|
#
|
|
# Runs the check; on any stale/down layer (exit != 0) posts an althing alert so
|
|
# the silent-failure class (the 2026-05-06→06-20 ana outage that went unnoticed
|
|
# ~6.5 weeks) can't recur. Installed as a systemd user timer on nh3-dev via
|
|
# scripts/install-backup-freshness-timer.sh.
|
|
#
|
|
# ⚠ 2026-09-19 — THIS SCRIPT'S OWN ALARM WIRE WAS CUT FOR THREE WEEKS.
|
|
# It called `althing-cli`, which the v3 post-office cutover (2026-08-28)
|
|
# DELETED rather than deprecated. The check kept working and kept detecting
|
|
# stale backups; it simply could not tell anyone, and the only trace was one
|
|
# WARN line inside a unit that was already showing `failed` for the stale
|
|
# backups themselves. Two lessons are wired into the code below:
|
|
#
|
|
# 1. A BROKEN ALARM IS A WORSE FAULT THAN THE THING IT WATCHES, so it gets
|
|
# its own exit code (2) instead of being folded into the check's (1).
|
|
# `systemctl status` can now tell "backups are stale" from "nobody was
|
|
# told".
|
|
# 2. THE HEALTHY PATH MUST BE EXERCISABLE. `--test-alert` sends a real
|
|
# message through the real path on demand — a positive control, so the
|
|
# wire is provably intact without waiting for a genuine stale backup.
|
|
#
|
|
# Alert recipient is infra-hermes: day-to-day checks and routine triage are
|
|
# his half of the split (operator ruling 2026-09-19). He escalates to
|
|
# infra-ops. ⚠ It must NOT be infra-ops — this runs AS infra-ops, so that
|
|
# would mail the alarm to itself, which is the mirror trap named in CLAUDE.md.
|
|
#
|
|
# Exit codes — mirrored from the check, so `systemctl status` distinguishes
|
|
# the three states that matter:
|
|
# 0 backups fresh
|
|
# 1 backups STALE/DOWN, and the alert was delivered
|
|
# 2 the ALERT PATH ITSELF FAILED (investigate this before the backups)
|
|
# 3 ERRORED-JOBS — every body fresh, a job errored recently; alert delivered
|
|
# 4 DEGRADED — no job errored, but an OK task logged an ERROR (fs-freeze refused =
|
|
# crash-consistent image only); alert delivered
|
|
#
|
|
# 3 is deliberately not 1. A nightly job that errored is worth a look; a stale
|
|
# backup body is worth a page. Reporting both as "STALE" over a fleet whose
|
|
# every backup is current is how an alarm teaches you to ignore it.
|
|
set -uo pipefail
|
|
|
|
REPO=/home/lkraven/development/eshpfi-management
|
|
POSTBOX=/home/lkraven/.local/bin/postbox
|
|
ALERT_TO=${BACKUP_ALERT_TO:-infra-hermes}
|
|
|
|
# Both vars are required and postbox has no default address by design — a
|
|
# systemd user unit inherits neither, so the unit sets them. Fail loudly here
|
|
# rather than discovering it at the moment an alert is needed.
|
|
if [ -z "${ALTHING_POST_OFFICE:-}" ] || [ -z "${ALTHING_HANDLE:-}" ]; then
|
|
echo "FATAL: ALTHING_POST_OFFICE and ALTHING_HANDLE must both be set —" >&2
|
|
echo " postbox has no default address. The alert path is DOWN." >&2
|
|
exit 2
|
|
fi
|
|
if [ ! -x "$POSTBOX" ]; then
|
|
echo "FATAL: $POSTBOX is missing or not executable. The alert path is DOWN." >&2
|
|
exit 2
|
|
fi
|
|
|
|
send_alert() { # $1 = subject, body on stdin
|
|
"$POSTBOX" send --to "$ALERT_TO" --subject "$1"
|
|
}
|
|
|
|
# --- positive control: prove the wire without waiting for a real fault ---
|
|
if [ "${1:-}" = "--test-alert" ]; then
|
|
printf '%s\n' \
|
|
"Positive control for the fleet backup-freshness alarm — no action needed." \
|
|
"" \
|
|
"This message exists to prove the alert path is intact. If you are reading" \
|
|
"it, backup-freshness-alert.sh can reach you. The path was silently dead" \
|
|
"from the 2026-08-28 althing v3 cutover until 2026-09-19 because this" \
|
|
"script still called the deleted althing-cli." \
|
|
| send_alert "✅ Backup-freshness alarm test ($(date '+%Y-%m-%d %H:%M'))" || {
|
|
echo "FAIL: the alert path is DOWN — that is the finding." >&2; exit 2; }
|
|
echo "OK: alert path verified to $ALERT_TO."
|
|
exit 0
|
|
fi
|
|
|
|
out=$("$REPO/scripts/check-backup-freshness.sh" 2>&1); rc=$?
|
|
printf '%s\n' "$out"
|
|
[ "$rc" -eq 0 ] && exit 0
|
|
|
|
# Match the words and the emoji to the finding. A job-errors-only state is not
|
|
# an outage and must not dress like one.
|
|
if [ "$rc" -eq 3 ]; then
|
|
subject="🟡 Backup jobs errored ($(date '+%Y-%m-%d')) — all bodies fresh"
|
|
lead="A vzdump job errored recently. Every backup body on the fleet is FRESH; nothing is stale."
|
|
elif [ "$rc" -eq 4 ]; then
|
|
subject="🟠 Backup DEGRADED ($(date '+%Y-%m-%d')) — an OK job logged an ERROR"
|
|
lead="Every backup body is FRESH and no job failed, but a backup logged an ERROR inside a task PVE calls OK. An fs-freeze failure there means that image is crash-consistent only, not application-consistent."
|
|
else
|
|
subject="🔴 Backup freshness ALERT ($(date '+%Y-%m-%d'))"
|
|
lead="Automated daily backup-freshness check found STALE or DOWN backup layer(s) on the PFI fleet."
|
|
fi
|
|
|
|
printf '%s\n' \
|
|
"$lead" \
|
|
"Runbook: docs/runbooks/backups.md (topology, 2-min check, rest-server-ana recovery)." \
|
|
"" \
|
|
"$out" \
|
|
| send_alert "$subject" || {
|
|
echo "FATAL: the check found problems *and* the alert post FAILED — nobody was told." >&2
|
|
echo " Fix the alert path first; that is the fault that hides the others." >&2
|
|
exit 2; }
|
|
|
|
exit "$rc"
|