#!/usr/bin/env bash
# backup-freshness-alert.sh — daily wrapper around check-backup-freshness.sh.
#
# Runs the check; on any stale/down layer (exit != 0) posts an althing alert so
# the silent-failure class (the 2026-05-06→06-20 ana outage that went unnoticed
# ~6.5 weeks) can't recur. Installed as a systemd user timer on nh3-dev via
# scripts/install-backup-freshness-timer.sh.
#
# ⚠ 2026-09-19 — THIS SCRIPT'S OWN ALARM WIRE WAS CUT FOR THREE WEEKS.
# It called `althing-cli`, which the v3 post-office cutover (2026-08-28)
# DELETED rather than deprecated. The check kept working and kept detecting
# stale backups; it simply could not tell anyone, and the only trace was one
# WARN line inside a unit that was already showing `failed` for the stale
# backups themselves. Two lessons are wired into the code below:
#
#   1. A BROKEN ALARM IS A WORSE FAULT THAN THE THING IT WATCHES, so it gets
#      its own exit code (2) instead of being folded into the check's (1).
#      `systemctl status` can now tell "backups are stale" from "nobody was
#      told".
#   2. THE HEALTHY PATH MUST BE EXERCISABLE. `--test-alert` sends a real
#      message through the real path on demand — a positive control, so the
#      wire is provably intact without waiting for a genuine stale backup.
#
# Alert recipient is infra-hermes: day-to-day checks and routine triage are
# his half of the split (operator ruling 2026-09-19). He escalates to
# infra-ops. ⚠ It must NOT be infra-ops — this runs AS infra-ops, so that
# would mail the alarm to itself, which is the mirror trap named in CLAUDE.md.
#
# Exit codes — mirrored from the check, so `systemctl status` distinguishes
# the three states that matter:
#   0  backups fresh
#   1  backups STALE/DOWN, and the alert was delivered
#   2  the ALERT PATH ITSELF FAILED (investigate this before the backups)
#   3  ERRORED-JOBS — every body fresh, a job errored recently; alert delivered
#
# 3 is deliberately not 1. A nightly job that errored is worth a look; a stale
# backup body is worth a page. Reporting both as "STALE" over a fleet whose
# every backup is current is how an alarm teaches you to ignore it.
set -uo pipefail

REPO=/home/lkraven/development/eshpfi-management
POSTBOX=/home/lkraven/.local/bin/postbox
ALERT_TO=${BACKUP_ALERT_TO:-infra-hermes}

# Both vars are required and postbox has no default address by design — a
# systemd user unit inherits neither, so the unit sets them. Fail loudly here
# rather than discovering it at the moment an alert is needed.
if [ -z "${ALTHING_POST_OFFICE:-}" ] || [ -z "${ALTHING_HANDLE:-}" ]; then
  echo "FATAL: ALTHING_POST_OFFICE and ALTHING_HANDLE must both be set —" >&2
  echo "       postbox has no default address. The alert path is DOWN." >&2
  exit 2
fi
if [ ! -x "$POSTBOX" ]; then
  echo "FATAL: $POSTBOX is missing or not executable. The alert path is DOWN." >&2
  exit 2
fi

send_alert() {   # $1 = subject, body on stdin
  "$POSTBOX" send --to "$ALERT_TO" --subject "$1"
}

# --- positive control: prove the wire without waiting for a real fault ---
if [ "${1:-}" = "--test-alert" ]; then
  printf '%s\n' \
    "Positive control for the fleet backup-freshness alarm — no action needed." \
    "" \
    "This message exists to prove the alert path is intact. If you are reading" \
    "it, backup-freshness-alert.sh can reach you. The path was silently dead" \
    "from the 2026-08-28 althing v3 cutover until 2026-09-19 because this" \
    "script still called the deleted althing-cli." \
    | send_alert "✅ Backup-freshness alarm test ($(date '+%Y-%m-%d %H:%M'))" || {
      echo "FAIL: the alert path is DOWN — that is the finding." >&2; exit 2; }
  echo "OK: alert path verified to $ALERT_TO."
  exit 0
fi

out=$("$REPO/scripts/check-backup-freshness.sh" 2>&1); rc=$?
printf '%s\n' "$out"
[ "$rc" -eq 0 ] && exit 0

# Match the words and the emoji to the finding. A job-errors-only state is not
# an outage and must not dress like one.
if [ "$rc" -eq 3 ]; then
  subject="🟡 Backup jobs errored ($(date '+%Y-%m-%d')) — all bodies fresh"
  lead="A vzdump job errored recently. Every backup body on the fleet is FRESH; nothing is stale."
else
  subject="🔴 Backup freshness ALERT ($(date '+%Y-%m-%d'))"
  lead="Automated daily backup-freshness check found STALE or DOWN backup layer(s) on the PFI fleet."
fi

printf '%s\n' \
  "$lead" \
  "Runbook: docs/runbooks/backups.md (topology, 2-min check, rest-server-ana recovery)." \
  "" \
  "$out" \
  | send_alert "$subject" || {
    echo "FATAL: the check found problems *and* the alert post FAILED — nobody was told." >&2
    echo "       Fix the alert path first; that is the fault that hides the others." >&2
    exit 2; }

exit "$rc"