diff --git a/services/althing-notify-failure/README.md b/services/althing-notify-failure/README.md new file mode 100644 index 0000000..ee117ba --- /dev/null +++ b/services/althing-notify-failure/README.md @@ -0,0 +1,67 @@ +# althing-notify-failure — failed-START alarms for fleet user units + +A systemd `OnFailure` template that reports any unit entering **failed** state +to the althing **infra-ops** inbox. Installed on nh3-dev; hooks 12 user units. + +```sh +services/althing-notify-failure/install.sh --dry-run # see what it would hook +services/althing-notify-failure/install.sh # idempotent; re-run to pick up new units +``` + +## The failure it exists to catch + +An uptime check cannot see the dangerous one. Measured by svos-dev, 2026-09-22: +a config change on 09-19 made `svos.service` refuse to boot, **the running +process predated the change and kept serving**, and the service sat one restart +from dark for three days. Every uptime probe was green and correct the whole +time — the thing *was* up. Nothing would have fired until the next restart, and +then it would have been an outage rather than a warning. + +The signal that catches it is **failed-START, not down**. A count taken after +that conversation: + +> **13 running user units on nh3-dev. Zero with an `OnFailure` hook.** + +Including `althing-po-herald` — whose silent failure cuts infra-ops's own mail +delivery, a blind spot in the notification path every other alarm depends on. + +## Why this is not noise + +`OnFailure` does not fire on a clean restart or a deliberate stop. svos-dev's +four restarts and three deploys in one day would have produced **zero** alerts. +With `Restart=on-failure`, a unit enters failed state only after exhausting its +start-limit burst, so a crash-loop yields **one** message per episode. + +## Two traps, both hit during the build + +⚠ **`%i`, never `%I`.** `%I` *unescapes* the instance name and systemd escaping +maps `-` to `/`. The acceptance test fired with `%I` and delivered a message for +`onfailure/selftest.service` — a unit that does not exist — with a spool path +that tried to create directories. `althing-po-herald.service` would have arrived +as `althing/po/herald.service`. The referring unit passes `%n` **raw**, so the +literal `%i` is correct. + +⚠ **The notifier must never report itself.** Guarded twice on purpose: the +template carries no `OnFailure` of its own, and the script bails on an instance +name matching itself. A notification loop is the one bug that pages you forever. + +## Delivery, honestly + +The script writes a **durable local record first** +(`~/.local/state/althing-notify-failure/`), then sends. `postbox` has no outbox — +a send that cannot reach the post office is dropped — and this alarm exists +precisely for moments nobody is watching, so the signal must survive the +post office being one of the things that is down. + +If `althing-po-herald` is the unit that failed, the message still **reaches** +the post office (postbox talks to it directly; the herald only delivers inbound +pokes). It will not be pushed into a live session, but it is stored and the next +`postbox read` finds it — a memo to the successor, which is the point. + +## Verified + +Acceptance-tested by firing a real unit that exits 42, twice: once to catch the +`%I` bug, once to confirm the fix. Delivered subject +`[systemd] onfailure-selftest.service FAILED on nh3-dev`, with state, result, +exit status, timestamp and the last 25 journal lines; spool file written. The +test unit was removed afterwards. diff --git a/services/althing-notify-failure/althing-notify-failure b/services/althing-notify-failure/althing-notify-failure new file mode 100755 index 0000000..7f0162a --- /dev/null +++ b/services/althing-notify-failure/althing-notify-failure @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# althing-notify-failure — report a systemd unit entering FAILED state to the +# althing infra-ops inbox. +# +# WHY THIS EXISTS. An uptime check cannot see the dangerous failure on this box. +# svos-dev measured it on 2026-09-22: a config change on 09-19 made svos.service +# refuse to boot, the RUNNING process predated the change and kept serving, and +# the service sat one restart away from dark for three days. Every uptime probe +# was green and correct the whole time -- the thing was up. The signal that +# would have fired is failed-START, not down. +# +# A count after that conversation: 13 running user units on nh3-dev, ZERO with +# an OnFailure hook. Including althing-po-herald, whose silent failure cuts +# infra-ops's own mail delivery -- a blind spot in the notification path that +# every other alarm on this fleet depends on. +# +# WHY OnFailure IS THE RIGHT TRIGGER AND NOT NOISE. It does not fire on a clean +# restart or a deliberate stop. svos-dev's four restarts and three deploys in +# one day would have produced zero alerts. With Restart=on-failure a unit enters +# failed state only after exhausting its start-limit burst, so a crash-loop +# yields ONE message per episode, not one per attempt. +set -uo pipefail + +UNIT="${1:-unknown.service}" +HOST="$(hostname)" +SPOOL="${HOME}/.local/state/althing-notify-failure" +POSTBOX="${POSTBOX:-${HOME}/.local/bin/postbox}" +RECIPIENT="${ALERT_RECIPIENT:-infra-ops}" + +# ⚠ NEVER report on ourselves. Without this, a notifier that fails while +# reporting a failure would be reported by another notifier, and so on. The +# template deliberately carries no OnFailure of its own either -- belt and +# braces, because a notification loop is the one bug that pages you forever. +case "$UNIT" in + althing-notify-failure@*) exit 0 ;; +esac + +state=$(systemctl --user show "$UNIT" -p ActiveState --value 2>/dev/null) +sub=$(systemctl --user show "$UNIT" -p SubState --value 2>/dev/null) +result=$(systemctl --user show "$UNIT" -p Result --value 2>/dev/null) +code=$(systemctl --user show "$UNIT" -p ExecMainStatus --value 2>/dev/null) +desc=$(systemctl --user show "$UNIT" -p Description --value 2>/dev/null) +since=$(systemctl --user show "$UNIT" -p ExecMainExitTimestamp --value 2>/dev/null) + +body=$(cat </dev/null | sed 's/^/ /') + +Triage: + systemctl --user status ${UNIT} + journalctl --user -u ${UNIT} -n 100 --no-pager +EOF +) + +# Durable local record FIRST, so the signal survives a post-office outage. +# postbox has no outbox: a send that cannot reach the post office is dropped, +# and this alarm exists precisely for the cases nobody is watching. +mkdir -p "$SPOOL" +stamp=$(date -u +%Y%m%dT%H%M%SZ) +printf '%s\n' "$body" > "${SPOOL}/${stamp}-${UNIT}.txt" + +if [ -x "$POSTBOX" ]; then + if printf '%s\n' "$body" | ALTHING_HANDLE="${ALTHING_HANDLE:-infra-ops}" \ + "$POSTBOX" send --to "$RECIPIENT" \ + --subject "[systemd] ${UNIT} FAILED on ${HOST}" >/dev/null 2>&1; then + printf 'delivered %s to %s\n' "$UNIT" "$RECIPIENT" + else + # ⚠ If althing-po-herald is the unit that failed, the message still + # REACHES the post office (postbox talks to it directly; the herald only + # delivers inbound pokes) -- it just will not be pushed into a live + # session. It is stored and the next `postbox read` finds it. A memo to + # the successor, which is the whole point. + printf 'POSTBOX DELIVERY FAILED for %s; spooled at %s\n' "$UNIT" "$SPOOL" >&2 + fi +else + printf 'postbox not executable at %s; spooled only\n' "$POSTBOX" >&2 +fi + +# Always succeed. A notifier that exits non-zero is itself a failed unit, and +# this one must never become the thing that needs reporting. +exit 0 diff --git a/services/althing-notify-failure/althing-notify-failure@.service b/services/althing-notify-failure/althing-notify-failure@.service new file mode 100644 index 0000000..eeb2b0c --- /dev/null +++ b/services/althing-notify-failure/althing-notify-failure@.service @@ -0,0 +1,27 @@ +# Template instantiated by another unit's OnFailure=. The instance name is the +# FAILING unit (passed as %n by the referrer), so one template covers every +# service on the box. +# +# ⚠ This unit deliberately has NO OnFailure of its own. A notifier that +# reported its own failures would be reported by another notifier, and a +# notification loop is the one bug that pages you forever. The script carries +# the same guard independently. +[Unit] +Description=Report %i entering failed state to the althing infra-ops inbox + +[Service] +Type=oneshot +Environment=ALTHING_HANDLE=infra-ops +Environment=ALTHING_POST_OFFICE=http://10.100.50.40:8390 +Environment=POSTBOX=%h/.local/bin/postbox +Environment=ALERT_RECIPIENT=infra-ops +# ⚠ %i, NEVER %I. %I UNESCAPES the instance name, and systemd escaping maps +# "-" to "/" -- so a hooked unit called althing-po-herald.service arrives as +# "althing/po/herald.service": wrong name in the subject, and a spool path that +# tries to create directories. Caught by the acceptance test on 2026-09-22, +# which fired with %I and delivered a message naming a unit that does not exist. +# The referring unit passes %n RAW (never escaped), so the literal %i is right. +ExecStart=%h/.local/bin/althing-notify-failure %i +# The notifier must not hang: a stuck postbox call would leave a oneshot +# activating forever and mask the very failure it was invoked for. +TimeoutStartSec=60 diff --git a/services/althing-notify-failure/install.sh b/services/althing-notify-failure/install.sh new file mode 100755 index 0000000..771d6fb --- /dev/null +++ b/services/althing-notify-failure/install.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Install the failure notifier and hook every fleet user unit on this box. +# services/althing-notify-failure/install.sh [--dry-run] +# +# Idempotent. Re-running adds hooks for units that appeared since last time. +set -euo pipefail +cd "$(dirname "$0")" +DRY="${1:-}" +UD="${HOME}/.config/systemd/user" +BIN="${HOME}/.local/bin" + +# dbus.service is systemd's own plumbing, not a fleet service -- excluded. +# althing-notify-failure@ is excluded because a notifier must never report +# itself (see the template's comment). +SKIP_RE='^(dbus|althing-notify-failure@).*' + +run() { if [ "$DRY" = "--dry-run" ]; then echo " would: $*"; else "$@"; fi; } + +echo "== installing the notifier ==" +run install -Dm755 althing-notify-failure "${BIN}/althing-notify-failure" +run install -Dm644 althing-notify-failure@.service "${UD}/althing-notify-failure@.service" + +echo "== hooking fleet user units ==" +mapfile -t UNITS < <(systemctl --user list-units --type=service --state=running --no-pager \ + | awk '{print $1}' | grep '\.service$' | grep -Ev "$SKIP_RE") +for u in "${UNITS[@]}"; do + d="${UD}/${u}.d" + run mkdir -p "$d" + run install -m644 onfailure-drop-in.conf "${d}/10-onfailure-althing.conf" + echo " hooked ${u}" +done + +run systemctl --user daemon-reload +echo "== ${#UNITS[@]} unit(s) hooked ==" diff --git a/services/althing-notify-failure/onfailure-drop-in.conf b/services/althing-notify-failure/onfailure-drop-in.conf new file mode 100644 index 0000000..5951c61 --- /dev/null +++ b/services/althing-notify-failure/onfailure-drop-in.conf @@ -0,0 +1,8 @@ +# Installed as .service.d/10-onfailure-althing.conf on every fleet user +# unit. A drop-in rather than an edit to the unit file so it is visible in +# `systemctl --user cat`, reversible by deleting one file, and survives the +# unit being reinstalled by its own deploy. +# +# %n is the failing unit's full name; the template receives it as %I. +[Unit] +OnFailure=althing-notify-failure@%n.service