From c2a70c13dbd04682a9faf5357fa69c71d79f7ad8 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Sun, 13 Sep 2026 23:11:46 -0700 Subject: [PATCH] feat(fv-ml1): daily drift alarm for the seat inventory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires scripts/seat-inventory.py --check to a user systemd timer on nh3-dev (09:15 daily, Persistent=true so a missed run fires on next boot) that posts to althing when the committed document stops matching the live box. Alarms rather than auto-committing. A drift means something changed on the HOST, which deserves a human look -- silently regenerating the doc would erase the evidence of when the change happened and why, which is how the char-rp substitution went unnoticed for three weeks. The alarm includes the changed table rows, not just the fact of divergence, so it does not send the reader hunting. ⚠ The post goes --to infra-ops, which is the fleet ops handle the reading session also runs as. That is the documented exception -- a memo from cron to a future session, the same pattern as the Beszel alerts -- so the message says so in its first line, to stop a future session triaging its own alarm as peer mail and trying to reply to it. SuccessExitStatus=0 1 because a detected drift is a deliberate non-zero exit, not a unit failure. --- scripts/seat-inventory-drift-check.sh | 55 +++++++++++++++++++++++++++ scripts/seat-inventory-drift.service | 9 +++++ scripts/seat-inventory-drift.timer | 10 +++++ 3 files changed, 74 insertions(+) create mode 100755 scripts/seat-inventory-drift-check.sh create mode 100644 scripts/seat-inventory-drift.service create mode 100644 scripts/seat-inventory-drift.timer diff --git a/scripts/seat-inventory-drift-check.sh b/scripts/seat-inventory-drift-check.sh new file mode 100755 index 0000000..60a0591 --- /dev/null +++ b/scripts/seat-inventory-drift-check.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Daily drift alarm for the fv-ml1 seat inventory. +# +# Runs scripts/seat-inventory.py --check against the LIVE box and posts to althing +# when the committed document no longer matches reality. This exists because the +# guarantee "the seat docs are current" cannot rest on anyone remembering to +# regenerate them -- seats change ON THE BOX, not through the repo, so nothing in +# the commit path would ever notice. +# +# Alarms, does not auto-commit: a drift is a signal that something changed on the +# host, and that deserves a human look rather than a silent doc update that would +# erase the evidence of when it happened. +set -uo pipefail +REPO=/home/lkraven/development/eshpfi-management +cd "$REPO" || exit 1 + +OUT=$(timeout 600 python3 scripts/seat-inventory.py --check 2>&1) +RC=$? + +if [ "$RC" -eq 0 ]; then + logger -t seat-inventory "current" + exit 0 +fi + +# Regenerate to a scratch copy so the alarm can name WHAT changed, not just that +# something did -- a drift alert with no diff sends you hunting. +DIFF=$(timeout 600 python3 scripts/seat-inventory.py --out /tmp/seat-inv-live.md >/dev/null 2>&1 \ + && diff -u docs/pfi/fv-ml1-gpu-seat-inventory.md /tmp/seat-inv-live.md \ + | grep -E '^[+-]\|' | grep -vE '^\+\+\+|^---' | head -20) +rm -f /tmp/seat-inv-live.md + +{ + echo "[AUTOMATED ALARM -- not peer correspondence. This is a memo from cron to" + echo " whichever infra-ops session reads it next. Do not reply to it; act on it" + echo " or ignore it. Same pattern as the Beszel alerts.]" + echo + echo "fv-ml1 seat inventory has DRIFTED from the committed document." + echo + echo "check output: $OUT" + echo + if [ -n "$DIFF" ]; then + echo "changed rows:" + echo "$DIFF" + else + echo "(no table rows differ -- change is in lineage, quant, spec-config or aliases)" + fi + echo + echo "A seat changed on the host. Regenerate and commit once you know why:" + echo " cd $REPO && python3 scripts/seat-inventory.py && git diff docs/pfi/" +} | ALTHING_POST_OFFICE=http://10.100.50.40:8390 ALTHING_HANDLE=infra-ops \ + postbox send --to infra-ops --subject "fv-ml1 seat inventory drift" 2>&1 \ + || logger -t seat-inventory "DRIFT detected but althing post FAILED" + +logger -t seat-inventory "DRIFT detected, alarm posted" +exit 1 diff --git a/scripts/seat-inventory-drift.service b/scripts/seat-inventory-drift.service new file mode 100644 index 0000000..929b82d --- /dev/null +++ b/scripts/seat-inventory-drift.service @@ -0,0 +1,9 @@ +[Unit] +Description=fv-ml1 seat inventory drift check (alarms to althing) +After=network-online.target + +[Service] +Type=oneshot +ExecStart=/home/lkraven/development/eshpfi-management/scripts/seat-inventory-drift-check.sh +# a drift is a non-zero exit by design; don't let systemd log it as a failure +SuccessExitStatus=0 1 diff --git a/scripts/seat-inventory-drift.timer b/scripts/seat-inventory-drift.timer new file mode 100644 index 0000000..892d7b3 --- /dev/null +++ b/scripts/seat-inventory-drift.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Daily fv-ml1 seat inventory drift check + +[Timer] +OnCalendar=*-*-* 09:15:00 +Persistent=true +RandomizedDelaySec=300 + +[Install] +WantedBy=timers.target