vh
c2a70c13db
Wires scripts/seat-inventory.py --check to a user systemd timer on nh3-dev (09:15 daily, Persistent=true so a missed run fires on next boot) that posts to althing when the committed document stops matching the live box. Alarms rather than auto-committing. A drift means something changed on the HOST, which deserves a human look -- silently regenerating the doc would erase the evidence of when the change happened and why, which is how the char-rp substitution went unnoticed for three weeks. The alarm includes the changed table rows, not just the fact of divergence, so it does not send the reader hunting. ⚠ The post goes --to infra-ops, which is the fleet ops handle the reading session also runs as. That is the documented exception -- a memo from cron to a future session, the same pattern as the Beszel alerts -- so the message says so in its first line, to stop a future session triaging its own alarm as peer mail and trying to reply to it. SuccessExitStatus=0 1 because a detected drift is a deliberate non-zero exit, not a unit failure.
56 lines
2.2 KiB
Bash
Executable File
56 lines
2.2 KiB
Bash
Executable File
#!/bin/bash
|
|
# Daily drift alarm for the fv-ml1 seat inventory.
|
|
#
|
|
# Runs scripts/seat-inventory.py --check against the LIVE box and posts to althing
|
|
# when the committed document no longer matches reality. This exists because the
|
|
# guarantee "the seat docs are current" cannot rest on anyone remembering to
|
|
# regenerate them -- seats change ON THE BOX, not through the repo, so nothing in
|
|
# the commit path would ever notice.
|
|
#
|
|
# Alarms, does not auto-commit: a drift is a signal that something changed on the
|
|
# host, and that deserves a human look rather than a silent doc update that would
|
|
# erase the evidence of when it happened.
|
|
set -uo pipefail
|
|
REPO=/home/lkraven/development/eshpfi-management
|
|
cd "$REPO" || exit 1
|
|
|
|
OUT=$(timeout 600 python3 scripts/seat-inventory.py --check 2>&1)
|
|
RC=$?
|
|
|
|
if [ "$RC" -eq 0 ]; then
|
|
logger -t seat-inventory "current"
|
|
exit 0
|
|
fi
|
|
|
|
# Regenerate to a scratch copy so the alarm can name WHAT changed, not just that
|
|
# something did -- a drift alert with no diff sends you hunting.
|
|
DIFF=$(timeout 600 python3 scripts/seat-inventory.py --out /tmp/seat-inv-live.md >/dev/null 2>&1 \
|
|
&& diff -u docs/pfi/fv-ml1-gpu-seat-inventory.md /tmp/seat-inv-live.md \
|
|
| grep -E '^[+-]\|' | grep -vE '^\+\+\+|^---' | head -20)
|
|
rm -f /tmp/seat-inv-live.md
|
|
|
|
{
|
|
echo "[AUTOMATED ALARM -- not peer correspondence. This is a memo from cron to"
|
|
echo " whichever infra-ops session reads it next. Do not reply to it; act on it"
|
|
echo " or ignore it. Same pattern as the Beszel alerts.]"
|
|
echo
|
|
echo "fv-ml1 seat inventory has DRIFTED from the committed document."
|
|
echo
|
|
echo "check output: $OUT"
|
|
echo
|
|
if [ -n "$DIFF" ]; then
|
|
echo "changed rows:"
|
|
echo "$DIFF"
|
|
else
|
|
echo "(no table rows differ -- change is in lineage, quant, spec-config or aliases)"
|
|
fi
|
|
echo
|
|
echo "A seat changed on the host. Regenerate and commit once you know why:"
|
|
echo " cd $REPO && python3 scripts/seat-inventory.py && git diff docs/pfi/"
|
|
} | ALTHING_POST_OFFICE=http://10.100.50.40:8390 ALTHING_HANDLE=infra-ops \
|
|
postbox send --to infra-ops --subject "fv-ml1 seat inventory drift" 2>&1 \
|
|
|| logger -t seat-inventory "DRIFT detected but althing post FAILED"
|
|
|
|
logger -t seat-inventory "DRIFT detected, alarm posted"
|
|
exit 1
|