diff --git a/scripts/seat-inventory-drift-check.sh b/scripts/seat-inventory-drift-check.sh new file mode 100755 index 0000000..60a0591 --- /dev/null +++ b/scripts/seat-inventory-drift-check.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Daily drift alarm for the fv-ml1 seat inventory. +# +# Runs scripts/seat-inventory.py --check against the LIVE box and posts to althing +# when the committed document no longer matches reality. This exists because the +# guarantee "the seat docs are current" cannot rest on anyone remembering to +# regenerate them -- seats change ON THE BOX, not through the repo, so nothing in +# the commit path would ever notice. +# +# Alarms, does not auto-commit: a drift is a signal that something changed on the +# host, and that deserves a human look rather than a silent doc update that would +# erase the evidence of when it happened. +set -uo pipefail +REPO=/home/lkraven/development/eshpfi-management +cd "$REPO" || exit 1 + +OUT=$(timeout 600 python3 scripts/seat-inventory.py --check 2>&1) +RC=$? + +if [ "$RC" -eq 0 ]; then + logger -t seat-inventory "current" + exit 0 +fi + +# Regenerate to a scratch copy so the alarm can name WHAT changed, not just that +# something did -- a drift alert with no diff sends you hunting. +DIFF=$(timeout 600 python3 scripts/seat-inventory.py --out /tmp/seat-inv-live.md >/dev/null 2>&1 \ + && diff -u docs/pfi/fv-ml1-gpu-seat-inventory.md /tmp/seat-inv-live.md \ + | grep -E '^[+-]\|' | grep -vE '^\+\+\+|^---' | head -20) +rm -f /tmp/seat-inv-live.md + +{ + echo "[AUTOMATED ALARM -- not peer correspondence. This is a memo from cron to" + echo " whichever infra-ops session reads it next. Do not reply to it; act on it" + echo " or ignore it. Same pattern as the Beszel alerts.]" + echo + echo "fv-ml1 seat inventory has DRIFTED from the committed document." + echo + echo "check output: $OUT" + echo + if [ -n "$DIFF" ]; then + echo "changed rows:" + echo "$DIFF" + else + echo "(no table rows differ -- change is in lineage, quant, spec-config or aliases)" + fi + echo + echo "A seat changed on the host. Regenerate and commit once you know why:" + echo " cd $REPO && python3 scripts/seat-inventory.py && git diff docs/pfi/" +} | ALTHING_POST_OFFICE=http://10.100.50.40:8390 ALTHING_HANDLE=infra-ops \ + postbox send --to infra-ops --subject "fv-ml1 seat inventory drift" 2>&1 \ + || logger -t seat-inventory "DRIFT detected but althing post FAILED" + +logger -t seat-inventory "DRIFT detected, alarm posted" +exit 1 diff --git a/scripts/seat-inventory-drift.service b/scripts/seat-inventory-drift.service new file mode 100644 index 0000000..929b82d --- /dev/null +++ b/scripts/seat-inventory-drift.service @@ -0,0 +1,9 @@ +[Unit] +Description=fv-ml1 seat inventory drift check (alarms to althing) +After=network-online.target + +[Service] +Type=oneshot +ExecStart=/home/lkraven/development/eshpfi-management/scripts/seat-inventory-drift-check.sh +# a drift is a non-zero exit by design; don't let systemd log it as a failure +SuccessExitStatus=0 1 diff --git a/scripts/seat-inventory-drift.timer b/scripts/seat-inventory-drift.timer new file mode 100644 index 0000000..892d7b3 --- /dev/null +++ b/scripts/seat-inventory-drift.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Daily fv-ml1 seat inventory drift check + +[Timer] +OnCalendar=*-*-* 09:15:00 +Persistent=true +RandomizedDelaySec=300 + +[Install] +WantedBy=timers.target