#!/bin/bash
# Daily drift alarm for the fv-ml1 seat inventory.
#
# Runs scripts/seat-inventory.py --check against the LIVE box and posts to althing
# when the committed document no longer matches reality. This exists because the
# guarantee "the seat docs are current" cannot rest on anyone remembering to
# regenerate them -- seats change ON THE BOX, not through the repo, so nothing in
# the commit path would ever notice.
#
# Alarms, does not auto-commit: a drift is a signal that something changed on the
# host, and that deserves a human look rather than a silent doc update that would
# erase the evidence of when it happened.
set -uo pipefail
REPO=/home/lkraven/development/eshpfi-management
cd "$REPO" || exit 1

OUT=$(timeout 600 python3 scripts/seat-inventory.py --check 2>&1)
RC=$?

if [ "$RC" -eq 0 ]; then
  logger -t seat-inventory "current"
  exit 0
fi

# Regenerate to a scratch copy so the alarm can name WHAT changed, not just that
# something did -- a drift alert with no diff sends you hunting.
DIFF=$(timeout 600 python3 scripts/seat-inventory.py --out /tmp/seat-inv-live.md >/dev/null 2>&1 \
       && diff -u docs/pfi/fv-ml1-gpu-seat-inventory.md /tmp/seat-inv-live.md \
          | grep -E '^[+-]\|' | grep -vE '^\+\+\+|^---' | head -20)
rm -f /tmp/seat-inv-live.md

{
  echo "[AUTOMATED ALARM -- not peer correspondence. This is a memo from cron to"
  echo " whichever infra-ops session reads it next. Do not reply to it; act on it"
  echo " or ignore it. Same pattern as the Beszel alerts.]"
  echo
  echo "fv-ml1 seat inventory has DRIFTED from the committed document."
  echo
  echo "check output: $OUT"
  echo
  if [ -n "$DIFF" ]; then
    echo "changed rows:"
    echo "$DIFF"
  else
    echo "(no table rows differ -- change is in lineage, quant, spec-config or aliases)"
  fi
  echo
  echo "A seat changed on the host. Regenerate and commit once you know why:"
  echo "  cd $REPO && python3 scripts/seat-inventory.py && git diff docs/pfi/"
} | ALTHING_POST_OFFICE=http://10.100.50.40:8390 ALTHING_HANDLE=infra-ops \
    postbox send --to infra-ops --subject "fv-ml1 seat inventory drift" 2>&1 \
  || logger -t seat-inventory "DRIFT detected but althing post FAILED"

logger -t seat-inventory "DRIFT detected, alarm posted"
exit 1