#!/usr/bin/env bash
# albok-check — an operator's manual check of the live Albok, in one screen:
#   1. the service: /health, the version it serves against the version in this repo
#   2. the wings this token can read, and their states
#   3. a search with a positive control (a query that MUST hit; an empty answer is a fault, not a null result)
#   4. Nemi: the newest walk report (age, version, outcomes, a re-update storm), the timer, a walk in progress
#   5. the container on nh3-docker, when ssh reaches it
# Environment (all optional): ALBOK_SERVICE, ALBOK_TOKEN (else the token in NEMI_CONFIG, else the vault),
# NEMI_CONFIG, ALBOK_CHECK_QUERY (the positive control), ALBOK_CHECK_STORM (updated-count that counts as a storm).
# Exit 0 when nothing failed; 1 when any line says FAIL. WARN lines never fail the run.
set -u
ALBOK_SERVICE="${ALBOK_SERVICE:-http://albok.nh3.internal:8392}"
NEMI_CONFIG="${NEMI_CONFIG:-$HOME/.config/albok/nemi.yaml}"
QUERY="${ALBOK_CHECK_QUERY:-recent decisions}"
STORM="${ALBOK_CHECK_STORM:-1000}"
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
FAILS=0
ok()   { printf '  ok    %s\n' "$*"; }
warn() { printf '  WARN  %s\n' "$*"; }
fail() { printf '  FAIL  %s\n' "$*"; FAILS=$((FAILS + 1)); }
albok() { if type -P albok >/dev/null 2>&1; then command albok "$@"; else uv --directory "$REPO" run -q albok "$@"; fi; }
nemi()  { if type -P nemi >/dev/null 2>&1; then command nemi "$@"; else uv --directory "$REPO" run -q nemi "$@"; fi; }

echo "albok-check $(date '+%Y-%m-%d %H%M %Z') — $ALBOK_SERVICE"

echo "service"
HDR="$(mktemp)"; BODY="$(mktemp)"; trap 'rm -f "$HDR" "$BODY"' EXIT
CODE="$(curl -s -m 10 -D "$HDR" -o "$BODY" -w '%{http_code}' "$ALBOK_SERVICE/health" 2>/dev/null || echo 000)"
if [ "$CODE" != "200" ]; then
  fail "/health answered HTTP $CODE (000 = unreachable: an OUTAGE, not an empty answer)"
else
  SERVED="$(awk 'tolower($1)=="x-albok-service:"{gsub(/\r/,"",$2);print $2}' "$HDR")"
  REPO_VERSION="$(python3 -c 'import tomllib,sys;print(tomllib.load(open(sys.argv[1],"rb"))["project"]["version"])' "$REPO/packages/albok-service/pyproject.toml" 2>/dev/null)"
  python3 - "$BODY" "$SERVED" "$REPO_VERSION" <<'PY'
import json, sys
body, served, repo = json.load(open(sys.argv[1])), sys.argv[2], sys.argv[3]
def ok(m): print(f"  ok    {m}")
def warn(m): print(f"  WARN  {m}")
def fail(m): print(f"  FAIL  {m}"); sys.exit(1)
(ok if body.get("status") == "ok" else fail)(f"/health status={body.get('status')} wings={body.get('wings')}")
(ok if served == repo else warn)(f"serves {served or '?'}; this repo is {repo or '?'}" + ("" if served == repo else "  (redeploy pending, or you are behind origin)"))
dead = [k for k, v in (body.get("canary") or {}).items() if v != "alive"]
(ok if not dead else fail)("canary: all arms alive" if not dead else f"canary arms not alive: {dead}")
emb = body.get("embedding") or {}
(ok if emb.get("reachable") else fail)(f"embedder {emb.get('model')} dim={emb.get('dim')} reachable={emb.get('reachable')}")
bad = {k: body.get(k) for k in ("drift", "head_regressed", "needs_reindex") if body.get(k)}
(ok if not bad else fail)("drift 0, head_regressed 0, needs_reindex 0" if not bad else f"wing trouble: {bad}")
PY
  [ $? -ne 0 ] && FAILS=$((FAILS + 1))
fi

echo "token"
TOKEN="${ALBOK_TOKEN:-}"
SRC="ALBOK_TOKEN"
if [ -z "$TOKEN" ] && [ -r "$NEMI_CONFIG" ]; then
  TOKEN="$(python3 -c 'import sys,yaml;print(yaml.safe_load(open(sys.argv[1]))["service"]["token"])' "$NEMI_CONFIG" 2>/dev/null)"; SRC="$NEMI_CONFIG"
fi
if [ -z "$TOKEN" ] && command -v secret >/dev/null 2>&1; then
  TOKEN="$(secret get albok/token-nemi 2>/dev/null)"; SRC="vault albok/token-nemi"
fi
if [ -z "$TOKEN" ]; then
  fail "no token (set ALBOK_TOKEN, or point NEMI_CONFIG at nemi.yaml); skipping the wing and search checks"
else
  ok "token from $SRC (Nemi's write token: read checks only here)"
  export ALBOK_SERVICE ALBOK_TOKEN="$TOKEN"

  echo "wings"
  STATUS="$(albok status 2>&1)"; RC=$?
  if [ $RC -ne 0 ]; then fail "albok status: $STATUS"; else
    while read -r wing state; do
      [ -z "$wing" ] && continue
      case "$state" in ready) ok "$wing $state" ;; locked) warn "$wing $state (encrypted wing not mounted)" ;; *) fail "$wing $state" ;; esac
    done <<< "$STATUS"
  fi

  echo "search (positive control: \"$QUERY\")"
  HITS="$(albok search --json --top-k 3 "$QUERY" 2>&1)"; RC=$?
  if [ $RC -ne 0 ]; then fail "albok search: $HITS"; else
    python3 - "$HITS" <<'PY'
import json, sys
r = json.loads(sys.argv[1]).get("results") or []
if not r: print("  FAIL  zero hits for the positive control: the index is empty or search is broken"); sys.exit(1)
print(f"  ok    {len(r)} hits; top: {r[0].get('wing')} {r[0].get('title')!r} score={r[0].get('score'):.3f} (never threshold on score: the null floor is ~0.031)")
PY
    [ $? -ne 0 ] && FAILS=$((FAILS + 1))
  fi
fi

echo "nemi"
STATE_DIR="$(python3 -c 'import sys,yaml;print(yaml.safe_load(open(sys.argv[1]))["state_dir"])' "$NEMI_CONFIG" 2>/dev/null || echo "$HOME/.local/state/albok-nemi")"
python3 - "$STATE_DIR" "$STORM" <<'PY'
import glob, json, os, sys
from datetime import datetime, timezone
state, storm = sys.argv[1], int(sys.argv[2])
reports = sorted(glob.glob(os.path.join(state, "reports", "*.json")), key=os.path.getmtime)
if not reports: print(f"  FAIL  no walk report under {state}/reports"); sys.exit(1)
d = json.load(open(reports[-1]))
age = datetime.now(timezone.utc) - datetime.fromisoformat(d["run_at"])
mins = int(age.total_seconds() // 60)
print(f"  {'ok   ' if mins <= 120 else 'WARN '} newest walk {d['run_at']} ({mins} min ago; hourly timer) nemi {d.get('nemi_version')} on {d.get('host')}")
rc = 0
for wing, w in (d.get("wings") or {}).items():
    o = w.get("outcomes") or {}
    line = " ".join(f"{k} {o.get(k, 0)}" for k in ("created", "updated", "unchanged", "tombstoned", "revived"))
    if w.get("error"):
        print(f"  FAIL  {wing}: {line}; the walk ended in an error: {w['error']}"); rc = 1
    elif o.get("updated", 0) >= storm:
        print(f"  WARN  {wing}: {line}  <- a re-update storm (>= {storm}): every object rewritten; a Nemi release before service 0.1.3 did this")
    else:
        print(f"  ok    {wing}: {line}")
for s in d.get("skipped") or []: print(f"  WARN  skipped: {s}")
sys.exit(rc)
PY
[ $? -ne 0 ] && FAILS=$((FAILS + 1))
if command -v systemctl >/dev/null 2>&1 && systemctl --user list-unit-files albok-nemi.timer >/dev/null 2>&1; then
  T="$(systemctl --user is-active albok-nemi.timer 2>/dev/null)"; S="$(systemctl --user is-active albok-nemi.service 2>/dev/null)"
  case "$T" in active) ok "timer active; $(systemctl --user show albok-nemi.timer -p LastTriggerUSec -p NextElapseUSecRealtime 2>/dev/null | sed -e 's/LastTriggerUSec=/last fired /' -e 's/NextElapseUSecRealtime=/next /' | paste -sd ';' | sed 's/;/; /')" ;; *) fail "albok-nemi.timer is $T" ;; esac
  case "$S" in activating|active) ok "a walk is running now (albok-nemi.service $S)" ;; failed) fail "albok-nemi.service failed: journalctl --user -u albok-nemi.service -n 30" ;; *) ok "no walk running (service $S)" ;; esac
else
  warn "no albok-nemi.timer on this box (it lives on nh3-dev as a user unit)"
fi
if [ -e "$STATE_DIR/run.lock" ] && command -v flock >/dev/null 2>&1; then
  flock -n "$STATE_DIR/run.lock" true 2>/dev/null && ok "run.lock free" || ok "run.lock held (a walk holds it; a second walk would refuse to start)"
fi

echo "container (nh3-docker)"
if PS="$(ssh -o ConnectTimeout=3 -o BatchMode=yes nh3-docker 'docker ps --filter name=albok-service --format "{{.Image}} {{.Status}}"' 2>/dev/null)" && [ -n "$PS" ]; then
  case "$PS" in *healthy*) ok "$PS" ;; *) fail "$PS" ;; esac
else
  warn "ssh nh3-docker not reachable from here, or no albok-service container; skipped"
fi

echo
if [ "$FAILS" -eq 0 ]; then echo "albok-check: nothing failed"; else echo "albok-check: $FAILS FAIL"; exit 1; fi
