#!/usr/bin/env bash
# albok-check — an operator's manual check of the live Albok, in one screen:
#   1. the service: /health, the version it serves against the version in this repo
#   2. the wings this token can read, and their states
#   3. a search with a positive control (a query that MUST hit; an empty answer is a fault, not a null result)
#   4. Nemi: the newest walk report (age, version, outcomes, a re-update storm), the timer, a walk in progress
#   5. the container on its host (nh3-albok since 2026-10-04), when ssh reaches it
# Environment (all optional): ALBOK_SERVICE, ALBOK_TOKEN (else the token in NEMI_CONFIG, else the vault),
# NEMI_CONFIG, ALBOK_CHECK_QUERY (the positive control), ALBOK_CHECK_STORM (updated-count that counts as a storm),
# ALBOK_HOST (the ssh target for the container check; default infra-ops@nh3-albok.nh3.internal).
# Exit 0 when nothing failed; 1 when any line says FAIL. WARN lines never fail the run.
set -u
ALBOK_SERVICE="${ALBOK_SERVICE:-http://albok.nh3.internal:8392}"
NEMI_CONFIG="${NEMI_CONFIG:-$HOME/.config/albok/nemi.yaml}"
QUERY="${ALBOK_CHECK_QUERY:-recent decisions}"
STORM="${ALBOK_CHECK_STORM:-1000}"
ALBOK_HOST="${ALBOK_HOST:-infra-ops@nh3-albok.nh3.internal}"
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
FAILS=0
ok()   { printf '  ok    %s\n' "$*"; }
warn() { printf '  WARN  %s\n' "$*"; }
fail() { printf '  FAIL  %s\n' "$*"; FAILS=$((FAILS + 1)); }
albok() { if type -P albok >/dev/null 2>&1; then command albok "$@"; else uv --directory "$REPO" run -q albok "$@"; fi; }
nemi()  { if type -P nemi >/dev/null 2>&1; then command nemi "$@"; else uv --directory "$REPO" run -q nemi "$@"; fi; }

echo "albok-check $(date '+%Y-%m-%d %H%M %Z') — $ALBOK_SERVICE"

echo "service"
HDR="$(mktemp)"; BODY="$(mktemp)"; trap 'rm -f "$HDR" "$BODY"' EXIT
CODE="$(curl -s -m 10 -D "$HDR" -o "$BODY" -w '%{http_code}' "$ALBOK_SERVICE/health" 2>/dev/null || echo 000)"
if [ "$CODE" != "200" ]; then
  fail "/health answered HTTP $CODE (000 = unreachable: an OUTAGE, not an empty answer)"
else
  SERVED="$(awk 'tolower($1)=="x-albok-service:"{gsub(/\r/,"",$2);print $2}' "$HDR")"
  REPO_VERSION="$(python3 -c 'import tomllib,sys;print(tomllib.load(open(sys.argv[1],"rb"))["project"]["version"])' "$REPO/packages/albok-service/pyproject.toml" 2>/dev/null)"
  python3 - "$BODY" "$SERVED" "$REPO_VERSION" <<'PY'
import json, sys
body, served, repo = json.load(open(sys.argv[1])), sys.argv[2], sys.argv[3]
def ok(m): print(f"  ok    {m}")
def warn(m): print(f"  WARN  {m}")
def fail(m): print(f"  FAIL  {m}"); sys.exit(1)
leaks = body.get("leaks")   # service >= 0.1.13: live objects carrying a credential pattern (ids on /health/leaks, behind a token)
if "leaks" in body and leaks is None:
    warn("leaks: unknown — the service could not count them (health is degraded; see its log)")
elif leaks is not None:
    (ok if not leaks else warn)(f"leaks: {leaks}" + ("" if not leaks else "  <- objects carrying a credential pattern; rotate the credential (ids below where this token can see them)"))
(ok if body.get("status") == "ok" else fail)(f"/health status={body.get('status')} wings={body.get('wings')}")
(ok if served == repo else warn)(f"serves {served or '?'}; this repo is {repo or '?'}" + ("" if served == repo else "  (redeploy pending, or you are behind origin)"))
dead = [k for k, v in (body.get("canary") or {}).items() if v != "alive"]
(ok if not dead else fail)("canary: all arms alive" if not dead else f"canary arms not alive: {dead}")
emb = body.get("embedding") or {}
(ok if emb.get("reachable") else fail)(f"embedder {emb.get('model')} dim={emb.get('dim')} reachable={emb.get('reachable')}")
w = emb.get("write")
if w is not None:   # the write path's own embedder (service >= 0.1.9): False refuses writes (health is degraded), None is unverified
    agrees = w.get("agrees")
    (ok if agrees is True else fail if agrees is False else warn)(
        f"write embedder {w.get('model')}: " + {True: "same vector space as the read embedder", False: "a DIFFERENT vector space: writes refused",
                                                None: "not yet verified (no write since start, or it was unreachable at the last one)"}[agrees])
bad = {k: body.get(k) for k in ("drift", "head_regressed", "needs_reindex", "hidden_map_stale") if body.get(k)}
(ok if not bad else fail)("drift 0, head_regressed 0, needs_reindex 0, hidden_map_stale 0" if not bad else f"wing trouble: {bad}")
PY
  [ $? -ne 0 ] && FAILS=$((FAILS + 1))
fi

echo "token"
TOKEN="${ALBOK_TOKEN:-}"
SRC="ALBOK_TOKEN"
if [ -z "$TOKEN" ] && [ -r "$NEMI_CONFIG" ]; then
  TOKEN="$(python3 -c 'import sys,yaml;print(yaml.safe_load(open(sys.argv[1]))["service"]["token"])' "$NEMI_CONFIG" 2>/dev/null)"; SRC="$NEMI_CONFIG"
fi
if [ -z "$TOKEN" ] && command -v secret >/dev/null 2>&1; then
  TOKEN="$(secret get albok/token-nemi 2>/dev/null)"; SRC="vault albok/token-nemi"
fi
if [ -z "$TOKEN" ]; then
  fail "no token (set ALBOK_TOKEN, or point NEMI_CONFIG at nemi.yaml); skipping the wing and search checks"
else
  ok "token from $SRC (Nemi's write token: read checks only here)"
  export ALBOK_SERVICE ALBOK_TOKEN="$TOKEN"

  echo "wings"
  STATUS="$(albok status 2>&1)"; RC=$?
  if [ $RC -ne 0 ]; then fail "albok status: $STATUS"; else
    while read -r wing state; do
      [ -z "$wing" ] && continue
      case "$state" in ready) ok "$wing $state" ;; locked) warn "$wing $state (encrypted wing not mounted)" ;; *) fail "$wing $state" ;; esac
    done <<< "$STATUS"
  fi
  # the live generation per wing (service >= 0.1.11; a reindex builds the next one beside it and swaps). The HTTP status
  # is checked: a JSON error body (401, 403, 500) is a FAIL, never an "ok" map; only a 404 means an older service
  GCODE="$(curl -s -m 10 -H "Authorization: Bearer $TOKEN" -o "$BODY" -w '%{http_code}' "$ALBOK_SERVICE/health/generations" 2>/dev/null || echo 000)"
  if [ "$GCODE" = "200" ]; then
    GENS="$(python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); assert all(isinstance(v, int) for v in d.values()), d; print(", ".join(f"{k} g{v}" for k, v in sorted(d.items())) or "none")' "$BODY" 2>/dev/null)"
    if [ -n "$GENS" ]; then ok "generations: $GENS"; else fail "/health/generations answered 200 with a body that is not a generations map: $(head -c 200 "$BODY")"; fi
  elif [ "$GCODE" = "404" ]; then
    warn "no /health/generations (service < 0.1.11)"
  else
    fail "/health/generations answered HTTP $GCODE: $(head -c 200 "$BODY" | tr -d '\n')"
  fi

  # the leak ids (service >= 0.1.13), where this token holds `secret` on a wing; a plain reader sees the count above only
  LCODE="$(curl -s -m 10 -H "Authorization: Bearer $TOKEN" -o "$BODY" -w '%{http_code}' "$ALBOK_SERVICE/health/leaks" 2>/dev/null || echo 000)"
  if [ "$LCODE" = "200" ]; then
    python3 - "$BODY" <<'PY'
import json, sys
try:
    listing = json.load(open(sys.argv[1]))
    assert isinstance(listing, dict)
except Exception:
    print("  FAIL  /health/leaks answered 200 with a body that is not a leak listing"); sys.exit(1)
for wing, objs in sorted(listing.items()):
    for o in objs:
        print(f"  WARN  leak in {wing}: {o.get('object_id')} ({', '.join(o.get('families') or [])} in {', '.join(o.get('fields') or [])})")
PY
    [ $? -ne 0 ] && FAILS=$((FAILS + 1))
  elif [ "$LCODE" != "404" ]; then                      # 404: a service older than 0.1.13
    fail "/health/leaks answered HTTP $LCODE: $(head -c 200 "$BODY" | tr -d '\n')"
  fi

  echo "search (positive control: \"$QUERY\")"
  HITS="$(albok search --json --top-k 3 "$QUERY" 2>&1)"; RC=$?
  if [ $RC -ne 0 ]; then fail "albok search: $HITS"; else
    python3 - "$HITS" <<'PY'
import json, sys
r = json.loads(sys.argv[1]).get("results") or []
if not r: print("  FAIL  zero hits for the positive control: the index is empty or search is broken"); sys.exit(1)
print(f"  ok    {len(r)} hits; top: {r[0].get('wing')} {r[0].get('title')!r} score={r[0].get('score'):.3f} (never threshold on score: the null floor is ~0.031)")
PY
    [ $? -ne 0 ] && FAILS=$((FAILS + 1))
  fi
fi

echo "nemi"
STATE_DIR="$(python3 -c 'import sys,yaml;print(yaml.safe_load(open(sys.argv[1]))["state_dir"])' "$NEMI_CONFIG" 2>/dev/null || echo "$HOME/.local/state/albok-nemi")"
python3 - "$STATE_DIR" "$STORM" <<'PY'
import glob, json, os, sys
from datetime import datetime, timezone
state, storm = sys.argv[1], int(sys.argv[2])
reports = sorted((p for p in glob.glob(os.path.join(state, "reports", "*.json")) if not os.path.basename(p).startswith("eval-")), key=os.path.getmtime)   # eval reports are not walks
if not reports: print(f"  FAIL  no walk report under {state}/reports"); sys.exit(1)
d = json.load(open(reports[-1]))
age = datetime.now(timezone.utc) - datetime.fromisoformat(d["run_at"])
mins = int(age.total_seconds() // 60)
print(f"  {'ok   ' if mins <= 120 else 'WARN '} newest walk {d['run_at']} ({mins} min ago; hourly timer) nemi {d.get('nemi_version')} on {d.get('host')}")
rc = 0
for wing, w in (d.get("wings") or {}).items():
    o = w.get("outcomes") or {}
    line = " ".join(f"{k} {o.get(k, 0)}" for k in ("created", "updated", "unchanged", "tombstoned", "revived"))
    if w.get("error"):
        print(f"  FAIL  {wing}: {line}; the walk ended in an error: {w['error']}"); rc = 1
    elif o.get("updated", 0) >= storm:
        print(f"  WARN  {wing}: {line}  <- many updates (>= {storm}): usually commits to big memory files (each restamps source_commit on all its sections); a Nemi version change against service < 0.1.3 did the same")
    else:
        print(f"  ok    {wing}: {line}")
    # unit 4.4: a partial walk is self-healing (retried hourly) but a persistent cause needs the operator — WARN, never FAIL
    held = {k: v for k, v in ((w.get("sensitivity") or {}).get("held") or {}).items() if v}
    if held:
        print(f"  WARN  {wing}: held {sum(held.values())} object(s) ({', '.join(f'{k} {v}' for k, v in held.items())}); the walk was partial and tombstoned nothing. "
              f"Retried every hour; a cause that persists (an invalid `visibility:` tag, a dead judge seat, a listing over the cap) is yours to remove — see the report's notes")
    if w.get("unresolved"):
        print(f"  WARN  {wing}: {len(w['unresolved'])} object(s) stayed unresolved (needs_bytes after their repair); the walk was partial. Retried next hour; persistent → the service's run_unresolved rows")
for s in d.get("skipped") or []: print(f"  WARN  skipped: {s}")
sys.exit(rc)
PY
[ $? -ne 0 ] && FAILS=$((FAILS + 1))
if command -v systemctl >/dev/null 2>&1 && systemctl --user list-unit-files albok-nemi.timer >/dev/null 2>&1; then
  T="$(systemctl --user is-active albok-nemi.timer 2>/dev/null)"; S="$(systemctl --user is-active albok-nemi.service 2>/dev/null)"
  case "$T" in active) ok "timer active; $(systemctl --user show albok-nemi.timer -p LastTriggerUSec -p NextElapseUSecRealtime 2>/dev/null | sed -e 's/LastTriggerUSec=/last fired /' -e 's/NextElapseUSecRealtime=/next /' | paste -sd ';' | sed 's/;/; /')" ;; *) fail "albok-nemi.timer is $T" ;; esac
  case "$S" in activating|active) ok "a walk is running now (albok-nemi.service $S)" ;; failed) fail "albok-nemi.service failed: journalctl --user -u albok-nemi.service -n 30" ;; *) ok "no walk running (service $S)" ;; esac
else
  warn "no albok-nemi.timer on this box (it lives on nh3-dev as a user unit)"
fi
if [ -e "$STATE_DIR/run.lock" ] && command -v flock >/dev/null 2>&1; then
  flock -n "$STATE_DIR/run.lock" true 2>/dev/null && ok "run.lock free" || ok "run.lock held (a walk holds it; a second walk would refuse to start)"
fi

echo "container ($ALBOK_HOST)"
if PS="$(ssh -o ConnectTimeout=3 -o BatchMode=yes "$ALBOK_HOST" 'docker ps --filter name=albok-service --format "{{.Image}} {{.Status}}"' 2>/dev/null)" && [ -n "$PS" ]; then
  case "$PS" in *healthy*) ok "$PS" ;; *) fail "$PS" ;; esac
else
  warn "ssh $ALBOK_HOST not reachable from here, or no albok-service container; skipped"
fi

echo
if [ "$FAILS" -eq 0 ]; then echo "albok-check: nothing failed"; else echo "albok-check: $FAILS FAIL"; exit 1; fi
