diff --git a/scripts/althing-seat-tests/exit1-behavior-test.py b/scripts/althing-seat-tests/exit1-behavior-test.py index 9ea4fad..9c03109 100755 --- a/scripts/althing-seat-tests/exit1-behavior-test.py +++ b/scripts/althing-seat-tests/exit1-behavior-test.py @@ -3,16 +3,14 @@ Calls cron.monitor.check_monitor and cron.scheduler._apply_monitor_gate directly against fabricated job dicts and throwaway monitor scripts. No live job touched, no agent woken. Exit codes: 0 = all assertions held, 1 = mismatch. + +Guarded behind __main__: importing this module (e.g. by pytest collection) +must not execute it or touch the live cron store. """ import os, sys, json, tempfile -sys.path.insert(0, os.path.expanduser("~/.hermes/hermes-agent")) -os.environ.setdefault("HERMES_HOME", os.path.expanduser("~/.hermes")) - -from cron.monitor import check_monitor -from cron.scheduler import _apply_monitor_gate SDIR = os.path.expanduser("~/.hermes/scripts") -results = [] + def mkscript(name, body): p = os.path.join(SDIR, name) @@ -21,74 +19,99 @@ def mkscript(name, body): os.chmod(p, 0o755) return os.path.relpath(p, SDIR) -# Script S1: emits the sentinel then exit 1 (proposed repaired failure path) -s1 = mkscript("_t_exit1.sh", - '#!/usr/bin/env bash\necho "PEEK-FAILED rc=2"\nexit 1\n') -# Script S2: emits the sentinel then exit 0 (current production behaviour) -s2 = mkscript("_t_exit0.sh", - '#!/usr/bin/env bash\necho "PEEK-FAILED rc=2"\nexit 0\n') def fake_job(jid, script, prior_hash=None): st = {"last_output_hash": prior_hash} if prior_hash else None return {"id": jid, "name": jid, "monitor_script": script, "monitor_state": st, "schedule": {"kind": "interval", "minutes": 5}} -# --- T1: exit 1 -> check_monitor ok=False, no state persisted -job = fake_job("_t_exit1_job", s1) -out = check_monitor(job) -results.append(("T1 exit1 => ok=False", out.ok is False)) -results.append(("T1 error carries sentinel", "PEEK-FAILED" in (out.error or ""))) -from cron.jobs import get_job -persisted = get_job("_t_exit1_job") -results.append(("T1 nothing persisted (no monitor_state on a real store)", - persisted is None or not (persisted.get("monitor_state") or {}).get("last_output_hash"))) -# --- T2: exit 0 -> ok=True changed=True (sentinel becomes a persisted hash) -job0 = fake_job("_t_exit0_job", s2) -out0 = check_monitor(job0) -results.append(("T2 exit0 => ok=True changed=True", out0.ok is True and out0.changed is True)) -# simulate what a real store would now hold (job not in the store, so update_job -# has nothing to write; fabricate the persisted hash the same way the monitor did) -from cron.monitor import hash_monitor_output -persisted_hash = hash_monitor_output("PEEK-FAILED rc=2") -suppressed = check_monitor(fake_job("_t_exit0_job", s2, prior_hash=persisted_hash)) -results.append(("T2 re-run with persisted hash suppressed (outage wakes once per transition)", - suppressed.ok is True and suppressed.changed is False)) +def main(): + sys.path.insert(0, os.path.expanduser("~/.hermes/hermes-agent")) + os.environ.setdefault("HERMES_HOME", os.path.expanduser("~/.hermes")) -# --- T3: _apply_monitor_gate on the exit-1 job returns the error early-result -early, prompt, ctx = _apply_monitor_gate(fake_job("_t_exit1_job", s1), "_t_exit1_job", "t", None) -ok_gate = (early is not None and early[0] is False and "source failed" in early[1]) -results.append(("T3 gate early-returns failure alert, no agent run", ok_gate)) -# repeated failing tick: gate must alert AGAIN (nothing persisted to suppress against) -early2, _, _ = _apply_monitor_gate(fake_job("_t_exit1_job", s1), "_t_exit1_job", "t", None) -results.append(("T3 second failing tick alerts again (no dedup state)", - early2 is not None and early2[0] is False)) + from cron.monitor import check_monitor, hash_monitor_output + from cron.scheduler import _apply_monitor_gate + from cron.jobs import get_job -# --- T4: sustained-failure then recovery: no backoff, no auto-pause, job keeps firing. -# Source claim (svos-dev): the monitor-failure branch early-returns only; -# _block_and_pause_job is unreachable from it. Behavioral close: N consecutive -# failing ticks each alert (no dedup/pause), a recovery tick passes the gate -# open (early=None), and the job is never left paused/absent-of-schedule. -fail_alerts = 0 -for _ in range(5): - early_t, _, ctx_t = _apply_monitor_gate(fake_job("_t_exit1_job", s1), "_t_exit1_job", "t", None) - if early_t is not None and early_t[0] is False: - fail_alerts += 1 -results.append(("T4 5 consecutive failing ticks each alert (no backoff/disable)", fail_alerts == 5)) -after = get_job("_t_exit1_job") -results.append(("T4 job not paused/disabled by failures (absent or state!=paused)", - after is None or after.get("state") not in ("paused", "blocked"))) -# recovery: a script that succeeds with a NEW output passes the gate open -s3 = mkscript("_t_recover.sh", '#!/usr/bin/env bash\necho "[9001]"\nexit 0\n') -early_r, _, ctx_r = _apply_monitor_gate(fake_job("_t_rec_job", s3), "_t_rec_job", "t", None) -results.append(("T4 recovery tick passes gate open (early=None, monitor ctx present)", - early_r is None and ctx_r is not None)) -os.remove(os.path.join(SDIR, s3)) + results = [] -# cleanup -for f in (s1, s2): - os.remove(os.path.join(SDIR, f)) + # Script S1: emits the sentinel then exit 1 (proposed repaired failure path) + s1 = mkscript("_t_exit1.sh", + '#!/usr/bin/env bash\necho "PEEK-FAILED rc=2"\nexit 1\n') + # Script S2: emits the sentinel then exit 0 (current production behaviour) + s2 = mkscript("_t_exit0.sh", + '#!/usr/bin/env bash\necho "PEEK-FAILED rc=2"\nexit 0\n') -for name, ok in results: - print(("PASS" if ok else "FAIL"), "-", name) -sys.exit(0 if all(ok for _, ok in results) else 1) + try: + # --- T1: exit 1 -> check_monitor ok=False, no state persisted + job = fake_job("_t_exit1_job", s1) + out = check_monitor(job) + results.append(("T1 exit1 => ok=False", out.ok is False)) + results.append(("T1 error carries sentinel", "PEEK-FAILED" in (out.error or ""))) + persisted = get_job("_t_exit1_job") + results.append(("T1 nothing persisted (no monitor_state on a real store)", + persisted is None or not (persisted.get("monitor_state") or {}).get("last_output_hash"))) + + # --- T2: exit 0 -> ok=True changed=True (sentinel becomes a persisted hash) + job0 = fake_job("_t_exit0_job", s2) + out0 = check_monitor(job0) + results.append(("T2 exit0 => ok=True changed=True", out0.ok is True and out0.changed is True)) + # simulate what a real store would now hold (job not in the store, so update_job + # has nothing to write; fabricate the persisted hash the same way the monitor did) + persisted_hash = hash_monitor_output("PEEK-FAILED rc=2") + suppressed = check_monitor(fake_job("_t_exit0_job", s2, prior_hash=persisted_hash)) + results.append(("T2 re-run with persisted hash suppressed (outage wakes once per transition)", + suppressed.ok is True and suppressed.changed is False)) + + # --- T3: _apply_monitor_gate on the exit-1 job returns the error early-result + early, prompt, ctx = _apply_monitor_gate(fake_job("_t_exit1_job", s1), "_t_exit1_job", "t", None) + ok_gate = (early is not None and early[0] is False and "source failed" in early[1]) + results.append(("T3 gate early-returns failure alert, no agent run", ok_gate)) + # repeated failing tick: gate must alert AGAIN (nothing persisted to suppress against) + early2, _, _ = _apply_monitor_gate(fake_job("_t_exit1_job", s1), "_t_exit1_job", "t", None) + results.append(("T3 second failing tick alerts again (no dedup state)", + early2 is not None and early2[0] is False)) + + # --- T4: sustained-failure then recovery: no backoff, no auto-pause, job keeps firing. + # Source claim (svos-dev): the monitor-failure branch early-returns only; + # _block_and_pause_job is unreachable from it. Behavioral close: N consecutive + # failing ticks each alert (no dedup/pause), a recovery tick passes the gate + # open (early=None), and the job is never left paused/absent-of-schedule. + fail_alerts = 0 + for _ in range(5): + early_t, _, ctx_t = _apply_monitor_gate(fake_job("_t_exit1_job", s1), "_t_exit1_job", "t", None) + if early_t is not None and early_t[0] is False: + fail_alerts += 1 + results.append(("T4 5 consecutive failing ticks each alert (no backoff/disable)", fail_alerts == 5)) + after = get_job("_t_exit1_job") + results.append(("T4 job not paused/disabled by failures (absent or state!=paused)", + after is None or after.get("state") not in ("paused", "blocked"))) + # recovery: a script that succeeds with a NEW output passes the gate open + s3 = mkscript("_t_recover.sh", '#!/usr/bin/env bash\necho "[9001]"\nexit 0\n') + early_r, _, ctx_r = _apply_monitor_gate(fake_job("_t_rec_job", s3), "_t_rec_job", "t", None) + results.append(("T4 recovery tick passes gate open (early=None, monitor ctx present)", + early_r is None and ctx_r is not None)) + finally: + for f in (s1, s2): + try: + os.remove(os.path.join(SDIR, f)) + except FileNotFoundError: + pass + try: + os.remove(os.path.join(SDIR, "_t_recover.sh")) + except FileNotFoundError: + pass + # cron output dirs the fabricated job ids produced (check_monitor persists + # last-output snapshots under ~/.hermes/cron/output//) + import shutil + for d in ("_t_exit1_job", "_t_exit0_job", "_t_rec_job"): + shutil.rmtree(os.path.expanduser(f"~/.hermes/cron/output/{d}"), ignore_errors=True) + + for name, ok in results: + print(("PASS" if ok else "FAIL"), "-", name) + sys.exit(0 if all(ok for _, ok in results) else 1) + + +if __name__ == "__main__": + main() diff --git a/scripts/althing-seat-tests/hash_gate_repro.py b/scripts/althing-seat-tests/hash_gate_repro.py index 3d1fcbd..96c0cdc 100644 --- a/scripts/althing-seat-tests/hash_gate_repro.py +++ b/scripts/althing-seat-tests/hash_gate_repro.py @@ -9,79 +9,89 @@ Controls per forseti (2740): - crash case: persist detection, simulate death before 'read', reload state as after restart, rerun with SAME id set -> suppressed (the finding) Each condition repeated 3x. + +Guarded behind __main__: importing this module (e.g. by pytest collection) +must not monkeypatch the live cron.monitor module or execute it. """ import sys, copy -sys.path.insert(0, "/home/lkraven/.hermes/hermes-agent") -from cron import monitor -STATE = {} # in-memory stand-in for jobs.json monitor_state -def fake_persist(job_id, new_hash, output): - STATE[job_id] = {"last_output_hash": new_hash, "snapshot": output} +def main(): + sys.path.insert(0, "/home/lkraven/.hermes/hermes-agent") + from cron import monitor -monitor._persist_monitor_state = fake_persist -monitor._read_last_output = lambda job_id: STATE.get(job_id, {}).get("snapshot", "") -monitor._write_last_output = lambda job_id, output: STATE[job_id].__setitem__("snapshot", output) + STATE = {} # in-memory stand-in for jobs.json monitor_state -OUTPUT = [""] # current observed monitor output, set per tick + def fake_persist(job_id, new_hash, output): + STATE[job_id] = {"last_output_hash": new_hash, "snapshot": output} -def fake_source(job): - return True, OUTPUT[0] + monitor._persist_monitor_state = fake_persist + monitor._read_last_output = lambda job_id: STATE.get(job_id, {}).get("snapshot", "") + monitor._write_last_output = lambda job_id, output: STATE[job_id].__setitem__("snapshot", output) -monitor._run_monitor_source = fake_source + OUTPUT = [""] # current observed monitor output, set per tick -def tick(ids): - """One monitor tick with the given unread id set. Returns (ok, changed).""" - OUTPUT[0] = "\n".join(f"[{i}]" for i in ids) + "\n" - job = {"id": "disposable-test-job", "monitor_script": "stub"} - if "disposable-test-job" in STATE: # simulate fresh load from store - job["monitor_state"] = copy.deepcopy(STATE["disposable-test-job"]) - out = monitor.check_monitor(job) - return out.ok, getattr(out, "changed", None) + def fake_source(job): + return True, OUTPUT[0] -results = [] -for rep in range(1, 4): - # --- consumption control: a SUCCESSFUL run consumes -> output genuinely changes. - # (Was: same ids after success, which only proved unchanged-input suppression. - # Fix: after the run consumes, the mailbox is empty, so the next observation - # is the idle sentinel; that transition admits once, stable idle suppresses.) - STATE.clear() - r1 = tick(["100"]) # first observation -> wake - r2 = tick([]) # agent consumed -> monitor now says idle - r3 = tick([]) # still idle -> suppressed - results.append((f"rep{rep} control-consume: first={r1} after-consume={r2} stable-idle={r3}", - r1 == (True, True) and r2 == (True, True) and r3 == (True, False))) + monitor._run_monitor_source = fake_source - # --- suppression control (renamed honestly): unchanged output suppresses --- - STATE.clear() - r1 = tick(["100"]) - r2 = tick(["100"]) # no consumption, same set next tick - results.append((f"rep{rep} control-unchanged-suppresses: first={r1} unchanged={r2}", - r1 == (True, True) and r2 == (True, False))) + def tick(ids): + """One monitor tick with the given unread id set. Returns (ok, changed).""" + OUTPUT[0] = "\n".join(f"[{i}]" for i in ids) + "\n" + job = {"id": "disposable-test-job", "monitor_script": "stub"} + if "disposable-test-job" in STATE: # simulate fresh load from store + job["monitor_state"] = copy.deepcopy(STATE["disposable-test-job"]) + out = monitor.check_monitor(job) + return out.ok, getattr(out, "changed", None) - # --- positive control: changed ID set admits another run --- - STATE.clear() - r1 = tick(["100"]) - r2 = tick(["100", "101"]) # new mail -> changed -> wake - results.append((f"rep{rep} control-change: first={r1} changed={r2}", - r1 == (True, True) and r2 == (True, True))) + results = [] + for rep in range(1, 4): + # --- consumption control: a SUCCESSFUL run consumes -> output genuinely changes. + # (Was: same ids after success, which only proved unchanged-input suppression. + # Fix: after the run consumes, the mailbox is empty, so the next observation + # is the idle sentinel; that transition admits once, stable idle suppresses.) + STATE.clear() + r1 = tick(["100"]) # first observation -> wake + r2 = tick([]) # agent consumed -> monitor now says idle + r3 = tick([]) # still idle -> suppressed + results.append((f"rep{rep} control-consume: first={r1} after-consume={r2} stable-idle={r3}", + r1 == (True, True) and r2 == (True, True) and r3 == (True, False))) - # --- crash case: detection persisted, agent died BEFORE consuming mail --- - STATE.clear() - r1 = tick(["200", "201"]) # detection: hash persisted pre-agent - # (agent session would run `postbox read` here; simulate it dying first) - r2 = tick(["200", "201"]) # next tick, same unread set, post-restart reload - r3 = tick(["200", "201"]) # and again: unbounded silence - results.append((f"rep{rep} crash-before-read: detect={r1} retry1={r2} retry2={r3}", - r1 == (True, True) and r2 == (True, False) and r3 == (True, False))) + # --- suppression control (renamed honestly): unchanged output suppresses --- + STATE.clear() + r1 = tick(["100"]) + r2 = tick(["100"]) # no consumption, same set next tick + results.append((f"rep{rep} control-unchanged-suppresses: first={r1} unchanged={r2}", + r1 == (True, True) and r2 == (True, False))) - # --- recovery path: stranded mail + a NEW message eventually wakes --- - r4 = tick(["200", "201", "202"]) # a genuinely new message re-wakes, - results.append((f"rep{rep} recovery-on-new-mail: {r4}", r4 == (True, True))) + # --- positive control: changed ID set admits another run --- + STATE.clear() + r1 = tick(["100"]) + r2 = tick(["100", "101"]) # new mail -> changed -> wake + results.append((f"rep{rep} control-change: first={r1} changed={r2}", + r1 == (True, True) and r2 == (True, True))) -ok = True -for line, passed in results: - print(("PASS " if passed else "FAIL ") + line) - ok = ok and passed -print("ALL PASS" if ok else "SOME FAILED") -sys.exit(0 if ok else 1) + # --- crash case: detection persisted, agent died BEFORE consuming mail --- + STATE.clear() + r1 = tick(["200", "201"]) # detection: hash persisted pre-agent + # (agent session would run `postbox read` here; simulate it dying first) + r2 = tick(["200", "201"]) # next tick, same unread set, post-restart reload + r3 = tick(["200", "201"]) # and again: unbounded silence + results.append((f"rep{rep} crash-before-read: detect={r1} retry1={r2} retry2={r3}", + r1 == (True, True) and r2 == (True, False) and r3 == (True, False))) + + # --- recovery path: stranded mail + a NEW message eventually wakes --- + r4 = tick(["200", "201", "202"]) # a genuinely new message re-wakes, + results.append((f"rep{rep} recovery-on-new-mail: {r4}", r4 == (True, True))) + + ok = True + for line, passed in results: + print(("PASS " if passed else "FAIL ") + line) + ok = ok and passed + print("ALL PASS" if ok else "SOME FAILED") + sys.exit(0 if ok else 1) + + +if __name__ == "__main__": + main()