"""Disposable behavioral test: monitor source exit!=0 -> ok=False, nothing persisted. Calls cron.monitor.check_monitor and cron.scheduler._apply_monitor_gate directly against fabricated job dicts and throwaway monitor scripts. No live job touched, no agent woken. Exit codes: 0 = all assertions held, 1 = mismatch. """ import os, sys, json, tempfile sys.path.insert(0, os.path.expanduser("~/.hermes/hermes-agent")) os.environ.setdefault("HERMES_HOME", os.path.expanduser("~/.hermes")) from cron.monitor import check_monitor from cron.scheduler import _apply_monitor_gate SDIR = os.path.expanduser("~/.hermes/scripts") results = [] def mkscript(name, body): p = os.path.join(SDIR, name) with open(p, "w") as f: f.write(body) os.chmod(p, 0o755) return os.path.relpath(p, SDIR) # Script S1: emits the sentinel then exit 1 (proposed repaired failure path) s1 = mkscript("_t_exit1.sh", '#!/usr/bin/env bash\necho "PEEK-FAILED rc=2"\nexit 1\n') # Script S2: emits the sentinel then exit 0 (current production behaviour) s2 = mkscript("_t_exit0.sh", '#!/usr/bin/env bash\necho "PEEK-FAILED rc=2"\nexit 0\n') def fake_job(jid, script, prior_hash=None): st = {"last_output_hash": prior_hash} if prior_hash else None return {"id": jid, "name": jid, "monitor_script": script, "monitor_state": st, "schedule": {"kind": "interval", "minutes": 5}} # --- T1: exit 1 -> check_monitor ok=False, no state persisted job = fake_job("_t_exit1_job", s1) out = check_monitor(job) results.append(("T1 exit1 => ok=False", out.ok is False)) results.append(("T1 error carries sentinel", "PEEK-FAILED" in (out.error or ""))) from cron.jobs import get_job persisted = get_job("_t_exit1_job") results.append(("T1 nothing persisted (no monitor_state on a real store)", persisted is None or not (persisted.get("monitor_state") or {}).get("last_output_hash"))) # --- T2: exit 0 -> ok=True changed=True (sentinel becomes a persisted hash) job0 = fake_job("_t_exit0_job", s2) out0 = check_monitor(job0) results.append(("T2 exit0 => ok=True changed=True", out0.ok is True and out0.changed is True)) # simulate what a real store would now hold (job not in the store, so update_job # has nothing to write; fabricate the persisted hash the same way the monitor did) from cron.monitor import hash_monitor_output persisted_hash = hash_monitor_output("PEEK-FAILED rc=2") suppressed = check_monitor(fake_job("_t_exit0_job", s2, prior_hash=persisted_hash)) results.append(("T2 re-run with persisted hash suppressed (outage wakes once per transition)", suppressed.ok is True and suppressed.changed is False)) # --- T3: _apply_monitor_gate on the exit-1 job returns the error early-result early, prompt, ctx = _apply_monitor_gate(fake_job("_t_exit1_job", s1), "_t_exit1_job", "t", None) ok_gate = (early is not None and early[0] is False and "source failed" in early[1]) results.append(("T3 gate early-returns failure alert, no agent run", ok_gate)) # repeated failing tick: gate must alert AGAIN (nothing persisted to suppress against) early2, _, _ = _apply_monitor_gate(fake_job("_t_exit1_job", s1), "_t_exit1_job", "t", None) results.append(("T3 second failing tick alerts again (no dedup state)", early2 is not None and early2[0] is False)) # --- T4: sustained-failure then recovery: no backoff, no auto-pause, job keeps firing. # Source claim (svos-dev): the monitor-failure branch early-returns only; # _block_and_pause_job is unreachable from it. Behavioral close: N consecutive # failing ticks each alert (no dedup/pause), a recovery tick passes the gate # open (early=None), and the job is never left paused/absent-of-schedule. fail_alerts = 0 for _ in range(5): early_t, _, ctx_t = _apply_monitor_gate(fake_job("_t_exit1_job", s1), "_t_exit1_job", "t", None) if early_t is not None and early_t[0] is False: fail_alerts += 1 results.append(("T4 5 consecutive failing ticks each alert (no backoff/disable)", fail_alerts == 5)) after = get_job("_t_exit1_job") results.append(("T4 job not paused/disabled by failures (absent or state!=paused)", after is None or after.get("state") not in ("paused", "blocked"))) # recovery: a script that succeeds with a NEW output passes the gate open s3 = mkscript("_t_recover.sh", '#!/usr/bin/env bash\necho "[9001]"\nexit 0\n') early_r, _, ctx_r = _apply_monitor_gate(fake_job("_t_rec_job", s3), "_t_rec_job", "t", None) results.append(("T4 recovery tick passes gate open (early=None, monitor ctx present)", early_r is None and ctx_r is not None)) os.remove(os.path.join(SDIR, s3)) # cleanup for f in (s1, s2): os.remove(os.path.join(SDIR, f)) for name, ok in results: print(("PASS" if ok else "FAIL"), "-", name) sys.exit(0 if all(ok for _, ok in results) else 1)