vh
e43e2626f1
Disposable harnesses from the wake-RFI thread (01M2WT3F): exit1-behavior-test.py covers the monitor exit-code/gate path including a sustained-failure/no-backoff assertion; hash_gate_repro.py covers the detection-time hash persistence crash-before-read boundary with consumption/unchanged/changed controls. Both fail nonzero on mismatch. Committing so seat artifact provenance rides git rather than file mtimes.
95 lines
4.7 KiB
Python
Executable File
95 lines
4.7 KiB
Python
Executable File
"""Disposable behavioral test: monitor source exit!=0 -> ok=False, nothing persisted.
|
|
|
|
Calls cron.monitor.check_monitor and cron.scheduler._apply_monitor_gate directly
|
|
against fabricated job dicts and throwaway monitor scripts. No live job touched,
|
|
no agent woken. Exit codes: 0 = all assertions held, 1 = mismatch.
|
|
"""
|
|
import os, sys, json, tempfile
|
|
sys.path.insert(0, os.path.expanduser("~/.hermes/hermes-agent"))
|
|
os.environ.setdefault("HERMES_HOME", os.path.expanduser("~/.hermes"))
|
|
|
|
from cron.monitor import check_monitor
|
|
from cron.scheduler import _apply_monitor_gate
|
|
|
|
SDIR = os.path.expanduser("~/.hermes/scripts")
|
|
results = []
|
|
|
|
def mkscript(name, body):
|
|
p = os.path.join(SDIR, name)
|
|
with open(p, "w") as f:
|
|
f.write(body)
|
|
os.chmod(p, 0o755)
|
|
return os.path.relpath(p, SDIR)
|
|
|
|
# Script S1: emits the sentinel then exit 1 (proposed repaired failure path)
|
|
s1 = mkscript("_t_exit1.sh",
|
|
'#!/usr/bin/env bash\necho "PEEK-FAILED rc=2"\nexit 1\n')
|
|
# Script S2: emits the sentinel then exit 0 (current production behaviour)
|
|
s2 = mkscript("_t_exit0.sh",
|
|
'#!/usr/bin/env bash\necho "PEEK-FAILED rc=2"\nexit 0\n')
|
|
|
|
def fake_job(jid, script, prior_hash=None):
|
|
st = {"last_output_hash": prior_hash} if prior_hash else None
|
|
return {"id": jid, "name": jid, "monitor_script": script,
|
|
"monitor_state": st, "schedule": {"kind": "interval", "minutes": 5}}
|
|
|
|
# --- T1: exit 1 -> check_monitor ok=False, no state persisted
|
|
job = fake_job("_t_exit1_job", s1)
|
|
out = check_monitor(job)
|
|
results.append(("T1 exit1 => ok=False", out.ok is False))
|
|
results.append(("T1 error carries sentinel", "PEEK-FAILED" in (out.error or "")))
|
|
from cron.jobs import get_job
|
|
persisted = get_job("_t_exit1_job")
|
|
results.append(("T1 nothing persisted (no monitor_state on a real store)",
|
|
persisted is None or not (persisted.get("monitor_state") or {}).get("last_output_hash")))
|
|
|
|
# --- T2: exit 0 -> ok=True changed=True (sentinel becomes a persisted hash)
|
|
job0 = fake_job("_t_exit0_job", s2)
|
|
out0 = check_monitor(job0)
|
|
results.append(("T2 exit0 => ok=True changed=True", out0.ok is True and out0.changed is True))
|
|
# simulate what a real store would now hold (job not in the store, so update_job
|
|
# has nothing to write; fabricate the persisted hash the same way the monitor did)
|
|
from cron.monitor import hash_monitor_output
|
|
persisted_hash = hash_monitor_output("PEEK-FAILED rc=2")
|
|
suppressed = check_monitor(fake_job("_t_exit0_job", s2, prior_hash=persisted_hash))
|
|
results.append(("T2 re-run with persisted hash suppressed (outage wakes once per transition)",
|
|
suppressed.ok is True and suppressed.changed is False))
|
|
|
|
# --- T3: _apply_monitor_gate on the exit-1 job returns the error early-result
|
|
early, prompt, ctx = _apply_monitor_gate(fake_job("_t_exit1_job", s1), "_t_exit1_job", "t", None)
|
|
ok_gate = (early is not None and early[0] is False and "source failed" in early[1])
|
|
results.append(("T3 gate early-returns failure alert, no agent run", ok_gate))
|
|
# repeated failing tick: gate must alert AGAIN (nothing persisted to suppress against)
|
|
early2, _, _ = _apply_monitor_gate(fake_job("_t_exit1_job", s1), "_t_exit1_job", "t", None)
|
|
results.append(("T3 second failing tick alerts again (no dedup state)",
|
|
early2 is not None and early2[0] is False))
|
|
|
|
# --- T4: sustained-failure then recovery: no backoff, no auto-pause, job keeps firing.
|
|
# Source claim (svos-dev): the monitor-failure branch early-returns only;
|
|
# _block_and_pause_job is unreachable from it. Behavioral close: N consecutive
|
|
# failing ticks each alert (no dedup/pause), a recovery tick passes the gate
|
|
# open (early=None), and the job is never left paused/absent-of-schedule.
|
|
fail_alerts = 0
|
|
for _ in range(5):
|
|
early_t, _, ctx_t = _apply_monitor_gate(fake_job("_t_exit1_job", s1), "_t_exit1_job", "t", None)
|
|
if early_t is not None and early_t[0] is False:
|
|
fail_alerts += 1
|
|
results.append(("T4 5 consecutive failing ticks each alert (no backoff/disable)", fail_alerts == 5))
|
|
after = get_job("_t_exit1_job")
|
|
results.append(("T4 job not paused/disabled by failures (absent or state!=paused)",
|
|
after is None or after.get("state") not in ("paused", "blocked")))
|
|
# recovery: a script that succeeds with a NEW output passes the gate open
|
|
s3 = mkscript("_t_recover.sh", '#!/usr/bin/env bash\necho "[9001]"\nexit 0\n')
|
|
early_r, _, ctx_r = _apply_monitor_gate(fake_job("_t_rec_job", s3), "_t_rec_job", "t", None)
|
|
results.append(("T4 recovery tick passes gate open (early=None, monitor ctx present)",
|
|
early_r is None and ctx_r is not None))
|
|
os.remove(os.path.join(SDIR, s3))
|
|
|
|
# cleanup
|
|
for f in (s1, s2):
|
|
os.remove(os.path.join(SDIR, f))
|
|
|
|
for name, ok in results:
|
|
print(("PASS" if ok else "FAIL"), "-", name)
|
|
sys.exit(0 if all(ok for _, ok in results) else 1)
|