From c0f53a865fc24db9a6530eee4d330143fa9224c9 Mon Sep 17 00:00:00 2001 From: ScottW514 Date: Sat, 15 Aug 2026 15:57:11 -0400 Subject: [PATCH] forgetest: the release acceptance tool and the bench diagnostics page A stdlib-only daemon on the dev image (HTTP :8090) that runs the acceptance catalog against the machine from a self-contained page, keeps the append-only result log under /data/forgetest, and exports the release artifact the gate reads. Tests declare kind (auto / operator / live), hardware (api / takeover), coverage globs, prerequisites, and core membership; a test's domain fingerprint is the hash of the manifest files its globs select plus the platform and its own implementation, so a PASS stays valid exactly while nothing it covers changed. Campaign rules: a FAIL ends the campaign, the core (image health, kernel latch and drills, one live emission witness) is never inherited, invalidate-all forces a full campaign, no SKIP. Live tests need the operator acknowledgment and the physical arm press through the controller; takeover tests stop forgectrl for the duration with a crash-recoverable marker; the tool never touches the laser latch. Catalog v1: 24 tests ported from the proven bench drills with their recorded pass criteria (image, kernel K1-K3 and fire A/B/U, forgectrl API and logs, motion incl. dead-man, cooling, live laser, camera, update, cloud). The bench tab lists every scripts/bench tool and runs the board-side ones as subprocesses (takeover tools wrapped). 44 host unit tests, including the gate verification fixtures. Installed only by forgefirm-image-dev, with the bench scripts under /usr/share/forgetest/bench. --- .gitignore | 6 + forgetest/README.md | 44 ++ forgetest/forgetest.init | 51 ++ forgetest/forgetest/__init__.py | 10 + forgetest/forgetest/__main__.py | 69 +++ forgetest/forgetest/artifact.py | 189 +++++++ forgetest/forgetest/bench.py | 227 ++++++++ forgetest/forgetest/campaign.py | 145 ++++++ forgetest/forgetest/catalog.py | 149 ++++++ forgetest/forgetest/coverage.py | 73 +++ forgetest/forgetest/hw.py | 313 +++++++++++ forgetest/forgetest/log.py | 79 +++ forgetest/forgetest/manifest.py | 167 ++++++ forgetest/forgetest/page.py | 217 ++++++++ forgetest/forgetest/runner.py | 448 ++++++++++++++++ forgetest/forgetest/server.py | 263 ++++++++++ forgetest/forgetest/suite/__init__.py | 13 + forgetest/forgetest/suite/camera.py | 52 ++ forgetest/forgetest/suite/cloud.py | 163 ++++++ forgetest/forgetest/suite/cooling.py | 114 ++++ forgetest/forgetest/suite/forgectrl.py | 182 +++++++ forgetest/forgetest/suite/image.py | 156 ++++++ forgetest/forgetest/suite/kernel.py | 486 ++++++++++++++++++ forgetest/forgetest/suite/laser.py | 426 +++++++++++++++ forgetest/forgetest/suite/logs.py | 59 +++ forgetest/forgetest/suite/motion.py | 459 +++++++++++++++++ forgetest/forgetest/suite/update.py | 47 ++ forgetest/tests/helpers.py | 78 +++ forgetest/tests/test_artifact.py | 143 ++++++ forgetest/tests/test_campaign.py | 160 ++++++ forgetest/tests/test_manifest.py | 153 ++++++ forgetest/tests/test_server.py | 309 +++++++++++ .../recipes-forgefirm/forgetest/forgetest.bb | 47 ++ .../images/forgefirm-image-dev.bb | 5 +- scripts/bench/README.md | 8 +- 35 files changed, 5508 insertions(+), 2 deletions(-) create mode 100644 forgetest/README.md create mode 100644 forgetest/forgetest.init create mode 100644 forgetest/forgetest/__init__.py create mode 100644 forgetest/forgetest/__main__.py create mode 100644 forgetest/forgetest/artifact.py create mode 100644 forgetest/forgetest/bench.py create mode 100644 forgetest/forgetest/campaign.py create mode 100644 forgetest/forgetest/catalog.py create mode 100644 forgetest/forgetest/coverage.py create mode 100644 forgetest/forgetest/hw.py create mode 100644 forgetest/forgetest/log.py create mode 100644 forgetest/forgetest/manifest.py create mode 100644 forgetest/forgetest/page.py create mode 100644 forgetest/forgetest/runner.py create mode 100644 forgetest/forgetest/server.py create mode 100644 forgetest/forgetest/suite/__init__.py create mode 100644 forgetest/forgetest/suite/camera.py create mode 100644 forgetest/forgetest/suite/cloud.py create mode 100644 forgetest/forgetest/suite/cooling.py create mode 100644 forgetest/forgetest/suite/forgectrl.py create mode 100644 forgetest/forgetest/suite/image.py create mode 100644 forgetest/forgetest/suite/kernel.py create mode 100644 forgetest/forgetest/suite/laser.py create mode 100644 forgetest/forgetest/suite/logs.py create mode 100644 forgetest/forgetest/suite/motion.py create mode 100644 forgetest/forgetest/suite/update.py create mode 100644 forgetest/tests/helpers.py create mode 100644 forgetest/tests/test_artifact.py create mode 100644 forgetest/tests/test_campaign.py create mode 100644 forgetest/tests/test_manifest.py create mode 100644 forgetest/tests/test_server.py create mode 100644 meta-forgefirm/recipes-forgefirm/forgetest/forgetest.bb diff --git a/.gitignore b/.gitignore index 3183dcb..836f69a 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,9 @@ # Python bytecode __pycache__/ + +# manifest-from-tree.py fetch cache +/.manifest-cache/ + +# manifest-from-tree.py default output +/tree-manifest.json diff --git a/forgetest/README.md b/forgetest/README.md new file mode 100644 index 0000000..3c5ed8f --- /dev/null +++ b/forgetest/README.md @@ -0,0 +1,44 @@ +# forgetest - the ForgeFIRM release acceptance tool + +The daemon behind `http://:8090/` on the dev image: runs the +acceptance catalog against the machine, keeps the append-only result log, +decides which results still apply to the image that is running, exports +the release artifact `scripts/release.sh` gates on, and serves the bench +diagnostics page. The contract - catalog, campaigns, fingerprints, +inheritance, the gate, the coverage rule - is +[`docs/ACCEPTANCE.md`](../docs/ACCEPTANCE.md). + +## Run the host tests + + cd forgetest + python3 -m unittest discover -s tests -v + +## Run the daemon on a workstation (against a mock or a manifest file) + + FORGETEST_DATA=/tmp/ft FORGETEST_MANIFEST=../tree-manifest.json \ + FORGECTRL_URL=http://:8080 python3 -m forgetest --port 8090 + +`scripts/manifest-from-tree.py` produces `tree-manifest.json` from the recipe +pins; the coverage lint is `python3 -m forgetest.coverage --manifest ...`. + +## Environment + +| Variable | Default | Purpose | +|---|---|---| +| `FORGETEST_DATA` | `/data/forgetest` | results.jsonl, bench.jsonl, token, export/ | +| `FORGETEST_MANIFEST` | `/etc/forgefirm-manifest.json` | the image manifest | +| `FORGETEST_PORT`, `FORGETEST_HOST` | 8090, 0.0.0.0 | listener | +| `FORGETEST_BENCH_DIR` | `/usr/share/forgetest/bench` | the installed bench scripts | +| `FORGETEST_MARKER` | `/run/forgetest.active` | takeover marker | +| `FORGECTRL_URL`, `FORGECTRL_TOKEN_FILE` | `http://127.0.0.1:8080`, `/data/forgefirm/panel.token` | forgectrl client | +| `GF_SYSFS_ROOT` | `/sys/glowforge/` | kernel module sysfs | +| `GRBL_HOST`, `GRBL_PORT` | 127.0.0.1, 23 | Grbl TCP | + +## Adding a test + +Register it in the subsystem module under `forgetest/suite/` with +`@test(...)`: id `subsystem.name`, kind, hardware, `covers`, `requires`, +`always`, steps. The body gets a `Context` (`log`, `check`, `fail`, +`prompt`, `confirm`, `instruct`, `sleep`, `evidence`, `forgectrl`, `sysfs`, +`grbl`, `takeover`). Return normally for PASS, raise `runner.Failed` for +FAIL. Then run the unit tests and the coverage lint. diff --git a/forgetest/forgetest.init b/forgetest/forgetest.init new file mode 100644 index 0000000..57cc2b2 --- /dev/null +++ b/forgetest/forgetest.init @@ -0,0 +1,51 @@ +#!/bin/sh +### BEGIN INIT INFO +# Provides: forgetest +# Required-Start: $network forgectrl +# Required-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: ForgeFIRM release acceptance tool (dev image, HTTP :8090) +### END INIT INFO + +# Dev-image only. The daemon reads /etc/forgefirm-manifest.json, keeps its +# state under /data/forgetest, and serves the acceptance + bench pages on +# port 8090 (FORGETEST_PORT). It starts after forgectrl and stops before +# the controllers at shutdown; a takeover left behind by a hard stop is +# recovered at the next start (forgectrl is started again). + +PIDFILE=/var/run/forgetest.pid +DATA=/data/forgetest +LOG=$DATA/daemon.log + +case "$1" in + start) + echo "Starting forgetest" + mkdir -p "$DATA" + start-stop-daemon -S -q -p $PIDFILE -m -b -x /bin/sh -- -c \ + "exec /usr/bin/python3 -m forgetest >> $LOG 2>&1" + ;; + stop) + echo "Stopping forgetest" + start-stop-daemon -K -q -p $PIDFILE + rm -f $PIDFILE + ;; + restart) + $0 stop + sleep 1 + $0 start + ;; + status) + if [ -f $PIDFILE ] && kill -0 "$(cat $PIDFILE)" 2>/dev/null; then + echo "forgetest is running" + exit 0 + fi + echo "forgetest is not running" + exit 3 + ;; + *) + echo "Usage: $0 {start|stop|restart|status}" + exit 1 + ;; +esac +exit 0 diff --git a/forgetest/forgetest/__init__.py b/forgetest/forgetest/__init__.py new file mode 100644 index 0000000..62995fc --- /dev/null +++ b/forgetest/forgetest/__init__.py @@ -0,0 +1,10 @@ +"""forgetest - the ForgeFIRM release acceptance tool. + +A daemon on the dev image (HTTP :8090) that runs the acceptance catalog +against the machine, keeps the append-only result log, decides which +results still apply to the image that is running, and exports the +release artifact the release gate reads. The bench diagnostics page rides +the same daemon. +""" + +VERSION = "0.1.0" diff --git a/forgetest/forgetest/__main__.py b/forgetest/forgetest/__main__.py new file mode 100644 index 0000000..9bff2a6 --- /dev/null +++ b/forgetest/forgetest/__main__.py @@ -0,0 +1,69 @@ +"""forgetest daemon entry point. + + python3 -m forgetest [--port 8090] [--host 0.0.0.0] [--manifest PATH] + +Environment: FORGETEST_DATA (state directory, default /data/forgetest), +FORGETEST_MANIFEST, FORGETEST_BENCH_DIR, FORGETEST_MARKER, and the hw.py +overrides. +""" +import argparse +import os +import signal +import sys +import threading + +from . import VERSION +from . import bench as _bench +from . import catalog as _catalog +from . import manifest as _manifest +from . import server as _server +from .log import Log, data_dir +from .runner import Runner + + +def main(argv=None): + ap = argparse.ArgumentParser(prog="forgetest") + ap.add_argument("--port", type=int, default=int(os.environ.get("FORGETEST_PORT") or 8090)) + ap.add_argument("--host", default=os.environ.get("FORGETEST_HOST") or "0.0.0.0") + ap.add_argument("--manifest", default=None) + ap.add_argument("--version", action="version", version="forgetest %s" % VERSION) + args = ap.parse_args(argv) + + try: + manifest = _manifest.Manifest.load(args.manifest) + except (OSError, ValueError) as e: + print("forgetest: cannot load the image manifest: %s" % e, file=sys.stderr) + return 2 + registry = _catalog.load_suite() + os.makedirs(data_dir(), exist_ok=True) + log = Log() + bench = _bench.Bench() + runner = Runner(log, manifest, registry, bench) + token = _server.load_token() + app = _server.App(runner, token) + srv = _server.make_server(app, args.host, args.port) + + stop = threading.Event() + + def on_signal(*_): + stop.set() + signal.signal(signal.SIGTERM, on_signal) + signal.signal(signal.SIGINT, on_signal) + print("forgetest %s: %d tests, image %s (%s), listening on %s:%d" + % (VERSION, len(registry), manifest.version, (manifest.content_sha or "")[:12], + args.host, args.port), file=sys.stderr, flush=True) + for m in runner.messages: + print("forgetest: %s" % m, file=sys.stderr, flush=True) + th = threading.Thread(target=srv.serve_forever, name="forgetest-http", daemon=True) + th.start() + try: + while not stop.is_set(): + stop.wait(1.0) + finally: + srv.shutdown() + srv.server_close() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/forgetest/forgetest/artifact.py b/forgetest/forgetest/artifact.py new file mode 100644 index 0000000..ab9518f --- /dev/null +++ b/forgetest/forgetest/artifact.py @@ -0,0 +1,189 @@ +"""The release artifact (acceptance.json / acceptance.md) and its verification. + +The exporter serializes the campaign state with, for every catalog test, +the winning result record (same-campaign PASS or inherited PASS, with its +origin) and the fingerprint it was recorded under. The gate +(scripts/acceptance-gate.py) calls verify() with the artifact, the +release build's manifest, and the catalog from the same source tree, and +recomputes every fingerprint - a PASS applies to the release exactly when +the recomputation matches. The artifact is self-hashed so an edited file +is caught before any of that. +""" +import json + +from . import VERSION +from . import campaign as _campaign +from . import manifest as _manifest +from .log import now_ts + +FORMAT = 1 +_RESULT_KEYS = ("ts", "campaign", "test", "result", "fingerprint", "manifest_sha", "image", + "duration_s", "message", "evidence", "answers", "log", "operator") + + +def _record(rec): + return {k: rec.get(k) for k in _RESULT_KEYS if k in rec} + + +def build(state, tests, manifest, records, catalog_hash): + """The artifact dict (self-hash included). records: the full log, + used to find the winning record for each test.""" + results = [r for r in records if r.get("t") == "result"] + by_key = {} + for r in results: + by_key[(r.get("test"), r.get("ts"), r.get("campaign"))] = r + + tests_out = [] + for t in tests: + st = state["tests"][t.id] + origin = st.get("origin") + rec = None + if origin: + rec = by_key.get((t.id, origin.get("ts"), origin.get("campaign"))) + entry = t.definition() + entry.update({ + "title": t.title, + "source_sha": t.source_sha, + "fingerprint": st["fingerprint"], + "satisfied": st["satisfied"], + "inherited": st["status"] == "inherited", + "record": _record(rec) if rec else None, + }) + tests_out.append(entry) + + art = { + "format": FORMAT, + "tool_version": VERSION, + "exported_at": now_ts(), + "image": {"name": manifest.image_name, "version": manifest.version}, + "manifest_sha": manifest.content_sha, + "identity_sha": manifest.identity_sha(), + "catalog_hash": catalog_hash, + "campaign": state["campaign"], + "invalidate": state["invalidate"], + "authorized": state["authorized"], + "counts": state["counts"], + "manifest": manifest.data, + "tests": tests_out, + } + art["sha256"] = _manifest.sha256_text(_manifest.canonical(art)) + return art + + +def to_json(art): + return json.dumps(art, sort_keys=True, indent=1) + "\n" + + +def to_markdown(art): + lines = [] + img = art.get("image", {}) + lines.append("# ForgeFIRM acceptance - %s" % img.get("version", "?")) + lines.append("") + lines.append("- Image: `%s` (%s)" % (img.get("version", "?"), img.get("name", "?"))) + lines.append("- Manifest identity: `%s`" % (art.get("manifest_sha") or "?")) + lines.append("- Catalog: `%s`" % (art.get("catalog_hash") or "?")) + c = art.get("campaign") or {} + lines.append("- Campaign: `%s` opened %s" % (c.get("id", "-"), c.get("ts", "-"))) + inv = art.get("invalidate") + if inv: + lines.append("- Full campaign required since %s: %s" % (inv.get("ts"), inv.get("reason"))) + lines.append("- Exported: %s" % art.get("exported_at")) + lines.append("- **Release authorized: %s**" % ("YES" if art.get("authorized") else "NO")) + cnt = art.get("counts", {}) + lines.append("- Tests: %s total, %s satisfied (%s inherited), %s required" + % (cnt.get("total"), cnt.get("satisfied"), cnt.get("inherited"), cnt.get("required"))) + lines.append("") + lines.append("| Test | Kind | Result | Run at | Campaign | Inherited |") + lines.append("|---|---|---|---|---|---|") + for t in art.get("tests", []): + rec = t.get("record") or {} + kind = t.get("kind", "") + if t.get("always"): + kind += ", core" + if t.get("hardware") == "takeover": + kind += ", takeover" + lines.append("| `%s` | %s | %s | %s | `%s` | %s |" % ( + t.get("id"), kind, rec.get("result", "-"), rec.get("ts", "-"), + rec.get("campaign", "-"), "yes" if t.get("inherited") else "no")) + lines.append("") + lines.append("Artifact sha256: `%s`" % art.get("sha256")) + lines.append("") + return "\n".join(lines) + + +def verify(art, release_manifest, tests, catalog_hash, expect_machine=None): + """Gate decision. Returns (ok, rows, problems). + + rows: per-test dicts for the report. problems: list of strings; empty + means the artifact authorizes the release manifest. + """ + problems = [] + rows = [] + + body = dict(art) + sha = body.pop("sha256", None) + if sha != _manifest.sha256_text(_manifest.canonical(body)): + problems.append("artifact self-hash mismatch (edited or truncated file)") + return False, rows, problems + if art.get("format") != FORMAT: + problems.append("artifact format %r, expected %r" % (art.get("format"), FORMAT)) + if not art.get("authorized"): + problems.append("artifact does not claim authorization") + + if expect_machine and release_manifest.platform.get("machine") != expect_machine: + problems.append("release manifest machine %r, expected %r" + % (release_manifest.platform.get("machine"), expect_machine)) + if art.get("catalog_hash") != catalog_hash: + problems.append("catalog changed since the campaign (artifact %s, tree %s)" + % ((art.get("catalog_hash") or "?")[:12], catalog_hash[:12])) + + tests = list(tests) + by_id = {t.id: t for t in tests} + art_tests = {t["id"]: t for t in art.get("tests", [])} + for tid in sorted(set(by_id) - set(art_tests)): + problems.append("test %s is in the catalog but not in the artifact" % tid) + for tid in sorted(set(art_tests) - set(by_id)): + problems.append("test %s is in the artifact but not in the catalog" % tid) + + epoch = (art.get("invalidate") or {}).get("ts") + for t in tests: + a = art_tests.get(t.id) + if a is None: + continue + row = {"id": t.id, "always": t.always, "inherited": bool(a.get("inherited"))} + rec = a.get("record") + ok = True + why = [] + if a.get("covers") != [list(c) for c in t.covers] or a.get("always") != t.always \ + or a.get("kind") != t.kind or a.get("requires") != list(t.requires): + ok = False + why.append("definition differs from the catalog in the tree") + if a.get("source_sha") != t.source_sha: + ok = False + why.append("test implementation changed since the campaign") + if not rec: + ok = False + why.append("no PASS recorded") + else: + if rec.get("result") != _campaign.PASS: + ok = False + why.append("recorded result is %s" % rec.get("result")) + fp = t.fingerprint(release_manifest) + row["fingerprint"] = fp + if rec.get("fingerprint") != fp: + ok = False + why.append("fingerprint differs from the release build (domain changed)") + if a.get("inherited"): + if t.always: + ok = False + why.append("always-required test may not be inherited") + if epoch and (rec.get("ts") or "") <= epoch: + ok = False + why.append("inherited PASS predates the invalidate-all") + row["ts"] = rec.get("ts") + row["ok"] = ok + row["why"] = why + rows.append(row) + if not ok: + problems.append("%s: %s" % (t.id, "; ".join(why))) + return not problems, rows, problems diff --git a/forgetest/forgetest/bench.py b/forgetest/forgetest/bench.py new file mode 100644 index 0000000..5ed3c06 --- /dev/null +++ b/forgetest/forgetest/bench.py @@ -0,0 +1,227 @@ +"""The bench diagnostics page: registry of the bench tools and the +subprocess runner behind the #bench tab. + +The registry lists every tool of scripts/bench (the README is the human +index; this is the machine one) with its safety class and argument spec. +A tool is runnable from the page once `ported` is set: the script is +installed under the tool directory (/usr/share/forgetest/bench on the +image, override FORGETEST_BENCH_DIR) and runs as a subprocess with the +form's arguments, output streamed to the page. Unported tools are listed +so the catalog of what exists is complete, with Start disabled. + +Safety classes: + dry reads or dry motion, no emission, forgectrl stays up + takeover needs forgectrl stopped and the pulse device free + live laser emission possible (operator acknowledgment required) + scope needs bench instrumentation on top of the class before it + +Bench runs are recorded in /bench.jsonl and never enter a campaign. +""" +import json +import os +import shlex +import sys +import threading + +from .log import data_dir, now_ts + +DEFAULT_TOOL_DIR = "/usr/share/forgetest/bench" + + +def _arg(name, type="str", default=None, help="", choices=None): + a = {"name": name, "type": type, "default": default, "help": help} + if choices: + a["choices"] = list(choices) + return a + + +TOOLS = [ + # -- board-side, dry ------------------------------------------------------ + {"id": "check-pwm", "title": "Laser PWM register check", "script": "check_pwm.py", + "safety": "dry", "where": "board", "ported": True, "args": [], + "desc": "Reads PWM2 PWMCR/PWMPR via /dev/mem; expects divider 13 x ~127 counts = ~40 kHz. Read-only."}, + {"id": "pacing-test", "title": "Protocol-loop pacing check", "script": "pacing_test.py", + "safety": "dry", "where": "board", "ported": True, + "args": [_arg("mm", "float", 30.0, "jog distance (+X first)"), _arg("feed", "float", 600.0, "feed rate")], + "desc": "Dry motion: idle/parked states coarse-paced, active motion tight-paced, hold/resume mid-move keeps position."}, + {"id": "bench-m2", "title": "Motion-quality bench", "script": "bench_m2.py", + "safety": "dry", "where": "board", "ported": True, "args": [], + "argv_fixed": ["127.0.0.1"], + "desc": "Bounded round-trip jogs (sanity, max-rate, diagonal) + feed-hold/resume; reports peak feed, transitions, drift."}, + {"id": "bench-phase2", "title": "End-of-data protocol bench", "script": "bench_phase2.py", + "safety": "takeover", "where": "board", "ported": True, "args": [], + "desc": "Underrun detection/ack, parked no-replay guard, resume(0), continuous feed, run/underrun cycles. Motors locked, laser latched."}, + {"id": "cp-watchdog", "title": "HV charge-pump watchdog timing", "script": "cp_watchdog_timing.py", + "safety": "dry", "where": "board", "ported": True, + "args": [_arg("duration_s", "float", 14.0, "capture length")], + "desc": "Latches CHG_PUMP feed pulses, polls the watchdog readbacks while commanding short local jogs. Motion only, laser locked."}, + {"id": "accel-fast", "title": "Head accelerometer sampler", "script": "accel_fast.py", + "safety": "dry", "where": "board", "ported": True, + "args": [_arg("duration_s", "float", 5.0, "capture length"), + _arg("jog1", "str", None, "optional mid-capture jog, e.g. $J=G91X20F2400"), + _arg("jog2", "str", None, "optional second jog")], + "desc": "Direct-I2C sampler for the head-bus LIS2HH12s with optional mid-capture jogs. CSV to /tmp/accel.csv."}, + {"id": "bump-seek", "title": "Accelerometer bump-seek homing prototype", "script": "bump_seek.py", + "safety": "dry", "where": "board", "ported": True, + "args": [_arg("direction", "choice", "-", "X direction", ["-", "+"]), + _arg("feed", "int", 120, "creep feed"), _arg("segment_mm", "float", 15.0, "jog segment"), + _arg("max_mm", "float", 200.0, "travel bound")], + "desc": "Creeps toward a rail in bounded jog segments, detects the contact jolt, jog-cancels and backs off."}, + # -- board-side, takeover / scope ------------------------------------------ + {"id": "pwm-sweep", "title": "LASER_PWM scope sweep", "script": "pwm_sweep.py", + "safety": "scope", "where": "board", "ported": False, + "args": [_arg("mode", "choice", "check", "check = read-only, sweep = duty staircase", ["check", "sweep"])], + "desc": "check: readbacks + PWM2 dump; sweep: PWMSAR through 50/25/75/6/100 percent with 4 s holds. Locked state only."}, + {"id": "pwm-hold", "title": "LASER_PWM scope hold", "script": "pwm_hold.py", + "safety": "scope", "where": "board", "ported": False, + "args": [_arg("sar", "int", 64, "PWMSAR value"), _arg("seconds", "int", 10, "hold time")], + "desc": "Holds one PWMSAR value for a scope window, then restores. Locked state only."}, + {"id": "fire-test", "title": "FIRE drop-timing test (A/B/U)", "script": "fire_test.py", + "safety": "takeover", "where": "board", "ported": True, + "args": [_arg("mode", "choice", "A", "A latch locked, B unlocked/unarmed, U true underrun", ["A", "B", "U"])], + "desc": "Duty 0 throughout; refuses to unlock if HV reports good. Software witnesses + the PSU-connector LASER_ON scope point."}, + {"id": "pwm-stream", "title": "LASER_PWM stream-path test", "script": "pwm_stream_test.py", + "safety": "takeover", "where": "board", "ported": False, "args": [], + "desc": "Streams power bytes only (no steps, no FIRE, motor_lock=15, latch locked) through /dev/glowforge."}, + {"id": "gate-a-kernel", "title": "Kernel laser-safety drills K1/K2/K3", "script": "gate_a_kernel_drills.py", + "safety": "takeover", "where": "board", "ported": True, + "args": [_arg("drill", "choice", "K1", "K1 stop floor, K2 resume honors latch, K3 mid-ramp unlock", ["K1", "K2", "K3"])], + "desc": "Software witnesses (cnc/state, laser_enable, laser_on, interlock bit 3); K3 refuses if HV reports good."}, + {"id": "platform-drills", "title": "Kernel platform drills", "script": "platform_drills.py", + "safety": "takeover", "where": "board", "ported": True, + "args": [_arg("drill", "choice", "decay", "deadman / rmmod / decay / led / all", + ["deadman", "rmmod", "decay", "led", "all"])], + "desc": "Dead-man trip readback, rmmod/modprobe cycles under load, decay/microstep readback, LED sequence."}, + # -- cooling ------------------------------------------------------------------ + {"id": "flow-confirm", "title": "Coolant flow suspicion/confirmation drill", "script": "flow_confirm_drill.py", + "safety": "dry", "where": "board", "ported": True, "args": [], + "desc": "One M8 session walks the verdict state machine through real pump-off transients; PASS/FAIL per transition."}, + {"id": "flow-escalate", "title": "Coolant starved re-check escalation drill", "script": "flow_escalate_drill.py", + "safety": "dry", "where": "board", "ported": False, "args": [], + "desc": "With the pump off the job-start check reads SUSPECT and the driver must escalate to FAULT."}, + {"id": "flow-characterize", "title": "Coolant flow characterization", "script": "flow_characterize.py", + "safety": "dry", "where": "host", "ported": False, + "args": [_arg("duty", "int", 30, "heater duty percent")], + "desc": "Baseline -> flow -> no-flow -> recovery with the factory temperature curve; aborts past 45 C downstream."}, + {"id": "flow-sustained", "title": "Coolant sustained re-check run", "script": "flow_sustained.py", + "safety": "dry", "where": "host", "ported": False, "args": [], + "desc": "Long run of the real re-check cadence via M8: verdicts, false faults, loop heat accumulation."}, + {"id": "flow-warm", "title": "Coolant warm-baseline validation", "script": "flow_warm_validate.py", + "safety": "dry", "where": "host", "ported": False, "args": [], + "desc": "Runs the real check from a heater-warmed baseline."}, + {"id": "flow-recheck", "title": "Coolant re-check characterization", "script": "flow_recheck_char.py", + "safety": "dry", "where": "host", "ported": False, "args": [], + "desc": "Short in-run re-checks and the differential metric."}, + {"id": "flow-sampler", "title": "Coolant sampler", "script": "flow_sampler.py", + "safety": "dry", "where": "board", "ported": True, + "args": [_arg("duration_s", "int", 30, "capture length"), _arg("interval_s", "float", 1.0, "sample interval")], + "desc": "Prints elapsed,raw_down,raw_up at the interval; the sampler behind the flow tools."}, + {"id": "temp-calibrate", "title": "Coolant temperature spot-check", "script": "temp_calibrate.py", + "safety": "dry", "where": "host", "ported": False, + "args": [_arg("mode", "choice", "watch", "watch / point / fit", ["watch", "point", "fit"]), + _arg("measured_c", "float", None, "thermometer reading for point")], + "desc": "Pairs a measured temperature with averaged raw readings; fits a per-machine line."}, + {"id": "fan-test", "title": "Fan/coolant bench", "script": "fan_test.py", + "safety": "dry", "where": "host", "ported": False, "args": [], + "desc": "Snapshots fan PWMs/tachs/temps, drives M8 -> cut fans, M9 -> cooldown -> idle. Host-side; port pending."}, + # -- laser (live) -------------------------------------------------------------- + {"id": "live-fire", "title": "LIVE laser drills", "script": "live_fire_drills.py", + "safety": "live", "where": "board", "ported": False, + "args": [_arg("drill", "choice", "witness", "witness / hold / faultpos", ["witness", "hold", "faultpos"])], + "argv_fixed_after": ["127.0.0.1"], + "desc": "Emission witness, disarm grace in Hold, stale-origin refusal. The operator's arm press is required for every drill."}, + # -- host-side harnesses (CI) ------------------------------------------------------ + {"id": "laser-stream-test", "title": "Laser pulse-stream emission harness", "script": "laser_stream_test.py", + "safety": "dry", "where": "host", "ported": False, "args": [], + "desc": "Null-sink controller stream capture against the feeder contract. Runs in the grblHAL repo's CI."}, + {"id": "laser-lifecycle-test", "title": "Armed-window lifecycle harness", "script": "laser_lifecycle_test.py", + "safety": "dry", "where": "host", "ported": False, "args": [], + "desc": "Arm/disarm lifecycle on the null-sink controller. Runs in the grblHAL repo's CI."}, + {"id": "puls-profile", "title": "Factory .puls profile decoder", "script": "puls_profile.py", + "safety": "dry", "where": "host", "ported": False, "args": [], + "desc": "Decodes factory pulse streams into velocity/accel profiles. Runs anywhere; needs a .puls file."}, +] + + +class Bench: + def __init__(self, tools=None, tool_dir=None, index_path=None): + self.tools = list(tools if tools is not None else TOOLS) + self._by_id = {t["id"]: t for t in self.tools} + self._tool_dir = tool_dir + self.index_path = index_path or os.path.join(data_dir(), "bench.jsonl") + self._lock = threading.Lock() + + def tool_dir(self): + return self._tool_dir or os.environ.get("FORGETEST_BENCH_DIR") or DEFAULT_TOOL_DIR + + def get(self, tool_id): + return self._by_id.get(tool_id) + + def command(self, tool, args): + """argv for a tool with the form's arguments. Returns + (ok, argv, error).""" + script = os.path.join(self.tool_dir(), tool["script"]) + if not os.path.exists(script): + return False, None, "script not installed: %s" % tool["script"] + argv = [sys.executable, script] + list(tool.get("argv_fixed", [])) + for spec in tool.get("args", []): + raw = args.get(spec["name"], spec.get("default")) + if raw is None or raw == "": + if spec.get("default") is None: + continue # optional and absent + raw = spec["default"] + try: + if spec["type"] == "int": + val = str(int(raw)) + elif spec["type"] == "float": + val = repr(float(raw)) + elif spec["type"] == "choice": + if str(raw) not in spec["choices"]: + return False, None, "%s must be one of %s" % (spec["name"], spec["choices"]) + val = str(raw) + else: + val = str(raw) + if any(ch in val for ch in "\0\n\r"): + return False, None, "%s: invalid characters" % spec["name"] + except (TypeError, ValueError): + return False, None, "%s: invalid %s" % (spec["name"], spec["type"]) + argv.append(val) + argv += list(tool.get("argv_fixed_after", [])) + return True, argv, None + + def record(self, tool, args, run): + rec = {"ts": run.started_ts, "tool": tool["id"], "args": args, "result": run.finished, + "log_tail": run.lines[-50:]} + with self._lock: + os.makedirs(os.path.dirname(self.index_path), exist_ok=True) + with open(self.index_path, "a", encoding="utf-8") as f: + f.write(json.dumps(rec, sort_keys=True, separators=(",", ":")) + "\n") + + def last_runs(self): + out = {} + try: + with open(self.index_path, "r", encoding="utf-8") as f: + for line in f: + try: + rec = json.loads(line) + except ValueError: + continue + out[rec.get("tool")] = {"ts": rec.get("ts"), "result": rec.get("result"), + "args": rec.get("args")} + except OSError: + pass + return out + + def listing(self): + last = self.last_runs() + items = [] + for t in self.tools: + item = {k: t[k] for k in ("id", "title", "script", "safety", "where", "ported", "args", "desc")} + item["installed"] = os.path.exists(os.path.join(self.tool_dir(), t["script"])) + item["last"] = last.get(t["id"]) + items.append(item) + return items + + def describe_command(self, tool, args): + ok, argv, err = self.command(tool, args) + return " ".join(shlex.quote(a) for a in argv) if ok else err diff --git a/forgetest/forgetest/campaign.py b/forgetest/forgetest/campaign.py new file mode 100644 index 0000000..14188ff --- /dev/null +++ b/forgetest/forgetest/campaign.py @@ -0,0 +1,145 @@ +"""Campaign rules: which results apply, what is required, and whether a +release is authorized. + +A campaign is bound to one image (manifest content hash) and one catalog +(catalog hash). It is open from its record until a FAIL/ERROR result in +it, an invalidate-all, an explicit reset, or a different image or +catalog. Rules per test T with domain fingerprint F(T): + + satisfied by campaign a PASS in the open campaign with fingerprint F(T) + satisfied by inheritance (never for the always-required core) the newest + PASS anywhere in the history with fingerprint F(T) + and newer than the last invalidate-all + required otherwise (reason: always / never-passed / + domain-changed) + + authorized <=> a campaign is open and every catalog test is satisfied + +Pure functions over the record list; nothing here touches hardware or +the filesystem, which keeps the rules unit-testable and lets the release +gate reason with the same code. +""" + +PASS, FAIL, ERROR, ABORTED = "PASS", "FAIL", "ERROR", "ABORTED" +CLOSING = (FAIL, ERROR) + + +def _summary(rec): + if rec is None: + return None + keys = ("ts", "result", "campaign", "image", "fingerprint", "duration_s", "message") + return {k: rec.get(k) for k in keys if k in rec} + + +def open_campaign(records, manifest_sha, catalog_hash): + """(open_campaign_record_or_None, last_campaign_record_or_None, + closed_by_or_None, invalidate_record_or_None).""" + current = None + closed_by = None + invalidate = None + for r in records: + t = r.get("t") + if t == "campaign": + current = r + closed_by = None + elif t == "result": + if current and r.get("campaign") == current.get("id") and r.get("result") in CLOSING: + closed_by = closed_by or "fail" + elif t == "invalidate": + invalidate = r + if current: + closed_by = closed_by or "invalidate" + elif t == "reset": + if current: + closed_by = closed_by or "reset" + if current and not closed_by: + if current.get("manifest_sha") != manifest_sha: + closed_by = "image" + elif current.get("catalog_hash") != catalog_hash: + closed_by = "catalog" + return (current if current and not closed_by else None), current, closed_by, invalidate + + +def compute(records, tests, manifest, catalog_hash, running=None): + """The state the page shows and the exporter serializes. + + tests: iterable of catalog.Test. running: id of the test in progress + (marked in the per-test status), or None. + """ + results = [r for r in records if r.get("t") == "result"] + campaign, last, closed_by, invalidate = open_campaign(records, manifest.content_sha, catalog_hash) + epoch = invalidate.get("ts") if invalidate else None + open_id = campaign.get("id") if campaign else None + + by_test = {} + for r in results: + by_test.setdefault(r.get("test"), []).append(r) + + out_tests = {} + tests = list(tests) + for t in tests: + fp = t.fingerprint(manifest) + hist = by_test.get(t.id, []) + last_r = hist[-1] if hist else None + origin = None + status = "none" + satisfied = False + reason = None + same = None + if open_id: + for r in reversed(hist): + if r.get("campaign") == open_id and r.get("result") == PASS and r.get("fingerprint") == fp: + same = r + break + if same is not None: + status, satisfied, origin, reason = "pass", True, same, "campaign" + elif t.always: + reason = "always" + else: + inh = None + for r in reversed(hist): + if r.get("result") != PASS or r.get("fingerprint") != fp: + continue + if epoch and (r.get("ts") or "") <= epoch: + continue + inh = r + break + if inh is not None: + status, satisfied, origin, reason = "inherited", True, inh, "inherited" + else: + reason = "domain-changed" if any(r.get("result") == PASS for r in hist) else "never-passed" + if not satisfied and last_r is not None: + status = {PASS: "stale", FAIL: "fail", ERROR: "error", ABORTED: "aborted"}.get(last_r.get("result"), "none") + if running == t.id: + status = "running" + out_tests[t.id] = { + "fingerprint": fp, + "status": status, + "satisfied": satisfied, + "required": not satisfied, + "reason": reason, + "last": _summary(last_r), + "origin": _summary(origin), + } + + # requires: a prerequisite counts when it is satisfied (campaign or + # inherited); the always-required core is the freshness mechanism. + for t in tests: + missing = [r for r in t.requires if not out_tests.get(r, {}).get("satisfied")] + out_tests[t.id]["missing_requires"] = missing + out_tests[t.id]["requires_met"] = not missing + + n_sat = sum(1 for v in out_tests.values() if v["satisfied"]) + n_inh = sum(1 for v in out_tests.values() if v["status"] == "inherited") + counts = {"total": len(tests), "satisfied": n_sat, "inherited": n_inh, + "required": len(tests) - n_sat} + authorized = bool(campaign) and n_sat == len(tests) and len(tests) > 0 + return { + "campaign": campaign, + "last_campaign": last, + "closed_by": closed_by, + "invalidate": invalidate, + "authorized": authorized, + "counts": counts, + "tests": out_tests, + } diff --git a/forgetest/forgetest/catalog.py b/forgetest/forgetest/catalog.py new file mode 100644 index 0000000..06d91f1 --- /dev/null +++ b/forgetest/forgetest/catalog.py @@ -0,0 +1,149 @@ +"""The acceptance catalog: test definitions and the registry. + +A test is a function decorated with @test(...). The decorator records +what the release gate needs to know without running anything: the id, +the subsystem, the kind (auto / operator / live), how it takes the +hardware (api / takeover), what source it covers, what it requires, and +whether it belongs to the always-required core. The function body runs +under the runner with a Context (log, prompts, evidence, hardware +helpers) and reports by returning normally (PASS) or raising +runner.Failed (FAIL). +""" +import hashlib +import inspect +import os +import re + +from . import manifest as _manifest + +KINDS = ("auto", "operator", "live") +HARDWARE = ("api", "takeover") +_ID_RX = re.compile(r"^[a-z][a-z0-9-]*\.[a-z][a-z0-9-]*$") + +REGISTRY = {} + + +class Test: + def __init__(self, id, title, subsystem, kind, hardware, covers, requires, + always, est_min, steps, description, fn): + self.id = id + self.title = title + self.subsystem = subsystem + self.kind = kind + self.hardware = hardware + self.covers = tuple((str(c), str(g)) for c, g in covers) + self.requires = tuple(requires) + self.always = bool(always) + self.est_min = est_min + self.steps = tuple(steps) + self.description = description or (fn.__doc__ or "").strip() + self.fn = fn + self._source_sha = None + + @property + def source_sha(self): + """sha256 of the module file that defines the test, line endings + normalized. Part of the fingerprint: a changed implementation + invalidates earlier passes of this test and no other.""" + if self._source_sha is None: + path = inspect.getsourcefile(self.fn) or inspect.getfile(self.fn) + self._source_sha = source_file_sha(path) + return self._source_sha + + def fingerprint(self, manifest): + return _manifest.fingerprint(manifest, self.covers, extra=[self.source_sha]) + + def definition(self): + """The gate-visible definition (no implementation, no prose).""" + return { + "id": self.id, + "subsystem": self.subsystem, + "kind": self.kind, + "hardware": self.hardware, + "covers": [list(c) for c in self.covers], + "requires": list(self.requires), + "always": self.always, + } + + def describe(self): + d = self.definition() + d.update({"title": self.title, "est_min": self.est_min, + "steps": list(self.steps), "description": self.description}) + return d + + +def source_file_sha(path): + with open(path, "rb") as f: + data = f.read().replace(b"\r\n", b"\n") + return hashlib.sha256(data).hexdigest() + + +def test(id, *, title, subsystem, kind="auto", hardware="api", covers=(), + requires=(), always=False, est_min=1, steps=(), description=""): + if not _ID_RX.match(id): + raise ValueError("test id %r must look like subsystem.name" % id) + if kind not in KINDS: + raise ValueError("test %s: kind %r" % (id, kind)) + if hardware not in HARDWARE: + raise ValueError("test %s: hardware %r" % (id, hardware)) + for c, g in covers: + if c in _manifest.DEV_ONLY_COMPONENTS: + raise ValueError("test %s: may not cover dev-only component %r" % (id, c)) + + def deco(fn): + if id in REGISTRY: + raise ValueError("duplicate test id %r" % id) + REGISTRY[id] = Test(id, title, subsystem, kind, hardware, covers, requires, + always, est_min, steps, description, fn) + return fn + return deco + + +def all_tests(registry=None): + """Tests in registration order (the suite modules import in + subsystem order, so this is the display order).""" + return list((registry if registry is not None else REGISTRY).values()) + + +def get(id, registry=None): + return (registry if registry is not None else REGISTRY).get(id) + + +def validate(registry=None): + """Every `requires` names a known test and there are no cycles.""" + reg = registry if registry is not None else REGISTRY + for t in reg.values(): + for r in t.requires: + if r not in reg: + raise ValueError("test %s requires unknown test %s" % (t.id, r)) + seen = {} + + def visit(tid, stack): + if tid in stack: + raise ValueError("requires cycle: %s" % " -> ".join(stack + [tid])) + if seen.get(tid): + return + for r in reg[tid].requires: + visit(r, stack + [tid]) + seen[tid] = True + for tid in reg: + visit(tid, []) + + +def catalog_hash(registry=None): + """Identity of the catalog's definitions (ids, kinds, coverage, + requirements, core membership) - not of the implementations, which + the per-test fingerprints carry.""" + defs = sorted((t.definition() for t in all_tests(registry)), key=lambda d: d["id"]) + return _manifest.sha256_text(_manifest.canonical(defs)) + + +def load_suite(): + """Import the suite modules (each registers its tests) and validate.""" + from . import suite # noqa: F401 (registers on import) + validate() + return REGISTRY + + +def suite_dir(): + return os.path.join(os.path.dirname(os.path.abspath(__file__)), "suite") diff --git a/forgetest/forgetest/coverage.py b/forgetest/forgetest/coverage.py new file mode 100644 index 0000000..367272d --- /dev/null +++ b/forgetest/forgetest/coverage.py @@ -0,0 +1,73 @@ +"""Coverage lint: every source path in the manifest must be selected by +some test's coverage globs, except the allowlisted non-behavioral paths. + +Why a hard rule: under the domain model an uncovered file is worse than an +untested one - a change there leaves every inherited PASS valid when it +should have invalidated them. The lint is the floor (the file is +fingerprinted by at least one test); whether that test exercises the +change stays with the change author. + + python3 -m forgetest.coverage [--manifest PATH] [--enforce] [--json] + +Exit 0 when nothing is uncovered (or when only reporting), 1 under +--enforce with uncovered paths, 2 on a usage or load error. +""" +import argparse +import json +import sys + +from . import catalog as _catalog +from . import manifest as _manifest + +# Non-behavioral paths that need no acceptance coverage. Reviewed with the +# catalog: widening this list is a change like any other. +ALLOW = [ + ("*", ".github/**"), + ("*", ".gitignore"), + ("*", ".gitmodules"), + ("*", "**/*.md"), + ("*", "LICENSE*"), + ("*", "COPYING*"), + ("*", "docs/**"), + ("*", "tests/**"), + ("*", "graphify-out/**"), + ("*", "**/.gitkeep"), + ("forgectrl", "tools/**"), # the host-side mock, no target behavior +] + + +def run(manifest, tests, allow=ALLOW): + return _manifest.coverage_report(manifest, tests, allow) + + +def main(argv=None): + ap = argparse.ArgumentParser(description="forgetest coverage lint") + ap.add_argument("--manifest", default=None, help="manifest JSON (default: the running image's)") + ap.add_argument("--enforce", action="store_true", help="exit 1 when any path is uncovered") + ap.add_argument("--json", action="store_true", help="machine-readable report") + args = ap.parse_args(argv) + try: + manifest = _manifest.Manifest.load(args.manifest) + except (OSError, ValueError) as e: + print("cannot load manifest: %s" % e, file=sys.stderr) + return 2 + registry = _catalog.load_suite() + tests = _catalog.all_tests(registry) + report = run(manifest, tests) + total = sum(len(v) for v in report.values()) + if args.json: + print(json.dumps({"uncovered": report, "total": total, "tests": len(tests)}, indent=1, sort_keys=True)) + else: + for comp in sorted(report): + print("%s: %d uncovered path(s)" % (comp, len(report[comp]))) + for p in report[comp]: + print(" %s" % p) + print("coverage: %d uncovered path(s) across %d component(s), %d tests" + % (total, len(report), len(tests))) + if args.enforce and total: + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/forgetest/forgetest/hw.py b/forgetest/forgetest/hw.py new file mode 100644 index 0000000..462a7ea --- /dev/null +++ b/forgetest/forgetest/hw.py @@ -0,0 +1,313 @@ +"""Hardware and service access for the suite: forgectrl's HTTP API, the +kernel module's sysfs, the init scripts, and the Grbl TCP port. + +Everything is reachable through environment overrides so the suite can be +exercised against a mock on a host: + FORGECTRL_URL default http://127.0.0.1:8080 + FORGECTRL_TOKEN_FILE default /data/forgefirm/panel.token + GF_SYSFS_ROOT default /sys/glowforge/ (must end with '/') + GRBL_HOST / GRBL_PORT default 127.0.0.1 / 23 + FORGETEST_INITD default /etc/init.d +""" +import json +import os +import socket +import subprocess +import time +import urllib.error +import urllib.parse +import urllib.request + + +class HwError(Exception): + pass + + +# ------------------------------------------------------------ forgectrl + +class Forgectrl: + """Thin client for the machine-services daemon.""" + + def __init__(self, base=None, token=None, timeout=10.0): + self.base = (base or os.environ.get("FORGECTRL_URL") or "http://127.0.0.1:8080").rstrip("/") + self.timeout = timeout + self._token = token + + @property + def token(self): + if self._token is None: + path = os.environ.get("FORGECTRL_TOKEN_FILE") or "/data/forgefirm/panel.token" + try: + with open(path, "r", encoding="utf-8") as f: + self._token = f.read().strip() + except OSError: + self._token = "" + return self._token + + def host_header(self): + return urllib.parse.urlsplit(self.base).netloc + + def request(self, method, path, params=None, data=None, headers=None, auth=True, raw=False): + """Returns (status, body). body is parsed JSON when the response + is JSON, else text (or bytes when raw=True). Never raises on an + HTTP error status - the suite asserts on codes.""" + url = self.base + path + if params: + url += ("&" if "?" in url else "?") + urllib.parse.urlencode(params) + body = None + hdrs = {"Host": self.host_header()} + if headers: + hdrs.update(headers) + if data is not None: + if isinstance(data, (dict, list)): + body = urllib.parse.urlencode(data).encode() + hdrs.setdefault("Content-Type", "application/x-www-form-urlencoded") + elif isinstance(data, str): + body = data.encode() + else: + body = data + if auth and self.token: + hdrs.setdefault("X-ForgeFIRM-Token", self.token) + req = urllib.request.Request(url, data=body, method=method, headers=hdrs) + try: + with urllib.request.urlopen(req, timeout=self.timeout) as resp: + status = resp.status + content = resp.read() + ctype = resp.headers.get("Content-Type", "") + except urllib.error.HTTPError as e: + status = e.code + content = e.read() + ctype = e.headers.get("Content-Type", "") if e.headers else "" + except (urllib.error.URLError, socket.timeout, OSError) as e: + raise HwError("forgectrl %s %s: %s" % (method, path, e)) + if raw: + return status, content + text = content.decode("utf-8", "replace") + if "json" in ctype: + try: + return status, json.loads(text) + except ValueError: + pass + return status, text + + def get(self, path, **kw): + return self.request("GET", path, **kw) + + def post(self, path, **kw): + return self.request("POST", path, **kw) + + def status(self): + st, body = self.get("/status") + if st != 200 or not isinstance(body, dict): + raise HwError("forgectrl /status -> %s" % st) + return body + + def settings(self): + st, body = self.get("/settings") + if st != 200 or not isinstance(body, dict): + raise HwError("forgectrl /settings -> %s" % st) + return body + + def wait_idle(self, timeout=60.0, poll=0.5, abort=None): + deadline = time.time() + timeout + while time.time() < deadline: + if abort and abort(): + raise HwError("aborted while waiting for idle") + try: + if self.status().get("state") == "idle": + return True + except HwError: + pass + time.sleep(poll) + return False + + +# ---------------------------------------------------------------- sysfs + +def sysfs_root(): + r = os.environ.get("GF_SYSFS_ROOT") or "/sys/glowforge/" + return r if r.endswith("/") else r + "/" + + +def sysfs_read(attr, default=None): + try: + with open(sysfs_root() + attr, "r") as f: + return f.read().strip() + except OSError: + return default + + +def sysfs_int(attr, default=None): + v = sysfs_read(attr) + if v is None or v == "": + return default + try: + return int(v.split()[0], 0) + except ValueError: + return default + + +def sysfs_write(attr, value): + with open(sysfs_root() + attr, "w") as f: + f.write(str(value)) + + +# --------------------------------------------------------------- init.d + +def initd(service, action, timeout=60): + """Run /etc/init.d/ ; returns (rc, output).""" + base = os.environ.get("FORGETEST_INITD") or "/etc/init.d" + script = os.path.join(base, service) + try: + p = subprocess.run([script, action], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + timeout=timeout) + except (OSError, subprocess.TimeoutExpired) as e: + return 127, str(e) + return p.returncode, p.stdout.decode("utf-8", "replace") + + +def pidof(comm): + """PIDs whose /proc//comm equals comm (15-char kernel limit applies).""" + out = [] + try: + for pid in os.listdir("/proc"): + if not pid.isdigit(): + continue + try: + with open("/proc/%s/comm" % pid) as f: + if f.read().strip() == comm: + out.append(int(pid)) + except OSError: + pass + except OSError: + pass + return out + + +def run(cmd, timeout=60): + """Run a command list; returns (rc, combined output).""" + try: + p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout) + except (OSError, subprocess.TimeoutExpired) as e: + return 127, str(e) + return p.returncode, p.stdout.decode("utf-8", "replace") + + +# ------------------------------------------------------------------ grbl + +class Grbl: + """Minimal Grbl-over-TCP client. The suite is the only client while a + motion test runs; nothing here is used to poll status when a sender + may be attached (position for display comes from forgectrl).""" + + def __init__(self, host=None, port=None, timeout=5.0): + self.host = host or os.environ.get("GRBL_HOST") or "127.0.0.1" + self.port = int(port or os.environ.get("GRBL_PORT") or 23) + self.timeout = timeout + self.sock = None + self.buf = b"" + + def __enter__(self): + self.connect() + return self + + def __exit__(self, *exc): + self.close() + + def connect(self): + self.sock = socket.create_connection((self.host, self.port), timeout=self.timeout) + # Drain the greeting. + time.sleep(0.3) + self.drain() + + def close(self): + if self.sock: + try: + self.sock.close() + except OSError: + pass + self.sock = None + + def drain(self): + self.sock.settimeout(0.05) + try: + while True: + d = self.sock.recv(4096) + if not d: + break + self.buf += d + except (socket.timeout, OSError): + pass + out, self.buf = self.buf, b"" + return out.decode("utf-8", "replace") + + def send_raw(self, data): + self.sock.settimeout(self.timeout) + self.sock.sendall(data) + + def command(self, line, timeout=None): + """Send one line, return the response lines up to ok/error.""" + self.send_raw((line.strip() + "\n").encode()) + return self.wait_response(timeout or self.timeout) + + def wait_response(self, timeout): + deadline = time.time() + timeout + lines = [] + self.sock.settimeout(0.2) + while time.time() < deadline: + try: + d = self.sock.recv(4096) + if not d: + break + self.buf += d + except socket.timeout: + pass + while b"\n" in self.buf: + raw, self.buf = self.buf.split(b"\n", 1) + s = raw.decode("utf-8", "replace").strip() + if not s: + continue + lines.append(s) + if s == "ok" or s.startswith("error:"): + return lines + return lines + + def realtime(self, byte): + self.send_raw(bytes([byte])) + + def status_report(self): + """One '?' report, parsed: {'state': 'Idle', 'MPos': (x,y,z), ...}.""" + self.drain() + self.send_raw(b"?") + deadline = time.time() + self.timeout + self.sock.settimeout(0.2) + while time.time() < deadline: + try: + d = self.sock.recv(4096) + if d: + self.buf += d + except socket.timeout: + pass + i = self.buf.find(b"<") + j = self.buf.find(b">", i + 1) if i >= 0 else -1 + if i >= 0 and j > i: + rep = self.buf[i + 1:j].decode("utf-8", "replace") + self.buf = self.buf[j + 1:] + return parse_report(rep) + raise HwError("no status report from grbl") + + +def parse_report(rep): + parts = rep.split("|") + out = {"state": parts[0]} + for p in parts[1:]: + if ":" in p: + k, v = p.split(":", 1) + if k in ("MPos", "WPos", "WCO"): + try: + out[k] = tuple(float(x) for x in v.split(",")) + except ValueError: + out[k] = v + else: + out[k] = v + return out diff --git a/forgetest/forgetest/log.py b/forgetest/forgetest/log.py new file mode 100644 index 0000000..ce2a47c --- /dev/null +++ b/forgetest/forgetest/log.py @@ -0,0 +1,79 @@ +"""The append-only result log (JSONL) and the export helpers. + +One JSON object per line under the data directory (default +/data/forgetest/results.jsonl, override FORGETEST_DATA). Records: + + campaign {"id","manifest_sha","catalog_hash","image"} a campaign opened + result {"campaign","test","result","fingerprint","manifest_sha", + "image","duration_s","evidence","answers","log","message"} + invalidate {"reason"} manual invalidate-all (full campaign required) + reset {"reason"} explicit campaign reset + export {"artifact_sha256","authorized","campaign"} + +Every record carries "t" (type) and "ts" (UTC, ISO 8601, seconds). +The file is only ever appended; a corrupt line is skipped, counted, and +reported, never repaired in place. +""" +import json +import os +import threading +import time + +DEFAULT_DATA_DIR = "/data/forgetest" + + +def now_ts(): + return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + + +def data_dir(): + return os.environ.get("FORGETEST_DATA") or DEFAULT_DATA_DIR + + +class Log: + def __init__(self, path=None): + self.path = path or os.path.join(data_dir(), "results.jsonl") + self._lock = threading.Lock() + self.corrupt = 0 + + def append(self, rec): + rec = dict(rec) + rec.setdefault("ts", now_ts()) + line = json.dumps(rec, sort_keys=True, separators=(",", ":")) + with self._lock: + os.makedirs(os.path.dirname(self.path), exist_ok=True) + with open(self.path, "a", encoding="utf-8") as f: + f.write(line + "\n") + f.flush() + os.fsync(f.fileno()) + return rec + + def read(self): + recs = [] + self.corrupt = 0 + if not os.path.exists(self.path): + return recs + with self._lock: + with open(self.path, "r", encoding="utf-8") as f: + lines = f.readlines() + for line in lines: + line = line.strip() + if not line: + continue + try: + rec = json.loads(line) + except ValueError: + self.corrupt += 1 + continue + if isinstance(rec, dict) and "t" in rec: + recs.append(rec) + else: + self.corrupt += 1 + return recs + + def raw(self): + if not os.path.exists(self.path): + return "" + with self._lock: + with open(self.path, "r", encoding="utf-8") as f: + return f.read() diff --git a/forgetest/forgetest/manifest.py b/forgetest/forgetest/manifest.py new file mode 100644 index 0000000..561909a --- /dev/null +++ b/forgetest/forgetest/manifest.py @@ -0,0 +1,167 @@ +"""The image manifest and the domain fingerprint. + +/etc/forgefirm-manifest.json (written by forgefirm-image-manifest.bbclass) +identifies the build's inputs: for every component the pinned revision and +one [path, blob-id] pair per source file, plus the platform identity +(machine, kernel modules directory, device tree hashes, layer content +hashes). This module loads it and computes a test's *domain fingerprint*: +the hash of the source files its coverage globs select, plus the platform, +plus the test's own implementation. A recorded PASS applies to a build +exactly when the fingerprint recomputed from that build's manifest is the +same - the same code runs on the board and in the release gate. +""" +import hashlib +import json +import os +import re + +DEFAULT_PATH = "/etc/forgefirm-manifest.json" + +# Components that ship only on the dev image. They can never be part of a +# fingerprint (the release manifest lacks them, so the gate could not +# recompute it); the test implementation is folded in separately. +DEV_ONLY_COMPONENTS = ("forgetest",) + + +def canonical(obj): + """Canonical JSON: sorted keys, no whitespace - the hashing form.""" + return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + +def sha256_text(text): + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def glob_to_regex(pattern): + """Coverage glob -> anchored regex. '**' spans directories, '*' and '?' + stay inside one path segment. Paths use '/' (git paths).""" + out = [] + i, n = 0, len(pattern) + while i < n: + c = pattern[i] + if c == "*": + if pattern[i:i + 2] == "**": + # '**/' also matches zero directories + if pattern[i:i + 3] == "**/": + out.append("(?:.*/)?") + i += 3 + continue + out.append(".*") + i += 2 + continue + out.append("[^/]*") + elif c == "?": + out.append("[^/]") + else: + out.append(re.escape(c)) + i += 1 + return re.compile("^" + "".join(out) + "$") + + +def match_files(files, pattern): + """(path, blob) pairs from a component's file list that the glob selects.""" + rx = glob_to_regex(pattern) + return [(p, b) for p, b in files if rx.match(p)] + + +class Manifest: + def __init__(self, data): + self.data = data + self.components = data.get("components", {}) or {} + self.platform = data.get("platform", {}) or {} + self.image = data.get("image", {}) or {} + self.content_sha = data.get("content_sha256") + + @classmethod + def load(cls, path=None): + path = path or os.environ.get("FORGETEST_MANIFEST") or DEFAULT_PATH + with open(path, "r", encoding="utf-8") as f: + return cls(json.load(f)) + + @classmethod + def from_json(cls, text): + return cls(json.loads(text)) + + @property + def version(self): + return self.image.get("version") or "unknown" + + @property + def image_name(self): + return self.image.get("name") or "unknown" + + def files(self, component): + """The [path, blob] list of a component, or None if the component + is not in this manifest.""" + c = self.components.get(component) + if c is None: + return None + return [tuple(x) for x in c.get("files", [])] + + def component_names(self): + return sorted(self.components) + + def identity_sha(self): + """sha256 of the acceptance-relevant identity: every component + except the dev-only ones, plus the platform. Informational (the + gate decides per test, by fingerprint).""" + comps = {k: v for k, v in self.components.items() if k not in DEV_ONLY_COMPONENTS} + return sha256_text(canonical({"components": comps, "platform": self.platform})) + + +def fingerprint(manifest, covers, extra=()): + """The domain fingerprint of a coverage map on a manifest. + + covers: iterable of (component, glob). extra: strings folded in after + the files (the test's own implementation hash). A component the + manifest lacks contributes a marker so the fingerprint is still + defined and distinct. + """ + parts = set() + for comp, pat in covers: + if comp in DEV_ONLY_COMPONENTS: + raise ValueError("coverage may not name the dev-only component %r" % comp) + files = manifest.files(comp) + if files is None: + parts.add((comp, "@missing", "")) + continue + for p, b in match_files(files, pat): + parts.add((comp, p, b)) + h = hashlib.sha256() + h.update(canonical(sorted(parts)).encode("utf-8")) + h.update(b"\n") + h.update(canonical(manifest.platform).encode("utf-8")) + for e in extra: + h.update(b"\n") + h.update(str(e).encode("utf-8")) + return h.hexdigest() + + +def coverage_report(manifest, tests, allow=()): + """Which manifest paths no test covers. + + tests: iterable with .covers. allow: iterable of (component, glob) + that need no coverage (docs, CI, licenses...). Returns + {component: [uncovered paths]} for the non-dev-only components. + """ + covered = {} + for t in tests: + for comp, pat in t.covers: + covered.setdefault(comp, []).append(glob_to_regex(pat)) + allowed = {} + for comp, pat in allow: + allowed.setdefault(comp, []).append(glob_to_regex(pat)) + report = {} + for comp in manifest.component_names(): + if comp in DEV_ONLY_COMPONENTS: + continue + rxs = covered.get(comp, []) + allowed.get(comp, []) + star = allowed.get("*", []) + missing = [] + for p, _b in manifest.files(comp): + if any(rx.match(p) for rx in rxs) or any(rx.match(p) for rx in star): + continue + missing.append(p) + if missing: + report[comp] = sorted(missing) + return report diff --git a/forgetest/forgetest/page.py b/forgetest/forgetest/page.py new file mode 100644 index 0000000..4d5c1e5 --- /dev/null +++ b/forgetest/forgetest/page.py @@ -0,0 +1,217 @@ +"""The single page: acceptance tab + bench tab. Self-contained (inline +CSS/JS, ES5, no external assets); the visual identity follows the +forgectrl control panel. State comes from GET /state on a 2 s poll; the +catalog and the bench listing are fetched once per tab and refreshed +after a run finishes.""" + +_HTML = r""" + +ForgeFIRM acceptance + +
ForgeFIRM acceptance
+ +
+
+
+

Campaign

+ +
+
+ + + + + + +
+ +
+

A release is authorized when a campaign is open on this image and every catalog test is satisfied - by a PASS in the campaign, or (never for the core) by an earlier PASS whose domain fingerprint is unchanged. A FAIL ends the campaign. Invalidate-all forces a full campaign; give the reason.

+
+
+
+ +
+ +
+ +""" + + +def render(token): + return _HTML.replace("__TOKEN__", token) diff --git a/forgetest/forgetest/runner.py b/forgetest/forgetest/runner.py new file mode 100644 index 0000000..f49c0b7 --- /dev/null +++ b/forgetest/forgetest/runner.py @@ -0,0 +1,448 @@ +"""The runner: executes one acceptance test or one bench tool at a time. + +Acceptance tests run in a worker thread with a Context: log lines, an +operator prompt channel (the page shows the question, the answer comes +back through the API), an abort flag, an evidence dict, hardware helpers, +and the takeover wrapper for tests that need forgectrl out of the way. +Results are appended to the log with the fingerprint the test ran under; +the campaign rules (campaign.py) do the rest. + +Bench tools are subprocesses (bench.py registry): same single slot, same +log pane, no campaign effect. + +Safety, in code rather than convention: a live test starts only with the +operator's acknowledgment in the request; the runner never touches the +laser latch; a takeover always ends with forgectrl started again, and a +marker file makes a crash mid-takeover recoverable at the next start. +""" +import os +import random +import subprocess +import threading +import time +import traceback + +from . import artifact as _artifact +from . import campaign as _campaign +from . import catalog as _catalog +from . import hw +from .log import now_ts, data_dir + +MAX_LINES = 4000 + + +class Aborted(Exception): + pass + + +class Failed(Exception): + pass + + +class Run: + def __init__(self, kind, id, title): + self.kind = kind # 'test' | 'bench' + self.id = id + self.title = title + self.started = time.time() + self.started_ts = now_ts() + self.lines = [] + self.dropped = 0 + self.prompt = None # {"id","question","options"} + self.answers = [] + self.evidence = {} + self.aborted = threading.Event() + self.finished = None # {"result","message","duration_s"} + self.proc = None + self._lock = threading.Lock() + self._cv = threading.Condition(self._lock) + self._answer = None + self._prompt_seq = 0 + + def log(self, msg): + line = "%s %s" % (time.strftime("%H:%M:%S"), msg) + with self._lock: + if len(self.lines) >= MAX_LINES: + self.lines.pop(0) + self.dropped += 1 + self.lines.append(line) + + def snapshot(self, tail=200): + with self._lock: + lines = self.lines[-tail:] + prompt = dict(self.prompt) if self.prompt else None + return { + "kind": self.kind, "id": self.id, "title": self.title, + "started": self.started_ts, "elapsed_s": int(time.time() - self.started), + "log": lines, "dropped": self.dropped, "prompt": prompt, + "finished": self.finished, "aborting": self.aborted.is_set(), + } + + # -- prompt channel ----------------------------------------------- + def ask(self, question, options): + with self._cv: + self._prompt_seq += 1 + pid = "p%d" % self._prompt_seq + self.prompt = {"id": pid, "question": question, "options": list(options)} + self._answer = None + while self._answer is None and not self.aborted.is_set(): + self._cv.wait(0.5) + self.prompt = None + if self.aborted.is_set() and self._answer is None: + raise Aborted("aborted at prompt") + ans = self._answer + self._answer = None + self.answers.append({"ts": now_ts(), "question": question, "answer": ans}) + return ans + + def answer(self, prompt_id, value): + with self._cv: + if not self.prompt or self.prompt["id"] != prompt_id: + return False + if value not in self.prompt["options"]: + return False + self._answer = value + self._cv.notify_all() + return True + + def abort(self): + self.aborted.set() + with self._cv: + self._cv.notify_all() + p = self.proc + if p is not None and p.poll() is None: + try: + p.terminate() + except OSError: + pass + + +class Context: + """What a test function gets.""" + + def __init__(self, run, runner, test): + self.run = run + self.runner = runner + self.test = test + self.evidence = run.evidence + self._forgectrl = None + + # -- reporting ----------------------------------------------------- + def log(self, msg, *args): + self.run.log(msg % args if args else msg) + + def check(self, cond, msg, *args): + if not cond: + raise Failed(msg % args if args else msg) + + def fail(self, msg, *args): + raise Failed(msg % args if args else msg) + + def aborted(self): + return self.run.aborted.is_set() + + def checkpoint(self): + if self.aborted(): + raise Aborted("aborted") + + def sleep(self, seconds): + deadline = time.time() + seconds + while time.time() < deadline: + self.checkpoint() + time.sleep(min(0.25, max(0.0, deadline - time.time()))) + + # -- operator ------------------------------------------------------ + def prompt(self, question, options=("Yes", "No")): + self.log("PROMPT: %s", question) + ans = self.run.ask(question, options) + self.log("ANSWER: %s", ans) + return ans + + def confirm(self, question): + """Yes/No question; a No is a test failure.""" + ans = self.prompt(question, ("Yes", "No")) + if ans != "Yes": + raise Failed("operator answered No: %s" % question) + return True + + def instruct(self, text): + """A step for the operator; continues on Done, fails on Cannot.""" + ans = self.prompt(text, ("Done", "Cannot")) + if ans != "Done": + raise Failed("operator could not: %s" % text) + + # -- hardware ------------------------------------------------------ + @property + def forgectrl(self): + if self._forgectrl is None: + self._forgectrl = hw.Forgectrl() + return self._forgectrl + + def sysfs(self, attr, default=None): + return hw.sysfs_read(attr, default) + + def sysfs_int(self, attr, default=None): + return hw.sysfs_int(attr, default) + + def grbl(self): + return hw.Grbl() + + def takeover(self): + return Takeover(self.run.log, self.test.id) + + +class Takeover: + """Hardware takeover: the controller is stopped through the supervisor, + forgectrl is stopped, the marker records the ownership, and forgectrl + is started again on every exit path. Used by takeover tests (through + Context.takeover()) and by takeover bench tools.""" + + def __init__(self, log, who): + self.log = log # callable(str) + self.who = who + self.marker = marker_path() + + def __enter__(self): + log = self.log + log("takeover: stopping the controller through forgectrl") + try: + st, body = hw.Forgectrl().post("/controller/stop") + log("takeover: POST /controller/stop -> %s" % st) + except hw.HwError as e: + log("takeover: forgectrl unreachable (%s)" % e) + with open(self.marker, "w") as f: + f.write("%s %s\n" % (now_ts(), self.who)) + rc, out = hw.initd("forgectrl", "stop") + log("takeover: forgectrl stop -> rc %s" % rc) + deadline = time.time() + 15 + while time.time() < deadline and (hw.pidof("forgectrl") or hw.pidof("grblHAL_glowfor")): + time.sleep(0.5) + left = hw.pidof("forgectrl") + hw.pidof("grblHAL_glowfor") + if left: + self.__exit__(None, None, None) + raise Failed("takeover: processes still alive after stop: %s" % left) + log("takeover: pulse device free") + return self + + def __exit__(self, exc_type, exc, tb): + rc, out = hw.initd("forgectrl", "start") + self.log("takeover: forgectrl start -> rc %s" % rc) + try: + os.remove(self.marker) + except OSError: + pass + return False + + +def marker_path(): + return os.environ.get("FORGETEST_MARKER") or "/run/forgetest.active" + + +class Runner: + def __init__(self, log, manifest, registry, bench=None): + self.log = log + self.manifest = manifest + self.registry = registry + self.bench = bench + self.catalog_hash = _catalog.catalog_hash(registry) + self._lock = threading.Lock() + self.current = None + self.last = None + self.messages = [] + self.recover() + + # -- startup recovery ------------------------------------------------ + def recover(self): + m = marker_path() + if os.path.exists(m): + try: + with open(m) as f: + who = f.read().strip() + except OSError: + who = "?" + self.messages.append("recovered a takeover left by '%s': starting forgectrl" % who) + hw.initd("forgectrl", "start") + try: + os.remove(m) + except OSError: + pass + + # -- state ----------------------------------------------------------- + def tests(self): + return _catalog.all_tests(self.registry) + + def running_id(self): + r = self.current + return r.id if r and r.kind == "test" and not r.finished else None + + def state(self): + records = self.log.read() + st = _campaign.compute(records, self.tests(), self.manifest, self.catalog_hash, + running=self.running_id()) + st["catalog_hash"] = self.catalog_hash + st["manifest"] = {"sha": self.manifest.content_sha, "identity": self.manifest.identity_sha(), + "image": self.manifest.image_name, "version": self.manifest.version} + st["log_corrupt"] = self.log.corrupt + st["messages"] = list(self.messages) + r = self.current + st["running"] = r.snapshot() if r and not r.finished else None + last = r if (r and r.finished) else self.last + st["last_run"] = last.snapshot() if last else None + return st, records + + def busy(self): + r = self.current + return bool(r and not r.finished) + + # -- campaign actions ------------------------------------------------- + def _open_campaign_if_needed(self, state): + if state["campaign"]: + return state["campaign"] + cid = "c-%s-%04x" % (time.strftime("%Y%m%d%H%M%S", time.gmtime()), random.randrange(1 << 16)) + rec = self.log.append({"t": "campaign", "id": cid, "manifest_sha": self.manifest.content_sha, + "catalog_hash": self.catalog_hash, "image": self.manifest.version}) + return rec + + def invalidate(self, reason): + reason = (reason or "").strip() + if not reason: + return False, "a reason is required" + if self.busy(): + return False, "a run is in progress" + self.log.append({"t": "invalidate", "reason": reason}) + return True, "all results invalidated; a full campaign is required" + + def reset(self, reason=""): + if self.busy(): + return False, "a run is in progress" + self.log.append({"t": "reset", "reason": (reason or "").strip()}) + return True, "campaign reset" + + def export(self): + state, records = self.state() + art = _artifact.build(state, self.tests(), self.manifest, records, self.catalog_hash) + self.log.append({"t": "export", "artifact_sha256": art["sha256"], "authorized": art["authorized"], + "campaign": (state["campaign"] or {}).get("id")}) + return art + + # -- starting ----------------------------------------------------------- + def start_test(self, test_id, ack_live=False): + t = _catalog.get(test_id, self.registry) + if t is None: + return False, "unknown test" + with self._lock: + if self.busy(): + return False, "a run is in progress" + state, _ = self.state() + ts = state["tests"][t.id] + if not ts["requires_met"]: + return False, "prerequisites not satisfied: %s" % ", ".join(ts["missing_requires"]) + if t.kind == "live" and not ack_live: + return False, "live test: acknowledge eye protection, fire watch, and exhaust first" + campaign = self._open_campaign_if_needed(state) + run = Run("test", t.id, t.title) + self.last = self.current + self.current = run + run.log("start %s (%s, %s) in campaign %s" % (t.id, t.kind, t.hardware, campaign["id"])) + if t.kind == "live": + run.evidence["operator"] = {"ack_live": True, "ts": now_ts()} + th = threading.Thread(target=self._exec_test, args=(t, run, campaign), daemon=True, + name="forgetest-run") + th.start() + return True, "started" + + def _exec_test(self, t, run, campaign): + ctx = Context(run, self, t) + fp = t.fingerprint(self.manifest) + result, message = _campaign.PASS, "" + try: + t.fn(ctx) + if run.aborted.is_set(): + result, message = _campaign.ABORTED, "aborted" + except Aborted as e: + result, message = _campaign.ABORTED, str(e) or "aborted" + except Failed as e: + result, message = _campaign.FAIL, str(e) + except Exception as e: # noqa: BLE001 - an erroring test is a failed test + result, message = _campaign.ERROR, "%s: %s" % (type(e).__name__, e) + run.log(traceback.format_exc().rstrip()) + duration = int(time.time() - run.started) + run.log("result %s%s" % (result, (": " + message) if message else "")) + rec = {"t": "result", "campaign": campaign["id"], "test": t.id, "result": result, + "fingerprint": fp, "manifest_sha": self.manifest.content_sha, + "image": self.manifest.version, "duration_s": duration, "message": message, + "evidence": run.evidence, "answers": run.answers, "log": list(run.lines)} + self.log.append(rec) + run.finished = {"result": result, "message": message, "duration_s": duration} + + # -- bench tools --------------------------------------------------------- + def start_bench(self, tool_id, args=None, ack_live=False): + if self.bench is None: + return False, "no bench registry" + tool = self.bench.get(tool_id) + if tool is None: + return False, "unknown tool" + if not tool.get("ported"): + return False, "tool not yet ported to the bench page" + ok, argv, err = self.bench.command(tool, args or {}) + if not ok: + return False, err + if tool.get("safety") == "live" and not ack_live: + return False, "live tool: acknowledge eye protection, fire watch, and exhaust first" + with self._lock: + if self.busy(): + return False, "a run is in progress" + run = Run("bench", tool["id"], tool["title"]) + self.last = self.current + self.current = run + run.log("bench %s: %s" % (tool["id"], " ".join(argv))) + th = threading.Thread(target=self._exec_bench, args=(tool, run, argv, args or {}), + daemon=True, name="forgetest-bench") + th.start() + return True, "started" + + def _exec_bench(self, tool, run, argv, args): + rc = None + message = "" + try: + env = dict(os.environ) + env.setdefault("PYTHONUNBUFFERED", "1") + takeover = Takeover(run.log, "bench:" + tool["id"]) if tool.get("safety") == "takeover" else None + if takeover is not None: + takeover.__enter__() + try: + run.proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + env=env, cwd=self.bench.tool_dir()) + for raw in iter(run.proc.stdout.readline, b""): + run.log(raw.decode("utf-8", "replace").rstrip()) + run.proc.stdout.close() + rc = run.proc.wait() + finally: + if takeover is not None: + takeover.__exit__(None, None, None) + if run.aborted.is_set(): + message = "aborted" + except Exception as e: # noqa: BLE001 + message = "%s: %s" % (type(e).__name__, e) + run.log(message) + duration = int(time.time() - run.started) + result = "ABORTED" if run.aborted.is_set() else ("OK" if rc == 0 else "EXIT %s" % rc) + run.log("bench %s finished: %s" % (tool["id"], result)) + run.finished = {"result": result, "message": message, "duration_s": duration, "rc": rc} + self.bench.record(tool, args, run) + + # -- control -------------------------------------------------------------- + def answer(self, prompt_id, value): + r = self.current + if not r or r.finished: + return False, "nothing is running" + if r.answer(prompt_id, value): + return True, "answered" + return False, "no such prompt (or the answer is not one of the options)" + + def abort(self): + r = self.current + if not r or r.finished: + return False, "nothing is running" + r.abort() + return True, "abort requested" diff --git a/forgetest/forgetest/server.py b/forgetest/forgetest/server.py new file mode 100644 index 0000000..6857bf0 --- /dev/null +++ b/forgetest/forgetest/server.py @@ -0,0 +1,263 @@ +"""HTTP: the page, the JSON API, and the access rules. + +Access follows forgectrl's panel (auth.c): the Host header must be an +address literal or localhost (no DNS names - the rebinding vehicle), a +cross-site Sec-Fetch-Site is refused, an Origin must itself be a literal, +and every state-changing call needs the bearer token that is generated on +first start, stored 0600 under the data directory, and embedded in the +page. Read-only calls need the origin checks only. + +Routes + GET / the page + GET /state campaign + per-test state + running run + GET /catalog test definitions (title, steps, covers...) + GET /bench bench tool listing + GET /result?test&ts one full result record (log, evidence) + GET /log the raw JSONL + GET /export/acceptance.json | .md the last export + POST /start {test, ack_live} start an acceptance test + POST /bench/start {tool, args, ack_live} + POST /answer {prompt_id, value} + POST /abort + POST /invalidate {reason} + POST /reset {reason} + POST /export build + save the artifact, returns it +""" +import hmac +import json +import os +import re +import secrets +import sys +import urllib.parse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from . import artifact as _artifact +from . import page as _page +from .log import data_dir + +TOKEN_HEX = 32 +_LITERAL_RX = re.compile(r"^[0-9.]+$") + + +def load_token(path=None): + path = path or os.path.join(data_dir(), "token") + try: + with open(path, "r", encoding="utf-8") as f: + tok = f.read().strip() + if len(tok) == TOKEN_HEX and all(c in "0123456789abcdef" for c in tok): + return tok + except OSError: + pass + tok = secrets.token_hex(TOKEN_HEX // 2) + os.makedirs(os.path.dirname(path), exist_ok=True) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as f: + f.write(tok + "\n") + return tok + + +def host_is_literal(h): + if not h: + return False + if h.startswith("["): + return True + host = h.split(":", 1)[0] + if host == "localhost": + return True + return bool(_LITERAL_RX.match(host)) + + +def origin_ok(headers): + if not host_is_literal(headers.get("Host")): + return False + sfs = headers.get("Sec-Fetch-Site") + if sfs and sfs not in ("same-origin", "none"): + return False + origin = headers.get("Origin") + if origin and origin != "null": + p = urllib.parse.urlsplit(origin) + if not host_is_literal(p.netloc): + return False + return True + + +class App: + """What the handler needs: the runner, the token, the export dir.""" + + def __init__(self, runner, token, export_dir=None): + self.runner = runner + self.token = token + self.export_dir = export_dir or os.path.join(data_dir(), "export") + + def token_ok(self, given): + return bool(given) and hmac.compare_digest(given, self.token) + + +class Handler(BaseHTTPRequestHandler): + server_version = "forgetest" + app = None # set on the server class + + # -- plumbing --------------------------------------------------------- + def log_message(self, fmt, *args): + if os.environ.get("FORGETEST_HTTP_LOG"): + sys.stderr.write("forgetest: %s - %s\n" % (self.address_string(), fmt % args)) + + def _send(self, status, body, ctype="application/json", extra=None): + if isinstance(body, (dict, list)): + body = json.dumps(body, sort_keys=True).encode("utf-8") + elif isinstance(body, str): + body = body.encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", ctype + ("; charset=utf-8" if ctype.startswith("text/") else "")) + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.send_header("X-Content-Type-Options", "nosniff") + for k, v in (extra or {}).items(): + self.send_header(k, v) + self.end_headers() + if self.command != "HEAD": + self.wfile.write(body) + + def _deny(self, status, msg): + self._send(status, {"error": msg}) + + def _read_ok(self): + if not origin_ok(self.headers): + self._deny(403, "request origin refused") + return False + return True + + def _write_ok(self, query): + if not origin_ok(self.headers): + self._deny(403, "request origin refused") + return False + tok = self.headers.get("X-ForgeFIRM-Token") or (query.get("token") or [None])[0] + if not self.app.token_ok(tok): + self._deny(403, "authentication required") + return False + return True + + def _body_json(self): + n = int(self.headers.get("Content-Length") or 0) + if n <= 0: + return {} + if n > 1 << 20: + raise ValueError("body too large") + raw = self.rfile.read(n) + ctype = self.headers.get("Content-Type", "") + if "json" in ctype: + data = json.loads(raw.decode("utf-8")) + return data if isinstance(data, dict) else {} + # form-encoded fallback + return {k: v[0] for k, v in urllib.parse.parse_qs(raw.decode("utf-8")).items()} + + # -- GET --------------------------------------------------------------- + def do_HEAD(self): + self.do_GET() + + def do_GET(self): + url = urllib.parse.urlsplit(self.path) + path = url.path + query = urllib.parse.parse_qs(url.query) + r = self.app.runner + if not self._read_ok(): + return + try: + if path == "/": + self._send(200, _page.render(self.app.token), "text/html") + elif path == "/state": + state, _ = r.state() + self._send(200, state) + elif path == "/catalog": + self._send(200, {"tests": [t.describe() for t in r.tests()], + "catalog_hash": r.catalog_hash}) + elif path == "/bench": + b = r.bench + self._send(200, {"tools": b.listing() if b else [], "tool_dir": b.tool_dir() if b else None}) + elif path == "/result": + test = (query.get("test") or [""])[0] + ts = (query.get("ts") or [""])[0] + rec = None + for x in reversed(r.log.read()): + if x.get("t") == "result" and x.get("test") == test and (not ts or x.get("ts") == ts): + rec = x + break + if rec is None: + self._deny(404, "no such result") + else: + self._send(200, rec) + elif path == "/log": + self._send(200, r.log.raw(), "text/plain") + elif path in ("/export/acceptance.json", "/export/acceptance.md"): + fn = os.path.join(self.app.export_dir, os.path.basename(path)) + if not os.path.exists(fn): + self._deny(404, "nothing exported yet") + return + with open(fn, "rb") as f: + data = f.read() + ctype = "application/json" if path.endswith(".json") else "text/markdown" + self._send(200, data, ctype, {"Content-Disposition": "attachment; filename=%s" + % os.path.basename(path)}) + else: + self._deny(404, "not found") + except Exception as e: # noqa: BLE001 + self._deny(500, "%s: %s" % (type(e).__name__, e)) + + # -- POST ----------------------------------------------------------------- + def do_POST(self): + url = urllib.parse.urlsplit(self.path) + path = url.path + query = urllib.parse.parse_qs(url.query) + r = self.app.runner + if not self._write_ok(query): + return + try: + body = self._body_json() + except ValueError as e: + self._deny(400, "bad body: %s" % e) + return + try: + if path == "/start": + ok, msg = r.start_test(str(body.get("test", "")), ack_live=bool(body.get("ack_live"))) + self._send(200 if ok else 409, {"ok": ok, "message": msg}) + elif path == "/bench/start": + args = body.get("args") or {} + if not isinstance(args, dict): + self._deny(400, "args must be an object") + return + ok, msg = r.start_bench(str(body.get("tool", "")), args, ack_live=bool(body.get("ack_live"))) + self._send(200 if ok else 409, {"ok": ok, "message": msg}) + elif path == "/answer": + ok, msg = r.answer(str(body.get("prompt_id", "")), str(body.get("value", ""))) + self._send(200 if ok else 409, {"ok": ok, "message": msg}) + elif path == "/abort": + ok, msg = r.abort() + self._send(200 if ok else 409, {"ok": ok, "message": msg}) + elif path == "/invalidate": + ok, msg = r.invalidate(str(body.get("reason", ""))) + self._send(200 if ok else 400, {"ok": ok, "message": msg}) + elif path == "/reset": + ok, msg = r.reset(str(body.get("reason", ""))) + self._send(200 if ok else 409, {"ok": ok, "message": msg}) + elif path == "/export": + art = r.export() + os.makedirs(self.app.export_dir, exist_ok=True) + with open(os.path.join(self.app.export_dir, "acceptance.json"), "w", encoding="utf-8") as f: + f.write(_artifact.to_json(art)) + with open(os.path.join(self.app.export_dir, "acceptance.md"), "w", encoding="utf-8") as f: + f.write(_artifact.to_markdown(art)) + self._send(200, {"ok": True, "authorized": art["authorized"], "sha256": art["sha256"], + "counts": art["counts"]}) + else: + self._deny(404, "not found") + except Exception as e: # noqa: BLE001 + self._deny(500, "%s: %s" % (type(e).__name__, e)) + + +def make_server(app, host="0.0.0.0", port=8090): + handler = type("ForgetestHandler", (Handler,), {"app": app}) + ThreadingHTTPServer.allow_reuse_address = True + srv = ThreadingHTTPServer((host, port), handler) + srv.daemon_threads = True + return srv diff --git a/forgetest/forgetest/suite/__init__.py b/forgetest/forgetest/suite/__init__.py new file mode 100644 index 0000000..b929d95 --- /dev/null +++ b/forgetest/forgetest/suite/__init__.py @@ -0,0 +1,13 @@ +"""The acceptance suite: one module per subsystem, imported in display +order. Each module registers its tests with @catalog.test.""" + +from . import image # noqa: F401,E402 +from . import kernel # noqa: F401,E402 +from . import forgectrl # noqa: F401,E402 +from . import logs # noqa: F401,E402 +from . import motion # noqa: F401,E402 +from . import cooling # noqa: F401,E402 +from . import laser # noqa: F401,E402 +from . import camera # noqa: F401,E402 +from . import update # noqa: F401,E402 +from . import cloud # noqa: F401,E402 diff --git a/forgetest/forgetest/suite/camera.py b/forgetest/forgetest/suite/camera.py new file mode 100644 index 0000000..f857bf8 --- /dev/null +++ b/forgetest/forgetest/suite/camera.py @@ -0,0 +1,52 @@ +"""camera.* - the lid camera pipeline through forgectrl.""" +from ..catalog import test + +_CAM_COVERS = [("forgectrl", "src/cam.*"), ("forgectrl", "src/debayer.*"), ("forgectrl", "src/vpu_jpeg.*"), + ("python3-gfhardware", "gfhardware/src/**"), ("python3-gfhardware", "gfhardware/cam*")] + + +@test("camera.snapshot", title="Lid camera snapshot and stream", subsystem="camera", + kind="operator", est_min=2, + covers=_CAM_COVERS, requires=["forgectrl.panel-serves"], + steps=["Lid closed. You will be asked to look at the control panel's Status tab."], + description="/cam/snapshot returns a JPEG of a plausible size (a black frame compresses " + "far smaller), the MJPEG stream starts and stops, and the operator confirms " + "the panel shows the bed.") +def snapshot(ctx): + fc = ctx.forgectrl + ev = ctx.evidence + st, body = fc.get("/cam/status") + ctx.check(st == 200 and isinstance(body, dict), "GET /cam/status -> %s", st) + ev["cam_status"] = body + ctx.log("cam status: %s", body) + + st, data = fc.get("/cam/snapshot", params={"cam": "lid", "res": "half"}, raw=True) + ctx.log("GET /cam/snapshot?cam=lid&res=half -> %s (%d bytes)", st, len(data) if data else 0) + ctx.check(st == 200, "snapshot -> %s %s", st, data[:120] if data else "") + ctx.check(data[:2] == b"\xff\xd8" and data[-2:] == b"\xff\xd9", "snapshot is not a complete JPEG") + ev["snapshot_bytes"] = len(data) + ctx.check(len(data) > 20000, "snapshot is only %d bytes - a dark or empty frame?", len(data)) + + st, data = fc.get("/cam/snapshot", params={"cam": "lid", "res": "full"}, raw=True) + ctx.log("GET /cam/snapshot?cam=lid&res=full -> %s (%d bytes)", st, len(data) if data else 0) + ctx.check(st == 200 and data[:2] == b"\xff\xd8", "full-resolution snapshot -> %s", st) + ev["snapshot_full_bytes"] = len(data) + + # the stream: fetch a little of it and let it close + import urllib.request + req = urllib.request.Request(fc.base + "/cam/stream", headers={"Host": fc.host_header()}) + try: + with urllib.request.urlopen(req, timeout=10) as r: + ctype = r.headers.get("Content-Type", "") + chunk = r.read(65536) + except Exception as e: # noqa: BLE001 + ctx.fail("stream did not open: %s", e) + ev["stream_ctype"] = ctype + ctx.log("stream: %s, first %d bytes", ctype, len(chunk)) + ctx.check("multipart" in ctype and b"\xff\xd8" in chunk, "stream is not an MJPEG multipart") + ctx.sleep(2) + st, body = fc.get("/cam/status") + ev["cam_status_after"] = body + ctx.log("cam status after: %s", body) + ctx.confirm("Open the control panel (port 8080), Status tab: does the lid snapshot show the bed " + "(not black, not frozen, roughly the right orientation)?") diff --git a/forgetest/forgetest/suite/cloud.py b/forgetest/forgetest/suite/cloud.py new file mode 100644 index 0000000..b349035 --- /dev/null +++ b/forgetest/forgetest/suite/cloud.py @@ -0,0 +1,163 @@ +"""cloud.* - the controller mode switch and the optional Glowforge web-service +mode (gfcloud daemon, gfhome homing runner).""" +import json +import os +import socket +import time + +from ..catalog import test +from .. import hw + +_CLOUD_COVERS = [("forgefirm-app", "**"), ("python3-gfhardware", "**"), ("python3-gfutilities", "**"), + ("forgectrl", "src/super.c"), ("forgectrl", "src/main.c")] + +GF_LATEST = "/data/forgefirm/gf-latest.json" + + +def wait_mode(ctx, fc, want_mode, want_controller="running", timeout=90): + t0 = time.time() + last = None + while time.time() - t0 < timeout: + ctx.checkpoint() + st, m = fc.get("/mode") + if st == 200 and isinstance(m, dict): + last = m + if m.get("mode") == want_mode and m.get("controller") == want_controller: + return m + if m.get("controller") == "motion-fault": + break + time.sleep(1) + return last + + +def grbl_port_open(timeout=5): + try: + s = socket.create_connection((os.environ.get("GRBL_HOST") or "127.0.0.1", + int(os.environ.get("GRBL_PORT") or 23)), timeout=timeout) + s.close() + return True + except OSError: + return False + + +@test("cloud.mode-switch", title="Controller mode switch grbl -> cloud -> grbl", subsystem="cloud", + kind="auto", est_min=4, + covers=_CLOUD_COVERS, requires=["forgectrl.auth", "motion.pacing"], + steps=["Bed clear (the supervisor's liveness probe may jog the head a few mm on a " + "controller spawn). Cloud credentials configured; the machine on the network."], + description="POST /mode switches to the cloud controller: gfcloud comes up under " + "supervision and records its connect-time service probe (/status gfsvc); the " + "camera service survives the switch; switching back brings grblHAL up with the " + "Grbl port open and Idle.") +def mode_switch(ctx): + fc = ctx.forgectrl + ev = ctx.evidence + st, m0 = fc.get("/mode") + ctx.check(st == 200 and isinstance(m0, dict), "GET /mode -> %s", st) + ev["mode_before"] = m0 + ctx.log("mode before: %s", m0) + ctx.check(m0.get("mode") == "grbl", "start this test in grbl mode (now %s)", m0.get("mode")) + st, cam0 = fc.get("/cam/status") + ev["cam_before"] = cam0 + probe_before = None + try: + probe_before = os.stat(GF_LATEST).st_mtime + except OSError: + pass + + st, body = fc.post("/mode", data={"controller": "cloud"}) + ctx.log("POST /mode controller=cloud -> %s %s", st, body) + ctx.check(st == 200, "mode switch to cloud refused: %s %s", st, body) + m = wait_mode(ctx, fc, "cloud", timeout=90) + ev["mode_cloud"] = m + ctx.log("mode after switch: %s", m) + ctx.check(m and m.get("mode") == "cloud" and m.get("controller") == "running", + "cloud controller did not come up: %s", m) + # the connect-time service probe is the evidence of a live cloud session + t0 = time.time() + probe = None + while time.time() - t0 < 120: + ctx.checkpoint() + try: + mt = os.stat(GF_LATEST).st_mtime + if probe_before is None or mt > probe_before: + with open(GF_LATEST) as f: + probe = json.load(f) + break + except (OSError, ValueError): + pass + time.sleep(2) + ev["gf_probe"] = probe + ctx.log("cloud service probe: %s", probe) + st, cam1 = fc.get("/cam/status") + ev["cam_during_cloud"] = cam1 + ctx.check(st == 200 and isinstance(cam1, dict), "camera status lost during cloud mode (%s)", st) + + st, body = fc.post("/mode", data={"controller": "grbl"}) + ctx.log("POST /mode controller=grbl -> %s %s", st, body) + ctx.check(st == 200, "mode switch back to grbl refused: %s %s", st, body) + m = wait_mode(ctx, fc, "grbl", timeout=120) + ev["mode_after"] = m + ctx.log("mode after switch back: %s", m) + ctx.check(m and m.get("mode") == "grbl" and m.get("controller") == "running", + "grbl controller did not come back: %s", m) + ctx.check(m.get("motion") != "fault", "motion fault after the switch") + ctx.sleep(3) + ctx.check(grbl_port_open(), "Grbl port not open after the switch back") + with ctx.grbl() as g: + st = g.status_report()["state"] + ev["grbl_state"] = st + ctx.log("grbl state after: %s", st) + ctx.check(st.startswith("Idle") or st.startswith("Alarm"), "grbl reports %s", st) + ctx.check(probe is not None, "the cloud controller never recorded a service probe " + "(no credentials, no network, or the service refused) - cloud mode not proven") + st, cam2 = fc.get("/cam/status") + ev["cam_after"] = cam2 + ctx.check(st == 200, "camera status lost after the switch back") + + +@test("cloud.gfhome-homing", title="Glowforge web-service homing ($H with homing_mode=gfcloud)", + subsystem="cloud", kind="operator", est_min=5, + covers=_CLOUD_COVERS + [("grblhal-glowforge", "src/**")], requires=["cloud.mode-switch"], + steps=["homing_mode = gfcloud and cloud credentials configured; bed clear, lid closed.", + "Watch the gantry: the service drives it to the corner with camera corrections."], + description="In grbl mode, $H runs gfhome: the web-service homing session with the " + "head-accelerometer motion witness. The controller returns to Idle with " + "homed:true within the session timeout, and the operator confirms the head " + "reached the home corner.") +def gfhome_homing(ctx): + fc = ctx.forgectrl + ev = ctx.evidence + settings = fc.settings() + hm = settings.get("homing_mode") + ev["homing_mode"] = hm + ctx.check(hm == "gfcloud", "homing_mode is %r, this test needs gfcloud", hm) + ctx.instruct("Bed clear, lid closed, head anywhere. Watch the gantry during homing.") + with ctx.grbl() as g: + st = g.status_report()["state"] + ctx.check(st.startswith("Idle") or st.startswith("Alarm"), "controller is %s", st) + if st.startswith("Alarm"): + g.command("$X") + t0 = time.time() + g.send_raw(b"$H\n") + homed = False + state = None + while time.time() - t0 < 600: + ctx.checkpoint() + s = fc.status() + state = s.get("state") + try: + gs = g.status_report()["state"] + except hw.HwError: + gs = "?" + if s.get("homed") and gs.startswith("Idle"): + homed = True + break + if gs.startswith("Alarm"): + break + time.sleep(2) + ev["homing_s"] = round(time.time() - t0, 1) + ev["homed"] = homed + ctx.log("homing: homed=%s after %.1f s (kernel %s, grbl %s)", homed, ev["homing_s"], state, gs) + ctx.check(homed, "homing did not complete (grbl %s)", gs) + ctx.confirm("Did the head travel to the home corner under camera corrections and stop there?") diff --git a/forgetest/forgetest/suite/cooling.py b/forgetest/forgetest/suite/cooling.py new file mode 100644 index 0000000..e3f5e26 --- /dev/null +++ b/forgetest/forgetest/suite/cooling.py @@ -0,0 +1,114 @@ +"""cooling.* - the cooling engine: flow verification through forgectrl's +diagnostics runner (the same check the fire gate runs), and the fan +profile returning to idle after motion.""" +import time + +from ..catalog import test +from .. import hw + +_COOL_COVERS = [("forgectrl", "src/cool.*"), ("forgectrl", "src/diag.*"), + ("grblhal-glowforge", "src/gfcool*"), ("kernel-module-glowforge", "src/thermal*"), + ("kernel-module-glowforge", "src/pic*")] + + +@test("cooling.flow-verify", title="Coolant flow check separates flow from no-flow", + subsystem="cooling", kind="auto", est_min=4, + covers=_COOL_COVERS, requires=["kernel.latch-locked-idle"], + steps=["Coolant loop normal (pump on); the machine idle. The controller is suspended by " + "forgectrl for the duration (about 3 minutes)."], + description="forgectrl's flow-verify diagnostic: one heater window with the pump on and " + "one with it commanded off, judged against the configured threshold. PASS = " + "the threshold separates the two readings with the margins forgectrl reports; " + "a thin margin is recorded as a warning.") +def flow_verify(ctx): + fc = ctx.forgectrl + ev = ctx.evidence + st, body = fc.get("/diag/status") + ctx.check(st == 200 and isinstance(body, dict), "GET /diag/status -> %s", st) + ctx.check(not body.get("running"), "a diagnostic is already running (%s)", body.get("tool")) + st, body = fc.post("/diag/flow-verify") + ctx.log("POST /diag/flow-verify -> %s %s", st, body if isinstance(body, dict) else "") + ctx.check(st == 200, "could not start flow-verify (%s %s)", st, body) + last_phase = None + result = None + t0 = time.time() + try: + while time.time() - t0 < 900: + ctx.checkpoint() + st, d = fc.get("/diag/status") + if st == 200 and isinstance(d, dict): + if d.get("phase") != last_phase: + last_phase = d.get("phase") + ctx.log("phase: %s (down %.1f C, up %.1f C)", last_phase, d.get("down_c", 0), d.get("up_c", 0)) + if not d.get("running") and d.get("result") is not None: + result = d.get("result") + for line in d.get("log", [])[-12:]: + ctx.log(" diag: %s", line) + break + time.sleep(2) + except BaseException: + fc.post("/diag/abort") + raise + ctx.check(result is not None, "flow-verify did not finish within 15 minutes") + ev["result"] = result + ctx.check("error" not in result, "flow-verify error: %s", result.get("error")) + ctx.log("verdict: pass=%s threshold=%s flow_rise=%s noflow_rise=%s margins %s/%s thin=%s", + result.get("pass"), result.get("threshold"), result.get("flow_rise"), + result.get("noflow_rise"), result.get("margin_flow"), result.get("margin_noflow"), + result.get("thin_margin")) + ctx.check(result.get("pass") is True, "the threshold does not separate flow from no-flow: %s", result) + if result.get("thin_margin"): + ctx.log("WARNING: thin margin - run flow-calibrate") + ctx.check(fc.wait_idle(120, abort=ctx.aborted), "machine did not return to idle after the diagnostic") + + +@test("cooling.fans-quiet-after-motion", title="Fan profile returns to idle after motion and after M8/M9", + subsystem="cooling", kind="auto", est_min=2, + covers=_COOL_COVERS + [("forgectrl", "src/super.c")], requires=["motion.pacing"], + steps=["Bed clear; the head needs 20 mm of free +X travel."], + description="A dry jog and an M8/M9 cycle must not leave the run fan profile on: within " + "the cooldown the exhaust/intake tachs return to the idle level seen before.") +def fans_quiet(ctx): + fc = ctx.forgectrl + ev = ctx.evidence + + def fans(): + s = fc.status() + return dict(s.get("fans") or {}) + + before = fans() + ev["before"] = before + ctx.log("fans before: %s", before) + with ctx.grbl() as g: + st = g.status_report() + ctx.check(st["state"].startswith("Idle"), "controller is %s", st["state"]) + g.command("G91") + g.command("$J=G91X20F2400") + t0 = time.time() + while time.time() - t0 < 30 and not g.status_report()["state"].startswith("Idle"): + ctx.sleep(0.2) + g.command("$J=G91X-20F2400") + while time.time() - t0 < 60 and not g.status_report()["state"].startswith("Idle"): + ctx.sleep(0.2) + g.command("M8") + ctx.sleep(3) + during = fans() + ev["during_m8"] = during + ctx.log("fans during M8: %s", during) + g.command("M9") + g.command("G90") + # cooldown: forgectrl's engine takes cool_cooldown_s (default tens of seconds) + settle = None + t0 = time.time() + while time.time() - t0 < 240: + ctx.sleep(5) + now = fans() + close = all(abs(now.get(k, 0) - before.get(k, 0)) <= max(150, 0.15 * max(before.get(k, 0), 1)) + for k in ("exhaust", "intake_1", "intake_2")) + if close: + settle = time.time() - t0 + break + ev["after"] = fans() + ev["settle_s"] = round(settle, 1) if settle is not None else None + ctx.log("fans after: %s (settled in %s s)", ev["after"], ev["settle_s"]) + ctx.check(settle is not None, "fans did not return to the idle profile within 240 s: %s", ev["after"]) diff --git a/forgetest/forgetest/suite/forgectrl.py b/forgetest/forgetest/suite/forgectrl.py new file mode 100644 index 0000000..5299038 --- /dev/null +++ b/forgetest/forgetest/suite/forgectrl.py @@ -0,0 +1,182 @@ +"""forgectrl.* - the machine-services daemon's API, access control, and panel.""" +import json +import socket + +from ..catalog import test +from .. import hw + +_COVERS_AUTH = [("forgectrl", "src/auth.*"), ("forgectrl", "src/main.c")] + + +def lan_ip(): + """The board's own non-loopback IPv4 (the address a LAN client would + use), or None.""" + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + s.connect(("10.255.255.255", 9)) + ip = s.getsockname()[0] + except OSError: + ip = None + finally: + s.close() + if ip and not ip.startswith("127."): + return ip + return None + + +@test("forgectrl.auth", title="API access control", subsystem="forgectrl", kind="auto", est_min=1, + covers=_COVERS_AUTH, + description="Every state-changing endpoint refuses an unauthenticated write; a non-literal " + "Host, a non-literal Origin and a cross-site Sec-Fetch-Site are refused; the " + "cooling report channel refuses a non-loopback peer; the fuse view is token-gated; " + "the flash and factory-restore chain is refused unauthenticated.") +def auth(ctx): + fc = ctx.forgectrl + ev = ctx.evidence + + st, body = fc.get("/status") + ctx.log("GET /status -> %s", st) + ctx.check(st == 200, "GET /status -> %s", st) + + # unauthenticated writes: every one must be refused before it acts + for path, params in (("/controller/stop", None), ("/controller/start", None), + ("/mode", {"controller": "grbl"}), ("/settings", {"ui_units": "mm"}), + ("/diag/flow-verify", None), ("/diag/abort", None), + ("/update/apply", None), ("/boot", {"slot": "a"}), + ("/system/reboot", None), ("/restore/factory", None)): + st, body = fc.post(path, params=params, auth=False) + ctx.log("POST %s (no token) -> %s %s", path, st, body if isinstance(body, dict) else "") + ev["noauth " + path] = st + ctx.check(st == 403, "POST %s without a token -> %s, expected 403", path, st) + ctx.check(isinstance(body, dict) and body.get("error") == "authentication required", + "POST %s without a token: unexpected body %r", path, body) + + # the upload sink refuses during body parse; only the status is asserted + st, body = fc.post("/update/upload", data=b"not a firmware archive", auth=False, + headers={"Content-Type": "application/octet-stream"}) + ev["noauth /update/upload"] = st + ctx.log("POST /update/upload (no token) -> %s", st) + ctx.check(st in (400, 403), "POST /update/upload without a token -> %s", st) + + # origin checks (read endpoint, so only the origin layer decides) + st, body = fc.get("/status", headers={"Host": "evil.example.net"}) + ev["host_name"] = st + ctx.log("GET /status Host=evil.example.net -> %s", st) + ctx.check(st == 403, "a DNS-name Host was accepted (%s)", st) + st, body = fc.get("/status", headers={"Origin": "http://evil.example.net"}) + ev["origin_name"] = st + ctx.log("GET /status Origin=http://evil.example.net -> %s", st) + ctx.check(st == 403, "a DNS-name Origin was accepted (%s)", st) + st, body = fc.get("/status", headers={"Sec-Fetch-Site": "cross-site"}) + ev["sfs_cross"] = st + ctx.log("GET /status Sec-Fetch-Site=cross-site -> %s", st) + ctx.check(st == 403, "a cross-site fetch was accepted (%s)", st) + st, body = fc.get("/status", headers={"Sec-Fetch-Site": "same-origin", "Origin": "http://127.0.0.1:8080"}) + ctx.check(st == 200, "same-origin literal Origin refused (%s)", st) + + # the token opens the fuse view; without it, refused + st, body = fc.get("/fuse-identity", auth=False) + ev["fuse_noauth"] = st + ctx.check(st == 403, "GET /fuse-identity without token -> %s", st) + st, body = fc.get("/fuse-identity") + ev["fuse_auth"] = st + ctx.log("GET /fuse-identity with token -> %s", st) + ctx.check(st == 200, "GET /fuse-identity with the token -> %s", st) + + # the cooling report channel: loopback only, even with a token + ip = lan_ip() + ev["lan_ip"] = ip + ctx.check(ip, "cannot determine the board's LAN address") + port = fc.base.rsplit(":", 1)[-1] + lan = hw.Forgectrl("http://%s:%s" % (ip, port), token=fc.token) + st, body = lan.post("/cool/state", params={"mode": "idle", "armed": "0"}) + ev["cool_state_from_lan"] = st + ctx.log("POST /cool/state from %s -> %s %s", ip, st, body if isinstance(body, dict) else "") + ctx.check(st == 403 and isinstance(body, dict) and body.get("error") == "loopback only", + "/cool/state accepted a non-loopback peer (%s %r)", st, body) + + +@test("forgectrl.settings-bounds", title="Settings validation and restore", subsystem="forgectrl", + kind="auto", est_min=1, + covers=[("forgectrl", "src/settings.*"), ("forgectrl", "src/main.c")], + description="An over-length value and an out-of-range value are refused (400) and leave the " + "settings byte-identical; an in-range value is accepted (200).") +def settings_bounds(ctx): + fc = ctx.forgectrl + ev = ctx.evidence + before = fc.settings() + ev["keys"] = len(before) + + st, body = fc.post("/settings", data={"gf_serial": "X" * 300}) + ev["overlong"] = st + ctx.log("POST /settings gf_serial=<300 chars> -> %s %s", st, body if isinstance(body, dict) else "") + ctx.check(st == 400, "over-length value -> %s, expected 400", st) + + st, body = fc.post("/settings", data={"laser_disarm_s": "99999"}) + ev["out_of_range"] = st + ctx.log("POST /settings laser_disarm_s=99999 -> %s", st) + ctx.check(st == 400, "out-of-range value -> %s, expected 400", st) + + st, body = fc.post("/settings", data={"no_such_key_forgetest": "1"}) + ev["unknown_key"] = st + ctx.log("POST /settings no_such_key_forgetest=1 -> %s", st) + ctx.check(st in (400, 404), "unknown key -> %s, expected 400", st) + + after = fc.settings() + ctx.check(json.dumps(after, sort_keys=True) == json.dumps(before, sort_keys=True), + "settings changed after refused writes") + ctx.log("settings unchanged after the refused writes") + + # an accepted in-range write: rewrite a present key with its own value + key = None + for k in ("ui_units", "laser_disarm_s", "cool_flow_rise", "rail_settle_s"): + v = before.get(k) + if isinstance(v, str) and v != "": + key = k + break + if key is None: + key, val = "ui_units", "mm" + ctx.log("no settable key is present; writing %s=%s (recorded in evidence)", key, val) + else: + val = before[key] + st, body = fc.post("/settings", data={key: val}) + ev["accepted"] = {"key": key, "value": val, "status": st} + ctx.log("POST /settings %s=%s -> %s", key, val, st) + ctx.check(st == 200, "in-range write -> %s, expected 200", st) + final = fc.settings() + others_before = {k: v for k, v in before.items() if k != key} + others_after = {k: v for k, v in final.items() if k != key} + ctx.check(others_before == others_after, "other settings changed by the write") + ctx.check(final.get(key) == val, "%s reads back %r, wrote %r", key, final.get(key), val) + + +@test("forgectrl.panel-serves", title="Control panel and status endpoints", subsystem="forgectrl", + kind="auto", est_min=1, + covers=[("forgectrl", "src/ui.*"), ("forgectrl", "src/status.*"), ("forgectrl", "src/cam.c"), + ("forgectrl", "src/main.c")], + description="The panel page is served, /status carries the machine telemetry the panel and " + "the acceptance tool read, and /cam/status answers.") +def panel_serves(ctx): + fc = ctx.forgectrl + ev = ctx.evidence + st, body = fc.get("/", raw=True) + ev["panel_status"] = st + ctx.log("GET / -> %s (%d bytes)", st, len(body) if body else 0) + ctx.check(st == 200, "GET / -> %s", st) + text = body.decode("utf-8", "replace") + ctx.check(" %s %s", st, cam) + ctx.check(st == 200 and isinstance(cam, dict) and "running" in cam, "GET /cam/status -> %s", st) diff --git a/forgetest/forgetest/suite/image.py b/forgetest/forgetest/suite/image.py new file mode 100644 index 0000000..1970d91 --- /dev/null +++ b/forgetest/forgetest/suite/image.py @@ -0,0 +1,156 @@ +"""image.* - the post-flash health of the running image (always-required).""" +import glob +import gzip +import os +import re +import stat + +from ..catalog import test +from .. import hw + + +def _read(path, default=None): + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + return f.read() + except OSError: + return default + + +def kernel_config(): + """The running kernel's config as a dict, from /proc/config.gz or + /boot/config-. None when neither is available.""" + try: + with gzip.open("/proc/config.gz", "rb") as f: + text = f.read().decode("utf-8", "replace") + except OSError: + text = None + if text is None: + rel = _read("/proc/sys/kernel/osrelease", "").strip() + text = _read("/boot/config-%s" % rel) if rel else None + if text is None: + return None + cfg = {} + for line in text.splitlines(): + m = re.match(r"^(CONFIG_[A-Z0-9_]+)=(.*)$", line) + if m: + cfg[m.group(1)] = m.group(2) + m = re.match(r"^# (CONFIG_[A-Z0-9_]+) is not set$", line) + if m: + cfg[m.group(1)] = "n" + return cfg + + +def fds_of(pid): + out = [] + try: + for fd in os.listdir("/proc/%d/fd" % pid): + try: + out.append(os.readlink("/proc/%d/fd/%s" % (pid, fd))) + except OSError: + pass + except OSError: + pass + return out + + +@test("image.health", title="Post-flash image health", subsystem="image", kind="auto", + always=True, est_min=1, + covers=[("forgectrl", "init/**"), ("forgectrl", "src/main.c"), ("forgectrl", "src/auth.c"), + ("forgectrl", "CMakeLists.txt"), ("grblhal-glowforge", "src/boards/**"), + ("grblhal-glowforge", "CMakeLists.txt"), ("kernel-module-glowforge", "**"), + ("linux-fslc", "**")], + description="The image that is running is the image the manifest describes, with the " + "kernel options, the module, the daemon ownership, the init ordering, and the " + "file modes the release depends on.") +def image_health(ctx): + ev = ctx.evidence + manifest = ctx.runner.manifest + + # 1. version stamp + ver = (_read("/etc/forgefirm-version", "") or "").strip() + ev["forgefirm_version"] = ver + ctx.log("forgefirm-version: %s (manifest: %s)", ver, manifest.version) + ctx.check(ver == manifest.version, "/etc/forgefirm-version %r != manifest %r", ver, manifest.version) + + # 2. kernel options + cfg = kernel_config() + ctx.check(cfg is not None, "kernel config unavailable (/proc/config.gz, /boot/config-*)") + for opt, want in (("CONFIG_PREEMPT", ("y",)), ("CONFIG_IMX2_WDT", ("y", "m")), + ("CONFIG_PANIC_ON_OOPS", ("y",))): + val = cfg.get(opt, "n") + ev[opt] = val + ctx.log("%s=%s", opt, val) + ctx.check(val in want, "%s=%s, expected one of %s", opt, val, want) + rel = _read("/proc/sys/kernel/osrelease", "").strip() + ev["kernel_release"] = rel + mods = manifest.platform.get("kernel_modules") or [] + ctx.log("kernel release %s (manifest modules dirs: %s)", rel, ",".join(mods)) + ctx.check(not mods or rel in mods, "running kernel %r is not the manifest's %s", rel, mods) + + # 3. the module and its sysfs + ctx.check(os.path.isdir("/sys/module/glowforge"), "glowforge.ko is not loaded") + state = hw.sysfs_read("cnc/state") + ev["cnc_state"] = state + ctx.check(state is not None, "/sys/glowforge/cnc/state unreadable") + ctx.log("cnc/state: %s", state) + free = hw.sysfs_int("cnc/free") + ev["cnc_free"] = free + ctx.check(free is not None and free > 0, "cnc/free unreadable or zero (ring not mapped?)") + ctx.log("cnc/free: %s bytes", free) + ctx.check(hw.sysfs_read("cnc/interlock_circuit") is not None, "cnc/interlock_circuit unreadable") + + # 4. forgectrl holds /dev/glowforge and supervises the controller + pids = hw.pidof("forgectrl") + ev["forgectrl_pids"] = pids + ctx.check(pids, "forgectrl is not running") + holders = [p for p in pids if any(l == "/dev/glowforge" for l in fds_of(p))] + ev["pulse_device_holders"] = holders + ctx.log("forgectrl pids %s, holding /dev/glowforge: %s", pids, holders) + ctx.check(holders, "no forgectrl process holds /dev/glowforge") + st, mode = ctx.forgectrl.get("/mode") + ev["mode"] = mode + ctx.check(st == 200 and isinstance(mode, dict), "GET /mode -> %s", st) + ctx.log("mode: %s", mode) + ctx.check(mode.get("controller") in ("running", "standby"), + "controller is %r (expected running or standby)", mode.get("controller")) + ctx.check(mode.get("motion") != "fault", "supervisor reports a motion fault") + + # 5. init ordering: controllers stop before the daemon + k = sorted(os.path.basename(p) for p in glob.glob("/etc/rc6.d/K*")) + ev["rc6_kill"] = k + kg = [x for x in k if "grblhal" in x] + kf = [x for x in k if x.endswith("forgectrl")] + ctx.log("rc6.d: %s", " ".join(k)) + ctx.check(kg and kf, "rc6.d lacks the grblhal/forgectrl kill links") + ctx.check(kg[0] < kf[0], "controller kill link %s must sort before forgectrl's %s", kg[0], kf[0]) + + # 6. logging lever present, no userspace watchdog daemon + logging = [n for n in ("forgefirm-logging", "forgefirm-logrotate") if os.path.exists("/etc/init.d/" + n)] + ev["logging_init"] = logging + ctx.check(logging, "no ForgeFIRM logging/logrotate init script") + ctx.check(not os.path.exists("/etc/init.d/watchdog"), "a userspace watchdog init script is present") + # kernel threads (the imx2_wdt kthread is expected) have an empty cmdline + user_wd = [p for p in hw.pidof("watchdog") if _read("/proc/%d/cmdline" % p, "")] + ev["userspace_watchdog_pids"] = user_wd + ctx.check(not user_wd, "a userspace watchdog daemon is running: %s", user_wd) + + # 7. file modes and space + for path in ("/data/forgefirm/panel.token", "/data/forgefirm.conf"): + if os.path.exists(path): + m = stat.S_IMODE(os.stat(path).st_mode) + ev[path] = "%o" % m + ctx.log("%s mode %o", path, m) + ctx.check(m == 0o600, "%s mode %o, expected 600", path, m) + if os.path.isdir("/data"): + s = os.statvfs("/data") + free_mb = s.f_bavail * s.f_frsize // (1024 * 1024) + ev["data_free_mb"] = free_mb + ctx.log("/data free: %d MiB", free_mb) + ctx.check(free_mb >= 20, "/data has only %d MiB free", free_mb) + + # 8. the manifest itself is coherent + ctx.check(manifest.content_sha and len(manifest.content_sha) == 64, "manifest content_sha256 missing") + ctx.check("kernel-module-glowforge" in manifest.components, "manifest lacks kernel-module-glowforge") + ctx.check("linux-fslc" in manifest.components, "manifest lacks the kernel entry") + ctx.log("manifest %s: %d components", manifest.content_sha[:12], len(manifest.components)) diff --git a/forgetest/forgetest/suite/kernel.py b/forgetest/forgetest/suite/kernel.py new file mode 100644 index 0000000..e9d5d28 --- /dev/null +++ b/forgetest/forgetest/suite/kernel.py @@ -0,0 +1,486 @@ +"""kernel.* - glowforge.ko safety readbacks and the pulse-engine drills. + +The drills are the bench scripts `scripts/bench/gate_a_kernel_drills.py` +(K1/K2/K3) and `scripts/bench/fire_test.py` (A/B/U) with their proven +sequences kept intact; they run under a hardware takeover (forgectrl and +the controller stopped, the pulse device free) and judge the software +witnesses: cnc/state, laser_enable (the FIRE line), laser_on and +laser_on_sampled (the gated LASER_ON output), interlock_circuit bit 3 +(the commanded latch), faults and underruns. Every drill forces duty to +zero before any FIRE bit, keeps motor_lock=15 (no axis moves), and +re-locks the latch on every exit path. K3 and fire B/U unlock the latch +for their run and therefore refuse to proceed while laser_pgood reports +the HV supply good (the operator opens the lid: the safety chain holds +HV off). +""" +import errno +import os +import struct +import time + +try: + import fcntl +except ImportError: # host unit tests import the suite off-target + fcntl = None + +from ..catalog import test +from .. import hw +from ..runner import Failed + +# interlock_circuit bits (UAPI.md): bit 3 = the driven latch line, set = locked. +LATCH_BIT = 1 << 3 + +TICK_HZ = 10000 +FIRE = b"\x10" +PAD = b"\x00" +XSTEP = b"\x01" +POWER0 = bytes([0x80]) # power byte, duty 0 + +_KERNEL_COVERS = [("kernel-module-glowforge", "**"), ("linux-fslc", "**")] + + +# ---------------------------------------------------------------- helpers + +def wr(attr, val): + hw.sysfs_write(attr, val) + + +def rd(attr): + v = hw.sysfs_read(attr) + if v is None: + raise Failed("cannot read %s" % attr) + return v + + +def rd_pos(): + with open(hw.sysfs_root() + "cnc/position", "rb") as f: + raw = f.read(32) + return struct.unpack("<5i", raw[:20]) + + +def snap(ctx, tag): + line = ("%s: state=%s laser_enable=%s laser_on=%s laser_on_sampled=%s interlock=%s" + % (tag, rd("cnc/state"), rd("cnc/laser_enable"), rd("cnc/laser_on"), + rd("cnc/laser_on_sampled"), rd("cnc/interlock_circuit"))) + ctx.log(line) + return line + + +def wait_state(ctx, want, timeout, poll=0.05): + t0 = time.time() + while time.time() - t0 < timeout: + ctx.checkpoint() + s = rd("cnc/state") + if s == want: + return s + time.sleep(poll) + return rd("cnc/state") + + +def watch_laser_until_idle(ctx, timeout): + """Tight-loop laser_enable/laser_on watch; returns (hits, end_state).""" + hits = [] + t0 = time.time() + state = "running" + n = 0 + while time.time() - t0 < timeout: + en = rd("cnc/laser_enable") + on = rd("cnc/laser_on") + if en != "0" or on != "0": + hits.append((round(time.time() - t0, 4), en, on)) + state = rd("cnc/state") + if state != "running": + break + n += 1 + if n % 200 == 0: + ctx.checkpoint() + return hits, state + + +class PulseDevice: + """Exclusive hold of /dev/glowforge for one drill.""" + + def __init__(self, ctx): + self.ctx = ctx + self.fd = None + + def __enter__(self): + try: + self.fd = os.open("/dev/glowforge", os.O_WRONLY) + except OSError as e: + if e.errno == errno.EBUSY: + raise Failed("/dev/glowforge is busy - the takeover did not free the pulse device") + raise + fcntl.flock(self.fd, fcntl.LOCK_EX) + return self + + def write(self, data): + os.write(self.fd, data) + + def rewind(self): + os.lseek(self.fd, 1, os.SEEK_SET) + + def __exit__(self, *exc): + try: + wr("cnc/laser_latch", 1) # re-lock unconditionally + finally: + fcntl.flock(self.fd, fcntl.LOCK_UN) + os.close(self.fd) + return False + + +def require_hv_not_good(ctx): + """K3 and fire B/U unlock the latch: refuse while HV reports good. + Called before the takeover; gives the operator one chance to drop it + (open the lid).""" + pgood = rd("cnc/laser_pgood") + if pgood != "0": + ctx.log("laser_pgood=%s: the HV supply reports good", pgood) + ctx.instruct("This drill unlocks the laser latch with a zero-duty stream and must run " + "with the HV supply NOT good. Open the lid (the safety chain holds HV off), " + "then Done.") + pgood = rd("cnc/laser_pgood") + ctx.evidence["laser_pgood"] = pgood + ctx.check(pgood == "0", "laser_pgood=%s (HV supply reports good) - refusing the latch unlock", pgood) + + +def check_hv_not_good(ctx): + """The hard check right before an unlock (no prompt: forgectrl is down).""" + pgood = rd("cnc/laser_pgood") + ctx.check(pgood == "0", "laser_pgood=%s (HV supply reports good) - refusing the latch unlock", pgood) + + +# ---------------------------------------------------------------- readbacks + +@test("kernel.latch-locked-idle", title="Laser latch locked at idle", subsystem="kernel", + kind="auto", always=True, est_min=1, + covers=_KERNEL_COVERS + [("forgectrl", "src/super.c"), + ("grblhal-glowforge", "src/glowforge_laser.c"), + ("grblhal-glowforge", "src/driver.c")], + description="With the machine idle the kernel latch reads locked, the FIRE line is not " + "driven, no LASER_ON sample is seen, and no stepper fault is pending; forgectrl " + "agrees.") +def latch_locked_idle(ctx): + ev = ctx.evidence + state = hw.sysfs_read("cnc/state") + ev["cnc_state"] = state + ctx.log("cnc/state: %s", state) + ctx.check(state is not None, "cnc/state unreadable") + ctx.check(state in ("idle", "disabled"), "machine is %r, run this test at idle", state) + + ilk = hw.sysfs_int("cnc/interlock_circuit") + ev["interlock_circuit"] = ilk + ctx.check(ilk is not None, "cnc/interlock_circuit unreadable") + ctx.log("interlock_circuit: %d (0x%x)", ilk, ilk) + ctx.check(ilk & LATCH_BIT, "latch line reads unlocked at idle (bit 3 clear)") + + fire = hw.sysfs_int("cnc/laser_enable") + ev["laser_enable"] = fire + ctx.log("laser_enable (FIRE line): %s", fire) + ctx.check(fire == 0, "FIRE line driven at idle (laser_enable=%s)", fire) + + on = hw.sysfs_int("cnc/laser_on") + on_s = hw.sysfs_int("cnc/laser_on_sampled") + ev["laser_on"] = on + ev["laser_on_sampled"] = on_s + ctx.log("laser_on: %s, laser_on_sampled: %s", on, on_s) + ctx.check(on == 0, "LASER_ON active at idle") + ctx.check(on_s == 0, "LASER_ON samples seen at idle (%s)", on_s) + + faults = hw.sysfs_int("cnc/faults") + ev["faults"] = faults + ctx.log("faults: %s", faults) + ctx.check(faults == 0, "stepper faults pending: %s", faults) + + st = ctx.forgectrl.status() + ev["forgectrl_laser_locked"] = st.get("laser_locked") + ctx.log("forgectrl /status laser_locked=%s state=%s", st.get("laser_locked"), st.get("state")) + ctx.check(st.get("laser_locked") is True, "forgectrl reports the latch unlocked") + + +# ---------------------------------------------------------------- K1 + K2 + +@test("kernel.k1-k2", title="Controlled-stop floor and resume honors the locked latch", + subsystem="kernel", kind="auto", hardware="takeover", always=True, est_min=2, + covers=_KERNEL_COVERS, + requires=["kernel.latch-locked-idle"], + description="K1: a controlled stop mid-run ramps the step frequency down (tens of ms), " + "never consumes the tail as a burst or hangs. K2: with the latch locked, a " + "stop inside the leading pads and a resume with a positive waypoint replays " + "a 2 s FIRE window with laser_enable/laser_on at 0 throughout. Motors locked; " + "duty zero.") +def k1_k2(ctx): + ev = ctx.evidence + with ctx.takeover(): + # ---- K1 + stream = POWER0 + PAD * (6 * TICK_HZ) + ctx.log("K1: %d bytes = %.1f s of pads at %d Hz, ramp 125000 Hz/s", + len(stream), len(stream) / TICK_HZ, TICK_HZ) + snap(ctx, "K1 pre") + wr("cnc/motor_lock", 15) + wr("cnc/laser_latch", 1) + wr("cnc/ramp_rate", 125000) + wr("cnc/step_freq", TICK_HZ) + with PulseDevice(ctx) as dev: + dev.rewind() + wr("cnc/enable", 1) + ctx.sleep(0.5) + dev.write(stream) + wr("cnc/run", 1) + ctx.sleep(1.5) # well past the accel ramp + st = rd("cnc/state") + ctx.check(st == "running", "K1: expected running before the stop, got %s", st) + t0 = time.time() + wr("cnc/stop", 1) + while time.time() - t0 < 5: + if rd("cnc/state") != "running": + break + dt = time.time() - t0 + state = rd("cnc/state") + faults = rd("cnc/faults") + ev["k1"] = {"stop_to_idle_s": round(dt, 4), "state": state, "faults": faults} + ctx.log("K1 controlled stop: state=%s after %.4f s, faults=%s", state, dt, faults) + # drain the paused remainder laser-less so the device ends clean + wr("cnc/resume", 0) + wait_state(ctx, "running", 2, poll=0.005) + wait_state(ctx, "idle", 10) + ctx.check(state == "idle", "K1: state %s after the stop", state) + ctx.check(dt >= 0.02, "K1: stop consumed the tail as a burst (%.4f s) - decel floor broken", dt) + ctx.check(dt <= 3.0, "K1: stop took %.4f s", dt) + ctx.check(faults == "0", "K1: faults=%s", faults) + ctx.log("K1 PASS: decelerating tail %.4f s, no burst, no fault", dt) + + # ---- K2 + step_sec = (XSTEP + PAD * 4) * 1000 # 1000 X steps at 2 kHz (masked) + stream = (POWER0 + PAD * TICK_HZ + step_sec + PAD * (TICK_HZ // 2) + + FIRE * (2 * TICK_HZ) + PAD * TICK_HZ) + ctx.log("K2: %d bytes = %.1f s; latch stays LOCKED; waypoint +200; motor_lock=15", + len(stream), len(stream) / TICK_HZ) + snap(ctx, "K2 pre") + wr("cnc/motor_lock", 15) + wr("cnc/laser_latch", 1) + wr("cnc/ramp_rate", 125000) + wr("cnc/step_freq", TICK_HZ) + with PulseDevice(ctx) as dev: + dev.rewind() + wr("cnc/enable", 1) + ctx.sleep(0.5) + dev.write(stream) + pos_before = rd_pos() + wr("cnc/run", 1) + ctx.sleep(0.4) # inside the initial pads + wr("cnc/stop", 1) + state = wait_state(ctx, "idle", 5, poll=0.01) + ctx.check(state == "idle", "K2: controlled stop did not reach idle (state=%s)", state) + ctx.log("K2: paused inside the pads; resuming with waypoint +200 (latch LOCKED)") + wr("cnc/resume", 200) + wait_state(ctx, "running", 2, poll=0.005) + hits, state = watch_laser_until_idle(ctx, 20) + pos_after = rd_pos() + snap(ctx, "K2 post") + ev["k2"] = {"hits": hits[:10], "end_state": state, "pos_before": pos_before, + "pos_after": pos_after, "laser_on_sampled": rd("cnc/laser_on_sampled"), + "underruns": rd("cnc/underruns"), "faults": rd("cnc/faults")} + ctx.log("K2 done: state=%s pos before=%s after=%s", state, pos_before, pos_after) + ctx.check(not hits, "K2: laser asserted with the latch locked: %s", hits[:10]) + if pos_before[:3] == pos_after[:3]: + ctx.log("K2 NOTE: position did not advance under motor_lock (waypoint completion " + "unconfirmed by the counter on this board)") + ctx.log("K2 PASS: FIRE window replayed after the resume waypoint with " + "laser_enable/laser_on at 0 throughout") + + +# ---------------------------------------------------------------- K3 + +@test("kernel.k3-unlock", title="Mid-run latch unlock never re-arms FIRE", subsystem="kernel", + kind="operator", hardware="takeover", always=True, est_min=2, + covers=_KERNEL_COVERS, + requires=["kernel.k1-k2"], + steps=["If the HV supply reports good the drill asks you to open the lid first (the " + "safety chain holds HV off); zero duty throughout."], + description="Stream of FIRE bits run with the latch locked (laser-less by the run-start " + "guard); the latch is unlocked during the accel ramp. The unlock drives the " + "latch pin (interlock bit 3 clears) but must not restore the FIRE drive while " + "the run is in flight: laser_enable stays 0 for the entire run.") +def k3_unlock(ctx): + ev = ctx.evidence + require_hv_not_good(ctx) + with ctx.takeover(): + check_hv_not_good(ctx) + stream = POWER0 + FIRE * (3 * TICK_HZ) + PAD * (TICK_HZ // 2) + ctx.log("K3: %d bytes = %.1f s of FIRE bits; ramp_rate 10000 Hz/s (~0.9 s accel " + "window); unlock at t=+0.15 s", len(stream), len(stream) / TICK_HZ) + snap(ctx, "K3 pre") + wr("cnc/motor_lock", 15) + wr("cnc/laser_latch", 1) + wr("cnc/step_freq", TICK_HZ) + wr("cnc/ramp_rate", 10000) + try: + with PulseDevice(ctx) as dev: + dev.rewind() + wr("cnc/enable", 1) + ctx.sleep(0.5) + dev.write(stream) + wr("cnc/run", 1) + time.sleep(0.15) # inside the accel ramp + wr("cnc/laser_latch", 0) + ilk = rd("cnc/interlock_circuit") + ctx.log("K3: latch UNLOCKED mid-ramp; interlock=%s (bit 3 should read 0)", ilk) + hits, state = watch_laser_until_idle(ctx, 20) + snap(ctx, "K3 post") + ev["k3"] = {"interlock_after_unlock": ilk, "hits": hits[:10], "end_state": state, + "laser_on_sampled": rd("cnc/laser_on_sampled"), + "underruns": rd("cnc/underruns"), "faults": rd("cnc/faults")} + finally: + wr("cnc/laser_latch", 1) + try: + wr("cnc/ramp_rate", 125000) + except OSError: + ctx.log("WARNING: could not restore ramp_rate=125000") + ctx.check((int(ilk) & LATCH_BIT) == 0, "K3: the unlock did not drive the latch pin (interlock=%s)", ilk) + ctx.check(not hits, "K3: FIRE drive re-armed by a mid-run unlock: %s", hits[:10]) + ctx.log("K3 PASS: laser_enable stayed 0 for the entire run after the mid-ramp unlock") + + +# ---------------------------------------------------------------- FIRE A/B/U + +def _fire_stream(): + return (POWER0 + # duty zero before any FIRE bit + PAD * TICK_HZ + # 1 s baseline + FIRE * (2 * TICK_HZ) + # 2.000 s FIRE window (bounded by pads) + PAD * TICK_HZ + # 1 s gap + FIRE * (2 * TICK_HZ)) # 2.000 s FIRE window ending AT end-of-data + + +def _fire_phase(ctx, mode): + """One phase of fire_test.py; returns the evidence dict.""" + unlock = mode in ("B", "U") + underrun_mode = mode == "U" + stream = _fire_stream() + ctx.log("fire %s: stream %d bytes = %.3f s", mode, len(stream), len(stream) / TICK_HZ) + if unlock: + check_hv_not_good(ctx) + snap(ctx, "fire %s pre" % mode) + wr("cnc/motor_lock", 15) + wr("cnc/step_freq", TICK_HZ) + wr("cnc/laser_latch", 1) + underruns_before = int(rd("cnc/underruns")) + mid = None + tail = None + with PulseDevice(ctx) as dev: + dev.rewind() + wr("cnc/enable", 1) + ctx.sleep(0.5) + dev.write(stream) + pos_before = rd_pos() + if underrun_mode: + wr("cnc/streaming", 1) # end-of-data mid-run = true underrun + ctx.log("fire U: streaming=1, the terminal end-of-data will be a TRUE UNDERRUN") + try: + if unlock: + wr("cnc/laser_latch", 0) + ctx.log("fire %s: latch UNLOCKED for this run", mode) + wr("cnc/run", 1) + t0 = time.time() + state = "" + samples = [] + while time.time() - t0 < 20: + ctx.checkpoint() + state = rd("cnc/state") + dt = time.time() - t0 + en, on, ons = rd("cnc/laser_enable"), rd("cnc/laser_on"), rd("cnc/laser_on_sampled") + samples.append((round(dt, 2), state, en, on, ons)) + if mid is None and 1.5 < dt < 3.0: + mid = {"t": round(dt, 2), "laser_enable": en, "laser_on": on, + "laser_on_sampled": ons, "interlock": rd("cnc/interlock_circuit")} + ctx.log("fire %s mid (inside FIRE window): laser_enable=%s laser_on=%s " + "laser_on_sampled=%s interlock=%s", mode, en, on, ons, mid["interlock"]) + if state != "running": + break + time.sleep(0.05) + end_dt = time.time() - t0 + tail = {"state": state, "after_s": round(end_dt, 2), "laser_enable": rd("cnc/laser_enable"), + "laser_on": rd("cnc/laser_on")} + ctx.log("fire %s done: state=%s after %.1f s (laser_enable=%s)", mode, state, end_dt, + tail["laser_enable"]) + if underrun_mode: + if state == "underrun": + ctx.log("fire U: underrun state reached as EXPECTED; acking via stop") + else: + ctx.log("fire U: WARNING expected underrun state, got %s", state) + wr("cnc/stop", 1) + wr("cnc/streaming", 0) + tail["acked_state"] = rd("cnc/state") + ctx.log("fire U: acked: state=%s", tail["acked_state"]) + finally: + wr("cnc/laser_latch", 1) + if underrun_mode: + try: + wr("cnc/streaming", 0) + except OSError: + pass + pos_after = rd_pos() + snap(ctx, "fire %s post" % mode) + ev = {"mid": mid, "tail": tail, "moved": pos_before[:3] != pos_after[:3], + "underruns_before": underruns_before, "underruns_after": int(rd("cnc/underruns")), + "faults": rd("cnc/faults"), + "any_laser_on": any(s[3] != "0" or s[4] != "0" for s in samples), + "any_fire_driven": any(s[2] != "0" for s in samples)} + ctx.log("fire %s: moved=%s underruns %d->%d faults=%s", mode, ev["moved"], + ev["underruns_before"], ev["underruns_after"], ev["faults"]) + return ev + + +@test("kernel.fire-abu", title="FIRE line: latch locked, unlocked-unarmed, true underrun", + subsystem="kernel", kind="operator", hardware="takeover", always=True, est_min=3, + covers=_KERNEL_COVERS, + requires=["kernel.k1-k2"], + steps=["Phases B and U unlock the latch with a zero-duty stream: if the HV supply reports " + "good the drill asks you to open the lid first (the safety chain holds HV off)."], + description="A: latch locked, 40 000 streamed FIRE bits, nothing on the FIRE/LASER_ON " + "nets. B: latch unlocked with the chain unarmed, the FIRE line is driven " + "mid-window and LASER_ON stays off (the safety AND-gate holds), FIRE clear at " + "end-of-data. U: streaming declared, the terminal end-of-data is a true " + "underrun, the backstop drops FIRE and stop acks it.") +def fire_abu(ctx): + ev = ctx.evidence + require_hv_not_good(ctx) + with ctx.takeover(): + try: + a = _fire_phase(ctx, "A") + ev["A"] = a + ctx.check(a["mid"] is not None, "A: no mid-window sample") + ctx.check(not a["any_fire_driven"], "A: FIRE line driven with the latch locked") + ctx.check(not a["any_laser_on"], "A: LASER_ON seen with the latch locked") + ctx.check(a["tail"]["state"] == "idle", "A: ended in %s", a["tail"]["state"]) + ctx.check(not a["moved"], "A: position moved with motors locked") + ctx.log("fire A PASS: latch locked, no FIRE drive, no LASER_ON") + + b = _fire_phase(ctx, "B") + ev["B"] = b + ctx.check(b["mid"] is not None, "B: no mid-window sample") + ctx.check(b["mid"]["laser_enable"] != "0", + "B: FIRE line not driven mid-window with the latch unlocked (%s)", b["mid"]) + ctx.check(not b["any_laser_on"], "B: LASER_ON active with the chain unarmed - the AND-gate did not hold") + ctx.check(b["tail"]["state"] == "idle", "B: ended in %s", b["tail"]["state"]) + ctx.check(b["tail"]["laser_enable"] == "0", "B: FIRE still driven after end-of-data") + ctx.check(b["underruns_after"] == b["underruns_before"], "B: underrun counted on a normal completion") + ctx.log("fire B PASS: FIRE driven mid-window, LASER_ON off, FIRE clear at end-of-data") + + u = _fire_phase(ctx, "U") + ev["U"] = u + ctx.check(u["tail"]["state"] == "underrun", "U: expected the underrun state, got %s", u["tail"]["state"]) + ctx.check(u["tail"]["laser_enable"] == "0", "U: FIRE still driven after the underrun") + ctx.check(not u["any_laser_on"], "U: LASER_ON active with the chain unarmed") + ctx.check(u["tail"].get("acked_state") == "idle", "U: stop did not ack the underrun (state %s)", + u["tail"].get("acked_state")) + ctx.check(u["underruns_after"] == u["underruns_before"] + 1, + "U: underrun counter %d -> %d", u["underruns_before"], u["underruns_after"]) + ctx.log("fire U PASS: true underrun, backstop dropped FIRE, stop acked") + finally: + wr("cnc/laser_latch", 1) + try: + wr("cnc/disable", 1) # the script's safe state + except OSError: + pass + ctx.log("safe state restored: state=%s latch=LOCKED", rd("cnc/state")) diff --git a/forgetest/forgetest/suite/laser.py b/forgetest/forgetest/suite/laser.py new file mode 100644 index 0000000..4299e48 --- /dev/null +++ b/forgetest/forgetest/suite/laser.py @@ -0,0 +1,426 @@ +"""laser.* - LIVE laser tests, ported from `scripts/bench/live_fire_drills.py`. + +Every test here can emit. The page starts one only with the operator's +eye-protection / fire-watch / exhaust acknowledgment; the test then +prompts for the scrap and the button, streams a small job through the +controller, and the machine fires only after the operator presses the +physical arm button - nothing here defeats that gate, and forgetest +never touches the laser latch. One emission per test; on abort or error +the job is soft-reset (`^X`: controlled stop, latch relocked). The +witnesses are forgectrl's `/status` (`laser.emission_samples` = the +kernel's LASER_ON sample count, `hv_current_raw`, `lid_ir`) and +`/cool/status` (`armed`), sampled at ~8 Hz through the arm -> fire -> +disarm lifecycle. +""" +import time + +from ..catalog import test +from .. import hw +from ..runner import Failed + +_LASER_COVERS = [("grblhal-glowforge", "src/**"), ("kernel-module-glowforge", "**"), + ("forgectrl", "src/super.c"), ("forgectrl", "src/cool.c"), + ("forgectrl", "src/status.c"), ("forgectrl", "src/main.c")] + +ARM_CUE = ("LIVE FIRE. Eye protection on, exhaust running, fire watch and extinguisher in reach, " + "scrap under the head with room to move (%s), lid closed. When the job starts the " + "white button lights and the stream blocks until you press the physical arm button; " + "the machine fires only after your press. Ready?") + + +def sample(ctx): + """One combined /status + /cool/status sample, or None on error.""" + fc = ctx.forgectrl + try: + st1, st = fc.get("/status") + st2, cs = fc.get("/cool/status") + except hw.HwError: + return None + if st1 != 200 or st2 != 200 or not isinstance(st, dict) or not isinstance(cs, dict): + return None + return { + "t": time.time(), + "kstate": st.get("state"), + "emission": (st.get("laser") or {}).get("emission_samples"), + "pgood": (st.get("laser") or {}).get("pgood_samples"), + "faults": st.get("faults"), + "hv": st.get("hv_current_raw"), + "ir": st.get("lid_ir"), + "homed": st.get("homed"), + "armed": cs.get("armed"), + "fire_watch": cs.get("fire_watch"), + "verdict": cs.get("verdict"), + } + + +def prepare(ctx, g): + """Guarantee a clean Idle start: clear a latched Door hold or an Alarm.""" + st = g.status_report() + if "Door" in st["state"] or "Hold" in st["state"]: + g.realtime(0x18) + ctx.sleep(2) + g.drain() + st = g.status_report() + if "Alarm" in st["state"]: + ctx.log("unlock: %s", g.command("$X")) + st = g.status_report() + ctx.log("connect: %s", st["state"]) + ctx.check(st["state"].startswith("Idle"), "controller is %s, expected Idle", st["state"]) + return st + + +def stream(g, lines): + for ln in lines: + g.send_raw((ln + "\n").encode()) + + +def run_and_sample(ctx, g, job, sample_hz=8, overall_timeout=200): + """Stream the job; sample forgectrl through arm -> fire -> disarm. + Completes on: emission seen then Idle > 3 s; or armed then disarmed + with no fire, Idle > 3 s, > 15 s in; or the overall timeout.""" + samples = [] + period = 1.0 / sample_hz + s0 = sample(ctx) + if s0: + samples.append(s0) + stream(g, job) + t_start = time.time() + next_t = t_start + seen_emission = seen_armed = disarmed_now = False + idle_since = None + while time.time() - t_start < overall_timeout: + ctx.checkpoint() + now = time.time() + if now >= next_t: + smp = sample(ctx) + if smp: + samples.append(smp) + if smp["emission"] and smp["emission"] > 0: + seen_emission = True + if smp["armed"]: + seen_armed = True + disarmed_now = seen_armed and not smp["armed"] + next_t = now + period + st = g.status_report()["state"] + if st.startswith("Idle"): + if idle_since is None: + idle_since = now + idle_for = now - idle_since + if seen_emission and idle_for > 3.0: + break + if disarmed_now and (now - t_start) > 15 and idle_for > 3.0: + break + else: + idle_since = None + time.sleep(0.05) + return samples + + +class LiveJob: + """Leaves the laser commanded off on every exit; a soft reset on + abort/failure stops motion, relocks, and closes the armed window.""" + + def __init__(self, ctx, g): + self.ctx, self.g = ctx, g + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + try: + if exc_type is not None: + self.ctx.log("stopping the job: soft reset (%s)", exc_type.__name__) + self.g.realtime(0x18) + time.sleep(1) + self.g.command("M5", timeout=1) + except Exception: # noqa: BLE001 - best effort on the way out + pass + return False + + +def wait_disarm(ctx, timeout): + t0 = time.time() + while time.time() - t0 < timeout: + ctx.checkpoint() + s = sample(ctx) + if s and not s["armed"]: + return time.time() - t0 + time.sleep(0.5) + return None + + +@test("laser.emission-witness", title="Live emission witness (S400 vector mark) and job-based disarm", + subsystem="laser", kind="live", always=True, est_min=5, + covers=_LASER_COVERS, + requires=["kernel.latch-locked-idle", "kernel.k1-k2", "motion.jog-roundtrip"], + steps=["Scrap under the head with 20 mm of free +X and +Y travel; lid closed; exhaust on.", + "Press the physical button when it lights white (the arm)."], + description="A 20 mm square outline at S400/F600 in dynamic laser mode: emission_samples " + "(the kernel's LASER_ON sample count) goes nonzero during the fire window and " + "returns to 0 at Idle, HV current rises during the burn, the armed window is " + "observed, and the M2 program end disarms promptly at Idle (job-based, not " + "the 60 s idle grace). The operator confirms the mark.") +def emission_witness(ctx): + ev = ctx.evidence + with ctx.grbl() as g, LiveJob(ctx, g): + prepare(ctx, g) + base = sample(ctx) + ctx.check(base, "forgectrl /status or /cool/status unavailable") + ev["pre_fire"] = base + ctx.log("pre-fire: emission=%s hv=%s armed=%s verdict=%s", base["emission"], base["hv"], + base["armed"], base["verdict"]) + ctx.check(not base["emission"], "emission_samples nonzero before the job (%s)", base["emission"]) + ctx.instruct(ARM_CUE % "20 mm +X and +Y") + job = ["G91", "G21", "M4", "S400", + "G1 X20 F600", "G1 Y20 F600", "G1 X-20 F600", "G1 Y-20 F600", + "M5", "G90", "M2"] + samples = run_and_sample(ctx, g, job) + emis = [s["emission"] for s in samples if s["emission"] is not None] + peak = max(emis) if emis else 0 + end = emis[-1] if emis else None + hv = [s["hv"] for s in samples if s["hv"] is not None] + ir_peak = [0, 0, 0, 0] + for s in samples: + if s["ir"] and len(s["ir"]) == 4: + for i in range(4): + ir_peak[i] = max(ir_peak[i], s["ir"][i]) + armed_seen = any(s["armed"] for s in samples) + ev.update({"samples": len(samples), "emission_peak": peak, "emission_end": end, + "hv_min": min(hv) if hv else None, "hv_max": max(hv) if hv else None, + "lid_ir_peak": ir_peak, "armed_seen": armed_seen, + "pgood_peak": max((s["pgood"] for s in samples if s["pgood"] is not None), default=None)}) + ctx.log("emission_samples peak=%s end=%s; hv %s..%s; lid_ir peak %s; armed seen %s", + peak, end, ev["hv_min"], ev["hv_max"], ir_peak, armed_seen) + # X-3: job-based disarm at Idle after M2 + dt = wait_disarm(ctx, 75) + ev["disarm_after_idle_s"] = round(dt, 1) if dt is not None else None + ctx.log("time-to-disarm after Idle: %s s", ev["disarm_after_idle_s"]) + ctx.check(armed_seen, "the armed window was never observed (arm refused, or no button press)") + ctx.check(peak > 0, "no emission witnessed (emission_samples stayed 0)") + ctx.check(end == 0, "emission_samples did not return to 0 at Idle (%s)", end) + ctx.check(hv and max(hv) > min(hv), "HV current did not rise during the burn (%s..%s)", + ev["hv_min"], ev["hv_max"]) + ctx.check(dt is not None and dt < 10.0, + "the M2 job did not disarm promptly at Idle (%s s; the idle grace is ~60 s)", + ev["disarm_after_idle_s"]) + ctx.confirm("Did the laser mark a 20 mm square outline on the scrap, and is the machine now " + "idle with the button dark?") + ctx.log("PASS: emission peak %s -> 0, HV %s..%s, disarmed %.1f s after Idle, mark confirmed", + peak, ev["hv_min"], ev["hv_max"], dt) + + +@test("laser.disarm-in-hold", title="Disarm grace counts down in Hold", subsystem="laser", + kind="live", est_min=4, + covers=_LASER_COVERS, + requires=["laser.emission-witness"], + steps=["Scrap under the head with 40 mm of free +X travel; lid closed; exhaust on.", + "Press the physical button when it lights white."], + description="Arm and start a +X move at S400/F300, feed-hold it after ~2 s of motion, and " + "hold: the disarm grace must count down while held and close the armed window " + "(armed -> false) without the job resuming.") +def disarm_in_hold(ctx): + ev = ctx.evidence + with ctx.grbl() as g, LiveJob(ctx, g): + prepare(ctx, g) + ctx.instruct(ARM_CUE % "40 mm +X") + stream(g, ["G91", "G21", "M4", "S400", "G1 X40 F300"]) + ctx.log("armed; waiting for motion to start (arm + your button press)...") + t0 = time.time() + st = None + while time.time() - t0 < 180: + ctx.checkpoint() + st = g.status_report()["state"] + if st.startswith("Run"): + break + time.sleep(0.1) + ctx.check(st and st.startswith("Run"), "motion never started (state=%s) - arm refused or no press", st) + ctx.log("moving under laser: %s; feed-hold in 2 s", st) + ctx.sleep(2) + g.realtime(ord("!")) + t1 = time.time() + while time.time() - t1 < 5: + st = g.status_report()["state"] + if st.startswith("Hold"): + break + time.sleep(0.1) + ev["held_state"] = st + ctx.log("feed-held mid-move: %s; watching the disarm grace count down IN HOLD", st) + ctx.check(st.startswith("Hold"), "feed hold did not park (state %s)", st) + t0 = time.time() + disarmed_at = None + left_hold = None + while time.time() - t0 < 120: + ctx.checkpoint() + s = sample(ctx) + held = g.status_report()["state"].startswith("Hold") + if s and not s["armed"]: + disarmed_at = time.time() - t0 + break + if not held and left_hold is None: + left_hold = g.status_report()["state"] + ctx.log("note: left Hold (state=%s) before disarm", left_hold) + time.sleep(1) + ev["disarmed_after_s"] = round(disarmed_at, 1) if disarmed_at is not None else None + ev["left_hold"] = left_hold + # recover: laser off, abort out of hold + g.command("M5", timeout=1) + g.realtime(0x18) + ctx.sleep(1) + if "Alarm" in g.status_report()["state"]: + g.command("$X") + ctx.check(disarmed_at is not None, "still armed after 120 s in Hold") + ctx.check(left_hold is None, "the job left Hold (%s) before the disarm", left_hold) + ctx.log("PASS: disarmed in Hold after %.1f s", disarmed_at) + ctx.confirm("Did the head stop after ~2 s of the +X move and stay stopped, with the button " + "going dark on its own about a minute later?") + + +@test("laser.expected-stop", title="Armed kill on the expected-stop path (POST /controller/stop)", + subsystem="laser", kind="live", est_min=4, + covers=_LASER_COVERS + [("forgectrl", "src/main.c")], + requires=["laser.emission-witness"], + steps=["Scrap under the head with 40 mm of free +X and +Y travel; lid closed; exhaust on.", + "Press the physical button when it lights white.", + "The controller is left stopped until you judge the stop; the test then restarts it."], + description="Start a mark job at S400/F200; once emission is live, POST /controller/stop. " + "Emission must drop to 0 within 2.5 s and stay 0, the kernel must not be " + "running, and the supervisor's restart is a separate, operator-judged step " + "(POST /controller/start, no motion, no laser).") +def expected_stop(ctx): + ev = ctx.evidence + fc = ctx.forgectrl + with ctx.grbl() as g, LiveJob(ctx, g): + prepare(ctx, g) + ctx.instruct(ARM_CUE % "40 mm +X and +Y") + stream(g, ["G91", "G21", "M4", "S400", + "G1 X40 F200", "G1 Y40 F200", "G1 X-40 F200", "G1 Y-40 F200", + "M5", "G90", "M2"]) + t0 = time.time() + smp = None + seen = False + while time.time() - t0 < 240: + ctx.checkpoint() + smp = sample(ctx) + if smp and smp["emission"] and smp["emission"] > 0: + seen = True + break + time.sleep(0.15) + if not seen: + g.realtime(0x18) + raise Failed("no emission seen within 240 s (arm refused, or no button press)") + ctx.log("emission live (%s) - stopping the controller NOW", smp["emission"]) + t_stop = time.time() + code, body = fc.post("/controller/stop") + post_dt = time.time() - t_stop + ctx.log("POST /controller/stop -> %s %s (%.2f s)", code, body, post_dt) + trail = [] + for _ in range(40): # ~5 s at 8 Hz + s = sample(ctx) + if s: + trail.append((round(time.time() - t_stop, 2), s["emission"], s["kstate"], s["armed"])) + time.sleep(0.12) + for t in trail: + ctx.log(" post-stop %s", t) + zero_at = next((t for t, e, _, _ in trail if e == 0), None) + tail_zero = all(e == 0 for _, e, _, _ in trail[-16:]) + not_running = all(k != "running" for _, _, k, _ in trail[-16:]) + st_mode, mode = fc.get("/mode") + ev.update({"post_status": code, "post_s": round(post_dt, 2), "zero_at_s": zero_at, + "tail_zero": tail_zero, "kernel_not_running": not_running, "mode_after_stop": mode, + "trail": trail}) + ctx.log("emission first 0 at +%s s; last 2 s all zero: %s; kernel not running: %s; /mode %s", + zero_at, tail_zero, not_running, mode) + ctx.check(code == 200, "POST /controller/stop -> %s", code) + ctx.check(zero_at is not None and zero_at < 2.5, "emission did not drop within 2.5 s (first 0 at %s)", zero_at) + ctx.check(tail_zero, "emission returned after the stop") + ctx.check(not_running, "the kernel was still running after the stop") + ctx.instruct("The controller is STOPPED (supervision held). Judge the stop on the scrap - a " + "short cut, then an abrupt end - and confirm the machine is quiet; then Done to " + "restart the controller (no motion, no laser).") + st, body = fc.post("/controller/start") + ctx.log("POST /controller/start -> %s %s", st, body) + ctx.check(st == 200, "POST /controller/start -> %s", st) + ctx.sleep(6) + st, mode = fc.get("/mode") + ev["mode_after_start"] = mode + ctx.log("/mode after start: %s", mode) + ctx.check(isinstance(mode, dict) and mode.get("controller") == "running", + "controller not running after the restart: %s", mode) + ctx.log("PASS: stop in %.2f s, emission 0 at +%s s, controller restarted", post_dt, zero_at) + + +@test("laser.kill-mid-fire", title="Armed kill: SIGKILL of the controller while emitting", + subsystem="laser", kind="live", est_min=4, + covers=_LASER_COVERS, + requires=["laser.expected-stop", "motion.deadman"], + steps=["Scrap under the head with 40 mm of free +X and +Y travel; lid closed; exhaust on.", + "Press the physical button when it lights white."], + description="Start a mark job at S400/F200; once emission is live, SIGKILL the controller. " + "The supervisor's exit safing must end the fire tail within the ring's in-flight " + "window: emission drops to 0 within 2.5 s and stays 0, the kernel is not " + "running, the latch reads locked, and the controller is respawned.") +def kill_mid_fire(ctx): + import os as _os + import signal as _signal + ev = ctx.evidence + fc = ctx.forgectrl + st, m0 = fc.get("/mode") + ctx.check(st == 200 and isinstance(m0, dict) and m0.get("controller") == "running", "controller not running: %s", m0) + pid = m0.get("pid") + with ctx.grbl() as g, LiveJob(ctx, g): + prepare(ctx, g) + ctx.instruct(ARM_CUE % "40 mm +X and +Y") + stream(g, ["G91", "G21", "M4", "S400", + "G1 X40 F200", "G1 Y40 F200", "G1 X-40 F200", "G1 Y-40 F200", + "M5", "G90", "M2"]) + t0 = time.time() + smp = None + seen = False + while time.time() - t0 < 240: + ctx.checkpoint() + smp = sample(ctx) + if smp and smp["emission"] and smp["emission"] > 0: + seen = True + break + time.sleep(0.15) + if not seen: + g.realtime(0x18) + raise Failed("no emission seen within 240 s (arm refused, or no button press)") + ctx.log("emission live (%s) - SIGKILL controller pid %s NOW", smp["emission"], pid) + t_kill = time.time() + _os.kill(pid, _signal.SIGKILL) + trail = [] + for _ in range(40): # ~5 s at 8 Hz + s = sample(ctx) + if s: + trail.append((round(time.time() - t_kill, 2), s["emission"], s["kstate"], s["armed"])) + time.sleep(0.12) + for t in trail: + ctx.log(" post-kill %s", t) + zero_at = next((t for t, e, _, _ in trail if e == 0), None) + tail_zero = all(e == 0 for _, e, _, _ in trail[-16:]) + not_running = all(k != "running" for _, _, k, _ in trail[-16:]) + ilk = hw.sysfs_int("cnc/interlock_circuit") + locked = ilk is not None and bool(ilk & (1 << 3)) + ev.update({"pid": pid, "zero_at_s": zero_at, "tail_zero": tail_zero, "kernel_not_running": not_running, + "latch_locked": locked, "trail": trail}) + ctx.log("emission first 0 at +%s s; last 2 s all zero: %s; kernel not running: %s; latch locked: %s", + zero_at, tail_zero, not_running, locked) + ctx.check(zero_at is not None and zero_at < 2.5, "emission did not drop within 2.5 s (first 0 at %s)", zero_at) + ctx.check(tail_zero, "emission returned after the kill") + ctx.check(not_running, "the kernel was still running after the kill") + ctx.check(locked, "latch not locked after the kill") + t0 = time.time() + m1 = None + while time.time() - t0 < 60: + st, m1 = fc.get("/mode") + if isinstance(m1, dict) and m1.get("controller") == "running" and m1.get("pid") != pid: + break + ctx.sleep(1) + ev["mode_after"] = m1 + ctx.log("/mode after the kill: %s", m1) + ctx.check(m1 and m1.get("controller") == "running" and m1.get("pid") != pid, + "supervisor did not respawn the controller: %s", m1) + ctx.confirm("Did the cut end abruptly at the kill (a short line, no run-on), with the machine " + "quiet and the button dark now?") + ctx.log("PASS: emission 0 at +%s s after SIGKILL, latch locked, controller respawned", zero_at) diff --git a/forgetest/forgetest/suite/logs.py b/forgetest/forgetest/suite/logs.py new file mode 100644 index 0000000..e601c4e --- /dev/null +++ b/forgetest/forgetest/suite/logs.py @@ -0,0 +1,59 @@ +"""logs.* - unified logging: the log tree, tail, and the sanitized export.""" +import gzip +import io +import tarfile + +from ..catalog import test + +_LOG_COVERS = [("forgectrl", "src/logs.*"), ("forgectrl", "src/fflog.*"), ("forgectrl", "src/sanitize.*"), + ("forgectrl", "src/main.c")] + + +@test("logs.tree-tail-export", title="Log tree, tail, and sanitized export", subsystem="logs", + kind="auto", est_min=1, + covers=_LOG_COVERS, requires=["forgectrl.auth"], + description="/logs lists the loggers with their levels and files, /logs/tail returns the " + "forgectrl logger's tail, and POST /logs/export streams a sanitized tar.gz " + "bundle that contains no panel token.") +def tree_tail_export(ctx): + fc = ctx.forgectrl + ev = ctx.evidence + st, body = fc.get("/logs") + ctx.log("GET /logs -> %s", st) + ctx.check(st == 200 and isinstance(body, dict), "GET /logs -> %s", st) + loggers = body.get("loggers") or [] + names = [l.get("name") for l in loggers] if isinstance(loggers, list) else list(loggers) + ev["loggers"] = names + ctx.log("loggers: %s", names) + ctx.check("forgectrl" in names, "/logs lacks the forgectrl logger: %s", names) + + st, tail = fc.get("/logs/tail", params={"name": "forgectrl", "lines": "20"}) + ctx.log("GET /logs/tail?name=forgectrl -> %s", st) + ctx.check(st == 200, "GET /logs/tail -> %s %s", st, tail if isinstance(tail, dict) else "") + st, tail = fc.get("/logs/tail", params={"name": "no-such-logger"}) + ctx.check(st == 404, "unknown logger -> %s, expected 404", st) + + st, data = fc.post("/logs/export", raw=True) + ctx.log("POST /logs/export -> %s (%d bytes)", st, len(data) if data else 0) + ctx.check(st == 200, "POST /logs/export -> %s", st) + ev["export_bytes"] = len(data) + try: + raw = gzip.decompress(data) + tf = tarfile.open(fileobj=io.BytesIO(raw)) + members = tf.getnames() + except (OSError, tarfile.TarError, EOFError) as e: + ctx.fail("export is not a readable tar.gz: %s", e) + ev["members"] = len(members) + ctx.log("bundle: %d members, e.g. %s", len(members), members[:5]) + ctx.check(members, "empty bundle") + ctx.check(any(m.endswith("README.txt") for m in members), "sanitized bundle lacks README.txt") + token = fc.token + if token: + leaked = [] + for m in tf.getmembers(): + if m.isfile(): + content = tf.extractfile(m).read() + if token.encode() in content: + leaked.append(m.name) + ev["token_leaks"] = leaked + ctx.check(not leaked, "the sanitized bundle contains the panel token: %s", leaked) diff --git a/forgetest/forgetest/suite/motion.py b/forgetest/forgetest/suite/motion.py new file mode 100644 index 0000000..ac9ebdd --- /dev/null +++ b/forgetest/forgetest/suite/motion.py @@ -0,0 +1,459 @@ +"""motion.* - the motion controller under grblHAL: dry motion, no emission. + +Ported from `scripts/bench/pacing_test.py` (protocol-loop pacing) and +`scripts/bench/bench_m2.py` (motion-quality bench). Every move is relative +and round-trip; the laser stays latched (the tests never touch it); the +suite is the only Grbl client while a test runs. +""" +import time + +from ..catalog import test +from .. import hw +from ..runner import Failed + +_MOTION_COVERS = [("grblhal-glowforge", "src/**"), ("kernel-module-glowforge", "**"), + ("forgectrl", "src/super.*"), ("forgectrl", "src/liveness.*")] + + +def controller_pid(): + pids = hw.pidof("grblHAL_glowfor") + if not pids: + raise Failed("controller process not found (grblHAL_glowforge)") + return pids[0] + + +def cpu_ticks(pid): + with open("/proc/%d/stat" % pid) as f: + s = f.read().split() + return int(s[13]) + int(s[14]) # utime + stime (all threads) + + +def cpu_percent(ctx, pid, window): + import os + hz = os.sysconf("SC_CLK_TCK") + a = cpu_ticks(pid) + ctx.sleep(window) + b = cpu_ticks(pid) + return 100.0 * (b - a) / (hz * window) + + +def wait_state(ctx, g, prefix, timeout): + end = time.time() + timeout + while time.time() < end: + ctx.checkpoint() + st = g.status_report() + if st["state"].startswith(prefix): + return st + time.sleep(0.1) + return None + + +def wait_idle(ctx, g, timeout=30.0, poll=0.05): + """Poll until Idle; returns (peak_feed_mm_min, states_seen, final_report).""" + peak = 0.0 + states = [] + deadline = time.time() + timeout + st = None + while time.time() < deadline: + ctx.checkpoint() + st = g.status_report() + state = st["state"] + if not states or states[-1] != state: + states.append(state) + f = st.get("FS") or st.get("F") + if f: + try: + peak = max(peak, float(str(f).split(",")[0])) + except ValueError: + pass + if state.startswith("Idle"): + return peak, states, st + time.sleep(poll) + return peak, states + ["TIMEOUT"], st + + +def clean_slate(ctx, g): + st = g.status_report() + ctx.log("connect: %s", st["state"]) + if any(k in st["state"] for k in ("Alarm", "Door", "Hold")): + g.realtime(0x18) # soft reset + ctx.sleep(2) + g.drain() + st = g.status_report() + if "Alarm" in st["state"]: + ctx.log("unlock: %s", g.command("$X")) + st = g.status_report() + ctx.check(st["state"].startswith("Idle"), "controller is %s, expected Idle", st["state"]) + return st + + +@test("motion.pacing", title="Protocol-loop pacing (idle, parked, moving) and hold/resume position", + subsystem="motion", kind="auto", est_min=1, + covers=_MOTION_COVERS, requires=["kernel.latch-locked-idle"], + steps=["Bed clear; the head needs 30 mm of free +X travel."], + description="Idle CPU is low; a job parked in a completed feed hold is coarse-paced (not " + "busy-spinning at the motion rate); active motion is tight-paced; a feed-hold " + "mid-move then resume preserves position (the feeder never starves).") +def pacing(ctx): + dist, feed = 30.0, 600.0 + pid = controller_pid() + ev = ctx.evidence + ev["controller_pid"] = pid + with ctx.grbl() as g: + clean_slate(ctx, g) + g.command("M5") + g.command("G91") + idle = cpu_percent(ctx, pid, 3) + ev["idle_cpu_pct"] = round(idle, 1) + ctx.log("[1] idle CPU = %.1f%%", idle) + + start = g.status_report().get("MPos") + ctx.check(start, "no MPos in the status report") + g.command("G1 X%.3f F%.0f" % (dist, feed), timeout=0.5) + ctx.sleep(0.6) + moving = cpu_percent(ctx, pid, 1.0) + st_mv = g.status_report()["state"] + ev["moving_cpu_pct"] = round(moving, 1) + ctx.log("[3] state=%s CPU during move = %.1f%%", st_mv, moving) + + g.realtime(ord("!")) # feed hold + wait_state(ctx, g, "Hold", 5) + ctx.sleep(1.5) # decel completes (Hold:1 -> Hold:0) + held = g.status_report() + parked = cpu_percent(ctx, pid, 3) + ev["held_state"] = held["state"] + ev["parked_cpu_pct"] = round(parked, 1) + ctx.log("[2] %s: CPU parked in Hold = %.1f%%", held["state"], parked) + + g.realtime(ord("~")) # resume + peak, states, st = wait_idle(ctx, g, 30) + ctx.check("TIMEOUT" not in states, "did not return to Idle after the resume: %s", states) + end = st.get("MPos") + moved = end[0] - start[0] + ev["moved_mm"] = round(moved, 3) + ctx.log("[4] start X=%.3f end X=%.3f moved=%.3f (expect %.1f)", start[0], end[0], moved, dist) + # return to the starting position + g.command("G1 X%.3f F%.0f" % (-dist, feed), timeout=0.5) + wait_idle(ctx, g, 30) + g.command("G90") + final = g.status_report().get("MPos") + ev["final_drift_mm"] = round(final[0] - start[0], 3) if final else None + + ctx.check(abs(moved - dist) < 0.05, "hold+resume lost steps: moved %.3f of %.1f mm", moved, dist) + ctx.check(parked < moving * 0.5 and parked < 8.0, + "parked Hold is not coarse-paced: %.1f%% (moving %.1f%%)", parked, moving) + ctx.check(idle < 8.0, "idle CPU %.1f%%", idle) + ctx.log("PASS: idle %.1f%%, moving %.1f%%, parked %.1f%%, hold/resume exact", idle, moving, parked) + + +@test("motion.jog-roundtrip", title="Motion quality: bounded jogs, max rate, diagonal, hold/resume", + subsystem="motion", kind="operator", est_min=3, + covers=_MOTION_COVERS, requires=["kernel.latch-locked-idle"], + steps=["Park the head with at least 60 mm of free +X and 40 mm of free +Y travel; bed clear.", + "Watch the gantry: it must move on every jog and end where it started."], + description="Sanity jogs (X, Y 40 mm out/back at F2400), max-rate X out/back (60 mm at " + "F12000), a diagonal out/back, then a G1 with a feed-hold/resume in the middle. " + "No jog refused, every move returns to Idle, position drift within 0.05 mm, and " + "the operator saw the gantry move.") +def jog_roundtrip(ctx): + ev = ctx.evidence + ctx.instruct("Head parked with >= 60 mm free +X and >= 40 mm free +Y, bed clear, lid closed. " + "Watch the gantry during this test.") + with ctx.grbl() as g: + st0 = clean_slate(ctx, g) + start = st0.get("MPos") + ctx.check(start, "no MPos in the status report") + ev["start"] = start + moves = [] + for name, out, back in ( + ("X sanity 40mm", "$J=G91X40F2400", "$J=G91X-40F2400"), + ("Y sanity 40mm", "$J=G91Y40F2400", "$J=G91Y-40F2400"), + ("X max-rate 60mm", "$J=G91X60F12000", "$J=G91X-60F12000"), + ("diag 40mm", "$J=G91X40Y40F8000", "$J=G91X-40Y-40F8000")): + for jog in (out, back): + r = g.command(jog) + ctx.check(not any(x.startswith("error") for x in r), "%s: jog refused: %s", name, r) + peak, states, _ = wait_idle(ctx, g) + leg = "out" if jog == out else "back" + ctx.log("%s %s: peak %.0f mm/min, states %s", name, leg, peak, states) + moves.append({"name": name, "leg": leg, "peak": peak, "states": states}) + ctx.check("TIMEOUT" not in states, "%s %s did not return to Idle", name, leg) + ev["moves"] = moves + maxrate = max(m["peak"] for m in moves if m["name"].startswith("X max-rate")) + ev["max_rate_peak"] = maxrate + ctx.check(maxrate >= 6000, "max-rate jog peaked at only %.0f mm/min", maxrate) + + # feed-hold mid-move: G1 at 600 mm/min takes 3 s for 30 mm + g.command("G91") + g.command("G1X30F600", timeout=0.5) + ctx.sleep(1.0) + g.realtime(ord("!")) + ctx.sleep(0.8) + held = g.status_report() + ev["held_state"] = held["state"] + ctx.log("after !: %s", held["state"]) + g.realtime(ord("~")) + peak, states, _ = wait_idle(ctx, g) + ctx.log("after ~: states %s", states) + g.command("G1X-30F2400", timeout=0.5) + wait_idle(ctx, g) + g.command("G90") + final = g.status_report().get("MPos") + ev["final"] = final + drift = max(abs(a - b) for a, b in zip(final[:2], start[:2])) + ev["drift_mm"] = round(drift, 3) + ctx.log("final drift %.3f mm (start %s, final %s)", drift, start, final) + ctx.check("Hold" in held["state"], "feed hold did not park (state %s)", held["state"]) + ctx.check(drift <= 0.05, "position drift %.3f mm", drift) + ctx.confirm("Did the gantry move on every jog (X, Y, the fast X, the diagonal, the held move) " + "and end where it started?") + ctx.log("PASS: %d jogs, peak %.0f mm/min, hold parked, drift %.3f mm, operator confirmed", + len(moves), maxrate, drift) + + +# ---------------------------------------------------------------- liveness + +@test("motion.liveness-probe", title="Supervisor motion-liveness verdict", subsystem="motion", + kind="auto", est_min=1, + covers=[("forgectrl", "src/super.c"), ("forgectrl", "src/liveness.c"), ("kernel-module-glowforge", "**")], + requires=["kernel.latch-locked-idle"], + steps=["Bed clear, lid closed: the probe jogs the head a few mm (+X first)."], + description="forgectrl's supervisor reports the head-accelerometer liveness probe as " + "verified for the running controller (the DRV8825s are not wedged); when the " + "probe was skipped at spawn, the controller is respawned once so it runs.") +def liveness_probe(ctx): + fc = ctx.forgectrl + ev = ctx.evidence + st, m = fc.get("/mode") + ctx.check(st == 200 and isinstance(m, dict), "GET /mode -> %s", st) + ev["mode_before"] = m + ctx.log("mode: %s", m) + ctx.check(m.get("motion") != "fault", "supervisor reports motion-fault: %s", m) + if m.get("motion") != "verified": + ctx.log("probe %s at the last spawn - respawning the controller so it runs", m.get("motion")) + st, body = fc.post("/controller/stop") + ctx.check(st == 200, "POST /controller/stop -> %s", st) + ctx.sleep(2) + st, body = fc.post("/controller/start") + ctx.check(st == 200, "POST /controller/start -> %s", st) + t0 = time.time() + while time.time() - t0 < 60: + ctx.sleep(1) + st, m = fc.get("/mode") + if isinstance(m, dict) and m.get("controller") == "running": + break + ctx.sleep(3) + st, m = fc.get("/mode") + ev["mode_after"] = m + ctx.log("mode after: %s", m) + ctx.check(m.get("controller") == "running", "controller is %s", m.get("controller")) + ctx.check(m.get("motion") == "verified", "liveness is %r, expected verified", m.get("motion")) + + +# ---------------------------------------------------------------- cancel / abort + +@test("motion.cancel-abort", title="Jog cancel and controlled abort recover cleanly", subsystem="motion", + kind="auto", est_min=2, + covers=_MOTION_COVERS, requires=["motion.pacing"], + steps=["Bed clear; the head needs 40 mm of free +X travel."], + description="A jog-cancel (0x85) stops a jog short of its target and returns to Idle with " + "position preserved; a ^X abort mid-move decelerates under control into Alarm " + "with machine position retained, $X recovers to Idle, and a subsequent jog runs " + "(no driver wedge: the rail never cycled).") +def cancel_abort(ctx): + ev = ctx.evidence + with ctx.grbl() as g: + clean_slate(ctx, g) + start = g.status_report()["MPos"] + # jog cancel + g.command("$J=G91X40F2400") + ctx.sleep(0.4) + g.realtime(0x85) + st = wait_state(ctx, g, "Idle", 5) + ctx.check(st is not None, "not Idle within 5 s of the jog cancel") + p1 = st["MPos"] + moved1 = p1[0] - start[0] + ev["cancel_moved_mm"] = round(moved1, 3) + ctx.log("jog cancel: moved %.3f mm of 40 (state %s)", moved1, st["state"]) + ctx.check(0.5 < moved1 < 39.0, "jog cancel did not stop short of the target (%.3f mm)", moved1) + # ^X abort mid-move + g.command("G91") + g.command("G1X30F600", timeout=0.5) + ctx.sleep(1.0) + g.realtime(0x18) + st = wait_state(ctx, g, "Alarm", 5) + ctx.check(st is not None, "^X did not land in Alarm within 5 s") + p2 = st["MPos"] + moved2 = p2[0] - p1[0] + ev["abort_moved_mm"] = round(moved2, 3) + ctx.log("^X abort: state %s, moved %.3f mm of 30, position retained %s", st["state"], moved2, p2) + ctx.check(0.5 < moved2 < 29.5, "abort position not retained/plausible (%.3f mm)", moved2) + r = g.command("$X") + ctx.log("$X -> %s", r) + st = wait_state(ctx, g, "Idle", 5) + ctx.check(st is not None, "$X did not recover to Idle") + # a jog after the abort proves the drivers are alive; return to start + back = -(p2[0] - start[0]) + r = g.command("$J=G91X%.3fF2400" % back) + ctx.check(not any(x.startswith("error") for x in r), "return jog refused: %s", r) + peak, states, st = wait_idle(ctx, g, 30) + ctx.check("TIMEOUT" not in states, "return jog did not complete: %s", states) + final = st["MPos"] + drift = abs(final[0] - start[0]) + ev["final_drift_mm"] = round(drift, 3) + ctx.log("returned: drift %.3f mm", drift) + ctx.check(drift <= 0.05, "position drift %.3f mm after cancel/abort/return", drift) + g.command("G90") + + +# ---------------------------------------------------------------- dead-man + +def _kernel_x_mm(ctx): + pos = (ctx.forgectrl.status().get("pos") or {}) + return pos.get("x") + + +def _return_x(ctx, delta_mm): + """Jog back by the kernel-measured X delta (grbl's own position may be + untrusted after a kill; the kernel counters kept counting).""" + if delta_mm is None or abs(delta_mm) < 0.05: + return + with ctx.grbl() as g: + st = g.status_report()["state"] + if st.startswith("Alarm"): + g.command("$X") + g.command("$J=G91X%.3fF1200" % (-delta_mm)) + wait_idle(ctx, g, 30) + + +@test("motion.deadman", title="Dead-man: controller kill, controller hang, forgectrl restart mid-move", + subsystem="motion", kind="auto", est_min=4, + covers=_MOTION_COVERS + [("forgectrl", "src/main.c"), ("forgectrl", "init/**")], + requires=["motion.cancel-abort", "kernel.k1-k2"], + steps=["Bed clear; the head needs 40 mm of free +X travel and must not be at the left rail."], + description="SIGKILL of the controller mid-move: the supervisor reaps it, safes (cnc/stop, " + "latch relocked - it never unlocked), and respawns within seconds. SIGSTOP (a " + "hang) mid-move: the ring drains into a kernel underrun (fast halt, latch " + "locked); the hung process is killed and the supervisor respawns. forgectrl " + "restart mid-move: the busy controller finishes the move unmanaged and the new " + "daemon retakes supervision at idle. After each drill the head is jogged back " + "by the kernel-measured distance.") +def deadman(ctx): + import os as _os + import signal as _signal + fc = ctx.forgectrl + ev = ctx.evidence + + def latch_locked(): + v = hw.sysfs_int("cnc/interlock_circuit") + return v is not None and bool(v & (1 << 3)) + + def wait_running(timeout=30): + t0 = time.time() + while time.time() - t0 < timeout: + st, m = fc.get("/mode") + if isinstance(m, dict) and m.get("controller") == "running" and m.get("pid"): + return m + ctx.sleep(0.5) + return None + + # ---- 1. SIGKILL mid-move + m0 = wait_running(10) + ctx.check(m0, "controller not running") + pid0 = m0["pid"] + x0 = _kernel_x_mm(ctx) + unlocked_seen = False + with ctx.grbl() as g: + clean_slate(ctx, g) + g.command("G91") + g.command("G1X30F300", timeout=0.5) + ctx.sleep(1.0) + _os.kill(pid0, _signal.SIGKILL) + ctx.log("SIGKILL sent to controller pid %d mid-move", pid0) + t0 = time.time() + while time.time() - t0 < 15: + if not latch_locked(): + unlocked_seen = True + st, m = fc.get("/mode") + if isinstance(m, dict) and m.get("controller") == "running" and m.get("pid") != pid0: + break + ctx.sleep(0.2) + m1 = wait_running(30) + respawn_s = round(time.time() - t0, 1) + ev["sigkill"] = {"old_pid": pid0, "new": m1, "respawn_s": respawn_s, "unlocked_seen": unlocked_seen} + ctx.log("after SIGKILL: respawned as %s in %s s; latch unlocked seen: %s", m1, respawn_s, unlocked_seen) + ctx.check(m1 and m1.get("pid") != pid0, "supervisor did not respawn the controller") + ctx.check(not unlocked_seen, "the latch unlocked during the kill/respawn") + ctx.check(m1.get("motion") != "fault", "motion fault after the respawn") + ctx.sleep(3) + x1 = _kernel_x_mm(ctx) + ctx.log("kernel X: %s -> %s mm", x0, x1) + _return_x(ctx, (x1 - x0) if (x0 is not None and x1 is not None) else None) + + # ---- 2. SIGSTOP (hang) mid-move -> kernel underrun + m1 = wait_running(10) + pid1 = m1["pid"] + x0 = _kernel_x_mm(ctx) + underruns0 = hw.sysfs_int("cnc/underruns", 0) + with ctx.grbl() as g: + clean_slate(ctx, g) + g.command("G91") + g.command("G1X30F300", timeout=0.5) + ctx.sleep(1.0) + _os.kill(pid1, _signal.SIGSTOP) + ctx.log("SIGSTOP sent to controller pid %d mid-move", pid1) + t0 = time.time() + kstate = None + while time.time() - t0 < 10: + kstate = hw.sysfs_read("cnc/state") + if kstate == "underrun": + break + ctx.sleep(0.05) + halt_s = round(time.time() - t0, 2) + ev["sigstop"] = {"kernel_state": kstate, "halt_s": halt_s, "latch_locked": latch_locked(), + "underruns": hw.sysfs_int("cnc/underruns", 0)} + ctx.log("after SIGSTOP: kernel %s in %s s, latch locked %s, underruns %s -> %s", + kstate, halt_s, latch_locked(), underruns0, ev["sigstop"]["underruns"]) + _os.kill(pid1, _signal.SIGKILL) # the hung controller cannot recover itself + ctx.check(kstate == "underrun", "the ring did not drain into a kernel underrun (state %s)", kstate) + ctx.check(latch_locked(), "latch unlocked after the underrun") + m2 = wait_running(30) + ev["sigstop"]["respawn"] = m2 + ctx.check(m2 and m2.get("pid") != pid1, "supervisor did not respawn after the hang") + ctx.sleep(3) + x1 = _kernel_x_mm(ctx) + _return_x(ctx, (x1 - x0) if (x0 is not None and x1 is not None) else None) + + # ---- 3. forgectrl restart mid-move: the move finishes, supervision retaken at idle + m2 = wait_running(10) + pid2 = m2["pid"] + x0 = _kernel_x_mm(ctx) + with ctx.grbl() as g: + clean_slate(ctx, g) + g.command("G91") + g.command("G1X30F300", timeout=0.5) # ~6 s of motion + ctx.sleep(1.0) + rc, out = hw.initd("forgectrl", "restart") + ctx.log("forgectrl restart mid-move -> rc %s", rc) + peak, states, st = wait_idle(ctx, g, 30) + ev["restart"] = {"rc": rc, "states": states} + ctx.log("move after the restart: states %s", states) + ctx.check("TIMEOUT" not in states, "the move did not finish after the forgectrl restart") + g.command("G90") + t0 = time.time() + m3 = None + while time.time() - t0 < 60: + st, m3 = fc.get("/mode") + if isinstance(m3, dict) and m3.get("controller") == "running": + break + ctx.sleep(1) + ev["restart"]["mode_after"] = m3 + ctx.log("mode after restart: %s", m3) + ctx.check(m3 and m3.get("controller") == "running", "supervision not retaken after the restart: %s", m3) + ctx.check(m3.get("pid") == pid2, "the busy controller was replaced (%s -> %s) instead of retaken", + pid2, m3.get("pid")) + ctx.check(latch_locked(), "latch unlocked after the restart drill") + x1 = _kernel_x_mm(ctx) + _return_x(ctx, (x1 - x0) if (x0 is not None and x1 is not None) else None) + ctx.log("PASS: kill respawned in %s s, hang -> underrun in %s s, restart retook pid %s", + respawn_s, halt_s, pid2) diff --git a/forgetest/forgetest/suite/update.py b/forgetest/forgetest/suite/update.py new file mode 100644 index 0000000..bfc250a --- /dev/null +++ b/forgetest/forgetest/suite/update.py @@ -0,0 +1,47 @@ +"""update.* - the A/B slot inventory and the firmware verification path.""" +import os +import tempfile + +from ..catalog import test +from .. import hw + +_UPDATE_COVERS = [("forgectrl", "src/update.c"), ("forgectrl", "src/update.h")] + + +@test("update.slots-and-signature", title="Boot slots readable, unsigned/tampered archives refused", + subsystem="update", kind="auto", est_min=1, + covers=_UPDATE_COVERS, requires=["forgectrl.auth"], + description="/slots reports the A/B inventory consistent with `ffboot -l`; /update/status " + "answers; `fwup` refuses a garbage archive and a tampered signature against the " + "shipped release key. Nothing is written to any slot.") +def slots_and_signature(ctx): + fc = ctx.forgectrl + ev = ctx.evidence + st, slots = fc.get("/slots") + ctx.log("GET /slots -> %s %s", st, slots) + ctx.check(st == 200 and isinstance(slots, dict), "GET /slots -> %s", st) + ev["slots"] = slots + rc, out = hw.run(["ffboot", "-l"]) + ev["ffboot_l_rc"] = rc + ctx.log("ffboot -l -> rc %s\n%s", rc, out.strip()) + ctx.check(rc == 0, "ffboot -l failed (%s)", rc) + text = str(slots).lower() + ctx.check("forgefirm" in text or "slot" in text, "/slots does not look like a slot inventory") + + st, us = fc.get("/update/status") + ctx.log("GET /update/status -> %s %s", st, us) + ctx.check(st == 200 and isinstance(us, dict) and "running" in us, "GET /update/status -> %s", st) + ctx.check(not us.get("running"), "an update is running") + + key = "/etc/forgefirm/keys/forgefirm-release.pub" + ctx.check(os.path.exists(key), "release key %s missing", key) + with tempfile.NamedTemporaryFile(prefix="forgetest-", suffix=".fw", delete=False) as f: + f.write(b"this is not a firmware archive" * 64) + garbage = f.name + try: + rc, out = hw.run(["fwup", "-V", "-i", garbage, "-p", key], timeout=30) + ev["fwup_garbage_rc"] = rc + ctx.log("fwup -V garbage -> rc %s: %s", rc, out.strip()[:200]) + ctx.check(rc != 0, "fwup accepted a garbage archive") + finally: + os.unlink(garbage) diff --git a/forgetest/tests/helpers.py b/forgetest/tests/helpers.py new file mode 100644 index 0000000..423a7c3 --- /dev/null +++ b/forgetest/tests/helpers.py @@ -0,0 +1,78 @@ +"""Shared fixtures for the forgetest unit tests: synthetic manifests and +catalog entries that never touch hardware.""" +import hashlib +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.dirname(HERE)) + +from forgetest import catalog as catalog_mod # noqa: E402 +from forgetest import manifest as manifest_mod # noqa: E402 + + +def blob(text): + """A git-style blob id for text (what ls-tree would report).""" + data = text.encode() + return hashlib.sha1(b"blob %d\0" % len(data) + data).hexdigest() + + +def make_manifest(components=None, platform=None, version="20260101000000 (dev)", name="forgefirm-image-dev"): + comps = components if components is not None else { + "forgectrl": {"srcrev": "aaa", "files": [["src/main.c", blob("main")], ["src/ui.c", blob("ui")], + ["src/auth.c", blob("auth")], ["src/cool.c", blob("cool")], + ["README.md", blob("readme")]]}, + "grblhal-glowforge": {"srcrev": "bbb", "files": [["src/driver.c", blob("drv")], + ["src/grbl", "cccc"], ["src/grbl/core.c", blob("core")]]}, + "kernel-module-glowforge": {"srcrev": "ddd", "files": [["src/cnc.c", blob("cnc")]]}, + "linux-fslc": {"srcrev": "eee", "files": [["@srcrev", "eee"], ["@config", "cfg1"]]}, + "forgetest": {"srcrev": None, "files": [["forgetest/x.py", blob("x")]]}, + } + plat = platform if platform is not None else { + "machine": "glowforge", "kernel_modules": ["6.12.20-fslc+g0e01ec9f0d3f"], + "dtb": {"glowforge.dtb": "d" * 64}, + "layers": {"meta-forgefirm": {"content_sha256": "1" * 64}, "poky": {"rev": "p" * 40}}, + } + data = {"format": 1, "image": {"name": name, "version": version}, + "components": comps, "platform": plat} + data["content_sha256"] = manifest_mod.sha256_text( + manifest_mod.canonical({"components": comps, "platform": plat})) + return manifest_mod.Manifest(data) + + +def with_file(manifest, component, path, text): + """A copy of the manifest with one file's content changed/added.""" + import copy + data = copy.deepcopy(manifest.data) + files = data["components"][component]["files"] + for f in files: + if f[0] == path: + f[1] = blob(text) + break + else: + files.append([path, blob(text)]) + data["content_sha256"] = manifest_mod.sha256_text( + manifest_mod.canonical({"components": data["components"], "platform": data["platform"]})) + return manifest_mod.Manifest(data) + + +def with_platform(manifest, **changes): + import copy + data = copy.deepcopy(manifest.data) + data["platform"].update(changes) + data["content_sha256"] = manifest_mod.sha256_text( + manifest_mod.canonical({"components": data["components"], "platform": data["platform"]})) + return manifest_mod.Manifest(data) + + +def _noop(ctx): + pass + + +def make_test(id, covers, always=False, requires=(), kind="auto", fn=None, subsystem=None): + return catalog_mod.Test(id, "Title " + id, subsystem or id.split(".")[0], kind, "api", + covers, requires, always, 1, (), "desc", fn or _noop) + + +def registry(*tests): + return {t.id: t for t in tests} diff --git a/forgetest/tests/test_artifact.py b/forgetest/tests/test_artifact.py new file mode 100644 index 0000000..2cd44eb --- /dev/null +++ b/forgetest/tests/test_artifact.py @@ -0,0 +1,143 @@ +import copy +import json +import unittest + +import helpers +from forgetest import artifact, campaign, catalog +from forgetest import manifest as manifest_mod +from test_campaign import rec_campaign, rec_result + + +class ArtifactTests(unittest.TestCase): + def setUp(self): + self.man = helpers.make_manifest() + self.core = helpers.make_test("image.health", [("linux-fslc", "**")], always=True) + self.ui = helpers.make_test("forgectrl.panel", [("forgectrl", "src/ui.c")]) + self.cool = helpers.make_test("cooling.flow", [("forgectrl", "src/cool.c")]) + self.reg = helpers.registry(self.core, self.ui, self.cool) + self.tests = list(self.reg.values()) + self.chash = catalog.catalog_hash(self.reg) + # campaign 1 on the base image: everything passes + self.recs = [rec_campaign("c1", self.man, self.chash, "2026-08-20T10:00:00Z")] + for i, t in enumerate(self.tests): + self.recs.append(rec_result("c1", t, self.man, "PASS", "2026-08-20T10:0%d:00Z" % (i + 1))) + # image 2: cool.c changed; campaign 2 reruns core + cool + self.man2 = helpers.with_file(self.man, "forgectrl", "src/cool.c", "cool v2") + self.recs.append(rec_campaign("c2", self.man2, self.chash, "2026-08-21T10:00:00Z")) + self.recs.append(rec_result("c2", self.core, self.man2, "PASS", "2026-08-21T10:01:00Z")) + self.recs.append(rec_result("c2", self.cool, self.man2, "PASS", "2026-08-21T10:02:00Z")) + + def build(self, man=None): + man = man or self.man2 + st = campaign.compute(self.recs, self.tests, man, self.chash) + return artifact.build(st, self.tests, man, self.recs, self.chash), st + + def release_manifest(self, man=None): + """The release build's manifest: same identity, no forgetest component.""" + man = man or self.man2 + data = copy.deepcopy(man.data) + data["components"].pop("forgetest", None) + data["image"] = {"name": "forgefirm-image", "version": "v0.1.0"} + return manifest_mod.Manifest(data) + + def test_build_and_verify(self): + art, st = self.build() + self.assertTrue(art["authorized"]) + by = {t["id"]: t for t in art["tests"]} + self.assertTrue(by["forgectrl.panel"]["inherited"]) + self.assertEqual(by["forgectrl.panel"]["record"]["campaign"], "c1") + self.assertFalse(by["image.health"]["inherited"]) + text = artifact.to_json(art) + art2 = json.loads(text) + ok, rows, problems = artifact.verify(art2, self.release_manifest(), self.tests, self.chash, + expect_machine="glowforge") + self.assertEqual(problems, []) + self.assertTrue(ok) + md = artifact.to_markdown(art) + self.assertIn("Release authorized: YES", md) + self.assertIn("forgectrl.panel", md) + + def test_tamper_detected(self): + art, _ = self.build() + art2 = json.loads(artifact.to_json(art)) + art2["tests"][1]["record"]["result"] = "PASS" + art2["counts"]["required"] = 0 + art2["authorized"] = True + art2["tests"][2]["record"]["ts"] = "2026-08-21T10:02:01Z" + ok, rows, problems = artifact.verify(art2, self.release_manifest(), self.tests, self.chash) + self.assertFalse(ok) + self.assertIn("self-hash", problems[0]) + + def test_release_differs_in_covered_file(self): + art, _ = self.build() + rel = self.release_manifest(helpers.with_file(self.man2, "forgectrl", "src/ui.c", "ui v3")) + ok, rows, problems = artifact.verify(art, rel, self.tests, self.chash) + self.assertFalse(ok) + self.assertTrue(any("forgectrl.panel" in p and "fingerprint" in p for p in problems), problems) + # an uncovered change is fine + rel2 = self.release_manifest(helpers.with_file(self.man2, "forgectrl", "README.md", "docs")) + ok, rows, problems = artifact.verify(art, rel2, self.tests, self.chash) + self.assertTrue(ok, problems) + + def test_release_platform_differs(self): + art, _ = self.build() + rel = self.release_manifest(helpers.with_platform(self.man2, dtb={"glowforge.dtb": "0" * 64})) + ok, rows, problems = artifact.verify(art, rel, self.tests, self.chash) + self.assertFalse(ok) + self.assertEqual(len([r for r in rows if not r["ok"]]), 3) + + def test_unauthorized_artifact(self): + self.recs.append(rec_result("c2", self.ui, self.man2, "FAIL", "2026-08-21T10:03:00Z")) + art, st = self.build() + self.assertFalse(art["authorized"]) + ok, rows, problems = artifact.verify(art, self.release_manifest(), self.tests, self.chash) + self.assertFalse(ok) + self.assertTrue(any("does not claim authorization" in p for p in problems)) + + def test_core_inherited_refused(self): + art, _ = self.build() + # forge an artifact whose core record is marked inherited (self-hash recomputed) + body = json.loads(artifact.to_json(art)) + body.pop("sha256") + for t in body["tests"]: + if t["id"] == "image.health": + t["inherited"] = True + body["sha256"] = manifest_mod.sha256_text(manifest_mod.canonical(body)) + ok, rows, problems = artifact.verify(body, self.release_manifest(), self.tests, self.chash) + self.assertFalse(ok) + self.assertTrue(any("always-required" in p for p in problems), problems) + + def test_invalidate_epoch(self): + art, _ = self.build() + body = json.loads(artifact.to_json(art)) + body.pop("sha256") + body["invalidate"] = {"t": "invalidate", "ts": "2026-08-20T12:00:00Z", "reason": "tube"} + body["sha256"] = manifest_mod.sha256_text(manifest_mod.canonical(body)) + ok, rows, problems = artifact.verify(body, self.release_manifest(), self.tests, self.chash) + self.assertFalse(ok) + self.assertTrue(any("predates the invalidate" in p for p in problems), problems) + + def test_catalog_changed(self): + art, _ = self.build() + reg = helpers.registry(self.core, self.ui, self.cool, helpers.make_test("new.one", [])) + tests = list(reg.values()) + ok, rows, problems = artifact.verify(art, self.release_manifest(), tests, catalog.catalog_hash(reg)) + self.assertFalse(ok) + self.assertTrue(any("catalog changed" in p for p in problems)) + self.assertTrue(any("new.one" in p for p in problems)) + + def test_implementation_changed(self): + art, _ = self.build() + body = json.loads(artifact.to_json(art)) + body.pop("sha256") + for t in body["tests"]: + if t["id"] == "cooling.flow": + t["source_sha"] = "0" * 64 + body["sha256"] = manifest_mod.sha256_text(manifest_mod.canonical(body)) + ok, rows, problems = artifact.verify(body, self.release_manifest(), self.tests, self.chash) + self.assertFalse(ok) + self.assertTrue(any("implementation changed" in p for p in problems), problems) + + +if __name__ == "__main__": + unittest.main() diff --git a/forgetest/tests/test_campaign.py b/forgetest/tests/test_campaign.py new file mode 100644 index 0000000..ed14cf8 --- /dev/null +++ b/forgetest/tests/test_campaign.py @@ -0,0 +1,160 @@ +import unittest + +import helpers +from forgetest import campaign +from forgetest import catalog + + +def rec_campaign(cid, man, chash, ts): + return {"t": "campaign", "ts": ts, "id": cid, "manifest_sha": man.content_sha, + "catalog_hash": chash, "image": man.version} + + +def rec_result(cid, t, man, result, ts, fp=None): + return {"t": "result", "ts": ts, "campaign": cid, "test": t.id, "result": result, + "fingerprint": fp or t.fingerprint(man), "manifest_sha": man.content_sha, + "image": man.version, "duration_s": 1, "message": ""} + + +class CampaignTests(unittest.TestCase): + def setUp(self): + self.man = helpers.make_manifest() + self.core = helpers.make_test("image.health", [("linux-fslc", "**")], always=True) + self.ui = helpers.make_test("forgectrl.panel", [("forgectrl", "src/ui.c")]) + self.cool = helpers.make_test("cooling.flow", [("forgectrl", "src/cool.c")]) + self.live = helpers.make_test("laser.witness", [("grblhal-glowforge", "src/**")], + requires=("forgectrl.panel",), kind="live") + self.reg = helpers.registry(self.core, self.ui, self.cool, self.live) + self.tests = list(self.reg.values()) + self.chash = catalog.catalog_hash(self.reg) + + def compute(self, records, man=None, chash=None): + return campaign.compute(records, self.tests, man or self.man, chash or self.chash) + + def test_empty(self): + st = self.compute([]) + self.assertIsNone(st["campaign"]) + self.assertFalse(st["authorized"]) + for t in self.tests: + self.assertTrue(st["tests"][t.id]["required"]) + self.assertEqual(st["tests"][t.id]["reason"], "always" if t.always else "never-passed") + self.assertFalse(st["tests"][self.live.id]["requires_met"]) + + def test_full_campaign_authorizes(self): + recs = [rec_campaign("c1", self.man, self.chash, "2026-08-20T10:00:00Z")] + for i, t in enumerate(self.tests): + recs.append(rec_result("c1", t, self.man, "PASS", "2026-08-20T10:0%d:00Z" % (i + 1))) + st = self.compute(recs) + self.assertTrue(st["authorized"]) + self.assertEqual(st["counts"], {"total": 4, "satisfied": 4, "inherited": 0, "required": 0}) + self.assertEqual(st["tests"][self.core.id]["status"], "pass") + + def test_fail_closes_campaign(self): + recs = [rec_campaign("c1", self.man, self.chash, "2026-08-20T10:00:00Z"), + rec_result("c1", self.core, self.man, "PASS", "2026-08-20T10:01:00Z"), + rec_result("c1", self.ui, self.man, "PASS", "2026-08-20T10:02:00Z"), + rec_result("c1", self.cool, self.man, "FAIL", "2026-08-20T10:03:00Z")] + st = self.compute(recs) + self.assertIsNone(st["campaign"]) + self.assertEqual(st["closed_by"], "fail") + self.assertFalse(st["authorized"]) + # the core PASS in the closed campaign no longer counts; the ui PASS is inheritable + self.assertTrue(st["tests"][self.core.id]["required"]) + self.assertEqual(st["tests"][self.core.id]["reason"], "always") + self.assertEqual(st["tests"][self.ui.id]["status"], "inherited") + self.assertEqual(st["tests"][self.cool.id]["status"], "fail") + self.assertEqual(st["tests"][self.cool.id]["reason"], "never-passed") + + def test_error_closes_aborted_does_not(self): + recs = [rec_campaign("c1", self.man, self.chash, "2026-08-20T10:00:00Z"), + rec_result("c1", self.ui, self.man, "ABORTED", "2026-08-20T10:01:00Z")] + st = self.compute(recs) + self.assertIsNotNone(st["campaign"]) + self.assertEqual(st["tests"][self.ui.id]["status"], "aborted") + recs.append(rec_result("c1", self.ui, self.man, "ERROR", "2026-08-20T10:02:00Z")) + st = self.compute(recs) + self.assertIsNone(st["campaign"]) + self.assertEqual(st["closed_by"], "fail") + + def test_inheritance_follows_fingerprint(self): + recs = [rec_campaign("c1", self.man, self.chash, "2026-08-20T10:00:00Z")] + for i, t in enumerate(self.tests): + recs.append(rec_result("c1", t, self.man, "PASS", "2026-08-20T10:0%d:00Z" % (i + 1))) + # a new image with only cool.c changed + man2 = helpers.with_file(self.man, "forgectrl", "src/cool.c", "cool v2") + st = self.compute(recs, man=man2) + self.assertIsNone(st["campaign"]) + self.assertEqual(st["closed_by"], "image") + self.assertEqual(st["tests"][self.ui.id]["status"], "inherited") + self.assertEqual(st["tests"][self.live.id]["status"], "inherited") + self.assertEqual(st["tests"][self.cool.id]["status"], "stale") + self.assertEqual(st["tests"][self.cool.id]["reason"], "domain-changed") + self.assertEqual(st["tests"][self.core.id]["reason"], "always") + # open a campaign on the new image, run the core + cool -> authorized + recs.append(rec_campaign("c2", man2, self.chash, "2026-08-21T10:00:00Z")) + recs.append(rec_result("c2", self.core, man2, "PASS", "2026-08-21T10:01:00Z")) + st = self.compute(recs, man=man2) + self.assertFalse(st["authorized"]) + recs.append(rec_result("c2", self.cool, man2, "PASS", "2026-08-21T10:02:00Z")) + st = self.compute(recs, man=man2) + self.assertTrue(st["authorized"]) + self.assertEqual(st["counts"]["inherited"], 2) + self.assertEqual(st["tests"][self.ui.id]["origin"]["campaign"], "c1") + + def test_platform_change_invalidates_everything(self): + recs = [rec_campaign("c1", self.man, self.chash, "2026-08-20T10:00:00Z")] + for i, t in enumerate(self.tests): + recs.append(rec_result("c1", t, self.man, "PASS", "2026-08-20T10:0%d:00Z" % (i + 1))) + man2 = helpers.with_platform(self.man, dtb={"glowforge.dtb": "f" * 64}) + st = self.compute(recs, man=man2) + for t in self.tests: + self.assertTrue(st["tests"][t.id]["required"], t.id) + + def test_invalidate_all(self): + recs = [rec_campaign("c1", self.man, self.chash, "2026-08-20T10:00:00Z")] + for i, t in enumerate(self.tests): + recs.append(rec_result("c1", t, self.man, "PASS", "2026-08-20T10:0%d:00Z" % (i + 1))) + recs.append({"t": "invalidate", "ts": "2026-08-22T09:00:00Z", "reason": "new tube"}) + st = self.compute(recs) + self.assertIsNone(st["campaign"]) + self.assertEqual(st["closed_by"], "invalidate") + self.assertEqual(st["invalidate"]["reason"], "new tube") + for t in self.tests: + self.assertTrue(st["tests"][t.id]["required"], t.id) + self.assertNotEqual(st["tests"][t.id]["status"], "inherited") + # after the invalidate, a new campaign's passes count and later ones inherit again + recs.append(rec_campaign("c2", self.man, self.chash, "2026-08-22T10:00:00Z")) + for i, t in enumerate(self.tests): + recs.append(rec_result("c2", t, self.man, "PASS", "2026-08-22T10:0%d:00Z" % (i + 1))) + st = self.compute(recs) + self.assertTrue(st["authorized"]) + + def test_reset_and_catalog_change(self): + recs = [rec_campaign("c1", self.man, self.chash, "2026-08-20T10:00:00Z"), + rec_result("c1", self.core, self.man, "PASS", "2026-08-20T10:01:00Z"), + {"t": "reset", "ts": "2026-08-20T11:00:00Z", "reason": "x"}] + st = self.compute(recs) + self.assertIsNone(st["campaign"]) + self.assertEqual(st["closed_by"], "reset") + recs = [rec_campaign("c1", self.man, self.chash, "2026-08-20T10:00:00Z")] + st = self.compute(recs, chash="different") + self.assertIsNone(st["campaign"]) + self.assertEqual(st["closed_by"], "catalog") + + def test_requires_uses_satisfied(self): + recs = [rec_campaign("c1", self.man, self.chash, "2026-08-20T10:00:00Z"), + rec_result("c1", self.ui, self.man, "PASS", "2026-08-20T10:01:00Z")] + st = self.compute(recs) + self.assertTrue(st["tests"][self.live.id]["requires_met"]) + man2 = helpers.with_file(self.man, "forgectrl", "src/ui.c", "ui v2") + st = self.compute(recs, man=man2) + self.assertFalse(st["tests"][self.live.id]["requires_met"]) + self.assertEqual(st["tests"][self.live.id]["missing_requires"], ["forgectrl.panel"]) + + def test_running_marker(self): + st = campaign.compute([], self.tests, self.man, self.chash, running=self.ui.id) + self.assertEqual(st["tests"][self.ui.id]["status"], "running") + + +if __name__ == "__main__": + unittest.main() diff --git a/forgetest/tests/test_manifest.py b/forgetest/tests/test_manifest.py new file mode 100644 index 0000000..f4e95c2 --- /dev/null +++ b/forgetest/tests/test_manifest.py @@ -0,0 +1,153 @@ +import unittest + +import helpers +from forgetest import manifest as m +from forgetest import catalog + + +class GlobTests(unittest.TestCase): + def test_star_and_doublestar(self): + rx = m.glob_to_regex("src/*.c") + self.assertTrue(rx.match("src/main.c")) + self.assertFalse(rx.match("src/sub/main.c")) + rx = m.glob_to_regex("src/**") + self.assertTrue(rx.match("src/main.c")) + self.assertTrue(rx.match("src/sub/deep/x.h")) + self.assertFalse(rx.match("docs/x")) + rx = m.glob_to_regex("**/*.md") + self.assertTrue(rx.match("README.md")) + self.assertTrue(rx.match("docs/a/b.md")) + self.assertFalse(rx.match("docs/a/b.txt")) + rx = m.glob_to_regex("**") + self.assertTrue(rx.match("anything/at/all")) + rx = m.glob_to_regex("src/gfcool*") + self.assertTrue(rx.match("src/gfcool_client.c")) + self.assertFalse(rx.match("src/x/gfcool.c")) + + def test_question_mark_and_escaping(self): + rx = m.glob_to_regex("a?c.d") + self.assertTrue(rx.match("abc.d")) + self.assertFalse(rx.match("abcxd")) + self.assertFalse(rx.match("a/c.d")) + + +class FingerprintTests(unittest.TestCase): + def setUp(self): + self.man = helpers.make_manifest() + + def test_stable_and_order_independent(self): + c1 = [("forgectrl", "src/ui.c"), ("forgectrl", "src/auth.c")] + c2 = list(reversed(c1)) + self.assertEqual(m.fingerprint(self.man, c1), m.fingerprint(self.man, c2)) + self.assertEqual(m.fingerprint(self.man, c1), m.fingerprint(self.man, c1)) + + def test_changes_only_when_covered_file_changes(self): + covers = [("forgectrl", "src/ui.c")] + base = m.fingerprint(self.man, covers) + other = helpers.with_file(self.man, "forgectrl", "src/cool.c", "cool v2") + self.assertEqual(base, m.fingerprint(other, covers), "an uncovered change must not move the fingerprint") + changed = helpers.with_file(self.man, "forgectrl", "src/ui.c", "ui v2") + self.assertNotEqual(base, m.fingerprint(changed, covers)) + added = helpers.with_file(self.man, "forgectrl", "src/ui_extra.c", "new") + self.assertEqual(base, m.fingerprint(added, covers), "an unmatched new file does not move it") + added2 = helpers.with_file(self.man, "forgectrl", "src/x.c", "new") + self.assertNotEqual(base, m.fingerprint(added2, [("forgectrl", "src/*.c")])) + + def test_platform_is_always_in(self): + covers = [("forgectrl", "src/ui.c")] + base = m.fingerprint(self.man, covers) + p2 = helpers.with_platform(self.man, dtb={"glowforge.dtb": "e" * 64}) + self.assertNotEqual(base, m.fingerprint(p2, covers)) + p3 = helpers.with_platform(self.man, layers={"meta-forgefirm": {"content_sha256": "2" * 64}, + "poky": {"rev": "p" * 40}}) + self.assertNotEqual(base, m.fingerprint(p3, covers)) + + def test_kernel_pseudo_files(self): + covers = [("linux-fslc", "**")] + base = m.fingerprint(self.man, covers) + k2 = helpers.with_file(self.man, "linux-fslc", "@config", "cfg2") + self.assertNotEqual(base, m.fingerprint(k2, covers)) + + def test_missing_component_marker(self): + f1 = m.fingerprint(self.man, [("nonexistent", "**")]) + f2 = m.fingerprint(self.man, [("forgectrl", "nothing-matches-*")]) + self.assertNotEqual(f1, f2) + + def test_dev_only_refused(self): + with self.assertRaises(ValueError): + m.fingerprint(self.man, [("forgetest", "**")]) + + def test_extra_folds_in(self): + covers = [("forgectrl", "src/ui.c")] + self.assertNotEqual(m.fingerprint(self.man, covers, extra=["a"]), + m.fingerprint(self.man, covers, extra=["b"])) + + def test_submodule_gitlink_and_files(self): + covers = [("grblhal-glowforge", "src/grbl/**")] + base = m.fingerprint(self.man, covers) + core = helpers.with_file(self.man, "grblhal-glowforge", "src/grbl/core.c", "core v2") + self.assertNotEqual(base, m.fingerprint(core, covers)) + link = [("grblhal-glowforge", "src/grbl")] + self.assertEqual(m.fingerprint(self.man, link), m.fingerprint(core, link), + "the gitlink alone does not see a file-level change (fixture keeps the link id)") + + def test_identity_sha_ignores_dev_only(self): + a = self.man.identity_sha() + b = helpers.with_file(self.man, "forgetest", "forgetest/x.py", "changed").identity_sha() + self.assertEqual(a, b) + c = helpers.with_file(self.man, "forgectrl", "src/ui.c", "changed").identity_sha() + self.assertNotEqual(a, c) + + +class CoverageReportTests(unittest.TestCase): + def test_report(self): + man = helpers.make_manifest() + t1 = helpers.make_test("a.one", [("forgectrl", "src/ui.c"), ("forgectrl", "src/auth.c")]) + t2 = helpers.make_test("a.two", [("grblhal-glowforge", "**"), ("kernel-module-glowforge", "**"), + ("linux-fslc", "**")]) + rep = m.coverage_report(man, [t1, t2], allow=[("*", "**/*.md")]) + self.assertEqual(set(rep), {"forgectrl"}) + self.assertEqual(rep["forgectrl"], ["src/cool.c", "src/main.c"]) + rep2 = m.coverage_report(man, [t1, t2, helpers.make_test("a.three", [("forgectrl", "src/**")])], + allow=[("*", "**/*.md")]) + self.assertEqual(rep2, {}) + self.assertNotIn("forgetest", m.coverage_report(man, [], allow=[]), + "dev-only components are outside the report") + + +class CatalogTests(unittest.TestCase): + def test_catalog_hash_is_definition_only(self): + r1 = helpers.registry(helpers.make_test("a.one", [("forgectrl", "src/ui.c")])) + r2 = helpers.registry(helpers.make_test("a.one", [("forgectrl", "src/ui.c")], fn=lambda ctx: 1)) + self.assertEqual(catalog.catalog_hash(r1), catalog.catalog_hash(r2)) + r3 = helpers.registry(helpers.make_test("a.one", [("forgectrl", "src/**")])) + self.assertNotEqual(catalog.catalog_hash(r1), catalog.catalog_hash(r3)) + r4 = helpers.registry(helpers.make_test("a.one", [("forgectrl", "src/ui.c")], always=True)) + self.assertNotEqual(catalog.catalog_hash(r1), catalog.catalog_hash(r4)) + + def test_validate(self): + r = helpers.registry(helpers.make_test("a.one", [], requires=("a.two",))) + with self.assertRaises(ValueError): + catalog.validate(r) + r = helpers.registry(helpers.make_test("a.one", [], requires=("a.two",)), + helpers.make_test("a.two", [], requires=("a.one",))) + with self.assertRaises(ValueError): + catalog.validate(r) + + def test_decorator_rules(self): + with self.assertRaises(ValueError): + catalog.test("bad id", title="x", subsystem="s")(lambda c: None) + with self.assertRaises(ValueError): + catalog.test("s.x", title="x", subsystem="s", covers=[("forgetest", "**")])(lambda c: None) + + def test_real_suite_loads_and_validates(self): + reg = catalog.load_suite() + self.assertIn("image.health", reg) + self.assertTrue(reg["image.health"].always) + catalog.validate(reg) + for t in catalog.all_tests(reg): + self.assertEqual(len(t.source_sha), 64) + + +if __name__ == "__main__": + unittest.main() diff --git a/forgetest/tests/test_server.py b/forgetest/tests/test_server.py new file mode 100644 index 0000000..c98800b --- /dev/null +++ b/forgetest/tests/test_server.py @@ -0,0 +1,309 @@ +"""Runner + HTTP API end to end on localhost with a fake catalog and a +fake bench tool. No hardware, no forgectrl.""" +import json +import os +import shutil +import sys +import tempfile +import threading +import time +import unittest +import urllib.error +import urllib.request + +import helpers +from forgetest import bench as bench_mod +from forgetest import catalog, server +from forgetest.log import Log +from forgetest.runner import Failed, Runner + + +def t_pass(ctx): + ctx.log("hello") + ctx.evidence["k"] = 1 + + +def t_prompt(ctx): + ans = ctx.prompt("Did the light blink?", ("Yes", "No")) + if ans != "Yes": + raise Failed("operator said no") + + +def t_fail(ctx): + ctx.check(False, "deliberate") + + +def t_slow(ctx): + ctx.sleep(30) + + +def t_error(ctx): + raise RuntimeError("boom") + + +class ServerTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.mkdtemp(prefix="forgetest-") + os.environ["FORGETEST_DATA"] = cls.tmp + os.environ["FORGETEST_MARKER"] = os.path.join(cls.tmp, "marker") + cls.man = helpers.make_manifest() + cls.reg = helpers.registry( + helpers.make_test("fake.pass", [("forgectrl", "src/ui.c")], always=True, fn=t_pass), + helpers.make_test("fake.prompt", [("forgectrl", "src/auth.c")], fn=t_prompt, kind="operator"), + helpers.make_test("fake.fail", [("forgectrl", "src/cool.c")], fn=t_fail), + helpers.make_test("fake.slow", [("forgectrl", "src/main.c")], fn=t_slow), + helpers.make_test("fake.error", [("forgectrl", "src/main.c")], fn=t_error), + helpers.make_test("fake.live", [("grblhal-glowforge", "src/**")], fn=t_pass, kind="live"), + helpers.make_test("fake.needs", [("kernel-module-glowforge", "**")], fn=t_pass, + requires=("fake.prompt",)), + ) + # a fake bench tool + cls.tooldir = os.path.join(cls.tmp, "bench") + os.makedirs(cls.tooldir) + with open(os.path.join(cls.tooldir, "echo_tool.py"), "w") as f: + f.write("import sys, time\nprint('args', sys.argv[1:])\nsys.stdout.flush()\n" + "if 'slow' in sys.argv: time.sleep(30)\nsys.exit(0 if 'fail' not in sys.argv else 3)\n") + tools = [{"id": "echo", "title": "Echo", "script": "echo_tool.py", "safety": "dry", "where": "board", + "ported": True, "desc": "echo", + "args": [{"name": "word", "type": "str", "default": "hi", "help": ""}, + {"name": "n", "type": "int", "default": 2, "help": ""}]}, + {"id": "unported", "title": "U", "script": "nope.py", "safety": "dry", "where": "host", + "ported": False, "desc": "", "args": []}, + {"id": "hot", "title": "H", "script": "echo_tool.py", "safety": "live", "where": "board", + "ported": True, "desc": "", "args": []}, + {"id": "tk", "title": "T", "script": "echo_tool.py", "safety": "takeover", "where": "board", + "ported": True, "desc": "", "args": []}] + cls.bench = bench_mod.Bench(tools, tool_dir=cls.tooldir, + index_path=os.path.join(cls.tmp, "bench.jsonl")) + cls.log = Log(os.path.join(cls.tmp, "results.jsonl")) + cls.runner = Runner(cls.log, cls.man, cls.reg, cls.bench) + cls.token = server.load_token(os.path.join(cls.tmp, "token")) + cls.app = server.App(cls.runner, cls.token, export_dir=os.path.join(cls.tmp, "export")) + cls.srv = server.make_server(cls.app, "127.0.0.1", 0) + cls.port = cls.srv.server_address[1] + cls.th = threading.Thread(target=cls.srv.serve_forever, daemon=True) + cls.th.start() + + @classmethod + def tearDownClass(cls): + cls.srv.shutdown() + cls.srv.server_close() + shutil.rmtree(cls.tmp, ignore_errors=True) + + # -- helpers ---------------------------------------------------------- + def call(self, method, path, body=None, token=True, headers=None): + url = "http://127.0.0.1:%d%s" % (self.port, path) + hdrs = {"Host": "127.0.0.1:%d" % self.port} + if token: + hdrs["X-ForgeFIRM-Token"] = self.token + data = None + if body is not None: + data = json.dumps(body).encode() + hdrs["Content-Type"] = "application/json" + if headers: + hdrs.update(headers) + req = urllib.request.Request(url, data=data, method=method, headers=hdrs) + try: + with urllib.request.urlopen(req, timeout=10) as r: + raw = r.read() + st = r.status + ct = r.headers.get("Content-Type", "") + except urllib.error.HTTPError as e: + raw = e.read() + st = e.code + ct = e.headers.get("Content-Type", "") + if "json" in ct: + return st, json.loads(raw.decode()) + return st, raw + + def wait_idle(self, timeout=10): + deadline = time.time() + timeout + while time.time() < deadline: + st, d = self.call("GET", "/state") + if not d["running"]: + return d + time.sleep(0.1) + self.fail("run did not finish") + + def wait_prompt(self, timeout=10): + deadline = time.time() + timeout + while time.time() < deadline: + st, d = self.call("GET", "/state") + if d["running"] and d["running"]["prompt"]: + return d["running"]["prompt"] + time.sleep(0.05) + self.fail("no prompt appeared") + + # -- tests ----------------------------------------------------------------- + def test_01_auth_and_page(self): + st, d = self.call("GET", "/state", token=False) + self.assertEqual(st, 200) + st, d = self.call("GET", "/", token=False) + self.assertEqual(st, 200) + self.assertIn(self.token.encode(), d) + st, d = self.call("GET", "/state", headers={"Host": "evil.example.net"}) + self.assertEqual(st, 403) + st, d = self.call("GET", "/state", headers={"Origin": "http://evil.example.net"}) + self.assertEqual(st, 403) + st, d = self.call("GET", "/state", headers={"Sec-Fetch-Site": "cross-site"}) + self.assertEqual(st, 403) + st, d = self.call("POST", "/start", {"test": "fake.pass"}, token=False) + self.assertEqual(st, 403) + self.assertEqual(d["error"], "authentication required") + st, d = self.call("GET", "/catalog") + self.assertEqual(st, 200) + self.assertEqual(len(d["tests"]), 7) + st, d = self.call("GET", "/nope") + self.assertEqual(st, 404) + + def test_02_run_pass_opens_campaign(self): + st, d = self.call("GET", "/state") + self.assertIsNone(d["campaign"]) + st, d = self.call("POST", "/start", {"test": "fake.pass"}) + self.assertEqual(st, 200, d) + state = self.wait_idle() + self.assertIsNotNone(state["campaign"]) + self.assertEqual(state["tests"]["fake.pass"]["status"], "pass") + self.assertEqual(state["last_run"]["finished"]["result"], "PASS") + st, rec = self.call("GET", "/result?test=fake.pass") + self.assertEqual(st, 200) + self.assertEqual(rec["evidence"], {"k": 1}) + self.assertTrue(any("hello" in l for l in rec["log"])) + + def test_03_requires_and_live_gate(self): + st, d = self.call("POST", "/start", {"test": "fake.needs"}) + self.assertEqual(st, 409) + self.assertIn("prerequisites", d["message"]) + st, d = self.call("POST", "/start", {"test": "fake.live"}) + self.assertEqual(st, 409) + self.assertIn("live", d["message"]) + st, d = self.call("POST", "/start", {"test": "fake.live", "ack_live": True}) + self.assertEqual(st, 200) + state = self.wait_idle() + self.assertEqual(state["tests"]["fake.live"]["status"], "pass") + st, rec = self.call("GET", "/result?test=fake.live") + self.assertTrue(rec["evidence"]["operator"]["ack_live"]) + + def test_04_prompt_flow(self): + st, d = self.call("POST", "/start", {"test": "fake.prompt"}) + self.assertEqual(st, 200) + p = self.wait_prompt() + self.assertEqual(p["options"], ["Yes", "No"]) + st, d = self.call("POST", "/answer", {"prompt_id": p["id"], "value": "Maybe"}) + self.assertEqual(st, 409) + st, d = self.call("POST", "/answer", {"prompt_id": p["id"], "value": "Yes"}) + self.assertEqual(st, 200) + state = self.wait_idle() + self.assertEqual(state["tests"]["fake.prompt"]["status"], "pass") + st, rec = self.call("GET", "/result?test=fake.prompt") + self.assertEqual(rec["answers"][0]["answer"], "Yes") + # now the dependent test may start + st, d = self.call("POST", "/start", {"test": "fake.needs"}) + self.assertEqual(st, 200) + self.wait_idle() + + def test_05_busy_abort(self): + st, d = self.call("POST", "/start", {"test": "fake.slow"}) + self.assertEqual(st, 200) + st, d = self.call("POST", "/start", {"test": "fake.pass"}) + self.assertEqual(st, 409) + st, d = self.call("POST", "/abort") + self.assertEqual(st, 200) + state = self.wait_idle() + self.assertEqual(state["tests"]["fake.slow"]["status"], "aborted") + self.assertIsNotNone(state["campaign"], "an abort does not close the campaign") + + def test_06_error_and_fail_close_campaign(self): + st, d = self.call("GET", "/state") + cid = d["campaign"]["id"] + st, d = self.call("POST", "/start", {"test": "fake.error"}) + state = self.wait_idle() + self.assertEqual(state["tests"]["fake.error"]["status"], "error") + self.assertIsNone(state["campaign"]) + self.assertEqual(state["closed_by"], "fail") + # inherited passes survive; the core is required again + self.assertEqual(state["tests"]["fake.prompt"]["status"], "inherited") + self.assertTrue(state["tests"]["fake.pass"]["required"]) + # a new start opens a new campaign + st, d = self.call("POST", "/start", {"test": "fake.pass"}) + state = self.wait_idle() + self.assertIsNotNone(state["campaign"]) + self.assertNotEqual(state["campaign"]["id"], cid) + + def test_07_export_and_invalidate(self): + st, d = self.call("POST", "/export") + self.assertEqual(st, 200) + self.assertFalse(d["authorized"]) + st, raw = self.call("GET", "/export/acceptance.json") + self.assertEqual(st, 200) + art = raw if isinstance(raw, dict) else json.loads(raw) + self.assertEqual(art["manifest_sha"], self.man.content_sha) + st, raw = self.call("GET", "/export/acceptance.md") + self.assertEqual(st, 200) + self.assertIn(b"Release authorized: NO", raw) + st, d = self.call("POST", "/invalidate", {"reason": ""}) + self.assertEqual(st, 400) + st, d = self.call("POST", "/invalidate", {"reason": "new tube"}) + self.assertEqual(st, 200) + st, d = self.call("GET", "/state") + self.assertEqual(d["invalidate"]["reason"], "new tube") + self.assertIsNone(d["campaign"]) + for tid, ts in d["tests"].items(): + self.assertNotEqual(ts["status"], "inherited", tid) + st, raw = self.call("GET", "/log", token=False) + self.assertEqual(st, 200) + self.assertIn(b'"t": "invalidate"'.replace(b" ", b""), raw.replace(b" ", b"")) + + def test_08_bench(self): + st, d = self.call("GET", "/bench") + self.assertEqual(st, 200) + ids = [t["id"] for t in d["tools"]] + self.assertEqual(ids, ["echo", "unported", "hot", "tk"]) + st, d = self.call("POST", "/bench/start", {"tool": "unported"}) + self.assertEqual(st, 409) + st, d = self.call("POST", "/bench/start", {"tool": "hot"}) + self.assertEqual(st, 409) + st, d = self.call("POST", "/bench/start", {"tool": "echo", "args": {"word": "yo", "n": "x"}}) + self.assertEqual(st, 409) + st, d = self.call("POST", "/bench/start", {"tool": "echo", "args": {"word": "yo", "n": 5}}) + self.assertEqual(st, 200, d) + state = self.wait_idle() + self.assertEqual(state["last_run"]["kind"], "bench") + self.assertEqual(state["last_run"]["finished"]["result"], "OK") + self.assertTrue(any("['yo', '5']" in l for l in state["last_run"]["log"])) + st, d = self.call("GET", "/bench") + self.assertEqual(d["tools"][0]["last"]["result"]["result"], "OK") + # a failing tool and an aborted one + st, d = self.call("POST", "/bench/start", {"tool": "echo", "args": {"word": "fail", "n": 1}}) + state = self.wait_idle() + self.assertEqual(state["last_run"]["finished"]["result"], "EXIT 3") + st, d = self.call("POST", "/bench/start", {"tool": "echo", "args": {"word": "slow", "n": 1}}) + self.assertEqual(st, 200) + time.sleep(0.5) + st, d = self.call("POST", "/abort") + state = self.wait_idle() + self.assertEqual(state["last_run"]["finished"]["result"], "ABORTED") + # a takeover tool runs inside the takeover wrapper (init.d is absent on the host: rc 127) + st, d = self.call("POST", "/bench/start", {"tool": "tk"}) + self.assertEqual(st, 200, d) + state = self.wait_idle() + log = "\n".join(state["last_run"]["log"]) + self.assertIn("takeover: pulse device free", log) + self.assertIn("takeover: forgectrl start", log) + self.assertFalse(os.path.exists(os.environ["FORGETEST_MARKER"])) + # bench runs never touched the acceptance log + recs = self.log.read() + self.assertFalse(any(r.get("t") == "result" and r.get("test") == "echo" for r in recs)) + + def test_09_recovery_marker(self): + marker = os.environ["FORGETEST_MARKER"] + with open(marker, "w") as f: + f.write("x fake.slow\n") + r = Runner(self.log, self.man, self.reg, self.bench) + self.assertTrue(any("recovered" in m for m in r.messages)) + self.assertFalse(os.path.exists(marker)) + + +if __name__ == "__main__": + unittest.main() diff --git a/meta-forgefirm/recipes-forgefirm/forgetest/forgetest.bb b/meta-forgefirm/recipes-forgefirm/forgetest/forgetest.bb new file mode 100644 index 0000000..59e4950 --- /dev/null +++ b/meta-forgefirm/recipes-forgefirm/forgetest/forgetest.bb @@ -0,0 +1,47 @@ +SUMMARY = "ForgeFIRM release acceptance tool (dev image)" +DESCRIPTION = "Runs the release acceptance catalog against the machine from a \ +self-contained web page (HTTP :8090), keeps the append-only result log under \ +/data/forgetest, exports the release artifact the release gate reads, and \ +carries the bench diagnostics page. Installed only on the dev image." +HOMEPAGE = "https://github.com/ScottW514/forgefirm" +LICENSE = "MIT" +LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + +# The tool's canonical home is forgetest/ in this repo; the bench scripts +# it drives live in scripts/bench/. Both are packaged from the same tree +# that builds the images, so the catalog and the gate agree by construction. +FILESEXTRAPATHS:prepend := "${THISDIR}/../../../:" + +SRC_URI = " \ + file://forgetest/ \ + file://scripts/bench/ \ +" + +S = "${WORKDIR}" + +inherit python3-dir update-rc.d forgefirm-manifest + +# Non-git sources: fingerprint the package directory (dev-only component; +# it identifies the campaign, never a test fingerprint). +FORGEFIRM_MANIFEST_SRC = "${WORKDIR}/forgetest/forgetest" + +RDEPENDS:${PN} = "python3 forgectrl" + +INITSCRIPT_NAME = "forgetest" +INITSCRIPT_PARAMS = "start 95 2 3 4 5 . stop 70 0 1 6 ." + +do_install() { + # the package (sources only; python compiles on first import) + for f in $(cd ${WORKDIR}/forgetest/forgetest && find . -name '*.py' -type f); do + install -Dm 0644 ${WORKDIR}/forgetest/forgetest/$f \ + ${D}${PYTHON_SITEPACKAGES_DIR}/forgetest/$f + done + # the bench scripts the #bench tab drives + install -d ${D}${datadir}/forgetest/bench + for f in ${WORKDIR}/scripts/bench/*.py; do + install -m 0755 $f ${D}${datadir}/forgetest/bench/ + done + install -Dm 0755 ${WORKDIR}/forgetest/forgetest.init ${D}${sysconfdir}/init.d/forgetest +} + +FILES:${PN} += "${PYTHON_SITEPACKAGES_DIR}/forgetest ${datadir}/forgetest" diff --git a/meta-forgefirm/recipes-forgefirm/images/forgefirm-image-dev.bb b/meta-forgefirm/recipes-forgefirm/images/forgefirm-image-dev.bb index 365bcab..f19a7d2 100644 --- a/meta-forgefirm/recipes-forgefirm/images/forgefirm-image-dev.bb +++ b/meta-forgefirm/recipes-forgefirm/images/forgefirm-image-dev.bb @@ -3,9 +3,12 @@ require forgefirm-image.bb DESCRIPTION = "OpenGlow/ForgeFIRM development image for Glowforge" # Strict superset of forgefirm-image: everything the main image ships, plus -# debug tooling. +# debug tooling. forgetest is the release acceptance tool (HTTP :8090) and +# the bench diagnostics page; it belongs to the bench, never to a release +# image (docs/ACCEPTANCE.md). IMAGE_INSTALL += " \ forgectrl \ + forgetest \ " # debug-tweaks (passwordless root, root SSH login) belongs ONLY to the dev diff --git a/scripts/bench/README.md b/scripts/bench/README.md index deb85c6..e625b88 100644 --- a/scripts/bench/README.md +++ b/scripts/bench/README.md @@ -1,7 +1,13 @@ # ForgeFIRM bench tools Hardware-verification tools for the ForgeFIRM bench. All run ON the -target board (dev image, python3 present) unless noted. Host-side tools +target board (dev image, python3 present) unless noted. The dev image +installs them under `/usr/share/forgetest/bench/`, and the acceptance +tool's **Bench diagnostics** tab (`http://:8090/#bench`, +`docs/ACCEPTANCE.md`) runs the board-side ones with their arguments and +the output on the page (takeover tools get forgectrl stopped and started +around the run); the acceptance catalog itself is built from ports of +these drills. Host-side tools take the machine address from `GF_HOST` (or `argv`, where stated); the ones that shell into the board over ssh use the `ssh` on `PATH`, or the client named by `GF_SSH` (for example `GF_SSH='wsl -d -- ssh'`