forgetest: the release acceptance tool and the bench diagnostics page

A stdlib-only daemon on the dev image (HTTP :8090) that runs the acceptance
catalog against the machine from a self-contained page, keeps the append-only
result log under /data/forgetest, and exports the release artifact the gate
reads. Tests declare kind (auto / operator / live), hardware (api / takeover),
coverage globs, prerequisites, and core membership; a test's domain
fingerprint is the hash of the manifest files its globs select plus the
platform and its own implementation, so a PASS stays valid exactly while
nothing it covers changed. Campaign rules: a FAIL ends the campaign, the core
(image health, kernel latch and drills, one live emission witness) is never
inherited, invalidate-all forces a full campaign, no SKIP. Live tests need the
operator acknowledgment and the physical arm press through the controller;
takeover tests stop forgectrl for the duration with a crash-recoverable
marker; the tool never touches the laser latch.

Catalog v1: 24 tests ported from the proven bench drills with their recorded
pass criteria (image, kernel K1-K3 and fire A/B/U, forgectrl API and logs,
motion incl. dead-man, cooling, live laser, camera, update, cloud). The bench
tab lists every scripts/bench tool and runs the board-side ones as
subprocesses (takeover tools wrapped). 44 host unit tests, including the gate
verification fixtures. Installed only by forgefirm-image-dev, with the bench
scripts under /usr/share/forgetest/bench.

forgetest/forgetest/manifest.py

@@ -0,0 +1,167 @@
+"""The image manifest and the domain fingerprint.
+
+/etc/forgefirm-manifest.json (written by forgefirm-image-manifest.bbclass)
+identifies the build's inputs: for every component the pinned revision and
+one [path, blob-id] pair per source file, plus the platform identity
+(machine, kernel modules directory, device tree hashes, layer content
+hashes). This module loads it and computes a test's *domain fingerprint*:
+the hash of the source files its coverage globs select, plus the platform,
+plus the test's own implementation. A recorded PASS applies to a build
+exactly when the fingerprint recomputed from that build's manifest is the
+same - the same code runs on the board and in the release gate.
+"""
+import hashlib
+import json
+import os
+import re
+
+DEFAULT_PATH = "/etc/forgefirm-manifest.json"
+
+# Components that ship only on the dev image. They can never be part of a
+# fingerprint (the release manifest lacks them, so the gate could not
+# recompute it); the test implementation is folded in separately.
+DEV_ONLY_COMPONENTS = ("forgetest",)
+
+
+def canonical(obj):
+    """Canonical JSON: sorted keys, no whitespace - the hashing form."""
+    return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
+
+
+def sha256_text(text):
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def glob_to_regex(pattern):
+    """Coverage glob -> anchored regex. '**' spans directories, '*' and '?'
+    stay inside one path segment. Paths use '/' (git paths)."""
+    out = []
+    i, n = 0, len(pattern)
+    while i < n:
+        c = pattern[i]
+        if c == "*":
+            if pattern[i:i + 2] == "**":
+                # '**/' also matches zero directories
+                if pattern[i:i + 3] == "**/":
+                    out.append("(?:.*/)?")
+                    i += 3
+                    continue
+                out.append(".*")
+                i += 2
+                continue
+            out.append("[^/]*")
+        elif c == "?":
+            out.append("[^/]")
+        else:
+            out.append(re.escape(c))
+        i += 1
+    return re.compile("^" + "".join(out) + "$")
+
+
+def match_files(files, pattern):
+    """(path, blob) pairs from a component's file list that the glob selects."""
+    rx = glob_to_regex(pattern)
+    return [(p, b) for p, b in files if rx.match(p)]
+
+
+class Manifest:
+    def __init__(self, data):
+        self.data = data
+        self.components = data.get("components", {}) or {}
+        self.platform = data.get("platform", {}) or {}
+        self.image = data.get("image", {}) or {}
+        self.content_sha = data.get("content_sha256")
+
+    @classmethod
+    def load(cls, path=None):
+        path = path or os.environ.get("FORGETEST_MANIFEST") or DEFAULT_PATH
+        with open(path, "r", encoding="utf-8") as f:
+            return cls(json.load(f))
+
+    @classmethod
+    def from_json(cls, text):
+        return cls(json.loads(text))
+
+    @property
+    def version(self):
+        return self.image.get("version") or "unknown"
+
+    @property
+    def image_name(self):
+        return self.image.get("name") or "unknown"
+
+    def files(self, component):
+        """The [path, blob] list of a component, or None if the component
+        is not in this manifest."""
+        c = self.components.get(component)
+        if c is None:
+            return None
+        return [tuple(x) for x in c.get("files", [])]
+
+    def component_names(self):
+        return sorted(self.components)
+
+    def identity_sha(self):
+        """sha256 of the acceptance-relevant identity: every component
+        except the dev-only ones, plus the platform. Informational (the
+        gate decides per test, by fingerprint)."""
+        comps = {k: v for k, v in self.components.items() if k not in DEV_ONLY_COMPONENTS}
+        return sha256_text(canonical({"components": comps, "platform": self.platform}))
+
+
+def fingerprint(manifest, covers, extra=()):
+    """The domain fingerprint of a coverage map on a manifest.
+
+    covers: iterable of (component, glob). extra: strings folded in after
+    the files (the test's own implementation hash). A component the
+    manifest lacks contributes a marker so the fingerprint is still
+    defined and distinct.
+    """
+    parts = set()
+    for comp, pat in covers:
+        if comp in DEV_ONLY_COMPONENTS:
+            raise ValueError("coverage may not name the dev-only component %r" % comp)
+        files = manifest.files(comp)
+        if files is None:
+            parts.add((comp, "@missing", ""))
+            continue
+        for p, b in match_files(files, pat):
+            parts.add((comp, p, b))
+    h = hashlib.sha256()
+    h.update(canonical(sorted(parts)).encode("utf-8"))
+    h.update(b"\n")
+    h.update(canonical(manifest.platform).encode("utf-8"))
+    for e in extra:
+        h.update(b"\n")
+        h.update(str(e).encode("utf-8"))
+    return h.hexdigest()
+
+
+def coverage_report(manifest, tests, allow=()):
+    """Which manifest paths no test covers.
+
+    tests: iterable with .covers. allow: iterable of (component, glob)
+    that need no coverage (docs, CI, licenses...). Returns
+    {component: [uncovered paths]} for the non-dev-only components.
+    """
+    covered = {}
+    for t in tests:
+        for comp, pat in t.covers:
+            covered.setdefault(comp, []).append(glob_to_regex(pat))
+    allowed = {}
+    for comp, pat in allow:
+        allowed.setdefault(comp, []).append(glob_to_regex(pat))
+    report = {}
+    for comp in manifest.component_names():
+        if comp in DEV_ONLY_COMPONENTS:
+            continue
+        rxs = covered.get(comp, []) + allowed.get(comp, [])
+        star = allowed.get("*", [])
+        missing = []
+        for p, _b in manifest.files(comp):
+            if any(rx.match(p) for rx in rxs) or any(rx.match(p) for rx in star):
+                continue
+            missing.append(p)
+        if missing:
+            report[comp] = sorted(missing)
+    return report