forgetest: a campaign's machine is extension-free

An extension package is software the image does not carry, and a result
taken beside one is not a result about the image. Two places hold the line.

The baseline (_ext_side, in every pre and post pass): a package under the
tests' own prefix (org.forgetest.) and an owner key named forgetest-*.pub
are what a test made and left behind; they are removed (forgeext remove,
the key's file), recorded as restored, and the host stops the service on
its next turn. A process that still runs under a pool account after that
belongs to the operator's own packages: it is recorded as unrestorable and
never touched. An installed package that does not run is nobody's
leftover. hw.pool_pids() reads each process's Uid line; hw.ext_packages()
lists the package directory.

image.health (5b): the extension host is one process (/usr/bin/forgeext
run), its start link sorts after forgectrl's and its kill link before it,
no package is installed, and nothing runs under a pool account.

Proven. test_baseline.py, ExtensionFreeTests, over a stand-in forgeext and
a fake package tree: an installed package that does not run leaves
nothing; a test's package and key are removed and the operator's package
and key stay; a removal that fails says so in the host's own words;
running extensions are reported and left alone. The unit suite passes (451
tests, 0 undefined names). The link order image.health asks for is the one
the built root filesystems of image 20260921014201 have: S90forgectrl
before S91forgeext, K09forgeext before K90forgectrl.

Acceptance. image.health gains forgeext's init/** in its covers map; it
runs first in every campaign and is the on-image proof of 5b. The
baseline is harness, outside the suite and outside every fingerprint.

forgetest/forgetest/baseline.py

@@ -41,6 +41,7 @@ that made it, not a reason to hand the dirt on.
 import json
 import os
 import struct
+import subprocess
 import time
 
 from . import hw
@@ -289,6 +290,11 @@ def read_program_total():
         return None
 
 
+# The extension host stops a removed package's service on its next
+# one-second turn; its group is killed and waited for inside that turn.
+EXT_STOP_S = 8
+
+
 class Leftover:
     def __init__(self, item, found, expected, action):
         self.item = item
@@ -489,6 +495,7 @@ class Baseline:
         self._forgectrl_side(left, captured)
         self._kernel_side(left)
         self._lamp_side(left)
+        self._ext_side(left)
         self._preserved(left, captured)
         if left:
             self.log("%s: %d leftover(s): %s" % (phase, len(left), "; ".join(str(x) for x in left)))
@@ -687,6 +694,37 @@ class Baseline:
         left.append(Leftover("cool", found, "idle/unarmed/no hold",
                              act if w is not None else "failed: still %s" % found))
 
+    def _ext_side(self, left):
+        """A campaign's machine is extension-free. What a test made and
+        left behind (a package under the tests' own prefix, the owner key
+        it was signed with) is removed, and the host stops its service.
+        The operator's own packages are theirs: running ones are reported
+        and not touched."""
+        for id_ in [p for p in hw.ext_packages() if p.startswith(hw.EXT_TEST_PREFIX)]:
+            try:
+                p = subprocess.run([hw.FORGEEXT, "remove", id_], capture_output=True, text=True, timeout=120)
+                ok = p.returncode == 0 and id_ not in hw.ext_packages()
+                act = "restored" if ok else "failed: %s" % (p.stdout.strip()[-120:] or p.returncode)
+            except (OSError, subprocess.SubprocessError) as e:
+                act = "failed: %s" % e
+            left.append(Leftover("ext.package", id_, "not installed", act))
+        keys = hw.EXT_ROOT + "/keys"
+        for name in sorted(os.listdir(keys)) if os.path.isdir(keys) else []:
+            if not name.startswith("forgetest-"):
+                continue
+            try:
+                os.remove(os.path.join(keys, name))
+                act = "restored"
+            except OSError as e:
+                act = "failed: %s" % e
+            left.append(Leftover("ext.owner_key", name, "absent", act))
+        # the host stops a removed package's service on its next turn
+        running = self._wait("no test package running", lambda: not hw.pool_pids() or None, EXT_STOP_S) is None \
+            and hw.pool_pids()
+        if running:
+            left.append(Leftover("ext.running", "%d process(es) under pool accounts, packages %s"
+                                 % (len(running), hw.ext_packages()), "an extension-free machine", "unrestorable"))
+
     def _lamp_side(self, left):
         """The lid lamp at forgectrl's idle level (the lid_lamp_idle setting).
         In cloud mode the cloud client owns the lamp (its lid-image level)."""

forgetest/forgetest/hw.py

@@ -324,6 +324,40 @@ def initd(service, action, timeout=60):
     return p.returncode, p.stdout.decode("utf-8", "replace")
 
 
+# The extension host's side of the machine (forgefirm-sandbox, forgeext).
+EXT_POOL_FIRST, EXT_POOL_LAST = 800, 831
+EXT_ROOT = "/data/forgefirm/ext"
+EXT_TEST_PREFIX = "org.forgetest."          # a package a test made; every other one is the operator's
+FORGEEXT = "/usr/bin/forgeext"
+
+
+def pool_pids():
+    """{pid: uid} of every process that runs under an extension pool account."""
+    out = {}
+    try:
+        pids = [p for p in os.listdir("/proc") if p.isdigit()]
+    except OSError:
+        return out
+    for pid in pids:
+        try:
+            with open("/proc/%s/status" % pid) as f:
+                line = next((x for x in f if x.startswith("Uid:")), "")
+            uid = int(line.split()[1])              # the real uid; a service has all four alike
+        except (OSError, IndexError, ValueError):
+            continue
+        if EXT_POOL_FIRST <= uid <= EXT_POOL_LAST:
+            out[int(pid)] = uid
+    return out
+
+
+def ext_packages():
+    """The ids of the installed extension packages."""
+    try:
+        return sorted(d for d in os.listdir(EXT_ROOT + "/pkg") if os.path.isdir(os.path.join(EXT_ROOT, "pkg", d)))
+    except OSError:
+        return []
+
+
 def pidof(comm):
     """PIDs whose /proc/<pid>/comm equals comm (15-char kernel limit applies)."""
     out = []

forgetest/forgetest/suite/image.py

@@ -111,7 +111,7 @@ def fds_of(pid):
 
 @test("image.health", title="Post-flash image health", subsystem="image", kind="auto",
       always=True, est_min=1,
-      covers=[("forgectrl", "init/**"), ("forgectrl", "src/main.c"), ("forgectrl", "src/auth.c"),
+      covers=[("forgectrl", "init/**"), ("forgeext", "init/**"), ("forgectrl", "src/main.c"), ("forgectrl", "src/auth.c"),
               ("forgectrl", "CMakeLists.txt"), ("grblhal-glowforge", "src/boards/**"),
               ("grblhal-glowforge", "CMakeLists.txt"), ("kernel-module-glowforge", "**"),
               ("linux-fslc", "**")],
@@ -120,7 +120,9 @@ def fds_of(pid):
                   "console banner and the motd, "
                   "the kernel options, the module, the pulse ring it maps and the SDMA clocks it holds, "
                   "the daemon ownership, "
-                  "the init ordering, the file modes the release depends on, and the mounts: the "
+                  "the init ordering, the extension host as one process that starts after forgectrl and "
+                  "stops before it on a machine with no package installed and nothing running under a "
+                  "pool account, the file modes the release depends on, and the mounts: the "
                   "rootfs read-only, /data writable, the account files and the banner rendered "
                   "into tmpfs, the sshd host keys on /data, the factory slots on the dev image only.")
 def image_health(ctx):
@@ -267,6 +269,29 @@ def image_health(ctx):
     ctx.check(kg and kf, "rc6.d lacks the grblhal/forgectrl kill links")
     ctx.check(kg[0] < kf[0], "controller kill link %s must sort before forgectrl's %s", kg[0], kf[0])
 
+    # 5b. the extension host: one process, up after forgectrl and down
+    # before it, and a campaign's machine is extension-free
+    host = []
+    for pid in hw.pidof("forgeext"):
+        argv = _read("/proc/%d/cmdline" % pid, "").split("\0")
+        if argv[:2] == [hw.FORGEEXT, "run"]:
+            host.append(pid)
+    ev["extension_host_pids"] = host
+    ctx.check(len(host) == 1, "the extension host is not one running process: %s", host)
+    s5 = sorted(os.path.basename(x) for x in glob.glob("/etc/rc5.d/S*"))
+    sx = [x for x in s5 if x.endswith("forgeext")]
+    sf = [x for x in s5 if x.endswith("forgectrl")]
+    kx = [x for x in k if x.endswith("forgeext")]
+    ev["forgeext_links"] = {"start": sx, "kill": kx}
+    ctx.check(sx and sf and sf[0] < sx[0], "the extension host must start after forgectrl: %s, %s", sf, sx)
+    ctx.check(kx and kx[0] < kf[0], "the extension host must stop before forgectrl: %s, %s", kx, kf)
+    ev["extension_packages"] = hw.ext_packages()
+    ev["pool_processes"] = len(hw.pool_pids())
+    ctx.log("extension host pid %s, packages %s, pool processes %d", host, ev["extension_packages"], ev["pool_processes"])
+    ctx.check(not ev["extension_packages"], "a campaign wants an extension-free machine: installed %s",
+              ev["extension_packages"])
+    ctx.check(not ev["pool_processes"], "%d process(es) run under extension pool accounts", ev["pool_processes"])
+
     # 6. logging lever present, no userspace watchdog daemon
     logging = [n for n in ("forgefirm-logging", "forgefirm-logrotate") if os.path.exists("/etc/init.d/" + n)]
     ev["logging_init"] = logging

forgetest/tests/test_baseline.py

@@ -457,6 +457,71 @@ if __name__ == "__main__":
     unittest.main()
 
 
+class ExtensionFreeTests(BaselineTests):
+    """A campaign's machine is extension-free: what a test made is removed,
+    what is the operator's is reported and left alone."""
+
+    def setUp(self):
+        super().setUp()
+        self.root = os.path.join(self.tmp, "ext")
+        for d in ("pkg", "keys", "data"):
+            os.makedirs(os.path.join(self.root, d))
+        # a stand-in for the host's command line: remove <id> takes the package away
+        self.forgeext = os.path.join(self.tmp, "forgeext")
+        with open(self.forgeext, "w") as f:
+            f.write('#!/bin/sh\n[ "$1" = remove ] && rm -rf "%s/pkg/$2" && echo \'{"ok": true}\'\n' % self.root)
+        os.chmod(self.forgeext, 0o755)
+        self.pool = {}
+        self._saved = (baseline.hw.EXT_ROOT, baseline.hw.FORGEEXT, baseline.hw.pool_pids, baseline.EXT_STOP_S)
+        baseline.hw.EXT_ROOT, baseline.hw.FORGEEXT = self.root, self.forgeext
+        baseline.hw.pool_pids = lambda: dict(self.pool)
+        baseline.EXT_STOP_S = 0.2
+
+    def tearDown(self):
+        baseline.hw.EXT_ROOT, baseline.hw.FORGEEXT, baseline.hw.pool_pids, baseline.EXT_STOP_S = self._saved
+        super().tearDown()
+
+    def _package(self, id_):
+        os.makedirs(os.path.join(self.root, "pkg", id_, "1.0.0"))
+
+    def _key(self, name):
+        with open(os.path.join(self.root, "keys", name), "w") as f:
+            f.write("x")
+
+    def test_an_installed_package_that_does_not_run_is_nobodys_leftover(self):
+        self._package("org.example.notify")
+        self._key("owner.pub")
+        self.assertEqual(self.bl().enforce("pre", captured=None), [])
+
+    def test_what_a_test_left_behind_is_removed_and_the_operators_is_not(self):
+        self._package("org.forgetest.reference")
+        self._package("org.example.notify")
+        self._key("forgetest-reference.pub")
+        self._key("owner.pub")
+        left = {x.item: x for x in self.bl().enforce("post", captured=None)}
+        self.assertEqual(sorted(left), ["ext.owner_key", "ext.package"])
+        self.assertEqual((left["ext.package"].found, left["ext.package"].action), ("org.forgetest.reference", "restored"))
+        self.assertEqual((left["ext.owner_key"].found, left["ext.owner_key"].action), ("forgetest-reference.pub", "restored"))
+        self.assertEqual(os.listdir(os.path.join(self.root, "pkg")), ["org.example.notify"])
+        self.assertEqual(os.listdir(os.path.join(self.root, "keys")), ["owner.pub"])
+
+    def test_a_package_that_cannot_be_removed_says_so(self):
+        self._package("org.forgetest.reference")
+        with open(self.forgeext, "w") as f:
+            f.write('#!/bin/sh\necho \'{"ok": false, "error": "the lock is held"}\'\nexit 1\n')
+        left = self.bl().enforce("post", captured=None)
+        self.assertEqual([x.item for x in left], ["ext.package"])
+        self.assertTrue(left[0].action.startswith("failed:") and "the lock is held" in left[0].action, left[0].action)
+
+    def test_running_extensions_are_reported_and_left_alone(self):
+        self._package("org.example.notify")
+        self.pool = {4242: 800}
+        left = self.bl().enforce("pre", captured=None)
+        self.assertEqual([(x.item, x.action) for x in left], [("ext.running", "unrestorable")])
+        self.assertIn("org.example.notify", left[0].found)
+        self.assertTrue(os.path.isdir(os.path.join(self.root, "pkg", "org.example.notify")))
+
+
 class TransientNotLeftoverTests(BaselineTests):
     """A leftover is what a run left behind, not the machine part-way
     through its own work. Found on the bench reference, where