diff --git a/forgetest/forgetest/baseline.py b/forgetest/forgetest/baseline.py index 400f306..b7dbde1 100644 --- a/forgetest/forgetest/baseline.py +++ b/forgetest/forgetest/baseline.py @@ -41,6 +41,7 @@ that made it, not a reason to hand the dirt on. import json import os import struct +import subprocess import time from . import hw @@ -289,6 +290,11 @@ def read_program_total(): return None +# The extension host stops a removed package's service on its next +# one-second turn; its group is killed and waited for inside that turn. +EXT_STOP_S = 8 + + class Leftover: def __init__(self, item, found, expected, action): self.item = item @@ -489,6 +495,7 @@ class Baseline: self._forgectrl_side(left, captured) self._kernel_side(left) self._lamp_side(left) + self._ext_side(left) self._preserved(left, captured) if left: self.log("%s: %d leftover(s): %s" % (phase, len(left), "; ".join(str(x) for x in left))) @@ -687,6 +694,37 @@ class Baseline: left.append(Leftover("cool", found, "idle/unarmed/no hold", act if w is not None else "failed: still %s" % found)) + def _ext_side(self, left): + """A campaign's machine is extension-free. What a test made and + left behind (a package under the tests' own prefix, the owner key + it was signed with) is removed, and the host stops its service. + The operator's own packages are theirs: running ones are reported + and not touched.""" + for id_ in [p for p in hw.ext_packages() if p.startswith(hw.EXT_TEST_PREFIX)]: + try: + p = subprocess.run([hw.FORGEEXT, "remove", id_], capture_output=True, text=True, timeout=120) + ok = p.returncode == 0 and id_ not in hw.ext_packages() + act = "restored" if ok else "failed: %s" % (p.stdout.strip()[-120:] or p.returncode) + except (OSError, subprocess.SubprocessError) as e: + act = "failed: %s" % e + left.append(Leftover("ext.package", id_, "not installed", act)) + keys = hw.EXT_ROOT + "/keys" + for name in sorted(os.listdir(keys)) if os.path.isdir(keys) else []: + if not name.startswith("forgetest-"): + continue + try: + os.remove(os.path.join(keys, name)) + act = "restored" + except OSError as e: + act = "failed: %s" % e + left.append(Leftover("ext.owner_key", name, "absent", act)) + # the host stops a removed package's service on its next turn + running = self._wait("no test package running", lambda: not hw.pool_pids() or None, EXT_STOP_S) is None \ + and hw.pool_pids() + if running: + left.append(Leftover("ext.running", "%d process(es) under pool accounts, packages %s" + % (len(running), hw.ext_packages()), "an extension-free machine", "unrestorable")) + def _lamp_side(self, left): """The lid lamp at forgectrl's idle level (the lid_lamp_idle setting). In cloud mode the cloud client owns the lamp (its lid-image level).""" diff --git a/forgetest/forgetest/hw.py b/forgetest/forgetest/hw.py index 1259d46..f389744 100644 --- a/forgetest/forgetest/hw.py +++ b/forgetest/forgetest/hw.py @@ -324,6 +324,40 @@ def initd(service, action, timeout=60): return p.returncode, p.stdout.decode("utf-8", "replace") +# The extension host's side of the machine (forgefirm-sandbox, forgeext). +EXT_POOL_FIRST, EXT_POOL_LAST = 800, 831 +EXT_ROOT = "/data/forgefirm/ext" +EXT_TEST_PREFIX = "org.forgetest." # a package a test made; every other one is the operator's +FORGEEXT = "/usr/bin/forgeext" + + +def pool_pids(): + """{pid: uid} of every process that runs under an extension pool account.""" + out = {} + try: + pids = [p for p in os.listdir("/proc") if p.isdigit()] + except OSError: + return out + for pid in pids: + try: + with open("/proc/%s/status" % pid) as f: + line = next((x for x in f if x.startswith("Uid:")), "") + uid = int(line.split()[1]) # the real uid; a service has all four alike + except (OSError, IndexError, ValueError): + continue + if EXT_POOL_FIRST <= uid <= EXT_POOL_LAST: + out[int(pid)] = uid + return out + + +def ext_packages(): + """The ids of the installed extension packages.""" + try: + return sorted(d for d in os.listdir(EXT_ROOT + "/pkg") if os.path.isdir(os.path.join(EXT_ROOT, "pkg", d))) + except OSError: + return [] + + def pidof(comm): """PIDs whose /proc//comm equals comm (15-char kernel limit applies).""" out = [] diff --git a/forgetest/forgetest/suite/image.py b/forgetest/forgetest/suite/image.py index d1f7d82..faeb91e 100644 --- a/forgetest/forgetest/suite/image.py +++ b/forgetest/forgetest/suite/image.py @@ -111,7 +111,7 @@ def fds_of(pid): @test("image.health", title="Post-flash image health", subsystem="image", kind="auto", always=True, est_min=1, - covers=[("forgectrl", "init/**"), ("forgectrl", "src/main.c"), ("forgectrl", "src/auth.c"), + covers=[("forgectrl", "init/**"), ("forgeext", "init/**"), ("forgectrl", "src/main.c"), ("forgectrl", "src/auth.c"), ("forgectrl", "CMakeLists.txt"), ("grblhal-glowforge", "src/boards/**"), ("grblhal-glowforge", "CMakeLists.txt"), ("kernel-module-glowforge", "**"), ("linux-fslc", "**")], @@ -120,7 +120,9 @@ def fds_of(pid): "console banner and the motd, " "the kernel options, the module, the pulse ring it maps and the SDMA clocks it holds, " "the daemon ownership, " - "the init ordering, the file modes the release depends on, and the mounts: the " + "the init ordering, the extension host as one process that starts after forgectrl and " + "stops before it on a machine with no package installed and nothing running under a " + "pool account, the file modes the release depends on, and the mounts: the " "rootfs read-only, /data writable, the account files and the banner rendered " "into tmpfs, the sshd host keys on /data, the factory slots on the dev image only.") def image_health(ctx): @@ -267,6 +269,29 @@ def image_health(ctx): ctx.check(kg and kf, "rc6.d lacks the grblhal/forgectrl kill links") ctx.check(kg[0] < kf[0], "controller kill link %s must sort before forgectrl's %s", kg[0], kf[0]) + # 5b. the extension host: one process, up after forgectrl and down + # before it, and a campaign's machine is extension-free + host = [] + for pid in hw.pidof("forgeext"): + argv = _read("/proc/%d/cmdline" % pid, "").split("\0") + if argv[:2] == [hw.FORGEEXT, "run"]: + host.append(pid) + ev["extension_host_pids"] = host + ctx.check(len(host) == 1, "the extension host is not one running process: %s", host) + s5 = sorted(os.path.basename(x) for x in glob.glob("/etc/rc5.d/S*")) + sx = [x for x in s5 if x.endswith("forgeext")] + sf = [x for x in s5 if x.endswith("forgectrl")] + kx = [x for x in k if x.endswith("forgeext")] + ev["forgeext_links"] = {"start": sx, "kill": kx} + ctx.check(sx and sf and sf[0] < sx[0], "the extension host must start after forgectrl: %s, %s", sf, sx) + ctx.check(kx and kx[0] < kf[0], "the extension host must stop before forgectrl: %s, %s", kx, kf) + ev["extension_packages"] = hw.ext_packages() + ev["pool_processes"] = len(hw.pool_pids()) + ctx.log("extension host pid %s, packages %s, pool processes %d", host, ev["extension_packages"], ev["pool_processes"]) + ctx.check(not ev["extension_packages"], "a campaign wants an extension-free machine: installed %s", + ev["extension_packages"]) + ctx.check(not ev["pool_processes"], "%d process(es) run under extension pool accounts", ev["pool_processes"]) + # 6. logging lever present, no userspace watchdog daemon logging = [n for n in ("forgefirm-logging", "forgefirm-logrotate") if os.path.exists("/etc/init.d/" + n)] ev["logging_init"] = logging diff --git a/forgetest/tests/test_baseline.py b/forgetest/tests/test_baseline.py index f965af5..aac783f 100644 --- a/forgetest/tests/test_baseline.py +++ b/forgetest/tests/test_baseline.py @@ -457,6 +457,71 @@ if __name__ == "__main__": unittest.main() +class ExtensionFreeTests(BaselineTests): + """A campaign's machine is extension-free: what a test made is removed, + what is the operator's is reported and left alone.""" + + def setUp(self): + super().setUp() + self.root = os.path.join(self.tmp, "ext") + for d in ("pkg", "keys", "data"): + os.makedirs(os.path.join(self.root, d)) + # a stand-in for the host's command line: remove takes the package away + self.forgeext = os.path.join(self.tmp, "forgeext") + with open(self.forgeext, "w") as f: + f.write('#!/bin/sh\n[ "$1" = remove ] && rm -rf "%s/pkg/$2" && echo \'{"ok": true}\'\n' % self.root) + os.chmod(self.forgeext, 0o755) + self.pool = {} + self._saved = (baseline.hw.EXT_ROOT, baseline.hw.FORGEEXT, baseline.hw.pool_pids, baseline.EXT_STOP_S) + baseline.hw.EXT_ROOT, baseline.hw.FORGEEXT = self.root, self.forgeext + baseline.hw.pool_pids = lambda: dict(self.pool) + baseline.EXT_STOP_S = 0.2 + + def tearDown(self): + baseline.hw.EXT_ROOT, baseline.hw.FORGEEXT, baseline.hw.pool_pids, baseline.EXT_STOP_S = self._saved + super().tearDown() + + def _package(self, id_): + os.makedirs(os.path.join(self.root, "pkg", id_, "1.0.0")) + + def _key(self, name): + with open(os.path.join(self.root, "keys", name), "w") as f: + f.write("x") + + def test_an_installed_package_that_does_not_run_is_nobodys_leftover(self): + self._package("org.example.notify") + self._key("owner.pub") + self.assertEqual(self.bl().enforce("pre", captured=None), []) + + def test_what_a_test_left_behind_is_removed_and_the_operators_is_not(self): + self._package("org.forgetest.reference") + self._package("org.example.notify") + self._key("forgetest-reference.pub") + self._key("owner.pub") + left = {x.item: x for x in self.bl().enforce("post", captured=None)} + self.assertEqual(sorted(left), ["ext.owner_key", "ext.package"]) + self.assertEqual((left["ext.package"].found, left["ext.package"].action), ("org.forgetest.reference", "restored")) + self.assertEqual((left["ext.owner_key"].found, left["ext.owner_key"].action), ("forgetest-reference.pub", "restored")) + self.assertEqual(os.listdir(os.path.join(self.root, "pkg")), ["org.example.notify"]) + self.assertEqual(os.listdir(os.path.join(self.root, "keys")), ["owner.pub"]) + + def test_a_package_that_cannot_be_removed_says_so(self): + self._package("org.forgetest.reference") + with open(self.forgeext, "w") as f: + f.write('#!/bin/sh\necho \'{"ok": false, "error": "the lock is held"}\'\nexit 1\n') + left = self.bl().enforce("post", captured=None) + self.assertEqual([x.item for x in left], ["ext.package"]) + self.assertTrue(left[0].action.startswith("failed:") and "the lock is held" in left[0].action, left[0].action) + + def test_running_extensions_are_reported_and_left_alone(self): + self._package("org.example.notify") + self.pool = {4242: 800} + left = self.bl().enforce("pre", captured=None) + self.assertEqual([(x.item, x.action) for x in left], [("ext.running", "unrestorable")]) + self.assertIn("org.example.notify", left[0].found) + self.assertTrue(os.path.isdir(os.path.join(self.root, "pkg", "org.example.notify"))) + + class TransientNotLeftoverTests(BaselineTests): """A leftover is what a run left behind, not the machine part-way through its own work. Found on the bench reference, where