exthost.armed-freeze: a package's service is frozen for a real armed window

The freeze is the rule that keeps a package off the core while a job can
fire, and it had a host test against a stand-in forgectrl and one drill by
hand. This is its test on the machine, against the real engine.

The reference package runs, its heartbeat advancing twice a second.
cloud.dark-print's own job and body (a 30 s square at S0, the fixture's
press, the latch locked at the button, unlocked for the run, locked after)
open a real armed window over it. A sampler reads five times a second: the
engine's armed flag (GET /cool/status), the group's frozen state as the
kernel reports it (cgroup.events), the heartbeat, the service's pid, and
the latch. The test asks for a window of 10 s or more; frozen in every
sample from 2 s in to the close, with the heartbeat still; the first frozen
sample no later than the first unlocked-latch sample, so the freeze is in
place before the run; thawed, with the heartbeat moving again, 3 s after
the close; and one process throughout.

exthost.service's put-back and its as-found checks became _put_back() and
_as_found(), shared by both tests. exthost's top-level imports pulled the
setup suite (and now the cloud suite) in ahead of it and reordered the
catalog, so exthost is imported last: image, kernel, forgectrl, setup,
logs, motion, cooling, laser, camera, update, cloud, exthost; 106 tests.

Proven. The unit suite: 451 tests, 0 undefined names. On the bench
reference, image 20260920211625 with the cross-built forgectrl and host
from /tmp, the coolant taken from 27.2 to 25.4 C with an M8 session first
(the cloud client starts no print above 27): exthost.armed-freeze PASS, the
window open 37.5 s, the group frozen 0.42 s after it opened and 4.97 s
before the latch unlocked for the run, frozen in all 172 samples from 2 s
in to the close with the heartbeat still, thawed 0.62 s after the close.
With the host suspended (SIGSTOP) across the window, so that nobody
freezes anything, it FAILS: not frozen in 177 of 177 samples. exthost.service
PASS again after the refactor (a killed host's service gone in 0.05 s, the
host back and the service running after 5.6 s). The bench was left as
found.

Acceptance. exthost.armed-freeze is new: kind operator (the fixture
presses), takeover; it covers forgeext's src/super.*, src/run.*,
src/machine.*, src/cgroup.*, and forgectrl's src/cool.*, and requires
exthost.service and cloud.dark-print.

forgetest/forgetest/suite/__init__.py

@@ -9,7 +9,6 @@ order. Each module registers its tests with @catalog.test."""
 from . import image      # noqa: F401,E402
 from . import kernel     # noqa: F401,E402
 from . import forgectrl  # noqa: F401,E402
-from . import exthost    # noqa: F401,E402
 from . import setup # noqa: F401,E402
 from . import setup_dark  # noqa: F401,E402
 from . import setup_sheet  # noqa: F401,E402
@@ -20,3 +19,5 @@ from . import laser      # noqa: F401,E402
 from . import camera     # noqa: F401,E402
 from . import update     # noqa: F401,E402
 from . import cloud      # noqa: F401,E402
+# Last: it stands on the setup record's consent and on the cloud suite's dark print.
+from . import exthost    # noqa: F401,E402

forgetest/forgetest/suite/exthost.py

@@ -15,6 +15,7 @@ import time
 from ..catalog import test
 from .forgectrl import lan_ip
 from .image import kernel_config
+from .cloud import OFFLINE_STEP, Offline, dark_print_body, enter_offline, latch_locked, offline_cleanup, offline_job
 from .setup import SAFETY_PHRASE, read_file, record_path, request, write_file
 
 CG = "/sys/fs/cgroup"
@@ -644,6 +645,44 @@ def _pack_reference(work, lan):
     return signed, key + ".pub"
 
 
+def _put_back(ctx, fc, work, prior, etag, raw, dir_mode):
+    """Everything a test of the host put on the machine, taken away again:
+    safe mode, the package, the owner key, the work directory, the
+    setting, the data directory's mode, and the setup record (under a
+    forgectrl restart)."""
+    import shutil
+    if os.path.exists(SAFE_FILE):
+        os.remove(SAFE_FILE)
+    st, reply = fc.post("/settings", data={"ext_enabled": "0"})
+    r = _forgeext("remove", REF_ID)
+    ctx.log("remove %s -> %s", REF_ID, "ok" if r.get("ok") else r.get("error"))
+    owner_key = os.path.join(EXT_ROOT, "keys", REF_KEY + ".pub")
+    if os.path.exists(owner_key):
+        os.remove(owner_key)
+    shutil.rmtree(work, ignore_errors=True)
+    if prior == "1":
+        st, reply = fc.post("/settings", data={"ext_enabled": "1", "advisory": etag, "phrase": SAFETY_PHRASE})
+    elif prior == "":
+        st, reply = fc.post("/settings", params={"ext_enabled": ""})
+    ctx.log("restore ext_enabled=%r -> %s", prior, st)
+    if prior != "1":
+        os.chmod(os.path.dirname(EXT_ROOT), dir_mode)
+    with ctx.takeover():
+        write_file(record_path(), raw)
+    ctx.log("the previous record is back under a restart")
+
+
+def _as_found(ctx, fc, prior, raw, dir_mode, found_tree):
+    ctx.check((fc.settings().get("ext_enabled") or "") == prior, "ext_enabled not restored: %r, was %r",
+              fc.settings().get("ext_enabled"), prior)
+    ctx.check(read_file(record_path()) == raw, "the setup record on disk is not the one found")
+    ctx.check(prior == "1" or os.stat(os.path.dirname(EXT_ROOT)).st_mode & 0o7777 == dir_mode,
+              "the data directory's mode is not the one found")
+    left = _tree(EXT_ROOT)
+    ctx.check(left == found_tree, "the extension root is not as found: %s", sorted(set(left) ^ set(found_tree)))
+    ctx.check(len(_host_pids()) == 1, "the extension host is not running at the end")
+
+
 @test("exthost.service", title="A package's service runs confined under the extension host",
       subsystem="exthost", kind="auto", hardware="takeover", est_min=5,
       covers=[("forgeext", "**"), ("forgectrl", "src/main.c"), ("forgectrl", "src/logs.*")],
@@ -815,30 +854,126 @@ def service(ctx):
         ctx.check("ext_enabled" in (_host_status().get("off_reason") or ""), "the host does not say why nothing runs: %s",
                   _host_status().get("off_reason"))
     finally:
-        if os.path.exists(SAFE_FILE):
+        _put_back(ctx, fc, work, prior, etag, raw, dir_mode)
-            os.remove(SAFE_FILE)
+    _as_found(ctx, fc, prior, raw, dir_mode, found_tree)
-        st, reply = fc.post("/settings", data={"ext_enabled": "0"})
-        r = _forgeext("remove", REF_ID)
-        ctx.log("remove %s -> %s", REF_ID, "ok" if r.get("ok") else r.get("error"))
-        if os.path.exists(owner_key):
-            os.remove(owner_key)
-        shutil.rmtree(work, ignore_errors=True)
-        if prior == "1":
-            st, reply = fc.post("/settings", data={"ext_enabled": "1", "advisory": etag, "phrase": SAFETY_PHRASE})
-        elif prior == "":
-            st, reply = fc.post("/settings", params={"ext_enabled": ""})
-        ctx.log("restore ext_enabled=%r -> %s", prior, st)
-        if prior != "1":
-            os.chmod(data_dir, dir_mode)
-        with ctx.takeover():
-            write_file(record_path(), raw)
-        ctx.log("the previous record is back under a restart")
 
-    ctx.check((fc.settings().get("ext_enabled") or "") == prior, "ext_enabled not restored: %r, was %r",
+
-              fc.settings().get("ext_enabled"), prior)
+# ------------------------------------------------- the armed window
-    ctx.check(read_file(record_path()) == raw, "the setup record on disk is not the one found")
+
-    ctx.check(prior == "1" or os.stat(data_dir).st_mode & 0o7777 == dir_mode, "the data directory's mode is not the one found")
+def _frozen(id_):
-    left = _tree(EXT_ROOT)
+    ev = _read("%s/%s/cgroup.events" % (POOL_GROUP, id_))
-    ctx.check(left == found_tree, "the extension root is not as found: %s", sorted(set(left) ^ set(found_tree)))
+    return "frozen 1" in ev if ev else None
-    ctx.check(len(_host_pids()) == 1, "the extension host is not running at the end")
+
+
+@test("exthost.armed-freeze", title="A package's service is frozen for the armed window",
+      subsystem="exthost", kind="operator", hardware="takeover", est_min=6,
+      covers=[("forgeext", "src/super.*"), ("forgeext", "src/run.*"), ("forgeext", "src/machine.*"),
+              ("forgeext", "src/cgroup.*"), ("forgectrl", "src/cool.*")],
+      requires=["exthost.service", "cloud.dark-print"], actions=["button"],
+      steps=[OFFLINE_STEP,
+             "Bed clear (the job is dark: a 30 s square at S0, nothing fires). Press the button "
+             "when it lights."],
+      description="The reference package runs, its heartbeat advancing twice a second, and a dark "
+                  "cloud print (cloud.dark-print's own job and checks) opens a real armed window over "
+                  "it. Sampled five times a second from before the button to after the end: the "
+                  "engine's armed flag, the group's frozen state as the kernel reports it, and the "
+                  "heartbeat. The window is open for at least 10 s. From 2 s after it opens to its "
+                  "close the group reads frozen in every sample and the heartbeat does not move, the "
+                  "freeze is in place before the run starts (the latch unlocks only for the run), and "
+                  "within 3 s of the close the group is thawed and the heartbeat advances again. The "
+                  "service is the same process throughout. Everything is put back as exthost.service "
+                  "puts it back.")
+def armed_freeze(ctx):
+    import tempfile
+    import threading
+    fc = ctx.forgectrl
+    ev = ctx.evidence
+    ctx.check(len(_host_pids()) == 1, "the extension host is not one running process: %s", _host_pids())
+    ctx.check(fc.wait_idle(timeout=30, abort=ctx.aborted), "machine not idle: settings are locked")
+    prior = fc.settings().get("ext_enabled") or ""
+    raw = read_file(record_path())
+    ctx.check(raw, "no setup record at %s", record_path())
+    ctx.check(not os.path.exists(SAFE_FILE), "%s exists: the machine is in safe mode", SAFE_FILE)
+    found_tree = _tree(EXT_ROOT)
+    dir_mode = os.stat(os.path.dirname(EXT_ROOT)).st_mode & 0o7777
+    st, body, hdrs = request(fc.base, "GET", "/advisories/extensions", headers={"Host": fc.host_header()})
+    etag = hdrs.get("etag")
+    ctx.check(st == 200 and etag, "GET /advisories/extensions -> %s", st)
+    work = tempfile.mkdtemp(prefix="forgetest-ffx.")
+    beat_path = os.path.join(EXT_ROOT, "data", REF_ID, "beat")
+    samples = []
+    stop = threading.Event()
+
+    def sampler():
+        while not stop.is_set():
+            st_, cool = fc.get("/cool/status")
+            samples.append((time.time(), bool(cool.get("armed")) if st_ == 200 and isinstance(cool, dict) else None,
+                            _frozen(REF_ID), _read(beat_path).strip(), (_svc(REF_ID) or {}).get("pid"),
+                            latch_locked()))
+            time.sleep(0.2)
+
+    try:
+        archive, pub = _pack_reference(work, lan_ip())
+        with open(pub, "rb") as f:
+            write_file(os.path.join(EXT_ROOT, "keys", REF_KEY + ".pub"), f.read())
+        os.chmod(os.path.join(EXT_ROOT, "keys", REF_KEY + ".pub"), 0o644)
+        r = _forgeext("install", archive, "--consent-community")
+        ctx.check(r.get("ok") is True, "the install -> %s", r.get("error"))
+        st, reply = fc.post("/settings", data={"ext_enabled": "1", "advisory": etag, "phrase": SAFETY_PHRASE})
+        ctx.check(st == 200, "ext_enabled=1 over the advisory -> %s %r", st, reply)
+        seen = set()
+        ctx.check(_until(ctx, lambda: seen.add(_read(beat_path)) or len(seen) >= 4, 120, poll=0.2),
+                  "the reference service's heartbeat does not advance: %s", _svc(REF_ID) or _host_status())
+        pid = _svc(REF_ID).get("pid")
+
+        offset = enter_offline(ctx)
+        job = offline_job(ctx, "dark.puls", seconds=30)
+        off = Offline().__enter__()
+        t = threading.Thread(target=sampler, daemon=True)
+        t.start()
+        try:
+            dark_print_body(ctx, ev, off, job, offset, fc)
+            ctx.sleep(6)                                # past the close, for the thaw
+        finally:
+            stop.set()
+            t.join(timeout=5)
+            off.__exit__(None, None, None)
+            offline_cleanup(ctx)
+
+        opens = [x for x in samples if x[1]]
+        ctx.check(opens, "the engine never read armed across the print: %d samples", len(samples))
+        t_open, t_close = opens[0][0], opens[-1][0]
+        ev["window_s"] = round(t_close - t_open, 1)
+        ctx.check(t_close - t_open >= 10, "the armed window was open for %.1f s only", t_close - t_open)
+        inside = [x for x in samples if t_open + 2.0 <= x[0] <= t_close]
+        thawed = [x for x in inside if x[2] is not True]
+        ev["samples"] = {"all": len(samples), "inside": len(inside), "not_frozen_inside": len(thawed)}
+        first_frozen = next((x[0] for x in samples if x[0] >= t_open and x[2] is True), None)
+        ev["freeze_lag_s"] = round(first_frozen - t_open, 2) if first_frozen else None
+        ctx.check(inside and not thawed, "inside the armed window the group read not frozen in %d of %d samples "
+                  "(the first %.1f s after it opened)", len(thawed), len(inside), (thawed[0][0] - t_open) if thawed else 0)
+        beats = {x[3] for x in inside}
+        ctx.check(len(beats) == 1, "the heartbeat moved inside the armed window: %s", sorted(beats)[:6])
+        # the latch unlocks only for the run: its first unlocked sample is the run's start
+        run_start = next((x[0] for x in samples if x[5] is False), None)
+        ctx.check(run_start is not None, "the latch never read unlocked: the print did not run under the samples")
+        ctx.check(first_frozen is not None and first_frozen <= run_start,
+                  "the run started before the freeze was in place (frozen %s, the latch unlocked %s after the window opened)",
+                  None if first_frozen is None else round(first_frozen - t_open, 2), round(run_start - t_open, 2))
+        ev["freeze_led_the_run_s"] = round(run_start - first_frozen, 2)
+        after = [x for x in samples if x[0] >= t_close + 3.0]
+        ctx.check(after and all(x[2] is False for x in after), "3 s after the close the group is not thawed: %s",
+                  [x[2] for x in after][:8])
+        ctx.check(len({x[3] for x in after}) >= 2, "the heartbeat did not advance again after the close")
+        last_frozen = max((x[0] for x in samples if x[2] is True), default=None)
+        ev["thaw_lag_s"] = round(last_frozen - t_close, 2) if last_frozen else None
+        ctx.check({x[4] for x in samples if x[4]} == {pid}, "the service did not stay the same process: %s",
+                  sorted({x[4] for x in samples if x[4]}))
+        ctx.log("the armed window was open %.1f s; frozen %.2f s after it opened and %.2f s before the latch unlocked "
+                "for the run, in every one of %d samples from 2 s in to the close, the heartbeat still; thawed %.2f s "
+                "after the close", ev["window_s"], ev["freeze_lag_s"], ev["freeze_led_the_run_s"], len(inside),
+                ev["thaw_lag_s"] if ev["thaw_lag_s"] is not None else -1)
+    finally:
+        _put_back(ctx, fc, work, prior, etag, raw, dir_mode)
+    _as_found(ctx, fc, prior, raw, dir_mode, found_tree)