forgetest: a takeover hands back the cloud client it found

In cloud mode forgectrl's start at the end of a takeover starts a cloud
client, and what that client is comes only from gfcloud's one-start
markers, which the client that starts reads and takes down. The takeover
started it bare: the online client, with the service's connect-time
hunt. Seen on the bench with exthost.armed-freeze, which runs its print
under the offline service and puts the setup record back under a
takeover: the machine came back online in cloud mode, the hunt homed the
head at 14:21:01, the run's result was written at 14:21:02, and the
service sent a head move at 14:21:08, after the run had ended, where the
next test's switch to GRBL mode would have cut it off. It also broke the
harness's own rule that every cloud client start it makes comes up
without the hunt.

The takeover now reads, on entry, whether the running cloud client is
the offline service: the client itself holds the listening socket at
/run/gfcloud-offline.sock (hw.listens_on reads /proc/net/unix and the
process's own descriptors, so a socket file another process left behind
does not count). forgectrl's start is made under the no-hunt marker in
cloud mode, and under the offline marker as well when the offline
service was found. After the start the client that came up must listen
on the offline socket within 60 s; a miss is recorded on the run's
baseline capture and the post pass turns it into a leftover ("cloud
client at the takeover end", not restored: a cloud test starts the
client it needs), which fails the run. A marker no client read (the
gate, standby, a fault) is taken down, so a later start never comes up
under it. The judge never breaks the exit path.

Proof: forgetest's unit tests pass on the host (481 OK). The new
tests/test_takeover_client.py drives a real enter and exit against the
fake forgectrl, a fake /proc, and an init script that plays gfcloud's
start: the offline service comes back offline, an online client after
it fails the run, a start that never happened takes the markers down,
an online client comes back without the hunt, a stale socket file is not
the offline service, GRBL mode starts under no marker. Negative
controls: without the markers four of them fail, without the judge
three. Bench drill on the bench reference with this package from /tmp:
GRBL mode, the offline service started as enter_offline starts it (pid
4834), a takeover; the client forgectrl brought back (pid 4898) read
both markers, logged "OFFLINE service", listened on the socket, and in
the 10 s after had no web session, no hunt, and no motion. The bench was
then handed back to GRBL mode as found.

Acceptance: runner.py, baseline.py and hw.py are in no test's
fingerprint, so no result moves; the catalog test that exercises the
change is exthost.armed-freeze, whose takeover now fails the run unless
the offline service comes back.

forgetest/forgetest/baseline.py

@@ -117,6 +117,12 @@ UNPRESERVED_SETTINGS = ("controller_mode",)
 # service starts its own client with the hunt.
 NOHUNT_MARKER = os.environ.get("FORGETEST_NOHUNT_MARKER", "/run/gfcloud-nohunt")
 
+# The offline service: gfcloud's one-start offline marker and the socket
+# the service listens on while it runs. A takeover that finds it running
+# makes forgectrl's start at its end under the marker again (runner.Takeover).
+OFFLINE_MARKER = os.environ.get("FORGETEST_OFFLINE_MARKER", "/run/gfcloud-offline")
+OFFLINE_SOCKET = os.environ.get("FORGETEST_OFFLINE_SOCKET", "/run/gfcloud-offline.sock")
+
 # Read-only readbacks with their idle values (no direct restore: the state
 # comes right through forgectrl - see restore_forgectrl - or is fatal).
 IDLE_READBACKS = [
@@ -913,6 +919,13 @@ class Baseline:
             left.append(Leftover("position at the %s" % r["where"], r["found"], r["expected"],
                                  "unrestorable: a controller restart re-zeroed the counters there, "
                                  "so the head is not moved"))
+        # The cloud client forgectrl's start brought back at the end of a
+        # takeover, when it is not the one the takeover found (the same
+        # judge, runner.Takeover). The client that came up is left as it
+        # is: the cloud test that needs a client starts its own.
+        for r in captured.get("restart_clients") or []:
+            left.append(Leftover("cloud client at the %s" % r["where"], r["found"], r["expected"],
+                                 "not restored: a cloud test starts the client it needs"))
         was = captured.get("settings")
         if was:
             st, body = self.fc_get("/settings")

forgetest/forgetest/hw.py

@@ -395,6 +395,44 @@ def pgrep_f(needle):
     return out
 
 
+def listens_on(pid, path):
+    """Process pid holds a listening UNIX socket bound at path: one of its
+    own descriptors is that socket, so a socket file another process left
+    behind does not count. Reads /proc (FORGETEST_PROC_ROOT for tests)."""
+    if not pid:
+        return False
+    proc = os.environ.get("FORGETEST_PROC_ROOT") or "/proc"
+    inodes = set()
+    try:
+        with open(os.path.join(proc, "net", "unix")) as f:
+            next(f, None)                       # the header line
+            for ln in f:
+                # Num RefCount Protocol Flags Type St Inode Path
+                parts = ln.split()
+                try:
+                    if len(parts) >= 8 and parts[7] == path and int(parts[3], 16) & 0x10000:
+                        inodes.add(parts[6])    # __SO_ACCEPTCON: a listener
+                except ValueError:
+                    continue
+    except OSError:
+        return False
+    if not inodes:
+        return False
+    fd_dir = os.path.join(proc, str(pid), "fd")
+    try:
+        fds = os.listdir(fd_dir)
+    except OSError:
+        return False
+    for fd in fds:
+        try:
+            link = os.readlink(os.path.join(fd_dir, fd))
+        except OSError:
+            continue
+        if link.startswith("socket:[") and link[8:-1] in inodes:
+            return True
+    return False
+
+
 def run(cmd, timeout=60):
     """Run a command list; returns (rc, combined output)."""
     try:

forgetest/forgetest/runner.py

@@ -589,6 +589,7 @@ class Takeover:
         self.saved = {}
         self.run = run            # the test's run, whose baseline the head is judged against
         self.cloud = False
+        self.offline = False      # the cloud client found is the offline service
         self.frame = None         # the counters' frame, read before the controller is stopped
         self.found = None         # (counters, microstep mode) once the controller is gone
 
@@ -620,8 +621,12 @@ class Takeover:
         try:
             st, mode = hw.Forgectrl().get("/mode")
             self.cloud = st == 200 and isinstance(mode, dict) and mode.get("mode") == "cloud"
+            self.offline = self.cloud and hw.listens_on(mode.get("pid"), _baseline.OFFLINE_SOCKET)
         except hw.HwError:
             pass
+        if self.offline:
+            log("takeover: the cloud client is the offline service (pid %s); forgectrl's start "
+                "brings it back offline" % mode.get("pid"))
         # The frame is read while the controller runs: its exit takes the
         # anchor with it (forgectrl unlinks it), though not the counters.
         self.frame = _baseline.counter_frame()
@@ -702,6 +707,83 @@ class Takeover:
         if now != was and not _baseline.position_quantized(was, now, _baseline.counter_steps_per_mm()):
             self._record("takeover end", now, was)
 
+    # In cloud mode forgectrl's start at the end starts a cloud client, and
+    # what that client is comes only from the one-start markers gfcloud
+    # reads and takes down as it starts. Started bare, it is the online
+    # client with the service's connect-time hunt, and the moves the
+    # service sends after the hunt are still running as the run ends and
+    # the next one begins, whatever the takeover found. The start is made
+    # under the no-hunt marker, as every client start the runner makes is,
+    # and under the offline marker as well when the takeover found the
+    # offline service; that client is then proven to be the offline
+    # service, and a miss is recorded for the baseline's post pass, which
+    # fails the run.
+    OFFLINE_BACK_S = 60       # the start to the offline service's listener
+
+    def client_markers(self):
+        """(path, name) of the markers forgectrl's start is made under."""
+        if not self.cloud:
+            return []
+        marks = [(_baseline.NOHUNT_MARKER, "no-hunt")]
+        if self.offline:
+            marks.append((_baseline.OFFLINE_MARKER, "offline"))
+        return marks
+
+    def markers_on(self):
+        for path, name in self.client_markers():
+            try:
+                with open(path, "w") as f:
+                    f.write("forgetest takeover %s\n" % self.who)
+            except OSError as e:
+                self.log("takeover: WARNING the %s marker could not be written: %s" % (name, e))
+
+    def wait_offline(self):
+        """The cloud client's pid once it listens on the offline socket, or
+        None after OFFLINE_BACK_S."""
+        deadline = time.time() + self.OFFLINE_BACK_S
+        while True:
+            try:
+                st, m = hw.Forgectrl().get("/mode")
+            except hw.HwError:
+                st, m = None, None
+            pid = m.get("pid") if st == 200 and isinstance(m, dict) else None
+            if hw.listens_on(pid, _baseline.OFFLINE_SOCKET):
+                return pid
+            if time.time() >= deadline:
+                return None
+            time.sleep(0.5)
+
+    def client_back(self, mode):
+        """After forgectrl's start: a marker no client read is taken down
+        (no client started - the gate, standby, a fault - and a later
+        start must not come up under it), and the offline service found is
+        judged against the client that came up."""
+        if not self.cloud:
+            return
+        mode = mode if isinstance(mode, dict) else {}
+        running = mode.get("mode") == "cloud" and mode.get("controller") == "running"
+        pid = self.wait_offline() if self.offline and running else None
+        for path, name in self.client_markers():
+            if (not running or name == "offline") and os.path.exists(path):
+                try:
+                    os.remove(path)
+                    self.log("takeover: the %s marker was never read - taken down" % name)
+                except OSError:
+                    pass
+        if not self.offline:
+            return
+        if pid:
+            self.log("takeover: the cloud client is back as the offline service (pid %s)" % pid)
+            return
+        found = ("a client that is not the offline service (pid %s)" % mode.get("pid") if running
+                 else "no cloud client running (mode %s, controller %s)"
+                 % (mode.get("mode"), mode.get("controller")))
+        cap = self.run.baseline_captured if self.run is not None else None
+        if cap is not None:
+            cap.setdefault("restart_clients", []).append(
+                {"where": "takeover end", "found": found, "expected": "the offline service"})
+        self.log("takeover: the offline service did not come back: %s; recorded as a leftover" % found)
+
     def __exit__(self, exc_type, exc, tb):
         try:
             self.position_at_end()
@@ -709,6 +791,7 @@ class Takeover:
             self.log("takeover: WARNING the head's position could not be judged: %s: %s"
                      % (type(e).__name__, e))
         self.restore_attrs()
+        self.markers_on()
         rc, out = hw.initd("forgectrl", "start")
         self.log("takeover: forgectrl start -> rc %s" % rc)
         try:
@@ -717,7 +800,12 @@ class Takeover:
             pass
         # leave the machine settled for whatever runs next: the probe
         # move done, the controller back (or the ladder's verdict logged)
-        self.wait_settled()
+        mode = self.wait_settled()
+        try:
+            self.client_back(mode)
+        except Exception as e:  # noqa: BLE001 - the judge never breaks the exit path
+            self.log("takeover: WARNING the cloud client could not be judged: %s: %s"
+                     % (type(e).__name__, e))
         return False
 
 

forgetest/tests/test_takeover_client.py

@@ -0,0 +1,200 @@
+# Copyright 2026 514 LLC d/b/a OpenGlow
+# Written by Scott Wiederhold
+# https://community.openglow.org
+# SPDX-License-Identifier:    MIT
+
+"""A takeover hands back the cloud client it found. In cloud mode
+forgectrl's start at the end of a takeover starts a cloud client, and what
+that client is comes only from gfcloud's one-start markers. Started bare it
+is the online client with the service's connect-time hunt, and the moves
+the service sends after the hunt outlast the run. The start is made under
+the no-hunt marker, and under the offline marker when the takeover found
+the offline service, which it then proves came back; a miss is a leftover
+that fails the run. Runs against the fake forgectrl, a fake /proc, and a
+fake init script that plays the client's start."""
+import os
+import shutil
+import tempfile
+import unittest
+
+from forgetest import baseline, hw, runner
+
+from helpers import FakeForgectrl
+
+
+class FakeRun:
+    def __init__(self, cap):
+        self.baseline_captured = cap
+        self.lines = []
+
+    def log(self, s):
+        self.lines.append(s)
+
+
+class TakeoverClientTests(unittest.TestCase):
+    OLD_PID, NEW_PID = 200, 201
+
+    def setUp(self):
+        self.tmp = tempfile.mkdtemp(prefix="forgetest-tkc-")
+        self.proc = os.path.join(self.tmp, "proc")
+        os.makedirs(os.path.join(self.proc, "net"))
+        self.sysfs = os.path.join(self.tmp, "sysfs") + os.sep
+        for group in ("cnc", "pic", "head"):
+            os.makedirs(self.sysfs + group)
+        os.environ["GF_SYSFS_ROOT"] = self.sysfs
+        os.environ["FORGETEST_PROC_ROOT"] = self.proc
+        os.environ["FORGETEST_MARKER"] = os.path.join(self.tmp, "forgetest.active")
+        self.saved = (baseline.NOHUNT_MARKER, baseline.OFFLINE_MARKER, baseline.OFFLINE_SOCKET,
+                      hw.initd, runner.Takeover.OFFLINE_BACK_S)
+        baseline.NOHUNT_MARKER = os.path.join(self.tmp, "gfcloud-nohunt")
+        baseline.OFFLINE_MARKER = os.path.join(self.tmp, "gfcloud-offline")
+        baseline.OFFLINE_SOCKET = "/run/gfcloud-offline.sock"
+        runner.Takeover.OFFLINE_BACK_S = 0
+        baseline.Baseline._unreachable_until = 0.0
+        self.unix = []                          # (pid, inode, path, listening)
+        self.fc = FakeForgectrl().start()
+        self.fc.state["mode"] = {"mode": "cloud", "controller": "running", "pid": self.OLD_PID,
+                                 "motion": "verified"}
+        self.calls = []
+        self.at_start = None                    # the markers present when forgectrl started
+        self.comes_up = "running"               # the controller forgectrl's start settles on
+        hw.initd = self._initd
+
+    def tearDown(self):
+        (baseline.NOHUNT_MARKER, baseline.OFFLINE_MARKER, baseline.OFFLINE_SOCKET,
+         hw.initd, runner.Takeover.OFFLINE_BACK_S) = self.saved
+        self.fc.stop()
+        for k in ("GF_SYSFS_ROOT", "FORGETEST_PROC_ROOT", "FORGETEST_MARKER"):
+            os.environ.pop(k, None)
+        shutil.rmtree(self.tmp, ignore_errors=True)
+
+    # -- the fakes ------------------------------------------------------
+    def _socket(self, pid, inode, path, listening=True):
+        self.unix.append((pid, inode, path, listening))
+        fd_dir = os.path.join(self.proc, str(pid), "fd")
+        os.makedirs(fd_dir, exist_ok=True)
+        os.symlink("socket:[%d]" % inode, os.path.join(fd_dir, str(len(os.listdir(fd_dir)) + 3)))
+        with open(os.path.join(self.proc, "net", "unix"), "w") as f:
+            f.write("Num       RefCount Protocol Flags    Type St Inode Path\n")
+            for _pid, ino, p, lis in self.unix:
+                f.write("00000000: 00000002 00000000 %08X 0001 %s %d %s\n"
+                        % (0x10000 if lis else 0, "01" if lis else "03", ino, p))
+
+    def _offline_client(self, pid):
+        self._socket(pid, 9000 + pid, baseline.OFFLINE_SOCKET)
+
+    def _initd(self, service, action, timeout=60):
+        """forgectrl stop/start; the start plays gfcloud's: it reads and
+        takes down the markers, and it is the offline service when the
+        offline marker was there."""
+        self.calls.append((service, action))
+        if action != "start":
+            return 0, ""
+        self.at_start = sorted(os.path.basename(p) for p in (baseline.NOHUNT_MARKER, baseline.OFFLINE_MARKER)
+                               if os.path.exists(p))
+        mode = self.fc.state["mode"]
+        if self.comes_up != "running":
+            self.fc.state["mode"] = dict(mode, controller=self.comes_up, pid=None, motion="unverified")
+            return 0, ""
+        offline = os.path.exists(baseline.OFFLINE_MARKER)
+        for p in (baseline.NOHUNT_MARKER, baseline.OFFLINE_MARKER):
+            if os.path.exists(p):
+                os.remove(p)
+        self.fc.state["mode"] = dict(mode, controller="running", pid=self.NEW_PID, motion="verified")
+        if offline and mode.get("mode") == "cloud":
+            self._offline_client(self.NEW_PID)
+        return 0, ""
+
+    def takeover(self):
+        cap = {"position": None, "mode": self.fc.state["mode"]["mode"]}
+        run = FakeRun(cap)
+        with runner.Takeover(run.log, "test.id", run=run):
+            pass
+        return cap, run
+
+    def markers_left(self):
+        return [p for p in (baseline.NOHUNT_MARKER, baseline.OFFLINE_MARKER) if os.path.exists(p)]
+
+    # -- the socket judge -------------------------------------------------
+    def test_the_listener_is_the_process_own_descriptor(self):
+        self._offline_client(self.OLD_PID)
+        self.assertTrue(hw.listens_on(self.OLD_PID, baseline.OFFLINE_SOCKET))
+        # the path alone is not enough: another process's socket, a
+        # connected (not listening) socket, another path, no pid
+        self.assertFalse(hw.listens_on(self.NEW_PID, baseline.OFFLINE_SOCKET))
+        self._socket(self.NEW_PID, 7001, baseline.OFFLINE_SOCKET, listening=False)
+        self.assertFalse(hw.listens_on(self.NEW_PID, baseline.OFFLINE_SOCKET))
+        self.assertFalse(hw.listens_on(self.OLD_PID, "/run/forgefirm/grbl.ctl"))
+        self.assertFalse(hw.listens_on(None, baseline.OFFLINE_SOCKET))
+
+    # -- the takeover ------------------------------------------------------
+    def test_the_offline_service_comes_back_offline(self):
+        # exthost.armed-freeze's case: the setup record put back under a
+        # takeover while the offline service ran
+        self._offline_client(self.OLD_PID)
+        cap, run = self.takeover()
+        self.assertEqual(self.calls, [("forgectrl", "stop"), ("forgectrl", "start")])
+        self.assertEqual(self.at_start, ["gfcloud-nohunt", "gfcloud-offline"])
+        self.assertTrue(hw.listens_on(self.NEW_PID, baseline.OFFLINE_SOCKET))
+        self.assertNotIn("restart_clients", cap)
+        self.assertEqual(self.markers_left(), [])
+        self.assertTrue(any("back as the offline service (pid %d)" % self.NEW_PID in l for l in run.lines),
+                        run.lines)
+
+    def test_an_online_client_after_the_offline_service_fails_the_run(self):
+        # the client that came up is not listening: an online client, or
+        # a marker the start never saw
+        self._offline_client(self.OLD_PID)
+        real = self._initd
+
+        def lost_marker(service, action, timeout=60):
+            if action == "start" and os.path.exists(baseline.OFFLINE_MARKER):
+                os.remove(baseline.OFFLINE_MARKER)
+            return real(service, action, timeout)
+        hw.initd = lost_marker
+        cap, run = self.takeover()
+        self.assertEqual(cap["restart_clients"],
+                         [{"where": "takeover end", "expected": "the offline service",
+                           "found": "a client that is not the offline service (pid %d)" % self.NEW_PID}])
+        left = baseline.Baseline(run.log)
+        items = []
+        left._preserved(items, cap)
+        self.assertEqual([x.item for x in items], ["cloud client at the takeover end"])
+        self.assertTrue(items[0].action.startswith("not restored"), items[0].action)
+
+    def test_no_client_started_takes_the_markers_down(self):
+        # the gate closed, or standby: nothing read the markers, and a
+        # later start must not come up offline or without its hunt
+        self._offline_client(self.OLD_PID)
+        self.comes_up = "gated"
+        cap, run = self.takeover()
+        self.assertEqual(self.at_start, ["gfcloud-nohunt", "gfcloud-offline"])
+        self.assertEqual(self.markers_left(), [])
+        self.assertEqual(len(cap["restart_clients"]), 1)
+        self.assertIn("no cloud client running", cap["restart_clients"][0]["found"])
+
+    def test_an_online_client_comes_back_without_the_hunt(self):
+        cap, run = self.takeover()
+        self.assertEqual(self.at_start, ["gfcloud-nohunt"])
+        self.assertFalse(hw.listens_on(self.NEW_PID, baseline.OFFLINE_SOCKET))
+        self.assertNotIn("restart_clients", cap)
+        self.assertEqual(self.markers_left(), [])
+
+    def test_a_stale_offline_socket_file_is_not_the_offline_service(self):
+        # a socket another process left at the path: the online client
+        # found stays online
+        self._socket(99, 7002, baseline.OFFLINE_SOCKET)
+        cap, run = self.takeover()
+        self.assertEqual(self.at_start, ["gfcloud-nohunt"])
+        self.assertNotIn("restart_clients", cap)
+
+    def test_grbl_mode_starts_under_no_marker(self):
+        self.fc.state["mode"] = {"mode": "grbl", "controller": "running", "pid": self.OLD_PID,
+                                 "motion": "verified"}
+        cap, run = self.takeover()
+        self.assertEqual(self.at_start, [])
+        self.assertNotIn("restart_clients", cap)
+
+
+if __name__ == "__main__":
+    unittest.main()