diff --git a/forgetest/forgetest/baseline.py b/forgetest/forgetest/baseline.py index d51ddd1..baa1827 100644 --- a/forgetest/forgetest/baseline.py +++ b/forgetest/forgetest/baseline.py @@ -117,6 +117,12 @@ UNPRESERVED_SETTINGS = ("controller_mode",) # service starts its own client with the hunt. NOHUNT_MARKER = os.environ.get("FORGETEST_NOHUNT_MARKER", "/run/gfcloud-nohunt") +# The offline service: gfcloud's one-start offline marker and the socket +# the service listens on while it runs. A takeover that finds it running +# makes forgectrl's start at its end under the marker again (runner.Takeover). +OFFLINE_MARKER = os.environ.get("FORGETEST_OFFLINE_MARKER", "/run/gfcloud-offline") +OFFLINE_SOCKET = os.environ.get("FORGETEST_OFFLINE_SOCKET", "/run/gfcloud-offline.sock") + # Read-only readbacks with their idle values (no direct restore: the state # comes right through forgectrl - see restore_forgectrl - or is fatal). IDLE_READBACKS = [ @@ -913,6 +919,13 @@ class Baseline: left.append(Leftover("position at the %s" % r["where"], r["found"], r["expected"], "unrestorable: a controller restart re-zeroed the counters there, " "so the head is not moved")) + # The cloud client forgectrl's start brought back at the end of a + # takeover, when it is not the one the takeover found (the same + # judge, runner.Takeover). The client that came up is left as it + # is: the cloud test that needs a client starts its own. + for r in captured.get("restart_clients") or []: + left.append(Leftover("cloud client at the %s" % r["where"], r["found"], r["expected"], + "not restored: a cloud test starts the client it needs")) was = captured.get("settings") if was: st, body = self.fc_get("/settings") diff --git a/forgetest/forgetest/hw.py b/forgetest/forgetest/hw.py index f389744..b8e3633 100644 --- a/forgetest/forgetest/hw.py +++ b/forgetest/forgetest/hw.py @@ -395,6 +395,44 @@ def pgrep_f(needle): return out +def listens_on(pid, path): + """Process pid holds a listening UNIX socket bound at path: one of its + own descriptors is that socket, so a socket file another process left + behind does not count. Reads /proc (FORGETEST_PROC_ROOT for tests).""" + if not pid: + return False + proc = os.environ.get("FORGETEST_PROC_ROOT") or "/proc" + inodes = set() + try: + with open(os.path.join(proc, "net", "unix")) as f: + next(f, None) # the header line + for ln in f: + # Num RefCount Protocol Flags Type St Inode Path + parts = ln.split() + try: + if len(parts) >= 8 and parts[7] == path and int(parts[3], 16) & 0x10000: + inodes.add(parts[6]) # __SO_ACCEPTCON: a listener + except ValueError: + continue + except OSError: + return False + if not inodes: + return False + fd_dir = os.path.join(proc, str(pid), "fd") + try: + fds = os.listdir(fd_dir) + except OSError: + return False + for fd in fds: + try: + link = os.readlink(os.path.join(fd_dir, fd)) + except OSError: + continue + if link.startswith("socket:[") and link[8:-1] in inodes: + return True + return False + + def run(cmd, timeout=60): """Run a command list; returns (rc, combined output).""" try: diff --git a/forgetest/forgetest/runner.py b/forgetest/forgetest/runner.py index 1ef6968..86d10ef 100644 --- a/forgetest/forgetest/runner.py +++ b/forgetest/forgetest/runner.py @@ -589,6 +589,7 @@ class Takeover: self.saved = {} self.run = run # the test's run, whose baseline the head is judged against self.cloud = False + self.offline = False # the cloud client found is the offline service self.frame = None # the counters' frame, read before the controller is stopped self.found = None # (counters, microstep mode) once the controller is gone @@ -620,8 +621,12 @@ class Takeover: try: st, mode = hw.Forgectrl().get("/mode") self.cloud = st == 200 and isinstance(mode, dict) and mode.get("mode") == "cloud" + self.offline = self.cloud and hw.listens_on(mode.get("pid"), _baseline.OFFLINE_SOCKET) except hw.HwError: pass + if self.offline: + log("takeover: the cloud client is the offline service (pid %s); forgectrl's start " + "brings it back offline" % mode.get("pid")) # The frame is read while the controller runs: its exit takes the # anchor with it (forgectrl unlinks it), though not the counters. self.frame = _baseline.counter_frame() @@ -702,6 +707,83 @@ class Takeover: if now != was and not _baseline.position_quantized(was, now, _baseline.counter_steps_per_mm()): self._record("takeover end", now, was) + # In cloud mode forgectrl's start at the end starts a cloud client, and + # what that client is comes only from the one-start markers gfcloud + # reads and takes down as it starts. Started bare, it is the online + # client with the service's connect-time hunt, and the moves the + # service sends after the hunt are still running as the run ends and + # the next one begins, whatever the takeover found. The start is made + # under the no-hunt marker, as every client start the runner makes is, + # and under the offline marker as well when the takeover found the + # offline service; that client is then proven to be the offline + # service, and a miss is recorded for the baseline's post pass, which + # fails the run. + OFFLINE_BACK_S = 60 # the start to the offline service's listener + + def client_markers(self): + """(path, name) of the markers forgectrl's start is made under.""" + if not self.cloud: + return [] + marks = [(_baseline.NOHUNT_MARKER, "no-hunt")] + if self.offline: + marks.append((_baseline.OFFLINE_MARKER, "offline")) + return marks + + def markers_on(self): + for path, name in self.client_markers(): + try: + with open(path, "w") as f: + f.write("forgetest takeover %s\n" % self.who) + except OSError as e: + self.log("takeover: WARNING the %s marker could not be written: %s" % (name, e)) + + def wait_offline(self): + """The cloud client's pid once it listens on the offline socket, or + None after OFFLINE_BACK_S.""" + deadline = time.time() + self.OFFLINE_BACK_S + while True: + try: + st, m = hw.Forgectrl().get("/mode") + except hw.HwError: + st, m = None, None + pid = m.get("pid") if st == 200 and isinstance(m, dict) else None + if hw.listens_on(pid, _baseline.OFFLINE_SOCKET): + return pid + if time.time() >= deadline: + return None + time.sleep(0.5) + + def client_back(self, mode): + """After forgectrl's start: a marker no client read is taken down + (no client started - the gate, standby, a fault - and a later + start must not come up under it), and the offline service found is + judged against the client that came up.""" + if not self.cloud: + return + mode = mode if isinstance(mode, dict) else {} + running = mode.get("mode") == "cloud" and mode.get("controller") == "running" + pid = self.wait_offline() if self.offline and running else None + for path, name in self.client_markers(): + if (not running or name == "offline") and os.path.exists(path): + try: + os.remove(path) + self.log("takeover: the %s marker was never read - taken down" % name) + except OSError: + pass + if not self.offline: + return + if pid: + self.log("takeover: the cloud client is back as the offline service (pid %s)" % pid) + return + found = ("a client that is not the offline service (pid %s)" % mode.get("pid") if running + else "no cloud client running (mode %s, controller %s)" + % (mode.get("mode"), mode.get("controller"))) + cap = self.run.baseline_captured if self.run is not None else None + if cap is not None: + cap.setdefault("restart_clients", []).append( + {"where": "takeover end", "found": found, "expected": "the offline service"}) + self.log("takeover: the offline service did not come back: %s; recorded as a leftover" % found) + def __exit__(self, exc_type, exc, tb): try: self.position_at_end() @@ -709,6 +791,7 @@ class Takeover: self.log("takeover: WARNING the head's position could not be judged: %s: %s" % (type(e).__name__, e)) self.restore_attrs() + self.markers_on() rc, out = hw.initd("forgectrl", "start") self.log("takeover: forgectrl start -> rc %s" % rc) try: @@ -717,7 +800,12 @@ class Takeover: pass # leave the machine settled for whatever runs next: the probe # move done, the controller back (or the ladder's verdict logged) - self.wait_settled() + mode = self.wait_settled() + try: + self.client_back(mode) + except Exception as e: # noqa: BLE001 - the judge never breaks the exit path + self.log("takeover: WARNING the cloud client could not be judged: %s: %s" + % (type(e).__name__, e)) return False diff --git a/forgetest/tests/test_takeover_client.py b/forgetest/tests/test_takeover_client.py new file mode 100644 index 0000000..1a7e82c --- /dev/null +++ b/forgetest/tests/test_takeover_client.py @@ -0,0 +1,200 @@ +# Copyright 2026 514 LLC d/b/a OpenGlow +# Written by Scott Wiederhold +# https://community.openglow.org +# SPDX-License-Identifier: MIT + +"""A takeover hands back the cloud client it found. In cloud mode +forgectrl's start at the end of a takeover starts a cloud client, and what +that client is comes only from gfcloud's one-start markers. Started bare it +is the online client with the service's connect-time hunt, and the moves +the service sends after the hunt outlast the run. The start is made under +the no-hunt marker, and under the offline marker when the takeover found +the offline service, which it then proves came back; a miss is a leftover +that fails the run. Runs against the fake forgectrl, a fake /proc, and a +fake init script that plays the client's start.""" +import os +import shutil +import tempfile +import unittest + +from forgetest import baseline, hw, runner + +from helpers import FakeForgectrl + + +class FakeRun: + def __init__(self, cap): + self.baseline_captured = cap + self.lines = [] + + def log(self, s): + self.lines.append(s) + + +class TakeoverClientTests(unittest.TestCase): + OLD_PID, NEW_PID = 200, 201 + + def setUp(self): + self.tmp = tempfile.mkdtemp(prefix="forgetest-tkc-") + self.proc = os.path.join(self.tmp, "proc") + os.makedirs(os.path.join(self.proc, "net")) + self.sysfs = os.path.join(self.tmp, "sysfs") + os.sep + for group in ("cnc", "pic", "head"): + os.makedirs(self.sysfs + group) + os.environ["GF_SYSFS_ROOT"] = self.sysfs + os.environ["FORGETEST_PROC_ROOT"] = self.proc + os.environ["FORGETEST_MARKER"] = os.path.join(self.tmp, "forgetest.active") + self.saved = (baseline.NOHUNT_MARKER, baseline.OFFLINE_MARKER, baseline.OFFLINE_SOCKET, + hw.initd, runner.Takeover.OFFLINE_BACK_S) + baseline.NOHUNT_MARKER = os.path.join(self.tmp, "gfcloud-nohunt") + baseline.OFFLINE_MARKER = os.path.join(self.tmp, "gfcloud-offline") + baseline.OFFLINE_SOCKET = "/run/gfcloud-offline.sock" + runner.Takeover.OFFLINE_BACK_S = 0 + baseline.Baseline._unreachable_until = 0.0 + self.unix = [] # (pid, inode, path, listening) + self.fc = FakeForgectrl().start() + self.fc.state["mode"] = {"mode": "cloud", "controller": "running", "pid": self.OLD_PID, + "motion": "verified"} + self.calls = [] + self.at_start = None # the markers present when forgectrl started + self.comes_up = "running" # the controller forgectrl's start settles on + hw.initd = self._initd + + def tearDown(self): + (baseline.NOHUNT_MARKER, baseline.OFFLINE_MARKER, baseline.OFFLINE_SOCKET, + hw.initd, runner.Takeover.OFFLINE_BACK_S) = self.saved + self.fc.stop() + for k in ("GF_SYSFS_ROOT", "FORGETEST_PROC_ROOT", "FORGETEST_MARKER"): + os.environ.pop(k, None) + shutil.rmtree(self.tmp, ignore_errors=True) + + # -- the fakes ------------------------------------------------------ + def _socket(self, pid, inode, path, listening=True): + self.unix.append((pid, inode, path, listening)) + fd_dir = os.path.join(self.proc, str(pid), "fd") + os.makedirs(fd_dir, exist_ok=True) + os.symlink("socket:[%d]" % inode, os.path.join(fd_dir, str(len(os.listdir(fd_dir)) + 3))) + with open(os.path.join(self.proc, "net", "unix"), "w") as f: + f.write("Num RefCount Protocol Flags Type St Inode Path\n") + for _pid, ino, p, lis in self.unix: + f.write("00000000: 00000002 00000000 %08X 0001 %s %d %s\n" + % (0x10000 if lis else 0, "01" if lis else "03", ino, p)) + + def _offline_client(self, pid): + self._socket(pid, 9000 + pid, baseline.OFFLINE_SOCKET) + + def _initd(self, service, action, timeout=60): + """forgectrl stop/start; the start plays gfcloud's: it reads and + takes down the markers, and it is the offline service when the + offline marker was there.""" + self.calls.append((service, action)) + if action != "start": + return 0, "" + self.at_start = sorted(os.path.basename(p) for p in (baseline.NOHUNT_MARKER, baseline.OFFLINE_MARKER) + if os.path.exists(p)) + mode = self.fc.state["mode"] + if self.comes_up != "running": + self.fc.state["mode"] = dict(mode, controller=self.comes_up, pid=None, motion="unverified") + return 0, "" + offline = os.path.exists(baseline.OFFLINE_MARKER) + for p in (baseline.NOHUNT_MARKER, baseline.OFFLINE_MARKER): + if os.path.exists(p): + os.remove(p) + self.fc.state["mode"] = dict(mode, controller="running", pid=self.NEW_PID, motion="verified") + if offline and mode.get("mode") == "cloud": + self._offline_client(self.NEW_PID) + return 0, "" + + def takeover(self): + cap = {"position": None, "mode": self.fc.state["mode"]["mode"]} + run = FakeRun(cap) + with runner.Takeover(run.log, "test.id", run=run): + pass + return cap, run + + def markers_left(self): + return [p for p in (baseline.NOHUNT_MARKER, baseline.OFFLINE_MARKER) if os.path.exists(p)] + + # -- the socket judge ------------------------------------------------- + def test_the_listener_is_the_process_own_descriptor(self): + self._offline_client(self.OLD_PID) + self.assertTrue(hw.listens_on(self.OLD_PID, baseline.OFFLINE_SOCKET)) + # the path alone is not enough: another process's socket, a + # connected (not listening) socket, another path, no pid + self.assertFalse(hw.listens_on(self.NEW_PID, baseline.OFFLINE_SOCKET)) + self._socket(self.NEW_PID, 7001, baseline.OFFLINE_SOCKET, listening=False) + self.assertFalse(hw.listens_on(self.NEW_PID, baseline.OFFLINE_SOCKET)) + self.assertFalse(hw.listens_on(self.OLD_PID, "/run/forgefirm/grbl.ctl")) + self.assertFalse(hw.listens_on(None, baseline.OFFLINE_SOCKET)) + + # -- the takeover ------------------------------------------------------ + def test_the_offline_service_comes_back_offline(self): + # exthost.armed-freeze's case: the setup record put back under a + # takeover while the offline service ran + self._offline_client(self.OLD_PID) + cap, run = self.takeover() + self.assertEqual(self.calls, [("forgectrl", "stop"), ("forgectrl", "start")]) + self.assertEqual(self.at_start, ["gfcloud-nohunt", "gfcloud-offline"]) + self.assertTrue(hw.listens_on(self.NEW_PID, baseline.OFFLINE_SOCKET)) + self.assertNotIn("restart_clients", cap) + self.assertEqual(self.markers_left(), []) + self.assertTrue(any("back as the offline service (pid %d)" % self.NEW_PID in l for l in run.lines), + run.lines) + + def test_an_online_client_after_the_offline_service_fails_the_run(self): + # the client that came up is not listening: an online client, or + # a marker the start never saw + self._offline_client(self.OLD_PID) + real = self._initd + + def lost_marker(service, action, timeout=60): + if action == "start" and os.path.exists(baseline.OFFLINE_MARKER): + os.remove(baseline.OFFLINE_MARKER) + return real(service, action, timeout) + hw.initd = lost_marker + cap, run = self.takeover() + self.assertEqual(cap["restart_clients"], + [{"where": "takeover end", "expected": "the offline service", + "found": "a client that is not the offline service (pid %d)" % self.NEW_PID}]) + left = baseline.Baseline(run.log) + items = [] + left._preserved(items, cap) + self.assertEqual([x.item for x in items], ["cloud client at the takeover end"]) + self.assertTrue(items[0].action.startswith("not restored"), items[0].action) + + def test_no_client_started_takes_the_markers_down(self): + # the gate closed, or standby: nothing read the markers, and a + # later start must not come up offline or without its hunt + self._offline_client(self.OLD_PID) + self.comes_up = "gated" + cap, run = self.takeover() + self.assertEqual(self.at_start, ["gfcloud-nohunt", "gfcloud-offline"]) + self.assertEqual(self.markers_left(), []) + self.assertEqual(len(cap["restart_clients"]), 1) + self.assertIn("no cloud client running", cap["restart_clients"][0]["found"]) + + def test_an_online_client_comes_back_without_the_hunt(self): + cap, run = self.takeover() + self.assertEqual(self.at_start, ["gfcloud-nohunt"]) + self.assertFalse(hw.listens_on(self.NEW_PID, baseline.OFFLINE_SOCKET)) + self.assertNotIn("restart_clients", cap) + self.assertEqual(self.markers_left(), []) + + def test_a_stale_offline_socket_file_is_not_the_offline_service(self): + # a socket another process left at the path: the online client + # found stays online + self._socket(99, 7002, baseline.OFFLINE_SOCKET) + cap, run = self.takeover() + self.assertEqual(self.at_start, ["gfcloud-nohunt"]) + self.assertNotIn("restart_clients", cap) + + def test_grbl_mode_starts_under_no_marker(self): + self.fc.state["mode"] = {"mode": "grbl", "controller": "running", "pid": self.OLD_PID, + "motion": "verified"} + cap, run = self.takeover() + self.assertEqual(self.at_start, []) + self.assertNotIn("restart_clients", cap) + + +if __name__ == "__main__": + unittest.main()