forgetest: the service's hunt paid only where it is the subject

A cloud client the tool starts for anything but homing comes up under
the /run/gfcloud-nohunt marker: the real client back after the
emulator, a mode the runner switches to or hands back, a controller it
restarts. The service keeps the head position it has. cloud.mode-switch
and cloud.service-protocol keep their hunts, and so does the one real
print: enter_cloud reuses a running session only when that client has
hunted the machine itself (session_hunted: never the emulator's, never
a no-hunt start), otherwise it restarts the client with the hunt, since
a print placed on a head position the service only believes can run the
gantry into a rail. The markers are one start, taken down by the client
that read them first thing; the tool's own removal stays for a start
that never happened. Catalog unchanged; the cloud tests' shared code
moved, so their implementation hashes move with it.
This commit is contained in:
ScottW514
2026-08-23 11:27:36 -04:00
parent 5c7727b9bf
commit 969bac6013
5 changed files with 294 additions and 37 deletions
+17
View File
@@ -171,6 +171,23 @@ bench, or one whose `/data` has been wiped, starts from a full campaign.
latch, so they stay `live`. The offline client is left running; the latch, so they stay `live`. The offline client is left running; the
next test that needs the service restarts it (`enter_cloud` does), as next test that needs the service restarts it (`enter_cloud` does), as
does a mode switch or a controller restart. does a mode switch or a controller restart.
**The service's connect-time hunt is paid only where it is the
subject.** A cloud client the tool starts for anything else (the real
client back after the emulator, a mode the runner switches to or hands
back, a controller it restarts) comes up under the `/run/gfcloud-nohunt`
marker: its first settings report is the reconnect form, and the
service keeps the head position it has instead of homing. The hunt
tests (`cloud.mode-switch`, `cloud.service-protocol`) get theirs, and
so does the one real print: `enter_cloud` reuses a running session
only when that client has hunted the machine itself (never the
emulator's, never a no-hunt start), otherwise it restarts the client
with the hunt, because a print placed on a head position the service
only believes can run the gantry into a rail. The same holds outside
the tool: a machine left in cloud mode by a campaign may not have
hunted since GRBL mode moved the head, so open and close the lid (the
service re-hunts) or restart the controller before printing from the
app. Every marker is one start: the client that reads it takes it
down.
4. Or hand the whole list to a queue. **Run what is left** offers two: 4. Or hand the whole list to a queue. **Run what is left** offers two:
**Unattended** takes every `auto` test the campaign does not already **Unattended** takes every `auto` test the campaign does not already
count as satisfied, and needs nobody in the room; **Operator and live** count as satisfied, and needs nobody in the room; **Operator and live**
+39
View File
@@ -78,6 +78,15 @@ GRBL_CONTROLLER_SYSFS = ("cnc/motor_lock", "cnc/x_mode", "cnc/y_mode", "cnc/x_de
# would leave the runtime in one mode and the boot in the other). # would leave the runtime in one mode and the boot in the other).
UNPRESERVED_SETTINGS = ("controller_mode",) UNPRESERVED_SETTINGS = ("controller_mode",)
# A cloud client the baseline starts (the mode a test declares, the mode
# or controller handed back after a run) comes up without the service's
# connect-time hunt: the marker gfcloud reads first thing at start and
# takes down (one start), so it is left in place once the start is
# accepted and removed only when the start never happened. The service
# keeps the head position it has; a test that homes or prints under the
# service starts its own client with the hunt.
NOHUNT_MARKER = os.environ.get("FORGETEST_NOHUNT_MARKER", "/run/gfcloud-nohunt")
# Read-only readbacks with their idle values (no direct restore: the state # Read-only readbacks with their idle values (no direct restore: the state
# comes right through forgectrl - see restore_forgectrl - or is fatal). # comes right through forgectrl - see restore_forgectrl - or is fatal).
IDLE_READBACKS = [ IDLE_READBACKS = [
@@ -260,6 +269,25 @@ class Baseline:
self.log("WARNING - forgectrl did not settle within %d s (last /mode: %s)" % (timeout, last)) self.log("WARNING - forgectrl did not settle within %d s (last /mode: %s)" % (timeout, last))
return last return last
# -- a cloud client without the hunt ---------------------------------
def nohunt_on(self, want):
"""The no-hunt marker before a start that brings the cloud client
up (a switch to cloud, a controller start in cloud mode)."""
if want != "cloud":
return
try:
with open(NOHUNT_MARKER, "w") as f:
f.write("forgetest baseline\n")
except OSError as e:
self.log("no-hunt marker not written (%s): the client will hunt" % e)
def nohunt_off(self):
"""The marker the client never read: a start that was refused."""
try:
os.remove(NOHUNT_MARKER)
except OSError:
pass
# -- the mode a test needs ------------------------------------------- # -- the mode a test needs -------------------------------------------
def switch_mode(self, want, timeout=SETTLE_S): def switch_mode(self, want, timeout=SETTLE_S):
"""Put the machine in controller mode `want` through the supervisor """Put the machine in controller mode `want` through the supervisor
@@ -276,8 +304,10 @@ class Baseline:
return True, "already in %s mode" % want return True, "already in %s mode" % want
self.log("switching to %s mode (found %s, controller %s)" self.log("switching to %s mode (found %s, controller %s)"
% (want, mode.get("mode"), mode.get("controller"))) % (want, mode.get("mode"), mode.get("controller")))
self.nohunt_on(want)
st, body = self.fc_post("/mode", data={"controller": want}) st, body = self.fc_post("/mode", data={"controller": want})
if st != 200: if st != 200:
self.nohunt_off()
return False, "POST /mode controller=%s -> %s %s" % (want, st, body) return False, "POST /mode controller=%s -> %s %s" % (want, st, body)
mode = self.wait_settled(timeout=timeout) or {} mode = self.wait_settled(timeout=timeout) or {}
self.mode = mode.get("mode") self.mode = mode.get("mode")
@@ -358,19 +388,28 @@ class Baseline:
# declaring it is handed back through the switch. # declaring it is handed back through the switch.
want = (captured or {}).get("mode") or mode.get("mode") or "grbl" want = (captured or {}).get("mode") or mode.get("mode") or "grbl"
if mode.get("mode") != want: if mode.get("mode") != want:
self.nohunt_on(want)
st, body = self.fc_post("/mode", data={"controller": want}) st, body = self.fc_post("/mode", data={"controller": want})
if st != 200:
self.nohunt_off()
mode = self.wait_settled() or mode mode = self.wait_settled() or mode
self.mode = mode.get("mode") self.mode = mode.get("mode")
left.append(Leftover("mode", mode.get("mode"), want, left.append(Leftover("mode", mode.get("mode"), want,
"restored" if mode.get("mode") == want else "failed: %s %s" % (st, body))) "restored" if mode.get("mode") == want else "failed: %s %s" % (st, body)))
if mode.get("controller") == "motion-fault": if mode.get("controller") == "motion-fault":
self.nohunt_on(want)
st, body = self.fc_post("/mode", data={"controller": want}) st, body = self.fc_post("/mode", data={"controller": want})
if st != 200:
self.nohunt_off()
mode = self.wait_settled() or mode mode = self.wait_settled() or mode
left.append(Leftover("controller", "motion-fault", "running", left.append(Leftover("controller", "motion-fault", "running",
"restored" if mode.get("controller") == "running" "restored" if mode.get("controller") == "running"
else "failed: %s" % mode.get("controller"))) else "failed: %s" % mode.get("controller")))
elif mode.get("controller") == "standby": elif mode.get("controller") == "standby":
self.nohunt_on(mode.get("mode"))
st, body = self.fc_post("/controller/start") st, body = self.fc_post("/controller/start")
if st != 200:
self.nohunt_off()
mode = self.wait_settled() or mode mode = self.wait_settled() or mode
left.append(Leftover("controller", "standby", "running", left.append(Leftover("controller", "standby", "running",
"restored" if mode.get("controller") == "running" "restored" if mode.get("controller") == "running"
+86 -28
View File
@@ -429,6 +429,13 @@ JOB_DIR = "/tmp/forgetest" # the jobs the offline tests write (tmpfs)
EMULATE_MARK = "EMULATE: the emulator" EMULATE_MARK = "EMULATE: the emulator"
EMULATE_MARKER = "/run/gfcloud-emulate" EMULATE_MARKER = "/run/gfcloud-emulate"
EMULATOR_WORK = "/tmp/gfcloud-emulate" EMULATOR_WORK = "/tmp/gfcloud-emulate"
# No connect-time hunt (gfcloud --no-hunt, or the marker at start): the
# first settings report in the reconnect form, the service keeps the head
# position it has. For a restart between tests that do not home; the
# homing tests and the one real print get their hunt.
NOHUNT_MARK = "NO-HUNT:"
NOHUNT_MARKER = "/run/gfcloud-nohunt"
HUNT_DONE = "hunt ["
WS_MARKS = ("RX-EVENT: ready", "RX-EVENT: closed", "RECONNECTING", "CLOSING", OFFLINE_MARK) WS_MARKS = ("RX-EVENT: ready", "RX-EVENT: closed", "RECONNECTING", "CLOSING", OFFLINE_MARK)
ACTIVITY_MARKS = ("start motion", "start return home", "starting run", "starting z homing cycle") ACTIVITY_MARKS = ("start motion", "start return home", "starting run", "starting z homing cycle")
LOG_TAIL_BYTES = 4 << 20 LOG_TAIL_BYTES = 4 << 20
@@ -596,6 +603,24 @@ def session_live(pid):
return last == "RX-EVENT: ready", "%s: last websocket state %s" % (where, last) return last == "RX-EVENT: ready", "%s: last websocket state %s" % (where, last)
def session_hunted(pid):
"""The running cloud client (gfcloud[pid]) has homed the machine under
the service: a hunt of its own finished (the connect-time one, or a
later re-hunt), and it is not the emulator. A client started without
the hunt, or one whose lines have left the log tail, has not - the
service may hold a head position the machine no longer has."""
tag = "gfcloud[%s]" % pid
hunted = False
for ln in log_tail(GFCLOUD_LOG):
if tag not in ln:
continue
if EMULATE_MARK in ln:
return False
if HUNT_DONE in ln and "finished with event" in ln and COMPLETED in ln:
hunted = True
return hunted
def client_offline(pid): def client_offline(pid):
"""The running cloud client is the offline service.""" """The running cloud client is the offline service."""
live, detail = session_live(pid) live, detail = session_live(pid)
@@ -637,16 +662,20 @@ def wait_session(ctx, offset, timeout=120):
return session return session
def fresh_cloud_connect(ctx): def fresh_cloud_connect(ctx, hunt=True):
"""A NEW cloud client with a fresh service session (its connect-time """A NEW cloud client with a fresh service session: in cloud mode the
hunt follows): in cloud mode the controller is restarted through the controller is restarted through the supervisor's stop/start lever; in
supervisor's stop/start lever; in GRBL mode the mode is switched (and GRBL mode the mode is switched (and the change declared - the machine
the change declared - the machine stays in cloud mode). Returns the log stays in cloud mode). Its connect-time hunt follows unless hunt is
offset from before the connect, so the caller sees the whole session.""" False (the no-hunt marker, for a start where the service already
knows the head). Returns the log offset from before the connect, so
the caller sees the whole session."""
fc = ctx.forgectrl fc = ctx.forgectrl
st, m = fc.get("/mode") st, m = fc.get("/mode")
ctx.check(st == 200 and isinstance(m, dict), "GET /mode -> %s", st) ctx.check(st == 200 and isinstance(m, dict), "GET /mode -> %s", st)
offset = log_size(GFCLOUD_LOG) offset = log_size(GFCLOUD_LOG)
if not hunt:
marker_on(ctx, NOHUNT_MARKER)
if m.get("mode") == "cloud": if m.get("mode") == "cloud":
ctx.log("cloud mode: restarting the cloud client for a fresh connect (was pid %s)", m.get("pid")) ctx.log("cloud mode: restarting the cloud client for a fresh connect (was pid %s)", m.get("pid"))
st, body = fc.post("/controller/stop") st, body = fc.post("/controller/stop")
@@ -672,25 +701,35 @@ def fresh_cloud_connect(ctx):
def enter_cloud(ctx): def enter_cloud(ctx):
"""Cloud mode with a live service session and a quiet machine. An """Cloud mode with a live service session, the machine homed under
existing cloud session is reused; from GRBL mode the switch is made it, and quiet. An existing cloud session is reused when the client
once (its connect-time hunt is waited out) and the machine stays in has hunted the machine itself (never the emulator's session, never a
cloud mode. Returns the log offset where the test's own window begins.""" no-hunt start: the service would be holding a head position the
machine may not have, and what follows is the one real print); from
GRBL mode the switch is made once (its connect-time hunt is waited
out) and the machine stays in cloud mode. Returns the log offset
where the test's own window begins."""
fc = ctx.forgectrl fc = ctx.forgectrl
st, m = fc.get("/mode") st, m = fc.get("/mode")
ctx.check(st == 200 and isinstance(m, dict), "GET /mode -> %s", st) ctx.check(st == 200 and isinstance(m, dict), "GET /mode -> %s", st)
offline_marker_off(ctx) offline_marker_off(ctx)
live = detail = None live = detail = None
hunted = False
if m.get("mode") == "cloud" and m.get("controller") == "running": if m.get("mode") == "cloud" and m.get("controller") == "running":
live, detail = session_live(m.get("pid")) live, detail = session_live(m.get("pid"))
ctx.check(live or "offline" in detail, ctx.check(live or "offline" in detail,
"cloud mode is up (pid %s) but the client has no live service session (%s) - " "cloud mode is up (pid %s) but the client has no live service session (%s) - "
"check credentials and network, or restart the controller", m.get("pid"), detail) "check credentials and network, or restart the controller", m.get("pid"), detail)
if live: hunted = live and session_hunted(m.get("pid"))
ctx.log("cloud mode already up (pid %s), service session live - reusing it", m.get("pid")) if live and hunted:
ctx.log("cloud mode already up (pid %s), service session live and the machine hunted "
"under it - reusing it", m.get("pid"))
offset = log_size(GFCLOUD_LOG) offset = log_size(GFCLOUD_LOG)
else: else:
if detail: if live:
ctx.log("the running client (pid %s) never hunted the machine under the service: "
"restarting it with the hunt", m.get("pid"))
elif detail:
ctx.log("the running client is the offline service: restarting it with the service") ctx.log("the running client is the offline service: restarting it with the service")
offset = fresh_cloud_connect(ctx) offset = fresh_cloud_connect(ctx)
hunt = wait_action_finished(ctx, offset, "hunt", HUNT_TIMEOUT_S) hunt = wait_action_finished(ctx, offset, "hunt", HUNT_TIMEOUT_S)
@@ -702,9 +741,20 @@ def enter_cloud(ctx):
return log_size(GFCLOUD_LOG) return log_size(GFCLOUD_LOG)
def marker_on(ctx, path):
"""A one-start marker for the next cloud client: the client that reads
it takes it down. Written before the start; offline_marker_off is the
belt-and-braces removal once the client's own line is in."""
with open(path, "w") as f:
f.write("forgetest %s\n" % ctx.test.id)
def offline_marker_off(ctx): def offline_marker_off(ctx):
"""No client started from here on comes up offline, or as the emulator.""" """No client started from here on comes up offline, as the emulator,
for path, what in ((OFFLINE_MARKER, "offline"), (EMULATE_MARKER, "emulate")): or without its hunt. The client takes a marker it read down itself;
this catches one it never read (a start that failed)."""
for path, what in ((OFFLINE_MARKER, "offline"), (EMULATE_MARKER, "emulate"),
(NOHUNT_MARKER, "no-hunt")):
try: try:
os.remove(path) os.remove(path)
ctx.log("%s marker removed", what) ctx.log("%s marker removed", what)
@@ -712,20 +762,23 @@ def offline_marker_off(ctx):
pass pass
def restart_client(ctx, marker=None, who=""): def restart_client(ctx, marker=None, who="", hunt=True):
"""The cloud client started fresh - restarted in cloud mode, switched to """The cloud client started fresh - restarted in cloud mode, switched to
from GRBL mode (the change declared) - under `marker` if one is given, from GRBL mode (the change declared) - under `marker` if one is given,
which stays in place until the client has read it (the caller waits and under the no-hunt marker when hunt is False. A marker stays in
for the client's own line and takes the marker down; the supervisor place until the client has read it (the client takes it down; the
reports the client running seconds before Python has finished caller waits for the client's own line and takes it down as well if
importing). Returns the log offset from before the start.""" the start failed - the supervisor reports the client running seconds
before Python has finished importing). Returns the log offset from
before the start."""
fc = ctx.forgectrl fc = ctx.forgectrl
st, m = fc.get("/mode") st, m = fc.get("/mode")
ctx.check(st == 200 and isinstance(m, dict), "GET /mode -> %s", st) ctx.check(st == 200 and isinstance(m, dict), "GET /mode -> %s", st)
offset = log_size(GFCLOUD_LOG) offset = log_size(GFCLOUD_LOG)
if marker: if marker:
with open(marker, "w") as f: marker_on(ctx, marker)
f.write("forgetest %s\n" % ctx.test.id) if not hunt:
marker_on(ctx, NOHUNT_MARKER)
if m.get("mode") == "cloud": if m.get("mode") == "cloud":
ctx.log("cloud mode: restarting the client%s (was pid %s)", who, m.get("pid")) ctx.log("cloud mode: restarting the client%s (was pid %s)", who, m.get("pid"))
st, body = fc.post("/controller/stop") st, body = fc.post("/controller/stop")
@@ -766,17 +819,22 @@ def enter_emulator(ctx):
def leave_emulator(ctx): def leave_emulator(ctx):
"""The real client back, its session waited for and its connect-time """The real client back without its connect-time hunt (nothing moved
hunt waited out, so the machine is what the next test expects.""" while the emulator had the session, and a test that prints hunts
first regardless), its session waited for, so the machine is what
the next test expects."""
offline_marker_off(ctx) offline_marker_off(ctx)
offset = restart_client(ctx, who=" with the machine") offset = restart_client(ctx, who=" with the machine, no hunt", hunt=False)
try:
got = wait_log(ctx, offset, [NOHUNT_MARK], 60)
ctx.check(got[NOHUNT_MARK], "the real client did not come up without the hunt within 60 s")
finally:
offline_marker_off(ctx)
session = wait_session(ctx, offset) session = wait_session(ctx, offset)
ctx.check(session_established(session), "the real client did not establish its session after the emulator") ctx.check(session_established(session), "the real client did not establish its session after the emulator")
hunt = wait_action_finished(ctx, offset, "hunt", HUNT_TIMEOUT_S)
ctx.check(hunt, "the real client's connect-time hunt did not finish within %d s", HUNT_TIMEOUT_S)
ctx.check(wait_quiet(ctx, offset), "the service was still moving the head %d s after the real " ctx.check(wait_quiet(ctx, offset), "the service was still moving the head %d s after the real "
"client came back", QUIET_TIMEOUT_S) "client came back", QUIET_TIMEOUT_S)
ctx.log("the machine is back under the real client") ctx.log("the machine is back under the real client, no hunt")
def enter_offline(ctx): def enter_offline(ctx):
+63
View File
@@ -293,6 +293,69 @@ class BaselineModeTests(BaselineTests):
self.fc.state["mode"] = {"mode": "cloud", "controller": "running", "pid": 7, "motion": "verified"} self.fc.state["mode"] = {"mode": "cloud", "controller": "running", "pid": 7, "motion": "verified"}
self.fc.state["settings"]["controller_mode"] = "cloud" self.fc.state["settings"]["controller_mode"] = "cloud"
def nohunt_marker(self, refuse=False):
"""The no-hunt marker under the fake tree; what each POST found.
The fake client takes the marker down as gfcloud does, first thing
at its start; with refuse the POST is refused and no client starts."""
baseline.NOHUNT_MARKER = os.path.join(self.tmp, "nohunt-marker")
seen = []
def on_post(path, form):
seen.append((path, dict(form), os.path.exists(baseline.NOHUNT_MARKER)))
if refuse:
return 409, {"error": "busy"}
if os.path.exists(baseline.NOHUNT_MARKER):
os.remove(baseline.NOHUNT_MARKER)
if path == "/mode":
self.fc.state["mode"] = dict(self.fc.state["mode"], mode=form["controller"], controller="running")
elif path == "/controller/start":
self.fc.state["mode"] = dict(self.fc.state["mode"], controller="running")
return None
self.fc.on_post = on_post
return seen
def test_a_refused_switch_leaves_no_marker_for_the_next_start(self):
seen = self.nohunt_marker(refuse=True)
ok, detail = self.bl().switch_mode("cloud")
self.assertFalse(ok)
self.assertEqual(seen, [("/mode", {"controller": "cloud"}, True)])
self.assertFalse(os.path.exists(baseline.NOHUNT_MARKER))
def test_a_switch_to_cloud_starts_the_client_without_the_hunt(self):
seen = self.nohunt_marker()
ok, detail = self.bl().switch_mode("cloud")
self.assertTrue(ok, detail)
self.assertEqual(seen, [("/mode", {"controller": "cloud"}, True)])
self.assertFalse(os.path.exists(baseline.NOHUNT_MARKER)) # one start, never left behind
def test_a_switch_to_grbl_sets_no_marker(self):
seen = self.nohunt_marker()
self.cloud()
ok, detail = self.bl().switch_mode("grbl")
self.assertFalse(ok) # the fake has no Grbl port: the switch itself happened
self.assertEqual(seen, [("/mode", {"controller": "grbl"}, False)])
self.assertFalse(os.path.exists(baseline.NOHUNT_MARKER))
def test_the_mode_handed_back_is_cloud_without_the_hunt(self):
seen = self.nohunt_marker()
self.cloud()
b = self.bl()
cap = b.capture() # found in cloud
self.fc.state["mode"] = dict(self.fc.state["mode"], mode="grbl") # the run left it in grbl
b.enforce("post", captured=cap)
self.assertEqual(seen[0], ("/mode", {"controller": "cloud"}, True))
self.assertFalse(os.path.exists(baseline.NOHUNT_MARKER))
def test_a_standby_cloud_client_is_started_without_the_hunt(self):
seen = self.nohunt_marker()
self.cloud()
b = self.bl()
cap = b.capture()
self.fc.state["mode"] = dict(self.fc.state["mode"], controller="standby")
b.enforce("post", captured=cap)
self.assertEqual(seen[0], ("/controller/start", {}, True))
self.assertFalse(os.path.exists(baseline.NOHUNT_MARKER))
def test_grbl_mode_restores_the_controller_values_and_the_lamp(self): def test_grbl_mode_restores_the_controller_values_and_the_lamp(self):
self._attr("cnc/step_freq", "10000") self._attr("cnc/step_freq", "10000")
self._attr("pic/lid_led", "77") self._attr("pic/lid_led", "77")
+89 -9
View File
@@ -171,9 +171,10 @@ class CloudSuiteTests(unittest.TestCase):
cloud.Offline = FakeOffline cloud.Offline = FakeOffline
cloud.OFFLINE_MARKER = os.path.join(self.tmp, "offline-marker") cloud.OFFLINE_MARKER = os.path.join(self.tmp, "offline-marker")
cloud.JOB_DIR = os.path.join(self.tmp, "jobs") cloud.JOB_DIR = os.path.join(self.tmp, "jobs")
self.saved_emu = (cloud.EMULATE_MARKER, cloud.EMULATOR_WORK) self.saved_emu = (cloud.EMULATE_MARKER, cloud.EMULATOR_WORK, cloud.NOHUNT_MARKER)
cloud.EMULATE_MARKER = os.path.join(self.tmp, "emulate-marker") cloud.EMULATE_MARKER = os.path.join(self.tmp, "emulate-marker")
cloud.EMULATOR_WORK = os.path.join(self.tmp, "emu-work") cloud.EMULATOR_WORK = os.path.join(self.tmp, "emu-work")
cloud.NOHUNT_MARKER = os.path.join(self.tmp, "nohunt-marker")
FakeOffline.sent = [] FakeOffline.sent = []
FakeOffline.on_send = None FakeOffline.on_send = None
self.engine_line = EFFECTIVE_LINE # what the engine logs at the print; None = nothing self.engine_line = EFFECTIVE_LINE # what the engine logs at the print; None = nothing
@@ -192,7 +193,7 @@ class CloudSuiteTests(unittest.TestCase):
(cloud.GFCLOUD_LOG, cloud.FORGECTRL_LOG, cloud.QUIET_S, cloud.QUIET_TIMEOUT_S, (cloud.GFCLOUD_LOG, cloud.FORGECTRL_LOG, cloud.QUIET_S, cloud.QUIET_TIMEOUT_S,
cloud.HUNT_TIMEOUT_S, cloud.GFHOME_LOG, cloud.Offline, cloud.OFFLINE_MARKER, cloud.HUNT_TIMEOUT_S, cloud.GFHOME_LOG, cloud.Offline, cloud.OFFLINE_MARKER,
cloud.JOB_DIR) = self.saved cloud.JOB_DIR) = self.saved
(cloud.EMULATE_MARKER, cloud.EMULATOR_WORK) = self.saved_emu (cloud.EMULATE_MARKER, cloud.EMULATOR_WORK, cloud.NOHUNT_MARKER) = self.saved_emu
os.environ.pop("GF_SYSFS_ROOT", None) os.environ.pop("GF_SYSFS_ROOT", None)
os.environ.pop("GF_LEDS_ROOT", None) os.environ.pop("GF_LEDS_ROOT", None)
shutil.rmtree(self.tmp, ignore_errors=True) shutil.rmtree(self.tmp, ignore_errors=True)
@@ -224,6 +225,12 @@ class CloudSuiteTests(unittest.TestCase):
def lid(self, closed): def lid(self, closed):
self.fc.state["status"]["switches"]["lid"] = bool(closed) self.fc.state["status"]["switches"]["lid"] = bool(closed)
def hunted(self, pid):
"""The running client homed the machine under the service (a test
that prints reuses only such a session)."""
self.append(["2026-08-17T09:41:20.000000+00:00 gfcloud[%d] INFO basemachine:_finish_action hunt "
"[1578200001]: finished with event \":completed\"" % pid])
def in_offline(self, pid=3100): def in_offline(self, pid=3100):
"""Cloud mode with the offline service already up (its mark is the """Cloud mode with the offline service already up (its mark is the
newest websocket-state line for the pid).""" newest websocket-state line for the pid)."""
@@ -283,11 +290,12 @@ class CloudSuiteTests(unittest.TestCase):
"47848 byte body", "47848 byte body",
"2026-08-22T23:51:02.000000+00:00 gfcloud[4100] INFO basemachine:_finish_action print [1578300003]: " "2026-08-22T23:51:02.000000+00:00 gfcloud[4100] INFO basemachine:_finish_action print [1578300003]: "
"finished with event \":completed\""] "finished with event \":completed\""]
REAL_BACK = ["2026-08-22T23:52:02.000000+00:00 gfcloud[4200] INFO authentication:authenticate_machine SUCCESS", REAL_BACK = ["2026-08-22T23:52:01.000000+00:00 gfcloud[4200] INFO ffmachine:inhibit_connect_hunt NO-HUNT: the "
"first settings report in the reconnect form; the service keeps its head position, no "
"connect-time hunt",
"2026-08-22T23:52:02.000000+00:00 gfcloud[4200] INFO authentication:authenticate_machine SUCCESS",
"2026-08-22T23:52:03.000000+00:00 gfcloud[4200] INFO websocket:ws_connect ESTABLISHED", "2026-08-22T23:52:03.000000+00:00 gfcloud[4200] INFO websocket:ws_connect ESTABLISHED",
"2026-08-22T23:52:03.500000+00:00 gfcloud[4200] INFO websocket:_on_open RX-EVENT: ready", "2026-08-22T23:52:03.500000+00:00 gfcloud[4200] INFO websocket:_on_open RX-EVENT: ready"]
"2026-08-22T23:52:15.000000+00:00 gfcloud[4200] INFO basemachine:_finish_action hunt [1578300010]: "
"finished with event \":completed\""]
def emulator_setup(self, print_lines=None, info=True): def emulator_setup(self, print_lines=None, info=True):
self.in_cloud(pid=2278) self.in_cloud(pid=2278)
@@ -298,7 +306,7 @@ class CloudSuiteTests(unittest.TestCase):
if path == "/controller/stop": if path == "/controller/stop":
self.fc.state["mode"] = dict(self.fc.state["mode"], controller="standby", pid=0) self.fc.state["mode"] = dict(self.fc.state["mode"], controller="standby", pid=0)
elif path == "/controller/start": elif path == "/controller/start":
starts.append(os.path.exists(cloud.EMULATE_MARKER)) starts.append((os.path.exists(cloud.EMULATE_MARKER), os.path.exists(cloud.NOHUNT_MARKER)))
if len(starts) == 1: if len(starts) == 1:
self.fc.state["mode"] = dict(self.fc.state["mode"], controller="running", pid=self.EMU_PID) self.fc.state["mode"] = dict(self.fc.state["mode"], controller="running", pid=self.EMU_PID)
self.append(self.EMU_START, delay=0.2) self.append(self.EMU_START, delay=0.2)
@@ -320,8 +328,11 @@ class CloudSuiteTests(unittest.TestCase):
starts, hooks = self.emulator_setup() starts, hooks = self.emulator_setup()
run = self.run_test(cloud.service_protocol, hooks=hooks, test_id="cloud.service-protocol") run = self.run_test(cloud.service_protocol, hooks=hooks, test_id="cloud.service-protocol")
ev = run.evidence ev = run.evidence
self.assertEqual(starts, [True, False]) # emulator under the marker, the real client without # the emulator under its marker with the hunt; the real client back without the hunt
self.assertEqual(starts, [(True, False), (False, True)])
self.assertFalse(os.path.exists(cloud.EMULATE_MARKER)) self.assertFalse(os.path.exists(cloud.EMULATE_MARKER))
self.assertFalse(os.path.exists(cloud.NOHUNT_MARKER))
self.assertTrue(any("no hunt" in l for l in run.lines), run.lines[-5:])
self.assertEqual([p for p, _ in self.fc.posts], self.assertEqual([p for p, _ in self.fc.posts],
["/controller/stop", "/controller/start", "/controller/stop", "/controller/start"]) ["/controller/stop", "/controller/start", "/controller/stop", "/controller/start"])
self.assertIn(":completed", ev["hunt"]) self.assertIn(":completed", ev["hunt"])
@@ -336,7 +347,25 @@ class CloudSuiteTests(unittest.TestCase):
failed = [l.replace(':completed"', ':failed"') for l in self.EMU_PRINT] failed = [l.replace(':completed"', ':failed"') for l in self.EMU_PRINT]
starts, hooks = self.emulator_setup(print_lines=failed) starts, hooks = self.emulator_setup(print_lines=failed)
self.assertFails(cloud.service_protocol, "did not complete", hooks=hooks) self.assertFails(cloud.service_protocol, "did not complete", hooks=hooks)
self.assertEqual(starts, [True, False]) # the real client came back regardless self.assertEqual(starts, [(True, False), (False, True)]) # the real client came back regardless
def test_markers_the_client_consumed_are_not_missed(self):
# gfcloud takes a marker it read down itself; the suite's own removal is then a no-op
starts, hooks = self.emulator_setup()
on_post = self.fc.on_post
def consuming(path, form):
r = on_post(path, form)
if path == "/controller/start":
for p in (cloud.EMULATE_MARKER, cloud.NOHUNT_MARKER):
if os.path.exists(p):
os.remove(p)
return r
self.fc.on_post = consuming
run = self.run_test(cloud.service_protocol, hooks=hooks, test_id="cloud.service-protocol")
self.assertEqual(starts, [(True, False), (False, True)])
self.assertTrue(any("PASS: session, hunt" in l for l in run.lines), run.lines[-5:])
self.assertFalse(any("marker removed" in l for l in run.lines), run.lines)
def test_the_emulators_session_is_not_the_machines(self): def test_the_emulators_session_is_not_the_machines(self):
self.append(self.EMU_START + ["2026-08-22T23:50:07.000000+00:00 gfcloud[4100] INFO websocket:_on_open " self.append(self.EMU_START + ["2026-08-22T23:50:07.000000+00:00 gfcloud[4100] INFO websocket:_on_open "
@@ -452,6 +481,50 @@ class CloudSuiteTests(unittest.TestCase):
self.assertEqual(self.fc.posts, []) self.assertEqual(self.fc.posts, [])
self.assertEqual(run.baseline_captured["mode"], "cloud") self.assertEqual(run.baseline_captured["mode"], "cloud")
NOHUNT_CLIENT = ["2026-08-22T23:52:01.000000+00:00 gfcloud[4200] INFO ffmachine:inhibit_connect_hunt NO-HUNT: the "
"first settings report in the reconnect form; the service keeps its head position, no "
"connect-time hunt",
"2026-08-22T23:52:02.000000+00:00 gfcloud[4200] INFO authentication:authenticate_machine SUCCESS",
"2026-08-22T23:52:03.000000+00:00 gfcloud[4200] INFO websocket:ws_connect ESTABLISHED",
"2026-08-22T23:52:03.500000+00:00 gfcloud[4200] INFO websocket:_on_open RX-EVENT: ready"]
def test_session_hunted_is_the_clients_own_hunt(self):
self.append(fixture("huntlid"))
self.assertTrue(cloud.session_hunted(2278)) # its connect-time hunt completed
self.assertFalse(cloud.session_hunted(9999)) # no lines: unknown is not hunted
self.append(self.NOHUNT_CLIENT)
self.assertTrue(cloud.session_live(4200)[0])
self.assertFalse(cloud.session_hunted(4200)) # live, never hunted
self.append(self.EMU_START)
self.assertFalse(cloud.session_hunted(4100)) # the emulator's hunt is not the machine's
# a later re-hunt (the lid closed) makes a no-hunt client hunted
self.append(["2026-08-22T23:55:00.000000+00:00 gfcloud[4200] INFO basemachine:_finish_action hunt [1578300020]: "
"finished with event \":completed\""])
self.assertTrue(cloud.session_hunted(4200))
def test_enter_cloud_restarts_a_live_session_that_never_hunted_with_the_hunt(self):
# the real print must not ride a head position the service only believes
self.in_cloud(pid=4200)
self.append(fixture("huntlid") + self.NOHUNT_CLIENT)
lines = fixture("huntlid")
_, post = cut(lines, "gfuiservice:__init__ INITIALIZED")
starts = []
def on_post(path, form):
if path == "/controller/stop":
self.fc.state["mode"] = dict(self.fc.state["mode"], controller="standby", pid=0)
elif path == "/controller/start":
starts.append(os.path.exists(cloud.NOHUNT_MARKER))
self.fc.state["mode"] = dict(self.fc.state["mode"], controller="running", pid=2278)
self.append(post, delay=0.2)
return None
self.fc.on_post = on_post
run = self.run_test(lambda ctx: cloud.enter_cloud(ctx))
self.assertEqual([p for p, _ in self.fc.posts], ["/controller/stop", "/controller/start"])
self.assertEqual(starts, [False]) # with the hunt: no marker
self.assertTrue(any("never hunted" in l for l in run.lines), run.lines)
self.assertTrue(any("connect-time hunt" in l and ":completed" in l for l in run.lines), run.lines)
def test_enter_cloud_refuses_a_dead_session(self): def test_enter_cloud_refuses_a_dead_session(self):
self.in_cloud(pid=1927) # the client that shut down in the excerpt self.in_cloud(pid=1927) # the client that shut down in the excerpt
self.append(fixture("huntlid")) self.append(fixture("huntlid"))
@@ -610,6 +683,7 @@ class CloudSuiteTests(unittest.TestCase):
def test_pause_resume_passes_on_the_machines_lines(self): def test_pause_resume_passes_on_the_machines_lines(self):
self.in_cloud(pid=1522) self.in_cloud(pid=1522)
self.hunted(1522)
self.append(["2026-08-17T09:41:00.000000+00:00 gfcloud[1522] INFO authentication:authenticate_machine SUCCESS", self.append(["2026-08-17T09:41:00.000000+00:00 gfcloud[1522] INFO authentication:authenticate_machine SUCCESS",
"2026-08-17T09:41:00.500000+00:00 gfcloud[1522] INFO websocket:_on_open RX-EVENT: ready", "2026-08-17T09:41:00.500000+00:00 gfcloud[1522] INFO websocket:_on_open RX-EVENT: ready",
"2026-08-17T09:41:00.900000+00:00 gfcloud[1522] INFO websocket:ws_connect ESTABLISHED"]) "2026-08-17T09:41:00.900000+00:00 gfcloud[1522] INFO websocket:ws_connect ESTABLISHED"])
@@ -636,6 +710,7 @@ class CloudSuiteTests(unittest.TestCase):
def test_pause_resume_fails_when_the_client_names_no_header_limits(self): def test_pause_resume_fails_when_the_client_names_no_header_limits(self):
self.client_limits = False self.client_limits = False
self.in_cloud(pid=1522) self.in_cloud(pid=1522)
self.hunted(1522)
self.append(["2026-08-17T09:41:00.500000+00:00 gfcloud[1522] INFO websocket:_on_open RX-EVENT: ready"]) self.append(["2026-08-17T09:41:00.500000+00:00 gfcloud[1522] INFO websocket:_on_open RX-EVENT: ready"])
hooks = self.replay_print("pause", "the press pauses the print", "current state: MachineState.IDLE") hooks = self.replay_print("pause", "the press pauses the print", "current state: MachineState.IDLE")
self.fc.state["cool"]["armed"] = True self.fc.state["cool"]["armed"] = True
@@ -644,6 +719,7 @@ class CloudSuiteTests(unittest.TestCase):
def test_pause_resume_fails_when_the_engine_resolves_against_no_header(self): def test_pause_resume_fails_when_the_engine_resolves_against_no_header(self):
self.engine_line = EFFECTIVE_LINE.replace("header 33.0", "header none 0.0") self.engine_line = EFFECTIVE_LINE.replace("header 33.0", "header none 0.0")
self.in_cloud(pid=1522) self.in_cloud(pid=1522)
self.hunted(1522)
self.append(["2026-08-17T09:41:00.500000+00:00 gfcloud[1522] INFO websocket:_on_open RX-EVENT: ready"]) self.append(["2026-08-17T09:41:00.500000+00:00 gfcloud[1522] INFO websocket:_on_open RX-EVENT: ready"])
hooks = self.replay_print("pause", "the press pauses the print", "current state: MachineState.IDLE") hooks = self.replay_print("pause", "the press pauses the print", "current state: MachineState.IDLE")
self.fc.state["cool"]["armed"] = True self.fc.state["cool"]["armed"] = True
@@ -651,6 +727,7 @@ class CloudSuiteTests(unittest.TestCase):
def test_pause_resume_fails_when_the_print_is_cancelled_instead(self): def test_pause_resume_fails_when_the_print_is_cancelled_instead(self):
self.in_cloud(pid=1522) self.in_cloud(pid=1522)
self.hunted(1522)
self.append(["2026-08-17T09:41:00.500000+00:00 gfcloud[1522] INFO websocket:_on_open RX-EVENT: ready"]) self.append(["2026-08-17T09:41:00.500000+00:00 gfcloud[1522] INFO websocket:_on_open RX-EVENT: ready"])
lines = fixture("pause") lines = fixture("pause")
lines = [l.replace('print [1576550507]: finished with event ":completed"', lines = [l.replace('print [1576550507]: finished with event ":completed"',
@@ -665,6 +742,7 @@ class CloudSuiteTests(unittest.TestCase):
"""The second press was seen but the retraced restart never logged """The second press was seen but the retraced restart never logged
(the app's resume is its own line now): the failure names that.""" (the app's resume is its own line now): the failure names that."""
self.in_cloud(pid=1522) self.in_cloud(pid=1522)
self.hunted(1522)
self.append(["2026-08-17T09:41:00.500000+00:00 gfcloud[1522] INFO websocket:_on_open RX-EVENT: ready"]) self.append(["2026-08-17T09:41:00.500000+00:00 gfcloud[1522] INFO websocket:_on_open RX-EVENT: ready"])
lines = [l for l in fixture("pause") if "resuming (laser lead" not in l] lines = [l for l in fixture("pause") if "resuming (laser lead" not in l]
pre, rest = cut(lines, "current state: MachineState.RUNNING") pre, rest = cut(lines, "current state: MachineState.RUNNING")
@@ -676,6 +754,7 @@ class CloudSuiteTests(unittest.TestCase):
def test_pause_resume_fails_when_the_kernel_refuses_the_resume(self): def test_pause_resume_fails_when_the_kernel_refuses_the_resume(self):
self.in_cloud(pid=1522) self.in_cloud(pid=1522)
self.hunted(1522)
self.append(["2026-08-17T09:41:00.500000+00:00 gfcloud[1522] INFO websocket:_on_open RX-EVENT: ready"]) self.append(["2026-08-17T09:41:00.500000+00:00 gfcloud[1522] INFO websocket:_on_open RX-EVENT: ready"])
lines = [l.replace("machine:_resume_retraced resuming (laser lead 1950 ticks)", lines = [l.replace("machine:_resume_retraced resuming (laser lead 1950 ticks)",
"machine:_resume_retraced resume refused ([Errno 22] Invalid argument); cancelling") "machine:_resume_retraced resume refused ([Errno 22] Invalid argument); cancelling")
@@ -706,6 +785,7 @@ class CloudSuiteTests(unittest.TestCase):
def test_pause_resume_fails_without_the_pause_line(self): def test_pause_resume_fails_without_the_pause_line(self):
self.in_cloud(pid=1522) self.in_cloud(pid=1522)
self.hunted(1522)
self.append(["2026-08-17T09:41:00.500000+00:00 gfcloud[1522] INFO websocket:_on_open RX-EVENT: ready"]) self.append(["2026-08-17T09:41:00.500000+00:00 gfcloud[1522] INFO websocket:_on_open RX-EVENT: ready"])
lines = [l for l in fixture("pause") if "button pressed mid-run" not in l] lines = [l for l in fixture("pause") if "button pressed mid-run" not in l]
pre, rest = cut(lines, "current state: MachineState.RUNNING") pre, rest = cut(lines, "current state: MachineState.RUNNING")