forgetest: the service's hunt paid only where it is the subject

A cloud client the tool starts for anything but homing comes up under
the /run/gfcloud-nohunt marker: the real client back after the
emulator, a mode the runner switches to or hands back, a controller it
restarts. The service keeps the head position it has. cloud.mode-switch
and cloud.service-protocol keep their hunts, and so does the one real
print: enter_cloud reuses a running session only when that client has
hunted the machine itself (session_hunted: never the emulator's, never
a no-hunt start), otherwise it restarts the client with the hunt, since
a print placed on a head position the service only believes can run the
gantry into a rail. The markers are one start, taken down by the client
that read them first thing; the tool's own removal stays for a start
that never happened. Catalog unchanged; the cloud tests' shared code
moved, so their implementation hashes move with it.
This commit is contained in:
ScottW514
2026-08-23 11:27:36 -04:00
parent 5c7727b9bf
commit 969bac6013
5 changed files with 294 additions and 37 deletions
+39
View File
@@ -78,6 +78,15 @@ GRBL_CONTROLLER_SYSFS = ("cnc/motor_lock", "cnc/x_mode", "cnc/y_mode", "cnc/x_de
# would leave the runtime in one mode and the boot in the other).
UNPRESERVED_SETTINGS = ("controller_mode",)
# A cloud client the baseline starts (the mode a test declares, the mode
# or controller handed back after a run) comes up without the service's
# connect-time hunt: the marker gfcloud reads first thing at start and
# takes down (one start), so it is left in place once the start is
# accepted and removed only when the start never happened. The service
# keeps the head position it has; a test that homes or prints under the
# service starts its own client with the hunt.
NOHUNT_MARKER = os.environ.get("FORGETEST_NOHUNT_MARKER", "/run/gfcloud-nohunt")
# Read-only readbacks with their idle values (no direct restore: the state
# comes right through forgectrl - see restore_forgectrl - or is fatal).
IDLE_READBACKS = [
@@ -260,6 +269,25 @@ class Baseline:
self.log("WARNING - forgectrl did not settle within %d s (last /mode: %s)" % (timeout, last))
return last
# -- a cloud client without the hunt ---------------------------------
def nohunt_on(self, want):
"""The no-hunt marker before a start that brings the cloud client
up (a switch to cloud, a controller start in cloud mode)."""
if want != "cloud":
return
try:
with open(NOHUNT_MARKER, "w") as f:
f.write("forgetest baseline\n")
except OSError as e:
self.log("no-hunt marker not written (%s): the client will hunt" % e)
def nohunt_off(self):
"""The marker the client never read: a start that was refused."""
try:
os.remove(NOHUNT_MARKER)
except OSError:
pass
# -- the mode a test needs -------------------------------------------
def switch_mode(self, want, timeout=SETTLE_S):
"""Put the machine in controller mode `want` through the supervisor
@@ -276,8 +304,10 @@ class Baseline:
return True, "already in %s mode" % want
self.log("switching to %s mode (found %s, controller %s)"
% (want, mode.get("mode"), mode.get("controller")))
self.nohunt_on(want)
st, body = self.fc_post("/mode", data={"controller": want})
if st != 200:
self.nohunt_off()
return False, "POST /mode controller=%s -> %s %s" % (want, st, body)
mode = self.wait_settled(timeout=timeout) or {}
self.mode = mode.get("mode")
@@ -358,19 +388,28 @@ class Baseline:
# declaring it is handed back through the switch.
want = (captured or {}).get("mode") or mode.get("mode") or "grbl"
if mode.get("mode") != want:
self.nohunt_on(want)
st, body = self.fc_post("/mode", data={"controller": want})
if st != 200:
self.nohunt_off()
mode = self.wait_settled() or mode
self.mode = mode.get("mode")
left.append(Leftover("mode", mode.get("mode"), want,
"restored" if mode.get("mode") == want else "failed: %s %s" % (st, body)))
if mode.get("controller") == "motion-fault":
self.nohunt_on(want)
st, body = self.fc_post("/mode", data={"controller": want})
if st != 200:
self.nohunt_off()
mode = self.wait_settled() or mode
left.append(Leftover("controller", "motion-fault", "running",
"restored" if mode.get("controller") == "running"
else "failed: %s" % mode.get("controller")))
elif mode.get("controller") == "standby":
self.nohunt_on(mode.get("mode"))
st, body = self.fc_post("/controller/start")
if st != 200:
self.nohunt_off()
mode = self.wait_settled() or mode
left.append(Leftover("controller", "standby", "running",
"restored" if mode.get("controller") == "running"
+86 -28
View File
@@ -429,6 +429,13 @@ JOB_DIR = "/tmp/forgetest" # the jobs the offline tests write (tmpfs)
EMULATE_MARK = "EMULATE: the emulator"
EMULATE_MARKER = "/run/gfcloud-emulate"
EMULATOR_WORK = "/tmp/gfcloud-emulate"
# No connect-time hunt (gfcloud --no-hunt, or the marker at start): the
# first settings report in the reconnect form, the service keeps the head
# position it has. For a restart between tests that do not home; the
# homing tests and the one real print get their hunt.
NOHUNT_MARK = "NO-HUNT:"
NOHUNT_MARKER = "/run/gfcloud-nohunt"
HUNT_DONE = "hunt ["
WS_MARKS = ("RX-EVENT: ready", "RX-EVENT: closed", "RECONNECTING", "CLOSING", OFFLINE_MARK)
ACTIVITY_MARKS = ("start motion", "start return home", "starting run", "starting z homing cycle")
LOG_TAIL_BYTES = 4 << 20
@@ -596,6 +603,24 @@ def session_live(pid):
return last == "RX-EVENT: ready", "%s: last websocket state %s" % (where, last)
def session_hunted(pid):
"""The running cloud client (gfcloud[pid]) has homed the machine under
the service: a hunt of its own finished (the connect-time one, or a
later re-hunt), and it is not the emulator. A client started without
the hunt, or one whose lines have left the log tail, has not - the
service may hold a head position the machine no longer has."""
tag = "gfcloud[%s]" % pid
hunted = False
for ln in log_tail(GFCLOUD_LOG):
if tag not in ln:
continue
if EMULATE_MARK in ln:
return False
if HUNT_DONE in ln and "finished with event" in ln and COMPLETED in ln:
hunted = True
return hunted
def client_offline(pid):
"""The running cloud client is the offline service."""
live, detail = session_live(pid)
@@ -637,16 +662,20 @@ def wait_session(ctx, offset, timeout=120):
return session
def fresh_cloud_connect(ctx):
"""A NEW cloud client with a fresh service session (its connect-time
hunt follows): in cloud mode the controller is restarted through the
supervisor's stop/start lever; in GRBL mode the mode is switched (and
the change declared - the machine stays in cloud mode). Returns the log
offset from before the connect, so the caller sees the whole session."""
def fresh_cloud_connect(ctx, hunt=True):
"""A NEW cloud client with a fresh service session: in cloud mode the
controller is restarted through the supervisor's stop/start lever; in
GRBL mode the mode is switched (and the change declared - the machine
stays in cloud mode). Its connect-time hunt follows unless hunt is
False (the no-hunt marker, for a start where the service already
knows the head). Returns the log offset from before the connect, so
the caller sees the whole session."""
fc = ctx.forgectrl
st, m = fc.get("/mode")
ctx.check(st == 200 and isinstance(m, dict), "GET /mode -> %s", st)
offset = log_size(GFCLOUD_LOG)
if not hunt:
marker_on(ctx, NOHUNT_MARKER)
if m.get("mode") == "cloud":
ctx.log("cloud mode: restarting the cloud client for a fresh connect (was pid %s)", m.get("pid"))
st, body = fc.post("/controller/stop")
@@ -672,25 +701,35 @@ def fresh_cloud_connect(ctx):
def enter_cloud(ctx):
"""Cloud mode with a live service session and a quiet machine. An
existing cloud session is reused; from GRBL mode the switch is made
once (its connect-time hunt is waited out) and the machine stays in
cloud mode. Returns the log offset where the test's own window begins."""
"""Cloud mode with a live service session, the machine homed under
it, and quiet. An existing cloud session is reused when the client
has hunted the machine itself (never the emulator's session, never a
no-hunt start: the service would be holding a head position the
machine may not have, and what follows is the one real print); from
GRBL mode the switch is made once (its connect-time hunt is waited
out) and the machine stays in cloud mode. Returns the log offset
where the test's own window begins."""
fc = ctx.forgectrl
st, m = fc.get("/mode")
ctx.check(st == 200 and isinstance(m, dict), "GET /mode -> %s", st)
offline_marker_off(ctx)
live = detail = None
hunted = False
if m.get("mode") == "cloud" and m.get("controller") == "running":
live, detail = session_live(m.get("pid"))
ctx.check(live or "offline" in detail,
"cloud mode is up (pid %s) but the client has no live service session (%s) - "
"check credentials and network, or restart the controller", m.get("pid"), detail)
if live:
ctx.log("cloud mode already up (pid %s), service session live - reusing it", m.get("pid"))
hunted = live and session_hunted(m.get("pid"))
if live and hunted:
ctx.log("cloud mode already up (pid %s), service session live and the machine hunted "
"under it - reusing it", m.get("pid"))
offset = log_size(GFCLOUD_LOG)
else:
if detail:
if live:
ctx.log("the running client (pid %s) never hunted the machine under the service: "
"restarting it with the hunt", m.get("pid"))
elif detail:
ctx.log("the running client is the offline service: restarting it with the service")
offset = fresh_cloud_connect(ctx)
hunt = wait_action_finished(ctx, offset, "hunt", HUNT_TIMEOUT_S)
@@ -702,9 +741,20 @@ def enter_cloud(ctx):
return log_size(GFCLOUD_LOG)
def marker_on(ctx, path):
"""A one-start marker for the next cloud client: the client that reads
it takes it down. Written before the start; offline_marker_off is the
belt-and-braces removal once the client's own line is in."""
with open(path, "w") as f:
f.write("forgetest %s\n" % ctx.test.id)
def offline_marker_off(ctx):
"""No client started from here on comes up offline, or as the emulator."""
for path, what in ((OFFLINE_MARKER, "offline"), (EMULATE_MARKER, "emulate")):
"""No client started from here on comes up offline, as the emulator,
or without its hunt. The client takes a marker it read down itself;
this catches one it never read (a start that failed)."""
for path, what in ((OFFLINE_MARKER, "offline"), (EMULATE_MARKER, "emulate"),
(NOHUNT_MARKER, "no-hunt")):
try:
os.remove(path)
ctx.log("%s marker removed", what)
@@ -712,20 +762,23 @@ def offline_marker_off(ctx):
pass
def restart_client(ctx, marker=None, who=""):
def restart_client(ctx, marker=None, who="", hunt=True):
"""The cloud client started fresh - restarted in cloud mode, switched to
from GRBL mode (the change declared) - under `marker` if one is given,
which stays in place until the client has read it (the caller waits
for the client's own line and takes the marker down; the supervisor
reports the client running seconds before Python has finished
importing). Returns the log offset from before the start."""
and under the no-hunt marker when hunt is False. A marker stays in
place until the client has read it (the client takes it down; the
caller waits for the client's own line and takes it down as well if
the start failed - the supervisor reports the client running seconds
before Python has finished importing). Returns the log offset from
before the start."""
fc = ctx.forgectrl
st, m = fc.get("/mode")
ctx.check(st == 200 and isinstance(m, dict), "GET /mode -> %s", st)
offset = log_size(GFCLOUD_LOG)
if marker:
with open(marker, "w") as f:
f.write("forgetest %s\n" % ctx.test.id)
marker_on(ctx, marker)
if not hunt:
marker_on(ctx, NOHUNT_MARKER)
if m.get("mode") == "cloud":
ctx.log("cloud mode: restarting the client%s (was pid %s)", who, m.get("pid"))
st, body = fc.post("/controller/stop")
@@ -766,17 +819,22 @@ def enter_emulator(ctx):
def leave_emulator(ctx):
"""The real client back, its session waited for and its connect-time
hunt waited out, so the machine is what the next test expects."""
"""The real client back without its connect-time hunt (nothing moved
while the emulator had the session, and a test that prints hunts
first regardless), its session waited for, so the machine is what
the next test expects."""
offline_marker_off(ctx)
offset = restart_client(ctx, who=" with the machine")
offset = restart_client(ctx, who=" with the machine, no hunt", hunt=False)
try:
got = wait_log(ctx, offset, [NOHUNT_MARK], 60)
ctx.check(got[NOHUNT_MARK], "the real client did not come up without the hunt within 60 s")
finally:
offline_marker_off(ctx)
session = wait_session(ctx, offset)
ctx.check(session_established(session), "the real client did not establish its session after the emulator")
hunt = wait_action_finished(ctx, offset, "hunt", HUNT_TIMEOUT_S)
ctx.check(hunt, "the real client's connect-time hunt did not finish within %d s", HUNT_TIMEOUT_S)
ctx.check(wait_quiet(ctx, offset), "the service was still moving the head %d s after the real "
"client came back", QUIET_TIMEOUT_S)
ctx.log("the machine is back under the real client")
ctx.log("the machine is back under the real client, no hunt")
def enter_offline(ctx):