Audit follow-through: runbook, bench tools, recipes, release tooling

BRINGUP describes the present: the 54-test catalog and its seven-test
always core, the tier counts, the shipped low-temperature gates, the
density floor ($35 = 10), the two local core commits, the ffboot env
write, the aa-offset route, the current bench image, and the bench
measurements the audit asks for (pooled into the next session). The
workstation shell notes and every em dash are gone.

forgetest: the takeover waits for the cloud client too (found by its
command line); the unauthenticated /boot probe names the endpoint's
parameter; the UI prose is American English. Recipes: forgetest
fetches its package directory and init script only and drops
__pycache__ at unpack; the dev image no longer re-adds forgectrl; the
release image's remove list drops the gfui-client the BSP no longer
has; the platform identity strips the kernel's local-version hash
from the modules directory name, so a re-patched kernel keeps its
fingerprints. grblhal restart is stop then start. release.sh --dev
packs the dev image. fixture.sh refuses a readable env file.

Bench tools: the live-fire drills measure the lid-IR baseline before
every run and point at the fire-watch thresholds the engine reads;
one thermistor conversion (gfbench.degc) serves every drill; the six
dated measurement records leave the tool directory; feeder.c names the
two sysfs writes its caller makes.

Host tests: forgetest 258 pass; the coverage lint reports no uncovered
path across 54 tests. Acceptance: forgectrl.auth covers the /boot
probe; update.* cover ffboot and the manifest identity; the runbook
and bench-tool changes have no catalog consequence.
This commit is contained in:
ScottW514
2026-09-02 09:51:23 -04:00
parent 6002da8d12
commit 88ec984e28
29 changed files with 265 additions and 5296 deletions
+19
View File
@@ -337,6 +337,25 @@ def pidof(comm):
return out
def pgrep_f(needle):
"""PIDs whose /proc/<pid>/cmdline contains needle (a script run by an
interpreter has the interpreter's comm, so pidof cannot see it)."""
out = []
try:
for pid in os.listdir("/proc"):
if not pid.isdigit():
continue
try:
with open("/proc/%s/cmdline" % pid, "rb") as f:
if needle.encode() in f.read():
out.append(int(pid))
except OSError:
pass
except OSError:
pass
return out
def run(cmd, timeout=60):
"""Run a command list; returns (rc, combined output)."""
try:
+6 -2
View File
@@ -500,9 +500,13 @@ class Takeover:
rc, out = hw.initd("forgectrl", "stop")
log("takeover: forgectrl stop -> rc %s" % rc)
deadline = time.time() + 15
while time.time() < deadline and (hw.pidof("forgectrl") or hw.pidof("grblHAL_glowfor")):
# Every holder of the pulse device: the daemon, the GRBL controller,
# and the cloud client (a script, so found by its command line).
def holders():
return hw.pidof("forgectrl") + hw.pidof("grblHAL_glowfor") + hw.pgrep_f("gfcloud.py")
while time.time() < deadline and holders():
time.sleep(0.5)
left = hw.pidof("forgectrl") + hw.pidof("grblHAL_glowfor")
left = holders()
if left:
self.__exit__(None, None, None)
raise Failed("takeover: processes still alive after stop: %s" % left)
+11 -11
View File
@@ -1215,7 +1215,7 @@ def lid_interlock_abort_body(ctx, ev, off, job, offset):
"The job is longer than the ring holds, so the ring is filled before the button "
"lights; that takes a minute. When the button lights white, do NOT press it - open "
"the lid, and close it when told. Nothing moves and nothing fires."],
description="A cloud print waiting for the button is cancelled by the lid: the wait ends "
description="A cloud print waiting for the button is canceled by the lid: the wait ends "
"with the lid named as the reason, the laser latch relocks, the armed window "
"closes, no run starts, and the job ends ':cancelled'. The job is longer than "
"the ring, so its feeder is alive through the wait with the rest of the print "
@@ -1285,7 +1285,7 @@ def lid_during_button_wait_body(ctx, ev, off, job, offset):
ctx.act("lid", "close")
settle_cloud(ctx, offset)
ev["events"] = offline_events(off)
ctx.log("PASS: lid open at the button prompt cancelled the print; latch locked, armed=false, "
ctx.log("PASS: lid open at the button prompt canceled the print; latch locked, armed=false, "
"button dark, ring empty with the feed stopped")
@@ -1422,7 +1422,7 @@ def pause_resume(ctx):
relocked = [ln for ln in log_lines_since(GFCLOUD_LOG, offset)
if "relocking the laser" in ln or ("print [" in ln and CANCELLED in ln)]
ev["relock_or_cancel_lines"] = len(relocked)
ctx.check(not relocked, "the pause relocked or cancelled the job (%s)", relocked[:2])
ctx.check(not relocked, "the pause relocked or canceled the job (%s)", relocked[:2])
# The job's lifecycle, from the same print: a warm-up before the first
# fire and a rest after the park are equipment protection the service
@@ -1587,8 +1587,8 @@ def oversize_stream_body(ctx, ev, off, job, offset, before):
# or interlock abort - stop, park back to the job start, relock,
# disarm, ':cancelled' - judged in full.
off.cancel(9004)
ctx.log("cancelled the print as the app would")
svc_stop = "action cancelled mid-run; stopping motion"
ctx.log("canceled the print as the app would")
svc_stop = "action canceled mid-run; stopping motion"
got = wait_log(ctx, offset, [svc_stop, "start return home", "return home complete"], 300)
fin = wait_action_finished(ctx, offset, "print", 60)
ev["service_cancel"] = {k: message(v) for k, v in got.items()}
@@ -1596,7 +1596,7 @@ def oversize_stream_body(ctx, ev, off, job, offset, before):
for k, v in ev["service_cancel"].items():
ctx.log(" [cancel] %s: %s", k, "seen" if v else "MISSING")
ctx.check(got[svc_stop], "the app's cancel did not stop the run")
ctx.check(got["return home complete"], "the cancelled print did not park to completion")
ctx.check(got["return home complete"], "the canceled print did not park to completion")
judge_abort_tail(ctx, ev, offset, "app", fin)
ctx.check(hw.sysfs_int("cnc/streaming", 0) == 0,
"the device was left in live-feed mode after the job ended")
@@ -1610,7 +1610,7 @@ def oversize_stream_body(ctx, ev, off, job, offset, before):
"stopped it, parked, relocked and reported ':cancelled'")
@test("cloud.paused-lid-cancel", title="A paused cloud print is cancelled by the lid",
@test("cloud.paused-lid-cancel", title="A paused cloud print is canceled by the lid",
subsystem="cloud", kind="live", est_min=6,
covers=_MACHINE_RUN, requires=["cloud.pause-resume", "cloud.lid-interlock-abort"],
actions=["button", "lid"],
@@ -1618,7 +1618,7 @@ def oversize_stream_body(ctx, ev, off, job, offset, before):
"The head needs 40 mm of free +X and +Y travel.",
"Press the button to start; when asked, press it again (pause), then open the lid and "
"leave it open until the head is back; close it when told."],
description="A job paused on the button is cancelled by the lid, from the state the factory "
description="A job paused on the button is canceled by the lid, from the state the factory "
"ends it in - there is no resume past a lid open. The same tail as every abort: "
"the motion stops, the head parks back at the job start, the latch relocks and "
"the armed window closes, the button goes dark, and the print ends ':cancelled'. "
@@ -1666,12 +1666,12 @@ def paused_lid_cancel_body(ctx, ev, off, job, offset):
st, cs = ctx.forgectrl.get("/cool/status")
ev["armed_after"] = cs.get("armed") if isinstance(cs, dict) else None
ev["latch_locked_after"] = latch_locked()
ctx.check(not ev["armed_after"], "armed window still open after the paused print was cancelled")
ctx.check(not ev["armed_after"], "armed window still open after the paused print was canceled")
ctx.check(ev["latch_locked_after"],
"kernel latch not locked after the paused print was cancelled")
"kernel latch not locked after the paused print was canceled")
ev["button_dark"] = hw.button_lit()
ctx.check(ev["button_dark"] is False, "the button is still lit after the cancel (%s)", ev["button_dark"])
ctx.act("lid", "close")
settle_cloud(ctx, offset)
ev["events"] = offline_events(off)
ctx.log("PASS: a paused print cancelled by the lid stopped, parked, relocked and reported ':cancelled'")
ctx.log("PASS: a paused print canceled by the lid stopped, parked, relocked and reported ':cancelled'")
+1 -1
View File
@@ -45,7 +45,7 @@ def auth(ctx):
for path, params in (("/controller/stop", None), ("/controller/start", None),
("/mode", {"controller": "grbl"}), ("/settings", {"ui_units": "mm"}),
("/diag/flow-verify", None), ("/diag/abort", None),
("/update/apply", None), ("/boot", {"slot": "a"}),
("/update/apply", None), ("/boot", {"target": "a"}),
("/system/reboot", None), ("/restore/factory", None)):
st, body = fc.post(path, params=params, auth=False)
ctx.log("POST %s (no token) -> %s %s", path, st, body if isinstance(body, dict) else "")
+2 -2
View File
@@ -715,7 +715,7 @@ def arm_wait_lid(ctx):
ev["messages"] = [ln for ln in text.splitlines() if ln.startswith("[MSG:") or ln.startswith("ALARM")]
ctx.log("controller: %s", ev["messages"])
ctx.check("lid opened during arm - job canceled" in text,
"the lid open was not reported as cancelling the arm")
"the lid open was not reported as canceling the arm")
ctx.check("help]" in text, "no reset banner after the lid-open cancel (it must be a clean cancel)")
ctx.check("ALARM" not in text, "an alarm was raised on the lid-open cancel")
# Armed window closed and the kernel latch locked.
@@ -869,7 +869,7 @@ def pause_resume_lid_cancel(ctx):
ctx.check(zero_at is not None and zero_at < 3.0,
"emission did not stop after the lid opened (first 0 at %s)", zero_at)
ctx.check(tail_zero, "emission returned after the lid opened")
ctx.check("lid opened - job canceled" in text, "the lid open was not reported as cancelling the job")
ctx.check("lid opened - job canceled" in text, "the lid open was not reported as canceling the job")
ctx.check("help]" in text, "no reset banner after the cancel")
ctx.check("ALARM" not in text, "an alarm was raised on the cancel (position should be kept)")
t3 = time.time()
+3 -3
View File
@@ -12,7 +12,7 @@
* only updated in place. A poll that rebuilt them would swallow the click
* it landed on: the button that took the mousedown would be gone before
* the mouseup, so no click event would ever be raised. Every action also
* greys its control out on the press and pulls the next poll forward, so
* grays its control out on the press and pulls the next poll forward, so
* the page answers the operator rather than the timer. The help popovers
* (help.js) sit on static elements only, so no rebuild ever orphans one.
*
@@ -56,7 +56,7 @@ function schedule(ms){if(pollTimer)window.clearTimeout(pollTimer);pollTimer=wind
function kick(){schedule(120)}
function poll(){if(polling){schedule(150);return}polling=true;
/* While an action is still in flight the state is asked for in full:
a 304 would skip the render that releases the greyed-out buttons. */
a 304 would skip the render that releases the grayed-out buttons. */
var h=(stateEtag&&!pending)?{'If-None-Match':stateEtag}:null;
api('GET','/state',null,function(s,d,x){polling=false;
if(s===200){stateEtag=x.getResponseHeader('ETag');state=d;
@@ -72,7 +72,7 @@ function setIgnoreReq(on){ignoreReq=!!on;try{window.localStorage.setItem('forget
$('ignreq').checked=ignoreReq;setHtml($('ignreqon'),ignoreReq?"<span class='on'>ON - prerequisites are not enforced</span>":'');
if(state&&catalog)renderGroups()}
/* A start already sent but not yet seen in the state counts as busy, so
the buttons grey out on the click rather than on the next poll. The
the buttons gray out on the click rather than on the next poll. The
window is capped in case the answer never arrives. A queue holds the
machine between its tests as well as during them. */
function batchActive(){return !!(state&&state.batch&&!state.batch.finished)}
+4 -4
View File
@@ -757,7 +757,7 @@ class CloudSuiteTests(unittest.TestCase):
self.hunted(1522)
self.append(["2026-08-17T09:41:00.500000+00:00 gfcloud[1522] INFO websocket:_on_open RX-EVENT: ready"])
lines = [l.replace("machine:_resume_retraced resuming (laser lead 1950 ticks)",
"machine:_resume_retraced resume refused ([Errno 22] Invalid argument); cancelling")
"machine:_resume_retraced resume refused ([Errno 22] Invalid argument); canceling")
for l in fixture("pause")]
pre, rest = cut(lines, "current state: MachineState.RUNNING")
pre, rest = pre + [rest[0]], rest[1:]
@@ -972,7 +972,7 @@ class CloudSuiteTests(unittest.TestCase):
hooks = {"press it. The print then waits": lambda: self.append(wait + run[:1], delay=0.05)}
self.assertFails(cloud.verdict_hold, "the run started under the warm-up hold", hooks=hooks)
# -- a paused print cancelled by the lid, a running one by the app --------
# -- a paused print canceled by the lid, a running one by the app --------
def cancel_parts(self):
"""(print prologue, the pause lines, the lid stop + park + cancel,
the same tail with the app's cancel as the trigger - what
@@ -986,7 +986,7 @@ class CloudSuiteTests(unittest.TestCase):
"2026-08-17T09:42:26.100000+00:00 gfcloud[1522] INFO machine:_run_loop "
"paused at Position(x=41.2, y=17.0, z=0.0)"]
app_cancel = [l.replace("lid opened mid-run; stopping motion",
"action cancelled mid-run; stopping motion") for l in rest]
"action canceled mid-run; stopping motion") for l in rest]
return pre, paused, rest, app_cancel
def test_paused_lid_cancel_on_the_bench_excerpt(self):
@@ -1009,7 +1009,7 @@ class CloudSuiteTests(unittest.TestCase):
self.assertFalse(ev["armed_after"])
self.assertFalse(ev["button_dark"])
self.assertEqual(self.fc.posts, [])
self.assertTrue(any("PASS: a paused print cancelled by the lid" in l for l in run.lines), run.lines)
self.assertTrue(any("PASS: a paused print canceled by the lid" in l for l in run.lines), run.lines)
def test_paused_lid_cancel_fails_when_the_paused_print_resumes_instead(self):
# a lid that resumed (or was ignored) leaves the print ':completed'