mirror of
https://github.com/openglow-org/forgefirm.git
synced 2026-09-28 09:11:11 -07:00
forgetest: witness the airflow behind the beam; harnesses carry the armed flag
The cooling verdict now carries the engine's own armed flag, so the stand-in engines in both null-sink harnesses publish it. The lifecycle harness gains two cases: an engine that never takes the armed window must produce a refused arm and no emission, and one that takes it a couple of seconds late must produce a wait and then a normal arm. The late case is the one that proves the controller keeps reading the verdict while it is blocked in the arm; without that every job would fail there. The emission witness gains the bench form of the same rule: no sample may show the laser firing while the cooling engine reports a phase that runs the fans at their idle duty. That is what a burn with no airflow looks like from the outside, and nothing in the catalog looked for it.
This commit is contained in:
@@ -90,6 +90,7 @@ def sample(ctx):
|
|||||||
"ir": st.get("lid_ir"),
|
"ir": st.get("lid_ir"),
|
||||||
"homed": st.get("homed"),
|
"homed": st.get("homed"),
|
||||||
"armed": cs.get("armed"),
|
"armed": cs.get("armed"),
|
||||||
|
"phase": cs.get("phase"),
|
||||||
"fire_watch": cs.get("fire_watch"),
|
"fire_watch": cs.get("fire_watch"),
|
||||||
"verdict": cs.get("verdict"),
|
"verdict": cs.get("verdict"),
|
||||||
"beam": hw.sysfs_int("head/beam_detect_analog"),
|
"beam": hw.sysfs_int("head/beam_detect_analog"),
|
||||||
@@ -110,8 +111,13 @@ def sample(ctx):
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
TRAIL_FIELDS = ("t", "gstate", "kstate", "emission", "armed", "il", "button_latch", "locked",
|
TRAIL_FIELDS = ("t", "gstate", "kstate", "emission", "armed", "phase", "il", "button_latch",
|
||||||
"lid", "button", "hv_enable", "beam", "hv", "msgs")
|
"locked", "lid", "button", "hv_enable", "beam", "hv", "msgs")
|
||||||
|
|
||||||
|
# The cooling phases that run the fans at their idle duty: the engine has
|
||||||
|
# no run session, so no cut airflow and no flow interrogation. Emission
|
||||||
|
# in one of these is the beam on the work with idle airflow.
|
||||||
|
IDLE_AIRFLOW_PHASES = ("idle", "warm-up")
|
||||||
|
|
||||||
|
|
||||||
def trail(samples):
|
def trail(samples):
|
||||||
@@ -465,9 +471,17 @@ def emission_witness(ctx):
|
|||||||
for i in range(4):
|
for i in range(4):
|
||||||
ir_peak[i] = max(ir_peak[i], s["ir"][i])
|
ir_peak[i] = max(ir_peak[i], s["ir"][i])
|
||||||
armed_seen = any(s["armed"] for s in samples)
|
armed_seen = any(s["armed"] for s in samples)
|
||||||
|
# The beam may only come on inside a run session. Until the engine
|
||||||
|
# has taken the armed window it is still holding the fans at their
|
||||||
|
# idle duty, and the verdict standing on file is the one it
|
||||||
|
# computed for the idle session before the arm.
|
||||||
|
idle_fire = [s for s in samples
|
||||||
|
if s["emission"] and s["phase"] in IDLE_AIRFLOW_PHASES]
|
||||||
ev.update({"samples": len(samples), "emission_peak": peak, "emission_end": end,
|
ev.update({"samples": len(samples), "emission_peak": peak, "emission_end": end,
|
||||||
"hv_min": min(hv) if hv else None, "hv_max": max(hv) if hv else None,
|
"hv_min": min(hv) if hv else None, "hv_max": max(hv) if hv else None,
|
||||||
"lid_ir_peak": ir_peak, "armed_seen": armed_seen,
|
"lid_ir_peak": ir_peak, "armed_seen": armed_seen,
|
||||||
|
"idle_airflow_fire": [(round(s["t"] - samples[0]["t"], 2),
|
||||||
|
s["phase"], s["emission"]) for s in idle_fire],
|
||||||
"pgood_peak": max((s["pgood"] for s in samples if s["pgood"] is not None), default=None)})
|
"pgood_peak": max((s["pgood"] for s in samples if s["pgood"] is not None), default=None)})
|
||||||
ctx.log("emission_samples peak=%s end=%s; hv %s..%s; lid_ir peak %s; armed seen %s",
|
ctx.log("emission_samples peak=%s end=%s; hv %s..%s; lid_ir peak %s; armed seen %s",
|
||||||
peak, end, ev["hv_min"], ev["hv_max"], ir_peak, armed_seen)
|
peak, end, ev["hv_min"], ev["hv_max"], ir_peak, armed_seen)
|
||||||
@@ -488,6 +502,11 @@ def emission_witness(ctx):
|
|||||||
ctx.log("time-to-disarm after Idle: %s s", ev["disarm_after_idle_s"])
|
ctx.log("time-to-disarm after Idle: %s s", ev["disarm_after_idle_s"])
|
||||||
ctx.check(armed_seen, "the armed window was never observed (arm refused, or no button press)")
|
ctx.check(armed_seen, "the armed window was never observed (arm refused, or no button press)")
|
||||||
ctx.check(peak > 0, "no emission witnessed (emission_samples stayed 0)")
|
ctx.check(peak > 0, "no emission witnessed (emission_samples stayed 0)")
|
||||||
|
first_idle_fire = ev["idle_airflow_fire"][0] if idle_fire else (None, None, None)
|
||||||
|
ctx.check(not idle_fire,
|
||||||
|
"the laser fired while the cooling engine had no run session, so the fans were at "
|
||||||
|
"their idle duty (phase %s at t=%s s, emission %s)",
|
||||||
|
first_idle_fire[1], first_idle_fire[0], first_idle_fire[2])
|
||||||
ctx.check(end == 0, "emission_samples did not return to 0 at Idle (%s)", end)
|
ctx.check(end == 0, "emission_samples did not return to 0 at Idle (%s)", end)
|
||||||
ctx.check(hv and max(hv) > min(hv), "HV current did not rise during the burn (%s..%s)",
|
ctx.check(hv and max(hv) > min(hv), "HV current did not rise during the burn (%s..%s)",
|
||||||
ev["hv_min"], ev["hv_max"])
|
ev["hv_min"], ev["hv_max"])
|
||||||
|
|||||||
@@ -141,12 +141,23 @@ def wait_idle(sock, log):
|
|||||||
fail("controller never returned to Idle")
|
fail("controller never returned to Idle")
|
||||||
|
|
||||||
|
|
||||||
def publish_verdicts(path, stop, fire_ok):
|
def publish_verdicts(path, stop, fire_ok, armed_ack=True, ack_after_s=0.0):
|
||||||
|
"""Stand in for the cooling engine. "armed" is the engine's
|
||||||
|
acknowledgment that it has taken the controller's armed window and
|
||||||
|
applied the run airflow; the controller refuses to fire on a verdict
|
||||||
|
that lacks it, so an engine that never acknowledges (armed_ack
|
||||||
|
False) must produce a refused arm and no emission. ack_after_s
|
||||||
|
withholds the acknowledgment for that long first, which is the real
|
||||||
|
engine's case: it answers on its next tick, and the controller has
|
||||||
|
to see the refreshed verdict to get past the arm."""
|
||||||
|
t0 = time.monotonic()
|
||||||
while not stop.is_set():
|
while not stop.is_set():
|
||||||
|
acked = armed_ack and time.monotonic() - t0 >= ack_after_s
|
||||||
body = ('{"ts_mono":%.3f,"fire_ok":%s,"hold":false,'
|
body = ('{"ts_mono":%.3f,"fire_ok":%s,"hold":false,'
|
||||||
'"resume_ok":true,"reason":""}'
|
'"resume_ok":true,"armed":%s,"reason":""}'
|
||||||
% (time.clock_gettime(time.CLOCK_MONOTONIC),
|
% (time.clock_gettime(time.CLOCK_MONOTONIC),
|
||||||
"true" if fire_ok else "false"))
|
"true" if fire_ok else "false",
|
||||||
|
"true" if acked else "false"))
|
||||||
tmp = path + ".tmp"
|
tmp = path + ".tmp"
|
||||||
with open(tmp, "w") as f:
|
with open(tmp, "w") as f:
|
||||||
f.write(body)
|
f.write(body)
|
||||||
@@ -157,7 +168,8 @@ def publish_verdicts(path, stop, fire_ok):
|
|||||||
class Session:
|
class Session:
|
||||||
"""One controller process with the lifecycle overrides applied."""
|
"""One controller process with the lifecycle overrides applied."""
|
||||||
|
|
||||||
def __init__(self, name, fire_ok=True, disarm_s=2, switches=None, conf_extra=""):
|
def __init__(self, name, fire_ok=True, disarm_s=2, switches=None, conf_extra="",
|
||||||
|
armed_ack=True, ack_after_s=0.0):
|
||||||
self.name = name
|
self.name = name
|
||||||
self.workdir = tempfile.mkdtemp(prefix="laser-lifecycle-")
|
self.workdir = tempfile.mkdtemp(prefix="laser-lifecycle-")
|
||||||
conf = os.path.join(self.workdir, "forgefirm.conf")
|
conf = os.path.join(self.workdir, "forgefirm.conf")
|
||||||
@@ -175,7 +187,7 @@ class Session:
|
|||||||
env["GF_SWITCH_FILE"] = self.switch_file
|
env["GF_SWITCH_FILE"] = self.switch_file
|
||||||
self.stop = threading.Event()
|
self.stop = threading.Event()
|
||||||
self.pub = threading.Thread(target=publish_verdicts,
|
self.pub = threading.Thread(target=publish_verdicts,
|
||||||
args=(verdict, self.stop, fire_ok),
|
args=(verdict, self.stop, fire_ok, armed_ack, ack_after_s),
|
||||||
daemon=True)
|
daemon=True)
|
||||||
self.pub.start()
|
self.pub.start()
|
||||||
self.proc = subprocess.Popen([BIN, "-p", str(PORT)],
|
self.proc = subprocess.Popen([BIN, "-p", str(PORT)],
|
||||||
@@ -622,6 +634,66 @@ def test_verdict_blocks_arm():
|
|||||||
s.close()
|
s.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_arm_waits_for_engine_ack():
|
||||||
|
"""Rule 5: the arm does not fire on a verdict computed before it.
|
||||||
|
|
||||||
|
The armed window is reported to the cooling engine when it opens,
|
||||||
|
and the engine answers by applying the cut airflow and the flow
|
||||||
|
interrogation for the job. The verdict standing on file until that
|
||||||
|
answer arrives is the one computed for the idle session that
|
||||||
|
preceded the arm, and at idle nothing is wrong, so it says fire is
|
||||||
|
fine. Firing on it puts the beam on the work with the fans still at
|
||||||
|
their idle duty - observed on the bench as a burn of a few
|
||||||
|
millimeters before the engine's first tick took the job into a hold.
|
||||||
|
|
||||||
|
Here the engine never acknowledges: fire_ok stays true throughout,
|
||||||
|
so the only thing that can refuse the job is the missing
|
||||||
|
acknowledgment. The arm must be refused and no window may open."""
|
||||||
|
s = Session("arm-ack", fire_ok=True, armed_ack=False)
|
||||||
|
try:
|
||||||
|
# The arm blocks in the acknowledgment wait, so the ok for these
|
||||||
|
# lines only arrives once it gives up: send without waiting.
|
||||||
|
s.send_raw("M4 S100")
|
||||||
|
s.send_raw("G1 X1 F600")
|
||||||
|
if not wait_for(s.log, BLOCKED, 15, s.sock):
|
||||||
|
fail("[arm-ack] a job whose cooling engine never took it was not "
|
||||||
|
"refused (the arm fired on the pre-arm verdict)")
|
||||||
|
if ARMED in "".join(s.log):
|
||||||
|
fail("[arm-ack] the armed window opened without the engine's "
|
||||||
|
"acknowledgment")
|
||||||
|
print("PASS [arm-ack]: the arm refused a verdict that predates it")
|
||||||
|
finally:
|
||||||
|
s.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_arm_proceeds_on_a_late_ack():
|
||||||
|
"""Rule 5: the acknowledgment arriving late is the ordinary case.
|
||||||
|
|
||||||
|
The real engine answers on its next tick, so the arm has to observe
|
||||||
|
a verdict refreshed under it and then go on to open the window. This
|
||||||
|
is the case that proves the wait is a wait and not a refusal: the
|
||||||
|
controller must keep polling the verdict file while it is blocked in
|
||||||
|
the arm, or every job on the machine would fail here."""
|
||||||
|
ack_s = 2.0
|
||||||
|
s = Session("arm-ack-late", fire_ok=True, ack_after_s=ack_s)
|
||||||
|
try:
|
||||||
|
t0 = time.time()
|
||||||
|
s.send_raw("M4 S100")
|
||||||
|
s.send_raw("G1 X1 F600")
|
||||||
|
if not wait_for(s.log, ARMED, 15, s.sock):
|
||||||
|
fail("[arm-ack-late] the arm never completed once the engine took "
|
||||||
|
"the job (the verdict is not refreshed inside the wait)")
|
||||||
|
dt = time.time() - t0
|
||||||
|
if dt < ack_s - 0.5:
|
||||||
|
fail("[arm-ack-late] armed after %.1f s, before the engine took the "
|
||||||
|
"job at %.1f s" % (dt, ack_s))
|
||||||
|
if BLOCKED in "".join(s.log):
|
||||||
|
fail("[arm-ack-late] a late acknowledgment was reported as a refusal")
|
||||||
|
print("PASS [arm-ack-late]: the arm waited %.1f s for the engine, then armed" % dt)
|
||||||
|
finally:
|
||||||
|
s.close()
|
||||||
|
|
||||||
|
|
||||||
def test_sigterm_mid_job():
|
def test_sigterm_mid_job():
|
||||||
"""Rule 5: a termination signal during an armed job is a STOP, not a
|
"""Rule 5: a termination signal during an armed job is a STOP, not a
|
||||||
"finish the job first". The supervisor's expected-stop path (mode
|
"finish the job first". The supervisor's expected-stop path (mode
|
||||||
@@ -931,6 +1003,8 @@ def main():
|
|||||||
test_pause_then_lid_cancel_returns_to_the_job_start()
|
test_pause_then_lid_cancel_returns_to_the_job_start()
|
||||||
test_lid_policy_hold()
|
test_lid_policy_hold()
|
||||||
test_verdict_blocks_arm()
|
test_verdict_blocks_arm()
|
||||||
|
test_arm_waits_for_engine_ack()
|
||||||
|
test_arm_proceeds_on_a_late_ack()
|
||||||
test_sigterm_mid_job()
|
test_sigterm_mid_job()
|
||||||
print("PASS: the armed-window lifecycle holds")
|
print("PASS: the armed-window lifecycle holds")
|
||||||
|
|
||||||
|
|||||||
@@ -375,10 +375,13 @@ def wait_state(sock, log, prefix, timeout=5.0):
|
|||||||
def publish_verdicts(path, stop):
|
def publish_verdicts(path, stop):
|
||||||
"""Publish a fresh, clean cooling verdict every 0.5 s (the arm flow
|
"""Publish a fresh, clean cooling verdict every 0.5 s (the arm flow
|
||||||
refuses without one; freshness window is 2 s). Same-host monotonic
|
refuses without one; freshness window is 2 s). Same-host monotonic
|
||||||
clock, atomic rename so the reader never sees a torn file."""
|
clock, atomic rename so the reader never sees a torn file. "armed"
|
||||||
|
is the engine's acknowledgment that it has taken the controller's
|
||||||
|
armed window; the arm waits for it, so a stand-in engine that means
|
||||||
|
to let jobs run must assert it."""
|
||||||
while not stop.is_set():
|
while not stop.is_set():
|
||||||
body = ('{"ts_mono":%.3f,"fire_ok":true,"hold":false,'
|
body = ('{"ts_mono":%.3f,"fire_ok":true,"hold":false,'
|
||||||
'"resume_ok":true,"reason":""}'
|
'"resume_ok":true,"armed":true,"reason":""}'
|
||||||
% time.clock_gettime(time.CLOCK_MONOTONIC))
|
% time.clock_gettime(time.CLOCK_MONOTONIC))
|
||||||
tmp = path + ".tmp"
|
tmp = path + ".tmp"
|
||||||
with open(tmp, "w") as f:
|
with open(tmp, "w") as f:
|
||||||
|
|||||||
Reference in New Issue
Block a user