From 0ceb4abc4a9a613d5ef2a3c989c33a3cb4752613 Mon Sep 17 00:00:00 2001 From: ScottW514 Date: Sat, 19 Sep 2026 17:04:49 -0400 Subject: [PATCH] forgetest: cooling.fail-tier-stop waits for the crash fault to end with its session The test raised a crash fault and handed the machine back with it still standing. The engine trips the crash tier on one 1 Hz tick and ends the run session on its next tick, which is when the fault and its hold clear ("head crash fault cleared with the run session"). The supervisor has the new controller up inside that second, and the test returned as soon as it saw the new pid and the two log lines, so the runner's hand-back check could read /cool/status before the clearing tick: phase run, hold true. A hold is the run's doing by the baseline's rule, so the check recorded a leftover and failed the test, though the wait that followed ended on its own. Seen on the bench reference in campaign c-20260919202934-3d3a, the only failure among 65 results: the trip at 20:54:44.871, the new controller at 20:54:45.094, the test's PASS line at 20:54:45, the session end at 20:54:45.863, and "cool=run/armed=False/hold=True ... -> waited" from the hand-back check. The engine opened no session after that one; the 16 s the check waited were the smoke phase that follows an armed session. The test now asserts what the engine documents: after the restart checks it waits up to 10 s for the verdict to leave CRASH, records how long that took (fault_cleared_s) with the phase, verdict, hold and armed flags it found, and fails when the fault does not end with the session. The machine is then handed back in the engine's own post-job phase, which the baseline already treats as the engine's work and not a leftover. Proven: the cooling, artifact, campaign and queue host tests pass and the catalog loads with the new source. No component source changed, so no pin moves; the test's own source hash does, so its result comes from this revision. --- forgetest/forgetest/suite/cooling.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/forgetest/forgetest/suite/cooling.py b/forgetest/forgetest/suite/cooling.py index d655654..72c71ac 100644 --- a/forgetest/forgetest/suite/cooling.py +++ b/forgetest/forgetest/suite/cooling.py @@ -1442,7 +1442,9 @@ def fan_duty_readback(ctx): "watch's thresholds at their lowest (1) make the head's own move trip the abort " "generator inside an armed, dark (S0) job. Expected: the engine logs the crash " "signal, the supervisor logs the stop and starts a new controller (new pid), the " - "latch is locked, the kernel idle, no emission, and the thresholds are put back.") + "latch is locked, the kernel idle, no emission, the crash fault ends with the run " + "session (the engine's next tick after the controller is gone), and the " + "thresholds are put back.") def fail_tier_stop(ctx): from .laser import prepare, stream, sample from .motion import FORGECTRL_LOG, _log_lines, _log_offset @@ -1501,6 +1503,16 @@ def fail_tier_stop(ctx): ctx.check(ev["result"]["latch_locked"], "latch not locked after the crash tier") ctx.check(not (smp and smp["emission"]), "emission during a dark job: %s", smp and smp["emission"]) ctx.check(fc.wait_idle(30, abort=ctx.aborted), "machine not idle after the restart") + # The crash fault is the run session's: the engine ends the session + # on its next 1 Hz tick after the controller is gone, and the fault + # and its hold go with it. The supervisor has the new controller up + # inside that second, so a test that returned here would hand the + # machine back with the fault it raised still standing. + cleared = ctx.wait_for(lambda: _cool(fc).get("verdict") != "CRASH", 10) + ev["result"]["fault_cleared_s"] = cleared + ctx.log("the crash fault ended with the run session after %s s: %s", cleared, + {k: _cool(fc).get(k) for k in ("phase", "verdict", "hold", "armed")}) + ctx.check(cleared is not None, "the crash fault did not end with the run session: %s", _cool(fc)) finally: fc.wait_idle(30, abort=ctx.aborted) st, body = fc.post("/settings", params=orig)