diff --git a/forgetest/forgetest/suite/cooling.py b/forgetest/forgetest/suite/cooling.py index b9e2d5b..0e27aab 100644 --- a/forgetest/forgetest/suite/cooling.py +++ b/forgetest/forgetest/suite/cooling.py @@ -83,6 +83,7 @@ SAMPLE_S = 5 # tach sampling period (the big exhaust fan coasts f STABLE_SAMPLES = 3 # consecutive agreeing samples that make an idle reference IDLE_REF_TIMEOUT_S = 180 # a previous test's cooldown + spin-down COOLDOWN_TIMEOUT_S = 240 # the engine's smoke phase + idle drop + spin-down +RESTART_IDLE_TIMEOUT_S = 90 # a daemon restart: the controller's start, its liveness probe, the engine's tick @test("cooling.flow-verify", title="Coolant flow check separates flow from no-flow", @@ -156,15 +157,19 @@ def _tach_stable(a, b): return all(abs(a.get(k, 0) - b.get(k, 0)) <= max(100, 0.10 * max(a.get(k, 0), 1)) for k in TACH_KEYS) -@test("cooling.fans-quiet-after-motion", title="Fan profile returns to idle after motion and after M8/M9", - subsystem="cooling", kind="auto", mode="grbl", est_min=3, +@test("cooling.fans-quiet-after-motion", title="Fan profile returns to idle after motion, after M8/M9, " + "and after a daemon restart on a busy machine", + subsystem="cooling", kind="auto", mode="grbl", est_min=5, covers=_COOL_COVERS + [("forgectrl", "src/super.c")], requires=["motion.pacing"], steps=["Bed clear; the head needs 20 mm of free +X travel."], description="A dry jog and an M8/M9 cycle must not leave the run fan profile on: within " "the cooldown the engine is back at its idle duty and the exhaust/intake tachs " "are back at (or below) the idle level they held before the test. The idle " "reference is taken only once the engine is idle and the tachs have stopped " - "changing, so a previous test's spin-down cannot be mistaken for idle.") + "changing, so a previous test's spin-down cannot be mistaken for idle. Then a " + "forgectrl restart with the kernel not idle (a takeover's safe state): the " + "engine starts in its cooldown posture and must take the idle duties back once " + "the machine is idle, not keep the exhaust at cooldown duty for good.") def fans_quiet(ctx): fc = ctx.forgectrl ev = ctx.evidence @@ -238,6 +243,35 @@ def fans_quiet(ctx): ctx.check(settle is not None, "fans did not return to the idle profile within %d s: %s, duty %s, " "phase %s (idle reference %s)", COOLDOWN_TIMEOUT_S, ev["after"], ev["duty_after"], phase(), before) + # A daemon restart while the kernel is not idle: forgectrl stopped, the + # kernel put in the takeover's safe state (disabled), forgectrl started. + # The engine judges the machine busy and takes the cooldown airflow; the + # controller's start enables the kernel and the machine is idle again, + # and the engine must follow it to the idle duties. + rc, out = hw.initd("forgectrl", "stop") + ctx.log("forgectrl stop -> rc %s", rc) + ctx.check(rc == 0, "forgectrl stop failed: %s", (out or "").strip()[:200]) + hw.sysfs_write("cnc/disable", "1") + ev["kstate_at_restart"] = hw.sysfs_read("cnc/state") + rc, out = hw.initd("forgectrl", "start") + ctx.log("forgectrl start with cnc/state=%s -> rc %s", ev["kstate_at_restart"], rc) + ctx.check(rc == 0, "forgectrl start failed: %s", (out or "").strip()[:200]) + back = None + t0 = time.time() + while time.time() - t0 < RESTART_IDLE_TIMEOUT_S: + ctx.sleep(SAMPLE_S) + st, m = fc.get("/mode") + d = _duties() + ctrl = m.get("controller") if isinstance(m, dict) else st + ctx.log(" after the restart +%3.0f s: controller %s duty %s", time.time() - t0, ctrl, d) + if ctrl == "running" and d == IDLE_DUTY: + back = time.time() - t0 + break + ev["restart_idle_s"] = round(back, 1) if back is not None else None + ev["duty_after_restart"] = _duties() + ctx.check(back is not None, "the engine kept the busy-start airflow after the restart (duty %s " + "after %d s)", ev["duty_after_restart"], RESTART_IDLE_TIMEOUT_S) + GATE_KEYS = ("cool_temp_max", "cool_temp_resume") VERDICT_WAIT_S = 20 # the engine reloads settings at run start and ticks at 1 Hz diff --git a/forgetest/tests/test_cooling_suite.py b/forgetest/tests/test_cooling_suite.py index 028bb44..3b83381 100644 --- a/forgetest/tests/test_cooling_suite.py +++ b/forgetest/tests/test_cooling_suite.py @@ -89,19 +89,38 @@ class FansQuietTests(unittest.TestCase): self.tmp = tempfile.mkdtemp(prefix="forgetest-cool-") self.sysfs = os.path.join(self.tmp, "sysfs") + os.sep os.makedirs(self.sysfs + "thermal") + os.makedirs(self.sysfs + "cnc") + with open(self.sysfs + "cnc/state", "w") as f: + f.write("idle") os.environ["GF_SYSFS_ROOT"] = self.sysfs self.fc = helpers.FakeForgectrl().start() self.grbl = FakeGrbl() - self.saved = (cooling.SAMPLE_S, cooling.IDLE_REF_TIMEOUT_S, cooling.COOLDOWN_TIMEOUT_S) + self.saved = (cooling.SAMPLE_S, cooling.IDLE_REF_TIMEOUT_S, cooling.COOLDOWN_TIMEOUT_S, + cooling.RESTART_IDLE_TIMEOUT_S, cooling.hw.initd) cooling.SAMPLE_S = 0.1 cooling.IDLE_REF_TIMEOUT_S = 3 cooling.COOLDOWN_TIMEOUT_S = 4 + cooling.RESTART_IDLE_TIMEOUT_S = 1 + # The daemon restart of the last phase, scripted: the init script + # is a stub, and the engine that comes up applies `after_start` + # (the idle duties by default; a daemon that keeps the busy-start + # cooldown airflow applies those instead). + self.initd_calls = [] + self.after_start = (0, 0) + + def initd(service, action, timeout=60): + self.initd_calls.append((service, action)) + if action == "start": + self.duty(*self.after_start) + return 0, "" + cooling.hw.initd = initd self.fc.state["cool"] = {"phase": "idle", "armed": False, "hold": False} self.duty(0, 0) self.fans(0, 736, 733) def tearDown(self): - cooling.SAMPLE_S, cooling.IDLE_REF_TIMEOUT_S, cooling.COOLDOWN_TIMEOUT_S = self.saved + (cooling.SAMPLE_S, cooling.IDLE_REF_TIMEOUT_S, cooling.COOLDOWN_TIMEOUT_S, + cooling.RESTART_IDLE_TIMEOUT_S, cooling.hw.initd) = self.saved self.grbl.close() self.fc.stop() os.environ.pop("GF_SYSFS_ROOT", None) @@ -174,6 +193,26 @@ class FansQuietTests(unittest.TestCase): self.assertIsNotNone(run.evidence["settle_s"]) self.assertTrue(any("idle ref:" in ln for ln in run.lines)) + def test_the_restart_phase_stops_disables_starts_and_sees_idle_duty(self): + self.run_engine() + run = self.run_test() + self.assertEqual(run.finished, None) + self.assertEqual(self.initd_calls, [("forgectrl", "stop"), ("forgectrl", "start")]) + with open(self.sysfs + "cnc/disable") as f: + self.assertEqual(f.read(), "1") + self.assertEqual(run.evidence["duty_after_restart"], cooling.IDLE_DUTY) + self.assertIsNotNone(run.evidence["restart_idle_s"]) + + def test_a_daemon_that_keeps_the_busy_start_airflow_fails(self): + # The bench case: after a takeover's restart the exhaust ran at + # cooldown duty on an idle machine for good. + self.run_engine() + self.after_start = (32768, 21639) + with self.assertRaises(Failed) as cm: + self.run_test() + self.assertIn("kept the busy-start airflow", str(cm.exception)) + self.assertIn("32768", str(cm.exception)) + def test_fans_left_running_fail_with_the_reference_in_the_message(self): def on_command(line): if line == "M8":