Hand the machine back, do not describe what is wrong with it

The hand-back is the promise that a run leaves the machine where it found
it. Two of its checks reported instead of restoring, and the machine sat
in the state the run left it in.

The cooling engine: a run that ends without ending its job leaves one
alive behind it - the engine armed and holding for a job that is never
coming back, the fans at run duty. The check waited two minutes for that
to resolve itself, which it cannot, and wrote "failed: still
run/armed=True/hold=True". On the bench reference the fans then ran for
an hour.

The pulse ring: bytes the last job never played sit there, and the next
run replays them before its own. The check refused the return jog and
wrote "clear the ring (controller restart) before moving" - the
instruction, to a log, instead of the action.

Both end the same way: stop the controller and let the supervisor bring
it back. The job goes, the arm and the hold go with it, and the ring is
empty on the way in. stand_down() does that and proves it settled;
the cooling check calls it when the engine will not idle on its own, and
the return jog calls it when the ring has residue, refusing only if bytes
survive a restart. A leftover still reports what it found - the record of
what the run did is the point - but it reports having fixed it.

Host-proven: 62 baseline unit tests, including one that the hand-back
calls the stand-down for ring residue rather than describing it.

forgetest/forgetest/baseline.py

@@ -456,6 +456,37 @@ class Baseline:
             time.sleep(1.0)
         return None
 
+    def stand_down(self, why):
+        """End whatever the machine is still doing, and prove it ended.
+
+        A run that dies mid-job leaves the job alive behind it: the
+        cooling engine armed and holding for a job nothing will finish,
+        the fans at run duty, and the rest of the program still queued in
+        the pulse ring. Waiting does not end any of that - the engine is
+        holding correctly, for a job that is never coming back.
+
+        Stopping the controller ends it. The supervisor brings the
+        controller back, the arm and the hold go with the job, and the
+        ring is empty on the way in. This is what the ring-residue
+        message has always told an operator to do; the hand-back does it
+        instead of saying it.
+
+        Returns the /mode body the machine settled on, or None."""
+        self.log("baseline: standing the machine down (%s): stopping the controller" % why)
+        st, _b = self.fc_post("/controller/stop")
+        if st != 200:
+            self.log("baseline: stand-down: /controller/stop -> %s" % st)
+        deadline = time.time() + 30
+        while time.time() < deadline and not self.abort():
+            body = self.fc_get("/mode")[1] or {}
+            if body.get("controller") != "running":
+                break
+            time.sleep(1.0)
+        st, _b = self.fc_post("/controller/start")
+        if st != 200:
+            self.log("baseline: stand-down: /controller/start -> %s" % st)
+        return self.wait_settled()
+
     def cloud_mode(self):
         return self.mode == "cloud"
 
@@ -542,11 +573,20 @@ class Baseline:
         if st == 200 and isinstance(c, dict):
             if c.get("phase") != "idle" or c.get("armed") or c.get("hold"):
                 found = "%s/armed=%s/hold=%s" % (c.get("phase"), c.get("armed"), c.get("hold"))
-                w = self._wait("cool idle", lambda: (lambda x: x.get("phase") == "idle" and not x.get("armed")
-                                                    and not x.get("hold"))(self.fc_get("/cool/status")[1] or {}),
-                               COOL_IDLE_S)
+                idle = lambda: (lambda x: x.get("phase") == "idle" and not x.get("armed")     # noqa: E731
+                                and not x.get("hold"))(self.fc_get("/cool/status")[1] or {})
+                w = self._wait("cool idle", idle, COOL_IDLE_S)
+                # The engine holds for a job. A run that ended without
+                # ending its job leaves one alive, and no amount of
+                # waiting ends it: stand the machine down and let the
+                # supervisor bring the controller back clean.
+                act = "waited"
+                if w is None:
+                    self.stand_down("the cooling engine is still %s" % found)
+                    w = self._wait("cool idle", idle, COOL_IDLE_S)
+                    act = "restored (stood the machine down)"
                 left.append(Leftover("cool", found, "idle/unarmed/no hold",
-                                     "waited" if w is not None else "failed: still %s" % found))
+                                     act if w is not None else "failed: still %s" % found))
 
     def _lamp_side(self, left):
         """The lid lamp at forgectrl's idle level (the lid_lamp_idle setting).
@@ -625,10 +665,18 @@ class Baseline:
         # Never jog on top of stale bytes: a run started now would replay
         # whatever the ring still holds before the jog, in a direction and
         # for a distance nobody asked for. Report and leave the head.
+        # Stale bytes in the ring are the last job's, and a jog now would
+        # replay them before its own. The ring empties when the
+        # controller restarts, so the hand-back restarts it rather than
+        # leaving the head where it is and the bytes where they are.
         residue = read_ring_residue()
         if residue:
-            return ("unrestorable: %d unplayed bytes queued in the kernel ring - a jog would "
-                    "replay them; clear the ring (controller restart) before moving" % residue)
+            self.log("baseline: %d unplayed bytes in the kernel ring before the return" % residue)
+            self.stand_down("%d unplayed bytes in the kernel ring" % residue)
+            residue = read_ring_residue()
+            if residue:
+                return ("unrestorable: %d unplayed bytes still queued in the kernel ring after a "
+                        "controller restart - a jog would replay them" % residue)
         # a controller may be inside a respawn backoff (seconds): wait for it
         mode = None
         deadline = time.time() + 30

forgetest/tests/test_baseline.py

@@ -154,16 +154,32 @@ class BaselineTests(unittest.TestCase):
         b = self.bl()
         cap = b.capture()
         # the run left 40 unplayed bytes queued in the kernel ring and the
-        # head 1000 counts out: the residue is reported, and the return jog
-        # is refused (it would replay the residue first)
+        # head 1000 counts out: the hand-back stands the machine down to
+        # empty the ring, and only refuses the return jog when the bytes
+        # are still there afterwards (they are here: no daemon to restart
+        # a controller)
         self._pos_bytes(1000, 0, 0, 100, 140)
         left = b.enforce("post", captured=cap)
         items = {x.item: x for x in left}
         self.assertIn("pulse ring", items)
         self.assertEqual(items["pulse ring"].found, "40 unplayed bytes")
-        self.assertIn("unplayed bytes queued", items["position"].action)
+        self.assertIn("still queued", items["position"].action)
+        self.assertIn("after a controller restart", items["position"].action)
         self.assertEqual(baseline.read_ring_residue(), 40)
 
+    def test_the_hand_back_stands_the_machine_down_for_residue(self):
+        """It acts, it does not report: a run that left bytes in the ring
+        gets a controller restart out of the hand-back, because that is
+        what empties the ring."""
+        b = self.bl()
+        cap = b.capture()
+        called = []
+        b.stand_down = lambda why: called.append(why)
+        self._pos_bytes(1000, 0, 0, 100, 140)
+        b.enforce("post", captured=cap)
+        self.assertEqual(len(called), 1)
+        self.assertIn("unplayed bytes", called[0])
+
     def test_clean_ring_reads_zero_residue(self):
         self.assertEqual(baseline.read_ring_residue(), 0)
 

meta-forgefirm/recipes-forgefirm/forgectrl/forgectrl-pin.inc

@@ -2,5 +2,5 @@
 # only SRCREV and PV here - the image manifest leaves *-pin.inc out of the
 # layer content hash because the component entry already identifies the
 # pinned source (forgefirm-image-manifest.bbclass).
-SRCREV = "6e694798903df6add92ec97876cda4851b739aa0"
-PV = "0.1.15"
+SRCREV = "e58aa8a5ad8b91c75896f0c53add8a72095b01d2"
+PV = "0.1.16"