From 7605a9094635c09004a7aa8d1ebed3b208627695 Mon Sep 17 00:00:00 2001 From: ScottW514 Date: Wed, 2 Sep 2026 17:02:33 -0400 Subject: [PATCH] forgetest: the update drill reads the daemon's reply, and a queue start waits for the fixture probe update.slots-and-signature's apply section required 200 from POST /update/apply, where the daemon answers 202 with started, like every job endpoint, so its first bench run on image 20260902144848 ended before the job did; the cleanup then deleted the staged archive under the running job. The drill requires 202 and started, and looks for the daemon's refusal ("archive is not signed with the ForgeFIRM release key"). Bench: the apply started, the job ended with that refusal, PASS. A queue started 4 s after a forgetest restart ran 7 tests instead of 10. The bench page's /state poll had a fixture probe in flight (an mDNS answer), probe_fixture stamped its time at its start, and the queue start read the stale fixture, none, so the three operator tests the fixture runs in the unattended queue were routed to nobody. The probe now runs under a lock and is stamped when it completes: a caller that arrives during a probe waits for its answer. tests/test_fixture.py holds the race with a slow scripted probe; it fails on the old code. Catalog consequence: the update implementation hash moves; the runner change is dev-only. --- forgetest/forgetest/runner.py | 43 ++++++++++++++++------------- forgetest/forgetest/suite/update.py | 3 +- forgetest/tests/test_fixture.py | 20 ++++++++++++++ 3 files changed, 46 insertions(+), 20 deletions(-) diff --git a/forgetest/forgetest/runner.py b/forgetest/forgetest/runner.py index 7710731..c3f1248 100644 --- a/forgetest/forgetest/runner.py +++ b/forgetest/forgetest/runner.py @@ -544,6 +544,7 @@ class Runner: self.batch = None self.fixture = None # the bench actuator, when one is up (fixture.py) self._fixture_probed = 0.0 + self._fixture_lock = threading.Lock() self._fixture_said = None self.boot_ref = None self.recover() @@ -568,25 +569,29 @@ class Runner: """The fixture, up and answering, or None; re-probed at most every FIXTURE_PROBE_S unless forced (before a run). A probe that finds it gone, or a config that appeared, changes the queues' routing - from then on.""" - now = time.time() - if not force and now - self._fixture_probed < FIXTURE_PROBE_S: - return self.fixture - self._fixture_probed = now - had = self.fixture - # a box that is not there is said once, not every probe - said = [] - fx = _fixture.probe(said.append) - for m in said: - if m != self._fixture_said: - self._note(m) - self._fixture_said = m - if fx is not None: - self._fixture_said = None - if fx is None and had is not None: - self._note("fixture: %s no longer answers - running without it" % had.hostname) - self.fixture = fx - return fx + from then on. One probe at a time, and the probe counts from its + completion: a caller that arrives while another's probe is in + flight (a page poll and a queue start right after the daemon + came up) waits for its answer instead of reading the stale + fixture, which routed the fixture's tests to nobody.""" + with self._fixture_lock: + if not force and time.time() - self._fixture_probed < FIXTURE_PROBE_S: + return self.fixture + had = self.fixture + # a box that is not there is said once, not every probe + said = [] + fx = _fixture.probe(said.append) + for m in said: + if m != self._fixture_said: + self._note(m) + self._fixture_said = m + if fx is not None: + self._fixture_said = None + if fx is None and had is not None: + self._note("fixture: %s no longer answers - running without it" % had.hostname) + self.fixture = fx + self._fixture_probed = time.time() + return fx def fixture_channels(self): """The channels the fixture covers right now (the button only diff --git a/forgetest/forgetest/suite/update.py b/forgetest/forgetest/suite/update.py index 312fed3..bf1b6e1 100644 --- a/forgetest/forgetest/suite/update.py +++ b/forgetest/forgetest/suite/update.py @@ -96,7 +96,8 @@ def slots_and_signature(ctx): ctx.log("slot %s is selected for the next boot: the apply is refused before the " "signature check, which is its own guard", target) else: - ctx.check(st == 200, "the apply job did not start: %s %s", st, body) + ctx.check(st == 202 and isinstance(body, dict) and body.get("started") is True, + "the apply job did not start: %s %s", st, body) result = None t0 = time.time() while time.time() - t0 < 60: diff --git a/forgetest/tests/test_fixture.py b/forgetest/tests/test_fixture.py index 6ae84ee..e7bcffb 100644 --- a/forgetest/tests/test_fixture.py +++ b/forgetest/tests/test_fixture.py @@ -416,6 +416,26 @@ class RoutingTests(unittest.TestCase): self.assertEqual(av["unattended"], ["r.auto"]) self.assertIsNone(summary) + def test_a_queue_started_during_a_probe_waits_for_the_fixture(self): + # The daemon just came up: nothing probed yet, a page poll starts + # the first probe (slow: an mDNS answer), and the queue start + # lands while it is in flight. The start must see the fixture. + stub = self.stub + + def slow_probe(log, path=None, resolver=None): + time.sleep(0.5) + return stub + fx.probe = slow_probe + self.runner._fixture_probed = 0.0 + self.runner.fixture = None + poll = threading.Thread(target=self.runner.probe_fixture) + poll.start() + time.sleep(0.1) + order = self.runner.batch_selection("unattended") + poll.join() + self.assertIn("r.lid", order) + self.assertIn("r.btn", order) + def run_queue(self, group, ack_live=False): ok, msg, order = self.runner.start_batch(group, ack_live=ack_live) self.assertTrue(ok, msg)