diff --git a/forgetest/forgetest/runner.py b/forgetest/forgetest/runner.py index 7710731..c3f1248 100644 --- a/forgetest/forgetest/runner.py +++ b/forgetest/forgetest/runner.py @@ -544,6 +544,7 @@ class Runner: self.batch = None self.fixture = None # the bench actuator, when one is up (fixture.py) self._fixture_probed = 0.0 + self._fixture_lock = threading.Lock() self._fixture_said = None self.boot_ref = None self.recover() @@ -568,25 +569,29 @@ class Runner: """The fixture, up and answering, or None; re-probed at most every FIXTURE_PROBE_S unless forced (before a run). A probe that finds it gone, or a config that appeared, changes the queues' routing - from then on.""" - now = time.time() - if not force and now - self._fixture_probed < FIXTURE_PROBE_S: - return self.fixture - self._fixture_probed = now - had = self.fixture - # a box that is not there is said once, not every probe - said = [] - fx = _fixture.probe(said.append) - for m in said: - if m != self._fixture_said: - self._note(m) - self._fixture_said = m - if fx is not None: - self._fixture_said = None - if fx is None and had is not None: - self._note("fixture: %s no longer answers - running without it" % had.hostname) - self.fixture = fx - return fx + from then on. One probe at a time, and the probe counts from its + completion: a caller that arrives while another's probe is in + flight (a page poll and a queue start right after the daemon + came up) waits for its answer instead of reading the stale + fixture, which routed the fixture's tests to nobody.""" + with self._fixture_lock: + if not force and time.time() - self._fixture_probed < FIXTURE_PROBE_S: + return self.fixture + had = self.fixture + # a box that is not there is said once, not every probe + said = [] + fx = _fixture.probe(said.append) + for m in said: + if m != self._fixture_said: + self._note(m) + self._fixture_said = m + if fx is not None: + self._fixture_said = None + if fx is None and had is not None: + self._note("fixture: %s no longer answers - running without it" % had.hostname) + self.fixture = fx + self._fixture_probed = time.time() + return fx def fixture_channels(self): """The channels the fixture covers right now (the button only diff --git a/forgetest/forgetest/suite/update.py b/forgetest/forgetest/suite/update.py index 312fed3..bf1b6e1 100644 --- a/forgetest/forgetest/suite/update.py +++ b/forgetest/forgetest/suite/update.py @@ -96,7 +96,8 @@ def slots_and_signature(ctx): ctx.log("slot %s is selected for the next boot: the apply is refused before the " "signature check, which is its own guard", target) else: - ctx.check(st == 200, "the apply job did not start: %s %s", st, body) + ctx.check(st == 202 and isinstance(body, dict) and body.get("started") is True, + "the apply job did not start: %s %s", st, body) result = None t0 = time.time() while time.time() - t0 < 60: diff --git a/forgetest/tests/test_fixture.py b/forgetest/tests/test_fixture.py index 6ae84ee..e7bcffb 100644 --- a/forgetest/tests/test_fixture.py +++ b/forgetest/tests/test_fixture.py @@ -416,6 +416,26 @@ class RoutingTests(unittest.TestCase): self.assertEqual(av["unattended"], ["r.auto"]) self.assertIsNone(summary) + def test_a_queue_started_during_a_probe_waits_for_the_fixture(self): + # The daemon just came up: nothing probed yet, a page poll starts + # the first probe (slow: an mDNS answer), and the queue start + # lands while it is in flight. The start must see the fixture. + stub = self.stub + + def slow_probe(log, path=None, resolver=None): + time.sleep(0.5) + return stub + fx.probe = slow_probe + self.runner._fixture_probed = 0.0 + self.runner.fixture = None + poll = threading.Thread(target=self.runner.probe_fixture) + poll.start() + time.sleep(0.1) + order = self.runner.batch_selection("unattended") + poll.join() + self.assertIn("r.lid", order) + self.assertIn("r.btn", order) + def run_queue(self, group, ack_live=False): ok, msg, order = self.runner.start_batch(group, ack_live=ack_live) self.assertTrue(ok, msg)