mirror of
https://github.com/openglow-org/forgefirm.git
synced 2026-09-28 09:11:11 -07:00
Let the page run a campaign's remaining tests as a queue
A campaign is mostly waiting for the next Start. The page now offers two queues, and each takes every test of its kinds the campaign does not already count as satisfied: Unattended for the auto tests, which need nobody in the room, and Operator and live for the ones that need somebody at the machine, since they prompt and they fire the laser. The buttons say how many they would run and ask before starting; the live queue names the tests that fire and takes the acknowledgment once, for all of them. A queue runs one test at a time through the runner's single slot, in prerequisite order. Registration order otherwise, so a run reads down the page, but a prerequisite inside the queue always goes first. It stops on the first result that is not a PASS: a FAIL closes the campaign, and carrying on would only open a second one behind the operator's back. A test the runner refuses to start is skipped with the reason on the page and the rest carry on, which is what happens to an auto test waiting on an operator one: run the attended queue, then the unattended one again. The queue lives in the runner, not in the tab, so reloading the page or closing it leaves the run alone. While one is up it holds the machine between its tests as well as during them, so a single Start and the bench tools are refused rather than cutting in. Stop the queue cancels what is still waiting and lets the run in progress finish; Abort ends that one too, and lands as the non-PASS that stops the queue. Every run a queue starts records which one put it there. Also moves the /state ETag test to the end of its class. It invalidates, timestamps are whole seconds, and a PASS stamped in the same second as an invalidate is deliberately not inheritable, so on a fast run it decided the inheritance an earlier test was checking. forgetest is a dev-only component and can never appear in a coverage map, so this has no acceptance catalog consequence.
This commit is contained in:
ScottW514
@@ -0,0 +1,342 @@
|
||||
"""The two queues: what they select, the order they run in, and where
|
||||
they stop.
|
||||
|
||||
A campaign is mostly waiting, so the page offers to run what is still
|
||||
owed in one go: the unattended tests for an empty room, the attended ones
|
||||
for an operator at the machine. The rules that matter here are that a
|
||||
queue takes exactly the unsatisfied tests of its kinds, runs a
|
||||
prerequisite before the test that names it, stops on the first result
|
||||
that is not a PASS (a FAIL closes the campaign, so going on would open a
|
||||
second one behind the operator), and never fires the laser without the
|
||||
acknowledgment.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
import helpers
|
||||
from forgetest import catalog, server
|
||||
from forgetest.log import Log
|
||||
from forgetest.runner import BATCH_GROUPS, Failed, Runner
|
||||
|
||||
|
||||
def t_pass(ctx):
|
||||
ctx.log("ok")
|
||||
|
||||
|
||||
def t_fail(ctx):
|
||||
ctx.check(False, "deliberate")
|
||||
|
||||
|
||||
def t_prompt(ctx):
|
||||
ctx.prompt("Ready?", ("Continue",))
|
||||
|
||||
|
||||
def t_slow(ctx):
|
||||
ctx.sleep(30)
|
||||
|
||||
|
||||
class OrderTests(unittest.TestCase):
|
||||
"""order_by_requires is a pure function over the requires graph."""
|
||||
|
||||
def order(self, tests, selected):
|
||||
return catalog.order_by_requires(tests, selected)
|
||||
|
||||
def test_prerequisite_first(self):
|
||||
a = helpers.make_test("s.a", [], fn=t_pass)
|
||||
b = helpers.make_test("s.b", [], requires=("s.a",), fn=t_pass)
|
||||
# selection order must not matter; registration order breaks ties
|
||||
self.assertEqual(self.order([b, a], ["s.b", "s.a"]), ["s.a", "s.b"])
|
||||
self.assertEqual(self.order([a, b], ["s.b", "s.a"]), ["s.a", "s.b"])
|
||||
|
||||
def test_chain(self):
|
||||
c = helpers.make_test("s.c", [], requires=("s.b",), fn=t_pass)
|
||||
b = helpers.make_test("s.b", [], requires=("s.a",), fn=t_pass)
|
||||
a = helpers.make_test("s.a", [], fn=t_pass)
|
||||
self.assertEqual(self.order([c, b, a], ["s.c", "s.b", "s.a"]), ["s.a", "s.b", "s.c"])
|
||||
|
||||
def test_registration_order_otherwise(self):
|
||||
ts = [helpers.make_test("s.%d" % i, [], fn=t_pass) for i in range(4)]
|
||||
self.assertEqual(self.order(ts, [t.id for t in ts]), ["s.0", "s.1", "s.2", "s.3"])
|
||||
|
||||
def test_prerequisite_outside_the_selection_places_nothing(self):
|
||||
a = helpers.make_test("s.a", [], fn=t_pass)
|
||||
b = helpers.make_test("s.b", [], requires=("s.a",), fn=t_pass)
|
||||
self.assertEqual(self.order([a, b], ["s.b"]), ["s.b"])
|
||||
|
||||
def test_unknown_and_empty(self):
|
||||
a = helpers.make_test("s.a", [], fn=t_pass)
|
||||
self.assertEqual(self.order([a], []), [])
|
||||
self.assertEqual(self.order([a], ["s.nope"]), [])
|
||||
|
||||
def test_a_cycle_does_not_recurse_forever(self):
|
||||
"""validate() rejects cycles; a malformed registry must still not
|
||||
hang the runner."""
|
||||
a = helpers.make_test("s.a", [], requires=("s.b",), fn=t_pass)
|
||||
b = helpers.make_test("s.b", [], requires=("s.a",), fn=t_pass)
|
||||
out = self.order([a, b], ["s.a", "s.b"])
|
||||
self.assertEqual(sorted(out), ["s.a", "s.b"])
|
||||
|
||||
|
||||
class QueueTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.tmp = tempfile.mkdtemp(prefix="forgetest-queue-")
|
||||
os.environ["FORGETEST_DATA"] = cls.tmp
|
||||
os.environ["FORGETEST_MARKER"] = os.path.join(cls.tmp, "marker")
|
||||
cls.man = helpers.make_manifest()
|
||||
cls.reg = helpers.registry(
|
||||
# auto: b requires a, so a queue must run a first even though
|
||||
# b registers earlier
|
||||
helpers.make_test("q.b", [("forgectrl", "src/ui.c")], requires=("q.a",), fn=t_pass),
|
||||
helpers.make_test("q.a", [("forgectrl", "src/auth.c")], fn=t_pass),
|
||||
helpers.make_test("q.c", [("forgectrl", "src/cool.c")], fn=t_pass),
|
||||
# attended
|
||||
helpers.make_test("q.op", [("forgectrl", "src/main.c")], kind="operator", fn=t_prompt),
|
||||
helpers.make_test("q.live", [("grblhal-glowforge", "src/**")], kind="live", fn=t_pass),
|
||||
)
|
||||
cls.log = Log(os.path.join(cls.tmp, "results.jsonl"))
|
||||
cls.runner = Runner(cls.log, cls.man, cls.reg)
|
||||
cls.token = server.load_token(os.path.join(cls.tmp, "token"))
|
||||
cls.app = server.App(cls.runner, cls.token, export_dir=os.path.join(cls.tmp, "export"))
|
||||
cls.srv = server.make_server(cls.app, "127.0.0.1", 0)
|
||||
cls.port = cls.srv.server_address[1]
|
||||
threading.Thread(target=cls.srv.serve_forever, daemon=True).start()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.srv.shutdown()
|
||||
cls.srv.server_close()
|
||||
shutil.rmtree(cls.tmp, ignore_errors=True)
|
||||
|
||||
def setUp(self):
|
||||
# A clean history per test: a PASS left by an earlier one would be
|
||||
# inherited and drop that test out of the selection. Truncating is
|
||||
# also the shrunk-file path Log.read has to notice.
|
||||
open(self.log.path, "w").close()
|
||||
self.assertEqual(self.log.read(), [])
|
||||
self.runner.batch = None
|
||||
|
||||
def call(self, method, path, body=None):
|
||||
url = "http://127.0.0.1:%d%s" % (self.port, path)
|
||||
hdrs = {"Host": "127.0.0.1:%d" % self.port, "X-ForgeFIRM-Token": self.token}
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode()
|
||||
hdrs["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(url, data=data, method=method, headers=hdrs)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as r:
|
||||
return r.status, json.loads(r.read().decode())
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, json.loads(e.read().decode())
|
||||
|
||||
def wait_queue(self, timeout=30):
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
st, d = self.call("GET", "/state")
|
||||
if d["batch"] and d["batch"]["finished"]:
|
||||
return d
|
||||
time.sleep(0.05)
|
||||
self.fail("the queue did not finish")
|
||||
|
||||
# -- selection ---------------------------------------------------------
|
||||
def test_groups_split_the_catalog_by_who_has_to_be_there(self):
|
||||
self.assertEqual(BATCH_GROUPS["unattended"], ("auto",))
|
||||
self.assertEqual(sorted(BATCH_GROUPS["attended"]), ["live", "operator"])
|
||||
st, d = self.call("GET", "/state")
|
||||
self.assertEqual(d["batch_available"]["unattended"], ["q.a", "q.b", "q.c"])
|
||||
self.assertEqual(d["batch_available"]["attended"], ["q.op", "q.live"])
|
||||
|
||||
def test_selection_runs_prerequisites_first(self):
|
||||
"""q.b registers before q.a but requires it."""
|
||||
st, d = self.call("GET", "/state")
|
||||
order = d["batch_available"]["unattended"]
|
||||
self.assertLess(order.index("q.a"), order.index("q.b"))
|
||||
|
||||
def test_satisfied_tests_drop_out(self):
|
||||
st, d = self.call("POST", "/start", {"test": "q.c"})
|
||||
self.assertEqual(st, 200)
|
||||
self.wait_idle()
|
||||
st, d = self.call("GET", "/state")
|
||||
self.assertEqual(d["tests"]["q.c"]["status"], "pass")
|
||||
self.assertNotIn("q.c", d["batch_available"]["unattended"])
|
||||
self.assertEqual(d["batch_available"]["unattended"], ["q.a", "q.b"])
|
||||
|
||||
def wait_idle(self, timeout=20):
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
st, d = self.call("GET", "/state")
|
||||
if not d["running"]:
|
||||
return d
|
||||
time.sleep(0.05)
|
||||
self.fail("run did not finish")
|
||||
|
||||
# -- running -----------------------------------------------------------
|
||||
def test_unattended_queue_runs_everything_in_order(self):
|
||||
st, d = self.call("POST", "/batch", {"group": "unattended"})
|
||||
self.assertEqual(st, 200)
|
||||
self.assertEqual(d["order"], ["q.a", "q.b", "q.c"])
|
||||
d = self.wait_queue()
|
||||
b = d["batch"]
|
||||
self.assertEqual([x["test"] for x in b["done"]], ["q.a", "q.b", "q.c"])
|
||||
self.assertEqual(set(x["result"] for x in b["done"]), {"PASS"})
|
||||
self.assertEqual(b["skipped"], [])
|
||||
self.assertIsNone(b["stopped"])
|
||||
for tid in ("q.a", "q.b", "q.c"):
|
||||
self.assertEqual(d["tests"][tid]["status"], "pass", tid)
|
||||
# and the run records say which queue put them there
|
||||
st, rec = self.call("GET", "/result?test=q.b")
|
||||
self.assertEqual(rec["evidence"]["batch"]["group"], "unattended")
|
||||
|
||||
def test_an_empty_queue_is_refused_not_started(self):
|
||||
self.call("POST", "/batch", {"group": "unattended"})
|
||||
self.wait_queue()
|
||||
st, d = self.call("POST", "/batch", {"group": "unattended"})
|
||||
self.assertEqual(st, 409)
|
||||
self.assertIn("already satisfied", d["message"])
|
||||
|
||||
def test_a_failure_stops_the_queue(self):
|
||||
reg = helpers.registry(
|
||||
helpers.make_test("f.a", [("forgectrl", "src/ui.c")], fn=t_pass),
|
||||
helpers.make_test("f.b", [("forgectrl", "src/auth.c")], fn=t_fail),
|
||||
helpers.make_test("f.c", [("forgectrl", "src/cool.c")], fn=t_pass),
|
||||
)
|
||||
tmp = tempfile.mkdtemp(prefix="forgetest-fail-")
|
||||
try:
|
||||
r = Runner(Log(os.path.join(tmp, "r.jsonl")), self.man, reg)
|
||||
ok, msg, order = r.start_batch("unattended")
|
||||
self.assertTrue(ok, msg)
|
||||
self.assertEqual(order, ["f.a", "f.b", "f.c"])
|
||||
deadline = time.time() + 20
|
||||
while not r.batch["finished"] and time.time() < deadline:
|
||||
time.sleep(0.05)
|
||||
b = r.batch_snapshot()
|
||||
self.assertEqual([x["test"] for x in b["done"]], ["f.a", "f.b"])
|
||||
self.assertEqual(b["done"][1]["result"], "FAIL")
|
||||
self.assertEqual(b["stopped"], "FAIL on f.b")
|
||||
self.assertEqual(b["pending"], ["f.c"], "f.c ran after a FAIL closed the campaign")
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
def test_a_test_that_cannot_start_is_skipped_not_silently_dropped(self):
|
||||
"""An unmet prerequisite outside the queue refuses the start; the
|
||||
queue records why and carries on."""
|
||||
reg = helpers.registry(
|
||||
helpers.make_test("k.op", [("forgectrl", "src/main.c")], kind="operator", fn=t_pass),
|
||||
helpers.make_test("k.a", [("forgectrl", "src/ui.c")], requires=("k.op",), fn=t_pass),
|
||||
helpers.make_test("k.b", [("forgectrl", "src/auth.c")], fn=t_pass),
|
||||
)
|
||||
tmp = tempfile.mkdtemp(prefix="forgetest-skip-")
|
||||
try:
|
||||
r = Runner(Log(os.path.join(tmp, "r.jsonl")), self.man, reg)
|
||||
ok, msg, order = r.start_batch("unattended")
|
||||
self.assertEqual(order, ["k.a", "k.b"])
|
||||
deadline = time.time() + 20
|
||||
while not r.batch["finished"] and time.time() < deadline:
|
||||
time.sleep(0.05)
|
||||
b = r.batch_snapshot()
|
||||
self.assertEqual([x["test"] for x in b["skipped"]], ["k.a"])
|
||||
self.assertIn("prerequisites not satisfied", b["skipped"][0]["reason"])
|
||||
self.assertEqual([x["test"] for x in b["done"]], ["k.b"])
|
||||
self.assertIsNone(b["stopped"])
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
# -- safety and exclusion ----------------------------------------------
|
||||
def test_an_attended_queue_with_a_live_test_needs_the_acknowledgment(self):
|
||||
st, d = self.call("POST", "/batch", {"group": "attended"})
|
||||
self.assertEqual(st, 409)
|
||||
self.assertIn("fires the laser", d["message"])
|
||||
self.assertIn("q.live", d["message"])
|
||||
st, d = self.call("GET", "/state")
|
||||
self.assertIsNone(d["batch"], "a refused queue must not exist")
|
||||
|
||||
def test_the_acknowledgment_reaches_every_live_run(self):
|
||||
st, d = self.call("POST", "/batch", {"group": "attended", "ack_live": True})
|
||||
self.assertEqual(st, 200)
|
||||
# the operator test prompts; answer it so the queue can reach the live one
|
||||
deadline = time.time() + 20
|
||||
while time.time() < deadline:
|
||||
st, s = self.call("GET", "/state")
|
||||
if s["running"] and s["running"]["prompt"]:
|
||||
self.call("POST", "/answer", {"prompt_id": s["running"]["prompt"]["id"],
|
||||
"value": "Continue"})
|
||||
break
|
||||
if s["batch"] and s["batch"]["finished"]:
|
||||
break
|
||||
time.sleep(0.05)
|
||||
d = self.wait_queue()
|
||||
self.assertEqual([x["test"] for x in d["batch"]["done"]], ["q.op", "q.live"])
|
||||
st, rec = self.call("GET", "/result?test=q.live")
|
||||
self.assertTrue(rec["evidence"]["operator"]["ack_live"])
|
||||
|
||||
def test_a_single_start_is_refused_while_a_queue_runs(self):
|
||||
reg = helpers.registry(
|
||||
helpers.make_test("s.slow", [("forgectrl", "src/ui.c")], fn=t_slow),
|
||||
helpers.make_test("s.other", [("forgectrl", "src/auth.c")], fn=t_pass),
|
||||
)
|
||||
tmp = tempfile.mkdtemp(prefix="forgetest-excl-")
|
||||
try:
|
||||
r = Runner(Log(os.path.join(tmp, "r.jsonl")), self.man, reg)
|
||||
r.start_batch("unattended")
|
||||
deadline = time.time() + 10
|
||||
while r.current is None and time.time() < deadline:
|
||||
time.sleep(0.02)
|
||||
ok, msg = r.start_test("s.other")
|
||||
self.assertFalse(ok)
|
||||
self.assertEqual(msg, "a queue is running")
|
||||
ok, msg = r.start_bench("anything")
|
||||
self.assertFalse(ok)
|
||||
self.assertEqual(msg, "a queue is running")
|
||||
ok, msg, _ = r.start_batch("unattended")
|
||||
self.assertFalse(ok)
|
||||
self.assertIn("already running", msg)
|
||||
r.stop_batch()
|
||||
r.abort()
|
||||
deadline = time.time() + 20
|
||||
while not r.batch["finished"] and time.time() < deadline:
|
||||
time.sleep(0.05)
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
def test_stop_cancels_what_is_waiting(self):
|
||||
reg = helpers.registry(
|
||||
helpers.make_test("p.slow", [("forgectrl", "src/ui.c")], fn=t_slow),
|
||||
helpers.make_test("p.next", [("forgectrl", "src/auth.c")], fn=t_pass),
|
||||
)
|
||||
tmp = tempfile.mkdtemp(prefix="forgetest-stop-")
|
||||
try:
|
||||
r = Runner(Log(os.path.join(tmp, "r.jsonl")), self.man, reg)
|
||||
r.start_batch("unattended")
|
||||
deadline = time.time() + 10
|
||||
while r.current is None and time.time() < deadline:
|
||||
time.sleep(0.02)
|
||||
ok, msg = r.stop_batch()
|
||||
self.assertTrue(ok, msg)
|
||||
self.assertTrue(r.batch_snapshot()["stopping"])
|
||||
r.abort() # the run in progress needs its own lever
|
||||
deadline = time.time() + 20
|
||||
while not r.batch["finished"] and time.time() < deadline:
|
||||
time.sleep(0.05)
|
||||
b = r.batch_snapshot()
|
||||
self.assertNotIn("p.next", [x["test"] for x in b["done"]])
|
||||
ok, msg = r.stop_batch()
|
||||
self.assertFalse(ok)
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
def test_unknown_group(self):
|
||||
st, d = self.call("POST", "/batch", {"group": "everything"})
|
||||
self.assertEqual(st, 409)
|
||||
self.assertEqual(d["message"], "unknown queue")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -184,6 +184,12 @@ class PageTests(unittest.TestCase):
|
||||
self.assertIn("function updateBench()", html)
|
||||
# the prompt buttons are rebuilt only when the prompt changes
|
||||
self.assertIn("if(pk!==promptKey)", html)
|
||||
# the queue controls are static markup: a poll relabels them and
|
||||
# flips disabled, it never replaces the node
|
||||
for bid in ("q-unattended", "q-attended", "q-stop"):
|
||||
self.assertIn("id='%s'" % bid, html)
|
||||
self.assertNotIn("id='" + bid + "-", html)
|
||||
self.assertIn("function renderQueue()", html)
|
||||
|
||||
def test_page_is_self_contained_ascii(self):
|
||||
html = page.render("0" * 32)
|
||||
|
||||
@@ -167,25 +167,6 @@ class ServerTests(unittest.TestCase):
|
||||
st, d = self.call("GET", "/nope")
|
||||
self.assertEqual(st, 404)
|
||||
|
||||
def test_01b_state_is_conditional(self):
|
||||
"""The page polls; an unchanged state must cost a 304, and the
|
||||
ETag must move as soon as the state does."""
|
||||
st, d, hdrs = self.call_h("GET", "/state")
|
||||
self.assertEqual(st, 200)
|
||||
etag = hdrs.get("ETag")
|
||||
self.assertTrue(etag and etag.startswith('"'), "no ETag on /state")
|
||||
st, _, hdrs2 = self.call_h("GET", "/state", headers={"If-None-Match": etag})
|
||||
self.assertEqual(st, 304)
|
||||
self.assertEqual(hdrs2.get("ETag"), etag)
|
||||
# a stale validator must not be honored
|
||||
st, _, _ = self.call_h("GET", "/state", headers={"If-None-Match": '"stale"'})
|
||||
self.assertEqual(st, 200)
|
||||
# and the validator moves with the state
|
||||
self.call("POST", "/invalidate", {"reason": "etag check"})
|
||||
st, _, hdrs3 = self.call_h("GET", "/state", headers={"If-None-Match": etag})
|
||||
self.assertEqual(st, 200)
|
||||
self.assertNotEqual(hdrs3.get("ETag"), etag)
|
||||
|
||||
def test_01c_connection_is_kept_alive(self):
|
||||
"""A poll per second over a fresh TCP connection each time is
|
||||
waste the board does not need to pay."""
|
||||
@@ -399,6 +380,31 @@ class ServerTests(unittest.TestCase):
|
||||
self.assertTrue(any("recovered" in m for m in r.messages))
|
||||
self.assertFalse(os.path.exists(marker))
|
||||
|
||||
def test_10_state_is_conditional(self):
|
||||
"""The page polls; an unchanged state must cost a 304, and the
|
||||
ETag must move as soon as the state does.
|
||||
|
||||
Runs last because it invalidates: timestamps are whole seconds, and
|
||||
a PASS stamped in the same second as an invalidate is deliberately
|
||||
not inheritable, so an invalidate early in this class would decide
|
||||
the inheritance the earlier tests are checking.
|
||||
"""
|
||||
st, d, hdrs = self.call_h("GET", "/state")
|
||||
self.assertEqual(st, 200)
|
||||
etag = hdrs.get("ETag")
|
||||
self.assertTrue(etag and etag.startswith('"'), "no ETag on /state")
|
||||
st, _, hdrs2 = self.call_h("GET", "/state", headers={"If-None-Match": etag})
|
||||
self.assertEqual(st, 304)
|
||||
self.assertEqual(hdrs2.get("ETag"), etag)
|
||||
# a stale validator must not be honored
|
||||
st, _, _ = self.call_h("GET", "/state", headers={"If-None-Match": '"stale"'})
|
||||
self.assertEqual(st, 200)
|
||||
# and the validator moves with the state
|
||||
self.call("POST", "/invalidate", {"reason": "etag check"})
|
||||
st, _, hdrs3 = self.call_h("GET", "/state", headers={"If-None-Match": etag})
|
||||
self.assertEqual(st, 200)
|
||||
self.assertNotEqual(hdrs3.get("ETag"), etag)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user