From 8d16b93e8689ef69715ac03cfc8a61d83fddc4b2 Mon Sep 17 00:00:00 2001 From: ScottW514 Date: Fri, 16 Oct 2026 14:17:00 -0400 Subject: [PATCH] forgetest: the shared judges and a test's own imports reach the hash the gate reads The sibling walk read the top-level statements of a module alone. Five tests import their judging helpers inside their own functions, so that no other test of the module moves with them: cloud.mode-switch (homeoff.py: judge_whole_motions, camera_home_return), cooling.flow-under-load and cooling.fail-tier-stop (laser.py's sample and run_and_sample, motion.py's log readers), forgectrl.auth (setup.py's request and decode) and motion.release (setup_dark.py's run_check). None of those helpers was in the hash of its test, and homeoff.py was in no hash at all: an edit to judge_whole_motions moved 0 of 43 tests, and an edit to laser.sample moved the three laser.* tests and neither cooling test, so an inherited PASS survived a change to its own judge. The guard that every package module a suite module imports is a listed judge read the top level alone too, and no unit test held the production path: Test.source_sha, which the fingerprint, the campaign and the gate read, could drop the shared judges with CI green. implementation_sha now adds to the hash of a test the suite modules its own @test function imports, at any depth, and what they import in turn; no other test of the module takes them. The shared text of a module gives the modules it imports at any depth too: a helper that imports inside its body brings that module into every test of the module, as the helper itself does. _test_defs finds the @test functions of a module for module_parts and for both walks. The guard walks the whole tree of each suite module, so a package import inside a function or a block is held to the list as well. test_a_shared_judge_edit_moves_every_test holds the production path on a suite test: its source_sha equals implementation_sha with SHARED_JUDGES and differs from it with none. Proof: forgetest-ci "Unit tests" (`python -B -m unittest discover -s tests -v` in forgetest): test_manifest.py runs 39 tests, OK, on Windows with python 3.14 and on Ubuntu 24.04 with python 3.12, with test_a_sibling_imported_inside_a_function_moves_the_tests_that_run_it (a judge imported in the body of one test moves that test and not the other; imported in the body of a helper, it moves both), the source_sha assertions, and the guard on the whole tree; test_artifact.py, test_campaign.py, test_redact.py and test_responsiveness.py pass. Negative controls on scratch copies: implementation_sha's default taking no judges fails the source_sha assertion, and so does Test.source_sha passing judges=(); the new case fails against catalog.py as it was, and against a shared-text walk of top-level statements alone; a `from .. import server` inside a function of suite/forgectrl.py, or in a try block at its top level, fails the guard, and the function-local one passes the guard that reads the top level alone. On the real catalog: an edit to judge_whole_motions moves cloud.mode-switch alone (0 tests before); an edit to laser.sample moves the three laser.* tests and both cooling tests (the three alone before); an edit to run_check moves the six setup.* tests and motion.release (the six alone before); an edit to setup.request moves forgectrl.auth and motion.release (motion.release through setup_dark.py, which imports setup.py) beside the 8 it moved before; an edit to hw.sysfs_int moves 43 of 43, as before. The coverage lint on the tree manifest from the recipe pins reports 0 uncovered paths, 0 host-tested row problems, 43 tests. pyflakes reports no undefined name. Acceptance: cloud.mode-switch, cooling.flow-under-load, cooling.fail-tier-stop, forgectrl.auth and motion.release: the implementation hash of each moves once, its own imports joining it; the other 38 do not move (source_sha compared before and after). Against the campaign this series already needs, that adds no run: 850498f moved all 43 once. tests/ are in no fingerprint. --- forgetest/forgetest/catalog.py | 101 +++++++++++++++++++++---------- forgetest/tests/test_manifest.py | 42 ++++++++++++- 2 files changed, 108 insertions(+), 35 deletions(-) diff --git a/forgetest/forgetest/catalog.py b/forgetest/forgetest/catalog.py index c41a738..26220a1 100644 --- a/forgetest/forgetest/catalog.py +++ b/forgetest/forgetest/catalog.py @@ -69,13 +69,14 @@ class Test: def source_sha(self): """The hash of the test's own implementation: its function (the decorator included) together with the code its module shares - among its tests - everything outside the @test functions - and - the shared judges (SHARED_JUDGES). Part of the fingerprint: a - change inside one test's body invalidates that test's earlier - passes and no other; a change to a helper invalidates the tests - of that module; a change to a shared judge invalidates every - test. A test the module does not define in the @test form hashes - its whole file.""" + among its tests (everything outside the @test functions), the + shared code of every sibling module that either of the two + imports, and the shared judges (SHARED_JUDGES). Part of the + fingerprint: a change inside one test's body invalidates that + test's earlier passes and no other; a change to a helper + invalidates the tests of that module and the tests that import + it; a change to a shared judge invalidates every test. A test the + module does not define in the @test form hashes its whole file.""" if self._source_sha is None: # One filler at a time: a second poll that arrives while the # first computes waits for it rather than repeating the work. @@ -200,6 +201,20 @@ def _parsed(path): return hit +def _test_defs(tree): + """{test id: its function} for the @test("...") functions at the top + level of a module's tree.""" + defs = {} + for node in (tree.body if tree is not None else []): + if not isinstance(node, ast.FunctionDef): + continue + for d in node.decorator_list: + if (isinstance(d, ast.Call) and isinstance(d.func, ast.Name) and d.func.id == "test" + and d.args and isinstance(d.args[0], ast.Constant) and isinstance(d.args[0].value, str)): + defs[d.args[0].value] = node + return defs + + def module_parts(path): """(shared sha, {test id: own sha}) for a suite module: the test ids are read from the @test("...") decorators on the module's top-level @@ -213,14 +228,9 @@ def module_parts(path): text, tree = _parsed(path) lines = text.split("\n") spans = {} - for node in (tree.body if tree is not None else []): - if not isinstance(node, ast.FunctionDef): - continue - for d in node.decorator_list: - if (isinstance(d, ast.Call) and isinstance(d.func, ast.Name) and d.func.id == "test" - and d.args and isinstance(d.args[0], ast.Constant) and isinstance(d.args[0].value, str)): - start = min([dd.lineno for dd in node.decorator_list] + [node.lineno]) - spans[d.args[0].value] = (start, node.end_lineno) + for tid, node in _test_defs(tree).items(): + start = min([dd.lineno for dd in node.decorator_list] + [node.lineno]) + spans[tid] = (start, node.end_lineno) owned = set() for a, b in spans.values(): owned.update(range(a, b + 1)) @@ -233,10 +243,10 @@ def module_parts(path): def sibling_imports(path, seen=None): - """The suite modules a module imports with `from .x import ...`, as - paths, transitively, in a stable order. A helper a test calls from - a sibling module is part of the judge, so its text belongs in the - test's fingerprint.""" + """The suite modules a module's shared text imports with `from .x + import ...`, as paths, transitively, in a stable order. A helper a + test calls from a sibling module is part of the judge, so its text + belongs in the test's fingerprint.""" if seen is None: seen = [] for sib in _direct_siblings(path): @@ -246,33 +256,60 @@ def sibling_imports(path, seen=None): return seen +def _imported(path, nodes): + """The suite modules the statements `nodes` import with `from .x + import`, at any depth (inside a function or a block too), in the + order they are written.""" + here = os.path.dirname(path) + found = sorted((n for top in nodes for n in ast.walk(top) + if isinstance(n, ast.ImportFrom) and n.level == 1 and n.module), + key=lambda n: (n.lineno, n.col_offset)) + out = [] + for node in found: + sib = os.path.join(here, node.module.replace(".", os.sep) + ".py") + if os.path.isfile(sib) and sib not in out: + out.append(sib) + return out + + def _direct_siblings(path): - """The suite modules one module imports itself, in import order.""" + """The suite modules one module's shared text imports itself: at its + top level, and inside its helpers and blocks. What a @test function + imports is that test's own (_own_siblings).""" with _PARTS_LOCK: hit = _SIBLINGS.get(path) if hit is None: _text, tree = _parsed(path) - here = os.path.dirname(path) - hit = [] - for node in (tree.body if tree is not None else []): - if isinstance(node, ast.ImportFrom) and node.level == 1 and node.module: - sib = os.path.join(here, node.module.replace(".", os.sep) + ".py") - if os.path.isfile(sib): - hit.append(sib) - _SIBLINGS[path] = hit + tests = list(_test_defs(tree).values()) + shared = [n for n in (tree.body if tree is not None else []) if not any(n is t for t in tests)] + hit = _SIBLINGS[path] = _imported(path, shared) return hit +def _own_siblings(path, test_id): + """The suite modules one test's own function imports: part of that + test's judge, and of no other test's.""" + _text, tree = _parsed(path) + node = _test_defs(tree).get(test_id) + return _imported(path, [node]) if node is not None else [] + + def implementation_sha(path, test_id, judges=None): """The implementation hash of one test (see Test.source_sha): the test's own text, the shared text of its module, the shared text of - every sibling suite module the module imports, and the text of the - shared judges. `judges` names files to take as the judges in place - of SHARED_JUDGES (the unit tests edit their own).""" + every sibling suite module the module's shared text or the test's + own function imports (transitively), and the text of the shared + judges. `judges` names files to take as the judges in place of + SHARED_JUDGES (the unit tests edit their own).""" shared, own = module_parts(path) if test_id not in own: return source_file_sha(path) - helpers = [module_parts(p)[0] for p in sorted(sibling_imports(path))] + sibs = sibling_imports(path) + for sib in _own_siblings(path, test_id): + if sib not in sibs and sib != path: + sibs.append(sib) + sibling_imports(sib, sibs) + helpers = [module_parts(p)[0] for p in sorted(sibs) if p != path] paths = shared_judge_paths() if judges is None else list(judges) judged = [module_parts(p)[0] for p in paths] return _manifest.sha256_text("%s:%s" % (":".join([shared] + helpers + judged), own[test_id])) diff --git a/forgetest/tests/test_manifest.py b/forgetest/tests/test_manifest.py index c9546bc..0aa8557 100644 --- a/forgetest/tests/test_manifest.py +++ b/forgetest/tests/test_manifest.py @@ -485,8 +485,9 @@ class CatalogTests(unittest.TestCase): def test_the_shared_judges_exist_and_take_every_package_import_of_the_suite(self): # The list is kept by hand: the Context reaches runner.py, hw.py # and baseline.py with no import to find. So every package module - # a suite module does import has to be on it (or be the registry, - # which judges nothing), or that judge would be in no fingerprint. + # a suite module does import, at its top level or inside a function + # or a block, has to be on it (or be the registry, which judges + # nothing), or that judge would be in no fingerprint. import ast import glob paths = catalog.shared_judge_paths() @@ -497,7 +498,7 @@ class CatalogTests(unittest.TestCase): for mod in sorted(glob.glob(os.path.join(catalog.suite_dir(), "*.py"))): with open(mod, encoding="utf-8") as f: tree = ast.parse(f.read()) - for node in tree.body: + for node in ast.walk(tree): if isinstance(node, ast.ImportFrom) and node.level == 2: for name in ([node.module] if node.module else [a.name for a in node.names]): self.assertIn(name + ".py", allowed, @@ -614,6 +615,41 @@ def two(ctx): with open(other, "w", newline="\n") as f: f.write("def serve():\n return 1\n") self.assertEqual((a2, b2), shas()) + # Test.source_sha, which the fingerprint, the campaign and the gate + # read, takes the shared judges itself: a suite test's hash is the + # one with SHARED_JUDGES folded in, not the one without them. + import inspect + t = catalog.load_suite()["image.health"] + src = inspect.getsourcefile(t.fn) + self.assertEqual(t.source_sha, catalog.implementation_sha(src, t.id, judges=catalog.shared_judge_paths())) + self.assertNotEqual(t.source_sha, catalog.implementation_sha(src, t.id, judges=[])) + + def test_a_sibling_imported_inside_a_function_moves_the_tests_that_run_it(self): + # A test that imports a judge inside its own function carries the + # judge in its hash, and no other test of its module does; a + # helper of the module that imports one inside its body carries it + # into every test of the module. + sib = os.path.join(self.tmp, "judge.py") + + def write_judge(text): + with open(sib, "w", newline="\n") as f: + f.write(text) + catalog.forget(sib) + write_judge("def judge(x):\n return x > 1\n") + self.write(self.MODULE.replace("def two(ctx):\n", + "def two(ctx):\n from .judge import judge\n judge(2)\n")) + a1, b1 = self.shas() + write_judge("def judge(x):\n return x > 2\n") + a2, b2 = self.shas() + self.assertEqual(a1, a2) + self.assertNotEqual(b1, b2) + self.write(self.MODULE.replace(" return x + 1\n", + " from .judge import judge\n return judge(x)\n")) + a3, b3 = self.shas() + write_judge("def judge(x):\n return x > 3\n") + a4, b4 = self.shas() + self.assertNotEqual(a3, a4) + self.assertNotEqual(b3, b4) def test_line_endings_do_not_count(self): a1, b1 = self.shas()