forgetest: a test's implementation hash is its own function and its module's shared code

The implementation half of a domain fingerprint was the whole suite
file, so a two-line witness fix in laser.py re-required every laser
test and a rename in cloud.py every cloud test: sixty attended minutes
for changes that touched two test bodies. Now the hash is the test's
own function (its decorator included) together with the module's text
outside every @test function. A body edit moves that test alone; a
helper edit moves the tests of its module, which is what a helper does;
a file that defines no test in the @test form hashes whole. The gate
computes it the same way, from the same code.

Every recorded fingerprint moves once with this, so the next campaign is
a full one: the price of every later fix costing one test.

Also carries the re-targeted cloud replays that the previous commit
left in the working tree (the CI failure on 296fd68).
This commit is contained in:
ScottW514
2026-08-22 18:06:27 -04:00
parent 296fd686e1
commit 2547a8eb68
4 changed files with 164 additions and 26 deletions
+9 -5
View File
@@ -65,9 +65,12 @@ each test's *details*.
A test's **domain fingerprint** is the hash of the `(component, path, A test's **domain fingerprint** is the hash of the `(component, path,
blob-id)` triples its coverage globs select in the image manifest, plus the blob-id)` triples its coverage globs select in the image manifest, plus the
platform identity, plus the hash of the test's own implementation. A PASS platform identity, plus the hash of the test's own implementation: its
recorded under fingerprint F applies to any build whose recomputed function (decorator included) together with the code its suite module
fingerprint is F - the same code computes it on the board and in the gate. shares among its tests, everything outside the module's `@test`
functions. A PASS recorded under fingerprint F applies to any build whose
recomputed fingerprint is F - the same code computes it on the board and
in the gate.
Consequences: Consequences:
@@ -85,8 +88,9 @@ Consequences:
init scripts, a third-party pin with no manifest entry) is layer content init scripts, a third-party pin with no manifest entry) is layer content
and invalidates everything; so does a pin written into a recipe body and invalidates everything; so does a pin written into a recipe body
instead of its pin file (the safe direction). instead of its pin file (the safe direction).
- A change to a test's implementation invalidates that test's earlier - A change inside a test's body invalidates that test's earlier passes
passes and no other. and no other; a change to a helper its module shares invalidates the
tests of that module.
- "Touched" is computed from content hashes carried in the image, never - "Touched" is computed from content hashes carried in the image, never
declared by hand. declared by hand.
+59 -4
View File
@@ -15,10 +15,12 @@ under the runner with a Context (log, prompts, evidence, hardware
helpers) and reports by returning normally (PASS) or raising helpers) and reports by returning normally (PASS) or raising
runner.Failed (FAIL). runner.Failed (FAIL).
""" """
import ast
import hashlib import hashlib
import inspect import inspect
import os import os
import re import re
import threading
from . import manifest as _manifest from . import manifest as _manifest
@@ -59,12 +61,16 @@ class Test:
@property @property
def source_sha(self): def source_sha(self):
"""sha256 of the module file that defines the test, line endings """The hash of the test's own implementation: its function (the
normalized. Part of the fingerprint: a changed implementation decorator included) together with the code its module shares
invalidates earlier passes of this test and no other.""" among its tests - everything outside the @test functions. Part
of the fingerprint: a change inside one test's body invalidates
that test's earlier passes and no other; a change to a helper
invalidates the tests of that module. A test the module does not
define in the @test form hashes its whole file."""
if self._source_sha is None: if self._source_sha is None:
path = inspect.getsourcefile(self.fn) or inspect.getfile(self.fn) path = inspect.getsourcefile(self.fn) or inspect.getfile(self.fn)
self._source_sha = source_file_sha(path) self._source_sha = implementation_sha(path, self.id)
return self._source_sha return self._source_sha
def fingerprint(self, manifest): def fingerprint(self, manifest):
@@ -117,6 +123,55 @@ def source_file_sha(path):
return hashlib.sha256(data).hexdigest() return hashlib.sha256(data).hexdigest()
_PARTS = {} # path -> (shared sha, {test id: own sha}); the files never change under a run
_PARTS_LOCK = threading.Lock()
def module_parts(path):
"""(shared sha, {test id: own sha}) for a suite module: the test ids
are read from the @test("...") decorators on the module's top-level
functions; a test's own text runs from its first decorator line to
the end of its body; the shared text is every other line of the
file. Line endings normalized."""
with _PARTS_LOCK:
hit = _PARTS.get(path)
if hit is not None:
return hit
with open(path, "rb") as f:
text = f.read().replace(b"\r\n", b"\n").decode("utf-8")
lines = text.split("\n")
spans = {}
try:
tree = ast.parse(text)
except SyntaxError:
tree = None
for node in (tree.body if tree is not None else []):
if not isinstance(node, ast.FunctionDef):
continue
for d in node.decorator_list:
if (isinstance(d, ast.Call) and isinstance(d.func, ast.Name) and d.func.id == "test"
and d.args and isinstance(d.args[0], ast.Constant) and isinstance(d.args[0].value, str)):
start = min([dd.lineno for dd in node.decorator_list] + [node.lineno])
spans[d.args[0].value] = (start, node.end_lineno)
owned = set()
for a, b in spans.values():
owned.update(range(a, b + 1))
shared = "\n".join(ln for i, ln in enumerate(lines, 1) if i not in owned)
own = {tid: _manifest.sha256_text("\n".join(lines[a - 1:b])) for tid, (a, b) in spans.items()}
parts = (_manifest.sha256_text(shared), own)
with _PARTS_LOCK:
_PARTS[path] = parts
return parts
def implementation_sha(path, test_id):
"""The implementation hash of one test (see Test.source_sha)."""
shared, own = module_parts(path)
if test_id not in own:
return source_file_sha(path)
return _manifest.sha256_text("%s:%s" % (shared, own[test_id]))
def test(id, *, title, subsystem, kind="auto", hardware="api", mode=None, covers=(), def test(id, *, title, subsystem, kind="auto", hardware="api", mode=None, covers=(),
requires=(), always=False, est_min=1, steps=(), description="", actions=(), requires=(), always=False, est_min=1, steps=(), description="", actions=(),
precheck=None): precheck=None):
+14 -17
View File
@@ -596,7 +596,8 @@ class CloudSuiteTests(unittest.TestCase):
# -- a paused print cancelled by the lid, a running one by the app -------- # -- a paused print cancelled by the lid, a running one by the app --------
def cancel_parts(self): def cancel_parts(self):
"""(print prologue, the pause lines, the lid stop + park + cancel, """(print prologue, the pause lines, the lid stop + park + cancel,
the same tail with the app's cancel as the trigger).""" the same tail with the app's cancel as the trigger - what
cloud.oversize-stream's ending looks like)."""
lines = fixture("lidabort") lines = fixture("lidabort")
pre, rest = cut(lines, "waiting for button") pre, rest = cut(lines, "waiting for button")
run_pre, rest = cut(rest, "machine:_run_loop starting run") run_pre, rest = cut(rest, "machine:_run_loop starting run")
@@ -609,46 +610,42 @@ class CloudSuiteTests(unittest.TestCase):
"action cancelled mid-run; stopping motion") for l in rest] "action cancelled mid-run; stopping motion") for l in rest]
return pre, paused, rest, app_cancel return pre, paused, rest, app_cancel
def test_pause_cancel_paths_on_the_bench_excerpt(self): def test_paused_lid_cancel_on_the_bench_excerpt(self):
self.in_cloud(pid=1522) self.in_cloud(pid=1522)
self.append(["2026-08-17T09:41:00.500000+00:00 gfcloud[1522] INFO websocket:_on_open RX-EVENT: ready"]) self.append(["2026-08-17T09:41:00.500000+00:00 gfcloud[1522] INFO websocket:_on_open RX-EVENT: ready"])
pre, paused, lid_tail, app_tail = self.cancel_parts() pre, paused, lid_tail, _app = self.cancel_parts()
prints = [] prints = []
def next_print(): def next_print():
prints.append(1) prints.append(1)
self.append(pre, delay=0.1) self.append(pre, delay=0.1)
run = self.run_test(cloud.pause_cancel_paths, run = self.run_test(cloud.paused_lid_cancel,
hooks={"Click Done here": next_print, hooks={"Click Done here": next_print,
"the press pauses the print": lambda: self.append(paused, delay=0.05), "the press pauses the print": lambda: self.append(paused, delay=0.05),
"The print is paused: leave the lid open": lambda: ( "The print is paused: leave the lid open": lambda: (
self.lid(False), self.append(lid_tail, delay=0.05)), self.lid(False), self.append(lid_tail, delay=0.05)),
"Close the lid.": lambda: self.lid(True), "Close the lid.": lambda: self.lid(True)},
"cancel the print from the app now": lambda: self.append(app_tail, test_id="cloud.paused-lid-cancel")
delay=0.05)},
test_id="cloud.pause-cancel-paths")
ev = run.evidence ev = run.evidence
self.assertEqual(len(prints), 2) # two prints, one cue each self.assertEqual(len(prints), 1) # one print, one cue
self.assertEqual(ev["paused"], {"button pressed mid-run; pausing": True, "paused at": True}) self.assertEqual(ev["paused"], {"button pressed mid-run; pausing": True, "paused at": True})
self.assertIn(":cancelled", ev["lid_from_pause"]["print finished"]) self.assertIn(":cancelled", ev["lid_from_pause"]["print finished"])
self.assertIn(":cancelled", ev["service_cancel"]["print finished"])
self.assertTrue(ev["lid_from_pause"]["return home complete"]) self.assertTrue(ev["lid_from_pause"]["return home complete"])
self.assertTrue(ev["service_cancel"]["return home complete"]) self.assertEqual(ev["counters_after"], [0, 0, 3])
self.assertEqual(ev["counters_after_print1"], [0, 0, 3]) self.assertTrue(ev["latch_locked_after"])
self.assertEqual(ev["counters_after_print2"], [0, 0, 3]) self.assertFalse(ev["armed_after"])
self.assertTrue(ev["latch_locked_after_print1"] and ev["latch_locked_after_print2"]) self.assertFalse(ev["button_dark"])
self.assertFalse(ev["armed_after_print1"] or ev["armed_after_print2"])
self.assertEqual(self.fc.posts, []) self.assertEqual(self.fc.posts, [])
self.assertTrue(any("PASS: a paused print cancelled by the lid" in l for l in run.lines), run.lines) self.assertTrue(any("PASS: a paused print cancelled by the lid" in l for l in run.lines), run.lines)
def test_pause_cancel_paths_fails_when_the_paused_print_resumes_instead(self): def test_paused_lid_cancel_fails_when_the_paused_print_resumes_instead(self):
# a lid that resumed (or was ignored) leaves the print ':completed' # a lid that resumed (or was ignored) leaves the print ':completed'
self.in_cloud(pid=1522) self.in_cloud(pid=1522)
self.append(["2026-08-17T09:41:00.500000+00:00 gfcloud[1522] INFO websocket:_on_open RX-EVENT: ready"]) self.append(["2026-08-17T09:41:00.500000+00:00 gfcloud[1522] INFO websocket:_on_open RX-EVENT: ready"])
pre, paused, lid_tail, _app = self.cancel_parts() pre, paused, lid_tail, _app = self.cancel_parts()
lid_tail = [l.replace(':cancelled"', ':completed"') for l in lid_tail] lid_tail = [l.replace(':cancelled"', ':completed"') for l in lid_tail]
self.assertFails( self.assertFails(
cloud.pause_cancel_paths, "print 1 did not end ':cancelled'", cloud.paused_lid_cancel, "the print did not end ':cancelled'",
hooks={"Click Done here": lambda: self.append(pre, delay=0.1), hooks={"Click Done here": lambda: self.append(pre, delay=0.1),
"the press pauses the print": lambda: self.append(paused, delay=0.05), "the press pauses the print": lambda: self.append(paused, delay=0.05),
"The print is paused: leave the lid open": lambda: ( "The print is paused: leave the lid open": lambda: (
+82
View File
@@ -1,3 +1,4 @@
import os
import unittest import unittest
import helpers import helpers
@@ -151,3 +152,84 @@ class CatalogTests(unittest.TestCase):
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
class ImplementationHashTests(unittest.TestCase):
"""A test's implementation hash is its own function plus the code its
module shares: a body edit moves one test, a helper edit moves them
all, and a file without @test functions hashes whole."""
MODULE = '''"""a suite module"""
from forgetest.catalog import test
def helper(x):
return x + 1
@test("m.one", title="one", subsystem="m")
def one(ctx):
ctx.log(helper(1))
@test("m.two", title="two", subsystem="m",
steps=["a step"])
def two(ctx):
ctx.log(helper(2))
'''
def setUp(self):
import tempfile
self.tmp = tempfile.mkdtemp(prefix="forgetest-impl-")
self.path = os.path.join(self.tmp, "mod.py")
self.write(self.MODULE)
def tearDown(self):
import shutil
shutil.rmtree(self.tmp, ignore_errors=True)
def write(self, text):
catalog._PARTS.pop(self.path, None)
with open(self.path, "w", newline="\n") as f:
f.write(text)
def shas(self):
return catalog.implementation_sha(self.path, "m.one"), catalog.implementation_sha(self.path, "m.two")
def test_a_body_edit_moves_that_test_only(self):
a1, b1 = self.shas()
self.write(self.MODULE.replace("ctx.log(helper(2))", "ctx.log(helper(3))"))
a2, b2 = self.shas()
self.assertEqual(a1, a2)
self.assertNotEqual(b1, b2)
def test_a_decorator_edit_moves_that_test_only(self):
a1, b1 = self.shas()
self.write(self.MODULE.replace('steps=["a step"]', 'steps=["another step"]'))
a2, b2 = self.shas()
self.assertEqual(a1, a2)
self.assertNotEqual(b1, b2)
def test_a_helper_edit_moves_every_test_of_the_module(self):
a1, b1 = self.shas()
self.write(self.MODULE.replace("return x + 1", "return x + 2"))
a2, b2 = self.shas()
self.assertNotEqual(a1, a2)
self.assertNotEqual(b1, b2)
def test_line_endings_do_not_count(self):
a1, b1 = self.shas()
catalog._PARTS.pop(self.path, None)
with open(self.path, "wb") as f:
f.write(self.MODULE.replace("\n", "\r\n").encode())
self.assertEqual((a1, b1), self.shas())
def test_an_unknown_id_hashes_the_whole_file(self):
self.assertEqual(catalog.implementation_sha(self.path, "m.none"), catalog.source_file_sha(self.path))
def test_every_suite_test_is_spanned(self):
import inspect
reg = catalog.load_suite()
for t in catalog.all_tests(reg):
path = inspect.getsourcefile(t.fn)
self.assertIn(t.id, catalog.module_parts(path)[1], t.id)