From a9c794be34bbc693b74b44c73fbff5375899b182 Mon Sep 17 00:00:00 2001 From: ScottW514 Date: Tue, 13 Oct 2026 07:40:12 -0400 Subject: [PATCH] forgetest: a path a host test proves needs no catalog test The coverage lint failed CI on any behavioral path no catalog test covered, so every software change pulled a new or wider acceptance test in with it, though the catalog is for what needs the machine or an operator. A path whose behavior a host test in CI proves now has a row in HOST_TESTED (manifest.py): the component, a glob, the host test, and the CI workflow that runs it, the last two as ":" - a file of a component in the manifest (its tests/ and .github/ files are listed there), or "forgefirm:" and a path in this repository (the null-sink harnesses, which grblhal-glowforge's workflow runs). - coverage_report counts a host-tested path as covered. Fingerprints are unchanged: a path no catalog test covers is in no fingerprint, so a change there makes no acceptance run necessary. - host_row_problems reports a row whose glob selects no behavioral file; a host test or workflow that does not exist (a component file missing from the manifest, a "forgefirm:" file missing from the checkout, the dev-only forgetest, no repository named); a workflow that should be readable and is not; one that does not check out the test's repository; and one with no step that runs the test. - The workflow is read at the pinned revision: read_source takes a component's blob from the bare repo manifest-from-tree.py fetched for its source URL (the checkout's .manifest-cache, or --sources), a "forgefirm:" file from the checkout. With no sources (an image) it is not read and the lint says so; with sources, a read that fails is a problem, never a pass. - workflow_steps reads the workflow's steps by indentation (uses, run, working-directory, with), comments dropped, whole lines and from " #"; a quoted scalar loses its quotes, a quote inside a command stays. checkout_path finds where the test's repository is checked out (its own: the checkout naming no repository; another: the checkout of /, the name from the component's source URL). - workflow_runs looks at what a step's run commands execute, split into simple commands (;, &&, ||, |, newlines, then/do/else; a backslash continuation joined), with sudo and its flags, env, time, exec and variable assignments passed over, each path resolved against the step's working-directory and compared with the test's workspace path: the command itself (a script run directly, a C test's built binary /); the script an interpreter runs (python, bash, sh) or a for loop's list, by path or a glob matching the whole path segment by segment; unittest discover starting in the test's own directory with a pattern its name matches (test*.py unless -p); pytest given the test or its own directory, for a name it collects. An argument to anything else (echo, cat, a build target), a step's name, a comment, a trigger's paths filter, or a path that only ends like the test's does not count. - The lint prints the row problems and fails --enforce on them, as on an uncovered path or an empty covers entry; --json carries them as host_tested, with workflows_read (false when a workflow that should have been read was not). It reads HOST_TESTED when it runs. - The table starts empty: rows come with the tests that leave the catalog. Proof: host tests: a host-tested row takes its paths out of the report and moves no fingerprint; good rows report nothing; each kind of bad row is reported with its reason; a workflow that runs one harness and names another only in its paths filter, a comment line and an inline comment runs the first and not the second; workflow_runs on a built binary, sudo and a binary, a script, a harness under another checkout, pytest on a directory, a for-loop glob and unittest discover under a step's working-directory, and not under another one; and not on a paths entry, an inline comment, a glob or a discover that the path only ends like (tools/tests/..., scripts/bench/tests/...), a longer name, another file, a step's name, or a commented line in a block; not on echo with the stem or the binary's path, a build target, cat of a script, unittest discover from a directory above the test or with a pattern its name misses, or pytest on a directory above it; on discover with -p '*_test.py', pytest given the file, a script run directly and by bash, and sudo -E with assignments continued onto the next line; checkout_path for its own and another repository and none; repo_name from a source URL; a component's workflow read from a bare repo's blob, judged, not judged without the sources, and a blob that cannot be read reported; the command line with a workflow it cannot read reports workflows_read false and the read failure, and fails --enforce. The manifest tests (34) pass on Windows and under Linux; the forgetest host suite passes under Linux. Against the recipe pins and their fetched workflows: rows for forgectrl's tray_test, forgeext's holdkeep_test, sandbox_test (under sudo) and run_test.py (sudo, env and assignments over three lines), the laser_stream_test harness (grblhal-glowforge's checkout of forgefirm), gfhardware's test_machine_lid_button (a for-loop glob under its working-directory), gfutilities' test_frame (pytest on tests) and forgetest's own test_manifest (unittest discover under forgefirm/forgetest) pass; rows for forgectrl's fflog_e2e.sh, the bench script fire_test.py and kas/source-bundle.yml (only in forgetest-ci's paths filter) are refused. The lint on that manifest reports 0 uncovered paths and 0 host-tested row problems. Acceptance: none. forgetest is dev-only and in no fingerprint; the lint runs in CI, not in a campaign. --- .github/workflows/forgetest-ci.yml | 7 +- forgetest/forgetest/coverage.py | 50 +++-- forgetest/forgetest/manifest.py | 319 ++++++++++++++++++++++++++++- forgetest/tests/test_manifest.py | 211 +++++++++++++++++++ 4 files changed, 569 insertions(+), 18 deletions(-) diff --git a/.github/workflows/forgetest-ci.yml b/.github/workflows/forgetest-ci.yml index 4398ff8..0475b9a 100644 --- a/.github/workflows/forgetest-ci.yml +++ b/.github/workflows/forgetest-ci.yml @@ -12,9 +12,10 @@ # - shared UI files: theme.css and the vendored Bootstrap the page # carries are byte-identical to forgectrl's at its pinned revision # - coverage lint: every source path of every component pinned by the -# recipes must be selected by some catalog test's coverage globs (the -# tree manifest is generated from the pins with git - no Yocto build); -# enforced: an uncovered path fails the job (https://docs.forgefirm.org/developers/acceptance/). +# recipes must be selected by some catalog test's coverage globs or by a +# host-tested row naming the host test that proves it (the tree manifest +# is generated from the pins with git - no Yocto build); enforced: an +# uncovered path fails the job (https://docs.forgefirm.org/developers/testing/). name: forgetest-ci diff --git a/forgetest/forgetest/coverage.py b/forgetest/forgetest/coverage.py index 343e336..6080b32 100644 --- a/forgetest/forgetest/coverage.py +++ b/forgetest/forgetest/coverage.py @@ -4,24 +4,33 @@ # SPDX-License-Identifier: MIT """Coverage lint: every source path in the manifest must be selected by -some test's coverage globs, except the allowlisted non-behavioral paths, -and every coverage glob must select at least one path of its component. +some test's coverage globs or by a host-tested row (manifest.HOST_TESTED), +except the allowlisted non-behavioral paths; every coverage glob and every +host-tested glob must select at least one path of its component; and every +host-tested row must name a host test that exists and a CI workflow that +runs it. Why a hard rule: under the domain model an uncovered file is worse than an untested one - a change there leaves every inherited PASS valid when it should have invalidated them. The lint is the floor (the file is -fingerprinted by at least one test); whether that test exercises the -change stays with the change author. An entry that selects nothing is -the same defect from the other side: the test's fingerprint ignores the -file it named. +fingerprinted by at least one test, or a named host test that CI runs +answers for it); whether that test exercises the change stays with the +change author. An entry that selects nothing is the same defect from the +other side: the test's fingerprint ignores the file it named. - python3 -m forgetest.coverage [--manifest PATH] [--enforce] [--json] +Whether a workflow runs its host test is read from the workflow's text at +the pinned revision, in the sources scripts/manifest-from-tree.py fetched +(--sources, default the checkout's .manifest-cache). Without them (on an +image) that part is not checked, and the report says so. + + python3 -m forgetest.coverage [--manifest PATH] [--sources DIR] [--enforce] [--json] Exit 0 when nothing is uncovered (or when only reporting), 1 under --enforce with uncovered paths, 2 on a usage or load error. """ import argparse import json +import os import sys from . import catalog as _catalog @@ -33,16 +42,22 @@ ALLOW = _manifest.NON_BEHAVIORAL def run(manifest, tests, allow=ALLOW): - return _manifest.coverage_report(manifest, tests, allow) + return _manifest.coverage_report(manifest, tests, allow, host=_manifest.HOST_TESTED) def empty(manifest, tests): return _manifest.empty_covers(manifest, tests) +def host_problems(manifest, sources=_manifest.SOURCES): + return _manifest.host_row_problems(manifest, host=_manifest.HOST_TESTED, sources=sources) + + def main(argv=None): ap = argparse.ArgumentParser(description="forgetest coverage lint") ap.add_argument("--manifest", default=None, help="manifest JSON (default: the running image's)") + ap.add_argument("--sources", default=_manifest.SOURCES, + help="the pinned sources manifest-from-tree.py fetched (default: the checkout's .manifest-cache)") ap.add_argument("--enforce", action="store_true", help="exit 1 when any path is uncovered") ap.add_argument("--json", action="store_true", help="machine-readable report") args = ap.parse_args(argv) @@ -56,19 +71,30 @@ def main(argv=None): report = run(manifest, tests) total = sum(len(v) for v in report.values()) hollow = empty(manifest, tests) + sources = args.sources if args.sources and os.path.isdir(args.sources) else None + rows = host_problems(manifest, sources) if args.json: print(json.dumps({"uncovered": report, "total": total, "tests": len(tests), - "empty": [list(e) for e in hollow]}, indent=1, sort_keys=True)) + "empty": [list(e) for e in hollow], + "host_tested": [list(r) for r in rows], + "workflows_read": sources is not None and not any("cannot be read" in r[3] for r in rows)}, + indent=1, sort_keys=True)) else: + if _manifest.HOST_TESTED and sources is None: + print("host-tested: no sources, so whether a workflow runs each host test is not checked") for comp in sorted(report): print("%s: %d uncovered path(s)" % (comp, len(report[comp]))) for p in report[comp]: print(" %s" % p) for tid, comp, pat in hollow: print("%s: covers (%s, %s) selects nothing" % (tid, comp, pat)) - print("coverage: %d uncovered path(s) across %d component(s), %d empty entr%s, %d tests" - % (total, len(report), len(hollow), "y" if len(hollow) == 1 else "ies", len(tests))) - if args.enforce and (total or hollow): + for comp, pat, ref, why in rows: + print("host-tested (%s, %s) by %s: %s" % (comp, pat, ref, why)) + print("coverage: %d uncovered path(s) across %d component(s), %d empty entr%s, " + "%d host-tested row problem(s), %d tests" + % (total, len(report), len(hollow), "y" if len(hollow) == 1 else "ies", len(rows), + len(tests))) + if args.enforce and (total or hollow or rows): return 1 return 0 diff --git a/forgetest/forgetest/manifest.py b/forgetest/forgetest/manifest.py index 6aaf78c..f26dae6 100644 --- a/forgetest/forgetest/manifest.py +++ b/forgetest/forgetest/manifest.py @@ -19,6 +19,7 @@ import functools import hashlib import json import os +import posixpath import re DEFAULT_PATH = "/etc/forgefirm-manifest.json" @@ -110,6 +111,33 @@ BEHAVIORAL = [ ] +# Paths a host test in CI proves, which no catalog test covers. The catalog +# is for what needs the machine or an operator; software whose inputs can be +# emulated is proven on a host. A row is (component, glob, host test, CI +# workflow), the last two named as ":": a component of +# the manifest (its tests/ and .github/ files are listed there), or +# "forgefirm" for this repository (the null-sink harnesses under +# scripts/bench/, which grblhal-glowforge's workflow runs). The workflow +# must run the host test: name it, or discover it (a glob, unittest +# discover, pytest on its directory). A host-tested path counts as covered +# in the coverage lint, and like every path no test covers it is in no +# fingerprint: a change there makes no acceptance run necessary. A path a +# catalog test exercises on the hardware belongs in that test's covers, not +# here. +HOST_TESTED = [ +] + +# This repository's root, when the package runs from a checkout of it; a +# "forgefirm:" file is looked for there. Installed on an image, the package +# is not in a checkout, and such a reference is not checked. +_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +REPO_ROOT = _ROOT if os.path.isdir(os.path.join(_ROOT, "meta-forgefirm")) else None +# The pinned sources scripts/manifest-from-tree.py fetches (one bare repo +# per source URL); a component's workflow is read from there. Without them +# (on an image) whether a workflow runs a host test is not checked. +SOURCES = os.path.join(REPO_ROOT, ".manifest-cache") if REPO_ROOT else None + + def non_behavioral(comp, path, allow=NON_BEHAVIORAL): for c, pat in BEHAVIORAL: if c == comp and glob_to_regex(pat).match(path): @@ -216,18 +244,303 @@ def empty_covers(manifest, tests): return out -def coverage_report(manifest, tests, allow=NON_BEHAVIORAL): - """Which manifest paths no test covers. +def _ref_problem(manifest, ref, what, root): + """Why a ":" reference names no file, or None. A + component's file must be in the manifest; a "forgefirm:" one must exist + under `root`, and is not checked without one.""" + repo, sep, path = ref.partition(":") + if not sep or not path: + return "names no repository:path for its %s" % what + if repo == "forgefirm": + if root is not None and not os.path.isfile(os.path.join(root, path)): + return "no such %s" % what + return None + if repo in DEV_ONLY_COMPONENTS or manifest.files(repo) is None: + return "no repository %r for its %s" % (repo, what) + if path not in {p for p, _b in manifest.files(repo)}: + return "no such %s" % what + return None + + +def read_source(manifest, ref, root=REPO_ROOT, sources=SOURCES): + """(text, why) for a ":" file: a "forgefirm:" file + from `root`, a component's from its pinned blob in the bare repo + `sources` holds for its source URL. (None, None) when it cannot be read + here (no root, no sources); (None, why) when it should have been and + was not, which is a defect, never a pass.""" + import subprocess + repo, _sep, path = ref.partition(":") + if repo == "forgefirm": + if root is None: + return None, None + try: + with open(os.path.join(root, path), encoding="utf-8") as f: + return f.read(), None + except (OSError, UnicodeDecodeError) as e: + return None, "cannot be read: %s" % e + if not sources: + return None, None + comp = manifest.components.get(repo) or {} + blob = dict(manifest.files(repo) or []).get(path) + url = comp.get("source") + if not (blob and url): + return None, "cannot be read: no blob or source URL in the manifest" + cache = os.path.join(sources, hashlib.sha1(url.encode()).hexdigest()[:16]) + try: + return subprocess.run(["git", "cat-file", "-p", blob], cwd=cache, capture_output=True, + check=True).stdout.decode("utf-8", "replace"), None + except (OSError, subprocess.CalledProcessError) as e: + return None, "cannot be read from %s: %s" % (cache, e) + + +def repo_name(manifest, repo): + """A repository's GitHub name: its source URL's last part, or this + repository's.""" + if repo == "forgefirm": + return "forgefirm" + url = ((manifest.components.get(repo) or {}).get("source") or "").rstrip("/") + name = url.rsplit("/", 1)[-1] + return name[:-4] if name.endswith(".git") else name + + +_STEP_KEYS = ("name", "uses", "run", "id", "if", "with", "env", "working-directory", "shell", + "continue-on-error", "timeout-minutes") +_STEP_RX = re.compile(r"^(\s*)-\s+(%s)\s*:" % "|".join(re.escape(k) for k in _STEP_KEYS)) +_KEY_RX = re.compile(r"^(\s*)([\w-]+)\s*:\s*(.*)$") + + +def _indent(line): + return len(line) - len(line.lstrip(" ")) + + +def workflow_steps(text): + """A workflow's steps, each a dict of its keys: "uses", "run" (the + command text, a block's lines joined), "working-directory", and "with" + (a dict). YAML read by its indentation, as far as these workflows use + it; comments, whole lines and from ' #' to the end of a line, are not + read. Anything outside a step (a trigger's paths, a job's name) is not + a step.""" + lines = [] + for ln in text.splitlines(): + if ln.lstrip().startswith("#"): + continue + lines.append(re.sub(r"\s+#.*$", "", ln).rstrip()) + steps, i = [], 0 + while i < len(lines): + m = _STEP_RX.match(lines[i]) + if not m: + i += 1 + continue + dash = len(m.group(1)) + body = [" " * (dash + 2) + lines[i][dash + 1:].lstrip()] + i += 1 + while i < len(lines) and (not lines[i].strip() or _indent(lines[i]) > dash): + body.append(lines[i]) + i += 1 + step, k = {}, 0 + key_col = _indent(body[0]) + while k < len(body): + km = _KEY_RX.match(body[k]) + if not km or _indent(body[k]) != key_col: + k += 1 + continue + key, val = km.group(2), km.group(3).strip() + if len(val) > 1 and val[0] == val[-1] and val[0] in "'\"": + val = val[1:-1] # a quoted scalar, not a quote inside a command + k += 1 + child = [] + while k < len(body) and (not body[k].strip() or _indent(body[k]) > key_col): + child.append(body[k]) + k += 1 + if key == "with": + step["with"] = {c.group(2): c.group(3).strip().strip("'\"") + for c in (_KEY_RX.match(x) for x in child) if c} + elif key == "run": + step["run"] = "\n".join(x.strip() for x in child) if val[:1] in ("|", ">") else val + else: + step[key] = val + steps.append(step) + return steps + + +def checkout_path(steps, name, own): + """Where a workflow checks the repository `name` out (its path, "" for + the workspace), or None when it does not: its own repository is the + checkout that names no repository, another one the checkout whose + repository is /.""" + for s in steps: + if not s.get("uses", "").startswith("actions/checkout"): + continue + w = s.get("with", {}) + repo = w.get("repository") + if (repo is None and own) or (repo is not None and repo.rsplit("/", 1)[-1].lower() == name.lower()): + return posixpath.normpath(w.get("path") or ".") + return None + + +_SPLIT_RX = re.compile(r"&&|\|\||[;|\n]|(?:^|(?<=\s))(?:then|do|else)(?=\s|$)") +_PREFIXES = ("sudo", "env", "time", "exec", "command", "nice") +_INTERPRETER_RX = re.compile(r"^(?:python[0-9.]*|bash|sh)$") + + +def _commands(run): + """The simple commands of a step's run text, each as its words, with + what only starts the real command taken off: sudo and its flags, env, + time, exec, and variable assignments.""" + import shlex + out = [] + for part in _SPLIT_RX.split(run.replace("\\\n", " ")): + try: + words = shlex.split(part) + except ValueError: + words = part.split() + while words: + if words[0] in _PREFIXES: + words = words[1:] + while words and words[0].startswith("-"): + words = words[1:] + elif re.match(r"^[A-Za-z_]\w*=", words[0]): + words = words[1:] + else: + break + if words: + out.append(words) + return out + + +def _segments_match(path, pattern): + """A path against a path or a glob, one segment against one segment + ('*' never crosses a '/').""" + import fnmatch + a, b = path.split("/"), pattern.split("/") + return len(a) == len(b) and all(fnmatch.fnmatchcase(x, y) for x, y in zip(a, b)) + + +def _discovers(args, full, at): + """Whether `python -m unittest discover ARGS` finds the test: its + directory is the start directory (-s, default the working directory) + and its file name matches the pattern (-p, default test*.py).""" + import fnmatch + start, pattern = ".", "test*.py" + for i, a in enumerate(args): + if a in ("-s", "--start-directory") and i + 1 < len(args): + start = args[i + 1] + elif a in ("-p", "--pattern") and i + 1 < len(args): + pattern = args[i + 1] + return posixpath.dirname(full) == at(start) and fnmatch.fnmatchcase(posixpath.basename(full), pattern) + + +def _pytest_runs(args, full, at): + """Whether pytest ARGS collects the test: named, or its directory given + and its file name one pytest collects (test_*.py, *_test.py).""" + import fnmatch + name = posixpath.basename(full) + collected = fnmatch.fnmatchcase(name, "test_*.py") or fnmatch.fnmatchcase(name, "*_test.py") + for a in args: + if a.startswith("-") or "$" in a: + continue + if at(a) == full or (collected and at(a) == posixpath.dirname(full)): + return True + return False + + +def workflow_runs(steps, test_path): + """Whether a step of the workflow runs the test at test_path (its path + in the workspace: the checkout's path joined to the test's path in its + repository). Only what a step's run commands execute counts, each path + resolved against the step's working-directory: the command itself + (a script run directly, or a C test's built binary, /); + the script an interpreter runs (python, bash, sh), a glob matching the + whole path; a for loop's list over the test; unittest discover from + the test's own directory with a pattern its name matches; pytest on + the test or its own directory. An argument to anything else (echo, a + build, an install) does not run the test.""" + full = posixpath.normpath(test_path) + stem, ext = posixpath.splitext(posixpath.basename(full)) + compiled = ext in (".c", ".cc", ".cpp") + for s in steps: + run = s.get("run") + if not run: + continue + wd = s.get("working-directory") or "." + + def at(p): + return posixpath.normpath(posixpath.join(wd, p)) + + def names(p): + return "$" not in p and _segments_match(full, at(p)) + for words in _commands(run): + cmd, args = words[0], words[1:] + if cmd == "for" and "in" in args: + if any(names(p) for p in args[args.index("in") + 1:]): + return True + elif _INTERPRETER_RX.match(posixpath.basename(cmd)): + if "-m" in args: + i = args.index("-m") + mod, rest = (args[i + 1], args[i + 2:]) if i + 1 < len(args) else ("", []) + if (mod == "unittest" and rest[:1] == ["discover"] and _discovers(rest[1:], full, at)) or \ + (mod == "pytest" and _pytest_runs(rest, full, at)): + return True + else: + script = next((a for a in args if not a.startswith("-")), None) + if script and names(script): + return True + elif posixpath.basename(cmd) == "pytest": + if _pytest_runs(args, full, at): + return True + elif names(cmd) or (compiled and "/" in cmd and posixpath.basename(cmd) == stem): + return True + return False + + +def host_row_problems(manifest, host=HOST_TESTED, root=REPO_ROOT, sources=SOURCES): + """HOST_TESTED rows that prove nothing: a glob that selects no + behavioral file of its component (the same defect as an empty covers + entry), a host test or a workflow that does not exist, a workflow that + cannot be read where it should be, one that does not check out the + test's repository, or one with no step that runs the test. Whether a + workflow runs its test is not checked where it cannot be read (see + read_source). Returns [(component, glob, host test, why)].""" + out = [] + for comp, pat, ref, ci in host: + files = manifest.files(comp) + if files is None or not [p for p, _b in match_files(files, pat) if not non_behavioral(comp, p)]: + out.append((comp, pat, ref, "selects nothing")) + why = _ref_problem(manifest, ref, "host test", root) or _ref_problem(manifest, ci, "workflow", root) + if why: + out.append((comp, pat, ref, why)) + continue + text, why = read_source(manifest, ci, root, sources) + if why: + out.append((comp, pat, ref, "%s %s" % (ci, why))) + continue + if text is None: + continue + steps = workflow_steps(text) + test_repo, _sep, test_path = ref.partition(":") + prefix = checkout_path(steps, repo_name(manifest, test_repo), test_repo == ci.partition(":")[0]) + if prefix is None: + out.append((comp, pat, ref, "%s does not check out %s" % (ci, test_repo))) + elif not workflow_runs(steps, posixpath.join(prefix, test_path)): + out.append((comp, pat, ref, "%s does not run it" % ci)) + return out + + +def coverage_report(manifest, tests, allow=NON_BEHAVIORAL, host=HOST_TESTED): + """Which manifest paths neither a test nor a host test covers. tests: iterable with .covers. allow: iterable of (component, glob) that need no coverage (docs, CI, licenses...); a BEHAVIORAL path is - never allowed away, as it is never taken out of a fingerprint. Returns + never allowed away, as it is never taken out of a fingerprint. host: + the HOST_TESTED rows, whose paths count as covered. Returns {component: [uncovered paths]} for the non-dev-only components. """ covered = {} for t in tests: for comp, pat in t.covers: covered.setdefault(comp, []).append(glob_to_regex(pat)) + for comp, pat, _ref, _ci in host: + covered.setdefault(comp, []).append(glob_to_regex(pat)) allowed = {} for comp, pat in allow: allowed.setdefault(comp, []).append(glob_to_regex(pat)) diff --git a/forgetest/tests/test_manifest.py b/forgetest/tests/test_manifest.py index 05d8881..4c15f47 100644 --- a/forgetest/tests/test_manifest.py +++ b/forgetest/tests/test_manifest.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: MIT import os +import sys import unittest import helpers @@ -169,6 +170,216 @@ class CoverageReportTests(unittest.TestCase): docs = helpers.make_test("a.three", [("forgectrl", "**/*.md")]) self.assertEqual(m.empty_covers(man, [docs]), [("a.three", "forgectrl", "**/*.md")]) + def test_a_host_tested_path_is_covered_and_in_no_fingerprint(self): + man = helpers.with_file(helpers.make_manifest(), "forgectrl", "tests/cool_test.c", "test") + t = helpers.make_test("a.one", [("forgectrl", "src/ui.c"), ("forgectrl", "src/auth.c"), + ("grblhal-glowforge", "**"), ("kernel-module-glowforge", "**"), + ("linux-fslc", "**")]) + self.assertEqual(m.coverage_report(man, [t], host=[]), {"forgectrl": ["src/cool.c", "src/main.c"]}) + ci = "forgectrl:.github/workflows/build.yml" + host = [("forgectrl", "src/cool.c", "forgectrl:tests/cool_test.c", ci), + ("forgectrl", "src/main.c", "forgectrl:tests/cool_test.c", ci)] + self.assertEqual(m.coverage_report(man, [t], host=host), {}) + # a change to a host-tested path makes no acceptance run necessary + self.assertEqual(m.fingerprint(man, t.covers), + m.fingerprint(helpers.with_file(man, "forgectrl", "src/cool.c", "changed"), t.covers)) + + def _root(self, files): + """A scratch checkout of this repository: {path: text}.""" + import shutil + import tempfile + root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, root) + for path, text in files.items(): + os.makedirs(os.path.dirname(os.path.join(root, path)), exist_ok=True) + with open(os.path.join(root, path), "w", encoding="utf-8") as f: + f.write(text) + return root + + # A workflow of this repository as the scratch checkouts carry it: it + # checks itself out at the workspace root and runs the one harness. + CI_YML = ("on:\n push:\n paths:\n - 'scripts/bench/drill.py'\n" + "jobs:\n test:\n steps:\n" + " - uses: actions/checkout@v4\n" + " # drill.py is a bench tool\n" + " - name: The stream\n" + " run: python3 scripts/bench/stream_test.py # not drill.py\n") + + def test_a_host_tested_row_selects_a_path_and_names_a_real_test_and_workflow(self): + man = helpers.make_manifest() + for path in ("tests/cool_test.c", ".github/workflows/build.yml"): + man = helpers.with_file(man, "forgectrl", path, path) + root = self._root({"scripts/bench/stream_test.py": "", ".github/workflows/ci.yml": self.CI_YML}) + ci = "forgectrl:.github/workflows/build.yml" + good = [("forgectrl", "src/cool.c", "forgectrl:tests/cool_test.c", ci), + ("grblhal-glowforge", "src/driver.c", "forgefirm:scripts/bench/stream_test.py", + "forgefirm:.github/workflows/ci.yml")] + self.assertEqual(m.host_row_problems(man, good, root=root, sources=None), []) + bad = [("forgectrl", "cool.c", "forgectrl:tests/cool_test.c", ci), # globs anchor at the root + ("forgectrl", "**/*.md", "forgectrl:tests/cool_test.c", ci), # docs only + ("forgectrl", "src/main.c", "forgectrl:tests/gone_test.c", ci), + ("forgectrl", "src/main.c", "tests/cool_test.c", ci), + ("forgectrl", "src/main.c", "forgetest:forgetest/x.py", ci), # the acceptance tool itself + ("forgectrl", "src/main.c", "forgefirm:scripts/bench/gone_test.py", ci), + ("forgectrl", "src/main.c", "forgectrl:tests/cool_test.c", "forgectrl:.github/workflows/gone.yml"), + ("forgectrl", "src/main.c", "forgectrl:tests/cool_test.c", "build.yml")] + self.assertEqual([r[3] for r in m.host_row_problems(man, bad, root=root, sources=None)], + ["selects nothing", "selects nothing", "no such host test", + "names no repository:path for its host test", "no repository 'forgetest' for its host test", + "no such host test", "no such workflow", "names no repository:path for its workflow"]) + # installed on an image the package is in no checkout: a "forgefirm:" file is not looked for + self.assertEqual(m.host_row_problems(man, bad[5:6], root=None, sources=None), []) + + def test_the_workflow_has_to_run_the_host_test(self): + man = helpers.make_manifest() + root = self._root({"scripts/bench/stream_test.py": "", "scripts/bench/drill.py": "", + ".github/workflows/ci.yml": self.CI_YML}) + ci = "forgefirm:.github/workflows/ci.yml" + runs = ("grblhal-glowforge", "src/driver.c", "forgefirm:scripts/bench/stream_test.py", ci) + # drill.py is in the trigger's paths, a comment line and an inline comment, and run by no step + idle = ("grblhal-glowforge", "src/driver.c", "forgefirm:scripts/bench/drill.py", ci) + self.assertEqual(m.host_row_problems(man, [runs], root=root, sources=None), []) + self.assertEqual(m.host_row_problems(man, [idle], root=root, sources=None), + [(idle[0], idle[1], idle[2], ci + " does not run it")]) + + def _runs(self, yml, path): + return m.workflow_runs(m.workflow_steps(yml), path) + + def test_what_running_a_host_test_looks_like(self): + step = "steps:\n - name: x\n run: %s\n" + self.assertTrue(self._runs(step % "./build/tray_test", "tests/tray_test.c")) + self.assertTrue(self._runs(step % "sudo ./build/sandbox_test", "tests/sandbox_test.c")) + self.assertTrue(self._runs(step % "python3 -B tests/run_test.py", "tests/run_test.py")) + self.assertTrue(self._runs(step % "python3 forgefirm/scripts/bench/ctlport_test.py build/x", + "forgefirm/scripts/bench/ctlport_test.py")) + self.assertTrue(self._runs(step % "python -m pytest -q tests", "tests/test_frame.py")) + # a block, with the step's working-directory: the words are its paths + block = ("steps:\n - name: every module\n working-directory: %s\n run: |\n" + " set -e\n for t in tests/test_*.py; do\n python -m unittest \"tests.$t\"\n" + " done\n - working-directory: forgefirm/forgetest\n" + " run: python -B -m unittest discover -s tests -v\n") + self.assertTrue(self._runs(block % "gfh", "gfh/tests/test_feeder.py")) + self.assertTrue(self._runs(block % "gfh", "forgefirm/forgetest/tests/test_x.py")) + self.assertFalse(self._runs(block % "other", "gfh/tests/test_feeder.py")) + # what the review found accepted: a trigger's paths, an inline comment, a glob or a discover + # that the path only ends like + self.assertFalse(self._runs("on:\n push:\n paths:\n - 'kas/source-bundle.yml'\n" + + step % "python3 x.py", "kas/source-bundle.yml")) + self.assertFalse(self._runs(step % "./build/lens_test # tray_test is manual", "tests/tray_test.c")) + self.assertFalse(self._runs(step % "for t in tests/test_*.py; do python $t; done", + "tools/tests/test_feeder.py")) + self.assertFalse(self._runs(step % "python -m unittest discover -s tests", + "scripts/bench/tests/foo_test.py")) + # a longer name, another file, a step's name, a comment line + self.assertFalse(self._runs(step % "./build/tray_test_more", "tests/tray_test.c")) + self.assertFalse(self._runs(step % "./build/lens_test", "tests/fflog_e2e.sh")) + self.assertFalse(self._runs("steps:\n - name: tests/run_test.py\n run: make\n", "tests/run_test.py")) + self.assertFalse(self._runs("steps:\n - run: |\n # python3 tests/run_test.py\n make\n", + "tests/run_test.py")) + # the second review: an argument to a command that is not the test does not run it, and a + # discover or pytest runs its own directory's matching files only + self.assertFalse(self._runs(step % "echo tray_test", "tests/tray_test.c")) + self.assertFalse(self._runs(step % "echo ./build/tray_test", "tests/tray_test.c")) + self.assertFalse(self._runs(step % "cmake --build build --target tray_test", "tests/tray_test.c")) + self.assertFalse(self._runs(step % "cat tests/run_test.py", "tests/run_test.py")) + self.assertFalse(self._runs(step % "python -m unittest discover -s forgefirm", + "forgefirm/scripts/bench/fire_test.py")) + self.assertFalse(self._runs(step % "python -m unittest discover -s tests", "tests/foo_test.py")) + self.assertFalse(self._runs(step % "python -m pytest forgefirm", "forgefirm/tests/test_a.py")) + self.assertTrue(self._runs(step % "python -m unittest discover -s tests -p '*_test.py'", "tests/foo_test.py")) + self.assertTrue(self._runs(step % "pytest -q tests/test_a.py", "tests/test_a.py")) + # a script run directly or by its shell, and a command continued across lines + self.assertTrue(self._runs(step % "./tests/fflog_e2e.sh", "tests/fflog_e2e.sh")) + self.assertTrue(self._runs(step % "bash tests/fflog_e2e.sh", "tests/fflog_e2e.sh")) + self.assertTrue(self._runs("steps:\n - run: |\n sudo -E FORGEEXT=build/forgeext \\\n" + " python3 -B tests/run_test.py\n", "tests/run_test.py")) + + def test_the_lint_says_when_it_could_not_read_a_workflow(self): + import contextlib + import io + import json + import tempfile + from unittest import mock + from forgetest import coverage + work = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, work) + comps = {"forgectrl": {"srcrev": "aaa", "source": "https://example.invalid/forgectrl.git", "files": [ + ["src/tray.c", "1" * 40], ["tests/tray_test.c", "2" * 40], [".github/workflows/build.yml", "3" * 40]]}} + path = os.path.join(work, "manifest.json") + with open(path, "w", encoding="utf-8") as f: + f.write(json.dumps(helpers.make_manifest(components=comps).data)) + row = ("forgectrl", "src/tray.c", "forgectrl:tests/tray_test.c", "forgectrl:.github/workflows/build.yml") + out = io.StringIO() + with mock.patch.object(m, "HOST_TESTED", [row]), contextlib.redirect_stdout(out): + rc = coverage.main(["--manifest", path, "--sources", work, "--json"]) + self.assertEqual(rc, 0) # reporting only + rep = json.loads(out.getvalue()) + self.assertFalse(rep["workflows_read"]) + self.assertTrue(any("cannot be read" in r[3] for r in rep["host_tested"])) + out = io.StringIO() + with mock.patch.object(m, "HOST_TESTED", [row]), contextlib.redirect_stdout(out): + self.assertEqual(coverage.main(["--manifest", path, "--sources", work, "--enforce"]), 1) + + def test_where_a_workflow_checks_a_repository_out(self): + steps = m.workflow_steps("jobs:\n t:\n steps:\n" + " - uses: actions/checkout@v4\n with:\n path: own\n" + " - name: Fetch the harnesses\n uses: actions/checkout@v4\n" + " with:\n repository: openglow-org/forgefirm\n path: forgefirm\n") + self.assertEqual(m.checkout_path(steps, "grblHAL-glowforge", True), "own") + self.assertEqual(m.checkout_path(steps, "forgefirm", False), "forgefirm") + self.assertIsNone(m.checkout_path(steps, "forgectrl", False)) + self.assertEqual(m.checkout_path(m.workflow_steps("steps:\n - uses: actions/checkout@v4\n"), "x", True), ".") + self.assertEqual(m.repo_name(helpers.make_manifest(components={"python3-gfutilities": { + "source": "https://github.com/openglow-org/Glowforge-Utilities.git", "files": []}}), + "python3-gfutilities"), "Glowforge-Utilities") + + def test_a_components_workflow_is_read_at_its_pinned_blob(self): + import hashlib + import shutil + import subprocess + import tempfile + if not shutil.which("git"): + self.skipTest("no git") + sources = tempfile.mkdtemp() + + def writable(func, path, _exc): # git writes its objects read-only + os.chmod(path, 0o700) + func(path) + self.addCleanup(shutil.rmtree, sources, **({"onexc": writable} if sys.version_info >= (3, 12) + else {"onerror": writable})) + url = "https://example.invalid/forgectrl.git" + cache = os.path.join(sources, hashlib.sha1(url.encode()).hexdigest()[:16]) + os.makedirs(cache) + subprocess.run(["git", "init", "-q", "--bare"], cwd=cache, check=True) + + def blob(text): + return subprocess.run(["git", "hash-object", "-w", "--stdin"], cwd=cache, input=text.encode(), + capture_output=True, check=True).stdout.decode().strip() + yml = "steps:\n - uses: actions/checkout@v4\n - run: ./build/tray_test\n" + comps = {"forgectrl": {"srcrev": "aaa", "source": url, "files": [ + ["src/tray.c", "1" * 40], ["tests/tray_test.c", "2" * 40], ["tests/lens_test.c", "3" * 40], + [".github/workflows/build.yml", blob(yml)], [".github/workflows/lost.yml", "4" * 40]]}} + man = helpers.make_manifest(components=comps) + ci = "forgectrl:.github/workflows/build.yml" + self.assertEqual(m.read_source(man, ci, root=None, sources=sources), (yml, None)) + tray = ("forgectrl", "src/tray.c", "forgectrl:tests/tray_test.c", ci) + lens = ("forgectrl", "src/tray.c", "forgectrl:tests/lens_test.c", ci) + self.assertEqual(m.host_row_problems(man, [tray, lens], root=None, sources=sources), + [(lens[0], lens[1], lens[2], ci + " does not run it")]) + # without the sources the workflow cannot be read, and is not judged + self.assertEqual(m.host_row_problems(man, [lens], root=None, sources=None), []) + # with them, a workflow that should be read and is not is a problem, never a pass + lost = ("forgectrl", "src/tray.c", "forgectrl:tests/tray_test.c", "forgectrl:.github/workflows/lost.yml") + probs = m.host_row_problems(man, [lost], root=None, sources=sources) + self.assertEqual(len(probs), 1) + self.assertIn("cannot be read", probs[0][3]) + + def test_the_real_host_tested_rows_are_well_formed(self): + for row in m.HOST_TESTED: + self.assertEqual(len(row), 4, row) + self.assertIn(":", row[2], row) + self.assertIn(":", row[3], row) + class CatalogTests(unittest.TestCase): def test_catalog_hash_is_definition_only(self):