"""Controls for the instrument that certifies every other falsifier. `scripts/mutation_check.py` exists because a green test proves nothing until it has seen the change it forbids. The same sentence applies to the tool: it shipped two defects in one session, each of which made it report a falsifier PROVED WITHOUT RUNNING IT (no green baseline; the pyc cache silently reverting byte-identical mutations). Both were found by accident. So the tool gets what CLAUDE.md demands of any measurement: a POSITIVE CONTROL it must detect, and a NEGATIVE CONTROL it must not fire on. An instrument that only ever sees unknowns cannot distinguish "absent" from "blind". """ from __future__ import annotations import pathlib import sys sys.path.insert(0, str(pathlib.Path(__file__).parent.parent / "scripts")) from mutation_check import check # noqa: E402 def _tree(tmp_path, source: str, test_body: str): """A throwaway repo: one module, one test file, both real on disk.""" (tmp_path / "mod.py").write_text(source) (tmp_path / "test_probe.py").write_text( "import sys, pathlib\n" "sys.path.insert(0, str(pathlib.Path(__file__).parent))\n" "from mod import f\n\n" + test_body ) return tmp_path def test_a_real_falsifier_is_reported_proved(tmp_path): """NEGATIVE CONTROL — the tool must not cry wolf on a sound test. `f` returns 2; the test asserts it. Flipping the constant must go red, and the tool must say so.""" repo = _tree(tmp_path, "def f():\n return 2\n", "def test_f():\n assert f() == 2\n") proved, note = check( {"label": "flip the constant", "file": "mod.py", "test": "test_probe.py::test_f", "old": "return 2", "new": "return 3"}, repo=repo) assert proved, note def test_a_vacuous_falsifier_is_caught(tmp_path): """POSITIVE CONTROL — the one that matters, and the one usually skipped. The test asserts only that `f()` is an int, so flipping the constant does NOT break it. The test cites the behaviour without forbidding it. The tool must report NOT PROVED; if it cannot detect a known-vacuous falsifier, its twelve `proved` lines are worth nothing.""" repo = _tree(tmp_path, "def f():\n return 2\n", "def test_f():\n assert isinstance(f(), int)\n") proved, note = check( {"label": "flip the constant", "file": "mod.py", "test": "test_probe.py::test_f", "old": "return 2", "new": "return 3"}, repo=repo) assert not proved assert "VACUOUS" in note def test_an_already_red_test_is_a_harness_failure_not_a_proof(tmp_path): """DEFECT 1, as a control. Before the baseline check this returned PROVED — a broken assertion reading as a certified falsifier.""" repo = _tree(tmp_path, "def f():\n return 2\n", "def test_f():\n assert f() == 99\n") proved, note = check( {"label": "flip the constant", "file": "mod.py", "test": "test_probe.py::test_f", "old": "return 2", "new": "return 3"}, repo=repo) assert not proved assert "BASELINE RED" in note def test_a_same_size_mutation_is_not_swallowed_by_the_bytecode_cache(tmp_path): """DEFECT 2, as a control. `< 2` -> `< 1` is byte-identical in size, so a mutation landing in the same mtime second as the revert before it used to run against cached bytecode and report PROVED having tested nothing. Run twice: the verdict must be stable. The original defect's tell was exactly a verdict that flipped between consecutive identical runs.""" repo = _tree(tmp_path, "def f(n):\n return n < 2\n", "def test_f():\n assert f(1) is True and f(2) is False\n") m = {"label": "off by one", "file": "mod.py", "test": "test_probe.py::test_f", "old": "return n < 2", "new": "return n < 1"} assert [check(m, repo=repo)[0] for _ in range(2)] == [True, True] def test_a_drifted_anchor_is_reported_not_skipped(tmp_path): """A table whose `old` no longer matches the source stops proving anything. Silently skipping it would shrink the denominator and keep the run green.""" repo = _tree(tmp_path, "def f():\n return 2\n", "def test_f():\n assert f() == 2\n") proved, note = check( {"label": "stale", "file": "mod.py", "test": "test_probe.py::test_f", "old": "return 2222", "new": "return 3"}, repo=repo) assert not proved assert "anchor not found" in note def test_the_source_is_restored_even_when_the_mutation_proves(tmp_path): """The tool writes to tracked source files. Leaving one mutated would put a defect in the tree that looks like authored code.""" repo = _tree(tmp_path, "def f():\n return 2\n", "def test_f():\n assert f() == 2\n") before = (repo / "mod.py").read_text() check({"label": "flip", "file": "mod.py", "test": "test_probe.py::test_f", "old": "return 2", "new": "return 3"}, repo=repo) assert (repo / "mod.py").read_text() == before def test_a_reverted_file_keeps_its_mtime(tmp_path): """The repo IS its own deployment root: nothing takes effect until the service restarts, so "is :8090 stale?" is answered by comparing the service's start time against source mtimes. A tool that rewrites a file with identical bytes still bumps its mtime and makes that check lie — it reported the live service 16 minutes stale when it was current. Defeating change: dropping the os.utime in the restore.""" import os repo = _tree(tmp_path, "def f():\n return 2\n", "def test_f():\n assert f() == 2\n") mod = repo / "mod.py" os.utime(mod, (1_000_000_000, 1_000_000_000)) before = mod.stat().st_mtime_ns check({"label": "flip", "file": "mod.py", "test": "test_probe.py::test_f", "old": "return 2", "new": "return 3"}, repo=repo) assert mod.stat().st_mtime_ns == before def test_an_ambiguous_anchor_is_reported_not_guessed(tmp_path): """An `old` that matches twice mutates whichever comes FIRST, so the row proves or fails by where the first match happens to fall rather than by the line it names. design-dev found two r2b rows proving that way (2026-09-28). A row must name exactly one place.""" repo = _tree(tmp_path, "def f():\n x = 2\n x = 2\n return x\n", "def test_f():\n assert f() == 2\n") proved, note = check( {"label": "twice", "file": "mod.py", "test": "test_probe.py::test_f", "old": " x = 2\n", "new": " x = 3\n"}, repo=repo) assert not proved assert "ambiguous" in note # ---- SPYRJA (heid bug-hunt, hulda, 2026-09-28): the instrument's own edges -- def test_a_mutation_that_stops_the_test_running_is_not_a_proof(tmp_path): """A mutation that breaks collection (a syntax error) exits non-zero without the assertion ever running. `rc != 0` certified that as PROVED: the tool's one claim, that the test caught the change, was never tested.""" repo = _tree(tmp_path, "def f():\n return 2\n", "def test_f():\n assert f() == 2\n") proved, note = check( {"label": "syntax", "file": "mod.py", "test": "test_probe.py::test_f", "old": "return 2", "new": "return ("}, repo=repo) assert not proved assert "did not run" in note def test_an_overlapping_anchor_is_ambiguous_too(tmp_path): """`str.count` counts NON-overlapping matches: `"aaa".count("aa") == 1` while `aa` starts at two places. The ambiguity guard must see both.""" repo = _tree(tmp_path, 'def f():\n return len("aaa")\n', "def test_f():\n assert f() == 3\n") proved, note = check( {"label": "overlap", "file": "mod.py", "test": "test_probe.py::test_f", "old": "aa", "new": "b"}, repo=repo) assert not proved assert "ambiguous" in note def test_a_crlf_source_is_restored_byte_for_byte(tmp_path): """Text-mode I/O read CRLF as LF and wrote LF back, then compared LF with LF and called it restored — and reset the mtime so nothing looked touched.""" repo = _tree(tmp_path, "def f():\n return 2\n", "def test_f():\n assert f() == 2\n") mod = repo / "mod.py" mod.write_bytes(b"def f():\r\n return 2\r\n") before = mod.read_bytes() check({"label": "flip", "file": "mod.py", "test": "test_probe.py::test_f", "old": "return 2", "new": "return 3"}, repo=repo) assert mod.read_bytes() == before def test_a_mutation_that_hangs_is_stopped_and_the_source_restored(tmp_path): """No timeout meant a mutation that loops forever held the checker — and the mutated file — until someone killed it by hand.""" repo = _tree(tmp_path, "def f():\n return 2\n", "def test_f():\n assert f() == 2\n") before = (repo / "mod.py").read_text() proved, note = check( {"label": "hang", "file": "mod.py", "test": "test_probe.py::test_f", "old": "return 2", "new": "while True: pass"}, repo=repo, timeout=10) assert not proved assert "timed out" in note.lower() assert (repo / "mod.py").read_text() == before def test_a_second_run_is_refused_while_one_holds_the_repo(tmp_path, monkeypatch): """Two checkers interleaving on one tree can restore each other's mutation back in. One run at a time, by lock.""" import fcntl import mutation_check as mc monkeypatch.setattr(mc, "REPO", tmp_path) monkeypatch.setattr(mc, "INFLIGHT", tmp_path / ".mutation-inflight") monkeypatch.setattr(mc, "TABLES", tmp_path / "tables") (tmp_path / "tables").mkdir() with open(tmp_path / ".mutation-lock", "w") as held: fcntl.flock(held, fcntl.LOCK_EX | fcntl.LOCK_NB) assert mc.main(["mutation_check.py"]) == 2