"""Controls for the instrument that certifies every other falsifier. `scripts/mutation_check.py` exists because a green test proves nothing until it has seen the change it forbids. The same sentence applies to the tool: it shipped two defects in one session, each of which made it report a falsifier PROVED WITHOUT RUNNING IT (no green baseline; the pyc cache silently reverting byte-identical mutations). Both were found by accident. So the tool gets what CLAUDE.md demands of any measurement: a POSITIVE CONTROL it must detect, and a NEGATIVE CONTROL it must not fire on. An instrument that only ever sees unknowns cannot distinguish "absent" from "blind". """ from __future__ import annotations import pathlib import sys sys.path.insert(0, str(pathlib.Path(__file__).parent.parent / "scripts")) from mutation_check import check # noqa: E402 def _tree(tmp_path, source: str, test_body: str): """A throwaway repo: one module, one test file, both real on disk.""" (tmp_path / "mod.py").write_text(source) (tmp_path / "test_probe.py").write_text( "import sys, pathlib\n" "sys.path.insert(0, str(pathlib.Path(__file__).parent))\n" "from mod import f\n\n" + test_body ) return tmp_path def test_a_real_falsifier_is_reported_proved(tmp_path): """NEGATIVE CONTROL — the tool must not cry wolf on a sound test. `f` returns 2; the test asserts it. Flipping the constant must go red, and the tool must say so.""" repo = _tree(tmp_path, "def f():\n return 2\n", "def test_f():\n assert f() == 2\n") proved, note = check( {"label": "flip the constant", "file": "mod.py", "test": "test_probe.py::test_f", "old": "return 2", "new": "return 3"}, repo=repo) assert proved, note def test_a_vacuous_falsifier_is_caught(tmp_path): """POSITIVE CONTROL — the one that matters, and the one usually skipped. The test asserts only that `f()` is an int, so flipping the constant does NOT break it. The test cites the behaviour without forbidding it. The tool must report NOT PROVED; if it cannot detect a known-vacuous falsifier, its twelve `proved` lines are worth nothing.""" repo = _tree(tmp_path, "def f():\n return 2\n", "def test_f():\n assert isinstance(f(), int)\n") proved, note = check( {"label": "flip the constant", "file": "mod.py", "test": "test_probe.py::test_f", "old": "return 2", "new": "return 3"}, repo=repo) assert not proved assert "VACUOUS" in note def test_an_already_red_test_is_a_harness_failure_not_a_proof(tmp_path): """DEFECT 1, as a control. Before the baseline check this returned PROVED — a broken assertion reading as a certified falsifier.""" repo = _tree(tmp_path, "def f():\n return 2\n", "def test_f():\n assert f() == 99\n") proved, note = check( {"label": "flip the constant", "file": "mod.py", "test": "test_probe.py::test_f", "old": "return 2", "new": "return 3"}, repo=repo) assert not proved assert "BASELINE RED" in note def test_a_same_size_mutation_is_not_swallowed_by_the_bytecode_cache(tmp_path): """DEFECT 2, as a control. `< 2` -> `< 1` is byte-identical in size, so a mutation landing in the same mtime second as the revert before it used to run against cached bytecode and report PROVED having tested nothing. Run twice: the verdict must be stable. The original defect's tell was exactly a verdict that flipped between consecutive identical runs.""" repo = _tree(tmp_path, "def f(n):\n return n < 2\n", "def test_f():\n assert f(1) is True and f(2) is False\n") m = {"label": "off by one", "file": "mod.py", "test": "test_probe.py::test_f", "old": "return n < 2", "new": "return n < 1"} assert [check(m, repo=repo)[0] for _ in range(2)] == [True, True] def test_a_drifted_anchor_is_reported_not_skipped(tmp_path): """A table whose `old` no longer matches the source stops proving anything. Silently skipping it would shrink the denominator and keep the run green.""" repo = _tree(tmp_path, "def f():\n return 2\n", "def test_f():\n assert f() == 2\n") proved, note = check( {"label": "stale", "file": "mod.py", "test": "test_probe.py::test_f", "old": "return 2222", "new": "return 3"}, repo=repo) assert not proved assert "anchor not found" in note def test_the_source_is_restored_even_when_the_mutation_proves(tmp_path): """The tool writes to tracked source files. Leaving one mutated would put a defect in the tree that looks like authored code.""" repo = _tree(tmp_path, "def f():\n return 2\n", "def test_f():\n assert f() == 2\n") before = (repo / "mod.py").read_text() check({"label": "flip", "file": "mod.py", "test": "test_probe.py::test_f", "old": "return 2", "new": "return 3"}, repo=repo) assert (repo / "mod.py").read_text() == before def test_a_reverted_file_keeps_its_mtime(tmp_path): """The repo IS its own deployment root: nothing takes effect until the service restarts, so "is :8090 stale?" is answered by comparing the service's start time against source mtimes. A tool that rewrites a file with identical bytes still bumps its mtime and makes that check lie — it reported the live service 16 minutes stale when it was current. Defeating change: dropping the os.utime in the restore.""" import os repo = _tree(tmp_path, "def f():\n return 2\n", "def test_f():\n assert f() == 2\n") mod = repo / "mod.py" os.utime(mod, (1_000_000_000, 1_000_000_000)) before = mod.stat().st_mtime_ns check({"label": "flip", "file": "mod.py", "test": "test_probe.py::test_f", "old": "return 2", "new": "return 3"}, repo=repo) assert mod.stat().st_mtime_ns == before