#!/usr/bin/env python3 """Prove a falsifier falsifies, by running the change it forbids. A green test is not evidence. A test that has never seen its own DEFEATING CHANGE is only evidence that the code and the assertion agree today; it may agree under the mutation too, in which case it forbids nothing and reads as though it forbids something. This repo has shipped that three times -- persistent-memory.d/2026-09-22-vacuous-falsifiers.md, persistent-memory.d/2026-09-22-seven-of-seven-falsifiers.md, and once more in U7 an hour after the second was written. So: for each declared mutation, apply it to the source, run the one test that claims to catch it, and require RED. Revert either way. .venv/bin/python scripts/mutation_check.py # every table .venv/bin/python scripts/mutation_check.py u7_navigation # one table Tables live in tests/mutations/*.toml and are committed, so a unit's proofs are an artifact rather than terminal scrollback. Adding a unit means adding a file, never editing this script. ⚠ TWO DEFECTS THIS TOOL HAD, both of which made it CERTIFY A FALSIFIER WITHOUT RUNNING IT. Neither is obvious and both cost real time: 1. NO GREEN BASELINE. A test that is ALREADY red reports red for every mutation thrown at it, so a broken assertion reads as a proven falsifier. Every run now checks the test passes unmutated first; a red baseline is a harness failure, reported as such, never as a proof. 2. THE BYTECODE CACHE. `< 2` -> `< 1` is BYTE-IDENTICAL IN SIZE, and CPython validates a .pyc against the source's (mtime, size) at ONE-SECOND granularity -- so a mutation landing in the same second as the revert before it is invisible and the unmutated code runs. The tell was a verdict that flipped between consecutive runs with nothing changed. Caches are dropped and PYTHONDONTWRITEBYTECODE is set for every run. This biases toward exactly the mutations most worth making: comparison flips, off-by-one constants, and/or swaps. """ from __future__ import annotations import os import shutil import subprocess import sys import tomllib from pathlib import Path REPO = Path(__file__).resolve().parent.parent TABLES = REPO / "tests" / "mutations" # Written before a source file is touched and removed after it is restored. Its # presence at startup means a previous run died between the two -- a `kill -9` # mid-mutation leaves a mutated tracked file that looks like authored code. INFLIGHT = REPO / ".mutation-inflight" def run(test: str, repo: Path = REPO) -> int: """Exit code of one test, with the bytecode cache defeated. See defect 2.""" for cache in repo.rglob("__pycache__"): shutil.rmtree(cache, ignore_errors=True) return subprocess.run( [sys.executable, "-m", "pytest", test, "-q", "--no-header", "-p", "no:warnings"], cwd=repo, capture_output=True, text=True, env=dict(os.environ, PYTHONDONTWRITEBYTECODE="1"), ).returncode def check(mutation: dict, repo: Path = REPO) -> tuple[bool, str]: """(proved, note) for one mutation. Never leaves the source mutated. `repo` is a parameter so the harness can be pointed at a throwaway tree and given KNOWN-vacuous and KNOWN-good falsifiers — see tests/test_mutation_check.py. An instrument that only ever sees unknowns cannot tell "nothing wrong here" from "I am blind", which is the whole of CLAUDE.md's positive-control rule applied to the tool that enforces it.""" test = mutation["test"] path = repo / mutation["file"] if run(test, repo) != 0: return False, f"BASELINE RED — {test} fails BEFORE the mutation" src = path.read_text() if mutation["old"] not in src: return False, f"anchor not found in {mutation['file']} — the table has drifted" INFLIGHT.write_text(f"{path}\n") try: path.write_text(src.replace(mutation["old"], mutation["new"], 1)) red = run(test, repo) != 0 finally: path.write_text(src) # Verified, not assumed: a restore that silently failed would leave a # mutation in a tracked file and the next run would measure it. assert path.read_text() == src, f"RESTORE FAILED for {path} — fix by hand" INFLIGHT.unlink(missing_ok=True) return red, "" if red else "VACUOUS — stayed green under the change it forbids" def main(argv: list[str]) -> int: if INFLIGHT.exists(): print(f"refusing to run: {INFLIGHT} exists, so a previous run died mid-mutation.") print(f"check `git diff {INFLIGHT.read_text().strip()}`, restore it, then delete the marker.") return 2 wanted = argv[1:] or None tables = sorted(TABLES.glob("*.toml")) if wanted: tables = [t for t in tables if t.stem in wanted] if not tables: print(f"no table matching {wanted} in {TABLES}") return 2 failed = [] for table in tables: doc = tomllib.loads(table.read_text()) print(f"\n### {table.stem} — {doc.get('unit', '')}") for m in doc.get("mutation", []): proved, note = check(m) print(f"{' proved' if proved else ' NOT PROVED':14s} {m['label']}") if not proved: print(f"{'':14s} ^ {note}") failed.append(m["label"]) total = sum(len(tomllib.loads(t.read_text()).get("mutation", [])) for t in tables) print(f"\n{total - len(failed)}/{total} falsifiers proved by running the change they forbid") return 1 if failed else 0 if __name__ == "__main__": raise SystemExit(main(sys.argv))