booth/scripts/mutation_check.py

#!/usr/bin/env python3
"""Prove a falsifier falsifies, by running the change it forbids.

A green test is not evidence. A test that has never seen its own DEFEATING
CHANGE is only evidence that the code and the assertion agree today; it may
agree under the mutation too, in which case it forbids nothing and reads as
though it forbids something. This repo has shipped that three times --
persistent-memory.d/2026-09-22-vacuous-falsifiers.md,
persistent-memory.d/2026-09-22-seven-of-seven-falsifiers.md, and once more in
U7 an hour after the second was written.

So: for each declared mutation, apply it to the source, run the one test that
claims to catch it, and require RED. Revert either way.

    .venv/bin/python scripts/mutation_check.py                # every table
    .venv/bin/python scripts/mutation_check.py u7_navigation  # one table

Tables live in tests/mutations/*.toml and are committed, so a unit's proofs are
an artifact rather than terminal scrollback. Adding a unit means adding a file,
never editing this script.

⚠ TWO DEFECTS THIS TOOL HAD, both of which made it CERTIFY A FALSIFIER WITHOUT
RUNNING IT. Neither is obvious and both cost real time:

1. NO GREEN BASELINE. A test that is ALREADY red reports red for every mutation
   thrown at it, so a broken assertion reads as a proven falsifier. Every run
   now checks the test passes unmutated first; a red baseline is a harness
   failure, reported as such, never as a proof.

2. THE BYTECODE CACHE. `< 2` -> `< 1` is BYTE-IDENTICAL IN SIZE, and CPython
   validates a .pyc against the source's (mtime, size) at ONE-SECOND
   granularity -- so a mutation landing in the same second as the revert before
   it is invisible and the unmutated code runs. The tell was a verdict that
   flipped between consecutive runs with nothing changed. Caches are dropped
   and PYTHONDONTWRITEBYTECODE is set for every run. This biases toward exactly
   the mutations most worth making: comparison flips, off-by-one constants,
   and/or swaps.
"""

from __future__ import annotations

import os
import shutil
import subprocess
import sys
import tomllib
from pathlib import Path

REPO = Path(__file__).resolve().parent.parent
TABLES = REPO / "tests" / "mutations"
# Written before a source file is touched and removed after it is restored. Its
# presence at startup means a previous run died between the two -- a `kill -9`
# mid-mutation leaves a mutated tracked file that looks like authored code.
INFLIGHT = REPO / ".mutation-inflight"


def run(test: str, repo: Path = REPO) -> int:
    """Exit code of one test, with the bytecode cache defeated. See defect 2."""
    for cache in repo.rglob("__pycache__"):
        shutil.rmtree(cache, ignore_errors=True)
    return subprocess.run(
        [sys.executable, "-m", "pytest", test, "-q", "--no-header", "-p", "no:warnings"],
        cwd=repo, capture_output=True, text=True,
        env=dict(os.environ, PYTHONDONTWRITEBYTECODE="1"),
    ).returncode


def check(mutation: dict, repo: Path = REPO) -> tuple[bool, str]:
    """(proved, note) for one mutation. Never leaves the source mutated.

    `repo` is a parameter so the harness can be pointed at a throwaway tree and
    given KNOWN-vacuous and KNOWN-good falsifiers — see
    tests/test_mutation_check.py. An instrument that only ever sees unknowns
    cannot tell "nothing wrong here" from "I am blind", which is the whole of
    CLAUDE.md's positive-control rule applied to the tool that enforces it."""
    test = mutation["test"]
    path = repo / mutation["file"]

    if run(test, repo) != 0:
        return False, f"BASELINE RED — {test} fails BEFORE the mutation"

    src = path.read_text()
    if mutation["old"] not in src:
        return False, f"anchor not found in {mutation['file']} — the table has drifted"

    INFLIGHT.write_text(f"{path}\n")
    try:
        path.write_text(src.replace(mutation["old"], mutation["new"], 1))
        red = run(test, repo) != 0
    finally:
        path.write_text(src)
        # Verified, not assumed: a restore that silently failed would leave a
        # mutation in a tracked file and the next run would measure it.
        assert path.read_text() == src, f"RESTORE FAILED for {path} — fix by hand"
        INFLIGHT.unlink(missing_ok=True)

    return red, "" if red else "VACUOUS — stayed green under the change it forbids"


def main(argv: list[str]) -> int:
    if INFLIGHT.exists():
        print(f"refusing to run: {INFLIGHT} exists, so a previous run died mid-mutation.")
        print(f"check `git diff {INFLIGHT.read_text().strip()}`, restore it, then delete the marker.")
        return 2

    wanted = argv[1:] or None
    tables = sorted(TABLES.glob("*.toml"))
    if wanted:
        tables = [t for t in tables if t.stem in wanted]
        if not tables:
            print(f"no table matching {wanted} in {TABLES}")
            return 2

    failed = []
    for table in tables:
        doc = tomllib.loads(table.read_text())
        print(f"\n### {table.stem} — {doc.get('unit', '')}")
        for m in doc.get("mutation", []):
            proved, note = check(m)
            print(f"{'  proved' if proved else '  NOT PROVED':14s}  {m['label']}")
            if not proved:
                print(f"{'':14s}  ^ {note}")
                failed.append(m["label"])

    total = sum(len(tomllib.loads(t.read_text()).get("mutation", [])) for t in tables)
    print(f"\n{total - len(failed)}/{total} falsifiers proved by running the change they forbid")
    return 1 if failed else 0


if __name__ == "__main__":
    raise SystemExit(main(sys.argv))