booth/tests/test_mutation_check.py

"""Controls for the instrument that certifies every other falsifier.

`scripts/mutation_check.py` exists because a green test proves nothing until it
has seen the change it forbids. The same sentence applies to the tool: it
shipped two defects in one session, each of which made it report a falsifier
PROVED WITHOUT RUNNING IT (no green baseline; the pyc cache silently reverting
byte-identical mutations). Both were found by accident.

So the tool gets what CLAUDE.md demands of any measurement: a POSITIVE CONTROL
it must detect, and a NEGATIVE CONTROL it must not fire on. An instrument that
only ever sees unknowns cannot distinguish "absent" from "blind".
"""

from __future__ import annotations

import pathlib
import sys

sys.path.insert(0, str(pathlib.Path(__file__).parent.parent / "scripts"))

from mutation_check import check  # noqa: E402


def _tree(tmp_path, source: str, test_body: str):
    """A throwaway repo: one module, one test file, both real on disk."""
    (tmp_path / "mod.py").write_text(source)
    (tmp_path / "test_probe.py").write_text(
        "import sys, pathlib\n"
        "sys.path.insert(0, str(pathlib.Path(__file__).parent))\n"
        "from mod import f\n\n" + test_body
    )
    return tmp_path


def test_a_real_falsifier_is_reported_proved(tmp_path):
    """NEGATIVE CONTROL — the tool must not cry wolf on a sound test.

    `f` returns 2; the test asserts it. Flipping the constant must go red, and
    the tool must say so."""
    repo = _tree(tmp_path, "def f():\n    return 2\n",
                 "def test_f():\n    assert f() == 2\n")
    proved, note = check(
        {"label": "flip the constant", "file": "mod.py", "test": "test_probe.py::test_f",
         "old": "return 2", "new": "return 3"}, repo=repo)
    assert proved, note


def test_a_vacuous_falsifier_is_caught(tmp_path):
    """POSITIVE CONTROL — the one that matters, and the one usually skipped.

    The test asserts only that `f()` is an int, so flipping the constant does
    NOT break it. The test cites the behaviour without forbidding it. The tool
    must report NOT PROVED; if it cannot detect a known-vacuous falsifier, its
    twelve `proved` lines are worth nothing."""
    repo = _tree(tmp_path, "def f():\n    return 2\n",
                 "def test_f():\n    assert isinstance(f(), int)\n")
    proved, note = check(
        {"label": "flip the constant", "file": "mod.py", "test": "test_probe.py::test_f",
         "old": "return 2", "new": "return 3"}, repo=repo)
    assert not proved
    assert "VACUOUS" in note


def test_an_already_red_test_is_a_harness_failure_not_a_proof(tmp_path):
    """DEFECT 1, as a control. Before the baseline check this returned PROVED —
    a broken assertion reading as a certified falsifier."""
    repo = _tree(tmp_path, "def f():\n    return 2\n",
                 "def test_f():\n    assert f() == 99\n")
    proved, note = check(
        {"label": "flip the constant", "file": "mod.py", "test": "test_probe.py::test_f",
         "old": "return 2", "new": "return 3"}, repo=repo)
    assert not proved
    assert "BASELINE RED" in note


def test_a_same_size_mutation_is_not_swallowed_by_the_bytecode_cache(tmp_path):
    """DEFECT 2, as a control. `< 2` -> `< 1` is byte-identical in size, so a
    mutation landing in the same mtime second as the revert before it used to
    run against cached bytecode and report PROVED having tested nothing.

    Run twice: the verdict must be stable. The original defect's tell was
    exactly a verdict that flipped between consecutive identical runs."""
    repo = _tree(tmp_path, "def f(n):\n    return n < 2\n",
                 "def test_f():\n    assert f(1) is True and f(2) is False\n")
    m = {"label": "off by one", "file": "mod.py", "test": "test_probe.py::test_f",
         "old": "return n < 2", "new": "return n < 1"}
    assert [check(m, repo=repo)[0] for _ in range(2)] == [True, True]


def test_a_drifted_anchor_is_reported_not_skipped(tmp_path):
    """A table whose `old` no longer matches the source stops proving anything.
    Silently skipping it would shrink the denominator and keep the run green."""
    repo = _tree(tmp_path, "def f():\n    return 2\n",
                 "def test_f():\n    assert f() == 2\n")
    proved, note = check(
        {"label": "stale", "file": "mod.py", "test": "test_probe.py::test_f",
         "old": "return 2222", "new": "return 3"}, repo=repo)
    assert not proved
    assert "anchor not found" in note


def test_the_source_is_restored_even_when_the_mutation_proves(tmp_path):
    """The tool writes to tracked source files. Leaving one mutated would put a
    defect in the tree that looks like authored code."""
    repo = _tree(tmp_path, "def f():\n    return 2\n",
                 "def test_f():\n    assert f() == 2\n")
    before = (repo / "mod.py").read_text()
    check({"label": "flip", "file": "mod.py", "test": "test_probe.py::test_f",
           "old": "return 2", "new": "return 3"}, repo=repo)
    assert (repo / "mod.py").read_text() == before


def test_a_reverted_file_keeps_its_mtime(tmp_path):
    """The repo IS its own deployment root: nothing takes effect until the
    service restarts, so "is :8090 stale?" is answered by comparing the
    service's start time against source mtimes. A tool that rewrites a file
    with identical bytes still bumps its mtime and makes that check lie — it
    reported the live service 16 minutes stale when it was current.

    Defeating change: dropping the os.utime in the restore."""
    import os

    repo = _tree(tmp_path, "def f():\n    return 2\n",
                 "def test_f():\n    assert f() == 2\n")
    mod = repo / "mod.py"
    os.utime(mod, (1_000_000_000, 1_000_000_000))
    before = mod.stat().st_mtime_ns
    check({"label": "flip", "file": "mod.py", "test": "test_probe.py::test_f",
           "old": "return 2", "new": "return 3"}, repo=repo)
    assert mod.stat().st_mtime_ns == before