vh
377e652670
Four items owed after S5b, reported by design-dev during the anti-slop run:
- carry() measured a sent-then-changed form against its OLD DEFAULTS. An
answer set back mid-flight to the value the page first showed read as
untouched, and the swap put the just-saved value over it. A form sent and
then changed is now measured against its sent snapshot (sentSet.snapOf).
- The embed's clean-batch reload saw only our own forms. A report's own
inputs lost whatever the operator had typed into them. Unsaved text in
any control we don't own now holds the reload, and the page says so.
- Two r2b.toml rows ("D3 a stored theme...", "D3 forced light...") matched
twice, so they proved only by where the first match fell. Both are
re-anchored, and scripts/mutation_check.py now refuses any anchor that
matches more than once. A new tool control covers that.
- The r2_flow contract's C3 steps 2 and 4 now say what S5b superseded. U3
gains the report-input rule.
Mutation rows: u3_submit_all +1, r2_submit_all +1. Four rows were
re-anchored onto the moved lines.
145 lines
6.5 KiB
Python
145 lines
6.5 KiB
Python
"""Controls for the instrument that certifies every other falsifier.
|
|
|
|
`scripts/mutation_check.py` exists because a green test proves nothing until it
|
|
has seen the change it forbids. The same sentence applies to the tool: it
|
|
shipped two defects in one session, each of which made it report a falsifier
|
|
PROVED WITHOUT RUNNING IT (no green baseline; the pyc cache silently reverting
|
|
byte-identical mutations). Both were found by accident.
|
|
|
|
So the tool gets what CLAUDE.md demands of any measurement: a POSITIVE CONTROL
|
|
it must detect, and a NEGATIVE CONTROL it must not fire on. An instrument that
|
|
only ever sees unknowns cannot distinguish "absent" from "blind".
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pathlib
|
|
import sys
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).parent.parent / "scripts"))
|
|
|
|
from mutation_check import check # noqa: E402
|
|
|
|
|
|
def _tree(tmp_path, source: str, test_body: str):
|
|
"""A throwaway repo: one module, one test file, both real on disk."""
|
|
(tmp_path / "mod.py").write_text(source)
|
|
(tmp_path / "test_probe.py").write_text(
|
|
"import sys, pathlib\n"
|
|
"sys.path.insert(0, str(pathlib.Path(__file__).parent))\n"
|
|
"from mod import f\n\n" + test_body
|
|
)
|
|
return tmp_path
|
|
|
|
|
|
def test_a_real_falsifier_is_reported_proved(tmp_path):
|
|
"""NEGATIVE CONTROL — the tool must not cry wolf on a sound test.
|
|
|
|
`f` returns 2; the test asserts it. Flipping the constant must go red, and
|
|
the tool must say so."""
|
|
repo = _tree(tmp_path, "def f():\n return 2\n",
|
|
"def test_f():\n assert f() == 2\n")
|
|
proved, note = check(
|
|
{"label": "flip the constant", "file": "mod.py", "test": "test_probe.py::test_f",
|
|
"old": "return 2", "new": "return 3"}, repo=repo)
|
|
assert proved, note
|
|
|
|
|
|
def test_a_vacuous_falsifier_is_caught(tmp_path):
|
|
"""POSITIVE CONTROL — the one that matters, and the one usually skipped.
|
|
|
|
The test asserts only that `f()` is an int, so flipping the constant does
|
|
NOT break it. The test cites the behaviour without forbidding it. The tool
|
|
must report NOT PROVED; if it cannot detect a known-vacuous falsifier, its
|
|
twelve `proved` lines are worth nothing."""
|
|
repo = _tree(tmp_path, "def f():\n return 2\n",
|
|
"def test_f():\n assert isinstance(f(), int)\n")
|
|
proved, note = check(
|
|
{"label": "flip the constant", "file": "mod.py", "test": "test_probe.py::test_f",
|
|
"old": "return 2", "new": "return 3"}, repo=repo)
|
|
assert not proved
|
|
assert "VACUOUS" in note
|
|
|
|
|
|
def test_an_already_red_test_is_a_harness_failure_not_a_proof(tmp_path):
|
|
"""DEFECT 1, as a control. Before the baseline check this returned PROVED —
|
|
a broken assertion reading as a certified falsifier."""
|
|
repo = _tree(tmp_path, "def f():\n return 2\n",
|
|
"def test_f():\n assert f() == 99\n")
|
|
proved, note = check(
|
|
{"label": "flip the constant", "file": "mod.py", "test": "test_probe.py::test_f",
|
|
"old": "return 2", "new": "return 3"}, repo=repo)
|
|
assert not proved
|
|
assert "BASELINE RED" in note
|
|
|
|
|
|
def test_a_same_size_mutation_is_not_swallowed_by_the_bytecode_cache(tmp_path):
|
|
"""DEFECT 2, as a control. `< 2` -> `< 1` is byte-identical in size, so a
|
|
mutation landing in the same mtime second as the revert before it used to
|
|
run against cached bytecode and report PROVED having tested nothing.
|
|
|
|
Run twice: the verdict must be stable. The original defect's tell was
|
|
exactly a verdict that flipped between consecutive identical runs."""
|
|
repo = _tree(tmp_path, "def f(n):\n return n < 2\n",
|
|
"def test_f():\n assert f(1) is True and f(2) is False\n")
|
|
m = {"label": "off by one", "file": "mod.py", "test": "test_probe.py::test_f",
|
|
"old": "return n < 2", "new": "return n < 1"}
|
|
assert [check(m, repo=repo)[0] for _ in range(2)] == [True, True]
|
|
|
|
|
|
def test_a_drifted_anchor_is_reported_not_skipped(tmp_path):
|
|
"""A table whose `old` no longer matches the source stops proving anything.
|
|
Silently skipping it would shrink the denominator and keep the run green."""
|
|
repo = _tree(tmp_path, "def f():\n return 2\n",
|
|
"def test_f():\n assert f() == 2\n")
|
|
proved, note = check(
|
|
{"label": "stale", "file": "mod.py", "test": "test_probe.py::test_f",
|
|
"old": "return 2222", "new": "return 3"}, repo=repo)
|
|
assert not proved
|
|
assert "anchor not found" in note
|
|
|
|
|
|
def test_the_source_is_restored_even_when_the_mutation_proves(tmp_path):
|
|
"""The tool writes to tracked source files. Leaving one mutated would put a
|
|
defect in the tree that looks like authored code."""
|
|
repo = _tree(tmp_path, "def f():\n return 2\n",
|
|
"def test_f():\n assert f() == 2\n")
|
|
before = (repo / "mod.py").read_text()
|
|
check({"label": "flip", "file": "mod.py", "test": "test_probe.py::test_f",
|
|
"old": "return 2", "new": "return 3"}, repo=repo)
|
|
assert (repo / "mod.py").read_text() == before
|
|
|
|
|
|
def test_a_reverted_file_keeps_its_mtime(tmp_path):
|
|
"""The repo IS its own deployment root: nothing takes effect until the
|
|
service restarts, so "is :8090 stale?" is answered by comparing the
|
|
service's start time against source mtimes. A tool that rewrites a file
|
|
with identical bytes still bumps its mtime and makes that check lie — it
|
|
reported the live service 16 minutes stale when it was current.
|
|
|
|
Defeating change: dropping the os.utime in the restore."""
|
|
import os
|
|
|
|
repo = _tree(tmp_path, "def f():\n return 2\n",
|
|
"def test_f():\n assert f() == 2\n")
|
|
mod = repo / "mod.py"
|
|
os.utime(mod, (1_000_000_000, 1_000_000_000))
|
|
before = mod.stat().st_mtime_ns
|
|
check({"label": "flip", "file": "mod.py", "test": "test_probe.py::test_f",
|
|
"old": "return 2", "new": "return 3"}, repo=repo)
|
|
assert mod.stat().st_mtime_ns == before
|
|
|
|
|
|
def test_an_ambiguous_anchor_is_reported_not_guessed(tmp_path):
|
|
"""An `old` that matches twice mutates whichever comes FIRST, so the row
|
|
proves or fails by where the first match happens to fall rather than by
|
|
the line it names. design-dev found two r2b rows proving that way
|
|
(2026-09-28). A row must name exactly one place."""
|
|
repo = _tree(tmp_path, "def f():\n x = 2\n x = 2\n return x\n",
|
|
"def test_f():\n assert f() == 2\n")
|
|
proved, note = check(
|
|
{"label": "twice", "file": "mod.py", "test": "test_probe.py::test_f",
|
|
"old": " x = 2\n", "new": " x = 3\n"}, repo=repo)
|
|
assert not proved
|
|
assert "ambiguous" in note
|