diff --git a/scripts/bronte-corpus/stoplist_bronte.json b/scripts/bronte-corpus/stoplist_bronte.json index f9f544d..42a6813 100644 --- a/scripts/bronte-corpus/stoplist_bronte.json +++ b/scripts/bronte-corpus/stoplist_bronte.json @@ -49,7 +49,6 @@ "Rousseau", "Chénier", "Timon", - "Pierrot", "Gytrash" ], "political_and_period_terms": [ @@ -89,9 +88,8 @@ "Notre", "Adam", "Saul", - "Daniel", - "Samuel", - "Leaven" + "Daniel" ], - "sub_threshold_triage_note": "Everything added in the 2026-09-16 second pass came from the gate's own sub-threshold report — the 84 surfaces below the rename floor that the gate lists separately because rename never saw them. Each was read before landing here. Deliberately NOT stoplisted, because renaming is the safe direction and these may be characters rather than scripture: David, Joseph, Thomas. `Gytrash` is Yorkshire folklore Brontë did not invent; `Pierrot` and `Timon` are stock figures." + "sub_threshold_triage_note": "Everything added in the 2026-09-16 second pass came from the gate's own sub-threshold report — the 84 surfaces below the rename floor that the gate lists separately because rename never saw them. Each was read before landing here. Deliberately NOT stoplisted, because renaming is the safe direction and these may be characters rather than scripture: David, Joseph, Thomas. `Gytrash` is Yorkshire folklore Brontë did not invent; `Pierrot` and `Timon` are stock figures.", + "stoplist_is_an_unchecked_assertion": "⭐ A stoplist entry removes a surface from the entity map, so rename never touches it and the leak gate never scans for it. The gate then reports 0 of N surviving and is telling the truth about the set it was given — which makes a wrongly stoplisted CHARACTER an undetectable leak. Three were wrong here and all three were caught by audit_stoplist.py, not by the gate: Leaven (\"Robert Leaven, the coachman\" — Bessie's married surname, filed as the bread noun), Pierrot (\"Madame Pierrot: she comes from Lisle\" — a teacher in The Professor, filed as the commedia figure) and Samuel (\"Mr. Samuel Wynne\", filed as scripture). The audit clears Wellington (\"that Baal of a Lord Wellington\" — the real Duke) and Moses (\"the Rev. Moses Barraclough\" — the documented dual-use). Run the audit before the gate, every corpus." } \ No newline at end of file diff --git a/scripts/r49-corpus/audit_stoplist.py b/scripts/r49-corpus/audit_stoplist.py new file mode 100644 index 0000000..bf3ef08 --- /dev/null +++ b/scripts/r49-corpus/audit_stoplist.py @@ -0,0 +1,89 @@ +"""Audit a stoplist for entries that are actually CHARACTERS. + +⭐ WHY THIS IS A SEPARATE INSTRUMENT. A stoplist entry is an assertion the leak gate +can no longer check. Stoplisting a surface removes it from the entity map, so rename +never touches it and the gate never scans for it — which is exactly what a stoplist is +FOR when the surface is a real-world referent, and exactly how a wrongly stoplisted +CHARACTER becomes an undetectable leak. The gate will report 0 of N surviving and be +telling the truth about the set it was given. + +Found on lv-bronte 2026-09-16, and found by luck: a generated beat said "Mrs. Leaven", +and `Leaven` had been filed under scripture (the bread noun). Reading it back: +"Robert Leaven, the coachman" — Bessie's married surname. Running this audit instead of +trusting the luck then caught two more, `Pierrot` ("Madame Pierrot: she comes from +Lisle") and `Samuel` ("Mr. Samuel Wynne"), and correctly cleared `Wellington` +("that Baal of a Lord Wellington" — the real Duke). + +THE SIGNAL: an honorific in front of it. Real-world referents are not addressed as +Mr/Mrs/Miss/Madame/Lord. It is a heuristic, not a proof — `Lord Wellington` is a real +person and `Rev. Moses Barraclough` is a genuine dual-use — so every hit is REPORTED +FOR READING, never auto-removed. Deciding what a surface is requires reading it in +context, which is the lesson this whole line keeps relearning. + +Exit 1 when anything is flagged, so it can gate a pipeline: a clean run is silence. +""" +from __future__ import annotations +import argparse, json, re +from pathlib import Path + +HONORIFIC = (r"(?:Mr|Mrs|Miss|Misses|Madame|Mdlle|Mademoiselle|Monsieur|Lord|Lady|Sir|Dr" + r"|Doctor|Captain|Colonel|Major|General|Aunt|Uncle|Rev|Reverend|Professor" + r"|Master|Saint|St)\.?\s+") + + +def load_text(corpus: Path) -> str: + man = json.loads((corpus / "manifest.json").read_text()) + parts = [] + for w in man["works"]: + for line in (corpus / w["path"]).read_text(encoding="utf-8").splitlines(): + if line.strip(): + parts.append(json.loads(line)["text"]) + return "\n\n".join(parts) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("corpus") + ap.add_argument("--stoplist", required=True) + ap.add_argument("--allow", default="", + help="comma-separated surfaces already READ and confirmed real-world " + "or accepted dual-use; they are still shown, but do not fail the run") + ap.add_argument("--context", type=int, default=46) + a = ap.parse_args() + + text = load_text(Path(a.corpus)) + blob = json.loads(Path(a.stoplist).read_text()) + surfaces = sorted({s for v in blob.values() if isinstance(v, list) for s in v}) + cleared = {s.strip() for s in a.allow.split(",") if s.strip()} + + flagged = [] + for s in surfaces: + pat = re.compile(HONORIFIC + re.escape(s) + r"\b") + hits = list(pat.finditer(text)) + if hits: + flagged.append((len(hits), s, hits[0])) + + print(f" {len(surfaces)} stoplisted surfaces scanned for a preceding honorific") + if not flagged: + print(" [PASS] none of them is ever addressed as a person") + return 0 + + unresolved = 0 + for n, s, m in sorted(flagged, reverse=True): + mark = "cleared" if s in cleared else "⚠ READ THIS" + if s not in cleared: + unresolved += 1 + ctx = text[max(0, m.start() - a.context): m.end() + a.context].replace("\n", " ") + print(f" {mark:<12} {s:<16} {n:>4} honorific hits …{ctx}…") + + if unresolved: + print(f"\n== {unresolved} stoplisted surface(s) look like PEOPLE and are not on --allow.") + print(" Read each in context. If it is a character, take it OUT of the stoplist and") + print(" re-run the rename — the leak gate cannot see it while it is stoplisted.") + return 1 + print("\n [PASS] every honorific hit is on the read-and-cleared list") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())