From 5e6611466ca5d1326de0ba9403ba191f46a5dc17 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Thu, 17 Sep 2026 02:37:35 -0700 Subject: [PATCH] audit_pairs_sourcenames: --filter-out, so the detector is also the fix An already-built pair set cannot be repaired by build_sft_pairs.py --source-entities; that flag only works at generation time. Hemingway's and Yarros's sets both predate it. The contamination is in the BEAT, so dropping the row removes it outright. Measured on the Hemingway train pairs: 7,094 -> 7,024, 70 dropped, 0.99% of the training data. That is cheaper and cleaner than regenerating 70 beats against a second generator session, which would leave the set mixed-provenance for the sake of 1% more data. Verified by read-back rather than by the write succeeding: re-auditing the filtered file reports 0 of 7,024 on both columns, controls green, GATE PASS. Two refusals rather than a best-effort write: - a contaminated RESPONSE column aborts. That is a different fault -- pairs built against an unrenamed corpus -- and dropping rows would hide it instead of fixing it. - more than one --pairs input aborts, because the output is a single file and would silently merge train and val into one. Also cross-validated the detector against the lv-bronte pair sets on real data, where the answer is already on the record: pairs-full + pairs-val (post-fix) 0 of 3,858 matches the recorded "0 leaks across 3,858 pairs" exactly pairs-full.CONTAMINATED 15 of 792 = 1.89%, Rochester x6, Jane, Brocklehurst x2, Beck, Fairfax, Burns, Helen, Eyre -- against a record of "13 of the first 714 beats (1.8%)" with the same names An independently written instrument reproducing a documented finding at the right magnitude, on the right names, is the control that says its zeroes mean absent and not blind. --- scripts/r49-corpus/audit_pairs_sourcenames.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/scripts/r49-corpus/audit_pairs_sourcenames.py b/scripts/r49-corpus/audit_pairs_sourcenames.py index b1bc598..3f16b84 100644 --- a/scripts/r49-corpus/audit_pairs_sourcenames.py +++ b/scripts/r49-corpus/audit_pairs_sourcenames.py @@ -65,6 +65,16 @@ def main() -> int: help="rename.py's renameable threshold; mirrored from leak_gate.py") ap.add_argument("--report", default=None, help="write the full JSON breakdown here") ap.add_argument("--show", type=int, default=25, help="example beats to print") + ap.add_argument("--filter-out", default=None, metavar="PATH", + help="write a copy of the pairs with every contaminated ROW removed. " + "Turns this detector into the fix for an already-built pair set: " + "the contamination is in the beat, so dropping the row removes it " + "outright, and on Hemingway that costs 0.96%% of the training data. " + "Cheaper and cleaner than regenerating 70 beats against a second " + "generator session, which would leave the set mixed-provenance. " + "Refuses to write when a RESPONSE is contaminated -- that is a " + "different fault (pairs built against an unrenamed corpus) and " + "dropping rows would hide it rather than fix it.") a = ap.parse_args() ents_all = json.loads(Path(a.entities).read_text()) @@ -157,6 +167,40 @@ def main() -> int: }, ensure_ascii=False, indent=2), encoding="utf-8") print(f"\n wrote {a.report}") + if a.filter_out: + if nr: + print("\n== REFUSING to write a filtered copy: the RESPONSE column is contaminated.") + print(" That is not the beat-generator fault and dropping rows would hide it.") + print(" Check the provenance `corpus` path -- the pairs were probably built") + print(" against an unrenamed corpus, and the fix is upstream of this file.") + return 2 + if not (pos_ok and neg_ok): + print("\n== REFUSING to write a filtered copy: the controls did not pass, so the") + print(" set of rows to drop is not trustworthy.") + return 2 + if len({r["_src"] for r in rows}) != 1: + print("\n== REFUSING to write a filtered copy from more than one --pairs file:") + print(" the output is a single file and would silently merge train and val.") + print(" Filter each input separately.") + return 2 + drop = set() + for i, r in enumerate(rows): + if removed_pat.search(r.get("beat") or ""): + drop.add(i) + out = Path(a.filter_out) + kept = 0 + with out.open("w", encoding="utf-8") as fh: + for i, r in enumerate(rows): + if i in drop: + continue + r = {k: v for k, v in r.items() if k != "_src"} + fh.write(json.dumps(r, ensure_ascii=False) + "\n") + kept += 1 + print(f"\n wrote {out}: {kept} pairs kept, {len(drop)} dropped " + f"({len(drop) / len(rows):.2%})") + print(" ⚠ Re-run this audit against the filtered file before training on it. A fix") + print(" that is not read back is a claim, not a result.") + ok = pos_ok and neg_ok and nb == 0 and nr == 0 print(f"\n GATE: {'PASS' if ok else 'FAIL'}") return 0 if ok else 2