diff --git a/scripts/r49-corpus/show_memorisation_matches.py b/scripts/r49-corpus/show_memorisation_matches.py new file mode 100644 index 0000000..b99e182 --- /dev/null +++ b/scripts/r49-corpus/show_memorisation_matches.py @@ -0,0 +1,112 @@ +"""Print the actual verbatim runs an arm shares with the training corpus, for READING. + +`memorization_check.py` answers HOW OFTEN. This answers WHAT — and the two are different +questions with different consequences. The lv-hemingway gate shipped an adapter at 7x the +author's own collision rate because **all 19 matched runs were read** and every one was +stock dialogue in the commonest words in English (`came over and sat down at the table`), +max 9 words, no proper noun, no imagery, no plot. Elevated rate, zero protectable content. + +That reading was done BY HAND and left no instrument, so the next gate had to either repeat +it by hand or skip it. This is it, committed. + +⚠ **Rate and exposure are different questions and the second one is the one that matters +legally.** A low rate hiding one 30-word run of distinctive imagery is worse than a high +rate of `he said and then he said`. Sort by length, read the top, and look for: + + * PROPER NOUNS -- flagged below. In a renamed corpus a surviving name is either an + INVENTED replacement (harmless, it is not the author's word) or a leak the rename + missed, so every one needs its origin checked rather than counted. + * distinctive imagery, plot specifics, or a phrase you would recognise out of context. + +Nothing here is a threshold. It is a reading aid, and the reading is the axis. +""" +from __future__ import annotations +import argparse, json, pathlib, re, sys + +WORD = re.compile(r"[A-Za-z']+") + + +def tokens(text: str) -> tuple[list[str], list[str]]: + """Return (cased, lowered) token lists that index in lockstep.""" + cased = WORD.findall(text) + return cased, [w.lower() for w in cased] + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--eval-dir", required=True) + ap.add_argument("--corpus", required=True, help="the RENAMED copies the adapter trained on") + ap.add_argument("--glob", default="beats5.*.jsonl") + ap.add_argument("--strip", default="beats5.") + ap.add_argument("-n", type=int, default=8, help="n-gram length; must match the check") + ap.add_argument("--train-only", action="store_true", + help="build the gram set from split=='train' records only, so a collision " + "with held-out text is not reported as memorisation of training text") + ap.add_argument("--arm", default=None, help="restrict to one arm") + ap.add_argument("--top", type=int, default=0, help="print only the N longest runs (0 = all)") + a = ap.parse_args() + + N = a.n + corpus_words: list[str] = [] + for f in sorted(pathlib.Path(a.corpus).glob("*.copy0.jsonl")): + for l in f.read_text(encoding="utf-8").splitlines(): + r = json.loads(l) + if a.train_only and r.get("split") != "train": + continue + corpus_words.extend(re.findall(r"[a-z']+", r["text"].lower())) + grams = {" ".join(corpus_words[i:i + N]) for i in range(len(corpus_words) - N + 1)} + scope = "train split only" if a.train_only else "all records" + print(f"corpus ({scope}): {len(corpus_words):,} words, {len(grams):,} distinct {N}-grams") + + # POSITIVE CONTROL, every run. A matcher that only ever sees candidate text cannot tell + # `no matches` from `I am blind`, which is the same rule the gates run under. + probe = corpus_words[1000:1000 + N] + if " ".join(probe) not in grams: + print("== REFUSING: a slice of the corpus does not match the corpus. The matcher is broken.") + return 1 + print(f" [PASS] positive control: a corpus slice matches itself\n") + + found = [] + for f in sorted(pathlib.Path(a.eval_dir).glob(a.glob)): + arm = f.stem.replace(a.strip, "") + if a.arm and arm != a.arm: + continue + for l in f.read_text(encoding="utf-8").splitlines(): + r = json.loads(l) + cased, low = tokens(r["raw"]) + i = 0 + while i <= len(low) - N: + if " ".join(low[i:i + N]) in grams: + k = N + while i + k < len(low) and " ".join(low[i + k - N + 1:i + k + 1]) in grams: + k += 1 + # a token capitalised anywhere but the first position of the RUN is a + # proper noun rather than a sentence opening -- crude, deliberately + # over-reports, and every flag is meant to be read not counted. + caps = [w for w in cased[i + 1:i + k] if w[:1].isupper()] + found.append({"arm": arm, "id": r["id"], "seed": r["seed"], "len": k, + "text": " ".join(cased[i:i + k]), "caps": caps}) + i += k + else: + i += 1 + if not found: + print(f"no runs of {N}+ words shared with the corpus in any arm scanned.") + return 0 + + found.sort(key=lambda d: -d["len"]) + shown = found[:a.top] if a.top else found + flagged = [d for d in found if d["caps"]] + print(f"{len(found)} matched run(s), longest {found[0]['len']} words. " + f"{len(flagged)} carry a mid-run capital.\n") + for d in shown: + flag = f" ⚠ CAPS: {', '.join(d['caps'])}" if d["caps"] else "" + print(f" [{d['len']:>3}w] {d['arm']:<12} {d['id']:>4} seed={d['seed']}{flag}") + print(f" {d['text']}") + if a.top and len(found) > a.top: + print(f"\n ... {len(found) - a.top} shorter run(s) not shown; re-run without --top.") + print("\n⚠ READ THESE. The rate is not the finding; what is being reproduced is.") + return 0 + + +if __name__ == "__main__": + sys.exit(main())