feat(r49): commit the instrument for READING memorisation matches
memorization_check.py answers HOW OFTEN an arm collides with the training text. It does not answer WHAT is being reproduced, and those have different consequences: a low rate hiding one 30-word run of distinctive imagery is worse than a high rate of stock dialogue in the commonest words in English. The lv-hemingway gate shipped at 7x the author's own collision rate precisely because that reading was done -- every matched run was stock dialogue, max 9 words, no proper noun. But it was done BY HAND and left no instrument, so the next gate had to repeat it by hand or skip it. This is it. Prints every maximal matched run sorted by length, with arm/id/seed, and flags any token capitalised mid-run as a possible proper noun. The flag deliberately over-reports (sentence-initial I, He, What all trip it) because it is a reading aid and over-reporting is the safe direction. --train-only matches memorization_check.py so a collision with held-out text is not reported as memorisation of training text. A corpus slice is matched against the corpus every run, and the script REFUSES if that positive control fails -- a matcher that only ever sees candidate text cannot tell "no matches" from "blind". Validated against the lv-hemingway record's documented reading, which it reproduces to the word: longest run 9 words, "swift tristan" flagged as the one name-shaped hit (it is the RENAMED invented name, not Hemingway's), and the record's quoted examples -- "came over and sat down at the table", "i don t think so the girl said" -- both present. Required by GATE-PREREG.md AMENDMENT 1, which makes reading the matches part of axis B rather than a follow-up, because McCarthy is in copyright with a living estate and a match carrying distinctive imagery or a proper noun is disqualifying in a way a rate number alone is not.
This commit is contained in:
Vuong Hoang
@@ -0,0 +1,112 @@
|
||||
"""Print the actual verbatim runs an arm shares with the training corpus, for READING.
|
||||
|
||||
`memorization_check.py` answers HOW OFTEN. This answers WHAT — and the two are different
|
||||
questions with different consequences. The lv-hemingway gate shipped an adapter at 7x the
|
||||
author's own collision rate because **all 19 matched runs were read** and every one was
|
||||
stock dialogue in the commonest words in English (`came over and sat down at the table`),
|
||||
max 9 words, no proper noun, no imagery, no plot. Elevated rate, zero protectable content.
|
||||
|
||||
That reading was done BY HAND and left no instrument, so the next gate had to either repeat
|
||||
it by hand or skip it. This is it, committed.
|
||||
|
||||
⚠ **Rate and exposure are different questions and the second one is the one that matters
|
||||
legally.** A low rate hiding one 30-word run of distinctive imagery is worse than a high
|
||||
rate of `he said and then he said`. Sort by length, read the top, and look for:
|
||||
|
||||
* PROPER NOUNS -- flagged below. In a renamed corpus a surviving name is either an
|
||||
INVENTED replacement (harmless, it is not the author's word) or a leak the rename
|
||||
missed, so every one needs its origin checked rather than counted.
|
||||
* distinctive imagery, plot specifics, or a phrase you would recognise out of context.
|
||||
|
||||
Nothing here is a threshold. It is a reading aid, and the reading is the axis.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import argparse, json, pathlib, re, sys
|
||||
|
||||
WORD = re.compile(r"[A-Za-z']+")
|
||||
|
||||
|
||||
def tokens(text: str) -> tuple[list[str], list[str]]:
|
||||
"""Return (cased, lowered) token lists that index in lockstep."""
|
||||
cased = WORD.findall(text)
|
||||
return cased, [w.lower() for w in cased]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--eval-dir", required=True)
|
||||
ap.add_argument("--corpus", required=True, help="the RENAMED copies the adapter trained on")
|
||||
ap.add_argument("--glob", default="beats5.*.jsonl")
|
||||
ap.add_argument("--strip", default="beats5.")
|
||||
ap.add_argument("-n", type=int, default=8, help="n-gram length; must match the check")
|
||||
ap.add_argument("--train-only", action="store_true",
|
||||
help="build the gram set from split=='train' records only, so a collision "
|
||||
"with held-out text is not reported as memorisation of training text")
|
||||
ap.add_argument("--arm", default=None, help="restrict to one arm")
|
||||
ap.add_argument("--top", type=int, default=0, help="print only the N longest runs (0 = all)")
|
||||
a = ap.parse_args()
|
||||
|
||||
N = a.n
|
||||
corpus_words: list[str] = []
|
||||
for f in sorted(pathlib.Path(a.corpus).glob("*.copy0.jsonl")):
|
||||
for l in f.read_text(encoding="utf-8").splitlines():
|
||||
r = json.loads(l)
|
||||
if a.train_only and r.get("split") != "train":
|
||||
continue
|
||||
corpus_words.extend(re.findall(r"[a-z']+", r["text"].lower()))
|
||||
grams = {" ".join(corpus_words[i:i + N]) for i in range(len(corpus_words) - N + 1)}
|
||||
scope = "train split only" if a.train_only else "all records"
|
||||
print(f"corpus ({scope}): {len(corpus_words):,} words, {len(grams):,} distinct {N}-grams")
|
||||
|
||||
# POSITIVE CONTROL, every run. A matcher that only ever sees candidate text cannot tell
|
||||
# `no matches` from `I am blind`, which is the same rule the gates run under.
|
||||
probe = corpus_words[1000:1000 + N]
|
||||
if " ".join(probe) not in grams:
|
||||
print("== REFUSING: a slice of the corpus does not match the corpus. The matcher is broken.")
|
||||
return 1
|
||||
print(f" [PASS] positive control: a corpus slice matches itself\n")
|
||||
|
||||
found = []
|
||||
for f in sorted(pathlib.Path(a.eval_dir).glob(a.glob)):
|
||||
arm = f.stem.replace(a.strip, "")
|
||||
if a.arm and arm != a.arm:
|
||||
continue
|
||||
for l in f.read_text(encoding="utf-8").splitlines():
|
||||
r = json.loads(l)
|
||||
cased, low = tokens(r["raw"])
|
||||
i = 0
|
||||
while i <= len(low) - N:
|
||||
if " ".join(low[i:i + N]) in grams:
|
||||
k = N
|
||||
while i + k < len(low) and " ".join(low[i + k - N + 1:i + k + 1]) in grams:
|
||||
k += 1
|
||||
# a token capitalised anywhere but the first position of the RUN is a
|
||||
# proper noun rather than a sentence opening -- crude, deliberately
|
||||
# over-reports, and every flag is meant to be read not counted.
|
||||
caps = [w for w in cased[i + 1:i + k] if w[:1].isupper()]
|
||||
found.append({"arm": arm, "id": r["id"], "seed": r["seed"], "len": k,
|
||||
"text": " ".join(cased[i:i + k]), "caps": caps})
|
||||
i += k
|
||||
else:
|
||||
i += 1
|
||||
if not found:
|
||||
print(f"no runs of {N}+ words shared with the corpus in any arm scanned.")
|
||||
return 0
|
||||
|
||||
found.sort(key=lambda d: -d["len"])
|
||||
shown = found[:a.top] if a.top else found
|
||||
flagged = [d for d in found if d["caps"]]
|
||||
print(f"{len(found)} matched run(s), longest {found[0]['len']} words. "
|
||||
f"{len(flagged)} carry a mid-run capital.\n")
|
||||
for d in shown:
|
||||
flag = f" ⚠ CAPS: {', '.join(d['caps'])}" if d["caps"] else ""
|
||||
print(f" [{d['len']:>3}w] {d['arm']:<12} {d['id']:>4} seed={d['seed']}{flag}")
|
||||
print(f" {d['text']}")
|
||||
if a.top and len(found) > a.top:
|
||||
print(f"\n ... {len(found) - a.top} shorter run(s) not shown; re-run without --top.")
|
||||
print("\n⚠ READ THESE. The rate is not the finding; what is being reproduced is.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user