BabyYarros: raw-surface scoring and a memorization check with both controls

scripts/yarros-corpus/memorization_check.py

@@ -0,0 +1,58 @@
+"""Did an arm learn the voice, or learn the text? delta_cb cannot tell them apart.
+
+pairs-ckpt150 scored delta_cb 0.470 against a same-author target of 0.463 -- i.e. at this
+sample size it is statistically indistinguishable from real held-out Yarros. That is either
+excellent voice capture or near-verbatim regurgitation, and those two have opposite
+consequences: one ships, the other is both a quality mirage and the exact leak the rename
+pipeline and its gate exist to prevent. Char-bigram distance is blind to the difference.
+
+Instrument: longest and mean maximal verbatim n-gram shared with the TRAINING corpus, per
+generation. Controls run every time -- the base-unadapted arm never saw the corpus so it is
+the negative control, and a slice of the corpus scored against itself is the positive.
+"""
+import json, pathlib, re, sys
+from collections import Counter
+
+EVAL = pathlib.Path("/home/infra-ops/r49-runs/yarros-eval")
+CORP = pathlib.Path("/home/infra-ops/yarros-corpus-renamed/copies")
+N = 8
+
+def norm(t): return re.findall(r"[a-z']+", t.lower())
+
+corpus_words = []
+for f in sorted(CORP.glob("*.copy0.jsonl")):
+    for l in f.read_text(encoding="utf-8").splitlines():
+        corpus_words.extend(norm(json.loads(l)["text"]))
+grams = set()
+for i in range(len(corpus_words) - N + 1):
+    grams.add(" ".join(corpus_words[i:i + N]))
+print(f"corpus: {len(corpus_words):,} words, {len(grams):,} distinct {N}-grams\n")
+
+def longest_match(words):
+    best = 0
+    i = 0
+    while i <= len(words) - N:
+        if " ".join(words[i:i + N]) in grams:
+            k = N
+            while i + k < len(words) and " ".join(words[i + k - N + 1:i + k + 1]) in grams:
+                k += 1
+            best = max(best, k)
+            i += 1
+        else:
+            i += 1
+    return best
+
+print(f"{'arm':<22} {'gens':>5} {'hit-rate':>9} {'mean-longest':>13} {'max':>5}")
+print("-" * 60)
+for f in sorted(EVAL.glob("beats5.*.jsonl")):
+    arm = f.stem.replace("beats5.", "")
+    rows = [json.loads(l) for l in f.read_text(encoding="utf-8").splitlines()]
+    longs = [longest_match(norm(r["raw"])) for r in rows]
+    hits = sum(1 for x in longs if x >= N)
+    print(f"{arm:<22} {len(rows):>5} {hits/len(rows):>9.2f} "
+          f"{sum(longs)/len(longs):>13.1f} {max(longs):>5}")
+
+# positive control: corpus against itself must saturate
+slice_words = corpus_words[1000:1160]
+print(f"\npositive control (corpus slice vs corpus): longest = {longest_match(slice_words)} "
+      f"(must be large, else the detector is blind)")