esh-pfi-infrastructure/scripts/yarros-corpus/memorization_check.py

"""Did an arm learn the voice, or learn the text? delta_cb cannot tell them apart.

pairs-ckpt150 scored delta_cb 0.470 against a same-author target of 0.463 -- i.e. at this
sample size it is statistically indistinguishable from real held-out Yarros. That is either
excellent voice capture or near-verbatim regurgitation, and those two have opposite
consequences: one ships, the other is both a quality mirage and the exact leak the rename
pipeline and its gate exist to prevent. Char-bigram distance is blind to the difference.

Instrument: longest and mean maximal verbatim n-gram shared with the TRAINING corpus, per
generation. Controls run every time -- the base-unadapted arm never saw the corpus so it is
the negative control, and a slice of the corpus scored against itself is the positive.

⚠⚠ THE BASE-UNADAPTED NEGATIVE CONTROL IS DEFECTIVE, and lv-hemingway is where it was
caught. Base writes SUMMARY (18,035 words) while the adapted arms write PASTICHE (27,413):
text that does not imitate the register cannot collide with that register's n-grams, so
base's 0.00 partly measures "different register" rather than "did not memorise". A negative
control that differs from the candidate in a way CORRELATED with the metric is not a control.

The correct innocent sample is THE AUTHOR HIMSELF -- held-out text no arm trained on, in the
same register by construction. On Hemingway it read 0.01 against the shipped adapter's 0.07,
which REFUTED the comfortable "his plain register makes collisions inevitable" story instead
of assuming it. That control was computed by hand during the lv-hemingway gate and never
committed, so it was not reproducible; `--heldout-reference` is it, committed.

⭐ Two opt-in flags added 2026-09-21 for lv-mccarthy, pre-registered in
`scripts/mccarthy-corpus/GATE-PREREG.md` AMENDMENT 1. Neither runs by default and neither
changes a byte of the default output, because the Yarros, Bronte and Hemingway records were
written by the default path and must stay reproducible.

  --train-only           Build the n-gram set from split=="train" records ONLY. The default
                         builds it from EVERY record including val -- so a collision with
                         held-out text is counted as memorisation of training text, which it
                         is not. Mandatory with --heldout-reference, where the default would
                         score the val text against a gram set containing itself.
  --heldout-reference    Score the val-split text as an extra row, chunked to the arms' own
                         median generation length so the comparison is like for like.
"""
import argparse, json, pathlib, re, statistics, sys
from collections import Counter

# ⚠ These were hardcoded to Yarros. Pointed at a Brontë arm they would have compared
# it against the YARROS corpus and reported a clean zero — a negative that means
# "different book", not "did not memorise". Defaults are unchanged so every Yarros
# number already recorded stays reproducible byte for byte.
_ap = argparse.ArgumentParser()
_ap.add_argument("--eval-dir", default="/home/infra-ops/r49-runs/yarros-eval")
_ap.add_argument("--corpus", default="/home/infra-ops/yarros-corpus-renamed/copies",
                 help="the RENAMED copies the adapter actually trained on")
_ap.add_argument("--glob", default="beats5.*.jsonl", help="arm files inside --eval-dir")
_ap.add_argument("--strip", default="beats5.", help="prefix trimmed to name the arm")
_ap.add_argument("-n", type=int, default=8, help="n-gram length")
_ap.add_argument("--train-only", action="store_true",
                 help="build the gram set from split=='train' records only; see module docstring")
_ap.add_argument("--heldout-reference", action="store_true",
                 help="add the held-out-author innocent-sample row; requires --train-only")
_a = _ap.parse_args()

if _a.heldout_reference and not _a.train_only:
    # Without --train-only the gram set contains the val text, so the held-out row would
    # score the author's own words against themselves and return a saturated number that
    # means nothing. Refuse rather than print it.
    print("== REFUSING: --heldout-reference requires --train-only, or the held-out text is")
    print("   scored against a gram set that contains it and the row is meaningless.")
    sys.exit(1)

EVAL = pathlib.Path(_a.eval_dir)
CORP = pathlib.Path(_a.corpus)
N = _a.n

def norm(t): return re.findall(r"[a-z']+", t.lower())

corpus_words = []
heldout_words = []
for f in sorted(CORP.glob("*.copy0.jsonl")):
    for l in f.read_text(encoding="utf-8").splitlines():
        r = json.loads(l)
        if _a.train_only and r.get("split") != "train":
            if r.get("split") == "val":
                heldout_words.append(norm(r["text"]))
            continue
        corpus_words.extend(norm(r["text"]))
grams = set()
for i in range(len(corpus_words) - N + 1):
    grams.add(" ".join(corpus_words[i:i + N]))
label = "corpus (train split only)" if _a.train_only else "corpus"
print(f"{label}: {len(corpus_words):,} words, {len(grams):,} distinct {N}-grams\n")

def longest_match(words):
    best = 0
    i = 0
    while i <= len(words) - N:
        if " ".join(words[i:i + N]) in grams:
            k = N
            while i + k < len(words) and " ".join(words[i + k - N + 1:i + k + 1]) in grams:
                k += 1
            best = max(best, k)
            i += 1
        else:
            i += 1
    return best

def row(name, samples):
    longs = [longest_match(w) for w in samples]
    hits = sum(1 for x in longs if x >= N)
    print(f"{name:<22} {len(samples):>5} {hits/len(samples):>9.2f} "
          f"{sum(longs)/len(longs):>13.1f} {max(longs):>5}")
    return longs

print(f"{'arm':<22} {'gens':>5} {'hit-rate':>9} {'mean-longest':>13} {'max':>5}")
print("-" * 60)
gen_lengths = []
for f in sorted(EVAL.glob(_a.glob)):
    arm = f.stem.replace(_a.strip, "")
    rows = [json.loads(l) for l in f.read_text(encoding="utf-8").splitlines()]
    samples = [norm(r["raw"]) for r in rows]
    gen_lengths.extend(len(s) for s in samples)
    row(arm, samples)

if _a.heldout_reference:
    # ⭐ THE CORRECT NEGATIVE CONTROL. Real author text, never trained on, in the same
    # register as the candidates by construction -- which is exactly what base-unadapted
    # is not. Chunked to the arms' own median generation length so a longer or shorter
    # sample is not being compared with theirs.
    if not heldout_words:
        print("\n== no split=='val' records found; the held-out reference CANNOT be built.")
        print("   This is not a pass -- the axis has no innocent sample.")
        sys.exit(1)
    chunk = int(statistics.median(gen_lengths)) if gen_lengths else 100
    flat = [w for unit in heldout_words for w in unit]
    samples = [flat[i:i + chunk] for i in range(0, len(flat), chunk)]
    samples = [s for s in samples if len(s) >= chunk // 2]
    print("-" * 60)
    row(f"HELD-OUT (never trained)", samples)
    print(f"  ^ the innocent-sample rate: real author text, same register, {chunk}-word chunks.")
    print("    THIS is what a candidate is compared against -- not base-unadapted, whose zero")
    print("    partly measures `different register` rather than `did not memorise`.")

# positive control: corpus against itself must saturate
slice_words = corpus_words[1000:1160]
print(f"\npositive control (corpus slice vs corpus): longest = {longest_match(slice_words)} "
      f"(must be large, else the detector is blind)")