"""What ARE the verbatim 8-gram hits? A rate is not a judgement.

0.09 against a 0.00 control reads alarming; 0.00 against 0.09 could also be an
artefact of the control writing a different register entirely (the base arm wrote
18,035 words of summary against the adapted arms' 27,413 of pastiche, and text that
does not imitate the style trivially fails to match its n-grams). The only way to
tell a memorised passage from a common English run is to read them.
"""
import json, re, sys, pathlib
from collections import Counter
CORP = pathlib.Path(sys.argv[1]); EVAL = pathlib.Path(sys.argv[2]); N = 8
def norm(t): return re.findall(r"[a-z']+", t.lower())
words = []
for f in sorted(CORP.glob("*.copy0.jsonl")):
    for l in f.read_text(encoding="utf-8").splitlines():
        words.extend(norm(json.loads(l)["text"]))
grams = {" ".join(words[i:i+N]) for i in range(len(words)-N+1)}
print(f"corpus {len(words):,} words, {len(grams):,} distinct {N}-grams\n")
for arm in ("base", "ckpt1750", "ckpt850"):
    p = EVAL/f"beats5.{arm}.jsonl"
    if not p.exists(): continue
    rows = [json.loads(l) for l in p.read_text(encoding="utf-8").splitlines() if l.strip()]
    found = Counter()
    for r in rows:
        w = norm(r["raw"]); i = 0
        while i <= len(w)-N:
            g = " ".join(w[i:i+N])
            if g in grams:
                k = N
                while i+k < len(w) and " ".join(w[i+k-N+1:i+k+1]) in grams: k += 1
                found[" ".join(w[i:i+k])] += 1
                i += k
            else:
                i += 1
    print(f"=== {arm}: {len(rows)} gens, {sum(found.values())} matched runs, {len(found)} distinct")
    for g, c in found.most_common(40):
        print(f"   x{c}  [{len(g.split())}w]  {g}")
    print()