59 lines
2.5 KiB
Python
59 lines
2.5 KiB
Python
"""Did an arm learn the voice, or learn the text? delta_cb cannot tell them apart.
|
|
|
|
pairs-ckpt150 scored delta_cb 0.470 against a same-author target of 0.463 -- i.e. at this
|
|
sample size it is statistically indistinguishable from real held-out Yarros. That is either
|
|
excellent voice capture or near-verbatim regurgitation, and those two have opposite
|
|
consequences: one ships, the other is both a quality mirage and the exact leak the rename
|
|
pipeline and its gate exist to prevent. Char-bigram distance is blind to the difference.
|
|
|
|
Instrument: longest and mean maximal verbatim n-gram shared with the TRAINING corpus, per
|
|
generation. Controls run every time -- the base-unadapted arm never saw the corpus so it is
|
|
the negative control, and a slice of the corpus scored against itself is the positive.
|
|
"""
|
|
import json, pathlib, re, sys
|
|
from collections import Counter
|
|
|
|
EVAL = pathlib.Path("/home/infra-ops/r49-runs/yarros-eval")
|
|
CORP = pathlib.Path("/home/infra-ops/yarros-corpus-renamed/copies")
|
|
N = 8
|
|
|
|
def norm(t): return re.findall(r"[a-z']+", t.lower())
|
|
|
|
corpus_words = []
|
|
for f in sorted(CORP.glob("*.copy0.jsonl")):
|
|
for l in f.read_text(encoding="utf-8").splitlines():
|
|
corpus_words.extend(norm(json.loads(l)["text"]))
|
|
grams = set()
|
|
for i in range(len(corpus_words) - N + 1):
|
|
grams.add(" ".join(corpus_words[i:i + N]))
|
|
print(f"corpus: {len(corpus_words):,} words, {len(grams):,} distinct {N}-grams\n")
|
|
|
|
def longest_match(words):
|
|
best = 0
|
|
i = 0
|
|
while i <= len(words) - N:
|
|
if " ".join(words[i:i + N]) in grams:
|
|
k = N
|
|
while i + k < len(words) and " ".join(words[i + k - N + 1:i + k + 1]) in grams:
|
|
k += 1
|
|
best = max(best, k)
|
|
i += 1
|
|
else:
|
|
i += 1
|
|
return best
|
|
|
|
print(f"{'arm':<22} {'gens':>5} {'hit-rate':>9} {'mean-longest':>13} {'max':>5}")
|
|
print("-" * 60)
|
|
for f in sorted(EVAL.glob("beats5.*.jsonl")):
|
|
arm = f.stem.replace("beats5.", "")
|
|
rows = [json.loads(l) for l in f.read_text(encoding="utf-8").splitlines()]
|
|
longs = [longest_match(norm(r["raw"])) for r in rows]
|
|
hits = sum(1 for x in longs if x >= N)
|
|
print(f"{arm:<22} {len(rows):>5} {hits/len(rows):>9.2f} "
|
|
f"{sum(longs)/len(longs):>13.1f} {max(longs):>5}")
|
|
|
|
# positive control: corpus against itself must saturate
|
|
slice_words = corpus_words[1000:1160]
|
|
print(f"\npositive control (corpus slice vs corpus): longest = {longest_match(slice_words)} "
|
|
f"(must be large, else the detector is blind)")
|