diff --git a/scripts/r49-corpus/gen_beats_chat_yarros.py b/scripts/r49-corpus/gen_beats_chat_yarros.py index 743663a..8787191 100644 --- a/scripts/r49-corpus/gen_beats_chat_yarros.py +++ b/scripts/r49-corpus/gen_beats_chat_yarros.py @@ -92,7 +92,9 @@ with out.open("w", encoding="utf-8") as fh: hit = sum(1 for k in kws if k[:5] in para.lower()) w = len(para.split()) fh.write(json.dumps({"format": a.arm, "id": b["id"], "beat": b["beat"], "seed": seed, - "prompt": a.user_prefix + b["beat"], "paragraph": para, + "prompt": a.user_prefix + b["beat"], "paragraph": para, "raw": raw, + "raw_words": len(raw.split()), + "raw_blocks": len(STOP.split(raw)), "ran_on": m is None, "words": w, "in_band": 90 <= w <= 140, "beat_keywords": kws, "keyword_hits": hit}) + "\n") diff --git a/scripts/yarros-corpus/memorization_check.py b/scripts/yarros-corpus/memorization_check.py new file mode 100644 index 0000000..bd3ce72 --- /dev/null +++ b/scripts/yarros-corpus/memorization_check.py @@ -0,0 +1,58 @@ +"""Did an arm learn the voice, or learn the text? delta_cb cannot tell them apart. + +pairs-ckpt150 scored delta_cb 0.470 against a same-author target of 0.463 -- i.e. at this +sample size it is statistically indistinguishable from real held-out Yarros. That is either +excellent voice capture or near-verbatim regurgitation, and those two have opposite +consequences: one ships, the other is both a quality mirage and the exact leak the rename +pipeline and its gate exist to prevent. Char-bigram distance is blind to the difference. + +Instrument: longest and mean maximal verbatim n-gram shared with the TRAINING corpus, per +generation. Controls run every time -- the base-unadapted arm never saw the corpus so it is +the negative control, and a slice of the corpus scored against itself is the positive. +""" +import json, pathlib, re, sys +from collections import Counter + +EVAL = pathlib.Path("/home/infra-ops/r49-runs/yarros-eval") +CORP = pathlib.Path("/home/infra-ops/yarros-corpus-renamed/copies") +N = 8 + +def norm(t): return re.findall(r"[a-z']+", t.lower()) + +corpus_words = [] +for f in sorted(CORP.glob("*.copy0.jsonl")): + for l in f.read_text(encoding="utf-8").splitlines(): + corpus_words.extend(norm(json.loads(l)["text"])) +grams = set() +for i in range(len(corpus_words) - N + 1): + grams.add(" ".join(corpus_words[i:i + N])) +print(f"corpus: {len(corpus_words):,} words, {len(grams):,} distinct {N}-grams\n") + +def longest_match(words): + best = 0 + i = 0 + while i <= len(words) - N: + if " ".join(words[i:i + N]) in grams: + k = N + while i + k < len(words) and " ".join(words[i + k - N + 1:i + k + 1]) in grams: + k += 1 + best = max(best, k) + i += 1 + else: + i += 1 + return best + +print(f"{'arm':<22} {'gens':>5} {'hit-rate':>9} {'mean-longest':>13} {'max':>5}") +print("-" * 60) +for f in sorted(EVAL.glob("beats5.*.jsonl")): + arm = f.stem.replace("beats5.", "") + rows = [json.loads(l) for l in f.read_text(encoding="utf-8").splitlines()] + longs = [longest_match(norm(r["raw"])) for r in rows] + hits = sum(1 for x in longs if x >= N) + print(f"{arm:<22} {len(rows):>5} {hits/len(rows):>9.2f} " + f"{sum(longs)/len(longs):>13.1f} {max(longs):>5}") + +# positive control: corpus against itself must saturate +slice_words = corpus_words[1000:1160] +print(f"\npositive control (corpus slice vs corpus): longest = {longest_match(slice_words)} " + f"(must be large, else the detector is blind)") diff --git a/scripts/yarros-corpus/score_beats.py b/scripts/yarros-corpus/score_beats.py index 41b1ae6..b89a19a 100644 --- a/scripts/yarros-corpus/score_beats.py +++ b/scripts/yarros-corpus/score_beats.py @@ -46,27 +46,50 @@ def load(path: Path) -> list[dict]: return [json.loads(l) for l in path.read_text(encoding="utf-8").splitlines() if l.strip()] -def per_seed(rows: list[dict]) -> dict[int, dict]: +def view(r: dict, source: str) -> tuple[str, int, bool]: + """Return (text, words, ran_on) for the chosen measurement surface. + + The harness has always scored `paragraph` -- the generation truncated at its first + blank line -- because every arm before this one was expected to emit ONE paragraph. + The pair arm emits a multi-paragraph passage by construction, so that surface reported + it as a 19-word off-beat fragment while the untruncated output was 90-132 words with + the beat rendered in a later block. Both views are kept and the verdict names which one + it used: truncated answers "does it emit one paragraph", raw answers "did it render the + beat at the requested length", and either alone answers a different question than the + reader assumes it does. + """ + if source == "raw": + txt = r.get("raw", r["paragraph"]) + w = r.get("raw_words", len(txt.split())) + return txt, w, w > 140 # ran-on := overshot the requested band + return r["paragraph"], r["words"], r["ran_on"] + + +def per_seed(rows: list[dict], source: str) -> dict[int, dict]: by = defaultdict(list) for r in rows: by[r["seed"]].append(r) out = {} for seed, rs in by.items(): - cov = [(r["keyword_hits"] / len(r["beat_keywords"])) if r["beat_keywords"] else 0.0 - for r in rs] + views = [view(r, source) for r in rs] + cov = [] + for r, (txt, _w, _ro) in zip(rs, views): + kws = r["beat_keywords"] + low = txt.lower() + cov.append((sum(1 for k in kws if k[:5] in low) / len(kws)) if kws else 0.0) out[seed] = { "n": len(rs), - "in_band": sum(1 for r in rs if r["in_band"]) / len(rs), - "ran_on": sum(1 for r in rs if r["ran_on"]) / len(rs), + "in_band": sum(1 for _t, w, _ro in views if 90 <= w <= 140) / len(rs), + "ran_on": sum(1 for _t, _w, ro in views if ro) / len(rs), "coverage": st.mean(cov), "on_beat": sum(1 for c in cov if c >= ON_BEAT_COVERAGE) / len(cov), - "words_median": st.median(r["words"] for r in rs), + "words_median": st.median(w for _t, w, _ro in views), } return out -def summarise(name: str, rows: list[dict]) -> dict: - seeds = per_seed(rows) +def summarise(name: str, rows: list[dict], source: str) -> dict: + seeds = per_seed(rows, source) agg = {"arm": name, "n": len(rows), "seeds": len(seeds)} for k in ("in_band", "ran_on", "coverage", "on_beat", "words_median"): vals = [s[k] for s in seeds.values()] @@ -81,16 +104,18 @@ def main() -> int: ap.add_argument("--baseline", required=True, help="arm name the rule compares against") ap.add_argument("--candidate", required=True) ap.add_argument("--out", default=None) + ap.add_argument("--metric-source", choices=["paragraph", "raw"], default="paragraph") a = ap.parse_args() arms = {} for spec in a.arm: name, path = spec.split("=", 1) - arms[name] = summarise(name, load(Path(path))) + arms[name] = summarise(name, load(Path(path)), a.metric_source) floor = max(max(v[k + "_spread"] for k in ("in_band", "ran_on", "on_beat", "coverage")) for v in arms.values()) + print(f"measurement surface: {a.metric_source}") hdr = f"{'arm':<28} {'n':>3} {'in-band':>8} {'on-beat':>8} {'cover':>7} {'ran-on':>7} {'words':>6}" print(hdr); print("-" * len(hdr)) for v in arms.values(): @@ -118,7 +143,7 @@ def main() -> int: print(f"\nVERDICT: {verdict}") if a.out: Path(a.out).write_text(json.dumps( - {"arms": arms, "noise_floor": floor, "on_beat_coverage_threshold": ON_BEAT_COVERAGE, + {"arms": arms, "noise_floor": floor, "metric_source": a.metric_source, "on_beat_coverage_threshold": ON_BEAT_COVERAGE, "baseline": a.baseline, "candidate": a.candidate, "deltas": {"in_band": d_band, "on_beat": d_beat, "ran_on": d_ran}, "criteria": {"in_band_gt_floor": c1, "on_beat_not_worse": c2, "ran_on_not_worse": c3},