BabyYarros: raw-surface scoring and a memorization check with both controls

scripts/r49-corpus/gen_beats_chat_yarros.py

@@ -92,7 +92,9 @@ with out.open("w", encoding="utf-8") as fh:
             hit = sum(1 for k in kws if k[:5] in para.lower())
             w = len(para.split())
             fh.write(json.dumps({"format": a.arm, "id": b["id"], "beat": b["beat"], "seed": seed,
-                                 "prompt": a.user_prefix + b["beat"], "paragraph": para,
+                                 "prompt": a.user_prefix + b["beat"], "paragraph": para, "raw": raw,
+                                 "raw_words": len(raw.split()),
+                                 "raw_blocks": len(STOP.split(raw)),
                                  "ran_on": m is None, "words": w,
                                  "in_band": 90 <= w <= 140,
                                  "beat_keywords": kws, "keyword_hits": hit}) + "\n")

scripts/yarros-corpus/memorization_check.py

@@ -0,0 +1,58 @@
+"""Did an arm learn the voice, or learn the text? delta_cb cannot tell them apart.
+
+pairs-ckpt150 scored delta_cb 0.470 against a same-author target of 0.463 -- i.e. at this
+sample size it is statistically indistinguishable from real held-out Yarros. That is either
+excellent voice capture or near-verbatim regurgitation, and those two have opposite
+consequences: one ships, the other is both a quality mirage and the exact leak the rename
+pipeline and its gate exist to prevent. Char-bigram distance is blind to the difference.
+
+Instrument: longest and mean maximal verbatim n-gram shared with the TRAINING corpus, per
+generation. Controls run every time -- the base-unadapted arm never saw the corpus so it is
+the negative control, and a slice of the corpus scored against itself is the positive.
+"""
+import json, pathlib, re, sys
+from collections import Counter
+
+EVAL = pathlib.Path("/home/infra-ops/r49-runs/yarros-eval")
+CORP = pathlib.Path("/home/infra-ops/yarros-corpus-renamed/copies")
+N = 8
+
+def norm(t): return re.findall(r"[a-z']+", t.lower())
+
+corpus_words = []
+for f in sorted(CORP.glob("*.copy0.jsonl")):
+    for l in f.read_text(encoding="utf-8").splitlines():
+        corpus_words.extend(norm(json.loads(l)["text"]))
+grams = set()
+for i in range(len(corpus_words) - N + 1):
+    grams.add(" ".join(corpus_words[i:i + N]))
+print(f"corpus: {len(corpus_words):,} words, {len(grams):,} distinct {N}-grams\n")
+
+def longest_match(words):
+    best = 0
+    i = 0
+    while i <= len(words) - N:
+        if " ".join(words[i:i + N]) in grams:
+            k = N
+            while i + k < len(words) and " ".join(words[i + k - N + 1:i + k + 1]) in grams:
+                k += 1
+            best = max(best, k)
+            i += 1
+        else:
+            i += 1
+    return best
+
+print(f"{'arm':<22} {'gens':>5} {'hit-rate':>9} {'mean-longest':>13} {'max':>5}")
+print("-" * 60)
+for f in sorted(EVAL.glob("beats5.*.jsonl")):
+    arm = f.stem.replace("beats5.", "")
+    rows = [json.loads(l) for l in f.read_text(encoding="utf-8").splitlines()]
+    longs = [longest_match(norm(r["raw"])) for r in rows]
+    hits = sum(1 for x in longs if x >= N)
+    print(f"{arm:<22} {len(rows):>5} {hits/len(rows):>9.2f} "
+          f"{sum(longs)/len(longs):>13.1f} {max(longs):>5}")
+
+# positive control: corpus against itself must saturate
+slice_words = corpus_words[1000:1160]
+print(f"\npositive control (corpus slice vs corpus): longest = {longest_match(slice_words)} "
+      f"(must be large, else the detector is blind)")

scripts/yarros-corpus/score_beats.py

@@ -46,27 +46,50 @@ def load(path: Path) -> list[dict]:
     return [json.loads(l) for l in path.read_text(encoding="utf-8").splitlines() if l.strip()]
 
 
-def per_seed(rows: list[dict]) -> dict[int, dict]:
+def view(r: dict, source: str) -> tuple[str, int, bool]:
+    """Return (text, words, ran_on) for the chosen measurement surface.
+
+    The harness has always scored `paragraph` -- the generation truncated at its first
+    blank line -- because every arm before this one was expected to emit ONE paragraph.
+    The pair arm emits a multi-paragraph passage by construction, so that surface reported
+    it as a 19-word off-beat fragment while the untruncated output was 90-132 words with
+    the beat rendered in a later block. Both views are kept and the verdict names which one
+    it used: truncated answers "does it emit one paragraph", raw answers "did it render the
+    beat at the requested length", and either alone answers a different question than the
+    reader assumes it does.
+    """
+    if source == "raw":
+        txt = r.get("raw", r["paragraph"])
+        w = r.get("raw_words", len(txt.split()))
+        return txt, w, w > 140          # ran-on := overshot the requested band
+    return r["paragraph"], r["words"], r["ran_on"]
+
+
+def per_seed(rows: list[dict], source: str) -> dict[int, dict]:
     by = defaultdict(list)
     for r in rows:
         by[r["seed"]].append(r)
     out = {}
     for seed, rs in by.items():
-        cov = [(r["keyword_hits"] / len(r["beat_keywords"])) if r["beat_keywords"] else 0.0
-               for r in rs]
+        views = [view(r, source) for r in rs]
+        cov = []
+        for r, (txt, _w, _ro) in zip(rs, views):
+            kws = r["beat_keywords"]
+            low = txt.lower()
+            cov.append((sum(1 for k in kws if k[:5] in low) / len(kws)) if kws else 0.0)
         out[seed] = {
             "n": len(rs),
-            "in_band": sum(1 for r in rs if r["in_band"]) / len(rs),
-            "ran_on": sum(1 for r in rs if r["ran_on"]) / len(rs),
+            "in_band": sum(1 for _t, w, _ro in views if 90 <= w <= 140) / len(rs),
+            "ran_on": sum(1 for _t, _w, ro in views if ro) / len(rs),
             "coverage": st.mean(cov),
             "on_beat": sum(1 for c in cov if c >= ON_BEAT_COVERAGE) / len(cov),
-            "words_median": st.median(r["words"] for r in rs),
+            "words_median": st.median(w for _t, w, _ro in views),
         }
     return out
 
 
-def summarise(name: str, rows: list[dict]) -> dict:
-    seeds = per_seed(rows)
+def summarise(name: str, rows: list[dict], source: str) -> dict:
+    seeds = per_seed(rows, source)
     agg = {"arm": name, "n": len(rows), "seeds": len(seeds)}
     for k in ("in_band", "ran_on", "coverage", "on_beat", "words_median"):
         vals = [s[k] for s in seeds.values()]
@@ -81,16 +104,18 @@ def main() -> int:
     ap.add_argument("--baseline", required=True, help="arm name the rule compares against")
     ap.add_argument("--candidate", required=True)
     ap.add_argument("--out", default=None)
+    ap.add_argument("--metric-source", choices=["paragraph", "raw"], default="paragraph")
     a = ap.parse_args()
 
     arms = {}
     for spec in a.arm:
         name, path = spec.split("=", 1)
-        arms[name] = summarise(name, load(Path(path)))
+        arms[name] = summarise(name, load(Path(path)), a.metric_source)
 
     floor = max(max(v[k + "_spread"] for k in ("in_band", "ran_on", "on_beat", "coverage"))
                 for v in arms.values())
 
+    print(f"measurement surface: {a.metric_source}")
     hdr = f"{'arm':<28} {'n':>3} {'in-band':>8} {'on-beat':>8} {'cover':>7} {'ran-on':>7} {'words':>6}"
     print(hdr); print("-" * len(hdr))
     for v in arms.values():
@@ -118,7 +143,7 @@ def main() -> int:
     print(f"\nVERDICT: {verdict}")
     if a.out:
         Path(a.out).write_text(json.dumps(
-            {"arms": arms, "noise_floor": floor, "on_beat_coverage_threshold": ON_BEAT_COVERAGE,
+            {"arms": arms, "noise_floor": floor, "metric_source": a.metric_source, "on_beat_coverage_threshold": ON_BEAT_COVERAGE,
              "baseline": a.baseline, "candidate": a.candidate,
              "deltas": {"in_band": d_band, "on_beat": d_beat, "ran_on": d_ran},
              "criteria": {"in_band_gt_floor": c1, "on_beat_not_worse": c2, "ran_on_not_worse": c3},