diff --git a/scripts/r49-corpus/gen_beats_chat_yarros.py b/scripts/r49-corpus/gen_beats_chat_yarros.py index c5f6d3f..743663a 100644 --- a/scripts/r49-corpus/gen_beats_chat_yarros.py +++ b/scripts/r49-corpus/gen_beats_chat_yarros.py @@ -39,6 +39,11 @@ ap.add_argument("--seeds", type=int, nargs="+", default=[1234, 5678]) ap.add_argument("--max-new-tokens", type=int, default=320) ap.add_argument("--temperature", type=float, default=0.9) ap.add_argument("--top-p", type=float, default=0.95) +# The pair-SFT arm was trained with the user turn rendered as "Beat: " while this +# harness has always sent "BEAT: ". Default is unchanged so every previously recorded +# arm stays byte-reproducible; the flag exists so the casing can be MEASURED as its own +# variable instead of being confounded with the adapter it is being used to judge. +ap.add_argument("--user-prefix", default="BEAT: ") a = ap.parse_args() beats = json.loads(Path(a.beats).read_text()) @@ -71,7 +76,7 @@ with out.open("w", encoding="utf-8") as fh: torch.manual_seed(seed) text = tok.apply_chat_template( [{"role": "system", "content": SYS}, - {"role": "user", "content": "BEAT: " + b["beat"]}], + {"role": "user", "content": a.user_prefix + b["beat"]}], tokenize=False, add_generation_prompt=True, **({"enable_thinking": False} if "enable_thinking" in (tok.chat_template or "") else {})) ids = tok(text, return_tensors="pt", add_special_tokens=False).to("cuda") @@ -87,7 +92,7 @@ with out.open("w", encoding="utf-8") as fh: hit = sum(1 for k in kws if k[:5] in para.lower()) w = len(para.split()) fh.write(json.dumps({"format": a.arm, "id": b["id"], "beat": b["beat"], "seed": seed, - "prompt": "BEAT: " + b["beat"], "paragraph": para, + "prompt": a.user_prefix + b["beat"], "paragraph": para, "ran_on": m is None, "words": w, "in_band": 90 <= w <= 140, "beat_keywords": kws, "keyword_hits": hit}) + "\n") diff --git a/scripts/yarros-corpus/score_beats.py b/scripts/yarros-corpus/score_beats.py new file mode 100644 index 0000000..41b1ae6 --- /dev/null +++ b/scripts/yarros-corpus/score_beats.py @@ -0,0 +1,130 @@ +"""Score beat arms against a decision rule fixed BEFORE any arm's output was read. + +⚠ THE RULE BELOW IS PRE-REGISTERED. It was written and committed while the generation +chain was still running, for the reason brokkr's frozen adjudication exists: a threshold +chosen after seeing the numbers is a threshold chosen to produce a verdict. If it needs to +change, change it in a commit that says so and re-run every arm -- do not edit it in place +between a read and a conclusion. + +THE QUESTION: does pair-SFT fix the length/close discipline that raw-text training cost, +without giving back the direction-following it bought? The raw-text instruct arm's known +profile is on-beat strong, in-band weak, ran-on frequent. In-band is therefore the axis the +pilot exists to move, and it is the axis the rule keys on. + +DECISION RULE — pairs replace raw-text for Skaldsong iff ALL THREE hold, same harness, +same n, same box, every arm re-measured in one session: + + 1. in-band rate is HIGHER than the raw-text arm by MORE than the pooled within-arm + seed spread. A gain inside the spread is noise, not a fix. + 2. on-beat coverage is not WORSE than the raw-text arm by more than that same spread -- + it must not have bought length by losing direction. + 3. ran-on rate is not HIGHER than the raw-text arm by more than that spread. + + (1) fails -> the pilot did not do its job. Do not scale. + (1) holds, 2 or 3 no -> mixed result. Surface to the operator; do not auto-scale. + all three -> scale to the full corpus. + +on-beat is scored as COVERAGE (fraction of the beat's content keywords that surface in the +paragraph), thresholded at >= 0.5 for the binary. Both the coverage mean and the binary are +reported: a rule that reads only the binary hides a large move inside a threshold, and a +rule that reads only the mean hides a bimodal arm. + +NOISE FLOOR: the spread is the max-minus-min of a metric across the four SEEDS within an +arm, pooled (max) over arms. It is measured per run and printed with the verdict, because a +between-arm gap smaller than it is not a finding. This is the A-vs-A floor; it is not the +same thing as the distance to real Yarros and must not be reported as if it were. +""" +from __future__ import annotations +import argparse, json, math, statistics as st +from collections import defaultdict +from pathlib import Path + +ON_BEAT_COVERAGE = 0.5 + + +def load(path: Path) -> list[dict]: + return [json.loads(l) for l in path.read_text(encoding="utf-8").splitlines() if l.strip()] + + +def per_seed(rows: list[dict]) -> dict[int, dict]: + by = defaultdict(list) + for r in rows: + by[r["seed"]].append(r) + out = {} + for seed, rs in by.items(): + cov = [(r["keyword_hits"] / len(r["beat_keywords"])) if r["beat_keywords"] else 0.0 + for r in rs] + out[seed] = { + "n": len(rs), + "in_band": sum(1 for r in rs if r["in_band"]) / len(rs), + "ran_on": sum(1 for r in rs if r["ran_on"]) / len(rs), + "coverage": st.mean(cov), + "on_beat": sum(1 for c in cov if c >= ON_BEAT_COVERAGE) / len(cov), + "words_median": st.median(r["words"] for r in rs), + } + return out + + +def summarise(name: str, rows: list[dict]) -> dict: + seeds = per_seed(rows) + agg = {"arm": name, "n": len(rows), "seeds": len(seeds)} + for k in ("in_band", "ran_on", "coverage", "on_beat", "words_median"): + vals = [s[k] for s in seeds.values()] + agg[k] = st.mean(vals) + agg[k + "_spread"] = max(vals) - min(vals) + return agg + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--arm", action="append", required=True, metavar="NAME=PATH") + ap.add_argument("--baseline", required=True, help="arm name the rule compares against") + ap.add_argument("--candidate", required=True) + ap.add_argument("--out", default=None) + a = ap.parse_args() + + arms = {} + for spec in a.arm: + name, path = spec.split("=", 1) + arms[name] = summarise(name, load(Path(path))) + + floor = max(max(v[k + "_spread"] for k in ("in_band", "ran_on", "on_beat", "coverage")) + for v in arms.values()) + + hdr = f"{'arm':<28} {'n':>3} {'in-band':>8} {'on-beat':>8} {'cover':>7} {'ran-on':>7} {'words':>6}" + print(hdr); print("-" * len(hdr)) + for v in arms.values(): + print(f"{v['arm']:<28} {v['n']:>3} {v['in_band']:>8.2f} {v['on_beat']:>8.2f} " + f"{v['coverage']:>7.2f} {v['ran_on']:>7.2f} {v['words_median']:>6.0f}") + print(f"\nnoise floor (max within-arm spread across seeds): {floor:.3f}") + print("⚠ any between-arm gap at or under that is NOT a finding\n") + + b, c = arms[a.baseline], arms[a.candidate] + d_band = c["in_band"] - b["in_band"] + d_beat = c["on_beat"] - b["on_beat"] + d_ran = c["ran_on"] - b["ran_on"] + c1 = d_band > floor + c2 = d_beat >= -floor + c3 = d_ran <= floor + print(f"1. in-band {c['in_band']:.2f} vs {b['in_band']:.2f} delta {d_band:+.2f} " + f"{'PASS' if c1 else 'FAIL'} (needs > +{floor:.3f})") + print(f"2. on-beat {c['on_beat']:.2f} vs {b['on_beat']:.2f} delta {d_beat:+.2f} " + f"{'PASS' if c2 else 'FAIL'} (needs >= -{floor:.3f})") + print(f"3. ran-on {c['ran_on']:.2f} vs {b['ran_on']:.2f} delta {d_ran:+.2f} " + f"{'PASS' if c3 else 'FAIL'} (needs <= +{floor:.3f})") + + verdict = ("SCALE" if (c1 and c2 and c3) else + "DO-NOT-SCALE" if not c1 else "MIXED-SURFACE-TO-OPERATOR") + print(f"\nVERDICT: {verdict}") + if a.out: + Path(a.out).write_text(json.dumps( + {"arms": arms, "noise_floor": floor, "on_beat_coverage_threshold": ON_BEAT_COVERAGE, + "baseline": a.baseline, "candidate": a.candidate, + "deltas": {"in_band": d_band, "on_beat": d_beat, "ran_on": d_ran}, + "criteria": {"in_band_gt_floor": c1, "on_beat_not_worse": c2, "ran_on_not_worse": c3}, + "verdict": verdict}, indent=2), encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())