vh/esh-pfi-infrastructure
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects Releases Wiki Activity

BabyYarros: pre-register the pair-vs-rawtext decision rule before the arms are read

Browse Source
This commit is contained in:
Vuong Hoang
2026-09-15 11:19:11 -07:00
parent 90ed506db3
commit 713e83dd5e
2 changed files with 137 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/r49-corpus/gen_beats_chat_yarros.py
+7 -2
View File
@@ -39,6 +39,11 @@ ap.add_argument("--seeds", type=int, nargs="+", default=[1234, 5678])
ap.add_argument("--max-new-tokens", type=int, default=320)
ap.add_argument("--temperature", type=float, default=0.9)
ap.add_argument("--top-p", type=float, default=0.95)
# The pair-SFT arm was trained with the user turn rendered as "Beat: <x>" while this
# harness has always sent "BEAT: <x>". Default is unchanged so every previously recorded
# arm stays byte-reproducible; the flag exists so the casing can be MEASURED as its own
# variable instead of being confounded with the adapter it is being used to judge.
ap.add_argument("--user-prefix", default="BEAT: ")
a = ap.parse_args()
beats = json.loads(Path(a.beats).read_text())
@@ -71,7 +76,7 @@ with out.open("w", encoding="utf-8") as fh:
torch.manual_seed(seed)
text = tok.apply_chat_template(
[{"role": "system", "content": SYS},
{"role": "user", "content": "BEAT: " + b["beat"]}],
{"role": "user", "content": a.user_prefix + b["beat"]}],
tokenize=False, add_generation_prompt=True,
**({"enable_thinking": False} if "enable_thinking" in (tok.chat_template or "") else {}))
ids = tok(text, return_tensors="pt", add_special_tokens=False).to("cuda")
@@ -87,7 +92,7 @@ with out.open("w", encoding="utf-8") as fh:
hit = sum(1 for k in kws if k[:5] in para.lower())
w = len(para.split())
fh.write(json.dumps({"format": a.arm, "id": b["id"], "beat": b["beat"], "seed": seed,
"prompt": "BEAT: " + b["beat"], "paragraph": para,
"prompt": a.user_prefix + b["beat"], "paragraph": para,
"ran_on": m is None, "words": w,
"in_band": 90 <= w <= 140,
"beat_keywords": kws, "keyword_hits": hit}) + "\n")
scripts/yarros-corpus/score_beats.py
+130
View File
@@ -0,0 +1,130 @@
"""Score beat arms against a decision rule fixed BEFORE any arm's output was read.
⚠ THE RULE BELOW IS PRE-REGISTERED. It was written and committed while the generation
chain was still running, for the reason brokkr's frozen adjudication exists: a threshold
chosen after seeing the numbers is a threshold chosen to produce a verdict. If it needs to
change, change it in a commit that says so and re-run every arm -- do not edit it in place
between a read and a conclusion.
THE QUESTION: does pair-SFT fix the length/close discipline that raw-text training cost,
without giving back the direction-following it bought? The raw-text instruct arm's known
profile is on-beat strong, in-band weak, ran-on frequent. In-band is therefore the axis the
pilot exists to move, and it is the axis the rule keys on.
DECISION RULE — pairs replace raw-text for Skaldsong iff ALL THREE hold, same harness,
same n, same box, every arm re-measured in one session:
1. in-band rate is HIGHER than the raw-text arm by MORE than the pooled within-arm
seed spread. A gain inside the spread is noise, not a fix.
2. on-beat coverage is not WORSE than the raw-text arm by more than that same spread --
it must not have bought length by losing direction.
3. ran-on rate is not HIGHER than the raw-text arm by more than that spread.
(1) fails -> the pilot did not do its job. Do not scale.
(1) holds, 2 or 3 no -> mixed result. Surface to the operator; do not auto-scale.
all three -> scale to the full corpus.
on-beat is scored as COVERAGE (fraction of the beat's content keywords that surface in the
paragraph), thresholded at >= 0.5 for the binary. Both the coverage mean and the binary are
reported: a rule that reads only the binary hides a large move inside a threshold, and a
rule that reads only the mean hides a bimodal arm.
NOISE FLOOR: the spread is the max-minus-min of a metric across the four SEEDS within an
arm, pooled (max) over arms. It is measured per run and printed with the verdict, because a
between-arm gap smaller than it is not a finding. This is the A-vs-A floor; it is not the
same thing as the distance to real Yarros and must not be reported as if it were.
"""
from __future__ import annotations
import argparse, json, math, statistics as st
from collections import defaultdict
from pathlib import Path
ON_BEAT_COVERAGE = 0.5
def load(path: Path) -> list[dict]:
return [json.loads(l) for l in path.read_text(encoding="utf-8").splitlines() if l.strip()]
def per_seed(rows: list[dict]) -> dict[int, dict]:
by = defaultdict(list)
for r in rows:
by[r["seed"]].append(r)
out = {}
for seed, rs in by.items():
cov = [(r["keyword_hits"] / len(r["beat_keywords"])) if r["beat_keywords"] else 0.0
for r in rs]
out[seed] = {
"n": len(rs),
"in_band": sum(1 for r in rs if r["in_band"]) / len(rs),
"ran_on": sum(1 for r in rs if r["ran_on"]) / len(rs),
"coverage": st.mean(cov),
"on_beat": sum(1 for c in cov if c >= ON_BEAT_COVERAGE) / len(cov),
"words_median": st.median(r["words"] for r in rs),
}
return out
def summarise(name: str, rows: list[dict]) -> dict:
seeds = per_seed(rows)
agg = {"arm": name, "n": len(rows), "seeds": len(seeds)}
for k in ("in_band", "ran_on", "coverage", "on_beat", "words_median"):
vals = [s[k] for s in seeds.values()]
agg[k] = st.mean(vals)
agg[k + "_spread"] = max(vals) - min(vals)
return agg
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--arm", action="append", required=True, metavar="NAME=PATH")
ap.add_argument("--baseline", required=True, help="arm name the rule compares against")
ap.add_argument("--candidate", required=True)
ap.add_argument("--out", default=None)
a = ap.parse_args()
arms = {}
for spec in a.arm:
name, path = spec.split("=", 1)
arms[name] = summarise(name, load(Path(path)))
floor = max(max(v[k + "_spread"] for k in ("in_band", "ran_on", "on_beat", "coverage"))
for v in arms.values())
hdr = f"{'arm':<28} {'n':>3} {'in-band':>8} {'on-beat':>8} {'cover':>7} {'ran-on':>7} {'words':>6}"
print(hdr); print("-" * len(hdr))
for v in arms.values():
print(f"{v['arm']:<28} {v['n']:>3} {v['in_band']:>8.2f} {v['on_beat']:>8.2f} "
f"{v['coverage']:>7.2f} {v['ran_on']:>7.2f} {v['words_median']:>6.0f}")
print(f"\nnoise floor (max within-arm spread across seeds): {floor:.3f}")
print("⚠ any between-arm gap at or under that is NOT a finding\n")
b, c = arms[a.baseline], arms[a.candidate]
d_band = c["in_band"] - b["in_band"]
d_beat = c["on_beat"] - b["on_beat"]
d_ran = c["ran_on"] - b["ran_on"]
c1 = d_band > floor
c2 = d_beat >= -floor
c3 = d_ran <= floor
print(f"1. in-band {c['in_band']:.2f} vs {b['in_band']:.2f} delta {d_band:+.2f} "
f"{'PASS' if c1 else 'FAIL'} (needs > +{floor:.3f})")
print(f"2. on-beat {c['on_beat']:.2f} vs {b['on_beat']:.2f} delta {d_beat:+.2f} "
f"{'PASS' if c2 else 'FAIL'} (needs >= -{floor:.3f})")
print(f"3. ran-on {c['ran_on']:.2f} vs {b['ran_on']:.2f} delta {d_ran:+.2f} "
f"{'PASS' if c3 else 'FAIL'} (needs <= +{floor:.3f})")
verdict = ("SCALE" if (c1 and c2 and c3) else
"DO-NOT-SCALE" if not c1 else "MIXED-SURFACE-TO-OPERATOR")
print(f"\nVERDICT: {verdict}")
if a.out:
Path(a.out).write_text(json.dumps(
{"arms": arms, "noise_floor": floor, "on_beat_coverage_threshold": ON_BEAT_COVERAGE,
"baseline": a.baseline, "candidate": a.candidate,
"deltas": {"in_band": d_band, "on_beat": d_beat, "ran_on": d_ran},
"criteria": {"in_band_gt_floor": c1, "on_beat_not_worse": c2, "ran_on_not_worse": c3},
"verdict": verdict}, indent=2), encoding="utf-8")
return 0
if __name__ == "__main__":
raise SystemExit(main())