BabyYarros: pre-register the pair-vs-rawtext decision rule before the arms are read

This commit is contained in:
Vuong Hoang
2026-09-15 11:19:11 -07:00
parent 90ed506db3
commit 713e83dd5e
2 changed files with 137 additions and 2 deletions
+7 -2
View File
@@ -39,6 +39,11 @@ ap.add_argument("--seeds", type=int, nargs="+", default=[1234, 5678])
ap.add_argument("--max-new-tokens", type=int, default=320)
ap.add_argument("--temperature", type=float, default=0.9)
ap.add_argument("--top-p", type=float, default=0.95)
# The pair-SFT arm was trained with the user turn rendered as "Beat: <x>" while this
# harness has always sent "BEAT: <x>". Default is unchanged so every previously recorded
# arm stays byte-reproducible; the flag exists so the casing can be MEASURED as its own
# variable instead of being confounded with the adapter it is being used to judge.
ap.add_argument("--user-prefix", default="BEAT: ")
a = ap.parse_args()
beats = json.loads(Path(a.beats).read_text())
@@ -71,7 +76,7 @@ with out.open("w", encoding="utf-8") as fh:
torch.manual_seed(seed)
text = tok.apply_chat_template(
[{"role": "system", "content": SYS},
{"role": "user", "content": "BEAT: " + b["beat"]}],
{"role": "user", "content": a.user_prefix + b["beat"]}],
tokenize=False, add_generation_prompt=True,
**({"enable_thinking": False} if "enable_thinking" in (tok.chat_template or "") else {}))
ids = tok(text, return_tensors="pt", add_special_tokens=False).to("cuda")
@@ -87,7 +92,7 @@ with out.open("w", encoding="utf-8") as fh:
hit = sum(1 for k in kws if k[:5] in para.lower())
w = len(para.split())
fh.write(json.dumps({"format": a.arm, "id": b["id"], "beat": b["beat"], "seed": seed,
"prompt": "BEAT: " + b["beat"], "paragraph": para,
"prompt": a.user_prefix + b["beat"], "paragraph": para,
"ran_on": m is None, "words": w,
"in_band": 90 <= w <= 140,
"beat_keywords": kws, "keyword_hits": hit}) + "\n")