BabyYarros: pre-register the pair-vs-rawtext decision rule before the arms are read
This commit is contained in:
@@ -39,6 +39,11 @@ ap.add_argument("--seeds", type=int, nargs="+", default=[1234, 5678])
|
||||
ap.add_argument("--max-new-tokens", type=int, default=320)
|
||||
ap.add_argument("--temperature", type=float, default=0.9)
|
||||
ap.add_argument("--top-p", type=float, default=0.95)
|
||||
# The pair-SFT arm was trained with the user turn rendered as "Beat: <x>" while this
|
||||
# harness has always sent "BEAT: <x>". Default is unchanged so every previously recorded
|
||||
# arm stays byte-reproducible; the flag exists so the casing can be MEASURED as its own
|
||||
# variable instead of being confounded with the adapter it is being used to judge.
|
||||
ap.add_argument("--user-prefix", default="BEAT: ")
|
||||
a = ap.parse_args()
|
||||
|
||||
beats = json.loads(Path(a.beats).read_text())
|
||||
@@ -71,7 +76,7 @@ with out.open("w", encoding="utf-8") as fh:
|
||||
torch.manual_seed(seed)
|
||||
text = tok.apply_chat_template(
|
||||
[{"role": "system", "content": SYS},
|
||||
{"role": "user", "content": "BEAT: " + b["beat"]}],
|
||||
{"role": "user", "content": a.user_prefix + b["beat"]}],
|
||||
tokenize=False, add_generation_prompt=True,
|
||||
**({"enable_thinking": False} if "enable_thinking" in (tok.chat_template or "") else {}))
|
||||
ids = tok(text, return_tensors="pt", add_special_tokens=False).to("cuda")
|
||||
@@ -87,7 +92,7 @@ with out.open("w", encoding="utf-8") as fh:
|
||||
hit = sum(1 for k in kws if k[:5] in para.lower())
|
||||
w = len(para.split())
|
||||
fh.write(json.dumps({"format": a.arm, "id": b["id"], "beat": b["beat"], "seed": seed,
|
||||
"prompt": "BEAT: " + b["beat"], "paragraph": para,
|
||||
"prompt": a.user_prefix + b["beat"], "paragraph": para,
|
||||
"ran_on": m is None, "words": w,
|
||||
"in_band": 90 <= w <= 140,
|
||||
"beat_keywords": kws, "keyword_hits": hit}) + "\n")
|
||||
|
||||
Reference in New Issue
Block a user