grok-token-broker: hold a refreshable session credential behind a rotation-safety gate

scripts/yarros-corpus/train_pairs_lora.py

@@ -26,6 +26,7 @@ The cross-arm comparison has to be behavioural (on-beat / in-band / ran-on) plus
 """
 from __future__ import annotations
 import argparse, hashlib, json, math, random, subprocess, time
+import pathlib
 from pathlib import Path
 
 import torch
@@ -36,10 +37,27 @@ from peft import LoraConfig, get_peft_model
 
 TARGETS = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]
 
-SYS = ("You expand a single story beat into ONE paragraph of prose in the manner of Rebecca "
-       "Yarros — contemporary first-person PRESENT-tense narration, emotionally charged, sensory "
-       "and physical, the voice of new-adult romantasy. Render the beat itself; do not move past "
-       "it, do not add a new scene, do not comment. Output the paragraph only, 90–140 words.")
+# ⚠ THE SYSTEM PROMPT MUST MATCH THE ONE THE PAIRS WERE BUILT UNDER, and the pair file now
+# records it. Reading it from the pairs' provenance rather than hardcoding it here is the fix
+# for the defect that shaped the whole BabyYarros pilot: the trainer said "ONE paragraph" while
+# the data was a median of four, and the carrier believed the data. A literal in this file is a
+# second place for that to drift.
+SYS_YARROS_FROZEN = (
+    "You expand a single story beat into ONE paragraph of prose in the manner of Rebecca "
+    "Yarros — contemporary first-person PRESENT-tense narration, emotionally charged, sensory "
+    "and physical, the voice of new-adult romantasy. Render the beat itself; do not move past "
+    "it, do not add a new scene, do not comment. Output the paragraph only, 90–140 words.")
+
+
+def resolve_sys(pairs_path: str) -> tuple[str, str]:
+    """Return (system_prompt, source). Prefer the pair build's own provenance."""
+    prov = pathlib.Path(str(pairs_path) + ".provenance.json")
+    if prov.exists():
+        d = json.loads(prov.read_text(encoding="utf-8"))
+        s = d.get("system_prompt")
+        if s:
+            return s, f"pairs provenance ({d.get('register', '?')})"
+    return SYS_YARROS_FROZEN, "frozen Yarros literal (no provenance found)"
 
 
 def user_msg(rec: dict) -> str:
@@ -57,11 +75,11 @@ class Pairs(Dataset):
     when a template changes its spacing, and it would mask the wrong span without erroring.
     """
 
-    def __init__(self, tok, records, seq_len):
+    def __init__(self, tok, records, seq_len, sys_prompt):
         self.rows = []
         dropped = 0
         for r in records:
-            msgs = [{"role": "system", "content": SYS}, {"role": "user", "content": user_msg(r)}]
+            msgs = [{"role": "system", "content": sys_prompt}, {"role": "user", "content": user_msg(r)}]
             # ⚠ transformers 5.16 returns a BatchEncoding from apply_chat_template(tokenize=True),
             # not a list of ids. Taking len() of it yields 2 (the number of keys), so every
             # example failed the `len(full) <= len(prefix)` test and the whole dataset was
@@ -146,6 +164,8 @@ def main() -> int:
     torch.manual_seed(a.seed); random.seed(a.seed)
     out = Path(a.out); out.mkdir(parents=True, exist_ok=True)
 
+    SYS, sys_src = resolve_sys(a.pairs)
+    print(f"[data] system prompt from {sys_src}")
     tok = AutoTokenizer.from_pretrained(a.base)
     if tok.chat_template is None:
         raise SystemExit("REFUSING: carrier has no chat template -- it is not an instruct build")
@@ -169,7 +189,8 @@ def main() -> int:
     if any(r.get("split") != "val" for r in val_recs):
         raise SystemExit("REFUSING: val pairs are not all from the corpus val split")
 
-    train_ds, val_ds = Pairs(tok, train_recs, a.seq_len), Pairs(tok, val_recs, a.seq_len)
+    train_ds = Pairs(tok, train_recs, a.seq_len, SYS)
+    val_ds = Pairs(tok, val_recs, a.seq_len, SYS)
     with_ctx = sum(1 for r in train_recs if r.get("context"))
     print(f"[data] {len(train_ds)} train pairs ({train_ds.dropped} dropped), "
           f"{len(val_ds)} val ({val_ds.dropped} dropped), "
@@ -213,7 +234,8 @@ def main() -> int:
             "trainable_pct": round(100 * trainable / total, 3),
             "steps_per_epoch": steps_per_epoch, "planned_steps": steps_per_epoch * int(a.epochs),
             "resolved": resolved, "harness_commit": git, "harness_dirty_at_launch": dirty,
-            "launched_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "system_prompt": SYS}
+            "launched_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "system_prompt": SYS,
+            "system_prompt_source": sys_src}
     (out / "provenance.json").write_text(json.dumps(prov, indent=2))
     print("[prov] " + json.dumps({k: prov[k] for k in
           ("pairs_sha256_16", "train_pairs", "planned_steps", "trainable_pct",