"""BabyYarros Option C — build instruction→response SFT pairs from the gated corpus. The architecture question was settled on 2026-09-11: the adapted completion carrier has the voice and cannot take direction; the instruct model takes direction and has no voice. Skaldsong needs both, which means the corpus has to be rebuilt as instruction→response pairs and trained through the chat template rather than as raw continuation text. ⭐ THE PROPERTY THAT MAKES THIS SAFE: the model writes the INSTRUCTION, never the RESPONSE. Every response is real (renamed) Yarros prose, so the voice is inherited from the corpus and not synthesised. Only the beat is machine-made, and a beat is a summary — the easiest job in the building. A pipeline that generated the prose would be training the carrier on a 27B's pastiche of Yarros, which is the opposite of the point. ⚠ PASSAGES, NOT PARAGRAPHS. Measured on copy0: the median Yarros paragraph is 39 words (p90 67) against the product's 90-140 band, so one paragraph is nowhere near one product output. Accumulating consecutive paragraphs to the band gives 6,992 passages at median 104 words / 4 paragraphs — in-band by construction, and multi-paragraph, which is also the continuity requirement the stitching failure raised (independently-generated paragraphs drift in POV; the pair corpus has to contain multi-paragraph examples). ⚠ ONE COPY ONLY. The rename produced 6 copies under different name maps. Six copies of the same passage would be six near-duplicate responses differing only in proper nouns, which is corpus inflation, not augmentation, at pair-training scale. ⚠ TRAIN SPLIT ONLY. The val split stays untouched so held-out loss remains comparable to the raw-text arms. Generator is the local `gen` seat: free, and abliterated, which matters here for a non-obvious reason — the corpus contains sex and violence, and a refusing generator would silently drop exactly the passages where the voice is most distinctive, biasing the dataset toward its tamest regions. A refusal is a sampling bias, not just a gap. """ from __future__ import annotations import argparse, json, random, re, sys, time, urllib.error, urllib.request from pathlib import Path GATEWAY = "http://10.250.50.70:4000" # Matches gen_beats_chat_yarros.py verbatim so the pairs are trained in the SAME shape the # evaluation harness measures. A pair corpus trained under a different system prompt than # the eval drives would confound the carrier change with a prompt change. # ⚠ DEFECT 1 of the BabyYarros pair build: the system prompt said "ONE paragraph" while every # response was a MEDIAN OF FOUR. The carrier believed the data, correctly, and then looked # broken against a harness that truncated at the first blank line. The prompt now describes # what the response actually is. Both are kept: `SYS` is frozen so the shipped BabyYarros # adapter can still be reproduced and re-evaluated under the prompt it was trained on. SYS = ("You expand a single story beat into ONE paragraph of prose in the manner of Rebecca " "Yarros — contemporary first-person PRESENT-tense narration, emotionally charged, sensory " "and physical, the voice of new-adult romantasy. Render the beat itself; do not move past " "it, do not add a new scene, do not comment. Output the paragraph only, 90–140 words.") SYS_PASSAGE = ("You expand a single story beat into a SHORT PASSAGE of prose in the manner of " "{author} — {register}. The passage may run to several paragraphs and should read " "as continuous scene, not a summary. Render the beat itself; do not move past it, " "do not begin a new scene, do not comment, do not write a chapter heading. Output " "the prose only, 90–140 words.") REGISTERS = { "yarros": ("Rebecca Yarros", "contemporary first-person PRESENT-tense narration, emotionally " "charged, sensory and physical, the voice of new-adult romantasy"), "hemingway": ("Ernest Hemingway", "spare declarative sentences, concrete physical detail, " "heavy unattributed dialogue, feeling shown through action and omission rather " "than stated"), } CONTEXT_BLOCK = """The passage is preceded by this, for reference only. Do NOT write a beat for it — it is here so you resolve names and pronouns correctly. PRECEDING: {prev} """ BEAT_PROMPT = """{context}Below is a passage from a novel. Write the single story BEAT that a writer would have been given to produce it. Rules: - ONE sentence, 8 to 30 words, present tense, third person. - Name the characters who appear, using the names exactly as the passage spells them. - The passage is first-person and its narrator is usually UNNAMED in it. Call that person "she" or "he" as the passage implies. Never write "the narrator", "the speaker" or "I". - State WHAT HAPPENS — the action, the turn, the decision, the reveal. Not how it is written. - Do NOT describe the passage ("this excerpt shows..."), do NOT mention prose, style, tone or the author. - Do NOT reuse distinctive phrases from the passage. Say the event in your own plain words. - Output the sentence alone, with no label, no quotes and no preamble. PASSAGE: {passage}""" ABBREV = re.compile(r"\b(Mr|Mrs|Ms|Dr|St|Sr|Jr|Lt|Col|Gen|Capt|Sgt|Prof|vs|etc|No)\.$") META = re.compile(r"\b(passage|excerpt|paragraph|prose|narrat(?:or|ion)|the (?:author|text|scene) (?:is|describes)|this (?:scene|chapter))\b", re.I) # ⚠ MEASURED, and it was a composition bias rather than a nuisance: of 23 `meta` rejects in # a 150-passage diagnostic, **22 were the single word "narrator"** and 1 was the story-word # "passage". Those beats were otherwise clean ("Miguel shouts a warning as the venin # advances, while the dark wielder taunts the narrator..."). The model reaches for "the # narrator" precisely when the POV character is an ACTOR, so rejecting on it silently drops # action scenes and keeps the ones where she only observes -- a systematic skew in what the # carrier would learn to render, invisible in any spot-read of the kept pairs. The fix is a # single bounded RETRY with the correction restated, not a substitution: the generator knows # the character's gender from the prose and this session does not, and guessing it wrong # across 780k words of first-person narration is the worse failure. NARRATOR_ONLY = re.compile(r"\bnarrat(?:or|ion)\b", re.I) RETRY_NOTE = ("\n\nYour previous answer used the word \"narrator\". Rewrite it referring to that " "person as \"she\" or \"he\" — whichever the passage implies — and change nothing else.") def ctx_block(prev: str | None) -> str: """The beat generator sees the PRECEDING passage even when the pair will not carry it. Found in the positive control: passage [4] of nova ch16 is genuinely ambiguous about who is buried and who is digging, and the hand beat and the model beat disagreed for exactly that reason. A mid-scene passage can underdetermine its own referents; showing the generator the previous passage fixes the pronouns without putting anything in the instruction that the response does not support. """ return CONTEXT_BLOCK.format(prev=prev) if prev else "" def post(path: str, payload: dict, key: str, timeout: int = 120) -> dict: req = urllib.request.Request( GATEWAY + path, data=json.dumps(payload).encode(), headers={"Content-Type": "application/json", "Authorization": f"Bearer {key}"}) with urllib.request.urlopen(req, timeout=timeout) as r: return json.loads(r.read()) HEADING_MAX_WORDS = 6 def drop_leading_heading(paras: list[str], enabled: bool) -> list[str]: """Drop a unit's opening block when it is a bare chapter heading. ⚠ DEFECT 3 of the BabyYarros pair build, and it is far worse on Hemingway: 20 of 600 Yarros responses carried a chapter heading (3.3%), but **315 of 318 Hemingway units open with one** because these editions set `I`, `27` or a story title on its own line. The corpus builder KEEPS those headings deliberately -- they are part of the form the voice lives in and dropping them would teach the model that chapters begin mid-scene -- so the removal belongs here, at pair construction, not upstream in the corpus. A response beginning `CHAPTER SIXTY-SIX` teaches the carrier to emit chapter headings when asked for prose, which is exactly what the pilot adapter did. """ if not enabled or not paras: return paras return paras[1:] if len(paras[0].split()) <= HEADING_MAX_WORDS else paras def chunk(corpus: Path, lo: int, hi: int, split: str, drop_heading: bool = False) -> list[dict]: """Accumulate consecutive paragraphs into product-band passages, per chapter. Chapter-bounded so a passage never straddles a chapter break. The trailing buffer of each chapter is DROPPED rather than emitted short -- an out-of-band response would teach the length the product is trying to hold. """ out = [] for f in sorted(corpus.glob("*.copy0.jsonl")): for line in f.read_text(encoding="utf-8").splitlines(): d = json.loads(line) if d.get("split") != split: continue paras = [p.strip() for p in re.split(r"\n\s*\n", d["text"]) if p.strip()] paras = drop_leading_heading(paras, drop_heading) buf, n, prev = [], 0, None for p in paras: buf.append(p); n += len(p.split()) if n >= lo: if n <= int(hi * 1.6): out.append({"work": d["work"], "chapter": d["chapter"], "split": d["split"], "response": "\n\n".join(buf), "words": n, "prev": prev}) prev = "\n\n".join(buf) else: prev = None # oversized run dropped; context would be a lie buf, n = [], 0 return out def ngrams(text: str, n: int) -> set[str]: w = re.findall(r"[a-z']+", text.lower()) return {" ".join(w[i:i + n]) for i in range(max(0, len(w) - n + 1))} def vet(beat: str, passage: str, overlap_n: int) -> tuple[str | None, str]: """Return (clean_beat, reason). reason is '' on accept. Every rejection is a dataset defect that would otherwise train silently: echo -- a beat quoting the passage teaches the model to copy its instruction back, not to expand it. This is the one that would look fine in a spot-read and poison the whole run. meta -- 'this passage shows...' is a description of text, not a story beat length -- a 60-word beat is a summary; a 4-word beat is a title """ beat = " ".join(beat.strip().split()) beat = re.sub(r'^(?:beat|answer)\s*[:\-]\s*', '', beat, flags=re.I).strip().strip('"“”') if not beat: return None, "empty" # First sentence only -- the model sometimes adds a second. # ⚠ An abbreviation is not a sentence end. The naive `.+?[.!?]\s` truncated # "Jeremias arrives with Mr. Derrick and he sizes him up" to "...with Mr." -- caught in the # Hemingway positive control, and it would have been near-invisible in the built pairs # because the result is still a short grammatical-looking fragment. Hemingway is full of # `Mr. Singh`, `Mr. Bobby`, `Mr. Derrick`, so this fires constantly on this corpus and # essentially never on Yarros -- a defect one corpus exposes and another hides. m = re.match(r"^(.+?[.!?])(?:\s|$)", beat) while m and ABBREV.search(m.group(1)): nxt = re.match(r"^(.+?[.!?])(?:\s|$)", beat[m.end():]) if not nxt: m = None break m = re.match(r"^(.{%d,}?[.!?])(?:\s|$)" % (m.end() + 1), beat) if not m: break if m: beat = m.group(1).strip() nw = len(beat.split()) if nw < 6 or nw > 34: return None, f"length({nw})" if META.search(beat): return None, "meta" if ngrams(beat, overlap_n) & ngrams(passage, overlap_n): return None, f"echo({overlap_n}gram)" return beat, "" NAME = re.compile(r"(? list[str]: """Names the BEAT introduces that the RESPONSE never spells out. Reported, deliberately NOT rejected. Giving the generator the preceding passage fixed the referent ambiguity the positive control exposed -- it corrected a hand-written beat that had the buried and the digging characters backwards -- but it also lets a beat name someone who appears only in that preceding text. Rejecting on this would drop exactly the passages whose POV character is unnamed, which is a sampling bias dressed as a guard (same argument as the refusing-generator note above). So it is counted and surfaced in provenance, and the rate decides whether it needs a rule. """ return sorted({n for n in NAME.findall(beat) if n not in response}) def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--corpus", required=True, help="dir holding *.copy0.jsonl") ap.add_argument("--out", required=True) ap.add_argument("--key-file", default=None, help="file holding the gateway key") ap.add_argument("--key", default=None) ap.add_argument("--model", default="gen") ap.add_argument("--n", type=int, default=600) ap.add_argument("--lo", type=int, default=90) ap.add_argument("--hi", type=int, default=150) ap.add_argument("--split", default="train") ap.add_argument("--seed", type=int, default=4919) # ⚠ DEFECT 2 of the BabyYarros pair build, and the one that shaped its output most. # The chunker starts each passage where the last one ended, so a passage opens MID-SCENE # with lead-in the beat does not describe -- the beat summarises the whole span. At 0.4 the # pair usually withheld the preceding text, so the carrier learned "open somewhere # unrelated, then drift toward the beat", which is exactly what the pilot generations did: # the beat material landed in block 3 or 4. # Setting this to 1.0 motivates the opening instead of hiding it, and it matches how # Skaldsong actually calls the model -- a stitcher always has the previous passage. A # passage at the START of a unit has no predecessor and stays a genuine cold open, which is # correct: a chapter's first passage IS one, and its beat legitimately describes its start. ap.add_argument("--context-frac", type=float, default=0.4, help="fraction of pairs carrying the preceding passage as context; " "use 1.0 for the corrected recipe (see DEFECT 2)") ap.add_argument("--overlap-n", type=int, default=6) ap.add_argument("--register", choices=sorted(REGISTERS), default=None, help="DEFECT 1 fix: emit the PASSAGE system prompt for this author instead " "of the frozen one-paragraph Yarros prompt. Omit to keep the original.") ap.add_argument("--temperature", type=float, default=0.3) ap.add_argument("--control-out", default=None, help="write the first --control-n passages out for hand-written beats") ap.add_argument("--control-n", type=int, default=10) ap.add_argument("--dump-passages", action="store_true", help="chunk and report, generate nothing") ap.add_argument("--drop-leading-heading", action="store_true", help="DEFECT 3 fix: drop a unit's opening block when it is a bare chapter " "heading. OFF by default so the BabyYarros pair set stays byte-reproducible.") ap.add_argument("--control-in", default=None, help="JSON of hand-written beats [{idx,beat}]; generate beats for the SAME " "passages and print side by side. The positive control -- a beat " "generator nobody checked against a human can produce a clean-looking " "dataset that teaches the wrong mapping, and nothing downstream would show it.") a = ap.parse_args() sys_prompt = SYS if a.register: author, register = REGISTERS[a.register] sys_prompt = SYS_PASSAGE.format(author=author, register=register) key = a.key or (Path(a.key_file).read_text().strip() if a.key_file else None) passages = chunk(Path(a.corpus), a.lo, a.hi, a.split, a.drop_leading_heading) print(f"[chunk] {len(passages)} passages in split={a.split}, band {a.lo}-{a.hi}", flush=True) if not passages: print("REFUSING: no passages -- wrong corpus dir or split", file=sys.stderr) return 2 random.seed(a.seed) random.shuffle(passages) if a.control_out: ctl = [{"idx": i, "work": p["work"], "chapter": p["chapter"], "response": p["response"], "beat_handwritten": ""} for i, p in enumerate(passages[:a.control_n])] Path(a.control_out).write_text(json.dumps(ctl, indent=2, ensure_ascii=False), encoding="utf-8") print(f"[control] wrote {len(ctl)} passages to {a.control_out} for hand-written beats") if a.dump_passages: return 0 if not key: print("REFUSING: no gateway key (--key or --key-file)", file=sys.stderr) return 2 # The alias is not the model. `gen` has pointed at different concrete backends over # time; counting by an alias once inflated an exposure figure 4.7x on this fleet. The # build fingerprint is resolved at START and again at END and both go in provenance -- # a seat repointed mid-run would otherwise be invisible in the artefact. def fingerprint() -> str: try: r = post("/v1/chat/completions", {"model": a.model, "max_tokens": 1, "messages": [{"role": "user", "content": "ok"}]}, key, 60) return r.get("system_fingerprint") or "unknown" except Exception as e: return f"unresolved:{e}" fp_start = fingerprint() print(f"[seat] {a.model} fingerprint at start: {fp_start}", flush=True) if a.control_in: hand = {h["idx"]: h["beat"] for h in json.loads(Path(a.control_in).read_text(encoding="utf-8"))} agree = 0 for i in sorted(hand): psg = passages[i] r = post("/v1/chat/completions", { "model": a.model, "temperature": a.temperature, "max_tokens": 80, "messages": [{"role": "user", "content": BEAT_PROMPT.format(passage=psg["response"], context=ctx_block(psg["prev"]))}], }, key) beat, why = vet(r["choices"][0]["message"]["content"] or "", psg["response"], a.overlap_n) print("=" * 72) print(f"[{i}] {psg['work']} ch{psg['chapter']}") print(f" HAND : {hand[i]}") print(f" MODEL: {beat if beat else '<>'}") if beat: agree += 1 print("=" * 72) print(f"[control] {agree}/{len(hand)} model beats passed the guards; " f"judge the EVENT match by reading, not by this count") return 0 out_path = Path(a.out) rejects: dict[str, int] = {} kept = 0 name_drift = 0 retried = 0 retry_saved = 0 drift_examples: list[dict] = [] t0 = time.time() with out_path.open("w", encoding="utf-8") as fh: for p in passages: if kept >= a.n: break try: r = post("/v1/chat/completions", { "model": a.model, "temperature": a.temperature, "max_tokens": 80, "messages": [{"role": "user", "content": BEAT_PROMPT.format(passage=p["response"], context=ctx_block(p["prev"]))}], }, key) raw = r["choices"][0]["message"]["content"] or "" except (urllib.error.URLError, urllib.error.HTTPError, KeyError, TimeoutError) as e: rejects["transport"] = rejects.get("transport", 0) + 1 print(f"[warn] {type(e).__name__}: {e}", flush=True) continue beat, why = vet(raw, p["response"], a.overlap_n) if beat is None and why == "meta" and NARRATOR_ONLY.search(raw): retried += 1 try: r2 = post("/v1/chat/completions", { "model": a.model, "temperature": a.temperature, "max_tokens": 80, "messages": [ {"role": "user", "content": BEAT_PROMPT.format( passage=p["response"], context=ctx_block(p["prev"]))}, {"role": "assistant", "content": raw}, {"role": "user", "content": RETRY_NOTE.strip()}, ]}, key) beat, why = vet(r2["choices"][0]["message"]["content"] or "", p["response"], a.overlap_n) if beat is not None: retry_saved += 1 except (urllib.error.URLError, urllib.error.HTTPError, KeyError, TimeoutError): pass if beat is None: rejects[why.split("(")[0]] = rejects.get(why.split("(")[0], 0) + 1 continue drift = names_not_in_response(beat, p["response"]) if drift: name_drift += 1 drift_examples.append({"beat": beat, "names": drift}) use_ctx = p["prev"] is not None and random.random() < a.context_frac fh.write(json.dumps({ "work": p["work"], "chapter": p["chapter"], "split": p["split"], "beat": beat, "context": p["prev"] if use_ctx else None, "response": p["response"], "words": p["words"], }, ensure_ascii=False) + "\n") kept += 1 if kept % 50 == 0: print(f"[gen] {kept}/{a.n} {time.time()-t0:.0f}s rejects={rejects}", flush=True) fp_end = fingerprint() prov = { "generated_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "generator_model_alias": a.model, "generator_fingerprint_start": fp_start, "generator_fingerprint_end": fp_end, "generator_repointed_midrun": fp_start != fp_end, "gateway": GATEWAY, "temperature": a.temperature, "corpus": str(a.corpus), "split": a.split, "copy": "copy0", "band_words": [a.lo, a.hi], "seed": a.seed, "context_frac": a.context_frac, "overlap_guard_n": a.overlap_n, "passages_available": len(passages), "pairs_kept": kept, "rejects": rejects, "narrator_retries": retried, "narrator_retries_recovered": retry_saved, "beats_naming_absent_entity": name_drift, "beats_naming_absent_entity_rate": round(name_drift / max(1, kept), 4), "beats_naming_absent_entity_examples": drift_examples[:15], "reject_rate": round(sum(rejects.values()) / max(1, kept + sum(rejects.values())), 4), "elapsed_s": round(time.time() - t0, 1), "system_prompt": sys_prompt, "register": a.register or "yarros-frozen-one-paragraph", "drop_leading_heading": a.drop_leading_heading, } Path(str(out_path) + ".provenance.json").write_text(json.dumps(prov, indent=2), encoding="utf-8") print(f"[done] {kept} pairs -> {out_path}") print(f"[done] rejects {rejects} (rate {prov['reject_rate']})") print(f"[done] narrator-retries {retried}, recovered {retry_saved}") print(f"[done] beats naming an entity absent from their response: {name_drift}/{kept} " f"({prov['beats_naming_absent_entity_rate']}) -- reported, not rejected") if prov["generator_repointed_midrun"]: print("⚠ SEAT REPOINTED MID-RUN -- the pairs are not from one generator", file=sys.stderr) return 0 if __name__ == "__main__": raise SystemExit(main())