From 628283366991367246750532392c601bff125b70 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Wed, 16 Sep 2026 08:12:01 -0700 Subject: [PATCH] pairs: fix the three construction defects and the abbreviation-truncation bug --- scripts/yarros-corpus/build_sft_pairs.py | 90 ++++++++++++++++++++++-- 1 file changed, 85 insertions(+), 5 deletions(-) diff --git a/scripts/yarros-corpus/build_sft_pairs.py b/scripts/yarros-corpus/build_sft_pairs.py index 3e2edae..a433a08 100644 --- a/scripts/yarros-corpus/build_sft_pairs.py +++ b/scripts/yarros-corpus/build_sft_pairs.py @@ -39,11 +39,30 @@ GATEWAY = "http://10.250.50.70:4000" # Matches gen_beats_chat_yarros.py verbatim so the pairs are trained in the SAME shape the # evaluation harness measures. A pair corpus trained under a different system prompt than # the eval drives would confound the carrier change with a prompt change. +# ⚠ DEFECT 1 of the BabyYarros pair build: the system prompt said "ONE paragraph" while every +# response was a MEDIAN OF FOUR. The carrier believed the data, correctly, and then looked +# broken against a harness that truncated at the first blank line. The prompt now describes +# what the response actually is. Both are kept: `SYS` is frozen so the shipped BabyYarros +# adapter can still be reproduced and re-evaluated under the prompt it was trained on. SYS = ("You expand a single story beat into ONE paragraph of prose in the manner of Rebecca " "Yarros — contemporary first-person PRESENT-tense narration, emotionally charged, sensory " "and physical, the voice of new-adult romantasy. Render the beat itself; do not move past " "it, do not add a new scene, do not comment. Output the paragraph only, 90–140 words.") +SYS_PASSAGE = ("You expand a single story beat into a SHORT PASSAGE of prose in the manner of " + "{author} — {register}. The passage may run to several paragraphs and should read " + "as continuous scene, not a summary. Render the beat itself; do not move past it, " + "do not begin a new scene, do not comment, do not write a chapter heading. Output " + "the prose only, 90–140 words.") + +REGISTERS = { + "yarros": ("Rebecca Yarros", "contemporary first-person PRESENT-tense narration, emotionally " + "charged, sensory and physical, the voice of new-adult romantasy"), + "hemingway": ("Ernest Hemingway", "spare declarative sentences, concrete physical detail, " + "heavy unattributed dialogue, feeling shown through action and omission rather " + "than stated"), +} + CONTEXT_BLOCK = """The passage is preceded by this, for reference only. Do NOT write a beat for it — it is here so you resolve names and pronouns correctly. @@ -67,6 +86,8 @@ Rules: PASSAGE: {passage}""" +ABBREV = re.compile(r"\b(Mr|Mrs|Ms|Dr|St|Sr|Jr|Lt|Col|Gen|Capt|Sgt|Prof|vs|etc|No)\.$") + META = re.compile(r"\b(passage|excerpt|paragraph|prose|narrat(?:or|ion)|the (?:author|text|scene) (?:is|describes)|this (?:scene|chapter))\b", re.I) # ⚠ MEASURED, and it was a composition bias rather than a nuisance: of 23 `meta` rejects in @@ -104,7 +125,28 @@ def post(path: str, payload: dict, key: str, timeout: int = 120) -> dict: return json.loads(r.read()) -def chunk(corpus: Path, lo: int, hi: int, split: str) -> list[dict]: +HEADING_MAX_WORDS = 6 + + +def drop_leading_heading(paras: list[str], enabled: bool) -> list[str]: + """Drop a unit's opening block when it is a bare chapter heading. + + ⚠ DEFECT 3 of the BabyYarros pair build, and it is far worse on Hemingway: 20 of 600 + Yarros responses carried a chapter heading (3.3%), but **315 of 318 Hemingway units open + with one** because these editions set `I`, `27` or a story title on its own line. The + corpus builder KEEPS those headings deliberately -- they are part of the form the voice + lives in and dropping them would teach the model that chapters begin mid-scene -- so the + removal belongs here, at pair construction, not upstream in the corpus. + + A response beginning `CHAPTER SIXTY-SIX` teaches the carrier to emit chapter headings when + asked for prose, which is exactly what the pilot adapter did. + """ + if not enabled or not paras: + return paras + return paras[1:] if len(paras[0].split()) <= HEADING_MAX_WORDS else paras + + +def chunk(corpus: Path, lo: int, hi: int, split: str, drop_heading: bool = False) -> list[dict]: """Accumulate consecutive paragraphs into product-band passages, per chapter. Chapter-bounded so a passage never straddles a chapter break. The trailing buffer of @@ -118,6 +160,7 @@ def chunk(corpus: Path, lo: int, hi: int, split: str) -> list[dict]: if d.get("split") != split: continue paras = [p.strip() for p in re.split(r"\n\s*\n", d["text"]) if p.strip()] + paras = drop_leading_heading(paras, drop_heading) buf, n, prev = [], 0, None for p in paras: buf.append(p); n += len(p.split()) @@ -151,8 +194,22 @@ def vet(beat: str, passage: str, overlap_n: int) -> tuple[str | None, str]: beat = re.sub(r'^(?:beat|answer)\s*[:\-]\s*', '', beat, flags=re.I).strip().strip('"“”') if not beat: return None, "empty" - # first sentence only -- the model sometimes adds a second + # First sentence only -- the model sometimes adds a second. + # ⚠ An abbreviation is not a sentence end. The naive `.+?[.!?]\s` truncated + # "Jeremias arrives with Mr. Derrick and he sizes him up" to "...with Mr." -- caught in the + # Hemingway positive control, and it would have been near-invisible in the built pairs + # because the result is still a short grammatical-looking fragment. Hemingway is full of + # `Mr. Singh`, `Mr. Bobby`, `Mr. Derrick`, so this fires constantly on this corpus and + # essentially never on Yarros -- a defect one corpus exposes and another hides. m = re.match(r"^(.+?[.!?])(?:\s|$)", beat) + while m and ABBREV.search(m.group(1)): + nxt = re.match(r"^(.+?[.!?])(?:\s|$)", beat[m.end():]) + if not nxt: + m = None + break + m = re.match(r"^(.{%d,}?[.!?])(?:\s|$)" % (m.end() + 1), beat) + if not m: + break if m: beat = m.group(1).strip() nw = len(beat.split()) @@ -194,14 +251,31 @@ def main() -> int: ap.add_argument("--hi", type=int, default=150) ap.add_argument("--split", default="train") ap.add_argument("--seed", type=int, default=4919) + # ⚠ DEFECT 2 of the BabyYarros pair build, and the one that shaped its output most. + # The chunker starts each passage where the last one ended, so a passage opens MID-SCENE + # with lead-in the beat does not describe -- the beat summarises the whole span. At 0.4 the + # pair usually withheld the preceding text, so the carrier learned "open somewhere + # unrelated, then drift toward the beat", which is exactly what the pilot generations did: + # the beat material landed in block 3 or 4. + # Setting this to 1.0 motivates the opening instead of hiding it, and it matches how + # Skaldsong actually calls the model -- a stitcher always has the previous passage. A + # passage at the START of a unit has no predecessor and stays a genuine cold open, which is + # correct: a chapter's first passage IS one, and its beat legitimately describes its start. ap.add_argument("--context-frac", type=float, default=0.4, - help="fraction of pairs carrying the preceding passage as context") + help="fraction of pairs carrying the preceding passage as context; " + "use 1.0 for the corrected recipe (see DEFECT 2)") ap.add_argument("--overlap-n", type=int, default=6) + ap.add_argument("--register", choices=sorted(REGISTERS), default=None, + help="DEFECT 1 fix: emit the PASSAGE system prompt for this author instead " + "of the frozen one-paragraph Yarros prompt. Omit to keep the original.") ap.add_argument("--temperature", type=float, default=0.3) ap.add_argument("--control-out", default=None, help="write the first --control-n passages out for hand-written beats") ap.add_argument("--control-n", type=int, default=10) ap.add_argument("--dump-passages", action="store_true", help="chunk and report, generate nothing") + ap.add_argument("--drop-leading-heading", action="store_true", + help="DEFECT 3 fix: drop a unit's opening block when it is a bare chapter " + "heading. OFF by default so the BabyYarros pair set stays byte-reproducible.") ap.add_argument("--control-in", default=None, help="JSON of hand-written beats [{idx,beat}]; generate beats for the SAME " "passages and print side by side. The positive control -- a beat " @@ -209,8 +283,12 @@ def main() -> int: "dataset that teaches the wrong mapping, and nothing downstream would show it.") a = ap.parse_args() + sys_prompt = SYS + if a.register: + author, register = REGISTERS[a.register] + sys_prompt = SYS_PASSAGE.format(author=author, register=register) key = a.key or (Path(a.key_file).read_text().strip() if a.key_file else None) - passages = chunk(Path(a.corpus), a.lo, a.hi, a.split) + passages = chunk(Path(a.corpus), a.lo, a.hi, a.split, a.drop_leading_heading) print(f"[chunk] {len(passages)} passages in split={a.split}, band {a.lo}-{a.hi}", flush=True) if not passages: print("REFUSING: no passages -- wrong corpus dir or split", file=sys.stderr) @@ -342,7 +420,9 @@ def main() -> int: "beats_naming_absent_entity_examples": drift_examples[:15], "reject_rate": round(sum(rejects.values()) / max(1, kept + sum(rejects.values())), 4), "elapsed_s": round(time.time() - t0, 1), - "system_prompt": SYS, + "system_prompt": sys_prompt, + "register": a.register or "yarros-frozen-one-paragraph", + "drop_leading_heading": a.drop_leading_heading, } Path(str(out_path) + ".provenance.json").write_text(json.dumps(prov, indent=2), encoding="utf-8") print(f"[done] {kept} pairs -> {out_path}")