vh/esh-pfi-infrastructure
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects Releases Wiki Activity

pairs: fix the three construction defects and the abbreviation-truncation bug

Browse Source
This commit is contained in:
Vuong Hoang
2026-09-16 08:12:01 -07:00
parent 03b4a3f62c
commit 6282833669
1 changed files with 85 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/yarros-corpus/build_sft_pairs.py
+85 -5
View File
@@ -39,11 +39,30 @@ GATEWAY = "http://10.250.50.70:4000"
# Matches gen_beats_chat_yarros.py verbatim so the pairs are trained in the SAME shape the
# evaluation harness measures. A pair corpus trained under a different system prompt than
# the eval drives would confound the carrier change with a prompt change.
# ⚠ DEFECT 1 of the BabyYarros pair build: the system prompt said "ONE paragraph" while every
# response was a MEDIAN OF FOUR. The carrier believed the data, correctly, and then looked
# broken against a harness that truncated at the first blank line. The prompt now describes
# what the response actually is. Both are kept: `SYS` is frozen so the shipped BabyYarros
# adapter can still be reproduced and re-evaluated under the prompt it was trained on.
SYS = ("You expand a single story beat into ONE paragraph of prose in the manner of Rebecca "
"Yarros — contemporary first-person PRESENT-tense narration, emotionally charged, sensory "
"and physical, the voice of new-adult romantasy. Render the beat itself; do not move past "
"it, do not add a new scene, do not comment. Output the paragraph only, 90–140 words.")
SYS_PASSAGE = ("You expand a single story beat into a SHORT PASSAGE of prose in the manner of "
"{author} — {register}. The passage may run to several paragraphs and should read "
"as continuous scene, not a summary. Render the beat itself; do not move past it, "
"do not begin a new scene, do not comment, do not write a chapter heading. Output "
"the prose only, 90–140 words.")
REGISTERS = {
"yarros": ("Rebecca Yarros", "contemporary first-person PRESENT-tense narration, emotionally "
"charged, sensory and physical, the voice of new-adult romantasy"),
"hemingway": ("Ernest Hemingway", "spare declarative sentences, concrete physical detail, "
"heavy unattributed dialogue, feeling shown through action and omission rather "
"than stated"),
}
CONTEXT_BLOCK = """The passage is preceded by this, for reference only. Do NOT write a beat for it — it is
here so you resolve names and pronouns correctly.
@@ -67,6 +86,8 @@ Rules:
PASSAGE:
{passage}"""
ABBREV = re.compile(r"\b(Mr|Mrs|Ms|Dr|St|Sr|Jr|Lt|Col|Gen|Capt|Sgt|Prof|vs|etc|No)\.$")
META = re.compile(r"\b(passage|excerpt|paragraph|prose|narrat(?:or|ion)|the (?:author|text|scene) (?:is|describes)|this (?:scene|chapter))\b", re.I)
# ⚠ MEASURED, and it was a composition bias rather than a nuisance: of 23 `meta` rejects in
@@ -104,7 +125,28 @@ def post(path: str, payload: dict, key: str, timeout: int = 120) -> dict:
return json.loads(r.read())
def chunk(corpus: Path, lo: int, hi: int, split: str) -> list[dict]:
HEADING_MAX_WORDS = 6
def drop_leading_heading(paras: list[str], enabled: bool) -> list[str]:
"""Drop a unit's opening block when it is a bare chapter heading.
⚠ DEFECT 3 of the BabyYarros pair build, and it is far worse on Hemingway: 20 of 600
Yarros responses carried a chapter heading (3.3%), but **315 of 318 Hemingway units open
with one** because these editions set `I`, `27` or a story title on its own line. The
corpus builder KEEPS those headings deliberately -- they are part of the form the voice
lives in and dropping them would teach the model that chapters begin mid-scene -- so the
removal belongs here, at pair construction, not upstream in the corpus.
A response beginning `CHAPTER SIXTY-SIX` teaches the carrier to emit chapter headings when
asked for prose, which is exactly what the pilot adapter did.
"""
if not enabled or not paras:
return paras
return paras[1:] if len(paras[0].split()) <= HEADING_MAX_WORDS else paras
def chunk(corpus: Path, lo: int, hi: int, split: str, drop_heading: bool = False) -> list[dict]:
"""Accumulate consecutive paragraphs into product-band passages, per chapter.
Chapter-bounded so a passage never straddles a chapter break. The trailing buffer of
@@ -118,6 +160,7 @@ def chunk(corpus: Path, lo: int, hi: int, split: str) -> list[dict]:
if d.get("split") != split:
continue
paras = [p.strip() for p in re.split(r"\n\s*\n", d["text"]) if p.strip()]
paras = drop_leading_heading(paras, drop_heading)
buf, n, prev = [], 0, None
for p in paras:
buf.append(p); n += len(p.split())
@@ -151,8 +194,22 @@ def vet(beat: str, passage: str, overlap_n: int) -> tuple[str | None, str]:
beat = re.sub(r'^(?:beat|answer)\s*[:\-]\s*', '', beat, flags=re.I).strip().strip('"“”')
if not beat:
return None, "empty"
# first sentence only -- the model sometimes adds a second
# First sentence only -- the model sometimes adds a second.
# ⚠ An abbreviation is not a sentence end. The naive `.+?[.!?]\s` truncated
# "Jeremias arrives with Mr. Derrick and he sizes him up" to "...with Mr." -- caught in the
# Hemingway positive control, and it would have been near-invisible in the built pairs
# because the result is still a short grammatical-looking fragment. Hemingway is full of
# `Mr. Singh`, `Mr. Bobby`, `Mr. Derrick`, so this fires constantly on this corpus and
# essentially never on Yarros -- a defect one corpus exposes and another hides.
m = re.match(r"^(.+?[.!?])(?:\s|$)", beat)
while m and ABBREV.search(m.group(1)):
nxt = re.match(r"^(.+?[.!?])(?:\s|$)", beat[m.end():])
if not nxt:
m = None
break
m = re.match(r"^(.{%d,}?[.!?])(?:\s|$)" % (m.end() + 1), beat)
if not m:
break
if m:
beat = m.group(1).strip()
nw = len(beat.split())
@@ -194,14 +251,31 @@ def main() -> int:
ap.add_argument("--hi", type=int, default=150)
ap.add_argument("--split", default="train")
ap.add_argument("--seed", type=int, default=4919)
# ⚠ DEFECT 2 of the BabyYarros pair build, and the one that shaped its output most.
# The chunker starts each passage where the last one ended, so a passage opens MID-SCENE
# with lead-in the beat does not describe -- the beat summarises the whole span. At 0.4 the
# pair usually withheld the preceding text, so the carrier learned "open somewhere
# unrelated, then drift toward the beat", which is exactly what the pilot generations did:
# the beat material landed in block 3 or 4.
# Setting this to 1.0 motivates the opening instead of hiding it, and it matches how
# Skaldsong actually calls the model -- a stitcher always has the previous passage. A
# passage at the START of a unit has no predecessor and stays a genuine cold open, which is
# correct: a chapter's first passage IS one, and its beat legitimately describes its start.
ap.add_argument("--context-frac", type=float, default=0.4,
help="fraction of pairs carrying the preceding passage as context")
help="fraction of pairs carrying the preceding passage as context; "
"use 1.0 for the corrected recipe (see DEFECT 2)")
ap.add_argument("--overlap-n", type=int, default=6)
ap.add_argument("--register", choices=sorted(REGISTERS), default=None,
help="DEFECT 1 fix: emit the PASSAGE system prompt for this author instead "
"of the frozen one-paragraph Yarros prompt. Omit to keep the original.")
ap.add_argument("--temperature", type=float, default=0.3)
ap.add_argument("--control-out", default=None,
help="write the first --control-n passages out for hand-written beats")
ap.add_argument("--control-n", type=int, default=10)
ap.add_argument("--dump-passages", action="store_true", help="chunk and report, generate nothing")
ap.add_argument("--drop-leading-heading", action="store_true",
help="DEFECT 3 fix: drop a unit's opening block when it is a bare chapter "
"heading. OFF by default so the BabyYarros pair set stays byte-reproducible.")
ap.add_argument("--control-in", default=None,
help="JSON of hand-written beats [{idx,beat}]; generate beats for the SAME "
"passages and print side by side. The positive control -- a beat "
@@ -209,8 +283,12 @@ def main() -> int:
"dataset that teaches the wrong mapping, and nothing downstream would show it.")
a = ap.parse_args()
sys_prompt = SYS
if a.register:
author, register = REGISTERS[a.register]
sys_prompt = SYS_PASSAGE.format(author=author, register=register)
key = a.key or (Path(a.key_file).read_text().strip() if a.key_file else None)
passages = chunk(Path(a.corpus), a.lo, a.hi, a.split)
passages = chunk(Path(a.corpus), a.lo, a.hi, a.split, a.drop_leading_heading)
print(f"[chunk] {len(passages)} passages in split={a.split}, band {a.lo}-{a.hi}", flush=True)
if not passages:
print("REFUSING: no passages -- wrong corpus dir or split", file=sys.stderr)
@@ -342,7 +420,9 @@ def main() -> int:
"beats_naming_absent_entity_examples": drift_examples[:15],
"reject_rate": round(sum(rejects.values()) / max(1, kept + sum(rejects.values())), 4),
"elapsed_s": round(time.time() - t0, 1),
"system_prompt": SYS,
"system_prompt": sys_prompt,
"register": a.register or "yarros-frozen-one-paragraph",
"drop_leading_heading": a.drop_leading_heading,
}
Path(str(out_path) + ".provenance.json").write_text(json.dumps(prov, indent=2), encoding="utf-8")
print(f"[done] {kept} pairs -> {out_path}")