diff --git a/scripts/yarros-corpus/build_sft_pairs.py b/scripts/yarros-corpus/build_sft_pairs.py index a433a08..8d3820c 100644 --- a/scripts/yarros-corpus/build_sft_pairs.py +++ b/scripts/yarros-corpus/build_sft_pairs.py @@ -61,6 +61,15 @@ REGISTERS = { "hemingway": ("Ernest Hemingway", "spare declarative sentences, concrete physical detail, " "heavy unattributed dialogue, feeling shown through action and omission rather " "than stated"), + # Brontë is the far end of the same axis from Hemingway, and the register has to say + # so or the beat-writer produces modern summary prose that the passages never match: + # long periodic sentences, an explicitly retrospective first person, and moral + # weather carried in the landscape rather than in the dialogue. + "bronte": ("Charlotte Brontë", "mid-nineteenth-century first-person retrospective narration, " + "long periodic sentences built on semicolons and dashes, heightened interior " + "analysis and moral reflection, occasional direct address to the reader, and " + "physical setting — Yorkshire weather, schoolrooms, Belgian pensionnats — " + "rendered with the feeling it carries"), } CONTEXT_BLOCK = """The passage is preceded by this, for reference only. Do NOT write a beat for it — it is @@ -180,15 +189,36 @@ def ngrams(text: str, n: int) -> set[str]: return {" ".join(w[i:i + n]) for i in range(max(0, len(w) - n + 1))} -def vet(beat: str, passage: str, overlap_n: int) -> tuple[str | None, str]: +def vet(beat: str, passage: str, overlap_n: int, + source_pat: "re.Pattern[str] | None" = None) -> tuple[str | None, str]: """Return (clean_beat, reason). reason is '' on accept. Every rejection is a dataset defect that would otherwise train silently: - echo -- a beat quoting the passage teaches the model to copy its instruction - back, not to expand it. This is the one that would look fine in a - spot-read and poison the whole run. - meta -- 'this passage shows...' is a description of text, not a story beat - length -- a 60-word beat is a summary; a 4-word beat is a title + echo -- a beat quoting the passage teaches the model to copy its instruction + back, not to expand it. This is the one that would look fine in a + spot-read and poison the whole run. + meta -- 'this passage shows...' is a description of text, not a story beat + length -- a 60-word beat is a summary; a 4-word beat is a title + sourcename -- ⭐ the beat names a character the RENAME removed. See below. + + ⚠⭐ `sourcename` IS A LEAK THE CORPUS GATE STRUCTURALLY CANNOT SEE, and it was found + on lv-bronte (2026-09-16). The rename strips the author's names from the prose and + leak_gate.py proves they are gone — but the beat is written by an LLM that READ THE + PASSAGE, and if it recognises the book it supplies the canonical names from its own + training. Measured on Brontë: 13 of the first 714 beats (1.8%) named Rochester (x6), + Jane (x3), Brocklehurst (x2), Beck, Fairfax, Helen, Burns, Eyre, Reed and Rivers, + while 0 of 714 RESPONSES did — the rename was perfect and the instruction side was + not. One beat read `Saoirse confirms Rochester's flaws`, mixing a renamed name and a + canonical one in a single sentence, which is the mechanism in miniature. + + The beat is the INSTRUCTION half of the pair, so training on it re-teaches exactly the + inventions the rename pipeline exists to remove, and the corpus gate never looks at it: + the gate reads the corpus and the renamed copies, never the generated beats. + + ⚠ Exposure scales with how well the generator knows the book, so it is WORST for + public-domain classics and mildest for recent work — which is precisely why Yarros and + Hemingway did not surface it and Brontë did. Do not read their clean runs as evidence + this cannot happen; pass --source-entities on every corpus. """ beat = " ".join(beat.strip().split()) beat = re.sub(r'^(?:beat|answer)\s*[:\-]\s*', '', beat, flags=re.I).strip().strip('"“”') @@ -219,6 +249,10 @@ def vet(beat: str, passage: str, overlap_n: int) -> tuple[str | None, str]: return None, "meta" if ngrams(beat, overlap_n) & ngrams(passage, overlap_n): return None, f"echo({overlap_n}gram)" + if source_pat is not None: + m = source_pat.search(beat) + if m: + return None, f"sourcename({m.group(1)})" return beat, "" @@ -265,6 +299,12 @@ def main() -> int: help="fraction of pairs carrying the preceding passage as context; " "use 1.0 for the corrected recipe (see DEFECT 2)") ap.add_argument("--overlap-n", type=int, default=6) + ap.add_argument("--source-entities", default=None, + help="entities json for the UNRENAMED source. Any beat naming a surface " + "from it is rejected as `sourcename`. Pass this on every corpus: the " + "generator reads the passage and will supply canonical names from its " + "own memory of the book if it recognises it, which the corpus leak " + "gate cannot see because it never reads the generated beats.") ap.add_argument("--register", choices=sorted(REGISTERS), default=None, help="DEFECT 1 fix: emit the PASSAGE system prompt for this author instead " "of the frozen one-paragraph Yarros prompt. Omit to keep the original.") @@ -346,6 +386,17 @@ def main() -> int: return 0 out_path = Path(a.out) + source_pat = None + if a.source_entities: + _e = json.loads(Path(a.source_entities).read_text()) + _surf = sorted({(v.get("surface") or k) + for w in _e.values() for k, v in w["entities"].items() + if "\u2019" not in k and "'" not in k}, + key=len, reverse=True) + if _surf: + source_pat = re.compile(r"\b(" + "|".join(re.escape(x) for x in _surf) + r")\b") + print(f" source-entity filter: {len(_surf)} surfaces the beat may not name") + rejects: dict[str, int] = {} kept = 0 name_drift = 0 @@ -367,7 +418,7 @@ def main() -> int: rejects["transport"] = rejects.get("transport", 0) + 1 print(f"[warn] {type(e).__name__}: {e}", flush=True) continue - beat, why = vet(raw, p["response"], a.overlap_n) + beat, why = vet(raw, p["response"], a.overlap_n, source_pat) if beat is None and why == "meta" and NARRATOR_ONLY.search(raw): retried += 1 try: @@ -380,7 +431,7 @@ def main() -> int: {"role": "user", "content": RETRY_NOTE.strip()}, ]}, key) beat, why = vet(r2["choices"][0]["message"]["content"] or "", - p["response"], a.overlap_n) + p["response"], a.overlap_n, source_pat) if beat is not None: retry_saved += 1 except (urllib.error.URLError, urllib.error.HTTPError, KeyError, TimeoutError):