From 6dba912324f8d93f2f520dd26eb746a0a268483b Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Fri, 11 Sep 2026 08:46:45 -0700 Subject: [PATCH] BabyYarros: corpus built, gender resolution fixed, rename blocked on leak gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Located the source: five Rebecca Yarros works in the Kvasir licensed library, with rights recorded as gated. Built D1 at 208 chapters and 780,744 words, which is 15% larger than the Brontë corpus. No unwrap step was needed because Kvasir's cleaner already emits flowing paragraphs, so the hard-wrap defect that cost a re-cut on Brontë does not exist here. The alphabet was re-derived rather than inherited: 23 non-ASCII letters across three forms, against F02's 4 on a smaller sample. Same ASCII-fold conclusion from a different measurement, which is the reason to re-derive per corpus. The interesting finding is a new pathology. In a rotating first-person POV corpus, every book's narrator gets the wrong gender. Measured against six names verified in the text, the pronoun resolver called Violet male, Leah male and Landon female -- three of eighteen wrong, and all three are the narrator of the book where they were misgendered. A narrator is "I" in her own book, so her name appears mostly inside the other lead's dialogue surrounded by his pronouns. This is Brontë's "Jane called male" amplified by rotating POV. Title-first resolution, which fixed it for Brontë, is nearly blind here because contemporary romance uses given names rather than honorifics. What works is the POV header: resolve each name from the chapters it does not narrate. Validated at 9 correct, 9 held, 0 wrong against the previous 7, 8 and 3 wrong, and the instrument refuses to write unless it beats what it replaces. Re-pointing rename.py surfaced three bugs, two of which would have silently corrupted the corpus. Gender came only from honorifics and the entities file's gender field was ignored, so the POV fix had no effect until wired through; that took wilder from 1 gendered entity to 13. The pool labels were hardcoded in a print statement, so any non-Brontë preset crashed. And the collision-filter log claimed it dropped names colliding with Brontë entities regardless of which corpus it filtered against -- the logic was right but the message named the wrong corpus, which is how a reader later concludes the filter ran on the wrong thing. D3 is blocked and nothing has been trained. The leak gate shows 86 of 232 renameable source entities surviving where the Brontë run reached 0 of 203. It decomposes into detector false positives that need a stopword filter rather than renaming, genuine misses among worldbuilding proper nouns, and a third class whose cause is not yet established. Training before the gate passes means fitting in-copyright text with 86 identifiable source entities intact, in a corpus F02 already flagged as small enough for leak to be a real concern. --- persistent-memory.md | 5 + scripts/r49-corpus/rename.py | 42 ++++-- scripts/yarros-corpus/build_corpus_yarros.py | 128 +++++++++++++++++++ scripts/yarros-corpus/corpus_alphabet.json | 67 ++++++++++ scripts/yarros-corpus/manifest.json | 61 +++++++++ scripts/yarros-corpus/pov_gender.py | 115 +++++++++++++++++ 6 files changed, 408 insertions(+), 10 deletions(-) create mode 100644 scripts/yarros-corpus/build_corpus_yarros.py create mode 100644 scripts/yarros-corpus/corpus_alphabet.json create mode 100644 scripts/yarros-corpus/manifest.json create mode 100644 scripts/yarros-corpus/pov_gender.py diff --git a/persistent-memory.md b/persistent-memory.md index 5568a1e..f79e64b 100644 --- a/persistent-memory.md +++ b/persistent-memory.md @@ -206,6 +206,11 @@ _As of 2026-09-10 10:25 PT._ ## Recent decisions +- `[2026-09-11]` **BabyYarros D1 BUILT, D2 gender FIXED, D3 rename BLOCKED on the leak gate.** Operator: *"train the instruct on the yarros corpus -- babyyarros."* Source located: **5 works in the Kvasir licensed library** (`data/library/catalog.sqlite`, `rights=gated`) — Fourth Wing, Iron Flame, Wilder, Nova, Rebel. **D1 built: 208 chapters · 780,744 words** (15% larger than Brontë's 680,291) at `nh3-dev:~/yarros-corpus`. ⚠ **No unwrap needed** — Kvasir's cleaner already emits flowing paragraphs (median line 102 chars), so the Brontë hard-wrap defect does not exist here. **Alphabet RE-DERIVED rather than inherited**: 23 non-ASCII letters across é/à/ï in 780k words. F02 measured 4 (all é) on a 455,800-word sample; same conclusion (ASCII-fold) from a different number, which is why it is re-derived per corpus. +- `[2026-09-11]` ⭐⭐ **NEW PATHOLOGY, worse than Brontë's: in a ROTATING first-person POV corpus, every book's narrator gets the WRONG gender.** Measured against 6 names verified in the text: the pronoun resolver called **Violet 'm'** (Fourth Wing's narrator), **Leah 'm'** (Wilder's), **Landon 'f'** (Rebel's) — 3 of 18 wrong, and all three are narrators. Mechanism is Brontë's "Jane called male" amplified: a narrator is *I* in her own book, so her name appears mostly inside the other lead's dialogue among HIS pronouns. ⚠ **And title-first, the Brontë fix, is nearly blind here** — contemporary romance says "Violet", not "Miss Sorrengail": 3 gendered entities per work. **The fix that works for this corpus is the POV header**: chapters open `Chapter One / Leah / Port of Miami`, so resolve each name from the chapters it does NOT narrate. Validated **9 correct / 9 held / 0 WRONG** against 7/8/**3-wrong**; the instrument refuses to write unless it beats what it replaces. `scripts/yarros-corpus/pov_gender.py`. ⚠ Fourth Wing and Iron Flame are SINGLE-POV so they have no headers — Violet is now *held* (neutral token) there rather than wrongly gendered, which is the safe direction. +- `[2026-09-11]` ⚠ **Three real bugs found in `rename.py` while re-pointing it, two of which would have silently corrupted BabyYarros:** (1) **gender came ONLY from honorifics** — the entities file's `gender` field was ignored entirely, so my POV fix had no effect until wired in; now `tg.get(key) or e.get("gender")`, titles first so Brontë is unchanged. Effect: 1 → 13 gendered on `wilder`. (2) the pool labels `pool['fr']`/`pool['en']` were hardcoded in a print, so any non-Brontë preset crashed; pools are now a `PRESETS` dict (`bronte` = fr/en excluding en_US for period register; `yarros` = en_US/en_CA + es/it/de/fr at 0.62 US). (3) the collision-filter log said *"dropped N pool names that are Bronte entities"* **regardless of corpus** — the logic was right but the message named the wrong one, which is how a future reader concludes the filter ran against the wrong corpus. +- `[2026-09-11]` ⛔ **D3 BLOCKED: leak gate at 86 of 232 renameable source entities surviving; Brontë's run reached 0 of 203.** Decomposes into (a) **detector false positives** — `Hopefully`, `Whoa`, `Hey`, `Hmm`, `Holy` are adverbs and interjections the cap/lowercase-ratio detector calls names, and they need a stopword filter rather than renaming; (b) **genuine misses** including worldbuilding proper nouns (`Krovlan`, `Poromish`, `Fuil`, `Iorson`) — the `Thornfield × 100` case, and holding a place leaks it; (c) names like `Elizabeth`/`Penelope`/`Messina` appearing as both pool draws and surviving source entities, cause not yet established. **Nothing has been trained.** ⚠ Training before this gate passes means fitting in-copyright text with 86 identifiable source entities intact, in a corpus F02 already flagged as small enough for leak to be real. + - `[2026-09-11]` ⭐⭐ **THE INSTRUCT PROBE ANSWERS ITS QUESTION: voice and instruction-following DO coexist. Option C is de-risked.** `Qwen3-4B` **instruct** (not `-Base`), same corpus/seed/steps so the carrier is the only variable; best checkpoint `checkpoint-150` picked by loss (applying the 4B-Base lesson automatically this time). **Voice installed at full strength — curly quotes 16/18, IDENTICAL to the 4B-Base tuned arm's 16/18**, against the unadapted control's 1/18, and **task-leak 0/18 vs the base carrier's 4/18**. So the assistant prior did NOT block Brontë, which was the central risk. **Instruction-following SURVIVED: 10/10 on-beat through the chat template**, same as the untuned control. ⚠ **The cost is length discipline, not comprehension** — in-band 10/10 → **6/10**, median 124w → 140w. Training on Victorian prose made it wordier, a soft degradation rather than a break. ⚠ **Held-out 2.908 vs 4B-Base's 2.814** — the instruct carrier fits the corpus **0.094 nats worse** and **plateaus without turning** where base overfit at step 75: the assistant prior competes for capacity, so it absorbs less rather than overfitting more. - `[2026-09-11]` ⚠ **What raw-continuation training on an instruct carrier does NOT fix: the plot furniture.** Reading the product artifact, the tuned-instruct arm renders the beat and then drags the referent — *"He licked her clean… my master thus—my husband thus"*, turning the dog into a man, because Brontë's corpus is about masters and husbands. Another beat ran 247w and gave the narrator a list of duties. **This is exactly what instruction-PAIR training is for** — pairs teach "render this and stop", continuation teaches "keep writing Victorian prose". So the probe de-risks option C without substituting for it. ⚠ Also: my `ran_on` metric is uninformative on this job (10/10 on BOTH arms) because a single paragraph contains no blank line — it measures "no paragraph break found", which is correct and useless here. Do not read it as a finding. diff --git a/scripts/r49-corpus/rename.py b/scripts/r49-corpus/rename.py index 92dd0e5..f4366c0 100644 --- a/scripts/r49-corpus/rename.py +++ b/scripts/r49-corpus/rename.py @@ -36,6 +36,19 @@ ENGLISH_LOCALES = ["en_GB", "en_IE"] #: orthography -- a Yorkshire mill town full of Parisian surnames reads wrong. FRENCH_SHARE = {"villette": 0.60, "the-professor": 0.60, "jane-eyre": 0.25, "shirley": 0.25} +#: The pool is now per-corpus rather than per-author-hardcoded, because the same +#: register argument points somewhere else for every corpus. Brontë EXCLUDES en_US +#: (modern surnames read wrong for the 1840s); contemporary American romance wants +#: exactly those, with the European admixture F02 found matches Yarros's register. +#: Defaults reproduce the Brontë run byte-for-byte, so this is additive. +PRESETS = { + "bronte": {"a": ("fr", FRENCH_LOCALES), "b": ("en", ENGLISH_LOCALES), + "share": FRENCH_SHARE, "default_share": 0.25}, + "yarros": {"a": ("us", ["en_US", "en_CA"]), + "b": ("eu", ["es_ES", "es_MX", "it_IT", "de_DE", "fr_FR"]), + "share": {}, "default_share": 0.62}, +} + def title_gender(text: str) -> dict[str, str]: mt = collections.Counter(m.group(1).lower() for m in @@ -52,10 +65,11 @@ def title_gender(text: str) -> dict[str, str]: return out -def build_pool(dict_path: Path, alphabet: set[str]) -> dict: +def build_pool(dict_path: Path, alphabet: set[str], preset: str = "bronte") -> dict: d = json.loads(dict_path.read_text()) pool = {} - for label, locales in (("fr", FRENCH_LOCALES), ("en", ENGLISH_LOCALES)): + cfg = PRESETS[preset] + for label, locales in (cfg["a"], cfg["b"]): m, f, s = set(), set(), set() for loc in locales: v = d["by_locale"].get(loc, {}) @@ -82,13 +96,17 @@ def main() -> int: ap.add_argument("--copies", type=int, default=6) ap.add_argument("--seed", type=int, default=4919) ap.add_argument("--holdout-chapter", type=int, default=10) + ap.add_argument("--preset", default="bronte", choices=sorted(PRESETS), + help="which corpus's name-pool register to draw from") a = ap.parse_args() corpus = Path(a.corpus) man = json.loads((corpus / "manifest.json").read_text()) alphabet = set(json.loads((corpus / "corpus_alphabet.json").read_text())["letters"]) ents_all = json.loads(Path(a.entities).read_text()) - pool = build_pool(Path(a.dictionary), alphabet) + pool = build_pool(Path(a.dictionary), alphabet, a.preset) + cfg = PRESETS[a.preset] + label_a, label_b = cfg["a"][0], cfg["b"][0] # ⚠ Collision filter, against THIS corpus. F02 dropped 35 names for colliding # with the Yarros source so a rename could never map one of the author's # entities onto another; that filter is corpus-specific and does not carry. @@ -113,10 +131,11 @@ def main() -> int: if n not in source_names and not (set(re.split(r"[-\s’']", n)) & source_names)] dropped += before - len(pool[lang][bucket]) - print(f" collision filter: dropped {dropped} pool names that are Bronte entities") - print(f" pool (alphabet-filtered): " - f"fr {len(pool['fr']['male'])}m/{len(pool['fr']['female'])}f/{len(pool['fr']['surname'])}s " - f"en {len(pool['en']['male'])}m/{len(pool['en']['female'])}f/{len(pool['en']['surname'])}s") + print(f" collision filter: dropped {dropped} pool names that collide with " + f"{len(source_names)} source entities in THIS corpus ({a.preset})") + print(" pool (alphabet-filtered): " + " ".join( + f"{lab} {len(pool[lab]['male'])}m/{len(pool[lab]['female'])}f/{len(pool[lab]['surname'])}s" + for lab in (label_a, label_b))) out = Path(a.out); (out / "copies").mkdir(parents=True, exist_ok=True) stats = {"copies": a.copies, "seed": a.seed, "works": {}, "renamed": 0, "held": 0} @@ -137,7 +156,7 @@ def main() -> int: for key, e in ents.items(): if "’" in key or "'" in key or e["cap"] < 8: continue # possessives/contractions are not entities - g = tg.get(key) + g = tg.get(key) or e.get("gender") if g: renameable[key] = {"surface": e["surface"], "kind": "given", "gender": g} else: @@ -162,11 +181,14 @@ def main() -> int: for c in range(a.copies): rng = random.Random(a.seed + c * 1000) for slug, rows in works.items(): - fr_share = FRENCH_SHARE[slug] + # Share of pool A for this work. Brontë sets it per novel (Brussels + # vs Yorkshire); Yarros uses one default, because the register does + # not split by book the way hers does. + share_a = cfg["share"].get(slug, cfg["default_share"]) used = set() def draw(kind: str, gender: str | None) -> str: - lang = "fr" if rng.random() < fr_share else "en" + lang = label_a if rng.random() < share_a else label_b bucket = {"m": "male", "f": "female"}.get(gender or "", "surname") for _ in range(200): n = rng.choice(pool[lang][bucket]) diff --git a/scripts/yarros-corpus/build_corpus_yarros.py b/scripts/yarros-corpus/build_corpus_yarros.py new file mode 100644 index 0000000..fedc798 --- /dev/null +++ b/scripts/yarros-corpus/build_corpus_yarros.py @@ -0,0 +1,128 @@ +"""D1 for BabyYarros: build the corpus from the licensed Kvasir masters. + +Deliberately emits the SAME record schema as the Brontë builder +({work, chapter, heading, words, text} per work file, plus corpus_alphabet.json and +manifest.json), so entities.py, rename.py and train_voice_lora.py all run unchanged. +Matching an existing schema beats teaching three downstream tools a new one. + +Differences from the Brontë build, and each is a property of the source rather than +a preference: + + * NO Gutenberg boilerplate strip and NO download -- Kvasir already extracted and + cleaned these, and the catalog records the cleaner and its version. + * NO unwrap step. The Brontë corpus came hard-wrapped at ~70 characters and the + adapter learned the line breaks; these masters are already flowing paragraphs + (median non-blank line 102 chars), so the defect does not exist here. + * Chapters are "Chapter One" style words, not roman numerals, and are followed by + a POV name and often a location on their own lines -- first-person contemporary + romance with rotating narrators. Those header lines are KEPT: they are part of + the form the voice lives in, and dropping them would teach the model that + chapters begin mid-scene. + * ASCII alphabet, confirmed on the text rather than inherited from F02: 2 + non-ASCII letters across this corpus. Under the F02 rule (a rename pool's + character inventory must be a SUBSET of the corpus's) that means an ASCII-only + pool -- the opposite of Brontë, who needed French accents kept. +""" +from __future__ import annotations +import argparse, collections, json, os, re, sqlite3, sys +from pathlib import Path + +CATALOG = "/home/lkraven/development/kvasir/data/library/catalog.sqlite" +KVASIR = "/home/lkraven/development/kvasir" + +SLUGS = { + "Fourth Wing (Exclusive Holiday Edition)": "fourth-wing", + "Iron Flame": "iron-flame", + "Wilder (The Renegades)": "wilder", + "Nova (The Renegades #2)": "nova", + "Rebel (The Renegades)": "rebel", +} +# "Chapter One" / "Chapter Twenty-Three" / "Chapter 12" / "Prologue" / "Epilogue". +CHAPTER = re.compile( + r"^[ \t]*((?:Chapter|CHAPTER)[ \t]+(?:[A-Za-z-]+|\d+)|Prologue|PROLOGUE|Epilogue|EPILOGUE)" + r"[ \t]*\.?[ \t]*$", re.M) + + +def masters(): + c = sqlite3.connect(CATALOG) + rows = c.execute("select title, master_path, rights, normalized_text_sha256 " + "from masters where lower(author) like '%yarros%'").fetchall() + out = [] + for title, path, rights, sha in rows: + p = Path(path if os.path.isabs(path) else os.path.join(KVASIR, path)) + if not p.exists(): + print(f" ⚠ MISSING master for {title}: {p}", file=sys.stderr) + continue + out.append({"title": title, "slug": SLUGS.get(title, re.sub(r"\W+", "-", title.lower()).strip("-")), + "path": p, "rights": rights, "sha256": sha}) + return sorted(out, key=lambda w: w["slug"]) + + +def split_chapters(text: str): + """Return [(heading, body)]. Everything before the first heading is front matter.""" + marks = [(m.start(), m.group(1).strip()) for m in CHAPTER.finditer(text)] + if not marks: + return [("(whole)", text.strip())] + out = [] + for i, (pos, head) in enumerate(marks): + end = marks[i + 1][0] if i + 1 < len(marks) else len(text) + body = text[pos:end].strip() + if len(body.split()) >= 150: # skip a bare heading with no chapter behind it + out.append((head, body)) + return out + + +ap = argparse.ArgumentParser() +ap.add_argument("--out", required=True) +ap.add_argument("--survey", action="store_true", help="report and write nothing") +a = ap.parse_args() + +works = masters() +if not works: + raise SystemExit("REFUSING: no Yarros masters resolved from the catalog") + +out = Path(a.out) +alphabet = collections.Counter() +total_words = total_chaps = 0 +manifest = {"corpus": "BabyYarros", "author": "Rebecca Yarros", + "source": "kvasir data/library masters (licensed, rights=gated)", + "built_at": __import__("datetime").date.today().isoformat(), "works": []} + +for w in works: + text = w["path"].read_text(encoding="utf-8", errors="replace") + chaps = split_chapters(text) + alphabet.update(ch for ch in text if ch.isalpha()) + words = sum(len(b.split()) for _, b in chaps) + total_words += words; total_chaps += len(chaps) + print(f" {w['slug']:14} {len(chaps):>3} chapters {words:>7,} words rights={w['rights']}") + manifest["works"].append({"slug": w["slug"], "title": w["title"], "rights": w["rights"], + "master_sha256": w["sha256"], "chapters": len(chaps), "words": words, + "path": f"works/{w['slug']}.jsonl"}) + if not a.survey: + (out / "works").mkdir(parents=True, exist_ok=True) + with (out / "works" / f"{w['slug']}.jsonl").open("w", encoding="utf-8") as fh: + for i, (head, body) in enumerate(chaps, 1): + fh.write(json.dumps({"work": w["slug"], "chapter": i, "heading": head, + "words": len(body.split()), "text": body}, + ensure_ascii=False) + "\n") + +non_ascii = {c: n for c, n in alphabet.items() if ord(c) > 127} +print(f"\n TOTAL {total_chaps} chapters · {total_words:,} words · {len(alphabet)} distinct letters") +print(f" non-ASCII letters: {sum(non_ascii.values())} across {len(non_ascii)} forms {non_ascii or ''}") +manifest["totals"] = {"chapters": total_chaps, "words": total_words, + "distinct_letters": len(alphabet), "non_ascii_letters": sum(non_ascii.values())} +manifest["total_words"] = total_words +manifest["total_chapters"] = total_chaps + +if not a.survey: + (out / "manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8") + (out / "corpus_alphabet.json").write_text(json.dumps({ + "derived_from": "Rebecca Yarros, 5 novels, Kvasir licensed library", + "derived_at": manifest["built_at"], + "note": ("R49 F02 rule: a rename pool's character inventory must be a SUBSET of this. " + f"Measured on the built text: {sum(non_ascii.values())} non-ASCII letters, so the " + "pool is ASCII-only -- the opposite of the Brontë corpus, which needed French " + "accents kept."), + "letters": sorted(alphabet), "non_ascii": {c: n for c, n in sorted(non_ascii.items())}, + }, indent=2, ensure_ascii=False), encoding="utf-8") + print(f" wrote {out}") diff --git a/scripts/yarros-corpus/corpus_alphabet.json b/scripts/yarros-corpus/corpus_alphabet.json new file mode 100644 index 0000000..67a465c --- /dev/null +++ b/scripts/yarros-corpus/corpus_alphabet.json @@ -0,0 +1,67 @@ +{ + "derived_from": "Rebecca Yarros, 5 novels, Kvasir licensed library", + "derived_at": "2026-09-11", + "note": "R49 F02 rule: a rename pool's character inventory must be a SUBSET of this. Measured on the built text: 23 non-ASCII letters, so the pool is ASCII-only -- the opposite of the Brontë corpus, which needed French accents kept.", + "letters": [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", + "Q", + "R", + "S", + "T", + "U", + "V", + "W", + "X", + "Y", + "Z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "à", + "é", + "ï" + ], + "non_ascii": { + "à": 2, + "é": 19, + "ï": 2 + } +} \ No newline at end of file diff --git a/scripts/yarros-corpus/manifest.json b/scripts/yarros-corpus/manifest.json new file mode 100644 index 0000000..6a54d28 --- /dev/null +++ b/scripts/yarros-corpus/manifest.json @@ -0,0 +1,61 @@ +{ + "corpus": "BabyYarros", + "author": "Rebecca Yarros", + "source": "kvasir data/library masters (licensed, rights=gated)", + "built_at": "2026-09-11", + "works": [ + { + "slug": "fourth-wing", + "title": "Fourth Wing (Exclusive Holiday Edition)", + "rights": "gated", + "master_sha256": "606420acb827122d700eb47c18b7612399d130fe770787b139b0704673b1b897", + "chapters": 41, + "words": 191289, + "path": "works/fourth-wing.jsonl" + }, + { + "slug": "iron-flame", + "title": "Iron Flame", + "rights": "gated", + "master_sha256": "e66db0cd13789bb0d6065888bc117362c8b3c25f8827dcbc6ffcd452a7359af6", + "chapters": 66, + "words": 251949, + "path": "works/iron-flame.jsonl" + }, + { + "slug": "nova", + "title": "Nova (The Renegades #2)", + "rights": "gated", + "master_sha256": "ccccf3d64dd5810c5135ac86223e5f3e679fe5d1cdacd88df1eb9ff0164cb61e", + "chapters": 34, + "words": 109571, + "path": "works/nova.jsonl" + }, + { + "slug": "rebel", + "title": "Rebel (The Renegades)", + "rights": "gated", + "master_sha256": "ed61fea84eab962cbf4c96870eaaa180d5ea92278241ad54ebd6d9f6ae2c4e7d", + "chapters": 36, + "words": 118270, + "path": "works/rebel.jsonl" + }, + { + "slug": "wilder", + "title": "Wilder (The Renegades)", + "rights": "gated", + "master_sha256": "c798c9a5d24595deb870e25c34478172cdfd7758249e41e6ecbfa8240cf51a41", + "chapters": 31, + "words": 109665, + "path": "works/wilder.jsonl" + } + ], + "totals": { + "chapters": 208, + "words": 780744, + "distinct_letters": 55, + "non_ascii_letters": 23 + }, + "total_words": 780744, + "total_chapters": 208 +} \ No newline at end of file diff --git a/scripts/yarros-corpus/pov_gender.py b/scripts/yarros-corpus/pov_gender.py new file mode 100644 index 0000000..4bab57a --- /dev/null +++ b/scripts/yarros-corpus/pov_gender.py @@ -0,0 +1,115 @@ +"""Fix gender resolution for a rotating first-person POV corpus. + +Neither existing method works on Yarros, and they fail for opposite structural +reasons: + + * TITLE-FIRST (what Brontë needed) finds almost nothing -- 3 gendered entities per + work. Contemporary romance does not say "Miss Sorrengail", it says "Violet". + * PRONOUN PROXIMITY is wrong specifically on the people who matter most. Measured + against six names whose gender I verified in the text: 3 of 18 WRONG, and the + three are Violet, Leah and Landon -- each of them the first-person NARRATOR of + the book where they were misgendered. A narrator is "I" in her own book, so her + name appears mostly inside the other character's dialogue, surrounded by HIS + pronouns. This is the Brontë "Jane called male" pathology, and it is worse here + because Yarros rotates POV, so every book has a narrator set up to fail. + +The signal this corpus actually offers is the POV header: chapters open "Chapter +One / Leah / Port of Miami", naming their narrator. So resolve each name's gender +from the chapters it does NOT narrate -- where other narrators refer to it in the +third person and the pronouns are trustworthy. + +Refuses to write unless it beats the method it replaces on the verified control, +because a fix that is merely different is not a fix. +""" +from __future__ import annotations +import argparse, collections, json, re +from pathlib import Path + +MASC = {"he", "him", "his", "himself"} +FEM = {"she", "her", "hers", "herself"} +# The POV name sits on its own short line just after the chapter heading. +HEAD = re.compile(r"^[ \t]*((?:Chapter|CHAPTER)[ \t]+(?:[A-Za-z-]+|\d+)|Prologue|Epilogue)" + r"[ \t]*\.?[ \t]*\n+[ \t]*([A-Z][A-Za-z'’-]{1,18})[ \t]*$", re.M) + +ap = argparse.ArgumentParser() +ap.add_argument("corpus") +ap.add_argument("--entities", required=True) +ap.add_argument("--out", required=True) +ap.add_argument("--window", type=int, default=60, help="chars either side of a mention") +ap.add_argument("--min-hits", type=int, default=6) +ap.add_argument("--ratio", type=float, default=2.5) +ap.add_argument("--control", required=True, help="Name=g,Name=g -- verified in the text") +a = ap.parse_args() + +corpus = Path(a.corpus) +man = json.loads((corpus / "manifest.json").read_text()) +ents = json.loads(Path(a.entities).read_text()) +truth = dict(p.split("=") for p in a.control.split(",")) + +chapters: dict[str, list[tuple[str | None, str]]] = {} +for w in man["works"]: + rows = [json.loads(l) for l in (corpus / w["path"]).read_text(encoding="utf-8").splitlines() if l.strip()] + out = [] + for r in rows: + m = HEAD.search(r["text"][:400]) + out.append((m.group(2) if m else None, r["text"])) + chapters[w["slug"]] = out + povs = collections.Counter(p for p, _ in out if p) + print(f" {w['slug']:14} {len(out):>3} chapters · POV headers found in " + f"{sum(1 for p, _ in out if p):>3} · narrators: {dict(povs.most_common(6))}") + + +def resolve(slug: str, name: str, exclude_own_pov: bool) -> str | None: + m = f = 0 + for pov, text in chapters[slug]: + if exclude_own_pov and pov == name: + continue + for mt in re.finditer(rf"\b{re.escape(name)}\b", text): + ctx = text[max(0, mt.start() - a.window): mt.end() + a.window].lower() + for w in re.findall(r"[a-z]+", ctx): + if w in MASC: m += 1 + elif w in FEM: f += 1 + if m + f < a.min_hits: + return None + if m >= a.ratio * max(f, 1): return "m" + if f >= a.ratio * max(m, 1): return "f" + return None + + +def score(exclude: bool): + ok = wrong = held = 0 + detail = [] + for slug in chapters: + for key, ent in ents[slug]["entities"].items(): + s = ent.get("surface") or key + if s not in truth: + continue + g = resolve(slug, s, exclude) + t = truth[s] + if g == t: ok += 1 + elif g is None: held += 1 + else: wrong += 1; detail.append(f"{slug}/{s}={g} (truth {t})") + return ok, held, wrong, detail + + +base_ok, base_held, base_wrong, base_d = score(False) +new_ok, new_held, new_wrong, new_d = score(True) +print(f"\n control, WITHOUT excluding own-POV chapters: {base_ok} correct · {base_held} held · {base_wrong} WRONG {base_d}") +print(f" control, EXCLUDING own-POV chapters: {new_ok} correct · {new_held} held · {new_wrong} WRONG {new_d}") + +if new_wrong > base_wrong or (new_wrong == base_wrong and new_ok <= base_ok): + raise SystemExit("\n REFUSING to write: excluding own-POV chapters did not beat the " + "method it replaces on the verified control. A fix that is merely " + "different is not a fix.") + +applied = 0 +for slug in chapters: + for key, ent in ents[slug]["entities"].items(): + g = resolve(slug, ent.get("surface") or key, True) + if g and g != ent.get("gender"): + applied += 1 + if g: + ent["gender"] = g +Path(a.out).write_text(json.dumps(ents, indent=1), encoding="utf-8") +tot = sum(1 for w in ents.values() for e in w["entities"].values() if e.get("gender")) +print(f"\n wrote {a.out}: {applied} genders changed/added · {tot} entities now gendered")