From 03b4a3f62c7da1620df32bd51273028babfb7cd1 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Wed, 16 Sep 2026 07:55:34 -0700 Subject: [PATCH] BabyHemingway D2+D3: entities, base-rate gender resolver, rename preset, leak gate passes --- .../build_corpus_hemingway.py | 43 +++- .../hemingway-corpus/gender_by_proximity.py | 184 ++++++++++++++++++ .../phrase_map_hemingway.json | 44 +++++ .../hemingway-corpus/stoplist_hemingway.json | 49 +++++ scripts/r49-corpus/rename.py | 10 + 5 files changed, 329 insertions(+), 1 deletion(-) create mode 100644 scripts/hemingway-corpus/gender_by_proximity.py create mode 100644 scripts/hemingway-corpus/phrase_map_hemingway.json create mode 100644 scripts/hemingway-corpus/stoplist_hemingway.json diff --git a/scripts/hemingway-corpus/build_corpus_hemingway.py b/scripts/hemingway-corpus/build_corpus_hemingway.py index fe167d5..89add04 100644 --- a/scripts/hemingway-corpus/build_corpus_hemingway.py +++ b/scripts/hemingway-corpus/build_corpus_hemingway.py @@ -214,6 +214,44 @@ def split_by_contents(text: str): return units or None +# ⚠⚠ PUBLISHER BACK MATTER RIDES INSIDE THE LAST UNIT, in 8 of 10 works -- the identical +# defect the Yarros build hit, and it is worth naming again because nothing about the source +# changed to cause it: a splitter cuts on headings, and nothing follows the last one, so the +# "About the Author" block lands inside the final chapter. Here it carries ERNEST HEMINGWAY'S +# OWN NAME 95 times across 7 works -- "Ernest Hemingway was one of America's foremost +# journalists... died in 1961" -- which is precisely the leak the rename pipeline and its gate +# exist to prevent, sitting in the training text before either one runs. +# Bounded: last unit only, and refuses if it would take more than 2% of the corpus. +# ⚠ The EDITOR'S apparatus counts as back matter too, and it is the bigger leak. Stripping +# only the publisher block left 18 "Hemingway" mentions in `true-at-first-light` -- all of +# them inside a CAST OF CHARACTERS and SWAHILI GLOSSARY written by Patrick Hemingway +# ("Mary Ernest Hemingway's fourth and last wife", "Ngui Hemingway's gun bearer"). That is +# an editor describing the author's real household, not the author's prose, and it names him +# directly. The markers are matched in FILE ORDER and the earliest one wins, so the whole +# apparatus goes rather than just its last section. +BACKMATTER = re.compile( + r"^[ \t]*(THE END|About the Author|ABOUT THE AUTHOR|About the Publisher|" + r"CAST OF CHARACTERS|SWAHILI GLOSSARY|GLOSSARY|EDITOR.S ACKNOWLEDGMENTS|" + r"ACKNOWLEDGMENTS|ACKNOWLEDGEMENTS|" + r"Books by [A-Z]|BOOKS BY |Copyright|COPYRIGHT|Also by [A-Z])[ \t]*$", re.M) + + +def strip_backmatter(units, title): + if not units: + return units, 0 + head, body = units[-1] + m = BACKMATTER.search(body) + if not m: + return units, 0 + cut = body[:m.start()].rstrip() + removed = len(body.split()) - len(cut.split()) + if len(cut.split()) < MIN_UNIT_WORDS: + print(f" ⚠ REFUSING back-matter strip on {title}: last unit would fall below the " + f"{MIN_UNIT_WORDS}-word floor", file=sys.stderr) + return units, 0 + return units[:-1] + [(head, cut)], removed + + def split_units(text: str): """Return (pattern_name, [(heading, body)]). @@ -270,6 +308,7 @@ for w in works: text, fixed_lines = repair_lines(text) text, front_removed = strip_foreign_front(text, w["title"]) pattern, units = split_units(text) + units, back_removed = strip_backmatter(units, w["title"]) alphabet.update(ch for ch in text if ch.isalpha()) words = sum(len(b.split()) for _, b in units) total_words += words; total_units += len(units) @@ -279,7 +318,8 @@ for w in works: if w["title"] in CONTINUOUS else "") post = " (posthumous/edited)" if w["title"] in POSTHUMOUS_EDITED else "" extra = (f" [smallcaps {fixed_lines}]" if fixed_lines else "") + \ - (f" [front -{front_removed}w]" if front_removed else "") + (f" [front -{front_removed}w]" if front_removed else "") + \ + (f" [back -{back_removed}w]" if back_removed else "") print(f" {w['slug']:26} {len(units):>4} units {words:>8,} words " f"via {pattern:<14}{post}{flag}{extra}") manifest["works"].append({"slug": w["slug"], "title": w["title"], "rights": w["rights"], @@ -287,6 +327,7 @@ for w in works: "heading_pattern": pattern, "smallcaps_lines_repaired": fixed_lines, "foreign_front_matter_words_removed": front_removed, + "publisher_back_matter_words_removed": back_removed, "posthumous_editor_shaped": w["title"] in POSTHUMOUS_EDITED, "path": f"works/{w['slug']}.jsonl"}) if not a.survey: diff --git a/scripts/hemingway-corpus/gender_by_proximity.py b/scripts/hemingway-corpus/gender_by_proximity.py new file mode 100644 index 0000000..d2ccb77 --- /dev/null +++ b/scripts/hemingway-corpus/gender_by_proximity.py @@ -0,0 +1,184 @@ +"""D2b for BabyHemingway: resolve entity gender by MAJORITY VOTE over nearby pronouns. + +The honorific-and-local-window resolver inherited from the Bronte/Yarros line fails badly +here. Measured on this corpus before writing a line of replacement: **397 male, 20 female** +across 1,102 entity records, with Catherine Barkley, Brett Ashley, Pilar, Maria, Marita and +Mary all held neutral and `Helen` and `Audrey` resolved outright WRONG. A corpus containing +those characters does not have twenty women in it. + +Why it fails is the same mechanism Yarros exposed from the other side: Hemingway's women +appear mostly inside male characters' scenes, so the pronouns nearest their names are +predominantly `he`. Yarros solved its version with the POV chapter header; Hemingway's +editions have no such header, so that fix does not transfer and a different signal is needed. + +⭐ THE SIGNAL THAT WORKS IS VOLUME. A major character is named hundreds of times, so instead +of trusting the nearest pronoun in one window, every occurrence votes and the majority wins. +A single window is dominated by whoever else is in the scene; three hundred windows are +dominated by the person being written about. + +⚠ THE INSTRUMENT REFUSES TO WRITE UNLESS IT BEATS WHAT IT REPLACES, scored against a +hand-verified control list. That is the same guard `pov_gender.py` carried, and it is the +only reason to believe a replacement is an improvement rather than a different set of errors. +""" +from __future__ import annotations +import argparse, json, re, sys +from collections import Counter +from pathlib import Path + +MALE = {"he", "him", "his", "himself"} +FEMALE = {"she", "her", "hers", "herself"} +WINDOW = 12 # words either side of the mention +MIN_VOTES = 6 # below this the evidence is too thin to overrule a hold +MARGIN = 0.60 # winning share required, else HELD neutral + + +def load_text(corpus: Path) -> str: + out = [] + for f in sorted((corpus / "works").glob("*.jsonl")): + for line in f.read_text(encoding="utf-8").splitlines(): + out.append(json.loads(line)["text"]) + return "\n".join(out) + + +def tally_all(text: str, surfaces: set) -> dict: + """One pass over the corpus for EVERY surface at once. + + The obvious shape -- rescan the text once per surface -- is 1,102 surfaces x 995,000 + words and does not finish in any useful time. Tokenise once, walk once, and carry running + prefix counts of male and female pronouns so a window costs two subtractions instead of a + 25-word inner loop. + """ + words = [w.lower() for w in re.findall(r"[A-Za-z'’]+", text)] + n = len(words) + pm = [0] * (n + 1) + pf = [0] * (n + 1) + for i, w in enumerate(words): + pm[i + 1] = pm[i] + (1 if w in MALE else 0) + pf[i + 1] = pf[i] + (1 if w in FEMALE else 0) + want = {s.lower(): s for s in surfaces} + out = {s: [0, 0] for s in surfaces} + for i, w in enumerate(words): + s = want.get(w) + if s is None: + continue + lo, hi = max(0, i - WINDOW), min(n, i + WINDOW + 1) + out[s][0] += pm[hi] - pm[lo] + out[s][1] += pf[hi] - pf[lo] + return {k: (v[0], v[1]) for k, v in out.items()} + + +def decide(m: int, f: int, base_m: float = 0.5): + """Score a name's local pronoun mix AGAINST THE CORPUS BASE RATE, not against 50:50. + + ⚠ MEASURED, and it is why the first version of this was refused by its own gate: a raw + majority vote scored 18 correct but FIVE wrong against the incumbent's one, and every + error was female-read-as-male -- Pilar m=426 f=243, Brett m=249 f=137. Both are strongly + female-associated; they merely appear in a corpus where male pronouns outnumber female + ones several times over, so a bare majority is dominated by the background rate rather + than by the character. + + The correction is to ask whether a name's neighbourhood is male-heavy RELATIVE TO the + corpus, which is what `base_m` supplies. A hold stays the safe outcome: the gate counts a + wrong answer as worse than no answer, because rename can leave a held entity neutral but + cannot undo a man's name given to a woman. + """ + tot = m + f + if tot < MIN_VOTES: + return None + base_f = 1.0 - base_m + # odds of the observed mix under each hypothesis, expressed as a share after dividing + # out the background. lift_m > lift_f means male-heavy beyond what the corpus explains. + lift_m = (m / tot) / base_m if base_m else 0.0 + lift_f = (f / tot) / base_f if base_f else 0.0 + s = lift_m + lift_f + if not s: + return None + if lift_m / s >= MARGIN: + return "m" + if lift_f / s >= MARGIN: + return "f" + return None + + +def score(resolved: dict, truth: dict) -> tuple[int, int, int]: + ok = bad = held = 0 + for name, want in truth.items(): + got = resolved.get(name, "__absent__") + if got == "__absent__": + continue + if got is None: + held += 1 + elif got == want: + ok += 1 + else: + bad += 1 + return ok, held, bad + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("corpus") + ap.add_argument("--entities", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--control", required=True, + help="Name=g,Name=g ... hand-verified, the gate this must beat") + a = ap.parse_args() + + truth = {} + for item in a.control.split(","): + n, _, g = item.partition("=") + truth[n.strip()] = g.strip() + + ents = json.loads(Path(a.entities).read_text(encoding="utf-8")) + text = load_text(Path(a.corpus)) + + incumbent = {} + for blob in ents.values(): + for v in blob["entities"].values(): + incumbent.setdefault(v["surface"], v.get("gender")) + + surfaces = {v["surface"] for blob in ents.values() for v in blob["entities"].values()} + tallies = tally_all(text, surfaces) + toks = [w.lower() for w in re.findall(r"[A-Za-z'’]+", text)] + gm = sum(1 for w in toks if w in MALE) + gf = sum(1 for w in toks if w in FEMALE) + base_m = gm / (gm + gf) + print(f" corpus pronoun base rate: male {gm:,} / female {gf:,} -> base_m {base_m:.3f}") + resolved, detail = {}, {} + for s in sorted(surfaces): + m, f = tallies[s] + g = decide(m, f, base_m) + resolved[s] = g + detail[s] = {"gender": g, "male_votes": m, "female_votes": f} + + ok_i, held_i, bad_i = score(incumbent, truth) + ok_n, held_n, bad_n = score(resolved, truth) + print(f" incumbent (honorific/window): correct {ok_i} held {held_i} WRONG {bad_i}") + print(f" proximity majority vote : correct {ok_n} held {held_n} WRONG {bad_n}") + for n, want in truth.items(): + if resolved.get(n) not in (want, None): + print(f" ⚠ still wrong: {n} want={want} got={resolved.get(n)} " + f"(m={detail[n]['male_votes']} f={detail[n]['female_votes']})") + + if bad_n > bad_i or (bad_n == bad_i and ok_n <= ok_i): + print(" REFUSING to write: does not beat the incumbent", file=sys.stderr) + return 1 + + counts = Counter(v["gender"] for v in detail.values()) + print(f" gender distribution: {dict(counts)}") + + out = json.loads(Path(a.entities).read_text(encoding="utf-8")) + changed = 0 + for blob in out.values(): + for v in blob["entities"].values(): + g = resolved.get(v["surface"]) + if g != v.get("gender"): + changed += 1 + v["gender"] = g + Path(a.out).write_text(json.dumps(out, indent=2, ensure_ascii=False), encoding="utf-8") + print(f" wrote {a.out} ({changed} gender fields changed)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/hemingway-corpus/phrase_map_hemingway.json b/scripts/hemingway-corpus/phrase_map_hemingway.json new file mode 100644 index 0000000..ec24610 --- /dev/null +++ b/scripts/hemingway-corpus/phrase_map_hemingway.json @@ -0,0 +1,44 @@ +{ + "corpus": "BabyHemingway", + "why": "The unigram entity scan structurally CANNOT reach a coined proper noun whose every component is an ordinary word -- the same class as Yarros's `Riders Quadrant` and `Fourth Wing`. The cap/lowercase detector correctly refuses to call `Gran` or `Maestro` a name, so the phrase survives a gate reading zero. It needs a MAP, not a detector: substituting a coined title is a choice about register, not a measurement.", + "how_applied": "rename.py runs this AFTER the entity substitution; leak_gate.py audits every recurring capitalised 2-3gram against `allow` and flags anything neither mapped nor allowed.", + "scale_note": "⭐ Hemingway needs THREE mapped phrases where Yarros needed 48, and that gap is the whole difference between the corpora: Yarros invented a world, Hemingway named the real one. Correspondingly the allow list here is long and hers was short.", + "phrases": { + "Gran Maestro": "Primo Servitore", + "Unknown Tongue": "Borrowed Speech", + "Sin House": "Gladness House" + }, + "phrases_why": { + "Gran Maestro": "The narrator's coined honorific for the head waiter at Harry's Bar in Across the River and Into the Trees -- an invented title for one specific character, not an Italian idiom. `Primo Servitore` holds the register: Italian, mock-formal, a title one man awards another in a private joke.", + "Unknown Tongue": "A running invention in True at First Light -- the narrator's private made-up language, capitalised and named ('you put up with Unknown Tongue for a long time'). A named thing he invented, so it renames.", + "Sin House": "Henry's brothel in Islands in the Stream, a named place ('Do you want the key to Sin House'). Invented proper noun built from two ordinary words." + }, + "tokens": {}, + "allow": [ + "New York", "San Sebastian", "San Dona", "South America", "East Africa", + "Gulf Stream", "Sand Key", "Coast Guard", "Civil War", "General Staff", + "Mau Mau", "Signor Tenente", "Thank God", "For Christ", "For God", + "Our Lord", "Our Lady", "Happy Hunting Grounds", "The Colonel", "The Lieutenant", + "The Queen", "The American", "The Italians", "The Austrians", "The Germans", + "The German", "The French", "The Indians", "The Basque", "The Masai", + "The Wakamba", "North Americans", "White Man", "White Hunters", "Kamba Shamba", + "Royal Game", "Big Picture", "White Heather", "Bwana Game", "Game Scouts", + "Game Scout", "Roman Soldier", "Wine Seller", "The Interpreter", "Comrade General" + ], + "allow_why": [ + "Real-world referents and generic English. Every entry was READ IN CONTEXT before being", + "allowed rather than classified by shape, which is how the three mapped coinages above were", + "separated from these. Some were not obvious:", + "", + " Royal Game a real colonial-Kenyan legal category for protected species, not a coinage", + " White Heather a real Scotch whisky brand, handed over in a square squat bottle", + " Bwana Game Swahili-English address for the Game Ranger -- a job, not a character name", + " Roman Soldier stage-direction labels (1st/2nd/3rd) in `Today is Friday`, a one-act play", + " Wine Seller the same play's `Hebrew Wine Seller` -- a generic role label", + " Big Picture an English idiom, used as one by the Colonel", + " The Interpreter a character named by role, same class as The Colonel and The Lieutenant", + "", + "⚠ SENSITIVITY FLOOR: phrases recurring fewer than 5 times were never audited, and are", + "therefore neither mapped nor allowed nor reported." + ] +} diff --git a/scripts/hemingway-corpus/stoplist_hemingway.json b/scripts/hemingway-corpus/stoplist_hemingway.json new file mode 100644 index 0000000..31ae37b --- /dev/null +++ b/scripts/hemingway-corpus/stoplist_hemingway.json @@ -0,0 +1,49 @@ +{ + "_why": [ + "Surfaces excluded from renaming because they are REAL-WORLD referents, not the author's", + "invented proper nouns. The R49 rule is that ambiguous cases are deliberately RENAMED --", + "renaming is the safe direction and leaving is the leaking one -- so this list is kept to", + "categories that are unambiguously real: geography, demonyms, brands, and foreign-language", + "titles or common nouns that function as ordinary address.", + "", + "⚠ SENSITIVITY FLOOR, stated because it is part of the result: every surface in the top 200", + "by count was classified, and several were read IN CONTEXT first (that is how `G` was caught).", + "Below the top 200 surfaces were NOT individually read, so a rare real-world referent may be", + "renamed. That is the safe direction and an accepted cost, not an oversight.", + "", + "⚠ Hemingway is a far harder case than Bronte or Yarros for this: his fiction is saturated", + "with real cities, real armies, real wars and four languages. The 23-form non-ASCII alphabet", + "measured on this corpus is the same fact from the other side." + ], + "_caught_by_reading_in_context": { + "G": "NOT a name. It is the fragment left by `B.G.`, `G.M.`, `G2`, `G3` -- 248 occurrences. Renaming it would have produced `B.Vasquez` inside the prose.", + "Gran": "fragment of real names: `Gran Sasso D'Italia`, `Gran Italia` (a Milan restaurant), `Gran Hotel`.", + "Shamba": "Swahili common noun for a farm or settlement, used throughout True at First Light as an ordinary place word.", + "Ingles": "KEPT RENAMEABLE, deliberately. It is the gypsy band's in-world nickname for Robert Jordan, exactly parallel to Yarros's `Violence` for Violet -- a nickname is the author's invention even when the word is not." + }, + "geography": [ + "Africa", "America", "Madrid", "Milan", "Havana", "Venice", "Cuba", "Nairobi", + "Segovia", "Valencia", "Cannes", "Biarritz", "Laitokitok", "Boise", "York", + "Sebastian", "San", "Gran", "States", "Paris", "Spain", "Italy", "Austria", + "France", "Switzerland", "Kenya", "Tanganyika", "Chicago", "Michigan", "Florida", + "Guadarrama", "Escorial", "Navacerrada", "Pamplona", "Burguete", "Bayonne", + "Kilimanjaro", "Ebro", "Tagus", "Adriatic", "Piave", "Isonzo", "Gorizia", "Udine", + "Torcello", "Trieste", "Bimini", "Nantucket", "Constantinople", "Smyrna" + ], + "demonyms_and_languages": [ + "American", "Americans", "Cuban", "Cubans", "Indian", "Indians", "Italian", "Italians", + "German", "Germans", "Austrian", "Austrians", "Russian", "Russians", "Spanish", + "English", "French", "Basque", "Extremaduran", "Kamba", "Wakamba", "Masai", "Mau", + "Arap", "Swahili", "Moorish", "Negro", "Gypsy", "Gypsies", "Croat", "Hungarian" + ], + "brands_and_products": [ + "Perrier", "Packard", "Ford", "Fiat", "Cinzano", "Anis", "Chianti", "Marsala", + "Bass", "Gordon", "Thermos", "Mannlicher", "Springfield", "Winchester", "Mauser" + ], + "foreign_titles_and_common_nouns": [ + "Tenente", "Signor", "Signora", "Signorina", "Bwana", "Memsahib", "Hapana", "Ngoma", + "Shamba", "Don", "Dona", "Senor", "Senora", "Senorita", "Maestro", "Matador", + "Picador", "Banderillero", "Guardia", "Ayuntamiento", "Mister", "Missus", "Doc" + ], + "acronym_fragments": ["G", "GI", "CP", "BG", "GM", "MP", "RAF", "PC", "SIM"] +} diff --git a/scripts/r49-corpus/rename.py b/scripts/r49-corpus/rename.py index dd118be..9024427 100644 --- a/scripts/r49-corpus/rename.py +++ b/scripts/r49-corpus/rename.py @@ -47,6 +47,16 @@ PRESETS = { "yarros": {"a": ("us", ["en_US", "en_CA"]), "b": ("eu", ["es_ES", "es_MX", "it_IT", "de_DE", "fr_FR"]), "share": {}, "default_share": 0.62}, + # Hemingway's cast is genuinely multilingual -- Spanish, Italian, Cuban and American + # characters in the same books -- and the corpus alphabet agrees: 1,496 non-ASCII letters + # across 23 forms, against Yarros's 2. The F02 rule (a pool's character inventory must be + # a SUBSET of the corpus's) therefore permits accents here where it forbade them for + # Yarros, and forbidding them would strand every Spanish and Italian name in the cast. + # Share favours the romance-language pool because the two largest works by far -- For Whom + # the Bell Tolls and Islands in the Stream -- are set in Spain and the Caribbean. + "hemingway": {"a": ("us", ["en_US", "en_GB"]), + "b": ("rom", ["es_ES", "es_MX", "it_IT", "fr_FR"]), + "share": {}, "default_share": 0.45}, }