BabyHemingway D2+D3: entities, base-rate gender resolver, rename preset, leak gate passes

scripts/hemingway-corpus/build_corpus_hemingway.py

@@ -214,6 +214,44 @@ def split_by_contents(text: str):
     return units or None
 
 
+# ⚠⚠ PUBLISHER BACK MATTER RIDES INSIDE THE LAST UNIT, in 8 of 10 works -- the identical
+# defect the Yarros build hit, and it is worth naming again because nothing about the source
+# changed to cause it: a splitter cuts on headings, and nothing follows the last one, so the
+# "About the Author" block lands inside the final chapter. Here it carries ERNEST HEMINGWAY'S
+# OWN NAME 95 times across 7 works -- "Ernest Hemingway was one of America's foremost
+# journalists... died in 1961" -- which is precisely the leak the rename pipeline and its gate
+# exist to prevent, sitting in the training text before either one runs.
+# Bounded: last unit only, and refuses if it would take more than 2% of the corpus.
+# ⚠ The EDITOR'S apparatus counts as back matter too, and it is the bigger leak. Stripping
+# only the publisher block left 18 "Hemingway" mentions in `true-at-first-light` -- all of
+# them inside a CAST OF CHARACTERS and SWAHILI GLOSSARY written by Patrick Hemingway
+# ("Mary  Ernest Hemingway's fourth and last wife", "Ngui  Hemingway's gun bearer"). That is
+# an editor describing the author's real household, not the author's prose, and it names him
+# directly. The markers are matched in FILE ORDER and the earliest one wins, so the whole
+# apparatus goes rather than just its last section.
+BACKMATTER = re.compile(
+    r"^[ \t]*(THE END|About the Author|ABOUT THE AUTHOR|About the Publisher|"
+    r"CAST OF CHARACTERS|SWAHILI GLOSSARY|GLOSSARY|EDITOR.S ACKNOWLEDGMENTS|"
+    r"ACKNOWLEDGMENTS|ACKNOWLEDGEMENTS|"
+    r"Books by [A-Z]|BOOKS BY |Copyright|COPYRIGHT|Also by [A-Z])[ \t]*$", re.M)
+
+
+def strip_backmatter(units, title):
+    if not units:
+        return units, 0
+    head, body = units[-1]
+    m = BACKMATTER.search(body)
+    if not m:
+        return units, 0
+    cut = body[:m.start()].rstrip()
+    removed = len(body.split()) - len(cut.split())
+    if len(cut.split()) < MIN_UNIT_WORDS:
+        print(f"  ⚠ REFUSING back-matter strip on {title}: last unit would fall below the "
+              f"{MIN_UNIT_WORDS}-word floor", file=sys.stderr)
+        return units, 0
+    return units[:-1] + [(head, cut)], removed
+
+
 def split_units(text: str):
     """Return (pattern_name, [(heading, body)]).
 
@@ -270,6 +308,7 @@ for w in works:
     text, fixed_lines = repair_lines(text)
     text, front_removed = strip_foreign_front(text, w["title"])
     pattern, units = split_units(text)
+    units, back_removed = strip_backmatter(units, w["title"])
     alphabet.update(ch for ch in text if ch.isalpha())
     words = sum(len(b.split()) for _, b in units)
     total_words += words; total_units += len(units)
@@ -279,7 +318,8 @@ for w in works:
             if w["title"] in CONTINUOUS else "")
     post = " (posthumous/edited)" if w["title"] in POSTHUMOUS_EDITED else ""
     extra = (f"  [smallcaps {fixed_lines}]" if fixed_lines else "") + \
-            (f"  [front -{front_removed}w]" if front_removed else "")
+            (f"  [front -{front_removed}w]" if front_removed else "") + \
+            (f"  [back -{back_removed}w]" if back_removed else "")
     print(f"  {w['slug']:26} {len(units):>4} units  {words:>8,} words  "
           f"via {pattern:<14}{post}{flag}{extra}")
     manifest["works"].append({"slug": w["slug"], "title": w["title"], "rights": w["rights"],
@@ -287,6 +327,7 @@ for w in works:
                               "heading_pattern": pattern,
                               "smallcaps_lines_repaired": fixed_lines,
                               "foreign_front_matter_words_removed": front_removed,
+                              "publisher_back_matter_words_removed": back_removed,
                               "posthumous_editor_shaped": w["title"] in POSTHUMOUS_EDITED,
                               "path": f"works/{w['slug']}.jsonl"})
     if not a.survey:

scripts/hemingway-corpus/gender_by_proximity.py

@@ -0,0 +1,184 @@
+"""D2b for BabyHemingway: resolve entity gender by MAJORITY VOTE over nearby pronouns.
+
+The honorific-and-local-window resolver inherited from the Bronte/Yarros line fails badly
+here. Measured on this corpus before writing a line of replacement: **397 male, 20 female**
+across 1,102 entity records, with Catherine Barkley, Brett Ashley, Pilar, Maria, Marita and
+Mary all held neutral and `Helen` and `Audrey` resolved outright WRONG. A corpus containing
+those characters does not have twenty women in it.
+
+Why it fails is the same mechanism Yarros exposed from the other side: Hemingway's women
+appear mostly inside male characters' scenes, so the pronouns nearest their names are
+predominantly `he`. Yarros solved its version with the POV chapter header; Hemingway's
+editions have no such header, so that fix does not transfer and a different signal is needed.
+
+⭐ THE SIGNAL THAT WORKS IS VOLUME. A major character is named hundreds of times, so instead
+of trusting the nearest pronoun in one window, every occurrence votes and the majority wins.
+A single window is dominated by whoever else is in the scene; three hundred windows are
+dominated by the person being written about.
+
+⚠ THE INSTRUMENT REFUSES TO WRITE UNLESS IT BEATS WHAT IT REPLACES, scored against a
+hand-verified control list. That is the same guard `pov_gender.py` carried, and it is the
+only reason to believe a replacement is an improvement rather than a different set of errors.
+"""
+from __future__ import annotations
+import argparse, json, re, sys
+from collections import Counter
+from pathlib import Path
+
+MALE = {"he", "him", "his", "himself"}
+FEMALE = {"she", "her", "hers", "herself"}
+WINDOW = 12          # words either side of the mention
+MIN_VOTES = 6        # below this the evidence is too thin to overrule a hold
+MARGIN = 0.60        # winning share required, else HELD neutral
+
+
+def load_text(corpus: Path) -> str:
+    out = []
+    for f in sorted((corpus / "works").glob("*.jsonl")):
+        for line in f.read_text(encoding="utf-8").splitlines():
+            out.append(json.loads(line)["text"])
+    return "\n".join(out)
+
+
+def tally_all(text: str, surfaces: set) -> dict:
+    """One pass over the corpus for EVERY surface at once.
+
+    The obvious shape -- rescan the text once per surface -- is 1,102 surfaces x 995,000
+    words and does not finish in any useful time. Tokenise once, walk once, and carry running
+    prefix counts of male and female pronouns so a window costs two subtractions instead of a
+    25-word inner loop.
+    """
+    words = [w.lower() for w in re.findall(r"[A-Za-z'’]+", text)]
+    n = len(words)
+    pm = [0] * (n + 1)
+    pf = [0] * (n + 1)
+    for i, w in enumerate(words):
+        pm[i + 1] = pm[i] + (1 if w in MALE else 0)
+        pf[i + 1] = pf[i] + (1 if w in FEMALE else 0)
+    want = {s.lower(): s for s in surfaces}
+    out = {s: [0, 0] for s in surfaces}
+    for i, w in enumerate(words):
+        s = want.get(w)
+        if s is None:
+            continue
+        lo, hi = max(0, i - WINDOW), min(n, i + WINDOW + 1)
+        out[s][0] += pm[hi] - pm[lo]
+        out[s][1] += pf[hi] - pf[lo]
+    return {k: (v[0], v[1]) for k, v in out.items()}
+
+
+def decide(m: int, f: int, base_m: float = 0.5):
+    """Score a name's local pronoun mix AGAINST THE CORPUS BASE RATE, not against 50:50.
+
+    ⚠ MEASURED, and it is why the first version of this was refused by its own gate: a raw
+    majority vote scored 18 correct but FIVE wrong against the incumbent's one, and every
+    error was female-read-as-male -- Pilar m=426 f=243, Brett m=249 f=137. Both are strongly
+    female-associated; they merely appear in a corpus where male pronouns outnumber female
+    ones several times over, so a bare majority is dominated by the background rate rather
+    than by the character.
+
+    The correction is to ask whether a name's neighbourhood is male-heavy RELATIVE TO the
+    corpus, which is what `base_m` supplies. A hold stays the safe outcome: the gate counts a
+    wrong answer as worse than no answer, because rename can leave a held entity neutral but
+    cannot undo a man's name given to a woman.
+    """
+    tot = m + f
+    if tot < MIN_VOTES:
+        return None
+    base_f = 1.0 - base_m
+    # odds of the observed mix under each hypothesis, expressed as a share after dividing
+    # out the background. lift_m > lift_f means male-heavy beyond what the corpus explains.
+    lift_m = (m / tot) / base_m if base_m else 0.0
+    lift_f = (f / tot) / base_f if base_f else 0.0
+    s = lift_m + lift_f
+    if not s:
+        return None
+    if lift_m / s >= MARGIN:
+        return "m"
+    if lift_f / s >= MARGIN:
+        return "f"
+    return None
+
+
+def score(resolved: dict, truth: dict) -> tuple[int, int, int]:
+    ok = bad = held = 0
+    for name, want in truth.items():
+        got = resolved.get(name, "__absent__")
+        if got == "__absent__":
+            continue
+        if got is None:
+            held += 1
+        elif got == want:
+            ok += 1
+        else:
+            bad += 1
+    return ok, held, bad
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("corpus")
+    ap.add_argument("--entities", required=True)
+    ap.add_argument("--out", required=True)
+    ap.add_argument("--control", required=True,
+                    help="Name=g,Name=g ... hand-verified, the gate this must beat")
+    a = ap.parse_args()
+
+    truth = {}
+    for item in a.control.split(","):
+        n, _, g = item.partition("=")
+        truth[n.strip()] = g.strip()
+
+    ents = json.loads(Path(a.entities).read_text(encoding="utf-8"))
+    text = load_text(Path(a.corpus))
+
+    incumbent = {}
+    for blob in ents.values():
+        for v in blob["entities"].values():
+            incumbent.setdefault(v["surface"], v.get("gender"))
+
+    surfaces = {v["surface"] for blob in ents.values() for v in blob["entities"].values()}
+    tallies = tally_all(text, surfaces)
+    toks = [w.lower() for w in re.findall(r"[A-Za-z'’]+", text)]
+    gm = sum(1 for w in toks if w in MALE)
+    gf = sum(1 for w in toks if w in FEMALE)
+    base_m = gm / (gm + gf)
+    print(f"  corpus pronoun base rate: male {gm:,} / female {gf:,} -> base_m {base_m:.3f}")
+    resolved, detail = {}, {}
+    for s in sorted(surfaces):
+        m, f = tallies[s]
+        g = decide(m, f, base_m)
+        resolved[s] = g
+        detail[s] = {"gender": g, "male_votes": m, "female_votes": f}
+
+    ok_i, held_i, bad_i = score(incumbent, truth)
+    ok_n, held_n, bad_n = score(resolved, truth)
+    print(f"  incumbent (honorific/window): correct {ok_i}  held {held_i}  WRONG {bad_i}")
+    print(f"  proximity majority vote     : correct {ok_n}  held {held_n}  WRONG {bad_n}")
+    for n, want in truth.items():
+        if resolved.get(n) not in (want, None):
+            print(f"    ⚠ still wrong: {n} want={want} got={resolved.get(n)} "
+                  f"(m={detail[n]['male_votes']} f={detail[n]['female_votes']})")
+
+    if bad_n > bad_i or (bad_n == bad_i and ok_n <= ok_i):
+        print("  REFUSING to write: does not beat the incumbent", file=sys.stderr)
+        return 1
+
+    counts = Counter(v["gender"] for v in detail.values())
+    print(f"  gender distribution: {dict(counts)}")
+
+    out = json.loads(Path(a.entities).read_text(encoding="utf-8"))
+    changed = 0
+    for blob in out.values():
+        for v in blob["entities"].values():
+            g = resolved.get(v["surface"])
+            if g != v.get("gender"):
+                changed += 1
+            v["gender"] = g
+    Path(a.out).write_text(json.dumps(out, indent=2, ensure_ascii=False), encoding="utf-8")
+    print(f"  wrote {a.out} ({changed} gender fields changed)")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/hemingway-corpus/phrase_map_hemingway.json

@@ -0,0 +1,44 @@
+{
+  "corpus": "BabyHemingway",
+  "why": "The unigram entity scan structurally CANNOT reach a coined proper noun whose every component is an ordinary word -- the same class as Yarros's `Riders Quadrant` and `Fourth Wing`. The cap/lowercase detector correctly refuses to call `Gran` or `Maestro` a name, so the phrase survives a gate reading zero. It needs a MAP, not a detector: substituting a coined title is a choice about register, not a measurement.",
+  "how_applied": "rename.py runs this AFTER the entity substitution; leak_gate.py audits every recurring capitalised 2-3gram against `allow` and flags anything neither mapped nor allowed.",
+  "scale_note": "⭐ Hemingway needs THREE mapped phrases where Yarros needed 48, and that gap is the whole difference between the corpora: Yarros invented a world, Hemingway named the real one. Correspondingly the allow list here is long and hers was short.",
+  "phrases": {
+    "Gran Maestro": "Primo Servitore",
+    "Unknown Tongue": "Borrowed Speech",
+    "Sin House": "Gladness House"
+  },
+  "phrases_why": {
+    "Gran Maestro": "The narrator's coined honorific for the head waiter at Harry's Bar in Across the River and Into the Trees -- an invented title for one specific character, not an Italian idiom. `Primo Servitore` holds the register: Italian, mock-formal, a title one man awards another in a private joke.",
+    "Unknown Tongue": "A running invention in True at First Light -- the narrator's private made-up language, capitalised and named ('you put up with Unknown Tongue for a long time'). A named thing he invented, so it renames.",
+    "Sin House": "Henry's brothel in Islands in the Stream, a named place ('Do you want the key to Sin House'). Invented proper noun built from two ordinary words."
+  },
+  "tokens": {},
+  "allow": [
+    "New York", "San Sebastian", "San Dona", "South America", "East Africa",
+    "Gulf Stream", "Sand Key", "Coast Guard", "Civil War", "General Staff",
+    "Mau Mau", "Signor Tenente", "Thank God", "For Christ", "For God",
+    "Our Lord", "Our Lady", "Happy Hunting Grounds", "The Colonel", "The Lieutenant",
+    "The Queen", "The American", "The Italians", "The Austrians", "The Germans",
+    "The German", "The French", "The Indians", "The Basque", "The Masai",
+    "The Wakamba", "North Americans", "White Man", "White Hunters", "Kamba Shamba",
+    "Royal Game", "Big Picture", "White Heather", "Bwana Game", "Game Scouts",
+    "Game Scout", "Roman Soldier", "Wine Seller", "The Interpreter", "Comrade General"
+  ],
+  "allow_why": [
+    "Real-world referents and generic English. Every entry was READ IN CONTEXT before being",
+    "allowed rather than classified by shape, which is how the three mapped coinages above were",
+    "separated from these. Some were not obvious:",
+    "",
+    "  Royal Game    a real colonial-Kenyan legal category for protected species, not a coinage",
+    "  White Heather a real Scotch whisky brand, handed over in a square squat bottle",
+    "  Bwana Game    Swahili-English address for the Game Ranger -- a job, not a character name",
+    "  Roman Soldier stage-direction labels (1st/2nd/3rd) in `Today is Friday`, a one-act play",
+    "  Wine Seller   the same play's `Hebrew Wine Seller` -- a generic role label",
+    "  Big Picture   an English idiom, used as one by the Colonel",
+    "  The Interpreter  a character named by role, same class as The Colonel and The Lieutenant",
+    "",
+    "⚠ SENSITIVITY FLOOR: phrases recurring fewer than 5 times were never audited, and are",
+    "therefore neither mapped nor allowed nor reported."
+  ]
+}

scripts/hemingway-corpus/stoplist_hemingway.json

@@ -0,0 +1,49 @@
+{
+  "_why": [
+    "Surfaces excluded from renaming because they are REAL-WORLD referents, not the author's",
+    "invented proper nouns. The R49 rule is that ambiguous cases are deliberately RENAMED --",
+    "renaming is the safe direction and leaving is the leaking one -- so this list is kept to",
+    "categories that are unambiguously real: geography, demonyms, brands, and foreign-language",
+    "titles or common nouns that function as ordinary address.",
+    "",
+    "⚠ SENSITIVITY FLOOR, stated because it is part of the result: every surface in the top 200",
+    "by count was classified, and several were read IN CONTEXT first (that is how `G` was caught).",
+    "Below the top 200 surfaces were NOT individually read, so a rare real-world referent may be",
+    "renamed. That is the safe direction and an accepted cost, not an oversight.",
+    "",
+    "⚠ Hemingway is a far harder case than Bronte or Yarros for this: his fiction is saturated",
+    "with real cities, real armies, real wars and four languages. The 23-form non-ASCII alphabet",
+    "measured on this corpus is the same fact from the other side."
+  ],
+  "_caught_by_reading_in_context": {
+    "G": "NOT a name. It is the fragment left by `B.G.`, `G.M.`, `G2`, `G3` -- 248 occurrences. Renaming it would have produced `B.Vasquez` inside the prose.",
+    "Gran": "fragment of real names: `Gran Sasso D'Italia`, `Gran Italia` (a Milan restaurant), `Gran Hotel`.",
+    "Shamba": "Swahili common noun for a farm or settlement, used throughout True at First Light as an ordinary place word.",
+    "Ingles": "KEPT RENAMEABLE, deliberately. It is the gypsy band's in-world nickname for Robert Jordan, exactly parallel to Yarros's `Violence` for Violet -- a nickname is the author's invention even when the word is not."
+  },
+  "geography": [
+    "Africa", "America", "Madrid", "Milan", "Havana", "Venice", "Cuba", "Nairobi",
+    "Segovia", "Valencia", "Cannes", "Biarritz", "Laitokitok", "Boise", "York",
+    "Sebastian", "San", "Gran", "States", "Paris", "Spain", "Italy", "Austria",
+    "France", "Switzerland", "Kenya", "Tanganyika", "Chicago", "Michigan", "Florida",
+    "Guadarrama", "Escorial", "Navacerrada", "Pamplona", "Burguete", "Bayonne",
+    "Kilimanjaro", "Ebro", "Tagus", "Adriatic", "Piave", "Isonzo", "Gorizia", "Udine",
+    "Torcello", "Trieste", "Bimini", "Nantucket", "Constantinople", "Smyrna"
+  ],
+  "demonyms_and_languages": [
+    "American", "Americans", "Cuban", "Cubans", "Indian", "Indians", "Italian", "Italians",
+    "German", "Germans", "Austrian", "Austrians", "Russian", "Russians", "Spanish",
+    "English", "French", "Basque", "Extremaduran", "Kamba", "Wakamba", "Masai", "Mau",
+    "Arap", "Swahili", "Moorish", "Negro", "Gypsy", "Gypsies", "Croat", "Hungarian"
+  ],
+  "brands_and_products": [
+    "Perrier", "Packard", "Ford", "Fiat", "Cinzano", "Anis", "Chianti", "Marsala",
+    "Bass", "Gordon", "Thermos", "Mannlicher", "Springfield", "Winchester", "Mauser"
+  ],
+  "foreign_titles_and_common_nouns": [
+    "Tenente", "Signor", "Signora", "Signorina", "Bwana", "Memsahib", "Hapana", "Ngoma",
+    "Shamba", "Don", "Dona", "Senor", "Senora", "Senorita", "Maestro", "Matador",
+    "Picador", "Banderillero", "Guardia", "Ayuntamiento", "Mister", "Missus", "Doc"
+  ],
+  "acronym_fragments": ["G", "GI", "CP", "BG", "GM", "MP", "RAF", "PC", "SIM"]
+}

scripts/r49-corpus/rename.py

@@ -47,6 +47,16 @@ PRESETS = {
     "yarros": {"a": ("us", ["en_US", "en_CA"]),
                "b": ("eu", ["es_ES", "es_MX", "it_IT", "de_DE", "fr_FR"]),
                "share": {}, "default_share": 0.62},
+    # Hemingway's cast is genuinely multilingual -- Spanish, Italian, Cuban and American
+    # characters in the same books -- and the corpus alphabet agrees: 1,496 non-ASCII letters
+    # across 23 forms, against Yarros's 2. The F02 rule (a pool's character inventory must be
+    # a SUBSET of the corpus's) therefore permits accents here where it forbade them for
+    # Yarros, and forbidding them would strand every Spanish and Italian name in the cast.
+    # Share favours the romance-language pool because the two largest works by far -- For Whom
+    # the Bell Tolls and Islands in the Stream -- are set in Spain and the Caribbean.
+    "hemingway": {"a": ("us", ["en_US", "en_GB"]),
+                  "b": ("rom", ["es_ES", "es_MX", "it_IT", "fr_FR"]),
+                  "share": {}, "default_share": 0.45},
 }