diff --git a/scripts/mccarthy-corpus/gender_overrides_mccarthy.json b/scripts/mccarthy-corpus/gender_overrides_mccarthy.json new file mode 100644 index 0000000..8eeaa7c --- /dev/null +++ b/scripts/mccarthy-corpus/gender_overrides_mccarthy.json @@ -0,0 +1,34 @@ +{ + "_why": [ + "The human pass entities.py asks for. The honorific/window resolver scored 21 correct,", + "3 held and 1 WRONG against a 26-name hand-verified control; the base-rate proximity", + "resolver built for Hemingway scored 18/6/1 on the same control and its own guard", + "REFUSED to write, correctly. So the incumbent stands and these four are fixed by hand.", + "", + "⚠ Every entry below is FEMALE, and every one looks male-dominated in raw pronoun counts.", + "The McCarthy corpus runs 29,144 male pronouns to 5,036 female — a base rate of 85.3%", + "male. Against that background each of these is a strong female signal, which is the", + "same arithmetic that recovered Pilar and Brett on Hemingway.", + "", + "Alfonsa was in the control set and is NOT here: she appears 4 times, below the", + "--min-count 5 detection floor, so she is correctly absent from the map. That was an", + "error in the control, not in the pipeline." + ], + "Carla": { + "gender": "f", + "correcting": true, + "why": "Carla Jean Moss, Llewelyn's wife in No Country for Old Men. Read MALE by the incumbent because her scenes are dominated by Moss, Chigurh and Bell. Nearby pronouns 31m/21f — at the corpus base rate that would be 44m/8f, so 21 female against an expected 8 is decisive." + }, + "Magdalena": { + "gender": "f", + "why": "The girl John Grady loves in Cities of the Plain. HELD by the incumbent. Nearby pronouns 17m/22f — a female MAJORITY inside an 85%-male corpus." + }, + "Socorro": { + "gender": "f", + "why": "The cook at Mac's ranch — 'Socorro came and took the plate of biscuits and carried them to the oven'. HELD by the incumbent. Nearby pronouns 80m/65f against an expected 124m/21f." + }, + "Luisa": { + "gender": "f", + "why": "A servant in Cities of the Plain — 'Luisa had gone to bed and the house was quiet'. HELD by the incumbent. Nearby pronouns 13m/10f against an expected 20m/3f." + } +} diff --git a/scripts/mccarthy-corpus/stoplist_mccarthy.json b/scripts/mccarthy-corpus/stoplist_mccarthy.json index 8ca4b43..2f4b95c 100644 --- a/scripts/mccarthy-corpus/stoplist_mccarthy.json +++ b/scripts/mccarthy-corpus/stoplist_mccarthy.json @@ -41,7 +41,10 @@ "Ford / Hashknives": "STOPLISTED. 'a Model A Ford truck'; 'went to work for the Hashknives' -- the real Aztec Land & Cattle outfit. Brands and real companies.", "Calle": "STOPLISTED. Spanish for 'street' -- 'the Calle de Noche Triste'. A common noun, like Avenue.", "Yaqui / Gilenos": "STOPLISTED. Real indigenous peoples -- 'lay clear of the Yaqui country', 'all trace of the Gilenos faded'. Demonyms.", - "Terrell": "STOPLISTED. 'nineteen felony charges were filed in the Terrell County Court' -- a real Texas county." + "Terrell": "STOPLISTED. 'nineteen felony charges were filed in the Terrell County Court' -- a real Texas county.", + "Hearst / Trias / Huerta / Madero": "STOPLISTED as REAL PEOPLE. 'a señor Hearst' (the real Hearst ranch at Babícora); 'a contract with Trias' (Ángel Trías, the real Chihuahua governor who paid Glanton for scalps); 'Victoriano Huerta'; 'Francisco Madero'. Blood Meridian and the Border Trilogy put real historical figures on the page under their own names.", + "Batopite / Buenaventura / Deming / Nacogdoches / Californy / Hog": "STOPLISTED as geography. Real Chihuahua, New Mexico and Texas towns; 'Californy' is dialect for California; 'Hog' is only ever 'Hog Canyon Springs'.", + "Doc / Dorsey / Gillian": "KEPT RENAMEABLE. 'Doc Irving' and 'John Dorsey' are men; 'Soto y Gillian' is a cattle company named for its owners, which is ambiguous and so takes the safe direction." }, "geography": [ "America", @@ -52,35 +55,46 @@ "Arkansas", "Avenue", "Babícora", + "Batopite", "Baviacora", "Bavispe", "Bexar", "Boquilla", + "Buenaventura", "Caborca", "California", + "Californy", "Calle", "Casas", "Chihuahua", "Cloverdale", "Coahuila", + "Colorado", + "Deming", "Diego", "Durango", "Eagle", "El", "Encantada", "Francisco", + "Frontera", + "Galveston", "Grandes", "Griffin", + "Hog", "Huisiachepic", "Janos", + "Kentucky", "Langtry", "Madera", "Mexico", "Missouri", + "Nacogdoches", "Nacozari", "Namiquipa", "Odessa", "Oklahoma", + "Parras", "Paso", "Pilares", "Purísima", @@ -169,5 +183,13 @@ "Venada", "Moderno" ] - } + }, + "real_people": [ + "Francisco", + "Hearst", + "Huerta", + "Madero", + "Trias", + "Victoriano" + ] } diff --git a/scripts/r49-corpus/apply_gender_overrides.py b/scripts/r49-corpus/apply_gender_overrides.py new file mode 100644 index 0000000..475e427 --- /dev/null +++ b/scripts/r49-corpus/apply_gender_overrides.py @@ -0,0 +1,86 @@ +"""Apply hand-verified genders to an entity map — the human pass the pipeline asks for. + +`entities.py` says it plainly: "Nothing here guesses. Unresolved entities block corpus +emission and go to a human pass: held is cheap, wrong is poison -- a silently mis-gendered +entity scrambles pronoun agreement through every renamed copy and nothing downstream would +catch it." This is that pass, written down instead of typed into a JSON by hand. + +⚠ RAW PRONOUN COUNTS ARE THE TRAP, AND THIS IS WHY EVERY OVERRIDE CARRIES ITS EVIDENCE. +Measured on McCarthy: the corpus runs 29,144 male pronouns to 5,036 female, a base rate of +**85.3% male**. So a name sitting at 31 male / 21 female nearby is not "male-dominated" — at +the base rate it would be 44/8, and 21 female against an expected 8 is a strong FEMALE +signal. Carla Jean Moss was read male by the honorific/window resolver for exactly that +reason. The same arithmetic recovered Pilar and Brett on Hemingway. + +TWO REFUSALS, because an override file is a place where a typo is invisible: + + * a name not present in the entity map is an ERROR, not a no-op. A silent skip means a + misspelled override looks like it applied and the entity stays mis-gendered. + * changing a gender the map already holds requires `"correcting": true` on that entry. + Filling a HELD entity is the ordinary case; overruling the detector is not, and the two + should not look the same in a diff. +""" +from __future__ import annotations +import argparse, json +from pathlib import Path + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--entities", required=True) + ap.add_argument("--overrides", required=True, + help='JSON: {"": {"gender": "f", "why": "...", ' + '"correcting": true}} — `why` is required, `correcting` only when ' + 'the map already holds a different gender') + ap.add_argument("--out", required=True) + a = ap.parse_args() + + ents = json.loads(Path(a.entities).read_text()) + blob = json.loads(Path(a.overrides).read_text()) + ov = {k: v for k, v in blob.items() if not k.startswith("_")} + + # surface -> [(work, key)] + where: dict[str, list[tuple[str, str]]] = {} + for w, blk in ents.items(): + for k, e in blk["entities"].items(): + where.setdefault(e.get("surface") or k, []).append((w, k)) + + missing = [n for n in ov if n not in where] + if missing: + print(f"== REFUSING: {len(missing)} override(s) name no entity in the map: {missing}") + print(" A misspelled override that silently does nothing leaves the entity") + print(" mis-gendered AND looks like it was handled.") + return 1 + + no_why = [n for n, v in ov.items() if not (v.get("why") or "").strip()] + if no_why: + print(f"== REFUSING: no `why` on {no_why}. An override without its evidence is a guess.") + return 1 + + changed = filled = 0 + for name, spec in sorted(ov.items()): + g = spec["gender"] + for w, k in where[name]: + cur = ents[w]["entities"][k].get("gender") + if cur == g: + continue + if cur and not spec.get("correcting"): + print(f"== REFUSING: {name} in {w} already reads {cur!r} and the override says " + f"{g!r} without \"correcting\": true. Overruling the detector is not the " + f"same act as filling a held entity.") + return 1 + ents[w]["entities"][k]["gender"] = g + ents[w]["entities"][k]["gender_source"] = "hand-verified" + changed += cur is not None + filled += cur is None + print(f" {name:<14} {w:<26} {str(cur):>6} -> {g} " + f"{'CORRECTION' if cur else 'filled held'}") + + Path(a.out).write_text(json.dumps(ents, ensure_ascii=False, indent=1), encoding="utf-8") + print(f"\n {filled} held entity/entities filled, {changed} detector reading(s) corrected") + print(f" wrote {a.out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/r49-corpus/rename.py b/scripts/r49-corpus/rename.py index 9024427..73237fb 100644 --- a/scripts/r49-corpus/rename.py +++ b/scripts/r49-corpus/rename.py @@ -57,6 +57,21 @@ PRESETS = { "hemingway": {"a": ("us", ["en_US", "en_GB"]), "b": ("rom", ["es_ES", "es_MX", "it_IT", "fr_FR"]), "share": {}, "default_share": 0.45}, + # McCarthy is NOT a narrower Hemingway, and reusing that preset would have been the + # easy wrong answer. Both are Spanish-inflected, but Hemingway's romance pool carries + # it_IT and fr_FR for his Italian and French casts -- and McCarthy writes neither + # language and has no such characters. Drawing from it would drop Italian and French + # surnames into a Texas-Mexico border novel, which is exactly the register error the + # per-corpus pool exists to prevent. en_GB goes for the same reason: Hemingway has + # English characters (Brett Ashley, "the Englishman"), McCarthy's Anglo cast is + # Texan and Tennessean throughout. + # Share is even. The six works split about half and half: The Crossing and much of + # All the Pretty Horses and Cities of the Plain are set in Mexico, while Blood + # Meridian's gang and No Country's cast are Anglo. The corpus alphabet agrees that + # accents belong -- 1,411 non-ASCII letters across 14 forms, all Spanish (á é í ñ ó ú ü). + "mccarthy": {"a": ("us", ["en_US"]), + "b": ("mex", ["es_MX", "es_ES"]), + "share": {}, "default_share": 0.50}, } @@ -105,7 +120,16 @@ def main() -> int: ap.add_argument("--out", required=True) ap.add_argument("--copies", type=int, default=6) ap.add_argument("--seed", type=int, default=4919) - ap.add_argument("--holdout-chapter", type=int, default=10) + # ⚠ THE VAL SPLIT IS ONE CHAPTER INDEX PER WORK, so its SIZE is set by how many WORKS a + # corpus has, not by how many words. That is why it takes a LIST now. Measured across the + # line: Hemingway has 10 works -> 9 val units -> 36,563 words per copy and a decisive + # gate; Brontë has 4 works -> 4 units -> 17,043 words and a gate that could not resolve + # its own effect. A 588k-word corpus of 6 works would land at Brontë's end of that on a + # single index, and no amount of corpus size fixes it. + ap.add_argument("--holdout-chapter", type=int, nargs="+", default=[10], + help="chapter index/indices held out as val in EVERY work. Space them " + "apart -- adjacent chapters are more correlated with each other " + "than two drawn from different parts of a book.") ap.add_argument("--preset", default="bronte", choices=sorted(PRESETS), help="which corpus's name-pool register to draw from") ap.add_argument("--scope", default="work", choices=("work", "corpus"), @@ -123,6 +147,7 @@ def main() -> int: man = json.loads((corpus / "manifest.json").read_text()) alphabet = set(json.loads((corpus / "corpus_alphabet.json").read_text())["letters"]) ents_all = json.loads(Path(a.entities).read_text()) + holdout = set(a.holdout_chapter) pool = build_pool(Path(a.dictionary), alphabet, a.preset) cfg = PRESETS[a.preset] label_a, label_b = cfg["a"][0], cfg["b"][0] @@ -276,7 +301,7 @@ def main() -> int: txt = pat.sub(lambda m: mapping[surf2key[m.group(1)]], r["text"]) if phrase_sub: txt = phrase_sub(txt) - split = "val" if r["chapter"] == a.holdout_chapter else "train" + split = "val" if r["chapter"] in holdout else "train" fh.write(json.dumps({"work": slug, "copy": c, "chapter": r["chapter"], "split": split, "text": txt}, ensure_ascii=False) + "\n") emitted += 1