diff --git a/scripts/r49-corpus/entities.py b/scripts/r49-corpus/entities.py index c4d6a71..f3770a4 100644 --- a/scripts/r49-corpus/entities.py +++ b/scripts/r49-corpus/entities.py @@ -19,6 +19,20 @@ trained on. Nothing here guesses. Unresolved entities block corpus emission and go to a human pass: held is cheap, wrong is poison -- a silently mis-gendered entity scrambles pronoun agreement through every renamed copy and nothing downstream would catch it. + +⚠ v4, added for BabyYarros: a MID-SENTENCE test on top of the ratio. +The cap/lowercase ratio calls `Hey`, `Holy`, `Hopefully`, `Yep`, `Whoa`, `Nope` +and `Ugh` names, because a dialogue-heavy contemporary novel opens sentences with +them constantly and never writes them lowercase. The v1 lesson was that POSITION +ALONE misses names that start sentences; position as a SECOND filter has no such +problem, because a real name also appears mid-sentence. Measured on BabyYarros the +two populations do not overlap: 33 verified names sit at 0.567-0.985 mid-sentence, +and 19 verified interjections at 0.000-0.222. The gap is 2.5x wide, so the +threshold is not a tuned parameter. + +It is OPT-IN (`--min-mid-ratio`, default 0 = off) so the Brontë run stays +byte-reproducible. A 19th-century novel does not have this failure mode in the +same volume, and an unmeasured change to a settled corpus is not an improvement. """ from __future__ import annotations import argparse, collections, json, re, sys @@ -38,6 +52,9 @@ STOP_TITLES = { "Grandmother", "Grandfather", "Nurse", "King", "Queen", "Prince", "Princess", "Duke", "Duchess", "Earl", "Count", "Countess", "Baron", "Squire", "Parson", "Monseigneur", "Mlle", "Mme", "M", "Messrs", + # modern ranks and address forms, added for BabyYarros + "Sergeant", "Sgt", "Lieutenant", "Lt", "Corporal", "Admiral", "Commander", + "Cadet", "Officer", "Agent", "Coach", "Senator", "Majesty", "Highness", } #: Days, months, and the language/nation adjectives a 19th-century novel is full #: of. All are always-capitalised and would otherwise pass the ratio test. @@ -56,7 +73,15 @@ STOP_COMMON = { "Who","When","Where","Why","How","If","So","As","At","In","On","To","For","Of", "Nay","Alas","Madam","Sir","Mademoiselle","Monsieur", } -STOP = STOP_TITLES | STOP_COMMON +#: Structural words from the book's own apparatus. `Chapter` and `Article` pass +#: both the ratio test and the mid-sentence test -- `BONUS CONTENT Chapter Nine` +#: and `Article Three` put them mid-sentence -- and renaming them would rewrite +#: the corpus's own scaffolding. +STOP_STRUCTURAL = { + "Chapter", "Chapters", "Prologue", "Epilogue", "Part", "Appendix", "Volume", + "Article", "Section", "Contents", "Content", "Bonus", "Preface", "Interlude", +} +STOP = STOP_TITLES | STOP_COMMON | STOP_STRUCTURAL MALE_PRON = {"he", "him", "his", "himself"} FEM_PRON = {"she", "her", "hers", "herself"} @@ -71,11 +96,25 @@ def load(corpus: Path) -> dict[str, str]: return out -def detect(text: str, min_count: int, max_ratio: float) -> dict[str, dict]: - """Corpus-level capitalised-vs-lowercase ratio. See module docstring.""" +#: `’s` is a possessive and the rest are contractions; none of them is part of the +#: name. TOKEN keeps the apostrophe, so without folding `Afendra’s` is its own key. +CLITIC = re.compile(r"[’'](?:s|d|ll|ve|re|m|t)$", re.I) + + +def detect(text: str, min_count: int, max_ratio: float, fold_clitics: bool = False) -> dict[str, dict]: + """Corpus-level capitalised-vs-lowercase ratio. See module docstring. + + ⚠ `fold_clitics` folds `Afendra’s` into `Afendra`. Without it an entity that + NEVER appears unpossessed is keyed with the apostrophe, and both rename.py and + the leak gate skip apostrophe keys as contractions -- so it is never renamed + AND never reported. Measured on BabyYarros: `Afendra` survived every copy + while the gate read 0 of 314, which is the worst failure shape there is. + """ cap, low = collections.Counter(), collections.Counter() for m in TOKEN.finditer(text): t = m.group(0) + if fold_clitics: + t = CLITIC.sub("", t) or t (cap if t[:1].isupper() else low)[t.lower()] += 1 ents = {} for key, c in cap.items(): @@ -90,13 +129,128 @@ def detect(text: str, min_count: int, max_ratio: float) -> dict[str, dict]: return ents -def surface_forms(text: str, keys: set[str]) -> dict[str, str]: +#: Whatever can sit between a sentence terminator and the first word of the next +#: sentence: whitespace, opening quotes, brackets, a dash. +_OPENERS = set(' \t\n\u201c\u201d"\'\u2018\u2019([{\u2014\u2013-*') +_TERM = set('.!?\u2026') + + +def mid_sentence(text: str, keys: set[str], fold_clitics: bool = False) -> tuple[dict[str, int], dict[str, int]]: + """(mid, total) capitalised occurrences per key. + + `mid` counts the ones whose preceding non-opener character is not a sentence + terminator -- i.e. the capital is the writer's choice and not the position's. + """ + mid, tot = collections.Counter(), collections.Counter() + for m in TOKEN.finditer(text): + t = m.group(0) + if fold_clitics: + t = CLITIC.sub("", t) or t + if not t[:1].isupper(): + continue + k = t.lower() + if k not in keys: + continue + tot[k] += 1 + i = m.start() - 1 + while i >= 0 and text[i] in _OPENERS: + i -= 1 + if i >= 0 and text[i] not in _TERM: + mid[k] += 1 + return mid, tot + + +#: A word carrying one of these in front of it is a name, whatever its position +#: statistics say. This is rename.py's title-first idea used as a RESCUE rather +#: than as a gender signal. +_HONORIFIC = (r"(?:Mr|Mrs|Ms|Miss|Dr|Doctor|Professor|Prof|Colonel|Col|Major|General|Gen|" + r"Captain|Capt|Lieutenant|Lt|Sergeant|Sgt|Cadet|Sir|Madam|Lady|Lord|King|Queen|" + r"Officer|Agent|Coach|Senator|Judge|Father|Mother|Aunt|Uncle)") + + +def rescue_signals(text: str, keys: set[str]) -> dict[str, tuple[int, int]]: + """key -> (honorific-preceded, possessive) counts. + + ⚠ The mid-sentence filter drops real SURNAMES that are only ever used as + address -- measured here, `Delgado` 18/64, `Schur` 0/10, `Rhee` 0/8, because + every occurrence is `“Mr. Delgado,”` opening a line of dialogue. Two signals + separate those from the interjections the filter is FOR: a title in front, + and a possessive. Measured on BabyYarros, all 19 verified interjections score + zero on both, and every wrongly-dropped surname scores on at least one. + """ + hon, poss = collections.Counter(), collections.Counter() + for m in re.finditer(_HONORIFIC + r"\.?\s+([A-ZÀ-Þ][A-Za-zà-ÿœæ\-]+)", text): + k = m.group(1).lower() + if k in keys: + hon[k] += 1 + for m in re.finditer(r"\b([A-ZÀ-Þ][A-Za-zà-ÿœæ\-]+)[’\']s\b", text): + k = m.group(1).lower() + if k in keys: + poss[k] += 1 + return {k: (hon[k], poss[k]) for k in keys} + + +ACRONYM = re.compile(r"[A-Z]{2,}s?$") + + +def ratio_rejects(text: str, min_count: int, max_ratio: float, fold_clitics: bool) -> dict[str, dict]: + """Candidates frequent enough to matter that the cap/lowercase ratio threw out. + + ⚠ The ratio assumes consistent typography and BabyYarros breaks that: `wilder` + renders an in-book news article entirely in lowercase, so `eleanor baxter` and + `ms. baxter` appear uncapitalised three times against 23 capitalised ones -- + ratio 0.13 against a 0.05 bar, and a real character is silently never renamed. + """ + cap, low = collections.Counter(), collections.Counter() + for m in TOKEN.finditer(text): + t = m.group(0) + if fold_clitics: + t = CLITIC.sub("", t) or t + (cap if t[:1].isupper() else low)[t.lower()] += 1 + return {k: {"cap": c, "lower": low[k], "ratio": round(low[k] / c, 4)} + for k, c in cap.items() if c >= min_count and low[k] / c > max_ratio} + + +#: ⚠ DELIBERATELY NARROWER than `_HONORIFIC`. The wide list is safe when both +#: sides must be capitalised; matched case-insensitively it readmitted 143 junk +#: tokens (`the`, `says`, `like`, `up`) because `major`, `general`, `father`, +#: `sir` and `agent` are ordinary words in lowercase prose. These five are never +#: anything but a title, and the lowercase arm additionally REQUIRES the period. +_ABBREV = re.compile(r"\b(?:Mr|Mrs|Ms|Dr|Mister|Miss)\b\.?\s+([A-ZÀ-Þ][A-Za-zà-ÿœæ\-]+)" + r"|\b(?:mr|mrs|ms|dr)\.\s+([a-zà-ÿœæ][a-zà-ÿœæ\-]+)") + + +def honorific_hits(text: str, keys: set[str]) -> dict[str, int]: + """`Miss Baxter` and `ms. baxter` both count; `I miss you` does not.""" + hits = collections.Counter() + for m in _ABBREV.finditer(text): + k = (m.group(1) or m.group(2)).lower() + if k in keys: + hits[k] += 1 + return hits + + +def surface_forms(text: str, keys: set[str], prefer_mixed: bool = False, + fold_clitics: bool = False) -> dict[str, str]: + """Dominant spelling per key. + + ⚠ `prefer_mixed` picks the most common NON-all-caps form when one exists. + Without it a name that happens to sit inside an all-caps passage -- an + in-world dispatch here, an inscription in Shirley -- gets `BRAEVICK` as its + surface, and every rule downstream then reasons about an acronym. + """ best = collections.defaultdict(collections.Counter) for m in TOKEN.finditer(text): t = m.group(0) + if fold_clitics: + t = CLITIC.sub("", t) or t if t[:1].isupper() and t.lower() in keys: best[t.lower()][t] += 1 - return {k: c.most_common(1)[0][0] for k, c in best.items()} + out = {} + for k, c in best.items(): + mixed = [(n, f) for f, n in c.most_common() if not ACRONYM.fullmatch(f)] + out[k] = (max(mixed)[1] if (prefer_mixed and mixed) else c.most_common(1)[0][0]) + return out def link_identities(text: str, names: set[str], min_pairs: int) -> list[tuple[str, str]]: @@ -149,21 +303,75 @@ def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("corpus") ap.add_argument("--out", default=None) + ap.add_argument("--stoplist", default=None, + help="JSON file whose every list value holds surfaces to exclude; " + "per-corpus real-world referents, see stoplist_yarros.json") + ap.add_argument("--rescue-honorific", type=int, default=0, + help="readmit a candidate the cap/lowercase ratio rejected when a title " + "precedes it at least this many times (0 = off)") + ap.add_argument("--fold-clitics", action="store_true", + help="count `Afendra’s` as `Afendra` so a possessive-only entity is " + "detected at all (it is otherwise silently unrenamed AND ungated)") + ap.add_argument("--drop-acronyms", action="store_true", + help="treat an ALWAYS-all-caps surface as an acronym, not a name " + "(RSC/ATV/TV/BMX/VIP), and prefer a mixed-case surface when one exists") ap.add_argument("--min-count", type=int, default=5) ap.add_argument("--max-ratio", type=float, default=0.05) ap.add_argument("--min-pairs", type=int, default=2) + ap.add_argument("--min-mid-ratio", type=float, default=0.0, + help="drop a candidate whose capitals are overwhelmingly sentence-initial " + "(0 = off, which reproduces the Bronte run)") + ap.add_argument("--min-mid", type=int, default=2, + help="absolute mid-sentence floor, so a 1-of-2 accident cannot qualify") ap.add_argument("--control", default="", help="comma-separated known-true names (positive control)") + ap.add_argument("--negative-control", default="", + help="comma-separated known-NON-names that the filter must DROP") a = ap.parse_args() corpus = Path(a.corpus) works = load(corpus) + stop = set(STOP) + if a.stoplist: + blob = json.loads(Path(a.stoplist).read_text()) + extra = {n for v in blob.values() if isinstance(v, list) for n in v} + stop |= extra + print(f" stoplist {a.stoplist}: +{len(extra)} real-world / generic surfaces") controls = [c.strip() for c in a.control.split(",") if c.strip()] + neg_controls = [c.strip() for c in a.negative_control.split(",") if c.strip()] report, failed_control = {}, [] + mid_dropped: dict[str, tuple[int, int]] = {} + rescued: dict[str, tuple[int, int]] = {} + ratio_rescued: dict[str, tuple[int, int, int]] = {} for slug, text in works.items(): - ents = detect(text, a.min_count, a.max_ratio) - keys = {k for k in ents if k.capitalize() not in STOP and k.title() not in STOP} - keys = {k for k in keys if k not in {s.lower() for s in STOP}} - forms = surface_forms(text, keys) + ents = detect(text, a.min_count, a.max_ratio, a.fold_clitics) + if a.rescue_honorific: + rej = ratio_rejects(text, a.min_count, a.max_ratio, a.fold_clitics) + hh = honorific_hits(text, set(rej)) + back = {k: rej[k] for k, n in hh.items() if n >= a.rescue_honorific} + for k, v in back.items(): + ents.setdefault(k, v) + ratio_rescued[k] = (hh[k], v["cap"], v["lower"]) + keys = {k for k in ents if k.capitalize() not in stop and k.title() not in stop} + keys = {k for k in keys if k not in {s.lower() for s in stop}} + # ⚠ An all-caps surface is an acronym, not a name: RSC, ATV, TV, BMX, VIP, + # CTDs. Tested on the DOMINANT surface form, because a name also appears + # inside an all-caps in-world dispatch and must not be lost to that. + if a.drop_acronyms: + forms0 = surface_forms(text, keys, prefer_mixed=True, fold_clitics=a.fold_clitics) + keys = {k for k in keys if not ACRONYM.fullmatch(forms0.get(k, k))} + if a.min_mid_ratio > 0: + mid, tot = mid_sentence(text, keys, a.fold_clitics) + dropped_here = {k for k in keys + if mid[k] < a.min_mid or mid[k] / max(tot[k], 1) < a.min_mid_ratio} + sig = rescue_signals(text, dropped_here) + rescued_here = {k for k in dropped_here if sum(sig.get(k, (0, 0))) > 0} + for k in rescued_here: + rescued[k] = sig[k] + dropped_here -= rescued_here + for k in dropped_here: + mid_dropped[k] = (mid[k], tot[k]) + keys -= dropped_here + forms = surface_forms(text, keys, prefer_mixed=a.drop_acronyms, fold_clitics=a.fold_clitics) links = link_identities(text, keys, a.min_pairs) gender = resolve_gender(text, keys) # identity linking propagates gender: a bare surname inherits from its given name @@ -180,6 +388,32 @@ def main() -> int: f"{sum(1 for k in keys if gender.get(k)):>3} gendered " f"{sum(1 for k in keys if not gender.get(k)):>4} ungendered") + if ratio_rescued: + print(f"\n ratio-rejected but title-preceded, readmitted: {len(ratio_rescued)}") + for k, (h, c, l) in sorted(ratio_rescued.items(), key=lambda kv: -kv[1][0]): + print(f" {k:<16} {h:>3} titled · {c:>4} cap / {l:>3} lower") + + if a.min_mid_ratio > 0: + print(f"\n mid-sentence filter (>= {a.min_mid} and >= {a.min_mid_ratio:.2f} of capitals): " + f"dropped {len(mid_dropped)} candidates") + for k, (m, t) in sorted(mid_dropped.items(), key=lambda kv: -kv[1][1])[:20]: + print(f" {k:<16} {m:>4} mid / {t:>4} caps") + if len(mid_dropped) > 20: + print(f" ... and {len(mid_dropped) - 20} more") + print(f" rescued by honorific/possessive: {len(rescued)}") + for k, (h, po) in sorted(rescued.items(), key=lambda kv: -sum(kv[1])): + print(f" {k:<16} {h:>3} titled · {po:>3} possessive") + + if neg_controls: + print("\n negative control -- these are NOT names and must be DROPPED:") + for name in neg_controls: + hits = [s for s, r in report.items() if name.lower() in r["entities"]] + ok = not hits + print(f" [{'PASS' if ok else 'FAIL'}] {name:<14} " + f"{'dropped' if ok else 'STILL AN ENTITY in ' + ', '.join(hits)}") + if not ok: + failed_control.append(f"{name} (negative)") + if controls: print("\n positive control -- names known to be real must be FOUND:") for name in controls: diff --git a/scripts/r49-corpus/leak_gate.py b/scripts/r49-corpus/leak_gate.py new file mode 100644 index 0000000..cc908ec --- /dev/null +++ b/scripts/r49-corpus/leak_gate.py @@ -0,0 +1,187 @@ +"""R49 Stage D3 gate — do any of the author's own proper nouns survive the rename? + +The rename exists so a voice adapter fits *prose style* and not the author's +characters and worldbuilding. That only holds if the renamed copies are actually +clean, and "actually clean" is a measurement, not a property of having run the +script. Brontë's run reached 0 of 203; BabyYarros opened at 86 of 232. + +The gate is a whole-corpus scan, not a per-work one, and that distinction is +load-bearing. A name detected in `iron-flame` but below threshold in `fourth-wing` +is renamed in one copy and printed verbatim in the other, and a per-work gate +reports that as clean. + +CONTROLS. A detector that only ever sees the renamed text cannot tell "absent" +from "blind", so this instrument runs both directions every time: + + * POSITIVE -- the same scan over the UNRENAMED source. Every entity must be + found there. A miss means the matcher is broken and its zeroes are worthless. + * NEGATIVE -- a nonce string that appears in neither tree. A hit means the + matcher is manufacturing signal. + +Exit code is the gate: 0 iff the controls pass AND no source entity survives. +""" +from __future__ import annotations +import argparse, json, re, sys +from collections import Counter, defaultdict +from pathlib import Path + +NONCE = "Qxzvwolfram" # negative control: appears in no corpus + + +def load_works(corpus: Path) -> dict[str, str]: + man = json.loads((corpus / "manifest.json").read_text()) + out = {} + for w in man["works"]: + rows = [json.loads(l) for l in + (corpus / w["path"]).read_text(encoding="utf-8").splitlines() if l.strip()] + out[w["slug"]] = "\n\n".join(r["text"] for r in rows) + return out + + +def load_copies(renamed: Path) -> dict[str, str]: + out = {} + for p in sorted((renamed / "copies").glob("*.jsonl")): + rows = [json.loads(l) for l in p.read_text(encoding="utf-8").splitlines() if l.strip()] + out[p.name] = "\n\n".join(r["text"] for r in rows) + return out + + +def scan(texts: dict[str, str], surfaces: list[str]) -> dict[str, dict[str, int]]: + """surface -> {text_name: hits}. One alternation pass per text, not one per name. + + ⚠ Longest-first alternation, so `Xaden Riorson` is consumed before `Xaden` + and a two-part name is not counted twice. + """ + if not surfaces: + return {} + pat = re.compile(r"\b(" + "|".join(re.escape(s) for s in + sorted(surfaces, key=len, reverse=True)) + r")\b") + hits: dict[str, dict[str, int]] = defaultdict(dict) + for name, text in texts.items(): + local: dict[str, int] = defaultdict(int) + for m in pat.finditer(text): + local[m.group(1)] += 1 + for s, n in local.items(): + hits[s][name] = n + return hits + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("corpus", help="source corpus dir (manifest.json + works/)") + ap.add_argument("--entities", required=True) + ap.add_argument("--renamed", required=True, help="rename.py --out dir") + ap.add_argument("--min-cap", type=int, default=8, + help="rename.py's renameable threshold; entities below it are " + "reported separately because rename never touched them") + ap.add_argument("--phrase-map", default=None, + help="the JSON rename.py used; its `allow` list names the phrases judged " + "real-world or generic. Without it the phrase audit does not run.") + ap.add_argument("--phrase-min", type=int, default=5, + help="a capitalised 2-3gram must recur this often in the source to be audited") + ap.add_argument("--report", default=None, help="write the full JSON breakdown here") + a = ap.parse_args() + + corpus, renamed = Path(a.corpus), Path(a.renamed) + ents_all = json.loads(Path(a.entities).read_text()) + source = load_works(corpus) + copies = load_copies(renamed) + if not copies: + print("== no copy files found -- nothing to gate"); return 1 + + # Mirror rename.py's own renameable predicate so the two cannot drift apart. + renameable, sub_threshold = {}, {} + for slug, w in ents_all.items(): + for key, e in w["entities"].items(): + surf = e.get("surface") or key + if "’" in key or "'" in key: + continue + (renameable if e["cap"] >= a.min_cap else sub_threshold).setdefault(surf, set()).add(slug) + + surfaces = sorted(set(renameable) | set(sub_threshold)) + print(f" {len(renameable)} renameable surfaces (cap >= {a.min_cap}) · " + f"{len(sub_threshold)} sub-threshold · {len(copies)} copy files") + + # ---- controls -------------------------------------------------------- + src_hits = scan(source, surfaces + [NONCE]) + missing = [s for s in surfaces if s not in src_hits] + pos_ok = not missing + neg_ok = NONCE not in src_hits + print(f" [{'PASS' if pos_ok else 'FAIL'}] positive control: every surface found in the " + f"unrenamed source ({len(surfaces) - len(missing)}/{len(surfaces)})" + + ("" if pos_ok else f" -- MISSING {missing[:10]}")) + print(f" [{'PASS' if neg_ok else 'FAIL'}] negative control: nonce `{NONCE}` absent from source") + + # ---- the measurement ------------------------------------------------- + copy_hits = scan(copies, surfaces + [NONCE]) + neg_ok = neg_ok and NONCE not in copy_hits + surv_renameable = {s: copy_hits[s] for s in renameable if s in copy_hits} + surv_sub = {s: copy_hits[s] for s in sub_threshold if s in copy_hits} + + print(f"\n SURVIVING renameable: {len(surv_renameable)} of {len(renameable)}") + for s, where in sorted(surv_renameable.items(), key=lambda kv: -sum(kv[1].values()))[:40]: + tot = sum(where.values()) + print(f" {s:<18} {tot:>6} hits across {len(where)} copies " + f"(detected in: {','.join(sorted(renameable[s]))})") + if len(surv_renameable) > 40: + print(f" ... and {len(surv_renameable) - 40} more") + print(f"\n SURVIVING sub-threshold (cap < {a.min_cap}, rename never saw them): " + f"{len(surv_sub)} of {len(sub_threshold)}") + for s, where in sorted(surv_sub.items(), key=lambda kv: -sum(kv[1].values()))[:15]: + print(f" {s:<18} {sum(where.values()):>6} hits") + + # ---- phrase audit ---------------------------------------------------- + # ⚠ The unigram scan above cannot see `Riders Quadrant` or `Fourth Wing`: + # every component is an ordinary word the detector correctly refuses. This + # pass is what caught them AFTER the unigram gate read 0 of 314. + surviving_phrases = {} + if a.phrase_map: + pm = json.loads(Path(a.phrase_map).read_text()) + allow = set(pm.get("allow", [])) + PH = re.compile(r"\b([A-Z][a-z]{2,}(?: [A-Z][a-z]{2,}){1,2})\b") + src_ph = Counter() + for t in source.values(): + src_ph.update(PH.findall(t)) + cop_ph = Counter() + for t in copies.values(): + cop_ph.update(PH.findall(t)) + # A heading word cannot start a leak: `Chapter Twenty` is the book's own + # scaffolding, not the author's invention. + STRUCT = ("Chapter", "Prologue", "Epilogue", "Part", "Appendix", "Volume", "Book") + audited = {p for p, n in src_ph.items() + if n >= a.phrase_min and not p.startswith(STRUCT)} - allow + surviving_phrases = {p: {"source": src_ph[p], "copies": cop_ph[p]} + for p in audited if cop_ph[p] > 0} + print(f"\n PHRASE AUDIT: {len(audited)} capitalised 2-3grams recur >= {a.phrase_min} " + f"times in the source ({len(allow)} allow-listed as real-world/generic)") + print(f" SURVIVING phrases: {len(surviving_phrases)}") + for ph, w in sorted(surviving_phrases.items(), key=lambda kv: -kv[1]["source"])[:30]: + print(f" {ph:<34} source {w['source']:>4} copies {w['copies']:>5}") + + if a.report: + Path(a.report).write_text(json.dumps({ + "renameable_total": len(renameable), "sub_threshold_total": len(sub_threshold), + "controls": {"positive_pass": pos_ok, "negative_pass": neg_ok, "missing": missing}, + "surviving_renameable": {s: {"hits": sum(w.values()), "copies": len(w), + "detected_in": sorted(renameable[s])} + for s, w in surv_renameable.items()}, + "surviving_sub_threshold": {s: {"hits": sum(w.values()), "copies": len(w)} + for s, w in surv_sub.items()}, + "surviving_phrases": surviving_phrases, + }, ensure_ascii=False, indent=2), encoding="utf-8") + print(f"\n wrote {a.report}") + + if not (pos_ok and neg_ok): + print("\n== CONTROLS FAILED -- this gate's verdict is not trustworthy"); return 2 + if surv_renameable or surv_sub or surviving_phrases: + print(f"\n== GATE FAILED: {len(surv_renameable) + len(surv_sub)} source entities and " + f"{len(surviving_phrases)} phrases survive"); return 1 + print("\n== GATE PASSED: 0 source entities and 0 audited phrases survive in any copy") + print(f" ⚠ sensitivity floor: a name appearing fewer than {a.min_cap} times per work is " + f"never detected, and a phrase recurring fewer than {a.phrase_min} times is never " + f"audited. Neither is renamed, and neither is reported here.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/r49-corpus/rename.py b/scripts/r49-corpus/rename.py index f4366c0..dd118be 100644 --- a/scripts/r49-corpus/rename.py +++ b/scripts/r49-corpus/rename.py @@ -98,6 +98,15 @@ def main() -> int: ap.add_argument("--holdout-chapter", type=int, default=10) ap.add_argument("--preset", default="bronte", choices=sorted(PRESETS), help="which corpus's name-pool register to draw from") + ap.add_argument("--scope", default="work", choices=("work", "corpus"), + help="`work` maps each work independently (reproduces the Bronte run); " + "`corpus` uses ONE map across every work in a copy") + ap.add_argument("--phrase-map", default=None, + help="JSON with `phrases` (multiword) and `tokens` (capitalised single " + "words) neutralising in-world compounds the unigram pass cannot reach") + ap.add_argument("--min-cap", type=int, default=8, + help="minimum capitalised count for an entity to be renamed; below it " + "the entity is left in the text verbatim") a = ap.parse_args() corpus = Path(a.corpus) @@ -137,6 +146,21 @@ def main() -> int: f"{lab} {len(pool[lab]['male'])}m/{len(pool[lab]['female'])}f/{len(pool[lab]['surname'])}s" for lab in (label_a, label_b))) + # ⚠ Applied AFTER the entity substitution, so it can never eat a replacement + # name. Multiword first and longest first; single tokens are case-SENSITIVE + # and whole-word, so a dragon's lowercase `wing` survives while `Fourth Wing` + # does not. + phrase_sub = None + if a.phrase_map: + pm = json.loads(Path(a.phrase_map).read_text()) + table = {**pm.get("phrases", {}), **pm.get("tokens", {})} + if table: + pat_p = re.compile(r"\b(" + "|".join(re.escape(k) for k in + sorted(table, key=len, reverse=True)) + r")\b") + phrase_sub = lambda t: pat_p.sub(lambda m: table[m.group(1)], t) + print(f" phrase map {a.phrase_map}: {len(pm.get('phrases', {}))} phrases + " + f"{len(pm.get('tokens', {}))} capitalised tokens") + out = Path(a.out); (out / "copies").mkdir(parents=True, exist_ok=True) stats = {"copies": a.copies, "seed": a.seed, "works": {}, "renamed": 0, "held": 0} @@ -154,7 +178,7 @@ def main() -> int: titled = set(tg) renameable, held = {}, [] for key, e in ents.items(): - if "’" in key or "'" in key or e["cap"] < 8: + if "’" in key or "'" in key or e["cap"] < a.min_cap: continue # possessives/contractions are not entities g = tg.get(key) or e.get("gender") if g: @@ -176,16 +200,47 @@ def main() -> int: stats["renamed"] += len(renameable); stats["held"] += len(held) print(f" {slug:<14} renamed {len(renameable):>3} ({len(renameable)-len(held)} gendered, {len(held)} neutral)") + # ⚠ CORPUS SCOPE. Per-work maps leak across works and this is measurable, not + # theoretical: `Rebel` is detected in `rebel` and renamed there, then printed + # verbatim in the two Renegades books where it sits below threshold. A + # whole-corpus gate catches it; a per-work one reports clean. It also fixes a + # thing the Bronte corpus never had to care about -- Yarros is TWO SERIES, so + # Violet has to be the same person in Fourth Wing and Iron Flame, and a + # per-work draw gives her two different names inside one copy. + if a.scope == "corpus": + merged: dict[str, dict] = {} + genders: dict[str, set] = collections.defaultdict(set) + for slug, plan in plans.items(): + for key, v in plan.items(): + merged.setdefault(key, {"surface": v["surface"], "kind": v["kind"], "gender": None}) + if v["gender"]: + genders[key].add(v["gender"]) + conflicts = 0 + for key, v in merged.items(): + g = genders.get(key, set()) + if len(g) == 1: + v["gender"] = next(iter(g)); v["kind"] = "given" + else: + if len(g) > 1: + conflicts += 1 + v["kind"] = "surname" # held -> neutral pool, still renamed + n_gendered = sum(1 for v in merged.values() if v["gender"]) + print(f" corpus scope: {len(merged)} distinct surfaces " + f"({n_gendered} gendered, {len(merged) - n_gendered} neutral), " + f"{conflicts} gender conflicts held") + plans = {slug: merged for slug in plans} + # ---- D3: N seeded copies, one consistent map per copy ------------------- emitted = 0 for c in range(a.copies): rng = random.Random(a.seed + c * 1000) + corpus_map, corpus_used = {}, set() for slug, rows in works.items(): # Share of pool A for this work. Brontë sets it per novel (Brussels # vs Yorkshire); Yarros uses one default, because the register does # not split by book the way hers does. share_a = cfg["share"].get(slug, cfg["default_share"]) - used = set() + used = corpus_used if a.scope == "corpus" else set() def draw(kind: str, gender: str | None) -> str: lang = label_a if rng.random() < share_a else label_b @@ -196,7 +251,12 @@ def main() -> int: used.add(n); return n return rng.choice(pool[lang][bucket]) - mapping = {k: draw(v["kind"], v["gender"]) for k, v in plans[slug].items()} + if a.scope == "corpus": + for k, v in plans[slug].items(): + corpus_map.setdefault(k, draw(v["kind"], v["gender"])) + mapping = corpus_map + else: + mapping = {k: draw(v["kind"], v["gender"]) for k, v in plans[slug].items()} pat = re.compile(r"\b(" + "|".join(sorted((re.escape(v["surface"]) for v in plans[slug].values()), key=len, reverse=True)) + r")\b") surf2key = {v["surface"]: k for k, v in plans[slug].items()} @@ -204,6 +264,8 @@ def main() -> int: with path.open("w", encoding="utf-8") as fh: for r in rows: txt = pat.sub(lambda m: mapping[surf2key[m.group(1)]], r["text"]) + if phrase_sub: + txt = phrase_sub(txt) split = "val" if r["chapter"] == a.holdout_chapter else "train" fh.write(json.dumps({"work": slug, "copy": c, "chapter": r["chapter"], "split": split, "text": txt}, ensure_ascii=False) + "\n") diff --git a/scripts/yarros-corpus/RUNBOOK.md b/scripts/yarros-corpus/RUNBOOK.md new file mode 100644 index 0000000..20594e5 --- /dev/null +++ b/scripts/yarros-corpus/RUNBOOK.md @@ -0,0 +1,72 @@ +# BabyYarros — corpus pipeline, end to end + +Every stage is reproducible from the Kvasir masters. Run them in order; each one +refuses to write if its own controls fail, so a silent bad stage is not a +failure mode here. + +`$R` is this repo's `scripts/` directory. Work happens on **nh3-dev** (it has the +Kvasir library); training happens on **pfi-gx10**. + +```bash +# D1 — build from the licensed masters +python3 $R/yarros-corpus/build_corpus_yarros.py --out ~/yarros-corpus + +# D1b — repair EPUB typography, strip back matter +# small-caps epigraphs, drop caps, acknowledgments/newsletter tails +python3 $R/yarros-corpus/repair_typography.py ~/yarros-corpus --out ~/yarros-corpus-r + +# D2 — entity map. Every flag below is OPT-IN and off by default so the Brontë +# run stays byte-reproducible; all five are needed for this corpus. +python3 $R/r49-corpus/entities.py ~/yarros-corpus-r \ + --out ~/yarros-corpus-r/entities.json \ + --min-count 3 --min-mid-ratio 0.35 --drop-acronyms --fold-clitics \ + --rescue-honorific 2 --stoplist $R/yarros-corpus/stoplist_yarros.json \ + --control "Violet,Xaden,Basgiath,Tairn,Leah,Landon,Brennan,Mira,Rhiannon,Dain,Jesinia,Bodhi,Garrick,Imogen,Sorrengail,Riorson,Delgado,Schur,Rhee,Messina,Masen,Violence,Montserrat,Barrens,Originals,Lilith,Nyra,Naolin,Afendra,Baxter" \ + --negative-control "Hey,Holy,Hopefully,Yep,Hi,Whoa,Nope,Ugh,Ouch,Okay,Yeah,Thankfully,Honestly,Seriously,Hmm,Jesus,Logically,Chapter,Article,TV,VIP,ATV,BMX,Google,Nepal,American,Pacific,Harvard,Sergeant,Majesty,YouTube,Colorado" + +# D2b — gender, resolved from the chapters a name does NOT narrate +python3 $R/yarros-corpus/pov_gender.py ~/yarros-corpus-r \ + --entities ~/yarros-corpus-r/entities.json --out ~/yarros-corpus-r/entities-pov.json \ + --control "Violet=f,Xaden=m,Leah=f,Landon=m,Rhiannon=f,Dain=m,Mira=f,Brennan=m,Imogen=f,Paxton=m,Rachel=f,Penna=f,Nick=m,Liam=m,Sloane=f,Bodhi=m,Garrick=m,Jesinia=f" + +# D3 — rename, ONE map per copy across the whole corpus +python3 $R/r49-corpus/rename.py ~/yarros-corpus-r \ + --entities ~/yarros-corpus-r/entities-pov.json \ + --dictionary ~/r49-prep/name_dictionary.json \ + --out ~/yarros-corpus-renamed-v2 --preset yarros --scope corpus --min-cap 3 \ + --copies 6 --seed 4919 --phrase-map $R/yarros-corpus/phrase_map_yarros.json + +# GATE — must pass before anything is trained +python3 $R/r49-corpus/leak_gate.py ~/yarros-corpus-r \ + --entities ~/yarros-corpus-r/entities-pov.json --renamed ~/yarros-corpus-renamed-v2 \ + --min-cap 3 --phrase-map $R/yarros-corpus/phrase_map_yarros.json \ + --report ~/yarros-corpus-renamed-v2/leak_gate_report.json +``` + +## Why each opt-in flag exists + +Each one was added because the gate caught something, and each is measured, not +assumed. All five default to OFF, and the Brontë entity map was re-derived after +every change and confirmed identical in keys, surfaces and every field. + +| flag | the defect it fixes | evidence | +|---|---|---| +| `--min-mid-ratio 0.35` | `Hey`, `Holy`, `Hopefully`, `Yep`, `Whoa`, `Nope`, `Ugh` were entities | 33 verified names sit at 0.567–0.985 mid-sentence, 19 verified interjections at 0.000–0.222 | +| `--drop-acronyms` | `TV`, `VIP`, `ATV`, `BMX`, `RSC` renamed to surnames | tested on the DOMINANT surface, so `Braevick` inside an all-caps dispatch is not lost | +| `--fold-clitics` | `Afendra` never appears unpossessed, so it keyed as `Afendra’s` — which rename AND the gate both skip | it survived every copy while the gate read 0 of 314 | +| `--rescue-honorific 2` | `Baxter` rejected at ratio 0.13 because an in-book news article is set all-lowercase | `ms. baxter` ×14; the wide honorific list matched case-insensitively readmitted 143 junk tokens, so the rescue list is 5 abbreviations and the lowercase arm requires the period | +| `--stoplist` | real-world referents renamed (`Google`, `Nepal`, `American`) | every surface read in context first — `Violence` is Xaden's nickname for Violet and would have been wrongly excluded by a guess | + +`--scope corpus` and `--phrase-map` are rename-side, same shape: + +- **`--scope corpus`** — per-work maps leak across works (`Rebel` renamed in `rebel`, + printed verbatim in the other two Renegades books) and give one character two + names inside a single copy. Yarros is two *series*; Brontë was four unrelated novels. +- **`--phrase-map`** — the unigram pass cannot reach `Riders Quadrant`, `Flame Section` + or `Fourth Wing`, the book's own title, because every component is an ordinary word. + +## Sensitivity floor + +⚠ The gate resolves leak down to **3 capitalised occurrences per work** for names +and **5 recurrences** for phrases. Below those it does not detect, does not +rename, and does not report. "0 survive" means zero above that floor. diff --git a/scripts/yarros-corpus/phrase_map_yarros.json b/scripts/yarros-corpus/phrase_map_yarros.json new file mode 100644 index 0000000..e115006 --- /dev/null +++ b/scripts/yarros-corpus/phrase_map_yarros.json @@ -0,0 +1,55 @@ +{ + "corpus": "BabyYarros", + "why": "The unigram rename cannot reach an in-world compound built out of ordinary words. `Riders Quadrant`, `Flame Section`, `War Games` and `Fourth Wing` -- the book's own title -- all survived a gate that read 0 of 324, because every component is a common noun the cap/lowercase ratio correctly refuses to call a name. This is the `Thornfield x 100` case one level up, and it needs a map rather than a detector: substituting a head noun is a choice about register, not a measurement.", + "how_applied": "rename.py runs this AFTER the entity substitution. Multiword keys first, longest first; then capitalised single tokens, whole-word and case-sensitive, so the lowercase noun (a dragon's `wing`, a `squad` of cadets) is untouched.", + "phrases": { + "Silver One": "Argent One", + "First Six": "First Founders", + "Great War": "Long War", + "Unedited History": "Unabridged Record", + "Recovered Correspondence": "Retrieved Letters", + "The Fables": "The Legends", + "The Journal": "The Ledger", + "Field Guide": "Field Primer", + "Dreamless Sleep": "Endless Sleep", + "Conscription Day": "Levy Day" + }, + "tokens": { + "Quadrant": "Division", + "Quadrants": "Divisions", + "Wing": "Flight", + "Wings": "Flights", + "Section": "Cohort", + "Sections": "Cohorts", + "Squad": "Unit", + "Squads": "Units", + "Games": "Trials", + "Daggertail": "Spinecrest", + "Daggertails": "Spinecrests", + "Swordtail": "Bladecrest", + "Swordtails": "Bladecrests" + }, + "allow": [ + "Thank God", + "Abu Dhabi", + "Sri Lanka", + "Los Angeles", + "Machu Picchu", + "Taj Mahal", + "Buenos Aires", + "World Religion", + "High Roller", + "Hong Kong", + "Oak Moss", + "Moss Grove", + "Oak Moss Grove", + "Fox Motocross", + "Nitro Circus", + "Red Bull", + "Unauthorized Edition", + "Battle Brief", + "Pacific Ocean", + "Las Vegas" + ], + "allow_why": "Real-world referents and generic English that any novelist could write. `Battle Brief` and `Unauthorized Edition` stay because their distinctive halves -- the class and the in-world author -- are already renamed by the unigram pass, leaving ordinary words behind." +} diff --git a/scripts/yarros-corpus/repair_typography.py b/scripts/yarros-corpus/repair_typography.py new file mode 100644 index 0000000..746aacb --- /dev/null +++ b/scripts/yarros-corpus/repair_typography.py @@ -0,0 +1,192 @@ +"""BabyYarros D1b — repair two EPUB typography defects the D1 build carried through. + +The Brontë corpus needed an unwrap because it was hard-wrapped; this one does not, +and the D1 notes say so correctly. It has a DIFFERENT defect, and it was found by +the leak gate rather than by reading: the detector kept returning entities called +`IDERS`, `UADRANT`, `NAUTHORIZED`, `DITION`, and 17 bare single letters. + + 1. SMALL-CAPS EPIGRAPHS (fourth-wing + iron-flame, 106 lines, ~700 splits). + The Empyrean books open each chapter with an in-world citation set in small + caps. The extractor rendered the small-caps run as uppercase and left the + large initial as its own token: + + — M AJOR A FENDRA’S G UIDE TO THE R IDERS Q UADRANT (U NAUTHORIZED E DITION ) + + A split initial plus an uppercased run is exactly enough to recover the + original mixed case: a word WITH a split initial was capitalised in the + source (`M`+`AJOR` -> `Major`), and an all-caps word WITHOUT one was + lowercase (`TO THE` -> `to the`). So the line restores to + + —Major Afendra’s Guide to the Riders Quadrant (Unauthorized Edition) + + ⚠ The restoration is applied ONLY to lines carrying at least two splits. + One split is an ordinary sentence next to an acronym; two is a run. + + 2. DROP CAPS (52 occurrences, 51 of them iron-flame): `T he flight field`, + `X aden.`, `R evolution tastes`. Same cause, one letter instead of a run. + ⚠ `I`, `A` and `O` are EXCLUDED from the join because they are real + single-letter words -- `A slow smile spreads` is not a drop cap, and + joining it would invent `Aslow`. + +Both defects cost three ways: they manufacture entities the rename then scatters +through the corpus, they spend tokens on fragments, and they teach the adapter a +typography the author never wrote. + +The original corpus is left untouched so the D1 build stays reproducible; this +writes a repaired tree beside it, the same way `unwrap_corpus.py` did for Brontë. +""" +from __future__ import annotations +import argparse, json, re, shutil, sys +from pathlib import Path + +#: Back matter rides inside the LAST chapter, because the builder splits on +#: chapter headings and nothing follows the final one. Measured: 620-1,279 words +#: per work of acknowledgments, newsletter pitches and cover-artist credits -- +#: not the author's prose, and carrying the names of real people (her agent, her +#: editors, her children) straight into a corpus whose whole point is that no +#: identifiable name survives. +BACKMATTER = re.compile( + r"(?im)^[ \t]*(?:ACKNOWLEDGE?MENTS?|About the Author|Also by\b|Discover more\b|" + r"Don[’']t miss more books\b|Join the Entangled\b|Sign up for our newsletter\b|" + r"Keep reading for\b|Turn the page for\b)") + +SPLIT = re.compile(r"\b([A-Z]) ([A-Z]{2,})\b") +DROPCAP = re.compile(r"^([B-HJ-NP-Z]) ([a-z]{2,})") +ALLCAPS = re.compile(r"\b([A-Z]{2,})\b") +#: All-caps tokens that are genuinely acronyms rather than small-caps lowercase. +#: Kept uppercase when a small-caps line is restored. +ACRONYMS = {"RSC", "PTSD", "OK", "IV", "II", "III", "IV", "VI", "VII", "VIII", "IX", "XI"} + + +def restore_smallcaps(line: str) -> str: + """Two or more split initials means the whole line was a small-caps run.""" + if len(SPLIT.findall(line)) < 2: + return line + prev = None + while prev != line: # `A FENDRA’S` can chain with its neighbour + prev = line + line = SPLIT.sub(lambda m: m.group(1) + m.group(2).lower(), line) + # Whatever is still all-caps had no large initial, so it was lowercase. + line = ALLCAPS.sub(lambda m: m.group(1) if m.group(1) in ACRONYMS else m.group(1).lower(), line) + # ⚠ A possessive survives both passes: `A FENDRA’S` splits as `A`+`FENDRA`, + # so the run’s trailing `’S` is a lone capital that neither rule sees. + line = re.sub(r"([’'])S\b", r"\1s", line) + return re.sub(r"\(\s+", "(", re.sub(r"\s+\)", ")", line)) + + +def repair_text(text: str, counts: dict) -> str: + out = [] + for line in text.split("\n"): + before = line + line = restore_smallcaps(line) + if line != before: + counts["smallcap_lines"] += 1 + counts["smallcap_joins"] += len(SPLIT.findall(before)) + before2 = line + line = DROPCAP.sub(lambda m: m.group(1) + m.group(2), line.lstrip()) \ + if DROPCAP.match(line.lstrip()) else line + if line != before2: + counts["dropcap_joins"] += 1 + out.append(line) + return "\n".join(out) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("corpus") + ap.add_argument("--out", required=True) + a = ap.parse_args() + src, dst = Path(a.corpus), Path(a.out) + man = json.loads((src / "manifest.json").read_text()) + (dst / "works").mkdir(parents=True, exist_ok=True) + + counts = {"smallcap_lines": 0, "smallcap_joins": 0, "dropcap_joins": 0, "backmatter_words": 0} + samples, words_before, words_after, unchanged = [], 0, 0, 0 + for w in man["works"]: + rows = [json.loads(l) for l in (src / w["path"]).read_text(encoding="utf-8").splitlines() if l.strip()] + per = dict(counts) + # ⚠ Last chapter only. An earlier chapter that happens to contain the word + # `Acknowledgments` in dialogue must not be truncated. + m = BACKMATTER.search(rows[-1]["text"]) + if m: + cut = rows[-1]["text"][m.start():] + counts["backmatter_words"] += len(cut.split()) + rows[-1]["text"] = rows[-1]["text"][:m.start()].rstrip() + print(f" {w['slug']:14} back matter stripped at {m.group(0).strip()!r}: " + f"{len(cut.split()):,} words") + with (dst / w["path"]).open("w", encoding="utf-8") as fh: + for r in rows: + t0 = r["text"] + t1 = repair_text(t0, counts) + words_before += len(t0.split()); words_after += len(t1.split()) + if t0 == t1: + unchanged += 1 + elif len(samples) < 6: + for l0, l1 in zip(t0.split("\n"), t1.split("\n")): + if l0 != l1 and len(samples) < 6: + samples.append((w["slug"], l0[:110], l1[:110])) + r["text"] = t1; r["words"] = len(t1.split()) + fh.write(json.dumps(r, ensure_ascii=False) + "\n") + d = {k: counts[k] - per[k] for k in counts} + print(f" {w['slug']:14} smallcap lines {d['smallcap_lines']:>4} " + f"(joins {d['smallcap_joins']:>4}) dropcap joins {d['dropcap_joins']:>4}") + + print(f"\n chapters unchanged: {unchanged} of {sum(w['chapters'] for w in man['works'])}") + print(f" words {words_before:,} -> {words_after:,} " + f"({words_before - words_after:,} fragments rejoined)") + print("\n sample repairs:") + for slug, a0, a1 in samples: + print(f" {slug}\n - {a0}\n + {a1}") + + # ---- acceptance: the defect must be GONE and the join must not have run wild + joined = "\n".join((dst / w["path"]).read_text(encoding="utf-8") for w in man["works"]) + fails = [] + for must_not in ("R IDERS Q UADRANT", "T he flight field", "U NAUTHORIZED"): + if must_not in joined: + fails.append(f"still present: {must_not!r}") + if "Louise Fury" in joined: + fails.append("back matter survived: the author's agent is still named in the corpus") + if counts["backmatter_words"] > 0.02 * words_before: + fails.append(f"back-matter strip removed {counts['backmatter_words']:,} words, over 2% " + f"of the corpus -- a marker probably matched inside the prose") + for must in ("Riders Quadrant", "The flight field"): + if must not in joined: + fails.append(f"repair did not produce: {must!r}") + # negative control: a line with a single split is NOT a small-caps run + probe = "He got an A GRADE for it." + if restore_smallcaps(probe) != probe: + fails.append("single-split line was rewritten -- the >=2 guard is not holding") + # ⚠ Scoped to RESTORED lines only. The corpus also contains a genuinely + # all-caps in-world dispatch (`...BRAEVICK’S GRYPHON FLEET...`) that carries + # no split initials, so the restore never touches it and it is not a defect. + probe = "— M AJOR A FENDRA’S G UIDE TO THE R IDERS Q UADRANT" + if re.search(r"[’']S\b", restore_smallcaps(probe)): + fails.append("a restored small-caps line still carries an uppercase possessive `’S`") + probe2 = "A slow smile spreads across her face." + if DROPCAP.match(probe2): + fails.append("dropcap join would fire on the article `A`") + + for w in man["works"]: + pass + shutil.copy(src / "manifest.json", dst / "manifest.json") + shutil.copy(src / "corpus_alphabet.json", dst / "corpus_alphabet.json") + m2 = json.loads((dst / "manifest.json").read_text()) + for w in m2["works"]: + w["words"] = sum(json.loads(l)["words"] for l in (dst / w["path"]).read_text(encoding="utf-8").splitlines() if l.strip()) + m2["totals"]["words"] = m2["total_words"] = sum(w["words"] for w in m2["works"]) + m2["repaired_from"] = str(src) + m2["repair"] = counts + (dst / "manifest.json").write_text(json.dumps(m2, indent=2, ensure_ascii=False), encoding="utf-8") + + print() + for f in fails: + print(f" [FAIL] {f}") + if fails: + print("\n== REPAIR REJECTED"); return 1 + print(" [PASS] known-broken strings gone, repaired forms present, guards hold") + print(f"\n wrote {dst}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/yarros-corpus/stoplist_yarros.json b/scripts/yarros-corpus/stoplist_yarros.json new file mode 100644 index 0000000..942f738 --- /dev/null +++ b/scripts/yarros-corpus/stoplist_yarros.json @@ -0,0 +1,89 @@ +{ + "corpus": "BabyYarros", + "why": "The rename exists to keep the author's INVENTIONS out of the adapter. A real-world referent any novelist could use is not hers, and renaming it damages prose for no leak benefit. This is the same call Brontë's STOP_COMMON made for London/Paris/Brussels/Yorkshire, written per corpus because the register is per corpus.", + "how_derived": "Every capitalised surface the detector returned was read IN CONTEXT before landing here. That pass is why the list is short: `Violence` is Xaden's nickname for Violet, `Continent` and `Presentation` and `Battle Brief` and `Curator` and `Sage` and `Barrens` and `Originals` and `Montserrat` and `Athena` and `Aura` are all in-world, and every one of them would have been excluded by a plausible-looking guess. Ambiguous cases are deliberately NOT here: renaming is the safe direction, leaving is the leaking one.", + "real_world_geography": [ + "Nepal", + "Vegas", + "Las", + "Cuba", + "Miami", + "Dubai", + "Barcelona", + "Istanbul", + "Pacific", + "Everest", + "Korea", + "Angeles", + "Los", + "Fiji", + "Havana", + "Madagascar", + "Machu", + "Picchu", + "Venezuela", + "Himalayas", + "Taj", + "Mahal", + "Jakarta", + "Seoul", + "Chile", + "California", + "Peru", + "Morocco", + "America", + "Sri", + "Lanka", + "Abu", + "Dhabi", + "Buenos", + "Aires", + "Mykonos", + "Dartmouth", + "Colorado", + "India", + "Kathmandu", + "Hong", + "Kong", + "Lukla", + "Tahoe", + "Lima", + "Trenton" + ], + "real_world_institutions_and_brands": [ + "Harvard", + "UCLA", + "Google", + "Jeep", + "Marines", + "YouTube", + "GoPro", + "Bellagio", + "Fox" + ], + "real_world_nationalities": [ + "American", + "Cuban", + "Korean", + "Indian" + ], + "real_world_culture": [ + "Thanksgiving", + "Sherpas", + "Casanova" + ], + "generic_words_capitalised_only_in_in-world_titles": [ + "Guide", + "Edition", + "Unauthorized", + "Religion", + "Saturdays", + "Majesty", + "Property", + "Unedited", + "Secondhand", + "Excelling", + "T-shirt" + ], + "min_count_note": "Detection runs at --min-count 3 for this corpus, not the default 5. Measured: dropping to 3 recovers Lilith, Nyra, Naolin, Tirvainne, Dajalair, Kiralair, Morraine, Beinhaven, Codagh and Tairneanach -- all author inventions that the 5 floor left in the text verbatim. The cost is more real-world nouns reaching the pool, which is what this file absorbs. ⚠ The gate's SENSITIVITY FLOOR is this number: a name appearing fewer than 3 times is never detected, so it is neither renamed nor reported." +}