"""BabyYarros D1b — repair two EPUB typography defects the D1 build carried through. The Brontë corpus needed an unwrap because it was hard-wrapped; this one does not, and the D1 notes say so correctly. It has a DIFFERENT defect, and it was found by the leak gate rather than by reading: the detector kept returning entities called `IDERS`, `UADRANT`, `NAUTHORIZED`, `DITION`, and 17 bare single letters. 1. SMALL-CAPS EPIGRAPHS (fourth-wing + iron-flame, 106 lines, ~700 splits). The Empyrean books open each chapter with an in-world citation set in small caps. The extractor rendered the small-caps run as uppercase and left the large initial as its own token: — M AJOR A FENDRA’S G UIDE TO THE R IDERS Q UADRANT (U NAUTHORIZED E DITION ) A split initial plus an uppercased run is exactly enough to recover the original mixed case: a word WITH a split initial was capitalised in the source (`M`+`AJOR` -> `Major`), and an all-caps word WITHOUT one was lowercase (`TO THE` -> `to the`). So the line restores to —Major Afendra’s Guide to the Riders Quadrant (Unauthorized Edition) ⚠ The restoration is applied ONLY to lines carrying at least two splits. One split is an ordinary sentence next to an acronym; two is a run. 2. DROP CAPS (52 occurrences, 51 of them iron-flame): `T he flight field`, `X aden.`, `R evolution tastes`. Same cause, one letter instead of a run. ⚠ `I`, `A` and `O` are EXCLUDED from the join because they are real single-letter words -- `A slow smile spreads` is not a drop cap, and joining it would invent `Aslow`. Both defects cost three ways: they manufacture entities the rename then scatters through the corpus, they spend tokens on fragments, and they teach the adapter a typography the author never wrote. The original corpus is left untouched so the D1 build stays reproducible; this writes a repaired tree beside it, the same way `unwrap_corpus.py` did for Brontë. """ from __future__ import annotations import argparse, json, re, shutil, sys from pathlib import Path #: Back matter rides inside the LAST chapter, because the builder splits on #: chapter headings and nothing follows the final one. Measured: 620-1,279 words #: per work of acknowledgments, newsletter pitches and cover-artist credits -- #: not the author's prose, and carrying the names of real people (her agent, her #: editors, her children) straight into a corpus whose whole point is that no #: identifiable name survives. BACKMATTER = re.compile( r"(?im)^[ \t]*(?:ACKNOWLEDGE?MENTS?|About the Author|Also by\b|Discover more\b|" r"Don[’']t miss more books\b|Join the Entangled\b|Sign up for our newsletter\b|" r"Keep reading for\b|Turn the page for\b)") SPLIT = re.compile(r"\b([A-Z]) ([A-Z]{2,})\b") DROPCAP = re.compile(r"^([B-HJ-NP-Z]) ([a-z]{2,})") ALLCAPS = re.compile(r"\b([A-Z]{2,})\b") #: All-caps tokens that are genuinely acronyms rather than small-caps lowercase. #: Kept uppercase when a small-caps line is restored. ACRONYMS = {"RSC", "PTSD", "OK", "IV", "II", "III", "IV", "VI", "VII", "VIII", "IX", "XI"} def restore_smallcaps(line: str) -> str: """Two or more split initials means the whole line was a small-caps run.""" if len(SPLIT.findall(line)) < 2: return line prev = None while prev != line: # `A FENDRA’S` can chain with its neighbour prev = line line = SPLIT.sub(lambda m: m.group(1) + m.group(2).lower(), line) # Whatever is still all-caps had no large initial, so it was lowercase. line = ALLCAPS.sub(lambda m: m.group(1) if m.group(1) in ACRONYMS else m.group(1).lower(), line) # ⚠ A possessive survives both passes: `A FENDRA’S` splits as `A`+`FENDRA`, # so the run’s trailing `’S` is a lone capital that neither rule sees. line = re.sub(r"([’'])S\b", r"\1s", line) return re.sub(r"\(\s+", "(", re.sub(r"\s+\)", ")", line)) def repair_text(text: str, counts: dict) -> str: out = [] for line in text.split("\n"): before = line line = restore_smallcaps(line) if line != before: counts["smallcap_lines"] += 1 counts["smallcap_joins"] += len(SPLIT.findall(before)) before2 = line line = DROPCAP.sub(lambda m: m.group(1) + m.group(2), line.lstrip()) \ if DROPCAP.match(line.lstrip()) else line if line != before2: counts["dropcap_joins"] += 1 out.append(line) return "\n".join(out) def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("corpus") ap.add_argument("--out", required=True) a = ap.parse_args() src, dst = Path(a.corpus), Path(a.out) man = json.loads((src / "manifest.json").read_text()) (dst / "works").mkdir(parents=True, exist_ok=True) counts = {"smallcap_lines": 0, "smallcap_joins": 0, "dropcap_joins": 0, "backmatter_words": 0} samples, words_before, words_after, unchanged = [], 0, 0, 0 for w in man["works"]: rows = [json.loads(l) for l in (src / w["path"]).read_text(encoding="utf-8").splitlines() if l.strip()] per = dict(counts) # ⚠ Last chapter only. An earlier chapter that happens to contain the word # `Acknowledgments` in dialogue must not be truncated. m = BACKMATTER.search(rows[-1]["text"]) if m: cut = rows[-1]["text"][m.start():] counts["backmatter_words"] += len(cut.split()) rows[-1]["text"] = rows[-1]["text"][:m.start()].rstrip() print(f" {w['slug']:14} back matter stripped at {m.group(0).strip()!r}: " f"{len(cut.split()):,} words") with (dst / w["path"]).open("w", encoding="utf-8") as fh: for r in rows: t0 = r["text"] t1 = repair_text(t0, counts) words_before += len(t0.split()); words_after += len(t1.split()) if t0 == t1: unchanged += 1 elif len(samples) < 6: for l0, l1 in zip(t0.split("\n"), t1.split("\n")): if l0 != l1 and len(samples) < 6: samples.append((w["slug"], l0[:110], l1[:110])) r["text"] = t1; r["words"] = len(t1.split()) fh.write(json.dumps(r, ensure_ascii=False) + "\n") d = {k: counts[k] - per[k] for k in counts} print(f" {w['slug']:14} smallcap lines {d['smallcap_lines']:>4} " f"(joins {d['smallcap_joins']:>4}) dropcap joins {d['dropcap_joins']:>4}") print(f"\n chapters unchanged: {unchanged} of {sum(w['chapters'] for w in man['works'])}") print(f" words {words_before:,} -> {words_after:,} " f"({words_before - words_after:,} fragments rejoined)") print("\n sample repairs:") for slug, a0, a1 in samples: print(f" {slug}\n - {a0}\n + {a1}") # ---- acceptance: the defect must be GONE and the join must not have run wild joined = "\n".join((dst / w["path"]).read_text(encoding="utf-8") for w in man["works"]) fails = [] for must_not in ("R IDERS Q UADRANT", "T he flight field", "U NAUTHORIZED"): if must_not in joined: fails.append(f"still present: {must_not!r}") if "Louise Fury" in joined: fails.append("back matter survived: the author's agent is still named in the corpus") if counts["backmatter_words"] > 0.02 * words_before: fails.append(f"back-matter strip removed {counts['backmatter_words']:,} words, over 2% " f"of the corpus -- a marker probably matched inside the prose") for must in ("Riders Quadrant", "The flight field"): if must not in joined: fails.append(f"repair did not produce: {must!r}") # negative control: a line with a single split is NOT a small-caps run probe = "He got an A GRADE for it." if restore_smallcaps(probe) != probe: fails.append("single-split line was rewritten -- the >=2 guard is not holding") # ⚠ Scoped to RESTORED lines only. The corpus also contains a genuinely # all-caps in-world dispatch (`...BRAEVICK’S GRYPHON FLEET...`) that carries # no split initials, so the restore never touches it and it is not a defect. probe = "— M AJOR A FENDRA’S G UIDE TO THE R IDERS Q UADRANT" if re.search(r"[’']S\b", restore_smallcaps(probe)): fails.append("a restored small-caps line still carries an uppercase possessive `’S`") probe2 = "A slow smile spreads across her face." if DROPCAP.match(probe2): fails.append("dropcap join would fire on the article `A`") for w in man["works"]: pass shutil.copy(src / "manifest.json", dst / "manifest.json") shutil.copy(src / "corpus_alphabet.json", dst / "corpus_alphabet.json") m2 = json.loads((dst / "manifest.json").read_text()) for w in m2["works"]: w["words"] = sum(json.loads(l)["words"] for l in (dst / w["path"]).read_text(encoding="utf-8").splitlines() if l.strip()) m2["totals"]["words"] = m2["total_words"] = sum(w["words"] for w in m2["works"]) m2["repaired_from"] = str(src) m2["repair"] = counts (dst / "manifest.json").write_text(json.dumps(m2, indent=2, ensure_ascii=False), encoding="utf-8") print() for f in fails: print(f" [FAIL] {f}") if fails: print("\n== REPAIR REJECTED"); return 1 print(" [PASS] known-broken strings gone, repaired forms present, guards hold") print(f"\n wrote {dst}") return 0 if __name__ == "__main__": sys.exit(main())