esh-pfi-infrastructure/scripts/yarros-corpus/repair_typography.py

"""BabyYarros D1b — repair two EPUB typography defects the D1 build carried through.

The Brontë corpus needed an unwrap because it was hard-wrapped; this one does not,
and the D1 notes say so correctly. It has a DIFFERENT defect, and it was found by
the leak gate rather than by reading: the detector kept returning entities called
`IDERS`, `UADRANT`, `NAUTHORIZED`, `DITION`, and 17 bare single letters.

  1. SMALL-CAPS EPIGRAPHS (fourth-wing + iron-flame, 106 lines, ~700 splits).
     The Empyrean books open each chapter with an in-world citation set in small
     caps. The extractor rendered the small-caps run as uppercase and left the
     large initial as its own token:

         — M AJOR A FENDRA’S G UIDE TO THE R IDERS Q UADRANT (U NAUTHORIZED E DITION )

     A split initial plus an uppercased run is exactly enough to recover the
     original mixed case: a word WITH a split initial was capitalised in the
     source (`M`+`AJOR` -> `Major`), and an all-caps word WITHOUT one was
     lowercase (`TO THE` -> `to the`). So the line restores to

         —Major Afendra’s Guide to the Riders Quadrant (Unauthorized Edition)

     ⚠ The restoration is applied ONLY to lines carrying at least two splits.
     One split is an ordinary sentence next to an acronym; two is a run.

  2. DROP CAPS (52 occurrences, 51 of them iron-flame): `T he flight field`,
     `X aden.`, `R evolution tastes`. Same cause, one letter instead of a run.
     ⚠ `I`, `A` and `O` are EXCLUDED from the join because they are real
     single-letter words -- `A slow smile spreads` is not a drop cap, and
     joining it would invent `Aslow`.

Both defects cost three ways: they manufacture entities the rename then scatters
through the corpus, they spend tokens on fragments, and they teach the adapter a
typography the author never wrote.

The original corpus is left untouched so the D1 build stays reproducible; this
writes a repaired tree beside it, the same way `unwrap_corpus.py` did for Brontë.
"""
from __future__ import annotations
import argparse, json, re, shutil, sys
from pathlib import Path

#: Back matter rides inside the LAST chapter, because the builder splits on
#: chapter headings and nothing follows the final one. Measured: 620-1,279 words
#: per work of acknowledgments, newsletter pitches and cover-artist credits --
#: not the author's prose, and carrying the names of real people (her agent, her
#: editors, her children) straight into a corpus whose whole point is that no
#: identifiable name survives.
BACKMATTER = re.compile(
    r"(?im)^[ \t]*(?:ACKNOWLEDGE?MENTS?|About the Author|Also by\b|Discover more\b|"
    r"Don[’']t miss more books\b|Join the Entangled\b|Sign up for our newsletter\b|"
    r"Keep reading for\b|Turn the page for\b)")

SPLIT = re.compile(r"\b([A-Z]) ([A-Z]{2,})\b")
DROPCAP = re.compile(r"^([B-HJ-NP-Z]) ([a-z]{2,})")
ALLCAPS = re.compile(r"\b([A-Z]{2,})\b")
#: All-caps tokens that are genuinely acronyms rather than small-caps lowercase.
#: Kept uppercase when a small-caps line is restored.
ACRONYMS = {"RSC", "PTSD", "OK", "IV", "II", "III", "IV", "VI", "VII", "VIII", "IX", "XI"}


def restore_smallcaps(line: str) -> str:
    """Two or more split initials means the whole line was a small-caps run."""
    if len(SPLIT.findall(line)) < 2:
        return line
    prev = None
    while prev != line:                    # `A FENDRA’S` can chain with its neighbour
        prev = line
        line = SPLIT.sub(lambda m: m.group(1) + m.group(2).lower(), line)
    # Whatever is still all-caps had no large initial, so it was lowercase.
    line = ALLCAPS.sub(lambda m: m.group(1) if m.group(1) in ACRONYMS else m.group(1).lower(), line)
    # ⚠ A possessive survives both passes: `A FENDRA’S` splits as `A`+`FENDRA`,
    # so the run’s trailing `’S` is a lone capital that neither rule sees.
    line = re.sub(r"([’'])S\b", r"\1s", line)
    return re.sub(r"\(\s+", "(", re.sub(r"\s+\)", ")", line))


def repair_text(text: str, counts: dict) -> str:
    out = []
    for line in text.split("\n"):
        before = line
        line = restore_smallcaps(line)
        if line != before:
            counts["smallcap_lines"] += 1
            counts["smallcap_joins"] += len(SPLIT.findall(before))
        before2 = line
        line = DROPCAP.sub(lambda m: m.group(1) + m.group(2), line.lstrip()) \
            if DROPCAP.match(line.lstrip()) else line
        if line != before2:
            counts["dropcap_joins"] += 1
        out.append(line)
    return "\n".join(out)


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("corpus")
    ap.add_argument("--out", required=True)
    a = ap.parse_args()
    src, dst = Path(a.corpus), Path(a.out)
    man = json.loads((src / "manifest.json").read_text())
    (dst / "works").mkdir(parents=True, exist_ok=True)

    counts = {"smallcap_lines": 0, "smallcap_joins": 0, "dropcap_joins": 0, "backmatter_words": 0}
    samples, words_before, words_after, unchanged = [], 0, 0, 0
    for w in man["works"]:
        rows = [json.loads(l) for l in (src / w["path"]).read_text(encoding="utf-8").splitlines() if l.strip()]
        per = dict(counts)
        # ⚠ Last chapter only. An earlier chapter that happens to contain the word
        # `Acknowledgments` in dialogue must not be truncated.
        m = BACKMATTER.search(rows[-1]["text"])
        if m:
            cut = rows[-1]["text"][m.start():]
            counts["backmatter_words"] += len(cut.split())
            rows[-1]["text"] = rows[-1]["text"][:m.start()].rstrip()
            print(f"  {w['slug']:14} back matter stripped at {m.group(0).strip()!r}: "
                  f"{len(cut.split()):,} words")
        with (dst / w["path"]).open("w", encoding="utf-8") as fh:
            for r in rows:
                t0 = r["text"]
                t1 = repair_text(t0, counts)
                words_before += len(t0.split()); words_after += len(t1.split())
                if t0 == t1:
                    unchanged += 1
                elif len(samples) < 6:
                    for l0, l1 in zip(t0.split("\n"), t1.split("\n")):
                        if l0 != l1 and len(samples) < 6:
                            samples.append((w["slug"], l0[:110], l1[:110]))
                r["text"] = t1; r["words"] = len(t1.split())
                fh.write(json.dumps(r, ensure_ascii=False) + "\n")
        d = {k: counts[k] - per[k] for k in counts}
        print(f"  {w['slug']:14} smallcap lines {d['smallcap_lines']:>4} "
              f"(joins {d['smallcap_joins']:>4})  dropcap joins {d['dropcap_joins']:>4}")

    print(f"\n  chapters unchanged: {unchanged} of {sum(w['chapters'] for w in man['works'])}")
    print(f"  words {words_before:,} -> {words_after:,} "
          f"({words_before - words_after:,} fragments rejoined)")
    print("\n  sample repairs:")
    for slug, a0, a1 in samples:
        print(f"    {slug}\n      -  {a0}\n      +  {a1}")

    # ---- acceptance: the defect must be GONE and the join must not have run wild
    joined = "\n".join((dst / w["path"]).read_text(encoding="utf-8") for w in man["works"])
    fails = []
    for must_not in ("R IDERS Q UADRANT", "T he flight field", "U NAUTHORIZED"):
        if must_not in joined:
            fails.append(f"still present: {must_not!r}")
    if "Louise Fury" in joined:
        fails.append("back matter survived: the author's agent is still named in the corpus")
    if counts["backmatter_words"] > 0.02 * words_before:
        fails.append(f"back-matter strip removed {counts['backmatter_words']:,} words, over 2% "
                     f"of the corpus -- a marker probably matched inside the prose")
    for must in ("Riders Quadrant", "The flight field"):
        if must not in joined:
            fails.append(f"repair did not produce: {must!r}")
    # negative control: a line with a single split is NOT a small-caps run
    probe = "He got an A GRADE for it."
    if restore_smallcaps(probe) != probe:
        fails.append("single-split line was rewritten -- the >=2 guard is not holding")
    # ⚠ Scoped to RESTORED lines only. The corpus also contains a genuinely
    # all-caps in-world dispatch (`...BRAEVICK’S GRYPHON FLEET...`) that carries
    # no split initials, so the restore never touches it and it is not a defect.
    probe = "— M AJOR A FENDRA’S G UIDE TO THE R IDERS Q UADRANT"
    if re.search(r"[’']S\b", restore_smallcaps(probe)):
        fails.append("a restored small-caps line still carries an uppercase possessive `’S`")
    probe2 = "A slow smile spreads across her face."
    if DROPCAP.match(probe2):
        fails.append("dropcap join would fire on the article `A`")

    for w in man["works"]:
        pass
    shutil.copy(src / "manifest.json", dst / "manifest.json")
    shutil.copy(src / "corpus_alphabet.json", dst / "corpus_alphabet.json")
    m2 = json.loads((dst / "manifest.json").read_text())
    for w in m2["works"]:
        w["words"] = sum(json.loads(l)["words"] for l in (dst / w["path"]).read_text(encoding="utf-8").splitlines() if l.strip())
    m2["totals"]["words"] = m2["total_words"] = sum(w["words"] for w in m2["works"])
    m2["repaired_from"] = str(src)
    m2["repair"] = counts
    (dst / "manifest.json").write_text(json.dumps(m2, indent=2, ensure_ascii=False), encoding="utf-8")

    print()
    for f in fails:
        print(f"  [FAIL] {f}")
    if fails:
        print("\n== REPAIR REJECTED"); return 1
    print("  [PASS] known-broken strings gone, repaired forms present, guards hold")
    print(f"\n  wrote {dst}")
    return 0


if __name__ == "__main__":
    sys.exit(main())