esh-pfi-infrastructure/scripts/r49-corpus/build_corpus.py

"""R49 Stage D1 — acquire and clean a public-domain author corpus.

Charlotte Brontë's four novels from Project Gutenberg, stripped of boilerplate,
chapter-segmented, typography-normalised, with the corpus's own character
inventory derived from the result.

The alphabet is not cosmetic. R49 F02's rule is that the rename pool's character
inventory must be a SUBSET of the source corpus's -- substituting a 26%-diacritic
name pool into prose the author wrote in plain ASCII teaches the adapter a false
orthographic habit, landing directly on the axis being trained. So the corpus
derives the constraint and the pool obeys it, per work.

Two stages on purpose. `--survey` reports what is actually in the text before any
normalisation is chosen; normalisation decided from a guess rather than from the
survey is how a cleanup silently deletes something. Run the survey, read it, then
run the build.

  python build_corpus.py --survey             # measure, change nothing
  python build_corpus.py --build --out DIR    # emit the cleaned corpus
"""
from __future__ import annotations
import argparse, collections, json, re, sys, unicodedata, urllib.request
from pathlib import Path

# Catalogue ids verified against gutenberg.org's own search 2026-09-10, not
# recalled. Charlotte only -- the Bell poems are co-authored and the Gaskell
# biography is a different hand, so neither belongs in a single-voice corpus.
WORKS = [
    {"id": 1260,  "slug": "jane-eyre",     "title": "Jane Eyre: An Autobiography"},
    {"id": 9182,  "slug": "villette",      "title": "Villette"},
    {"id": 30486, "slug": "shirley",       "title": "Shirley"},
    {"id": 1028,  "slug": "the-professor", "title": "The Professor"},
]
URLS = ["https://www.gutenberg.org/cache/epub/{id}/pg{id}.txt",
        "https://www.gutenberg.org/files/{id}/{id}-0.txt",
        "https://www.gutenberg.org/files/{id}/{id}.txt"]

START = re.compile(r"^\*\*\*\s*START OF (?:THE|THIS) PROJECT GUTENBERG EBOOK.*?\*\*\*\s*$", re.M | re.I)
END = re.compile(r"^\*\*\*\s*END OF (?:THE|THIS) PROJECT GUTENBERG EBOOK.*?\*\*\*\s*$", re.M | re.I)
CHAPTER = re.compile(r"^\s*(CHAPTER\s+[IVXLCDM]+|CHAPTER\s+\d+)\.?\s*(.*)$", re.M)


def fetch(work, cache: Path) -> str:
    cache.mkdir(parents=True, exist_ok=True)
    raw = cache / f"{work['slug']}.raw.txt"
    if raw.exists():
        return raw.read_text(encoding="utf-8")
    for tmpl in URLS:
        url = tmpl.format(id=work["id"])
        try:
            with urllib.request.urlopen(url, timeout=60) as r:
                if r.status != 200:
                    continue
                text = r.read().decode("utf-8-sig")
            raw.write_text(text, encoding="utf-8")
            print(f"  fetched {work['slug']:<14} {url}  {len(text):,} bytes")
            return text
        except Exception as e:
            print(f"  .. {url} -> {type(e).__name__}")
    raise SystemExit(f"REFUSING: could not fetch {work['slug']} (id {work['id']})")


def strip_boilerplate(text: str, slug: str) -> str:
    """Keep only what lies between Gutenberg's own START/END markers.

    Anchoring on the markers rather than on a line count is what makes this
    safe across editions -- the front matter length differs per work.
    """
    m1, m2 = START.search(text), END.search(text)
    if not m1 or not m2:
        raise SystemExit(f"REFUSING: {slug} has no START/END markers; refusing to guess where the text begins")
    body = text[m1.end():m2.start()]
    # A transcriber credit block sometimes sits just inside the START marker.
    body = re.sub(r"\A\s*(?:Produced by|E-text prepared by|Transcribed from).*?\n\s*\n", "", body, flags=re.S | re.I)
    return body.strip("\n")


ROMAN = {"I":1,"V":5,"X":10,"L":50,"C":100,"D":500,"M":1000}


def roman_to_int(r: str) -> int:
    total, prev = 0, 0
    for ch in reversed(r.upper()):
        v = ROMAN.get(ch, 0)
        total = total - v if v < prev else total + v
        prev = max(prev, v)
    return total


def find_chapters(body: str) -> list[tuple[int, str, int]]:
    """Body chapter headings only, with any table of contents discarded.

    Measured 2026-09-10: The Professor ships a TOC that puts TWO chapter names
    on one line, so a bare regex returns 38 headings for a 25-chapter novel and
    a naive minimum-gap filter still leaks the TOC's tail. The rule that works
    is structural rather than cosmetic -- the body's "CHAPTER I" is the LAST one
    in the file, because a TOC always precedes the text it indexes. From there,
    keep only headings that continue the sequence and are separated by prose.
    """
    hits = []
    for m in CHAPTER.finditer(body):
        num = m.group(1).split()[-1].rstrip(".")
        n = int(num) if num.isdigit() else roman_to_int(num)
        hits.append((m.start(), m.group(1).strip(), n))
    if not hits:
        return []
    ones = [i for i, h in enumerate(hits) if h[2] == 1]
    start = ones[-1] if ones else 0
    kept, expect, last_pos = [], 1, -10**9
    for pos, label, n in hits[start:]:
        if n == expect and pos - last_pos > 500:
            kept.append((pos, label, n))
            expect, last_pos = expect + 1, pos
    return kept


#: Normalisation is decided from the survey, not from a guess. Measured across
#: the four works: Jane Eyre and Villette use curly quotes and em-dashes;
#: SHIRLEY uses straight quotes and `--` with zero em-dashes; The Professor
#: mixes curly quotes with `--`. That split is a transcriber artefact, not
#: Charlotte Bronte's punctuation, and leaving it would teach the adapter that
#: this author "sometimes" writes each form -- a false habit on the exact axis
#: being trained. Normalise toward what the text MEANS: `--` is a transcription
#: of an em-dash, so it becomes one.
def normalise_quotes(text: str) -> str:
    """Straight quotes -> curly, paired by alternation within each paragraph."""
    out = []
    for para in text.split("\n\n"):
        buf, open_d = [], True
        for ch in para:
            if ch == '"':
                buf.append("\u201c" if open_d else "\u201d")
                open_d = not open_d
            else:
                buf.append(ch)
        para = "".join(buf)
        # single quotes: apostrophe if flanked by letters, else a quote mark
        para = re.sub(r"(?<=[A-Za-z])'(?=[A-Za-z])", "\u2019", para)
        buf, open_s = [], True
        for ch in para:
            if ch == "'":
                buf.append("\u2018" if open_s else "\u2019")
                open_s = not open_s
            else:
                buf.append(ch)
        out.append("".join(buf))
    return "\n\n".join(out)


def clean(text: str) -> str:
    text = text.replace("\u00a0", " ")
    text = re.sub(r"(?<!-)--(?!-)", "\u2014", text)
    text = normalise_quotes(text)
    text = re.sub(r"[ \t]+\n", "\n", text)
    text = re.sub(r"\n{3,}", "\n\n", text)
    return text.strip("\n")


def survey(bodies: dict[str, str]) -> None:
    print("\n== character inventory, BEFORE any normalisation")
    allchars = collections.Counter()
    for slug, b in bodies.items():
        allchars.update(b)
    letters = {c for c in allchars if c.isalpha()}
    ascii_letters = {c for c in letters if ord(c) < 128}
    non_ascii = sorted(c for c in allchars if ord(c) > 127)
    print(f"  distinct characters      : {len(allchars)}")
    print(f"  distinct letters         : {len(letters)}  (ascii {len(ascii_letters)}, non-ascii {len(letters - ascii_letters)})")
    print(f"  distinct non-ascii chars : {len(non_ascii)}")
    print("  non-ascii, by frequency:")
    for c in sorted(non_ascii, key=lambda c: -allchars[c]):
        name = unicodedata.name(c, "?")
        print(f"     U+{ord(c):04X} {c!r:<8} {allchars[c]:>7}  {name}")
    print("\n== structure")
    for slug, b in bodies.items():
        heads = find_chapters(b)
        words = len(b.split())
        print(f"  {slug:<14} {words:>8,} words  {len(heads):>3} chapters  last: {heads[-1][1] if heads else '-'}")
    print(f"  {'TOTAL':<14} {sum(len(b.split()) for b in bodies.values()):>8,} words")


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--survey", action="store_true")
    ap.add_argument("--build", action="store_true")
    ap.add_argument("--out", default="corpus")
    ap.add_argument("--cache", default="raw")
    a = ap.parse_args()
    if not (a.survey or a.build):
        ap.error("pick --survey or --build")

    cache = Path(a.cache)
    print("== fetch")
    bodies = {}
    for w in WORKS:
        bodies[w["slug"]] = strip_boilerplate(fetch(w, cache), w["slug"])
        assert "PROJECT GUTENBERG" not in bodies[w["slug"]][:2000].upper(), f"{w['slug']}: boilerplate survived"

    if a.survey:
        survey(bodies)
        return 0

    out = Path(a.out)
    (out / "works").mkdir(parents=True, exist_ok=True)
    manifest, alphabet = [], set()
    for w in WORKS:
        slug = w["slug"]
        body = clean(bodies[slug])
        chaps = find_chapters(body)
        if not chaps:
            raise SystemExit(f"REFUSING: no chapters found in {slug}")
        # Self-consistency: the count must equal the last heading's numeral, or
        # the segmentation has silently over- or under-matched.
        if len(chaps) != chaps[-1][2]:
            raise SystemExit(
                f"REFUSING: {slug} segmented into {len(chaps)} chapters but the last "
                f"heading is {chaps[-1][1]} (= {chaps[-1][2]}). Segmentation is wrong.")
        records = []
        for i, (pos, label, n) in enumerate(chaps):
            end = chaps[i + 1][0] if i + 1 < len(chaps) else len(body)
            text = body[pos:end].strip("\n")
            records.append({"work": slug, "chapter": n, "heading": label,
                            "words": len(text.split()), "text": text})
        path = out / "works" / f"{slug}.jsonl"
        with path.open("w", encoding="utf-8") as fh:
            for r in records:
                fh.write(json.dumps(r, ensure_ascii=False) + "\n")
        alphabet |= {c for c in body if c.isalpha()}
        # Relative to the corpus root, never absolute: the corpus is built on one
        # box and trained on another, and an absolute build path makes the
        # manifest unreadable the moment it moves.
        manifest.append({"slug": slug, "gutenberg_id": w["id"], "title": w["title"],
                         "chapters": len(records),
                         "words": sum(r["words"] for r in records),
                         "chars": len(body), "path": f"works/{slug}.jsonl"})
        print(f"  wrote {slug:<14} {len(records):>3} chapters  {sum(r['words'] for r in records):>8,} words")

    alpha = sorted(alphabet)
    (out / "corpus_alphabet.json").write_text(json.dumps({
        "derived_from": "Charlotte Bronte, 4 novels, Project Gutenberg",
        "derived_at": "2026-09-10",
        "note": ("R49 F02 rule: a rename pool's character inventory must be a SUBSET of "
                 "this. Bronte writes French constantly (Villette, Adele, Brussels), so "
                 "unlike the Yarros corpus this alphabet legitimately carries accents -- "
                 "but only FRENCH ones. Czech/Latvian/Slovak/Hungarian marks never appear "
                 "and must not enter the pool."),
        "count": len(alpha), "letters": alpha,
        "non_ascii": [c for c in alpha if ord(c) > 127],
    }, ensure_ascii=False, indent=2), encoding="utf-8")
    (out / "manifest.json").write_text(json.dumps({
        "corpus": "bronte-charlotte-v1", "built_at": "2026-09-10",
        "source": "Project Gutenberg (public domain)",
        "normalisation": ("no-break space -> space; `--` -> em dash; straight quotes -> "
                          "curly, paired per paragraph. Decided from the survey: Shirley "
                          "was transcribed with straight quotes and zero em-dashes while "
                          "Jane Eyre and Villette use curly and em-dash, a transcriber "
                          "split rather than the author's punctuation."),
        "works": manifest,
        "total_words": sum(m["words"] for m in manifest),
        "total_chapters": sum(m["chapters"] for m in manifest),
    }, ensure_ascii=False, indent=2), encoding="utf-8")
    print(f"\n  alphabet: {len(alpha)} letters ({len([c for c in alpha if ord(c)>127])} non-ascii)")
    print(f"  TOTAL   : {sum(m['words'] for m in manifest):,} words in "
          f"{sum(m['chapters'] for m in manifest)} chapters -> {out}")
    return 0


if __name__ == "__main__":
    sys.exit(main())