"""R49 Stage D1 — acquire and clean a public-domain author corpus. Charlotte Brontë's four novels from Project Gutenberg, stripped of boilerplate, chapter-segmented, typography-normalised, with the corpus's own character inventory derived from the result. The alphabet is not cosmetic. R49 F02's rule is that the rename pool's character inventory must be a SUBSET of the source corpus's -- substituting a 26%-diacritic name pool into prose the author wrote in plain ASCII teaches the adapter a false orthographic habit, landing directly on the axis being trained. So the corpus derives the constraint and the pool obeys it, per work. Two stages on purpose. `--survey` reports what is actually in the text before any normalisation is chosen; normalisation decided from a guess rather than from the survey is how a cleanup silently deletes something. Run the survey, read it, then run the build. python build_corpus.py --survey # measure, change nothing python build_corpus.py --build --out DIR # emit the cleaned corpus """ from __future__ import annotations import argparse, collections, json, re, sys, unicodedata, urllib.request from pathlib import Path # Catalogue ids verified against gutenberg.org's own search 2026-09-10, not # recalled. Charlotte only -- the Bell poems are co-authored and the Gaskell # biography is a different hand, so neither belongs in a single-voice corpus. WORKS = [ {"id": 1260, "slug": "jane-eyre", "title": "Jane Eyre: An Autobiography"}, {"id": 9182, "slug": "villette", "title": "Villette"}, {"id": 30486, "slug": "shirley", "title": "Shirley"}, {"id": 1028, "slug": "the-professor", "title": "The Professor"}, ] URLS = ["https://www.gutenberg.org/cache/epub/{id}/pg{id}.txt", "https://www.gutenberg.org/files/{id}/{id}-0.txt", "https://www.gutenberg.org/files/{id}/{id}.txt"] START = re.compile(r"^\*\*\*\s*START OF (?:THE|THIS) PROJECT GUTENBERG EBOOK.*?\*\*\*\s*$", re.M | re.I) END = re.compile(r"^\*\*\*\s*END OF (?:THE|THIS) PROJECT GUTENBERG EBOOK.*?\*\*\*\s*$", re.M | re.I) CHAPTER = re.compile(r"^\s*(CHAPTER\s+[IVXLCDM]+|CHAPTER\s+\d+)\.?\s*(.*)$", re.M) def fetch(work, cache: Path) -> str: cache.mkdir(parents=True, exist_ok=True) raw = cache / f"{work['slug']}.raw.txt" if raw.exists(): return raw.read_text(encoding="utf-8") for tmpl in URLS: url = tmpl.format(id=work["id"]) try: with urllib.request.urlopen(url, timeout=60) as r: if r.status != 200: continue text = r.read().decode("utf-8-sig") raw.write_text(text, encoding="utf-8") print(f" fetched {work['slug']:<14} {url} {len(text):,} bytes") return text except Exception as e: print(f" .. {url} -> {type(e).__name__}") raise SystemExit(f"REFUSING: could not fetch {work['slug']} (id {work['id']})") def strip_boilerplate(text: str, slug: str) -> str: """Keep only what lies between Gutenberg's own START/END markers. Anchoring on the markers rather than on a line count is what makes this safe across editions -- the front matter length differs per work. """ m1, m2 = START.search(text), END.search(text) if not m1 or not m2: raise SystemExit(f"REFUSING: {slug} has no START/END markers; refusing to guess where the text begins") body = text[m1.end():m2.start()] # A transcriber credit block sometimes sits just inside the START marker. body = re.sub(r"\A\s*(?:Produced by|E-text prepared by|Transcribed from).*?\n\s*\n", "", body, flags=re.S | re.I) return body.strip("\n") ROMAN = {"I":1,"V":5,"X":10,"L":50,"C":100,"D":500,"M":1000} def roman_to_int(r: str) -> int: total, prev = 0, 0 for ch in reversed(r.upper()): v = ROMAN.get(ch, 0) total = total - v if v < prev else total + v prev = max(prev, v) return total def find_chapters(body: str) -> list[tuple[int, str, int]]: """Body chapter headings only, with any table of contents discarded. Measured 2026-09-10: The Professor ships a TOC that puts TWO chapter names on one line, so a bare regex returns 38 headings for a 25-chapter novel and a naive minimum-gap filter still leaks the TOC's tail. The rule that works is structural rather than cosmetic -- the body's "CHAPTER I" is the LAST one in the file, because a TOC always precedes the text it indexes. From there, keep only headings that continue the sequence and are separated by prose. """ hits = [] for m in CHAPTER.finditer(body): num = m.group(1).split()[-1].rstrip(".") n = int(num) if num.isdigit() else roman_to_int(num) hits.append((m.start(), m.group(1).strip(), n)) if not hits: return [] ones = [i for i, h in enumerate(hits) if h[2] == 1] start = ones[-1] if ones else 0 kept, expect, last_pos = [], 1, -10**9 for pos, label, n in hits[start:]: if n == expect and pos - last_pos > 500: kept.append((pos, label, n)) expect, last_pos = expect + 1, pos return kept #: Normalisation is decided from the survey, not from a guess. Measured across #: the four works: Jane Eyre and Villette use curly quotes and em-dashes; #: SHIRLEY uses straight quotes and `--` with zero em-dashes; The Professor #: mixes curly quotes with `--`. That split is a transcriber artefact, not #: Charlotte Bronte's punctuation, and leaving it would teach the adapter that #: this author "sometimes" writes each form -- a false habit on the exact axis #: being trained. Normalise toward what the text MEANS: `--` is a transcription #: of an em-dash, so it becomes one. def normalise_quotes(text: str) -> str: """Straight quotes -> curly, paired by alternation within each paragraph.""" out = [] for para in text.split("\n\n"): buf, open_d = [], True for ch in para: if ch == '"': buf.append("\u201c" if open_d else "\u201d") open_d = not open_d else: buf.append(ch) para = "".join(buf) # single quotes: apostrophe if flanked by letters, else a quote mark para = re.sub(r"(?<=[A-Za-z])'(?=[A-Za-z])", "\u2019", para) buf, open_s = [], True for ch in para: if ch == "'": buf.append("\u2018" if open_s else "\u2019") open_s = not open_s else: buf.append(ch) out.append("".join(buf)) return "\n\n".join(out) def clean(text: str) -> str: text = text.replace("\u00a0", " ") text = re.sub(r"(? None: print("\n== character inventory, BEFORE any normalisation") allchars = collections.Counter() for slug, b in bodies.items(): allchars.update(b) letters = {c for c in allchars if c.isalpha()} ascii_letters = {c for c in letters if ord(c) < 128} non_ascii = sorted(c for c in allchars if ord(c) > 127) print(f" distinct characters : {len(allchars)}") print(f" distinct letters : {len(letters)} (ascii {len(ascii_letters)}, non-ascii {len(letters - ascii_letters)})") print(f" distinct non-ascii chars : {len(non_ascii)}") print(" non-ascii, by frequency:") for c in sorted(non_ascii, key=lambda c: -allchars[c]): name = unicodedata.name(c, "?") print(f" U+{ord(c):04X} {c!r:<8} {allchars[c]:>7} {name}") print("\n== structure") for slug, b in bodies.items(): heads = find_chapters(b) words = len(b.split()) print(f" {slug:<14} {words:>8,} words {len(heads):>3} chapters last: {heads[-1][1] if heads else '-'}") print(f" {'TOTAL':<14} {sum(len(b.split()) for b in bodies.values()):>8,} words") def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--survey", action="store_true") ap.add_argument("--build", action="store_true") ap.add_argument("--out", default="corpus") ap.add_argument("--cache", default="raw") a = ap.parse_args() if not (a.survey or a.build): ap.error("pick --survey or --build") cache = Path(a.cache) print("== fetch") bodies = {} for w in WORKS: bodies[w["slug"]] = strip_boilerplate(fetch(w, cache), w["slug"]) assert "PROJECT GUTENBERG" not in bodies[w["slug"]][:2000].upper(), f"{w['slug']}: boilerplate survived" if a.survey: survey(bodies) return 0 out = Path(a.out) (out / "works").mkdir(parents=True, exist_ok=True) manifest, alphabet = [], set() for w in WORKS: slug = w["slug"] body = clean(bodies[slug]) chaps = find_chapters(body) if not chaps: raise SystemExit(f"REFUSING: no chapters found in {slug}") # Self-consistency: the count must equal the last heading's numeral, or # the segmentation has silently over- or under-matched. if len(chaps) != chaps[-1][2]: raise SystemExit( f"REFUSING: {slug} segmented into {len(chaps)} chapters but the last " f"heading is {chaps[-1][1]} (= {chaps[-1][2]}). Segmentation is wrong.") records = [] for i, (pos, label, n) in enumerate(chaps): end = chaps[i + 1][0] if i + 1 < len(chaps) else len(body) text = body[pos:end].strip("\n") records.append({"work": slug, "chapter": n, "heading": label, "words": len(text.split()), "text": text}) path = out / "works" / f"{slug}.jsonl" with path.open("w", encoding="utf-8") as fh: for r in records: fh.write(json.dumps(r, ensure_ascii=False) + "\n") alphabet |= {c for c in body if c.isalpha()} # Relative to the corpus root, never absolute: the corpus is built on one # box and trained on another, and an absolute build path makes the # manifest unreadable the moment it moves. manifest.append({"slug": slug, "gutenberg_id": w["id"], "title": w["title"], "chapters": len(records), "words": sum(r["words"] for r in records), "chars": len(body), "path": f"works/{slug}.jsonl"}) print(f" wrote {slug:<14} {len(records):>3} chapters {sum(r['words'] for r in records):>8,} words") alpha = sorted(alphabet) (out / "corpus_alphabet.json").write_text(json.dumps({ "derived_from": "Charlotte Bronte, 4 novels, Project Gutenberg", "derived_at": "2026-09-10", "note": ("R49 F02 rule: a rename pool's character inventory must be a SUBSET of " "this. Bronte writes French constantly (Villette, Adele, Brussels), so " "unlike the Yarros corpus this alphabet legitimately carries accents -- " "but only FRENCH ones. Czech/Latvian/Slovak/Hungarian marks never appear " "and must not enter the pool."), "count": len(alpha), "letters": alpha, "non_ascii": [c for c in alpha if ord(c) > 127], }, ensure_ascii=False, indent=2), encoding="utf-8") (out / "manifest.json").write_text(json.dumps({ "corpus": "bronte-charlotte-v1", "built_at": "2026-09-10", "source": "Project Gutenberg (public domain)", "normalisation": ("no-break space -> space; `--` -> em dash; straight quotes -> " "curly, paired per paragraph. Decided from the survey: Shirley " "was transcribed with straight quotes and zero em-dashes while " "Jane Eyre and Villette use curly and em-dash, a transcriber " "split rather than the author's punctuation."), "works": manifest, "total_words": sum(m["words"] for m in manifest), "total_chapters": sum(m["chapters"] for m in manifest), }, ensure_ascii=False, indent=2), encoding="utf-8") print(f"\n alphabet: {len(alpha)} letters ({len([c for c in alpha if ord(c)>127])} non-ascii)") print(f" TOTAL : {sum(m['words'] for m in manifest):,} words in " f"{sum(m['chapters'] for m in manifest)} chapters -> {out}") return 0 if __name__ == "__main__": sys.exit(main())