diff --git a/scripts/hemingway-corpus/build_corpus_hemingway.py b/scripts/hemingway-corpus/build_corpus_hemingway.py new file mode 100644 index 0000000..fe167d5 --- /dev/null +++ b/scripts/hemingway-corpus/build_corpus_hemingway.py @@ -0,0 +1,321 @@ +"""D1 for BabyHemingway: build the FICTION-ONLY corpus from the licensed Kvasir masters. + +Emits the same record schema as the Brontë and Yarros builders +({work, chapter, heading, words, text} per work file, plus corpus_alphabet.json and +manifest.json) so entities.py, rename.py, leak_gate.py and the trainers run unchanged. + +⭐ THE WORK THIS BUILDER DOES THAT THE OTHER TWO DID NOT IS EXCLUSION. Hemingway's 23 +catalogued works are 2,105,679 words, and roughly half of that must not be trained on. +Every exclusion below is MEASURED or has a voice-specific reason; none is a preference. + + 1. NON-FICTION (8 works, ~911k words) -- operator's scope call: fiction only. + By-Line, Dateline: Toronto, Death in the Afternoon, Green Hills of Africa, + The Dangerous Summer, and the three posthumous "Hemingway on X" anthologies. + + 2. SUBSUMED STORY COLLECTIONS (4 works, 169,759 words) -- MEASURED, not assumed. + `Short Stories` is The Short Stories of Ernest Hemingway (the First Forty-Nine) and + it CONTAINS the others. 8-gram overlap as a fraction of the smaller work: + + Short Stories + Winner Take Nothing 96.0% + Short Stories + The Snows of Kilimanjaro and Other... 95.2% + Short Stories + Men Without Women 92.9% + Short Stories + In Our Time 90.6% + + Keeping all five would weight those stories roughly twice. ⚠ The catalog's + `near_dup_pairs` table CANNOT see this -- it holds whole-document simhashes (one row + in the entire library) and this is PARTIAL containment, a collection inside a larger + collection. Whole-document dedup is blind to it, which is why it was measured directly. + + 3. THE TORRENTS OF SPRING (21,505 words) -- excluded for a reason unrelated to overlap, + and the only exclusion here that a word count could never justify. It is a deliberate + PARODY of Sherwood Anderson. The prose is Hemingway imitating another writer badly on + purpose, so for a VOICE adapter it is mislabelled data: it would teach the target + author's name attached to a different author's style. + +⚠ KEPT, with the editorial hand recorded rather than hidden: `True at First Light`, +`The Garden of Eden` and `Islands in the Stream` are posthumous and editor-shaped -- cut +and assembled by Hemingway's son and his publishers from unfinished manuscripts. They are +presented and read as his fiction, so they stay; but a voice measured on this corpus is +partly a voice his editors chose, and that belongs in the manifest. + +⚠ CHAPTER SPLITTING IS DIFFERENT AGAIN. Brontë used roman numerals, Yarros used +"Chapter One" plus a POV name. Hemingway's editions vary between bare arabic numerals on +their own line, "Chapter N", and story collections where the unit is a TITLED STORY rather +than a chapter. The splitter accepts all three and records which fired per work, because a +splitter that silently found one chapter in a novel produces a single 174k-word record that +every downstream tool will happily accept. +""" +from __future__ import annotations +import argparse, collections, datetime, json, os, re, sqlite3, sys +from pathlib import Path + +CATALOG = "/home/lkraven/development/kvasir/data/library/catalog.sqlite" +KVASIR = "/home/lkraven/development/kvasir" + +# The fiction set, after the three exclusion passes documented above. +FICTION = { + "For Whom the Bell Tolls": "for-whom-the-bell-tolls", + "Islands in the Stream": "islands-in-the-stream", + "Short Stories": "short-stories", + "True at First Light": "true-at-first-light", + "A Farewell to Arms": "a-farewell-to-arms", + "Across the River and Into the Trees": "across-the-river", + "The Sun Also Rises": "the-sun-also-rises", + "The Garden of Eden": "the-garden-of-eden", + "To Have and Have Not": "to-have-and-have-not", + "The Old Man and the Sea": "the-old-man-and-the-sea", +} +EXCLUDED = { + "In Our Time": "subsumed by Short Stories (90.6% 8-gram containment)", + "Men Without Women": "subsumed by Short Stories (92.9%)", + "Winner Take Nothing": "subsumed by Short Stories (96.0%)", + "The Snows of Kilimanjaro and Other Stories": "subsumed by Short Stories (95.2%)", + "The Torrents of Spring": "parody of Sherwood Anderson -- not the target voice", +} +POSTHUMOUS_EDITED = {"True at First Light", "The Garden of Eden", "Islands in the Stream"} +# Known-continuous: no internal divisions in the source, so one unit is CORRECT and the +# splitter-found-nothing warning must not fire. A guard that cries wolf on a legitimate case +# is a guard someone eventually stops reading. +CONTINUOUS = {"The Old Man and the Sea"} + +# Three heading forms, tried in order. `which` is recorded per work. +HEAD_PATTERNS = [ + ("chapter-word", re.compile( + r"^[ \t]*((?:Chapter|CHAPTER)[ \t]+(?:[A-Za-z-]+|\d+)|Prologue|PROLOGUE|Epilogue|EPILOGUE)" + r"[ \t]*\.?[ \t]*$", re.M)), + ("roman-numeral", re.compile(r"^[ \t]*((?=[IVXL])[IVXL]{1,7})[ \t]*\.?[ \t]*$", re.M)), + ("bare-numeral", re.compile(r"^[ \t]*(\d{1,3})[ \t]*\.?[ \t]*$", re.M)), + ("caps-title", re.compile(r"^[ \t]*([A-Z][A-Z '\-,!\.]{4,60})[ \t]*$", re.M)), +] +MIN_UNIT_WORDS = 150 + + +def masters(): + c = sqlite3.connect(CATALOG) + rows = c.execute("select title, master_path, rights, normalized_text_sha256 " + "from masters where lower(author) like '%hemingway%'").fetchall() + out, seen = [], set() + for title, path, rights, sha in rows: + if title in EXCLUDED or title not in FICTION: + continue + if title in seen: + print(f" ⚠ duplicate catalog row for {title}, keeping the first", file=sys.stderr) + continue + p = Path(path if os.path.isabs(path) else os.path.join(KVASIR, path)) + if not p.exists(): + print(f" ⚠ MISSING master for {title}: {p}", file=sys.stderr) + continue + seen.add(title) + out.append({"title": title, "slug": FICTION[title], "path": p, + "rights": rights, "sha256": sha}) + return sorted(out, key=lambda w: w["slug"]) + + +# ⚠⚠ THE TYPOGRAPHY REPAIR MUST RUN BEFORE THE SPLIT HERE, WHICH INVERTS THE YARROS ORDER. +# Yarros ran D1 build then D1b repair, because there the defect was inside chapter epigraphs +# and the chapter HEADINGS were clean. In these Hemingway editions the headings are the +# damaged thing -- `T HE O LD M AN AND THE S EA`, `T HE S HORT H APPY L IFE ...` -- so a +# splitter run first simply does not see them. Measured: 10 of 66 contents entries failed to +# match their own body heading until this ran. Repairing after the split would have left a +# 193k-word collection as five units and looked like a source quirk rather than a bug. +SPLIT_INITIAL = re.compile(r"\b([A-Z]) ([A-Z]{2,})\b") + + +def restore_smallcaps(line: str) -> str: + """`T HE O LD M AN` -> `The Old Man`. Only fires on a run of >=2 split initials. + + One split initial is an ordinary sentence next to an acronym; two or more on one line is + the extractor having rendered a small-caps run as uppercase while leaving each large + initial its own token. The restore is exact rather than approximate: a word WITH a split + initial was capitalised in the source, and an all-caps word WITHOUT one was lowercase. + """ + if len(SPLIT_INITIAL.findall(line)) < 2: + return line + return SPLIT_INITIAL.sub(lambda m: m.group(1) + m.group(2).lower(), line) + + +def repair_lines(text: str) -> tuple[str, int]: + out, fixed = [], 0 + for ln in text.split("\n"): + r = restore_smallcaps(ln) + if r != ln: + fixed += 1 + out.append(r) + return "\n".join(out), fixed + + +# Front matter by SOMEONE ELSE. `The Old Man and the Sea` in this edition opens with an +# introduction signed by Charles Scribner Jr.; training a voice adapter on another man's +# prose under Hemingway's name is the same class of error as The Torrents of Spring, just +# smaller. Strip is bounded by an acceptance check -- refuse if it would remove more than +# 15% of the work, because a runaway strip that eats a novel must fail loudly. +FOREIGN_FRONT = re.compile(r"^[ \t]*(INTRODUCTION|FOREWORD|PREFACE)\b", re.M | re.I) + + +def strip_foreign_front(text: str, title: str) -> tuple[str, int]: + m = FOREIGN_FRONT.search(text[:40000]) + if not m: + return text, 0 + after = text[m.end():] + # the real work starts at the first heading-shaped line following the introduction + nxt = re.search(r"^[ \t]*([A-Z][A-Za-z '\-,!\.]{4,60}|[IVXL]{1,7}|\d{1,3})[ \t]*$", + after, re.M) + if not nxt: + return text, 0 + cut = text[:m.start()] + after[nxt.start():] + removed = len(text.split()) - len(cut.split()) + if removed > 0.15 * len(text.split()): + print(f" ⚠ REFUSING front-matter strip on {title}: would remove {removed:,} words " + f"({removed/len(text.split()):.0%})", file=sys.stderr) + return text, 0 + return cut, removed + + +def split_by_contents(text: str): + """Story collections: take the CONTENTS list as the authoritative heading set. + + A regex over 'lines that look like a title' cannot separate a story title from a line of + dialogue in a Hemingway collection -- both are short standalone lines and there are 4,044 + of them in this file. The book states its own structure; use that instead of inferring it. + """ + lines = [l.strip() for l in text.split("\n")] + try: + start = lines.index("CONTENTS") + except ValueError: + return None + toc, skip = [], {"copyright", "about the publisher", "about the author", "contents"} + for l in lines[start + 1:]: + if not l: + continue + if len(l) > 70: + break + if l.lower() not in skip: + toc.append(l) + if len(toc) < 5: + return None + # ⚠ DO NOT dedupe on first occurrence. This edition carries a SECOND contents listing + # partway through, so "first occurrence of the title" resolved to an index entry for 31 + # of 58 stories -- each yielding a 2-to-10-word span that then failed the floor and was + # dropped, leaving 27 units and a silently truncated collection. Keeping EVERY occurrence + # and letting the word floor decide is self-correcting: an index entry is followed by the + # next index entry and scores ~3 words, while the real heading is followed by the story. + tocset = set(toc) + marks = [(i, l) for i, l in enumerate(lines) if i > start and l in tocset] + if len(marks) < 5: + return None + units = [] + for j, (li, head) in enumerate(marks): + end = marks[j + 1][0] if j + 1 < len(marks) else len(lines) + body = "\n".join(lines[li:end]).strip() + if len(body.split()) >= MIN_UNIT_WORDS: + units.append((head, body)) + print(f" contents: {len(toc)} listed, {len(marks)} matched in body, " + f"{len(units)} above the {MIN_UNIT_WORDS}-word floor") + return units or None + + +def split_units(text: str): + """Return (pattern_name, [(heading, body)]). + + Tries each heading form and takes the one yielding the most units above the floor. + ⚠ A single unit for a novel means the splitter FOUND NOTHING -- reported, not hidden, + because one 174k-word record is silently accepted by every downstream tool. + """ + best = ("none", [("(whole)", text.strip())]) + toc_units = split_by_contents(text) + if toc_units: + best = ("contents", toc_units) + for name, pat in HEAD_PATTERNS: + marks = [(m.start(), m.group(1).strip()) for m in pat.finditer(text)] + if not marks: + continue + units = [] + for i, (pos, head) in enumerate(marks): + end = marks[i + 1][0] if i + 1 < len(marks) else len(text) + body = text[pos:end].strip() + if len(body.split()) >= MIN_UNIT_WORDS: + units.append((head, body)) + if len(units) > len(best[1]): + best = (name, units) + return best + + +ap = argparse.ArgumentParser() +ap.add_argument("--out", required=True) +ap.add_argument("--survey", action="store_true", help="report and write nothing") +a = ap.parse_args() + +works = masters() +if len(works) != len(FICTION): + print(f" ⚠ resolved {len(works)} of {len(FICTION)} fiction works", file=sys.stderr) +if not works: + raise SystemExit("REFUSING: no Hemingway fiction masters resolved from the catalog") + +out = Path(a.out) +alphabet = collections.Counter() +total_words = total_units = 0 +manifest = {"corpus": "BabyHemingway", "author": "Ernest Hemingway", + "scope": "FICTION ONLY (operator, 2026-09-16)", + "source": "kvasir data/library masters (licensed, rights=gated)", + "built_at": datetime.date.today().isoformat(), + "excluded": EXCLUDED, + "excluded_nonfiction": ("By-Line, Dateline: Toronto, Death in the Afternoon, " + "Green Hills of Africa, The Dangerous Summer, Hemingway on " + "War/Hunting/Fishing -- out of scope, ~911k words"), + "posthumous_editor_shaped": sorted(POSTHUMOUS_EDITED), + "works": []} + +for w in works: + text = w["path"].read_text(encoding="utf-8", errors="replace") + text, fixed_lines = repair_lines(text) + text, front_removed = strip_foreign_front(text, w["title"]) + pattern, units = split_units(text) + alphabet.update(ch for ch in text if ch.isalpha()) + words = sum(len(b.split()) for _, b in units) + total_words += words; total_units += len(units) + flag = (" ⚠ SPLITTER FOUND NOTHING" if pattern == "none" + and w["title"] not in CONTINUOUS else + " (continuous, no divisions in source — expected)" + if w["title"] in CONTINUOUS else "") + post = " (posthumous/edited)" if w["title"] in POSTHUMOUS_EDITED else "" + extra = (f" [smallcaps {fixed_lines}]" if fixed_lines else "") + \ + (f" [front -{front_removed}w]" if front_removed else "") + print(f" {w['slug']:26} {len(units):>4} units {words:>8,} words " + f"via {pattern:<14}{post}{flag}{extra}") + manifest["works"].append({"slug": w["slug"], "title": w["title"], "rights": w["rights"], + "master_sha256": w["sha256"], "units": len(units), "words": words, + "heading_pattern": pattern, + "smallcaps_lines_repaired": fixed_lines, + "foreign_front_matter_words_removed": front_removed, + "posthumous_editor_shaped": w["title"] in POSTHUMOUS_EDITED, + "path": f"works/{w['slug']}.jsonl"}) + if not a.survey: + (out / "works").mkdir(parents=True, exist_ok=True) + with (out / "works" / f"{w['slug']}.jsonl").open("w", encoding="utf-8") as fh: + for i, (head, body) in enumerate(units, 1): + fh.write(json.dumps({"work": w["slug"], "chapter": i, "heading": head, + "words": len(body.split()), "text": body}, + ensure_ascii=False) + "\n") + +non_ascii = {c: n for c, n in alphabet.items() if ord(c) > 127} +print(f"\n TOTAL {total_units} units · {total_words:,} words · {len(alphabet)} distinct letters") +print(f" non-ASCII letters: {sum(non_ascii.values())} across {len(non_ascii)} forms") +print(f" (Bronte 680,291 · Yarros 780,744 for scale)") +manifest["totals"] = {"units": total_units, "words": total_words, + "distinct_letters": len(alphabet), "non_ascii_letters": sum(non_ascii.values())} +manifest["total_words"] = total_words +manifest["total_chapters"] = total_units + +if not a.survey: + (out / "manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8") + (out / "corpus_alphabet.json").write_text(json.dumps({ + "derived_from": "Ernest Hemingway, 10 fiction works, Kvasir licensed library", + "derived_at": manifest["built_at"], + "note": ("R49 F02 rule: a rename pool's character inventory must be a SUBSET of this. " + "RE-DERIVED on this corpus rather than inherited from Bronte or Yarros -- " + f"measured {sum(non_ascii.values())} non-ASCII letters across " + f"{len(non_ascii)} forms. Hemingway writes Spanish, French and Italian " + "constantly, so do NOT assume the Yarros ASCII-only conclusion transfers."), + "letters": sorted(alphabet), "non_ascii": {c: n for c, n in sorted(non_ascii.items())}, + }, indent=2, ensure_ascii=False), encoding="utf-8") + print(f" wrote {out}")