BabyHemingway D1: fiction-only corpus builder with measured exclusions
This commit is contained in:
Vuong Hoang
@@ -0,0 +1,321 @@
|
||||
"""D1 for BabyHemingway: build the FICTION-ONLY corpus from the licensed Kvasir masters.
|
||||
|
||||
Emits the same record schema as the Brontë and Yarros builders
|
||||
({work, chapter, heading, words, text} per work file, plus corpus_alphabet.json and
|
||||
manifest.json) so entities.py, rename.py, leak_gate.py and the trainers run unchanged.
|
||||
|
||||
⭐ THE WORK THIS BUILDER DOES THAT THE OTHER TWO DID NOT IS EXCLUSION. Hemingway's 23
|
||||
catalogued works are 2,105,679 words, and roughly half of that must not be trained on.
|
||||
Every exclusion below is MEASURED or has a voice-specific reason; none is a preference.
|
||||
|
||||
1. NON-FICTION (8 works, ~911k words) -- operator's scope call: fiction only.
|
||||
By-Line, Dateline: Toronto, Death in the Afternoon, Green Hills of Africa,
|
||||
The Dangerous Summer, and the three posthumous "Hemingway on X" anthologies.
|
||||
|
||||
2. SUBSUMED STORY COLLECTIONS (4 works, 169,759 words) -- MEASURED, not assumed.
|
||||
`Short Stories` is The Short Stories of Ernest Hemingway (the First Forty-Nine) and
|
||||
it CONTAINS the others. 8-gram overlap as a fraction of the smaller work:
|
||||
|
||||
Short Stories + Winner Take Nothing 96.0%
|
||||
Short Stories + The Snows of Kilimanjaro and Other... 95.2%
|
||||
Short Stories + Men Without Women 92.9%
|
||||
Short Stories + In Our Time 90.6%
|
||||
|
||||
Keeping all five would weight those stories roughly twice. ⚠ The catalog's
|
||||
`near_dup_pairs` table CANNOT see this -- it holds whole-document simhashes (one row
|
||||
in the entire library) and this is PARTIAL containment, a collection inside a larger
|
||||
collection. Whole-document dedup is blind to it, which is why it was measured directly.
|
||||
|
||||
3. THE TORRENTS OF SPRING (21,505 words) -- excluded for a reason unrelated to overlap,
|
||||
and the only exclusion here that a word count could never justify. It is a deliberate
|
||||
PARODY of Sherwood Anderson. The prose is Hemingway imitating another writer badly on
|
||||
purpose, so for a VOICE adapter it is mislabelled data: it would teach the target
|
||||
author's name attached to a different author's style.
|
||||
|
||||
⚠ KEPT, with the editorial hand recorded rather than hidden: `True at First Light`,
|
||||
`The Garden of Eden` and `Islands in the Stream` are posthumous and editor-shaped -- cut
|
||||
and assembled by Hemingway's son and his publishers from unfinished manuscripts. They are
|
||||
presented and read as his fiction, so they stay; but a voice measured on this corpus is
|
||||
partly a voice his editors chose, and that belongs in the manifest.
|
||||
|
||||
⚠ CHAPTER SPLITTING IS DIFFERENT AGAIN. Brontë used roman numerals, Yarros used
|
||||
"Chapter One" plus a POV name. Hemingway's editions vary between bare arabic numerals on
|
||||
their own line, "Chapter N", and story collections where the unit is a TITLED STORY rather
|
||||
than a chapter. The splitter accepts all three and records which fired per work, because a
|
||||
splitter that silently found one chapter in a novel produces a single 174k-word record that
|
||||
every downstream tool will happily accept.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import argparse, collections, datetime, json, os, re, sqlite3, sys
|
||||
from pathlib import Path
|
||||
|
||||
CATALOG = "/home/lkraven/development/kvasir/data/library/catalog.sqlite"
|
||||
KVASIR = "/home/lkraven/development/kvasir"
|
||||
|
||||
# The fiction set, after the three exclusion passes documented above.
|
||||
FICTION = {
|
||||
"For Whom the Bell Tolls": "for-whom-the-bell-tolls",
|
||||
"Islands in the Stream": "islands-in-the-stream",
|
||||
"Short Stories": "short-stories",
|
||||
"True at First Light": "true-at-first-light",
|
||||
"A Farewell to Arms": "a-farewell-to-arms",
|
||||
"Across the River and Into the Trees": "across-the-river",
|
||||
"The Sun Also Rises": "the-sun-also-rises",
|
||||
"The Garden of Eden": "the-garden-of-eden",
|
||||
"To Have and Have Not": "to-have-and-have-not",
|
||||
"The Old Man and the Sea": "the-old-man-and-the-sea",
|
||||
}
|
||||
EXCLUDED = {
|
||||
"In Our Time": "subsumed by Short Stories (90.6% 8-gram containment)",
|
||||
"Men Without Women": "subsumed by Short Stories (92.9%)",
|
||||
"Winner Take Nothing": "subsumed by Short Stories (96.0%)",
|
||||
"The Snows of Kilimanjaro and Other Stories": "subsumed by Short Stories (95.2%)",
|
||||
"The Torrents of Spring": "parody of Sherwood Anderson -- not the target voice",
|
||||
}
|
||||
POSTHUMOUS_EDITED = {"True at First Light", "The Garden of Eden", "Islands in the Stream"}
|
||||
# Known-continuous: no internal divisions in the source, so one unit is CORRECT and the
|
||||
# splitter-found-nothing warning must not fire. A guard that cries wolf on a legitimate case
|
||||
# is a guard someone eventually stops reading.
|
||||
CONTINUOUS = {"The Old Man and the Sea"}
|
||||
|
||||
# Three heading forms, tried in order. `which` is recorded per work.
|
||||
HEAD_PATTERNS = [
|
||||
("chapter-word", re.compile(
|
||||
r"^[ \t]*((?:Chapter|CHAPTER)[ \t]+(?:[A-Za-z-]+|\d+)|Prologue|PROLOGUE|Epilogue|EPILOGUE)"
|
||||
r"[ \t]*\.?[ \t]*$", re.M)),
|
||||
("roman-numeral", re.compile(r"^[ \t]*((?=[IVXL])[IVXL]{1,7})[ \t]*\.?[ \t]*$", re.M)),
|
||||
("bare-numeral", re.compile(r"^[ \t]*(\d{1,3})[ \t]*\.?[ \t]*$", re.M)),
|
||||
("caps-title", re.compile(r"^[ \t]*([A-Z][A-Z '\-,!\.]{4,60})[ \t]*$", re.M)),
|
||||
]
|
||||
MIN_UNIT_WORDS = 150
|
||||
|
||||
|
||||
def masters():
|
||||
c = sqlite3.connect(CATALOG)
|
||||
rows = c.execute("select title, master_path, rights, normalized_text_sha256 "
|
||||
"from masters where lower(author) like '%hemingway%'").fetchall()
|
||||
out, seen = [], set()
|
||||
for title, path, rights, sha in rows:
|
||||
if title in EXCLUDED or title not in FICTION:
|
||||
continue
|
||||
if title in seen:
|
||||
print(f" ⚠ duplicate catalog row for {title}, keeping the first", file=sys.stderr)
|
||||
continue
|
||||
p = Path(path if os.path.isabs(path) else os.path.join(KVASIR, path))
|
||||
if not p.exists():
|
||||
print(f" ⚠ MISSING master for {title}: {p}", file=sys.stderr)
|
||||
continue
|
||||
seen.add(title)
|
||||
out.append({"title": title, "slug": FICTION[title], "path": p,
|
||||
"rights": rights, "sha256": sha})
|
||||
return sorted(out, key=lambda w: w["slug"])
|
||||
|
||||
|
||||
# ⚠⚠ THE TYPOGRAPHY REPAIR MUST RUN BEFORE THE SPLIT HERE, WHICH INVERTS THE YARROS ORDER.
|
||||
# Yarros ran D1 build then D1b repair, because there the defect was inside chapter epigraphs
|
||||
# and the chapter HEADINGS were clean. In these Hemingway editions the headings are the
|
||||
# damaged thing -- `T HE O LD M AN AND THE S EA`, `T HE S HORT H APPY L IFE ...` -- so a
|
||||
# splitter run first simply does not see them. Measured: 10 of 66 contents entries failed to
|
||||
# match their own body heading until this ran. Repairing after the split would have left a
|
||||
# 193k-word collection as five units and looked like a source quirk rather than a bug.
|
||||
SPLIT_INITIAL = re.compile(r"\b([A-Z]) ([A-Z]{2,})\b")
|
||||
|
||||
|
||||
def restore_smallcaps(line: str) -> str:
|
||||
"""`T HE O LD M AN` -> `The Old Man`. Only fires on a run of >=2 split initials.
|
||||
|
||||
One split initial is an ordinary sentence next to an acronym; two or more on one line is
|
||||
the extractor having rendered a small-caps run as uppercase while leaving each large
|
||||
initial its own token. The restore is exact rather than approximate: a word WITH a split
|
||||
initial was capitalised in the source, and an all-caps word WITHOUT one was lowercase.
|
||||
"""
|
||||
if len(SPLIT_INITIAL.findall(line)) < 2:
|
||||
return line
|
||||
return SPLIT_INITIAL.sub(lambda m: m.group(1) + m.group(2).lower(), line)
|
||||
|
||||
|
||||
def repair_lines(text: str) -> tuple[str, int]:
|
||||
out, fixed = [], 0
|
||||
for ln in text.split("\n"):
|
||||
r = restore_smallcaps(ln)
|
||||
if r != ln:
|
||||
fixed += 1
|
||||
out.append(r)
|
||||
return "\n".join(out), fixed
|
||||
|
||||
|
||||
# Front matter by SOMEONE ELSE. `The Old Man and the Sea` in this edition opens with an
|
||||
# introduction signed by Charles Scribner Jr.; training a voice adapter on another man's
|
||||
# prose under Hemingway's name is the same class of error as The Torrents of Spring, just
|
||||
# smaller. Strip is bounded by an acceptance check -- refuse if it would remove more than
|
||||
# 15% of the work, because a runaway strip that eats a novel must fail loudly.
|
||||
FOREIGN_FRONT = re.compile(r"^[ \t]*(INTRODUCTION|FOREWORD|PREFACE)\b", re.M | re.I)
|
||||
|
||||
|
||||
def strip_foreign_front(text: str, title: str) -> tuple[str, int]:
|
||||
m = FOREIGN_FRONT.search(text[:40000])
|
||||
if not m:
|
||||
return text, 0
|
||||
after = text[m.end():]
|
||||
# the real work starts at the first heading-shaped line following the introduction
|
||||
nxt = re.search(r"^[ \t]*([A-Z][A-Za-z '\-,!\.]{4,60}|[IVXL]{1,7}|\d{1,3})[ \t]*$",
|
||||
after, re.M)
|
||||
if not nxt:
|
||||
return text, 0
|
||||
cut = text[:m.start()] + after[nxt.start():]
|
||||
removed = len(text.split()) - len(cut.split())
|
||||
if removed > 0.15 * len(text.split()):
|
||||
print(f" ⚠ REFUSING front-matter strip on {title}: would remove {removed:,} words "
|
||||
f"({removed/len(text.split()):.0%})", file=sys.stderr)
|
||||
return text, 0
|
||||
return cut, removed
|
||||
|
||||
|
||||
def split_by_contents(text: str):
|
||||
"""Story collections: take the CONTENTS list as the authoritative heading set.
|
||||
|
||||
A regex over 'lines that look like a title' cannot separate a story title from a line of
|
||||
dialogue in a Hemingway collection -- both are short standalone lines and there are 4,044
|
||||
of them in this file. The book states its own structure; use that instead of inferring it.
|
||||
"""
|
||||
lines = [l.strip() for l in text.split("\n")]
|
||||
try:
|
||||
start = lines.index("CONTENTS")
|
||||
except ValueError:
|
||||
return None
|
||||
toc, skip = [], {"copyright", "about the publisher", "about the author", "contents"}
|
||||
for l in lines[start + 1:]:
|
||||
if not l:
|
||||
continue
|
||||
if len(l) > 70:
|
||||
break
|
||||
if l.lower() not in skip:
|
||||
toc.append(l)
|
||||
if len(toc) < 5:
|
||||
return None
|
||||
# ⚠ DO NOT dedupe on first occurrence. This edition carries a SECOND contents listing
|
||||
# partway through, so "first occurrence of the title" resolved to an index entry for 31
|
||||
# of 58 stories -- each yielding a 2-to-10-word span that then failed the floor and was
|
||||
# dropped, leaving 27 units and a silently truncated collection. Keeping EVERY occurrence
|
||||
# and letting the word floor decide is self-correcting: an index entry is followed by the
|
||||
# next index entry and scores ~3 words, while the real heading is followed by the story.
|
||||
tocset = set(toc)
|
||||
marks = [(i, l) for i, l in enumerate(lines) if i > start and l in tocset]
|
||||
if len(marks) < 5:
|
||||
return None
|
||||
units = []
|
||||
for j, (li, head) in enumerate(marks):
|
||||
end = marks[j + 1][0] if j + 1 < len(marks) else len(lines)
|
||||
body = "\n".join(lines[li:end]).strip()
|
||||
if len(body.split()) >= MIN_UNIT_WORDS:
|
||||
units.append((head, body))
|
||||
print(f" contents: {len(toc)} listed, {len(marks)} matched in body, "
|
||||
f"{len(units)} above the {MIN_UNIT_WORDS}-word floor")
|
||||
return units or None
|
||||
|
||||
|
||||
def split_units(text: str):
|
||||
"""Return (pattern_name, [(heading, body)]).
|
||||
|
||||
Tries each heading form and takes the one yielding the most units above the floor.
|
||||
⚠ A single unit for a novel means the splitter FOUND NOTHING -- reported, not hidden,
|
||||
because one 174k-word record is silently accepted by every downstream tool.
|
||||
"""
|
||||
best = ("none", [("(whole)", text.strip())])
|
||||
toc_units = split_by_contents(text)
|
||||
if toc_units:
|
||||
best = ("contents", toc_units)
|
||||
for name, pat in HEAD_PATTERNS:
|
||||
marks = [(m.start(), m.group(1).strip()) for m in pat.finditer(text)]
|
||||
if not marks:
|
||||
continue
|
||||
units = []
|
||||
for i, (pos, head) in enumerate(marks):
|
||||
end = marks[i + 1][0] if i + 1 < len(marks) else len(text)
|
||||
body = text[pos:end].strip()
|
||||
if len(body.split()) >= MIN_UNIT_WORDS:
|
||||
units.append((head, body))
|
||||
if len(units) > len(best[1]):
|
||||
best = (name, units)
|
||||
return best
|
||||
|
||||
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", required=True)
|
||||
ap.add_argument("--survey", action="store_true", help="report and write nothing")
|
||||
a = ap.parse_args()
|
||||
|
||||
works = masters()
|
||||
if len(works) != len(FICTION):
|
||||
print(f" ⚠ resolved {len(works)} of {len(FICTION)} fiction works", file=sys.stderr)
|
||||
if not works:
|
||||
raise SystemExit("REFUSING: no Hemingway fiction masters resolved from the catalog")
|
||||
|
||||
out = Path(a.out)
|
||||
alphabet = collections.Counter()
|
||||
total_words = total_units = 0
|
||||
manifest = {"corpus": "BabyHemingway", "author": "Ernest Hemingway",
|
||||
"scope": "FICTION ONLY (operator, 2026-09-16)",
|
||||
"source": "kvasir data/library masters (licensed, rights=gated)",
|
||||
"built_at": datetime.date.today().isoformat(),
|
||||
"excluded": EXCLUDED,
|
||||
"excluded_nonfiction": ("By-Line, Dateline: Toronto, Death in the Afternoon, "
|
||||
"Green Hills of Africa, The Dangerous Summer, Hemingway on "
|
||||
"War/Hunting/Fishing -- out of scope, ~911k words"),
|
||||
"posthumous_editor_shaped": sorted(POSTHUMOUS_EDITED),
|
||||
"works": []}
|
||||
|
||||
for w in works:
|
||||
text = w["path"].read_text(encoding="utf-8", errors="replace")
|
||||
text, fixed_lines = repair_lines(text)
|
||||
text, front_removed = strip_foreign_front(text, w["title"])
|
||||
pattern, units = split_units(text)
|
||||
alphabet.update(ch for ch in text if ch.isalpha())
|
||||
words = sum(len(b.split()) for _, b in units)
|
||||
total_words += words; total_units += len(units)
|
||||
flag = (" ⚠ SPLITTER FOUND NOTHING" if pattern == "none"
|
||||
and w["title"] not in CONTINUOUS else
|
||||
" (continuous, no divisions in source — expected)"
|
||||
if w["title"] in CONTINUOUS else "")
|
||||
post = " (posthumous/edited)" if w["title"] in POSTHUMOUS_EDITED else ""
|
||||
extra = (f" [smallcaps {fixed_lines}]" if fixed_lines else "") + \
|
||||
(f" [front -{front_removed}w]" if front_removed else "")
|
||||
print(f" {w['slug']:26} {len(units):>4} units {words:>8,} words "
|
||||
f"via {pattern:<14}{post}{flag}{extra}")
|
||||
manifest["works"].append({"slug": w["slug"], "title": w["title"], "rights": w["rights"],
|
||||
"master_sha256": w["sha256"], "units": len(units), "words": words,
|
||||
"heading_pattern": pattern,
|
||||
"smallcaps_lines_repaired": fixed_lines,
|
||||
"foreign_front_matter_words_removed": front_removed,
|
||||
"posthumous_editor_shaped": w["title"] in POSTHUMOUS_EDITED,
|
||||
"path": f"works/{w['slug']}.jsonl"})
|
||||
if not a.survey:
|
||||
(out / "works").mkdir(parents=True, exist_ok=True)
|
||||
with (out / "works" / f"{w['slug']}.jsonl").open("w", encoding="utf-8") as fh:
|
||||
for i, (head, body) in enumerate(units, 1):
|
||||
fh.write(json.dumps({"work": w["slug"], "chapter": i, "heading": head,
|
||||
"words": len(body.split()), "text": body},
|
||||
ensure_ascii=False) + "\n")
|
||||
|
||||
non_ascii = {c: n for c, n in alphabet.items() if ord(c) > 127}
|
||||
print(f"\n TOTAL {total_units} units · {total_words:,} words · {len(alphabet)} distinct letters")
|
||||
print(f" non-ASCII letters: {sum(non_ascii.values())} across {len(non_ascii)} forms")
|
||||
print(f" (Bronte 680,291 · Yarros 780,744 for scale)")
|
||||
manifest["totals"] = {"units": total_units, "words": total_words,
|
||||
"distinct_letters": len(alphabet), "non_ascii_letters": sum(non_ascii.values())}
|
||||
manifest["total_words"] = total_words
|
||||
manifest["total_chapters"] = total_units
|
||||
|
||||
if not a.survey:
|
||||
(out / "manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8")
|
||||
(out / "corpus_alphabet.json").write_text(json.dumps({
|
||||
"derived_from": "Ernest Hemingway, 10 fiction works, Kvasir licensed library",
|
||||
"derived_at": manifest["built_at"],
|
||||
"note": ("R49 F02 rule: a rename pool's character inventory must be a SUBSET of this. "
|
||||
"RE-DERIVED on this corpus rather than inherited from Bronte or Yarros -- "
|
||||
f"measured {sum(non_ascii.values())} non-ASCII letters across "
|
||||
f"{len(non_ascii)} forms. Hemingway writes Spanish, French and Italian "
|
||||
"constantly, so do NOT assume the Yarros ASCII-only conclusion transfers."),
|
||||
"letters": sorted(alphabet), "non_ascii": {c: n for c, n in sorted(non_ascii.items())},
|
||||
}, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
print(f" wrote {out}")
|
||||
Reference in New Issue
Block a user