diff --git a/scripts/krakauer-corpus/build_corpus_krakauer.py b/scripts/krakauer-corpus/build_corpus_krakauer.py new file mode 100644 index 0000000..9e0750c --- /dev/null +++ b/scripts/krakauer-corpus/build_corpus_krakauer.py @@ -0,0 +1,314 @@ +"""D1 for lv-krakauer: build the corpus from the licensed Kvasir masters. + +Same record schema as the Brontë, Yarros, Hemingway and McCarthy builders, so entities.py, +rename.py, leak_gate.py and the trainers run unchanged. + +⚠⚠⚠ READ THIS BEFORE USING THE CORPUS THIS BUILDS. IT IS THE FIRST NON-FICTION VOICE +CORPUS IN THIS LINE, AND AN UNMEASURED FRACTION OF IT IS OTHER PEOPLE'S PROSE. + +Krakauer quotes constantly and at length — Chris McCandless's journals and letters, Pat +Tillman's diaries, court transcripts, depositions, nineteenth-century Mormon documents, +newspaper reports, and full paragraphs of Jack London, Wallace Stegner and Thoreau at the +chapter heads. In print those are set as indented block quotes or italics. **The extraction +lost both**, so inside the master they are ordinary paragraphs, indistinguishable from +Krakauer's own sentences by any signal this builder can read. + +What IS detectable is stripped, and it is almost nothing — **52 words** across four works, +0.01% of the corpus: chapter-head epigraphs found by a comma-bearing all-caps attribution +line ("JACK LONDON, THE CALL OF THE WILD"). A first estimate put this at ~1,500 words by +walking back three paragraph blocks from each attribution; the shipped rule is bounded to +the chapter head and takes far less, which is the safer error to make. + +⭐ **THE REMAINING QUOTED MATERIAL IS NOT MEASURED AND THIS BUILDER CANNOT MEASURE IT.** +Stating the floor rather than letting a 0.3% number travel as if it were the answer: this +method detects only quotations whose attribution survived as an all-caps line, and it would +report 0.0% for a book made entirely of undated block quotes. Anyone deciding whether to +train on this needs that sentence, not the 0.3%. + +The parallel is exact: Hemingway's scope was cut to fiction-only by an operator ruling, and +`The Torrents of Spring` was excluded because a parody is "the target author's name on a +different author's style, i.e. mislabelled data for a voice adapter". Embedded quotation is +the same error, distributed rather than concentrated, and the fraction is unknown. + +STRUCTURE, which differs from every earlier author in this line. + +⚠ THE FRONT MATTER IS AT THE FRONT AND IT IS ENORMOUS. Three of the four ebooks open with +`Acclaim for`, `ALSO BY JON KRAKAUER`, `Copyright` and `About the Author` inside the first +0.6% of the file. A back-matter marker search that takes the earliest hit — which is what +the McCarthy builder does, correctly, for McCarthy — would cut 99.9% of the book here. So +both strips are WINDOWED: front matter is cut at the author's own signature inside the first +10% of the file, back matter at the earliest apparatus marker inside the last 25%. + +⚠ RELYING ON THE SPLITTER TO DROP THE FRONT MATTER WAS THE FIRST DESIGN AND IT DID NOT WORK. +Units begin at the first heading mark, so front matter is dropped only when no mark falls +inside it — and in Missoula and Where Men Win Glory the ebook's table of contents sits above +the author's note, giving the splitter a `Chapter Thirty-Two` to start on. Unit 1 then +swallowed the whole apparatus. The name guard is what caught it. + +⚠ THE SMALL-CAPS SPLIT-INITIAL DEFECT IS PRESENT AND MUST BE REPAIRED BEFORE THE SPLIT. +`J ACK L ONDON , W HITE F ANG` and `A LEXANDER.` are the Hemingway `T HE O LD M AN` defect +again. Two of the four works split on `caps-title`, so a damaged heading is a heading the +splitter cannot see — repairing afterwards would be too late. +""" +from __future__ import annotations +import argparse, collections, datetime, json, os, re, sqlite3, sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "r49-corpus")) +from split_units import choose_units, format_report # noqa: E402 + +CATALOG = "/home/lkraven/development/kvasir/data/library/catalog.sqlite" +KVASIR = "/home/lkraven/development/kvasir" + +WORKS = { + "Into the Wild": "into-the-wild", + "Missoula: Rape and the Justice System in a College Town": "missoula", + "Under the Banner of Heaven: A Story of Violent Faith": "under-the-banner-of-heaven", + "Where Men Win Glory": "where-men-win-glory", +} + +# Back matter, matched only inside the LAST unit. `NOTES` and `BIBLIOGRAPHY` are source +# apparatus: thousands of words of citations that are not prose in any register. +BACKMATTER = re.compile( + r"^[ \t]*(AUTHOR.S NOTE|ACKNOWLEDGMENTS|ACKNOWLEDGEMENTS|SELECTED BIBLIOGRAPHY|" + r"BIBLIOGRAPHY|NOTES|INDEX|ABOUT THE AUTHOR|ALSO BY|APPENDIX)[ \t]*.{0,40}$", re.M) + +SPLIT_INITIAL = re.compile(r"\b([A-Z]) ([A-Z]{2,})\b") +# ⚠ THE AUTHOR SIGNS HIS OWN FRONT MATTER. Missoula and Where Men Win Glory open with an +# author's note closing "Jon Krakauer , February 2015" immediately before PART ONE, and the +# ebook's table of contents sits above that — so the splitter's first heading mark lands +# inside the front matter and unit 1 swallows the lot. Cutting at the signature is exact +# where a marker list would be guesswork, and it is bounded to the first 10% of the file. +AUTHOR_SIGNATURE = re.compile(r"^[ \t]*Jon Krakauer[ \t]*,?[ \t]*" + r"(?:January|February|March|April|May|June|July|August|" + r"September|October|November|December)?[ \t]*\d{4}[ \t]*$", re.M) +# An all-caps line WITH a comma is a source attribution ("HENRY DAVID THOREAU, JOURNAL"); +# without one it is a chapter title ("THE STAMPEDE TRAIL", "DETRITAL WASH"). Verified by +# reading every caps line in the two works that split on caps-title. +ATTRIBUTION = re.compile(r"^[ \t]*([A-Z][A-Z '\-.!]{3,40},[A-Z '\-.!][A-Z '\-,.!]{3,60})[ \t]*$", re.M) +EPIGRAPH_WINDOW = 1200 # an epigraph sits at the chapter head, not in the middle of it + +# ⚠ ZERO IS THE WRONG BAR FOR THIS AUTHOR, and the difference matters. Hemingway's own name +# in his training text was always PUBLISHER APPARATUS — a jacket biography, an editor's cast +# list — and stripping it to 0 was correct. Krakauer writes about himself: Into the Wild +# devotes two chapters to his own youth, so `Lewis Krakauer loved his five children deeply` +# is his prose, not a jacket blurb, and the second survivor is a reader's letter he quotes +# calling him a kook. Both were read before being allowed. The rename pipeline treats +# `Krakauer` as an ordinary capitalised surface and renames it downstream like any other. +# The number is pinned so that a master change, or a strip that stops working, fails loudly +# instead of widening silently. +BODY_NAME_OCCURRENCES = 2 + + +def strip_front(text: str) -> tuple[str, int]: + """Cut everything up to and including the author's signed front-matter note.""" + window = text[:int(len(text) * 0.10)] + hits = list(AUTHOR_SIGNATURE.finditer(window)) + if not hits: + return text, 0 + cut = text[hits[-1].end():].lstrip() + return cut, len(text.split()) - len(cut.split()) + + +def restore_smallcaps(line: str) -> str: + if len(SPLIT_INITIAL.findall(line)) < 2: + return line + return SPLIT_INITIAL.sub(lambda m: m.group(1) + m.group(2).lower(), line) + + +def repair_lines(text: str) -> tuple[str, int]: + out, fixed = [], 0 + for ln in text.split("\n"): + r = restore_smallcaps(ln) + fixed += r != ln + out.append(r) + return "\n".join(out), fixed + + +def strip_epigraph(head: str, body: str) -> tuple[str, int]: + """Drop a chapter-head epigraph: everything from after the heading to its attribution. + + Bounded to the first EPIGRAPH_WINDOW characters so an attribution quoted in the middle + of a chapter cannot take the preceding page of Krakauer's own prose with it. + """ + m = ATTRIBUTION.search(body[:EPIGRAPH_WINDOW]) + if not m: + return body, 0 + start = body.find("\n", body.find(head) + len(head)) if head in body[:200] else 0 + if start < 0 or start >= m.start(): + start = 0 + kept = (body[:start] + "\n\n" + body[m.end():]).strip() + removed = len(body.split()) - len(kept.split()) + return (kept, removed) if removed > 0 else (body, 0) + + +def masters(): + c = sqlite3.connect(CATALOG) + rows = c.execute( + "select w.title, w.source_format, m.master_path, m.rights, m.normalized_text_sha256 " + "from masters m join works w on w.current_normalized_content_id = " + "m.normalized_content_id where w.author = 'Jon Krakauer'").fetchall() + out, seen = [], set() + for title, fmt, path, rights, sha in rows: + if title not in WORKS or title in seen: + continue + p = Path(path if os.path.isabs(path) else os.path.join(KVASIR, path)) + if not p.exists(): + print(f" ⚠ MISSING master for {title}: {p}", file=sys.stderr) + continue + seen.add(title) + out.append({"title": title, "slug": WORKS[title], "path": p, "fmt": fmt, + "rights": rights, "sha256": sha}) + return sorted(out, key=lambda w: w["slug"]) + + +def strip_backmatter(text: str, title: str) -> tuple[str, int]: + """Cut at the earliest back-matter marker in the TAIL of the work. + + ⚠ Searching only the LAST UNIT was not enough and the name guard caught it. Where Men + Win Glory's ACKNOWLEDGMENTS sits at 94.8% and the splitter made 41 units, so the + apparatus landed in unit 37 with four more units of NOTES and BIBLIOGRAPHY after it — + all of them past a strip that only ever looked at unit 41. Into the Wild was worse: its + acknowledgments, then a full-page advertisement for Under the Banner of Heaven, survived + untouched. The search window is the last 25% so the same marker words appearing in + Krakauer's FRONT matter (`ALSO BY`, `ABOUT THE AUTHOR`, at 0.0-0.6% of these files) + cannot be mistaken for the end of the book. + """ + start = int(len(text) * 0.75) + m = BACKMATTER.search(text[start:]) + if not m: + return text, 0 + cut = text[:start + m.start()].rstrip() + removed = len(text.split()) - len(cut.split()) + if removed > 0.20 * len(text.split()): + print(f" ⚠ REFUSING back-matter strip on {title}: would remove {removed:,} words " + f"({removed/len(text.split()):.0%})", file=sys.stderr) + return text, 0 + return cut, removed + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--out", required=True) + ap.add_argument("--survey", action="store_true") + a = ap.parse_args() + + works = masters() + if len(works) != len(WORKS): + print(f" ⚠ resolved {len(works)} of {len(WORKS)} works", file=sys.stderr) + if not works: + raise SystemExit("REFUSING: no Krakauer masters resolved from the catalog") + + out = Path(a.out) + alphabet = collections.Counter() + total_words = total_units = epi_total = 0 + own_before = own_after = 0 + manifest = {"corpus": "lv-krakauer", "author": "Jon Krakauer", + "scope": "all four works in the licensed library — ALL NON-FICTION", + "source": "kvasir data/library masters (licensed, rights=gated)", + "built_at": datetime.date.today().isoformat(), + "⚠ unresolved": ("An UNMEASURED fraction of this corpus is quoted material — " + "journals, letters, court transcripts, historical documents. " + "Extraction lost the indentation and italics that marked it, " + "so it cannot be separated from Krakauer's own prose by any " + "signal this builder can read. Only chapter-head epigraphs " + "with a surviving all-caps attribution were removed (0.3%). " + "Training on this teaches a blend of voices."), + "works": []} + + for w in works: + text = w["path"].read_text(encoding="utf-8", errors="replace") + own_before += len(re.findall(r"Krakauer", text)) + text, fixed = repair_lines(text) + text, front_removed = strip_front(text) + text, back_removed = strip_backmatter(text, w["title"]) + mode, units, report = choose_units(text) + epi = 0 + new_units = [] + for head, body in units: + body, n = strip_epigraph(head, body) + epi += n + new_units.append((head, body)) + units = new_units + epi_total += epi + own_after += sum(len(re.findall(r"Krakauer", b)) for _, b in units) + alphabet.update(ch for ch in text if ch.isalpha()) + words = sum(len(b.split()) for _, b in units) + total_words += words + total_units += len(units) + print(f"\n {w['slug']:<26} {len(units):>4} units {words:>8,} words via {mode}" + + (f" [smallcaps {fixed}]" if fixed else "") + + (f" [front -{front_removed}w]" if front_removed else "") + + (f" [back -{back_removed}w]" if back_removed else "") + + (f" [epigraphs -{epi}w]" if epi else "")) + print(format_report(report, mode)) + manifest["works"].append({"slug": w["slug"], "title": w["title"], "rights": w["rights"], + "source_format": w["fmt"], "master_sha256": w["sha256"], + "units": len(units), "words": words, "heading_pattern": mode, + "synthetic_sections": mode == "paragraph-blocks", + "smallcaps_lines_repaired": fixed, + "front_matter_words_removed": front_removed, + "back_matter_words_removed": back_removed, + "epigraph_words_removed": epi, + "mode_report": report, "path": f"works/{w['slug']}.jsonl"}) + if not a.survey: + (out / "works").mkdir(parents=True, exist_ok=True) + with (out / "works" / f"{w['slug']}.jsonl").open("w", encoding="utf-8") as fh: + for i, (head, body) in enumerate(units, 1): + fh.write(json.dumps({"work": w["slug"], "chapter": i, "heading": head, + "words": len(body.split()), "text": body}, + ensure_ascii=False) + "\n") + + print(f"\n NAME GUARD 'Krakauer' in the raw masters {own_before} -> " + f"in the built corpus {own_after} (allowed: {BODY_NAME_OCCURRENCES} body mentions)") + ok_name = own_after <= BODY_NAME_OCCURRENCES + print(f" [{'PASS' if ok_name else 'FAIL'}] the author's own name survives only where he " + f"writes about himself") + if own_after: + for wk in works: + f = out / "works" / f"{wk['slug']}.jsonl" + if a.survey or not f.exists(): + continue + for line in f.read_text(encoding="utf-8").splitlines(): + r = json.loads(line) + for m in re.finditer(r"Krakauer", r["text"]): + ctx = r["text"][max(0, m.start() - 70):m.start() + 50].replace("\n", " ") + print(f" {r['work']} ch{r['chapter']}: …{ctx}…") + if own_after > BODY_NAME_OCCURRENCES: + print(f" ⚠ MORE than the {BODY_NAME_OCCURRENCES} read-and-allowed body mentions. " + f"Read the lines above: publisher apparatus must be stripped, not allowed.") + print(f"\n ⚠ QUOTED-MATERIAL FLOOR — the number below is NOT the answer:") + print(f" removed {epi_total:,} words of chapter-head epigraph = " + f"{epi_total/max(1,total_words):.1%} of the corpus.") + print(f" This method detects ONLY quotations whose all-caps attribution line survived") + print(f" extraction. Journals, letters, depositions and court transcripts lost their") + print(f" indentation and italics and are NOT detected, NOT counted and NOT removed.") + print(f" The fraction of this corpus that is other people's prose is UNKNOWN.") + + non_ascii = {c: n for c, n in alphabet.items() if ord(c) > 127} + print(f"\n TOTAL {total_units} units · {total_words:,} words · {len(alphabet)} distinct letters") + print(f" (Brontë 680,291 · McCarthy 584,756 · Yarros 780,744 · Hemingway 994,760 for scale)") + manifest["totals"] = {"units": total_units, "words": total_words, + "distinct_letters": len(alphabet), + "non_ascii_letters": sum(non_ascii.values()), + "epigraph_words_removed": epi_total, + "quoted_material_fraction": "UNKNOWN — see ⚠ unresolved", + "author_name_occurrences": own_after} + manifest["total_words"] = total_words + manifest["total_chapters"] = total_units + + if not a.survey: + (out / "manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False), + encoding="utf-8") + (out / "corpus_alphabet.json").write_text(json.dumps({ + "derived_from": "Jon Krakauer, 4 non-fiction works, Kvasir licensed library", + "derived_at": manifest["built_at"], + "note": "R49 F02 rule: a rename pool's inventory must be a SUBSET of this.", + "letters": sorted(alphabet), "non_ascii": {c: n for c, n in sorted(non_ascii.items())}, + }, indent=2, ensure_ascii=False), encoding="utf-8") + print(f" wrote {out}") + return 0 if ok_name else 1 + + +if __name__ == "__main__": + raise SystemExit(main())