"""D1 for lv-krakauer: build the corpus from the licensed Kvasir masters. Same record schema as the Brontë, Yarros, Hemingway and McCarthy builders, so entities.py, rename.py, leak_gate.py and the trainers run unchanged. ⚠⚠⚠ READ THIS BEFORE USING THE CORPUS THIS BUILDS. IT IS THE FIRST NON-FICTION VOICE CORPUS IN THIS LINE, AND AN UNMEASURED FRACTION OF IT IS OTHER PEOPLE'S PROSE. Krakauer quotes constantly and at length — Chris McCandless's journals and letters, Pat Tillman's diaries, court transcripts, depositions, nineteenth-century Mormon documents, newspaper reports, and full paragraphs of Jack London, Wallace Stegner and Thoreau at the chapter heads. In print those are set as indented block quotes or italics. **The extraction lost both**, so inside the master they are ordinary paragraphs, indistinguishable from Krakauer's own sentences by any signal this builder can read. What IS detectable is stripped, and it is almost nothing — **52 words** across four works, 0.01% of the corpus: chapter-head epigraphs found by a comma-bearing all-caps attribution line ("JACK LONDON, THE CALL OF THE WILD"). A first estimate put this at ~1,500 words by walking back three paragraph blocks from each attribution; the shipped rule is bounded to the chapter head and takes far less, which is the safer error to make. ⭐ **THE REMAINING QUOTED MATERIAL IS NOT MEASURED AND THIS BUILDER CANNOT MEASURE IT.** Stating the floor rather than letting a 0.3% number travel as if it were the answer: this method detects only quotations whose attribution survived as an all-caps line, and it would report 0.0% for a book made entirely of undated block quotes. Anyone deciding whether to train on this needs that sentence, not the 0.3%. The parallel is exact: Hemingway's scope was cut to fiction-only by an operator ruling, and `The Torrents of Spring` was excluded because a parody is "the target author's name on a different author's style, i.e. mislabelled data for a voice adapter". Embedded quotation is the same error, distributed rather than concentrated, and the fraction is unknown. STRUCTURE, which differs from every earlier author in this line. ⚠ THE FRONT MATTER IS AT THE FRONT AND IT IS ENORMOUS. Three of the four ebooks open with `Acclaim for`, `ALSO BY JON KRAKAUER`, `Copyright` and `About the Author` inside the first 0.6% of the file. A back-matter marker search that takes the earliest hit — which is what the McCarthy builder does, correctly, for McCarthy — would cut 99.9% of the book here. So both strips are WINDOWED: front matter is cut at the author's own signature inside the first 10% of the file, back matter at the earliest apparatus marker inside the last 25%. ⚠ RELYING ON THE SPLITTER TO DROP THE FRONT MATTER WAS THE FIRST DESIGN AND IT DID NOT WORK. Units begin at the first heading mark, so front matter is dropped only when no mark falls inside it — and in Missoula and Where Men Win Glory the ebook's table of contents sits above the author's note, giving the splitter a `Chapter Thirty-Two` to start on. Unit 1 then swallowed the whole apparatus. The name guard is what caught it. ⚠ THE SMALL-CAPS SPLIT-INITIAL DEFECT IS PRESENT AND MUST BE REPAIRED BEFORE THE SPLIT. `J ACK L ONDON , W HITE F ANG` and `A LEXANDER.` are the Hemingway `T HE O LD M AN` defect again. Two of the four works split on `caps-title`, so a damaged heading is a heading the splitter cannot see — repairing afterwards would be too late. """ from __future__ import annotations import argparse, collections, datetime, json, os, re, sqlite3, sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "r49-corpus")) from split_units import choose_units, format_report # noqa: E402 CATALOG = "/home/lkraven/development/kvasir/data/library/catalog.sqlite" KVASIR = "/home/lkraven/development/kvasir" WORKS = { "Into the Wild": "into-the-wild", "Missoula: Rape and the Justice System in a College Town": "missoula", "Under the Banner of Heaven: A Story of Violent Faith": "under-the-banner-of-heaven", "Where Men Win Glory": "where-men-win-glory", } # Back matter, matched only inside the LAST unit. `NOTES` and `BIBLIOGRAPHY` are source # apparatus: thousands of words of citations that are not prose in any register. BACKMATTER = re.compile( r"^[ \t]*(AUTHOR.S NOTE|ACKNOWLEDGMENTS|ACKNOWLEDGEMENTS|SELECTED BIBLIOGRAPHY|" r"BIBLIOGRAPHY|NOTES|INDEX|ABOUT THE AUTHOR|ALSO BY|APPENDIX)[ \t]*.{0,40}$", re.M) SPLIT_INITIAL = re.compile(r"\b([A-Z]) ([A-Z]{2,})\b") # ⚠ THE AUTHOR SIGNS HIS OWN FRONT MATTER. Missoula and Where Men Win Glory open with an # author's note closing "Jon Krakauer , February 2015" immediately before PART ONE, and the # ebook's table of contents sits above that — so the splitter's first heading mark lands # inside the front matter and unit 1 swallows the lot. Cutting at the signature is exact # where a marker list would be guesswork, and it is bounded to the first 10% of the file. AUTHOR_SIGNATURE = re.compile(r"^[ \t]*Jon Krakauer[ \t]*,?[ \t]*" r"(?:January|February|March|April|May|June|July|August|" r"September|October|November|December)?[ \t]*\d{4}[ \t]*$", re.M) # An all-caps line WITH a comma is a source attribution ("HENRY DAVID THOREAU, JOURNAL"); # without one it is a chapter title ("THE STAMPEDE TRAIL", "DETRITAL WASH"). Verified by # reading every caps line in the two works that split on caps-title. ATTRIBUTION = re.compile(r"^[ \t]*([A-Z][A-Z '\-.!]{3,40},[A-Z '\-.!][A-Z '\-,.!]{3,60})[ \t]*$", re.M) EPIGRAPH_WINDOW = 1200 # an epigraph sits at the chapter head, not in the middle of it # ⚠ ZERO IS THE WRONG BAR FOR THIS AUTHOR, and the difference matters. Hemingway's own name # in his training text was always PUBLISHER APPARATUS — a jacket biography, an editor's cast # list — and stripping it to 0 was correct. Krakauer writes about himself: Into the Wild # devotes two chapters to his own youth, so `Lewis Krakauer loved his five children deeply` # is his prose, not a jacket blurb, and the second survivor is a reader's letter he quotes # calling him a kook. Both were read before being allowed. The rename pipeline treats # `Krakauer` as an ordinary capitalised surface and renames it downstream like any other. # The number is pinned so that a master change, or a strip that stops working, fails loudly # instead of widening silently. BODY_NAME_OCCURRENCES = 2 def strip_front(text: str) -> tuple[str, int]: """Cut everything up to and including the author's signed front-matter note.""" window = text[:int(len(text) * 0.10)] hits = list(AUTHOR_SIGNATURE.finditer(window)) if not hits: return text, 0 cut = text[hits[-1].end():].lstrip() return cut, len(text.split()) - len(cut.split()) def restore_smallcaps(line: str) -> str: if len(SPLIT_INITIAL.findall(line)) < 2: return line return SPLIT_INITIAL.sub(lambda m: m.group(1) + m.group(2).lower(), line) def repair_lines(text: str) -> tuple[str, int]: out, fixed = [], 0 for ln in text.split("\n"): r = restore_smallcaps(ln) fixed += r != ln out.append(r) return "\n".join(out), fixed def strip_epigraph(head: str, body: str) -> tuple[str, int]: """Drop a chapter-head epigraph: everything from after the heading to its attribution. Bounded to the first EPIGRAPH_WINDOW characters so an attribution quoted in the middle of a chapter cannot take the preceding page of Krakauer's own prose with it. """ m = ATTRIBUTION.search(body[:EPIGRAPH_WINDOW]) if not m: return body, 0 start = body.find("\n", body.find(head) + len(head)) if head in body[:200] else 0 if start < 0 or start >= m.start(): start = 0 kept = (body[:start] + "\n\n" + body[m.end():]).strip() removed = len(body.split()) - len(kept.split()) return (kept, removed) if removed > 0 else (body, 0) def masters(): c = sqlite3.connect(CATALOG) rows = c.execute( "select w.title, w.source_format, m.master_path, m.rights, m.normalized_text_sha256 " "from masters m join works w on w.current_normalized_content_id = " "m.normalized_content_id where w.author = 'Jon Krakauer'").fetchall() out, seen = [], set() for title, fmt, path, rights, sha in rows: if title not in WORKS or title in seen: continue p = Path(path if os.path.isabs(path) else os.path.join(KVASIR, path)) if not p.exists(): print(f" ⚠ MISSING master for {title}: {p}", file=sys.stderr) continue seen.add(title) out.append({"title": title, "slug": WORKS[title], "path": p, "fmt": fmt, "rights": rights, "sha256": sha}) return sorted(out, key=lambda w: w["slug"]) def strip_backmatter(text: str, title: str) -> tuple[str, int]: """Cut at the earliest back-matter marker in the TAIL of the work. ⚠ Searching only the LAST UNIT was not enough and the name guard caught it. Where Men Win Glory's ACKNOWLEDGMENTS sits at 94.8% and the splitter made 41 units, so the apparatus landed in unit 37 with four more units of NOTES and BIBLIOGRAPHY after it — all of them past a strip that only ever looked at unit 41. Into the Wild was worse: its acknowledgments, then a full-page advertisement for Under the Banner of Heaven, survived untouched. The search window is the last 25% so the same marker words appearing in Krakauer's FRONT matter (`ALSO BY`, `ABOUT THE AUTHOR`, at 0.0-0.6% of these files) cannot be mistaken for the end of the book. """ start = int(len(text) * 0.75) m = BACKMATTER.search(text[start:]) if not m: return text, 0 cut = text[:start + m.start()].rstrip() removed = len(text.split()) - len(cut.split()) if removed > 0.20 * len(text.split()): print(f" ⚠ REFUSING back-matter strip on {title}: would remove {removed:,} words " f"({removed/len(text.split()):.0%})", file=sys.stderr) return text, 0 return cut, removed def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--out", required=True) ap.add_argument("--survey", action="store_true") a = ap.parse_args() works = masters() if len(works) != len(WORKS): print(f" ⚠ resolved {len(works)} of {len(WORKS)} works", file=sys.stderr) if not works: raise SystemExit("REFUSING: no Krakauer masters resolved from the catalog") out = Path(a.out) alphabet = collections.Counter() total_words = total_units = epi_total = 0 own_before = own_after = 0 manifest = {"corpus": "lv-krakauer", "author": "Jon Krakauer", "scope": "all four works in the licensed library — ALL NON-FICTION", "source": "kvasir data/library masters (licensed, rights=gated)", "built_at": datetime.date.today().isoformat(), "⚠ unresolved": ("An UNMEASURED fraction of this corpus is quoted material — " "journals, letters, court transcripts, historical documents. " "Extraction lost the indentation and italics that marked it, " "so it cannot be separated from Krakauer's own prose by any " "signal this builder can read. Only chapter-head epigraphs " "with a surviving all-caps attribution were removed (0.3%). " "Training on this teaches a blend of voices."), "works": []} for w in works: text = w["path"].read_text(encoding="utf-8", errors="replace") own_before += len(re.findall(r"Krakauer", text)) text, fixed = repair_lines(text) text, front_removed = strip_front(text) text, back_removed = strip_backmatter(text, w["title"]) mode, units, report = choose_units(text) epi = 0 new_units = [] for head, body in units: body, n = strip_epigraph(head, body) epi += n new_units.append((head, body)) units = new_units epi_total += epi own_after += sum(len(re.findall(r"Krakauer", b)) for _, b in units) alphabet.update(ch for ch in text if ch.isalpha()) words = sum(len(b.split()) for _, b in units) total_words += words total_units += len(units) print(f"\n {w['slug']:<26} {len(units):>4} units {words:>8,} words via {mode}" + (f" [smallcaps {fixed}]" if fixed else "") + (f" [front -{front_removed}w]" if front_removed else "") + (f" [back -{back_removed}w]" if back_removed else "") + (f" [epigraphs -{epi}w]" if epi else "")) print(format_report(report, mode)) manifest["works"].append({"slug": w["slug"], "title": w["title"], "rights": w["rights"], "source_format": w["fmt"], "master_sha256": w["sha256"], "units": len(units), "words": words, "heading_pattern": mode, "synthetic_sections": mode == "paragraph-blocks", "smallcaps_lines_repaired": fixed, "front_matter_words_removed": front_removed, "back_matter_words_removed": back_removed, "epigraph_words_removed": epi, "mode_report": report, "path": f"works/{w['slug']}.jsonl"}) if not a.survey: (out / "works").mkdir(parents=True, exist_ok=True) with (out / "works" / f"{w['slug']}.jsonl").open("w", encoding="utf-8") as fh: for i, (head, body) in enumerate(units, 1): fh.write(json.dumps({"work": w["slug"], "chapter": i, "heading": head, "words": len(body.split()), "text": body}, ensure_ascii=False) + "\n") print(f"\n NAME GUARD 'Krakauer' in the raw masters {own_before} -> " f"in the built corpus {own_after} (allowed: {BODY_NAME_OCCURRENCES} body mentions)") ok_name = own_after <= BODY_NAME_OCCURRENCES print(f" [{'PASS' if ok_name else 'FAIL'}] the author's own name survives only where he " f"writes about himself") if own_after: for wk in works: f = out / "works" / f"{wk['slug']}.jsonl" if a.survey or not f.exists(): continue for line in f.read_text(encoding="utf-8").splitlines(): r = json.loads(line) for m in re.finditer(r"Krakauer", r["text"]): ctx = r["text"][max(0, m.start() - 70):m.start() + 50].replace("\n", " ") print(f" {r['work']} ch{r['chapter']}: …{ctx}…") if own_after > BODY_NAME_OCCURRENCES: print(f" ⚠ MORE than the {BODY_NAME_OCCURRENCES} read-and-allowed body mentions. " f"Read the lines above: publisher apparatus must be stripped, not allowed.") print(f"\n ⚠ QUOTED-MATERIAL FLOOR — the number below is NOT the answer:") print(f" removed {epi_total:,} words of chapter-head epigraph = " f"{epi_total/max(1,total_words):.1%} of the corpus.") print(f" This method detects ONLY quotations whose all-caps attribution line survived") print(f" extraction. Journals, letters, depositions and court transcripts lost their") print(f" indentation and italics and are NOT detected, NOT counted and NOT removed.") print(f" The fraction of this corpus that is other people's prose is UNKNOWN.") non_ascii = {c: n for c, n in alphabet.items() if ord(c) > 127} print(f"\n TOTAL {total_units} units · {total_words:,} words · {len(alphabet)} distinct letters") print(f" (Brontë 680,291 · McCarthy 584,756 · Yarros 780,744 · Hemingway 994,760 for scale)") manifest["totals"] = {"units": total_units, "words": total_words, "distinct_letters": len(alphabet), "non_ascii_letters": sum(non_ascii.values()), "epigraph_words_removed": epi_total, "quoted_material_fraction": "UNKNOWN — see ⚠ unresolved", "author_name_occurrences": own_after} manifest["total_words"] = total_words manifest["total_chapters"] = total_units if not a.survey: (out / "manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8") (out / "corpus_alphabet.json").write_text(json.dumps({ "derived_from": "Jon Krakauer, 4 non-fiction works, Kvasir licensed library", "derived_at": manifest["built_at"], "note": "R49 F02 rule: a rename pool's inventory must be a SUBSET of this.", "letters": sorted(alphabet), "non_ascii": {c: n for c, n in sorted(non_ascii.items())}, }, indent=2, ensure_ascii=False), encoding="utf-8") print(f" wrote {out}") return 0 if ok_name else 1 if __name__ == "__main__": raise SystemExit(main())