diff --git a/scripts/mccarthy-corpus/build_corpus_mccarthy.py b/scripts/mccarthy-corpus/build_corpus_mccarthy.py new file mode 100644 index 0000000..be67ffa --- /dev/null +++ b/scripts/mccarthy-corpus/build_corpus_mccarthy.py @@ -0,0 +1,263 @@ +"""D1 for lv-mccarthy: build the corpus from the licensed Kvasir masters. + +Emits the same record schema as the Brontë, Yarros and Hemingway builders +({work, chapter, heading, words, text} per work file, plus corpus_alphabet.json and +manifest.json) so entities.py, rename.py, leak_gate.py and the trainers run unchanged. + +⭐ THE WORK THIS BUILDER DOES THAT THE OTHERS DID NOT IS PROTECTING A STYLE THAT LOOKS LIKE +DAMAGE. McCarthy uses **no quotation marks at all** and drops the apostrophe from most +contractions — `dont`, `aint`, `wont`, `didnt`. Measured across the six works: + + quotation marks 1 per 10,000 words (Hemingway: 838) + apostrophes 123 per 10,000 words (Hemingway: 241) + semicolons 0 per 10,000 words + +Every one of those numbers is the voice. ⚠⚠ `repair_typography.py` normalises "toward what +the text does" and MUST NOT be run on this corpus — it would read an unquoted line of +dialogue as a defect and put the quotes back, deleting the single most identifiable thing +about the author before training ever starts. This builder therefore runs NO typography +normalisation, and asserts the quote density afterwards so a future well-meaning change +cannot quietly undo it. + +⚠ AND IT MAKES THE VOICE GATE HARDER TO READ, WHICH IS A SEPARATE PROBLEM FOR LATER. +`voice_distance.py` is Burrows's Delta over CHARACTER BIGRAMS. An adapter that learns only +"emit no quotation marks" will move delta_cb a long way without having learned a sentence. +Pre-register a punctuation-normalised secondary read before gating this one, or the axis +will pass for the least interesting reason available. + +WHAT WAS EXCLUDED, MEASURED RATHER THAN ASSUMED. + + 1. TWO TRUNCATED CATALOGUE ROWS. The catalogue holds eight McCarthy rows; two are + fragments that carry a real title, a real author and `triage_disposition = accepted`: + + Blood Meridian [epub] 1,167 words (the book is ~117,000) + The Crossing [epub] 222 words (the book is ~150,000) + + Both hold genuine McCarthy prose, so nothing about the text says "broken" — the epub + extraction simply stopped. Each has a complete **mobi** sibling, which is what this + builder takes. ⚠ The catalogue's `near_dup_pairs` cannot see this: it holds ONE row in + the entire 1,284-work library, and a whole-document simhash cannot match a 222-word + fragment to the book it came from. + + 2. NOTHING ELSE. Cross-work 8-gram containment was measured over all fifteen pairs, on the + Hemingway precedent where a collection turned out to hold four other works at 90-96%. + Worst pair here is **0.10%** (No Country in All the Pretty Horses). Six independent + works; no subsumption, no parody, no non-fiction. + +BACK MATTER RIDES INSIDE THE LAST UNIT IN FOUR OF SIX WORKS, the same defect Yarros and +Hemingway hit, and the marker differs every time: + + The Crossing `THE END` then a dumped `Table of Contents` + Blood Meridian a dumped `Table of Contents` + No Country the author's name on its own line, an About-the-Author, then blurbs + All the Pretty.. a `Reader's Guide`, promo copy for The Road, then the CIP page + +⚠ IT CARRIES THE AUTHOR'S OWN NAME — 19 times in All the Pretty Horses, 7 in No Country — +which is exactly the leak the rename pipeline exists to prevent, sitting in the training +text before the pipeline runs. Markers are matched in FILE ORDER and the earliest wins. + +⚠⚠ AND THE STRIP RUNS **BEFORE** THE SPLIT HERE, INVERTING THE HEMINGWAY ORDER. Blood +Meridian and The Crossing end with a dumped table of contents made of bare roman numerals on +their own lines — which is the exact shape of a chapter marker. Splitting first would feed +the TOC to the splitter as two dozen extra chapters; only the 150-word floor accidentally +saves it today. +""" +from __future__ import annotations +import argparse, collections, datetime, json, os, re, sqlite3, sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "r49-corpus")) +from split_units import choose_units, format_report # noqa: E402 + +CATALOG = "/home/lkraven/development/kvasir/data/library/catalog.sqlite" +KVASIR = "/home/lkraven/development/kvasir" + +# title -> (slug, the format whose extraction is COMPLETE) +WORKS = { + "All the Pretty Horses": ("all-the-pretty-horses", "epub"), + "Blood Meridian Or the Evening Redness in": ("blood-meridian", "mobi"), + "Cities of the Plain": ("cities-of-the-plain", "epub"), + "No country for old men": ("no-country-for-old-men", "epub"), + "The Crossing": ("the-crossing", "mobi"), + "The Road": ("the-road", "epub"), +} +TRUNCATED = { + ("Blood Meridian Or the Evening Redness in the West", "epub"): "1,167 words of a ~117,000-word book", + ("The Crossing", "epub"): "222 words of a ~150,000-word book", +} + +BACKMATTER = re.compile( + r"^[ \t]*(THE END|Table of Contents|TABLE OF CONTENTS|Reader.s Guide|READER.S GUIDE|" + r"Reading Group Guide|READING GROUP GUIDE|Questions? (?:for|and) Discussion|" + r"Cormac McCarthy|About the Author|ABOUT THE AUTHOR|A NOTE ABOUT THE AUTHOR|" + r"Acclaim for|ACCLAIM FOR|Praise for|PRAISE FOR|Also by|ALSO BY|Books by|BOOKS BY|" + r"VINTAGE INTERNATIONAL|FIRST VINTAGE|Copyright ©|COPYRIGHT|Library of Congress)" + r"[ \t]*.{0,60}$", re.M) + +# ⚠ A LOST DROP CAP, fixed by name rather than by heuristic. The All the Pretty Horses epub +# opens `HE CANDLEFLAME and the image of the candleflame...` — the decorative T of "THE" was +# a separate image the extractor dropped. A general "restore a missing initial" rule would +# have to guess the letter; this is one known work, one known word, so it is asserted and +# patched explicitly. If the master ever changes, the assertion fails loudly instead of +# silently patching something else. +LOST_INITIAL = {"all-the-pretty-horses": ("HE CANDLEFLAME", "THE CANDLEFLAME")} + + +def masters(): + c = sqlite3.connect(CATALOG) + rows = c.execute( + "select w.title, w.source_format, m.master_path, m.rights, m.normalized_text_sha256 " + "from masters m join works w on w.current_normalized_content_id = " + "m.normalized_content_id where w.author = 'Cormac McCarthy'").fetchall() + out, seen = [], set() + for title, fmt, path, rights, sha in rows: + if (title, fmt) in TRUNCATED: + print(f" excluded {title[:44]:<46} [{fmt}] — {TRUNCATED[(title, fmt)]}") + continue + if title not in WORKS or WORKS[title][1] != fmt: + continue + if title in seen: + print(f" ⚠ duplicate row for {title}, keeping the first", file=sys.stderr) + continue + p = Path(path if os.path.isabs(path) else os.path.join(KVASIR, path)) + if not p.exists(): + print(f" ⚠ MISSING master for {title}: {p}", file=sys.stderr) + continue + seen.add(title) + out.append({"title": title, "slug": WORKS[title][0], "path": p, "fmt": fmt, + "rights": rights, "sha256": sha}) + return sorted(out, key=lambda w: w["slug"]) + + +def strip_backmatter(text: str, title: str) -> tuple[str, int]: + """Cut at the earliest back-matter marker. Bounded: refuse a strip over 5% of the work.""" + m = BACKMATTER.search(text) + if not m: + return text, 0 + cut = text[:m.start()].rstrip() + removed = len(text.split()) - len(cut.split()) + if removed > 0.05 * len(text.split()): + print(f" ⚠ REFUSING back-matter strip on {title}: would remove {removed:,} words " + f"({removed/len(text.split()):.0%}) — marker {m.group(1)!r} is probably a false " + f"positive inside the novel", file=sys.stderr) + return text, 0 + return cut, removed + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--out", required=True) + ap.add_argument("--survey", action="store_true", help="report and write nothing") + a = ap.parse_args() + + works = masters() + if len(works) != len(WORKS): + print(f" ⚠ resolved {len(works)} of {len(WORKS)} works", file=sys.stderr) + if not works: + raise SystemExit("REFUSING: no McCarthy masters resolved from the catalog") + + out = Path(a.out) + alphabet = collections.Counter() + total_words = total_units = 0 + own_name_before = own_name_after = 0 + manifest = {"corpus": "lv-mccarthy", "author": "Cormac McCarthy", + "scope": "all six complete novels in the licensed library", + "source": "kvasir data/library masters (licensed, rights=gated)", + "built_at": datetime.date.today().isoformat(), + "excluded_truncated_rows": {f"{t} [{f}]": why for (t, f), why in TRUNCATED.items()}, + "containment_checked": "all 15 pairs, worst 0.10% — no work subsumes another", + "typography": ("NO normalisation applied. McCarthy's missing quotation marks " + "and apostrophes are the voice, not damage. Do not run " + "repair_typography.py on this corpus."), + "works": []} + + for w in works: + text = w["path"].read_text(encoding="utf-8", errors="replace") + own_name_before += len(re.findall(r"McCarthy", text)) + text, back_removed = strip_backmatter(text, w["title"]) + fixed_initial = 0 + if w["slug"] in LOST_INITIAL: + bad, good = LOST_INITIAL[w["slug"]] + if text.lstrip().startswith(bad): + text = text.replace(bad, good, 1) + fixed_initial = 1 + else: + print(f" ⚠ {w['slug']}: expected a lost drop cap {bad!r} at the start and did " + f"not find one — the master changed; re-check before trusting this build", + file=sys.stderr) + mode, units, report = choose_units(text) + own_name_after += sum(len(re.findall(r"McCarthy", b)) for _, b in units) + alphabet.update(ch for ch in text if ch.isalpha()) + words = sum(len(b.split()) for _, b in units) + total_words += words + total_units += len(units) + synth = " (synthetic sections — the source has no usable divisions)" \ + if mode == "paragraph-blocks" else "" + print(f"\n {w['slug']:<24} {len(units):>4} units {words:>8,} words via {mode}{synth}" + + (f" [back -{back_removed}w]" if back_removed else "") + + (" [lost drop cap restored]" if fixed_initial else "")) + print(format_report(report, mode)) + manifest["works"].append({"slug": w["slug"], "title": w["title"], "rights": w["rights"], + "source_format": w["fmt"], "master_sha256": w["sha256"], + "units": len(units), "words": words, "heading_pattern": mode, + "synthetic_sections": mode == "paragraph-blocks", + "back_matter_words_removed": back_removed, + "lost_drop_cap_restored": bool(fixed_initial), + "mode_report": report, + "path": f"works/{w['slug']}.jsonl"}) + if not a.survey: + (out / "works").mkdir(parents=True, exist_ok=True) + with (out / "works" / f"{w['slug']}.jsonl").open("w", encoding="utf-8") as fh: + for i, (head, body) in enumerate(units, 1): + fh.write(json.dumps({"work": w["slug"], "chapter": i, "heading": head, + "words": len(body.split()), "text": body}, + ensure_ascii=False) + "\n") + + # ---- GUARDS: assert the things this corpus is unusual about ---------- + body = "\n".join(b for w in works for _, b in + choose_units(strip_backmatter( + w["path"].read_text(encoding="utf-8", errors="replace"), + w["title"])[0])[1]) + per10k = len(body.split()) / 10000 or 1 + quotes = len(re.findall(r'["“”]', body)) / per10k + apos = len(re.findall(r"['’]", body)) / per10k + print(f"\n STYLE GUARD quotation marks {quotes:.1f}/10k · apostrophes {apos:.0f}/10k") + ok_style = quotes < 10 + print(f" [{'PASS' if ok_style else 'FAIL'}] the unquoted-dialogue style survived the build " + f"(expect <10 quote marks per 10k; Hemingway's corpus reads 838)") + print(f" NAME GUARD 'McCarthy' in the raw masters {own_name_before} -> " + f"in the built corpus {own_name_after}") + ok_name = own_name_after == 0 + print(f" [{'PASS' if ok_name else 'FAIL'}] the author's own name is out of the training text") + + non_ascii = {c: n for c, n in alphabet.items() if ord(c) > 127} + print(f"\n TOTAL {total_units} units · {total_words:,} words · {len(alphabet)} distinct letters") + print(f" non-ASCII letters: {sum(non_ascii.values())} across {len(non_ascii)} forms") + print(f" (Brontë 680,291 · Yarros 780,744 · Hemingway 994,760 for scale)") + manifest["totals"] = {"units": total_units, "words": total_words, + "distinct_letters": len(alphabet), + "non_ascii_letters": sum(non_ascii.values()), + "quote_marks_per_10k": round(quotes, 2), + "apostrophes_per_10k": round(apos, 1), + "author_name_occurrences": own_name_after} + manifest["total_words"] = total_words + manifest["total_chapters"] = total_units + + if not a.survey: + (out / "manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8") + (out / "corpus_alphabet.json").write_text(json.dumps({ + "derived_from": "Cormac McCarthy, 6 novels, Kvasir licensed library", + "derived_at": manifest["built_at"], + "note": ("R49 F02 rule: a rename pool's character inventory must be a SUBSET of " + "this. RE-DERIVED on this corpus rather than inherited — measured " + f"{sum(non_ascii.values())} non-ASCII letters across {len(non_ascii)} " + "forms. McCarthy writes Spanish constantly (the Border Trilogy is half " + "set in Mexico), so the Yarros ASCII-only conclusion does NOT transfer."), + "letters": sorted(alphabet), "non_ascii": {c: n for c, n in sorted(non_ascii.items())}, + }, indent=2, ensure_ascii=False), encoding="utf-8") + print(f" wrote {out}") + return 0 if (ok_style and ok_name) else 1 + + +if __name__ == "__main__": + raise SystemExit(main())