"""D1 for lv-mccarthy: build the corpus from the licensed Kvasir masters. Emits the same record schema as the Brontë, Yarros and Hemingway builders ({work, chapter, heading, words, text} per work file, plus corpus_alphabet.json and manifest.json) so entities.py, rename.py, leak_gate.py and the trainers run unchanged. ⭐ THE WORK THIS BUILDER DOES THAT THE OTHERS DID NOT IS PROTECTING A STYLE THAT LOOKS LIKE DAMAGE. McCarthy uses **no quotation marks at all** and drops the apostrophe from most contractions — `dont`, `aint`, `wont`, `didnt`. Measured across the six works: quotation marks 1 per 10,000 words (Hemingway: 838) apostrophes 123 per 10,000 words (Hemingway: 241) semicolons 0 per 10,000 words Every one of those numbers is the voice. ⚠⚠ `repair_typography.py` normalises "toward what the text does" and MUST NOT be run on this corpus — it would read an unquoted line of dialogue as a defect and put the quotes back, deleting the single most identifiable thing about the author before training ever starts. This builder therefore runs NO typography normalisation, and asserts the quote density afterwards so a future well-meaning change cannot quietly undo it. ⚠ AND IT MAKES THE VOICE GATE HARDER TO READ, WHICH IS A SEPARATE PROBLEM FOR LATER. `voice_distance.py` is Burrows's Delta over CHARACTER BIGRAMS. An adapter that learns only "emit no quotation marks" will move delta_cb a long way without having learned a sentence. Pre-register a punctuation-normalised secondary read before gating this one, or the axis will pass for the least interesting reason available. WHAT WAS EXCLUDED, MEASURED RATHER THAN ASSUMED. 1. TWO TRUNCATED CATALOGUE ROWS. The catalogue holds eight McCarthy rows; two are fragments that carry a real title, a real author and `triage_disposition = accepted`: Blood Meridian [epub] 1,167 words (the book is ~117,000) The Crossing [epub] 222 words (the book is ~150,000) Both hold genuine McCarthy prose, so nothing about the text says "broken" — the epub extraction simply stopped. Each has a complete **mobi** sibling, which is what this builder takes. ⚠ The catalogue's `near_dup_pairs` cannot see this: it holds ONE row in the entire 1,284-work library, and a whole-document simhash cannot match a 222-word fragment to the book it came from. 2. NOTHING ELSE. Cross-work 8-gram containment was measured over all fifteen pairs, on the Hemingway precedent where a collection turned out to hold four other works at 90-96%. Worst pair here is **0.10%** (No Country in All the Pretty Horses). Six independent works; no subsumption, no parody, no non-fiction. BACK MATTER RIDES INSIDE THE LAST UNIT IN FOUR OF SIX WORKS, the same defect Yarros and Hemingway hit, and the marker differs every time: The Crossing `THE END` then a dumped `Table of Contents` Blood Meridian a dumped `Table of Contents` No Country the author's name on its own line, an About-the-Author, then blurbs All the Pretty.. a `Reader's Guide`, promo copy for The Road, then the CIP page ⚠ IT CARRIES THE AUTHOR'S OWN NAME — 19 times in All the Pretty Horses, 7 in No Country — which is exactly the leak the rename pipeline exists to prevent, sitting in the training text before the pipeline runs. Markers are matched in FILE ORDER and the earliest wins. ⚠⚠ AND THE STRIP RUNS **BEFORE** THE SPLIT HERE, INVERTING THE HEMINGWAY ORDER. Blood Meridian and The Crossing end with a dumped table of contents made of bare roman numerals on their own lines — which is the exact shape of a chapter marker. Splitting first would feed the TOC to the splitter as two dozen extra chapters; only the 150-word floor accidentally saves it today. """ from __future__ import annotations import argparse, collections, datetime, json, os, re, sqlite3, sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "r49-corpus")) from split_units import choose_units, format_report # noqa: E402 CATALOG = "/home/lkraven/development/kvasir/data/library/catalog.sqlite" KVASIR = "/home/lkraven/development/kvasir" # title -> (slug, the format whose extraction is COMPLETE) WORKS = { "All the Pretty Horses": ("all-the-pretty-horses", "epub"), "Blood Meridian Or the Evening Redness in": ("blood-meridian", "mobi"), "Cities of the Plain": ("cities-of-the-plain", "epub"), "No country for old men": ("no-country-for-old-men", "epub"), "The Crossing": ("the-crossing", "mobi"), "The Road": ("the-road", "epub"), } TRUNCATED = { ("Blood Meridian Or the Evening Redness in the West", "epub"): "1,167 words of a ~117,000-word book", ("The Crossing", "epub"): "222 words of a ~150,000-word book", } BACKMATTER = re.compile( r"^[ \t]*(THE END|Table of Contents|TABLE OF CONTENTS|Reader.s Guide|READER.S GUIDE|" r"Reading Group Guide|READING GROUP GUIDE|Questions? (?:for|and) Discussion|" r"Cormac McCarthy|About the Author|ABOUT THE AUTHOR|A NOTE ABOUT THE AUTHOR|" r"Acclaim for|ACCLAIM FOR|Praise for|PRAISE FOR|Also by|ALSO BY|Books by|BOOKS BY|" r"VINTAGE INTERNATIONAL|FIRST VINTAGE|Copyright ©|COPYRIGHT|Library of Congress)" r"[ \t]*.{0,60}$", re.M) # ⚠ A LOST DROP CAP, fixed by name rather than by heuristic. The All the Pretty Horses epub # opens `HE CANDLEFLAME and the image of the candleflame...` — the decorative T of "THE" was # a separate image the extractor dropped. A general "restore a missing initial" rule would # have to guess the letter; this is one known work, one known word, so it is asserted and # patched explicitly. If the master ever changes, the assertion fails loudly instead of # silently patching something else. LOST_INITIAL = {"all-the-pretty-horses": ("HE CANDLEFLAME", "THE CANDLEFLAME")} def masters(): c = sqlite3.connect(CATALOG) rows = c.execute( "select w.title, w.source_format, m.master_path, m.rights, m.normalized_text_sha256 " "from masters m join works w on w.current_normalized_content_id = " "m.normalized_content_id where w.author = 'Cormac McCarthy'").fetchall() out, seen = [], set() for title, fmt, path, rights, sha in rows: if (title, fmt) in TRUNCATED: print(f" excluded {title[:44]:<46} [{fmt}] — {TRUNCATED[(title, fmt)]}") continue if title not in WORKS or WORKS[title][1] != fmt: continue if title in seen: print(f" ⚠ duplicate row for {title}, keeping the first", file=sys.stderr) continue p = Path(path if os.path.isabs(path) else os.path.join(KVASIR, path)) if not p.exists(): print(f" ⚠ MISSING master for {title}: {p}", file=sys.stderr) continue seen.add(title) out.append({"title": title, "slug": WORKS[title][0], "path": p, "fmt": fmt, "rights": rights, "sha256": sha}) return sorted(out, key=lambda w: w["slug"]) def strip_backmatter(text: str, title: str) -> tuple[str, int]: """Cut at the earliest back-matter marker. Bounded: refuse a strip over 5% of the work.""" m = BACKMATTER.search(text) if not m: return text, 0 cut = text[:m.start()].rstrip() removed = len(text.split()) - len(cut.split()) if removed > 0.05 * len(text.split()): print(f" ⚠ REFUSING back-matter strip on {title}: would remove {removed:,} words " f"({removed/len(text.split()):.0%}) — marker {m.group(1)!r} is probably a false " f"positive inside the novel", file=sys.stderr) return text, 0 return cut, removed def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--out", required=True) ap.add_argument("--survey", action="store_true", help="report and write nothing") a = ap.parse_args() works = masters() if len(works) != len(WORKS): print(f" ⚠ resolved {len(works)} of {len(WORKS)} works", file=sys.stderr) if not works: raise SystemExit("REFUSING: no McCarthy masters resolved from the catalog") out = Path(a.out) alphabet = collections.Counter() total_words = total_units = 0 own_name_before = own_name_after = 0 manifest = {"corpus": "lv-mccarthy", "author": "Cormac McCarthy", "scope": "all six complete novels in the licensed library", "source": "kvasir data/library masters (licensed, rights=gated)", "built_at": datetime.date.today().isoformat(), "excluded_truncated_rows": {f"{t} [{f}]": why for (t, f), why in TRUNCATED.items()}, "containment_checked": "all 15 pairs, worst 0.10% — no work subsumes another", "typography": ("NO normalisation applied. McCarthy's missing quotation marks " "and apostrophes are the voice, not damage. Do not run " "repair_typography.py on this corpus."), "works": []} for w in works: text = w["path"].read_text(encoding="utf-8", errors="replace") own_name_before += len(re.findall(r"McCarthy", text)) text, back_removed = strip_backmatter(text, w["title"]) fixed_initial = 0 if w["slug"] in LOST_INITIAL: bad, good = LOST_INITIAL[w["slug"]] if text.lstrip().startswith(bad): text = text.replace(bad, good, 1) fixed_initial = 1 else: print(f" ⚠ {w['slug']}: expected a lost drop cap {bad!r} at the start and did " f"not find one — the master changed; re-check before trusting this build", file=sys.stderr) mode, units, report = choose_units(text) own_name_after += sum(len(re.findall(r"McCarthy", b)) for _, b in units) alphabet.update(ch for ch in text if ch.isalpha()) words = sum(len(b.split()) for _, b in units) total_words += words total_units += len(units) synth = " (synthetic sections — the source has no usable divisions)" \ if mode == "paragraph-blocks" else "" print(f"\n {w['slug']:<24} {len(units):>4} units {words:>8,} words via {mode}{synth}" + (f" [back -{back_removed}w]" if back_removed else "") + (" [lost drop cap restored]" if fixed_initial else "")) print(format_report(report, mode)) manifest["works"].append({"slug": w["slug"], "title": w["title"], "rights": w["rights"], "source_format": w["fmt"], "master_sha256": w["sha256"], "units": len(units), "words": words, "heading_pattern": mode, "synthetic_sections": mode == "paragraph-blocks", "back_matter_words_removed": back_removed, "lost_drop_cap_restored": bool(fixed_initial), "mode_report": report, "path": f"works/{w['slug']}.jsonl"}) if not a.survey: (out / "works").mkdir(parents=True, exist_ok=True) with (out / "works" / f"{w['slug']}.jsonl").open("w", encoding="utf-8") as fh: for i, (head, body) in enumerate(units, 1): fh.write(json.dumps({"work": w["slug"], "chapter": i, "heading": head, "words": len(body.split()), "text": body}, ensure_ascii=False) + "\n") # ---- GUARDS: assert the things this corpus is unusual about ---------- body = "\n".join(b for w in works for _, b in choose_units(strip_backmatter( w["path"].read_text(encoding="utf-8", errors="replace"), w["title"])[0])[1]) per10k = len(body.split()) / 10000 or 1 quotes = len(re.findall(r'["“”]', body)) / per10k apos = len(re.findall(r"['’]", body)) / per10k print(f"\n STYLE GUARD quotation marks {quotes:.1f}/10k · apostrophes {apos:.0f}/10k") ok_style = quotes < 10 print(f" [{'PASS' if ok_style else 'FAIL'}] the unquoted-dialogue style survived the build " f"(expect <10 quote marks per 10k; Hemingway's corpus reads 838)") print(f" NAME GUARD 'McCarthy' in the raw masters {own_name_before} -> " f"in the built corpus {own_name_after}") ok_name = own_name_after == 0 print(f" [{'PASS' if ok_name else 'FAIL'}] the author's own name is out of the training text") non_ascii = {c: n for c, n in alphabet.items() if ord(c) > 127} print(f"\n TOTAL {total_units} units · {total_words:,} words · {len(alphabet)} distinct letters") print(f" non-ASCII letters: {sum(non_ascii.values())} across {len(non_ascii)} forms") print(f" (Brontë 680,291 · Yarros 780,744 · Hemingway 994,760 for scale)") manifest["totals"] = {"units": total_units, "words": total_words, "distinct_letters": len(alphabet), "non_ascii_letters": sum(non_ascii.values()), "quote_marks_per_10k": round(quotes, 2), "apostrophes_per_10k": round(apos, 1), "author_name_occurrences": own_name_after} manifest["total_words"] = total_words manifest["total_chapters"] = total_units if not a.survey: (out / "manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8") (out / "corpus_alphabet.json").write_text(json.dumps({ "derived_from": "Cormac McCarthy, 6 novels, Kvasir licensed library", "derived_at": manifest["built_at"], "note": ("R49 F02 rule: a rename pool's character inventory must be a SUBSET of " "this. RE-DERIVED on this corpus rather than inherited — measured " f"{sum(non_ascii.values())} non-ASCII letters across {len(non_ascii)} " "forms. McCarthy writes Spanish constantly (the Border Trilogy is half " "set in Mexico), so the Yarros ASCII-only conclusion does NOT transfer."), "letters": sorted(alphabet), "non_ascii": {c: n for c, n in sorted(non_ascii.items())}, }, indent=2, ensure_ascii=False), encoding="utf-8") print(f" wrote {out}") return 0 if (ok_style and ok_name) else 1 if __name__ == "__main__": raise SystemExit(main())