diff --git a/scripts/mccarthy-corpus/build_corpus_mccarthy.py b/scripts/mccarthy-corpus/build_corpus_mccarthy.py index be67ffa..a823f8f 100644 --- a/scripts/mccarthy-corpus/build_corpus_mccarthy.py +++ b/scripts/mccarthy-corpus/build_corpus_mccarthy.py @@ -94,13 +94,70 @@ BACKMATTER = re.compile( r"VINTAGE INTERNATIONAL|FIRST VINTAGE|Copyright ©|COPYRIGHT|Library of Congress)" r"[ \t]*.{0,60}$", re.M) -# ⚠ A LOST DROP CAP, fixed by name rather than by heuristic. The All the Pretty Horses epub -# opens `HE CANDLEFLAME and the image of the candleflame...` — the decorative T of "THE" was -# a separate image the extractor dropped. A general "restore a missing initial" rule would -# have to guess the letter; this is one known work, one known word, so it is asserted and -# patched explicitly. If the master ever changes, the assertion fails loudly instead of -# silently patching something else. -LOST_INITIAL = {"all-the-pretty-horses": ("HE CANDLEFLAME", "THE CANDLEFLAME")} +# ⚠⚠ THESE EDITIONS SET SECTION OPENINGS IN SMALL CAPS AND THE EXTRACTOR MANGLED THEM THREE +# DIFFERENT WAYS. The entity map is what surfaced it: `E`, `H`, `T` and `K` came back as +# renameable entities with 17-33 capitalised occurrences each, which is the `G` class from +# the Hemingway build — a bare initial is never a name. +# +# 1. SPLIT INITIAL `T HE HOUSE was built` -> `The house was built` 14 cases +# The drop cap survived as its own token. Hemingway's restore_smallcaps only fires on +# TWO or more split initials in a line, so it does not see these single ones. +# 2. UNMARKED RUN `THEY STOOD in the doorway` -> `They stood in the ...` 88 cases +# The small-caps run came through as ordinary capitals with no split at all. Concentrated +# in Cities of the Plain (49) and The Crossing (37). +# 3. LOST INITIAL `HE CANDLEFLAME` -> `THE CANDLEFLAME` 1 case +# The decorative letter was a separate image and is simply gone. +# +# ⚠⚠ RULE 3 IS ANCHORED TO A BLOCK START, AND THAT IS NOT DECORATION. A second entry was +# nearly added here for `HEY RODE` -> `THEY RODE` in Cities of the Plain, because the +# opening showed up in a survey of the BUILT corpus. The raw master has `THEY RODE` intact, +# twice — `HEY RODE` was matching as a SUBSTRING of the correct text, and an unanchored +# replace turned both into `TTHEY RODE`, which rule 2 then lowercased to `Tthey rode`. The +# count assertion caught it (expected 1, found 2) and reading the master settled it. The +# entry is gone and the survivor is anchored so a substring can never fire it. +# +# ⚠ THE `[a-z]` ANCHOR ON RULE 2 IS WHAT MAKES IT SAFE. Lowercasing every all-caps run at a +# block start would eat a genuine shout or a sign; requiring the run to be followed +# immediately by a lowercase word means it is a sentence continuing, which a sign is not. +# Checked before shipping: all 23 distinct first words of the 88 are real words (HE, WHEN, +# THE, THEY, QUINQUAGESIMA ...) except the one `HEY`, which is case 3. +# +# ⚠ AND RULE 1 REQUIRES A FOLLOWING ALL-CAPS WORD, because `A TV was playing` and +# `A Mexican was changing` are an article plus a capitalised word, not a drop cap. Verified +# against all four such probes: `A TV`, `A Mexican`, `A God`, `A Tennessean` are untouched. +SPLIT_INITIAL = re.compile(r"\b([A-Z]) ([A-Z]{2,})\b") +DROPCAP = re.compile(r"(?m)^([A-Z]) ([A-Z]+(?:[ ][A-Z']{2,})+)") +SMALLCAPS_OPENING = re.compile(r"(?m)^([A-Z]{2,}(?:[ ][A-Z']{2,})+)(?=[ ][a-z])") +# Case 3 is patched by name with an expected count, never by heuristic — restoring a letter +# the extractor deleted means guessing it, and a guess that fires silently is worse than a +# missing capital. A master change makes the count wrong and fails the build. +LOST_INITIALS = { + "all-the-pretty-horses": [(re.compile(r"(?m)^HE CANDLEFLAME"), "THE CANDLEFLAME", 1)], +} + + +def restore_smallcaps(line: str) -> str: + if len(SPLIT_INITIAL.findall(line)) < 2: + return line + return SPLIT_INITIAL.sub(lambda m: m.group(1) + m.group(2).lower(), line) + + +def repair_smallcaps(text: str, slug: str) -> tuple[str, dict]: + """Run the three repairs in order; the lost initials MUST go first.""" + stats = {"lost_initial": 0, "split_initial": 0, "unmarked_run": 0} + for pat, good, expect in LOST_INITIALS.get(slug, []): + text, n = pat.subn(good, text) + if n != expect: + print(f" ⚠ {slug}: expected {expect} block-anchored occurrence(s) of " + f"{pat.pattern!r} and replaced {n} — the master changed; re-check before " + f"trusting this build", file=sys.stderr) + stats["lost_initial"] += n + text = "\n".join(restore_smallcaps(ln) for ln in text.split("\n")) + text, stats["split_initial"] = DROPCAP.subn( + lambda m: m.group(1) + m.group(2).lower(), text) + text, stats["unmarked_run"] = SMALLCAPS_OPENING.subn( + lambda m: m.group(1)[0] + m.group(1)[1:].lower(), text) + return text, stats def masters(): @@ -175,16 +232,7 @@ def main() -> int: text = w["path"].read_text(encoding="utf-8", errors="replace") own_name_before += len(re.findall(r"McCarthy", text)) text, back_removed = strip_backmatter(text, w["title"]) - fixed_initial = 0 - if w["slug"] in LOST_INITIAL: - bad, good = LOST_INITIAL[w["slug"]] - if text.lstrip().startswith(bad): - text = text.replace(bad, good, 1) - fixed_initial = 1 - else: - print(f" ⚠ {w['slug']}: expected a lost drop cap {bad!r} at the start and did " - f"not find one — the master changed; re-check before trusting this build", - file=sys.stderr) + text, caps = repair_smallcaps(text, w["slug"]) mode, units, report = choose_units(text) own_name_after += sum(len(re.findall(r"McCarthy", b)) for _, b in units) alphabet.update(ch for ch in text if ch.isalpha()) @@ -193,16 +241,16 @@ def main() -> int: total_units += len(units) synth = " (synthetic sections — the source has no usable divisions)" \ if mode == "paragraph-blocks" else "" + capnote = "".join(f" [{k.replace('_', ' ')} {v}]" for k, v in caps.items() if v) print(f"\n {w['slug']:<24} {len(units):>4} units {words:>8,} words via {mode}{synth}" - + (f" [back -{back_removed}w]" if back_removed else "") - + (" [lost drop cap restored]" if fixed_initial else "")) + + (f" [back -{back_removed}w]" if back_removed else "") + capnote) print(format_report(report, mode)) manifest["works"].append({"slug": w["slug"], "title": w["title"], "rights": w["rights"], "source_format": w["fmt"], "master_sha256": w["sha256"], "units": len(units), "words": words, "heading_pattern": mode, "synthetic_sections": mode == "paragraph-blocks", "back_matter_words_removed": back_removed, - "lost_drop_cap_restored": bool(fixed_initial), + "smallcaps_repairs": caps, "mode_report": report, "path": f"works/{w['slug']}.jsonl"}) if not a.survey: @@ -214,10 +262,9 @@ def main() -> int: ensure_ascii=False) + "\n") # ---- GUARDS: assert the things this corpus is unusual about ---------- - body = "\n".join(b for w in works for _, b in - choose_units(strip_backmatter( - w["path"].read_text(encoding="utf-8", errors="replace"), - w["title"])[0])[1]) + body = "\n".join(b for w in works for _, b in choose_units(repair_smallcaps( + strip_backmatter(w["path"].read_text(encoding="utf-8", errors="replace"), + w["title"])[0], w["slug"])[0])[1]) per10k = len(body.split()) / 10000 or 1 quotes = len(re.findall(r'["“”]', body)) / per10k apos = len(re.findall(r"['’]", body)) / per10k