From 0fa68cb465d33ccfac42f916f3f7d70c97fd9cfc Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Thu, 17 Sep 2026 08:40:31 -0700 Subject: [PATCH] lv-mccarthy D1 fix: three small-caps defects the entity map caught, and one I nearly added MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit D2's entity map returned `E`, `H`, `T` and `K` as renameable entities with 17-33 capitalised occurrences each. A bare initial is never a name -- that is the `G` class from the Hemingway build, where `G` was about to be renamed to a surname 248 times. Reading them in context showed the McCarthy editions set section openings in small caps and the extractor mangled them three different ways, none of which the D1 build repaired: 1. SPLIT INITIAL `T HE HOUSE was built` -> `The house was built` 32 cases Hemingway's restore_smallcaps only fires on TWO or more split initials in a line, so it is structurally blind to these single ones. 2. UNMARKED RUN `THEY STOOD in the doorway` -> `They stood in the ...` 88 cases Concentrated in Cities of the Plain (49) and The Crossing (37). 3. LOST INITIAL `HE CANDLEFLAME` -> `THE CANDLEFLAME` 1 case Rule 1 requires a FOLLOWING all-caps word, because `A TV was playing` and `A Mexican was changing` are an article plus a capitalised word, not a drop cap. All four such probes verified untouched. Rule 2's `[a-z]` lookahead is what makes it safe: lowercasing every all-caps run at a block start would eat a genuine shout or a sign, and requiring the run to be followed immediately by a lowercase word means it is a sentence continuing. All 23 distinct first words of the 88 were checked and are real words -- HE, WHEN, THE, THEY, QUINQUAGESIMA -- except one, which was case 3. ⚠⚠ AND A SECOND LOST-INITIAL ENTRY WAS NEARLY SHIPPED THAT WOULD HAVE CORRUPTED THE TEXT. `HEY RODE` -> `THEY RODE` looked right from a survey of the BUILT corpus. The raw master has `THEY RODE` intact, twice: `HEY RODE` was matching as a SUBSTRING, and the unanchored replace produced `TTHEY RODE`, which rule 2 then lowercased to `Tthey rode`. Two things caught it -- the count assertion (expected 1, replaced 2) and then reading the master. Rule 3 is now a block-anchored regex rather than a string replace, so a substring cannot fire it. ⚠ My first corruption check also missed it, searching for `TTHEY` when the pipeline had already lowercased it to `Tthey`. Check the shape the pipeline actually emits, not the shape you imagined it would. Totals move 584,756 -> 584,716 words, 167 units unchanged. Both guards still pass: quote marks 0.0/10k, author's own name 26 -> 0. Entity map positive control is 14/14 on real McCarthy characters (Glanton, Toadvine, Rawlins, Blevins, Alejandra, Chigurh, Moss, Bell, Boyd, Holden, Tobin, Magdalena, Eduardo, Parham); `T` and `E` no longer appear as entities. --- .../mccarthy-corpus/build_corpus_mccarthy.py | 95 ++++++++++++++----- 1 file changed, 71 insertions(+), 24 deletions(-) diff --git a/scripts/mccarthy-corpus/build_corpus_mccarthy.py b/scripts/mccarthy-corpus/build_corpus_mccarthy.py index be67ffa..a823f8f 100644 --- a/scripts/mccarthy-corpus/build_corpus_mccarthy.py +++ b/scripts/mccarthy-corpus/build_corpus_mccarthy.py @@ -94,13 +94,70 @@ BACKMATTER = re.compile( r"VINTAGE INTERNATIONAL|FIRST VINTAGE|Copyright ©|COPYRIGHT|Library of Congress)" r"[ \t]*.{0,60}$", re.M) -# ⚠ A LOST DROP CAP, fixed by name rather than by heuristic. The All the Pretty Horses epub -# opens `HE CANDLEFLAME and the image of the candleflame...` — the decorative T of "THE" was -# a separate image the extractor dropped. A general "restore a missing initial" rule would -# have to guess the letter; this is one known work, one known word, so it is asserted and -# patched explicitly. If the master ever changes, the assertion fails loudly instead of -# silently patching something else. -LOST_INITIAL = {"all-the-pretty-horses": ("HE CANDLEFLAME", "THE CANDLEFLAME")} +# ⚠⚠ THESE EDITIONS SET SECTION OPENINGS IN SMALL CAPS AND THE EXTRACTOR MANGLED THEM THREE +# DIFFERENT WAYS. The entity map is what surfaced it: `E`, `H`, `T` and `K` came back as +# renameable entities with 17-33 capitalised occurrences each, which is the `G` class from +# the Hemingway build — a bare initial is never a name. +# +# 1. SPLIT INITIAL `T HE HOUSE was built` -> `The house was built` 14 cases +# The drop cap survived as its own token. Hemingway's restore_smallcaps only fires on +# TWO or more split initials in a line, so it does not see these single ones. +# 2. UNMARKED RUN `THEY STOOD in the doorway` -> `They stood in the ...` 88 cases +# The small-caps run came through as ordinary capitals with no split at all. Concentrated +# in Cities of the Plain (49) and The Crossing (37). +# 3. LOST INITIAL `HE CANDLEFLAME` -> `THE CANDLEFLAME` 1 case +# The decorative letter was a separate image and is simply gone. +# +# ⚠⚠ RULE 3 IS ANCHORED TO A BLOCK START, AND THAT IS NOT DECORATION. A second entry was +# nearly added here for `HEY RODE` -> `THEY RODE` in Cities of the Plain, because the +# opening showed up in a survey of the BUILT corpus. The raw master has `THEY RODE` intact, +# twice — `HEY RODE` was matching as a SUBSTRING of the correct text, and an unanchored +# replace turned both into `TTHEY RODE`, which rule 2 then lowercased to `Tthey rode`. The +# count assertion caught it (expected 1, found 2) and reading the master settled it. The +# entry is gone and the survivor is anchored so a substring can never fire it. +# +# ⚠ THE `[a-z]` ANCHOR ON RULE 2 IS WHAT MAKES IT SAFE. Lowercasing every all-caps run at a +# block start would eat a genuine shout or a sign; requiring the run to be followed +# immediately by a lowercase word means it is a sentence continuing, which a sign is not. +# Checked before shipping: all 23 distinct first words of the 88 are real words (HE, WHEN, +# THE, THEY, QUINQUAGESIMA ...) except the one `HEY`, which is case 3. +# +# ⚠ AND RULE 1 REQUIRES A FOLLOWING ALL-CAPS WORD, because `A TV was playing` and +# `A Mexican was changing` are an article plus a capitalised word, not a drop cap. Verified +# against all four such probes: `A TV`, `A Mexican`, `A God`, `A Tennessean` are untouched. +SPLIT_INITIAL = re.compile(r"\b([A-Z]) ([A-Z]{2,})\b") +DROPCAP = re.compile(r"(?m)^([A-Z]) ([A-Z]+(?:[ ][A-Z']{2,})+)") +SMALLCAPS_OPENING = re.compile(r"(?m)^([A-Z]{2,}(?:[ ][A-Z']{2,})+)(?=[ ][a-z])") +# Case 3 is patched by name with an expected count, never by heuristic — restoring a letter +# the extractor deleted means guessing it, and a guess that fires silently is worse than a +# missing capital. A master change makes the count wrong and fails the build. +LOST_INITIALS = { + "all-the-pretty-horses": [(re.compile(r"(?m)^HE CANDLEFLAME"), "THE CANDLEFLAME", 1)], +} + + +def restore_smallcaps(line: str) -> str: + if len(SPLIT_INITIAL.findall(line)) < 2: + return line + return SPLIT_INITIAL.sub(lambda m: m.group(1) + m.group(2).lower(), line) + + +def repair_smallcaps(text: str, slug: str) -> tuple[str, dict]: + """Run the three repairs in order; the lost initials MUST go first.""" + stats = {"lost_initial": 0, "split_initial": 0, "unmarked_run": 0} + for pat, good, expect in LOST_INITIALS.get(slug, []): + text, n = pat.subn(good, text) + if n != expect: + print(f" ⚠ {slug}: expected {expect} block-anchored occurrence(s) of " + f"{pat.pattern!r} and replaced {n} — the master changed; re-check before " + f"trusting this build", file=sys.stderr) + stats["lost_initial"] += n + text = "\n".join(restore_smallcaps(ln) for ln in text.split("\n")) + text, stats["split_initial"] = DROPCAP.subn( + lambda m: m.group(1) + m.group(2).lower(), text) + text, stats["unmarked_run"] = SMALLCAPS_OPENING.subn( + lambda m: m.group(1)[0] + m.group(1)[1:].lower(), text) + return text, stats def masters(): @@ -175,16 +232,7 @@ def main() -> int: text = w["path"].read_text(encoding="utf-8", errors="replace") own_name_before += len(re.findall(r"McCarthy", text)) text, back_removed = strip_backmatter(text, w["title"]) - fixed_initial = 0 - if w["slug"] in LOST_INITIAL: - bad, good = LOST_INITIAL[w["slug"]] - if text.lstrip().startswith(bad): - text = text.replace(bad, good, 1) - fixed_initial = 1 - else: - print(f" ⚠ {w['slug']}: expected a lost drop cap {bad!r} at the start and did " - f"not find one — the master changed; re-check before trusting this build", - file=sys.stderr) + text, caps = repair_smallcaps(text, w["slug"]) mode, units, report = choose_units(text) own_name_after += sum(len(re.findall(r"McCarthy", b)) for _, b in units) alphabet.update(ch for ch in text if ch.isalpha()) @@ -193,16 +241,16 @@ def main() -> int: total_units += len(units) synth = " (synthetic sections — the source has no usable divisions)" \ if mode == "paragraph-blocks" else "" + capnote = "".join(f" [{k.replace('_', ' ')} {v}]" for k, v in caps.items() if v) print(f"\n {w['slug']:<24} {len(units):>4} units {words:>8,} words via {mode}{synth}" - + (f" [back -{back_removed}w]" if back_removed else "") - + (" [lost drop cap restored]" if fixed_initial else "")) + + (f" [back -{back_removed}w]" if back_removed else "") + capnote) print(format_report(report, mode)) manifest["works"].append({"slug": w["slug"], "title": w["title"], "rights": w["rights"], "source_format": w["fmt"], "master_sha256": w["sha256"], "units": len(units), "words": words, "heading_pattern": mode, "synthetic_sections": mode == "paragraph-blocks", "back_matter_words_removed": back_removed, - "lost_drop_cap_restored": bool(fixed_initial), + "smallcaps_repairs": caps, "mode_report": report, "path": f"works/{w['slug']}.jsonl"}) if not a.survey: @@ -214,10 +262,9 @@ def main() -> int: ensure_ascii=False) + "\n") # ---- GUARDS: assert the things this corpus is unusual about ---------- - body = "\n".join(b for w in works for _, b in - choose_units(strip_backmatter( - w["path"].read_text(encoding="utf-8", errors="replace"), - w["title"])[0])[1]) + body = "\n".join(b for w in works for _, b in choose_units(repair_smallcaps( + strip_backmatter(w["path"].read_text(encoding="utf-8", errors="replace"), + w["title"])[0], w["slug"])[0])[1]) per10k = len(body.split()) / 10000 or 1 quotes = len(re.findall(r'["“”]', body)) / per10k apos = len(re.findall(r"['’]", body)) / per10k