diff --git a/scripts/r49-corpus/leak_gate.py b/scripts/r49-corpus/leak_gate.py index 6de00c9..621e80e 100644 --- a/scripts/r49-corpus/leak_gate.py +++ b/scripts/r49-corpus/leak_gate.py @@ -63,27 +63,39 @@ def split_scan(source: dict[str, str], copies: dict[str, str], surfaces: list[st src_tokens = Counter() for t in source.values(): src_tokens.update(TOKEN.findall(t)) + # ⚠ ONE ALTERNATION PASS PER TEXT, not one per name -- the same reason `scan()` does it. + # Per-surface scanning is O(surfaces x copies x corpus) and it is not a theoretical cost: + # lv-mccarthy (108 surfaces, 36 copies) finished in 8 s and lv-hemingway (881 surfaces, + # 10 copies) was still running at 5 minutes and had to be killed. A gate too slow to run + # is not a gate. + cands = [s for s in surfaces if len(s) >= 4 and TOKEN.fullmatch(s)] + if not cands: + return {} + cands.sort(key=len, reverse=True) + by_stripped = {} + for s in cands: + by_stripped.setdefault(s, s) # exact form maps to itself + pat = re.compile(r"(? frag_max for f in frags): - continue - forms[form] += 1 - seen_copies.add(name) - if forms: - out[s] = {"forms": dict(forms), "copies": len(seen_copies)} + forms: dict[str, Counter] = defaultdict(Counter) + seen: dict[str, set] = defaultdict(set) + for name, text in copies.items(): + for m in pat.finditer(text): + form = m.group(1) + surf = by_stripped.get(strip.sub("", form)) + if surf is None or form == surf or form in allow: + continue + frags = TOKEN.findall(form) + # A genuine split leaves a fragment that is not a word of this corpus. + if len(frags) < 2 or all(src_tokens[f] > frag_max for f in frags): + continue + forms[surf][form] += 1 + seen[surf].add(name) + for surf, f in forms.items(): + out[surf] = {"forms": dict(f), "copies": len(seen[surf])} return out