perf(leak_gate): one alternation pass for the split scan — lv-hemingway went from timing out at 5 min to 35 s
Per-surface scanning is O(surfaces x copies x corpus). lv-mccarthy (108 surfaces, 36 copies) finished in 8 s; lv-hemingway (881 surfaces, 10 copies) was still running at 5 minutes and had to be killed. A gate too slow to run is not a gate. Same trick scan() already uses: build one alternation, map the matched string back to its surface by stripping separators. Regression: identical verdict and identical per-surface hit counts on the pre-fix lv-mccarthy tree (5 surfaces, 78 hits) and on the fixed one (0). Re-derived on the two shipped corpora with the committed instrument rather than a scratch probe: lv-hemingway GATE FAILED Pasionaria, Primitivo, Chicote -- 6 hits each, all 6 copies lv-bronte GATE PASSED 0
This commit is contained in:
@@ -63,27 +63,39 @@ def split_scan(source: dict[str, str], copies: dict[str, str], surfaces: list[st
|
||||
src_tokens = Counter()
|
||||
for t in source.values():
|
||||
src_tokens.update(TOKEN.findall(t))
|
||||
# ⚠ ONE ALTERNATION PASS PER TEXT, not one per name -- the same reason `scan()` does it.
|
||||
# Per-surface scanning is O(surfaces x copies x corpus) and it is not a theoretical cost:
|
||||
# lv-mccarthy (108 surfaces, 36 copies) finished in 8 s and lv-hemingway (881 surfaces,
|
||||
# 10 copies) was still running at 5 minutes and had to be killed. A gate too slow to run
|
||||
# is not a gate.
|
||||
cands = [s for s in surfaces if len(s) >= 4 and TOKEN.fullmatch(s)]
|
||||
if not cands:
|
||||
return {}
|
||||
cands.sort(key=len, reverse=True)
|
||||
by_stripped = {}
|
||||
for s in cands:
|
||||
by_stripped.setdefault(s, s) # exact form maps to itself
|
||||
pat = re.compile(r"(?<![A-Za-z])(" +
|
||||
"|".join((SPLIT_SEP + "?").join(re.escape(c) for c in s) for s in cands) +
|
||||
r")(?![A-Za-z])")
|
||||
strip = re.compile(SPLIT_SEP)
|
||||
out: dict[str, dict] = {}
|
||||
for s in surfaces:
|
||||
if len(s) < 4 or not TOKEN.fullmatch(s):
|
||||
continue
|
||||
body = (SPLIT_SEP + "?").join(re.escape(c) for c in s)
|
||||
pat = re.compile(r"(?<![A-Za-z])(" + body + r")(?![A-Za-z])")
|
||||
forms: Counter = Counter()
|
||||
seen_copies = set()
|
||||
for name, text in copies.items():
|
||||
for m in pat.finditer(text):
|
||||
form = m.group(1)
|
||||
if form == s or form in allow:
|
||||
continue
|
||||
frags = TOKEN.findall(form)
|
||||
# A genuine split leaves a fragment that is not a word of this corpus.
|
||||
if len(frags) < 2 or all(src_tokens[f] > frag_max for f in frags):
|
||||
continue
|
||||
forms[form] += 1
|
||||
seen_copies.add(name)
|
||||
if forms:
|
||||
out[s] = {"forms": dict(forms), "copies": len(seen_copies)}
|
||||
forms: dict[str, Counter] = defaultdict(Counter)
|
||||
seen: dict[str, set] = defaultdict(set)
|
||||
for name, text in copies.items():
|
||||
for m in pat.finditer(text):
|
||||
form = m.group(1)
|
||||
surf = by_stripped.get(strip.sub("", form))
|
||||
if surf is None or form == surf or form in allow:
|
||||
continue
|
||||
frags = TOKEN.findall(form)
|
||||
# A genuine split leaves a fragment that is not a word of this corpus.
|
||||
if len(frags) < 2 or all(src_tokens[f] > frag_max for f in frags):
|
||||
continue
|
||||
forms[surf][form] += 1
|
||||
seen[surf].add(name)
|
||||
for surf, f in forms.items():
|
||||
out[surf] = {"forms": dict(f), "copies": len(seen[surf])}
|
||||
return out
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user