"""Re-measure the R49 name pool's token-split distribution under a given tokenizer.

The pool's multi-token property is an operator requirement -- multi-token names
force the drafter to reconstruct a name from the prefix rather than recall it as
one embedding. F02 measured that property with the Qwen3.5-2B tokenizer; the
carrier ruling moved the sweep to Qwen3, whose vocabulary is a different size, so
the property has to be re-measured rather than assumed to carry over.

Names are tokenized with a leading space, matching how they appear mid-sentence.

  python tokenize_name_pool.py <dict.json> <tokenizer-dir-or-repo> [...]
"""
import json, sys, collections
from transformers import AutoTokenizer

pool_path, *tok_paths = sys.argv[1:]
d = json.load(open(pool_path))

given_m, given_f, surnames = [], [], []
for loc, v in d["by_locale"].items():
    given_m += v.get("male", [])
    given_f += v.get("female", [])
    for k in ("surnames_neutral", "surnames_male", "surnames_female"):
        surnames += v.get(k, [])
groups = {"male_given": given_m, "female_given": given_f, "surnames": surnames}
print(f"pool: {sum(len(v) for v in groups.values())} strings "
      f"({', '.join(f'{k} {len(v)}' for k, v in groups.items())})")

for tp in tok_paths:
    tok = AutoTokenizer.from_pretrained(tp)
    print(f"\n== {tp.rstrip('/').split('/')[-1]}  vocab={tok.vocab_size}")
    for gname, names in groups.items():
        hist = collections.Counter()
        tot = 0
        for n in names:
            k = len(tok.encode(" " + n, add_special_tokens=False))
            hist[min(k, 6)] += 1
            tot += k
        n = len(names)
        multi = sum(c for k, c in hist.items() if k >= 2)
        dist = " ".join(f"{k}tok {100*hist[k]/n:4.1f}%" for k in sorted(hist))
        print(f"   {gname:<14} mean {tot/n:.2f}  multi-token {100*multi/n:5.1f}%   {dist}")
    allnames = given_m + given_f + surnames
    tot = sum(len(tok.encode(" " + x, add_special_tokens=False)) for x in allnames)
    multi = sum(1 for x in allnames if len(tok.encode(" " + x, add_special_tokens=False)) >= 2)
    print(f"   {'POOL':<14} mean {tot/len(allnames):.2f}  multi-token {100*multi/len(allnames):5.1f}%")