diff --git a/scripts/training-probes/name-pool-tokens-2026-09-09.txt b/scripts/training-probes/name-pool-tokens-2026-09-09.txt new file mode 100644 index 0000000..19ba581 --- /dev/null +++ b/scripts/training-probes/name-pool-tokens-2026-09-09.txt @@ -0,0 +1,36 @@ +R49 name-pool token-split re-measurement after the dense-Qwen3 carrier ruling. +Run 2026-09-09 23:2x PT on pfi-gx10 via scripts/training-probes/tokenize_name_pool.py +against brokkr-smithy research/R49-author-voice-adapters/tools/name_dictionary.json +(unmodified). Names tokenized with a leading space. Pool DEDUPED across locales, +which reconciles with the dictionary's own `totals` block: male_given 5,339 and +female_given 5,226 match exactly. + +dictionary totals block: {"male_given": 5339, "female_given": 5226, + "surnames_neutral": 11840, "surnames_gendered_pairs": 3, + "ambiguous_dropped": 167} +deduped measured : {"male_given": 5339, "female_given": 5226, + "surnames": 13549} sum 24114 + (surnames differs because this unions surnames_male/surnames_female in as well.) + +== Qwen3-1.7B-Base config vocab 151,936 tokenizer.vocab_size 151,643 + male_given mean 2.40 multi 87.1% 1tok 12.9% 2tok 47.2% 3tok 29.0% 4tok 9.1% 5tok 1.6% 6tok 0.2% + female_given mean 2.44 multi 92.3% 1tok 7.7% 2tok 51.7% 3tok 30.9% 4tok 8.5% 5tok 1.1% 6tok 0.2% + surnames mean 2.48 multi 90.7% 1tok 9.3% 2tok 44.7% 3tok 35.9% 4tok 9.0% 5tok 1.2% 6tok 0.1% + POOL mean 2.46 multi 90.3% + +== Qwen3.5-2B-Base config vocab 248,320 tokenizer.vocab_size 248,044 + male_given mean 2.25 multi 83.6% 1tok 16.4% 2tok 50.5% 3tok 25.5% 4tok 6.6% 5tok 0.9% 6tok 0.0% + female_given mean 2.33 multi 90.2% 1tok 9.8% 2tok 55.5% 3tok 27.8% 4tok 6.1% 5tok 0.9% 6tok 0.0% + surnames mean 2.36 multi 88.8% 1tok 11.2% 2tok 49.2% 3tok 32.9% 4tok 6.1% 5tok 0.5% 6tok 0.0% + POOL mean 2.33 multi 88.0% + +READ: the multi-token property STRENGTHENS on the dense carrier, 88.0% -> 90.3%, +mean 2.33 -> 2.46. A smaller vocabulary fragments more, so Qwen3's 151,936 splits +names into more pieces than Qwen3.5's 248,320. The operator's requirement -- +multi-token names forcing reconstruction from the prefix rather than recall of one +embedding -- is better served after the ruling, not worse. + +POSITIVE CONTROL: the Qwen3.5 column reproduces R49 F02's published figure for the +same pool on the same tokenizer (F02: 89% multi-token, mean 2.35; here: 88.0%, +2.33). Within a point on both, so the instrument recovers a known-true value +before being asked about an unknown one. diff --git a/scripts/training-probes/tokenize_name_pool.py b/scripts/training-probes/tokenize_name_pool.py new file mode 100644 index 0000000..479b100 --- /dev/null +++ b/scripts/training-probes/tokenize_name_pool.py @@ -0,0 +1,46 @@ +"""Re-measure the R49 name pool's token-split distribution under a given tokenizer. + +The pool's multi-token property is an operator requirement -- multi-token names +force the drafter to reconstruct a name from the prefix rather than recall it as +one embedding. F02 measured that property with the Qwen3.5-2B tokenizer; the +carrier ruling moved the sweep to Qwen3, whose vocabulary is a different size, so +the property has to be re-measured rather than assumed to carry over. + +Names are tokenized with a leading space, matching how they appear mid-sentence. + + python tokenize_name_pool.py [...] +""" +import json, sys, collections +from transformers import AutoTokenizer + +pool_path, *tok_paths = sys.argv[1:] +d = json.load(open(pool_path)) + +given_m, given_f, surnames = [], [], [] +for loc, v in d["by_locale"].items(): + given_m += v.get("male", []) + given_f += v.get("female", []) + for k in ("surnames_neutral", "surnames_male", "surnames_female"): + surnames += v.get(k, []) +groups = {"male_given": given_m, "female_given": given_f, "surnames": surnames} +print(f"pool: {sum(len(v) for v in groups.values())} strings " + f"({', '.join(f'{k} {len(v)}' for k, v in groups.items())})") + +for tp in tok_paths: + tok = AutoTokenizer.from_pretrained(tp) + print(f"\n== {tp.rstrip('/').split('/')[-1]} vocab={tok.vocab_size}") + for gname, names in groups.items(): + hist = collections.Counter() + tot = 0 + for n in names: + k = len(tok.encode(" " + n, add_special_tokens=False)) + hist[min(k, 6)] += 1 + tot += k + n = len(names) + multi = sum(c for k, c in hist.items() if k >= 2) + dist = " ".join(f"{k}tok {100*hist[k]/n:4.1f}%" for k in sorted(hist)) + print(f" {gname:<14} mean {tot/n:.2f} multi-token {100*multi/n:5.1f}% {dist}") + allnames = given_m + given_f + surnames + tot = sum(len(tok.encode(" " + x, add_special_tokens=False)) for x in allnames) + multi = sum(1 for x in allnames if len(tok.encode(" " + x, add_special_tokens=False)) >= 2) + print(f" {'POOL':<14} mean {tot/len(allnames):.2f} multi-token {100*multi/len(allnames):5.1f}%")