From 04950c28813102d63c26a72f88f56eef0a9184c1 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Wed, 9 Sep 2026 22:59:15 -0700 Subject: [PATCH] feat(training-probes): re-measure the R49 name pool under the Qwen3 tokenizer brokkr-smithy flagged that R49 F02's name-pool token splits were measured with the Qwen3.5-2B tokenizer, so the dense-Qwen3 carrier ruling invalidates them. Measured rather than left on their critical path; handed over as input to their re-check, since the dictionary and the adjudication are theirs. The multi-token property strengthens on the chosen carrier: pool multi-token 88.0% -> 90.3%, mean tokens 2.33 -> 2.46. A smaller vocabulary fragments more, so Qwen3's 151,936 splits names into more pieces than Qwen3.5's 248,320. The operator's requirement that names be multi-token, so the drafter reconstructs them from the prefix instead of recalling one embedding, is better served after the ruling. Positive control: the Qwen3.5 column reproduces F02's published figure on the same pool and tokenizer (F02 89% / mean 2.35; here 88.0% / 2.33), so the instrument recovers a known-true value before being asked about an unknown one. The pool is deduped across locales, which reconciles male_given and female_given exactly against the dictionary's own totals block. --- .../name-pool-tokens-2026-09-09.txt | 36 +++++++++++++++ scripts/training-probes/tokenize_name_pool.py | 46 +++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 scripts/training-probes/name-pool-tokens-2026-09-09.txt create mode 100644 scripts/training-probes/tokenize_name_pool.py diff --git a/scripts/training-probes/name-pool-tokens-2026-09-09.txt b/scripts/training-probes/name-pool-tokens-2026-09-09.txt new file mode 100644 index 0000000..19ba581 --- /dev/null +++ b/scripts/training-probes/name-pool-tokens-2026-09-09.txt @@ -0,0 +1,36 @@ +R49 name-pool token-split re-measurement after the dense-Qwen3 carrier ruling. +Run 2026-09-09 23:2x PT on pfi-gx10 via scripts/training-probes/tokenize_name_pool.py +against brokkr-smithy research/R49-author-voice-adapters/tools/name_dictionary.json +(unmodified). Names tokenized with a leading space. Pool DEDUPED across locales, +which reconciles with the dictionary's own `totals` block: male_given 5,339 and +female_given 5,226 match exactly. + +dictionary totals block: {"male_given": 5339, "female_given": 5226, + "surnames_neutral": 11840, "surnames_gendered_pairs": 3, + "ambiguous_dropped": 167} +deduped measured : {"male_given": 5339, "female_given": 5226, + "surnames": 13549} sum 24114 + (surnames differs because this unions surnames_male/surnames_female in as well.) + +== Qwen3-1.7B-Base config vocab 151,936 tokenizer.vocab_size 151,643 + male_given mean 2.40 multi 87.1% 1tok 12.9% 2tok 47.2% 3tok 29.0% 4tok 9.1% 5tok 1.6% 6tok 0.2% + female_given mean 2.44 multi 92.3% 1tok 7.7% 2tok 51.7% 3tok 30.9% 4tok 8.5% 5tok 1.1% 6tok 0.2% + surnames mean 2.48 multi 90.7% 1tok 9.3% 2tok 44.7% 3tok 35.9% 4tok 9.0% 5tok 1.2% 6tok 0.1% + POOL mean 2.46 multi 90.3% + +== Qwen3.5-2B-Base config vocab 248,320 tokenizer.vocab_size 248,044 + male_given mean 2.25 multi 83.6% 1tok 16.4% 2tok 50.5% 3tok 25.5% 4tok 6.6% 5tok 0.9% 6tok 0.0% + female_given mean 2.33 multi 90.2% 1tok 9.8% 2tok 55.5% 3tok 27.8% 4tok 6.1% 5tok 0.9% 6tok 0.0% + surnames mean 2.36 multi 88.8% 1tok 11.2% 2tok 49.2% 3tok 32.9% 4tok 6.1% 5tok 0.5% 6tok 0.0% + POOL mean 2.33 multi 88.0% + +READ: the multi-token property STRENGTHENS on the dense carrier, 88.0% -> 90.3%, +mean 2.33 -> 2.46. A smaller vocabulary fragments more, so Qwen3's 151,936 splits +names into more pieces than Qwen3.5's 248,320. The operator's requirement -- +multi-token names forcing reconstruction from the prefix rather than recall of one +embedding -- is better served after the ruling, not worse. + +POSITIVE CONTROL: the Qwen3.5 column reproduces R49 F02's published figure for the +same pool on the same tokenizer (F02: 89% multi-token, mean 2.35; here: 88.0%, +2.33). Within a point on both, so the instrument recovers a known-true value +before being asked about an unknown one. diff --git a/scripts/training-probes/tokenize_name_pool.py b/scripts/training-probes/tokenize_name_pool.py new file mode 100644 index 0000000..479b100 --- /dev/null +++ b/scripts/training-probes/tokenize_name_pool.py @@ -0,0 +1,46 @@ +"""Re-measure the R49 name pool's token-split distribution under a given tokenizer. + +The pool's multi-token property is an operator requirement -- multi-token names +force the drafter to reconstruct a name from the prefix rather than recall it as +one embedding. F02 measured that property with the Qwen3.5-2B tokenizer; the +carrier ruling moved the sweep to Qwen3, whose vocabulary is a different size, so +the property has to be re-measured rather than assumed to carry over. + +Names are tokenized with a leading space, matching how they appear mid-sentence. + + python tokenize_name_pool.py [...] +""" +import json, sys, collections +from transformers import AutoTokenizer + +pool_path, *tok_paths = sys.argv[1:] +d = json.load(open(pool_path)) + +given_m, given_f, surnames = [], [], [] +for loc, v in d["by_locale"].items(): + given_m += v.get("male", []) + given_f += v.get("female", []) + for k in ("surnames_neutral", "surnames_male", "surnames_female"): + surnames += v.get(k, []) +groups = {"male_given": given_m, "female_given": given_f, "surnames": surnames} +print(f"pool: {sum(len(v) for v in groups.values())} strings " + f"({', '.join(f'{k} {len(v)}' for k, v in groups.items())})") + +for tp in tok_paths: + tok = AutoTokenizer.from_pretrained(tp) + print(f"\n== {tp.rstrip('/').split('/')[-1]} vocab={tok.vocab_size}") + for gname, names in groups.items(): + hist = collections.Counter() + tot = 0 + for n in names: + k = len(tok.encode(" " + n, add_special_tokens=False)) + hist[min(k, 6)] += 1 + tot += k + n = len(names) + multi = sum(c for k, c in hist.items() if k >= 2) + dist = " ".join(f"{k}tok {100*hist[k]/n:4.1f}%" for k in sorted(hist)) + print(f" {gname:<14} mean {tot/n:.2f} multi-token {100*multi/n:5.1f}% {dist}") + allnames = given_m + given_f + surnames + tot = sum(len(tok.encode(" " + x, add_special_tokens=False)) for x in allnames) + multi = sum(1 for x in allnames if len(tok.encode(" " + x, add_special_tokens=False)) >= 2) + print(f" {'POOL':<14} mean {tot/len(allnames):.2f} multi-token {100*multi/len(allnames):5.1f}%")