vh/esh-pfi-infrastructure
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects Releases Wiki Activity
Files
0d9bb109b892f360b3c1772ded120ce9f22fa595
esh-pfi-infrastructure/services/parakeet-ab-2026-09-30/results/accuracy-summary.json
T
vh a6c1d3c454 docs(parakeet): seat A/B vs parakeet-unified-en-0.6b - latency is the int8-on-CPU runtime; unified wins WER 
A/B of the live STT seat (fv-ml1 GPU 0, sherpa-onnx int8 v3) against
nvidia/parakeet-unified-en-0.6b, measured on GPU 3 with the seat's own image,
k2-fsa's published unified int8 export, fp32/fp16 exports made with k2-fsa's
recipe, v2 int8, and NeMo 3.0.0 (fp32, bf16 autocast, bf16 weights).

- Seat int8 graph runs on one CPU thread (cpu/wall 1.00, GPU 2-9%).
- unified-en under NeMo: -121/-234/-530 ms vs the seat at 1-3/3-8/8-20 s
  (paired, n=120/bin; floor <=6 ms; +50 ms positive control reads +52-54).
- unified-en WER lower in every runtime: -0.7 pp clean, -1.5 pp other,
  -3.2 to -4.4 pp AMI (paired CIs exclude 0).
- Seat defects found: hard 400 s input ceiling (HTTP 500), truncation after
  a quiet 1.5 s pause, and severe long-window dropouts (int8 v3 only).
- B-bf16w needs +0.8 to +1.5 GB over the seat's 1,690 MiB on GPU 0.

Raw requests, hypotheses, manifests and the full harness under
services/parakeet-ab-2026-09-30/. No deploy; live seat untouched apart
from 240 light test requests.
2026-09-30 18:51:44 -07:00

1935 lines
30 KiB
JSON
Raw Blame History

{
"wer": [
{
"set": "ls-clean",
"arm": "ab-a1",
"utts": 400,
"missing": 0,
"ref_words": 8144,
"S": 160,
"D": 35,
"I": 25,
"wer": 2.701,
"empty_outputs": 1
},
{
"set": "ls-clean",
"arm": "ab-a2",
"utts": 400,
"missing": 0,
"ref_words": 8144,
"S": 160,
"D": 35,
"I": 25,
"wer": 2.701,
"empty_outputs": 1
},
{
"set": "ls-clean",
"arm": "ab-af32",
"utts": 400,
"missing": 0,
"ref_words": 8144,
"S": 151,
"D": 26,
"I": 21,
"wer": 2.431,
"empty_outputs": 0
},
{
"set": "ls-clean",
"arm": "ab-at16",
"utts": 400,
"missing": 0,
"ref_words": 8144,
"S": 160,
"D": 35,
"I": 25,
"wer": 2.701,
"empty_outputs": 1
},
{
"set": "ls-clean",
"arm": "ab-b16",
"utts": 400,
"missing": 0,
"ref_words": 8144,
"S": 120,
"D": 24,
"I": 17,
"wer": 1.977,
"empty_outputs": 0
},
{
"set": "ls-clean",
"arm": "ab-b16w",
"utts": 400,
"missing": 0,
"ref_words": 8144,
"S": 120,
"D": 23,
"I": 17,
"wer": 1.965,
"empty_outputs": 0
},
{
"set": "ls-clean",
"arm": "ab-b32",
"utts": 400,
"missing": 0,
"ref_words": 8144,
"S": 120,
"D": 24,
"I": 17,
"wer": 1.977,
"empty_outputs": 0
},
{
"set": "ls-clean",
"arm": "ab-b32c",
"utts": 400,
"missing": 0,
"ref_words": 8144,
"S": 120,
"D": 24,
"I": 17,
"wer": 1.977,
"empty_outputs": 0
},
{
"set": "ls-clean",
"arm": "ab-b32t",
"utts": 400,
"missing": 0,
"ref_words": 8144,
"S": 120,
"D": 24,
"I": 17,
"wer": 1.977,
"empty_outputs": 0
},
{
"set": "ls-clean",
"arm": "ab-c",
"utts": 400,
"missing": 0,
"ref_words": 8144,
"S": 120,
"D": 21,
"I": 17,
"wer": 1.94,
"empty_outputs": 0
},
{
"set": "ls-clean",
"arm": "ab-cf16",
"utts": 400,
"missing": 0,
"ref_words": 8144,
"S": 120,
"D": 17,
"I": 17,
"wer": 1.891,
"empty_outputs": 0
},
{
"set": "ls-clean",
"arm": "ab-cf32",
"utts": 400,
"missing": 0,
"ref_words": 8144,
"S": 120,
"D": 17,
"I": 17,
"wer": 1.891,
"empty_outputs": 0
},
{
"set": "ls-clean",
"arm": "ab-d",
"utts": 400,
"missing": 0,
"ref_words": 8144,
"S": 132,
"D": 23,
"I": 17,
"wer": 2.112,
"empty_outputs": 0
},
{
"set": "ls-other",
"arm": "ab-a1",
"utts": 400,
"missing": 0,
"ref_words": 7370,
"S": 269,
"D": 38,
"I": 29,
"wer": 4.559,
"empty_outputs": 0
},
{
"set": "ls-other",
"arm": "ab-af32",
"utts": 400,
"missing": 0,
"ref_words": 7370,
"S": 198,
"D": 22,
"I": 24,
"wer": 3.311,
"empty_outputs": 0
},
{
"set": "ls-other",
"arm": "ab-b16",
"utts": 400,
"missing": 0,
"ref_words": 7370,
"S": 179,
"D": 24,
"I": 26,
"wer": 3.107,
"empty_outputs": 0
},
{
"set": "ls-other",
"arm": "ab-b16w",
"utts": 400,
"missing": 0,
"ref_words": 7370,
"S": 178,
"D": 24,
"I": 26,
"wer": 3.094,
"empty_outputs": 0
},
{
"set": "ls-other",
"arm": "ab-b32",
"utts": 400,
"missing": 0,
"ref_words": 7370,
"S": 179,
"D": 24,
"I": 26,
"wer": 3.107,
"empty_outputs": 0
},
{
"set": "ls-other",
"arm": "ab-c",
"utts": 400,
"missing": 0,
"ref_words": 7370,
"S": 172,
"D": 21,
"I": 23,
"wer": 2.931,
"empty_outputs": 0
},
{
"set": "ls-other",
"arm": "ab-cf16",
"utts": 400,
"missing": 0,
"ref_words": 7370,
"S": 176,
"D": 22,
"I": 24,
"wer": 3.012,
"empty_outputs": 0
},
{
"set": "ls-other",
"arm": "ab-cf32",
"utts": 400,
"missing": 0,
"ref_words": 7370,
"S": 177,
"D": 22,
"I": 24,
"wer": 3.026,
"empty_outputs": 0
},
{
"set": "ls-other",
"arm": "ab-d",
"utts": 400,
"missing": 0,
"ref_words": 7370,
"S": 201,
"D": 24,
"I": 25,
"wer": 3.392,
"empty_outputs": 0
},
{
"set": "ami",
"arm": "ab-a1",
"utts": 400,
"missing": 0,
"ref_words": 4121,
"S": 181,
"D": 299,
"I": 43,
"wer": 12.691,
"empty_outputs": 26
},
{
"set": "ami",
"arm": "ab-af32",
"utts": 400,
"missing": 0,
"ref_words": 4121,
"S": 180,
"D": 182,
"I": 70,
"wer": 10.483,
"empty_outputs": 18
},
{
"set": "ami",
"arm": "ab-b16",
"utts": 400,
"missing": 0,
"ref_words": 4121,
"S": 155,
"D": 130,
"I": 56,
"wer": 8.275,
"empty_outputs": 9
},
{
"set": "ami",
"arm": "ab-b16w",
"utts": 400,
"missing": 0,
"ref_words": 4121,
"S": 155,
"D": 130,
"I": 57,
"wer": 8.299,
"empty_outputs": 9
},
{
"set": "ami",
"arm": "ab-b32",
"utts": 400,
"missing": 0,
"ref_words": 4121,
"S": 157,
"D": 131,
"I": 57,
"wer": 8.372,
"empty_outputs": 10
},
{
"set": "ami",
"arm": "ab-c",
"utts": 400,
"missing": 0,
"ref_words": 4121,
"S": 162,
"D": 179,
"I": 51,
"wer": 9.512,
"empty_outputs": 17
},
{
"set": "ami",
"arm": "ab-cf16",
"utts": 400,
"missing": 0,
"ref_words": 4121,
"S": 157,
"D": 174,
"I": 53,
"wer": 9.318,
"empty_outputs": 15
},
{
"set": "ami",
"arm": "ab-cf32",
"utts": 400,
"missing": 0,
"ref_words": 4121,
"S": 158,
"D": 173,
"I": 55,
"wer": 9.367,
"empty_outputs": 15
},
{
"set": "ami",
"arm": "ab-d",
"utts": 400,
"missing": 0,
"ref_words": 4121,
"S": 178,
"D": 154,
"I": 81,
"wer": 10.022,
"empty_outputs": 15
},
{
"set": "pc",
"arm": "ab-a1",
"utts": 40,
"missing": 0,
"ref_words": 1386,
"S": 36,
"D": 306,
"I": 2,
"wer": 24.82,
"empty_outputs": 0
},
{
"set": "pc",
"arm": "ab-af32",
"utts": 40,
"missing": 0,
"ref_words": 1386,
"S": 37,
"D": 185,
"I": 3,
"wer": 16.234,
"empty_outputs": 0
},
{
"set": "pc",
"arm": "ab-b16",
"utts": 40,
"missing": 0,
"ref_words": 1386,
"S": 29,
"D": 177,
"I": 5,
"wer": 15.224,
"empty_outputs": 0
},
{
"set": "pc",
"arm": "ab-b16w",
"utts": 40,
"missing": 0,
"ref_words": 1386,
"S": 28,
"D": 177,
"I": 5,
"wer": 15.152,
"empty_outputs": 0
},
{
"set": "pc",
"arm": "ab-b32",
"utts": 40,
"missing": 0,
"ref_words": 1386,
"S": 27,
"D": 178,
"I": 5,
"wer": 15.152,
"empty_outputs": 0
},
{
"set": "pc",
"arm": "ab-c",
"utts": 40,
"missing": 0,
"ref_words": 1386,
"S": 28,
"D": 181,
"I": 4,
"wer": 15.368,
"empty_outputs": 0
},
{
"set": "pc",
"arm": "ab-cf16",
"utts": 40,
"missing": 0,
"ref_words": 1386,
"S": 30,
"D": 178,
"I": 5,
"wer": 15.368,
"empty_outputs": 0
},
{
"set": "pc",
"arm": "ab-cf32",
"utts": 40,
"missing": 0,
"ref_words": 1386,
"S": 31,
"D": 176,
"I": 5,
"wer": 15.296,
"empty_outputs": 0
},
{
"set": "pc",
"arm": "ab-d",
"utts": 40,
"missing": 0,
"ref_words": 1386,
"S": 28,
"D": 167,
"I": 4,
"wer": 14.358,
"empty_outputs": 0
},
{
"set": "null",
"arm": "ab-a1",
"utts": 400,
"missing": 0,
"ref_words": 8144,
"S": 165,
"D": 34,
"I": 24,
"wer": 2.738,
"empty_outputs": 1
},
{
"set": "null",
"arm": "ab-af32",
"utts": 400,
"missing": 0,
"ref_words": 8144,
"S": 150,
"D": 26,
"I": 20,
"wer": 2.407,
"empty_outputs": 0
},
{
"set": "null",
"arm": "ab-b16",
"utts": 400,
"missing": 0,
"ref_words": 8144,
"S": 120,
"D": 24,
"I": 17,
"wer": 1.977,
"empty_outputs": 0
},
{
"set": "null",
"arm": "ab-b16w",
"utts": 400,
"missing": 0,
"ref_words": 8144,
"S": 121,
"D": 23,
"I": 17,
"wer": 1.977,
"empty_outputs": 0
},
{
"set": "null",
"arm": "ab-b32",
"utts": 400,
"missing": 0,
"ref_words": 8144,
"S": 120,
"D": 24,
"I": 17,
"wer": 1.977,
"empty_outputs": 0
},
{
"set": "null",
"arm": "ab-c",
"utts": 400,
"missing": 0,
"ref_words": 8144,
"S": 119,
"D": 19,
"I": 17,
"wer": 1.903,
"empty_outputs": 0
},
{
"set": "null",
"arm": "ab-cf16",
"utts": 400,
"missing": 0,
"ref_words": 8144,
"S": 120,
"D": 17,
"I": 17,
"wer": 1.891,
"empty_outputs": 0
},
{
"set": "null",
"arm": "ab-cf32",
"utts": 400,
"missing": 0,
"ref_words": 8144,
"S": 120,
"D": 17,
"I": 17,
"wer": 1.891,
"empty_outputs": 0
},
{
"set": "null",
"arm": "ab-d",
"utts": 400,
"missing": 0,
"ref_words": 8144,
"S": 129,
"D": 21,
"I": 17,
"wer": 2.051,
"empty_outputs": 0
}
],
"delta": [
{
"set": "ls-clean",
"x": "ab-a2",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": 0.0,
"ci95": [
0.0,
0.0
],
"utts_with_identical_edit_counts": 400
},
{
"set": "ls-clean",
"x": "ab-af32",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -0.27,
"ci95": [
-0.599,
0.048
],
"utts_with_identical_edit_counts": 340
},
{
"set": "ls-clean",
"x": "ab-at16",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": 0.0,
"ci95": [
0.0,
0.0
],
"utts_with_identical_edit_counts": 400
},
{
"set": "ls-clean",
"x": "ab-b16",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -0.724,
"ci95": [
-1.097,
-0.348
],
"utts_with_identical_edit_counts": 315
},
{
"set": "ls-clean",
"x": "ab-b16w",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -0.737,
"ci95": [
-1.111,
-0.361
],
"utts_with_identical_edit_counts": 314
},
{
"set": "ls-clean",
"x": "ab-b32",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -0.724,
"ci95": [
-1.097,
-0.348
],
"utts_with_identical_edit_counts": 315
},
{
"set": "ls-clean",
"x": "ab-b32c",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -0.724,
"ci95": [
-1.097,
-0.348
],
"utts_with_identical_edit_counts": 315
},
{
"set": "ls-clean",
"x": "ab-b32t",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -0.724,
"ci95": [
-1.097,
-0.348
],
"utts_with_identical_edit_counts": 315
},
{
"set": "ls-clean",
"x": "ab-c",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -0.761,
"ci95": [
-1.112,
-0.407
],
"utts_with_identical_edit_counts": 309
},
{
"set": "ls-clean",
"x": "ab-cf16",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -0.81,
"ci95": [
-1.177,
-0.441
],
"utts_with_identical_edit_counts": 313
},
{
"set": "ls-clean",
"x": "ab-cf32",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -0.81,
"ci95": [
-1.181,
-0.446
],
"utts_with_identical_edit_counts": 313
},
{
"set": "ls-clean",
"x": "ab-d",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -0.589,
"ci95": [
-0.955,
-0.256
],
"utts_with_identical_edit_counts": 308
},
{
"set": "ls-clean",
"x": "ab-a2",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": 0.724,
"ci95": [
0.35,
1.098
],
"utts_with_identical_edit_counts": 315
},
{
"set": "ls-clean",
"x": "ab-af32",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": 0.454,
"ci95": [
0.159,
0.758
],
"utts_with_identical_edit_counts": 326
},
{
"set": "ls-clean",
"x": "ab-at16",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": 0.724,
"ci95": [
0.35,
1.098
],
"utts_with_identical_edit_counts": 315
},
{
"set": "ls-clean",
"x": "ab-b16",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": 0.0,
"ci95": [
0.0,
0.0
],
"utts_with_identical_edit_counts": 400
},
{
"set": "ls-clean",
"x": "ab-b16w",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": -0.012,
"ci95": [
-0.039,
0.0
],
"utts_with_identical_edit_counts": 399
},
{
"set": "ls-clean",
"x": "ab-b32c",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": 0.0,
"ci95": [
0.0,
0.0
],
"utts_with_identical_edit_counts": 400
},
{
"set": "ls-clean",
"x": "ab-b32t",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": 0.0,
"ci95": [
0.0,
0.0
],
"utts_with_identical_edit_counts": 400
},
{
"set": "ls-clean",
"x": "ab-c",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": -0.037,
"ci95": [
-0.177,
0.101
],
"utts_with_identical_edit_counts": 383
},
{
"set": "ls-clean",
"x": "ab-cf16",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": -0.086,
"ci95": [
-0.218,
0.026
],
"utts_with_identical_edit_counts": 390
},
{
"set": "ls-clean",
"x": "ab-cf32",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": -0.086,
"ci95": [
-0.216,
0.025
],
"utts_with_identical_edit_counts": 390
},
{
"set": "ls-clean",
"x": "ab-d",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": 0.135,
"ci95": [
-0.122,
0.393
],
"utts_with_identical_edit_counts": 343
},
{
"set": "ls-other",
"x": "ab-af32",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -1.248,
"ci95": [
-1.701,
-0.847
],
"utts_with_identical_edit_counts": 304
},
{
"set": "ls-other",
"x": "ab-b16",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -1.452,
"ci95": [
-1.989,
-0.949
],
"utts_with_identical_edit_counts": 272
},
{
"set": "ls-other",
"x": "ab-b16w",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -1.465,
"ci95": [
-2.003,
-0.965
],
"utts_with_identical_edit_counts": 273
},
{
"set": "ls-other",
"x": "ab-b32",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -1.452,
"ci95": [
-1.989,
-0.949
],
"utts_with_identical_edit_counts": 272
},
{
"set": "ls-other",
"x": "ab-c",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -1.628,
"ci95": [
-2.148,
-1.139
],
"utts_with_identical_edit_counts": 275
},
{
"set": "ls-other",
"x": "ab-cf16",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -1.547,
"ci95": [
-2.073,
-1.056
],
"utts_with_identical_edit_counts": 277
},
{
"set": "ls-other",
"x": "ab-cf32",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -1.533,
"ci95": [
-2.059,
-1.036
],
"utts_with_identical_edit_counts": 276
},
{
"set": "ls-other",
"x": "ab-d",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -1.167,
"ci95": [
-1.678,
-0.681
],
"utts_with_identical_edit_counts": 279
},
{
"set": "ls-other",
"x": "ab-af32",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": 0.204,
"ci95": [
-0.202,
0.635
],
"utts_with_identical_edit_counts": 295
},
{
"set": "ls-other",
"x": "ab-b16",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": 0.0,
"ci95": [
0.0,
0.0
],
"utts_with_identical_edit_counts": 400
},
{
"set": "ls-other",
"x": "ab-b16w",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": -0.014,
"ci95": [
-0.043,
0.0
],
"utts_with_identical_edit_counts": 399
},
{
"set": "ls-other",
"x": "ab-c",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": -0.176,
"ci95": [
-0.388,
0.026
],
"utts_with_identical_edit_counts": 364
},
{
"set": "ls-other",
"x": "ab-cf16",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": -0.095,
"ci95": [
-0.261,
0.069
],
"utts_with_identical_edit_counts": 373
},
{
"set": "ls-other",
"x": "ab-cf32",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": -0.081,
"ci95": [
-0.25,
0.087
],
"utts_with_identical_edit_counts": 372
},
{
"set": "ls-other",
"x": "ab-d",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": 0.285,
"ci95": [
-0.08,
0.66
],
"utts_with_identical_edit_counts": 307
},
{
"set": "ami",
"x": "ab-af32",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -2.208,
"ci95": [
-3.682,
-0.893
],
"utts_with_identical_edit_counts": 292
},
{
"set": "ami",
"x": "ab-b16",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -4.416,
"ci95": [
-6.578,
-2.508
],
"utts_with_identical_edit_counts": 231
},
{
"set": "ami",
"x": "ab-b16w",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -4.392,
"ci95": [
-6.574,
-2.489
],
"utts_with_identical_edit_counts": 230
},
{
"set": "ami",
"x": "ab-b32",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -4.319,
"ci95": [
-6.495,
-2.399
],
"utts_with_identical_edit_counts": 230
},
{
"set": "ami",
"x": "ab-c",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -3.179,
"ci95": [
-5.127,
-1.483
],
"utts_with_identical_edit_counts": 234
},
{
"set": "ami",
"x": "ab-cf16",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -3.373,
"ci95": [
-5.289,
-1.643
],
"utts_with_identical_edit_counts": 230
},
{
"set": "ami",
"x": "ab-cf32",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -3.324,
"ci95": [
-5.249,
-1.604
],
"utts_with_identical_edit_counts": 229
},
{
"set": "ami",
"x": "ab-d",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -2.669,
"ci95": [
-4.423,
-1.045
],
"utts_with_identical_edit_counts": 244
},
{
"set": "ami",
"x": "ab-af32",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": 2.111,
"ci95": [
0.646,
3.692
],
"utts_with_identical_edit_counts": 245
},
{
"set": "ami",
"x": "ab-b16",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": -0.097,
"ci95": [
-0.222,
0.0
],
"utts_with_identical_edit_counts": 392
},
{
"set": "ami",
"x": "ab-b16w",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": -0.073,
"ci95": [
-0.181,
0.025
],
"utts_with_identical_edit_counts": 395
},
{
"set": "ami",
"x": "ab-c",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": 1.14,
"ci95": [
0.28,
2.158
],
"utts_with_identical_edit_counts": 328
},
{
"set": "ami",
"x": "ab-cf16",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": 0.946,
"ci95": [
0.195,
1.867
],
"utts_with_identical_edit_counts": 337
},
{
"set": "ami",
"x": "ab-cf32",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": 0.995,
"ci95": [
0.241,
1.922
],
"utts_with_identical_edit_counts": 337
},
{
"set": "ami",
"x": "ab-d",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": 1.65,
"ci95": [
0.38,
3.064
],
"utts_with_identical_edit_counts": 240
},
{
"set": "pc",
"x": "ab-af32",
"y": "ab-a1",
"n_utts": 40,
"delta_pp": -8.586,
"ci95": [
-15.696,
-2.422
],
"utts_with_identical_edit_counts": 15
},
{
"set": "pc",
"x": "ab-b16",
"y": "ab-a1",
"n_utts": 40,
"delta_pp": -9.596,
"ci95": [
-16.959,
-2.941
],
"utts_with_identical_edit_counts": 9
},
{
"set": "pc",
"x": "ab-b16w",
"y": "ab-a1",
"n_utts": 40,
"delta_pp": -9.668,
"ci95": [
-17.036,
-3.023
],
"utts_with_identical_edit_counts": 9
},
{
"set": "pc",
"x": "ab-b32",
"y": "ab-a1",
"n_utts": 40,
"delta_pp": -9.668,
"ci95": [
-17.036,
-3.023
],
"utts_with_identical_edit_counts": 9
},
{
"set": "pc",
"x": "ab-c",
"y": "ab-a1",
"n_utts": 40,
"delta_pp": -9.452,
"ci95": [
-16.961,
-2.753
],
"utts_with_identical_edit_counts": 11
},
{
"set": "pc",
"x": "ab-cf16",
"y": "ab-a1",
"n_utts": 40,
"delta_pp": -9.452,
"ci95": [
-16.786,
-2.784
],
"utts_with_identical_edit_counts": 9
},
{
"set": "pc",
"x": "ab-cf32",
"y": "ab-a1",
"n_utts": 40,
"delta_pp": -9.524,
"ci95": [
-16.929,
-2.788
],
"utts_with_identical_edit_counts": 9
},
{
"set": "pc",
"x": "ab-d",
"y": "ab-a1",
"n_utts": 40,
"delta_pp": -10.462,
"ci95": [
-17.816,
-3.869
],
"utts_with_identical_edit_counts": 14
},
{
"set": "pc",
"x": "ab-af32",
"y": "ab-b32",
"n_utts": 40,
"delta_pp": 1.082,
"ci95": [
-1.183,
4.857
],
"utts_with_identical_edit_counts": 17
},
{
"set": "pc",
"x": "ab-b16",
"y": "ab-b32",
"n_utts": 40,
"delta_pp": 0.072,
"ci95": [
0.0,
0.236
],
"utts_with_identical_edit_counts": 38
},
{
"set": "pc",
"x": "ab-b16w",
"y": "ab-b32",
"n_utts": 40,
"delta_pp": 0.0,
"ci95": [
0.0,
0.0
],
"utts_with_identical_edit_counts": 39
},
{
"set": "pc",
"x": "ab-c",
"y": "ab-b32",
"n_utts": 40,
"delta_pp": 0.216,
"ci95": [
-0.383,
0.871
],
"utts_with_identical_edit_counts": 30
},
{
"set": "pc",
"x": "ab-cf16",
"y": "ab-b32",
"n_utts": 40,
"delta_pp": 0.216,
"ci95": [
0.0,
0.571
],
"utts_with_identical_edit_counts": 38
},
{
"set": "pc",
"x": "ab-cf32",
"y": "ab-b32",
"n_utts": 40,
"delta_pp": 0.144,
"ci95": [
-0.152,
0.528
],
"utts_with_identical_edit_counts": 37
},
{
"set": "pc",
"x": "ab-d",
"y": "ab-b32",
"n_utts": 40,
"delta_pp": -0.794,
"ci95": [
-1.512,
-0.075
],
"utts_with_identical_edit_counts": 19
},
{
"set": "null",
"x": "ab-af32",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -0.332,
"ci95": [
-0.649,
-0.013
],
"utts_with_identical_edit_counts": 338
},
{
"set": "null",
"x": "ab-b16",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -0.761,
"ci95": [
-1.129,
-0.393
],
"utts_with_identical_edit_counts": 311
},
{
"set": "null",
"x": "ab-b16w",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -0.761,
"ci95": [
-1.13,
-0.396
],
"utts_with_identical_edit_counts": 309
},
{
"set": "null",
"x": "ab-b32",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -0.761,
"ci95": [
-1.129,
-0.393
],
"utts_with_identical_edit_counts": 311
},
{
"set": "null",
"x": "ab-c",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -0.835,
"ci95": [
-1.179,
-0.48
],
"utts_with_identical_edit_counts": 311
},
{
"set": "null",
"x": "ab-cf16",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -0.847,
"ci95": [
-1.21,
-0.488
],
"utts_with_identical_edit_counts": 309
},
{
"set": "null",
"x": "ab-cf32",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -0.847,
"ci95": [
-1.21,
-0.482
],
"utts_with_identical_edit_counts": 310
},
{
"set": "null",
"x": "ab-d",
"y": "ab-a1",
"n_utts": 400,
"delta_pp": -0.688,
"ci95": [
-1.025,
-0.363
],
"utts_with_identical_edit_counts": 309
},
{
"set": "null",
"x": "ab-af32",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": 0.43,
"ci95": [
0.137,
0.729
],
"utts_with_identical_edit_counts": 327
},
{
"set": "null",
"x": "ab-b16",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": 0.0,
"ci95": [
0.0,
0.0
],
"utts_with_identical_edit_counts": 400
},
{
"set": "null",
"x": "ab-b16w",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": 0.0,
"ci95": [
-0.036,
0.037
],
"utts_with_identical_edit_counts": 398
},
{
"set": "null",
"x": "ab-c",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": -0.074,
"ci95": [
-0.202,
0.046
],
"utts_with_identical_edit_counts": 386
},
{
"set": "null",
"x": "ab-cf16",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": -0.086,
"ci95": [
-0.218,
0.026
],
"utts_with_identical_edit_counts": 390
},
{
"set": "null",
"x": "ab-cf32",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": -0.086,
"ci95": [
-0.216,
0.025
],
"utts_with_identical_edit_counts": 390
},
{
"set": "null",
"x": "ab-d",
"y": "ab-b32",
"n_utts": 400,
"delta_pp": 0.074,
"ci95": [
-0.177,
0.336
],
"utts_with_identical_edit_counts": 344
}
],
"pc": [
{
"arm": "ab-a1",
"utts": 40,
"deletions_clean": 2,
"deletions_silenced": 306,
"errors_clean": 25,
"errors_silenced": 344,
"wer_clean": 1.8,
"wer_silenced": 24.82,
"utts_with_more_deletions": 40
},
{
"arm": "ab-af32",
"utts": 40,
"deletions_clean": 1,
"deletions_silenced": 185,
"errors_clean": 27,
"errors_silenced": 225,
"wer_clean": 1.95,
"wer_silenced": 16.23,
"utts_with_more_deletions": 40
},
{
"arm": "ab-b16",
"utts": 40,
"deletions_clean": 2,
"deletions_silenced": 177,
"errors_clean": 26,
"errors_silenced": 211,
"wer_clean": 1.88,
"wer_silenced": 15.22,
"utts_with_more_deletions": 40
},
{
"arm": "ab-b16w",
"utts": 40,
"deletions_clean": 2,
"deletions_silenced": 177,
"errors_clean": 26,
"errors_silenced": 210,
"wer_clean": 1.88,
"wer_silenced": 15.15,
"utts_with_more_deletions": 40
},
{
"arm": "ab-b32",
"utts": 40,
"deletions_clean": 2,
"deletions_silenced": 178,
"errors_clean": 26,
"errors_silenced": 210,
"wer_clean": 1.88,
"wer_silenced": 15.15,
"utts_with_more_deletions": 40
},
{
"arm": "ab-c",
"utts": 40,
"deletions_clean": 2,
"deletions_silenced": 181,
"errors_clean": 26,
"errors_silenced": 213,
"wer_clean": 1.88,
"wer_silenced": 15.37,
"utts_with_more_deletions": 40
},
{
"arm": "ab-cf16",
"utts": 40,
"deletions_clean": 2,
"deletions_silenced": 178,
"errors_clean": 28,
"errors_silenced": 213,
"wer_clean": 2.02,
"wer_silenced": 15.37,
"utts_with_more_deletions": 40
},
{
"arm": "ab-cf32",
"utts": 40,
"deletions_clean": 2,
"deletions_silenced": 176,
"errors_clean": 28,
"errors_silenced": 212,
"wer_clean": 2.02,
"wer_silenced": 15.3,
"utts_with_more_deletions": 40
},
{
"arm": "ab-d",
"utts": 40,
"deletions_clean": 1,
"deletions_silenced": 167,
"errors_clean": 26,
"errors_silenced": 199,
"wer_clean": 1.88,
"wer_silenced": 14.36,
"utts_with_more_deletions": 40
}
],
"null": [
{
"arm": "ab-a1",
"n_utts": 400,
"delta_pp": 0.037,
"ci95": [
-0.078,
0.156
],
"utts_with_identical_edit_counts": 383
},
{
"arm": "ab-af32",
"n_utts": 400,
"delta_pp": -0.025,
"ci95": [
-0.063,
0.0
],
"utts_with_identical_edit_counts": 398
},
{
"arm": "ab-b16",
"n_utts": 400,
"delta_pp": 0.0,
"ci95": [
0.0,
0.0
],
"utts_with_identical_edit_counts": 400
},
{
"arm": "ab-b16w",
"n_utts": 400,
"delta_pp": 0.012,
"ci95": [
0.0,
0.039
],
"utts_with_identical_edit_counts": 399
},
{
"arm": "ab-b32",
"n_utts": 400,
"delta_pp": 0.0,
"ci95": [
0.0,
0.0
],
"utts_with_identical_edit_counts": 400
},
{
"arm": "ab-c",
"n_utts": 400,
"delta_pp": -0.037,
"ci95": [
-0.135,
0.057
],
"utts_with_identical_edit_counts": 389
},
{
"arm": "ab-cf16",
"n_utts": 400,
"delta_pp": 0.0,
"ci95": [
0.0,
0.0
],
"utts_with_identical_edit_counts": 400
},
{
"arm": "ab-cf32",
"n_utts": 400,
"delta_pp": 0.0,
"ci95": [
0.0,
0.0
],
"utts_with_identical_edit_counts": 400
},
{
"arm": "ab-d",
"n_utts": 400,
"delta_pp": -0.061,
"ci95": [
-0.149,
0.024
],
"utts_with_identical_edit_counts": 391
}
],
"same": [
{
"a": "ab-a1",
"b": "ab-a2",
"utts": 400,
"identical_text": 400
},
{
"a": "ab-a1",
"b": "ab-at16",
"utts": 400,
"identical_text": 400
},
{
"a": "ab-b32",
"b": "ab-b32t",
"utts": 400,
"identical_text": 400
}
],
"features": [
{
"arm": "ab-a1",
"outputs": 400,
"with_sentence_punct": 89.0,
"with_comma": 73.0,
"with_upper": 94.5,
"all_lower": 5.2,
"empty": 1,
"non_ascii": 1
},
{
"arm": "ab-a2",
"outputs": 400,
"with_sentence_punct": 89.0,
"with_comma": 73.0,
"with_upper": 94.5,
"all_lower": 5.2,
"empty": 1,
"non_ascii": 1
},
{
"arm": "ab-af32",
"outputs": 400,
"with_sentence_punct": 88.0,
"with_comma": 74.5,
"with_upper": 96.0,
"all_lower": 4.0,
"empty": 0,
"non_ascii": 1
},
{
"arm": "ab-at16",
"outputs": 400,
"with_sentence_punct": 89.0,
"with_comma": 73.0,
"with_upper": 94.5,
"all_lower": 5.2,
"empty": 1,
"non_ascii": 1
},
{
"arm": "ab-b16",
"outputs": 400,
"with_sentence_punct": 50.0,
"with_comma": 62.8,
"with_upper": 96.8,
"all_lower": 3.2,
"empty": 0,
"non_ascii": 0
},
{
"arm": "ab-b16w",
"outputs": 400,
"with_sentence_punct": 50.8,
"with_comma": 63.0,
"with_upper": 96.8,
"all_lower": 3.2,
"empty": 0,
"non_ascii": 0
},
{
"arm": "ab-b32",
"outputs": 400,
"with_sentence_punct": 50.0,
"with_comma": 63.0,
"with_upper": 96.8,
"all_lower": 3.2,
"empty": 0,
"non_ascii": 0
},
{
"arm": "ab-b32c",
"outputs": 400,
"with_sentence_punct": 50.0,
"with_comma": 63.0,
"with_upper": 96.8,
"all_lower": 3.2,
"empty": 0,
"non_ascii": 0
},
{
"arm": "ab-b32t",
"outputs": 400,
"with_sentence_punct": 50.0,
"with_comma": 63.0,
"with_upper": 96.8,
"all_lower": 3.2,
"empty": 0,
"non_ascii": 0
},
{
"arm": "ab-c",
"outputs": 400,
"with_sentence_punct": 51.0,
"with_comma": 64.5,
"with_upper": 97.0,
"all_lower": 3.0,
"empty": 0,
"non_ascii": 0
},
{
"arm": "ab-cf16",
"outputs": 400,
"with_sentence_punct": 48.5,
"with_comma": 62.0,
"with_upper": 96.0,
"all_lower": 4.0,
"empty": 0,
"non_ascii": 0
},
{
"arm": "ab-cf32",
"outputs": 400,
"with_sentence_punct": 48.5,
"with_comma": 62.0,
"with_upper": 96.0,
"all_lower": 4.0,
"empty": 0,
"non_ascii": 0
},
{
"arm": "ab-d",
"outputs": 400,
"with_sentence_punct": 83.8,
"with_comma": 73.5,
"with_upper": 94.8,
"all_lower": 5.2,
"empty": 0,
"non_ascii": 0
}
]
}
Reference in New Issue View Git Blame Copy Permalink