{ "wer": [ { "set": "ls-clean", "arm": "ab-a1", "utts": 400, "missing": 0, "ref_words": 8144, "S": 160, "D": 35, "I": 25, "wer": 2.701, "empty_outputs": 1 }, { "set": "ls-clean", "arm": "ab-a2", "utts": 400, "missing": 0, "ref_words": 8144, "S": 160, "D": 35, "I": 25, "wer": 2.701, "empty_outputs": 1 }, { "set": "ls-clean", "arm": "ab-af32", "utts": 400, "missing": 0, "ref_words": 8144, "S": 151, "D": 26, "I": 21, "wer": 2.431, "empty_outputs": 0 }, { "set": "ls-clean", "arm": "ab-at16", "utts": 400, "missing": 0, "ref_words": 8144, "S": 160, "D": 35, "I": 25, "wer": 2.701, "empty_outputs": 1 }, { "set": "ls-clean", "arm": "ab-b16", "utts": 400, "missing": 0, "ref_words": 8144, "S": 120, "D": 24, "I": 17, "wer": 1.977, "empty_outputs": 0 }, { "set": "ls-clean", "arm": "ab-b16w", "utts": 400, "missing": 0, "ref_words": 8144, "S": 120, "D": 23, "I": 17, "wer": 1.965, "empty_outputs": 0 }, { "set": "ls-clean", "arm": "ab-b32", "utts": 400, "missing": 0, "ref_words": 8144, "S": 120, "D": 24, "I": 17, "wer": 1.977, "empty_outputs": 0 }, { "set": "ls-clean", "arm": "ab-b32c", "utts": 400, "missing": 0, "ref_words": 8144, "S": 120, "D": 24, "I": 17, "wer": 1.977, "empty_outputs": 0 }, { "set": "ls-clean", "arm": "ab-b32t", "utts": 400, "missing": 0, "ref_words": 8144, "S": 120, "D": 24, "I": 17, "wer": 1.977, "empty_outputs": 0 }, { "set": "ls-clean", "arm": "ab-c", "utts": 400, "missing": 0, "ref_words": 8144, "S": 120, "D": 21, "I": 17, "wer": 1.94, "empty_outputs": 0 }, { "set": "ls-clean", "arm": "ab-cf16", "utts": 400, "missing": 0, "ref_words": 8144, "S": 120, "D": 17, "I": 17, "wer": 1.891, "empty_outputs": 0 }, { "set": "ls-clean", "arm": "ab-cf32", "utts": 400, "missing": 0, "ref_words": 8144, "S": 120, "D": 17, "I": 17, "wer": 1.891, "empty_outputs": 0 }, { "set": "ls-clean", "arm": "ab-d", "utts": 400, "missing": 0, "ref_words": 8144, "S": 132, "D": 23, "I": 17, "wer": 2.112, "empty_outputs": 0 }, { "set": "ls-other", "arm": "ab-a1", "utts": 400, "missing": 0, "ref_words": 7370, "S": 269, "D": 38, "I": 29, "wer": 4.559, "empty_outputs": 0 }, { "set": "ls-other", "arm": "ab-af32", "utts": 400, "missing": 0, "ref_words": 7370, "S": 198, "D": 22, "I": 24, "wer": 3.311, "empty_outputs": 0 }, { "set": "ls-other", "arm": "ab-b16", "utts": 400, "missing": 0, "ref_words": 7370, "S": 179, "D": 24, "I": 26, "wer": 3.107, "empty_outputs": 0 }, { "set": "ls-other", "arm": "ab-b16w", "utts": 400, "missing": 0, "ref_words": 7370, "S": 178, "D": 24, "I": 26, "wer": 3.094, "empty_outputs": 0 }, { "set": "ls-other", "arm": "ab-b32", "utts": 400, "missing": 0, "ref_words": 7370, "S": 179, "D": 24, "I": 26, "wer": 3.107, "empty_outputs": 0 }, { "set": "ls-other", "arm": "ab-c", "utts": 400, "missing": 0, "ref_words": 7370, "S": 172, "D": 21, "I": 23, "wer": 2.931, "empty_outputs": 0 }, { "set": "ls-other", "arm": "ab-cf16", "utts": 400, "missing": 0, "ref_words": 7370, "S": 176, "D": 22, "I": 24, "wer": 3.012, "empty_outputs": 0 }, { "set": "ls-other", "arm": "ab-cf32", "utts": 400, "missing": 0, "ref_words": 7370, "S": 177, "D": 22, "I": 24, "wer": 3.026, "empty_outputs": 0 }, { "set": "ls-other", "arm": "ab-d", "utts": 400, "missing": 0, "ref_words": 7370, "S": 201, "D": 24, "I": 25, "wer": 3.392, "empty_outputs": 0 }, { "set": "ami", "arm": "ab-a1", "utts": 400, "missing": 0, "ref_words": 4121, "S": 181, "D": 299, "I": 43, "wer": 12.691, "empty_outputs": 26 }, { "set": "ami", "arm": "ab-af32", "utts": 400, "missing": 0, "ref_words": 4121, "S": 180, "D": 182, "I": 70, "wer": 10.483, "empty_outputs": 18 }, { "set": "ami", "arm": "ab-b16", "utts": 400, "missing": 0, "ref_words": 4121, "S": 155, "D": 130, "I": 56, "wer": 8.275, "empty_outputs": 9 }, { "set": "ami", "arm": "ab-b16w", "utts": 400, "missing": 0, "ref_words": 4121, "S": 155, "D": 130, "I": 57, "wer": 8.299, "empty_outputs": 9 }, { "set": "ami", "arm": "ab-b32", "utts": 400, "missing": 0, "ref_words": 4121, "S": 157, "D": 131, "I": 57, "wer": 8.372, "empty_outputs": 10 }, { "set": "ami", "arm": "ab-c", "utts": 400, "missing": 0, "ref_words": 4121, "S": 162, "D": 179, "I": 51, "wer": 9.512, "empty_outputs": 17 }, { "set": "ami", "arm": "ab-cf16", "utts": 400, "missing": 0, "ref_words": 4121, "S": 157, "D": 174, "I": 53, "wer": 9.318, "empty_outputs": 15 }, { "set": "ami", "arm": "ab-cf32", "utts": 400, "missing": 0, "ref_words": 4121, "S": 158, "D": 173, "I": 55, "wer": 9.367, "empty_outputs": 15 }, { "set": "ami", "arm": "ab-d", "utts": 400, "missing": 0, "ref_words": 4121, "S": 178, "D": 154, "I": 81, "wer": 10.022, "empty_outputs": 15 }, { "set": "pc", "arm": "ab-a1", "utts": 40, "missing": 0, "ref_words": 1386, "S": 36, "D": 306, "I": 2, "wer": 24.82, "empty_outputs": 0 }, { "set": "pc", "arm": "ab-af32", "utts": 40, "missing": 0, "ref_words": 1386, "S": 37, "D": 185, "I": 3, "wer": 16.234, "empty_outputs": 0 }, { "set": "pc", "arm": "ab-b16", "utts": 40, "missing": 0, "ref_words": 1386, "S": 29, "D": 177, "I": 5, "wer": 15.224, "empty_outputs": 0 }, { "set": "pc", "arm": "ab-b16w", "utts": 40, "missing": 0, "ref_words": 1386, "S": 28, "D": 177, "I": 5, "wer": 15.152, "empty_outputs": 0 }, { "set": "pc", "arm": "ab-b32", "utts": 40, "missing": 0, "ref_words": 1386, "S": 27, "D": 178, "I": 5, "wer": 15.152, "empty_outputs": 0 }, { "set": "pc", "arm": "ab-c", "utts": 40, "missing": 0, "ref_words": 1386, "S": 28, "D": 181, "I": 4, "wer": 15.368, "empty_outputs": 0 }, { "set": "pc", "arm": "ab-cf16", "utts": 40, "missing": 0, "ref_words": 1386, "S": 30, "D": 178, "I": 5, "wer": 15.368, "empty_outputs": 0 }, { "set": "pc", "arm": "ab-cf32", "utts": 40, "missing": 0, "ref_words": 1386, "S": 31, "D": 176, "I": 5, "wer": 15.296, "empty_outputs": 0 }, { "set": "pc", "arm": "ab-d", "utts": 40, "missing": 0, "ref_words": 1386, "S": 28, "D": 167, "I": 4, "wer": 14.358, "empty_outputs": 0 }, { "set": "null", "arm": "ab-a1", "utts": 400, "missing": 0, "ref_words": 8144, "S": 165, "D": 34, "I": 24, "wer": 2.738, "empty_outputs": 1 }, { "set": "null", "arm": "ab-af32", "utts": 400, "missing": 0, "ref_words": 8144, "S": 150, "D": 26, "I": 20, "wer": 2.407, "empty_outputs": 0 }, { "set": "null", "arm": "ab-b16", "utts": 400, "missing": 0, "ref_words": 8144, "S": 120, "D": 24, "I": 17, "wer": 1.977, "empty_outputs": 0 }, { "set": "null", "arm": "ab-b16w", "utts": 400, "missing": 0, "ref_words": 8144, "S": 121, "D": 23, "I": 17, "wer": 1.977, "empty_outputs": 0 }, { "set": "null", "arm": "ab-b32", "utts": 400, "missing": 0, "ref_words": 8144, "S": 120, "D": 24, "I": 17, "wer": 1.977, "empty_outputs": 0 }, { "set": "null", "arm": "ab-c", "utts": 400, "missing": 0, "ref_words": 8144, "S": 119, "D": 19, "I": 17, "wer": 1.903, "empty_outputs": 0 }, { "set": "null", "arm": "ab-cf16", "utts": 400, "missing": 0, "ref_words": 8144, "S": 120, "D": 17, "I": 17, "wer": 1.891, "empty_outputs": 0 }, { "set": "null", "arm": "ab-cf32", "utts": 400, "missing": 0, "ref_words": 8144, "S": 120, "D": 17, "I": 17, "wer": 1.891, "empty_outputs": 0 }, { "set": "null", "arm": "ab-d", "utts": 400, "missing": 0, "ref_words": 8144, "S": 129, "D": 21, "I": 17, "wer": 2.051, "empty_outputs": 0 } ], "delta": [ { "set": "ls-clean", "x": "ab-a2", "y": "ab-a1", "n_utts": 400, "delta_pp": 0.0, "ci95": [ 0.0, 0.0 ], "utts_with_identical_edit_counts": 400 }, { "set": "ls-clean", "x": "ab-af32", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.27, "ci95": [ -0.599, 0.048 ], "utts_with_identical_edit_counts": 340 }, { "set": "ls-clean", "x": "ab-at16", "y": "ab-a1", "n_utts": 400, "delta_pp": 0.0, "ci95": [ 0.0, 0.0 ], "utts_with_identical_edit_counts": 400 }, { "set": "ls-clean", "x": "ab-b16", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.724, "ci95": [ -1.097, -0.348 ], "utts_with_identical_edit_counts": 315 }, { "set": "ls-clean", "x": "ab-b16w", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.737, "ci95": [ -1.111, -0.361 ], "utts_with_identical_edit_counts": 314 }, { "set": "ls-clean", "x": "ab-b32", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.724, "ci95": [ -1.097, -0.348 ], "utts_with_identical_edit_counts": 315 }, { "set": "ls-clean", "x": "ab-b32c", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.724, "ci95": [ -1.097, -0.348 ], "utts_with_identical_edit_counts": 315 }, { "set": "ls-clean", "x": "ab-b32t", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.724, "ci95": [ -1.097, -0.348 ], "utts_with_identical_edit_counts": 315 }, { "set": "ls-clean", "x": "ab-c", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.761, "ci95": [ -1.112, -0.407 ], "utts_with_identical_edit_counts": 309 }, { "set": "ls-clean", "x": "ab-cf16", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.81, "ci95": [ -1.177, -0.441 ], "utts_with_identical_edit_counts": 313 }, { "set": "ls-clean", "x": "ab-cf32", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.81, "ci95": [ -1.181, -0.446 ], "utts_with_identical_edit_counts": 313 }, { "set": "ls-clean", "x": "ab-d", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.589, "ci95": [ -0.955, -0.256 ], "utts_with_identical_edit_counts": 308 }, { "set": "ls-clean", "x": "ab-a2", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.724, "ci95": [ 0.35, 1.098 ], "utts_with_identical_edit_counts": 315 }, { "set": "ls-clean", "x": "ab-af32", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.454, "ci95": [ 0.159, 0.758 ], "utts_with_identical_edit_counts": 326 }, { "set": "ls-clean", "x": "ab-at16", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.724, "ci95": [ 0.35, 1.098 ], "utts_with_identical_edit_counts": 315 }, { "set": "ls-clean", "x": "ab-b16", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.0, "ci95": [ 0.0, 0.0 ], "utts_with_identical_edit_counts": 400 }, { "set": "ls-clean", "x": "ab-b16w", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.012, "ci95": [ -0.039, 0.0 ], "utts_with_identical_edit_counts": 399 }, { "set": "ls-clean", "x": "ab-b32c", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.0, "ci95": [ 0.0, 0.0 ], "utts_with_identical_edit_counts": 400 }, { "set": "ls-clean", "x": "ab-b32t", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.0, "ci95": [ 0.0, 0.0 ], "utts_with_identical_edit_counts": 400 }, { "set": "ls-clean", "x": "ab-c", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.037, "ci95": [ -0.177, 0.101 ], "utts_with_identical_edit_counts": 383 }, { "set": "ls-clean", "x": "ab-cf16", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.086, "ci95": [ -0.218, 0.026 ], "utts_with_identical_edit_counts": 390 }, { "set": "ls-clean", "x": "ab-cf32", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.086, "ci95": [ -0.216, 0.025 ], "utts_with_identical_edit_counts": 390 }, { "set": "ls-clean", "x": "ab-d", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.135, "ci95": [ -0.122, 0.393 ], "utts_with_identical_edit_counts": 343 }, { "set": "ls-other", "x": "ab-af32", "y": "ab-a1", "n_utts": 400, "delta_pp": -1.248, "ci95": [ -1.701, -0.847 ], "utts_with_identical_edit_counts": 304 }, { "set": "ls-other", "x": "ab-b16", "y": "ab-a1", "n_utts": 400, "delta_pp": -1.452, "ci95": [ -1.989, -0.949 ], "utts_with_identical_edit_counts": 272 }, { "set": "ls-other", "x": "ab-b16w", "y": "ab-a1", "n_utts": 400, "delta_pp": -1.465, "ci95": [ -2.003, -0.965 ], "utts_with_identical_edit_counts": 273 }, { "set": "ls-other", "x": "ab-b32", "y": "ab-a1", "n_utts": 400, "delta_pp": -1.452, "ci95": [ -1.989, -0.949 ], "utts_with_identical_edit_counts": 272 }, { "set": "ls-other", "x": "ab-c", "y": "ab-a1", "n_utts": 400, "delta_pp": -1.628, "ci95": [ -2.148, -1.139 ], "utts_with_identical_edit_counts": 275 }, { "set": "ls-other", "x": "ab-cf16", "y": "ab-a1", "n_utts": 400, "delta_pp": -1.547, "ci95": [ -2.073, -1.056 ], "utts_with_identical_edit_counts": 277 }, { "set": "ls-other", "x": "ab-cf32", "y": "ab-a1", "n_utts": 400, "delta_pp": -1.533, "ci95": [ -2.059, -1.036 ], "utts_with_identical_edit_counts": 276 }, { "set": "ls-other", "x": "ab-d", "y": "ab-a1", "n_utts": 400, "delta_pp": -1.167, "ci95": [ -1.678, -0.681 ], "utts_with_identical_edit_counts": 279 }, { "set": "ls-other", "x": "ab-af32", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.204, "ci95": [ -0.202, 0.635 ], "utts_with_identical_edit_counts": 295 }, { "set": "ls-other", "x": "ab-b16", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.0, "ci95": [ 0.0, 0.0 ], "utts_with_identical_edit_counts": 400 }, { "set": "ls-other", "x": "ab-b16w", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.014, "ci95": [ -0.043, 0.0 ], "utts_with_identical_edit_counts": 399 }, { "set": "ls-other", "x": "ab-c", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.176, "ci95": [ -0.388, 0.026 ], "utts_with_identical_edit_counts": 364 }, { "set": "ls-other", "x": "ab-cf16", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.095, "ci95": [ -0.261, 0.069 ], "utts_with_identical_edit_counts": 373 }, { "set": "ls-other", "x": "ab-cf32", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.081, "ci95": [ -0.25, 0.087 ], "utts_with_identical_edit_counts": 372 }, { "set": "ls-other", "x": "ab-d", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.285, "ci95": [ -0.08, 0.66 ], "utts_with_identical_edit_counts": 307 }, { "set": "ami", "x": "ab-af32", "y": "ab-a1", "n_utts": 400, "delta_pp": -2.208, "ci95": [ -3.682, -0.893 ], "utts_with_identical_edit_counts": 292 }, { "set": "ami", "x": "ab-b16", "y": "ab-a1", "n_utts": 400, "delta_pp": -4.416, "ci95": [ -6.578, -2.508 ], "utts_with_identical_edit_counts": 231 }, { "set": "ami", "x": "ab-b16w", "y": "ab-a1", "n_utts": 400, "delta_pp": -4.392, "ci95": [ -6.574, -2.489 ], "utts_with_identical_edit_counts": 230 }, { "set": "ami", "x": "ab-b32", "y": "ab-a1", "n_utts": 400, "delta_pp": -4.319, "ci95": [ -6.495, -2.399 ], "utts_with_identical_edit_counts": 230 }, { "set": "ami", "x": "ab-c", "y": "ab-a1", "n_utts": 400, "delta_pp": -3.179, "ci95": [ -5.127, -1.483 ], "utts_with_identical_edit_counts": 234 }, { "set": "ami", "x": "ab-cf16", "y": "ab-a1", "n_utts": 400, "delta_pp": -3.373, "ci95": [ -5.289, -1.643 ], "utts_with_identical_edit_counts": 230 }, { "set": "ami", "x": "ab-cf32", "y": "ab-a1", "n_utts": 400, "delta_pp": -3.324, "ci95": [ -5.249, -1.604 ], "utts_with_identical_edit_counts": 229 }, { "set": "ami", "x": "ab-d", "y": "ab-a1", "n_utts": 400, "delta_pp": -2.669, "ci95": [ -4.423, -1.045 ], "utts_with_identical_edit_counts": 244 }, { "set": "ami", "x": "ab-af32", "y": "ab-b32", "n_utts": 400, "delta_pp": 2.111, "ci95": [ 0.646, 3.692 ], "utts_with_identical_edit_counts": 245 }, { "set": "ami", "x": "ab-b16", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.097, "ci95": [ -0.222, 0.0 ], "utts_with_identical_edit_counts": 392 }, { "set": "ami", "x": "ab-b16w", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.073, "ci95": [ -0.181, 0.025 ], "utts_with_identical_edit_counts": 395 }, { "set": "ami", "x": "ab-c", "y": "ab-b32", "n_utts": 400, "delta_pp": 1.14, "ci95": [ 0.28, 2.158 ], "utts_with_identical_edit_counts": 328 }, { "set": "ami", "x": "ab-cf16", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.946, "ci95": [ 0.195, 1.867 ], "utts_with_identical_edit_counts": 337 }, { "set": "ami", "x": "ab-cf32", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.995, "ci95": [ 0.241, 1.922 ], "utts_with_identical_edit_counts": 337 }, { "set": "ami", "x": "ab-d", "y": "ab-b32", "n_utts": 400, "delta_pp": 1.65, "ci95": [ 0.38, 3.064 ], "utts_with_identical_edit_counts": 240 }, { "set": "pc", "x": "ab-af32", "y": "ab-a1", "n_utts": 40, "delta_pp": -8.586, "ci95": [ -15.696, -2.422 ], "utts_with_identical_edit_counts": 15 }, { "set": "pc", "x": "ab-b16", "y": "ab-a1", "n_utts": 40, "delta_pp": -9.596, "ci95": [ -16.959, -2.941 ], "utts_with_identical_edit_counts": 9 }, { "set": "pc", "x": "ab-b16w", "y": "ab-a1", "n_utts": 40, "delta_pp": -9.668, "ci95": [ -17.036, -3.023 ], "utts_with_identical_edit_counts": 9 }, { "set": "pc", "x": "ab-b32", "y": "ab-a1", "n_utts": 40, "delta_pp": -9.668, "ci95": [ -17.036, -3.023 ], "utts_with_identical_edit_counts": 9 }, { "set": "pc", "x": "ab-c", "y": "ab-a1", "n_utts": 40, "delta_pp": -9.452, "ci95": [ -16.961, -2.753 ], "utts_with_identical_edit_counts": 11 }, { "set": "pc", "x": "ab-cf16", "y": "ab-a1", "n_utts": 40, "delta_pp": -9.452, "ci95": [ -16.786, -2.784 ], "utts_with_identical_edit_counts": 9 }, { "set": "pc", "x": "ab-cf32", "y": "ab-a1", "n_utts": 40, "delta_pp": -9.524, "ci95": [ -16.929, -2.788 ], "utts_with_identical_edit_counts": 9 }, { "set": "pc", "x": "ab-d", "y": "ab-a1", "n_utts": 40, "delta_pp": -10.462, "ci95": [ -17.816, -3.869 ], "utts_with_identical_edit_counts": 14 }, { "set": "pc", "x": "ab-af32", "y": "ab-b32", "n_utts": 40, "delta_pp": 1.082, "ci95": [ -1.183, 4.857 ], "utts_with_identical_edit_counts": 17 }, { "set": "pc", "x": "ab-b16", "y": "ab-b32", "n_utts": 40, "delta_pp": 0.072, "ci95": [ 0.0, 0.236 ], "utts_with_identical_edit_counts": 38 }, { "set": "pc", "x": "ab-b16w", "y": "ab-b32", "n_utts": 40, "delta_pp": 0.0, "ci95": [ 0.0, 0.0 ], "utts_with_identical_edit_counts": 39 }, { "set": "pc", "x": "ab-c", "y": "ab-b32", "n_utts": 40, "delta_pp": 0.216, "ci95": [ -0.383, 0.871 ], "utts_with_identical_edit_counts": 30 }, { "set": "pc", "x": "ab-cf16", "y": "ab-b32", "n_utts": 40, "delta_pp": 0.216, "ci95": [ 0.0, 0.571 ], "utts_with_identical_edit_counts": 38 }, { "set": "pc", "x": "ab-cf32", "y": "ab-b32", "n_utts": 40, "delta_pp": 0.144, "ci95": [ -0.152, 0.528 ], "utts_with_identical_edit_counts": 37 }, { "set": "pc", "x": "ab-d", "y": "ab-b32", "n_utts": 40, "delta_pp": -0.794, "ci95": [ -1.512, -0.075 ], "utts_with_identical_edit_counts": 19 }, { "set": "null", "x": "ab-af32", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.332, "ci95": [ -0.649, -0.013 ], "utts_with_identical_edit_counts": 338 }, { "set": "null", "x": "ab-b16", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.761, "ci95": [ -1.129, -0.393 ], "utts_with_identical_edit_counts": 311 }, { "set": "null", "x": "ab-b16w", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.761, "ci95": [ -1.13, -0.396 ], "utts_with_identical_edit_counts": 309 }, { "set": "null", "x": "ab-b32", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.761, "ci95": [ -1.129, -0.393 ], "utts_with_identical_edit_counts": 311 }, { "set": "null", "x": "ab-c", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.835, "ci95": [ -1.179, -0.48 ], "utts_with_identical_edit_counts": 311 }, { "set": "null", "x": "ab-cf16", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.847, "ci95": [ -1.21, -0.488 ], "utts_with_identical_edit_counts": 309 }, { "set": "null", "x": "ab-cf32", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.847, "ci95": [ -1.21, -0.482 ], "utts_with_identical_edit_counts": 310 }, { "set": "null", "x": "ab-d", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.688, "ci95": [ -1.025, -0.363 ], "utts_with_identical_edit_counts": 309 }, { "set": "null", "x": "ab-af32", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.43, "ci95": [ 0.137, 0.729 ], "utts_with_identical_edit_counts": 327 }, { "set": "null", "x": "ab-b16", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.0, "ci95": [ 0.0, 0.0 ], "utts_with_identical_edit_counts": 400 }, { "set": "null", "x": "ab-b16w", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.0, "ci95": [ -0.036, 0.037 ], "utts_with_identical_edit_counts": 398 }, { "set": "null", "x": "ab-c", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.074, "ci95": [ -0.202, 0.046 ], "utts_with_identical_edit_counts": 386 }, { "set": "null", "x": "ab-cf16", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.086, "ci95": [ -0.218, 0.026 ], "utts_with_identical_edit_counts": 390 }, { "set": "null", "x": "ab-cf32", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.086, "ci95": [ -0.216, 0.025 ], "utts_with_identical_edit_counts": 390 }, { "set": "null", "x": "ab-d", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.074, "ci95": [ -0.177, 0.336 ], "utts_with_identical_edit_counts": 344 } ], "pc": [ { "arm": "ab-a1", "utts": 40, "deletions_clean": 2, "deletions_silenced": 306, "errors_clean": 25, "errors_silenced": 344, "wer_clean": 1.8, "wer_silenced": 24.82, "utts_with_more_deletions": 40 }, { "arm": "ab-af32", "utts": 40, "deletions_clean": 1, "deletions_silenced": 185, "errors_clean": 27, "errors_silenced": 225, "wer_clean": 1.95, "wer_silenced": 16.23, "utts_with_more_deletions": 40 }, { "arm": "ab-b16", "utts": 40, "deletions_clean": 2, "deletions_silenced": 177, "errors_clean": 26, "errors_silenced": 211, "wer_clean": 1.88, "wer_silenced": 15.22, "utts_with_more_deletions": 40 }, { "arm": "ab-b16w", "utts": 40, "deletions_clean": 2, "deletions_silenced": 177, "errors_clean": 26, "errors_silenced": 210, "wer_clean": 1.88, "wer_silenced": 15.15, "utts_with_more_deletions": 40 }, { "arm": "ab-b32", "utts": 40, "deletions_clean": 2, "deletions_silenced": 178, "errors_clean": 26, "errors_silenced": 210, "wer_clean": 1.88, "wer_silenced": 15.15, "utts_with_more_deletions": 40 }, { "arm": "ab-c", "utts": 40, "deletions_clean": 2, "deletions_silenced": 181, "errors_clean": 26, "errors_silenced": 213, "wer_clean": 1.88, "wer_silenced": 15.37, "utts_with_more_deletions": 40 }, { "arm": "ab-cf16", "utts": 40, "deletions_clean": 2, "deletions_silenced": 178, "errors_clean": 28, "errors_silenced": 213, "wer_clean": 2.02, "wer_silenced": 15.37, "utts_with_more_deletions": 40 }, { "arm": "ab-cf32", "utts": 40, "deletions_clean": 2, "deletions_silenced": 176, "errors_clean": 28, "errors_silenced": 212, "wer_clean": 2.02, "wer_silenced": 15.3, "utts_with_more_deletions": 40 }, { "arm": "ab-d", "utts": 40, "deletions_clean": 1, "deletions_silenced": 167, "errors_clean": 26, "errors_silenced": 199, "wer_clean": 1.88, "wer_silenced": 14.36, "utts_with_more_deletions": 40 } ], "null": [ { "arm": "ab-a1", "n_utts": 400, "delta_pp": 0.037, "ci95": [ -0.078, 0.156 ], "utts_with_identical_edit_counts": 383 }, { "arm": "ab-af32", "n_utts": 400, "delta_pp": -0.025, "ci95": [ -0.063, 0.0 ], "utts_with_identical_edit_counts": 398 }, { "arm": "ab-b16", "n_utts": 400, "delta_pp": 0.0, "ci95": [ 0.0, 0.0 ], "utts_with_identical_edit_counts": 400 }, { "arm": "ab-b16w", "n_utts": 400, "delta_pp": 0.012, "ci95": [ 0.0, 0.039 ], "utts_with_identical_edit_counts": 399 }, { "arm": "ab-b32", "n_utts": 400, "delta_pp": 0.0, "ci95": [ 0.0, 0.0 ], "utts_with_identical_edit_counts": 400 }, { "arm": "ab-c", "n_utts": 400, "delta_pp": -0.037, "ci95": [ -0.135, 0.057 ], "utts_with_identical_edit_counts": 389 }, { "arm": "ab-cf16", "n_utts": 400, "delta_pp": 0.0, "ci95": [ 0.0, 0.0 ], "utts_with_identical_edit_counts": 400 }, { "arm": "ab-cf32", "n_utts": 400, "delta_pp": 0.0, "ci95": [ 0.0, 0.0 ], "utts_with_identical_edit_counts": 400 }, { "arm": "ab-d", "n_utts": 400, "delta_pp": -0.061, "ci95": [ -0.149, 0.024 ], "utts_with_identical_edit_counts": 391 } ], "same": [ { "a": "ab-a1", "b": "ab-a2", "utts": 400, "identical_text": 400 }, { "a": "ab-a1", "b": "ab-at16", "utts": 400, "identical_text": 400 }, { "a": "ab-b32", "b": "ab-b32t", "utts": 400, "identical_text": 400 } ], "features": [ { "arm": "ab-a1", "outputs": 400, "with_sentence_punct": 89.0, "with_comma": 73.0, "with_upper": 94.5, "all_lower": 5.2, "empty": 1, "non_ascii": 1 }, { "arm": "ab-a2", "outputs": 400, "with_sentence_punct": 89.0, "with_comma": 73.0, "with_upper": 94.5, "all_lower": 5.2, "empty": 1, "non_ascii": 1 }, { "arm": "ab-af32", "outputs": 400, "with_sentence_punct": 88.0, "with_comma": 74.5, "with_upper": 96.0, "all_lower": 4.0, "empty": 0, "non_ascii": 1 }, { "arm": "ab-at16", "outputs": 400, "with_sentence_punct": 89.0, "with_comma": 73.0, "with_upper": 94.5, "all_lower": 5.2, "empty": 1, "non_ascii": 1 }, { "arm": "ab-b16", "outputs": 400, "with_sentence_punct": 50.0, "with_comma": 62.8, "with_upper": 96.8, "all_lower": 3.2, "empty": 0, "non_ascii": 0 }, { "arm": "ab-b16w", "outputs": 400, "with_sentence_punct": 50.8, "with_comma": 63.0, "with_upper": 96.8, "all_lower": 3.2, "empty": 0, "non_ascii": 0 }, { "arm": "ab-b32", "outputs": 400, "with_sentence_punct": 50.0, "with_comma": 63.0, "with_upper": 96.8, "all_lower": 3.2, "empty": 0, "non_ascii": 0 }, { "arm": "ab-b32c", "outputs": 400, "with_sentence_punct": 50.0, "with_comma": 63.0, "with_upper": 96.8, "all_lower": 3.2, "empty": 0, "non_ascii": 0 }, { "arm": "ab-b32t", "outputs": 400, "with_sentence_punct": 50.0, "with_comma": 63.0, "with_upper": 96.8, "all_lower": 3.2, "empty": 0, "non_ascii": 0 }, { "arm": "ab-c", "outputs": 400, "with_sentence_punct": 51.0, "with_comma": 64.5, "with_upper": 97.0, "all_lower": 3.0, "empty": 0, "non_ascii": 0 }, { "arm": "ab-cf16", "outputs": 400, "with_sentence_punct": 48.5, "with_comma": 62.0, "with_upper": 96.0, "all_lower": 4.0, "empty": 0, "non_ascii": 0 }, { "arm": "ab-cf32", "outputs": 400, "with_sentence_punct": 48.5, "with_comma": 62.0, "with_upper": 96.0, "all_lower": 4.0, "empty": 0, "non_ascii": 0 }, { "arm": "ab-d", "outputs": 400, "with_sentence_punct": 83.8, "with_comma": 73.5, "with_upper": 94.8, "all_lower": 5.2, "empty": 0, "non_ascii": 0 } ] }