## wer
{"set": "ls-clean", "arm": "ab-a1", "utts": 400, "missing": 0, "ref_words": 8144, "S": 160, "D": 35, "I": 25, "wer": 2.701, "empty_outputs": 1}
{"set": "ls-clean", "arm": "ab-a2", "utts": 400, "missing": 0, "ref_words": 8144, "S": 160, "D": 35, "I": 25, "wer": 2.701, "empty_outputs": 1}
{"set": "ls-clean", "arm": "ab-af32", "utts": 400, "missing": 0, "ref_words": 8144, "S": 151, "D": 26, "I": 21, "wer": 2.431, "empty_outputs": 0}
{"set": "ls-clean", "arm": "ab-at16", "utts": 400, "missing": 0, "ref_words": 8144, "S": 160, "D": 35, "I": 25, "wer": 2.701, "empty_outputs": 1}
{"set": "ls-clean", "arm": "ab-b16", "utts": 400, "missing": 0, "ref_words": 8144, "S": 120, "D": 24, "I": 17, "wer": 1.977, "empty_outputs": 0}
{"set": "ls-clean", "arm": "ab-b16w", "utts": 400, "missing": 0, "ref_words": 8144, "S": 120, "D": 23, "I": 17, "wer": 1.965, "empty_outputs": 0}
{"set": "ls-clean", "arm": "ab-b32", "utts": 400, "missing": 0, "ref_words": 8144, "S": 120, "D": 24, "I": 17, "wer": 1.977, "empty_outputs": 0}
{"set": "ls-clean", "arm": "ab-b32c", "utts": 400, "missing": 0, "ref_words": 8144, "S": 120, "D": 24, "I": 17, "wer": 1.977, "empty_outputs": 0}
{"set": "ls-clean", "arm": "ab-b32t", "utts": 400, "missing": 0, "ref_words": 8144, "S": 120, "D": 24, "I": 17, "wer": 1.977, "empty_outputs": 0}
{"set": "ls-clean", "arm": "ab-c", "utts": 400, "missing": 0, "ref_words": 8144, "S": 120, "D": 21, "I": 17, "wer": 1.94, "empty_outputs": 0}
{"set": "ls-clean", "arm": "ab-cf16", "utts": 400, "missing": 0, "ref_words": 8144, "S": 120, "D": 17, "I": 17, "wer": 1.891, "empty_outputs": 0}
{"set": "ls-clean", "arm": "ab-cf32", "utts": 400, "missing": 0, "ref_words": 8144, "S": 120, "D": 17, "I": 17, "wer": 1.891, "empty_outputs": 0}
{"set": "ls-clean", "arm": "ab-d", "utts": 400, "missing": 0, "ref_words": 8144, "S": 132, "D": 23, "I": 17, "wer": 2.112, "empty_outputs": 0}
{"set": "ls-other", "arm": "ab-a1", "utts": 400, "missing": 0, "ref_words": 7370, "S": 269, "D": 38, "I": 29, "wer": 4.559, "empty_outputs": 0}
{"set": "ls-other", "arm": "ab-af32", "utts": 400, "missing": 0, "ref_words": 7370, "S": 198, "D": 22, "I": 24, "wer": 3.311, "empty_outputs": 0}
{"set": "ls-other", "arm": "ab-b16", "utts": 400, "missing": 0, "ref_words": 7370, "S": 179, "D": 24, "I": 26, "wer": 3.107, "empty_outputs": 0}
{"set": "ls-other", "arm": "ab-b16w", "utts": 400, "missing": 0, "ref_words": 7370, "S": 178, "D": 24, "I": 26, "wer": 3.094, "empty_outputs": 0}
{"set": "ls-other", "arm": "ab-b32", "utts": 400, "missing": 0, "ref_words": 7370, "S": 179, "D": 24, "I": 26, "wer": 3.107, "empty_outputs": 0}
{"set": "ls-other", "arm": "ab-c", "utts": 400, "missing": 0, "ref_words": 7370, "S": 172, "D": 21, "I": 23, "wer": 2.931, "empty_outputs": 0}
{"set": "ls-other", "arm": "ab-cf16", "utts": 400, "missing": 0, "ref_words": 7370, "S": 176, "D": 22, "I": 24, "wer": 3.012, "empty_outputs": 0}
{"set": "ls-other", "arm": "ab-cf32", "utts": 400, "missing": 0, "ref_words": 7370, "S": 177, "D": 22, "I": 24, "wer": 3.026, "empty_outputs": 0}
{"set": "ls-other", "arm": "ab-d", "utts": 400, "missing": 0, "ref_words": 7370, "S": 201, "D": 24, "I": 25, "wer": 3.392, "empty_outputs": 0}
{"set": "ami", "arm": "ab-a1", "utts": 400, "missing": 0, "ref_words": 4121, "S": 181, "D": 299, "I": 43, "wer": 12.691, "empty_outputs": 26}
{"set": "ami", "arm": "ab-af32", "utts": 400, "missing": 0, "ref_words": 4121, "S": 180, "D": 182, "I": 70, "wer": 10.483, "empty_outputs": 18}
{"set": "ami", "arm": "ab-b16", "utts": 400, "missing": 0, "ref_words": 4121, "S": 155, "D": 130, "I": 56, "wer": 8.275, "empty_outputs": 9}
{"set": "ami", "arm": "ab-b16w", "utts": 400, "missing": 0, "ref_words": 4121, "S": 155, "D": 130, "I": 57, "wer": 8.299, "empty_outputs": 9}
{"set": "ami", "arm": "ab-b32", "utts": 400, "missing": 0, "ref_words": 4121, "S": 157, "D": 131, "I": 57, "wer": 8.372, "empty_outputs": 10}
{"set": "ami", "arm": "ab-c", "utts": 400, "missing": 0, "ref_words": 4121, "S": 162, "D": 179, "I": 51, "wer": 9.512, "empty_outputs": 17}
{"set": "ami", "arm": "ab-cf16", "utts": 400, "missing": 0, "ref_words": 4121, "S": 157, "D": 174, "I": 53, "wer": 9.318, "empty_outputs": 15}
{"set": "ami", "arm": "ab-cf32", "utts": 400, "missing": 0, "ref_words": 4121, "S": 158, "D": 173, "I": 55, "wer": 9.367, "empty_outputs": 15}
{"set": "ami", "arm": "ab-d", "utts": 400, "missing": 0, "ref_words": 4121, "S": 178, "D": 154, "I": 81, "wer": 10.022, "empty_outputs": 15}
{"set": "pc", "arm": "ab-a1", "utts": 40, "missing": 0, "ref_words": 1386, "S": 36, "D": 306, "I": 2, "wer": 24.82, "empty_outputs": 0}
{"set": "pc", "arm": "ab-af32", "utts": 40, "missing": 0, "ref_words": 1386, "S": 37, "D": 185, "I": 3, "wer": 16.234, "empty_outputs": 0}
{"set": "pc", "arm": "ab-b16", "utts": 40, "missing": 0, "ref_words": 1386, "S": 29, "D": 177, "I": 5, "wer": 15.224, "empty_outputs": 0}
{"set": "pc", "arm": "ab-b16w", "utts": 40, "missing": 0, "ref_words": 1386, "S": 28, "D": 177, "I": 5, "wer": 15.152, "empty_outputs": 0}
{"set": "pc", "arm": "ab-b32", "utts": 40, "missing": 0, "ref_words": 1386, "S": 27, "D": 178, "I": 5, "wer": 15.152, "empty_outputs": 0}
{"set": "pc", "arm": "ab-c", "utts": 40, "missing": 0, "ref_words": 1386, "S": 28, "D": 181, "I": 4, "wer": 15.368, "empty_outputs": 0}
{"set": "pc", "arm": "ab-cf16", "utts": 40, "missing": 0, "ref_words": 1386, "S": 30, "D": 178, "I": 5, "wer": 15.368, "empty_outputs": 0}
{"set": "pc", "arm": "ab-cf32", "utts": 40, "missing": 0, "ref_words": 1386, "S": 31, "D": 176, "I": 5, "wer": 15.296, "empty_outputs": 0}
{"set": "pc", "arm": "ab-d", "utts": 40, "missing": 0, "ref_words": 1386, "S": 28, "D": 167, "I": 4, "wer": 14.358, "empty_outputs": 0}
{"set": "null", "arm": "ab-a1", "utts": 400, "missing": 0, "ref_words": 8144, "S": 165, "D": 34, "I": 24, "wer": 2.738, "empty_outputs": 1}
{"set": "null", "arm": "ab-af32", "utts": 400, "missing": 0, "ref_words": 8144, "S": 150, "D": 26, "I": 20, "wer": 2.407, "empty_outputs": 0}
{"set": "null", "arm": "ab-b16", "utts": 400, "missing": 0, "ref_words": 8144, "S": 120, "D": 24, "I": 17, "wer": 1.977, "empty_outputs": 0}
{"set": "null", "arm": "ab-b16w", "utts": 400, "missing": 0, "ref_words": 8144, "S": 121, "D": 23, "I": 17, "wer": 1.977, "empty_outputs": 0}
{"set": "null", "arm": "ab-b32", "utts": 400, "missing": 0, "ref_words": 8144, "S": 120, "D": 24, "I": 17, "wer": 1.977, "empty_outputs": 0}
{"set": "null", "arm": "ab-c", "utts": 400, "missing": 0, "ref_words": 8144, "S": 119, "D": 19, "I": 17, "wer": 1.903, "empty_outputs": 0}
{"set": "null", "arm": "ab-cf16", "utts": 400, "missing": 0, "ref_words": 8144, "S": 120, "D": 17, "I": 17, "wer": 1.891, "empty_outputs": 0}
{"set": "null", "arm": "ab-cf32", "utts": 400, "missing": 0, "ref_words": 8144, "S": 120, "D": 17, "I": 17, "wer": 1.891, "empty_outputs": 0}
{"set": "null", "arm": "ab-d", "utts": 400, "missing": 0, "ref_words": 8144, "S": 129, "D": 21, "I": 17, "wer": 2.051, "empty_outputs": 0}
## delta
{"set": "ls-clean", "x": "ab-a2", "y": "ab-a1", "n_utts": 400, "delta_pp": 0.0, "ci95": [0.0, 0.0], "utts_with_identical_edit_counts": 400}
{"set": "ls-clean", "x": "ab-af32", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.27, "ci95": [-0.599, 0.048], "utts_with_identical_edit_counts": 340}
{"set": "ls-clean", "x": "ab-at16", "y": "ab-a1", "n_utts": 400, "delta_pp": 0.0, "ci95": [0.0, 0.0], "utts_with_identical_edit_counts": 400}
{"set": "ls-clean", "x": "ab-b16", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.724, "ci95": [-1.097, -0.348], "utts_with_identical_edit_counts": 315}
{"set": "ls-clean", "x": "ab-b16w", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.737, "ci95": [-1.111, -0.361], "utts_with_identical_edit_counts": 314}
{"set": "ls-clean", "x": "ab-b32", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.724, "ci95": [-1.097, -0.348], "utts_with_identical_edit_counts": 315}
{"set": "ls-clean", "x": "ab-b32c", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.724, "ci95": [-1.097, -0.348], "utts_with_identical_edit_counts": 315}
{"set": "ls-clean", "x": "ab-b32t", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.724, "ci95": [-1.097, -0.348], "utts_with_identical_edit_counts": 315}
{"set": "ls-clean", "x": "ab-c", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.761, "ci95": [-1.112, -0.407], "utts_with_identical_edit_counts": 309}
{"set": "ls-clean", "x": "ab-cf16", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.81, "ci95": [-1.177, -0.441], "utts_with_identical_edit_counts": 313}
{"set": "ls-clean", "x": "ab-cf32", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.81, "ci95": [-1.181, -0.446], "utts_with_identical_edit_counts": 313}
{"set": "ls-clean", "x": "ab-d", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.589, "ci95": [-0.955, -0.256], "utts_with_identical_edit_counts": 308}
{"set": "ls-clean", "x": "ab-a2", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.724, "ci95": [0.35, 1.098], "utts_with_identical_edit_counts": 315}
{"set": "ls-clean", "x": "ab-af32", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.454, "ci95": [0.159, 0.758], "utts_with_identical_edit_counts": 326}
{"set": "ls-clean", "x": "ab-at16", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.724, "ci95": [0.35, 1.098], "utts_with_identical_edit_counts": 315}
{"set": "ls-clean", "x": "ab-b16", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.0, "ci95": [0.0, 0.0], "utts_with_identical_edit_counts": 400}
{"set": "ls-clean", "x": "ab-b16w", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.012, "ci95": [-0.039, 0.0], "utts_with_identical_edit_counts": 399}
{"set": "ls-clean", "x": "ab-b32c", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.0, "ci95": [0.0, 0.0], "utts_with_identical_edit_counts": 400}
{"set": "ls-clean", "x": "ab-b32t", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.0, "ci95": [0.0, 0.0], "utts_with_identical_edit_counts": 400}
{"set": "ls-clean", "x": "ab-c", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.037, "ci95": [-0.177, 0.101], "utts_with_identical_edit_counts": 383}
{"set": "ls-clean", "x": "ab-cf16", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.086, "ci95": [-0.218, 0.026], "utts_with_identical_edit_counts": 390}
{"set": "ls-clean", "x": "ab-cf32", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.086, "ci95": [-0.216, 0.025], "utts_with_identical_edit_counts": 390}
{"set": "ls-clean", "x": "ab-d", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.135, "ci95": [-0.122, 0.393], "utts_with_identical_edit_counts": 343}
{"set": "ls-other", "x": "ab-af32", "y": "ab-a1", "n_utts": 400, "delta_pp": -1.248, "ci95": [-1.701, -0.847], "utts_with_identical_edit_counts": 304}
{"set": "ls-other", "x": "ab-b16", "y": "ab-a1", "n_utts": 400, "delta_pp": -1.452, "ci95": [-1.989, -0.949], "utts_with_identical_edit_counts": 272}
{"set": "ls-other", "x": "ab-b16w", "y": "ab-a1", "n_utts": 400, "delta_pp": -1.465, "ci95": [-2.003, -0.965], "utts_with_identical_edit_counts": 273}
{"set": "ls-other", "x": "ab-b32", "y": "ab-a1", "n_utts": 400, "delta_pp": -1.452, "ci95": [-1.989, -0.949], "utts_with_identical_edit_counts": 272}
{"set": "ls-other", "x": "ab-c", "y": "ab-a1", "n_utts": 400, "delta_pp": -1.628, "ci95": [-2.148, -1.139], "utts_with_identical_edit_counts": 275}
{"set": "ls-other", "x": "ab-cf16", "y": "ab-a1", "n_utts": 400, "delta_pp": -1.547, "ci95": [-2.073, -1.056], "utts_with_identical_edit_counts": 277}
{"set": "ls-other", "x": "ab-cf32", "y": "ab-a1", "n_utts": 400, "delta_pp": -1.533, "ci95": [-2.059, -1.036], "utts_with_identical_edit_counts": 276}
{"set": "ls-other", "x": "ab-d", "y": "ab-a1", "n_utts": 400, "delta_pp": -1.167, "ci95": [-1.678, -0.681], "utts_with_identical_edit_counts": 279}
{"set": "ls-other", "x": "ab-af32", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.204, "ci95": [-0.202, 0.635], "utts_with_identical_edit_counts": 295}
{"set": "ls-other", "x": "ab-b16", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.0, "ci95": [0.0, 0.0], "utts_with_identical_edit_counts": 400}
{"set": "ls-other", "x": "ab-b16w", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.014, "ci95": [-0.043, 0.0], "utts_with_identical_edit_counts": 399}
{"set": "ls-other", "x": "ab-c", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.176, "ci95": [-0.388, 0.026], "utts_with_identical_edit_counts": 364}
{"set": "ls-other", "x": "ab-cf16", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.095, "ci95": [-0.261, 0.069], "utts_with_identical_edit_counts": 373}
{"set": "ls-other", "x": "ab-cf32", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.081, "ci95": [-0.25, 0.087], "utts_with_identical_edit_counts": 372}
{"set": "ls-other", "x": "ab-d", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.285, "ci95": [-0.08, 0.66], "utts_with_identical_edit_counts": 307}
{"set": "ami", "x": "ab-af32", "y": "ab-a1", "n_utts": 400, "delta_pp": -2.208, "ci95": [-3.682, -0.893], "utts_with_identical_edit_counts": 292}
{"set": "ami", "x": "ab-b16", "y": "ab-a1", "n_utts": 400, "delta_pp": -4.416, "ci95": [-6.578, -2.508], "utts_with_identical_edit_counts": 231}
{"set": "ami", "x": "ab-b16w", "y": "ab-a1", "n_utts": 400, "delta_pp": -4.392, "ci95": [-6.574, -2.489], "utts_with_identical_edit_counts": 230}
{"set": "ami", "x": "ab-b32", "y": "ab-a1", "n_utts": 400, "delta_pp": -4.319, "ci95": [-6.495, -2.399], "utts_with_identical_edit_counts": 230}
{"set": "ami", "x": "ab-c", "y": "ab-a1", "n_utts": 400, "delta_pp": -3.179, "ci95": [-5.127, -1.483], "utts_with_identical_edit_counts": 234}
{"set": "ami", "x": "ab-cf16", "y": "ab-a1", "n_utts": 400, "delta_pp": -3.373, "ci95": [-5.289, -1.643], "utts_with_identical_edit_counts": 230}
{"set": "ami", "x": "ab-cf32", "y": "ab-a1", "n_utts": 400, "delta_pp": -3.324, "ci95": [-5.249, -1.604], "utts_with_identical_edit_counts": 229}
{"set": "ami", "x": "ab-d", "y": "ab-a1", "n_utts": 400, "delta_pp": -2.669, "ci95": [-4.423, -1.045], "utts_with_identical_edit_counts": 244}
{"set": "ami", "x": "ab-af32", "y": "ab-b32", "n_utts": 400, "delta_pp": 2.111, "ci95": [0.646, 3.692], "utts_with_identical_edit_counts": 245}
{"set": "ami", "x": "ab-b16", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.097, "ci95": [-0.222, 0.0], "utts_with_identical_edit_counts": 392}
{"set": "ami", "x": "ab-b16w", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.073, "ci95": [-0.181, 0.025], "utts_with_identical_edit_counts": 395}
{"set": "ami", "x": "ab-c", "y": "ab-b32", "n_utts": 400, "delta_pp": 1.14, "ci95": [0.28, 2.158], "utts_with_identical_edit_counts": 328}
{"set": "ami", "x": "ab-cf16", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.946, "ci95": [0.195, 1.867], "utts_with_identical_edit_counts": 337}
{"set": "ami", "x": "ab-cf32", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.995, "ci95": [0.241, 1.922], "utts_with_identical_edit_counts": 337}
{"set": "ami", "x": "ab-d", "y": "ab-b32", "n_utts": 400, "delta_pp": 1.65, "ci95": [0.38, 3.064], "utts_with_identical_edit_counts": 240}
{"set": "pc", "x": "ab-af32", "y": "ab-a1", "n_utts": 40, "delta_pp": -8.586, "ci95": [-15.696, -2.422], "utts_with_identical_edit_counts": 15}
{"set": "pc", "x": "ab-b16", "y": "ab-a1", "n_utts": 40, "delta_pp": -9.596, "ci95": [-16.959, -2.941], "utts_with_identical_edit_counts": 9}
{"set": "pc", "x": "ab-b16w", "y": "ab-a1", "n_utts": 40, "delta_pp": -9.668, "ci95": [-17.036, -3.023], "utts_with_identical_edit_counts": 9}
{"set": "pc", "x": "ab-b32", "y": "ab-a1", "n_utts": 40, "delta_pp": -9.668, "ci95": [-17.036, -3.023], "utts_with_identical_edit_counts": 9}
{"set": "pc", "x": "ab-c", "y": "ab-a1", "n_utts": 40, "delta_pp": -9.452, "ci95": [-16.961, -2.753], "utts_with_identical_edit_counts": 11}
{"set": "pc", "x": "ab-cf16", "y": "ab-a1", "n_utts": 40, "delta_pp": -9.452, "ci95": [-16.786, -2.784], "utts_with_identical_edit_counts": 9}
{"set": "pc", "x": "ab-cf32", "y": "ab-a1", "n_utts": 40, "delta_pp": -9.524, "ci95": [-16.929, -2.788], "utts_with_identical_edit_counts": 9}
{"set": "pc", "x": "ab-d", "y": "ab-a1", "n_utts": 40, "delta_pp": -10.462, "ci95": [-17.816, -3.869], "utts_with_identical_edit_counts": 14}
{"set": "pc", "x": "ab-af32", "y": "ab-b32", "n_utts": 40, "delta_pp": 1.082, "ci95": [-1.183, 4.857], "utts_with_identical_edit_counts": 17}
{"set": "pc", "x": "ab-b16", "y": "ab-b32", "n_utts": 40, "delta_pp": 0.072, "ci95": [0.0, 0.236], "utts_with_identical_edit_counts": 38}
{"set": "pc", "x": "ab-b16w", "y": "ab-b32", "n_utts": 40, "delta_pp": 0.0, "ci95": [0.0, 0.0], "utts_with_identical_edit_counts": 39}
{"set": "pc", "x": "ab-c", "y": "ab-b32", "n_utts": 40, "delta_pp": 0.216, "ci95": [-0.383, 0.871], "utts_with_identical_edit_counts": 30}
{"set": "pc", "x": "ab-cf16", "y": "ab-b32", "n_utts": 40, "delta_pp": 0.216, "ci95": [0.0, 0.571], "utts_with_identical_edit_counts": 38}
{"set": "pc", "x": "ab-cf32", "y": "ab-b32", "n_utts": 40, "delta_pp": 0.144, "ci95": [-0.152, 0.528], "utts_with_identical_edit_counts": 37}
{"set": "pc", "x": "ab-d", "y": "ab-b32", "n_utts": 40, "delta_pp": -0.794, "ci95": [-1.512, -0.075], "utts_with_identical_edit_counts": 19}
{"set": "null", "x": "ab-af32", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.332, "ci95": [-0.649, -0.013], "utts_with_identical_edit_counts": 338}
{"set": "null", "x": "ab-b16", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.761, "ci95": [-1.129, -0.393], "utts_with_identical_edit_counts": 311}
{"set": "null", "x": "ab-b16w", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.761, "ci95": [-1.13, -0.396], "utts_with_identical_edit_counts": 309}
{"set": "null", "x": "ab-b32", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.761, "ci95": [-1.129, -0.393], "utts_with_identical_edit_counts": 311}
{"set": "null", "x": "ab-c", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.835, "ci95": [-1.179, -0.48], "utts_with_identical_edit_counts": 311}
{"set": "null", "x": "ab-cf16", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.847, "ci95": [-1.21, -0.488], "utts_with_identical_edit_counts": 309}
{"set": "null", "x": "ab-cf32", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.847, "ci95": [-1.21, -0.482], "utts_with_identical_edit_counts": 310}
{"set": "null", "x": "ab-d", "y": "ab-a1", "n_utts": 400, "delta_pp": -0.688, "ci95": [-1.025, -0.363], "utts_with_identical_edit_counts": 309}
{"set": "null", "x": "ab-af32", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.43, "ci95": [0.137, 0.729], "utts_with_identical_edit_counts": 327}
{"set": "null", "x": "ab-b16", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.0, "ci95": [0.0, 0.0], "utts_with_identical_edit_counts": 400}
{"set": "null", "x": "ab-b16w", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.0, "ci95": [-0.036, 0.037], "utts_with_identical_edit_counts": 398}
{"set": "null", "x": "ab-c", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.074, "ci95": [-0.202, 0.046], "utts_with_identical_edit_counts": 386}
{"set": "null", "x": "ab-cf16", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.086, "ci95": [-0.218, 0.026], "utts_with_identical_edit_counts": 390}
{"set": "null", "x": "ab-cf32", "y": "ab-b32", "n_utts": 400, "delta_pp": -0.086, "ci95": [-0.216, 0.025], "utts_with_identical_edit_counts": 390}
{"set": "null", "x": "ab-d", "y": "ab-b32", "n_utts": 400, "delta_pp": 0.074, "ci95": [-0.177, 0.336], "utts_with_identical_edit_counts": 344}
## pc
{"arm": "ab-a1", "utts": 40, "deletions_clean": 2, "deletions_silenced": 306, "errors_clean": 25, "errors_silenced": 344, "wer_clean": 1.8, "wer_silenced": 24.82, "utts_with_more_deletions": 40}
{"arm": "ab-af32", "utts": 40, "deletions_clean": 1, "deletions_silenced": 185, "errors_clean": 27, "errors_silenced": 225, "wer_clean": 1.95, "wer_silenced": 16.23, "utts_with_more_deletions": 40}
{"arm": "ab-b16", "utts": 40, "deletions_clean": 2, "deletions_silenced": 177, "errors_clean": 26, "errors_silenced": 211, "wer_clean": 1.88, "wer_silenced": 15.22, "utts_with_more_deletions": 40}
{"arm": "ab-b16w", "utts": 40, "deletions_clean": 2, "deletions_silenced": 177, "errors_clean": 26, "errors_silenced": 210, "wer_clean": 1.88, "wer_silenced": 15.15, "utts_with_more_deletions": 40}
{"arm": "ab-b32", "utts": 40, "deletions_clean": 2, "deletions_silenced": 178, "errors_clean": 26, "errors_silenced": 210, "wer_clean": 1.88, "wer_silenced": 15.15, "utts_with_more_deletions": 40}
{"arm": "ab-c", "utts": 40, "deletions_clean": 2, "deletions_silenced": 181, "errors_clean": 26, "errors_silenced": 213, "wer_clean": 1.88, "wer_silenced": 15.37, "utts_with_more_deletions": 40}
{"arm": "ab-cf16", "utts": 40, "deletions_clean": 2, "deletions_silenced": 178, "errors_clean": 28, "errors_silenced": 213, "wer_clean": 2.02, "wer_silenced": 15.37, "utts_with_more_deletions": 40}
{"arm": "ab-cf32", "utts": 40, "deletions_clean": 2, "deletions_silenced": 176, "errors_clean": 28, "errors_silenced": 212, "wer_clean": 2.02, "wer_silenced": 15.3, "utts_with_more_deletions": 40}
{"arm": "ab-d", "utts": 40, "deletions_clean": 1, "deletions_silenced": 167, "errors_clean": 26, "errors_silenced": 199, "wer_clean": 1.88, "wer_silenced": 14.36, "utts_with_more_deletions": 40}
## null
{"arm": "ab-a1", "n_utts": 400, "delta_pp": 0.037, "ci95": [-0.078, 0.156], "utts_with_identical_edit_counts": 383}
{"arm": "ab-af32", "n_utts": 400, "delta_pp": -0.025, "ci95": [-0.063, 0.0], "utts_with_identical_edit_counts": 398}
{"arm": "ab-b16", "n_utts": 400, "delta_pp": 0.0, "ci95": [0.0, 0.0], "utts_with_identical_edit_counts": 400}
{"arm": "ab-b16w", "n_utts": 400, "delta_pp": 0.012, "ci95": [0.0, 0.039], "utts_with_identical_edit_counts": 399}
{"arm": "ab-b32", "n_utts": 400, "delta_pp": 0.0, "ci95": [0.0, 0.0], "utts_with_identical_edit_counts": 400}
{"arm": "ab-c", "n_utts": 400, "delta_pp": -0.037, "ci95": [-0.135, 0.057], "utts_with_identical_edit_counts": 389}
{"arm": "ab-cf16", "n_utts": 400, "delta_pp": 0.0, "ci95": [0.0, 0.0], "utts_with_identical_edit_counts": 400}
{"arm": "ab-cf32", "n_utts": 400, "delta_pp": 0.0, "ci95": [0.0, 0.0], "utts_with_identical_edit_counts": 400}
{"arm": "ab-d", "n_utts": 400, "delta_pp": -0.061, "ci95": [-0.149, 0.024], "utts_with_identical_edit_counts": 391}
## same
{"a": "ab-a1", "b": "ab-a2", "utts": 400, "identical_text": 400}
{"a": "ab-a1", "b": "ab-at16", "utts": 400, "identical_text": 400}
{"a": "ab-b32", "b": "ab-b32t", "utts": 400, "identical_text": 400}
## features
{"arm": "ab-a1", "outputs": 400, "with_sentence_punct": 89.0, "with_comma": 73.0, "with_upper": 94.5, "all_lower": 5.2, "empty": 1, "non_ascii": 1}
{"arm": "ab-a2", "outputs": 400, "with_sentence_punct": 89.0, "with_comma": 73.0, "with_upper": 94.5, "all_lower": 5.2, "empty": 1, "non_ascii": 1}
{"arm": "ab-af32", "outputs": 400, "with_sentence_punct": 88.0, "with_comma": 74.5, "with_upper": 96.0, "all_lower": 4.0, "empty": 0, "non_ascii": 1}
{"arm": "ab-at16", "outputs": 400, "with_sentence_punct": 89.0, "with_comma": 73.0, "with_upper": 94.5, "all_lower": 5.2, "empty": 1, "non_ascii": 1}
{"arm": "ab-b16", "outputs": 400, "with_sentence_punct": 50.0, "with_comma": 62.8, "with_upper": 96.8, "all_lower": 3.2, "empty": 0, "non_ascii": 0}
{"arm": "ab-b16w", "outputs": 400, "with_sentence_punct": 50.8, "with_comma": 63.0, "with_upper": 96.8, "all_lower": 3.2, "empty": 0, "non_ascii": 0}
{"arm": "ab-b32", "outputs": 400, "with_sentence_punct": 50.0, "with_comma": 63.0, "with_upper": 96.8, "all_lower": 3.2, "empty": 0, "non_ascii": 0}
{"arm": "ab-b32c", "outputs": 400, "with_sentence_punct": 50.0, "with_comma": 63.0, "with_upper": 96.8, "all_lower": 3.2, "empty": 0, "non_ascii": 0}
{"arm": "ab-b32t", "outputs": 400, "with_sentence_punct": 50.0, "with_comma": 63.0, "with_upper": 96.8, "all_lower": 3.2, "empty": 0, "non_ascii": 0}
{"arm": "ab-c", "outputs": 400, "with_sentence_punct": 51.0, "with_comma": 64.5, "with_upper": 97.0, "all_lower": 3.0, "empty": 0, "non_ascii": 0}
{"arm": "ab-cf16", "outputs": 400, "with_sentence_punct": 48.5, "with_comma": 62.0, "with_upper": 96.0, "all_lower": 4.0, "empty": 0, "non_ascii": 0}
{"arm": "ab-cf32", "outputs": 400, "with_sentence_punct": 48.5, "with_comma": 62.0, "with_upper": 96.0, "all_lower": 4.0, "empty": 0, "non_ascii": 0}
{"arm": "ab-d", "outputs": 400, "with_sentence_punct": 83.8, "with_comma": 73.5, "with_upper": 94.8, "all_lower": 5.2, "empty": 0, "non_ascii": 0}
