vh/esh-pfi-infrastructure
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects Releases Wiki Activity
Files
0d9bb109b892f360b3c1772ded120ce9f22fa595
esh-pfi-infrastructure/services/parakeet-ab-2026-09-30/results/latency-summary-block1.json
T
vh a6c1d3c454 docs(parakeet): seat A/B vs parakeet-unified-en-0.6b - latency is the int8-on-CPU runtime; unified wins WER 
A/B of the live STT seat (fv-ml1 GPU 0, sherpa-onnx int8 v3) against
nvidia/parakeet-unified-en-0.6b, measured on GPU 3 with the seat's own image,
k2-fsa's published unified int8 export, fp32/fp16 exports made with k2-fsa's
recipe, v2 int8, and NeMo 3.0.0 (fp32, bf16 autocast, bf16 weights).

- Seat int8 graph runs on one CPU thread (cpu/wall 1.00, GPU 2-9%).
- unified-en under NeMo: -121/-234/-530 ms vs the seat at 1-3/3-8/8-20 s
  (paired, n=120/bin; floor <=6 ms; +50 ms positive control reads +52-54).
- unified-en WER lower in every runtime: -0.7 pp clean, -1.5 pp other,
  -3.2 to -4.4 pp AMI (paired CIs exclude 0).
- Seat defects found: hard 400 s input ceiling (HTTP 500), truncation after
  a quiet 1.5 s pause, and severe long-window dropouts (int8 v3 only).
- B-bf16w needs +0.8 to +1.5 GB over the seat's 1,690 MiB on GPU 0.

Raw requests, hypotheses, manifests and the full harness under
services/parakeet-ab-2026-09-30/. No deploy; live seat untouched apart
from 240 light test requests.
2026-09-30 18:51:44 -07:00

2765 lines
50 KiB
JSON
Raw Blame History

{
"single": [
{
"arm": "A-live",
"bin": "b1_3",
"n": 40,
"rounds": 2,
"e2e_p50": 186.9345,
"e2e_p50_ci": [
178.221,
197.256
],
"e2e_p90": 221.0589,
"e2e_p99": 230.22012,
"e2e_max": 232.596,
"srv_p50": NaN,
"srv_p90": NaN,
"srv_p99": NaN,
"overhead_p50": NaN
},
{
"arm": "A-live",
"bin": "b3_8",
"n": 40,
"rounds": 2,
"e2e_p50": 307.509,
"e2e_p50_ci": [
274.548,
329.219
],
"e2e_p90": 412.47610000000003,
"e2e_p99": 442.05215,
"e2e_max": 454.031,
"srv_p50": NaN,
"srv_p90": NaN,
"srv_p99": NaN,
"overhead_p50": NaN
},
{
"arm": "A-live",
"bin": "b8_20",
"n": 40,
"rounds": 2,
"e2e_p50": 625.7205,
"e2e_p50_ci": [
546.5305000000001,
678.4715
],
"e2e_p90": 861.7992,
"e2e_p99": 950.46855,
"e2e_max": 955.599,
"srv_p50": NaN,
"srv_p90": NaN,
"srv_p99": NaN,
"overhead_p50": NaN
},
{
"arm": "ab-a1",
"bin": "b1_3",
"n": 120,
"rounds": 6,
"e2e_p50": 144.154,
"e2e_p50_ci": [
141.152,
151.5335
],
"e2e_p90": 170.44,
"e2e_p99": 183.63064,
"e2e_max": 190.46,
"srv_p50": 142.201,
"srv_p90": 168.4833,
"srv_p99": 181.66496,
"overhead_p50": 1.927500000000009
},
{
"arm": "ab-a1",
"bin": "b3_8",
"n": 120,
"rounds": 6,
"e2e_p50": 259.841,
"e2e_p50_ci": [
250.612,
278.977
],
"e2e_p90": 370.1067,
"e2e_p99": 383.94183,
"e2e_max": 395.984,
"srv_p50": 257.788,
"srv_p90": 367.9278,
"srv_p99": 381.81712,
"overhead_p50": 2.003
},
{
"arm": "ab-a1",
"bin": "b8_20",
"n": 120,
"rounds": 6,
"e2e_p50": 564.7085,
"e2e_p50_ci": [
537.357,
599.117
],
"e2e_p90": 814.4145,
"e2e_p99": 896.03701,
"e2e_max": 926.619,
"srv_p50": 562.5115000000001,
"srv_p90": 811.998,
"srv_p99": 893.4515,
"overhead_p50": 2.227999999999952
},
{
"arm": "ab-a1",
"bin": "b20_60",
"n": 120,
"rounds": 6,
"e2e_p50": 2026.8395,
"e2e_p50_ci": [
1773.768,
2130.274
],
"e2e_p90": 2634.379700000001,
"e2e_p99": 2768.342,
"e2e_max": 2918.03,
"srv_p50": 2021.8535,
"srv_p90": 2627.9708000000005,
"srv_p99": 2763.06149,
"overhead_p50": 4.995000000000118
},
{
"arm": "ab-a2",
"bin": "b1_3",
"n": 120,
"rounds": 6,
"e2e_p50": 149.566,
"e2e_p50_ci": [
144.8485,
155.3301
],
"e2e_p90": 176.1361,
"e2e_p99": 206.50726,
"e2e_max": 231.298,
"srv_p50": 147.655,
"srv_p90": 174.0857,
"srv_p99": 204.21379000000002,
"overhead_p50": 1.811000000000007
},
{
"arm": "ab-a2",
"bin": "b3_8",
"n": 120,
"rounds": 6,
"e2e_p50": 265.1355,
"e2e_p50_ci": [
253.531,
283.94849999999997
],
"e2e_p90": 374.2432,
"e2e_p99": 394.31717000000003,
"e2e_max": 428.174,
"srv_p50": 262.9735,
"srv_p90": 372.1184,
"srv_p99": 392.27439,
"overhead_p50": 1.8965000000000174
},
{
"arm": "ab-a2",
"bin": "b8_20",
"n": 120,
"rounds": 6,
"e2e_p50": 567.6005,
"e2e_p50_ci": [
526.207,
604.238
],
"e2e_p90": 814.1804,
"e2e_p99": 901.4791600000001,
"e2e_max": 914.991,
"srv_p50": 565.627,
"srv_p90": 811.9197,
"srv_p99": 898.93475,
"overhead_p50": 2.143499999999989
},
{
"arm": "ab-a2",
"bin": "b20_60",
"n": 120,
"rounds": 6,
"e2e_p50": 2010.0405,
"e2e_p50_ci": [
1788.55,
2146.0622375
],
"e2e_p90": 2717.133,
"e2e_p99": 2826.1192300000002,
"e2e_max": 2888.491,
"srv_p50": 2004.8105,
"srv_p90": 2709.3567000000003,
"srv_p99": 2818.1929,
"overhead_p50": 4.9044999999999845
},
{
"arm": "ab-a50",
"bin": "b1_3",
"n": 120,
"rounds": 6,
"e2e_p50": 197.023,
"e2e_p50_ci": [
193.662,
204.151
],
"e2e_p90": 224.04549999999998,
"e2e_p99": 233.44844,
"e2e_max": 250.708,
"srv_p50": 194.59300000000002,
"srv_p90": 222.2456,
"srv_p99": 231.07265,
"overhead_p50": 1.8440000000000083
},
{
"arm": "ab-a50",
"bin": "b3_8",
"n": 120,
"rounds": 6,
"e2e_p50": 313.2585,
"e2e_p50_ci": [
300.89599999999996,
332.154
],
"e2e_p90": 424.4139,
"e2e_p99": 454.77856,
"e2e_max": 462.794,
"srv_p50": 311.30449999999996,
"srv_p90": 422.3881,
"srv_p99": 452.47714,
"overhead_p50": 1.9620000000000175
},
{
"arm": "ab-a50",
"bin": "b8_20",
"n": 120,
"rounds": 6,
"e2e_p50": 615.278,
"e2e_p50_ci": [
576.0724625,
648.8035
],
"e2e_p90": 861.398,
"e2e_p99": 943.18099,
"e2e_max": 948.791,
"srv_p50": 612.9805,
"srv_p90": 859.0556,
"srv_p99": 940.7565300000001,
"overhead_p50": 2.1954999999999814
},
{
"arm": "ab-a50",
"bin": "b20_60",
"n": 120,
"rounds": 6,
"e2e_p50": 2072.331,
"e2e_p50_ci": [
1835.062,
2252.6615
],
"e2e_p90": 2678.076400000001,
"e2e_p99": 2828.85959,
"e2e_max": 2864.541,
"srv_p50": 2067.0485,
"srv_p90": 2671.894100000001,
"srv_p99": 2822.65957,
"overhead_p50": 4.935500000000047
},
{
"arm": "ab-af32",
"bin": "b1_3",
"n": 120,
"rounds": 6,
"e2e_p50": 38.112,
"e2e_p50_ci": [
36.8429875,
39.1363875
],
"e2e_p90": 41.0281,
"e2e_p99": 43.473530000000004,
"e2e_max": 47.597,
"srv_p50": 36.5375,
"srv_p90": 39.440999999999995,
"srv_p99": 41.77168,
"overhead_p50": 1.5335000000000036
},
{
"arm": "ab-af32",
"bin": "b3_8",
"n": 120,
"rounds": 6,
"e2e_p50": 46.705,
"e2e_p50_ci": [
45.94,
47.681
],
"e2e_p90": 53.1248,
"e2e_p99": 57.111830000000005,
"e2e_max": 77.336,
"srv_p50": 44.9835,
"srv_p90": 51.2949,
"srv_p99": 55.34447,
"overhead_p50": 1.6799999999999997
},
{
"arm": "ab-af32",
"bin": "b8_20",
"n": 120,
"rounds": 6,
"e2e_p50": 66.9905,
"e2e_p50_ci": [
65.3001625,
71.325
],
"e2e_p90": 80.4449,
"e2e_p99": 87.23487,
"e2e_max": 95.301,
"srv_p50": 64.976,
"srv_p90": 78.2058,
"srv_p99": 84.4756,
"overhead_p50": 1.9495000000000005
},
{
"arm": "ab-af32",
"bin": "b20_60",
"n": 120,
"rounds": 6,
"e2e_p50": 166.9715,
"e2e_p50_ci": [
157.172,
175.13
],
"e2e_p90": 219.2497,
"e2e_p99": 231.12354,
"e2e_max": 233.096,
"srv_p50": 162.114,
"srv_p90": 214.1336,
"srv_p99": 225.92634,
"overhead_p50": 4.624499999999998
},
{
"arm": "ab-apl",
"bin": "b1_3",
"n": 120,
"rounds": 6,
"e2e_p50": 149.886,
"e2e_p50_ci": [
146.93425000000002,
157.685
],
"e2e_p90": 175.79729999999998,
"e2e_p99": 189.57841,
"e2e_max": 191.594,
"srv_p50": NaN,
"srv_p90": NaN,
"srv_p99": NaN,
"overhead_p50": NaN
},
{
"arm": "ab-apl",
"bin": "b3_8",
"n": 120,
"rounds": 6,
"e2e_p50": 265.201,
"e2e_p50_ci": [
253.837175,
286.95416249999994
],
"e2e_p90": 378.3254,
"e2e_p99": 406.32518000000005,
"e2e_max": 425.374,
"srv_p50": NaN,
"srv_p90": NaN,
"srv_p99": NaN,
"overhead_p50": NaN
},
{
"arm": "ab-apl",
"bin": "b8_20",
"n": 120,
"rounds": 6,
"e2e_p50": 577.4665,
"e2e_p50_ci": [
547.2420625,
610.3857
],
"e2e_p90": 815.6719,
"e2e_p99": 927.87559,
"e2e_max": 946.408,
"srv_p50": NaN,
"srv_p90": NaN,
"srv_p99": NaN,
"overhead_p50": NaN
},
{
"arm": "ab-apl",
"bin": "b20_60",
"n": 120,
"rounds": 6,
"e2e_p50": 2043.3555000000001,
"e2e_p50_ci": [
1819.7146249999998,
2134.46
],
"e2e_p90": 2742.311,
"e2e_p99": 2837.08723,
"e2e_max": 2917.145,
"srv_p50": NaN,
"srv_p90": NaN,
"srv_p99": NaN,
"overhead_p50": NaN
},
{
"arm": "ab-at16",
"bin": "b1_3",
"n": 120,
"rounds": 6,
"e2e_p50": 84.95349999999999,
"e2e_p50_ci": [
82.67349999999999,
88.567
],
"e2e_p90": 97.63430000000001,
"e2e_p99": 104.36352000000001,
"e2e_max": 105.672,
"srv_p50": 83.07650000000001,
"srv_p90": 95.6849,
"srv_p99": 100.39101,
"overhead_p50": 1.8945000000000007
},
{
"arm": "ab-at16",
"bin": "b3_8",
"n": 120,
"rounds": 6,
"e2e_p50": 142.571,
"e2e_p50_ci": [
135.609,
151.88
],
"e2e_p90": 185.2551,
"e2e_p99": 196.84121000000002,
"e2e_max": 198.395,
"srv_p50": 140.586,
"srv_p90": 183.0474,
"srv_p99": 194.60847,
"overhead_p50": 2.019999999999996
},
{
"arm": "ab-at16",
"bin": "b8_20",
"n": 120,
"rounds": 6,
"e2e_p50": 317.979,
"e2e_p50_ci": [
298.2786375,
332.4065
],
"e2e_p90": 437.7253,
"e2e_p99": 506.69892,
"e2e_max": 516.918,
"srv_p50": 315.749,
"srv_p90": 435.2111,
"srv_p99": 504.27875,
"overhead_p50": 2.242999999999995
},
{
"arm": "ab-at16",
"bin": "b20_60",
"n": 120,
"rounds": 6,
"e2e_p50": 1171.2555,
"e2e_p50_ci": [
1056.098,
1275.5875
],
"e2e_p90": 1545.6782,
"e2e_p99": 1610.03147,
"e2e_max": 1630.236,
"srv_p50": 1167.1295,
"srv_p90": 1540.6798,
"srv_p99": 1603.21842,
"overhead_p50": 4.711500000000001
},
{
"arm": "ab-b16",
"bin": "b1_3",
"n": 120,
"rounds": 6,
"e2e_p50": 28.463,
"e2e_p50_ci": [
28.2125,
28.603
],
"e2e_p90": 31.419,
"e2e_p99": 55.38369000000006,
"e2e_max": 63.621,
"srv_p50": 27.1635,
"srv_p90": 30.1987,
"srv_p99": 52.47626000000005,
"overhead_p50": 1.3045000000000009
},
{
"arm": "ab-b16",
"bin": "b3_8",
"n": 120,
"rounds": 6,
"e2e_p50": 32.4005,
"e2e_p50_ci": [
31.9405,
32.9685
],
"e2e_p90": 35.900800000000004,
"e2e_p99": 38.53325,
"e2e_max": 63.233,
"srv_p50": 30.97,
"srv_p90": 34.3775,
"srv_p99": 37.061040000000006,
"overhead_p50": 1.4450000000000003
},
{
"arm": "ab-b16",
"bin": "b8_20",
"n": 120,
"rounds": 6,
"e2e_p50": 38.8095,
"e2e_p50_ci": [
38.112,
39.274
],
"e2e_p90": 42.991800000000005,
"e2e_p99": 45.68859,
"e2e_max": 75.683,
"srv_p50": 37.069,
"srv_p90": 41.1485,
"srv_p99": 43.888760000000005,
"overhead_p50": 1.6769999999999996
},
{
"arm": "ab-b16",
"bin": "b20_60",
"n": 120,
"rounds": 6,
"e2e_p50": 66.5855,
"e2e_p50_ci": [
64.44415,
72.496
],
"e2e_p90": 82.503,
"e2e_p99": 94.87406000000003,
"e2e_max": 428.378,
"srv_p50": 62.4345,
"srv_p90": 77.921,
"srv_p99": 85.87378000000001,
"overhead_p50": 3.987000000000002
},
{
"arm": "ab-b32",
"bin": "b1_3",
"n": 120,
"rounds": 6,
"e2e_p50": 22.903,
"e2e_p50_ci": [
22.6585,
23.2421375
],
"e2e_p90": 24.8718,
"e2e_p99": 29.37038,
"e2e_max": 60.697,
"srv_p50": 21.544,
"srv_p90": 23.4292,
"srv_p99": 26.745040000000003,
"overhead_p50": 1.2954999999999988
},
{
"arm": "ab-b32",
"bin": "b3_8",
"n": 120,
"rounds": 6,
"e2e_p50": 26.6485,
"e2e_p50_ci": [
25.876,
26.882
],
"e2e_p90": 29.2093,
"e2e_p99": 30.40961,
"e2e_max": 51.308,
"srv_p50": 25.158,
"srv_p90": 27.591,
"srv_p99": 28.95377,
"overhead_p50": 1.4060000000000006
},
{
"arm": "ab-b32",
"bin": "b8_20",
"n": 120,
"rounds": 6,
"e2e_p50": 34.2,
"e2e_p50_ci": [
33.305499999999995,
35.075
],
"e2e_p90": 40.0555,
"e2e_p99": 61.93889000000005,
"e2e_max": 238.128,
"srv_p50": 32.447500000000005,
"srv_p90": 37.927,
"srv_p99": 42.586110000000005,
"overhead_p50": 1.6630000000000003
},
{
"arm": "ab-b32",
"bin": "b20_60",
"n": 120,
"rounds": 6,
"e2e_p50": 76.68549999999999,
"e2e_p50_ci": [
72.2521875,
84.5265
],
"e2e_p90": 97.91590000000001,
"e2e_p99": 257.7183600000004,
"e2e_max": 418.176,
"srv_p50": 72.1815,
"srv_p90": 92.69120000000001,
"srv_p99": 96.43465,
"overhead_p50": 4.279500000000006
},
{
"arm": "ab-c",
"bin": "b1_3",
"n": 120,
"rounds": 6,
"e2e_p50": 155.65699999999998,
"e2e_p50_ci": [
150.791,
161.78535
],
"e2e_p90": 180.2866,
"e2e_p99": 197.16908,
"e2e_max": 220.95,
"srv_p50": 153.7385,
"srv_p90": 178.2343,
"srv_p99": 195.21729,
"overhead_p50": 1.9159999999999968
},
{
"arm": "ab-c",
"bin": "b3_8",
"n": 120,
"rounds": 6,
"e2e_p50": 271.145,
"e2e_p50_ci": [
260.86,
296.165
],
"e2e_p90": 389.119,
"e2e_p99": 413.78441,
"e2e_max": 414.077,
"srv_p50": 269.05550000000005,
"srv_p90": 387.1416,
"srv_p99": 411.55464,
"overhead_p50": 2.0160000000000053
},
{
"arm": "ab-c",
"bin": "b8_20",
"n": 120,
"rounds": 6,
"e2e_p50": 591.341,
"e2e_p50_ci": [
557.2525,
626.8489999999999
],
"e2e_p90": 844.6139999999999,
"e2e_p99": 941.61649,
"e2e_max": 951.567,
"srv_p50": 589.1465000000001,
"srv_p90": 840.8199,
"srv_p99": 939.14156,
"overhead_p50": 2.2830000000000155
},
{
"arm": "ab-c",
"bin": "b20_60",
"n": 120,
"rounds": 6,
"e2e_p50": 2100.9815,
"e2e_p50_ci": [
1846.4465875,
2203.178749999999
],
"e2e_p90": 2795.3831999999998,
"e2e_p99": 2942.59801,
"e2e_max": 2968.875,
"srv_p50": 2096.2395,
"srv_p90": 2787.6408,
"srv_p99": 2934.8455200000003,
"overhead_p50": 5.7690000000000055
},
{
"arm": "ab-cf16",
"bin": "b1_3",
"n": 120,
"rounds": 6,
"e2e_p50": 88.578,
"e2e_p50_ci": [
88.1285,
89.052
],
"e2e_p90": 91.2507,
"e2e_p99": 115.78435000000006,
"e2e_max": 131.481,
"srv_p50": 86.587,
"srv_p90": 89.3857,
"srv_p99": 112.22772000000005,
"overhead_p50": 1.9029999999999987
},
{
"arm": "ab-cf16",
"bin": "b3_8",
"n": 120,
"rounds": 6,
"e2e_p50": 98.88550000000001,
"e2e_p50_ci": [
97.62795,
99.7232625
],
"e2e_p90": 107.0976,
"e2e_p99": 111.17457999999999,
"e2e_max": 126.52,
"srv_p50": 96.8685,
"srv_p90": 105.0089,
"srv_p99": 108.01398,
"overhead_p50": 2.022999999999996
},
{
"arm": "ab-cf16",
"bin": "b8_20",
"n": 120,
"rounds": 6,
"e2e_p50": 123.6275,
"e2e_p50_ci": [
122.52267499999999,
127.86
],
"e2e_p90": 141.2198,
"e2e_p99": 148.1162,
"e2e_max": 150.782,
"srv_p50": 121.4285,
"srv_p90": 138.69889999999998,
"srv_p99": 145.65817,
"overhead_p50": 2.2590000000000003
},
{
"arm": "ab-cf16",
"bin": "b20_60",
"n": 120,
"rounds": 6,
"e2e_p50": 244.0725,
"e2e_p50_ci": [
230.906,
253.70350000000002
],
"e2e_p90": 306.53790000000004,
"e2e_p99": 317.67001,
"e2e_max": 323.224,
"srv_p50": 238.515,
"srv_p90": 300.13259999999997,
"srv_p99": 310.33878000000004,
"overhead_p50": 5.397999999999996
},
{
"arm": "ab-cf32",
"bin": "b1_3",
"n": 120,
"rounds": 6,
"e2e_p50": 40.6635,
"e2e_p50_ci": [
39.560925000000005,
41.466
],
"e2e_p90": 43.2667,
"e2e_p99": 66.53483000000006,
"e2e_max": 73.82,
"srv_p50": 39.1565,
"srv_p90": 41.5168,
"srv_p99": 63.199780000000054,
"overhead_p50": 1.6235
},
{
"arm": "ab-cf32",
"bin": "b3_8",
"n": 120,
"rounds": 6,
"e2e_p50": 50.394499999999994,
"e2e_p50_ci": [
49.553,
51.823
],
"e2e_p90": 58.0653,
"e2e_p99": 62.87102,
"e2e_max": 63.125,
"srv_p50": 48.615,
"srv_p90": 56.236200000000004,
"srv_p99": 60.76035,
"overhead_p50": 1.7860000000000014
},
{
"arm": "ab-cf32",
"bin": "b8_20",
"n": 120,
"rounds": 6,
"e2e_p50": 75.4965,
"e2e_p50_ci": [
73.658,
79.011
],
"e2e_p90": 90.9606,
"e2e_p99": 97.81545,
"e2e_max": 99.687,
"srv_p50": 73.35,
"srv_p90": 88.6361,
"srv_p99": 95.39887,
"overhead_p50": 2.139000000000003
},
{
"arm": "ab-cf32",
"bin": "b20_60",
"n": 120,
"rounds": 6,
"e2e_p50": 186.20600000000002,
"e2e_p50_ci": [
173.726,
203.094
],
"e2e_p90": 247.7615,
"e2e_p99": 260.15941,
"e2e_max": 261.374,
"srv_p50": 180.607,
"srv_p90": 241.5284,
"srv_p99": 253.60642,
"overhead_p50": 4.670500000000004
},
{
"arm": "ab-d",
"bin": "b1_3",
"n": 120,
"rounds": 6,
"e2e_p50": 149.53750000000002,
"e2e_p50_ci": [
145.507,
156.103
],
"e2e_p90": 173.9009,
"e2e_p99": 187.67077,
"e2e_max": 192.486,
"srv_p50": 147.541,
"srv_p90": 171.9481,
"srv_p99": 185.70975,
"overhead_p50": 1.9740000000000038
},
{
"arm": "ab-d",
"bin": "b3_8",
"n": 120,
"rounds": 6,
"e2e_p50": 263.815,
"e2e_p50_ci": [
249.846,
285.065
],
"e2e_p90": 378.622,
"e2e_p99": 389.95542,
"e2e_max": 395.254,
"srv_p50": 261.595,
"srv_p90": 376.2022,
"srv_p99": 387.4682,
"overhead_p50": 2.0649999999999977
},
{
"arm": "ab-d",
"bin": "b8_20",
"n": 120,
"rounds": 6,
"e2e_p50": 564.932,
"e2e_p50_ci": [
533.175,
608.494
],
"e2e_p90": 805.5060000000001,
"e2e_p99": 900.63581,
"e2e_max": 920.288,
"srv_p50": 562.606,
"srv_p90": 803.0586000000001,
"srv_p99": 897.64123,
"overhead_p50": 2.320500000000095
},
{
"arm": "ab-d",
"bin": "b20_60",
"n": 120,
"rounds": 6,
"e2e_p50": 2003.4865,
"e2e_p50_ci": [
1769.0145,
2152.51
],
"e2e_p90": 2718.2608999999998,
"e2e_p99": 2767.16359,
"e2e_max": 2792.983,
"srv_p50": 1997.973,
"srv_p90": 2711.9016,
"srv_p99": 2760.80658,
"overhead_p50": 5.066500000000019
}
],
"paired": [
{
"arm": "A-live",
"ref": "ab-a1",
"bin": "b1_3",
"n_pairs": 40,
"median_diff": 36.583,
"ci95": [
34.261,
37.741000000000014
],
"p50_arm": 186.9345,
"p50_ref": 148.656
},
{
"arm": "A-live",
"ref": "ab-a1",
"bin": "b3_8",
"n_pairs": 40,
"median_diff": 39.56749999999998,
"ci95": [
38.15999999999998,
40.6935
],
"p50_arm": 307.509,
"p50_ref": 259.803
},
{
"arm": "A-live",
"ref": "ab-a1",
"bin": "b8_20",
"n_pairs": 40,
"median_diff": 42.188999999999965,
"ci95": [
36.21406250000007,
47.188499999999976
],
"p50_arm": 625.7205,
"p50_ref": 582.0975
},
{
"arm": "ab-a2",
"ref": "ab-a1",
"bin": "b1_3",
"n_pairs": 120,
"median_diff": 1.0319999999999965,
"ci95": [
0.6559999999999917,
2.0520000000000067
],
"p50_arm": 149.566,
"p50_ref": 144.154
},
{
"arm": "ab-a2",
"ref": "ab-a1",
"bin": "b3_8",
"n_pairs": 120,
"median_diff": 1.4590000000000032,
"ci95": [
0.8919999999999959,
2.7325000000000017
],
"p50_arm": 265.1355,
"p50_ref": 259.841
},
{
"arm": "ab-a2",
"ref": "ab-a1",
"bin": "b8_20",
"n_pairs": 120,
"median_diff": 1.8290000000000077,
"ci95": [
0.4339999999999691,
3.1550000000000296
],
"p50_arm": 567.6005,
"p50_ref": 564.7085
},
{
"arm": "ab-a2",
"ref": "ab-a1",
"bin": "b20_60",
"n_pairs": 120,
"median_diff": 2.198500000000081,
"ci95": [
-3.79200000000003,
7.5060000000000855
],
"p50_arm": 2010.0405,
"p50_ref": 2026.8395
},
{
"arm": "ab-a50",
"ref": "ab-a1",
"bin": "b1_3",
"n_pairs": 120,
"median_diff": 52.61449999999999,
"ci95": [
52.123999999999995,
53.105999999999995
],
"p50_arm": 197.023,
"p50_ref": 144.154
},
{
"arm": "ab-a50",
"ref": "ab-a1",
"bin": "b3_8",
"n_pairs": 120,
"median_diff": 53.23849999999999,
"ci95": [
52.535000000000025,
53.75849999999997
],
"p50_arm": 313.2585,
"p50_ref": 259.841
},
{
"arm": "ab-a50",
"ref": "ab-a1",
"bin": "b8_20",
"n_pairs": 120,
"median_diff": 52.29699999999997,
"ci95": [
50.75699999999995,
53.31899999999996
],
"p50_arm": 615.278,
"p50_ref": 564.7085
},
{
"arm": "ab-a50",
"ref": "ab-a1",
"bin": "b20_60",
"n_pairs": 120,
"median_diff": 53.86500000000012,
"ci95": [
50.621000000000095,
56.652000000000044
],
"p50_arm": 2072.331,
"p50_ref": 2026.8395
},
{
"arm": "ab-af32",
"ref": "ab-a1",
"bin": "b1_3",
"n_pairs": 120,
"median_diff": -106.821,
"ci95": [
-114.16300000000001,
-103.903
],
"p50_arm": 38.112,
"p50_ref": 144.154
},
{
"arm": "ab-af32",
"ref": "ab-a1",
"bin": "b3_8",
"n_pairs": 120,
"median_diff": -212.75900000000001,
"ci95": [
-234.20999999999998,
-203.9845
],
"p50_arm": 46.705,
"p50_ref": 259.841
},
{
"arm": "ab-af32",
"ref": "ab-a1",
"bin": "b8_20",
"n_pairs": 120,
"median_diff": -495.02599999999995,
"ci95": [
-528.886225,
-465.28233750000004
],
"p50_arm": 66.9905,
"p50_ref": 564.7085
},
{
"arm": "ab-af32",
"ref": "ab-a1",
"bin": "b20_60",
"n_pairs": 120,
"median_diff": -1858.6570000000002,
"ci95": [
-1962.2749999999999,
-1616.193
],
"p50_arm": 166.9715,
"p50_ref": 2026.8395
},
{
"arm": "ab-apl",
"ref": "ab-a1",
"bin": "b1_3",
"n_pairs": 120,
"median_diff": 4.434499999999986,
"ci95": [
3.889499999999998,
5.062787500000012
],
"p50_arm": 149.886,
"p50_ref": 144.154
},
{
"arm": "ab-apl",
"ref": "ab-a1",
"bin": "b3_8",
"n_pairs": 120,
"median_diff": 5.771000000000001,
"ci95": [
5.009999999999991,
6.772999999999996
],
"p50_arm": 265.201,
"p50_ref": 259.841
},
{
"arm": "ab-apl",
"ref": "ab-a1",
"bin": "b8_20",
"n_pairs": 120,
"median_diff": 6.570000000000022,
"ci95": [
4.9002750000000095,
10.368000000000052
],
"p50_arm": 577.4665,
"p50_ref": 564.7085
},
{
"arm": "ab-apl",
"ref": "ab-a1",
"bin": "b20_60",
"n_pairs": 120,
"median_diff": 16.79349999999988,
"ci95": [
13.241999999999962,
22.286000000000172
],
"p50_arm": 2043.3555000000001,
"p50_ref": 2026.8395
},
{
"arm": "ab-at16",
"ref": "ab-a1",
"bin": "b1_3",
"n_pairs": 120,
"median_diff": -60.775999999999996,
"ci95": [
-62.879000000000005,
-58.28099999999999
],
"p50_arm": 84.95349999999999,
"p50_ref": 144.154
},
{
"arm": "ab-at16",
"ref": "ab-a1",
"bin": "b3_8",
"n_pairs": 120,
"median_diff": -118.961,
"ci95": [
-128.795,
-112.71702500000006
],
"p50_arm": 142.571,
"p50_ref": 259.841
},
{
"arm": "ab-at16",
"ref": "ab-a1",
"bin": "b8_20",
"n_pairs": 120,
"median_diff": -248.7035,
"ci95": [
-266.34799999999996,
-231.736
],
"p50_arm": 317.979,
"p50_ref": 564.7085
},
{
"arm": "ab-at16",
"ref": "ab-a1",
"bin": "b20_60",
"n_pairs": 120,
"median_diff": -843.9864999999999,
"ci95": [
-909.7669999999998,
-748.9618625000003
],
"p50_arm": 1171.2555,
"p50_ref": 2026.8395
},
{
"arm": "ab-b16",
"ref": "ab-a1",
"bin": "b1_3",
"n_pairs": 120,
"median_diff": -114.77850000000001,
"ci95": [
-121.79950000000001,
-111.55949999999999
],
"p50_arm": 28.463,
"p50_ref": 144.154
},
{
"arm": "ab-b16",
"ref": "ab-a1",
"bin": "b3_8",
"n_pairs": 120,
"median_diff": -228.98700000000002,
"ci95": [
-246.0545,
-218.044
],
"p50_arm": 32.4005,
"p50_ref": 259.841
},
{
"arm": "ab-b16",
"ref": "ab-a1",
"bin": "b8_20",
"n_pairs": 120,
"median_diff": -526.6745000000001,
"ci95": [
-557.47,
-490.79100000000005
],
"p50_arm": 38.8095,
"p50_ref": 564.7085
},
{
"arm": "ab-b16",
"ref": "ab-a1",
"bin": "b20_60",
"n_pairs": 120,
"median_diff": -1959.2355,
"ci95": [
-2071.3,
-1710.1040500000001
],
"p50_arm": 66.5855,
"p50_ref": 2026.8395
},
{
"arm": "ab-b32",
"ref": "ab-a1",
"bin": "b1_3",
"n_pairs": 120,
"median_diff": -121.3715,
"ci95": [
-128.37800000000001,
-117.33950000000002
],
"p50_arm": 22.903,
"p50_ref": 144.154
},
{
"arm": "ab-b32",
"ref": "ab-a1",
"bin": "b3_8",
"n_pairs": 120,
"median_diff": -234.33550000000002,
"ci95": [
-252.8675,
-225.31431250000017
],
"p50_arm": 26.6485,
"p50_ref": 259.841
},
{
"arm": "ab-b32",
"ref": "ab-a1",
"bin": "b8_20",
"n_pairs": 120,
"median_diff": -530.0845,
"ci95": [
-564.6324999999999,
-498.892
],
"p50_arm": 34.2,
"p50_ref": 564.7085
},
{
"arm": "ab-b32",
"ref": "ab-a1",
"bin": "b20_60",
"n_pairs": 120,
"median_diff": -1947.0520000000001,
"ci95": [
-2059.2355,
-1686.012
],
"p50_arm": 76.68549999999999,
"p50_ref": 2026.8395
},
{
"arm": "ab-c",
"ref": "ab-a1",
"bin": "b1_3",
"n_pairs": 120,
"median_diff": 7.693000000000005,
"ci95": [
6.942999999999998,
8.323999999999998
],
"p50_arm": 155.65699999999998,
"p50_ref": 144.154
},
{
"arm": "ab-c",
"ref": "ab-a1",
"bin": "b3_8",
"n_pairs": 120,
"median_diff": 12.222499999999997,
"ci95": [
10.804500000000047,
14.154924999999972
],
"p50_arm": 271.145,
"p50_ref": 259.841
},
{
"arm": "ab-c",
"ref": "ab-a1",
"bin": "b8_20",
"n_pairs": 120,
"median_diff": 21.976500000000016,
"ci95": [
19.214999999999975,
24.27049999999997
],
"p50_arm": 591.341,
"p50_ref": 564.7085
},
{
"arm": "ab-c",
"ref": "ab-a1",
"bin": "b20_60",
"n_pairs": 120,
"median_diff": 70.86750000000006,
"ci95": [
61.45449999999994,
80.74699999999984
],
"p50_arm": 2100.9815,
"p50_ref": 2026.8395
},
{
"arm": "ab-cf16",
"ref": "ab-a1",
"bin": "b1_3",
"n_pairs": 120,
"median_diff": -55.29200000000001,
"ci95": [
-61.2018125,
-51.280000000000015
],
"p50_arm": 88.578,
"p50_ref": 144.154
},
{
"arm": "ab-cf16",
"ref": "ab-a1",
"bin": "b3_8",
"n_pairs": 120,
"median_diff": -160.31400000000002,
"ci95": [
-180.96633750000004,
-152.063
],
"p50_arm": 98.88550000000001,
"p50_ref": 259.841
},
{
"arm": "ab-cf16",
"ref": "ab-a1",
"bin": "b8_20",
"n_pairs": 120,
"median_diff": -438.71799999999996,
"ci95": [
-471.99699999999996,
-408.19649999999996
],
"p50_arm": 123.6275,
"p50_ref": 564.7085
},
{
"arm": "ab-cf16",
"ref": "ab-a1",
"bin": "b20_60",
"n_pairs": 120,
"median_diff": -1779.5700000000002,
"ci95": [
-1878.3415,
-1538.6575000000003
],
"p50_arm": 244.0725,
"p50_ref": 2026.8395
},
{
"arm": "ab-cf32",
"ref": "ab-a1",
"bin": "b1_3",
"n_pairs": 120,
"median_diff": -103.77049999999998,
"ci95": [
-109.28300000000002,
-101.415
],
"p50_arm": 40.6635,
"p50_ref": 144.154
},
{
"arm": "ab-cf32",
"ref": "ab-a1",
"bin": "b3_8",
"n_pairs": 120,
"median_diff": -208.57500000000002,
"ci95": [
-228.445,
-200.839
],
"p50_arm": 50.394499999999994,
"p50_ref": 259.841
},
{
"arm": "ab-cf32",
"ref": "ab-a1",
"bin": "b8_20",
"n_pairs": 120,
"median_diff": -488.121,
"ci95": [
-520.8575,
-460.53149999999994
],
"p50_arm": 75.4965,
"p50_ref": 564.7085
},
{
"arm": "ab-cf32",
"ref": "ab-a1",
"bin": "b20_60",
"n_pairs": 120,
"median_diff": -1838.504,
"ci95": [
-1935.317125,
-1594.4778875
],
"p50_arm": 186.20600000000002,
"p50_ref": 2026.8395
},
{
"arm": "ab-d",
"ref": "ab-a1",
"bin": "b1_3",
"n_pairs": 120,
"median_diff": 2.9339999999999975,
"ci95": [
2.358550000000009,
3.3930000000000007
],
"p50_arm": 149.53750000000002,
"p50_ref": 144.154
},
{
"arm": "ab-d",
"ref": "ab-a1",
"bin": "b3_8",
"n_pairs": 120,
"median_diff": 3.54849999999999,
"ci95": [
2.8689999999999998,
4.222312500000006
],
"p50_arm": 263.815,
"p50_ref": 259.841
},
{
"arm": "ab-d",
"ref": "ab-a1",
"bin": "b8_20",
"n_pairs": 120,
"median_diff": 2.8585000000000207,
"ci95": [
1.95799999999997,
4.734499999999969
],
"p50_arm": 564.932,
"p50_ref": 564.7085
},
{
"arm": "ab-d",
"ref": "ab-a1",
"bin": "b20_60",
"n_pairs": 120,
"median_diff": 4.29099999999994,
"ci95": [
0.3284999999999627,
10.138237499999947
],
"p50_arm": 2003.4865,
"p50_ref": 2026.8395
}
],
"conc": [
{
"arm": "ab-a1",
"bin": "b1_3",
"n": 40,
"errors": 0,
"e2e_p50": 568.5889999999999,
"e2e_p90": 617.4087000000001,
"e2e_p99": 646.62933,
"srv_p50": 144.483,
"req_per_s": 6.86,
"audio_x_realtime": 15.3
},
{
"arm": "ab-a1",
"bin": "b20_60",
"n": 40,
"errors": 0,
"e2e_p50": 8313.420999999998,
"e2e_p90": 12135.6592,
"e2e_p99": 13743.74259,
"srv_p50": 2057.652,
"req_per_s": 0.49,
"audio_x_realtime": 21.2
},
{
"arm": "ab-a1",
"bin": "b3_8",
"n": 40,
"errors": 0,
"e2e_p50": 1062.226,
"e2e_p90": 1406.11,
"e2e_p99": 1586.15528,
"srv_p50": 269.728,
"req_per_s": 3.68,
"audio_x_realtime": 18.3
},
{
"arm": "ab-a1",
"bin": "b8_20",
"n": 40,
"errors": 0,
"e2e_p50": 2303.7520000000004,
"e2e_p90": 2745.1814000000004,
"e2e_p99": 3574.6497499999996,
"srv_p50": 569.577,
"req_per_s": 1.7,
"audio_x_realtime": 20.3
},
{
"arm": "ab-a2",
"bin": "b1_3",
"n": 40,
"errors": 0,
"e2e_p50": 561.0535,
"e2e_p90": 613.9161,
"e2e_p99": 638.66297,
"srv_p50": 141.5555,
"req_per_s": 6.92,
"audio_x_realtime": 15.5
},
{
"arm": "ab-a2",
"bin": "b20_60",
"n": 40,
"errors": 0,
"e2e_p50": 7423.6285,
"e2e_p90": 12493.0409,
"e2e_p99": 15152.00797,
"srv_p50": 2050.5119999999997,
"req_per_s": 0.5,
"audio_x_realtime": 21.3
},
{
"arm": "ab-a2",
"bin": "b3_8",
"n": 40,
"errors": 0,
"e2e_p50": 1072.9099999999999,
"e2e_p90": 1260.2422000000001,
"e2e_p99": 1561.26732,
"srv_p50": 267.7085,
"req_per_s": 3.73,
"audio_x_realtime": 18.6
},
{
"arm": "ab-a2",
"bin": "b8_20",
"n": 40,
"errors": 0,
"e2e_p50": 2294.5365,
"e2e_p90": 2734.4753000000005,
"e2e_p99": 3555.9726199999996,
"srv_p50": 566.7735,
"req_per_s": 1.7,
"audio_x_realtime": 20.4
},
{
"arm": "ab-a50",
"bin": "b1_3",
"n": 40,
"errors": 0,
"e2e_p50": 773.693,
"e2e_p90": 825.6566,
"e2e_p99": 857.49358,
"srv_p50": 194.21550000000002,
"req_per_s": 5.07,
"audio_x_realtime": 11.3
},
{
"arm": "ab-a50",
"bin": "b20_60",
"n": 40,
"errors": 0,
"e2e_p50": 8686.378,
"e2e_p90": 12353.0173,
"e2e_p99": 16021.343239999997,
"srv_p50": 2135.733,
"req_per_s": 0.48,
"audio_x_realtime": 20.5
},
{
"arm": "ab-a50",
"bin": "b3_8",
"n": 40,
"errors": 0,
"e2e_p50": 1267.4155,
"e2e_p90": 1669.3302,
"e2e_p99": 1867.07691,
"srv_p50": 320.0155,
"req_per_s": 3.1,
"audio_x_realtime": 15.4
},
{
"arm": "ab-a50",
"bin": "b8_20",
"n": 40,
"errors": 0,
"e2e_p50": 2531.751,
"e2e_p90": 3318.7967,
"e2e_p99": 3462.20835,
"srv_p50": 620.4085,
"req_per_s": 1.55,
"audio_x_realtime": 18.6
},
{
"arm": "ab-af32",
"bin": "b1_3",
"n": 40,
"errors": 0,
"e2e_p50": 137.56400000000002,
"e2e_p90": 152.6648,
"e2e_p99": 158.35925,
"srv_p50": 36.1515,
"req_per_s": 28.29,
"audio_x_realtime": 63.3
},
{
"arm": "ab-af32",
"bin": "b20_60",
"n": 40,
"errors": 0,
"e2e_p50": 661.702,
"e2e_p90": 1119.5386,
"e2e_p99": 1284.22707,
"srv_p50": 160.77800000000002,
"req_per_s": 5.91,
"audio_x_realtime": 253.7
},
{
"arm": "ab-af32",
"bin": "b3_8",
"n": 40,
"errors": 0,
"e2e_p50": 181.28199999999998,
"e2e_p90": 220.03090000000003,
"e2e_p99": 263.65106999999995,
"srv_p50": 44.197,
"req_per_s": 21.89,
"audio_x_realtime": 109.0
},
{
"arm": "ab-af32",
"bin": "b8_20",
"n": 40,
"errors": 0,
"e2e_p50": 264.53049999999996,
"e2e_p90": 288.98089999999996,
"e2e_p99": 348.8487,
"srv_p50": 63.983000000000004,
"req_per_s": 14.95,
"audio_x_realtime": 179.2
},
{
"arm": "ab-apl",
"bin": "b1_3",
"n": 40,
"errors": 0,
"e2e_p50": 603.4245,
"e2e_p90": 658.3872,
"e2e_p99": 702.37157,
"srv_p50": NaN,
"req_per_s": 6.47,
"audio_x_realtime": 14.5
},
{
"arm": "ab-apl",
"bin": "b20_60",
"n": 40,
"errors": 0,
"e2e_p50": 8303.3565,
"e2e_p90": 13690.4516,
"e2e_p99": 15568.66462,
"srv_p50": NaN,
"req_per_s": 0.49,
"audio_x_realtime": 20.8
},
{
"arm": "ab-apl",
"bin": "b3_8",
"n": 40,
"errors": 0,
"e2e_p50": 1090.2849999999999,
"e2e_p90": 1484.0900000000001,
"e2e_p99": 1689.45278,
"srv_p50": NaN,
"req_per_s": 3.59,
"audio_x_realtime": 17.9
},
{
"arm": "ab-apl",
"bin": "b8_20",
"n": 40,
"errors": 0,
"e2e_p50": 2348.5485,
"e2e_p90": 3265.2349999999997,
"e2e_p99": 3707.8709099999996,
"srv_p50": NaN,
"req_per_s": 1.64,
"audio_x_realtime": 19.6
},
{
"arm": "ab-at16",
"bin": "b1_3",
"n": 40,
"errors": 0,
"e2e_p50": 335.096,
"e2e_p90": 359.0988,
"e2e_p99": 378.50183000000004,
"srv_p50": 83.71549999999999,
"req_per_s": 11.72,
"audio_x_realtime": 26.2
},
{
"arm": "ab-at16",
"bin": "b20_60",
"n": 40,
"errors": 0,
"e2e_p50": 4646.514499999999,
"e2e_p90": 7005.34,
"e2e_p99": 7903.2298599999995,
"srv_p50": 1165.404,
"req_per_s": 0.86,
"audio_x_realtime": 36.9
},
{
"arm": "ab-at16",
"bin": "b3_8",
"n": 40,
"errors": 0,
"e2e_p50": 571.6975,
"e2e_p90": 653.7156,
"e2e_p99": 830.8833,
"srv_p50": 144.80149999999998,
"req_per_s": 6.99,
"audio_x_realtime": 34.8
},
{
"arm": "ab-at16",
"bin": "b8_20",
"n": 40,
"errors": 0,
"e2e_p50": 1299.2824999999998,
"e2e_p90": 1530.1692,
"e2e_p99": 1788.58545,
"srv_p50": 326.1535,
"req_per_s": 3.06,
"audio_x_realtime": 36.6
},
{
"arm": "ab-b16",
"bin": "b1_3",
"n": 40,
"errors": 0,
"e2e_p50": 105.89,
"e2e_p90": 107.58160000000001,
"e2e_p99": 132.46159999999998,
"srv_p50": 25.912999999999997,
"req_per_s": 36.36,
"audio_x_realtime": 81.3
},
{
"arm": "ab-b16",
"bin": "b20_60",
"n": 40,
"errors": 0,
"e2e_p50": 253.256,
"e2e_p90": 383.2502,
"e2e_p99": 437.22360999999995,
"srv_p50": 60.9865,
"req_per_s": 15.28,
"audio_x_realtime": 655.9
},
{
"arm": "ab-b16",
"bin": "b3_8",
"n": 40,
"errors": 0,
"e2e_p50": 122.44,
"e2e_p90": 144.1701,
"e2e_p99": 173.87584999999999,
"srv_p50": 29.8345,
"req_per_s": 32.73,
"audio_x_realtime": 162.9
},
{
"arm": "ab-b16",
"bin": "b8_20",
"n": 40,
"errors": 0,
"e2e_p50": 146.526,
"e2e_p90": 152.33870000000002,
"e2e_p99": 189.15855,
"srv_p50": 35.5875,
"req_per_s": 27.17,
"audio_x_realtime": 325.8
},
{
"arm": "ab-b32",
"bin": "b1_3",
"n": 40,
"errors": 0,
"e2e_p50": 84.5545,
"e2e_p90": 86.4117,
"e2e_p99": 113.42971999999999,
"srv_p50": 20.674,
"req_per_s": 45.1,
"audio_x_realtime": 100.9
},
{
"arm": "ab-b32",
"bin": "b20_60",
"n": 40,
"errors": 0,
"e2e_p50": 308.135,
"e2e_p90": 455.70840000000004,
"e2e_p99": 538.73872,
"srv_p50": 71.97149999999999,
"req_per_s": 13.06,
"audio_x_realtime": 560.6
},
{
"arm": "ab-b32",
"bin": "b3_8",
"n": 40,
"errors": 0,
"e2e_p50": 98.363,
"e2e_p90": 119.9475,
"e2e_p99": 140.33652999999998,
"srv_p50": 24.2605,
"req_per_s": 40.28,
"audio_x_realtime": 200.5
},
{
"arm": "ab-b32",
"bin": "b8_20",
"n": 40,
"errors": 0,
"e2e_p50": 128.82,
"e2e_p90": 138.8294,
"e2e_p99": 167.21312,
"srv_p50": 31.479,
"req_per_s": 30.37,
"audio_x_realtime": 364.1
},
{
"arm": "ab-c",
"bin": "b1_3",
"n": 40,
"errors": 0,
"e2e_p50": 636.244,
"e2e_p90": 718.9582,
"e2e_p99": 720.15386,
"srv_p50": 160.411,
"req_per_s": 6.18,
"audio_x_realtime": 13.8
},
{
"arm": "ab-c",
"bin": "b20_60",
"n": 40,
"errors": 0,
"e2e_p50": 8762.8095,
"e2e_p90": 12716.7069,
"e2e_p99": 14721.249979999999,
"srv_p50": 2141.1144999999997,
"req_per_s": 0.47,
"audio_x_realtime": 20.4
},
{
"arm": "ab-c",
"bin": "b3_8",
"n": 40,
"errors": 0,
"e2e_p50": 1136.806,
"e2e_p90": 1510.2582,
"e2e_p99": 1753.80822,
"srv_p50": 284.6175,
"req_per_s": 3.45,
"audio_x_realtime": 17.2
},
{
"arm": "ab-c",
"bin": "b8_20",
"n": 40,
"errors": 0,
"e2e_p50": 2414.6245,
"e2e_p90": 2882.676200000001,
"e2e_p99": 3714.4756599999996,
"srv_p50": 602.0944999999999,
"req_per_s": 1.62,
"audio_x_realtime": 19.4
},
{
"arm": "ab-cf16",
"bin": "b1_3",
"n": 40,
"errors": 0,
"e2e_p50": 348.048,
"e2e_p90": 355.304,
"e2e_p99": 379.03837,
"srv_p50": 86.519,
"req_per_s": 11.53,
"audio_x_realtime": 25.8
},
{
"arm": "ab-cf16",
"bin": "b20_60",
"n": 40,
"errors": 0,
"e2e_p50": 1021.231,
"e2e_p90": 1560.8085,
"e2e_p99": 1847.89599,
"srv_p50": 239.78699999999998,
"req_per_s": 4.0,
"audio_x_realtime": 171.8
},
{
"arm": "ab-cf16",
"bin": "b3_8",
"n": 40,
"errors": 0,
"e2e_p50": 390.653,
"e2e_p90": 499.07680000000005,
"e2e_p99": 558.46701,
"srv_p50": 98.122,
"req_per_s": 10.15,
"audio_x_realtime": 50.5
},
{
"arm": "ab-cf16",
"bin": "b8_20",
"n": 40,
"errors": 0,
"e2e_p50": 490.23,
"e2e_p90": 522.181,
"e2e_p99": 645.0670299999999,
"srv_p50": 121.057,
"req_per_s": 8.06,
"audio_x_realtime": 96.6
},
{
"arm": "ab-cf32",
"bin": "b1_3",
"n": 40,
"errors": 0,
"e2e_p50": 148.44850000000002,
"e2e_p90": 163.0172,
"e2e_p99": 165.83822,
"srv_p50": 38.917,
"req_per_s": 26.44,
"audio_x_realtime": 59.1
},
{
"arm": "ab-cf32",
"bin": "b20_60",
"n": 40,
"errors": 0,
"e2e_p50": 785.539,
"e2e_p90": 1399.4742,
"e2e_p99": 1551.33486,
"srv_p50": 186.506,
"req_per_s": 5.08,
"audio_x_realtime": 218.2
},
{
"arm": "ab-cf32",
"bin": "b3_8",
"n": 40,
"errors": 0,
"e2e_p50": 199.423,
"e2e_p90": 226.1265,
"e2e_p99": 285.77407999999997,
"srv_p50": 50.17,
"req_per_s": 19.69,
"audio_x_realtime": 98.0
},
{
"arm": "ab-cf32",
"bin": "b8_20",
"n": 40,
"errors": 0,
"e2e_p50": 294.647,
"e2e_p90": 328.5892,
"e2e_p99": 394.29058,
"srv_p50": 72.07300000000001,
"req_per_s": 13.3,
"audio_x_realtime": 159.4
},
{
"arm": "ab-d",
"bin": "b1_3",
"n": 40,
"errors": 0,
"e2e_p50": 610.2655,
"e2e_p90": 658.2273,
"e2e_p99": 688.3057,
"srv_p50": 151.4425,
"req_per_s": 6.49,
"audio_x_realtime": 14.5
},
{
"arm": "ab-d",
"bin": "b20_60",
"n": 40,
"errors": 0,
"e2e_p50": 8643.511999999999,
"e2e_p90": 12435.248,
"e2e_p99": 14957.066749999998,
"srv_p50": 2062.8815,
"req_per_s": 0.49,
"audio_x_realtime": 21.0
},
{
"arm": "ab-d",
"bin": "b3_8",
"n": 40,
"errors": 0,
"e2e_p50": 1104.4279999999999,
"e2e_p90": 1462.3495000000003,
"e2e_p99": 1633.0075299999999,
"srv_p50": 274.822,
"req_per_s": 3.54,
"audio_x_realtime": 17.6
},
{
"arm": "ab-d",
"bin": "b8_20",
"n": 40,
"errors": 0,
"e2e_p50": 2350.3140000000003,
"e2e_p90": 2855.8324000000002,
"e2e_p99": 3505.13398,
"srv_p50": 577.109,
"req_per_s": 1.65,
"audio_x_realtime": 19.8
}
],
"first": [
{
"arm": "ab-a1",
"id": "b1_3_10",
"bin": "b1_3",
"dur": 2.555,
"t_wall": 1790811572.8162718,
"e2e_ms": 223.357,
"server_ms": 211.723,
"status": 200,
"err": null
},
{
"arm": "ab-a1",
"id": "b3_8_10",
"bin": "b3_8",
"dur": 5.05,
"t_wall": 1790811573.1746786,
"e2e_ms": 357.146,
"server_ms": 354.976,
"status": 200,
"err": null
},
{
"arm": "ab-a1",
"id": "b8_20_10",
"bin": "b8_20",
"dur": 11.64,
"t_wall": 1790811573.825034,
"e2e_ms": 648.487,
"server_ms": 645.411,
"status": 200,
"err": null
},
{
"arm": "ab-a1",
"id": "b20_60_10",
"bin": "b20_60",
"dur": 54.9,
"t_wall": 1790811576.7342896,
"e2e_ms": 2903.51,
"server_ms": 2893.909,
"status": 200,
"err": null
},
{
"arm": "ab-a2",
"id": "b1_3_10",
"bin": "b1_3",
"dur": 2.555,
"t_wall": 1790811627.8702877,
"e2e_ms": 216.276,
"server_ms": 202.644,
"status": 200,
"err": null
},
{
"arm": "ab-a2",
"id": "b3_8_10",
"bin": "b3_8",
"dur": 5.05,
"t_wall": 1790811628.164761,
"e2e_ms": 293.576,
"server_ms": 290.873,
"status": 200,
"err": null
},
{
"arm": "ab-a2",
"id": "b8_20_10",
"bin": "b8_20",
"dur": 11.64,
"t_wall": 1790811628.859162,
"e2e_ms": 693.055,
"server_ms": 689.739,
"status": 200,
"err": null
},
{
"arm": "ab-a2",
"id": "b20_60_10",
"bin": "b20_60",
"dur": 54.9,
"t_wall": 1790811631.7620885,
"e2e_ms": 2897.135,
"server_ms": 2887.724,
"status": 200,
"err": null
},
{
"arm": "ab-a50",
"id": "b1_3_10",
"bin": "b1_3",
"dur": 2.555,
"t_wall": 1790811689.895353,
"e2e_ms": 260.979,
"server_ms": 251.681,
"status": 200,
"err": null
},
{
"arm": "ab-a50",
"id": "b3_8_10",
"bin": "b3_8",
"dur": 5.05,
"t_wall": 1790811690.2542,
"e2e_ms": 358.01,
"server_ms": 355.108,
"status": 200,
"err": null
},
{
"arm": "ab-a50",
"id": "b8_20_10",
"bin": "b8_20",
"dur": 11.64,
"t_wall": 1790811690.9825685,
"e2e_ms": 726.984,
"server_ms": 724.065,
"status": 200,
"err": null
},
{
"arm": "ab-a50",
"id": "b20_60_10",
"bin": "b20_60",
"dur": 54.9,
"t_wall": 1790811693.9041705,
"e2e_ms": 2915.796,
"server_ms": 2905.895,
"status": 200,
"err": null
},
{
"arm": "ab-apl",
"id": "b1_3_10",
"bin": "b1_3",
"dur": 2.555,
"t_wall": 1790811751.7075408,
"e2e_ms": 205.271,
"server_ms": null,
"status": 200,
"err": null
},
{
"arm": "ab-apl",
"id": "b3_8_10",
"bin": "b3_8",
"dur": 5.05,
"t_wall": 1790811751.9997044,
"e2e_ms": 291.27,
"server_ms": null,
"status": 200,
"err": null
},
{
"arm": "ab-apl",
"id": "b8_20_10",
"bin": "b8_20",
"dur": 11.64,
"t_wall": 1790811752.6717346,
"e2e_ms": 670.646,
"server_ms": null,
"status": 200,
"err": null
},
{
"arm": "ab-apl",
"id": "b20_60_10",
"bin": "b20_60",
"dur": 54.9,
"t_wall": 1790811755.6003006,
"e2e_ms": 2922.753,
"server_ms": null,
"status": 200,
"err": null
},
{
"arm": "ab-at16",
"id": "b1_3_10",
"bin": "b1_3",
"dur": 2.555,
"t_wall": 1790811812.8083022,
"e2e_ms": 129.012,
"server_ms": 124.959,
"status": 200,
"err": null
},
{
"arm": "ab-at16",
"id": "b3_8_10",
"bin": "b3_8",
"dur": 5.05,
"t_wall": 1790811812.9687788,
"e2e_ms": 159.71,
"server_ms": 157.74,
"status": 200,
"err": null
},
{
"arm": "ab-at16",
"id": "b8_20_10",
"bin": "b8_20",
"dur": 11.64,
"t_wall": 1790811813.2948213,
"e2e_ms": 324.896,
"server_ms": 321.865,
"status": 200,
"err": null
},
{
"arm": "ab-at16",
"id": "b20_60_10",
"bin": "b20_60",
"dur": 54.9,
"t_wall": 1790811814.8389106,
"e2e_ms": 1540.444,
"server_ms": 1531.975,
"status": 200,
"err": null
},
{
"arm": "ab-c",
"id": "b1_3_10",
"bin": "b1_3",
"dur": 2.555,
"t_wall": 1790811866.5098884,
"e2e_ms": 223.541,
"server_ms": 210.645,
"status": 200,
"err": null
},
{
"arm": "ab-c",
"id": "b3_8_10",
"bin": "b3_8",
"dur": 5.05,
"t_wall": 1790811866.8653347,
"e2e_ms": 354.618,
"server_ms": 351.397,
"status": 200,
"err": null
},
{
"arm": "ab-c",
"id": "b8_20_10",
"bin": "b8_20",
"dur": 11.64,
"t_wall": 1790811867.5352015,
"e2e_ms": 668.441,
"server_ms": 665.287,
"status": 200,
"err": null
},
{
"arm": "ab-c",
"id": "b20_60_10",
"bin": "b20_60",
"dur": 54.9,
"t_wall": 1790811870.5005066,
"e2e_ms": 2959.405,
"server_ms": 2949.41,
"status": 200,
"err": null
},
{
"arm": "ab-d",
"id": "b1_3_10",
"bin": "b1_3",
"dur": 2.555,
"t_wall": 1790811921.618863,
"e2e_ms": 217.189,
"server_ms": 212.387,
"status": 200,
"err": null
},
{
"arm": "ab-d",
"id": "b3_8_10",
"bin": "b3_8",
"dur": 5.05,
"t_wall": 1790811921.905966,
"e2e_ms": 286.202,
"server_ms": 284.139,
"status": 200,
"err": null
},
{
"arm": "ab-d",
"id": "b8_20_10",
"bin": "b8_20",
"dur": 11.64,
"t_wall": 1790811922.5258143,
"e2e_ms": 618.419,
"server_ms": 616.048,
"status": 200,
"err": null
},
{
"arm": "ab-d",
"id": "b20_60_10",
"bin": "b20_60",
"dur": 54.9,
"t_wall": 1790811925.2671535,
"e2e_ms": 2735.659,
"server_ms": 2727.488,
"status": 200,
"err": null
},
{
"arm": "ab-cf32",
"id": "b1_3_10",
"bin": "b1_3",
"dur": 2.555,
"t_wall": 1790811973.740321,
"e2e_ms": 82.453,
"server_ms": 71.038,
"status": 200,
"err": null
},
{
"arm": "ab-cf32",
"id": "b3_8_10",
"bin": "b3_8",
"dur": 5.05,
"t_wall": 1790811973.792273,
"e2e_ms": 51.008,
"server_ms": 49.114,
"status": 200,
"err": null
},
{
"arm": "ab-cf32",
"id": "b8_20_10",
"bin": "b8_20",
"dur": 11.64,
"t_wall": 1790811973.8732834,
"e2e_ms": 79.593,
"server_ms": 77.421,
"status": 200,
"err": null
},
{
"arm": "ab-cf32",
"id": "b20_60_10",
"bin": "b20_60",
"dur": 54.9,
"t_wall": 1790811974.133343,
"e2e_ms": 254.278,
"server_ms": 245.859,
"status": 200,
"err": null
},
{
"arm": "ab-cf16",
"id": "b1_3_10",
"bin": "b1_3",
"dur": 2.555,
"t_wall": 1790812025.6972337,
"e2e_ms": 123.521,
"server_ms": 119.342,
"status": 200,
"err": null
},
{
"arm": "ab-cf16",
"id": "b3_8_10",
"bin": "b3_8",
"dur": 5.05,
"t_wall": 1790812025.7957647,
"e2e_ms": 97.643,
"server_ms": 95.538,
"status": 200,
"err": null
},
{
"arm": "ab-cf16",
"id": "b8_20_10",
"bin": "b8_20",
"dur": 11.64,
"t_wall": 1790812025.9317398,
"e2e_ms": 134.637,
"server_ms": 132.425,
"status": 200,
"err": null
},
{
"arm": "ab-cf16",
"id": "b20_60_10",
"bin": "b20_60",
"dur": 54.9,
"t_wall": 1790812026.2421963,
"e2e_ms": 304.539,
"server_ms": 298.103,
"status": 200,
"err": null
},
{
"arm": "ab-af32",
"id": "b1_3_10",
"bin": "b1_3",
"dur": 2.555,
"t_wall": 1790812081.0276036,
"e2e_ms": 77.379,
"server_ms": 73.332,
"status": 200,
"err": null
},
{
"arm": "ab-af32",
"id": "b3_8_10",
"bin": "b3_8",
"dur": 5.05,
"t_wall": 1790812081.0759206,
"e2e_ms": 47.811,
"server_ms": 45.966,
"status": 200,
"err": null
},
{
"arm": "ab-af32",
"id": "b8_20_10",
"bin": "b8_20",
"dur": 11.64,
"t_wall": 1790812081.144304,
"e2e_ms": 67.599,
"server_ms": 65.54,
"status": 200,
"err": null
},
{
"arm": "ab-af32",
"id": "b20_60_10",
"bin": "b20_60",
"dur": 54.9,
"t_wall": 1790812081.368576,
"e2e_ms": 218.673,
"server_ms": 211.82,
"status": 200,
"err": null
},
{
"arm": "ab-b32",
"id": "b1_3_10",
"bin": "b1_3",
"dur": 2.555,
"t_wall": 1790812099.6981435,
"e2e_ms": 58.892,
"server_ms": 55.234,
"status": 200,
"err": null
},
{
"arm": "ab-b32",
"id": "b3_8_10",
"bin": "b3_8",
"dur": 5.05,
"t_wall": 1790812099.742739,
"e2e_ms": 43.753,
"server_ms": 41.778,
"status": 200,
"err": null
},
{
"arm": "ab-b32",
"id": "b8_20_10",
"bin": "b8_20",
"dur": 11.64,
"t_wall": 1790812099.7807178,
"e2e_ms": 36.787,
"server_ms": 34.631,
"status": 200,
"err": null
},
{
"arm": "ab-b32",
"id": "b20_60_10",
"bin": "b20_60",
"dur": 54.9,
"t_wall": 1790812100.1957908,
"e2e_ms": 409.785,
"server_ms": 404.602,
"status": 200,
"err": null
},
{
"arm": "ab-b16",
"id": "b1_3_10",
"bin": "b1_3",
"dur": 2.555,
"t_wall": 1790812125.038448,
"e2e_ms": 66.532,
"server_ms": 63.429,
"status": 200,
"err": null
},
{
"arm": "ab-b16",
"id": "b3_8_10",
"bin": "b3_8",
"dur": 5.05,
"t_wall": 1790812125.0743783,
"e2e_ms": 35.432,
"server_ms": 34.011,
"status": 200,
"err": null
},
{
"arm": "ab-b16",
"id": "b8_20_10",
"bin": "b8_20",
"dur": 11.64,
"t_wall": 1790812125.1213822,
"e2e_ms": 46.311,
"server_ms": 44.775,
"status": 200,
"err": null
},
{
"arm": "ab-b16",
"id": "b20_60_10",
"bin": "b20_60",
"dur": 54.9,
"t_wall": 1790812125.496252,
"e2e_ms": 372.25,
"server_ms": 366.245,
"status": 200,
"err": null
}
],
"memory": [
{
"arm": "ab-a1",
"pid": 3208587,
"cold_s": 48.2,
"rest_after_warmup_mib": 932,
"peak_utterance_block_mib": 5798,
"peak_whole_life_mib": 38566,
"last_mib": 38566,
"samples": 30058
},
{
"arm": "ab-a2",
"pid": 3211852,
"cold_s": 47.8,
"rest_after_warmup_mib": 922,
"peak_utterance_block_mib": 4764,
"peak_whole_life_mib": 4764,
"last_mib": 4764,
"samples": 29036
},
{
"arm": "ab-a50",
"pid": 3215242,
"cold_s": 47.5,
"rest_after_warmup_mib": 922,
"peak_utterance_block_mib": 4764,
"peak_whole_life_mib": 4764,
"last_mib": 4764,
"samples": 22302
},
{
"arm": "ab-apl",
"pid": 3218510,
"cold_s": 48.0,
"rest_after_warmup_mib": 930,
"peak_utterance_block_mib": 4772,
"peak_whole_life_mib": 4772,
"last_mib": 4772,
"samples": 22003
},
{
"arm": "ab-at16",
"pid": 3221843,
"cold_s": 47.7,
"rest_after_warmup_mib": 1698,
"peak_utterance_block_mib": 4772,
"peak_whole_life_mib": 4772,
"last_mib": 4772,
"samples": 28137
},
{
"arm": "ab-c",
"pid": 3225020,
"cold_s": 48.3,
"rest_after_warmup_mib": 916,
"peak_utterance_block_mib": 4758,
"peak_whole_life_mib": 10902,
"last_mib": 10902,
"samples": 32579
},
{
"arm": "ab-d",
"pid": 3228280,
"cold_s": 47.8,
"rest_after_warmup_mib": 916,
"peak_utterance_block_mib": 4758,
"peak_whole_life_mib": 8854,
"last_mib": 8854,
"samples": 32309
},
{
"arm": "ab-cf32",
"pid": 3231568,
"cold_s": 45.3,
"rest_after_warmup_mib": 3874,
"peak_utterance_block_mib": 6946,
"peak_whole_life_mib": 6946,
"last_mib": 6946,
"samples": 20979
},
{
"arm": "ab-cf16",
"pid": 3234656,
"cold_s": 45.6,
"rest_after_warmup_mib": 2776,
"peak_utterance_block_mib": 4824,
"peak_whole_life_mib": 4824,
"last_mib": 4824,
"samples": 24866
},
{
"arm": "ab-af32",
"pid": 3237908,
"cold_s": 44.6,
"rest_after_warmup_mib": 3932,
"peak_utterance_block_mib": 7004,
"peak_whole_life_mib": 7004,
"last_mib": 7004,
"samples": 20764
},
{
"arm": "ab-b32",
"pid": 3240981,
"cold_s": 12.1,
"rest_after_warmup_mib": 3836,
"peak_utterance_block_mib": 4800,
"peak_whole_life_mib": 5620,
"last_mib": 4800,
"samples": 19712
},
{
"arm": "ab-b16",
"pid": 3242482,
"cold_s": 11.8,
"rest_after_warmup_mib": 4802,
"peak_utterance_block_mib": 5126,
"peak_whole_life_mib": 5620,
"last_mib": 5126,
"samples": 19923
}
]
}
Reference in New Issue View Git Blame Copy Permalink