vh
a6c1d3c454
A/B of the live STT seat (fv-ml1 GPU 0, sherpa-onnx int8 v3) against nvidia/parakeet-unified-en-0.6b, measured on GPU 3 with the seat's own image, k2-fsa's published unified int8 export, fp32/fp16 exports made with k2-fsa's recipe, v2 int8, and NeMo 3.0.0 (fp32, bf16 autocast, bf16 weights). - Seat int8 graph runs on one CPU thread (cpu/wall 1.00, GPU 2-9%). - unified-en under NeMo: -121/-234/-530 ms vs the seat at 1-3/3-8/8-20 s (paired, n=120/bin; floor <=6 ms; +50 ms positive control reads +52-54). - unified-en WER lower in every runtime: -0.7 pp clean, -1.5 pp other, -3.2 to -4.4 pp AMI (paired CIs exclude 0). - Seat defects found: hard 400 s input ceiling (HTTP 500), truncation after a quiet 1.5 s pause, and severe long-window dropouts (int8 v3 only). - B-bf16w needs +0.8 to +1.5 GB over the seat's 1,690 MiB on GPU 0. Raw requests, hypotheses, manifests and the full harness under services/parakeet-ab-2026-09-30/. No deploy; live seat untouched apart from 240 light test requests.
49 lines
7.6 KiB
JSON
49 lines
7.6 KiB
JSON
{"arm": "ab-a1", "id": "b1_3_10", "bin": "b1_3", "dur": 2.555, "t_wall": 1790811572.8162718, "e2e_ms": 223.357, "server_ms": 211.723, "status": 200, "err": null}
|
|
{"arm": "ab-a1", "id": "b3_8_10", "bin": "b3_8", "dur": 5.05, "t_wall": 1790811573.1746786, "e2e_ms": 357.146, "server_ms": 354.976, "status": 200, "err": null}
|
|
{"arm": "ab-a1", "id": "b8_20_10", "bin": "b8_20", "dur": 11.64, "t_wall": 1790811573.825034, "e2e_ms": 648.487, "server_ms": 645.411, "status": 200, "err": null}
|
|
{"arm": "ab-a1", "id": "b20_60_10", "bin": "b20_60", "dur": 54.9, "t_wall": 1790811576.7342896, "e2e_ms": 2903.51, "server_ms": 2893.909, "status": 200, "err": null}
|
|
{"arm": "ab-a2", "id": "b1_3_10", "bin": "b1_3", "dur": 2.555, "t_wall": 1790811627.8702877, "e2e_ms": 216.276, "server_ms": 202.644, "status": 200, "err": null}
|
|
{"arm": "ab-a2", "id": "b3_8_10", "bin": "b3_8", "dur": 5.05, "t_wall": 1790811628.164761, "e2e_ms": 293.576, "server_ms": 290.873, "status": 200, "err": null}
|
|
{"arm": "ab-a2", "id": "b8_20_10", "bin": "b8_20", "dur": 11.64, "t_wall": 1790811628.859162, "e2e_ms": 693.055, "server_ms": 689.739, "status": 200, "err": null}
|
|
{"arm": "ab-a2", "id": "b20_60_10", "bin": "b20_60", "dur": 54.9, "t_wall": 1790811631.7620885, "e2e_ms": 2897.135, "server_ms": 2887.724, "status": 200, "err": null}
|
|
{"arm": "ab-a50", "id": "b1_3_10", "bin": "b1_3", "dur": 2.555, "t_wall": 1790811689.895353, "e2e_ms": 260.979, "server_ms": 251.681, "status": 200, "err": null}
|
|
{"arm": "ab-a50", "id": "b3_8_10", "bin": "b3_8", "dur": 5.05, "t_wall": 1790811690.2542, "e2e_ms": 358.01, "server_ms": 355.108, "status": 200, "err": null}
|
|
{"arm": "ab-a50", "id": "b8_20_10", "bin": "b8_20", "dur": 11.64, "t_wall": 1790811690.9825685, "e2e_ms": 726.984, "server_ms": 724.065, "status": 200, "err": null}
|
|
{"arm": "ab-a50", "id": "b20_60_10", "bin": "b20_60", "dur": 54.9, "t_wall": 1790811693.9041705, "e2e_ms": 2915.796, "server_ms": 2905.895, "status": 200, "err": null}
|
|
{"arm": "ab-apl", "id": "b1_3_10", "bin": "b1_3", "dur": 2.555, "t_wall": 1790811751.7075408, "e2e_ms": 205.271, "server_ms": null, "status": 200, "err": null}
|
|
{"arm": "ab-apl", "id": "b3_8_10", "bin": "b3_8", "dur": 5.05, "t_wall": 1790811751.9997044, "e2e_ms": 291.27, "server_ms": null, "status": 200, "err": null}
|
|
{"arm": "ab-apl", "id": "b8_20_10", "bin": "b8_20", "dur": 11.64, "t_wall": 1790811752.6717346, "e2e_ms": 670.646, "server_ms": null, "status": 200, "err": null}
|
|
{"arm": "ab-apl", "id": "b20_60_10", "bin": "b20_60", "dur": 54.9, "t_wall": 1790811755.6003006, "e2e_ms": 2922.753, "server_ms": null, "status": 200, "err": null}
|
|
{"arm": "ab-at16", "id": "b1_3_10", "bin": "b1_3", "dur": 2.555, "t_wall": 1790811812.8083022, "e2e_ms": 129.012, "server_ms": 124.959, "status": 200, "err": null}
|
|
{"arm": "ab-at16", "id": "b3_8_10", "bin": "b3_8", "dur": 5.05, "t_wall": 1790811812.9687788, "e2e_ms": 159.71, "server_ms": 157.74, "status": 200, "err": null}
|
|
{"arm": "ab-at16", "id": "b8_20_10", "bin": "b8_20", "dur": 11.64, "t_wall": 1790811813.2948213, "e2e_ms": 324.896, "server_ms": 321.865, "status": 200, "err": null}
|
|
{"arm": "ab-at16", "id": "b20_60_10", "bin": "b20_60", "dur": 54.9, "t_wall": 1790811814.8389106, "e2e_ms": 1540.444, "server_ms": 1531.975, "status": 200, "err": null}
|
|
{"arm": "ab-c", "id": "b1_3_10", "bin": "b1_3", "dur": 2.555, "t_wall": 1790811866.5098884, "e2e_ms": 223.541, "server_ms": 210.645, "status": 200, "err": null}
|
|
{"arm": "ab-c", "id": "b3_8_10", "bin": "b3_8", "dur": 5.05, "t_wall": 1790811866.8653347, "e2e_ms": 354.618, "server_ms": 351.397, "status": 200, "err": null}
|
|
{"arm": "ab-c", "id": "b8_20_10", "bin": "b8_20", "dur": 11.64, "t_wall": 1790811867.5352015, "e2e_ms": 668.441, "server_ms": 665.287, "status": 200, "err": null}
|
|
{"arm": "ab-c", "id": "b20_60_10", "bin": "b20_60", "dur": 54.9, "t_wall": 1790811870.5005066, "e2e_ms": 2959.405, "server_ms": 2949.41, "status": 200, "err": null}
|
|
{"arm": "ab-d", "id": "b1_3_10", "bin": "b1_3", "dur": 2.555, "t_wall": 1790811921.618863, "e2e_ms": 217.189, "server_ms": 212.387, "status": 200, "err": null}
|
|
{"arm": "ab-d", "id": "b3_8_10", "bin": "b3_8", "dur": 5.05, "t_wall": 1790811921.905966, "e2e_ms": 286.202, "server_ms": 284.139, "status": 200, "err": null}
|
|
{"arm": "ab-d", "id": "b8_20_10", "bin": "b8_20", "dur": 11.64, "t_wall": 1790811922.5258143, "e2e_ms": 618.419, "server_ms": 616.048, "status": 200, "err": null}
|
|
{"arm": "ab-d", "id": "b20_60_10", "bin": "b20_60", "dur": 54.9, "t_wall": 1790811925.2671535, "e2e_ms": 2735.659, "server_ms": 2727.488, "status": 200, "err": null}
|
|
{"arm": "ab-cf32", "id": "b1_3_10", "bin": "b1_3", "dur": 2.555, "t_wall": 1790811973.740321, "e2e_ms": 82.453, "server_ms": 71.038, "status": 200, "err": null}
|
|
{"arm": "ab-cf32", "id": "b3_8_10", "bin": "b3_8", "dur": 5.05, "t_wall": 1790811973.792273, "e2e_ms": 51.008, "server_ms": 49.114, "status": 200, "err": null}
|
|
{"arm": "ab-cf32", "id": "b8_20_10", "bin": "b8_20", "dur": 11.64, "t_wall": 1790811973.8732834, "e2e_ms": 79.593, "server_ms": 77.421, "status": 200, "err": null}
|
|
{"arm": "ab-cf32", "id": "b20_60_10", "bin": "b20_60", "dur": 54.9, "t_wall": 1790811974.133343, "e2e_ms": 254.278, "server_ms": 245.859, "status": 200, "err": null}
|
|
{"arm": "ab-cf16", "id": "b1_3_10", "bin": "b1_3", "dur": 2.555, "t_wall": 1790812025.6972337, "e2e_ms": 123.521, "server_ms": 119.342, "status": 200, "err": null}
|
|
{"arm": "ab-cf16", "id": "b3_8_10", "bin": "b3_8", "dur": 5.05, "t_wall": 1790812025.7957647, "e2e_ms": 97.643, "server_ms": 95.538, "status": 200, "err": null}
|
|
{"arm": "ab-cf16", "id": "b8_20_10", "bin": "b8_20", "dur": 11.64, "t_wall": 1790812025.9317398, "e2e_ms": 134.637, "server_ms": 132.425, "status": 200, "err": null}
|
|
{"arm": "ab-cf16", "id": "b20_60_10", "bin": "b20_60", "dur": 54.9, "t_wall": 1790812026.2421963, "e2e_ms": 304.539, "server_ms": 298.103, "status": 200, "err": null}
|
|
{"arm": "ab-af32", "id": "b1_3_10", "bin": "b1_3", "dur": 2.555, "t_wall": 1790812081.0276036, "e2e_ms": 77.379, "server_ms": 73.332, "status": 200, "err": null}
|
|
{"arm": "ab-af32", "id": "b3_8_10", "bin": "b3_8", "dur": 5.05, "t_wall": 1790812081.0759206, "e2e_ms": 47.811, "server_ms": 45.966, "status": 200, "err": null}
|
|
{"arm": "ab-af32", "id": "b8_20_10", "bin": "b8_20", "dur": 11.64, "t_wall": 1790812081.144304, "e2e_ms": 67.599, "server_ms": 65.54, "status": 200, "err": null}
|
|
{"arm": "ab-af32", "id": "b20_60_10", "bin": "b20_60", "dur": 54.9, "t_wall": 1790812081.368576, "e2e_ms": 218.673, "server_ms": 211.82, "status": 200, "err": null}
|
|
{"arm": "ab-b32", "id": "b1_3_10", "bin": "b1_3", "dur": 2.555, "t_wall": 1790812099.6981435, "e2e_ms": 58.892, "server_ms": 55.234, "status": 200, "err": null}
|
|
{"arm": "ab-b32", "id": "b3_8_10", "bin": "b3_8", "dur": 5.05, "t_wall": 1790812099.742739, "e2e_ms": 43.753, "server_ms": 41.778, "status": 200, "err": null}
|
|
{"arm": "ab-b32", "id": "b8_20_10", "bin": "b8_20", "dur": 11.64, "t_wall": 1790812099.7807178, "e2e_ms": 36.787, "server_ms": 34.631, "status": 200, "err": null}
|
|
{"arm": "ab-b32", "id": "b20_60_10", "bin": "b20_60", "dur": 54.9, "t_wall": 1790812100.1957908, "e2e_ms": 409.785, "server_ms": 404.602, "status": 200, "err": null}
|
|
{"arm": "ab-b16", "id": "b1_3_10", "bin": "b1_3", "dur": 2.555, "t_wall": 1790812125.038448, "e2e_ms": 66.532, "server_ms": 63.429, "status": 200, "err": null}
|
|
{"arm": "ab-b16", "id": "b3_8_10", "bin": "b3_8", "dur": 5.05, "t_wall": 1790812125.0743783, "e2e_ms": 35.432, "server_ms": 34.011, "status": 200, "err": null}
|
|
{"arm": "ab-b16", "id": "b8_20_10", "bin": "b8_20", "dur": 11.64, "t_wall": 1790812125.1213822, "e2e_ms": 46.311, "server_ms": 44.775, "status": 200, "err": null}
|
|
{"arm": "ab-b16", "id": "b20_60_10", "bin": "b20_60", "dur": 54.9, "t_wall": 1790812125.496252, "e2e_ms": 372.25, "server_ms": 366.245, "status": 200, "err": null}
|