vh/esh-pfi-infrastructure
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects Releases Wiki Activity
Files
a6c1d3c4541448ea76c6d679eee04d2a65f2f2f1
esh-pfi-infrastructure/services/parakeet-ab-2026-09-30/results/raw/phases-block1.jsonl
T
vh a6c1d3c454 docs(parakeet): seat A/B vs parakeet-unified-en-0.6b - latency is the int8-on-CPU runtime; unified wins WER 
A/B of the live STT seat (fv-ml1 GPU 0, sherpa-onnx int8 v3) against
nvidia/parakeet-unified-en-0.6b, measured on GPU 3 with the seat's own image,
k2-fsa's published unified int8 export, fp32/fp16 exports made with k2-fsa's
recipe, v2 int8, and NeMo 3.0.0 (fp32, bf16 autocast, bf16 weights).

- Seat int8 graph runs on one CPU thread (cpu/wall 1.00, GPU 2-9%).
- unified-en under NeMo: -121/-234/-530 ms vs the seat at 1-3/3-8/8-20 s
  (paired, n=120/bin; floor <=6 ms; +50 ms positive control reads +52-54).
- unified-en WER lower in every runtime: -0.7 pp clean, -1.5 pp other,
  -3.2 to -4.4 pp AMI (paired CIs exclude 0).
- Seat defects found: hard 400 s input ceiling (HTTP 500), truncation after
  a quiet 1.5 s pause, and severe long-window dropouts (int8 v3 only).
- B-bf16w needs +0.8 to +1.5 GB over the seat's 1,690 MiB on GPU 0.

Raw requests, hypotheses, manifests and the full harness under
services/parakeet-ab-2026-09-30/. No deploy; live seat untouched apart
from 240 light test requests.
2026-09-30 18:51:44 -07:00

97 lines
8.4 KiB
JSON
Raw Blame History

{"arm": "ab-a1", "phase": "rest-after-warmup", "t0": 1790811569.5917542, "t1": 1790811572.5917592}
{"arm": "ab-a2", "phase": "rest-after-warmup", "t0": 1790811624.6530228, "t1": 1790811627.653029}
{"arm": "ab-a50", "phase": "rest-after-warmup", "t0": 1790811686.6333246, "t1": 1790811689.6333303}
{"arm": "ab-apl", "phase": "rest-after-warmup", "t0": 1790811748.5013003, "t1": 1790811751.501305}
{"arm": "ab-at16", "phase": "rest-after-warmup", "t0": 1790811809.678218, "t1": 1790811812.6782246}
{"arm": "ab-c", "phase": "rest-after-warmup", "t0": 1790811863.285389, "t1": 1790811866.285394}
{"arm": "ab-d", "phase": "rest-after-warmup", "t0": 1790811918.4005463, "t1": 1790811921.4005525}
{"arm": "ab-cf32", "phase": "rest-after-warmup", "t0": 1790811970.656792, "t1": 1790811973.656798}
{"arm": "ab-cf16", "phase": "rest-after-warmup", "t0": 1790812022.5726755, "t1": 1790812025.5726814}
{"arm": "ab-af32", "phase": "rest-after-warmup", "t0": 1790812077.9493158, "t1": 1790812080.949321}
{"arm": "ab-b32", "phase": "rest-after-warmup", "t0": 1790812096.638151, "t1": 1790812099.6381571}
{"arm": "ab-b16", "phase": "rest-after-warmup", "t0": 1790812121.9710329, "t1": 1790812124.971038}
{"arm": "ab-a2", "phase": "lat-r0", "t0": 1790812125.4965167, "t1": 1790812187.007135}
{"arm": "ab-d", "phase": "lat-r0", "t0": 1790812187.0072649, "t1": 1790812247.578683}
{"arm": "ab-a50", "phase": "lat-r0", "t0": 1790812247.5788965, "t1": 1790812312.7582943}
{"arm": "ab-cf16", "phase": "lat-r0", "t0": 1790812312.758496, "t1": 1790812324.3687835}
{"arm": "ab-cf32", "phase": "lat-r0", "t0": 1790812324.3689945, "t1": 1790812331.8374858}
{"arm": "ab-b16", "phase": "lat-r0", "t0": 1790812331.837698, "t1": 1790812336.0153196}
{"arm": "ab-c", "phase": "lat-r0", "t0": 1790812336.0154521, "t1": 1790812399.656281}
{"arm": "ab-b32", "phase": "lat-r0", "t0": 1790812399.6564975, "t1": 1790812403.616376}
{"arm": "ab-a1", "phase": "lat-r0", "t0": 1790812403.6165466, "t1": 1790812465.578666}
{"arm": "ab-at16", "phase": "lat-r0", "t0": 1790812465.5788743, "t1": 1790812500.7428703}
{"arm": "ab-apl", "phase": "lat-r0", "t0": 1790812500.743009, "t1": 1790812562.8481212}
{"arm": "ab-af32", "phase": "lat-r0", "t0": 1790812562.8483267, "t1": 1790812569.6138718}
{"arm": "ab-af32", "phase": "lat-r1", "t0": 1790812569.61402, "t1": 1790812576.3360198}
{"arm": "ab-cf32", "phase": "lat-r1", "t0": 1790812576.3362226, "t1": 1790812583.880805}
{"arm": "ab-apl", "phase": "lat-r1", "t0": 1790812583.8810163, "t1": 1790812644.794285}
{"arm": "ab-b32", "phase": "lat-r1", "t0": 1790812644.79448, "t1": 1790812648.2971654}
{"arm": "ab-b16", "phase": "lat-r1", "t0": 1790812648.297381, "t1": 1790812651.953531}
{"arm": "ab-c", "phase": "lat-r1", "t0": 1790812651.9537418, "t1": 1790812715.5186048}
{"arm": "ab-d", "phase": "lat-r1", "t0": 1790812715.5188184, "t1": 1790812776.678384}
{"arm": "ab-at16", "phase": "lat-r1", "t0": 1790812776.6785686, "t1": 1790812811.7173295}
{"arm": "ab-a1", "phase": "lat-r1", "t0": 1790812811.717519, "t1": 1790812872.9456182}
{"arm": "ab-a2", "phase": "lat-r1", "t0": 1790812872.9458008, "t1": 1790812933.6530173}
{"arm": "ab-a50", "phase": "lat-r1", "t0": 1790812933.65325, "t1": 1790812998.1032982}
{"arm": "ab-cf16", "phase": "lat-r1", "t0": 1790812998.1035042, "t1": 1790813009.6293406}
{"arm": "ab-at16", "phase": "lat-r2", "t0": 1790813009.62965, "t1": 1790813044.5689945}
{"arm": "ab-c", "phase": "lat-r2", "t0": 1790813044.569184, "t1": 1790813107.8070064}
{"arm": "ab-cf32", "phase": "lat-r2", "t0": 1790813107.8072178, "t1": 1790813115.233969}
{"arm": "ab-a2", "phase": "lat-r2", "t0": 1790813115.2341795, "t1": 1790813176.8812537}
{"arm": "ab-d", "phase": "lat-r2", "t0": 1790813176.881396, "t1": 1790813238.5842261}
{"arm": "ab-b32", "phase": "lat-r2", "t0": 1790813238.584434, "t1": 1790813242.0657837}
{"arm": "ab-a1", "phase": "lat-r2", "t0": 1790813242.0659711, "t1": 1790813302.64495}
{"arm": "ab-a50", "phase": "lat-r2", "t0": 1790813302.64516, "t1": 1790813367.392624}
{"arm": "ab-b16", "phase": "lat-r2", "t0": 1790813367.3928695, "t1": 1790813371.027336}
{"arm": "ab-apl", "phase": "lat-r2", "t0": 1790813371.0275292, "t1": 1790813432.7388752}
{"arm": "ab-af32", "phase": "lat-r2", "t0": 1790813432.7391152, "t1": 1790813439.430608}
{"arm": "ab-cf16", "phase": "lat-r2", "t0": 1790813439.430816, "t1": 1790813450.9556117}
{"arm": "ab-at16", "phase": "lat-r3", "t0": 1790813450.9559083, "t1": 1790813485.4256256}
{"arm": "ab-a1", "phase": "lat-r3", "t0": 1790813485.425798, "t1": 1790813545.7027633}
{"arm": "ab-d", "phase": "lat-r3", "t0": 1790813545.702973, "t1": 1790813606.988272}
{"arm": "ab-a2", "phase": "lat-r3", "t0": 1790813606.9885, "t1": 1790813669.9248126}
{"arm": "ab-cf16", "phase": "lat-r3", "t0": 1790813669.9249954, "t1": 1790813681.4729636}
{"arm": "ab-b32", "phase": "lat-r3", "t0": 1790813681.4731736, "t1": 1790813685.3903198}
{"arm": "ab-apl", "phase": "lat-r3", "t0": 1790813685.390532, "t1": 1790813747.246512}
{"arm": "ab-c", "phase": "lat-r3", "t0": 1790813747.246718, "t1": 1790813810.9303126}
{"arm": "ab-cf32", "phase": "lat-r3", "t0": 1790813810.9305224, "t1": 1790813818.3714337}
{"arm": "ab-af32", "phase": "lat-r3", "t0": 1790813818.3716667, "t1": 1790813825.0550656}
{"arm": "ab-a50", "phase": "lat-r3", "t0": 1790813825.055269, "t1": 1790813890.0087988}
{"arm": "ab-b16", "phase": "lat-r3", "t0": 1790813890.0089836, "t1": 1790813893.6641235}
{"arm": "ab-a1", "phase": "lat-r4", "t0": 1790813893.664362, "t1": 1790813954.1769922}
{"arm": "ab-cf16", "phase": "lat-r4", "t0": 1790813954.1772017, "t1": 1790813965.717562}
{"arm": "ab-d", "phase": "lat-r4", "t0": 1790813965.7177472, "t1": 1790814027.2049408}
{"arm": "ab-cf32", "phase": "lat-r4", "t0": 1790814027.2051857, "t1": 1790814034.6653776}
{"arm": "ab-at16", "phase": "lat-r4", "t0": 1790814034.6655593, "t1": 1790814068.5630045}
{"arm": "ab-c", "phase": "lat-r4", "t0": 1790814068.5631104, "t1": 1790814131.276856}
{"arm": "ab-b16", "phase": "lat-r4", "t0": 1790814131.2770677, "t1": 1790814134.8667376}
{"arm": "ab-b32", "phase": "lat-r4", "t0": 1790814134.86693, "t1": 1790814138.3525987}
{"arm": "ab-a2", "phase": "lat-r4", "t0": 1790814138.352793, "t1": 1790814198.9694974}
{"arm": "ab-af32", "phase": "lat-r4", "t0": 1790814198.9697266, "t1": 1790814205.6496642}
{"arm": "ab-apl", "phase": "lat-r4", "t0": 1790814205.6498177, "t1": 1790814267.690293}
{"arm": "ab-a50", "phase": "lat-r4", "t0": 1790814267.6904974, "t1": 1790814333.2101731}
{"arm": "ab-af32", "phase": "lat-r5", "t0": 1790814333.210449, "t1": 1790814340.001132}
{"arm": "ab-d", "phase": "lat-r5", "t0": 1790814340.0013206, "t1": 1790814400.5689385}
{"arm": "ab-b32", "phase": "lat-r5", "t0": 1790814400.5691476, "t1": 1790814404.0392427}
{"arm": "ab-cf16", "phase": "lat-r5", "t0": 1790814404.039377, "t1": 1790814415.6019518}
{"arm": "ab-cf32", "phase": "lat-r5", "t0": 1790814415.6021612, "t1": 1790814423.0993385}
{"arm": "ab-apl", "phase": "lat-r5", "t0": 1790814423.0995467, "t1": 1790814484.8875544}
{"arm": "ab-c", "phase": "lat-r5", "t0": 1790814484.8877637, "t1": 1790814548.3318098}
{"arm": "ab-a1", "phase": "lat-r5", "t0": 1790814548.3320324, "t1": 1790814608.4877155}
{"arm": "ab-b16", "phase": "lat-r5", "t0": 1790814608.4879506, "t1": 1790814612.0272424}
{"arm": "ab-a2", "phase": "lat-r5", "t0": 1790814612.02743, "t1": 1790814672.14974}
{"arm": "ab-at16", "phase": "lat-r5", "t0": 1790814672.149839, "t1": 1790814706.5802422}
{"arm": "ab-a50", "phase": "lat-r5", "t0": 1790814706.5804288, "t1": 1790814771.1039104}
{"arm": "ab-a1", "phase": "conc4", "t0": 1790814771.1041691, "t1": 1790814893.0691576}
{"arm": "ab-a2", "phase": "conc4", "t0": 1790814893.0694113, "t1": 1790815014.4060903}
{"arm": "ab-a50", "phase": "conc4", "t0": 1790815014.4063337, "t1": 1790815145.3641863}
{"arm": "ab-apl", "phase": "conc4", "t0": 1790815145.364455, "t1": 1790815270.1540234}
{"arm": "ab-at16", "phase": "conc4", "t0": 1790815270.1542823, "t1": 1790815339.4303842}
{"arm": "ab-c", "phase": "conc4", "t0": 1790815339.4305325, "t1": 1790815467.0962896}
{"arm": "ab-d", "phase": "conc4", "t0": 1790815467.0965424, "t1": 1790815591.2174613}
{"arm": "ab-cf32", "phase": "conc4", "t0": 1790815591.2177114, "t1": 1790815606.2531717}
{"arm": "ab-cf16", "phase": "conc4", "t0": 1790815606.2534478, "t1": 1790815629.2447588}
{"arm": "ab-af32", "phase": "conc4", "t0": 1790815629.2450092, "t1": 1790815642.547299}
{"arm": "ab-b32", "phase": "conc4", "t0": 1790815642.5475514, "t1": 1790815649.4173794}
{"arm": "ab-b16", "phase": "conc4", "t0": 1790815649.41753, "t1": 1790815656.4448721}
Reference in New Issue View Git Blame Copy Permalink