vh
a6c1d3c454
A/B of the live STT seat (fv-ml1 GPU 0, sherpa-onnx int8 v3) against nvidia/parakeet-unified-en-0.6b, measured on GPU 3 with the seat's own image, k2-fsa's published unified int8 export, fp32/fp16 exports made with k2-fsa's recipe, v2 int8, and NeMo 3.0.0 (fp32, bf16 autocast, bf16 weights). - Seat int8 graph runs on one CPU thread (cpu/wall 1.00, GPU 2-9%). - unified-en under NeMo: -121/-234/-530 ms vs the seat at 1-3/3-8/8-20 s (paired, n=120/bin; floor <=6 ms; +50 ms positive control reads +52-54). - unified-en WER lower in every runtime: -0.7 pp clean, -1.5 pp other, -3.2 to -4.4 pp AMI (paired CIs exclude 0). - Seat defects found: hard 400 s input ceiling (HTTP 500), truncation after a quiet 1.5 s pause, and severe long-window dropouts (int8 v3 only). - B-bf16w needs +0.8 to +1.5 GB over the seat's 1,690 MiB on GPU 0. Raw requests, hypotheses, manifests and the full harness under services/parakeet-ab-2026-09-30/. No deploy; live seat untouched apart from 240 light test requests.
33 lines
1.1 KiB
Python
33 lines
1.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Accuracy block: every set through every arm, one thread per arm (no latency is measured here, so
|
|
arms may run side by side). Per-arm files out/raw/acc/<set>--<arm>.jsonl.
|
|
usage: run_acc.py ARM:PORT[:SETS] ... SETS = comma list, default all five
|
|
"""
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
|
|
AB = "/tank/spikes/parakeet-ab"
|
|
ALL = ["ls-clean", "ls-other", "ami", "pc", "null"]
|
|
os.makedirs(f"{AB}/out/raw/acc", exist_ok=True)
|
|
|
|
|
|
def run(arm, port, sets):
|
|
for s in sets:
|
|
out = f"{AB}/out/raw/acc/{s}--{arm}.jsonl"
|
|
if os.path.exists(out):
|
|
os.rename(out, out + ".prev")
|
|
r = subprocess.run(["python3", f"{AB}/code/bench.py", "acc", f"http://127.0.0.1:{port}/v1/audio/transcriptions",
|
|
arm, f"{AB}/data/{s}.jsonl", out], capture_output=True, text=True)
|
|
print(arm, s, "rc", r.returncode, r.stderr[-300:], flush=True)
|
|
|
|
|
|
ts = []
|
|
for spec in sys.argv[1:]:
|
|
p = spec.split(":")
|
|
ts.append(threading.Thread(target=run, args=(p[0], p[1], p[2].split(",") if len(p) > 2 else ALL)))
|
|
[t.start() for t in ts]
|
|
[t.join() for t in ts]
|
|
print("ACC DONE", flush=True)
|