docs(parakeet): seat A/B vs parakeet-unified-en-0.6b - latency is the int8-on-CPU runtime; unified wins WER
A/B of the live STT seat (fv-ml1 GPU 0, sherpa-onnx int8 v3) against nvidia/parakeet-unified-en-0.6b, measured on GPU 3 with the seat's own image, k2-fsa's published unified int8 export, fp32/fp16 exports made with k2-fsa's recipe, v2 int8, and NeMo 3.0.0 (fp32, bf16 autocast, bf16 weights). - Seat int8 graph runs on one CPU thread (cpu/wall 1.00, GPU 2-9%). - unified-en under NeMo: -121/-234/-530 ms vs the seat at 1-3/3-8/8-20 s (paired, n=120/bin; floor <=6 ms; +50 ms positive control reads +52-54). - unified-en WER lower in every runtime: -0.7 pp clean, -1.5 pp other, -3.2 to -4.4 pp AMI (paired CIs exclude 0). - Seat defects found: hard 400 s input ceiling (HTTP 500), truncation after a quiet 1.5 s pause, and severe long-window dropouts (int8 v3 only). - B-bf16w needs +0.8 to +1.5 GB over the seat's 1,690 MiB on GPU 0. Raw requests, hypotheses, manifests and the full harness under services/parakeet-ab-2026-09-30/. No deploy; live seat untouched apart from 240 light test requests.
This commit is contained in:
@@ -0,0 +1,90 @@
|
||||
"""Accuracy summary over out/raw/acc/<set>--<arm>.jsonl. Run with envs/score.
|
||||
|
||||
- WER per arm per set (score.py: whole-string Whisper normaliser, exact Levenshtein).
|
||||
- Paired bootstrap deltas vs a reference arm (default ab-a1 = the seat's runtime + weights).
|
||||
- Positive control (pc: 1.5 s of digital silence inside 40 utterances, reference unchanged): the arm
|
||||
must register extra DELETIONS vs the same 40 utterances in ls-clean.
|
||||
- Null control (null: ls-clean at -0.5 dB): the WER change vs ls-clean must sit inside the paired CI.
|
||||
- Determinism: identical text across two instances / thread counts.
|
||||
- Output features: share of outputs with sentence punctuation, with an upper-case letter, all-lower.
|
||||
usage: acc_summary.py ACC_DIR DATA_DIR [REF_ARM]
|
||||
"""
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import score # noqa: E402
|
||||
|
||||
ACC, DATA = sys.argv[1], sys.argv[2]
|
||||
REF = sys.argv[3] if len(sys.argv) > 3 else "ab-a1"
|
||||
SETS = ["ls-clean", "ls-other", "ami", "pc", "null"]
|
||||
|
||||
|
||||
def load(set_name):
|
||||
ref = {json.loads(l)["id"]: json.loads(l) for l in open(f"{DATA}/{set_name}.jsonl")}
|
||||
by = {}
|
||||
for p in glob.glob(f"{ACC}/{set_name}--*.jsonl"):
|
||||
arm = p.rsplit("--", 1)[1][:-6]
|
||||
by[arm] = {json.loads(l)["id"]: json.loads(l) for l in open(p)}
|
||||
return ref, by
|
||||
|
||||
|
||||
out = dict(wer=[], delta=[], pc=[], null=[], same=[], features=[])
|
||||
pu = {}
|
||||
for s in SETS:
|
||||
ref, by = load(s)
|
||||
for arm, rows in sorted(by.items()):
|
||||
pu[(s, arm)] = score.per_utt(ref, rows)
|
||||
out["wer"].append(dict(set=s, arm=arm, **score.summary(pu[(s, arm)])))
|
||||
if REF in by:
|
||||
for arm in sorted(by):
|
||||
if arm != REF:
|
||||
out["delta"].append(dict(set=s, x=arm, y=REF, **score.boot(pu[(s, arm)], pu[(s, REF)])))
|
||||
if "ab-b32" in by:
|
||||
for arm in sorted(by):
|
||||
if arm not in ("ab-b32", REF):
|
||||
out["delta"].append(dict(set=s, x=arm, y="ab-b32", **score.boot(pu[(s, arm)], pu[(s, "ab-b32")])))
|
||||
|
||||
# positive control: deletions on the silenced copies vs the same utterances unsilenced
|
||||
pc_ref = {json.loads(l)["id"]: json.loads(l) for l in open(f"{DATA}/pc.jsonl")}
|
||||
for (s, arm), v in pu.items():
|
||||
if s != "pc" or ("ls-clean", arm) not in pu:
|
||||
continue
|
||||
base = pu[("ls-clean", arm)]
|
||||
ids = [k for k in pc_ref if v.get(k) and base.get(k)]
|
||||
d_pc = sum(v[k][1] for k in ids); d_base = sum(base[k][1] for k in ids)
|
||||
e_pc = sum(sum(v[k][:3]) for k in ids); e_base = sum(sum(base[k][:3]) for k in ids)
|
||||
n = sum(base[k][3] for k in ids)
|
||||
hit = sum(1 for k in ids if v[k][1] > base[k][1])
|
||||
out["pc"].append(dict(arm=arm, utts=len(ids), deletions_clean=d_base, deletions_silenced=d_pc,
|
||||
errors_clean=e_base, errors_silenced=e_pc, wer_clean=round(100 * e_base / n, 2),
|
||||
wer_silenced=round(100 * e_pc / n, 2), utts_with_more_deletions=hit))
|
||||
# null control
|
||||
for (s, arm), v in pu.items():
|
||||
if s == "null" and ("ls-clean", arm) in pu:
|
||||
b = score.boot(v, pu[("ls-clean", arm)])
|
||||
out["null"].append(dict(arm=arm, **b))
|
||||
# determinism: identical text between pairs of arms on ls-clean
|
||||
ref, by = load("ls-clean")
|
||||
for a, b in (("ab-a1", "ab-a2"), ("ab-a1", "ab-at16"), ("ab-b32", "ab-b32t"), ("ab-a1", "A-live")):
|
||||
if a in by and b in by:
|
||||
ids = [k for k in ref if k in by[a] and k in by[b]]
|
||||
out["same"].append(dict(a=a, b=b, utts=len(ids), identical_text=sum(by[a][k]["text"] == by[b][k]["text"] for k in ids)))
|
||||
# features
|
||||
for arm, rows in sorted(by.items()):
|
||||
txt = [r["text"] or "" for r in rows.values() if r.get("status") == 200]
|
||||
out["features"].append(dict(arm=arm, outputs=len(txt),
|
||||
with_sentence_punct=round(100 * sum(bool(re.search(r"[.?!]", t)) for t in txt) / len(txt), 1),
|
||||
with_comma=round(100 * sum("," in t for t in txt) / len(txt), 1),
|
||||
with_upper=round(100 * sum(any(c.isupper() for c in t) for t in txt) / len(txt), 1),
|
||||
all_lower=round(100 * sum(t == t.lower() and t.strip() != "" for t in txt) / len(txt), 1),
|
||||
empty=sum(t.strip() == "" for t in txt),
|
||||
non_ascii=sum(any(ord(c) > 127 for c in t) for t in txt)))
|
||||
for k, v in out.items():
|
||||
print(f"## {k}")
|
||||
for r in v:
|
||||
print(json.dumps(r))
|
||||
json.dump(out, open(f"{ACC}/../../accuracy-summary.json", "w"), indent=1)
|
||||
@@ -0,0 +1,179 @@
|
||||
"""Latency + memory analysis. numpy; run with envs/score.
|
||||
|
||||
Single stream: per arm x bin, p50/p90/p99 of server-side (x-ab-decode-ms) and end-to-end (client wall,
|
||||
new connection per request, fv-ml1 loopback). Deltas vs a reference arm are PAIRED: in round k every arm
|
||||
got byte-identical input for a clip (the tail trim is seeded by round and clip, never by arm), so the
|
||||
statistic is the median of per-request differences, with a 95 % bootstrap CI over requests.
|
||||
Floor = ab-a2 vs ab-a1 (identical second instance). Positive control = ab-a50 (+50 ms injected) vs ab-a1.
|
||||
Null = ab-apl (the image's own app, no timing header) vs ab-a1 (e2e only).
|
||||
usage: analyze_lat.py RAW_DIR [REF_ARM]
|
||||
"""
|
||||
import collections
|
||||
import json
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
|
||||
RAW = sys.argv[1]
|
||||
REF = sys.argv[2] if len(sys.argv) > 2 else "ab-a1"
|
||||
BINS = ["b1_3", "b3_8", "b8_20", "b20_60"]
|
||||
rng = np.random.default_rng(7)
|
||||
|
||||
|
||||
def rows(fn, mode=None):
|
||||
out = []
|
||||
try:
|
||||
for l in open(f"{RAW}/{fn}"):
|
||||
r = json.loads(l)
|
||||
if mode is None or r.get("mode") == mode:
|
||||
out.append(r)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def pct(x, q):
|
||||
return float(np.percentile(np.asarray(x, float), q)) if len(x) else float("nan")
|
||||
|
||||
|
||||
def boot_ci(x, stat=np.median, n=2000):
|
||||
x = np.asarray(x, float)
|
||||
if len(x) < 3:
|
||||
return (float("nan"), float("nan"))
|
||||
b = [stat(x[rng.integers(0, len(x), len(x))]) for _ in range(n)]
|
||||
return (float(np.percentile(b, 2.5)), float(np.percentile(b, 97.5)))
|
||||
|
||||
|
||||
def fmt(v):
|
||||
return "nan" if v != v else f"{v:.1f}"
|
||||
|
||||
|
||||
def single_stream(files, arms_order=None):
|
||||
lat = [r for f in files for r in rows(f, "lat") if r["status"] == 200]
|
||||
bad = [r for f in files for r in rows(f, "lat") if r["status"] != 200]
|
||||
by = collections.defaultdict(list)
|
||||
for r in lat:
|
||||
by[(r["arm"], r["bin"])].append(r)
|
||||
arms = arms_order or sorted({a for a, _ in by})
|
||||
table = []
|
||||
for a in arms:
|
||||
for b in BINS:
|
||||
xs = by.get((a, b), [])
|
||||
if not xs:
|
||||
continue
|
||||
e = [r["e2e_ms"] for r in xs]
|
||||
s = [r["server_ms"] for r in xs if r.get("server_ms") is not None]
|
||||
lo, hi = boot_ci(e)
|
||||
table.append(dict(arm=a, bin=b, n=len(xs), rounds=len({r["round"] for r in xs}),
|
||||
e2e_p50=pct(e, 50), e2e_p50_ci=[lo, hi], e2e_p90=pct(e, 90), e2e_p99=pct(e, 99), e2e_max=max(e),
|
||||
srv_p50=pct(s, 50), srv_p90=pct(s, 90), srv_p99=pct(s, 99),
|
||||
overhead_p50=pct([r["e2e_ms"] - r["server_ms"] for r in xs if r.get("server_ms") is not None], 50)))
|
||||
return table, by, bad
|
||||
|
||||
|
||||
def paired(by, a, ref, b, key="e2e_ms"):
|
||||
ra = {(r["round"], r["id"]): r for r in by.get((a, b), [])}
|
||||
rr = {(r["round"], r["id"]): r for r in by.get((ref, b), [])}
|
||||
ks = sorted(set(ra) & set(rr))
|
||||
if not ks:
|
||||
return None
|
||||
for k in ks: # the pairing is only valid if the two requests carried identical input
|
||||
assert ra[k].get("trim_ms") == rr[k].get("trim_ms"), (a, ref, k)
|
||||
d = [ra[k][key] - rr[k][key] for k in ks]
|
||||
lo, hi = boot_ci(d)
|
||||
return dict(arm=a, ref=ref, bin=b, n_pairs=len(ks), median_diff=float(np.median(d)), ci95=[lo, hi],
|
||||
p50_arm=pct([ra[k][key] for k in ks], 50), p50_ref=pct([rr[k][key] for k in ks], 50))
|
||||
|
||||
|
||||
def conc():
|
||||
rs = [r for r in rows("conc.jsonl", "conc")]
|
||||
bins = {(r["arm"], r["bin"]): r for r in rows("conc.jsonl", "conc-bin")}
|
||||
by = collections.defaultdict(list)
|
||||
for r in rs:
|
||||
by[(r["arm"], r["bin"])].append(r)
|
||||
out = []
|
||||
for (a, b), xs in sorted(by.items()):
|
||||
ok = [r for r in xs if r["status"] == 200]
|
||||
e = [r["e2e_ms"] for r in ok]
|
||||
wall = bins[(a, b)]["wall_s"]
|
||||
audio = sum(r["dur"] - r.get("trim_ms", 0) / 1000 for r in ok)
|
||||
out.append(dict(arm=a, bin=b, n=len(xs), errors=len(xs) - len(ok), e2e_p50=pct(e, 50), e2e_p90=pct(e, 90),
|
||||
e2e_p99=pct(e, 99), srv_p50=pct([r["server_ms"] for r in ok if r.get("server_ms")], 50),
|
||||
req_per_s=round(len(ok) / wall, 2), audio_x_realtime=round(audio / wall, 1)))
|
||||
return out
|
||||
|
||||
|
||||
def memory():
|
||||
arms = {r["arm"]: r for r in rows("arms.jsonl")}
|
||||
phases = rows("phases.jsonl")
|
||||
# every PID of each container (NeMo forks workers; the CUDA context is in the first python process)
|
||||
try:
|
||||
allp = json.load(open(f"{RAW}/pids.json"))
|
||||
except FileNotFoundError:
|
||||
allp = {a: [r["pid"]] for a, r in arms.items()}
|
||||
pid2arm = {p: a for a, ps in allp.items() for p in ps}
|
||||
for a in arms:
|
||||
arms[a]["pid"] = next((p for p in allp.get(a, []) if p in pid2arm), arms[a]["pid"])
|
||||
series = collections.defaultdict(list) # pid -> [(t, MiB)]
|
||||
import datetime
|
||||
try:
|
||||
for l in open(f"{RAW}/mem.csv"):
|
||||
p = [x.strip() for x in l.split(",")]
|
||||
if len(p) != 4 or not p[2].isdigit():
|
||||
continue
|
||||
pid = int(p[2])
|
||||
if pid in pid2arm:
|
||||
t = datetime.datetime.strptime(p[0], "%Y/%m/%d %H:%M:%S.%f").timestamp()
|
||||
series[pid].append((t, int(p[3])))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
out = []
|
||||
for a, r in arms.items():
|
||||
s = sorted(x for p in allp.get(a, [r["pid"]]) for x in series.get(p, []))
|
||||
rest = [m for t, m in s if any(ph["arm"] == a and ph["phase"] == "rest-after-warmup" and ph["t0"] <= t <= ph["t1"] + 1 for ph in phases)]
|
||||
lat_w = [ph for ph in phases if ph["arm"] == a and (ph["phase"].startswith("lat-") or ph["phase"] == "conc4")]
|
||||
t_lat0 = min((ph["t0"] for ph in lat_w), default=None)
|
||||
t_lat1 = max((ph["t1"] for ph in lat_w), default=None)
|
||||
lat_peak = max((m for t, m in s if t_lat0 and t_lat0 <= t <= t_lat1 + 2), default=None)
|
||||
out.append(dict(arm=a, pid=r["pid"], cold_s=r["cold_s"], rest_after_warmup_mib=max(rest) if rest else None,
|
||||
peak_utterance_block_mib=lat_peak, peak_whole_life_mib=max((m for _, m in s), default=None),
|
||||
last_mib=s[-1][1] if s else None, samples=len(s)))
|
||||
return out
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
what = sys.argv[3] if len(sys.argv) > 3 else "all"
|
||||
files = ["lat.jsonl"] + (["lat-live.jsonl"] if what in ("all", "live") else [])
|
||||
t, by, bad = single_stream(files)
|
||||
print("## single stream (ms)")
|
||||
for r in t:
|
||||
print(f"{r['arm']:10s} {r['bin']:7s} n={r['n']:3d} e2e p50 {fmt(r['e2e_p50'])} [{fmt(r['e2e_p50_ci'][0])},{fmt(r['e2e_p50_ci'][1])}] "
|
||||
f"p90 {fmt(r['e2e_p90'])} p99 {fmt(r['e2e_p99'])} max {fmt(r['e2e_max'])} | server p50 {fmt(r['srv_p50'])} p90 {fmt(r['srv_p90'])} "
|
||||
f"p99 {fmt(r['srv_p99'])} | http overhead p50 {fmt(r['overhead_p50'])}")
|
||||
if bad:
|
||||
print("NON-200:", collections.Counter((r["arm"], r["status"]) for r in bad))
|
||||
print(f"\n## paired median difference vs {REF} (e2e ms; 95% bootstrap CI)")
|
||||
pairs = []
|
||||
for a in sorted({a for a, _ in by}):
|
||||
if a == REF:
|
||||
continue
|
||||
for b in BINS:
|
||||
p = paired(by, a, REF, b)
|
||||
if p:
|
||||
pairs.append(p)
|
||||
print(f"{a:10s} {b:7s} pairs={p['n_pairs']:3d} diff {p['median_diff']:+8.1f} [{p['ci95'][0]:+8.1f},{p['ci95'][1]:+8.1f}]")
|
||||
c = conc()
|
||||
print("\n## concurrency 4 (ms)")
|
||||
for r in c:
|
||||
print(f"{r['arm']:10s} {r['bin']:7s} n={r['n']} err={r['errors']} e2e p50 {fmt(r['e2e_p50'])} p90 {fmt(r['e2e_p90'])} p99 {fmt(r['e2e_p99'])} "
|
||||
f"| server p50 {fmt(r['srv_p50'])} | {r['req_per_s']} req/s, {r['audio_x_realtime']}x realtime")
|
||||
first = rows("first.jsonl")
|
||||
print("\n## first call per bin after cold start (ms) vs that arm's warm e2e p50")
|
||||
warm = {(r["arm"], r["bin"]): r["e2e_p50"] for r in t}
|
||||
for r in first:
|
||||
print(f"{r['arm']:10s} {r['bin']:7s} first {r['e2e_ms']:8.1f} warm p50 {fmt(warm.get((r['arm'], r['bin']), float('nan')))}")
|
||||
m = memory()
|
||||
print("\n## GPU memory per process (MiB)")
|
||||
for r in m:
|
||||
print(json.dumps(r))
|
||||
json.dump(dict(single=t, paired=pairs, conc=c, first=first, memory=m), open(f"{RAW}/../latency-summary.json", "w"), indent=1)
|
||||
@@ -0,0 +1,138 @@
|
||||
# A/B harness copy of stacks/parakeet/app.py (2026-09-30): identical decode path; adds an
|
||||
# x-ab-decode-ms header, an optional injected delay (AB_DELAY_MS) and a model-suffix knob.
|
||||
"""Thin FastAPI wrapper around sherpa-onnx's OfflineRecognizer for Parakeet-TDT.
|
||||
|
||||
Load the encoder/decoder/joiner/tokens once at startup; serve:
|
||||
POST /transcribe — our native shape
|
||||
POST /v1/audio/transcriptions — OpenAI-compatible alias (returns {"text": ...})
|
||||
GET /healthz — used by the docker healthcheck
|
||||
|
||||
No VAD chunking, no Silero preprocessing — parakeet-tdt handles long-form natively
|
||||
and the int8 ONNX model is a rounding error against this host's 96 GB cards.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import sherpa_onnx
|
||||
import soundfile as sf
|
||||
from fastapi import FastAPI, File, HTTPException, UploadFile
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
MODEL_DIR = Path(os.environ.get("MODEL_DIR", "/models"))
|
||||
PROVIDER = os.environ.get("PROVIDER", "cuda")
|
||||
NUM_THREADS = int(os.environ.get("NUM_THREADS", "1"))
|
||||
# A/B harness only (not in the seat): an injected delay for the positive control, and the
|
||||
# model filenames so a non-int8 export can be served by the same code.
|
||||
AB_DELAY_S = float(os.environ.get("AB_DELAY_MS", "0")) / 1000.0
|
||||
AB_SUFFIX = os.environ.get("AB_SUFFIX", "int8.onnx")
|
||||
|
||||
REQUIRED_FILES = (
|
||||
f"encoder.{AB_SUFFIX}",
|
||||
f"decoder.{AB_SUFFIX}",
|
||||
f"joiner.{AB_SUFFIX}",
|
||||
"tokens.txt",
|
||||
)
|
||||
|
||||
logger = logging.getLogger("parakeet")
|
||||
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO"))
|
||||
|
||||
|
||||
def _ensure_model_present() -> None:
|
||||
missing = [f for f in REQUIRED_FILES if not (MODEL_DIR / f).exists()]
|
||||
if missing:
|
||||
raise RuntimeError(
|
||||
f"Missing model files in {MODEL_DIR}: {missing}. "
|
||||
"The entrypoint is responsible for downloading them before the server starts."
|
||||
)
|
||||
|
||||
|
||||
def _load_recognizer() -> sherpa_onnx.OfflineRecognizer:
|
||||
_ensure_model_present()
|
||||
logger.info("loading OfflineRecognizer (provider=%s, threads=%d)", PROVIDER, NUM_THREADS)
|
||||
return sherpa_onnx.OfflineRecognizer.from_transducer(
|
||||
encoder=str(MODEL_DIR / f"encoder.{AB_SUFFIX}"),
|
||||
decoder=str(MODEL_DIR / f"decoder.{AB_SUFFIX}"),
|
||||
joiner=str(MODEL_DIR / f"joiner.{AB_SUFFIX}"),
|
||||
tokens=str(MODEL_DIR / "tokens.txt"),
|
||||
model_type="nemo_transducer",
|
||||
provider=PROVIDER,
|
||||
num_threads=NUM_THREADS,
|
||||
)
|
||||
|
||||
|
||||
def _warm(rec: "sherpa_onnx.OfflineRecognizer") -> None:
|
||||
"""Decode one throwaway buffer before the server accepts traffic.
|
||||
|
||||
⚠ NOT an optimisation — it moves a 45 s stall out of the first real request.
|
||||
ONNX Runtime's CUDA EP compiles and autotunes its kernels lazily, on the first
|
||||
decode, and on this host (RTX PRO 6000 Blackwell, sm_120) that measured **45.7 s**
|
||||
while every subsequent call was ~0.48 s. Without this, the first caller after any
|
||||
container restart sees a 45 s hang and most clients — LiteLLM's default request
|
||||
timeout included — give up long before it returns, which reads as "the service is
|
||||
broken" rather than "the service is warming".
|
||||
|
||||
The healthcheck's `start_period` (300 s) is what makes paying it here safe.
|
||||
"""
|
||||
try:
|
||||
t0 = time.monotonic()
|
||||
stream = rec.create_stream()
|
||||
# 1 s of silence at 16 kHz: enough to force the full encoder/decoder/joiner
|
||||
# path to compile, cheap enough not to matter.
|
||||
stream.accept_waveform(16000, np.zeros(16000, dtype=np.float32))
|
||||
rec.decode_stream(stream)
|
||||
logger.info("warmup decode complete in %.1fs — CUDA kernels compiled", time.monotonic() - t0)
|
||||
except Exception:
|
||||
# A failed warmup must not stop the server: the model is loaded and real
|
||||
# requests would still work, just with the stall back on the first caller.
|
||||
logger.exception("warmup decode failed; first real request will absorb the stall")
|
||||
|
||||
|
||||
app = FastAPI(title="Parakeet ASR (sherpa-onnx)")
|
||||
recognizer = _load_recognizer()
|
||||
_warm(recognizer)
|
||||
|
||||
|
||||
def _decode(raw: bytes) -> str:
|
||||
try:
|
||||
samples, sample_rate = sf.read(io.BytesIO(raw), dtype="float32")
|
||||
except Exception as exc:
|
||||
raise HTTPException(400, f"Could not decode audio: {exc}") from exc
|
||||
if samples.ndim > 1:
|
||||
samples = samples.mean(axis=1).astype(np.float32)
|
||||
|
||||
stream = recognizer.create_stream()
|
||||
stream.accept_waveform(sample_rate, samples)
|
||||
recognizer.decode_stream(stream)
|
||||
return stream.result.text
|
||||
|
||||
|
||||
@app.get("/healthz")
|
||||
def healthz() -> dict[str, str]:
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
def _timed(raw: bytes) -> JSONResponse:
|
||||
t0 = time.perf_counter()
|
||||
text = _decode(raw)
|
||||
if AB_DELAY_S:
|
||||
time.sleep(AB_DELAY_S) # positive control: blocks the loop exactly as a slower decode would
|
||||
ms = (time.perf_counter() - t0) * 1000.0
|
||||
return JSONResponse({"text": text}, headers={"x-ab-decode-ms": f"{ms:.3f}"})
|
||||
|
||||
|
||||
@app.post("/transcribe")
|
||||
async def transcribe(file: UploadFile = File(...)):
|
||||
return _timed(await file.read())
|
||||
|
||||
|
||||
@app.post("/v1/audio/transcriptions")
|
||||
async def openai_transcriptions(file: UploadFile = File(...)):
|
||||
return _timed(await file.read())
|
||||
|
||||
Executable
+40
@@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env bash
|
||||
# Start one A/B arm as a transient container on fv-ml1 GPU 3 (never 0), bound to 127.0.0.1 only.
|
||||
# Prints: <name> <port> <t_started_epoch> <t_ready_epoch> <cold_s> once /healthz answers.
|
||||
#
|
||||
# arm.sh sherpa NAME PORT MODEL_DIR [NUM_THREADS=1] [SUFFIX=int8.onnx] [DELAY_MS=0] [PLAIN=0]
|
||||
# The SEAT'S OWN IMAGE (local/parakeet:sherpa-onnx-v4) with the seat's env (PROVIDER=cuda). The
|
||||
# harness app_timed.py is mounted over /app/app.py unless PLAIN=1 (image's app untouched).
|
||||
# arm.sh nemo NAME PORT DTYPE PATH_MODE [CUDA_GRAPHS=1] [LOAD_CPU=0] [LOCAL_ATT=]
|
||||
# parakeet-unified-en-0.6b (pinned fe53cd88) under NeMo 3.0.0 torch, serve_nemo.py.
|
||||
set -euo pipefail
|
||||
AB=/tank/spikes/parakeet-ab
|
||||
kind=$1 name=$2 port=$3
|
||||
common=(--rm -d --name "$name" --gpus '"device=3"' --label ab=parakeet-2026-09-30 -p 127.0.0.1:$port:8000)
|
||||
if [ "$kind" = sherpa ]; then
|
||||
mdir=$4 nt=${5:-1} suf=${6:-int8.onnx} delay=${7:-0} plain=${8:-0}
|
||||
app=(-v "$AB/code/app_timed.py:/app/app.py:ro"); [ "$plain" = 1 ] && app=()
|
||||
# The seat's entrypoint.sh only knows the *.int8.onnx names (and would try to download into the
|
||||
# read-only mount), so a non-int8 export skips it and execs the same final command it would.
|
||||
ep=(); cmd=(); [ "$suf" != int8.onnx ] && { ep=(--entrypoint /opt/venv/bin/uvicorn); cmd=(app:app --host 0.0.0.0 --port 8000); }
|
||||
docker run "${common[@]}" -e MODEL_DIR=/models -e PROVIDER=cuda -e NUM_THREADS=$nt -e LOG_LEVEL=INFO \
|
||||
-e AB_SUFFIX=$suf -e AB_DELAY_MS=$delay "${app[@]}" -v "$mdir":/models:ro "${ep[@]}" local/parakeet:sherpa-onnx-v4 "${cmd[@]}" >/dev/null
|
||||
elif [ "$kind" = nemo ]; then
|
||||
dt=$4 pm=$5 cg=${6:-1} lc=${7:-0}
|
||||
docker run "${common[@]}" --user 1002:1003 -e HOME=/tmp -e USER=infra-ops -e LOGNAME=infra-ops -e HF_HUB_OFFLINE=1 \
|
||||
-e MODEL_PATH=/hf/hub/models--nvidia--parakeet-unified-en-0.6b/snapshots/fe53cd885760c96b6a5f51a0bfd362cb4584a98b/parakeet-unified-en-0.6b.nemo \
|
||||
-e DTYPE=$dt -e PATH_MODE=$pm -e CUDA_GRAPHS=$cg -e LOAD_CPU=$lc -e LOCAL_ATT=${8:-} \
|
||||
-v /tank/aimodels/huggingface:/hf:ro -v $AB:/ab:ro -v $AB/tmp:/tmp \
|
||||
--entrypoint /ab/envs/nemo300-serve/bin/python scriberr:local-blackwell \
|
||||
-m uvicorn serve_nemo:app --app-dir /ab/code --host 0.0.0.0 --port 8000 >/dev/null
|
||||
else
|
||||
echo "usage: arm.sh sherpa|nemo ..." >&2; exit 2
|
||||
fi
|
||||
for i in $(seq 1 2400); do
|
||||
curl -sf -m 2 http://127.0.0.1:$port/healthz >/dev/null 2>&1 && break
|
||||
docker inspect "$name" >/dev/null 2>&1 || { echo "$name died" >&2; exit 1; }
|
||||
sleep 0.25
|
||||
done
|
||||
t1=$(date +%s.%N)
|
||||
t0=$(date -d "$(docker inspect -f '{{.State.StartedAt}}' "$name")" +%s.%N)
|
||||
echo "$name $port $t0 $t1 $(python3 -c "print(round($t1-$t0,1))")"
|
||||
+186
@@ -0,0 +1,186 @@
|
||||
#!/usr/bin/env python3
|
||||
"""A/B client. Stdlib only. One NEW HTTP connection per request (what talk's httpx.AsyncClient-per-call does).
|
||||
|
||||
bench.py lat URL ARM MANIFEST OUT --rounds R [--seed S] [--pause SEC] [--auth KEY] [--bins b1_3,b3_8]
|
||||
R rounds; each round sends every clip once, in a seeded shuffled order. Single stream.
|
||||
bench.py conc URL ARM MANIFEST OUT --per-bin N [--workers 4] [--seed S]
|
||||
Per bin: N requests drawn from that bin's clips, W concurrent workers, back to back.
|
||||
bench.py acc URL ARM MANIFEST OUT
|
||||
Every manifest row once, in manifest order; stores the text.
|
||||
Rows (jsonl): arm, mode, round, id, bin, dur, e2e_ms, server_ms (x-ab-decode-ms header; null on the live seat),
|
||||
status, text (acc/lat), t_wall (unix), gpu_snapshot (lat: per round, nvidia-smi compute apps).
|
||||
"""
|
||||
import argparse
|
||||
import sys
|
||||
import http.client
|
||||
import json
|
||||
import random
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
import uuid
|
||||
|
||||
|
||||
def multipart(wav: bytes):
|
||||
b = uuid.uuid4().hex
|
||||
body = (f"--{b}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\next-stt\r\n"
|
||||
f"--{b}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"turn.wav\"\r\n"
|
||||
f"Content-Type: audio/wav\r\n\r\n").encode() + wav + f"\r\n--{b}--\r\n".encode()
|
||||
return body, f"multipart/form-data; boundary={b}"
|
||||
|
||||
|
||||
def post(url, wav, auth=None, timeout=3600):
|
||||
u = urllib.parse.urlparse(url)
|
||||
body, ctype = multipart(wav)
|
||||
hdr = {"Content-Type": ctype, "Content-Length": str(len(body))}
|
||||
if auth:
|
||||
hdr["Authorization"] = f"Bearer {auth}"
|
||||
t0 = time.perf_counter()
|
||||
c = http.client.HTTPConnection(u.hostname, u.port or 80, timeout=timeout)
|
||||
try:
|
||||
c.request("POST", u.path, body=body, headers=hdr)
|
||||
r = c.getresponse()
|
||||
data = r.read()
|
||||
t1 = time.perf_counter()
|
||||
sm = r.getheader("x-ab-decode-ms")
|
||||
status = r.status
|
||||
finally:
|
||||
c.close()
|
||||
text = None
|
||||
try:
|
||||
text = json.loads(data).get("text")
|
||||
except Exception:
|
||||
text = None
|
||||
return dict(e2e_ms=round((t1 - t0) * 1000, 3), server_ms=float(sm) if sm else None, status=status, text=text,
|
||||
err=None if status == 200 else data[:300].decode("utf-8", "replace"))
|
||||
|
||||
|
||||
def gpu_snapshot():
|
||||
try:
|
||||
out = subprocess.run(["nvidia-smi", "--query-compute-apps=gpu_uuid,pid,used_memory", "--format=csv,noheader,nounits"],
|
||||
capture_output=True, text=True, timeout=20).stdout
|
||||
return [l.strip() for l in out.splitlines() if l.strip()]
|
||||
except Exception as e: # noqa: BLE001
|
||||
return [f"err {e}"]
|
||||
|
||||
|
||||
def trimmed(wav_bytes, trim_ms):
|
||||
"""Same clip, `trim_ms` shorter at the tail: a never-seen input length (production utterances are all
|
||||
unique lengths, and ORT / NeMo cache per shape). Built BEFORE timing starts."""
|
||||
import io, wave
|
||||
if not trim_ms:
|
||||
return wav_bytes
|
||||
with wave.open(io.BytesIO(wav_bytes)) as w:
|
||||
p = w.getparams()
|
||||
fr = w.readframes(p.nframes)
|
||||
keep = max(1, p.nframes - int(p.framerate * trim_ms / 1000)) * p.sampwidth * p.nchannels
|
||||
bio = io.BytesIO()
|
||||
with wave.open(bio, "wb") as o:
|
||||
o.setnchannels(p.nchannels); o.setsampwidth(p.sampwidth); o.setframerate(p.framerate)
|
||||
o.writeframes(fr[:keep])
|
||||
return bio.getvalue()
|
||||
|
||||
|
||||
def jitter_ms(seed, k, uid, max_ms):
|
||||
return 0 if not max_ms else random.Random(f"{seed}-{k}-{uid}").randint(0, max_ms // 10) * 10
|
||||
|
||||
|
||||
def hostpath(p):
|
||||
"""Manifests carry the container path (/ab/...); the client may run on the host."""
|
||||
import os
|
||||
return p if os.path.exists(p) else p.replace("/ab/", "/tank/spikes/parakeet-ab/", 1)
|
||||
|
||||
|
||||
def load(manifest, bins=None):
|
||||
rows = [json.loads(l) for l in open(manifest)]
|
||||
if bins:
|
||||
rows = [r for r in rows if r.get("bin") in bins]
|
||||
for r in rows:
|
||||
r["_wav"] = open(hostpath(r["wav"]), "rb").read()
|
||||
return rows
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("mode")
|
||||
ap.add_argument("url")
|
||||
ap.add_argument("arm")
|
||||
ap.add_argument("manifest")
|
||||
ap.add_argument("out")
|
||||
ap.add_argument("--rounds", type=int, default=1)
|
||||
ap.add_argument("--round-offset", type=int, default=0)
|
||||
ap.add_argument("--seed", type=int, default=20260930)
|
||||
ap.add_argument("--pause", type=float, default=0.0)
|
||||
ap.add_argument("--auth")
|
||||
ap.add_argument("--bins")
|
||||
ap.add_argument("--per-bin", type=int, default=40)
|
||||
ap.add_argument("--workers", type=int, default=4)
|
||||
ap.add_argument("--stop-on-error", action="store_true", help="abort at the first non-200 (used on the live seat)")
|
||||
ap.add_argument("--jitter-ms", type=int, default=0, help="trim a seeded 0..N ms (10 ms steps) off each request's tail")
|
||||
a = ap.parse_args()
|
||||
rows = load(a.manifest, a.bins.split(",") if a.bins else None)
|
||||
out = open(a.out, "a")
|
||||
|
||||
def emit(d):
|
||||
out.write(json.dumps(d) + "\n")
|
||||
out.flush()
|
||||
|
||||
if a.mode == "acc":
|
||||
for r in rows:
|
||||
res = post(a.url, r["_wav"], a.auth)
|
||||
emit(dict(arm=a.arm, mode="acc", id=r["id"], dur=r["dur"], t_wall=time.time(), **res))
|
||||
elif a.mode == "lat":
|
||||
for k in range(a.round_offset, a.round_offset + a.rounds):
|
||||
order = list(range(len(rows)))
|
||||
random.Random(a.seed * 1000 + k).shuffle(order)
|
||||
jit = {i: jitter_ms(a.seed, k, rows[i]["id"], a.jitter_ms) for i in order}
|
||||
body = {i: trimmed(rows[i]["_wav"], jit[i]) for i in order}
|
||||
snap0 = gpu_snapshot()
|
||||
for i in order:
|
||||
r = rows[i]
|
||||
res = post(a.url, body[i], a.auth)
|
||||
res["trim_ms"] = jit[i]
|
||||
res.pop("err", None) if res["status"] == 200 else None
|
||||
emit(dict(arm=a.arm, mode="lat", round=k, id=r["id"], bin=r.get("bin"), dur=r["dur"], t_wall=time.time(), **res))
|
||||
if a.stop_on_error and res["status"] != 200:
|
||||
print("STOP: non-200 from", a.url, res.get("err"), flush=True)
|
||||
sys.exit(3)
|
||||
if a.pause:
|
||||
time.sleep(a.pause)
|
||||
emit(dict(arm=a.arm, mode="lat-round", round=k, gpu_start=snap0, gpu_end=gpu_snapshot(), t_wall=time.time()))
|
||||
elif a.mode == "conc":
|
||||
for tag in sorted({r["bin"] for r in rows}, key=lambda t: int(t[1:].split("_")[0])):
|
||||
pool = [r for r in rows if r["bin"] == tag]
|
||||
rng = random.Random(a.seed)
|
||||
queue = []
|
||||
for q in range(a.per_bin):
|
||||
r = rng.choice(pool)
|
||||
t = jitter_ms(a.seed, 10_000 + q, r["id"], a.jitter_ms)
|
||||
queue.append(dict(r, _wav=trimmed(r["_wav"], t), _trim=t))
|
||||
lock = threading.Lock()
|
||||
snap0 = gpu_snapshot()
|
||||
|
||||
def worker(w):
|
||||
while True:
|
||||
with lock:
|
||||
if not queue:
|
||||
return
|
||||
r = queue.pop()
|
||||
res = post(a.url, r["_wav"], a.auth)
|
||||
res.pop("text", None)
|
||||
with lock:
|
||||
emit(dict(arm=a.arm, mode="conc", workers=a.workers, worker=w, id=r["id"], bin=tag, dur=r["dur"],
|
||||
trim_ms=r["_trim"], t_wall=time.time(), **res))
|
||||
|
||||
t0 = time.perf_counter()
|
||||
ts = [threading.Thread(target=worker, args=(w,)) for w in range(a.workers)]
|
||||
[t.start() for t in ts]
|
||||
[t.join() for t in ts]
|
||||
emit(dict(arm=a.arm, mode="conc-bin", bin=tag, workers=a.workers, n=a.per_bin,
|
||||
wall_s=round(time.perf_counter() - t0, 3), gpu_start=snap0, gpu_end=gpu_snapshot()))
|
||||
out.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,23 @@
|
||||
"""fp16 copy of an fp32 sherpa-onnx transducer export: onnxconverter-common float16, keep_io_types=True
|
||||
(inputs/outputs stay fp32, so sherpa-onnx feeds it exactly as before); metadata carried over.
|
||||
Shape inference runs first BY PATH (infer_shapes_path handles the >2 GB fp32 encoder), so the converter
|
||||
sees every intermediate type; without it a scalar Mul in pre_encode is left fp32 and the graph won't load.
|
||||
usage: convert_fp16.py SRC_DIR DST_DIR"""
|
||||
import os, shutil, sys, tempfile
|
||||
import onnx
|
||||
from onnx.shape_inference import infer_shapes_path
|
||||
from onnxconverter_common import float16
|
||||
src, dst = sys.argv[1:3]
|
||||
os.makedirs(dst, exist_ok=True)
|
||||
for m in ("encoder", "decoder", "joiner"):
|
||||
inferred = f"{src}/{m}.inferred.onnx"
|
||||
infer_shapes_path(f"{src}/{m}.onnx", inferred)
|
||||
model = onnx.load(inferred)
|
||||
# the conv subsampling front (pre_encode, ~0.1 % of the FLOPs) stays fp32: the converter mis-types its
|
||||
# length-mask Cast/Mul otherwise
|
||||
keep32 = [n.name for n in model.graph.node if n.name.startswith("/pre_encode/")]
|
||||
m16 = float16.convert_float_to_float16(model, keep_io_types=True, disable_shape_infer=True, node_block_list=keep32)
|
||||
onnx.save(m16, f"{dst}/{m}.fp16.onnx")
|
||||
os.remove(inferred)
|
||||
shutil.copy(f"{src}/tokens.txt", f"{dst}/tokens.txt")
|
||||
print("ok", dst)
|
||||
@@ -0,0 +1,15 @@
|
||||
"""CPU seconds the server process burns per request vs the request's wall time (host /proc; Linux CLK_TCK=100)."""
|
||||
import json, os, sys, time, subprocess
|
||||
sys.path.insert(0, "/tank/spikes/parakeet-ab/code")
|
||||
from bench import post, hostpath
|
||||
url, pid = sys.argv[1], int(sys.argv[2])
|
||||
ids = sys.argv[3].split(",")
|
||||
rows = {json.loads(l)["id"]: json.loads(l) for l in open("/tank/spikes/parakeet-ab/data/lat.jsonl")}
|
||||
def cpu(p):
|
||||
f = open(f"/proc/{p}/stat").read().rsplit(")", 1)[1].split()
|
||||
return (int(f[11]) + int(f[12])) / os.sysconf("SC_CLK_TCK")
|
||||
def threads(p): return len(os.listdir(f"/proc/{p}/task"))
|
||||
for i in ids:
|
||||
wav = open(hostpath(rows[i]["wav"]), "rb").read()
|
||||
c0 = cpu(pid); r = post(url, wav); c1 = cpu(pid)
|
||||
print(f"{i:10s} dur {rows[i]['dur']:6.2f}s server {r['server_ms']:8.1f} ms process CPU {1000*(c1-c0):8.1f} ms cpu/wall {(c1-c0)*1000/r['server_ms']:.2f} threads {threads(pid)}", flush=True)
|
||||
@@ -0,0 +1,67 @@
|
||||
"""fp32 ONNX export of a NeMo transducer for sherpa-onnx, reproducing k2-fsa's own export verbatim
|
||||
(k2-fsa/sherpa-onnx @040afe36, scripts/nemo/{parakeet-tdt-0.6b-v3,parakeet-unified-en-0.6b}/export_onnx.py):
|
||||
same encoder/decoder/joint .export() calls, same tokens.txt, same metadata, encoder weights as external data.
|
||||
The ONLY omission is their final quantize_dynamic() step: the point is the non-int8 graph on the CUDA EP.
|
||||
usage: export_onnx.py NEMO_PATH OUT_DIR URL COMMENT
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
|
||||
import onnx
|
||||
import torch
|
||||
import nemo.collections.asr as nemo_asr
|
||||
|
||||
|
||||
def add_meta_data(filename, meta_data):
|
||||
model = onnx.load(filename)
|
||||
while len(model.metadata_props):
|
||||
model.metadata_props.pop()
|
||||
for key, value in meta_data.items():
|
||||
meta = model.metadata_props.add()
|
||||
meta.key = key
|
||||
meta.value = str(value)
|
||||
if os.path.basename(filename) == "encoder.onnx":
|
||||
onnx.save(model, filename, save_as_external_data=True, all_tensors_to_one_file=True, location="encoder.weights")
|
||||
else:
|
||||
onnx.save(model, filename)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def main():
|
||||
nemo_path, out, url, comment = sys.argv[1:5]
|
||||
os.makedirs(out, exist_ok=True)
|
||||
os.chdir(out)
|
||||
m = nemo_asr.models.ASRModel.restore_from(restore_path=nemo_path, map_location="cpu")
|
||||
m.eval()
|
||||
if m.cfg.get("validation_ds") is None:
|
||||
m.cfg.validation_ds = dict()
|
||||
with open("./tokens.txt", "w", encoding="utf-8") as f:
|
||||
for i, s in enumerate(m.joint.vocabulary):
|
||||
f.write(f"{s} {i}\n")
|
||||
f.write(f"<blk> {i+1}\n")
|
||||
m.encoder.export("encoder.onnx")
|
||||
m.decoder.export("decoder.onnx")
|
||||
m.joint.export("joiner.onnx")
|
||||
normalize_type = m.cfg.preprocessor.normalize
|
||||
if normalize_type == "NA":
|
||||
normalize_type = ""
|
||||
meta = {
|
||||
"vocab_size": m.decoder.vocab_size,
|
||||
"normalize_type": normalize_type,
|
||||
"pred_rnn_layers": m.decoder.pred_rnn_layers,
|
||||
"pred_hidden": m.decoder.pred_hidden,
|
||||
"subsampling_factor": m.encoder.subsampling_factor,
|
||||
"model_type": "EncDecRNNTBPEModel",
|
||||
"version": "2",
|
||||
"model_author": "NeMo",
|
||||
"url": url,
|
||||
"comment": comment,
|
||||
"feat_dim": 128,
|
||||
}
|
||||
add_meta_data("encoder.onnx", meta)
|
||||
print("meta", meta)
|
||||
os.system("ls -la")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Fetch pinned artifacts; verify each against its published digest. Prints a provenance line per file."""
|
||||
import hashlib, json, os, sys, urllib.request
|
||||
TOKEN = open("/tank/aimodels/huggingface/token").read().strip()
|
||||
DL = "/tank/spikes/parakeet-ab/dl"
|
||||
def sha256(p):
|
||||
h = hashlib.sha256()
|
||||
with open(p, "rb") as f:
|
||||
for b in iter(lambda: f.read(1 << 24), b""): h.update(b)
|
||||
return h.hexdigest()
|
||||
def fetch(url, out, auth=False):
|
||||
if os.path.exists(out): return
|
||||
req = urllib.request.Request(url, headers={"Authorization": f"Bearer {TOKEN}"} if auth else {})
|
||||
with urllib.request.urlopen(req, timeout=120) as r, open(out + ".part", "wb") as f:
|
||||
while True:
|
||||
b = r.read(1 << 22)
|
||||
if not b: break
|
||||
f.write(b)
|
||||
os.rename(out + ".part", out)
|
||||
prov = []
|
||||
# GitHub release assets (digest from the GitHub API, recorded 2026-09-30)
|
||||
gh = {"sherpa-onnx-nemo-parakeet-unified-en-0.6b-int8-non-streaming.tar.bz2": "99f63605b3a85a54c250c0869670a687b7d6598a47bf2421515e1f839a76e150",
|
||||
"sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8.tar.bz2": "157c157bc51155e03e37d2466522a3a737dd9c72bb25f36eb18912964161e1ad"}
|
||||
for name, want in gh.items():
|
||||
out = f"{DL}/{name}"
|
||||
fetch(f"https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/{name}", out)
|
||||
got = sha256(out)
|
||||
prov.append(dict(src=f"github:k2-fsa/sherpa-onnx@asr-models/{name}", sha256=got, expected=want, ok=got == want))
|
||||
# HF datasets, revision-pinned; expected sha = LFS oid from the authenticated API
|
||||
hf = [("openslr/librispeech_asr", "71cacbfb7e2354c4226d01e70d77d5fca3d04ba1", ["all/test.clean/0000.parquet", "all/test.other/0000.parquet"]),
|
||||
("edinburghcstr/ami", "46f28f2503e2ec48f8867a84eef356c70476beab", [f"ihm/test-0000{i}-of-00004.parquet" for i in range(4)])]
|
||||
for repo, rev, files in hf:
|
||||
req = urllib.request.Request(f"https://huggingface.co/api/datasets/{repo}/revision/{rev}?blobs=true", headers={"Authorization": f"Bearer {TOKEN}"})
|
||||
meta = json.load(urllib.request.urlopen(req, timeout=60))
|
||||
oid = {s["rfilename"]: (s.get("lfs") or {}).get("sha256") for s in meta["siblings"]}
|
||||
for fn in files:
|
||||
out = f"{DL}/{repo.replace('/', '__')}__{fn.replace('/', '__')}"
|
||||
fetch(f"https://huggingface.co/datasets/{repo}/resolve/{rev}/{fn}", out, auth=True)
|
||||
got = sha256(out)
|
||||
prov.append(dict(src=f"hf-dataset:{repo}@{rev}/{fn}", sha256=got, expected=oid.get(fn), ok=got == oid.get(fn)))
|
||||
for p in prov: print(json.dumps(p))
|
||||
json.dump(prov, open("/tank/spikes/parakeet-ab/out/provenance-fetch.json", "w"), indent=1)
|
||||
@@ -0,0 +1,115 @@
|
||||
"""Score transcripts against ground truth (or any reference) and find dropped stretches.
|
||||
|
||||
Both sides are normalised word by word with Whisper's English normaliser
|
||||
(transformers 4.53.3, no spelling map), so a hypothesis token keeps the time of
|
||||
the word it came from. Alignment: difflib opcodes, then exact Levenshtein inside
|
||||
each non-matching block.
|
||||
|
||||
A GAP is a maximal stretch between matching blocks of >= ISLAND tokens (shorter
|
||||
matching islands inside a mismatch are treated as part of it, so one spurious
|
||||
"the" cannot split a skipped paragraph in two). A gap is a
|
||||
DROPOUT if it holds >= RUN reference words and the hypothesis emitted fewer
|
||||
than half as many there (the hypothesis skipped speech), and an
|
||||
INSERTION run if the mirror holds (the hypothesis added >= RUN words the
|
||||
reference does not have).
|
||||
Prints counts only.
|
||||
"""
|
||||
import difflib
|
||||
import json
|
||||
import sys
|
||||
|
||||
from transformers.models.whisper.english_normalizer import EnglishTextNormalizer
|
||||
|
||||
RUN = 10
|
||||
ISLAND = 3
|
||||
_norm = EnglishTextNormalizer({})
|
||||
|
||||
|
||||
def tokens_from_text(text):
|
||||
return [t for w in text.split() for t in _norm(w).split()]
|
||||
|
||||
|
||||
def tokens_from_words(words):
|
||||
"""[(token, start, end)] from word dicts, one entry per normalised token."""
|
||||
out = []
|
||||
for w in words:
|
||||
for t in _norm(w["word"]).split():
|
||||
out.append((t, float(w["start"]), float(w["end"])))
|
||||
return out
|
||||
|
||||
|
||||
def _edit_counts(a, b):
|
||||
n, m = len(a), len(b)
|
||||
d = [[0] * (m + 1) for _ in range(n + 1)]
|
||||
for i in range(n + 1):
|
||||
d[i][0] = i
|
||||
for j in range(m + 1):
|
||||
d[0][j] = j
|
||||
for i in range(1, n + 1):
|
||||
for j in range(1, m + 1):
|
||||
d[i][j] = min(d[i - 1][j] + 1, d[i][j - 1] + 1, d[i - 1][j - 1] + (a[i - 1] != b[j - 1]))
|
||||
s = i_ = de = 0
|
||||
i, j = n, m
|
||||
while i > 0 or j > 0:
|
||||
if i > 0 and j > 0 and d[i][j] == d[i - 1][j - 1] + (a[i - 1] != b[j - 1]):
|
||||
s += a[i - 1] != b[j - 1]
|
||||
i, j = i - 1, j - 1
|
||||
elif i > 0 and d[i][j] == d[i - 1][j] + 1:
|
||||
de += 1
|
||||
i -= 1
|
||||
else:
|
||||
i_ += 1
|
||||
j -= 1
|
||||
return s, i_, de
|
||||
|
||||
|
||||
def score(ref, hyp, hyp_times=None):
|
||||
"""ref, hyp: token lists. hyp_times: [(start, end)] per hyp token, optional."""
|
||||
sm = difflib.SequenceMatcher(None, ref, hyp, autojunk=False)
|
||||
ops = sm.get_opcodes()
|
||||
S = I = D = 0
|
||||
for tag, i1, i2, j1, j2 in ops:
|
||||
if tag != "equal":
|
||||
s, i_, de = _edit_counts(ref[i1:i2], hyp[j1:j2])
|
||||
S, I, D = S + s, I + i_, D + de
|
||||
# gaps between solid matching blocks
|
||||
solid = [(i1, i2, j1, j2) for tag, i1, i2, j1, j2 in ops if tag == "equal" and i2 - i1 >= ISLAND]
|
||||
bounds = [(0, 0, 0, 0)] + solid + [(len(ref), len(ref), len(hyp), len(hyp))]
|
||||
drops, ins = [], []
|
||||
for (_, a_i2, _, a_j2), (b_i1, _, b_j1, _) in zip(bounds, bounds[1:]):
|
||||
gl, hl = b_i1 - a_i2, b_j1 - a_j2
|
||||
t0 = t1 = None
|
||||
if hyp_times:
|
||||
t0 = hyp_times[a_j2 - 1][1] if a_j2 > 0 else 0.0
|
||||
t1 = hyp_times[b_j1][0] if b_j1 < len(hyp) else hyp_times[-1][1]
|
||||
gap = dict(ref_start=a_i2, ref_words=gl, hyp_start=a_j2, hyp_words=hl, t0=t0, t1=t1)
|
||||
if gl >= RUN and hl < 0.5 * gl:
|
||||
drops.append(gap)
|
||||
elif hl >= RUN and gl < 0.5 * hl:
|
||||
ins.append(gap)
|
||||
n = len(ref)
|
||||
return dict(ref_words=n, hyp_words=len(hyp), S=S, I=I, D=D,
|
||||
wer=round((S + I + D) / n, 4) if n else None,
|
||||
dropouts=len(drops), dropout_words=sum(g["ref_words"] for g in drops),
|
||||
insertion_runs=len(ins), insertion_words=sum(g["hyp_words"] for g in ins),
|
||||
drop_gaps=drops, ins_gaps=ins)
|
||||
|
||||
|
||||
def load_hyp(path):
|
||||
doc = json.load(open(path))
|
||||
if "word_timestamps" in doc and doc["word_timestamps"]:
|
||||
toks = tokens_from_words(doc["word_timestamps"])
|
||||
return [t for t, _, _ in toks], [(s, e) for _, s, e in toks]
|
||||
return tokens_from_text(doc.get("transcription") or doc.get("text", "")), None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
ref_path, hyp_path = sys.argv[1], sys.argv[2]
|
||||
ref = tokens_from_text(open(ref_path).read())
|
||||
hyp, times = load_hyp(hyp_path)
|
||||
r = score(ref, hyp, times)
|
||||
print(json.dumps({k: v for k, v in r.items() if not k.endswith("_gaps")}))
|
||||
for g in r["drop_gaps"]:
|
||||
print(" DROP", g)
|
||||
for g in r["ins_gaps"]:
|
||||
print(" INS ", g)
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Live seat (A), gateway and cross-site passes, paired against the GPU-3 arms on identical trimmed input.
|
||||
A-live: fv-ml1 loopback to :8300, rounds 0-1. A-gateway: nh3-dev -> LiteLLM ext-stt (ana-docker) -> :8300,
|
||||
round 0. A-nh3direct: nh3-dev -> :8300, round 0. 1-20 s bins only (the live seat never got 20-60 s clips).
|
||||
usage: gw_analysis.py RAW_DIR"""
|
||||
import json
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
|
||||
RAW = sys.argv[1]
|
||||
rows = [json.loads(l) for f in ("lat-live.jsonl", "lat-gw.jsonl", "lat.jsonl") for l in open(f"{RAW}/{f}")]
|
||||
rows = [r for r in rows if r.get("mode") == "lat" and r["status"] == 200]
|
||||
by = {}
|
||||
for r in rows:
|
||||
by.setdefault(r["arm"], {})[(r["round"], r["id"])] = r
|
||||
rng = np.random.default_rng(7)
|
||||
|
||||
|
||||
def ci(d):
|
||||
d = np.asarray(d)
|
||||
b = [np.median(d[rng.integers(0, len(d), len(d))]) for _ in range(2000)]
|
||||
return np.percentile(b, [2.5, 97.5])
|
||||
|
||||
|
||||
BINS = ("b1_3", "b3_8", "b8_20")
|
||||
for arm in ("A-live", "A-gateway", "A-nh3direct"):
|
||||
for b in BINS:
|
||||
x = [r["e2e_ms"] for r in by[arm].values() if r["bin"] == b]
|
||||
print(f"{arm:12s} {b:6s} n={len(x):3d} e2e p50 {np.median(x):7.1f} p90 {np.percentile(x, 90):7.1f} max {max(x):7.1f}")
|
||||
print("paired: median diff [95% CI]")
|
||||
for a, ref in (("A-live", "ab-a1"), ("A-gateway", "A-live"), ("A-nh3direct", "A-live"), ("A-gateway", "A-nh3direct"),
|
||||
("A-gateway", "ab-b32"), ("A-live", "ab-b32")):
|
||||
for b in BINS:
|
||||
ks = [k for k in by[a] if k in by[ref] and by[a][k]["bin"] == b]
|
||||
assert all(by[a][k].get("trim_ms") == by[ref][k].get("trim_ms") for k in ks)
|
||||
d = [by[a][k]["e2e_ms"] - by[ref][k]["e2e_ms"] for k in ks]
|
||||
lo, hi = ci(d)
|
||||
print(f"{a:12s} - {ref:12s} {b:6s} pairs={len(ks):3d} {np.median(d):+8.1f} [{lo:+7.1f},{hi:+7.1f}]")
|
||||
ks = [k for k in by["A-live"] if k in by["ab-a1"]]
|
||||
print("A-live vs ab-a1 identical text:", sum(by["A-live"][k]["text"] == by["ab-a1"][k]["text"] for k in ks), "/", len(ks))
|
||||
ks = [k for k in by["A-gateway"] if k in by["A-live"]]
|
||||
print("A-gateway vs A-live identical text:", sum(by["A-gateway"][k]["text"] == by["A-live"][k]["text"] for k in ks), "/", len(ks))
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Memory a FRESH seat-config instance (A') needs per input length: ascending lengths, one request each,
|
||||
GPU memory of its PID read 2 s after each response. ORT's arena never shrinks, so the reading after a
|
||||
request is the high-water mark so far. Prefixes of the public SCOTUS audio (16 kHz mono).
|
||||
usage: len_sweep.py ARM PORT SECONDS..."""
|
||||
import io
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
import soundfile as sf
|
||||
|
||||
sys.path.insert(0, "/tank/spikes/parakeet-ab/code")
|
||||
from bench import post # noqa: E402
|
||||
|
||||
arm, port = sys.argv[1], sys.argv[2]
|
||||
lens = [float(x) for x in sys.argv[3:]]
|
||||
pids = set(subprocess.run(["docker", "top", arm, "-eo", "pid"], capture_output=True, text=True).stdout.split()[1:])
|
||||
a, sr = sf.read("/tank/spikes/scriberr-slicer/public/scotus.wav", dtype="float32")
|
||||
|
||||
|
||||
def mem():
|
||||
out = subprocess.run(["nvidia-smi", "--query-compute-apps=pid,used_memory", "--format=csv,noheader,nounits"],
|
||||
capture_output=True, text=True).stdout
|
||||
return sum(int(l.split(",")[1]) for l in out.splitlines() if l.split(",")[0].strip() in pids)
|
||||
|
||||
|
||||
print(json.dumps(dict(arm=arm, seconds=0, status=None, mib=mem())), flush=True)
|
||||
for s in lens:
|
||||
bio = io.BytesIO()
|
||||
sf.write(bio, a[: int(s * sr)], sr, format="WAV", subtype="PCM_16")
|
||||
r = post(f"http://127.0.0.1:{port}/v1/audio/transcriptions", bio.getvalue(), timeout=1800)
|
||||
time.sleep(2)
|
||||
print(json.dumps(dict(arm=arm, seconds=s, status=r["status"], e2e_ms=r["e2e_ms"], mib=mem())), flush=True)
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
# Light pass on the LIVE seat (GPU 0): talk-shaped 1-20 s clips only, 1 s apart, abort on the first non-200.
|
||||
cd /tank/spikes/parakeet-ab
|
||||
st() { echo "$1 $(date +%T) restarts/started/health=$(docker inspect -f "{{.RestartCount}} {{.State.StartedAt}} {{.State.Health.Status}}" parakeet) seat=$(nvidia-smi --query-compute-apps=pid,used_memory --format=csv,noheader | grep 1594431) gpu0free=$(nvidia-smi -i 0 --query-gpu=memory.free --format=csv,noheader)"; }
|
||||
st BEFORE
|
||||
python3 code/bench.py lat http://127.0.0.1:8300/v1/audio/transcriptions A-live data/lat.jsonl out/raw/lat-live.jsonl --rounds 2 --round-offset 0 --jitter-ms 490 --pause 1.0 --bins b1_3,b3_8,b8_20 --stop-on-error
|
||||
echo "rc=$?"
|
||||
st AFTER
|
||||
echo LIVE PASS DONE
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
# Long-form for ONE arm, with a GPU 3 headroom guard for Scriberr (an on-demand tenant of GPU 3, ~5.5 GB
|
||||
# per job): a watcher kills the arm container if free memory on GPU 3 drops under 8 GiB.
|
||||
# usage: long_guarded.sh ARM PORT
|
||||
cd /tank/spikes/parakeet-ab
|
||||
arm=$1 port=$2
|
||||
( while docker inspect "$arm" >/dev/null 2>&1; do
|
||||
f=$(nvidia-smi -i 3 --query-gpu=memory.free --format=csv,noheader,nounits)
|
||||
if [ "$f" -lt 8192 ]; then echo "GUARD: GPU3 free ${f} MiB < 8192, stopping $arm" ; docker rm -f "$arm" >/dev/null; break; fi
|
||||
sleep 0.5
|
||||
done ) &
|
||||
w=$!
|
||||
echo "phase start $(date +%s.%N) $arm"
|
||||
python3 code/run_long.py "$arm:$port"
|
||||
echo "phase end $(date +%s.%N) $arm"
|
||||
kill $w 2>/dev/null
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Long-form scoring, identical to the dropout investigation's public-file method:
|
||||
gtscore.py (copied verbatim from /tank/spikes/scriberr-slicer/code/dropout/) against the timed ground truth,
|
||||
dropouts split into clean speech vs crosstalk with boot.py's rule (diarized overlap >= 10 % of the stretch
|
||||
= crosstalk). Our hypotheses are text-only (the seat returns no timestamps), so they are tokenised with
|
||||
gtscore.tokens_from_text; gap times come from the ground-truth token times.
|
||||
usage: longscore.py RAW_LONG_DIR -> json lines per (file, arm, k)
|
||||
"""
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from gtscore import score, tokens_from_text # noqa: E402
|
||||
|
||||
ROOT = "/tank/spikes/scriberr-slicer"
|
||||
|
||||
|
||||
def keeper(f):
|
||||
tg = json.load(open(f"{ROOT}/gt/{f}.timed.json"))
|
||||
gtt = tg["times"]
|
||||
segs = json.load(open(f"{ROOT}/public/{f}.diar.json"))["segments"]
|
||||
|
||||
def clean(g):
|
||||
i0, i1 = g["ref_start"], g["ref_start"] + g["ref_words"]
|
||||
t0, t1 = gtt[i0], (gtt[i1] if i1 < len(gtt) else gtt[-1] + 0.5)
|
||||
n = max(1, int((max(t1, t0 + 0.5) - t0) / 0.1))
|
||||
ov = sum(sum(1 for s in segs if s["start"] <= t0 + k * 0.1 < s["end"]) >= 2 for k in range(n)) / n
|
||||
return ov < 0.1, round(t0, 1), round(t1, 1)
|
||||
return tg["tokens"], clean
|
||||
|
||||
|
||||
for p in sorted(glob.glob(f"{sys.argv[1]}/*.json")):
|
||||
d = json.load(open(p))
|
||||
ref, clean = keeper(d["file"])
|
||||
if d.get("status") != 200 or d.get("text") is None:
|
||||
print(json.dumps(dict(file=d["file"], arm=d["arm"], k=d["k"], status=d.get("status"), err=d.get("err"))))
|
||||
continue
|
||||
r = score(ref, tokens_from_text(d["text"]))
|
||||
drops = [(g, *clean(g)) for g in r["drop_gaps"]]
|
||||
cl = [g for g, c, _, _ in drops if c]
|
||||
print(json.dumps(dict(file=d["file"], arm=d["arm"], k=d["k"], e2e_s=round(d["e2e_ms"] / 1000, 1),
|
||||
wer=round(100 * r["wer"], 2), S=r["S"], D=r["D"], I=r["I"], ref_words=r["ref_words"], hyp_words=r["hyp_words"],
|
||||
dropouts=len(drops), dropout_words=sum(g["ref_words"] for g, *_ in drops),
|
||||
clean_dropouts=len(cl), clean_dropout_words=sum(g["ref_words"] for g in cl),
|
||||
insertion_runs=r["insertion_runs"], insertion_words=r["insertion_words"],
|
||||
drop_spans_s=[[t0, t1, g["ref_words"], "clean" if c else "crosstalk"] for g, c, t0, t1 in drops])))
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Long-form inside the seat's envelope: every ~6-min window WHOLE, one request, per arm (arms one at a
|
||||
time; arms run in parallel threads only when told to). Then score each window with the investigation's
|
||||
gtscore.score against its slice of the timed ground truth, crosstalk split by boot.py's diarization rule.
|
||||
usage:
|
||||
longwin.py run ARM:PORT ... -> out/raw/longwin/<arm>.jsonl (text kept: public data)
|
||||
longwin.py score -> per arm: per-file WER (summed edits), dropouts, dropped words
|
||||
"""
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
AB = "/tank/spikes/parakeet-ab"
|
||||
ROOT = "/tank/spikes/scriberr-slicer"
|
||||
sys.path.insert(0, f"{AB}/code")
|
||||
WINS = os.environ.get("AB_WINS", "long") # long = placement 1, long2 = placement 2 (first cut at 180 s)
|
||||
OUTD = f"{AB}/out/raw/longwin" + ("" if WINS == "long" else "-" + WINS)
|
||||
os.makedirs(OUTD, exist_ok=True)
|
||||
wins = [json.loads(l) for l in open(f"{AB}/data/{WINS}.jsonl")]
|
||||
|
||||
if sys.argv[1] == "run":
|
||||
import threading
|
||||
from bench import post
|
||||
|
||||
def run(spec):
|
||||
arm, port = spec.split(":")
|
||||
with open(f"{OUTD}/{arm}.jsonl", "w") as fo:
|
||||
for w in wins:
|
||||
wav = open(w["wav"], "rb").read()
|
||||
t0 = time.time()
|
||||
try:
|
||||
res = post(f"http://127.0.0.1:{port}/v1/audio/transcriptions", wav, timeout=1800)
|
||||
except Exception as e: # noqa: BLE001
|
||||
res = dict(status=-1, err=repr(e)[:300], e2e_ms=None, server_ms=None, text=None)
|
||||
fo.write(json.dumps(dict(arm=arm, id=w["id"], t0=t0, t1=time.time(), **res)) + "\n")
|
||||
fo.flush()
|
||||
print(arm, w["id"], res.get("status"), res.get("e2e_ms"), flush=True)
|
||||
ts = [threading.Thread(target=run, args=(s,)) for s in sys.argv[2:]]
|
||||
[t.start() for t in ts]
|
||||
[t.join() for t in ts]
|
||||
else:
|
||||
from gtscore import score, tokens_from_text
|
||||
diar = {f: json.load(open(f"{ROOT}/public/{f}.diar.json"))["segments"] for f in ("wilde", "scotus")}
|
||||
gtt = {f: json.load(open(f"{ROOT}/gt/{f}.timed.json"))["times"] for f in ("wilde", "scotus")}
|
||||
for p in sorted(glob.glob(f"{OUTD}/*.jsonl")):
|
||||
res = {json.loads(l)["id"]: json.loads(l) for l in open(p)}
|
||||
arm = os.path.basename(p)[:-6]
|
||||
agg = {}
|
||||
for w in wins:
|
||||
r = res.get(w["id"])
|
||||
a = agg.setdefault(w["file"], dict(N=0, E=0, drops=0, dropw=0, clean=0, cleanw=0, ins=0, insw=0, fail=0, e2e=[]))
|
||||
if not r or r.get("status") != 200:
|
||||
a["fail"] += 1
|
||||
continue
|
||||
ref = w["ref_tokens"]
|
||||
s = score(ref, tokens_from_text(r["text"]))
|
||||
a["N"] += s["ref_words"]; a["E"] += s["S"] + s["D"] + s["I"]
|
||||
a["ins"] += s["insertion_runs"]; a["insw"] += s["insertion_words"]
|
||||
a["e2e"].append(r["e2e_ms"])
|
||||
# map window-relative token indices back to absolute GT times for the crosstalk rule
|
||||
times = [t for t in gtt[w["file"]] if w["t0"] <= t < w["t1"]]
|
||||
for g in s["drop_gaps"]:
|
||||
i0, i1 = g["ref_start"], g["ref_start"] + g["ref_words"]
|
||||
t0 = times[i0]
|
||||
t1 = times[i1] if i1 < len(times) else times[-1] + 0.5
|
||||
n = max(1, int((max(t1, t0 + 0.5) - t0) / 0.1))
|
||||
ov = sum(sum(1 for sg in diar[w["file"]] if sg["start"] <= t0 + k * 0.1 < sg["end"]) >= 2 for k in range(n)) / n
|
||||
a["drops"] += 1; a["dropw"] += g["ref_words"]
|
||||
if ov < 0.1:
|
||||
a["clean"] += 1; a["cleanw"] += g["ref_words"]
|
||||
for f, a in agg.items():
|
||||
print(json.dumps(dict(arm=arm, file=f, windows=sum(1 for w in wins if w["file"] == f), failed=a["fail"],
|
||||
wer=round(100 * a["E"] / a["N"], 2) if a["N"] else None, ref_words=a["N"],
|
||||
dropouts=a["drops"], dropout_words=a["dropw"], clean_dropouts=a["clean"],
|
||||
clean_dropout_words=a["cleanw"], insertion_runs=a["ins"], insertion_words=a["insw"],
|
||||
e2e_ms_median=sorted(a["e2e"])[len(a["e2e"]) // 2] if a["e2e"] else None)))
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
# Long-window run with a GPU 3 headroom guard (Scriberr is an on-demand tenant of GPU 3).
|
||||
cd /tank/spikes/parakeet-ab
|
||||
AB_WINS=${AB_WINS:-long} python3 code/longwin.py run "$@" > out/longwin-run.log 2>&1 &
|
||||
run=$!
|
||||
while kill -0 $run 2>/dev/null; do
|
||||
f=$(nvidia-smi -i 3 --query-gpu=memory.free --format=csv,noheader,nounits)
|
||||
if [ "$f" -lt 8192 ]; then
|
||||
echo "GUARD fired: GPU3 free $f MiB"; for c in $(docker ps --filter label=ab=parakeet-2026-09-30 --format "{{.Names}}"); do docker rm -f $c >/dev/null; done; break
|
||||
fi
|
||||
sleep 0.5
|
||||
done
|
||||
wait $run
|
||||
echo "LONGWIN DONE"
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Positive-control variants of the same 40 utterances: the 1.5 s gap filled with white noise at -60 and
|
||||
-50 dBFS RMS instead of digital zeros (a room-tone pause rather than a gated one), and a 0.75 s zero gap.
|
||||
Reference unchanged. Writes data/pc-n60.jsonl, data/pc-n50.jsonl, data/pc-z075.jsonl."""
|
||||
import json
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
|
||||
D = "/tank/spikes/parakeet-ab/data"
|
||||
pc = [json.loads(l) for l in open(f"{D}/pc.jsonl")]
|
||||
clean = {json.loads(l)["id"]: json.loads(l) for l in open(f"{D}/ls-clean.jsonl")}
|
||||
rng = np.random.default_rng(20260930)
|
||||
for tag, mode, level, span in (("pc-n60", "noise", -60, 1.5), ("pc-n50", "noise", -50, 1.5), ("pc-z075", "zero", None, 0.75)):
|
||||
os.makedirs(f"{D}/{tag}", exist_ok=True)
|
||||
rows = []
|
||||
for r in pc:
|
||||
a, sr = sf.read(clean[r["id"]]["wav"].replace("/ab/", "/tank/spikes/parakeet-ab/", 1), dtype="float32")
|
||||
s0 = int(0.40 * len(a)); s1 = s0 + int(span * sr)
|
||||
b = a.copy()
|
||||
b[s0:s1] = (rng.standard_normal(s1 - s0).astype(np.float32) * 10 ** (level / 20)) if mode == "noise" else 0.0
|
||||
p = f"{D}/{tag}/{r['id']}.wav"
|
||||
sf.write(p, b, sr, subtype="PCM_16")
|
||||
rows.append(dict(r, wav=p.replace("/tank/spikes/parakeet-ab/", "/ab/", 1), silence=[round(s0 / sr, 3), round(s1 / sr, 3)], fill=f"{mode} {level}"))
|
||||
with open(f"{D}/{tag}.jsonl", "w") as f:
|
||||
for r in rows:
|
||||
f.write(json.dumps(r) + "\n")
|
||||
print(tag, len(rows))
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Cut the two public long-form files into ~6-minute windows that the seat can actually take whole.
|
||||
|
||||
The seat's ONNX graph has a hard ceiling of 5,000 encoder frames (pos_emb_max_len 5000 -> a 9,999-wide
|
||||
relative-position table) = 400 s; anything longer returns HTTP 500. Windows: nominal boundaries every
|
||||
360 s, each moved to the widest gap between consecutive ground-truth word starts within +-10 s (a pause),
|
||||
cut at the middle of that gap; every window must be < 395 s. Ground truth per window = the timed GT tokens
|
||||
whose start time falls inside it (the investigation's gt/<file>.timed.json, unchanged).
|
||||
Writes data/long/<file>-w<k>.wav and data/long.jsonl {id, file, k, t0, t1, dur, wav, ref_tokens}.
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
|
||||
OFFSET = float(sys.argv[1]) if len(sys.argv) > 1 else 0.0 # first cut at OFFSET s (second placement)
|
||||
NAME = sys.argv[2] if len(sys.argv) > 2 else "long"
|
||||
import soundfile as sf
|
||||
|
||||
ROOT = "/tank/spikes/scriberr-slicer"
|
||||
OUT = "/tank/spikes/parakeet-ab/data"
|
||||
rows = []
|
||||
for f in ("wilde", "scotus"):
|
||||
tg = json.load(open(f"{ROOT}/gt/{f}.timed.json"))
|
||||
tok, tim = tg["tokens"], tg["times"]
|
||||
a, sr = sf.read(f"{ROOT}/public/{f}.wav", dtype="float32")
|
||||
total = len(a) / sr
|
||||
starts = np.asarray(tim, float)
|
||||
gaps = [(starts[i + 1] - starts[i], (starts[i] + starts[i + 1]) / 2) for i in range(len(starts) - 1)]
|
||||
cuts = [0.0]
|
||||
k = 1
|
||||
while total - cuts[-1] > 395:
|
||||
nominal = cuts[-1] + (OFFSET if (OFFSET and len(cuts) == 1) else 360)
|
||||
cand = [(g, m) for g, m in gaps if abs(m - nominal) <= 10]
|
||||
cut = max(cand)[1] if cand else nominal
|
||||
cuts.append(cut)
|
||||
k += 1
|
||||
cuts.append(total)
|
||||
for k, (t0, t1) in enumerate(zip(cuts, cuts[1:])):
|
||||
assert t1 - t0 < 395, (f, k, t1 - t0)
|
||||
seg = a[int(t0 * sr):int(t1 * sr)]
|
||||
wav = f"{OUT}/{NAME}/{f}-w{k}.wav"
|
||||
import os
|
||||
os.makedirs(f"{OUT}/{NAME}", exist_ok=True)
|
||||
sf.write(wav, seg, sr, subtype="PCM_16")
|
||||
ref = [t for t, s in zip(tok, tim) if t0 <= s < t1]
|
||||
rows.append(dict(id=f"{f}-w{k}", file=f, k=k, t0=round(t0, 2), t1=round(t1, 2), dur=round(t1 - t0, 2),
|
||||
wav=wav, ref_tokens=ref))
|
||||
print(f, k, round(t0, 1), round(t1, 1), "dur", round(t1 - t0, 1), "ref words", len(ref))
|
||||
with open(f"{OUT}/{NAME}.jsonl", "w") as fo:
|
||||
for r in rows:
|
||||
fo.write(json.dumps(r) + "\n")
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Trace one arm's GPU memory against the requests it had served: when did the arena grow, and after what?
|
||||
usage: memtrace.py ARM (reads out/raw/pids.json, mem.csv, first/lat/conc jsonl)"""
|
||||
import datetime, json, sys
|
||||
AB = "/tank/spikes/parakeet-ab"
|
||||
arm = sys.argv[1]
|
||||
pids = set(map(str, json.load(open(f"{AB}/out/raw/pids.json"))[arm]))
|
||||
series = []
|
||||
for l in open(f"{AB}/out/raw/mem.csv"):
|
||||
p = [x.strip() for x in l.split(",")]
|
||||
if len(p) == 4 and p[2] in pids:
|
||||
series.append((datetime.datetime.strptime(p[0], "%Y/%m/%d %H:%M:%S.%f").timestamp(), int(p[3])))
|
||||
reqs = []
|
||||
for f in ("first", "lat", "conc"):
|
||||
for l in open(f"{AB}/out/raw/{f}.jsonl"):
|
||||
r = json.loads(l)
|
||||
if r.get("arm") == arm and "e2e_ms" in r:
|
||||
reqs.append((r["t_wall"], round(r["dur"] - r.get("trim_ms", 0) / 1000, 2), r.get("mode", "first")))
|
||||
reqs.sort()
|
||||
prev, i, longest, last = None, 0, 0.0, None
|
||||
for t, m in series:
|
||||
while i < len(reqs) and reqs[i][0] <= t:
|
||||
longest = max(longest, reqs[i][1]); last = reqs[i]; i += 1
|
||||
if m != prev:
|
||||
ts = datetime.datetime.fromtimestamp(t).strftime("%H:%M:%S")
|
||||
print(f"{ts} {m:6d} MiB longest served so far {longest:5.1f}s last request {last[1] if last else '-'}s ({last[2] if last else '-'})")
|
||||
prev = m
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serving env for parakeet-unified-en-0.6b under NeMo torch: same pins as the investigation's nemo300
|
||||
# (torch 2.8 cu128, nemo_toolkit[asr]==3.0.0) plus the seat's HTTP stack. Built inside the scriberr image
|
||||
# (its /usr/bin/python3 is 3.13). Own uv cache; the investigation's env and cache are not touched.
|
||||
set -e
|
||||
export UV_CACHE_DIR=/ab/envs/.uvcache UV_LINK_MODE=copy
|
||||
cd /ab/envs
|
||||
uv venv -q --python /usr/bin/python3 nemo300-serve
|
||||
VIRTUAL_ENV=/ab/envs/nemo300-serve uv pip install -q --index-url https://download.pytorch.org/whl/cu128 --extra-index-url https://pypi.org/simple \
|
||||
"torch==2.8.*" "torchaudio==2.8.*" "nemo_toolkit[asr]==3.0.0" fastapi "uvicorn[standard]" python-multipart soundfile httpx 2>&1 | tail -3
|
||||
/ab/envs/nemo300-serve/bin/python -c "import nemo, torch, fastapi; print('nemo', nemo.__version__, 'torch', torch.__version__, 'fastapi', fastapi.__version__, torch.cuda.is_available())"
|
||||
@@ -0,0 +1,14 @@
|
||||
import json, sys, time
|
||||
sys.path.insert(0, "/tank/spikes/parakeet-ab/code")
|
||||
from bench import post
|
||||
url = sys.argv[1]
|
||||
rows = {json.loads(l)["id"]: json.loads(l) for l in open("/tank/spikes/parakeet-ab/data/lat.jsonl")}
|
||||
from bench import hostpath; w = lambda i: open(hostpath(rows[i]["wav"]), "rb").read()
|
||||
def show(tag, i):
|
||||
r = post(url, w(i)); print(f"{tag:10s} {i:12s} dur {rows[i]['dur']:6.2f}s e2e {r['e2e_ms']:8.1f} server {r['server_ms']}", flush=True)
|
||||
for k in range(5): show("repeat", "b3_8_10")
|
||||
for k in range(5): show("fresh", f"b3_8_{k:02d}")
|
||||
for k in range(5): show("fresh1-3", f"b1_3_{k:02d}")
|
||||
for k in range(3): show("repeat", "b1_3_00")
|
||||
for k in range(3): show("fresh20", f"b20_60_{k:02d}")
|
||||
for k in range(3): show("repeat", "b20_60_00")
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Build the A/B test sets from the pinned parquets. Seeded; writes 16 kHz mono PCM16 WAVs + manifests.
|
||||
|
||||
ls-clean, ls-other : 400 utterances each, uniform random sample (seed 20260930) of LibriSpeech test
|
||||
ami : 400 AMI IHM test utterances >= 1.0 s, uniform random sample (same seed)
|
||||
lat : latency clips, 20 per bin, from ALL of test-clean:
|
||||
1-3 s, 3-8 s, 8-20 s: single utterances at evenly spaced duration quantiles of the bin
|
||||
20-60 s: consecutive utterances of one chapter joined with 0.25 s silence, targets 20..58 s
|
||||
pc : positive control, 40 ls-clean sample utterances >= 6 s with 1.5 s of digital silence
|
||||
placed at 40 % of the utterance (the reference still holds the words)
|
||||
null : the 400 ls-clean utterances at -0.5 dB gain (a should-not-matter perturbation)
|
||||
Manifests: data/<set>.jsonl rows {id, wav, dur, ref, ...}. Prints counts only.
|
||||
"""
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
|
||||
import numpy as np
|
||||
import pyarrow.parquet as pq
|
||||
import soundfile as sf
|
||||
|
||||
DL = "/ab/dl"
|
||||
OUT = "/ab/data"
|
||||
SEED = 20260930
|
||||
SR = 16000
|
||||
|
||||
|
||||
def audio_of(cell):
|
||||
a, sr = sf.read(io.BytesIO(cell["bytes"]), dtype="float32")
|
||||
if a.ndim > 1:
|
||||
a = a.mean(axis=1)
|
||||
assert sr == SR, sr
|
||||
return a.astype(np.float32)
|
||||
|
||||
|
||||
def write(set_name, uid, a):
|
||||
d = f"{OUT}/{set_name}"
|
||||
os.makedirs(d, exist_ok=True)
|
||||
p = f"{d}/{uid}.wav"
|
||||
sf.write(p, np.clip(a, -1, 1), SR, subtype="PCM_16")
|
||||
return p
|
||||
|
||||
|
||||
def dump(set_name, rows):
|
||||
with open(f"{OUT}/{set_name}.jsonl", "w") as f:
|
||||
for r in rows:
|
||||
f.write(json.dumps(r) + "\n")
|
||||
print(set_name, len(rows), "utts", round(sum(r["dur"] for r in rows) / 60, 1), "min")
|
||||
|
||||
|
||||
def libri(split):
|
||||
t = pq.read_table(f"{DL}/openslr__librispeech_asr__all__test.{split}__0000.parquet").to_pylist()
|
||||
return t
|
||||
|
||||
|
||||
clean = libri("clean")
|
||||
other = libri("other")
|
||||
print("test-clean", len(clean), "test-other", len(other))
|
||||
|
||||
samples = {}
|
||||
for name, tab in (("ls-clean", clean), ("ls-other", other)):
|
||||
rng = random.Random(SEED)
|
||||
pick = rng.sample(range(len(tab)), 400)
|
||||
rows = []
|
||||
for i in pick:
|
||||
r = tab[i]
|
||||
a = audio_of(r["audio"])
|
||||
rows.append(dict(id=r["id"], wav=write(name, r["id"], a), dur=round(len(a) / SR, 3), ref=r["text"],
|
||||
speaker=r["speaker_id"], chapter=r["chapter_id"]))
|
||||
samples.setdefault(name, []).append((r, a))
|
||||
dump(name, rows)
|
||||
|
||||
# AMI IHM test
|
||||
ami = []
|
||||
for k in range(4):
|
||||
for r in pq.read_table(f"{DL}/edinburghcstr__ami__ihm__test-0000{k}-of-00004.parquet").to_pylist():
|
||||
dur = float(r["end_time"]) - float(r["begin_time"])
|
||||
if dur >= 1.0 and r["text"].strip():
|
||||
ami.append(r)
|
||||
rng = random.Random(SEED)
|
||||
rows = []
|
||||
for r in rng.sample(ami, 400):
|
||||
a = audio_of(r["audio"])
|
||||
rows.append(dict(id=r["audio_id"], wav=write("ami", r["audio_id"], a), dur=round(len(a) / SR, 3), ref=r["text"],
|
||||
meeting=r["meeting_id"], speaker=r["speaker_id"]))
|
||||
print("ami eligible (>=1 s)", len(ami), "meetings in sample", len({r["meeting"] for r in rows}))
|
||||
dump("ami", rows)
|
||||
|
||||
# latency clips
|
||||
durs = [(len(audio_of(r["audio"])) / SR, i) for i, r in enumerate(clean)]
|
||||
lat = []
|
||||
for lo, hi, tag in ((1, 3, "b1_3"), (3, 8, "b3_8"), (8, 20, "b8_20")):
|
||||
inbin = sorted((d, i) for d, i in durs if lo <= d < hi)
|
||||
for k in range(20):
|
||||
d, i = inbin[int((k + 0.5) / 20 * len(inbin))]
|
||||
r = clean[i]
|
||||
a = audio_of(r["audio"])
|
||||
uid = f"{tag}_{k:02d}"
|
||||
lat.append(dict(id=uid, wav=write("lat", uid, a), dur=round(len(a) / SR, 3), ref=r["text"], bin=tag, src=[r["id"]]))
|
||||
# 20-60 s: join consecutive utterances within a chapter (ordered by id)
|
||||
by_ch = {}
|
||||
for r in clean:
|
||||
by_ch.setdefault((r["speaker_id"], r["chapter_id"]), []).append(r)
|
||||
chapters = sorted(by_ch)
|
||||
random.Random(SEED).shuffle(chapters)
|
||||
gap = np.zeros(int(0.25 * SR), dtype=np.float32)
|
||||
targets = [20 + (58 - 20) * k / 19 for k in range(20)]
|
||||
ci = 0
|
||||
for k, tgt in enumerate(targets):
|
||||
while True:
|
||||
utts = sorted(by_ch[chapters[ci % len(chapters)]], key=lambda r: r["id"])
|
||||
ci += 1
|
||||
parts, refs, ids, n = [], [], [], 0
|
||||
for r in utts:
|
||||
a = audio_of(r["audio"])
|
||||
parts += [a, gap]
|
||||
refs.append(r["text"])
|
||||
ids.append(r["id"])
|
||||
n += len(a) + len(gap)
|
||||
if n / SR >= tgt:
|
||||
break
|
||||
if tgt <= n / SR <= 60:
|
||||
break
|
||||
a = np.concatenate(parts[:-1])
|
||||
uid = f"b20_60_{k:02d}"
|
||||
lat.append(dict(id=uid, wav=write("lat", uid, a), dur=round(len(a) / SR, 3), ref=" ".join(refs), bin="b20_60", src=ids))
|
||||
dump("lat", lat)
|
||||
for tag in ("b1_3", "b3_8", "b8_20", "b20_60"):
|
||||
ds = [r["dur"] for r in lat if r["bin"] == tag]
|
||||
print(" ", tag, "n", len(ds), "min", min(ds), "median", sorted(ds)[len(ds) // 2], "max", max(ds))
|
||||
|
||||
# positive control: 1.5 s of digital silence at 40 % of the utterance
|
||||
pc = []
|
||||
for r, a in samples["ls-clean"]:
|
||||
if len(a) / SR >= 6.0 and len(pc) < 40:
|
||||
b = a.copy()
|
||||
s0 = int(0.40 * len(a))
|
||||
s1 = s0 + int(1.5 * SR)
|
||||
b[s0:s1] = 0.0
|
||||
pc.append(dict(id=r["id"], wav=write("pc", r["id"], b), dur=round(len(b) / SR, 3), ref=r["text"],
|
||||
silence=[round(s0 / SR, 3), round(s1 / SR, 3)]))
|
||||
dump("pc", pc)
|
||||
|
||||
# null control: -0.5 dB gain
|
||||
g = 10 ** (-0.5 / 20)
|
||||
nul = []
|
||||
for r, a in samples["ls-clean"]:
|
||||
nul.append(dict(id=r["id"], wav=write("null", r["id"], a * g), dur=round(len(a) / SR, 3), ref=r["text"]))
|
||||
dump("null", nul)
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Accuracy block: every set through every arm, one thread per arm (no latency is measured here, so
|
||||
arms may run side by side). Per-arm files out/raw/acc/<set>--<arm>.jsonl.
|
||||
usage: run_acc.py ARM:PORT[:SETS] ... SETS = comma list, default all five
|
||||
"""
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
|
||||
AB = "/tank/spikes/parakeet-ab"
|
||||
ALL = ["ls-clean", "ls-other", "ami", "pc", "null"]
|
||||
os.makedirs(f"{AB}/out/raw/acc", exist_ok=True)
|
||||
|
||||
|
||||
def run(arm, port, sets):
|
||||
for s in sets:
|
||||
out = f"{AB}/out/raw/acc/{s}--{arm}.jsonl"
|
||||
if os.path.exists(out):
|
||||
os.rename(out, out + ".prev")
|
||||
r = subprocess.run(["python3", f"{AB}/code/bench.py", "acc", f"http://127.0.0.1:{port}/v1/audio/transcriptions",
|
||||
arm, f"{AB}/data/{s}.jsonl", out], capture_output=True, text=True)
|
||||
print(arm, s, "rc", r.returncode, r.stderr[-300:], flush=True)
|
||||
|
||||
|
||||
ts = []
|
||||
for spec in sys.argv[1:]:
|
||||
p = spec.split(":")
|
||||
ts.append(threading.Thread(target=run, args=(p[0], p[1], p[2].split(",") if len(p) > 2 else ALL)))
|
||||
[t.start() for t in ts]
|
||||
[t.join() for t in ts]
|
||||
print("ACC DONE", flush=True)
|
||||
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Latency block on fv-ml1 GPU 3. Host python3, stdlib only.
|
||||
|
||||
1. Cold: start each arm fresh and SEQUENTIALLY (so no two warm-ups compete for CPU), record
|
||||
container-start -> /healthz, then a first-call probe: one never-seen clip per bin, ascending length.
|
||||
2. Warm single stream: R rounds; in each round the arms run in a seeded shuffled order and each arm
|
||||
sends all 80 clips once (seeded order, seeded 0-490 ms tail trim so every request is a new length).
|
||||
Interleaving spreads drift (thermal, a Scriberr job on GPU 3) across arms instead of onto one.
|
||||
3. Concurrency 4: per arm, per bin, 40 requests from 4 workers.
|
||||
Writes out/raw/{arms,lat,conc,phases}.jsonl. Memory comes from the separate nvidia-smi sampler.
|
||||
usage: run_lat.py ROUNDS [ARM ...]
|
||||
"""
|
||||
import json
|
||||
import random
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
AB = "/tank/spikes/parakeet-ab"
|
||||
import os
|
||||
RAW = os.environ.get("AB_RAW", f"{AB}/out/raw")
|
||||
M = f"{AB}/models"
|
||||
ARMS = { # name: (port, arm.sh args, description)
|
||||
"ab-a1": (18301, ["sherpa", "/tank/parakeet/models", "1"], "A' #1: seat image, v3 int8, 1 thread"),
|
||||
"ab-a2": (18302, ["sherpa", "/tank/parakeet/models", "1"], "A' #2: identical second instance (floor)"),
|
||||
"ab-a50": (18303, ["sherpa", "/tank/parakeet/models", "1", "int8.onnx", "50"], "A' +50 ms injected (positive control)"),
|
||||
"ab-apl": (18309, ["sherpa", "/tank/parakeet/models", "1", "int8.onnx", "0", "1"], "A' with the image's app untouched (null)"),
|
||||
"ab-at16": (18326, ["sherpa", "/tank/parakeet/models", "16"], "A' with NUM_THREADS=16"),
|
||||
"ab-c": (18304, ["sherpa", f"{M}/unified-int8", "1"], "C: unified-en int8 (k2-fsa), seat runtime"),
|
||||
"ab-d": (18305, ["sherpa", f"{M}/v2-int8", "1"], "D: v2 int8 (k2-fsa), seat runtime"),
|
||||
"ab-cf32": (18312, ["sherpa", f"{M}/unified-f32", "1", "onnx"], "C-fp32: unified-en fp32 ONNX (k2-fsa export recipe), seat runtime"),
|
||||
"ab-cf16": (18310, ["sherpa", f"{M}/unified-f16", "1", "fp16.onnx"], "C-fp16: unified-en fp16 ONNX (pre_encode kept fp32), seat runtime"),
|
||||
"ab-af32": (18311, ["sherpa", f"{M}/v3-f32", "1", "onnx"], "A-fp32: the seat's v3 as fp32 ONNX (k2-fsa export recipe), seat runtime"),
|
||||
"ab-b32": (18306, ["nemo", "fp32", "direct", "1"], "B-fp32: unified-en, NeMo 3.0.0 torch"),
|
||||
"ab-b16": (18307, ["nemo", "bf16", "direct", "1"], "B-bf16: unified-en, NeMo 3.0.0 torch, autocast bf16"),
|
||||
"ab-b32c": (18314, ["nemo", "fp32", "direct", "1", "1"], "B-fp32, .nemo restored on CPU then moved to GPU"),
|
||||
"ab-b16w": (18315, ["nemo", "bf16w", "direct", "1", "1"], "B-bf16w: encoder/decoder/joint weights in bf16, CPU load"),
|
||||
}
|
||||
BINS = ["b1_3", "b3_8", "b8_20", "b20_60"]
|
||||
|
||||
|
||||
def sh(cmd, **kw):
|
||||
return subprocess.run(cmd, capture_output=True, text=True, **kw)
|
||||
|
||||
|
||||
def log(fn, d):
|
||||
with open(f"{RAW}/{fn}", "a") as f:
|
||||
f.write(json.dumps(d) + "\n")
|
||||
|
||||
|
||||
def url(name):
|
||||
return f"http://127.0.0.1:{ARMS[name][0]}/v1/audio/transcriptions"
|
||||
|
||||
|
||||
def bench(*args):
|
||||
r = sh(["python3", f"{AB}/code/bench.py", *args])
|
||||
if r.returncode:
|
||||
print("bench failed", args, r.stderr[-500:], flush=True)
|
||||
|
||||
|
||||
def main():
|
||||
R = int(sys.argv[1])
|
||||
arms = sys.argv[2:] or list(ARMS)
|
||||
sys.path.insert(0, f"{AB}/code")
|
||||
from bench import post, hostpath
|
||||
lat = {json.loads(l)["id"]: json.loads(l) for l in open(f"{AB}/data/lat.jsonl")}
|
||||
|
||||
# 1. cold
|
||||
for name in arms:
|
||||
port, args, desc = ARMS[name]
|
||||
sh(["docker", "rm", "-f", name])
|
||||
t = time.time()
|
||||
r = sh([f"{AB}/code/arm.sh", args[0], name, str(port), *args[1:]])
|
||||
if r.returncode:
|
||||
print("arm failed", name, r.stderr, flush=True)
|
||||
continue
|
||||
_, _, t0, t1, cold = r.stdout.split()
|
||||
pid = sh(["docker", "top", name, "-eo", "pid"]).stdout.split()[-1]
|
||||
log("arms.jsonl", dict(arm=name, desc=desc, port=port, args=args, pid=int(pid), t_started=float(t0), t_ready=float(t1), cold_s=float(cold)))
|
||||
time.sleep(3) # at-rest memory window
|
||||
log("phases.jsonl", dict(arm=name, phase="rest-after-warmup", t0=time.time() - 3, t1=time.time()))
|
||||
for tag in BINS: # first call at each length: clip _10 of each bin (never sent before to this container)
|
||||
uid = f"{tag}_10"
|
||||
res = post(url(name), open(hostpath(lat[uid]["wav"]), "rb").read())
|
||||
res.pop("text", None)
|
||||
log("first.jsonl", dict(arm=name, id=uid, bin=tag, dur=lat[uid]["dur"], t_wall=time.time(), **res))
|
||||
print(f"cold {name} {cold}s", flush=True)
|
||||
|
||||
# 2. warm single stream, interleaved
|
||||
t_ss = time.time()
|
||||
for k in range(R):
|
||||
order = list(arms)
|
||||
random.Random(f"round-{k}").shuffle(order)
|
||||
for name in order:
|
||||
t0 = time.time()
|
||||
bench("lat", url(name), name, f"{AB}/data/lat.jsonl", f"{RAW}/lat.jsonl", "--rounds", "1",
|
||||
"--round-offset", str(k), "--jitter-ms", "490")
|
||||
log("phases.jsonl", dict(arm=name, phase=f"lat-r{k}", t0=t0, t1=time.time()))
|
||||
print(f"round {k} done {time.time() - t_ss:.0f}s", flush=True)
|
||||
|
||||
# 3. concurrency 4
|
||||
for name in arms:
|
||||
t0 = time.time()
|
||||
bench("conc", url(name), name, f"{AB}/data/lat.jsonl", f"{RAW}/conc.jsonl", "--per-bin", "40",
|
||||
"--workers", "4", "--jitter-ms", "490")
|
||||
log("phases.jsonl", dict(arm=name, phase="conc4", t0=t0, t1=time.time()))
|
||||
print(f"conc {name} done", flush=True)
|
||||
print("LATENCY BLOCK DONE", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Long-form block: each public file WHOLE, in one request, exactly as the seat would receive it, twice per
|
||||
arm (determinism). Arms run one at a time so each one's long-form memory peak is its own.
|
||||
Writes out/raw/long/<file>--<arm>--<k>.json {arm, file, k, e2e_ms, server_ms, status, text, t0, t1}.
|
||||
usage: run_long.py ARM:PORT ...
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
AB = "/tank/spikes/parakeet-ab"
|
||||
sys.path.insert(0, f"{AB}/code")
|
||||
from bench import post # noqa: E402
|
||||
|
||||
FILES = {"wilde": "/tank/spikes/scriberr-slicer/public/wilde.wav", "scotus": "/tank/spikes/scriberr-slicer/public/scotus.wav"}
|
||||
os.makedirs(f"{AB}/out/raw/long", exist_ok=True)
|
||||
for spec in sys.argv[1:]:
|
||||
arm, port = spec.split(":")
|
||||
for f, path in FILES.items():
|
||||
wav = open(path, "rb").read()
|
||||
for k in (1, 2):
|
||||
t0 = time.time()
|
||||
try:
|
||||
res = post(f"http://127.0.0.1:{port}/v1/audio/transcriptions", wav, timeout=3600)
|
||||
except Exception as e: # noqa: BLE001
|
||||
res = dict(status=-1, err=repr(e)[:300], e2e_ms=None, server_ms=None, text=None)
|
||||
t1 = time.time()
|
||||
json.dump(dict(arm=arm, file=f, k=k, t0=t0, t1=t1, **res), open(f"{AB}/out/raw/long/{f}--{arm}--{k}.json", "w"))
|
||||
print(arm, f, k, res.get("status"), res.get("e2e_ms"), (res.get("err") or "")[:200], flush=True)
|
||||
print("LONG DONE", flush=True)
|
||||
@@ -0,0 +1,133 @@
|
||||
"""WER for the utterance sets, with paired-bootstrap deltas between arms.
|
||||
|
||||
Normaliser: Whisper's EnglishTextNormalizer, transformers 4.53.3 (the investigation's version), no
|
||||
spelling map, applied to the WHOLE utterance string on both sides (the standard Open-ASR-leaderboard
|
||||
use; the investigation applied it word by word only because it had to keep per-word timestamps on
|
||||
long-form, which is scored with its own gtscore.py, unchanged). Edit counts: exact Levenshtein with a
|
||||
backtrace for S/D/I. Corpus WER = sum(S+D+I) / sum(ref words).
|
||||
|
||||
usage:
|
||||
score.py wer MANIFEST RESULTS.jsonl [ARM ...] -> per-arm WER table (json lines)
|
||||
score.py delta MANIFEST RESULTS.jsonl ARM_X ARM_Y -> paired bootstrap of WER_X - WER_Y
|
||||
score.py selftest -> positive/null checks of the scorer itself
|
||||
"""
|
||||
import json
|
||||
import random
|
||||
import sys
|
||||
|
||||
from transformers.models.whisper.english_normalizer import EnglishTextNormalizer
|
||||
|
||||
_norm = EnglishTextNormalizer({})
|
||||
|
||||
|
||||
def toks(text):
|
||||
return _norm(text or "").split()
|
||||
|
||||
|
||||
def edits(ref, hyp):
|
||||
n, m = len(ref), len(hyp)
|
||||
d = [[0] * (m + 1) for _ in range(n + 1)]
|
||||
for i in range(n + 1):
|
||||
d[i][0] = i
|
||||
for j in range(m + 1):
|
||||
d[0][j] = j
|
||||
for i in range(1, n + 1):
|
||||
ri = ref[i - 1]
|
||||
row, prev = d[i], d[i - 1]
|
||||
for j in range(1, m + 1):
|
||||
row[j] = min(prev[j] + 1, row[j - 1] + 1, prev[j - 1] + (ri != hyp[j - 1]))
|
||||
s = ins = de = 0
|
||||
i, j = n, m
|
||||
while i > 0 or j > 0:
|
||||
if i > 0 and j > 0 and d[i][j] == d[i - 1][j - 1] + (ref[i - 1] != hyp[j - 1]):
|
||||
s += ref[i - 1] != hyp[j - 1]
|
||||
i, j = i - 1, j - 1
|
||||
elif i > 0 and d[i][j] == d[i - 1][j] + 1:
|
||||
de += 1
|
||||
i -= 1
|
||||
else:
|
||||
ins += 1
|
||||
j -= 1
|
||||
return s, de, ins
|
||||
|
||||
|
||||
def load(manifest, results, arms=None):
|
||||
ref = {}
|
||||
for l in open(manifest):
|
||||
r = json.loads(l)
|
||||
ref[r["id"]] = r
|
||||
by = {}
|
||||
for l in open(results):
|
||||
r = json.loads(l)
|
||||
if r.get("mode") != "acc" or (arms and r["arm"] not in arms):
|
||||
continue
|
||||
by.setdefault(r["arm"], {})[r["id"]] = r # last write wins (a rerun replaces)
|
||||
return ref, by
|
||||
|
||||
|
||||
def per_utt(ref, rows):
|
||||
out = {}
|
||||
for uid, rr in ref.items():
|
||||
h = rows.get(uid)
|
||||
R = toks(rr["ref"])
|
||||
H = toks(h["text"]) if h and h.get("status") == 200 else None
|
||||
if H is None:
|
||||
out[uid] = None
|
||||
continue
|
||||
s, de, ins = edits(R, H)
|
||||
out[uid] = (s, de, ins, len(R), len(H) == 0)
|
||||
return out
|
||||
|
||||
|
||||
def summary(pu):
|
||||
ok = [v for v in pu.values() if v is not None]
|
||||
S = sum(v[0] for v in ok); D = sum(v[1] for v in ok); I = sum(v[2] for v in ok); N = sum(v[3] for v in ok)
|
||||
return dict(utts=len(ok), missing=sum(v is None for v in pu.values()), ref_words=N, S=S, D=D, I=I,
|
||||
wer=round(100 * (S + D + I) / N, 3), empty_outputs=sum(v[4] for v in ok))
|
||||
|
||||
|
||||
def boot(pu_x, pu_y, n=4000, seed=7):
|
||||
ids = [k for k in pu_x if pu_x[k] is not None and pu_y.get(k) is not None]
|
||||
ex = [sum(pu_x[k][:3]) for k in ids]; ey = [sum(pu_y[k][:3]) for k in ids]; nw = [pu_x[k][3] for k in ids]
|
||||
point = 100 * (sum(ex) - sum(ey)) / sum(nw)
|
||||
rng = random.Random(seed)
|
||||
ds = []
|
||||
for _ in range(n):
|
||||
idx = [rng.randrange(len(ids)) for _ in ids]
|
||||
N = sum(nw[i] for i in idx)
|
||||
ds.append(100 * (sum(ex[i] for i in idx) - sum(ey[i] for i in idx)) / N)
|
||||
ds.sort()
|
||||
same = sum(1 for k in ids if pu_x[k][:3] == pu_y[k][:3])
|
||||
return dict(n_utts=len(ids), delta_pp=round(point, 3), ci95=[round(ds[int(.025 * n)], 3), round(ds[int(.975 * n)], 3)],
|
||||
utts_with_identical_edit_counts=same)
|
||||
|
||||
|
||||
def selftest():
|
||||
# scorer positive controls: one deleted word, one wrong word, one inserted word must each register exactly once
|
||||
ref = "the quick brown fox jumps over the lazy dog"
|
||||
cases = {"identical": (ref, (0, 0, 0)), "one deletion": ("the quick fox jumps over the lazy dog", (0, 1, 0)),
|
||||
"one substitution": ("the quick brown box jumps over the lazy dog", (1, 0, 0)),
|
||||
"one insertion": ("the quick brown fox jumps right over the lazy dog", (0, 0, 1)),
|
||||
"casing+punct only (null)": ("The quick, brown fox jumps over the lazy dog.", (0, 0, 0)),
|
||||
"filler only (null)": ("the quick brown fox uh jumps over the lazy dog", (0, 0, 0))}
|
||||
ok = True
|
||||
for name, (hyp, want) in cases.items():
|
||||
got = edits(toks(ref), toks(hyp))
|
||||
ok &= got == want
|
||||
print(f"{name:26s} want S/D/I {want} got {got} {'OK' if got == want else 'FAIL'}")
|
||||
print("SELFTEST", "PASS" if ok else "FAIL")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cmd = sys.argv[1]
|
||||
if cmd == "selftest":
|
||||
selftest()
|
||||
elif cmd == "wer":
|
||||
ref, by = load(sys.argv[2], sys.argv[3], sys.argv[4:] or None)
|
||||
for arm in sorted(by):
|
||||
print(json.dumps(dict(arm=arm, set=sys.argv[2].rsplit("/", 1)[-1].replace(".jsonl", ""), **summary(per_utt(ref, by[arm])))))
|
||||
elif cmd == "delta":
|
||||
ref, by = load(sys.argv[2], sys.argv[3], sys.argv[4:6])
|
||||
x, y = sys.argv[4], sys.argv[5]
|
||||
print(json.dumps(dict(set=sys.argv[2].rsplit("/", 1)[-1].replace(".jsonl", ""), x=x, y=y,
|
||||
**boot(per_utt(ref, by[x]), per_utt(ref, by[y])))))
|
||||
@@ -0,0 +1,153 @@
|
||||
"""A/B arm B: parakeet-unified-en-0.6b under NeMo torch, behind the SAME thin HTTP shape as the seat.
|
||||
|
||||
Mirrors stacks/parakeet/app.py: model loaded at import, one warm-up decode before traffic, and
|
||||
`async def` handlers that call a blocking decode (so, like the seat, requests are serialised on the
|
||||
event loop). Same endpoints, same {"text": ...} body, plus the harness's x-ab-decode-ms header.
|
||||
|
||||
Decode path (PATH_MODE):
|
||||
direct preprocessor -> encoder -> RNNT greedy_batch, no DataLoader (what a latency seat would run)
|
||||
transcribe ASRModel.transcribe([samples]) (NeMo's public API; used to check `direct` gives the same text)
|
||||
DTYPE: fp32 | bf16 (torch.autocast over encoder+decoding, fp32 weights) | bf16w (encoder, decoder and joint
|
||||
WEIGHTS cast to bf16, no autocast; the mel front end stays fp32).
|
||||
LOAD_CPU=1: restore the .nemo on the CPU, then move it to the GPU (avoids the ~1.8 GB load-time transient).
|
||||
LOCAL_ATT=L,R: switch the encoder to rel_pos_local_attn with that context (NeMo's documented long-audio
|
||||
mode; memory linear in length instead of quadratic). Unset = full attention, as the seat runs.
|
||||
CUDA_GRAPHS: 1 | 0 for NeMo's label-looping CUDA-graph greedy decoder.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
import torch
|
||||
from fastapi import FastAPI, File, HTTPException, UploadFile
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
MODEL_PATH = os.environ["MODEL_PATH"]
|
||||
DTYPE = os.environ.get("DTYPE", "fp32")
|
||||
CUDA_GRAPHS = os.environ.get("CUDA_GRAPHS", "1") == "1"
|
||||
PATH_MODE = os.environ.get("PATH_MODE", "direct")
|
||||
LOAD_CPU = os.environ.get("LOAD_CPU", "0") == "1"
|
||||
LOCAL_ATT = os.environ.get("LOCAL_ATT", "")
|
||||
SR = 16000
|
||||
|
||||
logger = logging.getLogger("parakeet-nemo")
|
||||
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO"))
|
||||
|
||||
|
||||
def _load():
|
||||
import nemo.collections.asr as nemo_asr
|
||||
from omegaconf import open_dict
|
||||
|
||||
t0 = time.monotonic()
|
||||
if LOAD_CPU:
|
||||
m = nemo_asr.models.ASRModel.restore_from(MODEL_PATH, map_location="cpu").to("cuda")
|
||||
else:
|
||||
m = nemo_asr.models.ASRModel.restore_from(MODEL_PATH, map_location="cuda")
|
||||
m.eval()
|
||||
if m.cfg.get("validation_ds") is None: # the unified .nemo ships without it; transcribe() reads it
|
||||
with open_dict(m.cfg):
|
||||
m.cfg.validation_ds = {}
|
||||
d = m.cfg.decoding
|
||||
with open_dict(d):
|
||||
d.strategy = "greedy_batch"
|
||||
d.greedy["use_cuda_graph_decoder"] = CUDA_GRAPHS
|
||||
m.change_decoding_strategy(d, verbose=False)
|
||||
# What transcribe() does on entry: no dither, no pad_to. The direct path must match it.
|
||||
m.preprocessor.featurizer.dither = 0.0
|
||||
m.preprocessor.featurizer.pad_to = 0
|
||||
if LOCAL_ATT:
|
||||
m.change_attention_model("rel_pos_local_attn", [int(x) for x in LOCAL_ATT.split(",")])
|
||||
if DTYPE == "bf16w":
|
||||
for mod in (m.encoder, m.decoder, m.joint):
|
||||
mod.to(torch.bfloat16)
|
||||
logger.info("loaded %s (%s) dtype=%s cuda_graphs=%s path=%s local_att=%s in %.1fs", os.path.basename(MODEL_PATH),
|
||||
type(m).__name__, DTYPE, CUDA_GRAPHS, PATH_MODE, LOCAL_ATT or "full", time.monotonic() - t0)
|
||||
return m
|
||||
|
||||
|
||||
model = _load()
|
||||
_AMP = dict(device_type="cuda", dtype=torch.bfloat16, enabled=(DTYPE == "bf16"))
|
||||
_FEAT_DTYPE = torch.bfloat16 if DTYPE == "bf16w" else torch.float32
|
||||
|
||||
|
||||
def _hyp_text(h) -> str:
|
||||
if isinstance(h, str):
|
||||
return h
|
||||
t = getattr(h, "text", None)
|
||||
if isinstance(t, str):
|
||||
return t
|
||||
return model.tokenizer.ids_to_text([int(i) for i in h.y_sequence])
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def _infer(samples: np.ndarray) -> str:
|
||||
if PATH_MODE == "transcribe":
|
||||
with torch.autocast(**_AMP):
|
||||
out = model.transcribe([samples], batch_size=1, verbose=False)
|
||||
if isinstance(out, tuple):
|
||||
out = out[0]
|
||||
return _hyp_text(out[0])
|
||||
x = torch.from_numpy(samples).to("cuda", non_blocking=True).unsqueeze(0)
|
||||
xl = torch.tensor([x.shape[1]], device="cuda", dtype=torch.long)
|
||||
feats, fl = model.preprocessor(input_signal=x, length=xl)
|
||||
feats = feats.to(_FEAT_DTYPE)
|
||||
with torch.autocast(**_AMP):
|
||||
enc, el = model.encoder(audio_signal=feats, length=fl)
|
||||
hyps = model.decoding.rnnt_decoder_predictions_tensor(encoder_output=enc, encoded_lengths=el,
|
||||
return_hypotheses=False)
|
||||
if isinstance(hyps, tuple):
|
||||
hyps = hyps[0]
|
||||
return _hyp_text(hyps[0])
|
||||
|
||||
|
||||
def _warm() -> None:
|
||||
"""Same as the seat: one throwaway 1 s of silence before traffic."""
|
||||
t0 = time.monotonic()
|
||||
_infer(np.zeros(SR, dtype=np.float32))
|
||||
torch.cuda.synchronize()
|
||||
logger.info("warmup decode complete in %.1fs", time.monotonic() - t0)
|
||||
|
||||
|
||||
_warm()
|
||||
app = FastAPI(title="Parakeet ASR (NeMo torch, A/B arm)")
|
||||
|
||||
|
||||
def _decode(raw: bytes) -> str:
|
||||
try:
|
||||
samples, sample_rate = sf.read(io.BytesIO(raw), dtype="float32")
|
||||
except Exception as exc:
|
||||
raise HTTPException(400, f"Could not decode audio: {exc}") from exc
|
||||
if samples.ndim > 1:
|
||||
samples = samples.mean(axis=1).astype(np.float32)
|
||||
if sample_rate != SR:
|
||||
import torchaudio.functional as AF
|
||||
samples = AF.resample(torch.from_numpy(samples), sample_rate, SR).numpy()
|
||||
return _infer(np.ascontiguousarray(samples, dtype=np.float32))
|
||||
|
||||
|
||||
def _timed(raw: bytes) -> JSONResponse:
|
||||
t0 = time.perf_counter()
|
||||
text = _decode(raw)
|
||||
torch.cuda.synchronize()
|
||||
ms = (time.perf_counter() - t0) * 1000.0
|
||||
return JSONResponse({"text": text}, headers={"x-ab-decode-ms": f"{ms:.3f}"})
|
||||
|
||||
|
||||
@app.get("/healthz")
|
||||
def healthz() -> dict[str, str]:
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.post("/transcribe")
|
||||
async def transcribe(file: UploadFile = File(...)):
|
||||
return _timed(await file.read())
|
||||
|
||||
|
||||
@app.post("/v1/audio/transcriptions")
|
||||
async def openai_transcriptions(file: UploadFile = File(...)):
|
||||
return _timed(await file.read())
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Stage compact raw results for the repo: per-request latency CSV (no text), all hypotheses gzipped,
|
||||
manifests without audio paths. Run from /tank/spikes/parakeet-ab."""
|
||||
import glob
|
||||
import gzip
|
||||
import json
|
||||
import os
|
||||
|
||||
S = "stage/results/raw"
|
||||
os.makedirs(S, exist_ok=True)
|
||||
os.makedirs("stage/manifests", exist_ok=True)
|
||||
with open(f"{S}/requests.csv", "w") as f:
|
||||
f.write("block,arm,mode,round,workers,id,bin,dur_s,trim_ms,e2e_ms,server_ms,status\n")
|
||||
for blk, fn in (("1", "out/raw/lat.jsonl"), ("1", "out/raw/lat-live.jsonl"), ("1", "out/raw/lat-gw.jsonl"),
|
||||
("1", "out/raw/conc.jsonl"), ("2", "out/raw2/lat.jsonl"), ("2", "out/raw2/conc.jsonl")):
|
||||
for l in open(fn):
|
||||
r = json.loads(l)
|
||||
if r.get("mode") not in ("lat", "conc"):
|
||||
continue
|
||||
vals = (blk, r["arm"], r["mode"], r.get("round", ""), r.get("workers", ""), r["id"], r.get("bin", ""), r["dur"],
|
||||
r.get("trim_ms", 0), r["e2e_ms"], r.get("server_ms"), r["status"])
|
||||
f.write(",".join("" if v is None else str(v) for v in vals) + "\n")
|
||||
with gzip.open(f"{S}/hypotheses.jsonl.gz", "wt") as g:
|
||||
for p in sorted(glob.glob("out/raw/acc/*.jsonl")):
|
||||
st, arm = os.path.basename(p)[:-6].split("--")
|
||||
for l in open(p):
|
||||
r = json.loads(l)
|
||||
g.write(json.dumps(dict(kind="utt", set=st, arm=arm, id=r["id"], status=r["status"], text=r.get("text"))) + "\n")
|
||||
for d, plc in (("out/raw/longwin", 1), ("out/raw/longwin-long2", 2)):
|
||||
for p in sorted(glob.glob(f"{d}/*.jsonl")):
|
||||
for l in open(p):
|
||||
r = json.loads(l)
|
||||
g.write(json.dumps(dict(kind="window", placement=plc, arm=r["arm"], id=r["id"], status=r["status"],
|
||||
e2e_ms=r.get("e2e_ms"), text=r.get("text"))) + "\n")
|
||||
for p in sorted(glob.glob("out/raw/long/*.json")):
|
||||
r = json.load(open(p))
|
||||
g.write(json.dumps(dict(kind="wholefile", arm=r["arm"], id=f"{r['file']}-{r['k']}", status=r.get("status"),
|
||||
e2e_ms=r.get("e2e_ms"), err=r.get("err"), text=r.get("text"))) + "\n")
|
||||
for p in sorted(glob.glob("data/*.jsonl")):
|
||||
with open(f"stage/manifests/{os.path.basename(p)}", "w") as f:
|
||||
for l in open(p):
|
||||
r = json.loads(l)
|
||||
r.pop("wav", None)
|
||||
f.write(json.dumps(r) + "\n")
|
||||
print("staged")
|
||||
Reference in New Issue
Block a user