vh
317868dc7e
Prime asked for augaman on fv-ml1's utility card, beside vllm-coder. Mirror augaman-dev's f77164f compose, which parameterises the GPU reservation (GPU_ID, default 0) and the Homepage card name (CARD_SUFFIX). esh-ml1's resolved config is unchanged: same config hash, no recreate. On fv-ml1: augaman:0.1.2 built on-box from the tag, GPU_ID=1, healthy on CUDA at 1264 MiB, and pytest -m gpu tests/vision passes 3/3 on the Blackwell. It has its own gallery and no gallery backup, so it is fixtures-only. The host's raw restic copy of /var/lib/docker/volumes is not a consistent SQLite backup. docs/pfi/augaman-speed-bench/ holds the harness (augaman-dev's recipe plus a no-face control frame and a face-count check on every response), the raw rows and the summary. Server-side, one face: - esh-ml1 GPU 144 ms - fv-ml1 GPU 75 ms - fv-ml1 CPU on 6 cores 152 ms - esh-ml1 CPU 888 ms It agrees with augaman-dev's independent esh-ml1 measurement once each harness's floor is subtracted. This is the before for v0.1.3's detector fix.
71 lines
3.2 KiB
Python
71 lines
3.2 KiB
Python
"""augaman /recognize latency bench, harness per augaman-dev (2026-09-27):
|
|
sequential from nh3-dev; per (target, frame, run): 3 warm-up requests, then 20 timed.
|
|
3 runs, interleaved across targets and frames so drift spreads over every condition.
|
|
Each request is a fresh connection (no keep-alive), timed around the whole POST.
|
|
Server-side share: delta of augaman_pipeline_seconds{op="probe"} sum/count over the
|
|
20 timed requests (a mean, not a p50). Every response is checked for the expected
|
|
face count; a mismatch invalidates the row. Floor: /health, 3 runs x 20, same client.
|
|
|
|
Targets come from BENCH_TARGETS as JSON [{name, url, token_env}]; tokens are read
|
|
from the environment and never printed.
|
|
"""
|
|
import json, os, re, sys, time
|
|
import httpx
|
|
|
|
FRAMES = [("a_1080p_1face", "frame_a_1080p_oneface.jpg", 1),
|
|
("b_960x1001_2face", "obama_biden_2015.jpg", 2),
|
|
("c_1080p_0face", "frame_c_1080p_noface.jpg", 0)]
|
|
RUNS, WARM, N = 3, 3, 20
|
|
targets = json.loads(os.environ["BENCH_TARGETS"])
|
|
data = {f: open(f, "rb").read() for _, f, _ in FRAMES}
|
|
PROBE = re.compile(r'^augaman_pipeline_seconds_(sum|count)\{[^}]*op="probe"[^}]*\}\s+([0-9.eE+-]+)$', re.M)
|
|
|
|
|
|
def probe_stats(url):
|
|
t = httpx.get(url + "/metrics", timeout=10).text
|
|
vals = {k: float(v) for k, v in PROBE.findall(t)}
|
|
return vals.get("sum", float("nan")), vals.get("count", float("nan"))
|
|
|
|
|
|
def recognize(url, tok, fname):
|
|
t0 = time.perf_counter()
|
|
r = httpx.post(url + "/recognize", headers={"Authorization": f"Bearer {tok}"},
|
|
files={"frame": (fname, data[fname], "image/jpeg")}, timeout=120)
|
|
dt = (time.perf_counter() - t0) * 1000
|
|
r.raise_for_status()
|
|
return dt, len(r.json()["faces"])
|
|
|
|
|
|
def pct(xs, p):
|
|
xs = sorted(xs); k = (len(xs) - 1) * p; f = int(k); c = min(f + 1, len(xs) - 1)
|
|
return xs[f] + (xs[c] - xs[f]) * (k - f)
|
|
|
|
|
|
rows = []
|
|
for run in range(1, RUNS + 1):
|
|
for t in targets:
|
|
url, tok = t["url"], os.environ[t["token_env"]]
|
|
# floor
|
|
h = []
|
|
for _ in range(N):
|
|
t0 = time.perf_counter(); httpx.get(url + "/health", timeout=10).raise_for_status()
|
|
h.append((time.perf_counter() - t0) * 1000)
|
|
rows.append({"target": t["name"], "frame": "health_floor", "run": run,
|
|
"p50": pct(h, .5), "p90": pct(h, .9), "server_probe_mean": None, "bad": 0})
|
|
for label, fname, want in FRAMES:
|
|
for _ in range(WARM):
|
|
recognize(url, tok, fname)
|
|
s0, c0 = probe_stats(url)
|
|
lat, bad = [], 0
|
|
for _ in range(N):
|
|
dt, faces = recognize(url, tok, fname)
|
|
lat.append(dt); bad += faces != want
|
|
s1, c1 = probe_stats(url)
|
|
srv = (s1 - s0) / (c1 - c0) * 1000 if c1 > c0 else None
|
|
rows.append({"target": t["name"], "frame": label, "run": run, "p50": pct(lat, .5),
|
|
"p90": pct(lat, .9), "server_probe_mean": srv, "bad": bad})
|
|
print(f"run{run} {t['name']:<16} {label:<17} p50 {pct(lat,.5):8.1f} p90 {pct(lat,.9):8.1f}"
|
|
f" srv {srv if srv is None else round(srv,1)} bad {bad}", flush=True)
|
|
|
|
json.dump(rows, open(sys.argv[1] if len(sys.argv) > 1 else "bench_rows.json", "w"), indent=1)
|