spike(semif): latency profile and order-averaging measurement (no service change)

Latency, measured from nh3-dev (3 runs x 20 per condition; network floor 31 ms):
- /decide short: 71 ms end to end, 38 ms server-side;
- /decide with a ~2,000-token state: 210 / 169 ms;
- shared, 3 rotations: 113 / 79 ms;
- shared, 6 orderings: 137 / 99 ms.
Qwen3.5's fast kernels (causal_conv1d, flash-linear-attention) are not installed,
so transformers falls back to its reference PyTorch paths. That is a speed lever,
and using it needs a parity re-check.

Averaging over option orderings, on SemIf authored144 + perturbations108 (252 rows,
72 groups):
- a single ordering scores 78.6%;
- log-mean over the 3 rotations scores 87.7% (+9.1 pts, group-bootstrap 95% CI +4.7
  to +13.8);
- all 6 permutations score 88.1%.
Rotations capture nearly all of the gain. Rows where the rotations agree unanimously
(161) are 94.4% accurate; split rows (91) are 75.8%.

services/semif-serve/spike/averaging_spike.py

@@ -0,0 +1,89 @@
+"""SPIKE (2026-09-27): does averaging over option orderings help, and is agreement a useful
+ambiguity signal? Throwaway measurement, no service change: every labelled row goes out as ONE
+/decide/shared request carrying all 3! = 6 orderings of its options.
+
+Data: SemIf authored144 + perturbations108 (labelled, 3 options each), pinned commit.
+Conditions derived from the same 6 scored orderings per row:
+  single-original   the caller's own order (what the service does today)
+  single-expected   mean accuracy over the 6 single orderings (a caller's expected luck)
+  rotations         the 3 cyclic shifts of the original order, log-mean combined
+  all-6             all 6 permutations, log-mean combined
+Paired group-bootstrap (resampling group_id, 10k) for each combined condition minus
+single-original, so the delta is judged against its own sampling noise.
+    SEMIF_DIR=... SEMIF_URL=... SEMIF_TOKEN=... uv run --with httpx python averaging_spike.py out.json
+"""
+import itertools, json, math, os, random, statistics as st, sys, time
+from pathlib import Path
+import httpx
+
+S, U = Path(os.environ["SEMIF_DIR"]), os.environ["SEMIF_URL"]
+H = {"Authorization": f"Bearer {os.environ['SEMIF_TOKEN']}"}
+rows = []
+for name in ("authored144", "perturbations108"):
+    rows += [dict(json.loads(l), set=name) for l in (S / f"benchmarks/data/{name}.jsonl").read_text().splitlines() if l.strip()]
+
+
+def logmean(dists):   # dists: list of {option_id: p}; mean log p per option, renormalised
+    ids = dists[0].keys()
+    m = {k: st.fmean(math.log(max(d[k], 1e-12)) for d in dists) for k in ids}
+    top = max(m.values())
+    z = sum(math.exp(v - top) for v in m.values())
+    return {k: math.exp(v - top) / z for k, v in m.items()}
+
+
+argmax = lambda d: max(d, key=d.get)
+out, t0 = [], time.time()
+for r in rows:
+    opts = r["options"]
+    perms = list(itertools.permutations(range(len(opts))))
+    body = {"state": r["state"], "decisions": [
+        {"id": f"p{i}", "question": r["question"], "options": [opts[j] for j in p]} for i, p in enumerate(perms)]}
+    res = httpx.post(f"{U}/decide/shared", headers=H, json=body, timeout=120)
+    res.raise_for_status()
+    dists = [dict(zip(x["option_ids"], x["probabilities"])) for x in res.json()["results"]]
+    by_perm = dict(zip(perms, dists))
+    ident = tuple(range(len(opts)))
+    rot = [tuple((k + i) % len(opts) for k in ident) for i in range(len(opts))]
+    gold = opts[r["label"]]["id"]
+    rot_d, all_d = logmean([by_perm[p] for p in rot]), logmean(dists)
+    out.append({
+        "id": r["id"], "set": r["set"], "group": r["group_id"], "gold": gold,
+        "single_original": argmax(by_perm[ident]) == gold,
+        "single_expected": st.fmean(argmax(d) == gold for d in dists),
+        "rotations": argmax(rot_d) == gold,
+        "all6": argmax(all_d) == gold,
+        "agree_rot": sum(argmax(by_perm[p]) == argmax(rot_d) for p in rot) / len(rot),
+        "agree_all6": sum(argmax(d) == argmax(all_d) for d in dists) / len(dists),
+        "first_position_wins": sum(argmax(d) == opts[p[0]]["id"] for p, d in by_perm.items()) / len(perms),
+    })
+wall = time.time() - t0
+
+
+def boot(key, reps=10000, seed=7):
+    groups = {}
+    for o in out:
+        groups.setdefault(o["group"], []).append(o)
+    keys, rng, deltas = list(groups), random.Random(seed), []
+    for _ in range(reps):
+        sample = [o for g in (rng.choice(keys) for _ in keys) for o in groups[g]]
+        deltas.append(st.fmean(o[key] for o in sample) - st.fmean(o["single_original"] for o in sample))
+    deltas.sort()
+    return round(deltas[int(0.025 * reps)], 4), round(deltas[int(0.975 * reps)], 4)
+
+
+acc = lambda key, sub=out: round(st.fmean(o[key] for o in sub), 4)
+report = {"rows": len(out), "groups": len({o['group'] for o in out}), "wall_s": round(wall, 1),
+          "accuracy": {k: acc(k) for k in ("single_original", "single_expected", "rotations", "all6")},
+          "delta_vs_single_original_95ci": {k: boot(k) for k in ("rotations", "all6")},
+          "first_position_win_rate_mean": acc("first_position_wins"),
+          "by_set": {s: {k: acc(k, [o for o in out if o["set"] == s]) for k in ("single_original", "rotations", "all6")}
+                     for s in ("authored144", "perturbations108")}}
+for key in ("agree_rot", "agree_all6"):
+    unan = [o for o in out if o[key] == 1.0]
+    split = [o for o in out if o[key] < 1.0]
+    cond = "rotations" if key == "agree_rot" else "all6"
+    report[f"{key}: accuracy when unanimous vs split"] = {
+        "unanimous": {"rows": len(unan), "accuracy": acc(cond, unan) if unan else None},
+        "split": {"rows": len(split), "accuracy": acc(cond, split) if split else None}}
+json.dump({"report": report, "rows": out}, open(sys.argv[1], "w"), indent=1)
+print(json.dumps(report, indent=1))

services/semif-serve/spike/latency-2026-09-27.txt

@@ -0,0 +1,7 @@
+condition                      e2e p50 (runs)               server p50 (runs)
+health (floor)                   31.1 [30.1-33.4]           -
+decide, short (~130 tok)         71.2 [70.6-71.4]             38.4 [38.4-38.4]
+decide, long (~2,000 tok)       210.0 [209.5-211.0]          168.7 [168.3-168.9]
+shared, 3 rotations (short)     112.6 [111.9-113.4]           78.8 [78.7-78.9]
+shared, 6 orderings (short)     136.6 [135.2-139.5]           98.9 [98.6-99.0]
+shared, 3 rotations (long)      272.1 [270.3-273.7]          228.1 [227.9-228.7]

services/semif-serve/spike/latency.py

@@ -0,0 +1,58 @@
+"""semif-serve latency, 2026-09-27. Sequential client on nh3-dev, fresh connection per request,
+3 runs x 20 timed after 3 warm-ups per condition, conditions interleaved per run.
+e2e = wall time around the request; srv = what SemIf reports (direct: total_seconds; shared:
+timing.total_seconds), i.e. no network or HTTP.
+    SEMIF_URL=... SEMIF_TOKEN=... uv run --with httpx python latency.py
+"""
+import os, statistics as st, time, httpx, itertools
+
+U, H = os.environ["SEMIF_URL"], {"Authorization": f"Bearer {os.environ['SEMIF_TOKEN']}"}
+OPTS = [{"id": "casual_outing", "description": "A casual outing"},
+        {"id": "romantic_date", "description": "A romantic date"},
+        {"id": "booty_call", "description": "A booty call"}]
+CALL = "She calls up and says, hey, what're you doing right now? It's 2AM and I'm bored."
+LONG = "The service logged a routine heartbeat from node alpha at the scheduled interval without incident. " * 120
+Q = "What kind of invitation is this?"
+BIN = [{"id": "yes", "description": "Yes"}, {"id": "no", "description": "No"}]
+
+
+def rot(opts):
+    return [opts[i:] + opts[:i] for i in range(len(opts))]
+
+
+CONDS = {
+    "health (floor)": ("GET", "/health", None),
+    "decide, short (~130 tok)": ("POST", "/decide", {"id": "a", "state": CALL, "question": Q, "options": OPTS}),
+    "decide, long (~2,000 tok)": ("POST", "/decide", {"id": "b", "state": LONG, "question": "Was there an incident?", "options": BIN}),
+    "shared, 3 rotations (short)": ("POST", "/decide/shared", {"state": CALL, "decisions": [
+        {"id": f"r{i}", "question": Q, "options": o} for i, o in enumerate(rot(OPTS))]}),
+    "shared, 6 orderings (short)": ("POST", "/decide/shared", {"state": CALL, "decisions": [
+        {"id": f"p{i}", "question": Q, "options": list(o)} for i, o in enumerate(itertools.permutations(OPTS))]}),
+    "shared, 3 rotations (long)": ("POST", "/decide/shared", {"state": LONG, "decisions": [
+        {"id": f"r{i}", "question": "Was there an incident?", "options": o} for i, o in enumerate(rot(BIN + [{"id": "unsure", "description": "Cannot tell"}]))]}),
+}
+
+
+def once(method, path, body):
+    t = time.perf_counter()
+    r = httpx.request(method, U + path, json=body, headers=H, timeout=120)
+    r.raise_for_status()
+    e2e = (time.perf_counter() - t) * 1000
+    j = r.json()
+    srv = (j["timing"]["total_seconds"] if "timing" in j else j.get("total_seconds")) if path != "/health" else None
+    return e2e, (srv * 1000 if srv is not None else None)
+
+
+res = {k: {"e2e": [], "srv": []} for k in CONDS}
+for run in range(3):
+    for name, (m, p, b) in CONDS.items():
+        for _ in range(3):
+            once(m, p, b)
+        e, s = zip(*(once(m, p, b) for _ in range(20)))
+        res[name]["e2e"].append(st.median(e))
+        if s[0] is not None:
+            res[name]["srv"].append(st.median(s))
+print(f"{'condition':<30} {'e2e p50 (runs)':<28} {'server p50 (runs)'}")
+for name, r in res.items():
+    fmt = lambda xs: f"{st.median(xs):6.1f} [{min(xs):.1f}-{max(xs):.1f}]" if xs else "-"
+    print(f"{name:<30} {fmt(r['e2e']):<28} {fmt(r['srv'])}")